From 9bd72f56224a8cc980620b17210d9b9b9ede6166 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 18 Jan 2011 18:33:04 -0800 Subject: various fixes to smoketests, including allowing admin tests to run as a user, better timing, and allowing volume tests to run on non-udev linux --- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/smoketests/flags.py b/smoketests/flags.py index 35f432a77..9dc310692 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -35,5 +35,5 @@ DEFINE_bool = DEFINE_bool # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 DEFINE_string('region', 'nova', 'Region to use') -DEFINE_string('test_image', 'ami-tiny', 'Image to use for launch tests') +DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') DEFINE_string('use_ipv6', True, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index d5a3a7556..f73ab7e1c 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -258,10 +258,15 @@ class VolumeTests(UserSmokeTestCase): instance = reservation.instances[0] self.data['instance'] = instance for x in xrange(120): - if self.can_ping(instance.private_dns_name): + time.sleep(1) + instance.update() + #if self.can_ping(instance.private_dns_name): + if instance.state == u'running': break else: self.fail('unable to start instance') + time.sleep(10) + instance.update() def test_001_can_create_volume(self): volume = self.conn.create_volume(1, 'nova') @@ -273,10 +278,11 @@ class VolumeTests(UserSmokeTestCase): def test_002_can_attach_volume(self): volume = self.data['volume'] - for x in xrange(10): - if volume.status == u'available': + for x in xrange(30): + print volume.status + if volume.status.startswith('available'): break - time.sleep(5) + time.sleep(1) volume.update() else: self.fail('cannot attach volume with state %s' % volume.status) @@ -285,12 +291,12 @@ class VolumeTests(UserSmokeTestCase): # Volumes seems to report "available" too soon. for x in xrange(10): - if volume.status == u'in-use': + if volume.status.startswith('in-use'): break time.sleep(5) volume.update() - self.assertEqual(volume.status, u'in-use') + self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. time.sleep(5) @@ -298,9 +304,15 @@ class VolumeTests(UserSmokeTestCase): def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name conn = self.connect_ssh(ip, TEST_KEY) + # NOTE(vish): this will create an dev for images that don't have + # udev rules + stdin, stdout, stderr = conn.exec_command( + 'grep %s /proc/partitions | ' + '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' + % self.device.rpartition('/')[2]) commands = [] commands.append('mkdir -p /mnt/vol') - commands.append('mkfs.ext2 %s' % self.device) + commands.append('/sbin/mke2fs %s' % self.device) commands.append('mount %s /mnt/vol' % self.device) commands.append('echo success') stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) @@ -327,7 +339,7 @@ class VolumeTests(UserSmokeTestCase): "df -h | grep %s | awk {'print $2'}" % self.device) out = stdout.read() conn.close() - if not out.strip() == '1008M': + if not out.strip() == '1007.9M': self.fail('Volume is not the right size: %s %s' % (out, stderr.read())) -- cgit From 2f4258d99e8d97ec70645cd2df2f4e54dc869e89 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 20 Jan 2011 00:14:42 -0800 Subject: more smoketest fixes --- nova/__init__.py | 2 -- smoketests/base.py | 7 +++++-- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 26 ++++++++++++++++---------- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/nova/__init__.py b/nova/__init__.py index 8745617bc..256db55a9 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -30,5 +30,3 @@ .. moduleauthor:: Manish Singh .. moduleauthor:: Andy Smith """ - -from exception import * diff --git a/smoketests/base.py b/smoketests/base.py index 610270c5c..89ee92840 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -17,12 +17,10 @@ # under the License. import boto -import boto_v6 import commands import httplib import os import paramiko -import random import sys import unittest from boto.ec2.regioninfo import RegionInfo @@ -30,6 +28,8 @@ from boto.ec2.regioninfo import RegionInfo from smoketests import flags FLAGS = flags.FLAGS +boto_v6 = None + class SmokeTestCase(unittest.TestCase): @@ -146,6 +146,9 @@ class SmokeTestCase(unittest.TestCase): def run_tests(suites): argv = FLAGS(sys.argv) + if FLAGS.use_ipv6: + global boto_v6 + boto_v6 = __import__('boto_v6') if not os.getenv('EC2_ACCESS_KEY'): print >> sys.stderr, 'Missing EC2 environment variables. Please ' \ diff --git a/smoketests/flags.py b/smoketests/flags.py index 9dc310692..dc756347b 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -36,4 +36,4 @@ DEFINE_bool = DEFINE_bool DEFINE_string('region', 'nova', 'Region to use') DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') -DEFINE_string('use_ipv6', True, 'use the ipv6 or not') +DEFINE_bool('use_ipv6', True, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index f73ab7e1c..e5bc98ede 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -189,8 +189,8 @@ class InstanceTests(UserSmokeTestCase): try: conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) conn.close() - except Exception: - time.sleep(1) + except Exception, e: + time.sleep(5) else: break else: @@ -224,7 +224,7 @@ class InstanceTests(UserSmokeTestCase): try: conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) conn.close() - except socket.error: + except Exception: time.sleep(1) else: break @@ -256,17 +256,24 @@ class VolumeTests(UserSmokeTestCase): instance_type='m1.tiny', key_name=TEST_KEY) instance = reservation.instances[0] - self.data['instance'] = instance for x in xrange(120): time.sleep(1) instance.update() - #if self.can_ping(instance.private_dns_name): - if instance.state == u'running': + if self.can_ping(instance.private_dns_name): break else: self.fail('unable to start instance') - time.sleep(10) - instance.update() + self.data['instance'] = instance + for x in xrange(30): + try: + conn = self.connect_ssh(instance.private_dns_name, TEST_KEY) + conn.close() + except Exception: + time.sleep(5) + else: + break + else: + self.fail('could not ssh to instance') def test_001_can_create_volume(self): volume = self.conn.create_volume(1, 'nova') @@ -279,7 +286,6 @@ class VolumeTests(UserSmokeTestCase): volume = self.data['volume'] for x in xrange(30): - print volume.status if volume.status.startswith('available'): break time.sleep(1) @@ -438,7 +444,7 @@ class SecurityGroupTests(UserSmokeTestCase): if __name__ == "__main__": suites = {'image': unittest.makeSuite(ImageTests), 'instance': unittest.makeSuite(InstanceTests), - 'security_group': unittest.makeSuite(SecurityGroupTests), + #'security_group': unittest.makeSuite(SecurityGroupTests), 'volume': unittest.makeSuite(VolumeTests) } sys.exit(base.run_tests(suites)) -- cgit From e6b7fa7ae31e90f2d7322445da3843281fff9a70 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 22 Jan 2011 16:20:36 -0800 Subject: fixes and refactoring of smoketests --- smoketests/base.py | 43 +++++++++++--- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 133 +++++++++++++----------------------------- 3 files changed, 77 insertions(+), 101 deletions(-) diff --git a/smoketests/base.py b/smoketests/base.py index 89ee92840..afc618074 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -22,6 +22,7 @@ import httplib import os import paramiko import sys +import time import unittest from boto.ec2.regioninfo import RegionInfo @@ -31,7 +32,6 @@ FLAGS = flags.FLAGS boto_v6 = None - class SmokeTestCase(unittest.TestCase): def connect_ssh(self, ip, key_name): # TODO(devcamcar): set a more reasonable connection timeout time @@ -39,12 +39,10 @@ class SmokeTestCase(unittest.TestCase): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.WarningPolicy()) client.connect(ip, username='root', pkey=key) - stdin, stdout, stderr = client.exec_command('uptime') - print 'uptime: ', stdout.read() return client - def can_ping(self, ip): - """ Attempt to ping the specified IP, and give up after 1 second. """ + def can_ping(self, ip, command="ping"): + """Attempt to ping the specified IP, and give up after 1 second.""" # NOTE(devcamcar): ping timeout flag is different in OSX. if sys.platform == 'darwin': @@ -52,10 +50,41 @@ class SmokeTestCase(unittest.TestCase): else: timeout_flag = 'w' - status, output = commands.getstatusoutput('ping -c1 -%s1 %s' % - (timeout_flag, ip)) + status, output = commands.getstatusoutput('%s -c1 -%s1 %s' % + (command, timeout_flag, ip)) return status == 0 + def wait_for_running(self, instance, tries=60, wait=1): + """Wait for instance to be running""" + for x in xrange(tries): + instance.update() + if instance.state.startswith('running'): + return True + time.sleep(wait) + else: + return False + + def wait_for_ping(self, ip, command="ping", tries=120): + """Wait for ip to be pingable""" + for x in xrange(tries): + if self.can_ping(ip, command): + return True + else: + return False + + def wait_for_ssh(self, ip, key_name, tries=30, wait=5): + """Wait for ip to be sshable""" + for x in xrange(tries): + try: + conn = self.connect_ssh(ip, key_name) + conn.close() + except Exception, e: + time.sleep(wait) + else: + return True + else: + return False + def connection_for_env(self, **kwargs): """ Returns a boto ec2 connection for the current environment. diff --git a/smoketests/flags.py b/smoketests/flags.py index dc756347b..5f3c8505e 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -36,4 +36,4 @@ DEFINE_bool = DEFINE_bool DEFINE_string('region', 'nova', 'Region to use') DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') -DEFINE_bool('use_ipv6', True, 'use the ipv6 or not') +DEFINE_bool('use_ipv6', False, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index e5bc98ede..26f6344f7 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -19,7 +19,6 @@ import commands import os import random -import socket import sys import time import unittest @@ -91,7 +90,6 @@ class ImageTests(UserSmokeTestCase): break time.sleep(1) else: - print image.state self.assert_(False) # wasn't available within 10 seconds self.assert_(image.type == 'machine') @@ -143,70 +141,36 @@ class InstanceTests(UserSmokeTestCase): key_name=TEST_KEY, instance_type='m1.tiny') self.assertEqual(len(reservation.instances), 1) - self.data['instance_id'] = reservation.instances[0].id + self.data['instance'] = reservation.instances[0] def test_003_instance_runs_within_60_seconds(self): - reservations = self.conn.get_all_instances([self.data['instance_id']]) - instance = reservations[0].instances[0] + instance = self.data['instance'] # allow 60 seconds to exit pending with IP - for x in xrange(60): - instance.update() - if instance.state == u'running': - break - time.sleep(1) - else: + if not self.wait_for_running(self.data['instance']): self.fail('instance failed to start') - ip = reservations[0].instances[0].private_dns_name + self.data['instance'].update() + ip = self.data['instance'].private_dns_name self.failIf(ip == '0.0.0.0') - self.data['private_ip'] = ip if FLAGS.use_ipv6: - ipv6 = reservations[0].instances[0].dns_name_v6 + ipv6 = self.data['instance'].dns_name_v6 self.failIf(ipv6 is None) - self.data['ip_v6'] = ipv6 def test_004_can_ping_private_ip(self): - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 %s' % self.data['private_ip']) - if status == 0: - break - else: + if not self.wait_for_ping(self.data['instance'].private_dns_name): self.fail('could not ping instance') if FLAGS.use_ipv6: - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping6 -c1 %s' % self.data['ip_v6']) - if status == 0: - break - else: - self.fail('could not ping instance') + if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + self.fail('could not ping instance v6') def test_005_can_ssh_to_private_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) - conn.close() - except Exception, e: - time.sleep(5) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): self.fail('could not ssh to instance') if FLAGS.use_ipv6: - for x in xrange(30): - try: - conn = self.connect_ssh( - self.data['ip_v6'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].ip_v6, + TEST_KEY): self.fail('could not ssh to instance v6') def test_006_can_allocate_elastic_ip(self): @@ -215,21 +179,13 @@ class InstanceTests(UserSmokeTestCase): self.data['public_ip'] = result.public_ip def test_007_can_associate_ip_with_instance(self): - result = self.conn.associate_address(self.data['instance_id'], + result = self.conn.associate_address(self.data['instance'].id, self.data['public_ip']) self.assertTrue(result) def test_008_can_ssh_with_public_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: - self.fail('could not ssh to instance') + if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): + self.fail('could not ssh to public ip') def test_009_can_disassociate_ip_from_instance(self): result = self.conn.disassociate_address(self.data['public_ip']) @@ -241,8 +197,7 @@ class InstanceTests(UserSmokeTestCase): def test_999_tearDown(self): self.delete_key_pair(self.conn, TEST_KEY) - if self.data.has_key('instance_id'): - self.conn.terminate_instances([self.data['instance_id']]) + self.conn.terminate_instances([self.data['instance'].id]) class VolumeTests(UserSmokeTestCase): @@ -255,24 +210,14 @@ class VolumeTests(UserSmokeTestCase): reservation = self.conn.run_instances(FLAGS.test_image, instance_type='m1.tiny', key_name=TEST_KEY) - instance = reservation.instances[0] - for x in xrange(120): - time.sleep(1) - instance.update() - if self.can_ping(instance.private_dns_name): - break - else: - self.fail('unable to start instance') - self.data['instance'] = instance - for x in xrange(30): - try: - conn = self.connect_ssh(instance.private_dns_name, TEST_KEY) - conn.close() - except Exception: - time.sleep(5) - else: - break - else: + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): self.fail('could not ssh to instance') def test_001_can_create_volume(self): @@ -280,32 +225,34 @@ class VolumeTests(UserSmokeTestCase): self.assertEqual(volume.size, 1) self.data['volume'] = volume # Give network time to find volume. - time.sleep(5) + time.sleep(10) def test_002_can_attach_volume(self): volume = self.data['volume'] - for x in xrange(30): + for x in xrange(10): + volume.update() if volume.status.startswith('available'): break time.sleep(1) - volume.update() else: self.fail('cannot attach volume with state %s' % volume.status) volume.attach(self.data['instance'].id, self.device) - # Volumes seems to report "available" too soon. + # wait for x in xrange(10): + volume.update() if volume.status.startswith('in-use'): break - time.sleep(5) - volume.update() + time.sleep(1) + else: + self.fail('volume never got to in use') self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. - time.sleep(5) + time.sleep(10) def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name @@ -316,12 +263,12 @@ class VolumeTests(UserSmokeTestCase): 'grep %s /proc/partitions | ' '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' % self.device.rpartition('/')[2]) - commands = [] - commands.append('mkdir -p /mnt/vol') - commands.append('/sbin/mke2fs %s' % self.device) - commands.append('mount %s /mnt/vol' % self.device) - commands.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) + exec_list = [] + exec_list.append('mkdir -p /mnt/vol') + exec_list.append('/sbin/mke2fs %s' % self.device) + exec_list.append('mount %s /mnt/vol' % self.device) + exec_list.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) out = stdout.read() conn.close() if not out.strip().endswith('success'): -- cgit From 8001f334221a16d8328289f6954ef549844f76f3 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Mon, 14 Feb 2011 14:26:32 +0100 Subject: Fix DescribeRegion answer by using specific 'listen' configuration parameter instead of overloading ec2_host --- bin/nova-api | 4 ++-- bin/nova-combined | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 11176a021..eb59d0191 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -59,12 +59,12 @@ def run_app(paste_config_file): LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) wsgi.paste_config_to_flags(config, { "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_host" % api: getattr(FLAGS, "%s_host" % api), "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) + config.get('listen', '0.0.0.0'))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) diff --git a/bin/nova-combined b/bin/nova-combined index 913c866bf..889600eb7 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -67,11 +67,11 @@ if __name__ == '__main__': continue wsgi.paste_config_to_flags(config, { "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_host" % api: getattr(FLAGS, "%s_host" % api), "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) app = wsgi.load_paste_app(paste_config_file, api) apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) + config.get('listen', '0.0.0.0'))) if len(apps) > 0: logging.basicConfig() server = wsgi.Server() -- cgit From 745b7b22f7b22a09e6c3bbc1cd8591eb3aa7f554 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 15 Feb 2011 21:38:47 +0100 Subject: removed flag --pidfile from nova/services.py --- nova/service.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/service.py b/nova/service.py index 59648adf2..8b1b91e90 100644 --- a/nova/service.py +++ b/nova/service.py @@ -50,10 +50,6 @@ flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) -flags.DEFINE_string('pidfile', None, - 'pidfile to use for this service') - - flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) -- cgit From 9a7213b615bcaa2127f76146d594f5247ea0d0a4 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 17 Feb 2011 15:00:18 -0800 Subject: Initial support for per-instance metadata, though the OpenStack API. Key/value pairs can be specified at instance creation time and are returned in the details view. Support limits based on quota system. --- nova/api/ec2/cloud.py | 6 +- nova/api/openstack/servers.py | 30 +++++++-- nova/compute/api.py | 29 +++++++- nova/db/sqlalchemy/api.py | 2 + .../versions/004_add_instance_metadata.py | 78 ++++++++++++++++++++++ nova/db/sqlalchemy/models.py | 18 ++++- nova/quota.py | 14 +++- nova/tests/api/openstack/test_servers.py | 11 ++- nova/tests/test_quota.py | 24 +++++++ run_tests.sh | 4 +- 10 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6919cd8d2..33eba5028 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -783,6 +783,9 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) + # NOTE(justinsb): the EC2 API doesn't support metadata here, but this + # is needed for the unit tests. Maybe the unit tests shouldn't be + # calling the EC2 code instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), @@ -797,7 +800,8 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone')) + 'AvailabilityZone'), + metadata=kwargs.get('metadata', [])) return self._format_run_instances(context, instances[0]['reservation_id']) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 009ef6db1..49611703a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -78,9 +78,14 @@ def _translate_detail_keys(inst): except KeyError: LOG.debug(_("Failed to read public ip(s)")) - inst_dict['metadata'] = {} inst_dict['hostId'] = '' + # Return the metadata as a dictionary + metadata = {} + for item in inst['metadata']: + metadata[item['key']] = item['value'] + inst_dict['metadata'] = metadata + return dict(server=inst_dict) @@ -162,14 +167,26 @@ class Controller(wsgi.Controller): if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) - key_pair = auth_manager.AuthManager.get_key_pairs( - req.environ['nova.context'])[0] + context = req.environ['nova.context'] + + key_pair = auth_manager.AuthManager.get_key_pairs(context)[0] image_id = common.get_image_id_from_image_hash(self._image_service, - req.environ['nova.context'], env['server']['imageId']) + context, env['server']['imageId']) kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( req, image_id) + + # Metadata is a list, not a Dictionary, because we allow duplicate keys + # (even though JSON can't encode this) + # In future, we may not allow duplicate keys. + # However, the CloudServers API is not definitive on this front, + # and we want to be compatible. + metadata = [] + if env['server']['metadata']: + for k, v in env['server']['metadata'].items(): + metadata.append({'key': k, 'value': v}) + instances = self.compute_api.create( - req.environ['nova.context'], + context, instance_types.get_by_flavor_id(env['server']['flavorId']), image_id, kernel_id=kernel_id, @@ -177,7 +194,8 @@ class Controller(wsgi.Controller): display_name=env['server']['name'], display_description=env['server']['name'], key_name=key_pair['name'], - key_data=key_pair['public_key']) + key_data=key_pair['public_key'], + metadata=metadata) return _translate_keys(instances[0]) def update(self, req, id): diff --git a/nova/compute/api.py b/nova/compute/api.py index ed6f0e34a..cad167f4d 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -85,7 +85,7 @@ class API(base.Base): min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', - availability_zone=None, user_data=None): + availability_zone=None, user_data=None, metadata=[]): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -99,6 +99,30 @@ class API(base.Base): "run %s more instances of this type.") % num_instances, "InstanceLimitExceeded") + num_metadata = len(metadata) + quota_metadata = quota.allowed_metadata_items(context, num_metadata) + if quota_metadata < num_metadata: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " tried to set %(num_metadata)s metadata properties") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + + # Because metadata is stored in the DB, we hard-code the size limits + # In future, we may support more variable length strings, so we act + # as if this is quota-controlled for forwards compatibility + for metadata_item in metadata: + k = metadata_item['key'] + v = metadata_item['value'] + if len(k) > 255 or len(v) > 255: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " metadata property key or value too long") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + is_vpn = image_id == FLAGS.vpn_image_id if not is_vpn: image = self.image_service.show(context, image_id) @@ -155,7 +179,8 @@ class API(base.Base): 'key_name': key_name, 'key_data': key_data, 'locked': False, - 'availability_zone': availability_zone} + 'availability_zone': availability_zone, + 'metadata': metadata} elevated = context.elevated() instances = [] diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..a6b8066b9 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -715,6 +715,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ filter_by(id=instance_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -723,6 +724,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ + options(joinedload('metadata')).\ filter_by(project_id=context.project_id).\ filter_by(id=instance_id).\ filter_by(deleted=False).\ diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py new file mode 100644 index 000000000..4cb07e0d8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +quotas = Table('quotas', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# + +instance_metadata_table = Table('instance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('value', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +# +# New columns +# +quota_metadata_items = Column('metadata_items', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (instance_metadata_table, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + quotas.create_column(quota_metadata_items) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 40a96fc17..a842e4cc4 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -256,6 +256,7 @@ class Quota(BASE, NovaBase): volumes = Column(Integer) gigabytes = Column(Integer) floating_ips = Column(Integer) + metadata_items = Column(Integer) class ExportDevice(BASE, NovaBase): @@ -536,6 +537,20 @@ class Console(BASE, NovaBase): pool = relationship(ConsolePool, backref=backref('consoles')) +class InstanceMetadata(BASE, NovaBase): + """Represents a metadata key/value pair for an instance""" + __tablename__ = 'instance_metadata' + id = Column(Integer, primary_key=True) + key = Column(String(255)) + value = Column(String(255)) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=False) + instance = relationship(Instance, backref="metadata", + foreign_keys=instance_id, + primaryjoin='and_(' + 'InstanceMetadata.instance_id == Instance.id,' + 'InstanceMetadata.deleted == False)') + + class Zone(BASE, NovaBase): """Represents a child zone of this zone.""" __tablename__ = 'zones' @@ -557,7 +572,8 @@ def register_models(): Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, - Project, Certificate, ConsolePool, Console, Zone) + Project, Certificate, ConsolePool, Console, Zone, + InstanceMetadata) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/quota.py b/nova/quota.py index 3884eb308..6b52a97fa 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -35,6 +35,8 @@ flags.DEFINE_integer('quota_gigabytes', 1000, 'number of volume gigabytes allowed per project') flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') +flags.DEFINE_integer('quota_metadata_items', 128, + 'number of metadata items allowed per instance') def get_quota(context, project_id): @@ -42,7 +44,8 @@ def get_quota(context, project_id): 'cores': FLAGS.quota_cores, 'volumes': FLAGS.quota_volumes, 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips} + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items} try: quota = db.quota_get(context, project_id) for key in rval.keys(): @@ -94,6 +97,15 @@ def allowed_floating_ips(context, num_floating_ips): return min(num_floating_ips, allowed_floating_ips) +def allowed_metadata_items(context, num_metadata_items): + """Check quota; return min(num_metadata_items,allowed_metadata_items)""" + project_id = context.project_id + context = context.elevated() + quota = get_quota(context, project_id) + num_allowed_metadata_items = quota['metadata_items'] + return min(num_metadata_items, num_allowed_metadata_items) + + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..7eb81c2b8 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -28,6 +28,7 @@ import nova.api.openstack from nova.api.openstack import servers import nova.db.api from nova.db.sqlalchemy.models import Instance +from nova.db.sqlalchemy.models import InstanceMetadata import nova.rpc from nova.tests.api.openstack import fakes @@ -64,6 +65,9 @@ def instance_address(context, instance_id): def stub_instance(id, user_id=1, private_address=None, public_addresses=None): + metadata = [] + metadata.append(InstanceMetadata(key='seq', value=id)) + if public_addresses == None: public_addresses = list() @@ -95,7 +99,8 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "availability_zone": "", "display_name": "server%s" % id, "display_description": "", - "locked": False} + "locked": False, + "metadata": metadata} instance["fixed_ip"] = { "address": private_address, @@ -214,7 +219,8 @@ class ServersTest(unittest.TestCase): "get_image_id_from_image_hash", image_id_from_hash) body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, metadata={}, + name='server_test', imageId=2, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, personality={})) req = webob.Request.blank('/v1.0/servers') req.method = 'POST' @@ -291,6 +297,7 @@ class ServersTest(unittest.TestCase): self.assertEqual(s['id'], i) self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s['imageId'], 10) + self.assertEqual(s['metadata']['seq'], i) i += 1 def test_server_pause(self): diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 9548a8c13..36ccc273e 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -87,6 +87,18 @@ class QuotaTestCase(test.TestCase): num_instances = quota.allowed_instances(self.context, 100, instance_types.INSTANCE_TYPES['m1.small']) self.assertEqual(num_instances, 10) + + # metadata_items + too_many_items = FLAGS.quota_metadata_items + 1000 + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, FLAGS.quota_metadata_items) + db.quota_update(self.context, self.project.id, {'metadata_items': 5}) + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, 5) + + # Cleanup db.quota_destroy(self.context, self.project.id) def test_too_many_instances(self): @@ -151,3 +163,15 @@ class QuotaTestCase(test.TestCase): self.assertRaises(quota.QuotaError, self.cloud.allocate_address, self.context) db.floating_ip_destroy(context.get_admin_context(), address) + + def test_too_many_metadata_items(self): + metadata = {} + for i in range(FLAGS.quota_metadata_items + 1): + metadata['key%s' % i] = 'value%s' % i + self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.context, + min_count=1, + max_count=1, + instance_type='m1.small', + image_id='fake', + metadata=metadata) diff --git a/run_tests.sh b/run_tests.sh index 4e21fe945..58e92c06b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -73,7 +73,9 @@ fi if [ -z "$noseargs" ]; then - run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py bin/* nova setup.py || exit 1 + srcfiles=`find bin -type f ! -name "nova.conf*"` + srcfiles+=" nova setup.py" + run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py ${srcfiles} || exit 1 else run_tests fi -- cgit From f0d58ea141116ccfd1c977b0f3e5fc669c0ea8a9 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Thu, 17 Feb 2011 18:06:34 -0600 Subject: moved inject network info to a function which accepts only instance, and call it from reset network --- nova/virt/xenapi/vmops.py | 96 +++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 842e08f22..d1ef95c6c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -104,46 +104,6 @@ class VMOps(object): instance, kernel, ramdisk, pv_kernel) VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) - # write network info - admin_context = context.get_admin_context() - - # TODO(tr3buchet) - remove comment in multi-nic - # I've decided to go ahead and consider multiple IPs and networks - # at this stage even though they aren't implemented because these will - # be needed for multi-nic and there was no sense writing it for single - # network/single IP and then having to turn around and re-write it - IPs = db.fixed_ip_get_all_by_instance(admin_context, instance['id']) - for network in db.network_get_all_by_instance(admin_context, - instance['id']): - network_IPs = [ip for ip in IPs if ip.network_id == network.id] - - def ip_dict(ip): - return {'netmask': network['netmask'], - 'enabled': '1', - 'ip': ip.address} - - mac_id = instance.mac_address.replace(':', '') - location = 'vm-data/networking/%s' % mac_id - mapping = {'label': network['label'], - 'gateway': network['gateway'], - 'mac': instance.mac_address, - 'dns': [network['dns']], - 'ips': [ip_dict(ip) for ip in network_IPs]} - self.write_to_param_xenstore(vm_ref, {location: mapping}) - - # TODO(tr3buchet) - remove comment in multi-nic - # this bit here about creating the vifs will be updated - # in multi-nic to handle multiple IPs on the same network - # and multiple networks - # for now it works as there is only one of each - bridge = network['bridge'] - network_ref = \ - NetworkHelper.find_network_with_bridge(self._session, bridge) - - if network_ref: - VMHelper.create_vif(self._session, vm_ref, - network_ref, instance.mac_address) - LOG.debug(_('Starting VM %s...'), vm_ref) self._session.call_xenapi('VM.start', vm_ref, False, False) instance_name = instance.name @@ -174,7 +134,7 @@ class VMOps(object): timer.f = _wait_for_boot - # call reset networking + # call to reset network to inject network info and configure self.reset_network(instance) return timer.start(interval=0.5, now=True) @@ -438,11 +398,65 @@ class VMOps(object): # TODO: implement this! return 'http://fakeajaxconsole/fake_url' + def inject_network_info(self, instance): + """ + Generate the network info and make calls to place it into the + xenstore and the xenstore param list + + """ + # TODO(tr3buchet) - remove comment in multi-nic + # I've decided to go ahead and consider multiple IPs and networks + # at this stage even though they aren't implemented because these will + # be needed for multi-nic and there was no sense writing it for single + # network/single IP and then having to turn around and re-write it + vm_opaque_ref = self._get_vm_opaque_ref(instance.id) + logging.debug(_("injecting network info to xenstore for vm: |%s|"), + vm_opaque_ref) + admin_context = context.get_admin_context() + IPs = db.fixed_ip_get_all_by_instance(admin_context, instance['id']) + for network in db.network_get_all_by_instance(admin_context, + instance['id']): + network_IPs = [ip for ip in IPs if ip.network_id == network.id] + + def ip_dict(ip): + return {'netmask': network['netmask'], + 'enabled': '1', + 'ip': ip.address} + + mac_id = instance.mac_address.replace(':', '') + location = 'vm-data/networking/%s' % mac_id + mapping = {'label': network['label'], + 'gateway': network['gateway'], + 'mac': instance.mac_address, + 'dns': [network['dns']], + 'ips': [ip_dict(ip) for ip in network_IPs]} + self.write_to_param_xenstore(vm_opaque_ref, {location: mapping}) + try: + self.write_to_xenstore(vm_opaque_ref, location, + mapping['location']) + except KeyError: + # catch KeyError for domid if instance isn't running + pass + + # TODO(tr3buchet) - remove comment in multi-nic + # this bit here about creating the vifs will be updated + # in multi-nic to handle multiple IPs on the same network + # and multiple networks + # for now it works as there is only one of each + bridge = network['bridge'] + network_ref = \ + NetworkHelper.find_network_with_bridge(self._session, bridge) + + if network_ref: + VMHelper.create_vif(self._session, vm_opaque_ref, + network_ref, instance.mac_address) + def reset_network(self, instance): """ Creates uuid arg to pass to make_agent_call and calls it. """ + self.inject_network_info(instance) args = {'id': str(uuid.uuid4())} resp = self._make_agent_call('resetnetwork', instance, '', args) -- cgit From debfca945627323c160b4ad9aa9b63b364deff99 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:02:55 +0100 Subject: Switch to API_listen and API_listen_port, drop wsgi.paste_config_to_flags --- bin/nova-api | 18 ++++++++++++------ etc/nova-api.conf | 3 --- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index eb59d0191..e7ee6f6fe 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -47,6 +47,12 @@ FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] +for api in API_ENDPOINTS: + flags.DEFINE_string("%s_api_listen" % api, "0.0.0.0", + "IP address to listen to for API %s" % api) + flags.DEFINE_integer("%s_api_listen_port" % api, + getattr(FLAGS, "%s_port" % api), + "Port to listen to for API %s" % api) def run_app(paste_config_file): LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) @@ -57,14 +63,10 @@ def run_app(paste_config_file): LOG.debug(_("No paste configuration for app: %s"), api) continue LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: getattr(FLAGS, "%s_host" % api), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - config.get('listen', '0.0.0.0'))) + apps.append((app, getattr(FLAGS, "%s_api_listen_port" % api), + getattr(FLAGS, "%s_api_listen" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) @@ -82,6 +84,10 @@ if __name__ == '__main__': FLAGS(sys.argv) LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) conf = wsgi.paste_config_file('nova-api.conf') if conf: run_app(conf) diff --git a/etc/nova-api.conf b/etc/nova-api.conf index f0e749805..9f7e93d4c 100644 --- a/etc/nova-api.conf +++ b/etc/nova-api.conf @@ -1,6 +1,3 @@ -[DEFAULT] -verbose = 1 - ####### # EC2 # ####### -- cgit From a0145eed239a7afb545def17f25a08e8e4c68824 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:08:33 +0100 Subject: Set up logging once FLAGS properly read, no need to redo logging config anymore (was inoperant anyway) --- bin/nova-api | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index e7ee6f6fe..46f695248 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -39,10 +39,6 @@ from nova import log as logging from nova import version from nova import wsgi -logging.basicConfig() -LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) - FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] @@ -72,8 +68,6 @@ def run_app(paste_config_file): paste_config_file) return - # NOTE(todd): redo logging config, verbose could be set in paste config - logging.basicConfig() server = wsgi.Server() for app in apps: server.start(*app) @@ -82,6 +76,8 @@ def run_app(paste_config_file): if __name__ == '__main__': FLAGS(sys.argv) + logging.basicConfig() + LOG = logging.getLogger('nova.api') LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) LOG.debug(_("Full set of FLAGS:")) -- cgit From 27c2de313a41bced77f7a4769deae089a70f5385 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:21:14 +0100 Subject: Port changes to nova-combined, rename flags to API_listen and API_listen_port --- bin/nova-api | 8 ++++---- bin/nova-combined | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 46f695248..8d47a656e 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -44,9 +44,9 @@ FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] for api in API_ENDPOINTS: - flags.DEFINE_string("%s_api_listen" % api, "0.0.0.0", + flags.DEFINE_string("%s_listen" % api, "0.0.0.0", "IP address to listen to for API %s" % api) - flags.DEFINE_integer("%s_api_listen_port" % api, + flags.DEFINE_integer("%s_listen_port" % api, getattr(FLAGS, "%s_port" % api), "Port to listen to for API %s" % api) @@ -61,8 +61,8 @@ def run_app(paste_config_file): LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_api_listen_port" % api), - getattr(FLAGS, "%s_api_listen" % api))) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) diff --git a/bin/nova-combined b/bin/nova-combined index 889600eb7..40dc2945d 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -45,6 +45,14 @@ from nova import wsgi FLAGS = flags.FLAGS +API_ENDPOINTS = ['ec2', 'osapi'] + +for api in API_ENDPOINTS: + flags.DEFINE_string("%s_listen" % api, "0.0.0.0", + "IP address to listen to for API %s" % api) + flags.DEFINE_integer("%s_listen_port" % api, + getattr(FLAGS, "%s_port" % api), + "Port to listen to for API %s" % api) if __name__ == '__main__': utils.default_flagfile() @@ -57,21 +65,17 @@ if __name__ == '__main__': scheduler = service.Service.create(binary='nova-scheduler') #objectstore = service.Service.create(binary='nova-objectstore') - service.serve(compute, network, volume, scheduler) +# service.serve(compute, network, volume, scheduler) apps = [] paste_config_file = wsgi.paste_config_file('nova-api.conf') - for api in ['osapi', 'ec2']: + for api in API_ENDPOINTS: config = wsgi.load_paste_configuration(paste_config_file, api) if config is None: continue - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: getattr(FLAGS, "%s_host" % api), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - config.get('listen', '0.0.0.0'))) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) if len(apps) > 0: logging.basicConfig() server = wsgi.Server() -- cgit From a02af158f781b61dba67c454afb59b34f27ca5d7 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:23:15 +0100 Subject: Remove paste_config_to_flags since it's now unused --- nova/wsgi.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index e01cc1e1e..d9b1cae86 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -515,10 +515,3 @@ def load_paste_app(filename, appname): except LookupError: pass return app - - -def paste_config_to_flags(config, mixins): - for k, v in mixins.iteritems(): - value = config.get(k, v) - converted_value = FLAGS[k].parser.Parse(value) - setattr(FLAGS, k, converted_value) -- cgit From 15d140992a980d959be6633b59d178a5ea273cdc Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:23:52 +0100 Subject: PEP8 fix --- bin/nova-api | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nova-api b/bin/nova-api index 8d47a656e..1228f723c 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -50,6 +50,7 @@ for api in API_ENDPOINTS: getattr(FLAGS, "%s_port" % api), "Port to listen to for API %s" % api) + def run_app(paste_config_file): LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) apps = [] -- cgit From bf570ca5f199091d505d96b91a3dc3acfbfc9fc7 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:37:00 +0100 Subject: Fixed testing mode leftover --- bin/nova-combined | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-combined b/bin/nova-combined index 40dc2945d..dd21c8df4 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -65,7 +65,7 @@ if __name__ == '__main__': scheduler = service.Service.create(binary='nova-scheduler') #objectstore = service.Service.create(binary='nova-objectstore') -# service.serve(compute, network, volume, scheduler) + service.serve(compute, network, volume, scheduler) apps = [] paste_config_file = wsgi.paste_config_file('nova-api.conf') -- cgit From bef44d7621db516a0f5d407655f5e76adfd5c06d Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 10:14:56 -0800 Subject: Rename migration 004 => 005 --- .../versions/004_add_instance_metadata.py | 78 ---------------------- .../versions/005_add_instance_metadata.py | 78 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 78 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py deleted file mode 100644 index 4cb07e0d8..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py +++ /dev/null @@ -1,78 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - - -# Just for the ForeignKey and column creation to succeed, these are not the -# actual definitions of instances or services. -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -quotas = Table('quotas', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - - -# -# New Tables -# - -instance_metadata_table = Table('instance_metadata', meta, - Column('created_at', DateTime(timezone=False)), - Column('updated_at', DateTime(timezone=False)), - Column('deleted_at', DateTime(timezone=False)), - Column('deleted', Boolean(create_constraint=True, name=None)), - Column('id', Integer(), primary_key=True, nullable=False), - Column('instance_id', - Integer(), - ForeignKey('instances.id'), - nullable=False), - Column('key', - String(length=255, convert_unicode=False, assert_unicode=None, - unicode_error=None, _warn_on_bytestring=False)), - Column('value', - String(length=255, convert_unicode=False, assert_unicode=None, - unicode_error=None, _warn_on_bytestring=False))) - - -# -# New columns -# -quota_metadata_items = Column('metadata_items', Integer()) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - for table in (instance_metadata_table, ): - try: - table.create() - except Exception: - logging.info(repr(table)) - logging.exception('Exception while creating table') - raise - - quotas.create_column(quota_metadata_items) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py new file mode 100644 index 000000000..4cb07e0d8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +quotas = Table('quotas', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# + +instance_metadata_table = Table('instance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('value', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +# +# New columns +# +quota_metadata_items = Column('metadata_items', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (instance_metadata_table, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + quotas.create_column(quota_metadata_items) -- cgit From f66840390bf6f4660a091663d306417e760735bc Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 10:57:19 -0800 Subject: Support for HP SAN --- .../versions/006_add_provider_data_to_volumes.py | 72 ++++++ nova/db/sqlalchemy/models.py | 3 + nova/volume/driver.py | 144 +++++++++-- nova/volume/manager.py | 8 +- nova/volume/san.py | 287 ++++++++++++++++++--- 5 files changed, 459 insertions(+), 55 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py b/nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py new file mode 100644 index 000000000..705fc8ff3 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py @@ -0,0 +1,72 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Table stub-definitions +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +# +volumes = Table('volumes', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# +# None + +# +# Tables to alter +# +# None + +# +# Columns to add to existing tables +# + +volumes_provider_location = Column('provider_location', + String(length=256, + convert_unicode=False, + assert_unicode=None, + unicode_error=None, + _warn_on_bytestring=False)) + +volumes_provider_auth = Column('provider_auth', + String(length=256, + convert_unicode=False, + assert_unicode=None, + unicode_error=None, + _warn_on_bytestring=False)) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + # Add columns to existing tables + volumes.create_column(volumes_provider_location) + volumes.create_column(volumes_provider_auth) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 40a96fc17..4485ee9e4 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -243,6 +243,9 @@ class Volume(BASE, NovaBase): display_name = Column(String(255)) display_description = Column(String(255)) + provider_location = Column(String(255)) + provider_auth = Column(String(255)) + class Quota(BASE, NovaBase): """Represents quota overrides for a project.""" diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 82f4c2f54..f172e2fdc 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -21,6 +21,7 @@ Drivers for volumes. """ import time +import os from nova import exception from nova import flags @@ -36,6 +37,8 @@ flags.DEFINE_string('aoe_eth_dev', 'eth0', 'Which device to export the volumes on') flags.DEFINE_string('num_shell_tries', 3, 'number of times to attempt to run flakey shell commands') +flags.DEFINE_string('num_iscsi_scan_tries', 3, + 'number of times to rescan iSCSI target to find volume') flags.DEFINE_integer('num_shelves', 100, 'Number of vblade shelves') @@ -294,40 +297,133 @@ class ISCSIDriver(VolumeDriver): self._execute("sudo ietadm --op delete --tid=%s" % iscsi_target) - def _get_name_and_portal(self, volume): - """Gets iscsi name and portal from volume name and host.""" + def _do_iscsi_discovery(self, volume): + #TODO(justinsb): Deprecate discovery and use stored info + #NOTE(justinsb): Discovery won't work with CHAP-secured targets (?) + LOG.warn(_("ISCSI provider_location not stored, using discovery")) + volume_name = volume['name'] - host = volume['host'] + (out, _err) = self._execute("sudo iscsiadm -m discovery -t " - "sendtargets -p %s" % host) + "sendtargets -p %s" % (volume['host'])) for target in out.splitlines(): if FLAGS.iscsi_ip_prefix in target and volume_name in target: - (location, _sep, iscsi_name) = target.partition(" ") - break - iscsi_portal = location.split(",")[0] - return (iscsi_name, iscsi_portal) + return target + return None + + def _get_iscsi_properties(self, volume): + """Gets iscsi configuration, ideally from saved information in the + volume entity, but falling back to discovery if need be.""" + + properties = {} + + location = volume['provider_location'] + + if location: + # provider_location is the same format as iSCSI discovery output + properties['target_discovered'] = False + else: + location = self._do_iscsi_discovery(volume) + + if not location: + raise exception.Error(_("Could not find iSCSI export " + " for volume %s") % + (volume['name'])) + + LOG.debug(_("ISCSI Discovery: Found %s") % (location)) + properties['target_discovered'] = True + + (iscsi_target, _sep, iscsi_name) = location.partition(" ") + + iscsi_portal = iscsi_target.split(",")[0] + + properties['target_iqn'] = iscsi_name + properties['target_portal'] = iscsi_portal + + auth = volume['provider_auth'] + + if auth: + (auth_method, auth_username, auth_secret) = auth.split() + + properties['auth_method'] = auth_method + properties['auth_username'] = auth_username + properties['auth_password'] = auth_secret + + return properties + + def _run_iscsiadm(self, iscsi_properties, iscsi_command): + command = ("sudo iscsiadm -m node -T %s -p %s %s" % + (iscsi_properties['target_iqn'], + iscsi_properties['target_portal'], + iscsi_command)) + (out, err) = self._execute(command) + LOG.debug("iscsiadm %s: stdout=%s stderr=%s" % + (iscsi_command, out, err)) + return (out, err) + + def _iscsiadm_update(self, iscsi_properties, property_key, property_value): + iscsi_command = ("--op update -n %s -v %s" % + (property_key, property_value)) + return self._run_iscsiadm(iscsi_properties, iscsi_command) def discover_volume(self, volume): """Discover volume on a remote host.""" - iscsi_name, iscsi_portal = self._get_name_and_portal(volume) - self._execute("sudo iscsiadm -m node -T %s -p %s --login" % - (iscsi_name, iscsi_portal)) - self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v automatic" % - (iscsi_name, iscsi_portal)) - return "/dev/disk/by-path/ip-%s-iscsi-%s-lun-0" % (iscsi_portal, - iscsi_name) + iscsi_properties = self._get_iscsi_properties(volume) + + if not iscsi_properties['target_discovered']: + self._run_iscsiadm(iscsi_properties, "--op new") + + if iscsi_properties.get('auth_method'): + self._iscsiadm_update(iscsi_properties, + "node.session.auth.authmethod", + iscsi_properties['auth_method']) + self._iscsiadm_update(iscsi_properties, + "node.session.auth.username", + iscsi_properties['auth_username']) + self._iscsiadm_update(iscsi_properties, + "node.session.auth.password", + iscsi_properties['auth_password']) + + self._run_iscsiadm(iscsi_properties, "--login") + + self._iscsiadm_update(iscsi_properties, "node.startup", "automatic") + + mount_device = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-0" % + (iscsi_properties['target_portal'], + iscsi_properties['target_iqn'])) + + # The /dev/disk/by-path/... node is not always present immediately + # TODO(justinsb): This retry-with-delay is a pattern, move to utils? + tries = 0 + while not os.path.exists(mount_device): + if tries >= FLAGS.num_iscsi_scan_tries: + raise exception.Error(_("iSCSI device not found at %s") % + (mount_device)) + + LOG.warn(_("ISCSI volume not yet found at: %(mount_device)s. " + "Will rescan & retry. Try number: %(tries)s") % + locals()) + + # The rescan isn't documented as being necessary(?), but it helps + self._run_iscsiadm(iscsi_properties, "--rescan") + + tries = tries + 1 + if not os.path.exists(mount_device): + time.sleep(tries ** 2) + + if tries != 0: + LOG.debug(_("Found iSCSI node %(mount_device)s " + "(after %(tries)s rescans)") % + locals()) + + return mount_device def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" - iscsi_name, iscsi_portal = self._get_name_and_portal(volume) - self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v manual" % - (iscsi_name, iscsi_portal)) - self._execute("sudo iscsiadm -m node -T %s -p %s --logout " % - (iscsi_name, iscsi_portal)) - self._execute("sudo iscsiadm -m node --op delete " - "--targetname %s" % iscsi_name) + iscsi_properties = self._get_iscsi_properties(volume) + self._iscsiadm_update(iscsi_properties, "node.startup", "manual") + self._run_iscsiadm(iscsi_properties, "--logout") + self._run_iscsiadm(iscsi_properties, "--op delete") class FakeISCSIDriver(ISCSIDriver): diff --git a/nova/volume/manager.py b/nova/volume/manager.py index d2f02e4e0..7193ece14 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -107,10 +107,14 @@ class VolumeManager(manager.Manager): vol_size = volume_ref['size'] LOG.debug(_("volume %(vol_name)s: creating lv of" " size %(vol_size)sG") % locals()) - self.driver.create_volume(volume_ref) + db_update = self.driver.create_volume(volume_ref) + if db_update: + self.db.volume_update(context, volume_ref['id'], db_update) LOG.debug(_("volume %s: creating export"), volume_ref['name']) - self.driver.create_export(context, volume_ref) + db_update = self.driver.create_export(context, volume_ref) + if db_update: + self.db.volume_update(context, volume_ref['id'], db_update) except Exception: self.db.volume_update(context, volume_ref['id'], {'status': 'error'}) diff --git a/nova/volume/san.py b/nova/volume/san.py index 26d6125e7..911ad096f 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -23,6 +23,8 @@ The unique thing about a SAN is that we don't expect that we can run the volume import os import paramiko +from xml.etree import ElementTree + from nova import exception from nova import flags from nova import log as logging @@ -41,37 +43,15 @@ flags.DEFINE_string('san_password', '', 'Password for SAN controller') flags.DEFINE_string('san_privatekey', '', 'Filename of private key to use for SSH authentication') +flags.DEFINE_string('san_clustername', '', + 'Cluster name to use for creating volumes') +flags.DEFINE_integer('san_ssh_port', 22, + 'SSH port to use with SAN') class SanISCSIDriver(ISCSIDriver): """ Base class for SAN-style storage volumes (storage providers we access over SSH)""" - #Override because SAN ip != host ip - def _get_name_and_portal(self, volume): - """Gets iscsi name and portal from volume name and host.""" - volume_name = volume['name'] - - # TODO(justinsb): store in volume, remerge with generic iSCSI code - host = FLAGS.san_ip - - (out, _err) = self._execute("sudo iscsiadm -m discovery -t " - "sendtargets -p %s" % host) - - location = None - find_iscsi_name = self._build_iscsi_target_name(volume) - for target in out.splitlines(): - if find_iscsi_name in target: - (location, _sep, iscsi_name) = target.partition(" ") - break - if not location: - raise exception.Error(_("Could not find iSCSI export " - " for volume %s") % - volume_name) - - iscsi_portal = location.split(",")[0] - LOG.debug("iscsi_name=%s, iscsi_portal=%s" % - (iscsi_name, iscsi_portal)) - return (iscsi_name, iscsi_portal) def _build_iscsi_target_name(self, volume): return "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) @@ -85,6 +65,7 @@ class SanISCSIDriver(ISCSIDriver): ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if FLAGS.san_password: ssh.connect(FLAGS.san_ip, + port=FLAGS.san_ssh_port, username=FLAGS.san_login, password=FLAGS.san_password) elif FLAGS.san_privatekey: @@ -92,10 +73,11 @@ class SanISCSIDriver(ISCSIDriver): # It sucks that paramiko doesn't support DSA keys privatekey = paramiko.RSAKey.from_private_key_file(privatekeyfile) ssh.connect(FLAGS.san_ip, + port=FLAGS.san_ssh_port, username=FLAGS.san_login, pkey=privatekey) else: - raise exception.Error("Specify san_password or san_privatekey") + raise exception.Error(_("Specify san_password or san_privatekey")) return ssh def _run_ssh(self, command, check_exit_code=True): @@ -124,10 +106,10 @@ class SanISCSIDriver(ISCSIDriver): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" if not (FLAGS.san_password or FLAGS.san_privatekey): - raise exception.Error("Specify san_password or san_privatekey") + raise exception.Error(_("Specify san_password or san_privatekey")) if not (FLAGS.san_ip): - raise exception.Error("san_ip must be set") + raise exception.Error(_("san_ip must be set")) def _collect_lines(data): @@ -306,6 +288,17 @@ class SolarisISCSIDriver(SanISCSIDriver): self._run_ssh("pfexec /usr/sbin/stmfadm add-view -t %s %s" % (target_group_name, luid)) + #TODO(justinsb): Is this always 1? Does it matter? + iscsi_portal_interface = '1' + iscsi_portal = FLAGS.san_ip + ":3260," + iscsi_portal_interface + + db_update = {} + db_update['provider_location'] = ("%s %s" % + (iscsi_portal, + iscsi_name)) + + return db_update + def remove_export(self, context, volume): """Removes an export for a logical volume.""" @@ -333,3 +326,239 @@ class SolarisISCSIDriver(SanISCSIDriver): if self._is_lu_created(volume): self._run_ssh("pfexec /usr/sbin/sbdadm delete-lu %s" % (luid)) + + +class HpSanISCSIDriver(SanISCSIDriver): + """Executes commands relating to HP/Lefthand SAN ISCSI volumes. + We use the CLIQ interface, over SSH. + + Rough overview of CLIQ commands used: + CLIQ createVolume (creates the volume) + CLIQ getVolumeInfo (to discover the IQN etc) + CLIQ getClusterInfo (to discover the iSCSI target IP address) + CLIQ assignVolumeChap (exports it with CHAP security) + + The 'trick' here is that the HP SAN enforces security by default, so + normally a volume mount would need both to configure the SAN in the volume + layer and do the mount on the compute layer. Multi-layer operations are + not catered for at the moment in the nova architecture, so instead we + share the volume using CHAP at volume creation time. Then the mount need + only use those CHAP credentials, so can take place exclusively in the + compute layer""" + + def _cliq_run(self, verb, cliq_args): + """Runs a CLIQ command over SSH, without doing any result parsing""" + cliq_arg_strings = [] + for k, v in cliq_args.items(): + cliq_arg_strings.append(" %s=%s" % (k, v)) + cmd = verb + ''.join(cliq_arg_strings) + + return self._run_ssh(cmd) + + def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True): + """Runs a CLIQ command over SSH, parsing and checking the output""" + cliq_args['output'] = 'XML' + (out, _err) = self._cliq_run(verb, cliq_args) + + LOG.debug(_("CLIQ command returned %s"), out) + + result_xml = ElementTree.fromstring(out) + if check_cliq_result: + response_node = result_xml.find("response") + if response_node is None: + msg = (_("Malformed response to CLIQ command " + "%(verb)s %(cliq_args)s. Result=%(out)s") % + locals()) + raise exception.Error(msg) + + result_code = response_node.attrib.get("result") + + if result_code != "0": + msg = (_("Error running CLIQ command %(verb)s %(cliq_args)s. " + " Result=%(out)s") % + locals()) + raise exception.Error(msg) + + return result_xml + + def _cliq_get_cluster_info(self, cluster_name): + """Queries for info about the cluster (including IP)""" + cliq_args = {} + cliq_args['clusterName'] = cluster_name + cliq_args['searchDepth'] = '1' + cliq_args['verbose'] = '0' + + result_xml = self._cliq_run_xml("getClusterInfo", cliq_args) + + return result_xml + + def _cliq_get_cluster_vip(self, cluster_name): + """Gets the IP on which a cluster shares iSCSI volumes""" + cluster_xml = self._cliq_get_cluster_info(cluster_name) + + vips = [] + for vip in cluster_xml.findall("response/cluster/vip"): + vips.append(vip.attrib.get('ipAddress')) + + if len(vips) == 1: + return vips[0] + + _xml = ElementTree.tostring(cluster_xml) + msg = (_("Unexpected number of virtual ips for cluster " + " %(cluster_name)s. Result=%(_xml)s") % + locals()) + raise exception.Error(msg) + + def _cliq_get_volume_info(self, volume_name): + """Gets the volume info, including IQN""" + cliq_args = {} + cliq_args['volumeName'] = volume_name + result_xml = self._cliq_run_xml("getVolumeInfo", cliq_args) + + # Result looks like this: + # + # + # + # + # + # + # + # + + # Flatten the nodes into a dictionary; use prefixes to avoid collisions + volume_attributes = {} + + volume_node = result_xml.find("response/volume") + for k, v in volume_node.attrib.items(): + volume_attributes["volume." + k] = v + + status_node = volume_node.find("status") + if not status_node is None: + for k, v in status_node.attrib.items(): + volume_attributes["status." + k] = v + + # We only consider the first permission node + permission_node = volume_node.find("permission") + if not permission_node is None: + for k, v in status_node.attrib.items(): + volume_attributes["permission." + k] = v + + LOG.debug(_("Volume info: %(volume_name)s => %(volume_attributes)s") % + locals()) + return volume_attributes + + def create_volume(self, volume): + """Creates a volume.""" + cliq_args = {} + cliq_args['clusterName'] = FLAGS.san_clustername + #TODO(justinsb): Should we default to inheriting thinProvision? + cliq_args['thinProvision'] = '1' if FLAGS.san_thin_provision else '0' + cliq_args['volumeName'] = volume['name'] + if int(volume['size']) == 0: + cliq_args['size'] = '100MB' + else: + cliq_args['size'] = '%sGB' % volume['size'] + + self._cliq_run_xml("createVolume", cliq_args) + + volume_info = self._cliq_get_volume_info(volume['name']) + cluster_name = volume_info['volume.clusterName'] + iscsi_iqn = volume_info['volume.iscsiIqn'] + + #TODO(justinsb): Is this always 1? Does it matter? + cluster_interface = '1' + + cluster_vip = self._cliq_get_cluster_vip(cluster_name) + iscsi_portal = cluster_vip + ":3260," + cluster_interface + + db_update = {} + db_update['provider_location'] = ("%s %s" % + (iscsi_portal, + iscsi_iqn)) + + return db_update + + def delete_volume(self, volume): + """Deletes a volume.""" + cliq_args = {} + cliq_args['volumeName'] = volume['name'] + cliq_args['prompt'] = 'false' # Don't confirm + + self._cliq_run_xml("deleteVolume", cliq_args) + + def local_path(self, volume): + # TODO(justinsb): Is this needed here? + raise exception.Error(_("local_path not supported")) + + def ensure_export(self, context, volume): + """Synchronously recreates an export for a logical volume.""" + return self._do_export(context, volume, force_create=False) + + def create_export(self, context, volume): + return self._do_export(context, volume, force_create=True) + + def _do_export(self, context, volume, force_create): + """Supports ensure_export and create_export""" + volume_info = self._cliq_get_volume_info(volume['name']) + + is_shared = 'permission.authGroup' in volume_info + + db_update = {} + + should_export = False + + if force_create or not is_shared: + should_export = True + # Check that we have a project_id + project_id = volume['project_id'] + if not project_id: + project_id = context.project_id + + if project_id: + #TODO(justinsb): Use a real per-project password here + chap_username = 'proj_' + project_id + # HP/Lefthand requires that the password be >= 12 characters + chap_password = 'project_secret_' + project_id + else: + msg = (_("Could not determine project for volume %s, " + "can't export") % + (volume['name'])) + if force_create: + raise exception.Error(msg) + else: + LOG.warn(msg) + should_export = False + + if should_export: + cliq_args = {} + cliq_args['volumeName'] = volume['name'] + cliq_args['chapName'] = chap_username + cliq_args['targetSecret'] = chap_password + + self._cliq_run_xml("assignVolumeChap", cliq_args) + + db_update['provider_auth'] = ("CHAP %s %s" % + (chap_username, chap_password)) + + return db_update + + def remove_export(self, context, volume): + """Removes an export for a logical volume.""" + cliq_args = {} + cliq_args['volumeName'] = volume['name'] + + self._cliq_run_xml("unassignVolume", cliq_args) -- cgit From fa29dc0433384d5aa47f5ac069a8dc650e23ccae Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Fri, 18 Feb 2011 15:48:49 -0600 Subject: moved creating vifs to its own function, moved inject network to its own function --- nova/virt/xenapi/vmops.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d1ef95c6c..76b88a8bd 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -135,6 +135,8 @@ class VMOps(object): timer.f = _wait_for_boot # call to reset network to inject network info and configure + networks = self.inject_network_info(instance) + self.create_vifs(instance, networks) self.reset_network(instance) return timer.start(interval=0.5, now=True) @@ -414,8 +416,9 @@ class VMOps(object): vm_opaque_ref) admin_context = context.get_admin_context() IPs = db.fixed_ip_get_all_by_instance(admin_context, instance['id']) - for network in db.network_get_all_by_instance(admin_context, - instance['id']): + networks = db.network_get_all_by_instance(admin_context, + instance['id']) + for network in networks: network_IPs = [ip for ip in IPs if ip.network_id == network.id] def ip_dict(ip): @@ -438,11 +441,23 @@ class VMOps(object): # catch KeyError for domid if instance isn't running pass - # TODO(tr3buchet) - remove comment in multi-nic - # this bit here about creating the vifs will be updated - # in multi-nic to handle multiple IPs on the same network - # and multiple networks - # for now it works as there is only one of each + return networks + + def create_vifs(self, instance, networks=None): + """ + Creates vifs for an instance + + """ + logging.debug(_("creating vif(s) for vm: |%s|"), vm_opaque_ref) + if networks is None: + networks = db.network_get_all_by_instance(admin_context, + instance['id']) + # TODO(tr3buchet) - remove comment in multi-nic + # this bit here about creating the vifs will be updated + # in multi-nic to handle multiple IPs on the same network + # and multiple networks + # for now it works as there is only one of each + for network in networks: bridge = network['bridge'] network_ref = \ NetworkHelper.find_network_with_bridge(self._session, bridge) @@ -456,7 +471,6 @@ class VMOps(object): Creates uuid arg to pass to make_agent_call and calls it. """ - self.inject_network_info(instance) args = {'id': str(uuid.uuid4())} resp = self._make_agent_call('resetnetwork', instance, '', args) -- cgit From e369f2842446876505ce528c5bb56a3d41215f8f Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Fri, 18 Feb 2011 16:42:26 -0600 Subject: added admin api call for injecting network info, added api test for inject network info --- nova/api/openstack/__init__.py | 1 + nova/api/openstack/servers.py | 14 ++++++++++++++ nova/compute/api.py | 7 +++++++ nova/compute/manager.py | 12 ++++++++++++ nova/tests/api/openstack/test_servers.py | 12 ++++++++++++ nova/virt/xenapi_conn.py | 4 ++++ 6 files changed, 50 insertions(+) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index dc3738d4a..cfa2da486 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -80,6 +80,7 @@ class APIRouter(wsgi.Router): server_members['suspend'] = 'POST' server_members['resume'] = 'POST' server_members['reset_network'] = 'POST' + server_members['inject_network_info'] = 'POST' mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 33cc3bbde..55fdb765b 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -263,6 +263,20 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + def inject_network_info(self, req, id): + """ + Inject network info for an instance (admin only). + + """ + context = req.environ['nova.context'] + try: + self.compute_api.inject_network_info(context, id) + except: + readable = traceback.format_exc() + LOG.exception(_("Compute.api::inject_network_info %s"), readable) + return faults.Fault(exc.HTTPUnprocessableEntity()) + return exc.HTTPAccepted() + def pause(self, req, id): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] diff --git a/nova/compute/api.py b/nova/compute/api.py index ed6f0e34a..3e5cd495e 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -473,6 +473,13 @@ class API(base.Base): """ self._cast_compute_message('reset_network', context, instance_id) + def inject_network_info(self, context, instance_id): + """ + Inject network info for the instance. + + """ + self._cast_compute_message('inject_network_info', context, instance_id) + def attach_volume(self, context, instance_id, volume_id, device): if not re.match("^/dev/[a-z]d[a-z]+$", device): raise exception.ApiError(_("Invalid device specified: %s. " diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6fab1a41c..ae3ab519f 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -510,6 +510,18 @@ class ComputeManager(manager.Manager): context=context) self.driver.reset_network(instance_ref) + @checks_instance_lock + def inject_network_info(self, context, instance_id): + """ + Inject network info for the instance. + + """ + context = context.elevated() + instance_ref = self.db.instance_get(context, instance_id) + LOG.debug(_('instance %s: inject network info'), instance_id, + context=context) + self.driver.inject_network_info(instance_ref) + @exception.wrap_exception def get_console_output(self, context, instance_id): """Send the console output for an instance.""" diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 89e192eed..58da12dcc 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -293,6 +293,18 @@ class ServersTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) + def test_server_inject_network_info(self): + FLAGS.allow_admin_api = True + body = dict(server=dict( + name='server_test', imageId=2, flavorId=2, metadata={}, + personality={})) + req = webob.Request.blank('/v1.0/servers/1/inject_network_info') + req.method = 'POST' + req.content_type = 'application/json' + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 202) + def test_server_diagnostics(self): req = webob.Request.blank("/v1.0/servers/1/diagnostics") req.method = "GET" diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 2720d175f..f72a12380 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -192,6 +192,10 @@ class XenAPIConnection(object): """reset networking for specified instance""" self._vmops.reset_network(instance) + def inject_network_info(self, instance): + """inject network info for specified instance""" + self._vmops.inject_network_info(instance) + def get_info(self, instance_id): """Return data about VM instance""" return self._vmops.get_info(instance_id) -- cgit From 8684eb3aa638883ea82bbaf8eb59076f1d7e6a05 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 17:17:51 -0800 Subject: ObjectStore doesn't use properties collection; kernel_id and ramdisk_id aren't required anyway --- nova/api/openstack/servers.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 486eca508..11a84687d 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -144,13 +144,11 @@ class Controller(wsgi.Controller): metadata stored in Glance as 'image_properties' """ def lookup(param): - _image_id = image_id - try: - return image['properties'][param] - except KeyError: - raise exception.NotFound( - _("%(param)s property not found for image %(_image_id)s") % - locals()) + properties = image.get('properties') + if properties: + return properties.get(param) + else: + return image.get(param) image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From aeab8eeb038ca1d1dde05705028144a78552c4f7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 17:27:25 -0800 Subject: Don't crash if there's no 'fixed_ip' attribute (was returning None, which was unsubscriptable) --- nova/api/openstack/servers.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 486eca508..b54e28c0c 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -63,20 +63,22 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - # grab single private fixed ip - try: - private_ip = inst['fixed_ip']['address'] - if private_ip: - inst_dict['addresses']['private'].append(private_ip) - except KeyError: - LOG.debug(_("Failed to read private ip")) - - # grab all public floating ips - try: - for floating in inst['fixed_ip']['floating_ips']: - inst_dict['addresses']['public'].append(floating['address']) - except KeyError: - LOG.debug(_("Failed to read public ip(s)")) + fixed_ip = inst['fixed_ip'] + if fixed_ip: + # grab single private fixed ip + try: + private_ip = fixed_ip['address'] + if private_ip: + inst_dict['addresses']['private'].append(private_ip) + except KeyError: + LOG.debug(_("Failed to read private ip")) + + # grab all public floating ips + try: + for floating in fixed_ip['floating_ips']: + inst_dict['addresses']['public'].append(floating['address']) + except KeyError: + LOG.debug(_("Failed to read public ip(s)")) inst_dict['metadata'] = {} inst_dict['hostId'] = '' -- cgit From a7eed42c57fe7eaf6f2981a88a74a81a6890198c Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 20 Feb 2011 20:56:14 +0100 Subject: puppet scripts only there as an example, should be moved to some other place if they are still necessary --- contrib/puppet/files/etc/default/nova-compute | 1 - contrib/puppet/files/etc/default/nova-volume | 1 - contrib/puppet/files/etc/issue | 5 - contrib/puppet/files/etc/libvirt/qemu.conf | 170 -------- contrib/puppet/files/etc/lvm/lvm.conf | 463 -------------------- contrib/puppet/files/etc/nova.conf | 28 -- contrib/puppet/files/production/boto.cfg | 3 - contrib/puppet/files/production/genvpn.sh | 35 -- .../files/production/libvirt.qemu.xml.template | 35 -- contrib/puppet/files/production/my.cnf | 137 ------ contrib/puppet/files/production/nova-iptables | 187 --------- contrib/puppet/files/production/nova-iscsi-dev.sh | 19 - contrib/puppet/files/production/setup_data.sh | 6 - contrib/puppet/files/production/slap.sh | 261 ------------ contrib/puppet/fileserver.conf | 8 - contrib/puppet/manifests/classes/apt.pp | 1 - contrib/puppet/manifests/classes/issue.pp | 14 - contrib/puppet/manifests/classes/kern_module.pp | 34 -- contrib/puppet/manifests/classes/loopback.pp | 6 - contrib/puppet/manifests/classes/lvm.pp | 8 - contrib/puppet/manifests/classes/lvmconf.pp | 8 - contrib/puppet/manifests/classes/nova.pp | 464 --------------------- contrib/puppet/manifests/classes/swift.pp | 7 - contrib/puppet/manifests/site.pp | 120 ------ contrib/puppet/manifests/templates.pp | 21 - contrib/puppet/puppet.conf | 11 - contrib/puppet/templates/haproxy.cfg.erb | 39 -- contrib/puppet/templates/monitrc-nova-api.erb | 138 ------ contrib/puppet/templates/nova-iptables.erb | 10 - .../templates/production/nova-common.conf.erb | 55 --- .../puppet/templates/production/nova-nova.conf.erb | 21 - 31 files changed, 2316 deletions(-) delete mode 100644 contrib/puppet/files/etc/default/nova-compute delete mode 100644 contrib/puppet/files/etc/default/nova-volume delete mode 100644 contrib/puppet/files/etc/issue delete mode 100644 contrib/puppet/files/etc/libvirt/qemu.conf delete mode 100644 contrib/puppet/files/etc/lvm/lvm.conf delete mode 100644 contrib/puppet/files/etc/nova.conf delete mode 100644 contrib/puppet/files/production/boto.cfg delete mode 100644 contrib/puppet/files/production/genvpn.sh delete mode 100644 contrib/puppet/files/production/libvirt.qemu.xml.template delete mode 100644 contrib/puppet/files/production/my.cnf delete mode 100755 contrib/puppet/files/production/nova-iptables delete mode 100644 contrib/puppet/files/production/nova-iscsi-dev.sh delete mode 100755 contrib/puppet/files/production/setup_data.sh delete mode 100755 contrib/puppet/files/production/slap.sh delete mode 100644 contrib/puppet/fileserver.conf delete mode 100644 contrib/puppet/manifests/classes/apt.pp delete mode 100644 contrib/puppet/manifests/classes/issue.pp delete mode 100644 contrib/puppet/manifests/classes/kern_module.pp delete mode 100644 contrib/puppet/manifests/classes/loopback.pp delete mode 100644 contrib/puppet/manifests/classes/lvm.pp delete mode 100644 contrib/puppet/manifests/classes/lvmconf.pp delete mode 100644 contrib/puppet/manifests/classes/nova.pp delete mode 100644 contrib/puppet/manifests/classes/swift.pp delete mode 100644 contrib/puppet/manifests/site.pp delete mode 100644 contrib/puppet/manifests/templates.pp delete mode 100644 contrib/puppet/puppet.conf delete mode 100644 contrib/puppet/templates/haproxy.cfg.erb delete mode 100644 contrib/puppet/templates/monitrc-nova-api.erb delete mode 100644 contrib/puppet/templates/nova-iptables.erb delete mode 100644 contrib/puppet/templates/production/nova-common.conf.erb delete mode 100644 contrib/puppet/templates/production/nova-nova.conf.erb diff --git a/contrib/puppet/files/etc/default/nova-compute b/contrib/puppet/files/etc/default/nova-compute deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-compute +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/default/nova-volume b/contrib/puppet/files/etc/default/nova-volume deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-volume +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/issue b/contrib/puppet/files/etc/issue deleted file mode 100644 index 8c567221b..000000000 --- a/contrib/puppet/files/etc/issue +++ /dev/null @@ -1,5 +0,0 @@ ------------------------------------------------ - - Welcome to your OpenStack installation! - ------------------------------------------------ diff --git a/contrib/puppet/files/etc/libvirt/qemu.conf b/contrib/puppet/files/etc/libvirt/qemu.conf deleted file mode 100644 index 7839f12e5..000000000 --- a/contrib/puppet/files/etc/libvirt/qemu.conf +++ /dev/null @@ -1,170 +0,0 @@ -# Master configuration file for the QEMU driver. -# All settings described here are optional - if omitted, sensible -# defaults are used. - -# VNC is configured to listen on 127.0.0.1 by default. -# To make it listen on all public interfaces, uncomment -# this next option. -# -# NB, strong recommendation to enable TLS + x509 certificate -# verification when allowing public access -# -# vnc_listen = "0.0.0.0" - - -# Enable use of TLS encryption on the VNC server. This requires -# a VNC client which supports the VeNCrypt protocol extension. -# Examples include vinagre, virt-viewer, virt-manager and vencrypt -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to setup CA and issue a server certificate -# before enabling this. -# -# vnc_tls = 1 - - -# Use of TLS requires that x509 certificates be issued. The -# default it to keep them in /etc/pki/libvirt-vnc. This directory -# must contain -# -# ca-cert.pem - the CA master certificate -# server-cert.pem - the server certificate signed with ca-cert.pem -# server-key.pem - the server private key -# -# This option allows the certificate directory to be changed -# -# vnc_tls_x509_cert_dir = "/etc/pki/libvirt-vnc" - - -# The default TLS configuration only uses certificates for the server -# allowing the client to verify the server's identity and establish -# and encrypted channel. -# -# It is possible to use x509 certificates for authentication too, by -# issuing a x509 certificate to every client who needs to connect. -# -# Enabling this option will reject any client who does not have a -# certificate signed by the CA in /etc/pki/libvirt-vnc/ca-cert.pem -# -# vnc_tls_x509_verify = 1 - - -# The default VNC password. Only 8 letters are significant for -# VNC passwords. This parameter is only used if the per-domain -# XML config does not already provide a password. To allow -# access without passwords, leave this commented out. An empty -# string will still enable passwords, but be rejected by QEMU -# effectively preventing any use of VNC. Obviously change this -# example here before you set this -# -# vnc_password = "XYZ12345" - - -# Enable use of SASL encryption on the VNC server. This requires -# a VNC client which supports the SASL protocol extension. -# Examples include vinagre, virt-viewer and virt-manager -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to configure /etc/sasl2/qemu.conf to choose -# the desired SASL plugin (eg, GSSPI for Kerberos) -# -# vnc_sasl = 1 - - -# The default SASL configuration file is located in /etc/sasl2/ -# When running libvirtd unprivileged, it may be desirable to -# override the configs in this location. Set this parameter to -# point to the directory, and create a qemu.conf in that location -# -# vnc_sasl_dir = "/some/directory/sasl2" - - - - -# The default security driver is SELinux. If SELinux is disabled -# on the host, then the security driver will automatically disable -# itself. If you wish to disable QEMU SELinux security driver while -# leaving SELinux enabled for the host in general, then set this -# to 'none' instead -# -# security_driver = "selinux" - - -# The user ID for QEMU processes run by the system instance -user = "root" - -# The group ID for QEMU processes run by the system instance -group = "root" - -# Whether libvirt should dynamically change file ownership -# to match the configured user/group above. Defaults to 1. -# Set to 0 to disable file ownership changes. -#dynamic_ownership = 1 - - -# What cgroup controllers to make use of with QEMU guests -# -# - 'cpu' - use for schedular tunables -# - 'devices' - use for device whitelisting -# -# NB, even if configured here, they won't be used unless -# the adminsitrator has mounted cgroups. eg -# -# mkdir /dev/cgroup -# mount -t cgroup -o devices,cpu none /dev/cgroup -# -# They can be mounted anywhere, and different controlers -# can be mounted in different locations. libvirt will detect -# where they are located. -# -# cgroup_controllers = [ "cpu", "devices" ] - -# This is the basic set of devices allowed / required by -# all virtual machines. -# -# As well as this, any configured block backed disks, -# all sound device, and all PTY devices are allowed. -# -# This will only need setting if newer QEMU suddenly -# wants some device we don't already know a bout. -# -#cgroup_device_acl = [ -# "/dev/null", "/dev/full", "/dev/zero", -# "/dev/random", "/dev/urandom", -# "/dev/ptmx", "/dev/kvm", "/dev/kqemu", -# "/dev/rtc", "/dev/hpet", "/dev/net/tun", -#] - -# The default format for Qemu/KVM guest save images is raw; that is, the -# memory from the domain is dumped out directly to a file. If you have -# guests with a large amount of memory, however, this can take up quite -# a bit of space. If you would like to compress the images while they -# are being saved to disk, you can also set "lzop", "gzip", "bzip2", or "xz" -# for save_image_format. Note that this means you slow down the process of -# saving a domain in order to save disk space; the list above is in descending -# order by performance and ascending order by compression ratio. -# -# save_image_format = "raw" - -# If provided by the host and a hugetlbfs mount point is configured, -# a guest may request huge page backing. When this mount point is -# unspecified here, determination of a host mount point in /proc/mounts -# will be attempted. Specifying an explicit mount overrides detection -# of the same in /proc/mounts. Setting the mount point to "" will -# disable guest hugepage backing. -# -# NB, within this mount point, guests will create memory backing files -# in a location of $MOUNTPOINT/libvirt/qemu - -# hugetlbfs_mount = "/dev/hugepages" - -# mac_filter enables MAC addressed based filtering on bridge ports. -# This currently requires ebtables to be installed. -# -# mac_filter = 1 - -# By default, PCI devices below non-ACS switch are not allowed to be assigned -# to guests. By setting relaxed_acs_check to 1 such devices will be allowed to -# be assigned to guests. -# -# relaxed_acs_check = 1 diff --git a/contrib/puppet/files/etc/lvm/lvm.conf b/contrib/puppet/files/etc/lvm/lvm.conf deleted file mode 100644 index 4e814ad49..000000000 --- a/contrib/puppet/files/etc/lvm/lvm.conf +++ /dev/null @@ -1,463 +0,0 @@ -# This is an example configuration file for the LVM2 system. -# It contains the default settings that would be used if there was no -# /etc/lvm/lvm.conf file. -# -# Refer to 'man lvm.conf' for further information including the file layout. -# -# To put this file in a different directory and override /etc/lvm set -# the environment variable LVM_SYSTEM_DIR before running the tools. - - -# This section allows you to configure which block devices should -# be used by the LVM system. -devices { - - # Where do you want your volume groups to appear ? - dir = "/dev" - - # An array of directories that contain the device nodes you wish - # to use with LVM2. - scan = [ "/dev" ] - - # If several entries in the scanned directories correspond to the - # same block device and the tools need to display a name for device, - # all the pathnames are matched against each item in the following - # list of regular expressions in turn and the first match is used. - preferred_names = [ ] - - # Try to avoid using undescriptive /dev/dm-N names, if present. - # preferred_names = [ "^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d" ] - - # A filter that tells LVM2 to only use a restricted set of devices. - # The filter consists of an array of regular expressions. These - # expressions can be delimited by a character of your choice, and - # prefixed with either an 'a' (for accept) or 'r' (for reject). - # The first expression found to match a device name determines if - # the device will be accepted or rejected (ignored). Devices that - # don't match any patterns are accepted. - - # Be careful if there there are symbolic links or multiple filesystem - # entries for the same device as each name is checked separately against - # the list of patterns. The effect is that if any name matches any 'a' - # pattern, the device is accepted; otherwise if any name matches any 'r' - # pattern it is rejected; otherwise it is accepted. - - # Don't have more than one filter line active at once: only one gets used. - - # Run vgscan after you change this parameter to ensure that - # the cache file gets regenerated (see below). - # If it doesn't do what you expect, check the output of 'vgscan -vvvv'. - - - # By default we accept every block device: - filter = [ "r|/dev/etherd/.*|", "r|/dev/block/.*|", "a/.*/" ] - - # Exclude the cdrom drive - # filter = [ "r|/dev/cdrom|" ] - - # When testing I like to work with just loopback devices: - # filter = [ "a/loop/", "r/.*/" ] - - # Or maybe all loops and ide drives except hdc: - # filter =[ "a|loop|", "r|/dev/hdc|", "a|/dev/ide|", "r|.*|" ] - - # Use anchors if you want to be really specific - # filter = [ "a|^/dev/hda8$|", "r/.*/" ] - - # The results of the filtering are cached on disk to avoid - # rescanning dud devices (which can take a very long time). - # By default this cache is stored in the /etc/lvm/cache directory - # in a file called '.cache'. - # It is safe to delete the contents: the tools regenerate it. - # (The old setting 'cache' is still respected if neither of - # these new ones is present.) - cache_dir = "/etc/lvm/cache" - cache_file_prefix = "" - - # You can turn off writing this cache file by setting this to 0. - write_cache_state = 1 - - # Advanced settings. - - # List of pairs of additional acceptable block device types found - # in /proc/devices with maximum (non-zero) number of partitions. - # types = [ "fd", 16 ] - - # If sysfs is mounted (2.6 kernels) restrict device scanning to - # the block devices it believes are valid. - # 1 enables; 0 disables. - sysfs_scan = 1 - - # By default, LVM2 will ignore devices used as components of - # software RAID (md) devices by looking for md superblocks. - # 1 enables; 0 disables. - md_component_detection = 1 - - # By default, if a PV is placed directly upon an md device, LVM2 - # will align its data blocks with the md device's stripe-width. - # 1 enables; 0 disables. - md_chunk_alignment = 1 - - # By default, the start of a PV's data area will be a multiple of - # the 'minimum_io_size' or 'optimal_io_size' exposed in sysfs. - # - minimum_io_size - the smallest request the device can perform - # w/o incurring a read-modify-write penalty (e.g. MD's chunk size) - # - optimal_io_size - the device's preferred unit of receiving I/O - # (e.g. MD's stripe width) - # minimum_io_size is used if optimal_io_size is undefined (0). - # If md_chunk_alignment is enabled, that detects the optimal_io_size. - # This setting takes precedence over md_chunk_alignment. - # 1 enables; 0 disables. - data_alignment_detection = 1 - - # Alignment (in KB) of start of data area when creating a new PV. - # If a PV is placed directly upon an md device and md_chunk_alignment or - # data_alignment_detection is enabled this parameter is ignored. - # Set to 0 for the default alignment of 64KB or page size, if larger. - data_alignment = 0 - - # By default, the start of the PV's aligned data area will be shifted by - # the 'alignment_offset' exposed in sysfs. This offset is often 0 but - # may be non-zero; e.g.: certain 4KB sector drives that compensate for - # windows partitioning will have an alignment_offset of 3584 bytes - # (sector 7 is the lowest aligned logical block, the 4KB sectors start - # at LBA -1, and consequently sector 63 is aligned on a 4KB boundary). - # 1 enables; 0 disables. - data_alignment_offset_detection = 1 - - # If, while scanning the system for PVs, LVM2 encounters a device-mapper - # device that has its I/O suspended, it waits for it to become accessible. - # Set this to 1 to skip such devices. This should only be needed - # in recovery situations. - ignore_suspended_devices = 0 -} - -# This section that allows you to configure the nature of the -# information that LVM2 reports. -log { - - # Controls the messages sent to stdout or stderr. - # There are three levels of verbosity, 3 being the most verbose. - verbose = 0 - - # Should we send log messages through syslog? - # 1 is yes; 0 is no. - syslog = 1 - - # Should we log error and debug messages to a file? - # By default there is no log file. - #file = "/var/log/lvm2.log" - - # Should we overwrite the log file each time the program is run? - # By default we append. - overwrite = 0 - - # What level of log messages should we send to the log file and/or syslog? - # There are 6 syslog-like log levels currently in use - 2 to 7 inclusive. - # 7 is the most verbose (LOG_DEBUG). - level = 0 - - # Format of output messages - # Whether or not (1 or 0) to indent messages according to their severity - indent = 1 - - # Whether or not (1 or 0) to display the command name on each line output - command_names = 0 - - # A prefix to use before the message text (but after the command name, - # if selected). Default is two spaces, so you can see/grep the severity - # of each message. - prefix = " " - - # To make the messages look similar to the original LVM tools use: - # indent = 0 - # command_names = 1 - # prefix = " -- " - - # Set this if you want log messages during activation. - # Don't use this in low memory situations (can deadlock). - # activation = 0 -} - -# Configuration of metadata backups and archiving. In LVM2 when we -# talk about a 'backup' we mean making a copy of the metadata for the -# *current* system. The 'archive' contains old metadata configurations. -# Backups are stored in a human readeable text format. -backup { - - # Should we maintain a backup of the current metadata configuration ? - # Use 1 for Yes; 0 for No. - # Think very hard before turning this off! - backup = 1 - - # Where shall we keep it ? - # Remember to back up this directory regularly! - backup_dir = "/etc/lvm/backup" - - # Should we maintain an archive of old metadata configurations. - # Use 1 for Yes; 0 for No. - # On by default. Think very hard before turning this off. - archive = 1 - - # Where should archived files go ? - # Remember to back up this directory regularly! - archive_dir = "/etc/lvm/archive" - - # What is the minimum number of archive files you wish to keep ? - retain_min = 10 - - # What is the minimum time you wish to keep an archive file for ? - retain_days = 30 -} - -# Settings for the running LVM2 in shell (readline) mode. -shell { - - # Number of lines of history to store in ~/.lvm_history - history_size = 100 -} - - -# Miscellaneous global LVM2 settings -global { - - # The file creation mask for any files and directories created. - # Interpreted as octal if the first digit is zero. - umask = 077 - - # Allow other users to read the files - #umask = 022 - - # Enabling test mode means that no changes to the on disk metadata - # will be made. Equivalent to having the -t option on every - # command. Defaults to off. - test = 0 - - # Default value for --units argument - units = "h" - - # Since version 2.02.54, the tools distinguish between powers of - # 1024 bytes (e.g. KiB, MiB, GiB) and powers of 1000 bytes (e.g. - # KB, MB, GB). - # If you have scripts that depend on the old behaviour, set this to 0 - # temporarily until you update them. - si_unit_consistency = 1 - - # Whether or not to communicate with the kernel device-mapper. - # Set to 0 if you want to use the tools to manipulate LVM metadata - # without activating any logical volumes. - # If the device-mapper kernel driver is not present in your kernel - # setting this to 0 should suppress the error messages. - activation = 1 - - # If we can't communicate with device-mapper, should we try running - # the LVM1 tools? - # This option only applies to 2.4 kernels and is provided to help you - # switch between device-mapper kernels and LVM1 kernels. - # The LVM1 tools need to be installed with .lvm1 suffices - # e.g. vgscan.lvm1 and they will stop working after you start using - # the new lvm2 on-disk metadata format. - # The default value is set when the tools are built. - # fallback_to_lvm1 = 0 - - # The default metadata format that commands should use - "lvm1" or "lvm2". - # The command line override is -M1 or -M2. - # Defaults to "lvm2". - # format = "lvm2" - - # Location of proc filesystem - proc = "/proc" - - # Type of locking to use. Defaults to local file-based locking (1). - # Turn locking off by setting to 0 (dangerous: risks metadata corruption - # if LVM2 commands get run concurrently). - # Type 2 uses the external shared library locking_library. - # Type 3 uses built-in clustered locking. - # Type 4 uses read-only locking which forbids any operations that might - # change metadata. - locking_type = 1 - - # Set to 0 to fail when a lock request cannot be satisfied immediately. - wait_for_locks = 1 - - # If using external locking (type 2) and initialisation fails, - # with this set to 1 an attempt will be made to use the built-in - # clustered locking. - # If you are using a customised locking_library you should set this to 0. - fallback_to_clustered_locking = 1 - - # If an attempt to initialise type 2 or type 3 locking failed, perhaps - # because cluster components such as clvmd are not running, with this set - # to 1 an attempt will be made to use local file-based locking (type 1). - # If this succeeds, only commands against local volume groups will proceed. - # Volume Groups marked as clustered will be ignored. - fallback_to_local_locking = 1 - - # Local non-LV directory that holds file-based locks while commands are - # in progress. A directory like /tmp that may get wiped on reboot is OK. - locking_dir = "/var/lock/lvm" - - # Whenever there are competing read-only and read-write access requests for - # a volume group's metadata, instead of always granting the read-only - # requests immediately, delay them to allow the read-write requests to be - # serviced. Without this setting, write access may be stalled by a high - # volume of read-only requests. - # NB. This option only affects locking_type = 1 viz. local file-based - # locking. - prioritise_write_locks = 1 - - # Other entries can go here to allow you to load shared libraries - # e.g. if support for LVM1 metadata was compiled as a shared library use - # format_libraries = "liblvm2format1.so" - # Full pathnames can be given. - - # Search this directory first for shared libraries. - # library_dir = "/lib/lvm2" - - # The external locking library to load if locking_type is set to 2. - # locking_library = "liblvm2clusterlock.so" -} - -activation { - # Set to 0 to disable udev syncronisation (if compiled into the binaries). - # Processes will not wait for notification from udev. - # They will continue irrespective of any possible udev processing - # in the background. You should only use this if udev is not running - # or has rules that ignore the devices LVM2 creates. - # The command line argument --nodevsync takes precedence over this setting. - # If set to 1 when udev is not running, and there are LVM2 processes - # waiting for udev, run 'dmsetup udevcomplete_all' manually to wake them up. - udev_sync = 1 - - # How to fill in missing stripes if activating an incomplete volume. - # Using "error" will make inaccessible parts of the device return - # I/O errors on access. You can instead use a device path, in which - # case, that device will be used to in place of missing stripes. - # But note that using anything other than "error" with mirrored - # or snapshotted volumes is likely to result in data corruption. - missing_stripe_filler = "error" - - # How much stack (in KB) to reserve for use while devices suspended - reserved_stack = 256 - - # How much memory (in KB) to reserve for use while devices suspended - reserved_memory = 8192 - - # Nice value used while devices suspended - process_priority = -18 - - # If volume_list is defined, each LV is only activated if there is a - # match against the list. - # "vgname" and "vgname/lvname" are matched exactly. - # "@tag" matches any tag set in the LV or VG. - # "@*" matches if any tag defined on the host is also set in the LV or VG - # - # volume_list = [ "vg1", "vg2/lvol1", "@tag1", "@*" ] - - # Size (in KB) of each copy operation when mirroring - mirror_region_size = 512 - - # Setting to use when there is no readahead value stored in the metadata. - # - # "none" - Disable readahead. - # "auto" - Use default value chosen by kernel. - readahead = "auto" - - # 'mirror_image_fault_policy' and 'mirror_log_fault_policy' define - # how a device failure affecting a mirror is handled. - # A mirror is composed of mirror images (copies) and a log. - # A disk log ensures that a mirror does not need to be re-synced - # (all copies made the same) every time a machine reboots or crashes. - # - # In the event of a failure, the specified policy will be used to determine - # what happens. This applies to automatic repairs (when the mirror is being - # monitored by dmeventd) and to manual lvconvert --repair when - # --use-policies is given. - # - # "remove" - Simply remove the faulty device and run without it. If - # the log device fails, the mirror would convert to using - # an in-memory log. This means the mirror will not - # remember its sync status across crashes/reboots and - # the entire mirror will be re-synced. If a - # mirror image fails, the mirror will convert to a - # non-mirrored device if there is only one remaining good - # copy. - # - # "allocate" - Remove the faulty device and try to allocate space on - # a new device to be a replacement for the failed device. - # Using this policy for the log is fast and maintains the - # ability to remember sync state through crashes/reboots. - # Using this policy for a mirror device is slow, as it - # requires the mirror to resynchronize the devices, but it - # will preserve the mirror characteristic of the device. - # This policy acts like "remove" if no suitable device and - # space can be allocated for the replacement. - # - # "allocate_anywhere" - Not yet implemented. Useful to place the log device - # temporarily on same physical volume as one of the mirror - # images. This policy is not recommended for mirror devices - # since it would break the redundant nature of the mirror. This - # policy acts like "remove" if no suitable device and space can - # be allocated for the replacement. - - mirror_log_fault_policy = "allocate" - mirror_device_fault_policy = "remove" -} - - -#################### -# Advanced section # -#################### - -# Metadata settings -# -# metadata { - # Default number of copies of metadata to hold on each PV. 0, 1 or 2. - # You might want to override it from the command line with 0 - # when running pvcreate on new PVs which are to be added to large VGs. - - # pvmetadatacopies = 1 - - # Approximate default size of on-disk metadata areas in sectors. - # You should increase this if you have large volume groups or - # you want to retain a large on-disk history of your metadata changes. - - # pvmetadatasize = 255 - - # List of directories holding live copies of text format metadata. - # These directories must not be on logical volumes! - # It's possible to use LVM2 with a couple of directories here, - # preferably on different (non-LV) filesystems, and with no other - # on-disk metadata (pvmetadatacopies = 0). Or this can be in - # addition to on-disk metadata areas. - # The feature was originally added to simplify testing and is not - # supported under low memory situations - the machine could lock up. - # - # Never edit any files in these directories by hand unless you - # you are absolutely sure you know what you are doing! Use - # the supplied toolset to make changes (e.g. vgcfgrestore). - - # dirs = [ "/etc/lvm/metadata", "/mnt/disk2/lvm/metadata2" ] -#} - -# Event daemon -# -dmeventd { - # mirror_library is the library used when monitoring a mirror device. - # - # "libdevmapper-event-lvm2mirror.so" attempts to recover from - # failures. It removes failed devices from a volume group and - # reconfigures a mirror as necessary. If no mirror library is - # provided, mirrors are not monitored through dmeventd. - - mirror_library = "libdevmapper-event-lvm2mirror.so" - - # snapshot_library is the library used when monitoring a snapshot device. - # - # "libdevmapper-event-lvm2snapshot.so" monitors the filling of - # snapshots and emits a warning through syslog, when the use of - # snapshot exceedes 80%. The warning is repeated when 85%, 90% and - # 95% of the snapshot are filled. - - snapshot_library = "libdevmapper-event-lvm2snapshot.so" -} diff --git a/contrib/puppet/files/etc/nova.conf b/contrib/puppet/files/etc/nova.conf deleted file mode 100644 index a0d64078c..000000000 --- a/contrib/puppet/files/etc/nova.conf +++ /dev/null @@ -1,28 +0,0 @@ ---ec2_url=http://192.168.255.1:8773/services/Cloud ---rabbit_host=192.168.255.1 ---redis_host=192.168.255.1 ---s3_host=192.168.255.1 ---vpn_ip=192.168.255.1 ---datastore_path=/var/lib/nova/keeper ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vlan_start=2000 ---vlan_end=3000 ---private_range=192.168.0.0/16 ---public_range=10.0.0.0/24 ---volume_group=vgdata ---storage_dev=/dev/sdc ---bridge_dev=eth2 ---aoe_eth_dev=eth2 ---public_interface=vlan0 ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---vpn_image_id=ami-cloudpipe ---daemonize ---verbose ---syslog ---prefix=nova diff --git a/contrib/puppet/files/production/boto.cfg b/contrib/puppet/files/production/boto.cfg deleted file mode 100644 index f4a2de2b6..000000000 --- a/contrib/puppet/files/production/boto.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[Boto] -debug = 0 -num_retries = 1 diff --git a/contrib/puppet/files/production/genvpn.sh b/contrib/puppet/files/production/genvpn.sh deleted file mode 100644 index 538c3cd33..000000000 --- a/contrib/puppet/files/production/genvpn.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This gets zipped and run on the cloudpipe-managed OpenVPN server -NAME=$1 -SUBJ=$2 - -mkdir -p projects/$NAME -cd projects/$NAME - -# generate a server priv key -openssl genrsa -out server.key 2048 - -# generate a server CSR -openssl req -new -key server.key -out server.csr -batch -subj "$SUBJ" - -if [ "`id -u`" != "`grep nova /etc/passwd | cut -d':' -f3`" ]; then - sudo chown -R nova:nogroup . -fi diff --git a/contrib/puppet/files/production/libvirt.qemu.xml.template b/contrib/puppet/files/production/libvirt.qemu.xml.template deleted file mode 100644 index 114dfdc01..000000000 --- a/contrib/puppet/files/production/libvirt.qemu.xml.template +++ /dev/null @@ -1,35 +0,0 @@ - - %(name)s - - hvm - %(basepath)s/kernel - %(basepath)s/ramdisk - root=/dev/vda1 console=ttyS0 - - - - - %(memory_kb)s - %(vcpus)s - - - - - - - - - - - - - - - - - diff --git a/contrib/puppet/files/production/my.cnf b/contrib/puppet/files/production/my.cnf deleted file mode 100644 index 8777bc480..000000000 --- a/contrib/puppet/files/production/my.cnf +++ /dev/null @@ -1,137 +0,0 @@ -# -# The MySQL database server configuration file. -# -# You can copy this to one of: -# - "/etc/mysql/my.cnf" to set global options, -# - "~/.my.cnf" to set user-specific options. -# -# One can use all long options that the program supports. -# Run program with --help to get a list of available options and with -# --print-defaults to see which it would actually understand and use. -# -# For explanations see -# http://dev.mysql.com/doc/mysql/en/server-system-variables.html - -# This will be passed to all mysql clients -# It has been reported that passwords should be enclosed with ticks/quotes -# escpecially if they contain "#" chars... -# Remember to edit /etc/mysql/debian.cnf when changing the socket location. -[client] -port = 3306 -socket = /var/run/mysqld/mysqld.sock - -# Here is entries for some specific programs -# The following values assume you have at least 32M ram - -# This was formally known as [safe_mysqld]. Both versions are currently parsed. -[mysqld_safe] -socket = /var/run/mysqld/mysqld.sock -nice = 0 - -[mysqld] -# -# * Basic Settings -# - -# -# * IMPORTANT -# If you make changes to these settings and your system uses apparmor, you may -# also need to also adjust /etc/apparmor.d/usr.sbin.mysqld. -# - -user = mysql -socket = /var/run/mysqld/mysqld.sock -port = 3306 -basedir = /usr -datadir = /var/lib/mysql -tmpdir = /tmp -skip-external-locking -# -# Instead of skip-networking the default is now to listen only on -# localhost which is more compatible and is not less secure. -# bind-address = 127.0.0.1 -# -# * Fine Tuning -# -innodb_buffer_pool_size = 12G -#innodb_log_file_size = 256M -innodb_log_buffer_size=4M -innodb_flush_log_at_trx_commit=2 -innodb_thread_concurrency=8 -innodb_flush_method=O_DIRECT -key_buffer = 128M -max_allowed_packet = 256M -thread_stack = 8196K -thread_cache_size = 32 -# This replaces the startup script and checks MyISAM tables if needed -# the first time they are touched -myisam-recover = BACKUP -max_connections = 1000 -table_cache = 1024 -#thread_concurrency = 10 -# -# * Query Cache Configuration -# -query_cache_limit = 32M -query_cache_size = 256M -# -# * Logging and Replication -# -# Both location gets rotated by the cronjob. -# Be aware that this log type is a performance killer. -# As of 5.1 you can enable the log at runtime! -#general_log_file = /var/log/mysql/mysql.log -#general_log = 1 - -log_error = /var/log/mysql/error.log - -# Here you can see queries with especially long duration -log_slow_queries = /var/log/mysql/mysql-slow.log -long_query_time = 2 -#log-queries-not-using-indexes -# -# The following can be used as easy to replay backup logs or for replication. -# note: if you are setting up a replication slave, see README.Debian about -# other settings you may need to change. -server-id = 1 -log_bin = /var/log/mysql/mysql-bin.log -expire_logs_days = 10 -max_binlog_size = 50M -#binlog_do_db = include_database_name -#binlog_ignore_db = include_database_name -# -# * InnoDB -# -sync_binlog=1 -# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. -# Read the manual for more InnoDB related options. There are many! -# -# * Security Features -# -# Read the manual, too, if you want chroot! -# chroot = /var/lib/mysql/ -# -# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". -# -# ssl-ca=/etc/mysql/cacert.pem -# ssl-cert=/etc/mysql/server-cert.pem -# ssl-key=/etc/mysql/server-key.pem - - - -[mysqldump] -quick -quote-names -max_allowed_packet = 256M - -[mysql] -#no-auto-rehash # faster start of mysql but no tab completition - -[isamchk] -key_buffer = 128M - -# -# * IMPORTANT: Additional settings that can override those from this file! -# The files must end with '.cnf', otherwise they'll be ignored. -# -!includedir /etc/mysql/conf.d/ diff --git a/contrib/puppet/files/production/nova-iptables b/contrib/puppet/files/production/nova-iptables deleted file mode 100755 index 61e2ca2b9..000000000 --- a/contrib/puppet/files/production/nova-iptables +++ /dev/null @@ -1,187 +0,0 @@ -#! /bin/sh - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# NOTE(vish): This script sets up some reasonable defaults for iptables and -# creates nova-specific chains. If you use this script you should -# run nova-network and nova-compute with --use_nova_chains=True - - -# NOTE(vish): If you run public nova-api on a different port, make sure to -# change the port here - -if [ -f /etc/default/nova-iptables ] ; then - . /etc/default/nova-iptables -fi - -export LC_ALL=C - -API_PORT=${API_PORT:-"8773"} - -if [ ! -n "$IP" ]; then - # NOTE(vish): IP address is what address the services ALLOW on. - # This will just get the first ip in the list, so if you - # have more than one eth device set up, this will fail, and - # you should explicitly pass in the ip of the instance - IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` -fi - -if [ ! -n "$PRIVATE_RANGE" ]; then - #NOTE(vish): PRIVATE_RANGE: range is ALLOW to access DHCP - PRIVATE_RANGE="192.168.0.0/12" -fi - -if [ ! -n "$MGMT_IP" ]; then - # NOTE(vish): Management IP is the ip over which to allow ssh traffic. It - # will also allow traffic to nova-api - MGMT_IP="$IP" -fi - -if [ ! -n "$DMZ_IP" ]; then - # NOTE(vish): DMZ IP is the ip over which to allow api & objectstore access - DMZ_IP="$IP" -fi - -clear_nova_iptables() { - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - iptables -F - iptables -t nat -F - iptables -F services - iptables -X services - # HACK: re-adding fail2ban rules :( - iptables -N fail2ban-ssh - iptables -A INPUT -p tcp -m multiport --dports 22 -j fail2ban-ssh - iptables -A fail2ban-ssh -j RETURN -} - -load_nova_iptables() { - - iptables -P INPUT DROP - iptables -A INPUT -m state --state INVALID -j DROP - iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - # NOTE(ja): allow localhost for everything - iptables -A INPUT -d 127.0.0.1/32 -j ACCEPT - # NOTE(ja): 22 only allowed MGMT_IP before, but we widened it to any - # address, since ssh should be listening only on internal - # before we re-add this rule we will need to add - # flexibility for RSYNC between omega/stingray - iptables -A INPUT -m tcp -p tcp --dport 22 -j ACCEPT - iptables -A INPUT -m udp -p udp --dport 123 -j ACCEPT - iptables -A INPUT -p icmp -j ACCEPT - iptables -N services - iptables -A INPUT -j services - iptables -A INPUT -p tcp -j REJECT --reject-with tcp-reset - iptables -A INPUT -j REJECT --reject-with icmp-port-unreachable - - iptables -P FORWARD DROP - iptables -A FORWARD -m state --state INVALID -j DROP - iptables -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT - iptables -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu - - # NOTE(vish): DROP on output is too restrictive for now. We need to add - # in a bunch of more specific output rules to use it. - # iptables -P OUTPUT DROP - iptables -A OUTPUT -m state --state INVALID -j DROP - iptables -A OUTPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - - if [ -n "$GANGLIA" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 8649 -j ACCEPT - iptables -A services -m udp -p udp -d $IP --dport 8649 -j ACCEPT - fi - - # if [ -n "$WEB" ] || [ -n "$ALL" ]; then - # # NOTE(vish): This opens up ports for web access, allowing web-based - # # dashboards to work. - # iptables -A services -m tcp -p tcp -d $IP --dport 80 -j ACCEPT - # iptables -A services -m tcp -p tcp -d $IP --dport 443 -j ACCEPT - # fi - - if [ -n "$OBJECTSTORE" ] || [ -n "$ALL" ]; then - # infrastructure - iptables -A services -m tcp -p tcp -d $IP --dport 3333 -j ACCEPT - # clients - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport 3333 -j ACCEPT - fi - - if [ -n "$API" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport $API_PORT -j ACCEPT - if [ "$IP" != "$DMZ_IP" ]; then - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport $API_PORT -j ACCEPT - fi - if [ "$IP" != "$MGMT_IP" ] && [ "$DMZ_IP" != "$MGMT_IP" ]; then - iptables -A services -m tcp -p tcp -d $MGMT_IP --dport $API_PORT -j ACCEPT - fi - fi - - if [ -n "$REDIS" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 6379 -j ACCEPT - fi - - if [ -n "$MYSQL" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3306 -j ACCEPT - fi - - if [ -n "$RABBITMQ" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 4369 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 5672 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 53284 -j ACCEPT - fi - - if [ -n "$DNSMASQ" ] || [ -n "$ALL" ]; then - # NOTE(vish): this could theoretically be setup per network - # for each host, but it seems like overkill - iptables -A services -m tcp -p tcp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp --dport 67 -j ACCEPT - fi - - if [ -n "$LDAP" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 389 -j ACCEPT - fi - - if [ -n "$ISCSI" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3260 -j ACCEPT - iptables -A services -m tcp -p tcp -d 127.0.0.0/16 --dport 3260 -j ACCEPT - fi -} - - -case "$1" in - start) - echo "Starting nova-iptables: " - load_nova_iptables - ;; - stop) - echo "Clearing nova-iptables: " - clear_nova_iptables - ;; - restart) - echo "Restarting nova-iptables: " - clear_nova_iptables - load_nova_iptables - ;; - *) - echo "Usage: $NAME {start|stop|restart}" >&2 - exit 1 - ;; -esac - -exit 0 diff --git a/contrib/puppet/files/production/nova-iscsi-dev.sh b/contrib/puppet/files/production/nova-iscsi-dev.sh deleted file mode 100644 index 8eda10d2e..000000000 --- a/contrib/puppet/files/production/nova-iscsi-dev.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# FILE: /etc/udev/scripts/iscsidev.sh - -BUS=${1} -HOST=${BUS%%:*} - -[ -e /sys/class/iscsi_host ] || exit 1 - -file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" - -target_name=$(cat ${file}) - -# This is not an open-scsi drive -if [ -z "${target_name}" ]; then - exit 1 -fi - -echo "${target_name##*:}" diff --git a/contrib/puppet/files/production/setup_data.sh b/contrib/puppet/files/production/setup_data.sh deleted file mode 100755 index 1fbbac41c..000000000 --- a/contrib/puppet/files/production/setup_data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -/root/slap.sh -mysql -e "DROP DATABASE nova" -mysql -e "CREATE DATABASE nova" -mysql -e "GRANT ALL on nova.* to nova@'%' identified by 'TODO:CHANGEME:CMON'" -touch /root/installed diff --git a/contrib/puppet/files/production/slap.sh b/contrib/puppet/files/production/slap.sh deleted file mode 100755 index f8ea16949..000000000 --- a/contrib/puppet/files/production/slap.sh +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# LDAP INSTALL SCRIPT - SHOULD BE IDEMPOTENT, but it SCRUBS all USERS - -apt-get install -y slapd ldap-utils python-ldap - -cat >/etc/ldap/schema/openssh-lpk_openldap.schema < -# -# Based on the proposal of : Mark Ruijter -# - - -# octetString SYNTAX -attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' - DESC 'MANDATORY: OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -# printableString SYNTAX yes|no -objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY - DESC 'MANDATORY: OpenSSH LPK objectclass' - MAY ( sshPublicKey $ uid ) - ) -LPK_SCHEMA_EOF - -cat >/etc/ldap/schema/nova.schema < -# -# - -# using internet experimental oid arc as per BP64 3.1 -objectidentifier novaSchema 1.3.6.1.3.1.666.666 -objectidentifier novaAttrs novaSchema:3 -objectidentifier novaOCs novaSchema:4 - -attributetype ( - novaAttrs:1 - NAME 'accessKey' - DESC 'Key for accessing data' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:2 - NAME 'secretKey' - DESC 'Secret key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - -objectClass ( - novaOCs:1 - NAME 'novaUser' - DESC 'access and secret keys' - AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) - ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) - ) - -NOVA_SCHEMA_EOF - -mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig -cat >/etc/ldap/slapd.conf </etc/ldap/ldap.conf </etc/ldap/base.ldif < "/usr/bin/apt-get update" } diff --git a/contrib/puppet/manifests/classes/issue.pp b/contrib/puppet/manifests/classes/issue.pp deleted file mode 100644 index 8bb37ee3f..000000000 --- a/contrib/puppet/manifests/classes/issue.pp +++ /dev/null @@ -1,14 +0,0 @@ -class issue { - file { "/etc/issue": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } - file { "/etc/issue.net": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } -} diff --git a/contrib/puppet/manifests/classes/kern_module.pp b/contrib/puppet/manifests/classes/kern_module.pp deleted file mode 100644 index 00ec0636c..000000000 --- a/contrib/puppet/manifests/classes/kern_module.pp +++ /dev/null @@ -1,34 +0,0 @@ -# via http://projects.puppetlabs.com/projects/puppet/wiki/Kernel_Modules_Patterns - -define kern_module ($ensure) { - $modulesfile = $operatingsystem ? { ubuntu => "/etc/modules", redhat => "/etc/rc.modules" } - case $operatingsystem { - redhat: { file { "/etc/rc.modules": ensure => file, mode => 755 } } - } - case $ensure { - present: { - exec { "insert_module_${name}": - command => $operatingsystem ? { - ubuntu => "/bin/echo '${name}' >> '${modulesfile}'", - redhat => "/bin/echo '/sbin/modprobe ${name}' >> '${modulesfile}' " - }, - unless => "/bin/grep -qFx '${name}' '${modulesfile}'" - } - exec { "/sbin/modprobe ${name}": unless => "/bin/grep -q '^${name} ' '/proc/modules'" } - } - absent: { - exec { "/sbin/modprobe -r ${name}": onlyif => "/bin/grep -q '^${name} ' '/proc/modules'" } - exec { "remove_module_${name}": - command => $operatingsystem ? { - ubuntu => "/usr/bin/perl -ni -e 'print unless /^\\Q${name}\\E\$/' '${modulesfile}'", - redhat => "/usr/bin/perl -ni -e 'print unless /^\\Q/sbin/modprobe ${name}\\E\$/' '${modulesfile}'" - }, - onlyif => $operatingsystem ? { - ubuntu => "/bin/grep -qFx '${name}' '${modulesfile}'", - redhat => "/bin/grep -q '^/sbin/modprobe ${name}' '${modulesfile}'" - } - } - } - default: { err ( "unknown ensure value ${ensure}" ) } - } -} diff --git a/contrib/puppet/manifests/classes/loopback.pp b/contrib/puppet/manifests/classes/loopback.pp deleted file mode 100644 index e0fa9d541..000000000 --- a/contrib/puppet/manifests/classes/loopback.pp +++ /dev/null @@ -1,6 +0,0 @@ -define loopback($num) { - exec { "mknod -m 0660 /dev/loop${num} b 7 ${num}; chown root:disk /dev/loop${num}": - creates => "/dev/loop${num}", - path => ["/usr/bin", "/usr/sbin", "/bin"] - } -} diff --git a/contrib/puppet/manifests/classes/lvm.pp b/contrib/puppet/manifests/classes/lvm.pp deleted file mode 100644 index 5a407abcb..000000000 --- a/contrib/puppet/manifests/classes/lvm.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvm { - file { "/etc/lvm/lvm.conf": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/lvm.conf", - } -} diff --git a/contrib/puppet/manifests/classes/lvmconf.pp b/contrib/puppet/manifests/classes/lvmconf.pp deleted file mode 100644 index 4aa7ddfdc..000000000 --- a/contrib/puppet/manifests/classes/lvmconf.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvmconf { - file { "/etc/lvm/lvm.conf": - owner => "root", group => "root", mode => 644, - source => "puppet://${puppet_server}/files/etc/lvm/lvm.conf", - ensure => present - } -} - diff --git a/contrib/puppet/manifests/classes/nova.pp b/contrib/puppet/manifests/classes/nova.pp deleted file mode 100644 index e942860f4..000000000 --- a/contrib/puppet/manifests/classes/nova.pp +++ /dev/null @@ -1,464 +0,0 @@ -import "kern_module" -import "apt" -import "loopback" - -#$head_node_ip = "undef" -#$rabbit_ip = "undef" -#$vpn_ip = "undef" -#$public_interface = "undef" -#$vlan_start = "5000" -#$vlan_end = "6000" -#$private_range = "10.0.0.0/16" -#$public_range = "192.168.177.0/24" - -define nova_iptables($services, $ip="", $private_range="", $mgmt_ip="", $dmz_ip="") { - file { "/etc/init.d/nova-iptables": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iptables", - } - - file { "/etc/default/nova-iptables": - owner => "root", mode => 644, - content => template("nova-iptables.erb") - } -} - -define nova_conf_pointer($name) { - file { "/etc/nova/nova-${name}.conf": - owner => "nova", mode => 400, - content => "--flagfile=/etc/nova/nova.conf" - } -} - -class novaconf { - file { "/etc/nova/nova.conf": - owner => "nova", mode => 400, - content => template("production/nova-common.conf.erb", "production/nova-${cluster_name}.conf.erb") - } - nova_conf_pointer{'manage': name => 'manage'} -} - -class novadata { - package { "rabbitmq-server": ensure => present } - - file { "/etc/rabbitmq/rabbitmq.conf": - owner => "root", mode => 644, - content => "NODENAME=rabbit@localhost", - } - - service { "rabbitmq-server": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/rabbitmq/rabbitmq.conf"], - Package["rabbitmq-server"] - ] - } - - package { "mysql-server": ensure => present } - - file { "/etc/mysql/my.cnf": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/my.cnf", - } - - service { "mysql": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/mysql/my.cnf"], - Package["mysql-server"] - ] - } - - file { "/root/slap.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/slap.sh", - } - - file { "/root/setup_data.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/setup_data.sh", - } - - # setup compute data - exec { "setup_data": - command => "/root/setup_data.sh", - path => "/usr/bin:/bin", - unless => "test -f /root/installed", - require => [ - Service["mysql"], - File["/root/slap.sh"], - File["/root/setup_data.sh"] - ] - } -} - -define nscheduler($version) { - package { "nova-scheduler": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'scheduler': name => 'scheduler'} - exec { "update-rc.d -f nova-scheduler remove; update-rc.d nova-scheduler defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-scheduler", - unless => "test -f /etc/rc2.d/S50nova-scheduler" - } - service { "nova-scheduler": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-scheduler"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-scheduler.conf"] - ] - } - -} - -define napi($version, $api_servers, $api_base_port) { - file { "/etc/boto.cfg": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/boto.cfg", - } - - file { "/var/lib/nova/CA/genvpn.sh": - owner => "nova", mode => 755, - source => "puppet://${puppet_server}/files/production/genvpn.sh", - } - - package { "python-greenlet": ensure => present } - package { "nova-api": ensure => $version, require => [Exec["update-apt"], Package["python-greenlet"]] } - nova_conf_pointer{'api': name => 'api'} - - exec { "update-rc.d -f nova-api remove; update-rc.d nova-api defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-api", - unless => "test -f /etc/rc2.d/S50nova-api" - } - - service { "nova-netsync": - start => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock start", - stop => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock stop", - ensure => running, - hasstatus => false, - pattern => "nova-netsync", - require => Service["nova-api"], - subscribe => File["/etc/nova/nova.conf"] - } - service { "nova-api": - start => "monit start all -g nova_api", - stop => "monit stop all -g nova_api", - restart => "monit restart all -g nova_api", - # ensure => running, - # hasstatus => true, - require => Service["monit"], - subscribe => [ - Package["nova-objectstore"], - File["/etc/boto.cfg"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } - - # the haproxy & monit's template use $api_servers and $api_base_port - - package { "haproxy": ensure => present } - file { "/etc/default/haproxy": - owner => "root", mode => 644, - content => "ENABLED=1", - require => Package['haproxy'] - } - file { "/etc/haproxy/haproxy.cfg": - owner => "root", mode => 644, - content => template("/srv/cloud/puppet/templates/haproxy.cfg.erb"), - require => Package['haproxy'] - } - service { "haproxy": - ensure => true, - enable => true, - hasstatus => true, - subscribe => [ - Package["haproxy"], - File["/etc/default/haproxy"], - File["/etc/haproxy/haproxy.cfg"], - ] - } - - package { "socat": ensure => present } - - file { "/usr/local/bin/gmetric_haproxy.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/ganglia/gmetric_scripts/gmetric_haproxy.sh", - } - - cron { "gmetric_haproxy": - command => "/usr/local/bin/gmetric_haproxy.sh", - user => root, - minute => "*/3", - } - - package { "monit": ensure => present } - - file { "/etc/default/monit": - owner => "root", mode => 644, - content => "startup=1", - require => Package['monit'] - } - file { "/etc/monit/monitrc": - owner => "root", mode => 600, - content => template("/srv/cloud/puppet/templates/monitrc-nova-api.erb"), - require => Package['monit'] - } - service { "monit": - ensure => true, - pattern => "sbin/monit", - subscribe => [ - Package["monit"], - File["/etc/default/monit"], - File["/etc/monit/monitrc"], - ] - } - -} - - -define nnetwork($version) { - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - # EVIL HACK: custom binary because dnsmasq 2.52 segfaulted accessing dereferenced object - file { "/usr/sbin/dnsmasq": - owner => "root", group => "root", - source => "puppet://${puppet_server}/files/production/dnsmasq", - } - - package { "nova-network": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'dhcpbridge': name => 'dhcpbridge'} - nova_conf_pointer{'network': name => "network" } - - exec { "update-rc.d -f nova-network remove; update-rc.d nova-network defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-network", - unless => "test -f /etc/rc2.d/S50nova-network" - } - service { "nova-network": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-network"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-network.conf"] - ] - } -} - -define nobjectstore($version) { - package { "nova-objectstore": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'objectstore': name => 'objectstore'} - exec { "update-rc.d -f nova-objectstore remove; update-rc.d nova-objectstore defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-objectstore", - unless => "test -f /etc/rc2.d/S50nova-objectstore" - } - service { "nova-objectstore": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-objectstore"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } -} - -define ncompute($version) { - include ganglia-python - include ganglia-compute - - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - - # LIBVIRT has to be restarted when ebtables / gawk is installed - service { "libvirt-bin": - ensure => running, - pattern => "sbin/libvirtd", - subscribe => [ - Package["ebtables"], - Kern_module["kvm_intel"] - ], - require => [ - Package["libvirt-bin"], - Package["ebtables"], - Package["gawk"], - Kern_module["kvm_intel"], - File["/dev/kvm"] - ] - } - - package { "libvirt-bin": ensure => "0.8.3-1ubuntu14~ppalucid2" } - package { "ebtables": ensure => present } - package { "gawk": ensure => present } - - # ensure proper permissions on /dev/kvm - file { "/dev/kvm": - owner => "root", - group => "kvm", - mode => 660 - } - - # require hardware virt - kern_module { "kvm_intel": - ensure => present, - } - - # increase loopback devices - file { "/etc/modprobe.d/loop.conf": - owner => "root", mode => 644, - content => "options loop max_loop=40" - } - - nova_conf_pointer{'compute': name => 'compute'} - - loopback{loop0: num => 0} - loopback{loop1: num => 1} - loopback{loop2: num => 2} - loopback{loop3: num => 3} - loopback{loop4: num => 4} - loopback{loop5: num => 5} - loopback{loop6: num => 6} - loopback{loop7: num => 7} - loopback{loop8: num => 8} - loopback{loop9: num => 9} - loopback{loop10: num => 10} - loopback{loop11: num => 11} - loopback{loop12: num => 12} - loopback{loop13: num => 13} - loopback{loop14: num => 14} - loopback{loop15: num => 15} - loopback{loop16: num => 16} - loopback{loop17: num => 17} - loopback{loop18: num => 18} - loopback{loop19: num => 19} - loopback{loop20: num => 20} - loopback{loop21: num => 21} - loopback{loop22: num => 22} - loopback{loop23: num => 23} - loopback{loop24: num => 24} - loopback{loop25: num => 25} - loopback{loop26: num => 26} - loopback{loop27: num => 27} - loopback{loop28: num => 28} - loopback{loop29: num => 29} - loopback{loop30: num => 30} - loopback{loop31: num => 31} - loopback{loop32: num => 32} - loopback{loop33: num => 33} - loopback{loop34: num => 34} - loopback{loop35: num => 35} - loopback{loop36: num => 36} - loopback{loop37: num => 37} - loopback{loop38: num => 38} - loopback{loop39: num => 39} - - package { "python-libvirt": ensure => "0.8.3-1ubuntu14~ppalucid2" } - - package { "nova-compute": - ensure => "$version", - require => Package["python-libvirt"] - } - - #file { "/usr/share/nova/libvirt.qemu.xml.template": - # owner => "nova", mode => 400, - # source => "puppet://${puppet_server}/files/production/libvirt.qemu.xml.template", - #} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-compute remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-compute" - } - service { "nova-compute": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-compute"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-compute.conf"], - #File["/usr/share/nova/libvirt.qemu.xml.template"], - Service["libvirt-bin"], - Kern_module["kvm_intel"] - ] - } -} - -define nvolume($version) { - - package { "nova-volume": ensure => $version, require => Exec["update-apt"] } - - nova_conf_pointer{'volume': name => 'volume'} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-volume remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-volume" - } - - file { "/etc/default/iscsitarget": - owner => "root", mode => 644, - content => "ISCSITARGET_ENABLE=true" - } - - package { "iscsitarget": ensure => present } - - file { "/dev/iscsi": ensure => directory } # FIXME(vish): owner / mode? - file { "/usr/sbin/nova-iscsi-dev.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iscsi-dev.sh" - } - file { "/etc/udev/rules.d/55-openiscsi.rules": - owner => "root", mode => 644, - content => 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/usr/sbin/nova-iscsi-dev.sh %b",SYMLINK+="iscsi/%c%n"' - } - - service { "iscsitarget": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/default/iscsitarget"], - Package["iscsitarget"] - ] - } - - service { "nova-volume": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-volume"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-volume.conf"] - ] - } -} - -class novaspool { - # This isn't in release yet - #cron { logspool: - # command => "/usr/bin/nova-logspool /var/log/nova.log /var/lib/nova/spool", - # user => "nova" - #} - #cron { spoolsentry: - # command => "/usr/bin/nova-spoolsentry ${sentry_url} ${sentry_key} /var/lib/nova/spool", - # user => "nova" - #} -} diff --git a/contrib/puppet/manifests/classes/swift.pp b/contrib/puppet/manifests/classes/swift.pp deleted file mode 100644 index 64ffb6fa3..000000000 --- a/contrib/puppet/manifests/classes/swift.pp +++ /dev/null @@ -1,7 +0,0 @@ -class swift { - package { "memcached": ensure => present } - service { "memcached": require => Package['memcached'] } - - package { "swift-proxy": ensure => present } -} - diff --git a/contrib/puppet/manifests/site.pp b/contrib/puppet/manifests/site.pp deleted file mode 100644 index ca07a34ad..000000000 --- a/contrib/puppet/manifests/site.pp +++ /dev/null @@ -1,120 +0,0 @@ -# site.pp - -import "templates" -import "classes/*" - -node novabase inherits default { -# $puppet_server = "192.168.0.10" - $cluster_name = "openstack001" - $ganglia_udp_send_channel = "openstack001.example.com" - $syslog = "192.168.0.10" - - # THIS STUFF ISN'T IN RELEASE YET - #$sentry_url = "http://192.168.0.19/sentry/store/" - #$sentry_key = "TODO:SENTRYPASS" - - $local_network = "192.168.0.0/16" - $vpn_ip = "192.168.0.2" - $public_interface = "eth0" - include novanode -# include nova-common - include opsmetrics - -# non-nova stuff such as nova-dash inherit from novanode -# novaspool needs a better home -# include novaspool -} - -# Builder -node "nova000.example.com" inherits novabase { - $syslog = "server" - include ntp - include syslog-server -} - -# Non-Nova nodes - -node - "blog.example.com", - "wiki.example.com" -inherits novabase { - include ganglia-python - include ganglia-apache - include ganglia-mysql -} - - -node "nova001.example.com" -inherits novabase { - include novabase - - nova_iptables { nova: - services => [ - "ganglia", - "mysql", - "rabbitmq", - "ldap", - "api", - "objectstore", - "nrpe", - ], - ip => "192.168.0.10", - } - - nobjectstore { nova: version => "0.9.0" } - nscheduler { nova: version => "0.9.0" } - napi { nova: - version => "0.9.0", - api_servers => 10, - api_base_port => 8000 - } -} - -node "nova002.example.com" -inherits novabase { - include novaconf - - nova_iptables { nova: - services => [ - "ganglia", - "dnsmasq", - "nrpe" - ], - ip => "192.168.4.2", - private_range => "192.168.0.0/16", - } - - nnetwork { nova: version => "0.9.0" } -} - -node - "nova003.example.com", - "nova004.example.com", - "nova005.example.com", - "nova006.example.com", - "nova007.example.com", - "nova008.example.com", - "nova009.example.com", - "nova010.example.com", - "nova011.example.com", - "nova012.example.com", - "nova013.example.com", - "nova014.example.com", - "nova015.example.com", - "nova016.example.com", - "nova017.example.com", - "nova018.example.com", - "nova019.example.com", -inherits novabase { - include novaconf - ncompute { nova: version => "0.9.0" } - nvolume { nova: version => "0.9.0" } -} - -#node -# "nova020.example.com" -# "nova021.example.com" -#inherits novanode { -# include novaconf - #ncompute { nova: version => "0.9.0" } -#} diff --git a/contrib/puppet/manifests/templates.pp b/contrib/puppet/manifests/templates.pp deleted file mode 100644 index 90e433013..000000000 --- a/contrib/puppet/manifests/templates.pp +++ /dev/null @@ -1,21 +0,0 @@ -# templates.pp - -import "classes/*" - -class baseclass { -# include dns-client # FIXME: missing resolv.conf.erb?? - include issue -} - -node default { - $nova_site = "undef" - $nova_ns1 = "undef" - $nova_ns2 = "undef" -# include baseclass -} - -# novanode handles the system-level requirements for Nova/Swift nodes -class novanode { - include baseclass - include lvmconf -} diff --git a/contrib/puppet/puppet.conf b/contrib/puppet/puppet.conf deleted file mode 100644 index 92af920e3..000000000 --- a/contrib/puppet/puppet.conf +++ /dev/null @@ -1,11 +0,0 @@ -[main] -logdir=/var/log/puppet -vardir=/var/lib/puppet -ssldir=/var/lib/puppet/ssl -rundir=/var/run/puppet -factpath=$vardir/lib/facter -pluginsync=false - -[puppetmasterd] -templatedir=/var/lib/nova/contrib/puppet/templates -autosign=true diff --git a/contrib/puppet/templates/haproxy.cfg.erb b/contrib/puppet/templates/haproxy.cfg.erb deleted file mode 100644 index bd9991de7..000000000 --- a/contrib/puppet/templates/haproxy.cfg.erb +++ /dev/null @@ -1,39 +0,0 @@ -# this config needs haproxy-1.1.28 or haproxy-1.2.1 - -global - log 127.0.0.1 local0 - log 127.0.0.1 local1 notice - #log loghost local0 info - maxconn 4096 - #chroot /usr/share/haproxy - stats socket /var/run/haproxy.sock - user haproxy - group haproxy - daemon - #debug - #quiet - -defaults - log global - mode http - option httplog - option dontlognull - retries 3 - option redispatch - stats enable - stats uri /haproxy - maxconn 2000 - contimeout 5000 - clitimeout 50000 - srvtimeout 50000 - - -listen nova-api 0.0.0.0:8773 - option httpchk GET / HTTP/1.0\r\nHost:\ example.com - option forwardfor - reqidel ^X-Forwarded-For:.* - balance roundrobin -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset -%> - server api_<%= port %> 127.0.0.1:<%= port %> maxconn 1 check -<% end -%> - option httpclose # disable keep-alive diff --git a/contrib/puppet/templates/monitrc-nova-api.erb b/contrib/puppet/templates/monitrc-nova-api.erb deleted file mode 100644 index fe2626327..000000000 --- a/contrib/puppet/templates/monitrc-nova-api.erb +++ /dev/null @@ -1,138 +0,0 @@ -############################################################################### -## Monit control file -############################################################################### -## -## Comments begin with a '#' and extend through the end of the line. Keywords -## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. -## -## Below you will find examples of some frequently used statements. For -## information about the control file, a complete list of statements and -## options please have a look in the monit manual. -## -## -############################################################################### -## Global section -############################################################################### -## -## Start monit in the background (run as a daemon): -# -set daemon 60 # check services at 1-minute intervals - with start delay 30 # optional: delay the first check by half a minute - # (by default check immediately after monit start) - - -## Set syslog logging with the 'daemon' facility. If the FACILITY option is -## omitted, monit will use 'user' facility by default. If you want to log to -## a stand alone log file instead, specify the path to a log file -# -set logfile syslog facility log_daemon -# -# -### Set the location of monit id file which saves the unique id specific for -### given monit. The id is generated and stored on first monit start. -### By default the file is placed in $HOME/.monit.id. -# -# set idfile /var/.monit.id -# -### Set the location of monit state file which saves the monitoring state -### on each cycle. By default the file is placed in $HOME/.monit.state. If -### state file is stored on persistent filesystem, monit will recover the -### monitoring state across reboots. If it is on temporary filesystem, the -### state will be lost on reboot. -# -# set statefile /var/.monit.state -# -## Set the list of mail servers for alert delivery. Multiple servers may be -## specified using comma separator. By default monit uses port 25 - this -## is possible to override with the PORT option. -# -# set mailserver mail.bar.baz, # primary mailserver -# backup.bar.baz port 10025, # backup mailserver on port 10025 -# localhost # fallback relay -# -# -## By default monit will drop alert events if no mail servers are available. -## If you want to keep the alerts for a later delivery retry, you can use the -## EVENTQUEUE statement. The base directory where undelivered alerts will be -## stored is specified by the BASEDIR option. You can limit the maximal queue -## size using the SLOTS option (if omitted, the queue is limited by space -## available in the back end filesystem). -# -# set eventqueue -# basedir /var/monit # set the base directory where events will be stored -# slots 100 # optionaly limit the queue size -# -# -## Send status and events to M/Monit (Monit central management: for more -## informations about M/Monit see http://www.tildeslash.com/mmonit). -# -# set mmonit http://monit:monit@192.168.1.10:8080/collector -# -# -## Monit by default uses the following alert mail format: -## -## --8<-- -## From: monit@$HOST # sender -## Subject: monit alert -- $EVENT $SERVICE # subject -## -## $EVENT Service $SERVICE # -## # -## Date: $DATE # -## Action: $ACTION # -## Host: $HOST # body -## Description: $DESCRIPTION # -## # -## Your faithful employee, # -## monit # -## --8<-- -## -## You can override this message format or parts of it, such as subject -## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. -## are expanded at runtime. For example, to override the sender: -# -# set mail-format { from: monit@foo.bar } -# -# -## You can set alert recipients here whom will receive alerts if/when a -## service defined in this file has errors. Alerts may be restricted on -## events by using a filter as in the second example below. -# -# set alert sysadm@foo.bar # receive all alerts -# set alert manager@foo.bar only on { timeout } # receive just service- -# # timeout alert -# -# -## Monit has an embedded web server which can be used to view status of -## services monitored, the current configuration, actual services parameters -## and manage services from a web interface. -# - set httpd port 2812 and - use address localhost # only accept connection from localhost - allow localhost # allow localhost to connect to the server and -# allow admin:monit # require user 'admin' with password 'monit' -# allow @monit # allow users of group 'monit' to connect (rw) -# allow @users readonly # allow users of group 'users' to connect readonly -# -# -############################################################################### -## Services -############################################################################### - -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset %> - -check process nova_api_<%= port %> with pidfile /var/run/nova/nova-api-<%= port %>.pid - group nova_api - start program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock start" - as uid nova - stop program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock stop" - as uid nova - if failed port <%= port %> protocol http - with timeout 15 seconds - for 4 cycles - then restart - if totalmem > 300 Mb then restart - if cpu is greater than 60% for 2 cycles then alert - if cpu > 80% for 3 cycles then restart - if 3 restarts within 5 cycles then timeout - -<% end %> diff --git a/contrib/puppet/templates/nova-iptables.erb b/contrib/puppet/templates/nova-iptables.erb deleted file mode 100644 index 2fc066305..000000000 --- a/contrib/puppet/templates/nova-iptables.erb +++ /dev/null @@ -1,10 +0,0 @@ -<% services.each do |service| -%> -<%= service.upcase %>=1 -<% end -%> -<% if ip && ip != "" %>IP="<%=ip%>"<% end %> -<% if private_range && private_range != "" %>PRIVATE_RANGE="<%=private_range%>"<% end %> -<% if mgmt_ip && mgmt_ip != "" %>MGMT_IP="<%=mgmt_ip%>"<% end %> -<% if dmz_ip && dmz_ip != "" %>DMZ_IP="<%=dmz_ip%>"<% end %> - -# warning: this file is auto-generated by puppet - diff --git a/contrib/puppet/templates/production/nova-common.conf.erb b/contrib/puppet/templates/production/nova-common.conf.erb deleted file mode 100644 index 23ee0c5e8..000000000 --- a/contrib/puppet/templates/production/nova-common.conf.erb +++ /dev/null @@ -1,55 +0,0 @@ -# global ---dmz_net=192.168.0.0 ---dmz_mask=255.255.0.0 ---dmz_cidr=192.168.0.0/16 ---ldap_user_dn=cn=Administrators,dc=example,dc=com ---ldap_user_unit=Users ---ldap_user_subtree=ou=Users,dc=example,dc=com ---ldap_project_subtree=ou=Groups,dc=example,dc=com ---role_project_subtree=ou=Groups,dc=example,dc=com ---ldap_cloudadmin=cn=NovaAdmins,ou=Groups,dc=example,dc=com ---ldap_itsec=cn=NovaSecurity,ou=Groups,dc=example,dc=com ---ldap_sysadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_netadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_developer=cn=developers,ou=Groups,dc=example,dc=com ---verbose ---daemonize ---syslog ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---scheduler_driver=nova.scheduler.simple.SimpleScheduler ---libvirt_xml_template=/usr/share/nova/libvirt.qemu.xml.template ---credentials_template=/usr/share/nova/novarc.template ---boot_script_template=/usr/share/nova/bootscript.template ---vpn_client_template=/usr/share/nova/client.ovpn.template ---max_cores=40 ---max_gigabytes=2000 ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vpn_start=11000 ---volume_group=vgdata ---volume_manager=nova.volume.manager.ISCSIManager ---volume_driver=nova.volume.driver.ISCSIDriver ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---dhcpbridge=/usr/bin/nova-dhcpbridge ---vpn_image_id=ami-cloudpipe ---dhcpbridge_flagfile=/etc/nova/nova.conf ---credential_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=NOVA/CN=%s-%s ---auth_driver=nova.auth.ldapdriver.LdapDriver ---quota_cores=17 ---quota_floating_ips=5 ---quota_instances=6 ---quota_volumes=10 ---quota_gigabytes=100 ---use_nova_chains=True ---input_chain=services ---use_project_ca=True ---fixed_ip_disassociate_timeout=300 ---api_max_requests=1 ---api_listen_ip=127.0.0.1 ---user_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=%s-%s-%s ---project_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-ca-%s-%s ---vpn_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-vpn-%s-%s diff --git a/contrib/puppet/templates/production/nova-nova.conf.erb b/contrib/puppet/templates/production/nova-nova.conf.erb deleted file mode 100644 index 8683fefde..000000000 --- a/contrib/puppet/templates/production/nova-nova.conf.erb +++ /dev/null @@ -1,21 +0,0 @@ ---fixed_range=192.168.0.0/16 ---iscsi_ip_prefix=192.168.4 ---floating_range=10.0.0.0/24 ---rabbit_host=192.168.0.10 ---s3_host=192.168.0.10 ---cc_host=192.168.0.10 ---cc_dmz=192.168.24.10 ---s3_dmz=192.168.24.10 ---ec2_url=http://192.168.0.1:8773/services/Cloud ---vpn_ip=192.168.0.2 ---ldap_url=ldap://192.168.0.10 ---sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---other_sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---routing_source_ip=192.168.0.2 ---bridge_dev=eth1 ---public_interface=eth0 ---vlan_start=3100 ---num_networks=700 ---rabbit_userid=TODO:RABBIT ---rabbit_password=TODO:CHANGEME ---ldap_password=TODO:CHANGEME -- cgit From 3392f6b4b060402c8d9a442f1a89a24fa31c9342 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 21 Feb 2011 14:27:37 -0600 Subject: Removing duplicate installation docs and adding flag file information, plus pointing to docs.openstack.org for Admin-audience docs --- doc/source/adminguide/binaries.rst | 57 ---- doc/source/adminguide/distros/others.rst | 88 ------ doc/source/adminguide/distros/ubuntu.10.04.rst | 40 --- doc/source/adminguide/distros/ubuntu.10.10.rst | 41 --- doc/source/adminguide/euca2ools.rst | 49 ---- doc/source/adminguide/flags.rst | 23 -- doc/source/adminguide/getting.started.rst | 167 ----------- doc/source/adminguide/index.rst | 91 ------ doc/source/adminguide/managing.images.rst | 21 -- doc/source/adminguide/managing.instances.rst | 59 ---- doc/source/adminguide/managing.networks.rst | 70 ----- doc/source/adminguide/managing.projects.rst | 68 ----- doc/source/adminguide/managing.users.rst | 82 ------ doc/source/adminguide/managingsecurity.rst | 39 --- doc/source/adminguide/monitoring.rst | 27 -- doc/source/adminguide/multi.node.install.rst | 392 ------------------------- doc/source/adminguide/network.flat.rst | 60 ---- doc/source/adminguide/network.vlan.rst | 179 ----------- doc/source/adminguide/nova.manage.rst | 239 --------------- doc/source/adminguide/single.node.install.rst | 362 ----------------------- doc/source/community.rst | 12 +- doc/source/index.rst | 20 +- doc/source/object.model.rst | 14 +- doc/source/quickstart.rst | 2 +- 24 files changed, 24 insertions(+), 2178 deletions(-) delete mode 100644 doc/source/adminguide/binaries.rst delete mode 100644 doc/source/adminguide/distros/others.rst delete mode 100644 doc/source/adminguide/distros/ubuntu.10.04.rst delete mode 100644 doc/source/adminguide/distros/ubuntu.10.10.rst delete mode 100644 doc/source/adminguide/euca2ools.rst delete mode 100644 doc/source/adminguide/flags.rst delete mode 100644 doc/source/adminguide/getting.started.rst delete mode 100644 doc/source/adminguide/index.rst delete mode 100644 doc/source/adminguide/managing.images.rst delete mode 100644 doc/source/adminguide/managing.instances.rst delete mode 100644 doc/source/adminguide/managing.networks.rst delete mode 100644 doc/source/adminguide/managing.projects.rst delete mode 100644 doc/source/adminguide/managing.users.rst delete mode 100644 doc/source/adminguide/managingsecurity.rst delete mode 100644 doc/source/adminguide/monitoring.rst delete mode 100644 doc/source/adminguide/multi.node.install.rst delete mode 100644 doc/source/adminguide/network.flat.rst delete mode 100644 doc/source/adminguide/network.vlan.rst delete mode 100644 doc/source/adminguide/nova.manage.rst delete mode 100644 doc/source/adminguide/single.node.install.rst diff --git a/doc/source/adminguide/binaries.rst b/doc/source/adminguide/binaries.rst deleted file mode 100644 index 5c50a51f1..000000000 --- a/doc/source/adminguide/binaries.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _binaries: - -Nova Daemons -============= - -The configuration of these binaries relies on "flagfiles" using the google -gflags package:: - - $ nova-xxxxx --flagfile flagfile - -The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. - -nova-api --------- - -Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. - -nova-objectstore ----------------- - -Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with glance and a simple image manager. - -nova-compute ------------- - -Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. - -nova-volume ------------ - -Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. - -nova-network ------------- - -Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: - - $ nova-network --network_manager=nova.network.manager.FlatManager - -IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/adminguide/distros/others.rst b/doc/source/adminguide/distros/others.rst deleted file mode 100644 index ec14a9abb..000000000 --- a/doc/source/adminguide/distros/others.rst +++ /dev/null @@ -1,88 +0,0 @@ -Installation on other distros (like Debian, Fedora or CentOS ) -============================================================== - -Feel free to add additional notes for additional distributions. - -Nova installation on CentOS 5.5 -------------------------------- - -These are notes for installing OpenStack Compute on CentOS 5.5 and will be updated but are NOT final. Please test for accuracy and edit as you see fit. - -The principle botleneck for running nova on centos in python 2.6. Nova is written in python 2.6 and CentOS 5.5. comes with python 2.4. We can not update python system wide as some core utilities (like yum) is dependent on python 2.4. Also very few python 2.6 modules are available in centos/epel repos. - -Pre-reqs --------- - -Add euca2ools and EPEL repo first.:: - - cat >/etc/yum.repos.d/euca2ools.repo << EUCA_REPO_CONF_EOF - [eucalyptus] - name=euca2ools - baseurl=http://www.eucalyptussoftware.com/downloads/repo/euca2ools/1.3.1/yum/centos/ - enabled=1 - gpgcheck=0 - - EUCA_REPO_CONF_EOF - -:: - - rpm -Uvh 'http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-4.noarch.rpm' - -Now install python2.6, kvm and few other libraries through yum:: - - yum -y install dnsmasq vblade kpartx kvm gawk iptables ebtables bzr screen euca2ools curl rabbitmq-server gcc gcc-c++ autoconf automake swig openldap openldap-servers nginx python26 python26-devel python26-distribute git openssl-devel python26-tools mysql-server qemu kmod-kvm libxml2 libxslt libxslt-devel mysql-devel - -Then download the latest aoetools and then build(and install) it, check for the latest version on sourceforge, exact url will change if theres a new release:: - - wget -c http://sourceforge.net/projects/aoetools/files/aoetools/32/aoetools-32.tar.gz/download - tar -zxvf aoetools-32.tar.gz - cd aoetools-32 - make - make install - -Add the udev rules for aoetools:: - - cat > /etc/udev/rules.d/60-aoe.rules << AOE_RULES_EOF - SUBSYSTEM=="aoe", KERNEL=="discover", NAME="etherd/%k", GROUP="disk", MODE="0220" - SUBSYSTEM=="aoe", KERNEL=="err", NAME="etherd/%k", GROUP="disk", MODE="0440" - SUBSYSTEM=="aoe", KERNEL=="interfaces", NAME="etherd/%k", GROUP="disk", MODE="0220" - SUBSYSTEM=="aoe", KERNEL=="revalidate", NAME="etherd/%k", GROUP="disk", MODE="0220" - # aoe block devices - KERNEL=="etherd*", NAME="%k", GROUP="disk" - AOE_RULES_EOF - -Load the kernel modules:: - - modprobe aoe - -:: - - modprobe kvm - -Now, install the python modules using easy_install-2.6, this ensures the installation are done against python 2.6 - - -easy_install-2.6 twisted sqlalchemy mox greenlet carrot daemon eventlet tornado IPy routes lxml MySQL-python -python-gflags need to be downloaded and installed manually, use these commands (check the exact url for newer releases ): - -:: - - wget -c "http://python-gflags.googlecode.com/files/python-gflags-1.4.tar.gz" - tar -zxvf python-gflags-1.4.tar.gz - cd python-gflags-1.4 - python2.6 setup.py install - cd .. - -Same for python2.6-libxml2 module, notice the --with-python and --prefix flags. --with-python ensures we are building it against python2.6 (otherwise it will build against python2.4, which is default):: - - wget -c "ftp://xmlsoft.org/libxml2/libxml2-2.7.3.tar.gz" - tar -zxvf libxml2-2.7.3.tar.gz - cd libxml2-2.7.3 - ./configure --with-python=/usr/bin/python26 --prefix=/usr - make all - make install - cd python - python2.6 setup.py install - cd .. - -Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/distros/ubuntu.10.04.rst b/doc/source/adminguide/distros/ubuntu.10.04.rst deleted file mode 100644 index bd0693c46..000000000 --- a/doc/source/adminguide/distros/ubuntu.10.04.rst +++ /dev/null @@ -1,40 +0,0 @@ -Installing on Ubuntu 10.04 (Lucid) -================================== - -Step 1: Install dependencies ----------------------------- -Grab the latest code from launchpad: - -:: - - bzr clone lp:nova - -Here's a script you can use to install (and then run) Nova on Ubuntu or Debian (when using Debian, edit nova.sh to have USE_PPA=0): - -.. todo:: give a link to a stable releases page - -Step 2: Install dependencies ----------------------------- - -Nova requires rabbitmq for messaging, so install that first. - -*Note:* You must have sudo installed to run these commands as shown here. - -:: - - sudo apt-get install rabbitmq-server - - -You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. - -If you're running on Ubuntu 10.04, you'll need to install Twisted and python-gflags which is included in the OpenStack PPA. - -:: - - sudo apt-get install python-software-properties - sudo add-apt-repository ppa:nova-core/trunk - sudo apt-get update - sudo apt-get install python-twisted python-gflags - - -Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/distros/ubuntu.10.10.rst b/doc/source/adminguide/distros/ubuntu.10.10.rst deleted file mode 100644 index a3fa2def1..000000000 --- a/doc/source/adminguide/distros/ubuntu.10.10.rst +++ /dev/null @@ -1,41 +0,0 @@ -Installing on Ubuntu 10.10 (Maverick) -===================================== -Single Machine Installation (Ubuntu 10.10) - -While we wouldn't expect you to put OpenStack Compute into production on a non-LTS version of Ubuntu, these instructions are up-to-date with the latest version of Ubuntu. - -Make sure you are running Ubuntu 10.10 so that the packages will be available. This install requires more than 70 MB of free disk space. - -These instructions are based on Soren Hansen's blog entry, Openstack on Maverick. A script is in progress as well. - -Step 1: Install required prerequisites --------------------------------------- -Nova requires rabbitmq for messaging and redis for storing state (for now), so we'll install these first.:: - - sudo apt-get install rabbitmq-server redis-server - -You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. - -Step 2: Install Nova packages available in Maverick Meerkat ------------------------------------------------------------ -Type or copy/paste in the following line to get the packages that you use to run OpenStack Compute.:: - - sudo apt-get install python-nova - sudo apt-get install nova-api nova-objectstore nova-compute nova-scheduler nova-network euca2ools unzip - -You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. This operation may take a while as many dependent packages will be installed. Note: there is a dependency problem with python-nova which can be worked around by installing first. - -When the installation is complete, you'll see the following lines confirming::: - - Adding system user `nova' (UID 106) ... - Adding new user `nova' (UID 106) with group `nogroup' ... - Not creating home directory `/var/lib/nova'. - Setting up nova-scheduler (0.9.1~bzr331-0ubuntu2) ... - * Starting nova scheduler nova-scheduler - WARNING:root:Starting scheduler node - ...done. - Processing triggers for libc-bin ... - ldconfig deferred processing now taking place - Processing triggers for python-support ... - -Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/euca2ools.rst b/doc/source/adminguide/euca2ools.rst deleted file mode 100644 index 6f0c57358..000000000 --- a/doc/source/adminguide/euca2ools.rst +++ /dev/null @@ -1,49 +0,0 @@ -Euca2ools -========= - -Nova is compatible with most of the euca2ools command line utilities. Both Administrators and Users will find these tools helpful for day-to-day administration. - -* euca-add-group -* euca-delete-bundle -* euca-describe-instances -* euca-register -* euca-add-keypair -* euca-delete-group -* euca-describe-keypairs -* euca-release-address -* euca-allocate-address -* euca-delete-keypair -* euca-describe-regions -* euca-reset-image-attribute -* euca-associate-address -* euca-delete-snapshot -* euca-describe-snapshots -* euca-revoke -* euca-attach-volume -* euca-delete-volume -* euca-describe-volumes -* euca-run-instances -* euca-authorize -* euca-deregister -* euca-detach-volume -* euca-terminate-instances -* euca-bundle-image -* euca-describe-addresses -* euca-disassociate-address -* euca-unbundle -* euca-bundle-vol -* euca-describe-availability-zones -* euca-download-bundle -* euca-upload-bundle -* euca-confirm-product-instance -* euca-describe-groups -* euca-get-console-output -* euca-version -* euca-create-snapshot -* euca-describe-image-attribute -* euca-modify-image-attribute -* euca-create-volume -* euca-describe-images -* euca-reboot-instances - - diff --git a/doc/source/adminguide/flags.rst b/doc/source/adminguide/flags.rst deleted file mode 100644 index 072f0a1a5..000000000 --- a/doc/source/adminguide/flags.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Flags and Flagfiles -=================== - -* python-gflags -* flagfiles -* list of flags by component (see concepts list) diff --git a/doc/source/adminguide/getting.started.rst b/doc/source/adminguide/getting.started.rst deleted file mode 100644 index 675d8e664..000000000 --- a/doc/source/adminguide/getting.started.rst +++ /dev/null @@ -1,167 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Getting Started with Nova -========================= - -This code base is continually changing, so dependencies also change. If you -encounter any problems, see the :doc:`../community` page. -The `contrib/nova.sh` script should be kept up to date, and may be a good -resource to review when debugging. - -The purpose of this document is to get a system installed that you can use to -test your setup assumptions. Working from this base installtion you can -tweak configurations and work with different flags to monitor interaction with -your hardware, network, and other factors that will allow you to determine -suitability for your deployment. After following this setup method, you should -be able to experiment with different managers, drivers, and flags to get the -best performance. - -Dependencies ------------- - -Related servers we rely on - -* **RabbitMQ**: messaging queue, used for all communication between components - -Optional servers - -* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by - setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP - (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. - There is a script in the sources (`nova/auth/slap.sh`) to install a very basic - openldap server on ubuntu. -* **ReDIS**: There is a fake ldap auth driver - `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was - created for testing ldap implementation on systems that don't have an easy - means to install ldap. -* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to - be avilable. Currently, only sqlite3 an mysql have been tested. - -Python libraries that we use (from pip-requires): - -.. literalinclude:: ../../../tools/pip-requires - -Other libraries: - -* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available - from http://wiki.xensource.com/xenwiki/XCP_SDK or - http://community.citrix.com/cdn/xs/sdks. - -External unix tools that are required: - -* iptables -* ebtables -* gawk -* curl -* kvm -* libvirt -* dnsmasq -* vlan -* open-iscsi and iscsitarget (if you use iscsi volumes) -* aoetools and vblade-persist (if you use aoe-volumes) - -Nova uses cutting-edge versions of many packages. There are ubuntu packages in -the nova-core trunk ppa. You can use add this ppa to your sources list on an -ubuntu machine with the following commands:: - - sudo apt-get install -y python-software-properties - sudo add-apt-repository ppa:nova-core/trunk - -Recommended ------------ - -* euca2ools: python implementation of aws ec2-tools and ami tools -* build tornado to use C module for evented section - - -Installation --------------- - -You can install from packages for your particular Linux distribution if they are -available. Otherwise you can install from source by checking out the source -files from the `Nova Source Code Repository `_ -and running:: - - python setup.py install - -Configuration ---------------- - -Configuring the host system -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As you read through the Administration Guide you will notice configuration hints -inline with documentation on the subsystem you are configuring. Presented in -this "Getting Started with Nova" document, we only provide what you need to -get started as quickly as possible. For a more detailed description of system -configuration, start reading through :doc:`multi.node.install`. - -* Create a volume group (you can use an actual disk for the volume group as - well):: - - # This creates a 1GB file to create volumes out of - dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 - losetup --show -f MY_FILE_PATH - # replace /dev/loop0 below with whatever losetup returns - # nova-volumes is the default for the --volume_group flag - vgcreate nova-volumes /dev/loop0 - - -Configuring Nova -~~~~~~~~~~~~~~~~ - -Configuration of the entire system is performed through python-gflags. The -best way to track configuration is through the use of a flagfile. - -A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary -when you launch it. Flagfiles for nova are typically stored in -``/etc/nova/nova.conf``, and flags specific to a certain program are stored in -``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another -flagfile, so typically a file like ``nova-manage.conf`` would have as its first -line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before -specifying overrides or additional options. - -A sample configuration to test the system follows:: - - --verbose - --nodaemon - --auth_driver=nova.auth.dbdriver.DbDriver - -Running ---------- - -There are many parts to the nova system, each with a specific function. They -are built to be highly-available, so there are may configurations they can be -run in (ie: on many machines, many listeners per machine, etc). This part -of the guide only gets you started quickly, to learn about HA options, see -:doc:`multi.node.install`. - -Launch supporting services - -* rabbitmq -* redis (optional) -* mysql (optional) -* openldap (optional) - -Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` - -* nova-api -* nova-compute -* nova-objectstore -* nova-volume -* nova-scheduler diff --git a/doc/source/adminguide/index.rst b/doc/source/adminguide/index.rst deleted file mode 100644 index 3bd72cfdc..000000000 --- a/doc/source/adminguide/index.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Administration Guide -==================== - -This guide describes the basics of running and managing Nova. - -Running the Cloud ------------------ - -The fastest way to get a test cloud running is by following the directions in the :doc:`../quickstart`. - -Nova's cloud works via the interaction of a series of daemon processes that reside persistently on the host machine(s). Fortunately, the :doc:`../quickstart` process launches sample versions of all these daemons for you. Once you are familiar with basic Nova usage, you can learn more about daemons by reading :doc:`../service.architecture` and :doc:`binaries`. - -Administration Utilities ------------------------- - -There are two main tools that a system administrator will find useful to manage their Nova cloud: - -.. toctree:: - :maxdepth: 1 - - nova.manage - euca2ools - -The nova-manage command may only be run by users with admin priviledges. Commands for euca2ools can be used by all users, though specific commands may be restricted by Role Based Access Control. You can read more about creating and managing users in :doc:`managing.users` - -User and Resource Management ----------------------------- - -The nova-manage and euca2ools commands provide the basic interface to perform a broad range of administration functions. In this section, you can read more about how to accomplish specific administration tasks. - -For background on the core objects referenced in this section, see :doc:`../object.model` - -.. toctree:: - :maxdepth: 1 - - managing.users - managing.projects - managing.instances - managing.images - managing.volumes - managing.networks - -Deployment ----------- - -For a starting multi-node architecture, you would start with two nodes - a cloud controller node and a compute node. The cloud controller node contains the nova- services plus the Nova database. The compute node installs all the nova-services but then refers to the database installation, which is hosted by the cloud controller node. Ensure that the nova.conf file is identical on each node. If you find performance issues not related to database reads or writes, but due to the messaging queue backing up, you could add additional messaging services (rabbitmq). - -.. toctree:: - :maxdepth: 1 - - multi.node.install - dbsync - - -Networking -^^^^^^^^^^ - -.. toctree:: - :maxdepth: 1 - - multi.node.install - network.vlan.rst - network.flat.rst - - -Advanced Topics ---------------- - -.. toctree:: - :maxdepth: 1 - - flags - monitoring - diff --git a/doc/source/adminguide/managing.images.rst b/doc/source/adminguide/managing.images.rst deleted file mode 100644 index c5d93a6e8..000000000 --- a/doc/source/adminguide/managing.images.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Managing Images -=============== - -.. todo:: Put info on managing images here! diff --git a/doc/source/adminguide/managing.instances.rst b/doc/source/adminguide/managing.instances.rst deleted file mode 100644 index e62352017..000000000 --- a/doc/source/adminguide/managing.instances.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Managing Instances -================== - -Keypairs --------- - -Images can be shared by many users, so it is dangerous to put passwords into the images. Nova therefore supports injecting ssh keys into instances before they are booted. This allows a user to login to the instances that he or she creates securely. Generally the first thing that a user does when using the system is create a keypair. Nova generates a public and private key pair, and sends the private key to the user. The public key is stored so that it can be injected into instances. - -Keypairs are created through the api. They can be created on the command line using the euca2ools script euca-add-keypair. Refer to the man page for the available options. Example usage:: - - euca-add-keypair test > test.pem - chmod 600 test.pem - euca-run-instances -k test -t m1.tiny ami-tiny - # wait for boot - ssh -i test.pem root@ip.of.instance - - -Basic Management ----------------- -Instance management can be accomplished with euca commands: - - -To run an instance: - -:: - - euca-run-instances - - -To terminate an instance: - -:: - - euca-terminate-instances - -To reboot an instance: - -:: - - euca-reboot-instances - -See the euca2ools documentation for more information diff --git a/doc/source/adminguide/managing.networks.rst b/doc/source/adminguide/managing.networks.rst deleted file mode 100644 index 9eea46d70..000000000 --- a/doc/source/adminguide/managing.networks.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - Overview Sections Copyright 2010-2011 Citrix - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Networking Overview -=================== -In Nova, users organize their cloud resources in projects. A Nova project consists of a number of VM instances created by a user. For each VM instance, Nova assigns to it a private IP address. (Currently, Nova only supports Linux bridge networking that allows the virtual interfaces to connect to the outside network through the physical interface. Other virtual network technologies, such as Open vSwitch, could be supported in the future.) The Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. - -Nova Network Strategies ------------------------ - -Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can co-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. - -Read more about Nova network strategies here: - -.. toctree:: - :maxdepth: 1 - - network.flat.rst - network.vlan.rst - - -Network Management Commands ---------------------------- - -Admins and Network Administrators can use the 'nova-manage' command to manage network resources: - -VPN Management -~~~~~~~~~~~~~~ - -* vpn list: Print a listing of the VPNs for all projects. - * arguments: none -* vpn run: Start the VPN for a given project. - * arguments: project -* vpn spawn: Run all VPNs. - * arguments: none - - -Floating IP Management -~~~~~~~~~~~~~~~~~~~~~~ - -* floating create: Creates floating ips for host by range - * arguments: host ip_range -* floating delete: Deletes floating ips by range - * arguments: range -* floating list: Prints a listing of all floating ips - * arguments: none - -Network Management -~~~~~~~~~~~~~~~~~~ - -* network create: Creates fixed ips for host by range - * arguments: [fixed_range=FLAG], [num_networks=FLAG], - [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG] - diff --git a/doc/source/adminguide/managing.projects.rst b/doc/source/adminguide/managing.projects.rst deleted file mode 100644 index 5dd7f2de9..000000000 --- a/doc/source/adminguide/managing.projects.rst +++ /dev/null @@ -1,68 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Managing Projects -================= - -Projects are isolated resource containers forming the principal organizational structure within Nova. They consist of a separate vlan, volumes, instances, images, keys, and users. - -Although the original ec2 api only supports users, nova adds the concept of projects. A user can specify which project he or she wishes to use by appending `:project_id` to his or her access key. If no project is specified in the api request, nova will attempt to use a project with the same id as the user. - -The api will return NotAuthorized if a normal user attempts to make requests for a project that he or she is not a member of. Note that admins or users with special admin roles skip this check and can make requests for any project. - -To create a project, use the `project create` command of nova-manage. The syntax is nova-manage project create projectname manager_id [description] You must specify a projectname and a manager_id. For example:: - nova-manage project create john_project john "This is a sample project" - -You can add and remove users from projects with `project add` and `project remove`:: - nova-manage project add john_project john - nova-manage project remove john_project john - -Project Commands ----------------- - -Admins and Project Managers can use the 'nova-manage project' command to manage project resources: - -* project add: Adds user to project - * arguments: project user -* project create: Creates a new project - * arguments: name project_manager [description] -* project delete: Deletes an existing project - * arguments: project_id -* project environment: Exports environment variables to an sourcable file - * arguments: project_id user_id [filename='novarc] -* project list: lists all projects - * arguments: none -* project remove: Removes user from project - * arguments: project user -* project scrub: Deletes data associated with project - * arguments: project -* project zipfile: Exports credentials for project to a zip file - * arguments: project_id user_id [filename='nova.zip] - -Setting Quotas --------------- -Nova utilizes a quota system at the project level to control resource consumption across available hardware resources. Current quota controls are available to limit the: - -* Number of volumes which may be created -* Total size of all volumes within a project as measured in GB -* Number of instances which may be launched -* Number of processor cores which may be allocated -* Publicly accessible IP addresses - -Use the following command to set quotas for a project -* project quota: Set or display quotas for project - * arguments: project_id [key] [value] diff --git a/doc/source/adminguide/managing.users.rst b/doc/source/adminguide/managing.users.rst deleted file mode 100644 index 392142e86..000000000 --- a/doc/source/adminguide/managing.users.rst +++ /dev/null @@ -1,82 +0,0 @@ -Managing Users -============== - - -Users and Access Keys ---------------------- - -Access to the ec2 api is controlled by an access and secret key. The user's access key needs to be included in the request, and the request must be signed with the secret key. Upon receipt of api requests, nova will verify the signature and execute commands on behalf of the user. - -In order to begin using nova, you will need a to create a user. This can be easily accomplished using the user create or user admin commands in nova-manage. `user create` will create a regular user, whereas `user admin` will create an admin user. The syntax of the command is nova-manage user create username [access] [secret]. For example:: - - nova-manage user create john my-access-key a-super-secret-key - -If you do not specify an access or secret key, a random uuid will be created automatically. - -Credentials ------------ - -Nova can generate a handy set of credentials for a user. These credentials include a CA for bundling images and a file for setting environment variables to be used by euca2ools. If you don't need to bundle images, just the environment script is required. You can export one with the `project environment` command. The syntax of the command is nova-manage project environment project_id user_id [filename]. If you don't specify a filename, it will be exported as novarc. After generating the file, you can simply source it in bash to add the variables to your environment:: - - nova-manage project environment john_project john - . novarc - -If you do need to bundle images, you will need to get all of the credentials using `project zipfile`. Note that zipfile will give you an error message if networks haven't been created yet. Otherwise zipfile has the same syntax as environment, only the default file name is nova.zip. Example usage:: - - nova-manage project zipfile john_project john - unzip nova.zip - . novarc - -Role Based Access Control -------------------------- -Roles control the api actions that a user is allowed to perform. For example, a user cannot allocate a public ip without the `netadmin` role. It is important to remember that a users de facto permissions in a project is the intersection of user (global) roles and project (local) roles. So for john to have netadmin permissions in his project, he needs to separate roles specified. You can add roles with `role add`. The syntax is nova-manage role add user_id role [project_id]. Let's give john the netadmin role for his project:: - - nova-manage role add john netadmin - nova-manage role add john netadmin john_project - -Role-based access control (RBAC) is an approach to restricting system access to authorized users based on an individual’s role within an organization. Various employee functions require certain levels of system access in order to be successful. These functions are mapped to defined roles and individuals are categorized accordingly. Since users are not assigned permissions directly, but only acquire them through their role (or roles), management of individual user rights becomes a matter of assigning appropriate roles to the user. This simplifies common operations, such as adding a user, or changing a user's department. - -Nova’s rights management system employs the RBAC model and currently supports the following five roles: - -* **Cloud Administrator.** (admin) Users of this class enjoy complete system access. -* **IT Security.** (itsec) This role is limited to IT security personnel. It permits role holders to quarantine instances. -* **Project Manager.** (projectmanager)The default for project owners, this role affords users the ability to add other users to a project, interact with project images, and launch and terminate instances. -* **Network Administrator.** (netadmin) Users with this role are permitted to allocate and assign publicly accessible IP addresses as well as create and modify firewall rules. -* **Developer.** This is a general purpose role that is assigned to users by default. - -RBAC management is exposed through the dashboard for simplified user management. - - -User Commands -~~~~~~~~~~~~ - -Users, including admins, are created through the ``user`` commands. - -* user admin: creates a new admin and prints exports - * arguments: name [access] [secret] -* user create: creates a new user and prints exports - * arguments: name [access] [secret] -* user delete: deletes an existing user - * arguments: name -* user exports: prints access and secrets for user in export format - * arguments: name -* user list: lists all users - * arguments: none -* user modify: update a users keys & admin flag - * arguments: accesskey secretkey admin - * leave any field blank to ignore it, admin should be 'T', 'F', or blank - - -User Role Management -~~~~~~~~~~~~~~~~~~~~ - -* role add: adds role to user - * if project is specified, adds project specific role - * arguments: user, role [project] -* role has: checks to see if user has role - * if project is specified, returns True if user has - the global role and the project role - * arguments: user, role [project] -* role remove: removes role from user - * if project is specified, removes project specific role - * arguments: user, role [project] diff --git a/doc/source/adminguide/managingsecurity.rst b/doc/source/adminguide/managingsecurity.rst deleted file mode 100644 index 7893925e7..000000000 --- a/doc/source/adminguide/managingsecurity.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Security Considerations -======================= - -.. todo:: This doc is vague and just high-level right now. Describe architecture that enables security. - -The goal of securing a cloud computing system involves both protecting the instances, data on the instances, and -ensuring users are authenticated for actions and that borders are understood by the users and the system. -Protecting the system from intrusion or attack involves authentication, network protections, and -compromise detection. - -Key Concepts ------------- - -Authentication - Each instance is authenticated with a key pair. - -Network - Instances can communicate with each other but you can configure the boundaries through firewall -configuration. - -Monitoring - Log all API commands and audit those logs. - -Encryption - Data transfer between instances is not encrypted. - diff --git a/doc/source/adminguide/monitoring.rst b/doc/source/adminguide/monitoring.rst deleted file mode 100644 index 2c93c71b5..000000000 --- a/doc/source/adminguide/monitoring.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Monitoring -========== - -* components -* throughput -* exceptions -* hardware - -* ganglia -* syslog diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst deleted file mode 100644 index c53455e3e..000000000 --- a/doc/source/adminguide/multi.node.install.rst +++ /dev/null @@ -1,392 +0,0 @@ - -Installing Nova on Multiple Servers -=================================== - -When you move beyond evaluating the technology and into building an actual -production environment, you will need to know how to configure your datacenter -and how to deploy components across your clusters. This guide should help you -through that process. - -You can install multiple nodes to increase performance and availability of the OpenStack Compute installation. - -This setup is based on an Ubuntu Lucid 10.04 installation with the latest updates. Most of this works around issues that need to be resolved either in packaging or bug-fixing. It also needs to eventually be generalized, but the intent here is to get the multi-node configuration bootstrapped so folks can move forward. - -For a starting architecture, these instructions describing installing a cloud controller node and a compute node. The cloud controller node contains the nova- services plus the database. The compute node installs all the nova-services but then refers to the database installation, which is hosted by the cloud controller node. - -Requirements for a multi-node installation ------------------------------------------- - -* You need a real database, compatible with SQLAlchemy (mysql, postgresql) There's not a specific reason to choose one over another, it basically depends what you know. MySQL is easier to do High Availability (HA) with, but people may already know PostgreSQL. We should document both configurations, though. -* For a recommended HA setup, consider a MySQL master/slave replication, with as many slaves as you like, and probably a heartbeat to kick one of the slaves into being a master if it dies. -* For performance optimization, split reads and writes to the database. MySQL proxy is the easiest way to make this work if running MySQL. - -Assumptions ------------ - -* Networking is configured between/through the physical machines on a single subnet. -* Installation and execution are both performed by ROOT user. - - -Scripted Installation ---------------------- -A script available to get your OpenStack cloud running quickly. You can copy the file to the server where you want to install OpenStack Compute services - typically you would install a compute node and a cloud controller node. - -You must run these scripts with root permissions. - -From a server you intend to use as a cloud controller node, use this command to get the cloud controller script. This script is a work-in-progress and the maintainer plans to keep it up, but it is offered "as-is." Feel free to collaborate on it in GitHub - https://github.com/dubsquared/OpenStack-NOVA-Installer-Script/. - -:: - - wget --no-check-certificate https://github.com/dubsquared/OpenStack-NOVA-Installer-Script/raw/master/nova-CC-install-v1.1.sh - -Ensure you can execute the script by modifying the permissions on the script file. - -:: - - sudo chmod 755 nova-CC-install-v1.1.sh - - -:: - - sudo ./nova-CC-install-v1.1.sh - -Next, from a server you intend to use as a compute node (doesn't contain the database), install the nova services. You can use the nova-NODE-installer.sh script from the above github-hosted project for the compute node installation. - -Copy the nova.conf from the cloud controller node to the compute node. - -Restart related services:: - - libvirtd restart; service nova-network restart; service nova-compute restart; service nova-api restart; service nova-objectstore restart; service nova-scheduler restart - -You can go to the `Configuration section`_ for next steps. - -Manual Installation - Step-by-Step ----------------------------------- -The following sections show you how to install Nova manually with a cloud controller node and a separate compute node. The cloud controller node contains the database plus all nova- services, and the compute node runs nova- services only. - -Cloud Controller Installation -````````````````````````````` -On the cloud controller node, you install nova services and the related helper applications, and then configure with the nova.conf file. You will then copy the nova.conf file to the compute node, which you install as a second node in the `Compute Installation`_. - -Step 1 - Use apt-get to get the latest code -------------------------------------------- - -1. Setup Nova PPA with https://launchpad.net/~nova-core/+archive/trunk. The ‘python-software-properties’ package is a pre-requisite for setting up the nova package repo: - -:: - - sudo apt-get install python-software-properties - sudo add-apt-repository ppa:nova-core/trunk - -2. Run update. - -:: - - sudo apt-get update - -3. Install python required packages, nova-packages, and helper apps. - -:: - - sudo apt-get install python-greenlet python-mysqldb python-nova nova-common nova-doc nova-api nova-network nova-objectstore nova-scheduler nova-compute euca2ools unzip - -It is highly likely that there will be errors when the nova services come up since they are not yet configured. Don't worry, you're only at step 1! - -Step 2 Set up configuration file (installed in /etc/nova) ---------------------------------------------------------- - -1. Nova development has consolidated all config files to nova.conf as of November 2010. There is a default set of options that are already configured in nova.conf: - -:: - ---daemonize=1 ---dhcpbridge_flagfile=/etc/nova/nova.conf ---dhcpbridge=/usr/bin/nova-dhcpbridge ---logdir=/var/log/nova ---state_path=/var/lib/nova - -The following items ALSO need to be defined in /etc/nova/nova.conf. I’ve added some explanation of the variables, as comments CANNOT be in nova.conf. There seems to be an issue with nova-manage not processing the comments/whitespace correctly: - ---sql_connection ### Location of Nova SQL DB - ---s3_host ### This is where Nova is hosting the objectstore service, which will contain the VM images and buckets - ---rabbit_host ### This is where the rabbit AMQP messaging service is hosted - ---cc_host ### This is where the the nova-api service lives - ---verbose ### Optional but very helpful during initial setup - ---ec2_url ### The location to interface nova-api - ---network_manager ### Many options here, discussed below. This is how your controller will communicate with additional Nova nodes and VMs: - -nova.network.manager.FlatManager # Simple, no-vlan networking type -nova.network.manager. FlatDHCPManager # Flat networking with DHCP -nova.network.manager.VlanManager # Vlan networking with DHCP – /DEFAULT/ if no network manager is defined in nova.conf - ---fixed_range= ### This will be the IP network that ALL the projects for future VM guests will reside on. E.g. 192.168.0.0/12 - ---network_size=<# of addrs> ### This is the total number of IP Addrs to use for VM guests, of all projects. E.g. 5000 - -The following code can be cut and paste, and edited to your setup: - -Note: CC_ADDR= - -Detailed explanation of the following example is available above. - -:: - ---sql_connection=mysql://root:nova@/nova ---s3_host= ---rabbit_host= ---cc_host= ---verbose ---ec2_url=http://:8773/services/Cloud ---network_manager=nova.network.manager.VlanManager ---fixed_range= ---network_size=<# of addrs> - -2. Create a “nova” group, and set permissions:: - - addgroup nova - -The Nova config file should have its owner set to root:nova, and mode set to 0644, since they contain your MySQL server's root password. :: - - chown -R root:nova /etc/nova - chmod 644 /etc/nova/nova.conf - -Step 3 - Setup the SQL DB (MySQL for this setup) ------------------------------------------------- - -1. First you 'preseed' to bypass all the installation prompts:: - - bash - MYSQL_PASS=nova - cat < - # The loopback network interface - auto lo - iface lo inet loopback - - # Networking for NOVA - auto br100 - - iface br100 inet dhcp - bridge_ports eth0 - bridge_stp off - bridge_maxwait 0 - bridge_fd 0 - < end /etc/network/interfaces > - -Next, restart networking to apply the changes:: - - sudo /etc/init.d/networking restart - -Configuration -````````````` - -On the Compute node, you should continue with these configuration steps. - -Step 1 - Set up the Nova environment ------------------------------------- - -These are the commands you run to update the database if needed, and then set up a user and project:: - - /usr/bin/python /usr/bin/nova-manage db sync - /usr/bin/python /usr/bin/nova-manage user admin - /usr/bin/python /usr/bin/nova-manage project create - /usr/bin/python /usr/bin/nova-manage network create - -Here is an example of what this looks like with real data:: - - /usr/bin/python /usr/bin/nova-manage db sync - /usr/bin/python /usr/bin/nova-manage user admin dub - /usr/bin/python /usr/bin/nova-manage project create dubproject dub - /usr/bin/python /usr/bin/nova-manage network create 192.168.0.0/24 1 255 - -(I chose a /24 since that falls inside my /12 range I set in ‘fixed-range’ in nova.conf. Currently, there can only be one network, and I am using the max IP’s available in a /24. You can choose to use any valid amount that you would like.) - -Note: The nova-manage service assumes that the first IP address is your network (like 192.168.0.0), that the 2nd IP is your gateway (192.168.0.1), and that the broadcast is the very last IP in the range you defined (192.168.0.255). If this is not the case you will need to manually edit the sql db 'networks' table.o. - -On running the "nova-manage network create" command, entries are made in the 'networks' and 'fixed_ips' table. However, one of the networks listed in the 'networks' table needs to be marked as bridge in order for the code to know that a bridge exists. The Network is marked as bridged automatically based on the type of network manager selected. You only need to mark the network as a bridge if you chose FlatManager as your network type. More information can be found at the end of this document discussing setting up the bridge device. - - -Step 2 - Create Nova certifications ------------------------------------ - -1. Generate the certs as a zip file. These are the certs you will use to launch instances, bundle images, and all the other assorted api functions. - -:: - - mkdir –p /root/creds - /usr/bin/python /usr/bin/nova-manage project zipfile $NOVA_PROJECT $NOVA_PROJECT_USER /root/creds/novacreds.zip - -2. Unzip them in your home directory, and add them to your environment. - -:: - - unzip /root/creds/novacreds.zip -d /root/creds/ - cat /root/creds/novarc >> ~/.bashrc - source ~/.bashrc - -Step 3 - Restart all relevant services --------------------------------------- - -Restart all six services in total, just to cover the entire spectrum:: - - libvirtd restart; service nova-network restart; service nova-compute restart; service nova-api restart; service nova-objectstore restart; service nova-scheduler restart - -Step 4 - Closing steps, and cleaning up ---------------------------------------- - -One of the most commonly missed configuration areas is not allowing the proper access to VMs. Use the 'euca-authorize' command to enable access. Below, you will find the commands to allow 'ping' and 'ssh' to your VMs:: - - euca-authorize -P icmp -t -1:-1 default - euca-authorize -P tcp -p 22 default - -Another common issue is you cannot ping or SSH your instances after issusing the 'euca-authorize' commands. Something to look at is the amount of 'dnsmasq' processes that are running. If you have a running instance, check to see that TWO 'dnsmasq' processes are running. If not, perform the following:: - - killall dnsmasq - service nova-network restart - -To avoid issues with KVM and permissions with Nova, run the following commands to ensure we have VM's that are running optimally:: - - chgrp kvm /dev/kvm - chmod g+rwx /dev/kvm - -If you want to use the 10.04 Ubuntu Enterprise Cloud images that are readily available at http://uec-images.ubuntu.com/releases/10.04/release/, you may run into delays with booting. Any server that does not have nova-api running on it needs this iptables entry so that UEC images can get metadata info. On compute nodes, configure the iptables with this next step:: - - # iptables -t nat -A PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $NOVA_API_IP:8773 - -Testing the Installation -```````````````````````` - -You can confirm that your compute node is talking to your cloud controller. From the cloud controller, run this database query:: - - mysql -u$MYSQL_USER -p$MYSQL_PASS nova -e 'select * from services;' - -In return, you should see something similar to this:: - +---------------------+---------------------+------------+---------+----+----------+----------------+-----------+--------------+----------+-------------------+ - | created_at | updated_at | deleted_at | deleted | id | host | binary | topic | report_count | disabled | availability_zone | - +---------------------+---------------------+------------+---------+----+----------+----------------+-----------+--------------+----------+-------------------+ - | 2011-01-28 22:52:46 | 2011-02-03 06:55:48 | NULL | 0 | 1 | osdemo02 | nova-network | network | 46064 | 0 | nova | - | 2011-01-28 22:52:48 | 2011-02-03 06:55:57 | NULL | 0 | 2 | osdemo02 | nova-compute | compute | 46056 | 0 | nova | - | 2011-01-28 22:52:52 | 2011-02-03 06:55:50 | NULL | 0 | 3 | osdemo02 | nova-scheduler | scheduler | 46065 | 0 | nova | - | 2011-01-29 23:49:29 | 2011-02-03 06:54:26 | NULL | 0 | 4 | osdemo01 | nova-compute | compute | 37050 | 0 | nova | - | 2011-01-30 23:42:24 | 2011-02-03 06:55:44 | NULL | 0 | 9 | osdemo04 | nova-compute | compute | 28484 | 0 | nova | - | 2011-01-30 21:27:28 | 2011-02-03 06:54:23 | NULL | 0 | 8 | osdemo05 | nova-compute | compute | 29284 | 0 | nova | - +---------------------+---------------------+------------+---------+----+----------+----------------+-----------+--------------+----------+-------------------+ -You can see that 'osdemo0{1,2,4,5} are all running 'nova-compute.' When you start spinning up instances, they will allocate on any node that is running nova-compute from this list. - -You can then use `euca2ools` to test some items:: - - euca-describe-images - euca-describe-instances - -If you have issues with the API key, you may need to re-source your creds file:: - - . /root/creds/novarc - -If you don’t get any immediate errors, you’re successfully making calls to your cloud! - -Spinning up a VM for Testing -```````````````````````````` - -(This excerpt is from Thierry Carrez's blog, with reference to http://wiki.openstack.org/GettingImages.) - -The image that you will use here will be a ttylinux image, so this is a limited function server. You will be able to ping and SSH to this instance, but it is in no way a full production VM. - -UPDATE: Due to `bug 661159 `_, we can’t use images without ramdisks yet, so we can’t use the classic Ubuntu cloud images from http://uec-images.ubuntu.com/releases/ yet. For the sake of this tutorial, we’ll use the `ttylinux images from Scott Moser instead `_. - -Download the image, and publish to your bucket: - -:: - - image="ttylinux-uec-amd64-12.1_2.6.35-22_1.tar.gz" - wget http://smoser.brickies.net/ubuntu/ttylinux-uec/$image - uec-publish-tarball $image mybucket - -This will output three references, an "emi", an "eri" and an "eki." (Image, ramdisk, and kernel) The emi is the one we use to launch instances, so take note of this. - -Create a keypair to SSH to the server: - -:: - - euca-add-keypair mykey > mykey.priv - - chmod 0600 mykey.priv - -Boot your instance: - -:: - - euca-run-instances $emi -k mykey -t m1.tiny - -($emi is replaced with the output from the previous command) - -Checking status, and confirming communication: - -Once you have booted the instance, you can check the status the the `euca-describe-instances` command. Here you can view the instance ID, IP, and current status of the VM. - -:: - - euca-describe-instances - -Once in a "running" state, you can use your SSH key connect: - -:: - - ssh -i mykey.priv root@$ipaddress - -When you are ready to terminate the instance, you may do so with the `euca-terminate-instances` command: - -:: - - euca-terminate-instances $instance-id - -You can determine the instance-id with `euca-describe-instances`, and the format is "i-" with a series of letter and numbers following: e.g. i-a4g9d. - -For more information in creating you own custom (production ready) instance images, please visit http://wiki.openstack.org/GettingImages for more information! - -Enjoy your new private cloud, and play responsibly! diff --git a/doc/source/adminguide/network.flat.rst b/doc/source/adminguide/network.flat.rst deleted file mode 100644 index 3d8680c6f..000000000 --- a/doc/source/adminguide/network.flat.rst +++ /dev/null @@ -1,60 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -Flat Network Mode (Original and Flat) -===================================== - -Flat network mode removes most of the complexity of VLAN mode by simply -bridging all instance interfaces onto a single network. - -There are two variations of flat mode that differ mostly in how IP addresses -are given to instances. - - -Original Flat Mode ------------------- -IP addresses for VM instances are grabbed from a subnet specified by the network administrator, and injected into the image on launch. All instances of the system are attached to the same Linux networking bridge, configured manually by the network administrator both on the network controller hosting the network and on the computer controllers hosting the instances. To recap: - -* Each compute host creates a single bridge for all instances to use to attach to the external network. -* The networking configuration is injected into the instance before it is booted or it is obtained by a guest agent installed in the instance. - -Note that the configuration injection currently only works on linux-style systems that keep networking -configuration in /etc/network/interfaces. - - -Flat DHCP Mode --------------- -IP addresses for VM instances are grabbed from a subnet specified by the network administrator. Similar to the flat network, a single Linux networking bridge is created and configured manually by the network administrator and used for all instances. A DHCP server is started to pass out IP addresses to VM instances from the specified subnet. To recap: - -* Like flat mode, all instances are attached to a single bridge on the compute node. -* In addition a DHCP server is running to configure instances. - -Implementation --------------- - -The network nodes do not act as a default gateway in flat mode. Instances -are given public IP addresses. - -Compute nodes have iptables/ebtables entries created per project and -instance to protect against IP/MAC address spoofing and ARP poisoning. - - -Examples --------- - -.. todo:: add flat network mode configuration examples diff --git a/doc/source/adminguide/network.vlan.rst b/doc/source/adminguide/network.vlan.rst deleted file mode 100644 index c06ce8e8b..000000000 --- a/doc/source/adminguide/network.vlan.rst +++ /dev/null @@ -1,179 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -VLAN Network Mode -================= -VLAN Network Mode is the default mode for Nova. It provides a private network -segment for each project's instances that can be accessed via a dedicated -VPN connection from the Internet. - -In this mode, each project gets its own VLAN, Linux networking bridge, and subnet. The subnets are specified by the network administrator, and are assigned dynamically to a project when required. A DHCP Server is started for each VLAN to pass out IP addresses to VM instances from the subnet assigned to the project. All instances belonging to one project are bridged into the same VLAN for that project. The Linux networking bridges and VLANs are created by Nova when required, described in more detail in Nova VLAN Network Management Implementation. - -.. - (this text revised above) - Because the flat network and flat DhCP network are simple to understand and yet do not scale well enough for real-world cloud systems, this section focuses on the VLAN network implementation by the VLAN Network Manager. - - - In the VLAN network mode, all the VM instances of a project are connected together in a VLAN with the specified private subnet. Each running VM instance is assigned an IP address within the given private subnet. - -.. image:: /images/Novadiagram.png - :width: 790 - -While network traffic between VM instances belonging to the same VLAN is always open, Nova can enforce isolation of network traffic between different projects by enforcing one VLAN per project. - -In addition, the network administrator can specify a pool of public IP addresses that users may allocate and then assign to VMs, either at boot or dynamically at run-time. This capability is similar to Amazon's 'elastic IPs'. A public IP address may be associated with a running instances, allowing the VM instance to be accessed from the public network. The public IP addresses are accessible from the network host and NATed to the private IP address of the project. - -.. todo:: Describe how a public IP address could be associated with a project (a VLAN) - -This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code named :ref:`cloudpipe `) needs to be created. Nova generates a certificate and key for the user to access the vpn and starts the vpn automatically. More information on cloudpipe can be found :ref:`here `. - -The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) - -.. image:: /images/cloudpipe.png - :width: 100% - -Goals ------ - -For our implementation of Nova, our goal is that each project is in a protected network segment. Here are the specifications we keep in mind for meeting this goal. - - * RFC-1918 IP space - * public IP via NAT - * no default inbound Internet access without public NAT - * limited (project-admin controllable) outbound Internet access - * limited (project-admin controllable) access to other project segments - * all connectivity to instance and cloud API is via VPN into the project segment - -We also keep as a goal a common DMZ segment for support services, meaning these items are only visible from project segment: - - * metadata - * dashboard - -Limitations ------------ - -We kept in mind some of these limitations: - -* Projects / cluster limited to available VLANs in switching infrastructure -* Requires VPN for access to project segment - -Implementation --------------- -Currently Nova segregates project VLANs using 802.1q VLAN tagging in the -switching layer. Compute hosts create VLAN-specific interfaces and bridges -as required. - -The network nodes act as default gateway for project networks and contain -all of the routing and firewall rules implementing security groups. The -network node also handles DHCP to provide instance IPs for each project. - -VPN access is provided by running a small instance called CloudPipe -on the IP immediately following the gateway IP for each project. The -network node maps a dedicated public IP/port to the CloudPipe instance. - -Compute nodes have per-VLAN interfaces and bridges created as required. -These do NOT have IP addresses in the host to protect host access. -Compute nodes have iptables/ebtables entries created per project and -instance to protect against IP/MAC address spoofing and ARP poisoning. - -The network assignment to a project, and IP address assignment to a VM instance, are triggered when a user starts to run a VM instance. When running a VM instance, a user needs to specify a project for the instances, and the security groups (described in Security Groups) when the instance wants to join. If this is the first instance to be created for the project, then Nova (the cloud controller) needs to find a network controller to be the network host for the project; it then sets up a private network by finding an unused VLAN id, an unused subnet, and then the controller assigns them to the project, it also assigns a name to the project's Linux bridge (br100 stored in the Nova database), and allocating a private IP within the project's subnet for the new instance. - -If the instance the user wants to start is not the project's first, a subnet and a VLAN must have already been assigned to the project; therefore the system needs only to find an available IP address within the subnet and assign it to the new starting instance. If there is no private IP available within the subnet, an exception will be raised to the cloud controller, and the VM creation cannot proceed. - - -External Infrastructure ------------------------ - -Nova assumes the following is available: - -* DNS -* NTP -* Internet connectivity - - -Example -------- - -This example network configuration demonstrates most of the capabilities -of VLAN Mode. It splits administrative access to the nodes onto a dedicated -management network and uses dedicated network nodes to handle all -routing and gateway functions. - -It uses a 10GB network for instance traffic and a 1GB network for management. - - -Hardware -~~~~~~~~ - -* All nodes have a minimum of two NICs for management and production. - - * management is 1GB - * production is 10GB - * add additional NICs for bonding or HA/performance - -* network nodes should have an additional NIC dedicated to public Internet traffic -* switch needs to support enough simultaneous VLANs for number of projects -* production network configured as 802.1q trunk on switch - - -Operation -~~~~~~~~~ - -The network node controls the project network configuration: - -* assigns each project a VLAN and private IP range -* starts dnsmasq on project VLAN to serve private IP range -* configures iptables on network node for default project access -* launches CloudPipe instance and configures iptables access - -When starting an instance the network node: - -* sets up a VLAN interface and bridge on each host as required when an - instance is started on that host -* assigns private IP to instance -* generates MAC address for instance -* update dnsmasq with IP/MAC for instance - -When starting an instance the compute node: - -* sets up a VLAN interface and bridge on each host as required when an - instance is started on that host - - -Setup -~~~~~ - -* Assign VLANs in the switch: - - * public Internet segment - * production network - * management network - * cluster DMZ - -* Assign a contiguous range of VLANs to Nova for project use. -* Configure management NIC ports as management VLAN access ports. -* Configure management VLAN with Internet access as required -* Configure production NIC ports as 802.1q trunk ports. -* Configure Nova (need to add specifics here) - - * public IPs - * instance IPs - * project network size - * DMZ network - -.. todo:: need specific Nova configuration added diff --git a/doc/source/adminguide/nova.manage.rst b/doc/source/adminguide/nova.manage.rst deleted file mode 100644 index 0e9a29b6b..000000000 --- a/doc/source/adminguide/nova.manage.rst +++ /dev/null @@ -1,239 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -The nova-manage command -======================= - -Introduction -~~~~~~~~~~~~ - -The nova-manage command is used to perform many essential functions for -administration and ongoing maintenance of nova, such as user creation, -vpn management, and much more. - -The standard pattern for executing a nova-manage command is: -``nova-manage []`` - -For example, to obtain a list of all projects: -``nova-manage project list`` - -Run without arguments to see a list of available command categories: -``nova-manage`` - -Categories are user, project, role, shell, vpn, and floating. Detailed descriptions are below. - -You can also run with a category argument such as user to see a list of all commands in that category: -``nova-manage user`` - -These sections describe the available categories and arguments for nova-manage. - -Nova Db -~~~~~~~ - -``nova-manage db version`` - - Print the current database version. - -``nova-manage db sync`` - - Sync the database up to the most recent version. This is the standard way to create the db as well. - -Nova User -~~~~~~~~~ - -``nova-manage user admin `` - - Create an admin user with the name . - -``nova-manage user create `` - - Create a normal user with the name . - -``nova-manage user delete `` - - Delete the user with the name . - -``nova-manage user exports `` - - Outputs a list of access key and secret keys for user to the screen - -``nova-manage user list`` - - Outputs a list of all the user names to the screen. - -``nova-manage user modify `` - - Updates the indicated user keys, indicating with T or F if the user is an admin user. Leave any argument blank if you do not want to update it. - -Nova Project -~~~~~~~~~~~~ - -``nova-manage project add `` - - Add a nova project with the name to the database. - -``nova-manage project create `` - - Create a new nova project with the name (you still need to do nova-manage project add to add it to the database). - -``nova-manage project delete `` - - Delete a nova project with the name . - -``nova-manage project environment `` - - Exports environment variables for the named project to a file named novarc. - -``nova-manage project list`` - - Outputs a list of all the projects to the screen. - -``nova-manage project quota `` - - Outputs the size and specs of the project's instances including gigabytes, instances, floating IPs, volumes, and cores. - -``nova-manage project remove `` - - Deletes the project with the name . - -``nova-manage project zipfile`` - - Compresses all related files for a created project into a zip file nova.zip. - -Nova Role -~~~~~~~~~ - -nova-manage role [] -``nova-manage role add <(optional) projectname>`` - - Add a user to either a global or project-based role with the indicated assigned to the named user. Role names can be one of the following five roles: admin, itsec, projectmanager, netadmin, developer. If you add the project name as the last argument then the role is assigned just for that project, otherwise the user is assigned the named role for all projects. - -``nova-manage role has `` - Checks the user or project and responds with True if the user has a global role with a particular project. - -``nova-manage role remove `` - Remove the indicated role from the user. - -Nova Shell -~~~~~~~~~~ - -``nova-manage shell bpython`` - - Starts a new bpython shell. - -``nova-manage shell ipython`` - - Starts a new ipython shell. - -``nova-manage shell python`` - - Starts a new python shell. - -``nova-manage shell run`` - - Starts a new shell using python. - -``nova-manage shell script `` - - Runs the named script from the specified path with flags set. - -Nova VPN -~~~~~~~~ - -``nova-manage vpn list`` - - Displays a list of projects, their IP prot numbers, and what state they're in. - -``nova-manage vpn run `` - - Starts the VPN for the named project. - -``nova-manage vpn spawn`` - - Runs all VPNs. - -Nova Floating IPs -~~~~~~~~~~~~~~~~~ - -``nova-manage floating create `` - - Creates floating IP addresses for the named host by the given range. - -``nova-manage floating delete `` - - Deletes floating IP addresses in the range given. - -``nova-manage floating list`` - - Displays a list of all floating IP addresses. - -Concept: Flags --------------- - -python-gflags - - -Concept: Plugins ----------------- - -* Managers/Drivers: utils.import_object from string flag -* virt/connections: conditional loading from string flag -* db: LazyPluggable via string flag -* auth_manager: utils.import_class based on string flag -* Volumes: moving to pluggable driver instead of manager -* Network: pluggable managers -* Compute: same driver used, but pluggable at connection - - -Concept: IPC/RPC ----------------- - -Rabbit! - - -Concept: Fakes --------------- - -* auth -* ldap - - -Concept: Scheduler ------------------- - -* simple -* random - - -Concept: Security Groups ------------------------- - -Security groups - - -Concept: Certificate Authority ------------------------------- - -Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns <../cloudpipe>` and decrypting bundled images. - - -Concept: Images ---------------- - -* launching -* bundling diff --git a/doc/source/adminguide/single.node.install.rst b/doc/source/adminguide/single.node.install.rst deleted file mode 100644 index ff43aa90b..000000000 --- a/doc/source/adminguide/single.node.install.rst +++ /dev/null @@ -1,362 +0,0 @@ -Installing Nova on a Single Host -================================ - -Nova can be run on a single machine, and it is recommended that new users practice managing this type of installation before graduating to multi node systems. - -The fastest way to get a test cloud running is through our :doc:`../quickstart`. But for more detail on installing the system read this doc. - - -Step 1 and 2: Get the latest Nova code system software ------------------------------------------------------- - -Depending on your system, the method for accomplishing this varies - -.. toctree:: - :maxdepth: 1 - - distros/ubuntu.10.04 - distros/ubuntu.10.10 - distros/others - - -Step 3: Build and install Nova services ---------------------------------------- - -Switch to the base nova source directory. - -Then type or copy/paste in the following line to compile the Python code for OpenStack Compute. - -:: - - sudo python setup.py build - sudo python setup.py install - - -When the installation is complete, you'll see the following lines: - -:: - - Installing nova-network script to /usr/local/bin - Installing nova-volume script to /usr/local/bin - Installing nova-objectstore script to /usr/local/bin - Installing nova-manage script to /usr/local/bin - Installing nova-scheduler script to /usr/local/bin - Installing nova-dhcpbridge script to /usr/local/bin - Installing nova-compute script to /usr/local/bin - Installing nova-instancemonitor script to /usr/local/bin - Installing nova-api script to /usr/local/bin - Installing nova-import-canonical-imagestore script to /usr/local/bin - - Installed /usr/local/lib/python2.6/dist-packages/nova-2010.1-py2.6.egg - Processing dependencies for nova==2010.1 - Finished processing dependencies for nova==2010.1 - - -Step 4: Create the Nova Database --------------------------------- -Type or copy/paste in the following line to create your nova db:: - - sudo nova-manage db sync - -Step 5: Create a Nova administrator ------------------------------------ -Type or copy/paste in the following line to create a user named "anne.":: - - sudo nova-manage user admin anne - -You see an access key and a secret key export, such as these made-up ones::: - - export EC2_ACCESS_KEY=4e6498a2-blah-blah-blah-17d1333t97fd - export EC2_SECRET_KEY=0a520304-blah-blah-blah-340sp34k05bbe9a7 - -Step 6: Create the network --------------------------- - -Type or copy/paste in the following line to create a network prior to creating a project. - -:: - - sudo nova-manage network create 10.0.0.0/8 1 64 - -For this command, the IP address is the cidr notation for your netmask, such as 192.168.1.0/24. The value 1 is the total number of networks you want made, and the 64 value is the total number of ips in all networks. - -After running this command, entries are made in the 'networks' and 'fixed_ips' table in the database. - -Step 7: Create a project with the user you created --------------------------------------------------- -Type or copy/paste in the following line to create a project named IRT (for Ice Road Truckers, of course) with the newly-created user named anne. - -:: - - sudo nova-manage project create IRT anne - -:: - - Generating RSA private key, 1024 bit long modulus - .....++++++ - ..++++++ - e is 65537 (0x10001) - Using configuration from ./openssl.cnf - Check that the request matches the signature - Signature ok - The Subject's Distinguished Name is as follows - countryName :PRINTABLE:'US' - stateOrProvinceName :PRINTABLE:'California' - localityName :PRINTABLE:'MountainView' - organizationName :PRINTABLE:'AnsoLabs' - organizationalUnitName:PRINTABLE:'NovaDev' - commonName :PRINTABLE:'anne-2010-10-12T21:12:35Z' - Certificate is to be certified until Oct 12 21:12:35 2011 GMT (365 days) - - Write out database with 1 new entries - Data Base Updated - - -Step 8: Unzip the nova.zip --------------------------- - -You should have a nova.zip file in your current working directory. Unzip it with this command: - -:: - - unzip nova.zip - - -You'll see these files extract. - -:: - - Archive: nova.zip - extracting: novarc - extracting: pk.pem - extracting: cert.pem - extracting: nova-vpn.conf - extracting: cacert.pem - - -Step 9: Source the rc file --------------------------- -Type or copy/paste the following to source the novarc file in your current working directory. - -:: - - . novarc - - -Step 10: Pat yourself on the back :) ------------------------------------ -Congratulations, your cloud is up and running, you’ve created an admin user, created a network, retrieved the user's credentials and put them in your environment. - -Now you need an image. - - -Step 11: Get an image --------------------- -To make things easier, we've provided a small image on the Rackspace CDN. Use this command to get it on your server. - -:: - - wget http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz - - -:: - - --2010-10-12 21:40:55-- http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz - Resolving cblah2.cdn.cloudfiles.rackspacecloud.com... 208.111.196.6, 208.111.196.7 - Connecting to cblah2.cdn.cloudfiles.rackspacecloud.com|208.111.196.6|:80... connected. - HTTP request sent, awaiting response... 200 OK - Length: 58520278 (56M) [application/x-gzip] - Saving to: `images.tgz' - - 100%[======================================>] 58,520,278 14.1M/s in 3.9s - - 2010-10-12 21:40:59 (14.1 MB/s) - `images.tgz' saved [58520278/58520278] - - - -Step 12: Decompress the image file ----------------------------------- -Use this command to extract the image files::: - - tar xvzf images.tgz - -You get a directory listing like so::: - - images - |-- aki-lucid - | |-- image - | `-- info.json - |-- ami-tiny - | |-- image - | `-- info.json - `-- ari-lucid - |-- image - `-- info.json - -Step 13: Send commands to upload sample image to the cloud ----------------------------------------------------------- - -Type or copy/paste the following commands to create a manifest for the kernel.:: - - euca-bundle-image -i images/aki-lucid/image -p kernel --kernel true - -You should see this in response::: - - Checking image - Tarring image - Encrypting image - Splitting image... - Part: kernel.part.0 - Generating manifest /tmp/kernel.manifest.xml - -Type or copy/paste the following commands to create a manifest for the ramdisk.:: - - euca-bundle-image -i images/ari-lucid/image -p ramdisk --ramdisk true - -You should see this in response::: - - Checking image - Tarring image - Encrypting image - Splitting image... - Part: ramdisk.part.0 - Generating manifest /tmp/ramdisk.manifest.xml - -Type or copy/paste the following commands to upload the kernel bundle.:: - - euca-upload-bundle -m /tmp/kernel.manifest.xml -b mybucket - -You should see this in response::: - - Checking bucket: mybucket - Creating bucket: mybucket - Uploading manifest file - Uploading part: kernel.part.0 - Uploaded image as mybucket/kernel.manifest.xml - -Type or copy/paste the following commands to upload the ramdisk bundle.:: - - euca-upload-bundle -m /tmp/ramdisk.manifest.xml -b mybucket - -You should see this in response::: - - Checking bucket: mybucket - Uploading manifest file - Uploading part: ramdisk.part.0 - Uploaded image as mybucket/ramdisk.manifest.xml - -Type or copy/paste the following commands to register the kernel and get its ID.:: - - euca-register mybucket/kernel.manifest.xml - -You should see this in response::: - - IMAGE ami-fcbj2non - -Type or copy/paste the following commands to register the ramdisk and get its ID.:: - - euca-register mybucket/ramdisk.manifest.xml - -You should see this in response::: - - IMAGE ami-orukptrc - -Type or copy/paste the following commands to create a manifest for the machine image associated with the ramdisk and kernel IDs that you got from the previous commands.:: - - euca-bundle-image -i images/ami-tiny/image -p machine --kernel ami-fcbj2non --ramdisk ami-orukptrc - -You should see this in response::: - - Checking image - Tarring image - Encrypting image - Splitting image... - Part: machine.part.0 - Part: machine.part.1 - Part: machine.part.2 - Part: machine.part.3 - Part: machine.part.4 - Generating manifest /tmp/machine.manifest.xml - -Type or copy/paste the following commands to upload the machine image bundle.:: - - euca-upload-bundle -m /tmp/machine.manifest.xml -b mybucket - -You should see this in response::: - - Checking bucket: mybucket - Uploading manifest file - Uploading part: machine.part.0 - Uploading part: machine.part.1 - Uploading part: machine.part.2 - Uploading part: machine.part.3 - Uploading part: machine.part.4 - Uploaded image as mybucket/machine.manifest.xml - -Type or copy/paste the following commands to register the machine image and get its ID.:: - - euca-register mybucket/machine.manifest.xml - -You should see this in response::: - - IMAGE ami-g06qbntt - -Type or copy/paste the following commands to register a SSH keypair for use in starting and accessing the instances.:: - - euca-add-keypair mykey > mykey.priv - chmod 600 mykey.priv - -Type or copy/paste the following commands to run an instance using the keypair and IDs that we previously created.:: - - euca-run-instances ami-g06qbntt --kernel ami-fcbj2non --ramdisk ami-orukptrc -k mykey - -You should see this in response::: - - RESERVATION r-0at28z12 IRT - INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 scheduling mykey (IRT, None) m1.small 2010-10-18 19:02:10.443599 - -Type or copy/paste the following commands to watch as the scheduler launches, and completes booting your instance.:: - - euca-describe-instances - -You should see this in response::: - - RESERVATION r-0at28z12 IRT - INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 launching mykey (IRT, cloud02) m1.small 2010-10-18 19:02:10.443599 - -Type or copy/paste the following commands to see when loading is completed and the instance is running.:: - - euca-describe-instances - -You should see this in response::: - - RESERVATION r-0at28z12 IRT - INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 running mykey (IRT, cloud02) 0 m1.small 2010-10-18 19:02:10.443599 - -Type or copy/paste the following commands to check that the virtual machine is running.:: - - virsh list - -You should see this in response::: - - Id Name State - ---------------------------------- - 1 2842445831 running - -Type or copy/paste the following commands to ssh to the instance using your private key.:: - - ssh -i mykey.priv root@10.0.0.3 - - -Troubleshooting Installation ----------------------------- - -If you see an "error loading the config file './openssl.cnf'" it means you can copy the openssl.cnf file to the location where Nova expects it and reboot, then try the command again. - -:: - - cp /etc/ssl/openssl.cnf ~ - sudo reboot - - - diff --git a/doc/source/community.rst b/doc/source/community.rst index 4ae32f1eb..e925a47bd 100644 --- a/doc/source/community.rst +++ b/doc/source/community.rst @@ -18,7 +18,7 @@ Getting Involved ================ -The Nova community is a very friendly group and there are places online to join in with the +The OpenStack community for Nova is a very friendly group and there are places online to join in with the community. Feel free to ask questions. This document points you to some of the places where you can communicate with people. @@ -83,3 +83,13 @@ Twitter Because all the cool kids do it: `@openstack `_. Also follow the `#openstack `_ tag for relevant tweets. + +OpenStack Docs Site +------------------- + +The `nova.openstack.org `_ site is geared towards developer documentation, +and the `docs.openstack.org `_ site is intended for cloud administrators +who are standing up and running OpenStack Compute in production. You can contribute to the Docs Site +by using bzr and Launchpad and contributing to the openstack-manuals project at http://launchpad.net/openstack-manuals. + + diff --git a/doc/source/index.rst b/doc/source/index.rst index d337fb69f..846d3cfcd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,11 +32,13 @@ Nova is written with the following design guidelines in mind: * **API Compatibility**: Nova strives to provide API-compatible with popular systems like Amazon EC2 This documentation is generated by the Sphinx toolkit and lives in the source -tree. Additional documentation on Nova and other components of OpenStack can -be found on the `OpenStack wiki`_. Also see the :doc:`community` page for -other ways to interact with the community. +tree. Additional draft and project documentation on Nova and other components of OpenStack can +be found on the `OpenStack wiki`_. Cloud administrators, refer to `docs.openstack.org`_. + +Also see the :doc:`community` page for other ways to interact with the community. .. _`OpenStack wiki`: http://wiki.openstack.org +.. _`docs.openstack.org`: http://docs.openstack.org Key Concepts @@ -50,17 +52,7 @@ Key Concepts service.architecture nova.object.model swift.object.model - -Administrator's Documentation -============================= - -.. toctree:: - :maxdepth: 1 - - livecd - adminguide/index - adminguide/single.node.install - adminguide/multi.node.install + runnova/index Developer Docs ============== diff --git a/doc/source/object.model.rst b/doc/source/object.model.rst index d02f151fd..419e89b0c 100644 --- a/doc/source/object.model.rst +++ b/doc/source/object.model.rst @@ -18,8 +18,6 @@ Object Model ============ -.. todo:: Add brief description for core models - .. graphviz:: digraph foo { @@ -42,27 +40,27 @@ Object Model Users ----- -Each Nova User is authorized based on their access key and secret key, assigned per-user. Read more at :doc:`/adminguide/managing.users`. +Each Nova User is authorized based on their access key and secret key, assigned per-user. Read more at :doc:`/runnova/managing.users`. Projects -------- -For Nova, access to images is based on the project. Read more at :doc:`/adminguide/managing.projects`. +For Nova, access to images is based on the project. Read more at :doc:`/runnova/managing.projects`. Images ------ -Images are binary files that run the operating system. Read more at :doc:`/adminguide/managing.images`. +Images are binary files that run the operating system. Read more at :doc:`/runnova/managing.images`. Instances --------- -Instances are running virtual servers. Read more at :doc:`/adminguide/managing.instances`. +Instances are running virtual servers. Read more at :doc:`/runnova/managing.instances`. Volumes ------- -.. todo:: Write doc about volumes +Volumes offer extra block level storage to instances. Read more at `Managing Volumes `_. Security Groups --------------- @@ -72,7 +70,7 @@ In Nova, a security group is a named collection of network access rules, like fi VLANs ----- -VLAN is the default network mode for Nova. Read more at :doc:`/adminguide/network.vlan`. +VLAN is the default network mode for Nova. Read more at :doc:`/runnova/network.vlan`. IP Addresses ------------ diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 17c9e10a8..84ed3fe01 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -54,7 +54,7 @@ Environment Variables By tweaking the environment that nova.sh run in, you can build slightly different configurations (though for more complex setups you should see -:doc:`/adminguide/getting.started` and :doc:`/adminguide/multi.node.install`). +`Installing and Configuring OpenStack Compute `_). * HOST_IP * Default: address of first interface from the ifconfig command -- cgit From df1213b8091c9a63a25b15c6eb82272255f96cb6 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 21 Feb 2011 14:30:20 -0600 Subject: Updated to remove built docs --- doc/.autogenerated | 406 ++++++ doc/build/.DS_Store | Bin 0 -> 6148 bytes doc/build/html/.DS_Store | Bin 0 -> 6148 bytes doc/build/html/.buildinfo | 4 + doc/source/api/autoindex.rst | 138 ++ doc/source/api/nova..adminclient.rst | 6 + doc/source/api/nova..api.direct.rst | 6 + doc/source/api/nova..api.ec2.admin.rst | 6 + doc/source/api/nova..api.ec2.apirequest.rst | 6 + doc/source/api/nova..api.ec2.cloud.rst | 6 + .../api/nova..api.ec2.metadatarequesthandler.rst | 6 + doc/source/api/nova..api.openstack.auth.rst | 6 + .../api/nova..api.openstack.backup_schedules.rst | 6 + doc/source/api/nova..api.openstack.common.rst | 6 + doc/source/api/nova..api.openstack.consoles.rst | 6 + doc/source/api/nova..api.openstack.faults.rst | 6 + doc/source/api/nova..api.openstack.flavors.rst | 6 + doc/source/api/nova..api.openstack.images.rst | 6 + doc/source/api/nova..api.openstack.servers.rst | 6 + .../api/nova..api.openstack.shared_ip_groups.rst | 6 + doc/source/api/nova..api.openstack.zones.rst | 6 + doc/source/api/nova..auth.dbdriver.rst | 6 + doc/source/api/nova..auth.fakeldap.rst | 6 + doc/source/api/nova..auth.ldapdriver.rst | 6 + doc/source/api/nova..auth.manager.rst | 6 + doc/source/api/nova..auth.signer.rst | 6 + doc/source/api/nova..cloudpipe.pipelib.rst | 6 + doc/source/api/nova..compute.api.rst | 6 + doc/source/api/nova..compute.instance_types.rst | 6 + doc/source/api/nova..compute.manager.rst | 6 + doc/source/api/nova..compute.monitor.rst | 6 + doc/source/api/nova..compute.power_state.rst | 6 + doc/source/api/nova..console.api.rst | 6 + doc/source/api/nova..console.fake.rst | 6 + doc/source/api/nova..console.manager.rst | 6 + doc/source/api/nova..console.xvp.rst | 6 + doc/source/api/nova..context.rst | 6 + doc/source/api/nova..crypto.rst | 6 + doc/source/api/nova..db.api.rst | 6 + doc/source/api/nova..db.base.rst | 6 + doc/source/api/nova..db.migration.rst | 6 + doc/source/api/nova..db.sqlalchemy.api.rst | 6 + .../nova..db.sqlalchemy.migrate_repo.manage.rst | 6 + ...sqlalchemy.migrate_repo.versions.001_austin.rst | 6 + ....sqlalchemy.migrate_repo.versions.002_bexar.rst | 6 + ...ate_repo.versions.003_add_label_to_networks.rst | 6 + ...y.migrate_repo.versions.004_add_zone_tables.rst | 6 + doc/source/api/nova..db.sqlalchemy.migration.rst | 6 + doc/source/api/nova..db.sqlalchemy.models.rst | 6 + doc/source/api/nova..db.sqlalchemy.session.rst | 6 + doc/source/api/nova..exception.rst | 6 + doc/source/api/nova..fakememcache.rst | 6 + doc/source/api/nova..fakerabbit.rst | 6 + doc/source/api/nova..flags.rst | 6 + doc/source/api/nova..image.glance.rst | 6 + doc/source/api/nova..image.local.rst | 6 + doc/source/api/nova..image.s3.rst | 6 + doc/source/api/nova..image.service.rst | 6 + doc/source/api/nova..log.rst | 6 + doc/source/api/nova..manager.rst | 6 + doc/source/api/nova..network.api.rst | 6 + doc/source/api/nova..network.linux_net.rst | 6 + doc/source/api/nova..network.manager.rst | 6 + doc/source/api/nova..objectstore.bucket.rst | 6 + doc/source/api/nova..objectstore.handler.rst | 6 + doc/source/api/nova..objectstore.image.rst | 6 + doc/source/api/nova..objectstore.stored.rst | 6 + doc/source/api/nova..quota.rst | 6 + doc/source/api/nova..rpc.rst | 6 + doc/source/api/nova..scheduler.chance.rst | 6 + doc/source/api/nova..scheduler.driver.rst | 6 + doc/source/api/nova..scheduler.manager.rst | 6 + doc/source/api/nova..scheduler.simple.rst | 6 + doc/source/api/nova..scheduler.zone.rst | 6 + doc/source/api/nova..service.rst | 6 + doc/source/api/nova..test.rst | 6 + doc/source/api/nova..tests.api.openstack.fakes.rst | 6 + .../nova..tests.api.openstack.test_adminapi.rst | 6 + .../api/nova..tests.api.openstack.test_api.rst | 6 + .../api/nova..tests.api.openstack.test_auth.rst | 6 + .../api/nova..tests.api.openstack.test_common.rst | 6 + .../api/nova..tests.api.openstack.test_faults.rst | 6 + .../api/nova..tests.api.openstack.test_flavors.rst | 6 + .../api/nova..tests.api.openstack.test_images.rst | 6 + ...nova..tests.api.openstack.test_ratelimiting.rst | 6 + .../api/nova..tests.api.openstack.test_servers.rst | 6 + .....tests.api.openstack.test_shared_ip_groups.rst | 6 + .../api/nova..tests.api.openstack.test_zones.rst | 6 + doc/source/api/nova..tests.api.test_wsgi.rst | 6 + doc/source/api/nova..tests.db.fakes.rst | 6 + doc/source/api/nova..tests.declare_flags.rst | 6 + doc/source/api/nova..tests.fake_flags.rst | 6 + doc/source/api/nova..tests.glance.stubs.rst | 6 + doc/source/api/nova..tests.hyperv_unittest.rst | 6 + .../api/nova..tests.objectstore_unittest.rst | 6 + doc/source/api/nova..tests.real_flags.rst | 6 + doc/source/api/nova..tests.runtime_flags.rst | 6 + doc/source/api/nova..tests.test_access.rst | 6 + doc/source/api/nova..tests.test_api.rst | 6 + doc/source/api/nova..tests.test_auth.rst | 6 + doc/source/api/nova..tests.test_cloud.rst | 6 + doc/source/api/nova..tests.test_compute.rst | 6 + doc/source/api/nova..tests.test_console.rst | 6 + doc/source/api/nova..tests.test_direct.rst | 6 + doc/source/api/nova..tests.test_flags.rst | 6 + doc/source/api/nova..tests.test_localization.rst | 6 + doc/source/api/nova..tests.test_log.rst | 6 + doc/source/api/nova..tests.test_middleware.rst | 6 + doc/source/api/nova..tests.test_misc.rst | 6 + doc/source/api/nova..tests.test_network.rst | 6 + doc/source/api/nova..tests.test_quota.rst | 6 + doc/source/api/nova..tests.test_rpc.rst | 6 + doc/source/api/nova..tests.test_scheduler.rst | 6 + doc/source/api/nova..tests.test_service.rst | 6 + doc/source/api/nova..tests.test_twistd.rst | 6 + doc/source/api/nova..tests.test_virt.rst | 6 + doc/source/api/nova..tests.test_volume.rst | 6 + doc/source/api/nova..tests.test_xenapi.rst | 6 + doc/source/api/nova..tests.xenapi.stubs.rst | 6 + doc/source/api/nova..twistd.rst | 6 + doc/source/api/nova..utils.rst | 6 + doc/source/api/nova..version.rst | 6 + doc/source/api/nova..virt.connection.rst | 6 + doc/source/api/nova..virt.disk.rst | 6 + doc/source/api/nova..virt.fake.rst | 6 + doc/source/api/nova..virt.hyperv.rst | 6 + doc/source/api/nova..virt.images.rst | 6 + doc/source/api/nova..virt.libvirt_conn.rst | 6 + doc/source/api/nova..virt.xenapi.fake.rst | 6 + doc/source/api/nova..virt.xenapi.network_utils.rst | 6 + doc/source/api/nova..virt.xenapi.vm_utils.rst | 6 + doc/source/api/nova..virt.xenapi.vmops.rst | 6 + doc/source/api/nova..virt.xenapi.volume_utils.rst | 6 + doc/source/api/nova..virt.xenapi.volumeops.rst | 6 + doc/source/api/nova..virt.xenapi_conn.rst | 6 + doc/source/api/nova..volume.api.rst | 6 + doc/source/api/nova..volume.driver.rst | 6 + doc/source/api/nova..volume.manager.rst | 6 + doc/source/api/nova..volume.san.rst | 6 + doc/source/api/nova..wsgi.rst | 6 + doc/source/runnova/binaries.rst | 57 + doc/source/runnova/euca2ools.rst | 49 + doc/source/runnova/flags.rst | 193 +++ doc/source/runnova/getting.started.rst | 168 +++ doc/source/runnova/index.rst | 90 ++ doc/source/runnova/managing.images.rst | 21 + doc/source/runnova/managing.instances.rst | 59 + doc/source/runnova/managing.networks.rst | 70 + doc/source/runnova/managing.projects.rst | 68 + doc/source/runnova/managing.users.rst | 82 ++ doc/source/runnova/managingsecurity.rst | 39 + doc/source/runnova/monitoring.rst | 27 + doc/source/runnova/network.flat.rst | 60 + doc/source/runnova/network.vlan.rst | 179 +++ doc/source/runnova/nova.manage.rst | 239 ++++ test/.Python | 1 + test/bin/activate | 76 ++ test/bin/activate.csh | 32 + test/bin/activate.fish | 79 ++ test/bin/activate_this.py | 32 + test/bin/easy_install | 9 + test/bin/easy_install-2.6 | 9 + test/bin/pip | 9 + test/bin/pip-2.6 | 9 + test/bin/python | Bin 0 -> 50720 bytes test/bin/python2.6 | 1 + test/include/python2.6 | 1 + test/lib/python2.6/UserDict.py | 1 + test/lib/python2.6/_abcoll.py | 1 + test/lib/python2.6/abc.py | 1 + test/lib/python2.6/codecs.py | 1 + test/lib/python2.6/config | 1 + test/lib/python2.6/copy_reg.py | 1 + test/lib/python2.6/distutils/__init__.py | 91 ++ test/lib/python2.6/distutils/distutils.cfg | 6 + test/lib/python2.6/encodings | 1 + test/lib/python2.6/fnmatch.py | 1 + test/lib/python2.6/genericpath.py | 1 + test/lib/python2.6/lib-dynload | 1 + test/lib/python2.6/linecache.py | 1 + test/lib/python2.6/locale.py | 1 + test/lib/python2.6/ntpath.py | 1 + test/lib/python2.6/orig-prefix.txt | 1 + test/lib/python2.6/os.py | 1 + test/lib/python2.6/posixpath.py | 1 + test/lib/python2.6/re.py | 1 + test/lib/python2.6/site-packages/easy-install.pth | 4 + .../pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO | 348 +++++ .../pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt | 57 + .../EGG-INFO/dependency_links.txt | 1 + .../pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt | 4 + .../pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe | 1 + .../pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt | 1 + .../pip-0.8.1-py2.6.egg/pip/__init__.py | 261 ++++ .../pip-0.8.1-py2.6.egg/pip/_pkgutil.py | 589 ++++++++ .../pip-0.8.1-py2.6.egg/pip/backwardcompat.py | 55 + .../pip-0.8.1-py2.6.egg/pip/basecommand.py | 203 +++ .../pip-0.8.1-py2.6.egg/pip/baseparser.py | 231 ++++ .../pip-0.8.1-py2.6.egg/pip/commands/__init__.py | 1 + .../pip-0.8.1-py2.6.egg/pip/commands/bundle.py | 33 + .../pip-0.8.1-py2.6.egg/pip/commands/completion.py | 60 + .../pip-0.8.1-py2.6.egg/pip/commands/freeze.py | 109 ++ .../pip-0.8.1-py2.6.egg/pip/commands/help.py | 32 + .../pip-0.8.1-py2.6.egg/pip/commands/install.py | 247 ++++ .../pip-0.8.1-py2.6.egg/pip/commands/search.py | 116 ++ .../pip-0.8.1-py2.6.egg/pip/commands/uninstall.py | 42 + .../pip-0.8.1-py2.6.egg/pip/commands/unzip.py | 9 + .../pip-0.8.1-py2.6.egg/pip/commands/zip.py | 346 +++++ .../pip-0.8.1-py2.6.egg/pip/download.py | 470 +++++++ .../pip-0.8.1-py2.6.egg/pip/exceptions.py | 17 + .../site-packages/pip-0.8.1-py2.6.egg/pip/index.py | 686 ++++++++++ .../pip-0.8.1-py2.6.egg/pip/locations.py | 45 + .../site-packages/pip-0.8.1-py2.6.egg/pip/log.py | 181 +++ .../site-packages/pip-0.8.1-py2.6.egg/pip/req.py | 1432 ++++++++++++++++++++ .../pip-0.8.1-py2.6.egg/pip/runner.py | 18 + .../site-packages/pip-0.8.1-py2.6.egg/pip/util.py | 479 +++++++ .../pip-0.8.1-py2.6.egg/pip/vcs/__init__.py | 238 ++++ .../pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py | 138 ++ .../pip-0.8.1-py2.6.egg/pip/vcs/git.py | 204 +++ .../pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py | 162 +++ .../pip-0.8.1-py2.6.egg/pip/vcs/subversion.py | 260 ++++ .../site-packages/pip-0.8.1-py2.6.egg/pip/venv.py | 53 + .../site-packages/setuptools-0.6c11-py2.6.egg | Bin 0 -> 333447 bytes test/lib/python2.6/site-packages/setuptools.pth | 1 + test/lib/python2.6/site.py | 713 ++++++++++ test/lib/python2.6/sre.py | 1 + test/lib/python2.6/sre_compile.py | 1 + test/lib/python2.6/sre_constants.py | 1 + test/lib/python2.6/sre_parse.py | 1 + test/lib/python2.6/stat.py | 1 + test/lib/python2.6/types.py | 1 + test/lib/python2.6/warnings.py | 1 + 232 files changed, 10985 insertions(+) create mode 100644 doc/.autogenerated create mode 100644 doc/build/.DS_Store create mode 100644 doc/build/html/.DS_Store create mode 100644 doc/build/html/.buildinfo create mode 100644 doc/source/api/autoindex.rst create mode 100644 doc/source/api/nova..adminclient.rst create mode 100644 doc/source/api/nova..api.direct.rst create mode 100644 doc/source/api/nova..api.ec2.admin.rst create mode 100644 doc/source/api/nova..api.ec2.apirequest.rst create mode 100644 doc/source/api/nova..api.ec2.cloud.rst create mode 100644 doc/source/api/nova..api.ec2.metadatarequesthandler.rst create mode 100644 doc/source/api/nova..api.openstack.auth.rst create mode 100644 doc/source/api/nova..api.openstack.backup_schedules.rst create mode 100644 doc/source/api/nova..api.openstack.common.rst create mode 100644 doc/source/api/nova..api.openstack.consoles.rst create mode 100644 doc/source/api/nova..api.openstack.faults.rst create mode 100644 doc/source/api/nova..api.openstack.flavors.rst create mode 100644 doc/source/api/nova..api.openstack.images.rst create mode 100644 doc/source/api/nova..api.openstack.servers.rst create mode 100644 doc/source/api/nova..api.openstack.shared_ip_groups.rst create mode 100644 doc/source/api/nova..api.openstack.zones.rst create mode 100644 doc/source/api/nova..auth.dbdriver.rst create mode 100644 doc/source/api/nova..auth.fakeldap.rst create mode 100644 doc/source/api/nova..auth.ldapdriver.rst create mode 100644 doc/source/api/nova..auth.manager.rst create mode 100644 doc/source/api/nova..auth.signer.rst create mode 100644 doc/source/api/nova..cloudpipe.pipelib.rst create mode 100644 doc/source/api/nova..compute.api.rst create mode 100644 doc/source/api/nova..compute.instance_types.rst create mode 100644 doc/source/api/nova..compute.manager.rst create mode 100644 doc/source/api/nova..compute.monitor.rst create mode 100644 doc/source/api/nova..compute.power_state.rst create mode 100644 doc/source/api/nova..console.api.rst create mode 100644 doc/source/api/nova..console.fake.rst create mode 100644 doc/source/api/nova..console.manager.rst create mode 100644 doc/source/api/nova..console.xvp.rst create mode 100644 doc/source/api/nova..context.rst create mode 100644 doc/source/api/nova..crypto.rst create mode 100644 doc/source/api/nova..db.api.rst create mode 100644 doc/source/api/nova..db.base.rst create mode 100644 doc/source/api/nova..db.migration.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.api.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migration.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.models.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.session.rst create mode 100644 doc/source/api/nova..exception.rst create mode 100644 doc/source/api/nova..fakememcache.rst create mode 100644 doc/source/api/nova..fakerabbit.rst create mode 100644 doc/source/api/nova..flags.rst create mode 100644 doc/source/api/nova..image.glance.rst create mode 100644 doc/source/api/nova..image.local.rst create mode 100644 doc/source/api/nova..image.s3.rst create mode 100644 doc/source/api/nova..image.service.rst create mode 100644 doc/source/api/nova..log.rst create mode 100644 doc/source/api/nova..manager.rst create mode 100644 doc/source/api/nova..network.api.rst create mode 100644 doc/source/api/nova..network.linux_net.rst create mode 100644 doc/source/api/nova..network.manager.rst create mode 100644 doc/source/api/nova..objectstore.bucket.rst create mode 100644 doc/source/api/nova..objectstore.handler.rst create mode 100644 doc/source/api/nova..objectstore.image.rst create mode 100644 doc/source/api/nova..objectstore.stored.rst create mode 100644 doc/source/api/nova..quota.rst create mode 100644 doc/source/api/nova..rpc.rst create mode 100644 doc/source/api/nova..scheduler.chance.rst create mode 100644 doc/source/api/nova..scheduler.driver.rst create mode 100644 doc/source/api/nova..scheduler.manager.rst create mode 100644 doc/source/api/nova..scheduler.simple.rst create mode 100644 doc/source/api/nova..scheduler.zone.rst create mode 100644 doc/source/api/nova..service.rst create mode 100644 doc/source/api/nova..test.rst create mode 100644 doc/source/api/nova..tests.api.openstack.fakes.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_adminapi.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_api.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_auth.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_common.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_faults.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_flavors.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_images.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_servers.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_zones.rst create mode 100644 doc/source/api/nova..tests.api.test_wsgi.rst create mode 100644 doc/source/api/nova..tests.db.fakes.rst create mode 100644 doc/source/api/nova..tests.declare_flags.rst create mode 100644 doc/source/api/nova..tests.fake_flags.rst create mode 100644 doc/source/api/nova..tests.glance.stubs.rst create mode 100644 doc/source/api/nova..tests.hyperv_unittest.rst create mode 100644 doc/source/api/nova..tests.objectstore_unittest.rst create mode 100644 doc/source/api/nova..tests.real_flags.rst create mode 100644 doc/source/api/nova..tests.runtime_flags.rst create mode 100644 doc/source/api/nova..tests.test_access.rst create mode 100644 doc/source/api/nova..tests.test_api.rst create mode 100644 doc/source/api/nova..tests.test_auth.rst create mode 100644 doc/source/api/nova..tests.test_cloud.rst create mode 100644 doc/source/api/nova..tests.test_compute.rst create mode 100644 doc/source/api/nova..tests.test_console.rst create mode 100644 doc/source/api/nova..tests.test_direct.rst create mode 100644 doc/source/api/nova..tests.test_flags.rst create mode 100644 doc/source/api/nova..tests.test_localization.rst create mode 100644 doc/source/api/nova..tests.test_log.rst create mode 100644 doc/source/api/nova..tests.test_middleware.rst create mode 100644 doc/source/api/nova..tests.test_misc.rst create mode 100644 doc/source/api/nova..tests.test_network.rst create mode 100644 doc/source/api/nova..tests.test_quota.rst create mode 100644 doc/source/api/nova..tests.test_rpc.rst create mode 100644 doc/source/api/nova..tests.test_scheduler.rst create mode 100644 doc/source/api/nova..tests.test_service.rst create mode 100644 doc/source/api/nova..tests.test_twistd.rst create mode 100644 doc/source/api/nova..tests.test_virt.rst create mode 100644 doc/source/api/nova..tests.test_volume.rst create mode 100644 doc/source/api/nova..tests.test_xenapi.rst create mode 100644 doc/source/api/nova..tests.xenapi.stubs.rst create mode 100644 doc/source/api/nova..twistd.rst create mode 100644 doc/source/api/nova..utils.rst create mode 100644 doc/source/api/nova..version.rst create mode 100644 doc/source/api/nova..virt.connection.rst create mode 100644 doc/source/api/nova..virt.disk.rst create mode 100644 doc/source/api/nova..virt.fake.rst create mode 100644 doc/source/api/nova..virt.hyperv.rst create mode 100644 doc/source/api/nova..virt.images.rst create mode 100644 doc/source/api/nova..virt.libvirt_conn.rst create mode 100644 doc/source/api/nova..virt.xenapi.fake.rst create mode 100644 doc/source/api/nova..virt.xenapi.network_utils.rst create mode 100644 doc/source/api/nova..virt.xenapi.vm_utils.rst create mode 100644 doc/source/api/nova..virt.xenapi.vmops.rst create mode 100644 doc/source/api/nova..virt.xenapi.volume_utils.rst create mode 100644 doc/source/api/nova..virt.xenapi.volumeops.rst create mode 100644 doc/source/api/nova..virt.xenapi_conn.rst create mode 100644 doc/source/api/nova..volume.api.rst create mode 100644 doc/source/api/nova..volume.driver.rst create mode 100644 doc/source/api/nova..volume.manager.rst create mode 100644 doc/source/api/nova..volume.san.rst create mode 100644 doc/source/api/nova..wsgi.rst create mode 100644 doc/source/runnova/binaries.rst create mode 100644 doc/source/runnova/euca2ools.rst create mode 100644 doc/source/runnova/flags.rst create mode 100644 doc/source/runnova/getting.started.rst create mode 100644 doc/source/runnova/index.rst create mode 100644 doc/source/runnova/managing.images.rst create mode 100644 doc/source/runnova/managing.instances.rst create mode 100644 doc/source/runnova/managing.networks.rst create mode 100644 doc/source/runnova/managing.projects.rst create mode 100644 doc/source/runnova/managing.users.rst create mode 100644 doc/source/runnova/managingsecurity.rst create mode 100644 doc/source/runnova/monitoring.rst create mode 100644 doc/source/runnova/network.flat.rst create mode 100644 doc/source/runnova/network.vlan.rst create mode 100644 doc/source/runnova/nova.manage.rst create mode 120000 test/.Python create mode 100644 test/bin/activate create mode 100644 test/bin/activate.csh create mode 100644 test/bin/activate.fish create mode 100644 test/bin/activate_this.py create mode 100755 test/bin/easy_install create mode 100755 test/bin/easy_install-2.6 create mode 100755 test/bin/pip create mode 100755 test/bin/pip-2.6 create mode 100755 test/bin/python create mode 120000 test/bin/python2.6 create mode 120000 test/include/python2.6 create mode 120000 test/lib/python2.6/UserDict.py create mode 120000 test/lib/python2.6/_abcoll.py create mode 120000 test/lib/python2.6/abc.py create mode 120000 test/lib/python2.6/codecs.py create mode 120000 test/lib/python2.6/config create mode 120000 test/lib/python2.6/copy_reg.py create mode 100644 test/lib/python2.6/distutils/__init__.py create mode 100644 test/lib/python2.6/distutils/distutils.cfg create mode 120000 test/lib/python2.6/encodings create mode 120000 test/lib/python2.6/fnmatch.py create mode 120000 test/lib/python2.6/genericpath.py create mode 120000 test/lib/python2.6/lib-dynload create mode 120000 test/lib/python2.6/linecache.py create mode 120000 test/lib/python2.6/locale.py create mode 120000 test/lib/python2.6/ntpath.py create mode 100644 test/lib/python2.6/orig-prefix.txt create mode 120000 test/lib/python2.6/os.py create mode 120000 test/lib/python2.6/posixpath.py create mode 120000 test/lib/python2.6/re.py create mode 100644 test/lib/python2.6/site-packages/easy-install.pth create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py create mode 100644 test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg create mode 100644 test/lib/python2.6/site-packages/setuptools.pth create mode 100644 test/lib/python2.6/site.py create mode 120000 test/lib/python2.6/sre.py create mode 120000 test/lib/python2.6/sre_compile.py create mode 120000 test/lib/python2.6/sre_constants.py create mode 120000 test/lib/python2.6/sre_parse.py create mode 120000 test/lib/python2.6/stat.py create mode 120000 test/lib/python2.6/types.py create mode 120000 test/lib/python2.6/warnings.py diff --git a/doc/.autogenerated b/doc/.autogenerated new file mode 100644 index 000000000..e4c98ec9b --- /dev/null +++ b/doc/.autogenerated @@ -0,0 +1,406 @@ +source/api/nova..adminclient.rst +source/api/nova..api.direct.rst +source/api/nova..api.ec2.admin.rst +source/api/nova..api.ec2.apirequest.rst +source/api/nova..api.ec2.cloud.rst +source/api/nova..api.ec2.metadatarequesthandler.rst +source/api/nova..api.openstack.auth.rst +source/api/nova..api.openstack.backup_schedules.rst +source/api/nova..api.openstack.common.rst +source/api/nova..api.openstack.consoles.rst +source/api/nova..api.openstack.faults.rst +source/api/nova..api.openstack.flavors.rst +source/api/nova..api.openstack.images.rst +source/api/nova..api.openstack.servers.rst +source/api/nova..api.openstack.shared_ip_groups.rst +source/api/nova..api.openstack.zones.rst +source/api/nova..auth.dbdriver.rst +source/api/nova..auth.fakeldap.rst +source/api/nova..auth.ldapdriver.rst +source/api/nova..auth.manager.rst +source/api/nova..auth.signer.rst +source/api/nova..cloudpipe.pipelib.rst +source/api/nova..compute.api.rst +source/api/nova..compute.instance_types.rst +source/api/nova..compute.manager.rst +source/api/nova..compute.monitor.rst +source/api/nova..compute.power_state.rst +source/api/nova..console.api.rst +source/api/nova..console.fake.rst +source/api/nova..console.manager.rst +source/api/nova..console.xvp.rst +source/api/nova..context.rst +source/api/nova..crypto.rst +source/api/nova..db.api.rst +source/api/nova..db.base.rst +source/api/nova..db.migration.rst +source/api/nova..db.sqlalchemy.api.rst +source/api/nova..db.sqlalchemy.migrate_repo.manage.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst +source/api/nova..db.sqlalchemy.migration.rst +source/api/nova..db.sqlalchemy.models.rst +source/api/nova..db.sqlalchemy.session.rst +source/api/nova..exception.rst +source/api/nova..fakememcache.rst +source/api/nova..fakerabbit.rst +source/api/nova..flags.rst +source/api/nova..image.glance.rst +source/api/nova..image.local.rst +source/api/nova..image.s3.rst +source/api/nova..image.service.rst +source/api/nova..log.rst +source/api/nova..manager.rst +source/api/nova..network.api.rst +source/api/nova..network.linux_net.rst +source/api/nova..network.manager.rst +source/api/nova..objectstore.bucket.rst +source/api/nova..objectstore.handler.rst +source/api/nova..objectstore.image.rst +source/api/nova..objectstore.stored.rst +source/api/nova..quota.rst +source/api/nova..rpc.rst +source/api/nova..scheduler.chance.rst +source/api/nova..scheduler.driver.rst +source/api/nova..scheduler.manager.rst +source/api/nova..scheduler.simple.rst +source/api/nova..scheduler.zone.rst +source/api/nova..service.rst +source/api/nova..test.rst +source/api/nova..tests.api.openstack.fakes.rst +source/api/nova..tests.api.openstack.test_adminapi.rst +source/api/nova..tests.api.openstack.test_api.rst +source/api/nova..tests.api.openstack.test_auth.rst +source/api/nova..tests.api.openstack.test_common.rst +source/api/nova..tests.api.openstack.test_faults.rst +source/api/nova..tests.api.openstack.test_flavors.rst +source/api/nova..tests.api.openstack.test_images.rst +source/api/nova..tests.api.openstack.test_ratelimiting.rst +source/api/nova..tests.api.openstack.test_servers.rst +source/api/nova..tests.api.openstack.test_shared_ip_groups.rst +source/api/nova..tests.api.openstack.test_zones.rst +source/api/nova..tests.api.test_wsgi.rst +source/api/nova..tests.db.fakes.rst +source/api/nova..tests.declare_flags.rst +source/api/nova..tests.fake_flags.rst +source/api/nova..tests.glance.stubs.rst +source/api/nova..tests.hyperv_unittest.rst +source/api/nova..tests.objectstore_unittest.rst +source/api/nova..tests.real_flags.rst +source/api/nova..tests.runtime_flags.rst +source/api/nova..tests.test_access.rst +source/api/nova..tests.test_api.rst +source/api/nova..tests.test_auth.rst +source/api/nova..tests.test_cloud.rst +source/api/nova..tests.test_compute.rst +source/api/nova..tests.test_console.rst +source/api/nova..tests.test_direct.rst +source/api/nova..tests.test_flags.rst +source/api/nova..tests.test_localization.rst +source/api/nova..tests.test_log.rst +source/api/nova..tests.test_middleware.rst +source/api/nova..tests.test_misc.rst +source/api/nova..tests.test_network.rst +source/api/nova..tests.test_quota.rst +source/api/nova..tests.test_rpc.rst +source/api/nova..tests.test_scheduler.rst +source/api/nova..tests.test_service.rst +source/api/nova..tests.test_twistd.rst +source/api/nova..tests.test_virt.rst +source/api/nova..tests.test_volume.rst +source/api/nova..tests.test_xenapi.rst +source/api/nova..tests.xenapi.stubs.rst +source/api/nova..twistd.rst +source/api/nova..utils.rst +source/api/nova..version.rst +source/api/nova..virt.connection.rst +source/api/nova..virt.disk.rst +source/api/nova..virt.fake.rst +source/api/nova..virt.hyperv.rst +source/api/nova..virt.images.rst +source/api/nova..virt.libvirt_conn.rst +source/api/nova..virt.xenapi.fake.rst +source/api/nova..virt.xenapi.network_utils.rst +source/api/nova..virt.xenapi.vm_utils.rst +source/api/nova..virt.xenapi.vmops.rst +source/api/nova..virt.xenapi.volume_utils.rst +source/api/nova..virt.xenapi.volumeops.rst +source/api/nova..virt.xenapi_conn.rst +source/api/nova..volume.api.rst +source/api/nova..volume.driver.rst +source/api/nova..volume.manager.rst +source/api/nova..volume.san.rst +source/api/nova..wsgi.rst +source/api/autoindex.rst +source/api/nova..adminclient.rst +source/api/nova..api.direct.rst +source/api/nova..api.ec2.admin.rst +source/api/nova..api.ec2.apirequest.rst +source/api/nova..api.ec2.cloud.rst +source/api/nova..api.ec2.metadatarequesthandler.rst +source/api/nova..api.openstack.auth.rst +source/api/nova..api.openstack.backup_schedules.rst +source/api/nova..api.openstack.common.rst +source/api/nova..api.openstack.consoles.rst +source/api/nova..api.openstack.faults.rst +source/api/nova..api.openstack.flavors.rst +source/api/nova..api.openstack.images.rst +source/api/nova..api.openstack.servers.rst +source/api/nova..api.openstack.shared_ip_groups.rst +source/api/nova..api.openstack.zones.rst +source/api/nova..auth.dbdriver.rst +source/api/nova..auth.fakeldap.rst +source/api/nova..auth.ldapdriver.rst +source/api/nova..auth.manager.rst +source/api/nova..auth.signer.rst +source/api/nova..cloudpipe.pipelib.rst +source/api/nova..compute.api.rst +source/api/nova..compute.instance_types.rst +source/api/nova..compute.manager.rst +source/api/nova..compute.monitor.rst +source/api/nova..compute.power_state.rst +source/api/nova..console.api.rst +source/api/nova..console.fake.rst +source/api/nova..console.manager.rst +source/api/nova..console.xvp.rst +source/api/nova..context.rst +source/api/nova..crypto.rst +source/api/nova..db.api.rst +source/api/nova..db.base.rst +source/api/nova..db.migration.rst +source/api/nova..db.sqlalchemy.api.rst +source/api/nova..db.sqlalchemy.migrate_repo.manage.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst +source/api/nova..db.sqlalchemy.migration.rst +source/api/nova..db.sqlalchemy.models.rst +source/api/nova..db.sqlalchemy.session.rst +source/api/nova..exception.rst +source/api/nova..fakememcache.rst +source/api/nova..fakerabbit.rst +source/api/nova..flags.rst +source/api/nova..image.glance.rst +source/api/nova..image.local.rst +source/api/nova..image.s3.rst +source/api/nova..image.service.rst +source/api/nova..log.rst +source/api/nova..manager.rst +source/api/nova..network.api.rst +source/api/nova..network.linux_net.rst +source/api/nova..network.manager.rst +source/api/nova..objectstore.bucket.rst +source/api/nova..objectstore.handler.rst +source/api/nova..objectstore.image.rst +source/api/nova..objectstore.stored.rst +source/api/nova..quota.rst +source/api/nova..rpc.rst +source/api/nova..scheduler.chance.rst +source/api/nova..scheduler.driver.rst +source/api/nova..scheduler.manager.rst +source/api/nova..scheduler.simple.rst +source/api/nova..scheduler.zone.rst +source/api/nova..service.rst +source/api/nova..test.rst +source/api/nova..tests.api.openstack.fakes.rst +source/api/nova..tests.api.openstack.test_adminapi.rst +source/api/nova..tests.api.openstack.test_api.rst +source/api/nova..tests.api.openstack.test_auth.rst +source/api/nova..tests.api.openstack.test_common.rst +source/api/nova..tests.api.openstack.test_faults.rst +source/api/nova..tests.api.openstack.test_flavors.rst +source/api/nova..tests.api.openstack.test_images.rst +source/api/nova..tests.api.openstack.test_ratelimiting.rst +source/api/nova..tests.api.openstack.test_servers.rst +source/api/nova..tests.api.openstack.test_shared_ip_groups.rst +source/api/nova..tests.api.openstack.test_zones.rst +source/api/nova..tests.api.test_wsgi.rst +source/api/nova..tests.db.fakes.rst +source/api/nova..tests.declare_flags.rst +source/api/nova..tests.fake_flags.rst +source/api/nova..tests.glance.stubs.rst +source/api/nova..tests.hyperv_unittest.rst +source/api/nova..tests.objectstore_unittest.rst +source/api/nova..tests.real_flags.rst +source/api/nova..tests.runtime_flags.rst +source/api/nova..tests.test_access.rst +source/api/nova..tests.test_api.rst +source/api/nova..tests.test_auth.rst +source/api/nova..tests.test_cloud.rst +source/api/nova..tests.test_compute.rst +source/api/nova..tests.test_console.rst +source/api/nova..tests.test_direct.rst +source/api/nova..tests.test_flags.rst +source/api/nova..tests.test_localization.rst +source/api/nova..tests.test_log.rst +source/api/nova..tests.test_middleware.rst +source/api/nova..tests.test_misc.rst +source/api/nova..tests.test_network.rst +source/api/nova..tests.test_quota.rst +source/api/nova..tests.test_rpc.rst +source/api/nova..tests.test_scheduler.rst +source/api/nova..tests.test_service.rst +source/api/nova..tests.test_twistd.rst +source/api/nova..tests.test_virt.rst +source/api/nova..tests.test_volume.rst +source/api/nova..tests.test_xenapi.rst +source/api/nova..tests.xenapi.stubs.rst +source/api/nova..twistd.rst +source/api/nova..utils.rst +source/api/nova..version.rst +source/api/nova..virt.connection.rst +source/api/nova..virt.disk.rst +source/api/nova..virt.fake.rst +source/api/nova..virt.hyperv.rst +source/api/nova..virt.images.rst +source/api/nova..virt.libvirt_conn.rst +source/api/nova..virt.xenapi.fake.rst +source/api/nova..virt.xenapi.network_utils.rst +source/api/nova..virt.xenapi.vm_utils.rst +source/api/nova..virt.xenapi.vmops.rst +source/api/nova..virt.xenapi.volume_utils.rst +source/api/nova..virt.xenapi.volumeops.rst +source/api/nova..virt.xenapi_conn.rst +source/api/nova..volume.api.rst +source/api/nova..volume.driver.rst +source/api/nova..volume.manager.rst +source/api/nova..volume.san.rst +source/api/nova..wsgi.rst +source/api/nova..adminclient.rst +source/api/nova..api.direct.rst +source/api/nova..api.ec2.admin.rst +source/api/nova..api.ec2.apirequest.rst +source/api/nova..api.ec2.cloud.rst +source/api/nova..api.ec2.metadatarequesthandler.rst +source/api/nova..api.openstack.auth.rst +source/api/nova..api.openstack.backup_schedules.rst +source/api/nova..api.openstack.common.rst +source/api/nova..api.openstack.consoles.rst +source/api/nova..api.openstack.faults.rst +source/api/nova..api.openstack.flavors.rst +source/api/nova..api.openstack.images.rst +source/api/nova..api.openstack.servers.rst +source/api/nova..api.openstack.shared_ip_groups.rst +source/api/nova..api.openstack.zones.rst +source/api/nova..auth.dbdriver.rst +source/api/nova..auth.fakeldap.rst +source/api/nova..auth.ldapdriver.rst +source/api/nova..auth.manager.rst +source/api/nova..auth.signer.rst +source/api/nova..cloudpipe.pipelib.rst +source/api/nova..compute.api.rst +source/api/nova..compute.instance_types.rst +source/api/nova..compute.manager.rst +source/api/nova..compute.monitor.rst +source/api/nova..compute.power_state.rst +source/api/nova..console.api.rst +source/api/nova..console.fake.rst +source/api/nova..console.manager.rst +source/api/nova..console.xvp.rst +source/api/nova..context.rst +source/api/nova..crypto.rst +source/api/nova..db.api.rst +source/api/nova..db.base.rst +source/api/nova..db.migration.rst +source/api/nova..db.sqlalchemy.api.rst +source/api/nova..db.sqlalchemy.migrate_repo.manage.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst +source/api/nova..db.sqlalchemy.migration.rst +source/api/nova..db.sqlalchemy.models.rst +source/api/nova..db.sqlalchemy.session.rst +source/api/nova..exception.rst +source/api/nova..fakememcache.rst +source/api/nova..fakerabbit.rst +source/api/nova..flags.rst +source/api/nova..image.glance.rst +source/api/nova..image.local.rst +source/api/nova..image.s3.rst +source/api/nova..image.service.rst +source/api/nova..log.rst +source/api/nova..manager.rst +source/api/nova..network.api.rst +source/api/nova..network.linux_net.rst +source/api/nova..network.manager.rst +source/api/nova..objectstore.bucket.rst +source/api/nova..objectstore.handler.rst +source/api/nova..objectstore.image.rst +source/api/nova..objectstore.stored.rst +source/api/nova..quota.rst +source/api/nova..rpc.rst +source/api/nova..scheduler.chance.rst +source/api/nova..scheduler.driver.rst +source/api/nova..scheduler.manager.rst +source/api/nova..scheduler.simple.rst +source/api/nova..scheduler.zone.rst +source/api/nova..service.rst +source/api/nova..test.rst +source/api/nova..tests.api.openstack.fakes.rst +source/api/nova..tests.api.openstack.test_adminapi.rst +source/api/nova..tests.api.openstack.test_api.rst +source/api/nova..tests.api.openstack.test_auth.rst +source/api/nova..tests.api.openstack.test_common.rst +source/api/nova..tests.api.openstack.test_faults.rst +source/api/nova..tests.api.openstack.test_flavors.rst +source/api/nova..tests.api.openstack.test_images.rst +source/api/nova..tests.api.openstack.test_ratelimiting.rst +source/api/nova..tests.api.openstack.test_servers.rst +source/api/nova..tests.api.openstack.test_shared_ip_groups.rst +source/api/nova..tests.api.openstack.test_zones.rst +source/api/nova..tests.api.test_wsgi.rst +source/api/nova..tests.db.fakes.rst +source/api/nova..tests.declare_flags.rst +source/api/nova..tests.fake_flags.rst +source/api/nova..tests.glance.stubs.rst +source/api/nova..tests.hyperv_unittest.rst +source/api/nova..tests.objectstore_unittest.rst +source/api/nova..tests.real_flags.rst +source/api/nova..tests.runtime_flags.rst +source/api/nova..tests.test_access.rst +source/api/nova..tests.test_api.rst +source/api/nova..tests.test_auth.rst +source/api/nova..tests.test_cloud.rst +source/api/nova..tests.test_compute.rst +source/api/nova..tests.test_console.rst +source/api/nova..tests.test_direct.rst +source/api/nova..tests.test_flags.rst +source/api/nova..tests.test_localization.rst +source/api/nova..tests.test_log.rst +source/api/nova..tests.test_middleware.rst +source/api/nova..tests.test_misc.rst +source/api/nova..tests.test_network.rst +source/api/nova..tests.test_quota.rst +source/api/nova..tests.test_rpc.rst +source/api/nova..tests.test_scheduler.rst +source/api/nova..tests.test_service.rst +source/api/nova..tests.test_twistd.rst +source/api/nova..tests.test_virt.rst +source/api/nova..tests.test_volume.rst +source/api/nova..tests.test_xenapi.rst +source/api/nova..tests.xenapi.stubs.rst +source/api/nova..twistd.rst +source/api/nova..utils.rst +source/api/nova..version.rst +source/api/nova..virt.connection.rst +source/api/nova..virt.disk.rst +source/api/nova..virt.fake.rst +source/api/nova..virt.hyperv.rst +source/api/nova..virt.images.rst +source/api/nova..virt.libvirt_conn.rst +source/api/nova..virt.xenapi.fake.rst +source/api/nova..virt.xenapi.network_utils.rst +source/api/nova..virt.xenapi.vm_utils.rst +source/api/nova..virt.xenapi.vmops.rst +source/api/nova..virt.xenapi.volume_utils.rst +source/api/nova..virt.xenapi.volumeops.rst +source/api/nova..virt.xenapi_conn.rst +source/api/nova..volume.api.rst +source/api/nova..volume.driver.rst +source/api/nova..volume.manager.rst +source/api/nova..volume.san.rst +source/api/nova..wsgi.rst diff --git a/doc/build/.DS_Store b/doc/build/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/doc/build/.DS_Store differ diff --git a/doc/build/html/.DS_Store b/doc/build/html/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/doc/build/html/.DS_Store differ diff --git a/doc/build/html/.buildinfo b/doc/build/html/.buildinfo new file mode 100644 index 000000000..091736d4f --- /dev/null +++ b/doc/build/html/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. +config: 2a2fe6198f4be4a4d6f289b09d16d74a +tags: fbb0d17656682115ca4d033fb2f83ba1 diff --git a/doc/source/api/autoindex.rst b/doc/source/api/autoindex.rst new file mode 100644 index 000000000..41fc1f4a9 --- /dev/null +++ b/doc/source/api/autoindex.rst @@ -0,0 +1,138 @@ +.. toctree:: + :maxdepth: 1 + + nova..adminclient.rst + nova..api.direct.rst + nova..api.ec2.admin.rst + nova..api.ec2.apirequest.rst + nova..api.ec2.cloud.rst + nova..api.ec2.metadatarequesthandler.rst + nova..api.openstack.auth.rst + nova..api.openstack.backup_schedules.rst + nova..api.openstack.common.rst + nova..api.openstack.consoles.rst + nova..api.openstack.faults.rst + nova..api.openstack.flavors.rst + nova..api.openstack.images.rst + nova..api.openstack.servers.rst + nova..api.openstack.shared_ip_groups.rst + nova..api.openstack.zones.rst + nova..auth.dbdriver.rst + nova..auth.fakeldap.rst + nova..auth.ldapdriver.rst + nova..auth.manager.rst + nova..auth.signer.rst + nova..cloudpipe.pipelib.rst + nova..compute.api.rst + nova..compute.instance_types.rst + nova..compute.manager.rst + nova..compute.monitor.rst + nova..compute.power_state.rst + nova..console.api.rst + nova..console.fake.rst + nova..console.manager.rst + nova..console.xvp.rst + nova..context.rst + nova..crypto.rst + nova..db.api.rst + nova..db.base.rst + nova..db.migration.rst + nova..db.sqlalchemy.api.rst + nova..db.sqlalchemy.migrate_repo.manage.rst + nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst + nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst + nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst + nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst + nova..db.sqlalchemy.migration.rst + nova..db.sqlalchemy.models.rst + nova..db.sqlalchemy.session.rst + nova..exception.rst + nova..fakememcache.rst + nova..fakerabbit.rst + nova..flags.rst + nova..image.glance.rst + nova..image.local.rst + nova..image.s3.rst + nova..image.service.rst + nova..log.rst + nova..manager.rst + nova..network.api.rst + nova..network.linux_net.rst + nova..network.manager.rst + nova..objectstore.bucket.rst + nova..objectstore.handler.rst + nova..objectstore.image.rst + nova..objectstore.stored.rst + nova..quota.rst + nova..rpc.rst + nova..scheduler.chance.rst + nova..scheduler.driver.rst + nova..scheduler.manager.rst + nova..scheduler.simple.rst + nova..scheduler.zone.rst + nova..service.rst + nova..test.rst + nova..tests.api.openstack.fakes.rst + nova..tests.api.openstack.test_adminapi.rst + nova..tests.api.openstack.test_api.rst + nova..tests.api.openstack.test_auth.rst + nova..tests.api.openstack.test_common.rst + nova..tests.api.openstack.test_faults.rst + nova..tests.api.openstack.test_flavors.rst + nova..tests.api.openstack.test_images.rst + nova..tests.api.openstack.test_ratelimiting.rst + nova..tests.api.openstack.test_servers.rst + nova..tests.api.openstack.test_shared_ip_groups.rst + nova..tests.api.openstack.test_zones.rst + nova..tests.api.test_wsgi.rst + nova..tests.db.fakes.rst + nova..tests.declare_flags.rst + nova..tests.fake_flags.rst + nova..tests.glance.stubs.rst + nova..tests.hyperv_unittest.rst + nova..tests.objectstore_unittest.rst + nova..tests.real_flags.rst + nova..tests.runtime_flags.rst + nova..tests.test_access.rst + nova..tests.test_api.rst + nova..tests.test_auth.rst + nova..tests.test_cloud.rst + nova..tests.test_compute.rst + nova..tests.test_console.rst + nova..tests.test_direct.rst + nova..tests.test_flags.rst + nova..tests.test_localization.rst + nova..tests.test_log.rst + nova..tests.test_middleware.rst + nova..tests.test_misc.rst + nova..tests.test_network.rst + nova..tests.test_quota.rst + nova..tests.test_rpc.rst + nova..tests.test_scheduler.rst + nova..tests.test_service.rst + nova..tests.test_twistd.rst + nova..tests.test_virt.rst + nova..tests.test_volume.rst + nova..tests.test_xenapi.rst + nova..tests.xenapi.stubs.rst + nova..twistd.rst + nova..utils.rst + nova..version.rst + nova..virt.connection.rst + nova..virt.disk.rst + nova..virt.fake.rst + nova..virt.hyperv.rst + nova..virt.images.rst + nova..virt.libvirt_conn.rst + nova..virt.xenapi.fake.rst + nova..virt.xenapi.network_utils.rst + nova..virt.xenapi.vm_utils.rst + nova..virt.xenapi.vmops.rst + nova..virt.xenapi.volume_utils.rst + nova..virt.xenapi.volumeops.rst + nova..virt.xenapi_conn.rst + nova..volume.api.rst + nova..volume.driver.rst + nova..volume.manager.rst + nova..volume.san.rst + nova..wsgi.rst diff --git a/doc/source/api/nova..adminclient.rst b/doc/source/api/nova..adminclient.rst new file mode 100644 index 000000000..35fa839e1 --- /dev/null +++ b/doc/source/api/nova..adminclient.rst @@ -0,0 +1,6 @@ +The :mod:`nova..adminclient` Module +============================================================================== +.. automodule:: nova..adminclient + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.direct.rst b/doc/source/api/nova..api.direct.rst new file mode 100644 index 000000000..a1705c707 --- /dev/null +++ b/doc/source/api/nova..api.direct.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.direct` Module +============================================================================== +.. automodule:: nova..api.direct + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.admin.rst b/doc/source/api/nova..api.ec2.admin.rst new file mode 100644 index 000000000..4e9ab308b --- /dev/null +++ b/doc/source/api/nova..api.ec2.admin.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.admin` Module +============================================================================== +.. automodule:: nova..api.ec2.admin + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.apirequest.rst b/doc/source/api/nova..api.ec2.apirequest.rst new file mode 100644 index 000000000..c17a2ff3a --- /dev/null +++ b/doc/source/api/nova..api.ec2.apirequest.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.apirequest` Module +============================================================================== +.. automodule:: nova..api.ec2.apirequest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.cloud.rst b/doc/source/api/nova..api.ec2.cloud.rst new file mode 100644 index 000000000..f6145c217 --- /dev/null +++ b/doc/source/api/nova..api.ec2.cloud.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.cloud` Module +============================================================================== +.. automodule:: nova..api.ec2.cloud + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.metadatarequesthandler.rst b/doc/source/api/nova..api.ec2.metadatarequesthandler.rst new file mode 100644 index 000000000..75f5169e5 --- /dev/null +++ b/doc/source/api/nova..api.ec2.metadatarequesthandler.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.metadatarequesthandler` Module +============================================================================== +.. automodule:: nova..api.ec2.metadatarequesthandler + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.auth.rst b/doc/source/api/nova..api.openstack.auth.rst new file mode 100644 index 000000000..8c3f8f2da --- /dev/null +++ b/doc/source/api/nova..api.openstack.auth.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.auth` Module +============================================================================== +.. automodule:: nova..api.openstack.auth + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.backup_schedules.rst b/doc/source/api/nova..api.openstack.backup_schedules.rst new file mode 100644 index 000000000..6b406f12d --- /dev/null +++ b/doc/source/api/nova..api.openstack.backup_schedules.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.backup_schedules` Module +============================================================================== +.. automodule:: nova..api.openstack.backup_schedules + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.common.rst b/doc/source/api/nova..api.openstack.common.rst new file mode 100644 index 000000000..4fd734790 --- /dev/null +++ b/doc/source/api/nova..api.openstack.common.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.common` Module +============================================================================== +.. automodule:: nova..api.openstack.common + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.consoles.rst b/doc/source/api/nova..api.openstack.consoles.rst new file mode 100644 index 000000000..1e3e09599 --- /dev/null +++ b/doc/source/api/nova..api.openstack.consoles.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.consoles` Module +============================================================================== +.. automodule:: nova..api.openstack.consoles + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.faults.rst b/doc/source/api/nova..api.openstack.faults.rst new file mode 100644 index 000000000..7b25561f7 --- /dev/null +++ b/doc/source/api/nova..api.openstack.faults.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.faults` Module +============================================================================== +.. automodule:: nova..api.openstack.faults + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.flavors.rst b/doc/source/api/nova..api.openstack.flavors.rst new file mode 100644 index 000000000..0deb724de --- /dev/null +++ b/doc/source/api/nova..api.openstack.flavors.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.flavors` Module +============================================================================== +.. automodule:: nova..api.openstack.flavors + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.images.rst b/doc/source/api/nova..api.openstack.images.rst new file mode 100644 index 000000000..82bd5f1e8 --- /dev/null +++ b/doc/source/api/nova..api.openstack.images.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.images` Module +============================================================================== +.. automodule:: nova..api.openstack.images + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.servers.rst b/doc/source/api/nova..api.openstack.servers.rst new file mode 100644 index 000000000..c36856ea2 --- /dev/null +++ b/doc/source/api/nova..api.openstack.servers.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.servers` Module +============================================================================== +.. automodule:: nova..api.openstack.servers + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.shared_ip_groups.rst b/doc/source/api/nova..api.openstack.shared_ip_groups.rst new file mode 100644 index 000000000..4b1f44efe --- /dev/null +++ b/doc/source/api/nova..api.openstack.shared_ip_groups.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.shared_ip_groups` Module +============================================================================== +.. automodule:: nova..api.openstack.shared_ip_groups + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.zones.rst b/doc/source/api/nova..api.openstack.zones.rst new file mode 100644 index 000000000..ebe4569c5 --- /dev/null +++ b/doc/source/api/nova..api.openstack.zones.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.zones` Module +============================================================================== +.. automodule:: nova..api.openstack.zones + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.dbdriver.rst b/doc/source/api/nova..auth.dbdriver.rst new file mode 100644 index 000000000..7de68b6e0 --- /dev/null +++ b/doc/source/api/nova..auth.dbdriver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.dbdriver` Module +============================================================================== +.. automodule:: nova..auth.dbdriver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.fakeldap.rst b/doc/source/api/nova..auth.fakeldap.rst new file mode 100644 index 000000000..ca8a3ad4d --- /dev/null +++ b/doc/source/api/nova..auth.fakeldap.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.fakeldap` Module +============================================================================== +.. automodule:: nova..auth.fakeldap + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.ldapdriver.rst b/doc/source/api/nova..auth.ldapdriver.rst new file mode 100644 index 000000000..c44463522 --- /dev/null +++ b/doc/source/api/nova..auth.ldapdriver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.ldapdriver` Module +============================================================================== +.. automodule:: nova..auth.ldapdriver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.manager.rst b/doc/source/api/nova..auth.manager.rst new file mode 100644 index 000000000..bc5ce2ec3 --- /dev/null +++ b/doc/source/api/nova..auth.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.manager` Module +============================================================================== +.. automodule:: nova..auth.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.signer.rst b/doc/source/api/nova..auth.signer.rst new file mode 100644 index 000000000..aad824ead --- /dev/null +++ b/doc/source/api/nova..auth.signer.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.signer` Module +============================================================================== +.. automodule:: nova..auth.signer + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..cloudpipe.pipelib.rst b/doc/source/api/nova..cloudpipe.pipelib.rst new file mode 100644 index 000000000..054aaf484 --- /dev/null +++ b/doc/source/api/nova..cloudpipe.pipelib.rst @@ -0,0 +1,6 @@ +The :mod:`nova..cloudpipe.pipelib` Module +============================================================================== +.. automodule:: nova..cloudpipe.pipelib + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.api.rst b/doc/source/api/nova..compute.api.rst new file mode 100644 index 000000000..caa66313a --- /dev/null +++ b/doc/source/api/nova..compute.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.api` Module +============================================================================== +.. automodule:: nova..compute.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.instance_types.rst b/doc/source/api/nova..compute.instance_types.rst new file mode 100644 index 000000000..d206ff3a4 --- /dev/null +++ b/doc/source/api/nova..compute.instance_types.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.instance_types` Module +============================================================================== +.. automodule:: nova..compute.instance_types + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.manager.rst b/doc/source/api/nova..compute.manager.rst new file mode 100644 index 000000000..33a337c39 --- /dev/null +++ b/doc/source/api/nova..compute.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.manager` Module +============================================================================== +.. automodule:: nova..compute.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.monitor.rst b/doc/source/api/nova..compute.monitor.rst new file mode 100644 index 000000000..a91169ecd --- /dev/null +++ b/doc/source/api/nova..compute.monitor.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.monitor` Module +============================================================================== +.. automodule:: nova..compute.monitor + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.power_state.rst b/doc/source/api/nova..compute.power_state.rst new file mode 100644 index 000000000..41b1080e5 --- /dev/null +++ b/doc/source/api/nova..compute.power_state.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.power_state` Module +============================================================================== +.. automodule:: nova..compute.power_state + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.api.rst b/doc/source/api/nova..console.api.rst new file mode 100644 index 000000000..82a51d4c7 --- /dev/null +++ b/doc/source/api/nova..console.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.api` Module +============================================================================== +.. automodule:: nova..console.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.fake.rst b/doc/source/api/nova..console.fake.rst new file mode 100644 index 000000000..f053f85d6 --- /dev/null +++ b/doc/source/api/nova..console.fake.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.fake` Module +============================================================================== +.. automodule:: nova..console.fake + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.manager.rst b/doc/source/api/nova..console.manager.rst new file mode 100644 index 000000000..f9283a6c3 --- /dev/null +++ b/doc/source/api/nova..console.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.manager` Module +============================================================================== +.. automodule:: nova..console.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.xvp.rst b/doc/source/api/nova..console.xvp.rst new file mode 100644 index 000000000..a0887009e --- /dev/null +++ b/doc/source/api/nova..console.xvp.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.xvp` Module +============================================================================== +.. automodule:: nova..console.xvp + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..context.rst b/doc/source/api/nova..context.rst new file mode 100644 index 000000000..9de1adb24 --- /dev/null +++ b/doc/source/api/nova..context.rst @@ -0,0 +1,6 @@ +The :mod:`nova..context` Module +============================================================================== +.. automodule:: nova..context + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..crypto.rst b/doc/source/api/nova..crypto.rst new file mode 100644 index 000000000..af9f63634 --- /dev/null +++ b/doc/source/api/nova..crypto.rst @@ -0,0 +1,6 @@ +The :mod:`nova..crypto` Module +============================================================================== +.. automodule:: nova..crypto + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.api.rst b/doc/source/api/nova..db.api.rst new file mode 100644 index 000000000..6d998fbb2 --- /dev/null +++ b/doc/source/api/nova..db.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.api` Module +============================================================================== +.. automodule:: nova..db.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.base.rst b/doc/source/api/nova..db.base.rst new file mode 100644 index 000000000..29fb417d6 --- /dev/null +++ b/doc/source/api/nova..db.base.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.base` Module +============================================================================== +.. automodule:: nova..db.base + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.migration.rst b/doc/source/api/nova..db.migration.rst new file mode 100644 index 000000000..71dfea301 --- /dev/null +++ b/doc/source/api/nova..db.migration.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.migration` Module +============================================================================== +.. automodule:: nova..db.migration + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.api.rst b/doc/source/api/nova..db.sqlalchemy.api.rst new file mode 100644 index 000000000..76d0c1bd3 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.api` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst new file mode 100644 index 000000000..93decfb27 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.manage` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.manage + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst new file mode 100644 index 000000000..4b1219edb --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.001_austin` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.001_austin + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst new file mode 100644 index 000000000..82f1f4680 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.002_bexar` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.002_bexar + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst new file mode 100644 index 000000000..98f3e8da7 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst new file mode 100644 index 000000000..5cbb81191 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migration.rst b/doc/source/api/nova..db.sqlalchemy.migration.rst new file mode 100644 index 000000000..3a9b01b9a --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migration.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migration` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migration + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.models.rst b/doc/source/api/nova..db.sqlalchemy.models.rst new file mode 100644 index 000000000..9c795d7f5 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.models.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.models` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.models + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.session.rst b/doc/source/api/nova..db.sqlalchemy.session.rst new file mode 100644 index 000000000..cbfd6416a --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.session.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.session` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.session + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..exception.rst b/doc/source/api/nova..exception.rst new file mode 100644 index 000000000..97ac6b752 --- /dev/null +++ b/doc/source/api/nova..exception.rst @@ -0,0 +1,6 @@ +The :mod:`nova..exception` Module +============================================================================== +.. automodule:: nova..exception + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..fakememcache.rst b/doc/source/api/nova..fakememcache.rst new file mode 100644 index 000000000..7e7ffb98b --- /dev/null +++ b/doc/source/api/nova..fakememcache.rst @@ -0,0 +1,6 @@ +The :mod:`nova..fakememcache` Module +============================================================================== +.. automodule:: nova..fakememcache + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..fakerabbit.rst b/doc/source/api/nova..fakerabbit.rst new file mode 100644 index 000000000..f1e27c266 --- /dev/null +++ b/doc/source/api/nova..fakerabbit.rst @@ -0,0 +1,6 @@ +The :mod:`nova..fakerabbit` Module +============================================================================== +.. automodule:: nova..fakerabbit + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..flags.rst b/doc/source/api/nova..flags.rst new file mode 100644 index 000000000..08165be44 --- /dev/null +++ b/doc/source/api/nova..flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..flags` Module +============================================================================== +.. automodule:: nova..flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.glance.rst b/doc/source/api/nova..image.glance.rst new file mode 100644 index 000000000..b0882d5ec --- /dev/null +++ b/doc/source/api/nova..image.glance.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.glance` Module +============================================================================== +.. automodule:: nova..image.glance + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.local.rst b/doc/source/api/nova..image.local.rst new file mode 100644 index 000000000..b6ad5470b --- /dev/null +++ b/doc/source/api/nova..image.local.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.local` Module +============================================================================== +.. automodule:: nova..image.local + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.s3.rst b/doc/source/api/nova..image.s3.rst new file mode 100644 index 000000000..e5b236127 --- /dev/null +++ b/doc/source/api/nova..image.s3.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.s3` Module +============================================================================== +.. automodule:: nova..image.s3 + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.service.rst b/doc/source/api/nova..image.service.rst new file mode 100644 index 000000000..78ef1ecca --- /dev/null +++ b/doc/source/api/nova..image.service.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.service` Module +============================================================================== +.. automodule:: nova..image.service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..log.rst b/doc/source/api/nova..log.rst new file mode 100644 index 000000000..ff209709f --- /dev/null +++ b/doc/source/api/nova..log.rst @@ -0,0 +1,6 @@ +The :mod:`nova..log` Module +============================================================================== +.. automodule:: nova..log + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..manager.rst b/doc/source/api/nova..manager.rst new file mode 100644 index 000000000..576902491 --- /dev/null +++ b/doc/source/api/nova..manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..manager` Module +============================================================================== +.. automodule:: nova..manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..network.api.rst b/doc/source/api/nova..network.api.rst new file mode 100644 index 000000000..b63be2ba3 --- /dev/null +++ b/doc/source/api/nova..network.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..network.api` Module +============================================================================== +.. automodule:: nova..network.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..network.linux_net.rst b/doc/source/api/nova..network.linux_net.rst new file mode 100644 index 000000000..7af78d5ad --- /dev/null +++ b/doc/source/api/nova..network.linux_net.rst @@ -0,0 +1,6 @@ +The :mod:`nova..network.linux_net` Module +============================================================================== +.. automodule:: nova..network.linux_net + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..network.manager.rst b/doc/source/api/nova..network.manager.rst new file mode 100644 index 000000000..0ea705533 --- /dev/null +++ b/doc/source/api/nova..network.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..network.manager` Module +============================================================================== +.. automodule:: nova..network.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.bucket.rst b/doc/source/api/nova..objectstore.bucket.rst new file mode 100644 index 000000000..3bfdf639c --- /dev/null +++ b/doc/source/api/nova..objectstore.bucket.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.bucket` Module +============================================================================== +.. automodule:: nova..objectstore.bucket + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.handler.rst b/doc/source/api/nova..objectstore.handler.rst new file mode 100644 index 000000000..0eb8c4efb --- /dev/null +++ b/doc/source/api/nova..objectstore.handler.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.handler` Module +============================================================================== +.. automodule:: nova..objectstore.handler + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.image.rst b/doc/source/api/nova..objectstore.image.rst new file mode 100644 index 000000000..fa4c971f1 --- /dev/null +++ b/doc/source/api/nova..objectstore.image.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.image` Module +============================================================================== +.. automodule:: nova..objectstore.image + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.stored.rst b/doc/source/api/nova..objectstore.stored.rst new file mode 100644 index 000000000..2b1d997a3 --- /dev/null +++ b/doc/source/api/nova..objectstore.stored.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.stored` Module +============================================================================== +.. automodule:: nova..objectstore.stored + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..quota.rst b/doc/source/api/nova..quota.rst new file mode 100644 index 000000000..4140d95d6 --- /dev/null +++ b/doc/source/api/nova..quota.rst @@ -0,0 +1,6 @@ +The :mod:`nova..quota` Module +============================================================================== +.. automodule:: nova..quota + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..rpc.rst b/doc/source/api/nova..rpc.rst new file mode 100644 index 000000000..5b2a9b8e2 --- /dev/null +++ b/doc/source/api/nova..rpc.rst @@ -0,0 +1,6 @@ +The :mod:`nova..rpc` Module +============================================================================== +.. automodule:: nova..rpc + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.chance.rst b/doc/source/api/nova..scheduler.chance.rst new file mode 100644 index 000000000..89c074c8f --- /dev/null +++ b/doc/source/api/nova..scheduler.chance.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.chance` Module +============================================================================== +.. automodule:: nova..scheduler.chance + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.driver.rst b/doc/source/api/nova..scheduler.driver.rst new file mode 100644 index 000000000..793ed9c7b --- /dev/null +++ b/doc/source/api/nova..scheduler.driver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.driver` Module +============================================================================== +.. automodule:: nova..scheduler.driver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.manager.rst b/doc/source/api/nova..scheduler.manager.rst new file mode 100644 index 000000000..d0fc7c423 --- /dev/null +++ b/doc/source/api/nova..scheduler.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.manager` Module +============================================================================== +.. automodule:: nova..scheduler.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.simple.rst b/doc/source/api/nova..scheduler.simple.rst new file mode 100644 index 000000000..dacc2cf30 --- /dev/null +++ b/doc/source/api/nova..scheduler.simple.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.simple` Module +============================================================================== +.. automodule:: nova..scheduler.simple + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.zone.rst b/doc/source/api/nova..scheduler.zone.rst new file mode 100644 index 000000000..54c4bf201 --- /dev/null +++ b/doc/source/api/nova..scheduler.zone.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.zone` Module +============================================================================== +.. automodule:: nova..scheduler.zone + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..service.rst b/doc/source/api/nova..service.rst new file mode 100644 index 000000000..2d2dfcf2e --- /dev/null +++ b/doc/source/api/nova..service.rst @@ -0,0 +1,6 @@ +The :mod:`nova..service` Module +============================================================================== +.. automodule:: nova..service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..test.rst b/doc/source/api/nova..test.rst new file mode 100644 index 000000000..a6bdb6f1f --- /dev/null +++ b/doc/source/api/nova..test.rst @@ -0,0 +1,6 @@ +The :mod:`nova..test` Module +============================================================================== +.. automodule:: nova..test + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.fakes.rst b/doc/source/api/nova..tests.api.openstack.fakes.rst new file mode 100644 index 000000000..4a9ff5938 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.fakes.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.fakes` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.fakes + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_adminapi.rst b/doc/source/api/nova..tests.api.openstack.test_adminapi.rst new file mode 100644 index 000000000..19a85ca0f --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_adminapi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_adminapi` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_adminapi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_api.rst b/doc/source/api/nova..tests.api.openstack.test_api.rst new file mode 100644 index 000000000..68106d221 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_api` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_auth.rst b/doc/source/api/nova..tests.api.openstack.test_auth.rst new file mode 100644 index 000000000..9f0011669 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_auth.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_auth` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_auth + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_common.rst b/doc/source/api/nova..tests.api.openstack.test_common.rst new file mode 100644 index 000000000..82f40ecb8 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_common.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_common` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_common + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_faults.rst b/doc/source/api/nova..tests.api.openstack.test_faults.rst new file mode 100644 index 000000000..b839ae8a3 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_faults.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_faults` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_faults + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_flavors.rst b/doc/source/api/nova..tests.api.openstack.test_flavors.rst new file mode 100644 index 000000000..471fac56e --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_flavors.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_flavors` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_flavors + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_images.rst b/doc/source/api/nova..tests.api.openstack.test_images.rst new file mode 100644 index 000000000..57ae93c8c --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_images.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_images` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_images + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst b/doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst new file mode 100644 index 000000000..9a857f795 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_ratelimiting` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_ratelimiting + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_servers.rst b/doc/source/api/nova..tests.api.openstack.test_servers.rst new file mode 100644 index 000000000..ea602e6ab --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_servers.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_servers` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_servers + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst b/doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst new file mode 100644 index 000000000..48814af00 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_shared_ip_groups` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_shared_ip_groups + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_zones.rst b/doc/source/api/nova..tests.api.openstack.test_zones.rst new file mode 100644 index 000000000..ba7078e63 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_zones.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_zones` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_zones + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.test_wsgi.rst b/doc/source/api/nova..tests.api.test_wsgi.rst new file mode 100644 index 000000000..8e79caa4d --- /dev/null +++ b/doc/source/api/nova..tests.api.test_wsgi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.test_wsgi` Module +============================================================================== +.. automodule:: nova..tests.api.test_wsgi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.db.fakes.rst b/doc/source/api/nova..tests.db.fakes.rst new file mode 100644 index 000000000..cc79e55e2 --- /dev/null +++ b/doc/source/api/nova..tests.db.fakes.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.db.fakes` Module +============================================================================== +.. automodule:: nova..tests.db.fakes + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.declare_flags.rst b/doc/source/api/nova..tests.declare_flags.rst new file mode 100644 index 000000000..524e72e91 --- /dev/null +++ b/doc/source/api/nova..tests.declare_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.declare_flags` Module +============================================================================== +.. automodule:: nova..tests.declare_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.fake_flags.rst b/doc/source/api/nova..tests.fake_flags.rst new file mode 100644 index 000000000..a8dc3df36 --- /dev/null +++ b/doc/source/api/nova..tests.fake_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.fake_flags` Module +============================================================================== +.. automodule:: nova..tests.fake_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.glance.stubs.rst b/doc/source/api/nova..tests.glance.stubs.rst new file mode 100644 index 000000000..7ef5fccbe --- /dev/null +++ b/doc/source/api/nova..tests.glance.stubs.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.glance.stubs` Module +============================================================================== +.. automodule:: nova..tests.glance.stubs + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.hyperv_unittest.rst b/doc/source/api/nova..tests.hyperv_unittest.rst new file mode 100644 index 000000000..c08443121 --- /dev/null +++ b/doc/source/api/nova..tests.hyperv_unittest.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.hyperv_unittest` Module +============================================================================== +.. automodule:: nova..tests.hyperv_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.objectstore_unittest.rst b/doc/source/api/nova..tests.objectstore_unittest.rst new file mode 100644 index 000000000..0ae252f04 --- /dev/null +++ b/doc/source/api/nova..tests.objectstore_unittest.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.objectstore_unittest` Module +============================================================================== +.. automodule:: nova..tests.objectstore_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.real_flags.rst b/doc/source/api/nova..tests.real_flags.rst new file mode 100644 index 000000000..e9c0d1abd --- /dev/null +++ b/doc/source/api/nova..tests.real_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.real_flags` Module +============================================================================== +.. automodule:: nova..tests.real_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.runtime_flags.rst b/doc/source/api/nova..tests.runtime_flags.rst new file mode 100644 index 000000000..984e21199 --- /dev/null +++ b/doc/source/api/nova..tests.runtime_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.runtime_flags` Module +============================================================================== +.. automodule:: nova..tests.runtime_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_access.rst b/doc/source/api/nova..tests.test_access.rst new file mode 100644 index 000000000..300d8109e --- /dev/null +++ b/doc/source/api/nova..tests.test_access.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_access` Module +============================================================================== +.. automodule:: nova..tests.test_access + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_api.rst b/doc/source/api/nova..tests.test_api.rst new file mode 100644 index 000000000..f9473062e --- /dev/null +++ b/doc/source/api/nova..tests.test_api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_api` Module +============================================================================== +.. automodule:: nova..tests.test_api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_auth.rst b/doc/source/api/nova..tests.test_auth.rst new file mode 100644 index 000000000..ff4445ae4 --- /dev/null +++ b/doc/source/api/nova..tests.test_auth.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_auth` Module +============================================================================== +.. automodule:: nova..tests.test_auth + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_cloud.rst b/doc/source/api/nova..tests.test_cloud.rst new file mode 100644 index 000000000..7bd03db9a --- /dev/null +++ b/doc/source/api/nova..tests.test_cloud.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_cloud` Module +============================================================================== +.. automodule:: nova..tests.test_cloud + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_compute.rst b/doc/source/api/nova..tests.test_compute.rst new file mode 100644 index 000000000..90fd6e9d1 --- /dev/null +++ b/doc/source/api/nova..tests.test_compute.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_compute` Module +============================================================================== +.. automodule:: nova..tests.test_compute + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_console.rst b/doc/source/api/nova..tests.test_console.rst new file mode 100644 index 000000000..f695f5d17 --- /dev/null +++ b/doc/source/api/nova..tests.test_console.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_console` Module +============================================================================== +.. automodule:: nova..tests.test_console + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_direct.rst b/doc/source/api/nova..tests.test_direct.rst new file mode 100644 index 000000000..4f7adef19 --- /dev/null +++ b/doc/source/api/nova..tests.test_direct.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_direct` Module +============================================================================== +.. automodule:: nova..tests.test_direct + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_flags.rst b/doc/source/api/nova..tests.test_flags.rst new file mode 100644 index 000000000..2ec35d6c2 --- /dev/null +++ b/doc/source/api/nova..tests.test_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_flags` Module +============================================================================== +.. automodule:: nova..tests.test_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_localization.rst b/doc/source/api/nova..tests.test_localization.rst new file mode 100644 index 000000000..d93c83ba7 --- /dev/null +++ b/doc/source/api/nova..tests.test_localization.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_localization` Module +============================================================================== +.. automodule:: nova..tests.test_localization + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_log.rst b/doc/source/api/nova..tests.test_log.rst new file mode 100644 index 000000000..04ff5ead1 --- /dev/null +++ b/doc/source/api/nova..tests.test_log.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_log` Module +============================================================================== +.. automodule:: nova..tests.test_log + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_middleware.rst b/doc/source/api/nova..tests.test_middleware.rst new file mode 100644 index 000000000..2f9df5832 --- /dev/null +++ b/doc/source/api/nova..tests.test_middleware.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_middleware` Module +============================================================================== +.. automodule:: nova..tests.test_middleware + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_misc.rst b/doc/source/api/nova..tests.test_misc.rst new file mode 100644 index 000000000..4975f89d7 --- /dev/null +++ b/doc/source/api/nova..tests.test_misc.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_misc` Module +============================================================================== +.. automodule:: nova..tests.test_misc + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_network.rst b/doc/source/api/nova..tests.test_network.rst new file mode 100644 index 000000000..3a4b04ea4 --- /dev/null +++ b/doc/source/api/nova..tests.test_network.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_network` Module +============================================================================== +.. automodule:: nova..tests.test_network + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_quota.rst b/doc/source/api/nova..tests.test_quota.rst new file mode 100644 index 000000000..24ebf9ca3 --- /dev/null +++ b/doc/source/api/nova..tests.test_quota.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_quota` Module +============================================================================== +.. automodule:: nova..tests.test_quota + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_rpc.rst b/doc/source/api/nova..tests.test_rpc.rst new file mode 100644 index 000000000..c141d6889 --- /dev/null +++ b/doc/source/api/nova..tests.test_rpc.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_rpc` Module +============================================================================== +.. automodule:: nova..tests.test_rpc + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_scheduler.rst b/doc/source/api/nova..tests.test_scheduler.rst new file mode 100644 index 000000000..1cd9991db --- /dev/null +++ b/doc/source/api/nova..tests.test_scheduler.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_scheduler` Module +============================================================================== +.. automodule:: nova..tests.test_scheduler + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_service.rst b/doc/source/api/nova..tests.test_service.rst new file mode 100644 index 000000000..a264fbb55 --- /dev/null +++ b/doc/source/api/nova..tests.test_service.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_service` Module +============================================================================== +.. automodule:: nova..tests.test_service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_twistd.rst b/doc/source/api/nova..tests.test_twistd.rst new file mode 100644 index 000000000..cae0c0a28 --- /dev/null +++ b/doc/source/api/nova..tests.test_twistd.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_twistd` Module +============================================================================== +.. automodule:: nova..tests.test_twistd + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_virt.rst b/doc/source/api/nova..tests.test_virt.rst new file mode 100644 index 000000000..9b0dc1e46 --- /dev/null +++ b/doc/source/api/nova..tests.test_virt.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_virt` Module +============================================================================== +.. automodule:: nova..tests.test_virt + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_volume.rst b/doc/source/api/nova..tests.test_volume.rst new file mode 100644 index 000000000..b5affe53c --- /dev/null +++ b/doc/source/api/nova..tests.test_volume.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_volume` Module +============================================================================== +.. automodule:: nova..tests.test_volume + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_xenapi.rst b/doc/source/api/nova..tests.test_xenapi.rst new file mode 100644 index 000000000..7128baee4 --- /dev/null +++ b/doc/source/api/nova..tests.test_xenapi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_xenapi` Module +============================================================================== +.. automodule:: nova..tests.test_xenapi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.xenapi.stubs.rst b/doc/source/api/nova..tests.xenapi.stubs.rst new file mode 100644 index 000000000..356eed9a7 --- /dev/null +++ b/doc/source/api/nova..tests.xenapi.stubs.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.xenapi.stubs` Module +============================================================================== +.. automodule:: nova..tests.xenapi.stubs + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..twistd.rst b/doc/source/api/nova..twistd.rst new file mode 100644 index 000000000..d4145396d --- /dev/null +++ b/doc/source/api/nova..twistd.rst @@ -0,0 +1,6 @@ +The :mod:`nova..twistd` Module +============================================================================== +.. automodule:: nova..twistd + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..utils.rst b/doc/source/api/nova..utils.rst new file mode 100644 index 000000000..1131d1080 --- /dev/null +++ b/doc/source/api/nova..utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..utils` Module +============================================================================== +.. automodule:: nova..utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..version.rst b/doc/source/api/nova..version.rst new file mode 100644 index 000000000..4b0fc078f --- /dev/null +++ b/doc/source/api/nova..version.rst @@ -0,0 +1,6 @@ +The :mod:`nova..version` Module +============================================================================== +.. automodule:: nova..version + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.connection.rst b/doc/source/api/nova..virt.connection.rst new file mode 100644 index 000000000..caf766765 --- /dev/null +++ b/doc/source/api/nova..virt.connection.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.connection` Module +============================================================================== +.. automodule:: nova..virt.connection + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.disk.rst b/doc/source/api/nova..virt.disk.rst new file mode 100644 index 000000000..4a6c0f406 --- /dev/null +++ b/doc/source/api/nova..virt.disk.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.disk` Module +============================================================================== +.. automodule:: nova..virt.disk + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.fake.rst b/doc/source/api/nova..virt.fake.rst new file mode 100644 index 000000000..06ecdbf7d --- /dev/null +++ b/doc/source/api/nova..virt.fake.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.fake` Module +============================================================================== +.. automodule:: nova..virt.fake + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.hyperv.rst b/doc/source/api/nova..virt.hyperv.rst new file mode 100644 index 000000000..48d89378e --- /dev/null +++ b/doc/source/api/nova..virt.hyperv.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.hyperv` Module +============================================================================== +.. automodule:: nova..virt.hyperv + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.images.rst b/doc/source/api/nova..virt.images.rst new file mode 100644 index 000000000..4fdeb7af8 --- /dev/null +++ b/doc/source/api/nova..virt.images.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.images` Module +============================================================================== +.. automodule:: nova..virt.images + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.libvirt_conn.rst b/doc/source/api/nova..virt.libvirt_conn.rst new file mode 100644 index 000000000..7fb8aed5f --- /dev/null +++ b/doc/source/api/nova..virt.libvirt_conn.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.libvirt_conn` Module +============================================================================== +.. automodule:: nova..virt.libvirt_conn + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.fake.rst b/doc/source/api/nova..virt.xenapi.fake.rst new file mode 100644 index 000000000..752dabb14 --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.fake.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.fake` Module +============================================================================== +.. automodule:: nova..virt.xenapi.fake + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.network_utils.rst b/doc/source/api/nova..virt.xenapi.network_utils.rst new file mode 100644 index 000000000..15f52973e --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.network_utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.network_utils` Module +============================================================================== +.. automodule:: nova..virt.xenapi.network_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.vm_utils.rst b/doc/source/api/nova..virt.xenapi.vm_utils.rst new file mode 100644 index 000000000..18745dc71 --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.vm_utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.vm_utils` Module +============================================================================== +.. automodule:: nova..virt.xenapi.vm_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.vmops.rst b/doc/source/api/nova..virt.xenapi.vmops.rst new file mode 100644 index 000000000..30662c58d --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.vmops.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.vmops` Module +============================================================================== +.. automodule:: nova..virt.xenapi.vmops + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.volume_utils.rst b/doc/source/api/nova..virt.xenapi.volume_utils.rst new file mode 100644 index 000000000..413e4dc4b --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.volume_utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.volume_utils` Module +============================================================================== +.. automodule:: nova..virt.xenapi.volume_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.volumeops.rst b/doc/source/api/nova..virt.xenapi.volumeops.rst new file mode 100644 index 000000000..626f164df --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.volumeops.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.volumeops` Module +============================================================================== +.. automodule:: nova..virt.xenapi.volumeops + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi_conn.rst b/doc/source/api/nova..virt.xenapi_conn.rst new file mode 100644 index 000000000..14ac5147f --- /dev/null +++ b/doc/source/api/nova..virt.xenapi_conn.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi_conn` Module +============================================================================== +.. automodule:: nova..virt.xenapi_conn + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.api.rst b/doc/source/api/nova..volume.api.rst new file mode 100644 index 000000000..8ad36e049 --- /dev/null +++ b/doc/source/api/nova..volume.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.api` Module +============================================================================== +.. automodule:: nova..volume.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.driver.rst b/doc/source/api/nova..volume.driver.rst new file mode 100644 index 000000000..51f5c0729 --- /dev/null +++ b/doc/source/api/nova..volume.driver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.driver` Module +============================================================================== +.. automodule:: nova..volume.driver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.manager.rst b/doc/source/api/nova..volume.manager.rst new file mode 100644 index 000000000..91a192a8f --- /dev/null +++ b/doc/source/api/nova..volume.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.manager` Module +============================================================================== +.. automodule:: nova..volume.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.san.rst b/doc/source/api/nova..volume.san.rst new file mode 100644 index 000000000..1de068928 --- /dev/null +++ b/doc/source/api/nova..volume.san.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.san` Module +============================================================================== +.. automodule:: nova..volume.san + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..wsgi.rst b/doc/source/api/nova..wsgi.rst new file mode 100644 index 000000000..0bff1c332 --- /dev/null +++ b/doc/source/api/nova..wsgi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..wsgi` Module +============================================================================== +.. automodule:: nova..wsgi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/runnova/binaries.rst b/doc/source/runnova/binaries.rst new file mode 100644 index 000000000..023831021 --- /dev/null +++ b/doc/source/runnova/binaries.rst @@ -0,0 +1,57 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _binaries: + +Nova Daemons +============= + +The configuration of these binaries relies on "flagfiles" using the google +gflags package:: + + $ nova-xxxxx --flagfile flagfile + +The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. + +nova-api +-------- + +Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. + +nova-objectstore +---------------- + +Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with Glance (http://glance.openstack.org) and a simple image manager. + +nova-compute +------------ + +Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. + +nova-volume +----------- + +Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. + +nova-network +------------ + +Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: + + $ nova-network --network_manager=nova.network.manager.FlatManager + +IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/runnova/euca2ools.rst b/doc/source/runnova/euca2ools.rst new file mode 100644 index 000000000..6f0c57358 --- /dev/null +++ b/doc/source/runnova/euca2ools.rst @@ -0,0 +1,49 @@ +Euca2ools +========= + +Nova is compatible with most of the euca2ools command line utilities. Both Administrators and Users will find these tools helpful for day-to-day administration. + +* euca-add-group +* euca-delete-bundle +* euca-describe-instances +* euca-register +* euca-add-keypair +* euca-delete-group +* euca-describe-keypairs +* euca-release-address +* euca-allocate-address +* euca-delete-keypair +* euca-describe-regions +* euca-reset-image-attribute +* euca-associate-address +* euca-delete-snapshot +* euca-describe-snapshots +* euca-revoke +* euca-attach-volume +* euca-delete-volume +* euca-describe-volumes +* euca-run-instances +* euca-authorize +* euca-deregister +* euca-detach-volume +* euca-terminate-instances +* euca-bundle-image +* euca-describe-addresses +* euca-disassociate-address +* euca-unbundle +* euca-bundle-vol +* euca-describe-availability-zones +* euca-download-bundle +* euca-upload-bundle +* euca-confirm-product-instance +* euca-describe-groups +* euca-get-console-output +* euca-version +* euca-create-snapshot +* euca-describe-image-attribute +* euca-modify-image-attribute +* euca-create-volume +* euca-describe-images +* euca-reboot-instances + + diff --git a/doc/source/runnova/flags.rst b/doc/source/runnova/flags.rst new file mode 100644 index 000000000..1bfa022d9 --- /dev/null +++ b/doc/source/runnova/flags.rst @@ -0,0 +1,193 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Flags and Flagfiles +=================== + +Nova uses a configuration file containing flags located in /etc/nova/nova.conf. You can get the most recent listing of avaialble flags by running nova-(servicename) --help, for example, nova-api --help. + +Here's a list of available flags and their default settings. + + --ajax_console_proxy_port: port that ajax_console_proxy binds + (default: '8000') + --ajax_console_proxy_topic: the topic ajax proxy nodes listen on + (default: 'ajax_proxy') + --ajax_console_proxy_url: location of ajax console proxy, in the form + "http://127.0.0.1:8000" + (default: 'http://127.0.0.1:8000') + --auth_token_ttl: Seconds for auth tokens to linger + (default: '3600') + (an integer) + --aws_access_key_id: AWS Access ID + (default: 'admin') + --aws_secret_access_key: AWS Access Key + (default: 'admin') + --compute_manager: Manager for compute + (default: 'nova.compute.manager.ComputeManager') + --compute_topic: the topic compute nodes listen on + (default: 'compute') + --connection_type: libvirt, xenapi or fake + (default: 'libvirt') + --console_manager: Manager for console proxy + (default: 'nova.console.manager.ConsoleProxyManager') + --console_topic: the topic console proxy nodes listen on + (default: 'console') + --control_exchange: the main exchange to connect to + (default: 'nova') + --db_backend: The backend to use for db + (default: 'sqlalchemy') + --default_image: default image to use, testing only + (default: 'ami-11111') + --default_instance_type: default instance type to use, testing only + (default: 'm1.small') + --default_log_levels: list of logger=LEVEL pairs + (default: 'amqplib=WARN,sqlalchemy=WARN,eventlet.wsgi.server=WARN') + (a comma separated list) + --default_project: default project for openstack + (default: 'openstack') + --ec2_dmz_host: internal ip of api server + (default: '$my_ip') + --ec2_host: ip of api server + (default: '$my_ip') + --ec2_path: suffix for ec2 + (default: '/services/Cloud') + --ec2_port: cloud controller port + (default: '8773') + (an integer) + --ec2_scheme: prefix for ec2 + (default: 'http') + --[no]enable_new_services: Services to be added to the available pool on + create + (default: 'true') + --[no]fake_network: should we use fake network devices and addresses + (default: 'false') + --[no]fake_rabbit: use a fake rabbit + (default: 'false') + --glance_host: glance host + (default: '$my_ip') + --glance_port: glance port + (default: '9292') + (an integer) + -?,--[no]help: show this help + --[no]helpshort: show usage only for this module + --[no]helpxml: like --help, but generates XML output + --host: name of this node + (default: 'osdemo03') + --image_service: The service to use for retrieving and searching for images. + (default: 'nova.image.s3.S3ImageService') + --instance_name_template: Template string to be used to generate instance + names + (default: 'instance-%08x') + --logfile: output to named file + --logging_context_format_string: format string to use for log messages with + context + (default: '%(asctime)s %(levelname)s %(name)s [%(request_id)s %(user)s + %(project)s] %(message)s') + --logging_debug_format_suffix: data to append to log format when level is + DEBUG + (default: 'from %(processName)s (pid=%(process)d) %(funcName)s + %(pathname)s:%(lineno)d') + --logging_default_format_string: format string to use for log messages without + context + (default: '%(asctime)s %(levelname)s %(name)s [-] %(message)s') + --logging_exception_prefix: prefix each line of exception output with this + format + (default: '(%(name)s): TRACE: ') + --my_ip: host ip address + (default: '184.106.73.68') + --network_manager: Manager for network + (default: 'nova.network.manager.VlanManager') + --network_topic: the topic network nodes listen on + (default: 'network') + --node_availability_zone: availability zone of this node + (default: 'nova') + --null_kernel: kernel image that indicates not to use a kernel, but to use a + raw disk image instead + (default: 'nokernel') + --osapi_host: ip of api server + (default: '$my_ip') + --osapi_path: suffix for openstack + (default: '/v1.0/') + --osapi_port: OpenStack API port + (default: '8774') + (an integer) + --osapi_scheme: prefix for openstack + (default: 'http') + --periodic_interval: seconds between running periodic tasks + (default: '60') + (a positive integer) + --pidfile: pidfile to use for this service + --rabbit_host: rabbit host + (default: 'localhost') + --rabbit_max_retries: rabbit connection attempts + (default: '12') + (an integer) + --rabbit_password: rabbit password + (default: 'guest') + --rabbit_port: rabbit port + (default: '5672') + (an integer) + --rabbit_retry_interval: rabbit connection retry interval + (default: '10') + (an integer) + --rabbit_userid: rabbit userid + (default: 'guest') + --rabbit_virtual_host: rabbit virtual host + (default: '/') + --region_list: list of region=fqdn pairs separated by commas + (default: '') + (a comma separated list) + --report_interval: seconds between nodes reporting state to datastore + (default: '10') + (a positive integer) + --s3_dmz: s3 dmz ip (for instances) + (default: '$my_ip') + --s3_host: s3 host (for infrastructure) + (default: '$my_ip') + --s3_port: s3 port + (default: '3333') + (an integer) + --scheduler_manager: Manager for scheduler + (default: 'nova.scheduler.manager.SchedulerManager') + --scheduler_topic: the topic scheduler nodes listen on + (default: 'scheduler') + --sql_connection: connection string for sql database + (default: 'sqlite:///$state_path/nova.sqlite') + --sql_idle_timeout: timeout for idle sql database connections + (default: '3600') + --sql_max_retries: sql connection attempts + (default: '12') + (an integer) + --sql_retry_interval: sql connection retry interval + (default: '10') + (an integer) + --state_path: Top-level directory for maintaining nova's state + (default: '/usr/lib/pymodules/python2.6/nova/../') + --[no]use_syslog: output to syslog + (default: 'false') + --[no]verbose: show debug output + (default: 'false') + --volume_manager: Manager for volume + (default: 'nova.volume.manager.VolumeManager') + --volume_name_template: Template string to be used to generate instance names + (default: 'volume-%08x') + --volume_topic: the topic volume nodes listen on + (default: 'volume') + --vpn_image_id: AMI for cloudpipe vpn server + (default: 'ami-cloudpipe') + --vpn_key_suffix: Suffix to add to project name for vpn key and secgroups + (default: '-vpn') \ No newline at end of file diff --git a/doc/source/runnova/getting.started.rst b/doc/source/runnova/getting.started.rst new file mode 100644 index 000000000..4cc7307b0 --- /dev/null +++ b/doc/source/runnova/getting.started.rst @@ -0,0 +1,168 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Getting Started with Nova +========================= + +This code base is continually changing, so dependencies also change. If you +encounter any problems, see the :doc:`../community` page. +The `contrib/nova.sh` script should be kept up to date, and may be a good +resource to review when debugging. + +The purpose of this document is to get a system installed that you can use to +test your setup assumptions. Working from this base installtion you can +tweak configurations and work with different flags to monitor interaction with +your hardware, network, and other factors that will allow you to determine +suitability for your deployment. After following this setup method, you should +be able to experiment with different managers, drivers, and flags to get the +best performance. + +Dependencies +------------ + +Related servers we rely on + +* **RabbitMQ**: messaging queue, used for all communication between components + +Optional servers + +* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by + setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP + (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. + There is a script in the sources (`nova/auth/slap.sh`) to install a very basic + openldap server on ubuntu. +* **ReDIS**: There is a fake ldap auth driver + `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was + created for testing ldap implementation on systems that don't have an easy + means to install ldap. +* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to + be avilable. Currently, only sqlite3 an mysql have been tested. + +Python libraries that we use (from pip-requires): + +.. literalinclude:: ../../../tools/pip-requires + +Other libraries: + +* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available + from http://wiki.xensource.com/xenwiki/XCP_SDK or + http://community.citrix.com/cdn/xs/sdks. + +External unix tools that are required: + +* iptables +* ebtables +* gawk +* curl +* kvm +* libvirt +* dnsmasq +* vlan +* open-iscsi and iscsitarget (if you use iscsi volumes) +* aoetools and vblade-persist (if you use aoe-volumes) + +Nova uses cutting-edge versions of many packages. There are ubuntu packages in +the nova-core trunk ppa. You can use add this ppa to your sources list on an +ubuntu machine with the following commands:: + + sudo apt-get install -y python-software-properties + sudo add-apt-repository ppa:nova-core/trunk + +Recommended +----------- + +* euca2ools: python implementation of aws ec2-tools and ami tools +* build tornado to use C module for evented section + + +Installation +-------------- + +You can install from packages for your particular Linux distribution if they are +available. Otherwise you can install from source by checking out the source +files from the `Nova Source Code Repository `_ +and running:: + + python setup.py install + +Configuration +--------------- + +Configuring the host system +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Nova can be configured in many different ways. In this "Getting Started with Nova" document, we only provide what you need to get started as quickly as possible. For a more detailed description of system +configuration, start reading through `Installing and Configuring OpenStack Compute `_. + +`Detailed instructions for creating a volume group are available `_, or use these quick instructions. + +* Create a volume group (you can use an actual disk for the volume group as + well):: + + # This creates a 1GB file to create volumes out of + dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 + losetup --show -f MY_FILE_PATH + # replace /dev/loop0 below with whatever losetup returns + # nova-volumes is the default for the --volume_group flag + vgcreate nova-volumes /dev/loop0 + + +Configuring Nova +~~~~~~~~~~~~~~~~ + +Configuration of the entire system is performed through python-gflags. The +best way to track configuration is through the use of a flagfile. + +A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary +when you launch it. Flagfiles for nova are typically stored in +``/etc/nova/nova.conf``, and flags specific to a certain program are stored in +``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another +flagfile, so typically a file like ``nova-manage.conf`` would have as its first +line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before +specifying overrides or additional options. + +To get a current comprehensive list of flag file options, run bin/nova- --help, or refer to a static list at `Reference for Flags in nova.conf `_. + +A sample configuration to test the system follows:: + + --verbose + --nodaemon + --auth_driver=nova.auth.dbdriver.DbDriver + +Running +------- + +There are many parts to the nova system, each with a specific function. They +are built to be highly-available, so there are may configurations they can be +run in (ie: on many machines, many listeners per machine, etc). This part +of the guide only gets you started quickly, to learn about HA options, see +`Installing and Configuring OpenStack Compute `_. + +Launch supporting services + +* rabbitmq +* redis (optional) +* mysql (optional) +* openldap (optional) + +Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` + +* nova-api +* nova-compute +* nova-objectstore +* nova-volume +* nova-scheduler diff --git a/doc/source/runnova/index.rst b/doc/source/runnova/index.rst new file mode 100644 index 000000000..283d268ce --- /dev/null +++ b/doc/source/runnova/index.rst @@ -0,0 +1,90 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Running Nova +============ + +This guide describes the basics of running and managing Nova. For more administrator's documentation, refer to `docs.openstack.org `_. + +Running the Cloud +----------------- + +The fastest way to get a test cloud running is by following the directions in the :doc:`../quickstart`. It relies on a nova.sh script to run on a single machine. + +Nova's cloud works via the interaction of a series of daemon processes that reside persistently on the host machine(s). Fortunately, the :doc:`../quickstart` process launches sample versions of all these daemons for you. Once you are familiar with basic Nova usage, you can learn more about daemons by reading :doc:`../service.architecture` and :doc:`binaries`. + +Administration Utilities +------------------------ + +There are two main tools that a system administrator will find useful to manage their Nova cloud: + +.. toctree:: + :maxdepth: 1 + + nova.manage + euca2ools + +The nova-manage command may only be run by users with admin priviledges. Commands for euca2ools can be used by all users, though specific commands may be restricted by Role Based Access Control. You can read more about creating and managing users in :doc:`managing.users` + +User and Resource Management +---------------------------- + +The nova-manage and euca2ools commands provide the basic interface to perform a broad range of administration functions. In this section, you can read more about how to accomplish specific administration tasks. + +For background on the core objects referenced in this section, see :doc:`../object.model` + +.. toctree:: + :maxdepth: 1 + + managing.users + managing.projects + managing.instances + managing.images + managing.volumes + managing.networks + +Deployment +---------- + +For a starting multi-node architecture, you would start with two nodes - a cloud controller node and a compute node. The cloud controller node contains the nova- services plus the Nova database. The compute node installs all the nova-services but then refers to the database installation, which is hosted by the cloud controller node. Ensure that the nova.conf file is identical on each node. If you find performance issues not related to database reads or writes, but due to the messaging queue backing up, you could add additional messaging services (rabbitmq). For instructions on multi-server installations, refer to `Installing and Configuring OpenStack Compute `_. + + +.. toctree:: + :maxdepth: 1 + + dbsync + + +Networking +^^^^^^^^^^ + +.. toctree:: + :maxdepth: 1 + + network.vlan.rst + network.flat.rst + + +Advanced Topics +--------------- + +.. toctree:: + :maxdepth: 1 + + flags + monitoring + diff --git a/doc/source/runnova/managing.images.rst b/doc/source/runnova/managing.images.rst new file mode 100644 index 000000000..c5d93a6e8 --- /dev/null +++ b/doc/source/runnova/managing.images.rst @@ -0,0 +1,21 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Images +=============== + +.. todo:: Put info on managing images here! diff --git a/doc/source/runnova/managing.instances.rst b/doc/source/runnova/managing.instances.rst new file mode 100644 index 000000000..e62352017 --- /dev/null +++ b/doc/source/runnova/managing.instances.rst @@ -0,0 +1,59 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Instances +================== + +Keypairs +-------- + +Images can be shared by many users, so it is dangerous to put passwords into the images. Nova therefore supports injecting ssh keys into instances before they are booted. This allows a user to login to the instances that he or she creates securely. Generally the first thing that a user does when using the system is create a keypair. Nova generates a public and private key pair, and sends the private key to the user. The public key is stored so that it can be injected into instances. + +Keypairs are created through the api. They can be created on the command line using the euca2ools script euca-add-keypair. Refer to the man page for the available options. Example usage:: + + euca-add-keypair test > test.pem + chmod 600 test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + # wait for boot + ssh -i test.pem root@ip.of.instance + + +Basic Management +---------------- +Instance management can be accomplished with euca commands: + + +To run an instance: + +:: + + euca-run-instances + + +To terminate an instance: + +:: + + euca-terminate-instances + +To reboot an instance: + +:: + + euca-reboot-instances + +See the euca2ools documentation for more information diff --git a/doc/source/runnova/managing.networks.rst b/doc/source/runnova/managing.networks.rst new file mode 100644 index 000000000..9eea46d70 --- /dev/null +++ b/doc/source/runnova/managing.networks.rst @@ -0,0 +1,70 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + Overview Sections Copyright 2010-2011 Citrix + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Networking Overview +=================== +In Nova, users organize their cloud resources in projects. A Nova project consists of a number of VM instances created by a user. For each VM instance, Nova assigns to it a private IP address. (Currently, Nova only supports Linux bridge networking that allows the virtual interfaces to connect to the outside network through the physical interface. Other virtual network technologies, such as Open vSwitch, could be supported in the future.) The Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. + +Nova Network Strategies +----------------------- + +Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can co-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. + +Read more about Nova network strategies here: + +.. toctree:: + :maxdepth: 1 + + network.flat.rst + network.vlan.rst + + +Network Management Commands +--------------------------- + +Admins and Network Administrators can use the 'nova-manage' command to manage network resources: + +VPN Management +~~~~~~~~~~~~~~ + +* vpn list: Print a listing of the VPNs for all projects. + * arguments: none +* vpn run: Start the VPN for a given project. + * arguments: project +* vpn spawn: Run all VPNs. + * arguments: none + + +Floating IP Management +~~~~~~~~~~~~~~~~~~~~~~ + +* floating create: Creates floating ips for host by range + * arguments: host ip_range +* floating delete: Deletes floating ips by range + * arguments: range +* floating list: Prints a listing of all floating ips + * arguments: none + +Network Management +~~~~~~~~~~~~~~~~~~ + +* network create: Creates fixed ips for host by range + * arguments: [fixed_range=FLAG], [num_networks=FLAG], + [network_size=FLAG], [vlan_start=FLAG], + [vpn_start=FLAG] + diff --git a/doc/source/runnova/managing.projects.rst b/doc/source/runnova/managing.projects.rst new file mode 100644 index 000000000..5dd7f2de9 --- /dev/null +++ b/doc/source/runnova/managing.projects.rst @@ -0,0 +1,68 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Projects +================= + +Projects are isolated resource containers forming the principal organizational structure within Nova. They consist of a separate vlan, volumes, instances, images, keys, and users. + +Although the original ec2 api only supports users, nova adds the concept of projects. A user can specify which project he or she wishes to use by appending `:project_id` to his or her access key. If no project is specified in the api request, nova will attempt to use a project with the same id as the user. + +The api will return NotAuthorized if a normal user attempts to make requests for a project that he or she is not a member of. Note that admins or users with special admin roles skip this check and can make requests for any project. + +To create a project, use the `project create` command of nova-manage. The syntax is nova-manage project create projectname manager_id [description] You must specify a projectname and a manager_id. For example:: + nova-manage project create john_project john "This is a sample project" + +You can add and remove users from projects with `project add` and `project remove`:: + nova-manage project add john_project john + nova-manage project remove john_project john + +Project Commands +---------------- + +Admins and Project Managers can use the 'nova-manage project' command to manage project resources: + +* project add: Adds user to project + * arguments: project user +* project create: Creates a new project + * arguments: name project_manager [description] +* project delete: Deletes an existing project + * arguments: project_id +* project environment: Exports environment variables to an sourcable file + * arguments: project_id user_id [filename='novarc] +* project list: lists all projects + * arguments: none +* project remove: Removes user from project + * arguments: project user +* project scrub: Deletes data associated with project + * arguments: project +* project zipfile: Exports credentials for project to a zip file + * arguments: project_id user_id [filename='nova.zip] + +Setting Quotas +-------------- +Nova utilizes a quota system at the project level to control resource consumption across available hardware resources. Current quota controls are available to limit the: + +* Number of volumes which may be created +* Total size of all volumes within a project as measured in GB +* Number of instances which may be launched +* Number of processor cores which may be allocated +* Publicly accessible IP addresses + +Use the following command to set quotas for a project +* project quota: Set or display quotas for project + * arguments: project_id [key] [value] diff --git a/doc/source/runnova/managing.users.rst b/doc/source/runnova/managing.users.rst new file mode 100644 index 000000000..392142e86 --- /dev/null +++ b/doc/source/runnova/managing.users.rst @@ -0,0 +1,82 @@ +Managing Users +============== + + +Users and Access Keys +--------------------- + +Access to the ec2 api is controlled by an access and secret key. The user's access key needs to be included in the request, and the request must be signed with the secret key. Upon receipt of api requests, nova will verify the signature and execute commands on behalf of the user. + +In order to begin using nova, you will need a to create a user. This can be easily accomplished using the user create or user admin commands in nova-manage. `user create` will create a regular user, whereas `user admin` will create an admin user. The syntax of the command is nova-manage user create username [access] [secret]. For example:: + + nova-manage user create john my-access-key a-super-secret-key + +If you do not specify an access or secret key, a random uuid will be created automatically. + +Credentials +----------- + +Nova can generate a handy set of credentials for a user. These credentials include a CA for bundling images and a file for setting environment variables to be used by euca2ools. If you don't need to bundle images, just the environment script is required. You can export one with the `project environment` command. The syntax of the command is nova-manage project environment project_id user_id [filename]. If you don't specify a filename, it will be exported as novarc. After generating the file, you can simply source it in bash to add the variables to your environment:: + + nova-manage project environment john_project john + . novarc + +If you do need to bundle images, you will need to get all of the credentials using `project zipfile`. Note that zipfile will give you an error message if networks haven't been created yet. Otherwise zipfile has the same syntax as environment, only the default file name is nova.zip. Example usage:: + + nova-manage project zipfile john_project john + unzip nova.zip + . novarc + +Role Based Access Control +------------------------- +Roles control the api actions that a user is allowed to perform. For example, a user cannot allocate a public ip without the `netadmin` role. It is important to remember that a users de facto permissions in a project is the intersection of user (global) roles and project (local) roles. So for john to have netadmin permissions in his project, he needs to separate roles specified. You can add roles with `role add`. The syntax is nova-manage role add user_id role [project_id]. Let's give john the netadmin role for his project:: + + nova-manage role add john netadmin + nova-manage role add john netadmin john_project + +Role-based access control (RBAC) is an approach to restricting system access to authorized users based on an individual’s role within an organization. Various employee functions require certain levels of system access in order to be successful. These functions are mapped to defined roles and individuals are categorized accordingly. Since users are not assigned permissions directly, but only acquire them through their role (or roles), management of individual user rights becomes a matter of assigning appropriate roles to the user. This simplifies common operations, such as adding a user, or changing a user's department. + +Nova’s rights management system employs the RBAC model and currently supports the following five roles: + +* **Cloud Administrator.** (admin) Users of this class enjoy complete system access. +* **IT Security.** (itsec) This role is limited to IT security personnel. It permits role holders to quarantine instances. +* **Project Manager.** (projectmanager)The default for project owners, this role affords users the ability to add other users to a project, interact with project images, and launch and terminate instances. +* **Network Administrator.** (netadmin) Users with this role are permitted to allocate and assign publicly accessible IP addresses as well as create and modify firewall rules. +* **Developer.** This is a general purpose role that is assigned to users by default. + +RBAC management is exposed through the dashboard for simplified user management. + + +User Commands +~~~~~~~~~~~~ + +Users, including admins, are created through the ``user`` commands. + +* user admin: creates a new admin and prints exports + * arguments: name [access] [secret] +* user create: creates a new user and prints exports + * arguments: name [access] [secret] +* user delete: deletes an existing user + * arguments: name +* user exports: prints access and secrets for user in export format + * arguments: name +* user list: lists all users + * arguments: none +* user modify: update a users keys & admin flag + * arguments: accesskey secretkey admin + * leave any field blank to ignore it, admin should be 'T', 'F', or blank + + +User Role Management +~~~~~~~~~~~~~~~~~~~~ + +* role add: adds role to user + * if project is specified, adds project specific role + * arguments: user, role [project] +* role has: checks to see if user has role + * if project is specified, returns True if user has + the global role and the project role + * arguments: user, role [project] +* role remove: removes role from user + * if project is specified, removes project specific role + * arguments: user, role [project] diff --git a/doc/source/runnova/managingsecurity.rst b/doc/source/runnova/managingsecurity.rst new file mode 100644 index 000000000..7893925e7 --- /dev/null +++ b/doc/source/runnova/managingsecurity.rst @@ -0,0 +1,39 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Security Considerations +======================= + +.. todo:: This doc is vague and just high-level right now. Describe architecture that enables security. + +The goal of securing a cloud computing system involves both protecting the instances, data on the instances, and +ensuring users are authenticated for actions and that borders are understood by the users and the system. +Protecting the system from intrusion or attack involves authentication, network protections, and +compromise detection. + +Key Concepts +------------ + +Authentication - Each instance is authenticated with a key pair. + +Network - Instances can communicate with each other but you can configure the boundaries through firewall +configuration. + +Monitoring - Log all API commands and audit those logs. + +Encryption - Data transfer between instances is not encrypted. + diff --git a/doc/source/runnova/monitoring.rst b/doc/source/runnova/monitoring.rst new file mode 100644 index 000000000..2c93c71b5 --- /dev/null +++ b/doc/source/runnova/monitoring.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Monitoring +========== + +* components +* throughput +* exceptions +* hardware + +* ganglia +* syslog diff --git a/doc/source/runnova/network.flat.rst b/doc/source/runnova/network.flat.rst new file mode 100644 index 000000000..3d8680c6f --- /dev/null +++ b/doc/source/runnova/network.flat.rst @@ -0,0 +1,60 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +Flat Network Mode (Original and Flat) +===================================== + +Flat network mode removes most of the complexity of VLAN mode by simply +bridging all instance interfaces onto a single network. + +There are two variations of flat mode that differ mostly in how IP addresses +are given to instances. + + +Original Flat Mode +------------------ +IP addresses for VM instances are grabbed from a subnet specified by the network administrator, and injected into the image on launch. All instances of the system are attached to the same Linux networking bridge, configured manually by the network administrator both on the network controller hosting the network and on the computer controllers hosting the instances. To recap: + +* Each compute host creates a single bridge for all instances to use to attach to the external network. +* The networking configuration is injected into the instance before it is booted or it is obtained by a guest agent installed in the instance. + +Note that the configuration injection currently only works on linux-style systems that keep networking +configuration in /etc/network/interfaces. + + +Flat DHCP Mode +-------------- +IP addresses for VM instances are grabbed from a subnet specified by the network administrator. Similar to the flat network, a single Linux networking bridge is created and configured manually by the network administrator and used for all instances. A DHCP server is started to pass out IP addresses to VM instances from the specified subnet. To recap: + +* Like flat mode, all instances are attached to a single bridge on the compute node. +* In addition a DHCP server is running to configure instances. + +Implementation +-------------- + +The network nodes do not act as a default gateway in flat mode. Instances +are given public IP addresses. + +Compute nodes have iptables/ebtables entries created per project and +instance to protect against IP/MAC address spoofing and ARP poisoning. + + +Examples +-------- + +.. todo:: add flat network mode configuration examples diff --git a/doc/source/runnova/network.vlan.rst b/doc/source/runnova/network.vlan.rst new file mode 100644 index 000000000..c06ce8e8b --- /dev/null +++ b/doc/source/runnova/network.vlan.rst @@ -0,0 +1,179 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +VLAN Network Mode +================= +VLAN Network Mode is the default mode for Nova. It provides a private network +segment for each project's instances that can be accessed via a dedicated +VPN connection from the Internet. + +In this mode, each project gets its own VLAN, Linux networking bridge, and subnet. The subnets are specified by the network administrator, and are assigned dynamically to a project when required. A DHCP Server is started for each VLAN to pass out IP addresses to VM instances from the subnet assigned to the project. All instances belonging to one project are bridged into the same VLAN for that project. The Linux networking bridges and VLANs are created by Nova when required, described in more detail in Nova VLAN Network Management Implementation. + +.. + (this text revised above) + Because the flat network and flat DhCP network are simple to understand and yet do not scale well enough for real-world cloud systems, this section focuses on the VLAN network implementation by the VLAN Network Manager. + + + In the VLAN network mode, all the VM instances of a project are connected together in a VLAN with the specified private subnet. Each running VM instance is assigned an IP address within the given private subnet. + +.. image:: /images/Novadiagram.png + :width: 790 + +While network traffic between VM instances belonging to the same VLAN is always open, Nova can enforce isolation of network traffic between different projects by enforcing one VLAN per project. + +In addition, the network administrator can specify a pool of public IP addresses that users may allocate and then assign to VMs, either at boot or dynamically at run-time. This capability is similar to Amazon's 'elastic IPs'. A public IP address may be associated with a running instances, allowing the VM instance to be accessed from the public network. The public IP addresses are accessible from the network host and NATed to the private IP address of the project. + +.. todo:: Describe how a public IP address could be associated with a project (a VLAN) + +This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code named :ref:`cloudpipe `) needs to be created. Nova generates a certificate and key for the user to access the vpn and starts the vpn automatically. More information on cloudpipe can be found :ref:`here `. + +The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) + +.. image:: /images/cloudpipe.png + :width: 100% + +Goals +----- + +For our implementation of Nova, our goal is that each project is in a protected network segment. Here are the specifications we keep in mind for meeting this goal. + + * RFC-1918 IP space + * public IP via NAT + * no default inbound Internet access without public NAT + * limited (project-admin controllable) outbound Internet access + * limited (project-admin controllable) access to other project segments + * all connectivity to instance and cloud API is via VPN into the project segment + +We also keep as a goal a common DMZ segment for support services, meaning these items are only visible from project segment: + + * metadata + * dashboard + +Limitations +----------- + +We kept in mind some of these limitations: + +* Projects / cluster limited to available VLANs in switching infrastructure +* Requires VPN for access to project segment + +Implementation +-------------- +Currently Nova segregates project VLANs using 802.1q VLAN tagging in the +switching layer. Compute hosts create VLAN-specific interfaces and bridges +as required. + +The network nodes act as default gateway for project networks and contain +all of the routing and firewall rules implementing security groups. The +network node also handles DHCP to provide instance IPs for each project. + +VPN access is provided by running a small instance called CloudPipe +on the IP immediately following the gateway IP for each project. The +network node maps a dedicated public IP/port to the CloudPipe instance. + +Compute nodes have per-VLAN interfaces and bridges created as required. +These do NOT have IP addresses in the host to protect host access. +Compute nodes have iptables/ebtables entries created per project and +instance to protect against IP/MAC address spoofing and ARP poisoning. + +The network assignment to a project, and IP address assignment to a VM instance, are triggered when a user starts to run a VM instance. When running a VM instance, a user needs to specify a project for the instances, and the security groups (described in Security Groups) when the instance wants to join. If this is the first instance to be created for the project, then Nova (the cloud controller) needs to find a network controller to be the network host for the project; it then sets up a private network by finding an unused VLAN id, an unused subnet, and then the controller assigns them to the project, it also assigns a name to the project's Linux bridge (br100 stored in the Nova database), and allocating a private IP within the project's subnet for the new instance. + +If the instance the user wants to start is not the project's first, a subnet and a VLAN must have already been assigned to the project; therefore the system needs only to find an available IP address within the subnet and assign it to the new starting instance. If there is no private IP available within the subnet, an exception will be raised to the cloud controller, and the VM creation cannot proceed. + + +External Infrastructure +----------------------- + +Nova assumes the following is available: + +* DNS +* NTP +* Internet connectivity + + +Example +------- + +This example network configuration demonstrates most of the capabilities +of VLAN Mode. It splits administrative access to the nodes onto a dedicated +management network and uses dedicated network nodes to handle all +routing and gateway functions. + +It uses a 10GB network for instance traffic and a 1GB network for management. + + +Hardware +~~~~~~~~ + +* All nodes have a minimum of two NICs for management and production. + + * management is 1GB + * production is 10GB + * add additional NICs for bonding or HA/performance + +* network nodes should have an additional NIC dedicated to public Internet traffic +* switch needs to support enough simultaneous VLANs for number of projects +* production network configured as 802.1q trunk on switch + + +Operation +~~~~~~~~~ + +The network node controls the project network configuration: + +* assigns each project a VLAN and private IP range +* starts dnsmasq on project VLAN to serve private IP range +* configures iptables on network node for default project access +* launches CloudPipe instance and configures iptables access + +When starting an instance the network node: + +* sets up a VLAN interface and bridge on each host as required when an + instance is started on that host +* assigns private IP to instance +* generates MAC address for instance +* update dnsmasq with IP/MAC for instance + +When starting an instance the compute node: + +* sets up a VLAN interface and bridge on each host as required when an + instance is started on that host + + +Setup +~~~~~ + +* Assign VLANs in the switch: + + * public Internet segment + * production network + * management network + * cluster DMZ + +* Assign a contiguous range of VLANs to Nova for project use. +* Configure management NIC ports as management VLAN access ports. +* Configure management VLAN with Internet access as required +* Configure production NIC ports as 802.1q trunk ports. +* Configure Nova (need to add specifics here) + + * public IPs + * instance IPs + * project network size + * DMZ network + +.. todo:: need specific Nova configuration added diff --git a/doc/source/runnova/nova.manage.rst b/doc/source/runnova/nova.manage.rst new file mode 100644 index 000000000..0e9a29b6b --- /dev/null +++ b/doc/source/runnova/nova.manage.rst @@ -0,0 +1,239 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +The nova-manage command +======================= + +Introduction +~~~~~~~~~~~~ + +The nova-manage command is used to perform many essential functions for +administration and ongoing maintenance of nova, such as user creation, +vpn management, and much more. + +The standard pattern for executing a nova-manage command is: +``nova-manage []`` + +For example, to obtain a list of all projects: +``nova-manage project list`` + +Run without arguments to see a list of available command categories: +``nova-manage`` + +Categories are user, project, role, shell, vpn, and floating. Detailed descriptions are below. + +You can also run with a category argument such as user to see a list of all commands in that category: +``nova-manage user`` + +These sections describe the available categories and arguments for nova-manage. + +Nova Db +~~~~~~~ + +``nova-manage db version`` + + Print the current database version. + +``nova-manage db sync`` + + Sync the database up to the most recent version. This is the standard way to create the db as well. + +Nova User +~~~~~~~~~ + +``nova-manage user admin `` + + Create an admin user with the name . + +``nova-manage user create `` + + Create a normal user with the name . + +``nova-manage user delete `` + + Delete the user with the name . + +``nova-manage user exports `` + + Outputs a list of access key and secret keys for user to the screen + +``nova-manage user list`` + + Outputs a list of all the user names to the screen. + +``nova-manage user modify `` + + Updates the indicated user keys, indicating with T or F if the user is an admin user. Leave any argument blank if you do not want to update it. + +Nova Project +~~~~~~~~~~~~ + +``nova-manage project add `` + + Add a nova project with the name to the database. + +``nova-manage project create `` + + Create a new nova project with the name (you still need to do nova-manage project add to add it to the database). + +``nova-manage project delete `` + + Delete a nova project with the name . + +``nova-manage project environment `` + + Exports environment variables for the named project to a file named novarc. + +``nova-manage project list`` + + Outputs a list of all the projects to the screen. + +``nova-manage project quota `` + + Outputs the size and specs of the project's instances including gigabytes, instances, floating IPs, volumes, and cores. + +``nova-manage project remove `` + + Deletes the project with the name . + +``nova-manage project zipfile`` + + Compresses all related files for a created project into a zip file nova.zip. + +Nova Role +~~~~~~~~~ + +nova-manage role [] +``nova-manage role add <(optional) projectname>`` + + Add a user to either a global or project-based role with the indicated assigned to the named user. Role names can be one of the following five roles: admin, itsec, projectmanager, netadmin, developer. If you add the project name as the last argument then the role is assigned just for that project, otherwise the user is assigned the named role for all projects. + +``nova-manage role has `` + Checks the user or project and responds with True if the user has a global role with a particular project. + +``nova-manage role remove `` + Remove the indicated role from the user. + +Nova Shell +~~~~~~~~~~ + +``nova-manage shell bpython`` + + Starts a new bpython shell. + +``nova-manage shell ipython`` + + Starts a new ipython shell. + +``nova-manage shell python`` + + Starts a new python shell. + +``nova-manage shell run`` + + Starts a new shell using python. + +``nova-manage shell script `` + + Runs the named script from the specified path with flags set. + +Nova VPN +~~~~~~~~ + +``nova-manage vpn list`` + + Displays a list of projects, their IP prot numbers, and what state they're in. + +``nova-manage vpn run `` + + Starts the VPN for the named project. + +``nova-manage vpn spawn`` + + Runs all VPNs. + +Nova Floating IPs +~~~~~~~~~~~~~~~~~ + +``nova-manage floating create `` + + Creates floating IP addresses for the named host by the given range. + +``nova-manage floating delete `` + + Deletes floating IP addresses in the range given. + +``nova-manage floating list`` + + Displays a list of all floating IP addresses. + +Concept: Flags +-------------- + +python-gflags + + +Concept: Plugins +---------------- + +* Managers/Drivers: utils.import_object from string flag +* virt/connections: conditional loading from string flag +* db: LazyPluggable via string flag +* auth_manager: utils.import_class based on string flag +* Volumes: moving to pluggable driver instead of manager +* Network: pluggable managers +* Compute: same driver used, but pluggable at connection + + +Concept: IPC/RPC +---------------- + +Rabbit! + + +Concept: Fakes +-------------- + +* auth +* ldap + + +Concept: Scheduler +------------------ + +* simple +* random + + +Concept: Security Groups +------------------------ + +Security groups + + +Concept: Certificate Authority +------------------------------ + +Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns <../cloudpipe>` and decrypting bundled images. + + +Concept: Images +--------------- + +* launching +* bundling diff --git a/test/.Python b/test/.Python new file mode 120000 index 000000000..6cce156bd --- /dev/null +++ b/test/.Python @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/Python \ No newline at end of file diff --git a/test/bin/activate b/test/bin/activate new file mode 100644 index 000000000..588b54105 --- /dev/null +++ b/test/bin/activate @@ -0,0 +1,76 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "$_OLD_VIRTUAL_PATH" ] ; then + PATH="$_OLD_VIRTUAL_PATH" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then + PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then + hash -r + fi + + if [ -n "$_OLD_VIRTUAL_PS1" ] ; then + PS1="$_OLD_VIRTUAL_PS1" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "$1" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelavent variables +deactivate nondestructive + +VIRTUAL_ENV="/Users/anne.gentle/src/nova/docslice/test" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "$PYTHONHOME" ] ; then + _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" + unset PYTHONHOME +fi + +if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then + _OLD_VIRTUAL_PS1="$PS1" + if [ "x" != x ] ; then + PS1="$PS1" + else + if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" + else + PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" + fi + fi + export PS1 +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then + hash -r +fi diff --git a/test/bin/activate.csh b/test/bin/activate.csh new file mode 100644 index 000000000..d29ac339a --- /dev/null +++ b/test/bin/activate.csh @@ -0,0 +1,32 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelavent variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if ("" != "") then + set env_name = "" +else + if (`basename "$VIRTUAL_ENV"` == "__") then + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` + else + set env_name = `basename "$VIRTUAL_ENV"` + endif +endif +set prompt = "[$env_name] $prompt" +unset env_name + +rehash + diff --git a/test/bin/activate.fish b/test/bin/activate.fish new file mode 100644 index 000000000..de3a8901e --- /dev/null +++ b/test/bin/activate.fish @@ -0,0 +1,79 @@ +# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) +# you cannot run it directly + +function deactivate -d "Exit virtualenv and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + end + + set -e VIRTUAL_ENV + if test "$argv[1]" != "nondestructive" + # Self destruct! + functions -e deactivate + end +end + +# unset irrelavent variables +deactivate nondestructive + +set -gx VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# unset PYTHONHOME if set +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish shell uses a function, instead of env vars, + # to produce the prompt. Overriding the existing function is easy. + # However, adding to the current prompt, instead of clobbering it, + # is a little more work. + set -l oldpromptfile (tempfile) + if test $status + # save the current fish_prompt function... + echo "function _old_fish_prompt" >> $oldpromptfile + echo -n \# >> $oldpromptfile + functions fish_prompt >> $oldpromptfile + # we've made the "_old_fish_prompt" file, source it. + . $oldpromptfile + rm -f $oldpromptfile + + if test -n "" + # We've been given us a prompt override. + # + # FIXME: Unsure how to handle this *safely*. We could just eval() + # whatever is given, but the risk is a bit much. + echo "activate.fish: Alternative prompt prefix is not supported under fish-shell." 1>&2 + echo "activate.fish: Alter the fish_prompt in this file as needed." 1>&2 + end + + # with the original prompt function renamed, we can override with our own. + function fish_prompt + set -l _checkbase (basename "$VIRTUAL_ENV") + if test $_checkbase = "__" + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + printf "%s[%s]%s %s" (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) (_old_fish_prompt) + else + printf "%s(%s)%s%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) (_old_fish_prompt) + end + end + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" + end +end + diff --git a/test/bin/activate_this.py b/test/bin/activate_this.py new file mode 100644 index 000000000..aff6927d6 --- /dev/null +++ b/test/bin/activate_this.py @@ -0,0 +1,32 @@ +"""By using execfile(this_file, dict(__file__=this_file)) you will +activate this virtualenv environment. + +This can be used when you must use an existing Python interpreter, not +the virtualenv bin/python +""" + +try: + __file__ +except NameError: + raise AssertionError( + "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))") +import sys +import os + +base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if sys.platform == 'win32': + site_packages = os.path.join(base, 'Lib', 'site-packages') +else: + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') +prev_sys_path = list(sys.path) +import site +site.addsitedir(site_packages) +sys.real_prefix = sys.prefix +sys.prefix = base +# Move the added items to the front of the path: +new_sys_path = [] +for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) +sys.path[:0] = new_sys_path diff --git a/test/bin/easy_install b/test/bin/easy_install new file mode 100755 index 000000000..8544573fc --- /dev/null +++ b/test/bin/easy_install @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install' +__requires__ = 'setuptools==0.6c11' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install')() +) diff --git a/test/bin/easy_install-2.6 b/test/bin/easy_install-2.6 new file mode 100755 index 000000000..bad01d2d7 --- /dev/null +++ b/test/bin/easy_install-2.6 @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install-2.6' +__requires__ = 'setuptools==0.6c11' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install-2.6')() +) diff --git a/test/bin/pip b/test/bin/pip new file mode 100755 index 000000000..a3d2ed311 --- /dev/null +++ b/test/bin/pip @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip' +__requires__ = 'pip==0.8.1' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('pip==0.8.1', 'console_scripts', 'pip')() +) diff --git a/test/bin/pip-2.6 b/test/bin/pip-2.6 new file mode 100755 index 000000000..4ad06add1 --- /dev/null +++ b/test/bin/pip-2.6 @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip-2.6' +__requires__ = 'pip==0.8.1' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('pip==0.8.1', 'console_scripts', 'pip-2.6')() +) diff --git a/test/bin/python b/test/bin/python new file mode 100755 index 000000000..271b6ca30 Binary files /dev/null and b/test/bin/python differ diff --git a/test/bin/python2.6 b/test/bin/python2.6 new file mode 120000 index 000000000..d8654aa0e --- /dev/null +++ b/test/bin/python2.6 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/test/include/python2.6 b/test/include/python2.6 new file mode 120000 index 000000000..788ac55ba --- /dev/null +++ b/test/include/python2.6 @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/include/python2.6 \ No newline at end of file diff --git a/test/lib/python2.6/UserDict.py b/test/lib/python2.6/UserDict.py new file mode 120000 index 000000000..93e0864b1 --- /dev/null +++ b/test/lib/python2.6/UserDict.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/UserDict.py \ No newline at end of file diff --git a/test/lib/python2.6/_abcoll.py b/test/lib/python2.6/_abcoll.py new file mode 120000 index 000000000..ff1769246 --- /dev/null +++ b/test/lib/python2.6/_abcoll.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/_abcoll.py \ No newline at end of file diff --git a/test/lib/python2.6/abc.py b/test/lib/python2.6/abc.py new file mode 120000 index 000000000..79fa108b2 --- /dev/null +++ b/test/lib/python2.6/abc.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/abc.py \ No newline at end of file diff --git a/test/lib/python2.6/codecs.py b/test/lib/python2.6/codecs.py new file mode 120000 index 000000000..9e4bfb7fa --- /dev/null +++ b/test/lib/python2.6/codecs.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/codecs.py \ No newline at end of file diff --git a/test/lib/python2.6/config b/test/lib/python2.6/config new file mode 120000 index 000000000..eef498c62 --- /dev/null +++ b/test/lib/python2.6/config @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/config \ No newline at end of file diff --git a/test/lib/python2.6/copy_reg.py b/test/lib/python2.6/copy_reg.py new file mode 120000 index 000000000..7285fda0e --- /dev/null +++ b/test/lib/python2.6/copy_reg.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/copy_reg.py \ No newline at end of file diff --git a/test/lib/python2.6/distutils/__init__.py b/test/lib/python2.6/distutils/__init__.py new file mode 100644 index 000000000..7ebb41c04 --- /dev/null +++ b/test/lib/python2.6/distutils/__init__.py @@ -0,0 +1,91 @@ +import os +import sys +import warnings +import opcode # opcode is not a virtualenv module, so we can use it to find the stdlib + # Important! To work on pypy, this must be a module that resides in the + # lib-python/modified-x.y.z directory + +dirname = os.path.dirname + +distutils_path = os.path.join(os.path.dirname(opcode.__file__), 'distutils') +if os.path.normpath(distutils_path) == os.path.dirname(os.path.normpath(__file__)): + warnings.warn( + "The virtualenv distutils package at %s appears to be in the same location as the system distutils?") +else: + __path__.insert(0, distutils_path) + exec open(os.path.join(distutils_path, '__init__.py')).read() + +import dist +import sysconfig + + +## patch build_ext (distutils doesn't know how to get the libs directory +## path on windows - it hardcodes the paths around the patched sys.prefix) + +if sys.platform == 'win32': + from distutils.command.build_ext import build_ext as old_build_ext + class build_ext(old_build_ext): + def finalize_options (self): + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, basestring): + self.library_dirs = self.library_dirs.split(os.pathsep) + + self.library_dirs.insert(0, os.path.join(sys.real_prefix, "Libs")) + old_build_ext.finalize_options(self) + + from distutils.command import build_ext as build_ext_module + build_ext_module.build_ext = build_ext + +## distutils.dist patches: + +old_find_config_files = dist.Distribution.find_config_files +def find_config_files(self): + found = old_find_config_files(self) + system_distutils = os.path.join(distutils_path, 'distutils.cfg') + #if os.path.exists(system_distutils): + # found.insert(0, system_distutils) + # What to call the per-user config file + if os.name == 'posix': + user_filename = ".pydistutils.cfg" + else: + user_filename = "pydistutils.cfg" + user_filename = os.path.join(sys.prefix, user_filename) + if os.path.isfile(user_filename): + for item in list(found): + if item.endswith('pydistutils.cfg'): + found.remove(item) + found.append(user_filename) + return found +dist.Distribution.find_config_files = find_config_files + +## distutils.sysconfig patches: + +old_get_python_inc = sysconfig.get_python_inc +def sysconfig_get_python_inc(plat_specific=0, prefix=None): + if prefix is None: + prefix = sys.real_prefix + return old_get_python_inc(plat_specific, prefix) +sysconfig_get_python_inc.__doc__ = old_get_python_inc.__doc__ +sysconfig.get_python_inc = sysconfig_get_python_inc + +old_get_python_lib = sysconfig.get_python_lib +def sysconfig_get_python_lib(plat_specific=0, standard_lib=0, prefix=None): + if standard_lib and prefix is None: + prefix = sys.real_prefix + return old_get_python_lib(plat_specific, standard_lib, prefix) +sysconfig_get_python_lib.__doc__ = old_get_python_lib.__doc__ +sysconfig.get_python_lib = sysconfig_get_python_lib + +old_get_config_vars = sysconfig.get_config_vars +def sysconfig_get_config_vars(*args): + real_vars = old_get_config_vars(*args) + if sys.platform == 'win32': + lib_dir = os.path.join(sys.real_prefix, "libs") + if isinstance(real_vars, dict) and 'LIBDIR' not in real_vars: + real_vars['LIBDIR'] = lib_dir # asked for all + elif isinstance(real_vars, list) and 'LIBDIR' in args: + real_vars = real_vars + [lib_dir] # asked for list + return real_vars +sysconfig_get_config_vars.__doc__ = old_get_config_vars.__doc__ +sysconfig.get_config_vars = sysconfig_get_config_vars diff --git a/test/lib/python2.6/distutils/distutils.cfg b/test/lib/python2.6/distutils/distutils.cfg new file mode 100644 index 000000000..1af230ec9 --- /dev/null +++ b/test/lib/python2.6/distutils/distutils.cfg @@ -0,0 +1,6 @@ +# This is a config file local to this virtualenv installation +# You may include options that will be used by all distutils commands, +# and by easy_install. For instance: +# +# [easy_install] +# find_links = http://mylocalsite diff --git a/test/lib/python2.6/encodings b/test/lib/python2.6/encodings new file mode 120000 index 000000000..89f28f824 --- /dev/null +++ b/test/lib/python2.6/encodings @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/encodings \ No newline at end of file diff --git a/test/lib/python2.6/fnmatch.py b/test/lib/python2.6/fnmatch.py new file mode 120000 index 000000000..cce0594f2 --- /dev/null +++ b/test/lib/python2.6/fnmatch.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/fnmatch.py \ No newline at end of file diff --git a/test/lib/python2.6/genericpath.py b/test/lib/python2.6/genericpath.py new file mode 120000 index 000000000..b14e1bc26 --- /dev/null +++ b/test/lib/python2.6/genericpath.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/genericpath.py \ No newline at end of file diff --git a/test/lib/python2.6/lib-dynload b/test/lib/python2.6/lib-dynload new file mode 120000 index 000000000..4644b7072 --- /dev/null +++ b/test/lib/python2.6/lib-dynload @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/lib-dynload \ No newline at end of file diff --git a/test/lib/python2.6/linecache.py b/test/lib/python2.6/linecache.py new file mode 120000 index 000000000..783624da8 --- /dev/null +++ b/test/lib/python2.6/linecache.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/linecache.py \ No newline at end of file diff --git a/test/lib/python2.6/locale.py b/test/lib/python2.6/locale.py new file mode 120000 index 000000000..4e674c7b6 --- /dev/null +++ b/test/lib/python2.6/locale.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/locale.py \ No newline at end of file diff --git a/test/lib/python2.6/ntpath.py b/test/lib/python2.6/ntpath.py new file mode 120000 index 000000000..9b6b40f48 --- /dev/null +++ b/test/lib/python2.6/ntpath.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/ntpath.py \ No newline at end of file diff --git a/test/lib/python2.6/orig-prefix.txt b/test/lib/python2.6/orig-prefix.txt new file mode 100644 index 000000000..535eb0f0e --- /dev/null +++ b/test/lib/python2.6/orig-prefix.txt @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6 \ No newline at end of file diff --git a/test/lib/python2.6/os.py b/test/lib/python2.6/os.py new file mode 120000 index 000000000..92e6e9a7c --- /dev/null +++ b/test/lib/python2.6/os.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/os.py \ No newline at end of file diff --git a/test/lib/python2.6/posixpath.py b/test/lib/python2.6/posixpath.py new file mode 120000 index 000000000..c095d16a1 --- /dev/null +++ b/test/lib/python2.6/posixpath.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/posixpath.py \ No newline at end of file diff --git a/test/lib/python2.6/re.py b/test/lib/python2.6/re.py new file mode 120000 index 000000000..b4710c5f7 --- /dev/null +++ b/test/lib/python2.6/re.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/re.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/easy-install.pth b/test/lib/python2.6/site-packages/easy-install.pth new file mode 100644 index 000000000..7a6ae2b6d --- /dev/null +++ b/test/lib/python2.6/site-packages/easy-install.pth @@ -0,0 +1,4 @@ +import sys; sys.__plen = len(sys.path) +./setuptools-0.6c11-py2.6.egg +./pip-0.8.1-py2.6.egg +import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO new file mode 100644 index 000000000..29c30cb8c --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO @@ -0,0 +1,348 @@ +Metadata-Version: 1.0 +Name: pip +Version: 0.8.1 +Summary: pip installs packages. Python packages. An easy_install replacement +Home-page: http://pip.openplans.org +Author: Ian Bicking +Author-email: python-virtualenv@groups.google.com +License: MIT +Description: The main website for pip is `pip.openplans.org + `_. You can also install + the `in-development version `_ + of pip with ``easy_install pip==dev``. + + + Introduction + ------------ + + pip installs packages. Python packages. + + If you use `virtualenv `__ -- a tool + for installing libraries in a local and isolated manner -- you'll + automatically get a copy of pip. Free bonus! + + Once you have pip, you can use it like this:: + + $ pip install SomePackage + + SomePackage is some package you'll find on `PyPI + `_. This installs the package and all + its dependencies. + + pip does other stuff too, with packages, but install is the biggest + one. You can ``pip uninstall`` too. + + You can also install from a URL (that points to a tar or zip file), + install from some version control system (use URLs like + ``hg+http://domain/repo`` -- or prefix ``git+``, ``svn+`` etc). pip + knows a bunch of stuff about revisions and stuff, so if you need to do + things like install a very specific revision from a repository pip can + do that too. + + If you've ever used ``python setup.py develop``, you can do something + like that with ``pip install -e ./`` -- this works with packages that + use ``distutils`` too (usually this only works with Setuptools + projects). + + You can use ``pip install --upgrade SomePackage`` to upgrade to a + newer version, or ``pip install SomePackage==1.0.4`` to install a very + specific version. + + Pip Compared To easy_install + ---------------------------- + + pip is a replacement for `easy_install + `_. It uses mostly the + same techniques for finding packages, so packages that were made + easy_installable should be pip-installable as well. + + pip is meant to improve on easy_install. Some of the improvements: + + * All packages are downloaded before installation. Partially-completed + installation doesn't occur as a result. + + * Care is taken to present useful output on the console. + + * The reasons for actions are kept track of. For instance, if a package is + being installed, pip keeps track of why that package was required. + + * Error messages should be useful. + + * The code is relatively concise and cohesive, making it easier to use + programmatically. + + * Packages don't have to be installed as egg archives, they can be installed + flat (while keeping the egg metadata). + + * Native support for other version control systems (Git, Mercurial and Bazaar) + + * Uninstallation of packages. + + * Simple to define fixed sets of requirements and reliably reproduce a + set of packages. + + pip doesn't do everything that easy_install does. Specifically: + + * It cannot install from eggs. It only installs from source. (In the + future it would be good if it could install binaries from Windows ``.exe`` + or ``.msi`` -- binary install on other platforms is not a priority.) + + * It doesn't understand Setuptools extras (like ``package[test]``). This should + be added eventually. + + * It is incompatible with some packages that extensively customize distutils + or setuptools in their ``setup.py`` files. + + pip is complementary with `virtualenv + `__, and it is encouraged that you use + virtualenv to isolate your installation. + + Community + --------- + + The homepage for pip is temporarily located `on PyPI + `_ -- a more proper homepage will + follow. Bugs can go on the `pip issue tracker + `_. Discussion should happen on the + `virtualenv email group + `_. + + Uninstall + --------- + + pip is able to uninstall most installed packages with ``pip uninstall + package-name``. + + Known exceptions include pure-distutils packages installed with + ``python setup.py install`` (such packages leave behind no metadata allowing + determination of what files were installed), and script wrappers installed + by develop-installs (``python setup.py develop``). + + pip also performs an automatic uninstall of an old version of a package + before upgrading to a newer version, so outdated files (and egg-info data) + from conflicting versions aren't left hanging around to cause trouble. The + old version of the package is automatically restored if the new version + fails to download or install. + + .. _`requirements file`: + + Requirements Files + ------------------ + + When installing software, and Python packages in particular, it's common that + you get a lot of libraries installed. You just did ``easy_install MyPackage`` + and you get a dozen packages. Each of these packages has its own version. + + Maybe you ran that installation and it works. Great! Will it keep working? + Did you have to provide special options to get it to find everything? Did you + have to install a bunch of other optional pieces? Most of all, will you be able + to do it again? Requirements files give you a way to create an *environment*: + a *set* of packages that work together. + + If you've ever tried to setup an application on a new system, or with slightly + updated pieces, and had it fail, pip requirements are for you. If you + haven't had this problem then you will eventually, so pip requirements are + for you too -- requirements make explicit, repeatable installation of packages. + + So what are requirements files? They are very simple: lists of packages to + install. Instead of running something like ``pip MyApp`` and getting + whatever libraries come along, you can create a requirements file something like:: + + MyApp + Framework==0.9.4 + Library>=0.2 + + Then, regardless of what MyApp lists in ``setup.py``, you'll get a + specific version of Framework (0.9.4) and at least the 0.2 version of + Library. (You might think you could list these specific versions in + MyApp's ``setup.py`` -- but if you do that you'll have to edit MyApp + if you want to try a new version of Framework, or release a new + version of MyApp if you determine that Library 0.3 doesn't work with + your application.) You can also add optional libraries and support + tools that MyApp doesn't strictly require, giving people a set of + recommended libraries. + + You can also include "editable" packages -- packages that are checked out from + Subversion, Git, Mercurial and Bazaar. These are just like using the ``-e`` + option to pip. They look like:: + + -e svn+http://myrepo/svn/MyApp#egg=MyApp + + You have to start the URL with ``svn+`` (``git+``, ``hg+`` or ``bzr+``), and + you have to include ``#egg=Package`` so pip knows what to expect at that URL. + You can also include ``@rev`` in the URL, e.g., ``@275`` to check out + revision 275. + + Requirement files are mostly *flat*. Maybe ``MyApp`` requires + ``Framework``, and ``Framework`` requires ``Library``. I encourage + you to still list all these in a single requirement file; it is the + nature of Python programs that there are implicit bindings *directly* + between MyApp and Library. For instance, Framework might expose one + of Library's objects, and so if Library is updated it might directly + break MyApp. If that happens you can update the requirements file to + force an earlier version of Library, and you can do that without + having to re-release MyApp at all. + + Read the `requirements file format `_ to + learn about other features. + + Freezing Requirements + --------------------- + + So you have a working set of packages, and you want to be able to install them + elsewhere. `Requirements files`_ let you install exact versions, but it won't + tell you what all the exact versions are. + + To create a new requirements file from a known working environment, use:: + + $ pip freeze > stable-req.txt + + This will write a listing of *all* installed libraries to ``stable-req.txt`` + with exact versions for every library. You may want to edit the file down after + generating (e.g., to eliminate unnecessary libraries), but it'll give you a + stable starting point for constructing your requirements file. + + You can also give it an existing requirements file, and it will use that as a + sort of template for the new file. So if you do:: + + $ pip freeze -r devel-req.txt > stable-req.txt + + it will keep the packages listed in ``devel-req.txt`` in order and preserve + comments. + + Bundles + ------- + + Another way to distribute a set of libraries is a bundle format (specific to + pip). This format is not stable at this time (there simply hasn't been + any feedback, nor a great deal of thought). A bundle file contains all the + source for your package, and you can have pip install them all together. + Once you have the bundle file further network access won't be necessary. To + build a bundle file, do:: + + $ pip bundle MyApp.pybundle MyApp + + (Using a `requirements file`_ would be wise.) Then someone else can get the + file ``MyApp.pybundle`` and run:: + + $ pip install MyApp.pybundle + + This is *not* a binary format. This only packages source. If you have binary + packages, then the person who installs the files will have to have a compiler, + any necessary headers installed, etc. Binary packages are hard, this is + relatively easy. + + Using pip with virtualenv + ------------------------- + + pip is most nutritious when used with `virtualenv + `__. One of the reasons pip + doesn't install "multi-version" eggs is that virtualenv removes much of the need + for it. Because pip is installed by virtualenv, just use + ``path/to/my/environment/bin/pip`` to install things into that + specific environment. + + To tell pip to only run if there is a virtualenv currently activated, + and to bail if not, use:: + + export PIP_REQUIRE_VIRTUALENV=true + + To tell pip to automatically use the currently active virtualenv:: + + export PIP_RESPECT_VIRTUALENV=true + + Providing an environment with ``-E`` will be ignored. + + Using pip with virtualenvwrapper + --------------------------------- + + If you are using `virtualenvwrapper + `_, you might + want pip to automatically create its virtualenvs in your + ``$WORKON_HOME``. + + You can tell pip to do so by defining ``PIP_VIRTUALENV_BASE`` in your + environment and setting it to the same value as that of + ``$WORKON_HOME``. + + Do so by adding the line:: + + export PIP_VIRTUALENV_BASE=$WORKON_HOME + + in your .bashrc under the line starting with ``export WORKON_HOME``. + + Using pip with buildout + ----------------------- + + If you are using `zc.buildout + `_ you should look at + `gp.recipe.pip `_ as an + option to use pip and virtualenv in your buildouts. + + Command line completion + ----------------------- + + pip comes with support for command line completion in bash and zsh and + allows you tab complete commands and options. To enable it you simply + need copy the required shell script to the your shell startup file + (e.g. ``.profile`` or ``.zprofile``) by running the special ``completion`` + command, e.g. for bash:: + + $ pip completion --bash >> ~/.profile + + And for zsh:: + + $ pip completion --zsh >> ~/.zprofile + + Alternatively, you can use the result of the ``completion`` command + directly with the eval function of you shell, e.g. by adding:: + + eval "`pip completion --bash`" + + to your startup file. + + Searching for packages + ---------------------- + + pip can search the `Python Package Index `_ (PyPI) + for packages using the ``pip search`` command. To search, run:: + + $ pip search "query" + + The query will be used to search the names and summaries of all packages + indexed. + + pip searches http://pypi.python.org/pypi by default but alternative indexes + can be searched by using the ``--index`` flag. + + Mirror support + -------------- + + The `PyPI mirroring infrastructure `_ as + described in `PEP 381 `_ can be + used by passing the ``--use-mirrors`` option to the install command. + Alternatively, you can use the other ways to configure pip, e.g.:: + + $ export PIP_USE_MIRRORS=true + + If enabled, pip will automatically query the DNS entry of the mirror index URL + to find the list of mirrors to use. In case you want to override this list, + please use the ``--mirrors`` option of the install command, or add to your pip + configuration file:: + + [install] + use-mirrors = true + mirrors = + http://d.pypi.python.org + http://b.pypi.python.org + +Keywords: easy_install distutils setuptools egg virtualenv +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Topic :: Software Development :: Build Tools +Classifier: Programming Language :: Python :: 2.4 +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt new file mode 100644 index 000000000..3a068547e --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt @@ -0,0 +1,57 @@ +MANIFEST.in +setup.cfg +setup.py +docs/branches.txt +docs/ci-server-step-by-step.txt +docs/configuration.txt +docs/how-to-contribute.txt +docs/index.txt +docs/license.txt +docs/news.txt +docs/requirement-format.txt +docs/running-tests.txt +docs/_build/branches.html +docs/_build/ci-server-step-by-step.html +docs/_build/configuration.html +docs/_build/how-to-contribute.html +docs/_build/index.html +docs/_build/license.html +docs/_build/news.html +docs/_build/requirement-format.html +docs/_build/running-tests.html +docs/_build/search.html +pip/__init__.py +pip/_pkgutil.py +pip/backwardcompat.py +pip/basecommand.py +pip/baseparser.py +pip/download.py +pip/exceptions.py +pip/index.py +pip/locations.py +pip/log.py +pip/req.py +pip/runner.py +pip/util.py +pip/venv.py +pip.egg-info/PKG-INFO +pip.egg-info/SOURCES.txt +pip.egg-info/dependency_links.txt +pip.egg-info/entry_points.txt +pip.egg-info/not-zip-safe +pip.egg-info/top_level.txt +pip/commands/__init__.py +pip/commands/bundle.py +pip/commands/completion.py +pip/commands/freeze.py +pip/commands/help.py +pip/commands/install.py +pip/commands/search.py +pip/commands/uninstall.py +pip/commands/unzip.py +pip/commands/zip.py +pip/vcs/__init__.py +pip/vcs/bazaar.py +pip/vcs/git.py +pip/vcs/mercurial.py +pip/vcs/subversion.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt new file mode 100644 index 000000000..2b0afba73 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt @@ -0,0 +1,4 @@ +[console_scripts] +pip = pip:main +pip-2.6 = pip:main + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe @@ -0,0 +1 @@ + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt new file mode 100644 index 000000000..a1b589e38 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt @@ -0,0 +1 @@ +pip diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py new file mode 100644 index 000000000..c5de5c9a8 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +import os +import optparse + +import subprocess +import sys +import re +import difflib + +from pip.basecommand import command_dict, load_command, load_all_commands, command_names +from pip.baseparser import parser +from pip.exceptions import InstallationError +from pip.log import logger +from pip.util import get_installed_distributions +from pip.backwardcompat import walk_packages + + +def autocomplete(): + """Command and option completion for the main option parser (and options) + and its subcommands (and options). + + Enable by sourcing one of the completion shell scripts (bash or zsh). + """ + # Don't complete if user hasn't sourced bash_completion file. + if 'PIP_AUTO_COMPLETE' not in os.environ: + return + cwords = os.environ['COMP_WORDS'].split()[1:] + cword = int(os.environ['COMP_CWORD']) + try: + current = cwords[cword-1] + except IndexError: + current = '' + load_all_commands() + subcommands = [cmd for cmd, cls in command_dict.items() if not cls.hidden] + options = [] + # subcommand + try: + subcommand_name = [w for w in cwords if w in subcommands][0] + except IndexError: + subcommand_name = None + # subcommand options + if subcommand_name: + # special case: 'help' subcommand has no options + if subcommand_name == 'help': + sys.exit(1) + # special case: list locally installed dists for uninstall command + if subcommand_name == 'uninstall' and not current.startswith('-'): + installed = [] + lc = current.lower() + for dist in get_installed_distributions(local_only=True): + if dist.key.startswith(lc) and dist.key not in cwords[1:]: + installed.append(dist.key) + # if there are no dists installed, fall back to option completion + if installed: + for dist in installed: + print dist + sys.exit(1) + subcommand = command_dict.get(subcommand_name) + options += [(opt.get_opt_string(), opt.nargs) + for opt in subcommand.parser.option_list + if opt.help != optparse.SUPPRESS_HELP] + # filter out previously specified options from available options + prev_opts = [x.split('=')[0] for x in cwords[1:cword-1]] + options = filter(lambda (x, v): x not in prev_opts, options) + # filter options by current input + options = [(k, v) for k, v in options if k.startswith(current)] + for option in options: + opt_label = option[0] + # append '=' to options which require args + if option[1]: + opt_label += '=' + print opt_label + else: + # show options of main parser only when necessary + if current.startswith('-') or current.startswith('--'): + subcommands += [opt.get_opt_string() + for opt in parser.option_list + if opt.help != optparse.SUPPRESS_HELP] + print ' '.join(filter(lambda x: x.startswith(current), subcommands)) + sys.exit(1) + + +def version_control(): + # Import all the version control support modules: + from pip import vcs + for importer, modname, ispkg in \ + walk_packages(path=vcs.__path__, prefix=vcs.__name__+'.'): + __import__(modname) + + +def main(initial_args=None): + if initial_args is None: + initial_args = sys.argv[1:] + autocomplete() + version_control() + options, args = parser.parse_args(initial_args) + if options.help and not args: + args = ['help'] + if not args: + parser.error('You must give a command (use "pip help" to see a list of commands)') + command = args[0].lower() + load_command(command) + if command not in command_dict: + close_commands = difflib.get_close_matches(command, command_names()) + if close_commands: + guess = close_commands[0] + if args[1:]: + guess = "%s %s" % (guess, " ".join(args[1:])) + else: + guess = 'install %s' % command + error_dict = {'arg': command, 'guess': guess, + 'script': os.path.basename(sys.argv[0])} + parser.error('No command by the name %(script)s %(arg)s\n ' + '(maybe you meant "%(script)s %(guess)s")' % error_dict) + command = command_dict[command] + return command.main(initial_args, args[1:], options) + + +############################################################ +## Writing freeze files + + +class FrozenRequirement(object): + + def __init__(self, name, req, editable, comments=()): + self.name = name + self.req = req + self.editable = editable + self.comments = comments + + _rev_re = re.compile(r'-r(\d+)$') + _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') + + @classmethod + def from_dist(cls, dist, dependency_links, find_tags=False): + location = os.path.normcase(os.path.abspath(dist.location)) + comments = [] + from pip.vcs import vcs, get_src_requirement + if vcs.get_backend_name(location): + editable = True + req = get_src_requirement(dist, location, find_tags) + if req is None: + logger.warn('Could not determine repository location of %s' % location) + comments.append('## !! Could not determine repository location') + req = dist.as_requirement() + editable = False + else: + editable = False + req = dist.as_requirement() + specs = req.specs + assert len(specs) == 1 and specs[0][0] == '==' + version = specs[0][1] + ver_match = cls._rev_re.search(version) + date_match = cls._date_re.search(version) + if ver_match or date_match: + svn_backend = vcs.get_backend('svn') + if svn_backend: + svn_location = svn_backend( + ).get_location(dist, dependency_links) + if not svn_location: + logger.warn( + 'Warning: cannot find svn location for %s' % req) + comments.append('## FIXME: could not find svn URL in dependency_links for this package:') + else: + comments.append('# Installing as editable to satisfy requirement %s:' % req) + if ver_match: + rev = ver_match.group(1) + else: + rev = '{%s}' % date_match.group(1) + editable = True + req = '%s@%s#egg=%s' % (svn_location, rev, cls.egg_name(dist)) + return cls(dist.project_name, req, editable, comments) + + @staticmethod + def egg_name(dist): + name = dist.egg_name() + match = re.search(r'-py\d\.\d$', name) + if match: + name = name[:match.start()] + return name + + def __str__(self): + req = self.req + if self.editable: + req = '-e %s' % req + return '\n'.join(list(self.comments)+[str(req)])+'\n' + +############################################################ +## Requirement files + + +def call_subprocess(cmd, show_stdout=True, + filter_stdout=None, cwd=None, + raise_on_returncode=True, + command_level=logger.DEBUG, command_desc=None, + extra_environ=None): + if command_desc is None: + cmd_parts = [] + for part in cmd: + if ' ' in part or '\n' in part or '"' in part or "'" in part: + part = '"%s"' % part.replace('"', '\\"') + cmd_parts.append(part) + command_desc = ' '.join(cmd_parts) + if show_stdout: + stdout = None + else: + stdout = subprocess.PIPE + logger.log(command_level, "Running command %s" % command_desc) + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + try: + proc = subprocess.Popen( + cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, + cwd=cwd, env=env) + except Exception, e: + logger.fatal( + "Error %s while executing command %s" % (e, command_desc)) + raise + all_output = [] + if stdout is not None: + stdout = proc.stdout + while 1: + line = stdout.readline() + if not line: + break + line = line.rstrip() + all_output.append(line + '\n') + if filter_stdout: + level = filter_stdout(line) + if isinstance(level, tuple): + level, line = level + logger.log(level, line) + if not logger.stdout_level_matches(level): + logger.show_progress() + else: + logger.info(line) + else: + returned_stdout, returned_stderr = proc.communicate() + all_output = [returned_stdout or ''] + proc.wait() + if proc.returncode: + if raise_on_returncode: + if all_output: + logger.notify('Complete output from command %s:' % command_desc) + logger.notify('\n'.join(all_output) + '\n----------------------------------------') + raise InstallationError( + "Command %s failed with error code %s" + % (command_desc, proc.returncode)) + else: + logger.warn( + "Command %s had error code %s" + % (command_desc, proc.returncode)) + if stdout is not None: + return ''.join(all_output) + + +if __name__ == '__main__': + exit = main() + if exit: + sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py new file mode 100644 index 000000000..f8fb8aa61 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py @@ -0,0 +1,589 @@ +"""Utilities to support packages.""" + +# NOTE: This module must remain compatible with Python 2.3, as it is shared +# by setuptools for distribution with Python 2.3 and up. + +import os +import sys +import imp +import os.path +from types import ModuleType + +__all__ = [ + 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'walk_packages', 'iter_modules', + 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', +] + + +def read_code(stream): + # This helper is needed in order for the PEP 302 emulation to + # correctly handle compiled files + import marshal + + magic = stream.read(4) + if magic != imp.get_magic(): + return None + + stream.read(4) # Skip timestamp + return marshal.load(stream) + + +def simplegeneric(func): + """Make a trivial single-dispatch generic function""" + registry = {} + + def wrapper(*args, **kw): + ob = args[0] + try: + cls = ob.__class__ + except AttributeError: + cls = type(ob) + try: + mro = cls.__mro__ + except AttributeError: + try: + + class cls(cls, object): + pass + + mro = cls.__mro__[1:] + except TypeError: + mro = object, # must be an ExtensionClass or some such :( + for t in mro: + if t in registry: + return registry[t](*args, **kw) + else: + return func(*args, **kw) + try: + wrapper.__name__ = func.__name__ + except (TypeError, AttributeError): + pass # Python 2.3 doesn't allow functions to be renamed + + def register(typ, func=None): + if func is None: + return lambda f: register(typ, f) + registry[typ] = func + return func + + wrapper.__dict__ = func.__dict__ + wrapper.__doc__ = func.__doc__ + wrapper.register = register + return wrapper + + +def walk_packages(path=None, prefix='', onerror=None): + """Yields (module_loader, name, ispkg) for all modules recursively + on path, or, if path is None, all accessible modules. + + 'path' should be either None or a list of paths to look for + modules in. + + 'prefix' is a string to output on the front of every module name + on output. + + Note that this function must import all *packages* (NOT all + modules!) on the given path, in order to access the __path__ + attribute to find submodules. + + 'onerror' is a function which gets called with one argument (the + name of the package which was being imported) if any exception + occurs while trying to import a package. If no onerror function is + supplied, ImportErrors are caught and ignored, while all other + exceptions are propagated, terminating the search. + + Examples: + + # list all modules python can access + walk_packages() + + # list all submodules of ctypes + walk_packages(ctypes.__path__, ctypes.__name__+'.') + """ + + def seen(p, m={}): + if p in m: + return True + m[p] = True + + for importer, name, ispkg in iter_modules(path, prefix): + yield importer, name, ispkg + + if ispkg: + try: + __import__(name) + except ImportError: + if onerror is not None: + onerror(name) + except Exception: + if onerror is not None: + onerror(name) + else: + raise + else: + path = getattr(sys.modules[name], '__path__', None) or [] + + # don't traverse path items we've seen before + path = [p for p in path if not seen(p)] + + for item in walk_packages(path, name+'.', onerror): + yield item + + +def iter_modules(path=None, prefix=''): + """Yields (module_loader, name, ispkg) for all submodules on path, + or, if path is None, all top-level modules on sys.path. + + 'path' should be either None or a list of paths to look for + modules in. + + 'prefix' is a string to output on the front of every module name + on output. + """ + + if path is None: + importers = iter_importers() + else: + importers = map(get_importer, path) + + yielded = {} + for i in importers: + for name, ispkg in iter_importer_modules(i, prefix): + if name not in yielded: + yielded[name] = 1 + yield i, name, ispkg + + +#@simplegeneric +def iter_importer_modules(importer, prefix=''): + if not hasattr(importer, 'iter_modules'): + return [] + return importer.iter_modules(prefix) + +iter_importer_modules = simplegeneric(iter_importer_modules) + + +class ImpImporter: + """PEP 302 Importer that wraps Python's "classic" import algorithm + + ImpImporter(dirname) produces a PEP 302 importer that searches that + directory. ImpImporter(None) produces a PEP 302 importer that searches + the current sys.path, plus any modules that are frozen or built-in. + + Note that ImpImporter does not currently support being used by placement + on sys.meta_path. + """ + + def __init__(self, path=None): + self.path = path + + def find_module(self, fullname, path=None): + # Note: we ignore 'path' argument since it is only used via meta_path + subname = fullname.split(".")[-1] + if subname != fullname and self.path is None: + return None + if self.path is None: + path = None + else: + path = [os.path.realpath(self.path)] + try: + file, filename, etc = imp.find_module(subname, path) + except ImportError: + return None + return ImpLoader(fullname, file, filename, etc) + + def iter_modules(self, prefix=''): + if self.path is None or not os.path.isdir(self.path): + return + + yielded = {} + import inspect + + filenames = os.listdir(self.path) + filenames.sort() # handle packages before same-named modules + + for fn in filenames: + modname = inspect.getmodulename(fn) + if modname=='__init__' or modname in yielded: + continue + + path = os.path.join(self.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + for fn in os.listdir(path): + subname = inspect.getmodulename(fn) + if subname=='__init__': + ispkg = True + break + else: + continue # not a package + + if modname and '.' not in modname: + yielded[modname] = 1 + yield prefix + modname, ispkg + + +class ImpLoader: + """PEP 302 Loader that wraps Python's "classic" import algorithm + """ + code = source = None + + def __init__(self, fullname, file, filename, etc): + self.file = file + self.filename = filename + self.fullname = fullname + self.etc = etc + + def load_module(self, fullname): + self._reopen() + try: + mod = imp.load_module(fullname, self.file, self.filename, self.etc) + finally: + if self.file: + self.file.close() + # Note: we don't set __loader__ because we want the module to look + # normal; i.e. this is just a wrapper for standard import machinery + return mod + + def get_data(self, pathname): + return open(pathname, "rb").read() + + def _reopen(self): + if self.file and self.file.closed: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + self.file = open(self.filename, 'rU') + elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): + self.file = open(self.filename, 'rb') + + def _fix_name(self, fullname): + if fullname is None: + fullname = self.fullname + elif fullname != self.fullname: + raise ImportError("Loader for module %s cannot handle " + "module %s" % (self.fullname, fullname)) + return fullname + + def is_package(self, fullname): + fullname = self._fix_name(fullname) + return self.etc[2]==imp.PKG_DIRECTORY + + def get_code(self, fullname=None): + fullname = self._fix_name(fullname) + if self.code is None: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + source = self.get_source(fullname) + self.code = compile(source, self.filename, 'exec') + elif mod_type==imp.PY_COMPILED: + self._reopen() + try: + self.code = read_code(self.file) + finally: + self.file.close() + elif mod_type==imp.PKG_DIRECTORY: + self.code = self._get_delegate().get_code() + return self.code + + def get_source(self, fullname=None): + fullname = self._fix_name(fullname) + if self.source is None: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + self._reopen() + try: + self.source = self.file.read() + finally: + self.file.close() + elif mod_type==imp.PY_COMPILED: + if os.path.exists(self.filename[:-1]): + f = open(self.filename[:-1], 'rU') + self.source = f.read() + f.close() + elif mod_type==imp.PKG_DIRECTORY: + self.source = self._get_delegate().get_source() + return self.source + + def _get_delegate(self): + return ImpImporter(self.filename).find_module('__init__') + + def get_filename(self, fullname=None): + fullname = self._fix_name(fullname) + mod_type = self.etc[2] + if self.etc[2]==imp.PKG_DIRECTORY: + return self._get_delegate().get_filename() + elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): + return self.filename + return None + + +try: + import zipimport + from zipimport import zipimporter + + def iter_zipimport_modules(importer, prefix=''): + dirlist = zipimport._zip_directory_cache[importer.archive].keys() + dirlist.sort() + _prefix = importer.prefix + plen = len(_prefix) + yielded = {} + import inspect + for fn in dirlist: + if not fn.startswith(_prefix): + continue + + fn = fn[plen:].split(os.sep) + + if len(fn)==2 and fn[1].startswith('__init__.py'): + if fn[0] not in yielded: + yielded[fn[0]] = 1 + yield fn[0], True + + if len(fn)!=1: + continue + + modname = inspect.getmodulename(fn[0]) + if modname=='__init__': + continue + + if modname and '.' not in modname and modname not in yielded: + yielded[modname] = 1 + yield prefix + modname, False + + iter_importer_modules.register(zipimporter, iter_zipimport_modules) + +except ImportError: + pass + + +def get_importer(path_item): + """Retrieve a PEP 302 importer for the given path item + + The returned importer is cached in sys.path_importer_cache + if it was newly created by a path hook. + + If there is no importer, a wrapper around the basic import + machinery is returned. This wrapper is never inserted into + the importer cache (None is inserted instead). + + The cache (or part of it) can be cleared manually if a + rescan of sys.path_hooks is necessary. + """ + try: + importer = sys.path_importer_cache[path_item] + except KeyError: + for path_hook in sys.path_hooks: + try: + importer = path_hook(path_item) + break + except ImportError: + pass + else: + importer = None + sys.path_importer_cache.setdefault(path_item, importer) + + if importer is None: + try: + importer = ImpImporter(path_item) + except ImportError: + importer = None + return importer + + +def iter_importers(fullname=""): + """Yield PEP 302 importers for the given module name + + If fullname contains a '.', the importers will be for the package + containing fullname, otherwise they will be importers for sys.meta_path, + sys.path, and Python's "classic" import machinery, in that order. If + the named module is in a package, that package is imported as a side + effect of invoking this function. + + Non PEP 302 mechanisms (e.g. the Windows registry) used by the + standard import machinery to find files in alternative locations + are partially supported, but are searched AFTER sys.path. Normally, + these locations are searched BEFORE sys.path, preventing sys.path + entries from shadowing them. + + For this to cause a visible difference in behaviour, there must + be a module or package name that is accessible via both sys.path + and one of the non PEP 302 file system mechanisms. In this case, + the emulation will find the former version, while the builtin + import mechanism will find the latter. + + Items of the following types can be affected by this discrepancy: + imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY + """ + if fullname.startswith('.'): + raise ImportError("Relative module names not supported") + if '.' in fullname: + # Get the containing package's __path__ + pkg = '.'.join(fullname.split('.')[:-1]) + if pkg not in sys.modules: + __import__(pkg) + path = getattr(sys.modules[pkg], '__path__', None) or [] + else: + for importer in sys.meta_path: + yield importer + path = sys.path + for item in path: + yield get_importer(item) + if '.' not in fullname: + yield ImpImporter() + + +def get_loader(module_or_name): + """Get a PEP 302 "loader" object for module_or_name + + If the module or package is accessible via the normal import + mechanism, a wrapper around the relevant part of that machinery + is returned. Returns None if the module cannot be found or imported. + If the named module is not already imported, its containing package + (if any) is imported, in order to establish the package __path__. + + This function uses iter_importers(), and is thus subject to the same + limitations regarding platform-specific special import locations such + as the Windows registry. + """ + if module_or_name in sys.modules: + module_or_name = sys.modules[module_or_name] + if isinstance(module_or_name, ModuleType): + module = module_or_name + loader = getattr(module, '__loader__', None) + if loader is not None: + return loader + fullname = module.__name__ + else: + fullname = module_or_name + return find_loader(fullname) + + +def find_loader(fullname): + """Find a PEP 302 "loader" object for fullname + + If fullname contains dots, path must be the containing package's __path__. + Returns None if the module cannot be found or imported. This function uses + iter_importers(), and is thus subject to the same limitations regarding + platform-specific special import locations such as the Windows registry. + """ + for importer in iter_importers(fullname): + loader = importer.find_module(fullname) + if loader is not None: + return loader + + return None + + +def extend_path(path, name): + """Extend a package's path. + + Intended use is to place the following code in a package's __init__.py: + + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) + + This will add to the package's __path__ all subdirectories of + directories on sys.path named after the package. This is useful + if one wants to distribute different parts of a single logical + package as multiple directories. + + It also looks for *.pkg files beginning where * matches the name + argument. This feature is similar to *.pth files (see site.py), + except that it doesn't special-case lines starting with 'import'. + A *.pkg file is trusted at face value: apart from checking for + duplicates, all entries found in a *.pkg file are added to the + path, regardless of whether they are exist the filesystem. (This + is a feature.) + + If the input path is not a list (as is the case for frozen + packages) it is returned unchanged. The input path is not + modified; an extended copy is returned. Items are only appended + to the copy at the end. + + It is assumed that sys.path is a sequence. Items of sys.path that + are not (unicode or 8-bit) strings referring to existing + directories are ignored. Unicode items of sys.path that cause + errors when used as filenames may cause this function to raise an + exception (in line with os.path.isdir() behavior). + """ + + if not isinstance(path, list): + # This could happen e.g. when this is called from inside a + # frozen package. Return the path unchanged in that case. + return path + + pname = os.path.join(*name.split('.')) # Reconstitute as relative path + # Just in case os.extsep != '.' + sname = os.extsep.join(name.split('.')) + sname_pkg = sname + os.extsep + "pkg" + init_py = "__init__" + os.extsep + "py" + + path = path[:] # Start with a copy of the existing path + + for dir in sys.path: + if not isinstance(dir, basestring) or not os.path.isdir(dir): + continue + subdir = os.path.join(dir, pname) + # XXX This may still add duplicate entries to path on + # case-insensitive filesystems + initfile = os.path.join(subdir, init_py) + if subdir not in path and os.path.isfile(initfile): + path.append(subdir) + # XXX Is this the right thing for subpackages like zope.app? + # It looks for a file named "zope.app.pkg" + pkgfile = os.path.join(dir, sname_pkg) + if os.path.isfile(pkgfile): + try: + f = open(pkgfile) + except IOError, msg: + sys.stderr.write("Can't open %s: %s\n" % + (pkgfile, msg)) + else: + for line in f: + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + path.append(line) # Don't check for existence! + f.close() + + return path + + +def get_data(package, resource): + """Get a resource from a package. + + This is a wrapper round the PEP 302 loader get_data API. The package + argument should be the name of a package, in standard module format + (foo.bar). The resource argument should be in the form of a relative + filename, using '/' as the path separator. The parent directory name '..' + is not allowed, and nor is a rooted name (starting with a '/'). + + The function returns a binary string, which is the contents of the + specified resource. + + For packages located in the filesystem, which have already been imported, + this is the rough equivalent of + + d = os.path.dirname(sys.modules[package].__file__) + data = open(os.path.join(d, resource), 'rb').read() + + If the package cannot be located or loaded, or it uses a PEP 302 loader + which does not support get_data(), then None is returned. + """ + + loader = get_loader(package) + if loader is None or not hasattr(loader, 'get_data'): + return None + mod = sys.modules.get(package) or loader.load_module(package) + if mod is None or not hasattr(mod, '__file__'): + return None + + # Modify the resource name to be compatible with the loader.get_data + # signature - an os.path format "filename" starting with the dirname of + # the package's __file__ + parts = resource.split('/') + parts.insert(0, os.path.dirname(mod.__file__)) + resource_name = os.path.join(*parts) + return loader.get_data(resource_name) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py new file mode 100644 index 000000000..e7c11f1d3 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py @@ -0,0 +1,55 @@ +"""Stuff that isn't in some old versions of Python""" + +import sys +import os +import shutil + +__all__ = ['any', 'WindowsError', 'md5', 'copytree'] + +try: + WindowsError = WindowsError +except NameError: + WindowsError = None +try: + from hashlib import md5 +except ImportError: + import md5 as md5_module + md5 = md5_module.new + +try: + from pkgutil import walk_packages +except ImportError: + # let's fall back as long as we can + from _pkgutil import walk_packages + +try: + any = any +except NameError: + + def any(seq): + for item in seq: + if item: + return True + return False + + +def copytree(src, dst): + if sys.version_info < (2, 5): + before_last_dir = os.path.dirname(dst) + if not os.path.exists(before_last_dir): + os.makedirs(before_last_dir) + shutil.copytree(src, dst) + shutil.copymode(src, dst) + else: + shutil.copytree(src, dst) + + +def product(*args, **kwds): + # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy + # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py new file mode 100644 index 000000000..f450e8393 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py @@ -0,0 +1,203 @@ +"""Base Command class, and related routines""" + +from cStringIO import StringIO +import getpass +import os +import socket +import sys +import traceback +import time +import urllib +import urllib2 + +from pip import commands +from pip.log import logger +from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpFormatter +from pip.download import urlopen +from pip.exceptions import BadCommand, InstallationError, UninstallationError +from pip.venv import restart_in_venv +from pip.backwardcompat import walk_packages + +__all__ = ['command_dict', 'Command', 'load_all_commands', + 'load_command', 'command_names'] + +command_dict = {} + +# for backwards compatibiliy +get_proxy = urlopen.get_proxy + + +class Command(object): + name = None + usage = None + hidden = False + + def __init__(self): + assert self.name + self.parser = ConfigOptionParser( + usage=self.usage, + prog='%s %s' % (sys.argv[0], self.name), + version=parser.version, + formatter=UpdatingDefaultsHelpFormatter(), + name=self.name) + for option in parser.option_list: + if not option.dest or option.dest == 'help': + # -h, --version, etc + continue + self.parser.add_option(option) + command_dict[self.name] = self + + def merge_options(self, initial_options, options): + # Make sure we have all global options carried over + for attr in ['log', 'venv', 'proxy', 'venv_base', 'require_venv', + 'respect_venv', 'log_explicit_levels', 'log_file', + 'timeout', 'default_vcs', 'skip_requirements_regex', + 'no_input']: + setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr)) + options.quiet += initial_options.quiet + options.verbose += initial_options.verbose + + def setup_logging(self): + pass + + def main(self, complete_args, args, initial_options): + options, args = self.parser.parse_args(args) + self.merge_options(initial_options, options) + + level = 1 # Notify + level += options.verbose + level -= options.quiet + level = logger.level_for_integer(4-level) + complete_log = [] + logger.consumers.extend( + [(level, sys.stdout), + (logger.DEBUG, complete_log.append)]) + if options.log_explicit_levels: + logger.explicit_levels = True + + self.setup_logging() + + if options.require_venv and not options.venv: + # If a venv is required check if it can really be found + if not os.environ.get('VIRTUAL_ENV'): + logger.fatal('Could not find an activated virtualenv (required).') + sys.exit(3) + # Automatically install in currently activated venv if required + options.respect_venv = True + + if args and args[-1] == '___VENV_RESTART___': + ## FIXME: We don't do anything this this value yet: + args = args[:-2] + options.venv = None + else: + # If given the option to respect the activated environment + # check if no venv is given as a command line parameter + if options.respect_venv and os.environ.get('VIRTUAL_ENV'): + if options.venv and os.path.exists(options.venv): + # Make sure command line venv and environmental are the same + if (os.path.realpath(os.path.expanduser(options.venv)) != + os.path.realpath(os.environ.get('VIRTUAL_ENV'))): + logger.fatal("Given virtualenv (%s) doesn't match " + "currently activated virtualenv (%s)." + % (options.venv, os.environ.get('VIRTUAL_ENV'))) + sys.exit(3) + else: + options.venv = os.environ.get('VIRTUAL_ENV') + logger.info('Using already activated environment %s' % options.venv) + if options.venv: + logger.info('Running in environment %s' % options.venv) + site_packages=False + if options.site_packages: + site_packages=True + restart_in_venv(options.venv, options.venv_base, site_packages, + complete_args) + # restart_in_venv should actually never return, but for clarity... + return + + ## FIXME: not sure if this sure come before or after venv restart + if options.log: + log_fp = open_logfile(options.log, 'a') + logger.consumers.append((logger.DEBUG, log_fp)) + else: + log_fp = None + + socket.setdefaulttimeout(options.timeout or None) + + urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) + + exit = 0 + try: + self.run(options, args) + except (InstallationError, UninstallationError), e: + logger.fatal(str(e)) + logger.info('Exception information:\n%s' % format_exc()) + exit = 1 + except BadCommand, e: + logger.fatal(str(e)) + logger.info('Exception information:\n%s' % format_exc()) + exit = 1 + except: + logger.fatal('Exception:\n%s' % format_exc()) + exit = 2 + + if log_fp is not None: + log_fp.close() + if exit: + log_fn = options.log_file + text = '\n'.join(complete_log) + logger.fatal('Storing complete log in %s' % log_fn) + log_fp = open_logfile(log_fn, 'w') + log_fp.write(text) + log_fp.close() + return exit + + + + +def format_exc(exc_info=None): + if exc_info is None: + exc_info = sys.exc_info() + out = StringIO() + traceback.print_exception(*exc_info, **dict(file=out)) + return out.getvalue() + + +def open_logfile(filename, mode='a'): + """Open the named log file in append mode. + + If the file already exists, a separator will also be printed to + the file to separate past activity from current activity. + """ + filename = os.path.expanduser(filename) + filename = os.path.abspath(filename) + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + exists = os.path.exists(filename) + + log_fp = open(filename, mode) + if exists: + print >> log_fp, '-'*60 + print >> log_fp, '%s run on %s' % (sys.argv[0], time.strftime('%c')) + return log_fp + + +def load_command(name): + full_name = 'pip.commands.%s' % name + if full_name in sys.modules: + return + try: + __import__(full_name) + except ImportError: + pass + + +def load_all_commands(): + for name in command_names(): + load_command(name) + + +def command_names(): + names = set((pkg[1] for pkg in walk_packages(path=commands.__path__))) + return list(names) + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py new file mode 100644 index 000000000..a8bd6ce4f --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py @@ -0,0 +1,231 @@ +"""Base option parser setup""" + +import sys +import optparse +import pkg_resources +import ConfigParser +import os +from distutils.util import strtobool +from pip.locations import default_config_file, default_log_file + + +class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter): + """Custom help formatter for use in ConfigOptionParser that updates + the defaults before expanding them, allowing them to show up correctly + in the help listing""" + + def expand_default(self, option): + if self.parser is not None: + self.parser.update_defaults(self.parser.defaults) + return optparse.IndentedHelpFormatter.expand_default(self, option) + + +class ConfigOptionParser(optparse.OptionParser): + """Custom option parser which updates its defaults by by checking the + configuration files and environmental variables""" + + def __init__(self, *args, **kwargs): + self.config = ConfigParser.RawConfigParser() + self.name = kwargs.pop('name') + self.files = self.get_config_files() + self.config.read(self.files) + assert self.name + optparse.OptionParser.__init__(self, *args, **kwargs) + + def get_config_files(self): + config_file = os.environ.get('PIP_CONFIG_FILE', False) + if config_file and os.path.exists(config_file): + return [config_file] + return [default_config_file] + + def update_defaults(self, defaults): + """Updates the given defaults with values from the config files and + the environ. Does a little special handling for certain types of + options (lists).""" + # Then go and look for the other sources of configuration: + config = {} + # 1. config files + for section in ('global', self.name): + config.update(dict(self.get_config_section(section))) + # 2. environmental variables + config.update(dict(self.get_environ_vars())) + # Then set the options with those values + for key, val in config.iteritems(): + key = key.replace('_', '-') + if not key.startswith('--'): + key = '--%s' % key # only prefer long opts + option = self.get_option(key) + if option is not None: + # ignore empty values + if not val: + continue + # handle multiline configs + if option.action == 'append': + val = val.split() + else: + option.nargs = 1 + if option.action in ('store_true', 'store_false', 'count'): + val = strtobool(val) + try: + val = option.convert_value(key, val) + except optparse.OptionValueError, e: + print ("An error occured during configuration: %s" % e) + sys.exit(3) + defaults[option.dest] = val + return defaults + + def get_config_section(self, name): + """Get a section of a configuration""" + if self.config.has_section(name): + return self.config.items(name) + return [] + + def get_environ_vars(self, prefix='PIP_'): + """Returns a generator with all environmental vars with prefix PIP_""" + for key, val in os.environ.iteritems(): + if key.startswith(prefix): + yield (key.replace(prefix, '').lower(), val) + + def get_default_values(self): + """Overridding to make updating the defaults after instantiation of + the option parser possible, update_defaults() does the dirty work.""" + if not self.process_default_values: + # Old, pre-Optik 1.5 behaviour. + return optparse.Values(self.defaults) + + defaults = self.update_defaults(self.defaults.copy()) # ours + for option in self._get_all_options(): + default = defaults.get(option.dest) + if isinstance(default, basestring): + opt_str = option.get_opt_string() + defaults[option.dest] = option.check_value(opt_str, default) + return optparse.Values(defaults) + +try: + pip_dist = pkg_resources.get_distribution('pip') + version = '%s from %s (python %s)' % ( + pip_dist, pip_dist.location, sys.version[:3]) +except pkg_resources.DistributionNotFound: + # when running pip.py without installing + version=None + +parser = ConfigOptionParser( + usage='%prog COMMAND [OPTIONS]', + version=version, + add_help_option=False, + formatter=UpdatingDefaultsHelpFormatter(), + name='global') + +parser.add_option( + '-h', '--help', + dest='help', + action='store_true', + help='Show help') +parser.add_option( + '-E', '--environment', + dest='venv', + metavar='DIR', + help='virtualenv environment to run pip in (either give the ' + 'interpreter or the environment base directory)') +parser.add_option( + '-s', '--enable-site-packages', + dest='site_packages', + action='store_true', + help='Include site-packages in virtualenv if one is to be ' + 'created. Ignored if --environment is not used or ' + 'the virtualenv already exists.') +parser.add_option( + # Defines a default root directory for virtualenvs, relative + # virtualenvs names/paths are considered relative to it. + '--virtualenv-base', + dest='venv_base', + type='str', + default='', + help=optparse.SUPPRESS_HELP) +parser.add_option( + # Run only if inside a virtualenv, bail if not. + '--require-virtualenv', '--require-venv', + dest='require_venv', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) +parser.add_option( + # Use automatically an activated virtualenv instead of installing + # globally. -E will be ignored if used. + '--respect-virtualenv', '--respect-venv', + dest='respect_venv', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) + +parser.add_option( + '-v', '--verbose', + dest='verbose', + action='count', + default=0, + help='Give more output') +parser.add_option( + '-q', '--quiet', + dest='quiet', + action='count', + default=0, + help='Give less output') +parser.add_option( + '--log', + dest='log', + metavar='FILENAME', + help='Log file where a complete (maximum verbosity) record will be kept') +parser.add_option( + # Writes the log levels explicitely to the log' + '--log-explicit-levels', + dest='log_explicit_levels', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) +parser.add_option( + # The default log file + '--local-log', '--log-file', + dest='log_file', + metavar='FILENAME', + default=default_log_file, + help=optparse.SUPPRESS_HELP) +parser.add_option( + # Don't ask for input + '--no-input', + dest='no_input', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) + +parser.add_option( + '--proxy', + dest='proxy', + type='str', + default='', + help="Specify a proxy in the form user:passwd@proxy.server:port. " + "Note that the user:password@ is optional and required only if you " + "are behind an authenticated proxy. If you provide " + "user@proxy.server:port then you will be prompted for a password.") +parser.add_option( + '--timeout', '--default-timeout', + metavar='SECONDS', + dest='timeout', + type='float', + default=15, + help='Set the socket timeout (default %default seconds)') +parser.add_option( + # The default version control system for editables, e.g. 'svn' + '--default-vcs', + dest='default_vcs', + type='str', + default='', + help=optparse.SUPPRESS_HELP) +parser.add_option( + # A regex to be used to skip requirements + '--skip-requirements-regex', + dest='skip_requirements_regex', + type='str', + default='', + help=optparse.SUPPRESS_HELP) + +parser.disable_interspersed_args() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py new file mode 100644 index 000000000..792d60054 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py @@ -0,0 +1 @@ +# diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py new file mode 100644 index 000000000..fb0f75704 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py @@ -0,0 +1,33 @@ +from pip.locations import build_prefix, src_prefix +from pip.util import display_path, backup_dir +from pip.log import logger +from pip.exceptions import InstallationError +from pip.commands.install import InstallCommand + + +class BundleCommand(InstallCommand): + name = 'bundle' + usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...' + summary = 'Create pybundles (archives containing multiple packages)' + bundle = True + + def __init__(self): + super(BundleCommand, self).__init__() + + def run(self, options, args): + if not args: + raise InstallationError('You must give a bundle filename') + if not options.build_dir: + options.build_dir = backup_dir(build_prefix, '-bundle') + if not options.src_dir: + options.src_dir = backup_dir(src_prefix, '-bundle') + # We have to get everything when creating a bundle: + options.ignore_installed = True + logger.notify('Putting temporary build files in %s and source/develop files in %s' + % (display_path(options.build_dir), display_path(options.src_dir))) + self.bundle_filename = args.pop(0) + requirement_set = super(BundleCommand, self).run(options, args) + return requirement_set + + +BundleCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py new file mode 100644 index 000000000..d003b9ae3 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py @@ -0,0 +1,60 @@ +import sys +from pip.basecommand import Command + +BASE_COMPLETION = """ +# pip %(shell)s completion start%(script)s# pip %(shell)s completion end +""" + +COMPLETION_SCRIPTS = { + 'bash': """ +_pip_completion() +{ + COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ + COMP_CWORD=$COMP_CWORD \\ + PIP_AUTO_COMPLETE=1 $1 ) ) +} +complete -o default -F _pip_completion pip +""", 'zsh': """ +function _pip_completion { + local words cword + read -Ac words + read -cn cword + reply=( $( COMP_WORDS="$words[*]" \\ + COMP_CWORD=$(( cword-1 )) \\ + PIP_AUTO_COMPLETE=1 $words[1] ) ) +} +compctl -K _pip_completion pip +"""} + + +class CompletionCommand(Command): + name = 'completion' + summary = 'A helper command to be used for command completion' + hidden = True + + def __init__(self): + super(CompletionCommand, self).__init__() + self.parser.add_option( + '--bash', '-b', + action='store_const', + const='bash', + dest='shell', + help='Emit completion code for bash') + self.parser.add_option( + '--zsh', '-z', + action='store_const', + const='zsh', + dest='shell', + help='Emit completion code for zsh') + + def run(self, options, args): + """Prints the completion code of the given shell""" + shells = COMPLETION_SCRIPTS.keys() + shell_options = ['--'+shell for shell in sorted(shells)] + if options.shell in shells: + script = COMPLETION_SCRIPTS.get(options.shell, '') + print BASE_COMPLETION % {'script': script, 'shell': options.shell} + else: + sys.stderr.write('ERROR: You must pass %s\n' % ' or '.join(shell_options)) + +CompletionCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py new file mode 100644 index 000000000..01b5df934 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py @@ -0,0 +1,109 @@ +import re +import sys +import pkg_resources +import pip +from pip.req import InstallRequirement +from pip.log import logger +from pip.basecommand import Command +from pip.util import get_installed_distributions + + +class FreezeCommand(Command): + name = 'freeze' + usage = '%prog [OPTIONS]' + summary = 'Output all currently installed packages (exact versions) to stdout' + + def __init__(self): + super(FreezeCommand, self).__init__() + self.parser.add_option( + '-r', '--requirement', + dest='requirement', + action='store', + default=None, + metavar='FILENAME', + help='Use the given requirements file as a hint about how to generate the new frozen requirements') + self.parser.add_option( + '-f', '--find-links', + dest='find_links', + action='append', + default=[], + metavar='URL', + help='URL for finding packages, which will be added to the frozen requirements file') + self.parser.add_option( + '-l', '--local', + dest='local', + action='store_true', + default=False, + help='If in a virtualenv, do not report globally-installed packages') + + def setup_logging(self): + logger.move_stdout_to_stderr() + + def run(self, options, args): + requirement = options.requirement + find_links = options.find_links or [] + local_only = options.local + ## FIXME: Obviously this should be settable: + find_tags = False + skip_match = None + + skip_regex = options.skip_requirements_regex + if skip_regex: + skip_match = re.compile(skip_regex) + + dependency_links = [] + + f = sys.stdout + + for dist in pkg_resources.working_set: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + for link in find_links: + if '#egg=' in link: + dependency_links.append(link) + for link in find_links: + f.write('-f %s\n' % link) + installations = {} + for dist in get_installed_distributions(local_only=local_only): + req = pip.FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags) + installations[req.name] = req + if requirement: + req_f = open(requirement) + for line in req_f: + if not line.strip() or line.strip().startswith('#'): + f.write(line) + continue + if skip_match and skip_match.search(line): + f.write(line) + continue + elif line.startswith('-e') or line.startswith('--editable'): + if line.startswith('-e'): + line = line[2:].strip() + else: + line = line[len('--editable'):].strip().lstrip('=') + line_req = InstallRequirement.from_editable(line, default_vcs=options.default_vcs) + elif (line.startswith('-r') or line.startswith('--requirement') + or line.startswith('-Z') or line.startswith('--always-unzip') + or line.startswith('-f') or line.startswith('-i') + or line.startswith('--extra-index-url')): + f.write(line) + continue + else: + line_req = InstallRequirement.from_line(line) + if not line_req.name: + logger.notify("Skipping line because it's not clear what it would install: %s" + % line.strip()) + logger.notify(" (add #egg=PackageName to the URL to avoid this warning)") + continue + if line_req.name not in installations: + logger.warn("Requirement file contains %s, but that package is not installed" + % line.strip()) + continue + f.write(str(installations[line_req.name])) + del installations[line_req.name] + f.write('## The following requirements were added by pip --freeze:\n') + for installation in sorted(installations.values(), key=lambda x: x.name): + f.write(str(installation)) + + +FreezeCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py new file mode 100644 index 000000000..b0b366112 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py @@ -0,0 +1,32 @@ +from pip.basecommand import Command, command_dict, load_all_commands +from pip.exceptions import InstallationError +from pip.baseparser import parser + + +class HelpCommand(Command): + name = 'help' + usage = '%prog' + summary = 'Show available commands' + + def run(self, options, args): + load_all_commands() + if args: + ## FIXME: handle errors better here + command = args[0] + if command not in command_dict: + raise InstallationError('No command with the name: %s' % command) + command = command_dict[command] + command.parser.print_help() + return + parser.print_help() + print + print 'Commands available:' + commands = list(set(command_dict.values())) + commands.sort(key=lambda x: x.name) + for command in commands: + if command.hidden: + continue + print ' %s: %s' % (command.name, command.summary) + + +HelpCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py new file mode 100644 index 000000000..861c332bf --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py @@ -0,0 +1,247 @@ +import os, sys +from pip.req import InstallRequirement, RequirementSet +from pip.req import parse_requirements +from pip.log import logger +from pip.locations import build_prefix, src_prefix +from pip.basecommand import Command +from pip.index import PackageFinder +from pip.exceptions import InstallationError + + +class InstallCommand(Command): + name = 'install' + usage = '%prog [OPTIONS] PACKAGE_NAMES...' + summary = 'Install packages' + bundle = False + + def __init__(self): + super(InstallCommand, self).__init__() + self.parser.add_option( + '-e', '--editable', + dest='editables', + action='append', + default=[], + metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE', + help='Install a package directly from a checkout. Source will be checked ' + 'out into src/PACKAGE (lower-case) and installed in-place (using ' + 'setup.py develop). You can run this on an existing directory/checkout (like ' + 'pip install -e src/mycheckout). This option may be provided multiple times. ' + 'Possible values for VCS are: svn, git, hg and bzr.') + self.parser.add_option( + '-r', '--requirement', + dest='requirements', + action='append', + default=[], + metavar='FILENAME', + help='Install all the packages listed in the given requirements file. ' + 'This option can be used multiple times.') + self.parser.add_option( + '-f', '--find-links', + dest='find_links', + action='append', + default=[], + metavar='URL', + help='URL to look for packages at') + self.parser.add_option( + '-i', '--index-url', '--pypi-url', + dest='index_url', + metavar='URL', + default='http://pypi.python.org/simple/', + help='Base URL of Python Package Index (default %default)') + self.parser.add_option( + '--extra-index-url', + dest='extra_index_urls', + metavar='URL', + action='append', + default=[], + help='Extra URLs of package indexes to use in addition to --index-url') + self.parser.add_option( + '--no-index', + dest='no_index', + action='store_true', + default=False, + help='Ignore package index (only looking at --find-links URLs instead)') + self.parser.add_option( + '-M', '--use-mirrors', + dest='use_mirrors', + action='store_true', + default=False, + help='Use the PyPI mirrors as a fallback in case the main index is down.') + self.parser.add_option( + '--mirrors', + dest='mirrors', + metavar='URL', + action='append', + default=[], + help='Specific mirror URLs to query when --use-mirrors is used') + + self.parser.add_option( + '-b', '--build', '--build-dir', '--build-directory', + dest='build_dir', + metavar='DIR', + default=None, + help='Unpack packages into DIR (default %s) and build from there' % build_prefix) + self.parser.add_option( + '-d', '--download', '--download-dir', '--download-directory', + dest='download_dir', + metavar='DIR', + default=None, + help='Download packages into DIR instead of installing them') + self.parser.add_option( + '--download-cache', + dest='download_cache', + metavar='DIR', + default=None, + help='Cache downloaded packages in DIR') + self.parser.add_option( + '--src', '--source', '--source-dir', '--source-directory', + dest='src_dir', + metavar='DIR', + default=None, + help='Check out --editable packages into DIR (default %s)' % src_prefix) + + self.parser.add_option( + '-U', '--upgrade', + dest='upgrade', + action='store_true', + help='Upgrade all packages to the newest available version') + self.parser.add_option( + '-I', '--ignore-installed', + dest='ignore_installed', + action='store_true', + help='Ignore the installed packages (reinstalling instead)') + self.parser.add_option( + '--no-deps', '--no-dependencies', + dest='ignore_dependencies', + action='store_true', + default=False, + help='Ignore package dependencies') + self.parser.add_option( + '--no-install', + dest='no_install', + action='store_true', + help="Download and unpack all packages, but don't actually install them") + self.parser.add_option( + '--no-download', + dest='no_download', + action="store_true", + help="Don't download any packages, just install the ones already downloaded " + "(completes an install run with --no-install)") + + self.parser.add_option( + '--install-option', + dest='install_options', + action='append', + help="Extra arguments to be supplied to the setup.py install " + "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). " + "Use multiple --install-option options to pass multiple options to setup.py install. " + "If you are using an option with a directory path, be sure to use absolute path.") + + self.parser.add_option( + '--global-option', + dest='global_options', + action='append', + help="Extra global options to be supplied to the setup.py" + "call before the install command") + + self.parser.add_option( + '--user', + dest='use_user_site', + action='store_true', + help='Install to user-site') + + def _build_package_finder(self, options, index_urls): + """ + Create a package finder appropriate to this install command. + This method is meant to be overridden by subclasses, not + called directly. + """ + return PackageFinder(find_links=options.find_links, + index_urls=index_urls, + use_mirrors=options.use_mirrors, + mirrors=options.mirrors) + + def run(self, options, args): + if not options.build_dir: + options.build_dir = build_prefix + if not options.src_dir: + options.src_dir = src_prefix + if options.download_dir: + options.no_install = True + options.ignore_installed = True + options.build_dir = os.path.abspath(options.build_dir) + options.src_dir = os.path.abspath(options.src_dir) + install_options = options.install_options or [] + if options.use_user_site: + install_options.append('--user') + global_options = options.global_options or [] + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index: + logger.notify('Ignoring indexes: %s' % ','.join(index_urls)) + index_urls = [] + + finder = self._build_package_finder(options, index_urls) + + requirement_set = RequirementSet( + build_dir=options.build_dir, + src_dir=options.src_dir, + download_dir=options.download_dir, + download_cache=options.download_cache, + upgrade=options.upgrade, + ignore_installed=options.ignore_installed, + ignore_dependencies=options.ignore_dependencies) + for name in args: + requirement_set.add_requirement( + InstallRequirement.from_line(name, None)) + for name in options.editables: + requirement_set.add_requirement( + InstallRequirement.from_editable(name, default_vcs=options.default_vcs)) + for filename in options.requirements: + for req in parse_requirements(filename, finder=finder, options=options): + requirement_set.add_requirement(req) + + if not requirement_set.has_requirements: + if options.find_links: + raise InstallationError('You must give at least one ' + 'requirement to %s (maybe you meant "pip install %s"?)' + % (self.name, " ".join(options.find_links))) + raise InstallationError('You must give at least one requirement ' + 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) + + if (options.use_user_site and + sys.version_info < (2, 6)): + raise InstallationError('--user is only supported in Python version 2.6 and newer') + + import setuptools + if (options.use_user_site and + requirement_set.has_editables and + not getattr(setuptools, '_distribute', False)): + + raise InstallationError('--user --editable not supported with setuptools, use distribute') + + if not options.no_download: + requirement_set.prepare_files(finder, force_root_egg_info=self.bundle, bundle=self.bundle) + else: + requirement_set.locate_files() + + if not options.no_install and not self.bundle: + requirement_set.install(install_options, global_options) + installed = ' '.join([req.name for req in + requirement_set.successfully_installed]) + if installed: + logger.notify('Successfully installed %s' % installed) + elif not self.bundle: + downloaded = ' '.join([req.name for req in + requirement_set.successfully_downloaded]) + if downloaded: + logger.notify('Successfully downloaded %s' % downloaded) + elif self.bundle: + requirement_set.create_bundle(self.bundle_filename) + logger.notify('Created bundle in %s' % self.bundle_filename) + # Clean up + if not options.no_install: + requirement_set.cleanup_files(bundle=self.bundle) + return requirement_set + + +InstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py new file mode 100644 index 000000000..73da58ac6 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py @@ -0,0 +1,116 @@ +import sys +import xmlrpclib +import textwrap +import pkg_resources +import pip.download +from pip.basecommand import Command +from pip.util import get_terminal_size +from pip.log import logger +from distutils.version import StrictVersion, LooseVersion + + +class SearchCommand(Command): + name = 'search' + usage = '%prog QUERY' + summary = 'Search PyPI' + + def __init__(self): + super(SearchCommand, self).__init__() + self.parser.add_option( + '--index', + dest='index', + metavar='URL', + default='http://pypi.python.org/pypi', + help='Base URL of Python Package Index (default %default)') + + def run(self, options, args): + if not args: + logger.warn('ERROR: Missing required argument (search query).') + return + query = ' '.join(args) + index_url = options.index + + pypi_hits = self.search(query, index_url) + hits = transform_hits(pypi_hits) + + terminal_width = None + if sys.stdout.isatty(): + terminal_width = get_terminal_size()[0] + + print_results(hits, terminal_width=terminal_width) + + def search(self, query, index_url): + pypi = xmlrpclib.ServerProxy(index_url, pip.download.xmlrpclib_transport) + hits = pypi.search({'name': query, 'summary': query}, 'or') + return hits + + +def transform_hits(hits): + """ + The list from pypi is really a list of versions. We want a list of + packages with the list of versions stored inline. This converts the + list from pypi into one we can use. + """ + packages = {} + for hit in hits: + name = hit['name'] + summary = hit['summary'] + version = hit['version'] + score = hit['_pypi_ordering'] + + if name not in packages.keys(): + packages[name] = {'name': name, 'summary': summary, 'versions': [version], 'score': score} + else: + packages[name]['versions'].append(version) + + # if this is the highest version, replace summary and score + if version == highest_version(packages[name]['versions']): + packages[name]['summary'] = summary + packages[name]['score'] = score + + # each record has a unique name now, so we will convert the dict into a list sorted by score + package_list = sorted(packages.values(), lambda x, y: cmp(y['score'], x['score'])) + return package_list + + +def print_results(hits, name_column_width=25, terminal_width=None): + installed_packages = [p.project_name for p in pkg_resources.working_set] + for hit in hits: + name = hit['name'] + summary = hit['summary'] or '' + if terminal_width is not None: + # wrap and indent summary to fit terminal + summary = textwrap.wrap(summary, terminal_width - name_column_width - 5) + summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) + line = '%s - %s' % (name.ljust(name_column_width), summary) + try: + logger.notify(line) + if name in installed_packages: + dist = pkg_resources.get_distribution(name) + logger.indent += 2 + try: + latest = highest_version(hit['versions']) + if dist.version == latest: + logger.notify('INSTALLED: %s (latest)' % dist.version) + else: + logger.notify('INSTALLED: %s' % dist.version) + logger.notify('LATEST: %s' % latest) + finally: + logger.indent -= 2 + except UnicodeEncodeError: + pass + + +def compare_versions(version1, version2): + try: + return cmp(StrictVersion(version1), StrictVersion(version2)) + # in case of abnormal version number, fall back to LooseVersion + except ValueError: + return cmp(LooseVersion(version1), LooseVersion(version2)) + + +def highest_version(versions): + return reduce((lambda v1, v2: compare_versions(v1, v2) == 1 and v1 or v2), versions) + + +SearchCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py new file mode 100644 index 000000000..7effd844e --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py @@ -0,0 +1,42 @@ +from pip.req import InstallRequirement, RequirementSet, parse_requirements +from pip.basecommand import Command +from pip.exceptions import InstallationError + +class UninstallCommand(Command): + name = 'uninstall' + usage = '%prog [OPTIONS] PACKAGE_NAMES ...' + summary = 'Uninstall packages' + + def __init__(self): + super(UninstallCommand, self).__init__() + self.parser.add_option( + '-r', '--requirement', + dest='requirements', + action='append', + default=[], + metavar='FILENAME', + help='Uninstall all the packages listed in the given requirements file. ' + 'This option can be used multiple times.') + self.parser.add_option( + '-y', '--yes', + dest='yes', + action='store_true', + help="Don't ask for confirmation of uninstall deletions.") + + def run(self, options, args): + requirement_set = RequirementSet( + build_dir=None, + src_dir=None, + download_dir=None) + for name in args: + requirement_set.add_requirement( + InstallRequirement.from_line(name)) + for filename in options.requirements: + for req in parse_requirements(filename, options=options): + requirement_set.add_requirement(req) + if not requirement_set.has_requirements: + raise InstallationError('You must give at least one requirement ' + 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) + requirement_set.uninstall(auto_confirm=options.yes) + +UninstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py new file mode 100644 index 000000000..f83e18205 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py @@ -0,0 +1,9 @@ +from pip.commands.zip import ZipCommand + + +class UnzipCommand(ZipCommand): + name = 'unzip' + summary = 'Unzip individual packages' + + +UnzipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py new file mode 100644 index 000000000..346fc0519 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py @@ -0,0 +1,346 @@ +import sys +import re +import fnmatch +import os +import shutil +import zipfile +from pip.util import display_path, backup_dir +from pip.log import logger +from pip.exceptions import InstallationError +from pip.basecommand import Command + + +class ZipCommand(Command): + name = 'zip' + usage = '%prog [OPTIONS] PACKAGE_NAMES...' + summary = 'Zip individual packages' + + def __init__(self): + super(ZipCommand, self).__init__() + if self.name == 'zip': + self.parser.add_option( + '--unzip', + action='store_true', + dest='unzip', + help='Unzip (rather than zip) a package') + else: + self.parser.add_option( + '--zip', + action='store_false', + dest='unzip', + default=True, + help='Zip (rather than unzip) a package') + self.parser.add_option( + '--no-pyc', + action='store_true', + dest='no_pyc', + help='Do not include .pyc files in zip files (useful on Google App Engine)') + self.parser.add_option( + '-l', '--list', + action='store_true', + dest='list', + help='List the packages available, and their zip status') + self.parser.add_option( + '--sort-files', + action='store_true', + dest='sort_files', + help='With --list, sort packages according to how many files they contain') + self.parser.add_option( + '--path', + action='append', + dest='paths', + help='Restrict operations to the given paths (may include wildcards)') + self.parser.add_option( + '-n', '--simulate', + action='store_true', + help='Do not actually perform the zip/unzip operation') + + def paths(self): + """All the entries of sys.path, possibly restricted by --path""" + if not self.select_paths: + return sys.path + result = [] + match_any = set() + for path in sys.path: + path = os.path.normcase(os.path.abspath(path)) + for match in self.select_paths: + match = os.path.normcase(os.path.abspath(match)) + if '*' in match: + if re.search(fnmatch.translate(match+'*'), path): + result.append(path) + match_any.add(match) + break + else: + if path.startswith(match): + result.append(path) + match_any.add(match) + break + else: + logger.debug("Skipping path %s because it doesn't match %s" + % (path, ', '.join(self.select_paths))) + for match in self.select_paths: + if match not in match_any and '*' not in match: + result.append(match) + logger.debug("Adding path %s because it doesn't match anything already on sys.path" + % match) + return result + + def run(self, options, args): + self.select_paths = options.paths + self.simulate = options.simulate + if options.list: + return self.list(options, args) + if not args: + raise InstallationError( + 'You must give at least one package to zip or unzip') + packages = [] + for arg in args: + module_name, filename = self.find_package(arg) + if options.unzip and os.path.isdir(filename): + raise InstallationError( + 'The module %s (in %s) is not a zip file; cannot be unzipped' + % (module_name, filename)) + elif not options.unzip and not os.path.isdir(filename): + raise InstallationError( + 'The module %s (in %s) is not a directory; cannot be zipped' + % (module_name, filename)) + packages.append((module_name, filename)) + last_status = None + for module_name, filename in packages: + if options.unzip: + last_status = self.unzip_package(module_name, filename) + else: + last_status = self.zip_package(module_name, filename, options.no_pyc) + return last_status + + def unzip_package(self, module_name, filename): + zip_filename = os.path.dirname(filename) + if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename): + raise InstallationError( + 'Module %s (in %s) isn\'t located in a zip file in %s' + % (module_name, filename, zip_filename)) + package_path = os.path.dirname(zip_filename) + if not package_path in self.paths(): + logger.warn( + 'Unpacking %s into %s, but %s is not on sys.path' + % (display_path(zip_filename), display_path(package_path), + display_path(package_path))) + logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename))) + if self.simulate: + logger.notify('Skipping remaining operations because of --simulate') + return + logger.indent += 2 + try: + ## FIXME: this should be undoable: + zip = zipfile.ZipFile(zip_filename) + to_save = [] + for name in zip.namelist(): + if name.startswith(module_name + os.path.sep): + content = zip.read(name) + dest = os.path.join(package_path, name) + if not os.path.exists(os.path.dirname(dest)): + os.makedirs(os.path.dirname(dest)) + if not content and dest.endswith(os.path.sep): + if not os.path.exists(dest): + os.makedirs(dest) + else: + f = open(dest, 'wb') + f.write(content) + f.close() + else: + to_save.append((name, zip.read(name))) + zip.close() + if not to_save: + logger.info('Removing now-empty zip file %s' % display_path(zip_filename)) + os.unlink(zip_filename) + self.remove_filename_from_pth(zip_filename) + else: + logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename))) + zip = zipfile.ZipFile(zip_filename, 'w') + for name, content in to_save: + zip.writestr(name, content) + zip.close() + finally: + logger.indent -= 2 + + def zip_package(self, module_name, filename, no_pyc): + orig_filename = filename + logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) + logger.indent += 2 + if filename.endswith('.egg'): + dest_filename = filename + else: + dest_filename = filename + '.zip' + try: + ## FIXME: I think this needs to be undoable: + if filename == dest_filename: + filename = backup_dir(orig_filename) + logger.notify('Moving %s aside to %s' % (orig_filename, filename)) + if not self.simulate: + shutil.move(orig_filename, filename) + try: + logger.info('Creating zip file in %s' % display_path(dest_filename)) + if not self.simulate: + zip = zipfile.ZipFile(dest_filename, 'w') + zip.writestr(module_name + '/', '') + for dirpath, dirnames, filenames in os.walk(filename): + if no_pyc: + filenames = [f for f in filenames + if not f.lower().endswith('.pyc')] + for fns, is_dir in [(dirnames, True), (filenames, False)]: + for fn in fns: + full = os.path.join(dirpath, fn) + dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn) + if is_dir: + zip.writestr(dest+'/', '') + else: + zip.write(full, dest) + zip.close() + logger.info('Removing old directory %s' % display_path(filename)) + if not self.simulate: + shutil.rmtree(filename) + except: + ## FIXME: need to do an undo here + raise + ## FIXME: should also be undone: + self.add_filename_to_pth(dest_filename) + finally: + logger.indent -= 2 + + def remove_filename_from_pth(self, filename): + for pth in self.pth_files(): + f = open(pth, 'r') + lines = f.readlines() + f.close() + new_lines = [ + l for l in lines if l.strip() != filename] + if lines != new_lines: + logger.info('Removing reference to %s from .pth file %s' + % (display_path(filename), display_path(pth))) + if not filter(None, new_lines): + logger.info('%s file would be empty: deleting' % display_path(pth)) + if not self.simulate: + os.unlink(pth) + else: + if not self.simulate: + f = open(pth, 'wb') + f.writelines(new_lines) + f.close() + return + logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename)) + + def add_filename_to_pth(self, filename): + path = os.path.dirname(filename) + dest = os.path.join(path, filename + '.pth') + if path not in self.paths(): + logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest)) + if not self.simulate: + if os.path.exists(dest): + f = open(dest) + lines = f.readlines() + f.close() + if lines and not lines[-1].endswith('\n'): + lines[-1] += '\n' + lines.append(filename+'\n') + else: + lines = [filename + '\n'] + f = open(dest, 'wb') + f.writelines(lines) + f.close() + + def pth_files(self): + for path in self.paths(): + if not os.path.exists(path) or not os.path.isdir(path): + continue + for filename in os.listdir(path): + if filename.endswith('.pth'): + yield os.path.join(path, filename) + + def find_package(self, package): + for path in self.paths(): + full = os.path.join(path, package) + if os.path.exists(full): + return package, full + if not os.path.isdir(path) and zipfile.is_zipfile(path): + zip = zipfile.ZipFile(path, 'r') + try: + zip.read(os.path.join(package, '__init__.py')) + except KeyError: + pass + else: + zip.close() + return package, full + zip.close() + ## FIXME: need special error for package.py case: + raise InstallationError( + 'No package with the name %s found' % package) + + def list(self, options, args): + if args: + raise InstallationError( + 'You cannot give an argument with --list') + for path in sorted(self.paths()): + if not os.path.exists(path): + continue + basename = os.path.basename(path.rstrip(os.path.sep)) + if os.path.isfile(path) and zipfile.is_zipfile(path): + if os.path.dirname(path) not in self.paths(): + logger.notify('Zipped egg: %s' % display_path(path)) + continue + if (basename != 'site-packages' and basename != 'dist-packages' + and not path.replace('\\', '/').endswith('lib/python')): + continue + logger.notify('In %s:' % display_path(path)) + logger.indent += 2 + zipped = [] + unzipped = [] + try: + for filename in sorted(os.listdir(path)): + ext = os.path.splitext(filename)[1].lower() + if ext in ('.pth', '.egg-info', '.egg-link'): + continue + if ext == '.py': + logger.info('Not displaying %s: not a package' % display_path(filename)) + continue + full = os.path.join(path, filename) + if os.path.isdir(full): + unzipped.append((filename, self.count_package(full))) + elif zipfile.is_zipfile(full): + zipped.append(filename) + else: + logger.info('Unknown file: %s' % display_path(filename)) + if zipped: + logger.notify('Zipped packages:') + logger.indent += 2 + try: + for filename in zipped: + logger.notify(filename) + finally: + logger.indent -= 2 + else: + logger.notify('No zipped packages.') + if unzipped: + if options.sort_files: + unzipped.sort(key=lambda x: -x[1]) + logger.notify('Unzipped packages:') + logger.indent += 2 + try: + for filename, count in unzipped: + logger.notify('%s (%i files)' % (filename, count)) + finally: + logger.indent -= 2 + else: + logger.notify('No unzipped packages.') + finally: + logger.indent -= 2 + + def count_package(self, path): + total = 0 + for dirpath, dirnames, filenames in os.walk(path): + filenames = [f for f in filenames + if not f.lower().endswith('.pyc')] + total += len(filenames) + return total + + +ZipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py new file mode 100644 index 000000000..f1b63936b --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py @@ -0,0 +1,470 @@ +import xmlrpclib +import re +import getpass +import urllib +import urllib2 +import urlparse +import os +import mimetypes +import shutil +import tempfile +from pip.backwardcompat import md5, copytree +from pip.exceptions import InstallationError +from pip.util import (splitext, + format_size, display_path, backup_dir, ask, + unpack_file, create_download_cache_folder, cache_download) +from pip.vcs import vcs +from pip.log import logger + + +__all__ = ['xmlrpclib_transport', 'get_file_content', 'urlopen', + 'is_url', 'url_to_path', 'path_to_url', 'path_to_url2', + 'geturl', 'is_archive_file', 'unpack_vcs_link', + 'unpack_file_url', 'is_vcs_url', 'is_file_url', 'unpack_http_url'] + + +xmlrpclib_transport = xmlrpclib.Transport() + + +def get_file_content(url, comes_from=None): + """Gets the content of a file; it may be a filename, file: URL, or + http: URL. Returns (location, content)""" + match = _scheme_re.search(url) + if match: + scheme = match.group(1).lower() + if (scheme == 'file' and comes_from + and comes_from.startswith('http')): + raise InstallationError( + 'Requirements file %s references URL %s, which is local' + % (comes_from, url)) + if scheme == 'file': + path = url.split(':', 1)[1] + path = path.replace('\\', '/') + match = _url_slash_drive_re.match(path) + if match: + path = match.group(1) + ':' + path.split('|', 1)[1] + path = urllib.unquote(path) + if path.startswith('/'): + path = '/' + path.lstrip('/') + url = path + else: + ## FIXME: catch some errors + resp = urlopen(url) + return geturl(resp), resp.read() + try: + f = open(url) + content = f.read() + except IOError, e: + raise InstallationError('Could not open requirements file: %s' % str(e)) + else: + f.close() + return url, content + + +_scheme_re = re.compile(r'^(http|https|file):', re.I) +_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) + +class URLOpener(object): + """ + pip's own URL helper that adds HTTP auth and proxy support + """ + def __init__(self): + self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + + def __call__(self, url): + """ + If the given url contains auth info or if a normal request gets a 401 + response, an attempt is made to fetch the resource using basic HTTP + auth. + + """ + url, username, password = self.extract_credentials(url) + if username is None: + try: + response = urllib2.urlopen(self.get_request(url)) + except urllib2.HTTPError, e: + if e.code != 401: + raise + response = self.get_response(url) + else: + response = self.get_response(url, username, password) + return response + + def get_request(self, url): + """ + Wraps the URL to retrieve to protects against "creative" + interpretation of the RFC: http://bugs.python.org/issue8732 + """ + if isinstance(url, basestring): + url = urllib2.Request(url, headers={'Accept-encoding': 'identity'}) + return url + + def get_response(self, url, username=None, password=None): + """ + does the dirty work of actually getting the rsponse object using urllib2 + and its HTTP auth builtins. + """ + scheme, netloc, path, query, frag = urlparse.urlsplit(url) + pass_url = urlparse.urlunsplit(('_none_', netloc, path, query, frag)).replace('_none_://', '', 1) + req = self.get_request(url) + + stored_username, stored_password = self.passman.find_user_password(None, netloc) + # see if we have a password stored + if stored_username is None: + if username is None and self.prompting: + username = urllib.quote(raw_input('User for %s: ' % netloc)) + password = urllib.quote(getpass.getpass('Password: ')) + if username and password: + self.passman.add_password(None, netloc, username, password) + stored_username, stored_password = self.passman.find_user_password(None, netloc) + authhandler = urllib2.HTTPBasicAuthHandler(self.passman) + opener = urllib2.build_opener(authhandler) + # FIXME: should catch a 401 and offer to let the user reenter credentials + return opener.open(req) + + def setup(self, proxystr='', prompting=True): + """ + Sets the proxy handler given the option passed on the command + line. If an empty string is passed it looks at the HTTP_PROXY + environment variable. + """ + self.prompting = prompting + proxy = self.get_proxy(proxystr) + if proxy: + proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy}) + opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler) + urllib2.install_opener(opener) + + def parse_credentials(self, netloc): + if "@" in netloc: + userinfo = netloc.rsplit("@", 1)[0] + if ":" in userinfo: + return userinfo.split(":", 1) + return userinfo, None + return None, None + + def extract_credentials(self, url): + """ + Extracts user/password from a url. + + Returns a tuple: + (url-without-auth, username, password) + """ + if isinstance(url, urllib2.Request): + result = urlparse.urlsplit(url.get_full_url()) + else: + result = urlparse.urlsplit(url) + scheme, netloc, path, query, frag = result + + username, password = self.parse_credentials(netloc) + if username is None: + return url, None, None + elif password is None and self.prompting: + # remove the auth credentials from the url part + netloc = netloc.replace('%s@' % username, '', 1) + # prompt for the password + prompt = 'Password for %s@%s: ' % (username, netloc) + password = urllib.quote(getpass.getpass(prompt)) + else: + # remove the auth credentials from the url part + netloc = netloc.replace('%s:%s@' % (username, password), '', 1) + + target_url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + return target_url, username, password + + def get_proxy(self, proxystr=''): + """ + Get the proxy given the option passed on the command line. + If an empty string is passed it looks at the HTTP_PROXY + environment variable. + """ + if not proxystr: + proxystr = os.environ.get('HTTP_PROXY', '') + if proxystr: + if '@' in proxystr: + user_password, server_port = proxystr.split('@', 1) + if ':' in user_password: + user, password = user_password.split(':', 1) + else: + user = user_password + prompt = 'Password for %s@%s: ' % (user, server_port) + password = urllib.quote(getpass.getpass(prompt)) + return '%s:%s@%s' % (user, password, server_port) + else: + return proxystr + else: + return None + +urlopen = URLOpener() + + +def is_url(name): + """Returns true if the name looks like a URL""" + if ':' not in name: + return False + scheme = name.split(':', 1)[0].lower() + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + +def url_to_path(url): + """ + Convert a file: URL to a path. + """ + assert url.startswith('file:'), ( + "You can only turn file: urls into filenames (not %r)" % url) + path = url[len('file:'):].lstrip('/') + path = urllib.unquote(path) + if _url_drive_re.match(path): + path = path[0] + ':' + path[2:] + else: + path = '/' + path + return path + + +_drive_re = re.compile('^([a-z]):', re.I) +_url_drive_re = re.compile('^([a-z])[:|]', re.I) + + +def path_to_url(path): + """ + Convert a path to a file: URL. The path will be made absolute. + """ + path = os.path.normcase(os.path.abspath(path)) + if _drive_re.match(path): + path = path[0] + '|' + path[2:] + url = urllib.quote(path) + url = url.replace(os.path.sep, '/') + url = url.lstrip('/') + return 'file:///' + url + + +def path_to_url2(path): + """ + Convert a path to a file: URL. The path will be made absolute and have + quoted path parts. + """ + path = os.path.normpath(os.path.abspath(path)) + drive, path = os.path.splitdrive(path) + filepath = path.split(os.path.sep) + url = '/'.join([urllib.quote(part) for part in filepath]) + if not drive: + url = url.lstrip('/') + return 'file:///' + drive + url + + +def geturl(urllib2_resp): + """ + Use instead of urllib.addinfourl.geturl(), which appears to have + some issues with dropping the double slash for certain schemes + (e.g. file://). This implementation is probably over-eager, as it + always restores '://' if it is missing, and it appears some url + schemata aren't always followed by '//' after the colon, but as + far as I know pip doesn't need any of those. + The URI RFC can be found at: http://tools.ietf.org/html/rfc1630 + + This function assumes that + scheme:/foo/bar + is the same as + scheme:///foo/bar + """ + url = urllib2_resp.geturl() + scheme, rest = url.split(':', 1) + if rest.startswith('//'): + return url + else: + # FIXME: write a good test to cover it + return '%s://%s' % (scheme, rest) + + +def is_archive_file(name): + """Return True if `name` is a considered as an archive file.""" + archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle') + ext = splitext(name)[1].lower() + if ext in archives: + return True + return False + + +def unpack_vcs_link(link, location, only_download=False): + vcs_backend = _get_used_vcs_backend(link) + if only_download: + vcs_backend.export(location) + else: + vcs_backend.unpack(location) + + +def unpack_file_url(link, location): + source = url_to_path(link.url) + content_type = mimetypes.guess_type(source)[0] + if os.path.isdir(source): + # delete the location since shutil will create it again :( + if os.path.isdir(location): + shutil.rmtree(location) + copytree(source, location) + else: + unpack_file(source, location, content_type, link) + + +def _get_used_vcs_backend(link): + for backend in vcs.backends: + if link.scheme in backend.schemes: + vcs_backend = backend(link.url) + return vcs_backend + + +def is_vcs_url(link): + return bool(_get_used_vcs_backend(link)) + + +def is_file_url(link): + return link.url.lower().startswith('file:') + + +def _check_md5(download_hash, link): + download_hash = download_hash.hexdigest() + if download_hash != link.md5_hash: + logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!" + % (link, download_hash, link.md5_hash)) + raise InstallationError('Bad MD5 hash for package %s' % link) + + +def _get_md5_from_file(target_file, link): + download_hash = md5() + fp = open(target_file, 'rb') + while 1: + chunk = fp.read(4096) + if not chunk: + break + download_hash.update(chunk) + fp.close() + return download_hash + + +def _download_url(resp, link, temp_location): + fp = open(temp_location, 'wb') + download_hash = None + if link.md5_hash: + download_hash = md5() + try: + total_length = int(resp.info()['content-length']) + except (ValueError, KeyError): + total_length = 0 + downloaded = 0 + show_progress = total_length > 40*1000 or not total_length + show_url = link.show_url + try: + if show_progress: + ## FIXME: the URL can get really long in this message: + if total_length: + logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) + else: + logger.start_progress('Downloading %s (unknown size): ' % show_url) + else: + logger.notify('Downloading %s' % show_url) + logger.debug('Downloading from URL %s' % link) + + while 1: + chunk = resp.read(4096) + if not chunk: + break + downloaded += len(chunk) + if show_progress: + if not total_length: + logger.show_progress('%s' % format_size(downloaded)) + else: + logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded))) + if link.md5_hash: + download_hash.update(chunk) + fp.write(chunk) + fp.close() + finally: + if show_progress: + logger.end_progress('%s downloaded' % format_size(downloaded)) + return download_hash + + +def _copy_file(filename, location, content_type, link): + copy = True + download_location = os.path.join(location, link.filename) + if os.path.exists(download_location): + response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' + % display_path(download_location), ('i', 'w', 'b')) + if response == 'i': + copy = False + elif response == 'w': + logger.warn('Deleting %s' % display_path(download_location)) + os.remove(download_location) + elif response == 'b': + dest_file = backup_dir(download_location) + logger.warn('Backing up %s to %s' + % (display_path(download_location), display_path(dest_file))) + shutil.move(download_location, dest_file) + if copy: + shutil.copy(filename, download_location) + logger.indent -= 2 + logger.notify('Saved %s' % display_path(download_location)) + + +def unpack_http_url(link, location, download_cache, only_download): + temp_dir = tempfile.mkdtemp('-unpack', 'pip-') + target_url = link.url.split('#', 1)[0] + target_file = None + download_hash = None + if download_cache: + target_file = os.path.join(download_cache, + urllib.quote(target_url, '')) + if not os.path.isdir(download_cache): + create_download_cache_folder(download_cache) + if (target_file + and os.path.exists(target_file) + and os.path.exists(target_file+'.content-type')): + fp = open(target_file+'.content-type') + content_type = fp.read().strip() + fp.close() + if link.md5_hash: + download_hash = _get_md5_from_file(target_file, link) + temp_location = target_file + logger.notify('Using download cache from %s' % target_file) + else: + resp = _get_response_from_url(target_url, link) + content_type = resp.info()['content-type'] + filename = link.filename + ext = splitext(filename)[1] + if not ext: + ext = mimetypes.guess_extension(content_type) + if ext: + filename += ext + if not ext and link.url != geturl(resp): + ext = os.path.splitext(geturl(resp))[1] + if ext: + filename += ext + temp_location = os.path.join(temp_dir, filename) + download_hash = _download_url(resp, link, temp_location) + if link.md5_hash: + _check_md5(download_hash, link) + if only_download: + _copy_file(temp_location, location, content_type, link) + else: + unpack_file(temp_location, location, content_type, link) + if target_file and target_file != temp_location: + cache_download(target_file, temp_location, content_type) + if target_file is None: + os.unlink(temp_location) + os.rmdir(temp_dir) + + +def _get_response_from_url(target_url, link): + try: + resp = urlopen(target_url) + except urllib2.HTTPError, e: + logger.fatal("HTTP error %s while getting %s" % (e.code, link)) + raise + except IOError, e: + # Typically an FTP error + logger.fatal("Error %s while getting %s" % (e, link)) + raise + return resp + +class Urllib2HeadRequest(urllib2.Request): + def get_method(self): + return "HEAD" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py new file mode 100644 index 000000000..1ad1a616d --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py @@ -0,0 +1,17 @@ +"""Exceptions used throughout package""" + + +class InstallationError(Exception): + """General exception during installation""" + + +class UninstallationError(Exception): + """General exception during uninstallation""" + + +class DistributionNotFound(InstallationError): + """Raised when a distribution cannot be found to satisfy a requirement""" + + +class BadCommand(Exception): + """Raised when virtualenv or a command is not found""" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py new file mode 100644 index 000000000..e42d8c868 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py @@ -0,0 +1,686 @@ +"""Routines related to PyPI, indexes""" + +import sys +import os +import re +import mimetypes +import threading +import posixpath +import pkg_resources +import urllib +import urllib2 +import urlparse +import httplib +import random +import socket +import string +from Queue import Queue +from Queue import Empty as QueueEmpty +from pip.log import logger +from pip.util import Inf +from pip.util import normalize_name, splitext +from pip.exceptions import DistributionNotFound +from pip.backwardcompat import WindowsError, product +from pip.download import urlopen, path_to_url2, url_to_path, geturl, Urllib2HeadRequest + +__all__ = ['PackageFinder'] + + +DEFAULT_MIRROR_URL = "last.pypi.python.org" + + +class PackageFinder(object): + """This finds packages. + + This is meant to match easy_install's technique for looking for + packages, by reading pages and looking for appropriate links + """ + + def __init__(self, find_links, index_urls, + use_mirrors=False, mirrors=None, main_mirror_url=None): + self.find_links = find_links + self.index_urls = index_urls + self.dependency_links = [] + self.cache = PageCache() + # These are boring links that have already been logged somehow: + self.logged_links = set() + if use_mirrors: + self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) + logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) + else: + self.mirror_urls = [] + + def add_dependency_links(self, links): + ## FIXME: this shouldn't be global list this, it should only + ## apply to requirements of the package that specifies the + ## dependency_links value + ## FIXME: also, we should track comes_from (i.e., use Link) + self.dependency_links.extend(links) + + @staticmethod + def _sort_locations(locations): + """ + Sort locations into "files" (archives) and "urls", and return + a pair of lists (files,urls) + """ + files = [] + urls = [] + + # puts the url for the given file path into the appropriate + # list + def sort_path(path): + url = path_to_url2(path) + if mimetypes.guess_type(url, strict=False)[0] == 'text/html': + urls.append(url) + else: + files.append(url) + + for url in locations: + if url.startswith('file:'): + path = url_to_path(url) + if os.path.isdir(path): + path = os.path.realpath(path) + for item in os.listdir(path): + sort_path(os.path.join(path, item)) + elif os.path.isfile(path): + sort_path(path) + else: + urls.append(url) + return files, urls + + def find_requirement(self, req, upgrade): + url_name = req.url_name + # Only check main index if index URL is given: + main_index_url = None + if self.index_urls: + # Check that we have the url_name correctly spelled: + main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) + # This will also cache the page, so it's okay that we get it again later: + page = self._get_page(main_index_url, req) + if page is None: + url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name + + # Combine index URLs with mirror URLs here to allow + # adding more index URLs from requirements files + all_index_urls = self.index_urls + self.mirror_urls + + def mkurl_pypi_url(url): + loc = posixpath.join(url, url_name) + # For maximum compatibility with easy_install, ensure the path + # ends in a trailing slash. Although this isn't in the spec + # (and PyPI can handle it without the slash) some other index + # implementations might break if they relied on easy_install's behavior. + if not loc.endswith('/'): + loc = loc + '/' + return loc + if url_name is not None: + locations = [ + mkurl_pypi_url(url) + for url in all_index_urls] + self.find_links + else: + locations = list(self.find_links) + locations.extend(self.dependency_links) + for version in req.absolute_versions: + if url_name is not None and main_index_url is not None: + locations = [ + posixpath.join(main_index_url.url, version)] + locations + + file_locations, url_locations = self._sort_locations(locations) + + locations = [Link(url) for url in url_locations] + logger.debug('URLs to search for versions for %s:' % req) + for location in locations: + logger.debug('* %s' % location) + found_versions = [] + found_versions.extend( + self._package_versions( + [Link(url, '-f') for url in self.find_links], req.name.lower())) + page_versions = [] + for page in self._get_pages(locations, req): + logger.debug('Analyzing links from page %s' % page.url) + logger.indent += 2 + try: + page_versions.extend(self._package_versions(page.links, req.name.lower())) + finally: + logger.indent -= 2 + dependency_versions = list(self._package_versions( + [Link(url) for url in self.dependency_links], req.name.lower())) + if dependency_versions: + logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) + file_versions = list(self._package_versions( + [Link(url) for url in file_locations], req.name.lower())) + if not found_versions and not page_versions and not dependency_versions and not file_versions: + logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) + raise DistributionNotFound('No distributions at all found for %s' % req) + if req.satisfied_by is not None: + found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) + if file_versions: + file_versions.sort(reverse=True) + logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) + found_versions = file_versions + found_versions + all_versions = found_versions + page_versions + dependency_versions + applicable_versions = [] + for (parsed_version, link, version) in all_versions: + if version not in req.req: + logger.info("Ignoring link %s, version %s doesn't match %s" + % (link, version, ','.join([''.join(s) for s in req.req.specs]))) + continue + applicable_versions.append((link, version)) + applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) + existing_applicable = bool([link for link, version in applicable_versions if link is Inf]) + if not upgrade and existing_applicable: + if applicable_versions[0][1] is Inf: + logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' + % req.satisfied_by.version) + else: + logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' + % (req.satisfied_by.version, applicable_versions[0][1])) + return None + if not applicable_versions: + logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' + % (req, ', '.join([version for parsed_version, link, version in found_versions]))) + raise DistributionNotFound('No distributions matching the version for %s' % req) + if applicable_versions[0][0] is Inf: + # We have an existing version, and its the best version + logger.info('Installed version (%s) is most up-to-date (past versions: %s)' + % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none')) + return None + if len(applicable_versions) > 1: + logger.info('Using version %s (newest of versions: %s)' % + (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions]))) + return applicable_versions[0][0] + + def _find_url_name(self, index_url, url_name, req): + """Finds the true URL name of a package, when the given name isn't quite correct. + This is usually used to implement case-insensitivity.""" + if not index_url.url.endswith('/'): + # Vaguely part of the PyPI API... weird but true. + ## FIXME: bad to modify this? + index_url.url += '/' + page = self._get_page(index_url, req) + if page is None: + logger.fatal('Cannot fetch index base URL %s' % index_url) + return + norm_name = normalize_name(req.url_name) + for link in page.links: + base = posixpath.basename(link.path.rstrip('/')) + if norm_name == normalize_name(base): + logger.notify('Real name of requirement %s is %s' % (url_name, base)) + return base + return None + + def _get_pages(self, locations, req): + """Yields (page, page_url) from the given locations, skipping + locations that have errors, and adding download/homepage links""" + pending_queue = Queue() + for location in locations: + pending_queue.put(location) + done = [] + seen = set() + threads = [] + for i in range(min(10, len(locations))): + t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen)) + t.setDaemon(True) + threads.append(t) + t.start() + for t in threads: + t.join() + return done + + _log_lock = threading.Lock() + + def _get_queued_page(self, req, pending_queue, done, seen): + while 1: + try: + location = pending_queue.get(False) + except QueueEmpty: + return + if location in seen: + continue + seen.add(location) + page = self._get_page(location, req) + if page is None: + continue + done.append(page) + for link in page.rel_links(): + pending_queue.put(link) + + _egg_fragment_re = re.compile(r'#egg=([^&]*)') + _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I) + _py_version_re = re.compile(r'-py([123]\.[0-9])$') + + def _sort_links(self, links): + "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates" + eggs, no_eggs = [], [] + seen = set() + for link in links: + if link not in seen: + seen.add(link) + if link.egg_fragment: + eggs.append(link) + else: + no_eggs.append(link) + return no_eggs + eggs + + def _package_versions(self, links, search_name): + for link in self._sort_links(links): + for v in self._link_package_versions(link, search_name): + yield v + + def _link_package_versions(self, link, search_name): + """ + Return an iterable of triples (pkg_resources_version_key, + link, python_version) that can be extracted from the given + link. + + Meant to be overridden by subclasses, not called by clients. + """ + if link.egg_fragment: + egg_info = link.egg_fragment + else: + egg_info, ext = link.splitext() + if not ext: + if link not in self.logged_links: + logger.debug('Skipping link %s; not a file' % link) + self.logged_links.add(link) + return [] + if egg_info.endswith('.tar'): + # Special double-extension case: + egg_info = egg_info[:-4] + ext = '.tar' + ext + if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'): + if link not in self.logged_links: + logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext)) + self.logged_links.add(link) + return [] + version = self._egg_info_matches(egg_info, search_name, link) + if version is None: + logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name)) + return [] + match = self._py_version_re.search(version) + if match: + version = version[:match.start()] + py_version = match.group(1) + if py_version != sys.version[:3]: + logger.debug('Skipping %s because Python version is incorrect' % link) + return [] + logger.debug('Found link %s, version: %s' % (link, version)) + return [(pkg_resources.parse_version(version), + link, + version)] + + def _egg_info_matches(self, egg_info, search_name, link): + match = self._egg_info_re.search(egg_info) + if not match: + logger.debug('Could not parse version from link: %s' % link) + return None + name = match.group(0).lower() + # To match the "safe" name that pkg_resources creates: + name = name.replace('_', '-') + if name.startswith(search_name.lower()): + return match.group(0)[len(search_name):].lstrip('-') + else: + return None + + def _get_page(self, link, req): + return HTMLPage.get_page(link, req, cache=self.cache) + + def _get_mirror_urls(self, mirrors=None, main_mirror_url=None): + """Retrieves a list of URLs from the main mirror DNS entry + unless a list of mirror URLs are passed. + """ + if not mirrors: + mirrors = get_mirrors(main_mirror_url) + # Should this be made "less random"? E.g. netselect like? + random.shuffle(mirrors) + + mirror_urls = set() + for mirror_url in mirrors: + # Make sure we have a valid URL + if not ("http://" or "https://" or "file://") in mirror_url: + mirror_url = "http://%s" % mirror_url + if not mirror_url.endswith("/simple"): + mirror_url = "%s/simple/" % mirror_url + mirror_urls.add(mirror_url) + + return list(mirror_urls) + + +class PageCache(object): + """Cache of HTML pages""" + + failure_limit = 3 + + def __init__(self): + self._failures = {} + self._pages = {} + self._archives = {} + + def too_many_failures(self, url): + return self._failures.get(url, 0) >= self.failure_limit + + def get_page(self, url): + return self._pages.get(url) + + def is_archive(self, url): + return self._archives.get(url, False) + + def set_is_archive(self, url, value=True): + self._archives[url] = value + + def add_page_failure(self, url, level): + self._failures[url] = self._failures.get(url, 0)+level + + def add_page(self, urls, page): + for url in urls: + self._pages[url] = page + + +class HTMLPage(object): + """Represents one page, along with its URL""" + + ## FIXME: these regexes are horrible hacks: + _homepage_re = re.compile(r'\s*home\s*page', re.I) + _download_re = re.compile(r'\s*download\s+url', re.I) + ## These aren't so aweful: + _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I) + _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S) + _base_re = re.compile(r"""]+)""", re.I) + + def __init__(self, content, url, headers=None): + self.content = content + self.url = url + self.headers = headers + + def __str__(self): + return self.url + + @classmethod + def get_page(cls, link, req, cache=None, skip_archives=True): + url = link.url + url = url.split('#', 1)[0] + if cache.too_many_failures(url): + return None + + # Check for VCS schemes that do not support lookup as web pages. + from pip.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals()) + return None + + if cache is not None: + inst = cache.get_page(url) + if inst is not None: + return inst + try: + if skip_archives: + if cache is not None: + if cache.is_archive(url): + return None + filename = link.filename + for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']: + if filename.endswith(bad_ext): + content_type = cls._get_content_type(url) + if content_type.lower().startswith('text/html'): + break + else: + logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type)) + if cache is not None: + cache.set_is_archive(url) + return None + logger.debug('Getting page %s' % url) + + # Tack index.html onto file:// URLs that point to directories + (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) + if scheme == 'file' and os.path.isdir(urllib.url2pathname(path)): + # add trailing slash if not present so urljoin doesn't trim final segment + if not url.endswith('/'): + url += '/' + url = urlparse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s' % url) + + resp = urlopen(url) + + real_url = geturl(resp) + headers = resp.info() + inst = cls(resp.read(), real_url, headers) + except (urllib2.HTTPError, urllib2.URLError, socket.timeout, socket.error, OSError, WindowsError), e: + desc = str(e) + if isinstance(e, socket.timeout): + log_meth = logger.info + level =1 + desc = 'timed out' + elif isinstance(e, urllib2.URLError): + log_meth = logger.info + if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout): + desc = 'timed out' + level = 1 + else: + level = 2 + elif isinstance(e, urllib2.HTTPError) and e.code == 404: + ## FIXME: notify? + log_meth = logger.info + level = 2 + else: + log_meth = logger.info + level = 1 + log_meth('Could not fetch URL %s: %s' % (link, desc)) + log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req)) + if cache is not None: + cache.add_page_failure(url, level) + return None + if cache is not None: + cache.add_page([url, real_url], inst) + return inst + + @staticmethod + def _get_content_type(url): + """Get the Content-Type of the given url, using a HEAD request""" + scheme, netloc, path, query, fragment = urlparse.urlsplit(url) + if not scheme in ('http', 'https', 'ftp', 'ftps'): + ## FIXME: some warning or something? + ## assertion error? + return '' + req = Urllib2HeadRequest(url, headers={'Host': netloc}) + resp = urlopen(req) + try: + if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'): + ## FIXME: doesn't handle redirects + return '' + return resp.info().get('content-type', '') + finally: + resp.close() + + @property + def base_url(self): + if not hasattr(self, "_base_url"): + match = self._base_re.search(self.content) + if match: + self._base_url = match.group(1) + else: + self._base_url = self.url + return self._base_url + + @property + def links(self): + """Yields all links in the page""" + for match in self._href_re.finditer(self.content): + url = match.group(1) or match.group(2) or match.group(3) + url = self.clean_link(urlparse.urljoin(self.base_url, url)) + yield Link(url, self) + + def rel_links(self): + for url in self.explicit_rel_links(): + yield url + for url in self.scraped_rel_links(): + yield url + + def explicit_rel_links(self, rels=('homepage', 'download')): + """Yields all links with the given relations""" + for match in self._rel_re.finditer(self.content): + found_rels = match.group(1).lower().split() + for rel in rels: + if rel in found_rels: + break + else: + continue + match = self._href_re.search(match.group(0)) + if not match: + continue + url = match.group(1) or match.group(2) or match.group(3) + url = self.clean_link(urlparse.urljoin(self.base_url, url)) + yield Link(url, self) + + def scraped_rel_links(self): + for regex in (self._homepage_re, self._download_re): + match = regex.search(self.content) + if not match: + continue + href_match = self._href_re.search(self.content, pos=match.end()) + if not href_match: + continue + url = match.group(1) or match.group(2) or match.group(3) + if not url: + continue + url = self.clean_link(urlparse.urljoin(self.base_url, url)) + yield Link(url, self) + + _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) + + def clean_link(self, url): + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + return self._clean_re.sub( + lambda match: '%%%2x' % ord(match.group(0)), url) + + +class Link(object): + + def __init__(self, url, comes_from=None): + self.url = url + self.comes_from = comes_from + + def __str__(self): + if self.comes_from: + return '%s (from %s)' % (self.url, self.comes_from) + else: + return self.url + + def __repr__(self): + return '' % self + + def __eq__(self, other): + return self.url == other.url + + def __hash__(self): + return hash(self.url) + + @property + def filename(self): + url = self.url + url = url.split('#', 1)[0] + url = url.split('?', 1)[0] + url = url.rstrip('/') + name = posixpath.basename(url) + assert name, ( + 'URL %r produced no filename' % url) + return name + + @property + def scheme(self): + return urlparse.urlsplit(self.url)[0] + + @property + def path(self): + return urlparse.urlsplit(self.url)[2] + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + _egg_fragment_re = re.compile(r'#egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _md5_re = re.compile(r'md5=([a-f0-9]+)') + + @property + def md5_hash(self): + match = self._md5_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + +def get_requirement_from_url(url): + """Get a requirement from the URL, if possible. This looks for #egg + in the URL""" + link = Link(url) + egg_info = link.egg_fragment + if not egg_info: + egg_info = splitext(link.filename)[0] + return package_to_requirement(egg_info) + + +def package_to_requirement(package_name): + """Translate a name like Foo-1.2 to Foo==1.3""" + match = re.search(r'^(.*?)(-dev|-\d.*)', package_name) + if match: + name = match.group(1) + version = match.group(2) + else: + name = package_name + version = '' + if version: + return '%s==%s' % (name, version) + else: + return name + + +def get_mirrors(hostname=None): + """Return the list of mirrors from the last record found on the DNS + entry:: + + >>> from pip.index import get_mirrors + >>> get_mirrors() + ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', + 'd.pypi.python.org'] + + Originally written for the distutils2 project by Alexis Metaireau. + """ + if hostname is None: + hostname = DEFAULT_MIRROR_URL + + # return the last mirror registered on PyPI. + try: + hostname = socket.gethostbyname_ex(hostname)[0] + except socket.gaierror: + return [] + end_letter = hostname.split(".", 1) + + # determine the list from the last one. + return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] + + +def string_range(last): + """Compute the range of string between "a" and last. + + This works for simple "a to z" lists, but also for "a to zz" lists. + """ + for k in range(len(last)): + for x in product(string.ascii_lowercase, repeat=k+1): + result = ''.join(x) + yield result + if result == last: + return + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py new file mode 100644 index 000000000..4254ef2fb --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py @@ -0,0 +1,45 @@ +"""Locations where we look for configs, install stuff, etc""" + +import sys +import os +from distutils import sysconfig + + +def running_under_virtualenv(): + """ + Return True if we're running inside a virtualenv, False otherwise. + + """ + return hasattr(sys, 'real_prefix') + + +if running_under_virtualenv(): + ## FIXME: is build/ a good name? + build_prefix = os.path.join(sys.prefix, 'build') + src_prefix = os.path.join(sys.prefix, 'src') +else: + ## FIXME: this isn't a very good default + build_prefix = os.path.join(os.getcwd(), 'build') + src_prefix = os.path.join(os.getcwd(), 'src') + +# FIXME doesn't account for venv linked to global site-packages + +site_packages = sysconfig.get_python_lib() +user_dir = os.path.expanduser('~') +if sys.platform == 'win32': + bin_py = os.path.join(sys.prefix, 'Scripts') + # buildout uses 'bin' on Windows too? + if not os.path.exists(bin_py): + bin_py = os.path.join(sys.prefix, 'bin') + user_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming + default_storage_dir = os.path.join(user_dir, 'pip') + default_config_file = os.path.join(default_storage_dir, 'pip.ini') + default_log_file = os.path.join(default_storage_dir, 'pip.log') +else: + bin_py = os.path.join(sys.prefix, 'bin') + default_storage_dir = os.path.join(user_dir, '.pip') + default_config_file = os.path.join(default_storage_dir, 'pip.conf') + default_log_file = os.path.join(default_storage_dir, 'pip.log') + # Forcing to use /usr/local/bin for standard Mac OS X framework installs + if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': + bin_py = '/usr/local/bin' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py new file mode 100644 index 000000000..0218ab1ae --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py @@ -0,0 +1,181 @@ +"""Logging +""" + +import sys +import logging + + +class Logger(object): + + """ + Logging object for use in command-line script. Allows ranges of + levels, to avoid some redundancy of displayed information. + """ + + VERBOSE_DEBUG = logging.DEBUG-1 + DEBUG = logging.DEBUG + INFO = logging.INFO + NOTIFY = (logging.INFO+logging.WARN)/2 + WARN = WARNING = logging.WARN + ERROR = logging.ERROR + FATAL = logging.FATAL + + LEVELS = [VERBOSE_DEBUG, DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL] + + def __init__(self): + self.consumers = [] + self.indent = 0 + self.explicit_levels = False + self.in_progress = None + self.in_progress_hanging = False + + def debug(self, msg, *args, **kw): + self.log(self.DEBUG, msg, *args, **kw) + + def info(self, msg, *args, **kw): + self.log(self.INFO, msg, *args, **kw) + + def notify(self, msg, *args, **kw): + self.log(self.NOTIFY, msg, *args, **kw) + + def warn(self, msg, *args, **kw): + self.log(self.WARN, msg, *args, **kw) + + def error(self, msg, *args, **kw): + self.log(self.WARN, msg, *args, **kw) + + def fatal(self, msg, *args, **kw): + self.log(self.FATAL, msg, *args, **kw) + + def log(self, level, msg, *args, **kw): + if args: + if kw: + raise TypeError( + "You may give positional or keyword arguments, not both") + args = args or kw + rendered = None + for consumer_level, consumer in self.consumers: + if self.level_matches(level, consumer_level): + if (self.in_progress_hanging + and consumer in (sys.stdout, sys.stderr)): + self.in_progress_hanging = False + sys.stdout.write('\n') + sys.stdout.flush() + if rendered is None: + if args: + rendered = msg % args + else: + rendered = msg + rendered = ' '*self.indent + rendered + if self.explicit_levels: + ## FIXME: should this be a name, not a level number? + rendered = '%02i %s' % (level, rendered) + if hasattr(consumer, 'write'): + consumer.write(rendered+'\n') + else: + consumer(rendered) + + def start_progress(self, msg): + assert not self.in_progress, ( + "Tried to start_progress(%r) while in_progress %r" + % (msg, self.in_progress)) + if self.level_matches(self.NOTIFY, self._stdout_level()): + sys.stdout.write(' '*self.indent + msg) + sys.stdout.flush() + self.in_progress_hanging = True + else: + self.in_progress_hanging = False + self.in_progress = msg + self.last_message = None + + def end_progress(self, msg='done.'): + assert self.in_progress, ( + "Tried to end_progress without start_progress") + if self.stdout_level_matches(self.NOTIFY): + if not self.in_progress_hanging: + # Some message has been printed out since start_progress + sys.stdout.write('...' + self.in_progress + msg + '\n') + sys.stdout.flush() + else: + # These erase any messages shown with show_progress (besides .'s) + logger.show_progress('') + logger.show_progress('') + sys.stdout.write(msg + '\n') + sys.stdout.flush() + self.in_progress = None + self.in_progress_hanging = False + + def show_progress(self, message=None): + """If we are in a progress scope, and no log messages have been + shown, write out another '.'""" + if self.in_progress_hanging: + if message is None: + sys.stdout.write('.') + sys.stdout.flush() + else: + if self.last_message: + padding = ' ' * max(0, len(self.last_message)-len(message)) + else: + padding = '' + sys.stdout.write('\r%s%s%s%s' % (' '*self.indent, self.in_progress, message, padding)) + sys.stdout.flush() + self.last_message = message + + def stdout_level_matches(self, level): + """Returns true if a message at this level will go to stdout""" + return self.level_matches(level, self._stdout_level()) + + def _stdout_level(self): + """Returns the level that stdout runs at""" + for level, consumer in self.consumers: + if consumer is sys.stdout: + return level + return self.FATAL + + def level_matches(self, level, consumer_level): + """ + >>> l = Logger() + >>> l.level_matches(3, 4) + False + >>> l.level_matches(3, 2) + True + >>> l.level_matches(slice(None, 3), 3) + False + >>> l.level_matches(slice(None, 3), 2) + True + >>> l.level_matches(slice(1, 3), 1) + True + >>> l.level_matches(slice(2, 3), 1) + False + """ + if isinstance(level, slice): + start, stop = level.start, level.stop + if start is not None and start > consumer_level: + return False + if stop is not None or stop <= consumer_level: + return False + return True + else: + return level >= consumer_level + + @classmethod + def level_for_integer(cls, level): + levels = cls.LEVELS + if level < 0: + return levels[0] + if level >= len(levels): + return levels[-1] + return levels[level] + + def move_stdout_to_stderr(self): + to_remove = [] + to_add = [] + for consumer_level, consumer in self.consumers: + if consumer == sys.stdout: + to_remove.append((consumer_level, consumer)) + to_add.append((consumer_level, sys.stderr)) + for item in to_remove: + self.consumers.remove(item) + self.consumers.extend(to_add) + +logger = Logger() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py new file mode 100644 index 000000000..444e7252b --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py @@ -0,0 +1,1432 @@ +import sys +import os +import shutil +import re +import zipfile +import pkg_resources +import tempfile +import urlparse +import urllib2 +import urllib +import ConfigParser +from distutils.sysconfig import get_python_version +from email.FeedParser import FeedParser +from pip.locations import bin_py, running_under_virtualenv +from pip.exceptions import InstallationError, UninstallationError +from pip.vcs import vcs +from pip.log import logger +from pip.util import display_path, rmtree +from pip.util import ask, backup_dir +from pip.util import is_installable_dir, is_local, dist_is_local +from pip.util import renames, normalize_path, egg_link_path +from pip.util import make_path_relative +from pip import call_subprocess +from pip.backwardcompat import any, copytree +from pip.index import Link +from pip.locations import build_prefix +from pip.download import (get_file_content, is_url, url_to_path, + path_to_url, is_archive_file, + unpack_vcs_link, is_vcs_url, is_file_url, + unpack_file_url, unpack_http_url) + + +PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' + + +class InstallRequirement(object): + + def __init__(self, req, comes_from, source_dir=None, editable=False, + url=None, update=True): + if isinstance(req, basestring): + req = pkg_resources.Requirement.parse(req) + self.req = req + self.comes_from = comes_from + self.source_dir = source_dir + self.editable = editable + self.url = url + self._egg_info_path = None + # This holds the pkg_resources.Distribution object if this requirement + # is already available: + self.satisfied_by = None + # This hold the pkg_resources.Distribution object if this requirement + # conflicts with another installed distribution: + self.conflicts_with = None + self._temp_build_dir = None + self._is_bundle = None + # True if the editable should be updated: + self.update = update + # Set to True after successful installation + self.install_succeeded = None + # UninstallPathSet of uninstalled distribution (for possible rollback) + self.uninstalled = None + + @classmethod + def from_editable(cls, editable_req, comes_from=None, default_vcs=None): + name, url = parse_editable(editable_req, default_vcs) + if url.startswith('file:'): + source_dir = url_to_path(url) + else: + source_dir = None + return cls(name, comes_from, source_dir=source_dir, editable=True, url=url) + + @classmethod + def from_line(cls, name, comes_from=None): + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + """ + url = None + name = name.strip() + req = name + path = os.path.normpath(os.path.abspath(name)) + + if is_url(name): + url = name + ## FIXME: I think getting the requirement here is a bad idea: + #req = get_requirement_from_url(url) + req = None + elif os.path.isdir(path) and (os.path.sep in name or name.startswith('.')): + if not is_installable_dir(path): + raise InstallationError("Directory %r is not installable. File 'setup.py' not found." + % name) + url = path_to_url(name) + #req = get_requirement_from_url(url) + req = None + elif is_archive_file(path): + if not os.path.isfile(path): + logger.warn('Requirement %r looks like a filename, but the file does not exist' + % name) + url = path_to_url(name) + #req = get_requirement_from_url(url) + req = None + return cls(req, comes_from, url=url) + + def __str__(self): + if self.req: + s = str(self.req) + if self.url: + s += ' from %s' % self.url + else: + s = self.url + if self.satisfied_by is not None: + s += ' in %s' % display_path(self.satisfied_by.location) + if self.comes_from: + if isinstance(self.comes_from, basestring): + comes_from = self.comes_from + else: + comes_from = self.comes_from.from_path() + if comes_from: + s += ' (from %s)' % comes_from + return s + + def from_path(self): + if self.req is None: + return None + s = str(self.req) + if self.comes_from: + if isinstance(self.comes_from, basestring): + comes_from = self.comes_from + else: + comes_from = self.comes_from.from_path() + if comes_from: + s += '->' + comes_from + return s + + def build_location(self, build_dir, unpack=True): + if self._temp_build_dir is not None: + return self._temp_build_dir + if self.req is None: + self._temp_build_dir = tempfile.mkdtemp('-build', 'pip-') + self._ideal_build_dir = build_dir + return self._temp_build_dir + if self.editable: + name = self.name.lower() + else: + name = self.name + # FIXME: Is there a better place to create the build_dir? (hg and bzr need this) + if not os.path.exists(build_dir): + _make_build_dir(build_dir) + return os.path.join(build_dir, name) + + def correct_build_location(self): + """If the build location was a temporary directory, this will move it + to a new more permanent location""" + if self.source_dir is not None: + return + assert self.req is not None + assert self._temp_build_dir + old_location = self._temp_build_dir + new_build_dir = self._ideal_build_dir + del self._ideal_build_dir + if self.editable: + name = self.name.lower() + else: + name = self.name + new_location = os.path.join(new_build_dir, name) + if not os.path.exists(new_build_dir): + logger.debug('Creating directory %s' % new_build_dir) + _make_build_dir(new_build_dir) + if os.path.exists(new_location): + raise InstallationError( + 'A package already exists in %s; please remove it to continue' + % display_path(new_location)) + logger.debug('Moving package %s from %s to new location %s' + % (self, display_path(old_location), display_path(new_location))) + shutil.move(old_location, new_location) + self._temp_build_dir = new_location + self.source_dir = new_location + self._egg_info_path = None + + @property + def name(self): + if self.req is None: + return None + return self.req.project_name + + @property + def url_name(self): + if self.req is None: + return None + return urllib.quote(self.req.unsafe_name) + + @property + def setup_py(self): + return os.path.join(self.source_dir, 'setup.py') + + def run_egg_info(self, force_root_egg_info=False): + assert self.source_dir + if self.name: + logger.notify('Running setup.py egg_info for package %s' % self.name) + else: + logger.notify('Running setup.py egg_info for package from %s' % self.url) + logger.indent += 2 + try: + script = self._run_setup_py + script = script.replace('__SETUP_PY__', repr(self.setup_py)) + script = script.replace('__PKG_NAME__', repr(self.name)) + # We can't put the .egg-info files at the root, because then the source code will be mistaken + # for an installed egg, causing problems + if self.editable or force_root_egg_info: + egg_base_option = [] + else: + egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') + if not os.path.exists(egg_info_dir): + os.makedirs(egg_info_dir) + egg_base_option = ['--egg-base', 'pip-egg-info'] + call_subprocess( + [sys.executable, '-c', script, 'egg_info'] + egg_base_option, + cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False, + command_level=logger.VERBOSE_DEBUG, + command_desc='python setup.py egg_info') + finally: + logger.indent -= 2 + if not self.req: + self.req = pkg_resources.Requirement.parse(self.pkg_info()['Name']) + self.correct_build_location() + + ## FIXME: This is a lame hack, entirely for PasteScript which has + ## a self-provided entry point that causes this awkwardness + _run_setup_py = """ +__file__ = __SETUP_PY__ +from setuptools.command import egg_info +def replacement_run(self): + self.mkpath(self.egg_info) + installer = self.distribution.fetch_build_egg + for ep in egg_info.iter_entry_points('egg_info.writers'): + # require=False is the change we're making: + writer = ep.load(require=False) + if writer: + writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name)) + self.find_sources() +egg_info.egg_info.run = replacement_run +execfile(__file__) +""" + + def egg_info_data(self, filename): + if self.satisfied_by is not None: + if not self.satisfied_by.has_metadata(filename): + return None + return self.satisfied_by.get_metadata(filename) + assert self.source_dir + filename = self.egg_info_path(filename) + if not os.path.exists(filename): + return None + fp = open(filename, 'r') + data = fp.read() + fp.close() + return data + + def egg_info_path(self, filename): + if self._egg_info_path is None: + if self.editable: + base = self.source_dir + else: + base = os.path.join(self.source_dir, 'pip-egg-info') + filenames = os.listdir(base) + if self.editable: + filenames = [] + for root, dirs, files in os.walk(base): + for dir in vcs.dirnames: + if dir in dirs: + dirs.remove(dir) + for dir in dirs: + # Don't search in anything that looks like a virtualenv environment + if (os.path.exists(os.path.join(root, dir, 'bin', 'python')) + or os.path.exists(os.path.join(root, dir, 'Scripts', 'Python.exe'))): + dirs.remove(dir) + # Also don't search through tests + if dir == 'test' or dir == 'tests': + dirs.remove(dir) + filenames.extend([os.path.join(root, dir) + for dir in dirs]) + filenames = [f for f in filenames if f.endswith('.egg-info')] + + if not filenames: + raise InstallationError('No files/directores in %s (from %s)' % (base, filename)) + assert filenames, "No files/directories in %s (from %s)" % (base, filename) + + # if we have more than one match, we pick the toplevel one. This can + # easily be the case if there is a dist folder which contains an + # extracted tarball for testing purposes. + if len(filenames) > 1: + filenames.sort(key=lambda x: x.count(os.path.sep) + + (os.path.altsep and + x.count(os.path.altsep) or 0)) + self._egg_info_path = os.path.join(base, filenames[0]) + return os.path.join(self._egg_info_path, filename) + + def egg_info_lines(self, filename): + data = self.egg_info_data(filename) + if not data: + return [] + result = [] + for line in data.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + result.append(line) + return result + + def pkg_info(self): + p = FeedParser() + data = self.egg_info_data('PKG-INFO') + if not data: + logger.warn('No PKG-INFO file found in %s' % display_path(self.egg_info_path('PKG-INFO'))) + p.feed(data or '') + return p.close() + + @property + def dependency_links(self): + return self.egg_info_lines('dependency_links.txt') + + _requirements_section_re = re.compile(r'\[(.*?)\]') + + def requirements(self, extras=()): + in_extra = None + for line in self.egg_info_lines('requires.txt'): + match = self._requirements_section_re.match(line) + if match: + in_extra = match.group(1) + continue + if in_extra and in_extra not in extras: + # Skip requirement for an extra we aren't requiring + continue + yield line + + @property + def absolute_versions(self): + for qualifier, version in self.req.specs: + if qualifier == '==': + yield version + + @property + def installed_version(self): + return self.pkg_info()['version'] + + def assert_source_matches_version(self): + assert self.source_dir + if self.comes_from is None: + # We don't check the versions of things explicitly installed. + # This makes, e.g., "pip Package==dev" possible + return + version = self.installed_version + if version not in self.req: + logger.fatal( + 'Source in %s has the version %s, which does not match the requirement %s' + % (display_path(self.source_dir), version, self)) + raise InstallationError( + 'Source in %s has version %s that conflicts with %s' + % (display_path(self.source_dir), version, self)) + else: + logger.debug('Source in %s has version %s, which satisfies requirement %s' + % (display_path(self.source_dir), version, self)) + + def update_editable(self, obtain=True): + if not self.url: + logger.info("Cannot update repository at %s; repository location is unknown" % self.source_dir) + return + assert self.editable + assert self.source_dir + if self.url.startswith('file:'): + # Static paths don't get updated + return + assert '+' in self.url, "bad url: %r" % self.url + if not self.update: + return + vc_type, url = self.url.split('+', 1) + backend = vcs.get_backend(vc_type) + if backend: + vcs_backend = backend(self.url) + if obtain: + vcs_backend.obtain(self.source_dir) + else: + vcs_backend.export(self.source_dir) + else: + assert 0, ( + 'Unexpected version control type (in %s): %s' + % (self.url, vc_type)) + + def uninstall(self, auto_confirm=False): + """ + Uninstall the distribution currently satisfying this requirement. + + Prompts before removing or modifying files unless + ``auto_confirm`` is True. + + Refuses to delete or modify files outside of ``sys.prefix`` - + thus uninstallation within a virtual environment can only + modify that virtual environment, even if the virtualenv is + linked to global site-packages. + + """ + if not self.check_if_exists(): + raise UninstallationError("Cannot uninstall requirement %s, not installed" % (self.name,)) + dist = self.satisfied_by or self.conflicts_with + + paths_to_remove = UninstallPathSet(dist) + + pip_egg_info_path = os.path.join(dist.location, + dist.egg_name()) + '.egg-info' + easy_install_egg = dist.egg_name() + '.egg' + develop_egg_link = egg_link_path(dist) + if os.path.exists(pip_egg_info_path): + # package installed by pip + paths_to_remove.add(pip_egg_info_path) + if dist.has_metadata('installed-files.txt'): + for installed_file in dist.get_metadata('installed-files.txt').splitlines(): + path = os.path.normpath(os.path.join(pip_egg_info_path, installed_file)) + paths_to_remove.add(path) + if dist.has_metadata('top_level.txt'): + if dist.has_metadata('namespace_packages.txt'): + namespaces = dist.get_metadata('namespace_packages.txt') + else: + namespaces = [] + for top_level_pkg in [p for p + in dist.get_metadata('top_level.txt').splitlines() + if p and p not in namespaces]: + path = os.path.join(dist.location, top_level_pkg) + paths_to_remove.add(path) + paths_to_remove.add(path + '.py') + paths_to_remove.add(path + '.pyc') + + elif dist.location.endswith(easy_install_egg): + # package installed by easy_install + paths_to_remove.add(dist.location) + easy_install_pth = os.path.join(os.path.dirname(dist.location), + 'easy-install.pth') + paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) + + elif os.path.isfile(develop_egg_link): + # develop egg + fh = open(develop_egg_link, 'r') + link_pointer = os.path.normcase(fh.readline().strip()) + fh.close() + assert (link_pointer == dist.location), 'Egg-link %s does not match installed location of %s (at %s)' % (link_pointer, self.name, dist.location) + paths_to_remove.add(develop_egg_link) + easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), + 'easy-install.pth') + paths_to_remove.add_pth(easy_install_pth, dist.location) + + # find distutils scripts= scripts + if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): + for script in dist.metadata_listdir('scripts'): + paths_to_remove.add(os.path.join(bin_py, script)) + if sys.platform == 'win32': + paths_to_remove.add(os.path.join(bin_py, script) + '.bat') + + # find console_scripts + if dist.has_metadata('entry_points.txt'): + config = ConfigParser.SafeConfigParser() + config.readfp(FakeFile(dist.get_metadata_lines('entry_points.txt'))) + if config.has_section('console_scripts'): + for name, value in config.items('console_scripts'): + paths_to_remove.add(os.path.join(bin_py, name)) + if sys.platform == 'win32': + paths_to_remove.add(os.path.join(bin_py, name) + '.exe') + paths_to_remove.add(os.path.join(bin_py, name) + '.exe.manifest') + paths_to_remove.add(os.path.join(bin_py, name) + '-script.py') + + paths_to_remove.remove(auto_confirm) + self.uninstalled = paths_to_remove + + def rollback_uninstall(self): + if self.uninstalled: + self.uninstalled.rollback() + else: + logger.error("Can't rollback %s, nothing uninstalled." + % (self.project_name,)) + + def commit_uninstall(self): + if self.uninstalled: + self.uninstalled.commit() + else: + logger.error("Can't commit %s, nothing uninstalled." + % (self.project_name,)) + + def archive(self, build_dir): + assert self.source_dir + create_archive = True + archive_name = '%s-%s.zip' % (self.name, self.installed_version) + archive_path = os.path.join(build_dir, archive_name) + if os.path.exists(archive_path): + response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' + % display_path(archive_path), ('i', 'w', 'b')) + if response == 'i': + create_archive = False + elif response == 'w': + logger.warn('Deleting %s' % display_path(archive_path)) + os.remove(archive_path) + elif response == 'b': + dest_file = backup_dir(archive_path) + logger.warn('Backing up %s to %s' + % (display_path(archive_path), display_path(dest_file))) + shutil.move(archive_path, dest_file) + if create_archive: + zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) + dir = os.path.normcase(os.path.abspath(self.source_dir)) + for dirpath, dirnames, filenames in os.walk(dir): + if 'pip-egg-info' in dirnames: + dirnames.remove('pip-egg-info') + for dirname in dirnames: + dirname = os.path.join(dirpath, dirname) + name = self._clean_zip_name(dirname, dir) + zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') + zipdir.external_attr = 0755 << 16L + zip.writestr(zipdir, '') + for filename in filenames: + if filename == PIP_DELETE_MARKER_FILENAME: + continue + filename = os.path.join(dirpath, filename) + name = self._clean_zip_name(filename, dir) + zip.write(filename, self.name + '/' + name) + zip.close() + logger.indent -= 2 + logger.notify('Saved %s' % display_path(archive_path)) + + def _clean_zip_name(self, name, prefix): + assert name.startswith(prefix+os.path.sep), ( + "name %r doesn't start with prefix %r" % (name, prefix)) + name = name[len(prefix)+1:] + name = name.replace(os.path.sep, '/') + return name + + def install(self, install_options, global_options=()): + if self.editable: + self.install_editable(install_options, global_options) + return + temp_location = tempfile.mkdtemp('-record', 'pip-') + record_filename = os.path.join(temp_location, 'install-record.txt') + try: + + install_args = [ + sys.executable, '-c', + "import setuptools;__file__=%r;"\ + "execfile(__file__)" % self.setup_py] +\ + list(global_options) + [ + 'install', + '--single-version-externally-managed', + '--record', record_filename] + + if running_under_virtualenv(): + ## FIXME: I'm not sure if this is a reasonable location; probably not + ## but we can't put it in the default location, as that is a virtualenv symlink that isn't writable + install_args += ['--install-headers', + os.path.join(sys.prefix, 'include', 'site', + 'python' + get_python_version())] + logger.notify('Running setup.py install for %s' % self.name) + logger.indent += 2 + try: + call_subprocess(install_args + install_options, + cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False) + finally: + logger.indent -= 2 + if not os.path.exists(record_filename): + logger.notify('Record file %s not found' % record_filename) + return + self.install_succeeded = True + f = open(record_filename) + for line in f: + line = line.strip() + if line.endswith('.egg-info'): + egg_info_dir = line + break + else: + logger.warn('Could not find .egg-info directory in install record for %s' % self) + ## FIXME: put the record somewhere + ## FIXME: should this be an error? + return + f.close() + new_lines = [] + f = open(record_filename) + for line in f: + filename = line.strip() + if os.path.isdir(filename): + filename += os.path.sep + new_lines.append(make_path_relative(filename, egg_info_dir)) + f.close() + f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') + f.write('\n'.join(new_lines)+'\n') + f.close() + finally: + if os.path.exists(record_filename): + os.remove(record_filename) + os.rmdir(temp_location) + + def remove_temporary_source(self): + """Remove the source files from this requirement, if they are marked + for deletion""" + if self.is_bundle or os.path.exists(self.delete_marker_filename): + logger.info('Removing source in %s' % self.source_dir) + if self.source_dir: + rmtree(self.source_dir) + self.source_dir = None + if self._temp_build_dir and os.path.exists(self._temp_build_dir): + rmtree(self._temp_build_dir) + self._temp_build_dir = None + + def install_editable(self, install_options, global_options=()): + logger.notify('Running setup.py develop for %s' % self.name) + logger.indent += 2 + try: + ## FIXME: should we do --install-headers here too? + call_subprocess( + [sys.executable, '-c', + "import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py)] + + list(global_options) + ['develop', '--no-deps'] + list(install_options), + + cwd=self.source_dir, filter_stdout=self._filter_install, + show_stdout=False) + finally: + logger.indent -= 2 + self.install_succeeded = True + + def _filter_install(self, line): + level = logger.NOTIFY + for regex in [r'^running .*', r'^writing .*', '^creating .*', '^[Cc]opying .*', + r'^reading .*', r"^removing .*\.egg-info' \(and everything under it\)$", + r'^byte-compiling ', + # Not sure what this warning is, but it seems harmless: + r"^warning: manifest_maker: standard file '-c' not found$"]: + if re.search(regex, line.strip()): + level = logger.INFO + break + return (level, line) + + def check_if_exists(self): + """Find an installed distribution that satisfies or conflicts + with this requirement, and set self.satisfied_by or + self.conflicts_with appropriately.""" + if self.req is None: + return False + try: + self.satisfied_by = pkg_resources.get_distribution(self.req) + except pkg_resources.DistributionNotFound: + return False + except pkg_resources.VersionConflict: + self.conflicts_with = pkg_resources.get_distribution(self.req.project_name) + return True + + @property + def is_bundle(self): + if self._is_bundle is not None: + return self._is_bundle + base = self._temp_build_dir + if not base: + ## FIXME: this doesn't seem right: + return False + self._is_bundle = (os.path.exists(os.path.join(base, 'pip-manifest.txt')) + or os.path.exists(os.path.join(base, 'pyinstall-manifest.txt'))) + return self._is_bundle + + def bundle_requirements(self): + for dest_dir in self._bundle_editable_dirs: + package = os.path.basename(dest_dir) + ## FIXME: svnism: + for vcs_backend in vcs.backends: + url = rev = None + vcs_bundle_file = os.path.join( + dest_dir, vcs_backend.bundle_file) + if os.path.exists(vcs_bundle_file): + vc_type = vcs_backend.name + fp = open(vcs_bundle_file) + content = fp.read() + fp.close() + url, rev = vcs_backend().parse_vcs_bundle_file(content) + break + if url: + url = '%s+%s@%s' % (vc_type, url, rev) + else: + url = None + yield InstallRequirement( + package, self, editable=True, url=url, + update=False, source_dir=dest_dir) + for dest_dir in self._bundle_build_dirs: + package = os.path.basename(dest_dir) + yield InstallRequirement( + package, self, + source_dir=dest_dir) + + def move_bundle_files(self, dest_build_dir, dest_src_dir): + base = self._temp_build_dir + assert base + src_dir = os.path.join(base, 'src') + build_dir = os.path.join(base, 'build') + bundle_build_dirs = [] + bundle_editable_dirs = [] + for source_dir, dest_dir, dir_collection in [ + (src_dir, dest_src_dir, bundle_editable_dirs), + (build_dir, dest_build_dir, bundle_build_dirs)]: + if os.path.exists(source_dir): + for dirname in os.listdir(source_dir): + dest = os.path.join(dest_dir, dirname) + dir_collection.append(dest) + if os.path.exists(dest): + logger.warn('The directory %s (containing package %s) already exists; cannot move source from bundle %s' + % (dest, dirname, self)) + continue + if not os.path.exists(dest_dir): + logger.info('Creating directory %s' % dest_dir) + os.makedirs(dest_dir) + shutil.move(os.path.join(source_dir, dirname), dest) + if not os.listdir(source_dir): + os.rmdir(source_dir) + self._temp_build_dir = None + self._bundle_build_dirs = bundle_build_dirs + self._bundle_editable_dirs = bundle_editable_dirs + + @property + def delete_marker_filename(self): + assert self.source_dir + return os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME) + + +DELETE_MARKER_MESSAGE = '''\ +This file is placed here by pip to indicate the source was put +here by pip. + +Once this package is successfully installed this source code will be +deleted (unless you remove this file). +''' + + +class RequirementSet(object): + + def __init__(self, build_dir, src_dir, download_dir, download_cache=None, + upgrade=False, ignore_installed=False, + ignore_dependencies=False): + self.build_dir = build_dir + self.src_dir = src_dir + self.download_dir = download_dir + self.download_cache = download_cache + self.upgrade = upgrade + self.ignore_installed = ignore_installed + self.requirements = {} + # Mapping of alias: real_name + self.requirement_aliases = {} + self.unnamed_requirements = [] + self.ignore_dependencies = ignore_dependencies + self.successfully_downloaded = [] + self.successfully_installed = [] + self.reqs_to_cleanup = [] + + def __str__(self): + reqs = [req for req in self.requirements.values() + if not req.comes_from] + reqs.sort(key=lambda req: req.name.lower()) + return ' '.join([str(req.req) for req in reqs]) + + def add_requirement(self, install_req): + name = install_req.name + if not name: + self.unnamed_requirements.append(install_req) + else: + if self.has_requirement(name): + raise InstallationError( + 'Double requirement given: %s (aready in %s, name=%r)' + % (install_req, self.get_requirement(name), name)) + self.requirements[name] = install_req + ## FIXME: what about other normalizations? E.g., _ vs. -? + if name.lower() != name: + self.requirement_aliases[name.lower()] = name + + def has_requirement(self, project_name): + for name in project_name, project_name.lower(): + if name in self.requirements or name in self.requirement_aliases: + return True + return False + + @property + def has_requirements(self): + return self.requirements.values() or self.unnamed_requirements + + @property + def has_editables(self): + if any(req.editable for req in self.requirements.values()): + return True + if any(req.editable for req in self.unnamed_requirements): + return True + return False + + @property + def is_download(self): + if self.download_dir: + self.download_dir = os.path.expanduser(self.download_dir) + if os.path.exists(self.download_dir): + return True + else: + logger.fatal('Could not find download directory') + raise InstallationError( + "Could not find or access download directory '%s'" + % display_path(self.download_dir)) + return False + + def get_requirement(self, project_name): + for name in project_name, project_name.lower(): + if name in self.requirements: + return self.requirements[name] + if name in self.requirement_aliases: + return self.requirements[self.requirement_aliases[name]] + raise KeyError("No project with the name %r" % project_name) + + def uninstall(self, auto_confirm=False): + for req in self.requirements.values(): + req.uninstall(auto_confirm=auto_confirm) + req.commit_uninstall() + + def locate_files(self): + ## FIXME: duplicates code from install_files; relevant code should + ## probably be factored out into a separate method + unnamed = list(self.unnamed_requirements) + reqs = self.requirements.values() + while reqs or unnamed: + if unnamed: + req_to_install = unnamed.pop(0) + else: + req_to_install = reqs.pop(0) + install_needed = True + if not self.ignore_installed and not req_to_install.editable: + req_to_install.check_if_exists() + if req_to_install.satisfied_by: + if self.upgrade: + req_to_install.conflicts_with = req_to_install.satisfied_by + req_to_install.satisfied_by = None + else: + install_needed = False + if req_to_install.satisfied_by: + logger.notify('Requirement already satisfied ' + '(use --upgrade to upgrade): %s' + % req_to_install) + + if req_to_install.editable: + if req_to_install.source_dir is None: + req_to_install.source_dir = req_to_install.build_location(self.src_dir) + elif install_needed: + req_to_install.source_dir = req_to_install.build_location(self.build_dir, not self.is_download) + + if req_to_install.source_dir is not None and not os.path.isdir(req_to_install.source_dir): + raise InstallationError('Could not install requirement %s ' + 'because source folder %s does not exist ' + '(perhaps --no-download was used without first running ' + 'an equivalent install with --no-install?)' + % (req_to_install, req_to_install.source_dir)) + + def prepare_files(self, finder, force_root_egg_info=False, bundle=False): + """Prepare process. Create temp directories, download and/or unpack files.""" + unnamed = list(self.unnamed_requirements) + reqs = self.requirements.values() + while reqs or unnamed: + if unnamed: + req_to_install = unnamed.pop(0) + else: + req_to_install = reqs.pop(0) + install = True + if not self.ignore_installed and not req_to_install.editable: + req_to_install.check_if_exists() + if req_to_install.satisfied_by: + if self.upgrade: + req_to_install.conflicts_with = req_to_install.satisfied_by + req_to_install.satisfied_by = None + else: + install = False + if req_to_install.satisfied_by: + logger.notify('Requirement already satisfied ' + '(use --upgrade to upgrade): %s' + % req_to_install) + if req_to_install.editable: + logger.notify('Obtaining %s' % req_to_install) + elif install: + if req_to_install.url and req_to_install.url.lower().startswith('file:'): + logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) + else: + logger.notify('Downloading/unpacking %s' % req_to_install) + logger.indent += 2 + try: + is_bundle = False + if req_to_install.editable: + if req_to_install.source_dir is None: + location = req_to_install.build_location(self.src_dir) + req_to_install.source_dir = location + else: + location = req_to_install.source_dir + if not os.path.exists(self.build_dir): + _make_build_dir(self.build_dir) + req_to_install.update_editable(not self.is_download) + if self.is_download: + req_to_install.run_egg_info() + req_to_install.archive(self.download_dir) + else: + req_to_install.run_egg_info() + elif install: + ##@@ if filesystem packages are not marked + ##editable in a req, a non deterministic error + ##occurs when the script attempts to unpack the + ##build directory + + location = req_to_install.build_location(self.build_dir, not self.is_download) + ## FIXME: is the existance of the checkout good enough to use it? I don't think so. + unpack = True + if not os.path.exists(os.path.join(location, 'setup.py')): + ## FIXME: this won't upgrade when there's an existing package unpacked in `location` + if req_to_install.url is None: + url = finder.find_requirement(req_to_install, upgrade=self.upgrade) + else: + ## FIXME: should req_to_install.url already be a link? + url = Link(req_to_install.url) + assert url + if url: + try: + self.unpack_url(url, location, self.is_download) + except urllib2.HTTPError, e: + logger.fatal('Could not install requirement %s because of error %s' + % (req_to_install, e)) + raise InstallationError( + 'Could not install requirement %s because of HTTP error %s for URL %s' + % (req_to_install, e, url)) + else: + unpack = False + if unpack: + is_bundle = req_to_install.is_bundle + url = None + if is_bundle: + req_to_install.move_bundle_files(self.build_dir, self.src_dir) + for subreq in req_to_install.bundle_requirements(): + reqs.append(subreq) + self.add_requirement(subreq) + elif self.is_download: + req_to_install.source_dir = location + if url and url.scheme in vcs.all_schemes: + req_to_install.run_egg_info() + req_to_install.archive(self.download_dir) + else: + req_to_install.source_dir = location + req_to_install.run_egg_info() + if force_root_egg_info: + # We need to run this to make sure that the .egg-info/ + # directory is created for packing in the bundle + req_to_install.run_egg_info(force_root_egg_info=True) + req_to_install.assert_source_matches_version() + #@@ sketchy way of identifying packages not grabbed from an index + if bundle and req_to_install.url: + self.copy_to_build_dir(req_to_install) + if not is_bundle and not self.is_download: + ## FIXME: shouldn't be globally added: + finder.add_dependency_links(req_to_install.dependency_links) + ## FIXME: add extras in here: + if not self.ignore_dependencies: + for req in req_to_install.requirements(): + try: + name = pkg_resources.Requirement.parse(req).project_name + except ValueError, e: + ## FIXME: proper warning + logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) + continue + if self.has_requirement(name): + ## FIXME: check for conflict + continue + subreq = InstallRequirement(req, req_to_install) + reqs.append(subreq) + self.add_requirement(subreq) + if req_to_install.name not in self.requirements: + self.requirements[req_to_install.name] = req_to_install + else: + self.reqs_to_cleanup.append(req_to_install) + if install: + self.successfully_downloaded.append(req_to_install) + if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): + self.copy_to_build_dir(req_to_install) + finally: + logger.indent -= 2 + + def cleanup_files(self, bundle=False): + """Clean up files, remove builds.""" + logger.notify('Cleaning up...') + logger.indent += 2 + for req in self.reqs_to_cleanup: + req.remove_temporary_source() + + remove_dir = [] + if self._pip_has_created_build_dir(): + remove_dir.append(self.build_dir) + + # The source dir of a bundle can always be removed. + if bundle: + remove_dir.append(self.src_dir) + + for dir in remove_dir: + if os.path.exists(dir): + logger.info('Removing temporary dir %s...' % dir) + rmtree(dir) + + logger.indent -= 2 + + def _pip_has_created_build_dir(self): + return (self.build_dir == build_prefix and + os.path.exists(os.path.join(self.build_dir, PIP_DELETE_MARKER_FILENAME))) + + def copy_to_build_dir(self, req_to_install): + target_dir = req_to_install.editable and self.src_dir or self.build_dir + logger.info("Copying %s to %s" %(req_to_install.name, target_dir)) + dest = os.path.join(target_dir, req_to_install.name) + copytree(req_to_install.source_dir, dest) + call_subprocess(["python", "%s/setup.py"%dest, "clean"]) + + def unpack_url(self, link, location, only_download=False): + if only_download: + location = self.download_dir + if is_vcs_url(link): + return unpack_vcs_link(link, location, only_download) + elif is_file_url(link): + return unpack_file_url(link, location) + else: + if self.download_cache: + self.download_cache = os.path.expanduser(self.download_cache) + return unpack_http_url(link, location, self.download_cache, only_download) + + def install(self, install_options, global_options=()): + """Install everything in this set (after having downloaded and unpacked the packages)""" + to_install = sorted([r for r in self.requirements.values() + if self.upgrade or not r.satisfied_by], + key=lambda p: p.name.lower()) + if to_install: + logger.notify('Installing collected packages: %s' % (', '.join([req.name for req in to_install]))) + logger.indent += 2 + try: + for requirement in to_install: + if requirement.conflicts_with: + logger.notify('Found existing installation: %s' + % requirement.conflicts_with) + logger.indent += 2 + try: + requirement.uninstall(auto_confirm=True) + finally: + logger.indent -= 2 + try: + requirement.install(install_options, global_options) + except: + # if install did not succeed, rollback previous uninstall + if requirement.conflicts_with and not requirement.install_succeeded: + requirement.rollback_uninstall() + raise + else: + if requirement.conflicts_with and requirement.install_succeeded: + requirement.commit_uninstall() + requirement.remove_temporary_source() + finally: + logger.indent -= 2 + self.successfully_installed = to_install + + def create_bundle(self, bundle_filename): + ## FIXME: can't decide which is better; zip is easier to read + ## random files from, but tar.bz2 is smaller and not as lame a + ## format. + + ## FIXME: this file should really include a manifest of the + ## packages, maybe some other metadata files. It would make + ## it easier to detect as well. + zip = zipfile.ZipFile(bundle_filename, 'w', zipfile.ZIP_DEFLATED) + vcs_dirs = [] + for dir, basename in (self.build_dir, 'build'), (self.src_dir, 'src'): + dir = os.path.normcase(os.path.abspath(dir)) + for dirpath, dirnames, filenames in os.walk(dir): + for backend in vcs.backends: + vcs_backend = backend() + vcs_url = vcs_rev = None + if vcs_backend.dirname in dirnames: + for vcs_dir in vcs_dirs: + if dirpath.startswith(vcs_dir): + # vcs bundle file already in parent directory + break + else: + vcs_url, vcs_rev = vcs_backend.get_info( + os.path.join(dir, dirpath)) + vcs_dirs.append(dirpath) + vcs_bundle_file = vcs_backend.bundle_file + vcs_guide = vcs_backend.guide % {'url': vcs_url, + 'rev': vcs_rev} + dirnames.remove(vcs_backend.dirname) + break + if 'pip-egg-info' in dirnames: + dirnames.remove('pip-egg-info') + for dirname in dirnames: + dirname = os.path.join(dirpath, dirname) + name = self._clean_zip_name(dirname, dir) + zip.writestr(basename + '/' + name + '/', '') + for filename in filenames: + if filename == PIP_DELETE_MARKER_FILENAME: + continue + filename = os.path.join(dirpath, filename) + name = self._clean_zip_name(filename, dir) + zip.write(filename, basename + '/' + name) + if vcs_url: + name = os.path.join(dirpath, vcs_bundle_file) + name = self._clean_zip_name(name, dir) + zip.writestr(basename + '/' + name, vcs_guide) + + zip.writestr('pip-manifest.txt', self.bundle_requirements()) + zip.close() + + BUNDLE_HEADER = '''\ +# This is a pip bundle file, that contains many source packages +# that can be installed as a group. You can install this like: +# pip this_file.zip +# The rest of the file contains a list of all the packages included: +''' + + def bundle_requirements(self): + parts = [self.BUNDLE_HEADER] + for req in sorted( + [req for req in self.requirements.values() + if not req.comes_from], + key=lambda x: x.name): + parts.append('%s==%s\n' % (req.name, req.installed_version)) + parts.append('# These packages were installed to satisfy the above requirements:\n') + for req in sorted( + [req for req in self.requirements.values() + if req.comes_from], + key=lambda x: x.name): + parts.append('%s==%s\n' % (req.name, req.installed_version)) + ## FIXME: should we do something with self.unnamed_requirements? + return ''.join(parts) + + def _clean_zip_name(self, name, prefix): + assert name.startswith(prefix+os.path.sep), ( + "name %r doesn't start with prefix %r" % (name, prefix)) + name = name[len(prefix)+1:] + name = name.replace(os.path.sep, '/') + return name + + +def _make_build_dir(build_dir): + os.makedirs(build_dir) + _write_delete_marker_message(os.path.join(build_dir, PIP_DELETE_MARKER_FILENAME)) + + +def _write_delete_marker_message(filepath): + marker_fp = open(filepath, 'w') + marker_fp.write(DELETE_MARKER_MESSAGE) + marker_fp.close() + + +_scheme_re = re.compile(r'^(http|https|file):', re.I) + + +def parse_requirements(filename, finder=None, comes_from=None, options=None): + skip_match = None + skip_regex = options.skip_requirements_regex + if skip_regex: + skip_match = re.compile(skip_regex) + filename, content = get_file_content(filename, comes_from=comes_from) + for line_number, line in enumerate(content.splitlines()): + line_number += 1 + line = line.strip() + if not line or line.startswith('#'): + continue + if skip_match and skip_match.search(line): + continue + if line.startswith('-r') or line.startswith('--requirement'): + if line.startswith('-r'): + req_url = line[2:].strip() + else: + req_url = line[len('--requirement'):].strip().strip('=') + if _scheme_re.search(filename): + # Relative to a URL + req_url = urlparse.urljoin(req_url, filename) + elif not _scheme_re.search(req_url): + req_url = os.path.join(os.path.dirname(filename), req_url) + for item in parse_requirements(req_url, finder, comes_from=filename, options=options): + yield item + elif line.startswith('-Z') or line.startswith('--always-unzip'): + # No longer used, but previously these were used in + # requirement files, so we'll ignore. + pass + elif line.startswith('-f') or line.startswith('--find-links'): + if line.startswith('-f'): + line = line[2:].strip() + else: + line = line[len('--find-links'):].strip().lstrip('=') + ## FIXME: it would be nice to keep track of the source of + ## the find_links: + if finder: + finder.find_links.append(line) + elif line.startswith('-i') or line.startswith('--index-url'): + if line.startswith('-i'): + line = line[2:].strip() + else: + line = line[len('--index-url'):].strip().lstrip('=') + if finder: + finder.index_urls = [line] + elif line.startswith('--extra-index-url'): + line = line[len('--extra-index-url'):].strip().lstrip('=') + if finder: + finder.index_urls.append(line) + else: + comes_from = '-r %s (line %s)' % (filename, line_number) + if line.startswith('-e') or line.startswith('--editable'): + if line.startswith('-e'): + line = line[2:].strip() + else: + line = line[len('--editable'):].strip() + req = InstallRequirement.from_editable( + line, comes_from=comes_from, default_vcs=options.default_vcs) + else: + req = InstallRequirement.from_line(line, comes_from) + yield req + + +def parse_editable(editable_req, default_vcs=None): + """Parses svn+http://blahblah@rev#egg=Foobar into a requirement + (Foobar) and a URL""" + url = editable_req + if os.path.isdir(url) and os.path.exists(os.path.join(url, 'setup.py')): + # Treating it as code that has already been checked out + url = path_to_url(url) + if url.lower().startswith('file:'): + return None, url + for version_control in vcs: + if url.lower().startswith('%s:' % version_control): + url = '%s+%s' % (version_control, url) + if '+' not in url: + if default_vcs: + url = default_vcs + '+' + url + else: + raise InstallationError( + '--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req) + vc_type = url.split('+', 1)[0].lower() + if not vcs.get_backend(vc_type): + raise InstallationError( + 'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req) + match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req) + if (not match or not match.group(1)) and vcs.get_backend(vc_type): + parts = [p for p in editable_req.split('#', 1)[0].split('/') if p] + if parts[-2] in ('tags', 'branches', 'tag', 'branch'): + req = parts[-3] + elif parts[-1] == 'trunk': + req = parts[-2] + else: + raise InstallationError( + '--editable=%s is not the right format; it must have #egg=Package' + % editable_req) + else: + req = match.group(1) + ## FIXME: use package_to_requirement? + match = re.search(r'^(.*?)(?:-dev|-\d.*)', req) + if match: + # Strip off -dev, -0.2, etc. + req = match.group(1) + return req, url + + +class UninstallPathSet(object): + """A set of file paths to be removed in the uninstallation of a + requirement.""" + def __init__(self, dist): + self.paths = set() + self._refuse = set() + self.pth = {} + self.dist = dist + self.save_dir = None + self._moved_paths = [] + + def _permitted(self, path): + """ + Return True if the given path is one we are permitted to + remove/modify, False otherwise. + + """ + return is_local(path) + + def _can_uninstall(self): + if not dist_is_local(self.dist): + logger.notify("Not uninstalling %s at %s, outside environment %s" + % (self.dist.project_name, normalize_path(self.dist.location), sys.prefix)) + return False + return True + + def add(self, path): + path = normalize_path(path) + if not os.path.exists(path): + return + if self._permitted(path): + self.paths.add(path) + else: + self._refuse.add(path) + + def add_pth(self, pth_file, entry): + pth_file = normalize_path(pth_file) + if self._permitted(pth_file): + if pth_file not in self.pth: + self.pth[pth_file] = UninstallPthEntries(pth_file) + self.pth[pth_file].add(entry) + else: + self._refuse.add(pth_file) + + def compact(self, paths): + """Compact a path set to contain the minimal number of paths + necessary to contain all paths in the set. If /a/path/ and + /a/path/to/a/file.txt are both in the set, leave only the + shorter path.""" + short_paths = set() + for path in sorted(paths, key=len): + if not any([(path.startswith(shortpath) and + path[len(shortpath.rstrip(os.path.sep))] == os.path.sep) + for shortpath in short_paths]): + short_paths.add(path) + return short_paths + + def _stash(self, path): + return os.path.join( + self.save_dir, os.path.splitdrive(path)[1].lstrip(os.path.sep)) + + def remove(self, auto_confirm=False): + """Remove paths in ``self.paths`` with confirmation (unless + ``auto_confirm`` is True).""" + if not self._can_uninstall(): + return + logger.notify('Uninstalling %s:' % self.dist.project_name) + logger.indent += 2 + paths = sorted(self.compact(self.paths)) + try: + if auto_confirm: + response = 'y' + else: + for path in paths: + logger.notify(path) + response = ask('Proceed (y/n)? ', ('y', 'n')) + if self._refuse: + logger.notify('Not removing or modifying (outside of prefix):') + for path in self.compact(self._refuse): + logger.notify(path) + if response == 'y': + self.save_dir = tempfile.mkdtemp(suffix='-uninstall', + prefix='pip-') + for path in paths: + new_path = self._stash(path) + logger.info('Removing file or directory %s' % path) + self._moved_paths.append(path) + renames(path, new_path) + for pth in self.pth.values(): + pth.remove() + logger.notify('Successfully uninstalled %s' % self.dist.project_name) + + finally: + logger.indent -= 2 + + def rollback(self): + """Rollback the changes previously made by remove().""" + if self.save_dir is None: + logger.error("Can't roll back %s; was not uninstalled" % self.dist.project_name) + return False + logger.notify('Rolling back uninstall of %s' % self.dist.project_name) + for path in self._moved_paths: + tmp_path = self._stash(path) + logger.info('Replacing %s' % path) + renames(tmp_path, path) + for pth in self.pth: + pth.rollback() + + def commit(self): + """Remove temporary save dir: rollback will no longer be possible.""" + if self.save_dir is not None: + shutil.rmtree(self.save_dir) + self.save_dir = None + self._moved_paths = [] + + +class UninstallPthEntries(object): + def __init__(self, pth_file): + if not os.path.isfile(pth_file): + raise UninstallationError("Cannot remove entries from nonexistent file %s" % pth_file) + self.file = pth_file + self.entries = set() + self._saved_lines = None + + def add(self, entry): + entry = os.path.normcase(entry) + # On Windows, os.path.normcase converts the entry to use + # backslashes. This is correct for entries that describe absolute + # paths outside of site-packages, but all the others use forward + # slashes. + if sys.platform == 'win32' and not os.path.splitdrive(entry)[0]: + entry = entry.replace('\\', '/') + self.entries.add(entry) + + def remove(self): + logger.info('Removing pth entries from %s:' % self.file) + fh = open(self.file, 'r') + lines = fh.readlines() + self._saved_lines = lines + fh.close() + try: + for entry in self.entries: + logger.info('Removing entry: %s' % entry) + try: + lines.remove(entry + '\n') + except ValueError: + pass + finally: + pass + fh = open(self.file, 'wb') + fh.writelines(lines) + fh.close() + + def rollback(self): + if self._saved_lines is None: + logger.error('Cannot roll back changes to %s, none were made' % self.file) + return False + logger.info('Rolling %s back to previous state' % self.file) + fh = open(self.file, 'wb') + fh.writelines(self._saved_lines) + fh.close() + return True + + +class FakeFile(object): + """Wrap a list of lines in an object with readline() to make + ConfigParser happy.""" + def __init__(self, lines): + self._gen = (l for l in lines) + + def readline(self): + try: + return self._gen.next() + except StopIteration: + return '' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py new file mode 100644 index 000000000..be830ad9a --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py @@ -0,0 +1,18 @@ +import sys +import os + + +def run(): + base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ## FIXME: this is kind of crude; if we could create a fake pip + ## module, then exec into it and update pip.__path__ properly, we + ## wouldn't have to update sys.path: + sys.path.insert(0, base) + import pip + return pip.main() + + +if __name__ == '__main__': + exit = run() + if exit: + sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py new file mode 100644 index 000000000..1eab34c06 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py @@ -0,0 +1,479 @@ +import sys +import shutil +import os +import stat +import re +import posixpath +import pkg_resources +import zipfile +import tarfile +from pip.exceptions import InstallationError +from pip.backwardcompat import WindowsError +from pip.locations import site_packages, running_under_virtualenv +from pip.log import logger + +__all__ = ['rmtree', 'display_path', 'backup_dir', + 'find_command', 'ask', 'Inf', + 'normalize_name', 'splitext', + 'format_size', 'is_installable_dir', + 'is_svn_page', 'file_contents', + 'split_leading_dir', 'has_leading_dir', + 'make_path_relative', 'normalize_path', + 'renames', 'get_terminal_size', + 'unzip_file', 'untar_file', 'create_download_cache_folder', + 'cache_download', 'unpack_file'] + + +def rmtree(dir): + shutil.rmtree(dir, ignore_errors=True, + onerror=rmtree_errorhandler) + + +def rmtree_errorhandler(func, path, exc_info): + """On Windows, the files in .svn are read-only, so when rmtree() tries to + remove them, an exception is thrown. We catch that here, remove the + read-only attribute, and hopefully continue without problems.""" + exctype, value = exc_info[:2] + # lookin for a windows error + if exctype is not WindowsError or 'Access is denied' not in str(value): + raise + # file type should currently be read only + if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD): + raise + # convert to read/write + os.chmod(path, stat.S_IWRITE) + # use the original function to repeat the operation + func(path) + + +def display_path(path): + """Gives the display value for a given path, making it relative to cwd + if possible.""" + path = os.path.normcase(os.path.abspath(path)) + if path.startswith(os.getcwd() + os.path.sep): + path = '.' + path[len(os.getcwd()):] + return path + + +def backup_dir(dir, ext='.bak'): + """Figure out the name of a directory to back up the given dir to + (adding .bak, .bak2, etc)""" + n = 1 + extension = ext + while os.path.exists(dir + extension): + n += 1 + extension = ext + str(n) + return dir + extension + + +def find_command(cmd, paths=None, pathext=None): + """Searches the PATH for the given command and returns its path""" + if paths is None: + paths = os.environ.get('PATH', []).split(os.pathsep) + if isinstance(paths, basestring): + paths = [paths] + # check if there are funny path extensions for executables, e.g. Windows + if pathext is None: + pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD') + pathext = [ext for ext in pathext.lower().split(os.pathsep)] + # don't use extensions if the command ends with one of them + if os.path.splitext(cmd)[1].lower() in pathext: + pathext = [''] + # check if we find the command on PATH + for path in paths: + # try without extension first + cmd_path = os.path.join(path, cmd) + for ext in pathext: + # then including the extension + cmd_path_ext = cmd_path + ext + if os.path.exists(cmd_path_ext): + return cmd_path_ext + if os.path.exists(cmd_path): + return cmd_path + return None + + +def ask(message, options): + """Ask the message interactively, with the given possible responses""" + while 1: + if os.environ.get('PIP_NO_INPUT'): + raise Exception('No input was expected ($PIP_NO_INPUT set); question: %s' % message) + response = raw_input(message) + response = response.strip().lower() + if response not in options: + print 'Your response (%r) was not one of the expected responses: %s' % ( + response, ', '.join(options)) + else: + return response + + +class _Inf(object): + """I am bigger than everything!""" + def __cmp__(self, a): + if self is a: + return 0 + return 1 + + def __repr__(self): + return 'Inf' + +Inf = _Inf() +del _Inf + + +_normalize_re = re.compile(r'[^a-z]', re.I) + + +def normalize_name(name): + return _normalize_re.sub('-', name.lower()) + + +def format_size(bytes): + if bytes > 1000*1000: + return '%.1fMb' % (bytes/1000.0/1000) + elif bytes > 10*1000: + return '%iKb' % (bytes/1000) + elif bytes > 1000: + return '%.1fKb' % (bytes/1000.0) + else: + return '%ibytes' % bytes + + +def is_installable_dir(path): + """Return True if `path` is a directory containing a setup.py file.""" + if not os.path.isdir(path): + return False + setup_py = os.path.join(path, 'setup.py') + if os.path.isfile(setup_py): + return True + return False + + +def is_svn_page(html): + """Returns true if the page appears to be the index page of an svn repository""" + return (re.search(r'[^<]*Revision \d+:', html) + and re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I)) + + +def file_contents(filename): + fp = open(filename, 'rb') + try: + return fp.read() + finally: + fp.close() + + +def split_leading_dir(path): + path = str(path) + path = path.lstrip('/').lstrip('\\') + if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) + or '\\' not in path): + return path.split('/', 1) + elif '\\' in path: + return path.split('\\', 1) + else: + return path, '' + + +def has_leading_dir(paths): + """Returns true if all the paths have the same leading path name + (i.e., everything is in one subdirectory in an archive)""" + common_prefix = None + for path in paths: + prefix, rest = split_leading_dir(path) + if not prefix: + return False + elif common_prefix is None: + common_prefix = prefix + elif prefix != common_prefix: + return False + return True + + +def make_path_relative(path, rel_to): + """ + Make a filename relative, where the filename path, and it is + relative to rel_to + + >>> make_relative_path('/usr/share/something/a-file.pth', + ... '/usr/share/another-place/src/Directory') + '../../../something/a-file.pth' + >>> make_relative_path('/usr/share/something/a-file.pth', + ... '/home/user/src/Directory') + '../../../usr/share/something/a-file.pth' + >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/') + 'a-file.pth' + """ + path_filename = os.path.basename(path) + path = os.path.dirname(path) + path = os.path.normpath(os.path.abspath(path)) + rel_to = os.path.normpath(os.path.abspath(rel_to)) + path_parts = path.strip(os.path.sep).split(os.path.sep) + rel_to_parts = rel_to.strip(os.path.sep).split(os.path.sep) + while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]: + path_parts.pop(0) + rel_to_parts.pop(0) + full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename] + if full_parts == ['']: + return '.' + os.path.sep + return os.path.sep.join(full_parts) + + +def normalize_path(path): + """ + Convert a path to its canonical, case-normalized, absolute version. + + """ + return os.path.normcase(os.path.realpath(path)) + + +def splitext(path): + """Like os.path.splitext, but take off .tar too""" + base, ext = posixpath.splitext(path) + if base.lower().endswith('.tar'): + ext = base[-4:] + ext + base = base[:-4] + return base, ext + + +def renames(old, new): + """Like os.renames(), but handles renaming across devices.""" + # Implementation borrowed from os.renames(). + head, tail = os.path.split(new) + if head and tail and not os.path.exists(head): + os.makedirs(head) + + shutil.move(old, new) + + head, tail = os.path.split(old) + if head and tail: + try: + os.removedirs(head) + except OSError: + pass + + +def is_local(path): + """ + Return True if path is within sys.prefix, if we're running in a virtualenv. + + If we're not in a virtualenv, all paths are considered "local." + + """ + if not running_under_virtualenv(): + return True + return normalize_path(path).startswith(normalize_path(sys.prefix)) + + +def dist_is_local(dist): + """ + Return True if given Distribution object is installed locally + (i.e. within current virtualenv). + + Always True if we're not in a virtualenv. + + """ + return is_local(dist_location(dist)) + + +def get_installed_distributions(local_only=True, skip=('setuptools', 'pip', 'python')): + """ + Return a list of installed Distribution objects. + + If ``local_only`` is True (default), only return installations + local to the current virtualenv, if in a virtualenv. + + ``skip`` argument is an iterable of lower-case project names to + ignore; defaults to ('setuptools', 'pip', 'python'). [FIXME also + skip virtualenv?] + + """ + if local_only: + local_test = dist_is_local + else: + local_test = lambda d: True + return [d for d in pkg_resources.working_set if local_test(d) and d.key not in skip] + + +def egg_link_path(dist): + """ + Return the path where we'd expect to find a .egg-link file for + this distribution. (There doesn't seem to be any metadata in the + Distribution object for a develop egg that points back to its + .egg-link and easy-install.pth files). + + This won't find a globally-installed develop egg if we're in a + virtualenv. + + """ + return os.path.join(site_packages, dist.project_name) + '.egg-link' + + +def dist_location(dist): + """ + Get the site-packages location of this distribution. Generally + this is dist.location, except in the case of develop-installed + packages, where dist.location is the source code location, and we + want to know where the egg-link file is. + + """ + egg_link = egg_link_path(dist) + if os.path.exists(egg_link): + return egg_link + return dist.location + + +def get_terminal_size(): + """Returns a tuple (x, y) representing the width(x) and the height(x) + in characters of the terminal window.""" + def ioctl_GWINSZ(fd): + try: + import fcntl + import termios + import struct + cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, + '1234')) + except: + return None + if cr == (0, 0): + return None + if cr == (0, 0): + return None + return cr + cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + if not cr: + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + cr = ioctl_GWINSZ(fd) + os.close(fd) + except: + pass + if not cr: + cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) + return int(cr[1]), int(cr[0]) + + +def unzip_file(filename, location, flatten=True): + """Unzip the file (zip file located at filename) to the destination + location""" + if not os.path.exists(location): + os.makedirs(location) + zipfp = open(filename, 'rb') + try: + zip = zipfile.ZipFile(zipfp) + leading = has_leading_dir(zip.namelist()) and flatten + for name in zip.namelist(): + data = zip.read(name) + fn = name + if leading: + fn = split_leading_dir(name)[1] + fn = os.path.join(location, fn) + dir = os.path.dirname(fn) + if not os.path.exists(dir): + os.makedirs(dir) + if fn.endswith('/') or fn.endswith('\\'): + # A directory + if not os.path.exists(fn): + os.makedirs(fn) + else: + fp = open(fn, 'wb') + try: + fp.write(data) + finally: + fp.close() + finally: + zipfp.close() + + +def untar_file(filename, location): + """Untar the file (tar file located at filename) to the destination location""" + if not os.path.exists(location): + os.makedirs(location) + if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): + mode = 'r:gz' + elif filename.lower().endswith('.bz2') or filename.lower().endswith('.tbz'): + mode = 'r:bz2' + elif filename.lower().endswith('.tar'): + mode = 'r' + else: + logger.warn('Cannot determine compression type for file %s' % filename) + mode = 'r:*' + tar = tarfile.open(filename, mode) + try: + # note: python<=2.5 doesnt seem to know about pax headers, filter them + leading = has_leading_dir([ + member.name for member in tar.getmembers() + if member.name != 'pax_global_header' + ]) + for member in tar.getmembers(): + fn = member.name + if fn == 'pax_global_header': + continue + if leading: + fn = split_leading_dir(fn)[1] + path = os.path.join(location, fn) + if member.isdir(): + if not os.path.exists(path): + os.makedirs(path) + else: + try: + fp = tar.extractfile(member) + except (KeyError, AttributeError), e: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warn( + 'In the tar file %s the member %s is invalid: %s' + % (filename, member.name, e)) + continue + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + destfp = open(path, 'wb') + try: + shutil.copyfileobj(fp, destfp) + finally: + destfp.close() + fp.close() + finally: + tar.close() + + +def create_download_cache_folder(folder): + logger.indent -= 2 + logger.notify('Creating supposed download cache at %s' % folder) + logger.indent += 2 + os.makedirs(folder) + + +def cache_download(target_file, temp_location, content_type): + logger.notify('Storing download in cache at %s' % display_path(target_file)) + shutil.copyfile(temp_location, target_file) + fp = open(target_file+'.content-type', 'w') + fp.write(content_type) + fp.close() + os.unlink(temp_location) + + +def unpack_file(filename, location, content_type, link): + if (content_type == 'application/zip' + or filename.endswith('.zip') + or filename.endswith('.pybundle') + or zipfile.is_zipfile(filename)): + unzip_file(filename, location, flatten=not filename.endswith('.pybundle')) + elif (content_type == 'application/x-gzip' + or tarfile.is_tarfile(filename) + or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')): + untar_file(filename, location) + elif (content_type and content_type.startswith('text/html') + and is_svn_page(file_contents(filename))): + # We don't really care about this + from pip.vcs.subversion import Subversion + Subversion('svn+' + link.url).unpack(location) + else: + ## FIXME: handle? + ## FIXME: magic signatures? + logger.fatal('Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format' + % (filename, location, content_type)) + raise InstallationError('Cannot determine archive format of %s' % location) + + + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py new file mode 100644 index 000000000..e110440c7 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py @@ -0,0 +1,238 @@ +"""Handles all VCS (version control) support""" + +import os +import shutil +import urlparse +import urllib + +from pip.exceptions import BadCommand +from pip.log import logger +from pip.util import display_path, backup_dir, find_command, ask + + +__all__ = ['vcs', 'get_src_requirement', 'import_vcs_support'] + + +class VcsSupport(object): + _registry = {} + schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp'] + + def __init__(self): + # Register more schemes with urlparse for various version control systems + urlparse.uses_netloc.extend(self.schemes) + urlparse.uses_fragment.extend(self.schemes) + super(VcsSupport, self).__init__() + + def __iter__(self): + return self._registry.__iter__() + + @property + def backends(self): + return self._registry.values() + + @property + def dirnames(self): + return [backend.dirname for backend in self.backends] + + @property + def all_schemes(self): + schemes = [] + for backend in self.backends: + schemes.extend(backend.schemes) + return schemes + + def register(self, cls): + if not hasattr(cls, 'name'): + logger.warn('Cannot register VCS %s' % cls.__name__) + return + if cls.name not in self._registry: + self._registry[cls.name] = cls + + def unregister(self, cls=None, name=None): + if name in self._registry: + del self._registry[name] + elif cls in self._registry.values(): + del self._registry[cls.name] + else: + logger.warn('Cannot unregister because no class or name given') + + def get_backend_name(self, location): + """ + Return the name of the version control backend if found at given + location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') + """ + for vc_type in self._registry.values(): + path = os.path.join(location, vc_type.dirname) + if os.path.exists(path): + return vc_type.name + return None + + def get_backend(self, name): + name = name.lower() + if name in self._registry: + return self._registry[name] + + def get_backend_from_location(self, location): + vc_type = self.get_backend_name(location) + if vc_type: + return self.get_backend(vc_type) + return None + + +vcs = VcsSupport() + + +class VersionControl(object): + name = '' + dirname = '' + + def __init__(self, url=None, *args, **kwargs): + self.url = url + self._cmd = None + super(VersionControl, self).__init__(*args, **kwargs) + + def _filter(self, line): + return (logger.INFO, line) + + def _is_local_repository(self, repo): + """ + posix absolute paths start with os.path.sep, + win32 ones ones start with drive (like c:\\folder) + """ + drive, tail = os.path.splitdrive(repo) + return repo.startswith(os.path.sep) or drive + + @property + def cmd(self): + if self._cmd is not None: + return self._cmd + command = find_command(self.name) + if command is None: + raise BadCommand('Cannot find command %r' % self.name) + logger.info('Found command %r at %r' % (self.name, command)) + self._cmd = command + return command + + def get_url_rev(self): + """ + Returns the correct repository URL and revision by parsing the given + repository URL + """ + url = self.url.split('+', 1)[1] + scheme, netloc, path, query, frag = urlparse.urlsplit(url) + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + url = urlparse.urlunsplit((scheme, netloc, path, query, '')) + return url, rev + + def get_info(self, location): + """ + Returns (url, revision), where both are strings + """ + assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location + return self.get_url(location), self.get_revision(location) + + def normalize_url(self, url): + """ + Normalize a URL for comparison by unquoting it and removing any trailing slash. + """ + return urllib.unquote(url).rstrip('/') + + def compare_urls(self, url1, url2): + """ + Compare two repo URLs for identity, ignoring incidental differences. + """ + return (self.normalize_url(url1) == self.normalize_url(url2)) + + def parse_vcs_bundle_file(self, content): + """ + Takes the contents of the bundled text file that explains how to revert + the stripped off version control data of the given package and returns + the URL and revision of it. + """ + raise NotImplementedError + + def obtain(self, dest): + """ + Called when installing or updating an editable package, takes the + source path of the checkout. + """ + raise NotImplementedError + + def switch(self, dest, url, rev_options): + """ + Switch the repo at ``dest`` to point to ``URL``. + """ + raise NotImplemented + + def update(self, dest, rev_options): + """ + Update an already-existing repo to the given ``rev_options``. + """ + raise NotImplementedError + + def check_destination(self, dest, url, rev_options, rev_display): + """ + Prepare a location to receive a checkout/clone. + + Return True if the location is ready for (and requires) a + checkout/clone, False otherwise. + """ + checkout = True + prompt = False + if os.path.exists(dest): + checkout = False + if os.path.exists(os.path.join(dest, self.dirname)): + existing_url = self.get_url(dest) + if self.compare_urls(existing_url, url): + logger.info('%s in %s exists, and has correct URL (%s)' + % (self.repo_name.title(), display_path(dest), url)) + logger.notify('Updating %s %s%s' + % (display_path(dest), self.repo_name, rev_display)) + self.update(dest, rev_options) + else: + logger.warn('%s %s in %s exists with URL %s' + % (self.name, self.repo_name, display_path(dest), existing_url)) + prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) + else: + logger.warn('Directory %s already exists, and is not a %s %s.' + % (dest, self.name, self.repo_name)) + prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) + if prompt: + logger.warn('The plan is to install the %s repository %s' + % (self.name, url)) + response = ask('What to do? %s' % prompt[0], prompt[1]) + + if response == 's': + logger.notify('Switching %s %s to %s%s' + % (self.repo_name, display_path(dest), url, rev_display)) + self.switch(dest, url, rev_options) + elif response == 'i': + # do nothing + pass + elif response == 'w': + logger.warn('Deleting %s' % display_path(dest)) + shutil.rmtree(dest) + checkout = True + elif response == 'b': + dest_dir = backup_dir(dest) + logger.warn('Backing up %s to %s' + % (display_path(dest), dest_dir)) + shutil.move(dest, dest_dir) + checkout = True + return checkout + + def unpack(self, location): + raise NotImplementedError + + def get_src_requirement(self, dist, location, find_tags=False): + raise NotImplementedError + + +def get_src_requirement(dist, location, find_tags): + version_control = vcs.get_backend_from_location(location) + if version_control: + return version_control().get_src_requirement(dist, location, find_tags) + logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location) + return dist.as_requirement() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py new file mode 100644 index 000000000..3b6ea8f04 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py @@ -0,0 +1,138 @@ +import os +import shutil +import tempfile +import re +from pip import call_subprocess +from pip.log import logger +from pip.util import rmtree, display_path +from pip.vcs import vcs, VersionControl +from pip.download import path_to_url2 + + +class Bazaar(VersionControl): + name = 'bzr' + dirname = '.bzr' + repo_name = 'branch' + bundle_file = 'bzr-branch.txt' + schemes = ('bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp') + guide = ('# This was a Bazaar branch; to make it a branch again run:\n' + 'bzr branch -r %(rev)s %(url)s .\n') + + def parse_vcs_bundle_file(self, content): + url = rev = None + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + match = re.search(r'^bzr\s*branch\s*-r\s*(\d*)', line) + if match: + rev = match.group(1).strip() + url = line[match.end():].strip().split(None, 1)[0] + if url and rev: + return url, rev + return None, None + + def unpack(self, location): + """Get the bzr branch at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Checking out bzr repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + os.rmdir(location) + call_subprocess( + [self.cmd, 'branch', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the Bazaar repository at the url to the destination location""" + temp_dir = tempfile.mkdtemp('-export', 'pip-') + self.unpack(temp_dir) + if os.path.exists(location): + # Remove the location to make sure Bazaar can export it correctly + rmtree(location) + try: + call_subprocess([self.cmd, 'export', location], cwd=temp_dir, + filter_stdout=self._filter, show_stdout=False) + finally: + shutil.rmtree(temp_dir) + + def switch(self, dest, url, rev_options): + call_subprocess([self.cmd, 'switch', url], cwd=dest) + + def update(self, dest, rev_options): + call_subprocess( + [self.cmd, 'pull', '-q'] + rev_options, cwd=dest) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = ['-r', rev] + rev_display = ' (to revision %s)' % rev + else: + rev_options = [] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Checking out %s%s to %s' + % (url, rev_display, display_path(dest))) + call_subprocess( + [self.cmd, 'branch', '-q'] + rev_options + [url, dest]) + + def get_url_rev(self): + # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it + url, rev = super(Bazaar, self).get_url_rev() + if url.startswith('ssh://'): + url = 'bzr+' + url + return url, rev + + def get_url(self, location): + urls = call_subprocess( + [self.cmd, 'info'], show_stdout=False, cwd=location) + for line in urls.splitlines(): + line = line.strip() + for x in ('checkout of branch: ', + 'parent branch: '): + if line.startswith(x): + repo = line.split(x)[1] + if self._is_local_repository(repo): + return path_to_url2(repo) + return repo + return None + + def get_revision(self, location): + revision = call_subprocess( + [self.cmd, 'revno'], show_stdout=False, cwd=location) + return revision.splitlines()[-1] + + def get_tag_revs(self, location): + tags = call_subprocess( + [self.cmd, 'tags'], show_stdout=False, cwd=location) + tag_revs = [] + for line in tags.splitlines(): + tags_match = re.search(r'([.\w-]+)\s*(.*)$', line) + if tags_match: + tag = tags_match.group(1) + rev = tags_match.group(2) + tag_revs.append((rev.strip(), tag.strip())) + return dict(tag_revs) + + def get_src_requirement(self, dist, location, find_tags): + repo = self.get_url(location) + if not repo.lower().startswith('bzr:'): + repo = 'bzr+' + repo + egg_project_name = dist.egg_name().split('-', 1)[0] + if not repo: + return None + current_rev = self.get_revision(location) + tag_revs = self.get_tag_revs(location) + + if current_rev in tag_revs: + # It's a tag + full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) + else: + full_egg_name = '%s-dev_r%s' % (dist.egg_name(), current_rev) + return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) + + +vcs.register(Bazaar) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py new file mode 100644 index 000000000..0701e49ec --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py @@ -0,0 +1,204 @@ +import os +import shutil +import tempfile +import re +from pip import call_subprocess +from pip.util import display_path +from pip.vcs import vcs, VersionControl +from pip.log import logger +from urllib import url2pathname +from urlparse import urlsplit, urlunsplit + + +class Git(VersionControl): + name = 'git' + dirname = '.git' + repo_name = 'clone' + schemes = ('git', 'git+http', 'git+ssh', 'git+git', 'git+file') + bundle_file = 'git-clone.txt' + guide = ('# This was a Git repo; to make it a repo again run:\n' + 'git init\ngit remote add origin %(url)s -f\ngit checkout %(rev)s\n') + + def __init__(self, url=None, *args, **kwargs): + + # Works around an apparent Git bug + # (see http://article.gmane.org/gmane.comp.version-control.git/146500) + if url: + scheme, netloc, path, query, fragment = urlsplit(url) + if scheme.endswith('file'): + initial_slashes = path[:-len(path.lstrip('/'))] + newpath = initial_slashes + url2pathname(path).replace('\\', '/').lstrip('/') + url = urlunsplit((scheme, netloc, newpath, query, fragment)) + after_plus = scheme.find('+')+1 + url = scheme[:after_plus]+ urlunsplit((scheme[after_plus:], netloc, newpath, query, fragment)) + + super(Git, self).__init__(url, *args, **kwargs) + + def parse_vcs_bundle_file(self, content): + url = rev = None + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + url_match = re.search(r'git\s*remote\s*add\s*origin(.*)\s*-f', line) + if url_match: + url = url_match.group(1).strip() + rev_match = re.search(r'^git\s*checkout\s*-q\s*(.*)\s*', line) + if rev_match: + rev = rev_match.group(1).strip() + if url and rev: + return url, rev + return None, None + + def unpack(self, location): + """Clone the Git repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Cloning Git repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + os.rmdir(location) + call_subprocess( + [self.cmd, 'clone', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the Git repository at the url to the destination location""" + temp_dir = tempfile.mkdtemp('-export', 'pip-') + self.unpack(temp_dir) + try: + if not location.endswith('/'): + location = location + '/' + call_subprocess( + [self.cmd, 'checkout-index', '-a', '-f', '--prefix', location], + filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) + finally: + shutil.rmtree(temp_dir) + + def check_rev_options(self, rev, dest, rev_options): + """Check the revision options before checkout to compensate that tags + and branches may need origin/ as a prefix. + Returns the SHA1 of the branch or tag if found. + """ + revisions = self.get_tag_revs(dest) + revisions.update(self.get_branch_revs(dest)) + inverse_revisions = dict((v, k) for k, v in revisions.iteritems()) + # Check if rev is a branch name + origin_rev = 'origin/%s' % rev + if origin_rev in inverse_revisions: + return [inverse_revisions[origin_rev]] + elif rev in inverse_revisions: + return [inverse_revisions[rev]] + else: + logger.warn("Could not find a tag or branch '%s', assuming commit." % rev) + return rev_options + + def switch(self, dest, url, rev_options): + call_subprocess( + [self.cmd, 'config', 'remote.origin.url', url], cwd=dest) + call_subprocess( + [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) + + def update(self, dest, rev_options): + call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) + call_subprocess( + [self.cmd, 'checkout', '-q', '-f'] + rev_options, cwd=dest) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = [rev] + rev_display = ' (to %s)' % rev + else: + rev_options = ['master'] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest))) + call_subprocess([self.cmd, 'clone', '-q', url, dest]) + if rev: + rev_options = self.check_rev_options(rev, dest, rev_options) + # Only do a checkout if rev_options differs from HEAD + if not self.get_revision(dest).startswith(rev_options[0]): + call_subprocess([self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) + + def get_url(self, location): + url = call_subprocess( + [self.cmd, 'config', 'remote.origin.url'], + show_stdout=False, cwd=location) + return url.strip() + + def get_revision(self, location): + current_rev = call_subprocess( + [self.cmd, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location) + return current_rev.strip() + + def get_tag_revs(self, location): + tags = call_subprocess( + [self.cmd, 'tag', '-l'], + show_stdout=False, raise_on_returncode=False, cwd=location) + tag_revs = [] + for line in tags.splitlines(): + tag = line.strip() + rev = call_subprocess( + [self.cmd, 'rev-parse', tag], show_stdout=False, cwd=location) + tag_revs.append((rev.strip(), tag)) + tag_revs = dict(tag_revs) + return tag_revs + + def get_branch_revs(self, location): + branches = call_subprocess( + [self.cmd, 'branch', '-r'], show_stdout=False, cwd=location) + branch_revs = [] + for line in branches.splitlines(): + line = line.split('->')[0].strip() + branch = "".join([b for b in line.split() if b != '*']) + rev = call_subprocess( + [self.cmd, 'rev-parse', branch], show_stdout=False, cwd=location) + branch_revs.append((rev.strip(), branch)) + branch_revs = dict(branch_revs) + return branch_revs + + def get_src_requirement(self, dist, location, find_tags): + repo = self.get_url(location) + if not repo.lower().startswith('git:'): + repo = 'git+' + repo + egg_project_name = dist.egg_name().split('-', 1)[0] + if not repo: + return None + current_rev = self.get_revision(location) + tag_revs = self.get_tag_revs(location) + branch_revs = self.get_branch_revs(location) + + if current_rev in tag_revs: + # It's a tag + full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) + elif (current_rev in branch_revs and + branch_revs[current_rev] != 'origin/master'): + # It's the head of a branch + full_egg_name = '%s-%s' % (dist.egg_name(), + branch_revs[current_rev].replace('origin/', '')) + else: + full_egg_name = '%s-dev' % dist.egg_name() + + return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) + + def get_url_rev(self): + """ + Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. + That's required because although they use SSH they sometimes doesn't + work with a ssh:// scheme (e.g. Github). But we need a scheme for + parsing. Hence we remove it again afterwards and return it as a stub. + """ + if not '://' in self.url: + assert not 'file:' in self.url + self.url = self.url.replace('git+', 'git+ssh://') + url, rev = super(Git, self).get_url_rev() + url = url.replace('ssh://', '') + else: + url, rev = super(Git, self).get_url_rev() + + return url, rev + + +vcs.register(Git) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py new file mode 100644 index 000000000..70c8c833d --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py @@ -0,0 +1,162 @@ +import os +import shutil +import tempfile +import re +import ConfigParser +from pip import call_subprocess +from pip.util import display_path +from pip.log import logger +from pip.vcs import vcs, VersionControl +from pip.download import path_to_url2 + + +class Mercurial(VersionControl): + name = 'hg' + dirname = '.hg' + repo_name = 'clone' + schemes = ('hg', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http') + bundle_file = 'hg-clone.txt' + guide = ('# This was a Mercurial repo; to make it a repo again run:\n' + 'hg init\nhg pull %(url)s\nhg update -r %(rev)s\n') + + def parse_vcs_bundle_file(self, content): + url = rev = None + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + url_match = re.search(r'hg\s*pull\s*(.*)\s*', line) + if url_match: + url = url_match.group(1).strip() + rev_match = re.search(r'^hg\s*update\s*-r\s*(.*)\s*', line) + if rev_match: + rev = rev_match.group(1).strip() + if url and rev: + return url, rev + return None, None + + def unpack(self, location): + """Clone the Hg repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Cloning Mercurial repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + os.rmdir(location) + call_subprocess( + [self.cmd, 'clone', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the Hg repository at the url to the destination location""" + temp_dir = tempfile.mkdtemp('-export', 'pip-') + self.unpack(temp_dir) + try: + call_subprocess( + [self.cmd, 'archive', location], + filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) + finally: + shutil.rmtree(temp_dir) + + def switch(self, dest, url, rev_options): + repo_config = os.path.join(dest, self.dirname, 'hgrc') + config = ConfigParser.SafeConfigParser() + try: + config.read(repo_config) + config.set('paths', 'default', url) + config_file = open(repo_config, 'w') + config.write(config_file) + config_file.close() + except (OSError, ConfigParser.NoSectionError), e: + logger.warn( + 'Could not switch Mercurial repository to %s: %s' + % (url, e)) + else: + call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) + + def update(self, dest, rev_options): + call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) + call_subprocess( + [self.cmd, 'update', '-q'] + rev_options, cwd=dest) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = [rev] + rev_display = ' (to revision %s)' % rev + else: + rev_options = [] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Cloning hg %s%s to %s' + % (url, rev_display, display_path(dest))) + call_subprocess([self.cmd, 'clone', '--noupdate', '-q', url, dest]) + call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) + + def get_url(self, location): + url = call_subprocess( + [self.cmd, 'showconfig', 'paths.default'], + show_stdout=False, cwd=location).strip() + if self._is_local_repository(url): + url = path_to_url2(url) + return url.strip() + + def get_tag_revs(self, location): + tags = call_subprocess( + [self.cmd, 'tags'], show_stdout=False, cwd=location) + tag_revs = [] + for line in tags.splitlines(): + tags_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) + if tags_match: + tag = tags_match.group(1) + rev = tags_match.group(2) + tag_revs.append((rev.strip(), tag.strip())) + return dict(tag_revs) + + def get_branch_revs(self, location): + branches = call_subprocess( + [self.cmd, 'branches'], show_stdout=False, cwd=location) + branch_revs = [] + for line in branches.splitlines(): + branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) + if branches_match: + branch = branches_match.group(1) + rev = branches_match.group(2) + branch_revs.append((rev.strip(), branch.strip())) + return dict(branch_revs) + + def get_revision(self, location): + current_revision = call_subprocess( + [self.cmd, 'parents', '--template={rev}'], + show_stdout=False, cwd=location).strip() + return current_revision + + def get_revision_hash(self, location): + current_rev_hash = call_subprocess( + [self.cmd, 'parents', '--template={node}'], + show_stdout=False, cwd=location).strip() + return current_rev_hash + + def get_src_requirement(self, dist, location, find_tags): + repo = self.get_url(location) + if not repo.lower().startswith('hg:'): + repo = 'hg+' + repo + egg_project_name = dist.egg_name().split('-', 1)[0] + if not repo: + return None + current_rev = self.get_revision(location) + current_rev_hash = self.get_revision_hash(location) + tag_revs = self.get_tag_revs(location) + branch_revs = self.get_branch_revs(location) + if current_rev in tag_revs: + # It's a tag + full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) + elif current_rev in branch_revs: + # It's the tip of a branch + full_egg_name = '%s-%s' % (dist.egg_name(), branch_revs[current_rev]) + else: + full_egg_name = '%s-dev' % dist.egg_name() + return '%s@%s#egg=%s' % (repo, current_rev_hash, full_egg_name) + +vcs.register(Mercurial) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py new file mode 100644 index 000000000..85715d97a --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py @@ -0,0 +1,260 @@ +import os +import re +from pip import call_subprocess +from pip.index import Link +from pip.util import rmtree, display_path +from pip.log import logger +from pip.vcs import vcs, VersionControl + +_svn_xml_url_re = re.compile('url="([^"]+)"') +_svn_rev_re = re.compile('committed-rev="(\d+)"') +_svn_url_re = re.compile(r'URL: (.+)') +_svn_revision_re = re.compile(r'Revision: (.+)') + + +class Subversion(VersionControl): + name = 'svn' + dirname = '.svn' + repo_name = 'checkout' + schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https') + bundle_file = 'svn-checkout.txt' + guide = ('# This was an svn checkout; to make it a checkout again run:\n' + 'svn checkout --force -r %(rev)s %(url)s .\n') + + def get_info(self, location): + """Returns (url, revision), where both are strings""" + assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location + output = call_subprocess( + [self.cmd, 'info', location], show_stdout=False, extra_environ={'LANG': 'C'}) + match = _svn_url_re.search(output) + if not match: + logger.warn('Cannot determine URL of svn checkout %s' % display_path(location)) + logger.info('Output that cannot be parsed: \n%s' % output) + return None, None + url = match.group(1).strip() + match = _svn_revision_re.search(output) + if not match: + logger.warn('Cannot determine revision of svn checkout %s' % display_path(location)) + logger.info('Output that cannot be parsed: \n%s' % output) + return url, None + return url, match.group(1) + + def parse_vcs_bundle_file(self, content): + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + match = re.search(r'^-r\s*([^ ])?', line) + if not match: + return None, None + rev = match.group(1) + rest = line[match.end():].strip().split(None, 1)[0] + return rest, rev + return None, None + + def unpack(self, location): + """Check out the svn repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Checking out svn repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + # Subversion doesn't like to check out over an existing directory + # --force fixes this, but was only added in svn 1.5 + rmtree(location) + call_subprocess( + [self.cmd, 'checkout', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the svn repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Exporting svn repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + # Subversion doesn't like to check out over an existing directory + # --force fixes this, but was only added in svn 1.5 + rmtree(location) + call_subprocess( + [self.cmd, 'export', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def switch(self, dest, url, rev_options): + call_subprocess( + [self.cmd, 'switch'] + rev_options + [url, dest]) + + def update(self, dest, rev_options): + call_subprocess( + [self.cmd, 'update'] + rev_options + [dest]) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = ['-r', rev] + rev_display = ' (to revision %s)' % rev + else: + rev_options = [] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Checking out %s%s to %s' + % (url, rev_display, display_path(dest))) + call_subprocess( + [self.cmd, 'checkout', '-q'] + rev_options + [url, dest]) + + def get_location(self, dist, dependency_links): + for url in dependency_links: + egg_fragment = Link(url).egg_fragment + if not egg_fragment: + continue + if '-' in egg_fragment: + ## FIXME: will this work when a package has - in the name? + key = '-'.join(egg_fragment.split('-')[:-1]).lower() + else: + key = egg_fragment + if key == dist.key: + return url.split('#', 1)[0] + return None + + def get_revision(self, location): + """ + Return the maximum revision for all files under a given location + """ + # Note: taken from setuptools.command.egg_info + revision = 0 + + for base, dirs, files in os.walk(location): + if self.dirname not in dirs: + dirs[:] = [] + continue # no sense walking uncontrolled subdirs + dirs.remove(self.dirname) + entries_fn = os.path.join(base, self.dirname, 'entries') + if not os.path.exists(entries_fn): + ## FIXME: should we warn? + continue + f = open(entries_fn) + data = f.read() + f.close() + + if data.startswith('8') or data.startswith('9') or data.startswith('10'): + data = map(str.splitlines, data.split('\n\x0c\n')) + del data[0][0] # get rid of the '8' + dirurl = data[0][3] + revs = [int(d[9]) for d in data if len(d)>9 and d[9]]+[0] + if revs: + localrev = max(revs) + else: + localrev = 0 + elif data.startswith('<?xml'): + dirurl = _svn_xml_url_re.search(data).group(1) # get repository URL + revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)]+[0] + if revs: + localrev = max(revs) + else: + localrev = 0 + else: + logger.warn("Unrecognized .svn/entries format; skipping %s", base) + dirs[:] = [] + continue + if base == location: + base_url = dirurl+'/' # save the root url + elif not dirurl.startswith(base_url): + dirs[:] = [] + continue # not part of the same svn tree, skip it + revision = max(revision, localrev) + return revision + + def get_url_rev(self): + # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it + url, rev = super(Subversion, self).get_url_rev() + if url.startswith('ssh://'): + url = 'svn+' + url + return url, rev + + def get_url(self, location): + # In cases where the source is in a subdirectory, not alongside setup.py + # we have to look up in the location until we find a real setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without finding setup.py + logger.warn("Could not find setup.py for directory %s (tried all parent directories)" + % orig_location) + return None + f = open(os.path.join(location, self.dirname, 'entries')) + data = f.read() + f.close() + if data.startswith('8') or data.startswith('9') or data.startswith('10'): + data = map(str.splitlines, data.split('\n\x0c\n')) + del data[0][0] # get rid of the '8' + return data[0][3] + elif data.startswith('<?xml'): + match = _svn_xml_url_re.search(data) + if not match: + raise ValueError('Badly formatted data: %r' % data) + return match.group(1) # get repository URL + else: + logger.warn("Unrecognized .svn/entries format in %s" % location) + # Or raise exception? + return None + + def get_tag_revs(self, svn_tag_url): + stdout = call_subprocess( + [self.cmd, 'ls', '-v', svn_tag_url], show_stdout=False) + results = [] + for line in stdout.splitlines(): + parts = line.split() + rev = int(parts[0]) + tag = parts[-1].strip('/') + results.append((tag, rev)) + return results + + def find_tag_match(self, rev, tag_revs): + best_match_rev = None + best_tag = None + for tag, tag_rev in tag_revs: + if (tag_rev > rev and + (best_match_rev is None or best_match_rev > tag_rev)): + # FIXME: Is best_match > tag_rev really possible? + # or is it a sign something is wacky? + best_match_rev = tag_rev + best_tag = tag + return best_tag + + def get_src_requirement(self, dist, location, find_tags=False): + repo = self.get_url(location) + if repo is None: + return None + parts = repo.split('/') + ## FIXME: why not project name? + egg_project_name = dist.egg_name().split('-', 1)[0] + rev = self.get_revision(location) + if parts[-2] in ('tags', 'tag'): + # It's a tag, perfect! + full_egg_name = '%s-%s' % (egg_project_name, parts[-1]) + elif parts[-2] in ('branches', 'branch'): + # It's a branch :( + full_egg_name = '%s-%s-r%s' % (dist.egg_name(), parts[-1], rev) + elif parts[-1] == 'trunk': + # Trunk :-/ + full_egg_name = '%s-dev_r%s' % (dist.egg_name(), rev) + if find_tags: + tag_url = '/'.join(parts[:-1]) + '/tags' + tag_revs = self.get_tag_revs(tag_url) + match = self.find_tag_match(rev, tag_revs) + if match: + logger.notify('trunk checkout %s seems to be equivalent to tag %s' % match) + repo = '%s/%s' % (tag_url, match) + full_egg_name = '%s-%s' % (egg_project_name, match) + else: + # Don't know what it is + logger.warn('svn URL does not fit normal structure (tags/branches/trunk): %s' % repo) + full_egg_name = '%s-dev_r%s' % (egg_project_name, rev) + return 'svn+%s@%s#egg=%s' % (repo, rev, full_egg_name) + +vcs.register(Subversion) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py new file mode 100644 index 000000000..708abb05b --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py @@ -0,0 +1,53 @@ +"""Tools for working with virtualenv environments""" + +import os +import sys +import subprocess +from pip.exceptions import BadCommand +from pip.log import logger + + +def restart_in_venv(venv, base, site_packages, args): + """ + Restart this script using the interpreter in the given virtual environment + """ + if base and not os.path.isabs(venv) and not venv.startswith('~'): + base = os.path.expanduser(base) + # ensure we have an abs basepath at this point: + # a relative one makes no sense (or does it?) + if os.path.isabs(base): + venv = os.path.join(base, venv) + + if venv.startswith('~'): + venv = os.path.expanduser(venv) + + if not os.path.exists(venv): + try: + import virtualenv + except ImportError: + print 'The virtual environment does not exist: %s' % venv + print 'and virtualenv is not installed, so a new environment cannot be created' + sys.exit(3) + print 'Creating new virtualenv environment in %s' % venv + virtualenv.logger = logger + logger.indent += 2 + virtualenv.create_environment(venv, site_packages=site_packages) + if sys.platform == 'win32': + python = os.path.join(venv, 'Scripts', 'python.exe') + # check for bin directory which is used in buildouts + if not os.path.exists(python): + python = os.path.join(venv, 'bin', 'python.exe') + else: + python = os.path.join(venv, 'bin', 'python') + if not os.path.exists(python): + python = venv + if not os.path.exists(python): + raise BadCommand('Cannot find virtual environment interpreter at %s' % python) + base = os.path.dirname(os.path.dirname(python)) + file = os.path.join(os.path.dirname(__file__), 'runner.py') + if file.endswith('.pyc'): + file = file[:-1] + proc = subprocess.Popen( + [python, file] + args + [base, '___VENV_RESTART___']) + proc.wait() + sys.exit(proc.returncode) diff --git a/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg b/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg new file mode 100644 index 000000000..3c72d15b5 Binary files /dev/null and b/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg differ diff --git a/test/lib/python2.6/site-packages/setuptools.pth b/test/lib/python2.6/site-packages/setuptools.pth new file mode 100644 index 000000000..bf7602056 --- /dev/null +++ b/test/lib/python2.6/site-packages/setuptools.pth @@ -0,0 +1 @@ +./setuptools-0.6c11-py2.6.egg diff --git a/test/lib/python2.6/site.py b/test/lib/python2.6/site.py new file mode 100644 index 000000000..a49cfc349 --- /dev/null +++ b/test/lib/python2.6/site.py @@ -0,0 +1,713 @@ +"""Append module search paths for third-party packages to sys.path. + +**************************************************************** +* This module is automatically imported during initialization. * +**************************************************************** + +In earlier versions of Python (up to 1.5a3), scripts or modules that +needed to use site-specific modules would place ``import site'' +somewhere near the top of their code. Because of the automatic +import, this is no longer necessary (but code that does it still +works). + +This will append site-specific paths to the module search path. On +Unix, it starts with sys.prefix and sys.exec_prefix (if different) and +appends lib/python<version>/site-packages as well as lib/site-python. +It also supports the Debian convention of +lib/python<version>/dist-packages. On other platforms (mainly Mac and +Windows), it uses just sys.prefix (and sys.exec_prefix, if different, +but this is unlikely). The resulting directories, if they exist, are +appended to sys.path, and also inspected for path configuration files. + +FOR DEBIAN, this sys.path is augmented with directories in /usr/local. +Local addons go into /usr/local/lib/python<version>/site-packages +(resp. /usr/local/lib/site-python), Debian addons install into +/usr/{lib,share}/python<version>/dist-packages. + +A path configuration file is a file whose name has the form +<package>.pth; its contents are additional directories (one per line) +to be added to sys.path. Non-existing directories (or +non-directories) are never added to sys.path; no directory is added to +sys.path more than once. Blank lines and lines beginning with +'#' are skipped. Lines starting with 'import' are executed. + +For example, suppose sys.prefix and sys.exec_prefix are set to +/usr/local and there is a directory /usr/local/lib/python2.X/site-packages +with three subdirectories, foo, bar and spam, and two path +configuration files, foo.pth and bar.pth. Assume foo.pth contains the +following: + + # foo package configuration + foo + bar + bletch + +and bar.pth contains: + + # bar package configuration + bar + +Then the following directories are added to sys.path, in this order: + + /usr/local/lib/python2.X/site-packages/bar + /usr/local/lib/python2.X/site-packages/foo + +Note that bletch is omitted because it doesn't exist; bar precedes foo +because bar.pth comes alphabetically before foo.pth; and spam is +omitted because it is not mentioned in either path configuration file. + +After these path manipulations, an attempt is made to import a module +named sitecustomize, which can perform arbitrary additional +site-specific customizations. If this import fails with an +ImportError exception, it is silently ignored. + +""" + +import sys +import os +import __builtin__ +try: + set +except NameError: + from sets import Set as set + +# Prefixes for site-packages; add additional prefixes like /usr/local here +PREFIXES = [sys.prefix, sys.exec_prefix] +# Enable per user site-packages directory +# set it to False to disable the feature or True to force the feature +ENABLE_USER_SITE = None +# for distutils.commands.install +USER_SITE = None +USER_BASE = None + +_is_pypy = hasattr(sys, 'pypy_version_info') +_is_jython = sys.platform[:4] == 'java' +if _is_jython: + ModuleType = type(os) + +def makepath(*paths): + dir = os.path.join(*paths) + if _is_jython and (dir == '__classpath__' or + dir.startswith('__pyclasspath__')): + return dir, dir + dir = os.path.abspath(dir) + return dir, os.path.normcase(dir) + +def abs__file__(): + """Set all module' __file__ attribute to an absolute path""" + for m in sys.modules.values(): + if ((_is_jython and not isinstance(m, ModuleType)) or + hasattr(m, '__loader__')): + # only modules need the abspath in Jython. and don't mess + # with a PEP 302-supplied __file__ + continue + f = getattr(m, '__file__', None) + if f is None: + continue + m.__file__ = os.path.abspath(f) + +def removeduppaths(): + """ Remove duplicate entries from sys.path along with making them + absolute""" + # This ensures that the initial path provided by the interpreter contains + # only absolute pathnames, even if we're running from the build directory. + L = [] + known_paths = set() + for dir in sys.path: + # Filter out duplicate paths (on case-insensitive file systems also + # if they only differ in case); turn relative paths into absolute + # paths. + dir, dircase = makepath(dir) + if not dircase in known_paths: + L.append(dir) + known_paths.add(dircase) + sys.path[:] = L + return known_paths + +# XXX This should not be part of site.py, since it is needed even when +# using the -S option for Python. See http://www.python.org/sf/586680 +def addbuilddir(): + """Append ./build/lib.<platform> in case we're running in the build dir + (especially for Guido :-)""" + from distutils.util import get_platform + s = "build/lib.%s-%.3s" % (get_platform(), sys.version) + if hasattr(sys, 'gettotalrefcount'): + s += '-pydebug' + s = os.path.join(os.path.dirname(sys.path[-1]), s) + sys.path.append(s) + +def _init_pathinfo(): + """Return a set containing all existing directory entries from sys.path""" + d = set() + for dir in sys.path: + try: + if os.path.isdir(dir): + dir, dircase = makepath(dir) + d.add(dircase) + except TypeError: + continue + return d + +def addpackage(sitedir, name, known_paths): + """Add a new path to known_paths by combining sitedir and 'name' or execute + sitedir if it starts with 'import'""" + if known_paths is None: + _init_pathinfo() + reset = 1 + else: + reset = 0 + fullname = os.path.join(sitedir, name) + try: + f = open(fullname, "rU") + except IOError: + return + try: + for line in f: + if line.startswith("#"): + continue + if line.startswith("import"): + exec line + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + finally: + f.close() + if reset: + known_paths = None + return known_paths + +def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" + if known_paths is None: + known_paths = _init_pathinfo() + reset = 1 + else: + reset = 0 + sitedir, sitedircase = makepath(sitedir) + if not sitedircase in known_paths: + sys.path.append(sitedir) # Add path component + try: + names = os.listdir(sitedir) + except os.error: + return + names.sort() + for name in names: + if name.endswith(os.extsep + "pth"): + addpackage(sitedir, name, known_paths) + if reset: + known_paths = None + return known_paths + +def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix): + """Add site-packages (and possibly site-python) to sys.path""" + prefixes = [os.path.join(sys_prefix, "local"), sys_prefix] + if exec_prefix != sys_prefix: + prefixes.append(os.path.join(exec_prefix, "local")) + + for prefix in prefixes: + if prefix: + if sys.platform in ('os2emx', 'riscos') or _is_jython: + sitedirs = [os.path.join(prefix, "Lib", "site-packages")] + elif _is_pypy: + sitedirs = [os.path.join(prefix, 'site-packages')] + elif sys.platform == 'darwin' and prefix == sys_prefix: + + if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python + + sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"), + os.path.join(prefix, "Extras", "lib", "python")] + + else: # any other Python distros on OSX work this way + sitedirs = [os.path.join(prefix, "lib", + "python" + sys.version[:3], "site-packages")] + + elif os.sep == '/': + sitedirs = [os.path.join(prefix, + "lib", + "python" + sys.version[:3], + "site-packages"), + os.path.join(prefix, "lib", "site-python"), + os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")] + lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages") + if (os.path.exists(lib64_dir) and + os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]): + sitedirs.append(lib64_dir) + try: + # sys.getobjects only available in --with-pydebug build + sys.getobjects + sitedirs.insert(0, os.path.join(sitedirs[0], 'debug')) + except AttributeError: + pass + # Debian-specific dist-packages directories: + sitedirs.append(os.path.join(prefix, "lib", + "python" + sys.version[:3], + "dist-packages")) + sitedirs.append(os.path.join(prefix, "local/lib", + "python" + sys.version[:3], + "dist-packages")) + sitedirs.append(os.path.join(prefix, "lib", "dist-python")) + else: + sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")] + if sys.platform == 'darwin': + # for framework builds *only* we add the standard Apple + # locations. Currently only per-user, but /Library and + # /Network/Library could be added too + if 'Python.framework' in prefix: + home = os.environ.get('HOME') + if home: + sitedirs.append( + os.path.join(home, + 'Library', + 'Python', + sys.version[:3], + 'site-packages')) + for sitedir in sitedirs: + if os.path.isdir(sitedir): + addsitedir(sitedir, known_paths) + return None + +def check_enableusersite(): + """Check if user site directory is safe for inclusion + + The function tests for the command line flag (including environment var), + process uid/gid equal to effective uid/gid. + + None: Disabled for security reasons + False: Disabled by user (command line option) + True: Safe and enabled + """ + if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False): + return False + + if hasattr(os, "getuid") and hasattr(os, "geteuid"): + # check process uid == effective uid + if os.geteuid() != os.getuid(): + return None + if hasattr(os, "getgid") and hasattr(os, "getegid"): + # check process gid == effective gid + if os.getegid() != os.getgid(): + return None + + return True + +def addusersitepackages(known_paths): + """Add a per user site-package to sys.path + + Each user has its own python directory with site-packages in the + home directory. + + USER_BASE is the root directory for all Python versions + + USER_SITE is the user specific site-packages directory + + USER_SITE/.. can be used for data. + """ + global USER_BASE, USER_SITE, ENABLE_USER_SITE + env_base = os.environ.get("PYTHONUSERBASE", None) + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + #if sys.platform in ('os2emx', 'riscos'): + # # Don't know what to put here + # USER_BASE = '' + # USER_SITE = '' + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser(base, "Python") + USER_SITE = os.path.join(USER_BASE, + "Python" + sys.version[0] + sys.version[2], + "site-packages") + else: + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser("~", ".local") + USER_SITE = os.path.join(USER_BASE, "lib", + "python" + sys.version[:3], + "site-packages") + + if ENABLE_USER_SITE and os.path.isdir(USER_SITE): + addsitedir(USER_SITE, known_paths) + if ENABLE_USER_SITE: + for dist_libdir in ("lib", "local/lib"): + user_site = os.path.join(USER_BASE, dist_libdir, + "python" + sys.version[:3], + "dist-packages") + if os.path.isdir(user_site): + addsitedir(user_site, known_paths) + return known_paths + + + +def setBEGINLIBPATH(): + """The OS/2 EMX port has optional extension modules that do double duty + as DLLs (and must use the .DLL file extension) for other extensions. + The library search path needs to be amended so these will be found + during module import. Use BEGINLIBPATH so that these are at the start + of the library search path. + + """ + dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload") + libpath = os.environ['BEGINLIBPATH'].split(';') + if libpath[-1]: + libpath.append(dllpath) + else: + libpath[-1] = dllpath + os.environ['BEGINLIBPATH'] = ';'.join(libpath) + + +def setquit(): + """Define new built-ins 'quit' and 'exit'. + These are simply strings that display a hint on how to exit. + + """ + if os.sep == ':': + eof = 'Cmd-Q' + elif os.sep == '\\': + eof = 'Ctrl-Z plus Return' + else: + eof = 'Ctrl-D (i.e. EOF)' + + class Quitter(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return 'Use %s() or %s to exit' % (self.name, eof) + def __call__(self, code=None): + # Shells like IDLE catch the SystemExit, but listen when their + # stdin wrapper is closed. + try: + sys.stdin.close() + except: + pass + raise SystemExit(code) + __builtin__.quit = Quitter('quit') + __builtin__.exit = Quitter('exit') + + +class _Printer(object): + """interactive prompt objects for printing the license text, a list of + contributors and the copyright notice.""" + + MAXLINES = 23 + + def __init__(self, name, data, files=(), dirs=()): + self.__name = name + self.__data = data + self.__files = files + self.__dirs = dirs + self.__lines = None + + def __setup(self): + if self.__lines: + return + data = None + for dir in self.__dirs: + for filename in self.__files: + filename = os.path.join(dir, filename) + try: + fp = file(filename, "rU") + data = fp.read() + fp.close() + break + except IOError: + pass + if data: + break + if not data: + data = self.__data + self.__lines = data.split('\n') + self.__linecnt = len(self.__lines) + + def __repr__(self): + self.__setup() + if len(self.__lines) <= self.MAXLINES: + return "\n".join(self.__lines) + else: + return "Type %s() to see the full %s text" % ((self.__name,)*2) + + def __call__(self): + self.__setup() + prompt = 'Hit Return for more, or q (and Return) to quit: ' + lineno = 0 + while 1: + try: + for i in range(lineno, lineno + self.MAXLINES): + print self.__lines[i] + except IndexError: + break + else: + lineno += self.MAXLINES + key = None + while key is None: + key = raw_input(prompt) + if key not in ('', 'q'): + key = None + if key == 'q': + break + +def setcopyright(): + """Set 'copyright' and 'credits' in __builtin__""" + __builtin__.copyright = _Printer("copyright", sys.copyright) + if _is_jython: + __builtin__.credits = _Printer( + "credits", + "Jython is maintained by the Jython developers (www.jython.org).") + elif _is_pypy: + __builtin__.credits = _Printer( + "credits", + "PyPy is maintained by the PyPy developers: http://codespeak.net/pypy") + else: + __builtin__.credits = _Printer("credits", """\ + Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands + for supporting Python development. See www.python.org for more information.""") + here = os.path.dirname(os.__file__) + __builtin__.license = _Printer( + "license", "See http://www.python.org/%.3s/license.html" % sys.version, + ["LICENSE.txt", "LICENSE"], + [os.path.join(here, os.pardir), here, os.curdir]) + + +class _Helper(object): + """Define the built-in 'help'. + This is a wrapper around pydoc.help (with a twist). + + """ + + def __repr__(self): + return "Type help() for interactive help, " \ + "or help(object) for help about object." + def __call__(self, *args, **kwds): + import pydoc + return pydoc.help(*args, **kwds) + +def sethelper(): + __builtin__.help = _Helper() + +def aliasmbcs(): + """On Windows, some default encodings are not provided by Python, + while they are always available as "mbcs" in each locale. Make + them usable by aliasing to "mbcs" in such a case.""" + if sys.platform == 'win32': + import locale, codecs + enc = locale.getdefaultlocale()[1] + if enc.startswith('cp'): # "cp***" ? + try: + codecs.lookup(enc) + except LookupError: + import encodings + encodings._cache[enc] = encodings._unknown + encodings.aliases.aliases[enc] = 'mbcs' + +def setencoding(): + """Set the string encoding used by the Unicode implementation. The + default is 'ascii', but if you're willing to experiment, you can + change this.""" + encoding = "ascii" # Default value set by _PyUnicode_Init() + if 0: + # Enable to support locale aware default string encodings. + import locale + loc = locale.getdefaultlocale() + if loc[1]: + encoding = loc[1] + if 0: + # Enable to switch off string to Unicode coercion and implicit + # Unicode to string conversion. + encoding = "undefined" + if encoding != "ascii": + # On Non-Unicode builds this will raise an AttributeError... + sys.setdefaultencoding(encoding) # Needs Python Unicode build ! + + +def execsitecustomize(): + """Run custom site specific code, if available.""" + try: + import sitecustomize + except ImportError: + pass + +def virtual_install_main_packages(): + f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt')) + sys.real_prefix = f.read().strip() + f.close() + pos = 2 + if sys.path[0] == '': + pos += 1 + if sys.platform == 'win32': + paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')] + elif _is_jython: + paths = [os.path.join(sys.real_prefix, 'Lib')] + elif _is_pypy: + cpyver = '%d.%d.%d' % sys.version_info[:3] + paths = [os.path.join(sys.real_prefix, 'lib_pypy'), + os.path.join(sys.real_prefix, 'lib-python', 'modified-%s' % cpyver), + os.path.join(sys.real_prefix, 'lib-python', cpyver)] + else: + paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])] + lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3]) + if os.path.exists(lib64_path): + paths.append(lib64_path) + # This is hardcoded in the Python executable, but relative to sys.prefix: + plat_path = os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], + 'plat-%s' % sys.platform) + if os.path.exists(plat_path): + paths.append(plat_path) + # This is hardcoded in the Python executable, but + # relative to sys.prefix, so we have to fix up: + for path in list(paths): + tk_dir = os.path.join(path, 'lib-tk') + if os.path.exists(tk_dir): + paths.append(tk_dir) + + # These are hardcoded in the Apple's Python executable, + # but relative to sys.prefix, so we have to fix them up: + if sys.platform == 'darwin': + hardcoded_paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], module) + for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')] + + for path in hardcoded_paths: + if os.path.exists(path): + paths.append(path) + + sys.path.extend(paths) + +def force_global_eggs_after_local_site_packages(): + """ + Force easy_installed eggs in the global environment to get placed + in sys.path after all packages inside the virtualenv. This + maintains the "least surprise" result that packages in the + virtualenv always mask global packages, never the other way + around. + + """ + egginsert = getattr(sys, '__egginsert', 0) + for i, path in enumerate(sys.path): + if i > egginsert and path.startswith(sys.prefix): + egginsert = i + sys.__egginsert = egginsert + 1 + +def virtual_addsitepackages(known_paths): + force_global_eggs_after_local_site_packages() + return addsitepackages(known_paths, sys_prefix=sys.real_prefix) + +def fixclasspath(): + """Adjust the special classpath sys.path entries for Jython. These + entries should follow the base virtualenv lib directories. + """ + paths = [] + classpaths = [] + for path in sys.path: + if path == '__classpath__' or path.startswith('__pyclasspath__'): + classpaths.append(path) + else: + paths.append(path) + sys.path = paths + sys.path.extend(classpaths) + +def execusercustomize(): + """Run custom user specific code, if available.""" + try: + import usercustomize + except ImportError: + pass + + +def main(): + global ENABLE_USER_SITE + virtual_install_main_packages() + abs__file__() + paths_in_sys = removeduppaths() + if (os.name == "posix" and sys.path and + os.path.basename(sys.path[-1]) == "Modules"): + addbuilddir() + if _is_jython: + fixclasspath() + GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt')) + if not GLOBAL_SITE_PACKAGES: + ENABLE_USER_SITE = False + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + paths_in_sys = addsitepackages(paths_in_sys) + paths_in_sys = addusersitepackages(paths_in_sys) + if GLOBAL_SITE_PACKAGES: + paths_in_sys = virtual_addsitepackages(paths_in_sys) + if sys.platform == 'os2emx': + setBEGINLIBPATH() + setquit() + setcopyright() + sethelper() + aliasmbcs() + setencoding() + execsitecustomize() + if ENABLE_USER_SITE: + execusercustomize() + # Remove sys.setdefaultencoding() so that users cannot change the + # encoding after initialization. The test for presence is needed when + # this module is run as a script, because this code is executed twice. + if hasattr(sys, "setdefaultencoding"): + del sys.setdefaultencoding + +main() + +def _script(): + help = """\ + %s [--user-base] [--user-site] + + Without arguments print some useful information + With arguments print the value of USER_BASE and/or USER_SITE separated + by '%s'. + + Exit codes with --user-base or --user-site: + 0 - user site directory is enabled + 1 - user site directory is disabled by user + 2 - uses site directory is disabled by super user + or for security reasons + >2 - unknown error + """ + args = sys.argv[1:] + if not args: + print "sys.path = [" + for dir in sys.path: + print " %r," % (dir,) + print "]" + def exists(path): + if os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + print "USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE)) + print "USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE)) + print "ENABLE_USER_SITE: %r" % ENABLE_USER_SITE + sys.exit(0) + + buffer = [] + if '--user-base' in args: + buffer.append(USER_BASE) + if '--user-site' in args: + buffer.append(USER_SITE) + + if buffer: + print os.pathsep.join(buffer) + if ENABLE_USER_SITE: + sys.exit(0) + elif ENABLE_USER_SITE is False: + sys.exit(1) + elif ENABLE_USER_SITE is None: + sys.exit(2) + else: + sys.exit(3) + else: + import textwrap + print textwrap.dedent(help % (sys.argv[0], os.pathsep)) + sys.exit(10) + +if __name__ == '__main__': + _script() diff --git a/test/lib/python2.6/sre.py b/test/lib/python2.6/sre.py new file mode 120000 index 000000000..3ade4e80e --- /dev/null +++ b/test/lib/python2.6/sre.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_compile.py b/test/lib/python2.6/sre_compile.py new file mode 120000 index 000000000..fc13a37d5 --- /dev/null +++ b/test/lib/python2.6/sre_compile.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_compile.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_constants.py b/test/lib/python2.6/sre_constants.py new file mode 120000 index 000000000..98b0d5543 --- /dev/null +++ b/test/lib/python2.6/sre_constants.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_constants.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_parse.py b/test/lib/python2.6/sre_parse.py new file mode 120000 index 000000000..76b24c687 --- /dev/null +++ b/test/lib/python2.6/sre_parse.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_parse.py \ No newline at end of file diff --git a/test/lib/python2.6/stat.py b/test/lib/python2.6/stat.py new file mode 120000 index 000000000..68d61e662 --- /dev/null +++ b/test/lib/python2.6/stat.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/stat.py \ No newline at end of file diff --git a/test/lib/python2.6/types.py b/test/lib/python2.6/types.py new file mode 120000 index 000000000..72b00ba35 --- /dev/null +++ b/test/lib/python2.6/types.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/types.py \ No newline at end of file diff --git a/test/lib/python2.6/warnings.py b/test/lib/python2.6/warnings.py new file mode 120000 index 000000000..1af4e2833 --- /dev/null +++ b/test/lib/python2.6/warnings.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/warnings.py \ No newline at end of file -- cgit From ec7ab22b8b08820f7ffa3b05e40b09a925f1ef95 Mon Sep 17 00:00:00 2001 From: Anne Gentle <anne@openstack.org> Date: Mon, 21 Feb 2011 14:35:30 -0600 Subject: Removing pesky DS_Store files too. Begone. --- .bzrignore | 1 + doc/build/.DS_Store | Bin 6148 -> 0 bytes doc/build/html/.DS_Store | Bin 6148 -> 0 bytes 3 files changed, 1 insertion(+) delete mode 100644 doc/build/.DS_Store delete mode 100644 doc/build/html/.DS_Store diff --git a/.bzrignore b/.bzrignore index b271561a3..d22b62629 100644 --- a/.bzrignore +++ b/.bzrignore @@ -13,3 +13,4 @@ CA/serial* CA/newcerts/*.pem CA/private/cakey.pem nova/vcsversion.py +*.DS_Store diff --git a/doc/build/.DS_Store b/doc/build/.DS_Store deleted file mode 100644 index 5008ddfcf..000000000 Binary files a/doc/build/.DS_Store and /dev/null differ diff --git a/doc/build/html/.DS_Store b/doc/build/html/.DS_Store deleted file mode 100644 index 5008ddfcf..000000000 Binary files a/doc/build/html/.DS_Store and /dev/null differ -- cgit From 02e196192ea1f8be22c31828266b177d14d123cd Mon Sep 17 00:00:00 2001 From: Ken Pepple <ken.pepple@gmail.com> Date: Mon, 21 Feb 2011 12:41:15 -0800 Subject: make sure that ec2 response times are xs:dateTime parsable --- nova/api/ec2/apirequest.py | 8 +++++++- nova/tests/test_api.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 00b527d62..2b1acba5a 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -46,6 +46,11 @@ def _underscore_to_xmlcase(str): return res[:1].lower() + res[1:] +def _database_to_isoformat(datetimeobj): + """Return a xs:dateTime parsable string from datatime""" + return datetimeobj.strftime("%Y-%m-%dT%H:%M:%SZ") + + def _try_convert(value): """Return a non-string if possible""" if value == 'None': @@ -173,7 +178,8 @@ class APIRequest(object): elif isinstance(data, bool): data_el.appendChild(xml.createTextNode(str(data).lower())) elif isinstance(data, datetime.datetime): - data_el.appendChild(xml.createTextNode(data.isoformat())) + data_el.appendChild( + xml.createTextNode(_database_to_isoformat(data))) elif data != None: data_el.appendChild(xml.createTextNode(str(data))) diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index fa27825cd..d5c54a1c3 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -20,6 +20,7 @@ import boto from boto.ec2 import regioninfo +import datetime import httplib import random import StringIO @@ -127,6 +128,28 @@ class ApiEc2TestCase(test.TestCase): self.ec2.new_http_connection(host, is_secure).AndReturn(self.http) return self.http + def test_return_valid_isoformat(self): + """ + Ensure that the ec2 api returns datetime in xs:dateTime + (which apparently isn't datetime.isoformat()) + NOTE(ken-pepple): https://bugs.launchpad.net/nova/+bug/721297 + """ + conv = apirequest._database_to_isoformat + # sqlite database representation with microseconds + time_to_convert = datetime.datetime.strptime( + "2011-02-21 20:14:10.634276", + "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T20:14:10Z') + # mysqlite database representation + time_to_convert = datetime.datetime.strptime( + "2011-02-21 19:56:18", + "%Y-%m-%d %H:%M:%S") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T19:56:18Z') + def test_xmlns_version_matches_request_version(self): self.expect_http(api_version='2010-10-30') self.mox.ReplayAll() -- cgit From f797d6c6464f8ee2816d56ee771ad718418def64 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Mon, 21 Feb 2011 15:52:41 -0800 Subject: Renamed db_update to model_update, and lots more documentation --- nova/volume/driver.py | 26 ++++++++++++++++++++++---- nova/volume/manager.py | 12 ++++++------ nova/volume/san.py | 18 +++++++++--------- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index f172e2fdc..687bc99d0 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -91,7 +91,8 @@ class VolumeDriver(object): % FLAGS.volume_group) def create_volume(self, volume): - """Creates a logical volume.""" + """Creates a logical volume. Can optionally return a Dictionary of + changes to the volume object to be persisted.""" if int(volume['size']) == 0: sizestr = '100M' else: @@ -126,7 +127,8 @@ class VolumeDriver(object): raise NotImplementedError() def create_export(self, context, volume): - """Exports the volume.""" + """Exports the volume. Can optionally return a Dictionary of changes + to the volume object to be persisted.""" raise NotImplementedError() def remove_export(self, context, volume): @@ -225,7 +227,14 @@ class FakeAOEDriver(AOEDriver): class ISCSIDriver(VolumeDriver): - """Executes commands relating to ISCSI volumes.""" + """Executes commands relating to ISCSI volumes. We make use of model + provider properties as follows: + provider_location - if present, contains the iSCSI target information + in the same format as an ietadm discovery + i.e. '<target iqn>,<target portal> <target name>' + provider_auth - if present, contains a space-separated triple: + '<auth method> <auth username> <auth password>'. CHAP is the only + auth_method in use at the moment.""" def ensure_export(self, context, volume): """Synchronously recreates an export for a logical volume.""" @@ -313,7 +322,16 @@ class ISCSIDriver(VolumeDriver): def _get_iscsi_properties(self, volume): """Gets iscsi configuration, ideally from saved information in the - volume entity, but falling back to discovery if need be.""" + volume entity, but falling back to discovery if need be. The + properties are: + target_discovered - boolean indicating whether discovery was used, + target_iqn - the IQN of the iSCSI target, + target_portal - the portal of the iSCSI target, + and auth_method, auth_username and auth_password + - the authentication details. Right now, either + auth_method is not present meaning no authentication, or + auth_method == 'CHAP' meaning use CHAP with the specified + credentials.""" properties = {} diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 7193ece14..3e8bc16b3 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -107,14 +107,14 @@ class VolumeManager(manager.Manager): vol_size = volume_ref['size'] LOG.debug(_("volume %(vol_name)s: creating lv of" " size %(vol_size)sG") % locals()) - db_update = self.driver.create_volume(volume_ref) - if db_update: - self.db.volume_update(context, volume_ref['id'], db_update) + model_update = self.driver.create_volume(volume_ref) + if model_update: + self.db.volume_update(context, volume_ref['id'], model_update) LOG.debug(_("volume %s: creating export"), volume_ref['name']) - db_update = self.driver.create_export(context, volume_ref) - if db_update: - self.db.volume_update(context, volume_ref['id'], db_update) + model_update = self.driver.create_export(context, volume_ref) + if model_update: + self.db.volume_update(context, volume_ref['id'], model_update) except Exception: self.db.volume_update(context, volume_ref['id'], {'status': 'error'}) diff --git a/nova/volume/san.py b/nova/volume/san.py index 911ad096f..09192bc9f 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -485,12 +485,12 @@ class HpSanISCSIDriver(SanISCSIDriver): cluster_vip = self._cliq_get_cluster_vip(cluster_name) iscsi_portal = cluster_vip + ":3260," + cluster_interface - db_update = {} - db_update['provider_location'] = ("%s %s" % - (iscsi_portal, - iscsi_iqn)) + model_update = {} + model_update['provider_location'] = ("%s %s" % + (iscsi_portal, + iscsi_iqn)) - return db_update + return model_update def delete_volume(self, volume): """Deletes a volume.""" @@ -517,7 +517,7 @@ class HpSanISCSIDriver(SanISCSIDriver): is_shared = 'permission.authGroup' in volume_info - db_update = {} + model_update = {} should_export = False @@ -551,10 +551,10 @@ class HpSanISCSIDriver(SanISCSIDriver): self._cliq_run_xml("assignVolumeChap", cliq_args) - db_update['provider_auth'] = ("CHAP %s %s" % - (chap_username, chap_password)) + model_update['provider_auth'] = ("CHAP %s %s" % + (chap_username, chap_password)) - return db_update + return model_update def remove_export(self, context, volume): """Removes an export for a logical volume.""" -- cgit From 90007cd9085909a1e1ac59732a0b371dd79c2557 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Mon, 21 Feb 2011 15:55:50 -0800 Subject: pretty colors for logs and a few optimizations --- bin/nova-dhcpbridge | 2 + run_tests.py | 202 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 2 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index eda2dc072..04a1771f0 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -113,6 +113,8 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', + 'nova', + 'tests', 'tests.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] diff --git a/run_tests.py b/run_tests.py index 82345433a..43508394f 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,6 +17,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + import gettext import os import unittest @@ -28,14 +51,186 @@ from nose import core from nova import log as logging +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm(fd=stream.fileno()) + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_ERROR_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_ERROR_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_ERROR_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) + class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None + self.colorizer = None + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + # NOTE(vish): nose does funky stuff with stdout, so use stderr + # to setup the colorizer + if colorizer.supported(sys.stderr): + self.colorizer = colorizer(self.stream) + break def getDescription(self, test): return str(test) + def addSuccess(self, test): + if self.showAll: + self.colorizer.write("OK", 'green') + self.stream.writeln() + elif self.dots: + self.stream.write('.') + self.stream.flush() + + def addFailure(self, test): + if self.showAll: + self.colorizer.write("FAIL", 'red') + self.stream.writeln() + elif self.dots: + self.stream.write('F') + self.stream.flush() + + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + if self.showAll: + self.colorizer.write("ERROR", 'red') + self.stream.writeln() + elif self.dots: + stream.write('E') + def startTest(self, test): unittest.TestResult.startTest(self, test) current_case = test.test.__class__.__name__ @@ -59,12 +254,15 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - if os.path.exists("tests.sqlite"): - os.unlink("tests.sqlite") logging.setup() + testdir = os.path.abspath(os.path.join("nova","tests")) + testdb = os.path.join(testdir, "tests.sqlite") + if os.path.exists(testdb): + os.unlink(testdb) c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, + workingDir=testdir, plugins=core.DefaultPluginManager()) runner = NovaTestRunner(stream=c.stream, -- cgit From 305ef6bf5f8f8926fdaa8db5f75a0680fbd8a2be Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Mon, 21 Feb 2011 16:02:38 -0800 Subject: Fixed my confusion in documenting the syntax of iSCSI discovery --- nova/volume/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 687bc99d0..22c2c2fc3 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -231,7 +231,7 @@ class ISCSIDriver(VolumeDriver): provider properties as follows: provider_location - if present, contains the iSCSI target information in the same format as an ietadm discovery - i.e. '<target iqn>,<target portal> <target name>' + i.e. '<target ip/port>,<target portal> <target IQN>' provider_auth - if present, contains a space-separated triple: '<auth method> <auth username> <auth password>'. CHAP is the only auth_method in use at the moment.""" -- cgit From 0f402b72cbf80d1adde503eb532a578944fa0c79 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Mon, 21 Feb 2011 16:22:09 -0800 Subject: update based on prereq branch --- nova/tests/fake_flags.py | 2 +- run_tests.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index cfa65c137..575fefff6 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -39,6 +39,6 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///nova.sqlite' +FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True FLAGS.logfile = 'tests.log' diff --git a/run_tests.sh b/run_tests.sh index 70212cc6a..e8433bc06 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -39,7 +39,6 @@ done function run_tests { # Just run the test suites in current environment - ${wrapper} rm -f nova.sqlite ${wrapper} $NOSETESTS } -- cgit From 6e6c3fcfe97d60b6262bf90d423a77fa250bd383 Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Tue, 22 Feb 2011 08:22:42 +0100 Subject: added disabled services to the list of displayed services in bin/nova-manage brontes:~ # nova-manage service list ares nova-scheduler enabled :-) 2011-02-22 07:21:29 ares nova-network enabled :-) 2011-02-22 07:21:29 ares nova-volume enabled XXX 2011-02-16 19:04:29 brontes nova-volume enabled XXX 2011-02-12 18:31:43 brontes nova-network enabled :-) 2011-02-22 07:21:29 ares nova-compute disabled :-) 2011-02-22 07:21:25 brontes nova-compute disabled :-) 2011-02-22 07:21:24 --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 6d67252b8..af8432441 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -552,7 +552,7 @@ class ServiceCommands(object): args: [host] [service]""" ctxt = context.get_admin_context() now = datetime.datetime.utcnow() - services = db.service_get_all(ctxt) + services = db.service_get_all(ctxt) + db.service_get_all(ctxt, True) if host: services = [s for s in services if s['host'] == host] if service: -- cgit From 3125d978fec27608064dd3dd8d3696f2219fbf12 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Mon, 21 Feb 2011 23:26:03 -0800 Subject: use a different flag for listen port for apis --- bin/nova-api | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 11176a021..cb9b41725 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -44,6 +44,8 @@ LOG = logging.getLogger('nova.api') LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS +flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') +flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') API_ENDPOINTS = ['ec2', 'osapi'] @@ -60,10 +62,10 @@ def run_app(paste_config_file): wsgi.paste_config_to_flags(config, { "verbose": FLAGS.verbose, "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) + "%s_listen_port" % api: getattr(FLAGS, "%s_listen_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), getattr(FLAGS, "%s_host" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), -- cgit From 9e2942931b5381d3ba0e8cc4f9846160b003f45b Mon Sep 17 00:00:00 2001 From: Thierry Carrez <thierry@openstack.org> Date: Tue, 22 Feb 2011 17:18:04 +0100 Subject: Get rid of nova-combined, see rationale on ML --- bin/nova-combined | 83 ------------------------------------------------------- 1 file changed, 83 deletions(-) delete mode 100755 bin/nova-combined diff --git a/bin/nova-combined b/bin/nova-combined deleted file mode 100755 index 22f0d5cb7..000000000 --- a/bin/nova-combined +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Combined starter script for Nova services.""" - -import eventlet -eventlet.monkey_patch() - -import gettext -import os -import sys - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import wsgi - - -FLAGS = flags.FLAGS - -API_ENDPOINTS = ['ec2', 'osapi'] - -for api in API_ENDPOINTS: - flags.DEFINE_string("%s_listen" % api, "0.0.0.0", - "IP address to listen to for API %s" % api) - flags.DEFINE_integer("%s_listen_port" % api, - getattr(FLAGS, "%s_port" % api), - "Port to listen to for API %s" % api) - -if __name__ == '__main__': - utils.default_flagfile() - FLAGS(sys.argv) - logging.setup() - - compute = service.Service.create(binary='nova-compute') - network = service.Service.create(binary='nova-network') - volume = service.Service.create(binary='nova-volume') - scheduler = service.Service.create(binary='nova-scheduler') - #objectstore = service.Service.create(binary='nova-objectstore') - - service.serve(compute, network, volume, scheduler) - - apps = [] - paste_config_file = wsgi.paste_config_file('nova-api.conf') - for api in API_ENDPOINTS: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - continue - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_listen_port" % api), - getattr(FLAGS, "%s_listen" % api))) - if len(apps) > 0: - server = wsgi.Server() - for app in apps: - server.start(*app) - server.wait() -- cgit From 912e762c9baf3cb17a24bc0d9feba4b26892dbbc Mon Sep 17 00:00:00 2001 From: Thierry Carrez <thierry@openstack.org> Date: Tue, 22 Feb 2011 17:37:12 +0100 Subject: Also remove nova-combined from setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 4ab8f386b..3b48990ac 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,6 @@ DistUtilsExtra.auto.setup(name='nova', test_suite='nose.collector', scripts=['bin/nova-ajax-console-proxy', 'bin/nova-api', - 'bin/nova-combined', 'bin/nova-compute', 'bin/nova-console', 'bin/nova-dhcpbridge', -- cgit From 5b2ec209d07d7df45f9b7ca6eebfcbc9443de94e Mon Sep 17 00:00:00 2001 From: termie <github@anarkystic.com> Date: Tue, 22 Feb 2011 17:10:34 -0800 Subject: don't make a syslog handler if we didn't ask for one --- nova/log.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/log.py b/nova/log.py index 10c14d74b..591d26c63 100644 --- a/nova/log.py +++ b/nova/log.py @@ -236,16 +236,17 @@ class NovaRootLogger(NovaLogger): def __init__(self, name, level=NOTSET): self.logpath = None self.filelog = None - self.syslog = SysLogHandler(address='/dev/log') self.streamlog = StreamHandler() + self.syslog = None NovaLogger.__init__(self, name, level) def setup_from_flags(self): """Setup logger from flags""" global _filelog if FLAGS.use_syslog: + self.syslog = SysLogHandler(address='/dev/log') self.addHandler(self.syslog) - else: + elif self.syslog: self.removeHandler(self.syslog) logpath = _get_log_file_path() if logpath: -- cgit From 18793c2e184713d33bc93306d464cf443584ffd6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Tue, 22 Feb 2011 17:44:07 -0800 Subject: test that shows error on filtering groups --- nova/tests/test_cloud.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 445cc6e8b..2bce64353 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -133,6 +133,20 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_security_groups(self): + """Makes sure describe_security_groups works and filters results.""" + sec = db.security_group_create(self.context, {'name': 'test'}) + result = self.cloud.describe_security_groups(self.context) + # NOTE(vish): should have the default group as well + self.assertEqual(len(result['securityGroupInfo']), 2) + result = self.cloud.describe_security_groups(self.context, + group_name=[sec['name']]) + self.assertEqual(len(result['securityGroupInfo']), 1) + self.assertEqual( + cloud.ec2_id_to_id(result['securityGroupInfo'][0]['name']), + sec['name']) + db.security_group_destroy(self.context, sec['id']) + def test_describe_volumes(self): """Makes sure describe_volumes works and filters results.""" vol1 = db.volume_create(self.context, {}) @@ -286,19 +300,6 @@ class CloudTestCase(test.TestCase): LOG.debug(_("Terminating instance %s"), instance_id) rv = self.compute.terminate_instance(instance_id) - def test_describe_instances(self): - """Makes sure describe_instances works.""" - instance1 = db.instance_create(self.context, {'host': 'host2'}) - comp1 = db.service_create(self.context, {'host': 'host2', - 'availability_zone': 'zone1', - 'topic': "compute"}) - result = self.cloud.describe_instances(self.context) - self.assertEqual(result['reservationSet'][0] - ['instancesSet'][0] - ['placement']['availabilityZone'], 'zone1') - db.instance_destroy(self.context, instance1['id']) - db.service_destroy(self.context, comp1['id']) - def test_instance_update_state(self): # TODO(termie): what is this code even testing? def instance(num): -- cgit From 2610a522d26351686612058a6da0300bce731112 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Tue, 22 Feb 2011 17:49:38 -0800 Subject: fix test --- nova/tests/test_cloud.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 2bce64353..afdbb80a9 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -135,7 +135,9 @@ class CloudTestCase(test.TestCase): def test_describe_security_groups(self): """Makes sure describe_security_groups works and filters results.""" - sec = db.security_group_create(self.context, {'name': 'test'}) + sec = db.security_group_create(self.context, + {'project_id': self.context.project_id, + 'name': 'test'}) result = self.cloud.describe_security_groups(self.context) # NOTE(vish): should have the default group as well self.assertEqual(len(result['securityGroupInfo']), 2) @@ -143,7 +145,7 @@ class CloudTestCase(test.TestCase): group_name=[sec['name']]) self.assertEqual(len(result['securityGroupInfo']), 1) self.assertEqual( - cloud.ec2_id_to_id(result['securityGroupInfo'][0]['name']), + result['securityGroupInfo'][0]['groupName'], sec['name']) db.security_group_destroy(self.context, sec['id']) -- cgit From 828e3ea3f29f57767a4e25ad40b275c886cb7968 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Tue, 22 Feb 2011 18:02:25 -0800 Subject: fix and optimize security group filtering --- nova/api/ec2/cloud.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 882cdcfc9..fc9c13d91 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -318,14 +318,19 @@ class CloudController(object): def describe_security_groups(self, context, group_name=None, **kwargs): self.compute_api.ensure_default_security_group(context) - if context.is_admin: + if group_name: + groups = [] + for name in group_name: + group = db.security_group_get_by_name(context, + context.project_id, + name) + groups.append(group) + elif context.is_admin: groups = db.security_group_get_all(context) else: groups = db.security_group_get_by_project(context, context.project_id) groups = [self._format_security_group(context, g) for g in groups] - if not group_name is None: - groups = [g for g in groups if g.name in group_name] return {'securityGroupInfo': list(sorted(groups, -- cgit From 2fd33bdd50b933dc14fea065c823f5a73324129b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Tue, 22 Feb 2011 18:04:08 -0800 Subject: separate out smoketests and add updated nova.sh --- contrib/nova.sh | 12 +- smoketests/base.py | 12 + smoketests/public_network_smoketests.py | 11 +- smoketests/sysadmin_smoketests.py | 293 +++++++++++++++++++++++ smoketests/user_smoketests.py | 397 -------------------------------- 5 files changed, 321 insertions(+), 404 deletions(-) create mode 100644 smoketests/sysadmin_smoketests.py delete mode 100644 smoketests/user_smoketests.py diff --git a/contrib/nova.sh b/contrib/nova.sh index 9259035ca..1187f2728 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -66,7 +66,7 @@ if [ "$CMD" == "install" ]; then sudo apt-get install -y user-mode-linux kvm libvirt-bin sudo apt-get install -y screen euca2ools vlan curl rabbitmq-server sudo apt-get install -y lvm2 iscsitarget open-iscsi - sudo apt-get install -y socat + sudo apt-get install -y socat unzip echo "ISCSITARGET_ENABLE=true" | sudo tee /etc/default/iscsitarget sudo /etc/init.d/iscsitarget restart sudo modprobe kvm @@ -111,8 +111,7 @@ if [ "$CMD" == "run" ]; then --nodaemon --dhcpbridge_flagfile=$NOVA_DIR/bin/nova.conf --network_manager=nova.network.manager.$NET_MAN ---cc_host=$HOST_IP ---routing_source_ip=$HOST_IP +--my_ip=$HOST_IP --sql_connection=$SQL_CONN --auth_driver=nova.auth.$AUTH --libvirt_type=$LIBVIRT_TYPE @@ -151,7 +150,6 @@ NOVA_CONF_EOF mkdir -p $NOVA_DIR/instances rm -rf $NOVA_DIR/networks mkdir -p $NOVA_DIR/networks - $NOVA_DIR/tools/clean-vlans if [ ! -d "$NOVA_DIR/images" ]; then ln -s $DIR/images $NOVA_DIR/images fi @@ -169,10 +167,14 @@ NOVA_CONF_EOF # create a project called 'admin' with project manager of 'admin' $NOVA_DIR/bin/nova-manage project create admin admin # export environment variables for project 'admin' and user 'admin' - $NOVA_DIR/bin/nova-manage project environment admin admin $NOVA_DIR/novarc + $NOVA_DIR/bin/nova-manage project zipfile admin admin $NOVA_DIR/nova.zip + unzip -o $NOVA_DIR/nova.zip -d $NOVA_DIR/ # create a small network $NOVA_DIR/bin/nova-manage network create 10.0.0.0/8 1 32 + # create some floating ips + $NOVA_DIR/bin/nova-manage floating create `hostname` 10.6.0.0/27 + # nova api crashes if we start it with a regular screen command, # so send the start command by forcing text into the window. screen_it api "$NOVA_DIR/bin/nova-api" diff --git a/smoketests/base.py b/smoketests/base.py index afc618074..204b4a1eb 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -28,7 +28,9 @@ from boto.ec2.regioninfo import RegionInfo from smoketests import flags +SUITE_NAMES = '[image, instance, volume]' FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) boto_v6 = None @@ -173,6 +175,16 @@ class SmokeTestCase(unittest.TestCase): return True +TEST_DATA = {} + + +class UserSmokeTestCase(SmokeTestCase): + def setUp(self): + global TEST_DATA + self.conn = self.connection_for_env() + self.data = TEST_DATA + + def run_tests(suites): argv = FLAGS(sys.argv) if FLAGS.use_ipv6: diff --git a/smoketests/public_network_smoketests.py b/smoketests/public_network_smoketests.py index bfc2b20ba..5a4c67642 100644 --- a/smoketests/public_network_smoketests.py +++ b/smoketests/public_network_smoketests.py @@ -24,9 +24,16 @@ import sys import time import unittest +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from smoketests import flags from smoketests import base -from smoketests import user_smoketests #Note that this test should run from #public network (outside of private network segments) @@ -42,7 +49,7 @@ TEST_KEY2 = '%s_key2' % TEST_PREFIX TEST_DATA = {} -class InstanceTestsFromPublic(user_smoketests.UserSmokeTestCase): +class InstanceTestsFromPublic(base.UserSmokeTestCase): def test_001_can_create_keypair(self): key = self.create_key_pair(self.conn, TEST_KEY) self.assertEqual(key.name, TEST_KEY) diff --git a/smoketests/sysadmin_smoketests.py b/smoketests/sysadmin_smoketests.py new file mode 100644 index 000000000..e3b84d3d3 --- /dev/null +++ b/smoketests/sysadmin_smoketests.py @@ -0,0 +1,293 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import sys +import time +import unittest + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from smoketests import flags +from smoketests import base + + + +FLAGS = flags.FLAGS +flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', + 'Local kernel file to use for bundling tests') +flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', + 'Local image file to use for bundling tests') + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_GROUP = '%s_group' % TEST_PREFIX +class ImageTests(base.UserSmokeTestCase): + def test_001_can_bundle_image(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_image)) + + def test_002_can_upload_image(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) + + def test_003_can_register_image(self): + image_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_image)) + self.assert_(image_id is not None) + self.data['image_id'] = image_id + + def test_004_can_bundle_kernel(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) + + def test_005_can_upload_kernel(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) + + def test_006_can_register_kernel(self): + kernel_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_kernel)) + self.assert_(kernel_id is not None) + self.data['kernel_id'] = kernel_id + + def test_007_images_are_available_within_10_seconds(self): + for i in xrange(10): + image = self.conn.get_image(self.data['image_id']) + if image and image.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(image.type == 'machine') + + for i in xrange(10): + kernel = self.conn.get_image(self.data['kernel_id']) + if kernel and kernel.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(kernel.type == 'kernel') + + def test_008_can_describe_image_attribute(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + + def test_009_can_modify_image_launch_permission(self): + self.conn.modify_image_attribute(image_id=self.data['image_id'], + operation='add', + attribute='launchPermission', + groups='all') + image = self.conn.get_image(self.data['image_id']) + self.assertEqual(image.id, self.data['image_id']) + + def test_010_can_see_launch_permission(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + self.assert_(attrs.attrs['groups'][0], 'all') + + def test_011_user_can_deregister_kernel(self): + self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) + + def test_012_can_deregister_image(self): + self.assertTrue(self.conn.deregister_image(self.data['image_id'])) + + def test_013_can_delete_bundle(self): + self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) + + +class InstanceTests(base.UserSmokeTestCase): + def test_001_can_create_keypair(self): + key = self.create_key_pair(self.conn, TEST_KEY) + self.assertEqual(key.name, TEST_KEY) + + def test_002_can_create_instance_with_keypair(self): + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + instance_type='m1.tiny') + self.assertEqual(len(reservation.instances), 1) + self.data['instance'] = reservation.instances[0] + + def test_003_instance_runs_within_60_seconds(self): + instance = self.data['instance'] + # allow 60 seconds to exit pending with IP + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + ip = self.data['instance'].private_dns_name + self.failIf(ip == '0.0.0.0') + if FLAGS.use_ipv6: + ipv6 = self.data['instance'].dns_name_v6 + self.failIf(ipv6 is None) + + def test_004_can_ping_private_ip(self): + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + + if FLAGS.use_ipv6: + if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + self.fail('could not ping instance v6') + + def test_005_can_ssh_to_private_ip(self): + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + if FLAGS.use_ipv6: + if not self.wait_for_ssh(self.data['instance'].ip_v6, + TEST_KEY): + self.fail('could not ssh to instance v6') + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + self.conn.terminate_instances([self.data['instance'].id]) + + +class VolumeTests(base.UserSmokeTestCase): + def setUp(self): + super(VolumeTests, self).setUp() + self.device = '/dev/vdb' + + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_001_can_create_volume(self): + volume = self.conn.create_volume(1, 'nova') + self.assertEqual(volume.size, 1) + self.data['volume'] = volume + # Give network time to find volume. + time.sleep(10) + + def test_002_can_attach_volume(self): + volume = self.data['volume'] + + for x in xrange(10): + volume.update() + if volume.status.startswith('available'): + break + time.sleep(1) + else: + self.fail('cannot attach volume with state %s' % volume.status) + + volume.attach(self.data['instance'].id, self.device) + + # wait + for x in xrange(10): + volume.update() + if volume.status.startswith('in-use'): + break + time.sleep(1) + else: + self.fail('volume never got to in use') + + self.assertTrue(volume.status.startswith('in-use')) + + # Give instance time to recognize volume. + time.sleep(10) + + def test_003_can_mount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # NOTE(vish): this will create an dev for images that don't have + # udev rules + stdin, stdout, stderr = conn.exec_command( + 'grep %s /proc/partitions | ' + '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' + % self.device.rpartition('/')[2]) + exec_list = [] + exec_list.append('mkdir -p /mnt/vol') + exec_list.append('/sbin/mke2fs %s' % self.device) + exec_list.append('mount %s /mnt/vol' % self.device) + exec_list.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) + out = stdout.read() + conn.close() + if not out.strip().endswith('success'): + self.fail('Unable to mount: %s %s' % (out, stderr.read())) + + def test_004_can_write_to_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted + stdin, stdout, stderr = conn.exec_command( + 'echo hello > /mnt/vol/test.txt') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to write to mount: %s' % (err)) + + def test_005_volume_is_correct_size(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command( + "df -h | grep %s | awk {'print $2'}" % self.device) + out = stdout.read() + conn.close() + if not out.strip() == '1007.9M': + self.fail('Volume is not the right size: %s %s' % + (out, stderr.read())) + + def test_006_me_can_umount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to unmount: %s' % (err)) + + def test_007_me_can_detach_volume(self): + result = self.conn.detach_volume(volume_id=self.data['volume'].id) + self.assertTrue(result) + time.sleep(5) + + def test_008_me_can_delete_volume(self): + result = self.conn.delete_volume(self.data['volume'].id) + self.assertTrue(result) + + def test_999_tearDown(self): + self.conn.terminate_instances([self.data['instance'].id]) + self.conn.delete_key_pair(TEST_KEY) + + +if __name__ == "__main__": + suites = {'image': unittest.makeSuite(ImageTests), + 'instance': unittest.makeSuite(InstanceTests), + 'volume': unittest.makeSuite(VolumeTests) + } + sys.exit(base.run_tests(suites)) diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py deleted file mode 100644 index 26f6344f7..000000000 --- a/smoketests/user_smoketests.py +++ /dev/null @@ -1,397 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import commands -import os -import random -import sys -import time -import unittest - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -from smoketests import flags -from smoketests import base - - -SUITE_NAMES = '[image, instance, volume]' - -FLAGS = flags.FLAGS -flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) -flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', - 'Local kernel file to use for bundling tests') -flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', - 'Local image file to use for bundling tests') - -TEST_PREFIX = 'test%s' % int(random.random() * 1000000) -TEST_BUCKET = '%s_bucket' % TEST_PREFIX -TEST_KEY = '%s_key' % TEST_PREFIX -TEST_GROUP = '%s_group' % TEST_PREFIX -TEST_DATA = {} - - -class UserSmokeTestCase(base.SmokeTestCase): - def setUp(self): - global TEST_DATA - self.conn = self.connection_for_env() - self.data = TEST_DATA - - -class ImageTests(UserSmokeTestCase): - def test_001_can_bundle_image(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_image)) - - def test_002_can_upload_image(self): - self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) - - def test_003_can_register_image(self): - image_id = self.conn.register_image('%s/%s.manifest.xml' % - (TEST_BUCKET, FLAGS.bundle_image)) - self.assert_(image_id is not None) - self.data['image_id'] = image_id - - def test_004_can_bundle_kernel(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) - - def test_005_can_upload_kernel(self): - self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) - - def test_006_can_register_kernel(self): - kernel_id = self.conn.register_image('%s/%s.manifest.xml' % - (TEST_BUCKET, FLAGS.bundle_kernel)) - self.assert_(kernel_id is not None) - self.data['kernel_id'] = kernel_id - - def test_007_images_are_available_within_10_seconds(self): - for i in xrange(10): - image = self.conn.get_image(self.data['image_id']) - if image and image.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(image.type == 'machine') - - for i in xrange(10): - kernel = self.conn.get_image(self.data['kernel_id']) - if kernel and kernel.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(kernel.type == 'kernel') - - def test_008_can_describe_image_attribute(self): - attrs = self.conn.get_image_attribute(self.data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - - def test_009_can_modify_image_launch_permission(self): - self.conn.modify_image_attribute(image_id=self.data['image_id'], - operation='add', - attribute='launchPermission', - groups='all') - image = self.conn.get_image(self.data['image_id']) - self.assertEqual(image.id, self.data['image_id']) - - def test_010_can_see_launch_permission(self): - attrs = self.conn.get_image_attribute(self.data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - self.assert_(attrs.attrs['groups'][0], 'all') - - def test_011_user_can_deregister_kernel(self): - self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) - - def test_012_can_deregister_image(self): - self.assertTrue(self.conn.deregister_image(self.data['image_id'])) - - def test_013_can_delete_bundle(self): - self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) - - -class InstanceTests(UserSmokeTestCase): - def test_001_can_create_keypair(self): - key = self.create_key_pair(self.conn, TEST_KEY) - self.assertEqual(key.name, TEST_KEY) - - def test_002_can_create_instance_with_keypair(self): - reservation = self.conn.run_instances(FLAGS.test_image, - key_name=TEST_KEY, - instance_type='m1.tiny') - self.assertEqual(len(reservation.instances), 1) - self.data['instance'] = reservation.instances[0] - - def test_003_instance_runs_within_60_seconds(self): - instance = self.data['instance'] - # allow 60 seconds to exit pending with IP - if not self.wait_for_running(self.data['instance']): - self.fail('instance failed to start') - self.data['instance'].update() - ip = self.data['instance'].private_dns_name - self.failIf(ip == '0.0.0.0') - if FLAGS.use_ipv6: - ipv6 = self.data['instance'].dns_name_v6 - self.failIf(ipv6 is None) - - def test_004_can_ping_private_ip(self): - if not self.wait_for_ping(self.data['instance'].private_dns_name): - self.fail('could not ping instance') - - if FLAGS.use_ipv6: - if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): - self.fail('could not ping instance v6') - - def test_005_can_ssh_to_private_ip(self): - if not self.wait_for_ssh(self.data['instance'].private_dns_name, - TEST_KEY): - self.fail('could not ssh to instance') - - if FLAGS.use_ipv6: - if not self.wait_for_ssh(self.data['instance'].ip_v6, - TEST_KEY): - self.fail('could not ssh to instance v6') - - def test_006_can_allocate_elastic_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - def test_007_can_associate_ip_with_instance(self): - result = self.conn.associate_address(self.data['instance'].id, - self.data['public_ip']) - self.assertTrue(result) - - def test_008_can_ssh_with_public_ip(self): - if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): - self.fail('could not ssh to public ip') - - def test_009_can_disassociate_ip_from_instance(self): - result = self.conn.disassociate_address(self.data['public_ip']) - self.assertTrue(result) - - def test_010_can_deallocate_elastic_ip(self): - result = self.conn.release_address(self.data['public_ip']) - self.assertTrue(result) - - def test_999_tearDown(self): - self.delete_key_pair(self.conn, TEST_KEY) - self.conn.terminate_instances([self.data['instance'].id]) - - -class VolumeTests(UserSmokeTestCase): - def setUp(self): - super(VolumeTests, self).setUp() - self.device = '/dev/vdb' - - def test_000_setUp(self): - self.create_key_pair(self.conn, TEST_KEY) - reservation = self.conn.run_instances(FLAGS.test_image, - instance_type='m1.tiny', - key_name=TEST_KEY) - self.data['instance'] = reservation.instances[0] - if not self.wait_for_running(self.data['instance']): - self.fail('instance failed to start') - self.data['instance'].update() - if not self.wait_for_ping(self.data['instance'].private_dns_name): - self.fail('could not ping instance') - if not self.wait_for_ssh(self.data['instance'].private_dns_name, - TEST_KEY): - self.fail('could not ssh to instance') - - def test_001_can_create_volume(self): - volume = self.conn.create_volume(1, 'nova') - self.assertEqual(volume.size, 1) - self.data['volume'] = volume - # Give network time to find volume. - time.sleep(10) - - def test_002_can_attach_volume(self): - volume = self.data['volume'] - - for x in xrange(10): - volume.update() - if volume.status.startswith('available'): - break - time.sleep(1) - else: - self.fail('cannot attach volume with state %s' % volume.status) - - volume.attach(self.data['instance'].id, self.device) - - # wait - for x in xrange(10): - volume.update() - if volume.status.startswith('in-use'): - break - time.sleep(1) - else: - self.fail('volume never got to in use') - - self.assertTrue(volume.status.startswith('in-use')) - - # Give instance time to recognize volume. - time.sleep(10) - - def test_003_can_mount_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - # NOTE(vish): this will create an dev for images that don't have - # udev rules - stdin, stdout, stderr = conn.exec_command( - 'grep %s /proc/partitions | ' - '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' - % self.device.rpartition('/')[2]) - exec_list = [] - exec_list.append('mkdir -p /mnt/vol') - exec_list.append('/sbin/mke2fs %s' % self.device) - exec_list.append('mount %s /mnt/vol' % self.device) - exec_list.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) - out = stdout.read() - conn.close() - if not out.strip().endswith('success'): - self.fail('Unable to mount: %s %s' % (out, stderr.read())) - - def test_004_can_write_to_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted - stdin, stdout, stderr = conn.exec_command( - 'echo hello > /mnt/vol/test.txt') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to write to mount: %s' % (err)) - - def test_005_volume_is_correct_size(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - stdin, stdout, stderr = conn.exec_command( - "df -h | grep %s | awk {'print $2'}" % self.device) - out = stdout.read() - conn.close() - if not out.strip() == '1007.9M': - self.fail('Volume is not the right size: %s %s' % - (out, stderr.read())) - - def test_006_me_can_umount_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to unmount: %s' % (err)) - - def test_007_me_can_detach_volume(self): - result = self.conn.detach_volume(volume_id=self.data['volume'].id) - self.assertTrue(result) - time.sleep(5) - - def test_008_me_can_delete_volume(self): - result = self.conn.delete_volume(self.data['volume'].id) - self.assertTrue(result) - - def test_999_tearDown(self): - self.conn.terminate_instances([self.data['instance'].id]) - self.conn.delete_key_pair(TEST_KEY) - - -class SecurityGroupTests(UserSmokeTestCase): - - def __public_instance_is_accessible(self): - id_url = "latest/meta-data/instance-id" - options = "-s --max-time 1" - command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) - instance_id = commands.getoutput(command).strip() - if not instance_id: - return False - if instance_id != self.data['instance_id']: - raise Exception("Wrong instance id") - return True - - def test_001_can_create_security_group(self): - self.conn.create_security_group(TEST_GROUP, description='test') - - groups = self.conn.get_all_security_groups() - self.assertTrue(TEST_GROUP in [group.name for group in groups]) - - def test_002_can_launch_instance_in_security_group(self): - self.create_key_pair(self.conn, TEST_KEY) - reservation = self.conn.run_instances(FLAGS.test_image, - key_name=TEST_KEY, - security_groups=[TEST_GROUP], - instance_type='m1.tiny') - - self.data['instance_id'] = reservation.instances[0].id - - def test_003_can_authorize_security_group_ingress(self): - self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - - def test_004_can_access_instance_over_public_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - result = self.conn.associate_address(self.data['instance_id'], - self.data['public_ip']) - start_time = time.time() - while not self.__public_instance_is_accessible(): - # 1 minute to launch - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_005_can_revoke_security_group_ingress(self): - self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - start_time = time.time() - while self.__public_instance_is_accessible(): - # 1 minute to teardown - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_999_tearDown(self): - self.conn.delete_key_pair(TEST_KEY) - self.conn.delete_security_group(TEST_GROUP) - groups = self.conn.get_all_security_groups() - self.assertFalse(TEST_GROUP in [group.name for group in groups]) - self.conn.terminate_instances([self.data['instance_id']]) - self.assertTrue(self.conn.release_address(self.data['public_ip'])) - - -if __name__ == "__main__": - suites = {'image': unittest.makeSuite(ImageTests), - 'instance': unittest.makeSuite(InstanceTests), - #'security_group': unittest.makeSuite(SecurityGroupTests), - 'volume': unittest.makeSuite(VolumeTests) - } - sys.exit(base.run_tests(suites)) -- cgit From ef37833e6f45f99b1d16143d29685974a191c387 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Tue, 22 Feb 2011 18:04:32 -0800 Subject: add netadmin smoketests --- smoketests/netadmin_smoketests.py | 194 ++++++++++++++++++++++++++++++++++++++ smoketests/proxy.sh | 22 +++++ 2 files changed, 216 insertions(+) create mode 100644 smoketests/netadmin_smoketests.py create mode 100755 smoketests/proxy.sh diff --git a/smoketests/netadmin_smoketests.py b/smoketests/netadmin_smoketests.py new file mode 100644 index 000000000..38beb8fdc --- /dev/null +++ b/smoketests/netadmin_smoketests.py @@ -0,0 +1,194 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import sys +import time +import unittest + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from smoketests import flags +from smoketests import base + + +FLAGS = flags.FLAGS + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_GROUP = '%s_group' % TEST_PREFIX + + +class AddressTests(base.UserSmokeTestCase): + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_001_can_allocate_floating_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + def test_002_can_associate_ip_with_instance(self): + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + self.assertTrue(result) + + def test_003_can_ssh_with_public_ip(self): + ssh_authorized = False + groups = self.conn.get_all_security_groups(['default']) + for rule in groups[0].rules: + if (rule.ip_protocol == 'tcp' and + rule.from_port <= 22 and rule.to_port >= 22): + ssh_authorized = True + if not ssh_authorized: + self.conn.authorize_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + try: + if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): + self.fail('could not ssh to public ip') + finally: + if not ssh_authorized: + self.conn.revoke_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + + def test_004_can_disassociate_ip_from_instance(self): + result = self.conn.disassociate_address(self.data['public_ip']) + self.assertTrue(result) + + def test_005_can_deallocate_floating_ip(self): + result = self.conn.release_address(self.data['public_ip']) + self.assertTrue(result) + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + self.conn.terminate_instances([self.data['instance'].id]) + + +class SecurityGroupTests(base.UserSmokeTestCase): + + def __public_instance_is_accessible(self): + id_url = "latest/meta-data/instance-id" + options = "-s --max-time 1" + command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) + instance_id = commands.getoutput(command).strip() + if not instance_id: + return False + if instance_id != self.data['instance'].id: + raise Exception("Wrong instance id") + return True + + def test_001_can_create_security_group(self): + self.conn.create_security_group(TEST_GROUP, description='test') + + groups = self.conn.get_all_security_groups() + self.assertTrue(TEST_GROUP in [group.name for group in groups]) + + def test_002_can_launch_instance_in_security_group(self): + with open("proxy.sh") as f: + user_data = f.read() + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + security_groups=[TEST_GROUP], + user_data=user_data, + instance_type='m1.tiny') + + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_003_can_authorize_security_group_ingress(self): + self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + + def test_004_can_access_metadata_over_public_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + start_time = time.time() + try: + while not self.__public_instance_is_accessible(): + # 1 minute to launch + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + finally: + result = self.conn.disassociate_address(self.data['public_ip']) + + def test_005_can_revoke_security_group_ingress(self): + self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + start_time = time.time() + while self.__public_instance_is_accessible(): + # 1 minute to teardown + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + + def test_999_tearDown(self): + self.conn.delete_key_pair(TEST_KEY) + self.conn.delete_security_group(TEST_GROUP) + groups = self.conn.get_all_security_groups() + self.assertFalse(TEST_GROUP in [group.name for group in groups]) + self.conn.terminate_instances([self.data['instance'].id]) + self.assertTrue(self.conn.release_address(self.data['public_ip'])) + + +if __name__ == "__main__": + suites = {'address': unittest.makeSuite(AddressTests), + 'security_group': unittest.makeSuite(SecurityGroupTests) + } + sys.exit(base.run_tests(suites)) diff --git a/smoketests/proxy.sh b/smoketests/proxy.sh new file mode 100755 index 000000000..9b3f3108a --- /dev/null +++ b/smoketests/proxy.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This is a simple shell script that uses netcat to set up a proxy to the +# metadata server on port 80 and to a google ip on port 8080. This is meant +# to be passed in by a script to an instance via user data, so that +# automatic testing of network connectivity can be performed. + +# Example usage: +# euca-run-instances -t m1.tiny -f proxy.sh ami-tty + +mkfifo backpipe1 +mkfifo backpipe2 + +# NOTE(vish): proxy metadata on port 80 +while true; do + nc -l -p 80 0<backpipe1 | nc 169.254.169.254 80 1>backpipe1 +done & + +# NOTE(vish): proxy google on port 8080 +while true; do + nc -l -p 8080 0<backpipe2 | nc 74.125.19.99 80 1>backpipe2 +done & -- cgit From 9f169fdef93898097e33b5e1c0318f543ced672e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Tue, 22 Feb 2011 18:41:41 -0800 Subject: Reverted change to focus on the core bug - kernel_id and ramdisk_id are optional --- nova/api/openstack/servers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 11a84687d..41b05cbb4 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -144,11 +144,13 @@ class Controller(wsgi.Controller): metadata stored in Glance as 'image_properties' """ def lookup(param): - properties = image.get('properties') - if properties: - return properties.get(param) - else: - return image.get(param) + _image_id = image_id + try: + return image['properties'][param] + except KeyError: + LOG.debug( + _("%(param)s property not found for image %(_image_id)s") % + locals()) image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From 3ef3dfc2f6c8b9cc14119793df4990432ff74ea2 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Tue, 22 Feb 2011 18:42:23 -0800 Subject: Return null if no kernel_id / ramdisk_id --- nova/api/openstack/servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 41b05cbb4..d83bd34ab 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -151,6 +151,7 @@ class Controller(wsgi.Controller): LOG.debug( _("%(param)s property not found for image %(_image_id)s") % locals()) + return None image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From 409ee5ff22bbd62d94a7afb1df1e6b7353c95d83 Mon Sep 17 00:00:00 2001 From: Todd Willey <todd@ansolabs.com> Date: Tue, 22 Feb 2011 23:42:49 -0500 Subject: Update the admin client to deal with VPNs and have a function host list. --- nova/adminclient.py | 63 ++++++++++++++++++++++++---- nova/api/ec2/admin.py | 111 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 152 insertions(+), 22 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index c614b274c..fe2aca351 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -23,6 +23,8 @@ import base64 import boto import boto.exception import httplib +import re +import string from boto.ec2.regioninfo import RegionInfo @@ -165,19 +167,20 @@ class HostInfo(object): **Fields Include** - * Disk stats - * Running Instances - * Memory stats - * CPU stats - * Network address info - * Firewall info - * Bridge and devices - + * Hostname + * Compute service status + * Volume service status + * Instance count + * Volume count """ def __init__(self, connection=None): self.connection = connection self.hostname = None + self.compute = None + self.volume = None + self.instance_count = 0 + self.volume_count = 0 def __repr__(self): return 'Host:%s' % self.hostname @@ -188,7 +191,39 @@ class HostInfo(object): # this is needed by the sax parser, so ignore the ugly name def endElement(self, name, value, connection): - setattr(self, name, value) + fixed_name = string.lower(re.sub(r'([A-Z])', r'_\1', name)) + setattr(self, fixed_name, value) + + +class Vpn(object): + """ + Information about a Vpn, as parsed through SAX + + **Fields Include** + + * instance_id + * project_id + * public_ip + * public_port + * created_at + * internal_ip + * state + """ + + def __init__(self, connection=None): + self.connection = connection + self.instance_id = None + self.project_id = None + + def __repr__(self): + return 'Vpn:%s:%s' % (self.project_id, self.instance_id) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + fixed_name = string.lower(re.sub(r'([A-Z])', r'_\1', name)) + setattr(self, fixed_name, value) class InstanceType(object): @@ -422,6 +457,16 @@ class NovaAdminClient(object): zip = self.apiconn.get_object('GenerateX509ForUser', params, UserInfo) return zip.file + def start_vpn(self, project): + """ + Starts the vpn for a user + """ + return self.apiconn.get_object('StartVpn', {'Project': project}, Vpn) + + def get_vpns(self): + """Return a list of vpn with project name""" + return self.apiconn.get_list('DescribeVpns', {}, [('item', Vpn)]) + def get_hosts(self): return self.apiconn.get_list('DescribeHosts', {}, [('item', HostInfo)]) diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index 735951082..e2a05fce1 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -21,14 +21,18 @@ Admin API controller, exposed through http via the api worker. """ import base64 +import datetime from nova import db from nova import exception +from nova import flags from nova import log as logging +from nova import utils from nova.auth import manager from nova.compute import instance_types +FLAGS = flags.FLAGS LOG = logging.getLogger('nova.api.ec2.admin') @@ -55,12 +59,25 @@ def project_dict(project): return {} -def host_dict(host): +def host_dict(host, compute_service, instances, volume_service, volumes, now): """Convert a host model object to a result dict""" - if host: - return host.state - else: - return {} + rv = {'hostanme': host, 'instance_count': len(instances), + 'volume_count': len(volumes)} + if compute_service: + latest = compute_service['updated_at'] or compute_service['created_at'] + delta = now - latest + if delta.seconds <= FLAGS.service_down_time: + rv['compute'] = 'up' + else: + rv['compute'] = 'down' + if volume_service: + latest = volume_service['updated_at'] or volume_service['created_at'] + delta = now - latest + if delta.seconds <= FLAGS.service_down_time: + rv['volume'] = 'up' + else: + rv['volume'] = 'down' + return rv def instance_dict(name, inst): @@ -71,6 +88,25 @@ def instance_dict(name, inst): 'flavor_id': inst['flavorid']} +def vpn_dict(project, vpn_instance): + rv = {'project_id': project.id, + 'public_ip': project.vpn_ip, + 'public_port': project.vpn_port} + if vpn_instance: + rv['instance_id'] = vpn_instance['ec2_id'] + rv['created_at'] = utils.isotime(vpn_instance['created_at']) + address = vpn_instance.get('fixed_ip', None) + if address: + rv['internal_ip'] = address['address'] + if utils.vpn_ping(project.vpn_ip, project.vpn_port): + rv['state'] = 'running' + else: + rv['state'] = 'down' + else: + rv['state'] = 'pending' + return rv + + class AdminController(object): """ API Controller for users, hosts, nodes, and workers. @@ -223,19 +259,68 @@ class AdminController(object): raise exception.ApiError(_('operation must be add or remove')) return True + def _vpn_for(self, context, project_id): + """Get the VPN instance for a project ID.""" + for instance in db.instance_get_all_by_project(context, project_id): + if (instance['image_id'] == FLAGS.vpn_image_id + and not instance['state_description'] in + ['shutting_down', 'shutdown']): + return instance + + def start_vpn(self, context, project): + instance = self._vpn_for(context, project) + if not instance: + # NOTE(vish) import delayed because of __init__.py + from nova.cloudpipe import pipelib + pipe = pipelib.CloudPipe() + try: + pipe.launch_vpn_instance(project) + except db.NoMoreNetworks: + raise exception.ApiError("Unable to claim IP for VPN instance" + ", ensure it isn't running, and try " + "again in a few minutes") + instance = self._vpn_for(context, project) + return {'instance_id': instance['ec2_id']} + + def describe_vpns(self, context): + vpns = [] + for project in manager.AuthManager().get_projects(): + instance = self._vpn_for(context, project.id) + vpns.append(vpn_dict(project, instance)) + return {'items': vpns} + # FIXME(vish): these host commands don't work yet, perhaps some of the # required data can be retrieved from service objects? - def describe_hosts(self, _context, **_kwargs): + def describe_hosts(self, context, **_kwargs): """Returns status info for all nodes. Includes: - * Disk Space - * Instance List - * RAM used - * CPU used - * DHCP servers running - * Iptables / bridges + * Hostname + * Compute (up, down, None) + * Instance count + * Volume (up, down, None) + * Volume Count """ - return {'hostSet': [host_dict(h) for h in db.host_get_all()]} + services = db.service_get_all(context) + now = datetime.datetime.utcnow() + hosts = [] + rv = [] + for host in [service['host'] for service in services]: + if not host in hosts: + hosts.append(host) + for host in hosts: + compute = [s for s in services if s['host'] == host \ + and s['binary'] == 'nova-compute'] + if compute: + compute = compute[0] + instances = db.instance_get_all_by_host(context, host) + volume = [s for s in services if s['host'] == host \ + and s['binary'] == 'nova-volume'] + if volume: + volume = volume[0] + volumes = db.volume_get_all_by_host(context, host) + rv.append(host_dict(host, compute, instances, volume, volumes, + now)) + return {'hosts': rv} def describe_host(self, _context, name, **_kwargs): """Returns status info for single node.""" -- cgit From 943b863bef09a4e2b3de36c26a3fabbcc6093411 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Tue, 22 Feb 2011 23:21:01 -0800 Subject: Lots of test fixing --- nova/api/ec2/cloud.py | 5 ++- nova/compute/api.py | 2 +- nova/db/sqlalchemy/api.py | 3 +- nova/tests/api/openstack/test_servers.py | 2 +- nova/tests/test_cloud.py | 75 ++++++++------------------------ nova/tests/test_network.py | 3 ++ nova/tests/test_scheduler.py | 3 ++ nova/tests/test_virt.py | 3 ++ nova/virt/fake.py | 4 +- 9 files changed, 36 insertions(+), 64 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 882cdcfc9..99b6d5cb6 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -529,8 +529,9 @@ class CloudController(object): def get_ajax_console(self, context, instance_id, **kwargs): ec2_id = instance_id[0] - internal_id = ec2_id_to_id(ec2_id) - return self.compute_api.get_ajax_console(context, internal_id) + instance_id = ec2_id_to_id(ec2_id) + return self.compute_api.get_ajax_console(context, + instance_id=instance_id) def describe_volumes(self, context, volume_id=None, **kwargs): if volume_id: diff --git a/nova/compute/api.py b/nova/compute/api.py index 81ea6dc53..0caadc32e 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -447,7 +447,7 @@ class API(base.Base): {'method': 'authorize_ajax_console', 'args': {'token': output['token'], 'host': output['host'], 'port': output['port']}}) - return {'url': '%s?token=%s' % (FLAGS.ajax_console_proxy_url, + return {'url': '%s/?token=%s' % (FLAGS.ajax_console_proxy_url, output['token'])} def get_console_output(self, context, instance_id): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..2ab402e1c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1046,7 +1046,8 @@ def network_create_safe(context, values): @require_admin_context def network_disassociate(context, network_id): - network_update(context, network_id, {'project_id': None}) + network_update(context, network_id, {'project_id': None, + 'host': None}) @require_admin_context diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..589f3d3eb 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -84,7 +84,7 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "vcpus": 0, "local_gb": 0, "hostname": "", - "host": "", + "host": None, "instance_type": "", "user_data": "", "reservation_id": "", diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 1824d24bc..2c6dc5973 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -66,6 +66,7 @@ class CloudTestCase(test.TestCase): # set up services self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') self.network = self.start_service('network') self.manager = manager.AuthManager() @@ -73,8 +74,12 @@ class CloudTestCase(test.TestCase): self.project = self.manager.create_project('proj', 'admin', 'proj') self.context = context.RequestContext(user=self.user, project=self.project) + host = self.network.get_network_host(self.context.elevated()) def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.compute.kill() @@ -201,27 +206,32 @@ class CloudTestCase(test.TestCase): 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) + greenthread.sleep(0.3) instance_id = rv['instancesSet'][0]['instanceId'] output = self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) + instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_ajax_console(self): + image_id = FLAGS.default_image kwargs = {'image_id': image_id} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) - self.assertEquals(b64decode(output['output']), - 'http://fakeajaxconsole.com/?token=FAKETOKEN') + greenthread.sleep(0.3) + output = self.cloud.get_ajax_console(context=self.context, + instance_id=[instance_id]) + self.assertEquals(output['url'], + '%s/?token=FAKETOKEN' % FLAGS.ajax_console_proxy_url) # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) - rv = yield self.cloud.terminate_instances(self.context, [instance_id]) + rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_key_generation(self): result = self._create_key('test') @@ -297,57 +307,6 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, instance1['id']) db.service_destroy(self.context, comp1['id']) - def test_instance_update_state(self): - # TODO(termie): what is this code even testing? - def instance(num): - return { - 'reservation_id': 'r-1', - 'instance_id': 'i-%s' % num, - 'image_id': 'ami-%s' % num, - 'private_dns_name': '10.0.0.%s' % num, - 'dns_name': '10.0.0%s' % num, - 'ami_launch_index': str(num), - 'instance_type': 'fake', - 'availability_zone': 'fake', - 'key_name': None, - 'kernel_id': 'fake', - 'ramdisk_id': 'fake', - 'groups': ['default'], - 'product_codes': None, - 'state': 0x01, - 'user_data': ''} - rv = self.cloud._format_describe_instances(self.context) - logging.error(str(rv)) - self.assertEqual(len(rv['reservationSet']), 0) - - # simulate launch of 5 instances - # self.cloud.instances['pending'] = {} - #for i in xrange(5): - # inst = instance(i) - # self.cloud.instances['pending'][inst['instance_id']] = inst - - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - # report 4 nodes each having 1 of the instances - #for i in xrange(4): - # self.cloud.update_state('instances', - # {('node-%s' % i): {('i-%s' % i): - # instance(i)}}) - - # one instance should be pending still - #self.assert_(len(self.cloud.instances['pending'].keys()) == 1) - - # check that the reservations collapse - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - - # check that we can get metadata for each instance - #for i in xrange(4): - # data = self.cloud.get_metadata(instance(i)['private_dns_name']) - # self.assert_(data['meta-data']['ami-id'] == 'ami-%s' % i) - @staticmethod def _fake_set_image_description(ctxt, image_id, description): from nova.objectstore import handler diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f3..53cfea276 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -117,6 +117,9 @@ class NetworkTestCase(test.TestCase): utils.to_global_ipv6( network_ref['cidr_v6'], instance_ref['mac_address'])) + self._deallocate_address(0, address) + db.instance_destroy(context.get_admin_context(), + instance_ref['id']) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 250170072..8e4a4daf5 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -118,6 +118,7 @@ class ZoneSchedulerTestCase(test.TestCase): arg = IgnoreArg() db.service_get_all_by_topic(arg, arg).AndReturn(service_list) self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + self.mox.StubOutWithMock(db, 'instance_create', use_mock_anything=True) rpc.cast(ctxt, 'compute.host1', {'method': 'run_instance', @@ -150,6 +151,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" @@ -270,6 +272,7 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.driver.schedule_run_instance, self.context, instance_id) + db.instance_destroy(self.context, instance_id) for instance_id in instance_ids1: compute1.terminate_instance(self.context, instance_id) for instance_id in instance_ids2: diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..5b3247df9 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -204,6 +204,7 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, testuri) + db.instance_destroy(user_context, instance_ref['id']) def tearDown(self): super(LibvirtConnTestCase, self).tearDown() @@ -365,6 +366,7 @@ class IptablesFirewallTestCase(test.TestCase): '--dports 80:81 -j ACCEPT' % security_group_chain \ in self.out_rules, "TCP port 80/81 acceptance rule wasn't added") + db.instance_destroy(admin_ctxt, instance_ref['id']) class NWFilterTestCase(test.TestCase): @@ -514,3 +516,4 @@ class NWFilterTestCase(test.TestCase): self.fw.apply_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 92749f38a..4346dffc1 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -319,7 +319,9 @@ class FakeConnection(object): return 'FAKE CONSOLE OUTPUT' def get_ajax_console(self, instance): - return 'http://fakeajaxconsole.com/?token=FAKETOKEN' + return {'token': 'FAKETOKEN', + 'host': 'fakeajaxconsole.com', + 'port': 6969} def get_console_pool_info(self, console_type): return {'address': '127.0.0.1', -- cgit From 015900b215805808d8cc3138b0f4deb2c0941f76 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Tue, 22 Feb 2011 23:30:52 -0800 Subject: remove unnecessary stubout --- nova/tests/test_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 8e4a4daf5..b6888c4d2 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -118,7 +118,6 @@ class ZoneSchedulerTestCase(test.TestCase): arg = IgnoreArg() db.service_get_all_by_topic(arg, arg).AndReturn(service_list) self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) - self.mox.StubOutWithMock(db, 'instance_create', use_mock_anything=True) rpc.cast(ctxt, 'compute.host1', {'method': 'run_instance', -- cgit From 60ed7c9c52306d08b1ad3e759e173931b0a495a8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 00:59:15 -0800 Subject: fix failures --- run_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 4084d8d80..01a1cf8bc 100644 --- a/run_tests.py +++ b/run_tests.py @@ -181,6 +181,7 @@ class NovaTestResult(result.TextTestResult): return str(test) def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) if self.showAll: self.colorizer.write("OK", 'green') self.stream.writeln() @@ -188,7 +189,8 @@ class NovaTestResult(result.TextTestResult): self.stream.write('.') self.stream.flush() - def addFailure(self, test): + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) if self.showAll: self.colorizer.write("FAIL", 'red') self.stream.writeln() -- cgit From 2bec58e35ab1f2df543e50d399433f76e98210d7 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 01:13:41 -0800 Subject: move db creation into fixtures and clean db for each test --- nova/test.py | 17 ++--------------- nova/tests/__init__.py | 15 +++++++++++++++ nova/tests/api/openstack/__init__.py | 4 ++-- nova/tests/api/openstack/test_adminapi.py | 11 +++++------ nova/tests/api/openstack/test_api.py | 4 ++-- nova/tests/api/openstack/test_auth.py | 14 +++++++------- nova/tests/api/openstack/test_common.py | 5 +++-- nova/tests/api/openstack/test_faults.py | 4 ++-- nova/tests/api/openstack/test_flavors.py | 10 ++++------ nova/tests/api/openstack/test_images.py | 14 ++++++++++---- nova/tests/api/openstack/test_ratelimiting.py | 15 +++++++-------- nova/tests/api/openstack/test_servers.py | 10 ++++------ nova/tests/api/openstack/test_shared_ip_groups.py | 7 ++++--- nova/tests/api/openstack/test_zones.py | 10 ++++------ nova/tests/api/test_wsgi.py | 6 +++--- nova/tests/objectstore_unittest.py | 1 + nova/tests/test_direct.py | 1 + nova/tests/test_scheduler.py | 1 + nova/tests/test_virt.py | 3 ++- 19 files changed, 79 insertions(+), 73 deletions(-) diff --git a/nova/test.py b/nova/test.py index bff43b6c7..42accffa7 100644 --- a/nova/test.py +++ b/nova/test.py @@ -26,15 +26,14 @@ import datetime import unittest import mox +import shutil import stubout from nova import context from nova import db from nova import fakerabbit from nova import flags -from nova import log as logging from nova import rpc -from nova.network import manager as network_manager FLAGS = flags.FLAGS @@ -64,15 +63,7 @@ class TestCase(unittest.TestCase): # now that we have some required db setup for the system # to work properly. self.start = datetime.datetime.utcnow() - ctxt = context.get_admin_context() - if db.network_count(ctxt) != 5: - network_manager.VlanManager().create_networks(ctxt, - FLAGS.fixed_range, - 5, 16, - FLAGS.fixed_range_v6, - FLAGS.vlan_start, - FLAGS.vpn_start, - ) + shutil.copyfile("clean.sqlite", "tests.sqlite") # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators @@ -93,9 +84,6 @@ class TestCase(unittest.TestCase): self.mox.VerifyAll() # NOTE(vish): Clean up any ips associated during the test. ctxt = context.get_admin_context() - db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, - self.start) - db.network_disassociate_all(ctxt) rpc.Consumer.attach_to_eventlet = self.originalAttach for x in self.injected: try: @@ -106,7 +94,6 @@ class TestCase(unittest.TestCase): if FLAGS.fake_rabbit: fakerabbit.reset_all() - db.security_group_destroy_all(ctxt) super(TestCase, self).tearDown() finally: self.reset_flags() diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index 592d5bea9..5472bdaf2 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -37,5 +37,20 @@ setattr(__builtin__, '_', lambda x: x) def setup(): + import shutil + from nova import context + from nova import flags from nova.db import migration + from nova.network import manager as network_manager + from nova.tests import fake_flags + FLAGS = flags.FLAGS migration.db_sync() + ctxt = context.get_admin_context() + network_manager.VlanManager().create_networks(ctxt, + FLAGS.fixed_range, + 5, 16, + FLAGS.fixed_range_v6, + FLAGS.vlan_start, + FLAGS.vpn_start, + ) + shutil.copyfile("tests.sqlite", "clean.sqlite") diff --git a/nova/tests/api/openstack/__init__.py b/nova/tests/api/openstack/__init__.py index 77b1dd37f..e18120285 100644 --- a/nova/tests/api/openstack/__init__.py +++ b/nova/tests/api/openstack/__init__.py @@ -16,7 +16,7 @@ # under the License. import webob.dec -import unittest +from nova import test from nova import context from nova import flags @@ -33,7 +33,7 @@ def simple_wsgi(req): return "" -class RateLimitingMiddlewareTest(unittest.TestCase): +class RateLimitingMiddlewareTest(test.TestCase): def test_get_action_name(self): middleware = RateLimitingMiddleware(simple_wsgi) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 73120c31d..dfce1b127 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -15,13 +15,13 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import stubout import webob from paste import urlmap from nova import flags +from nova import test from nova.api import openstack from nova.api.openstack import ratelimiting from nova.api.openstack import auth @@ -30,9 +30,10 @@ from nova.tests.api.openstack import fakes FLAGS = flags.FLAGS -class AdminAPITest(unittest.TestCase): +class AdminAPITest(test.TestCase): def setUp(self): + super(AdminAPITest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -44,6 +45,7 @@ class AdminAPITest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + super(AdminAPITest, self).tearDown() def test_admin_enabled(self): FLAGS.allow_admin_api = True @@ -58,8 +60,5 @@ class AdminAPITest(unittest.TestCase): # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) - self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are unavailable. - -if __name__ == '__main__': - unittest.main() + self.assertEqual(res.status_int, 200) diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index db0fe1060..5112c486f 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -15,17 +15,17 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import webob.exc import webob.dec from webob import Request +from nova import test from nova.api import openstack from nova.api.openstack import faults -class APITest(unittest.TestCase): +class APITest(test.TestCase): def _wsgi_app(self, inner_app): # simpler version of the app than fakes.wsgi_app diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 0dd65d321..13f6c3a1c 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -16,7 +16,6 @@ # under the License. import datetime -import unittest import stubout import webob @@ -27,12 +26,14 @@ import nova.api.openstack.auth import nova.auth.manager from nova import auth from nova import context +from nova import test from nova.tests.api.openstack import fakes -class Test(unittest.TestCase): +class Test(test.TestCase): def setUp(self): + super(Test, self).setUp() self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) @@ -45,6 +46,7 @@ class Test(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() fakes.fake_data_store = {} + super(Test, self).tearDown() def test_authorize_user(self): f = fakes.FakeAuthManager() @@ -128,8 +130,9 @@ class Test(unittest.TestCase): self.assertEqual(result.status, '401 Unauthorized') -class TestLimiter(unittest.TestCase): +class TestLimiter(test.TestCase): def setUp(self): + super(TestLimiter, self).setUp() self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) @@ -141,6 +144,7 @@ class TestLimiter(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() fakes.fake_data_store = {} + super(TestLimiter, self).tearDown() def test_authorize_token(self): f = fakes.FakeAuthManager() @@ -161,7 +165,3 @@ class TestLimiter(unittest.TestCase): result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 9d9837cc9..59d850157 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -19,14 +19,14 @@ Test suites for 'common' code used throughout the OpenStack HTTP API. """ -import unittest from webob import Request +from nova import test from nova.api.openstack.common import limited -class LimiterTest(unittest.TestCase): +class LimiterTest(test.TestCase): """ Unit tests for the `nova.api.openstack.common.limited` method which takes in a list of items and, depending on the 'offset' and 'limit' GET params, @@ -37,6 +37,7 @@ class LimiterTest(unittest.TestCase): """ Run before each test. """ + super(LimiterTest, self).setUp() self.tiny = range(1) self.small = range(10) self.medium = range(1000) diff --git a/nova/tests/api/openstack/test_faults.py b/nova/tests/api/openstack/test_faults.py index fda2b5ede..7667753f4 100644 --- a/nova/tests/api/openstack/test_faults.py +++ b/nova/tests/api/openstack/test_faults.py @@ -15,15 +15,15 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import webob import webob.dec import webob.exc +from nova import test from nova.api.openstack import faults -class TestFaults(unittest.TestCase): +class TestFaults(test.TestCase): def test_fault_parts(self): req = webob.Request.blank('/.xml') diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 1bdaea161..761265965 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -15,18 +15,18 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest - import stubout import webob +from nova import test import nova.api from nova.api.openstack import flavors from nova.tests.api.openstack import fakes -class FlavorsTest(unittest.TestCase): +class FlavorsTest(test.TestCase): def setUp(self): + super(FlavorsTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -36,6 +36,7 @@ class FlavorsTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() + super(FlavorsTest, self).tearDown() def test_get_flavor_list(self): req = webob.Request.blank('/v1.0/flavors') @@ -43,6 +44,3 @@ class FlavorsTest(unittest.TestCase): def test_get_flavor_by_id(self): pass - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 8ab4d7569..e232bc3d5 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -22,7 +22,6 @@ and as a WSGI layer import json import datetime -import unittest import stubout import webob @@ -30,6 +29,7 @@ import webob from nova import context from nova import exception from nova import flags +from nova import test from nova import utils import nova.api.openstack from nova.api.openstack import images @@ -130,12 +130,13 @@ class BaseImageServiceTests(object): self.assertEquals(1, num_images) -class LocalImageServiceTest(unittest.TestCase, +class LocalImageServiceTest(test.TestCase, BaseImageServiceTests): """Tests the local image service""" def setUp(self): + super(LocalImageServiceTest, self).setUp() self.stubs = stubout.StubOutForTesting() service_class = 'nova.image.local.LocalImageService' self.service = utils.import_object(service_class) @@ -145,14 +146,16 @@ class LocalImageServiceTest(unittest.TestCase, self.service.delete_all() self.service.delete_imagedir() self.stubs.UnsetAll() + super(LocalImageServiceTest, self).tearDown() -class GlanceImageServiceTest(unittest.TestCase, +class GlanceImageServiceTest(test.TestCase, BaseImageServiceTests): """Tests the local image service""" def setUp(self): + super(GlanceImageServiceTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.stub_out_glance(self.stubs) fakes.stub_out_compute_api_snapshot(self.stubs) @@ -163,9 +166,10 @@ class GlanceImageServiceTest(unittest.TestCase, def tearDown(self): self.stubs.UnsetAll() + super(GlanceImageServiceTest, self).tearDown() -class ImageControllerWithGlanceServiceTest(unittest.TestCase): +class ImageControllerWithGlanceServiceTest(test.TestCase): """Test of the OpenStack API /images application controller""" @@ -194,6 +198,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): 'image_type': 'ramdisk'}] def setUp(self): + super(ImageControllerWithGlanceServiceTest, self).setUp() self.orig_image_service = FLAGS.image_service FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.stubs = stubout.StubOutForTesting() @@ -208,6 +213,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.image_service = self.orig_image_service + super(ImageControllerWithGlanceServiceTest, self).tearDown() def test_get_image_index(self): req = webob.Request.blank('/v1.0/images') diff --git a/nova/tests/api/openstack/test_ratelimiting.py b/nova/tests/api/openstack/test_ratelimiting.py index 4c9d6bc23..9ae90ee20 100644 --- a/nova/tests/api/openstack/test_ratelimiting.py +++ b/nova/tests/api/openstack/test_ratelimiting.py @@ -1,15 +1,16 @@ import httplib import StringIO import time -import unittest import webob +from nova import test import nova.api.openstack.ratelimiting as ratelimiting -class LimiterTest(unittest.TestCase): +class LimiterTest(test.TestCase): def setUp(self): + super(LimiterTest, self).setUp() self.limits = { 'a': (5, ratelimiting.PER_SECOND), 'b': (5, ratelimiting.PER_MINUTE), @@ -83,9 +84,10 @@ class FakeLimiter(object): return self._delay -class WSGIAppTest(unittest.TestCase): +class WSGIAppTest(test.TestCase): def setUp(self): + super(WSGIAppTest, self).setUp() self.limiter = FakeLimiter(self) self.app = ratelimiting.WSGIApp(self.limiter) @@ -206,7 +208,7 @@ def wire_HTTPConnection_to_WSGI(host, app): httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection) -class WSGIAppProxyTest(unittest.TestCase): +class WSGIAppProxyTest(test.TestCase): def setUp(self): """Our WSGIAppProxy is going to call across an HTTPConnection to a @@ -218,6 +220,7 @@ class WSGIAppProxyTest(unittest.TestCase): at the WSGIApp. And the limiter isn't real -- it's a fake that behaves the way we tell it to. """ + super(WSGIAppProxyTest, self).setUp() self.limiter = FakeLimiter(self) app = ratelimiting.WSGIApp(self.limiter) wire_HTTPConnection_to_WSGI('100.100.100.100:80', app) @@ -238,7 +241,3 @@ class WSGIAppProxyTest(unittest.TestCase): self.limiter.mock('murder', 'brutus', None) self.proxy.perform('stab', 'brutus') self.assertRaises(AssertionError, shouldRaise) - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..ea29dcf9b 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -17,13 +17,13 @@ import datetime import json -import unittest import stubout import webob from nova import db from nova import flags +from nova import test import nova.api.openstack from nova.api.openstack import servers import nova.db.api @@ -108,9 +108,10 @@ def fake_compute_api(cls, req, id): return True -class ServersTest(unittest.TestCase): +class ServersTest(test.TestCase): def setUp(self): + super(ServersTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -141,6 +142,7 @@ class ServersTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + super(ServersTest, self).tearDown() def test_get_server_by_id(self): req = webob.Request.blank('/v1.0/servers/1') @@ -410,7 +412,3 @@ class ServersTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) - - -if __name__ == "__main__": - unittest.main() diff --git a/nova/tests/api/openstack/test_shared_ip_groups.py b/nova/tests/api/openstack/test_shared_ip_groups.py index c2fc3a203..b4de2ef41 100644 --- a/nova/tests/api/openstack/test_shared_ip_groups.py +++ b/nova/tests/api/openstack/test_shared_ip_groups.py @@ -15,19 +15,20 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest - import stubout +from nova import test from nova.api.openstack import shared_ip_groups -class SharedIpGroupsTest(unittest.TestCase): +class SharedIpGroupsTest(test.TestCase): def setUp(self): + super(SharedIpGroupsTest, self).setUp() self.stubs = stubout.StubOutForTesting() def tearDown(self): self.stubs.UnsetAll() + super(SharedIpGroupsTest, self).tearDown() def test_get_shared_ip_groups(self): pass diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index df497ef1b..555b206b9 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import stubout import webob @@ -22,6 +21,7 @@ import json import nova.db from nova import context from nova import flags +from nova import test from nova.api.openstack import zones from nova.tests.api.openstack import fakes @@ -60,8 +60,9 @@ def zone_get_all(context): password='qwerty')] -class ZonesTest(unittest.TestCase): +class ZonesTest(test.TestCase): def setUp(self): + super(ZonesTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -81,6 +82,7 @@ class ZonesTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + super(ZonesTest, self).tearDown() def test_get_zone_list(self): req = webob.Request.blank('/v1.0/zones') @@ -134,7 +136,3 @@ class ZonesTest(unittest.TestCase): self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('username' in res_dict['zone']) - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 44e2d615c..2c7852214 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -21,7 +21,7 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ -import unittest +from nova import test import routes import webob @@ -29,7 +29,7 @@ import webob from nova import wsgi -class Test(unittest.TestCase): +class Test(test.TestCase): def test_debug(self): @@ -92,7 +92,7 @@ class Test(unittest.TestCase): self.assertNotEqual(result.body, "123") -class SerializerTest(unittest.TestCase): +class SerializerTest(test.TestCase): def match(self, url, accept, expect): input_dict = dict(servers=dict(a=(2, 3))) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index da86e6e11..5a1be08eb 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -311,4 +311,5 @@ class S3APITestCase(test.TestCase): self.auth_manager.delete_user('admin') self.auth_manager.delete_project('admin') stop_listening = defer.maybeDeferred(self.listening_port.stopListening) + super(S3APITestCase, self).tearDown() return defer.DeferredList([stop_listening]) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 7656f5396..b6bfab534 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -52,6 +52,7 @@ class DirectTestCase(test.TestCase): def tearDown(self): direct.ROUTES = {} + super(DirectTestCase, self).tearDown() def test_delegated_auth(self): req = webob.Request.blank('/fake/context') diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 9d458244b..1bad364e5 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -150,6 +150,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..7aadd65d5 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -206,9 +206,9 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) def tearDown(self): - super(LibvirtConnTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + super(LibvirtConnTestCase, self).tearDown() class IptablesFirewallTestCase(test.TestCase): @@ -388,6 +388,7 @@ class NWFilterTestCase(test.TestCase): def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) + super(NWFilterTestCase, self).tearDown() def test_cidr_rule_nwfilter_xml(self): cloud_controller = cloud.CloudController() -- cgit From 3b2a8b516fd9dbd08563c709e14323d571b8efee Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 01:52:07 -0800 Subject: speed up network tests --- nova/tests/__init__.py | 3 ++- nova/tests/fake_flags.py | 4 ++-- nova/tests/test_network.py | 8 +++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index 5472bdaf2..5afd9389d 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -48,7 +48,8 @@ def setup(): ctxt = context.get_admin_context() network_manager.VlanManager().create_networks(ctxt, FLAGS.fixed_range, - 5, 16, + FLAGS.num_networks, + FLAGS.network_size, FLAGS.fixed_range_v6, FLAGS.vlan_start, FLAGS.vpn_start, diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 575fefff6..a8291a968 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -29,8 +29,8 @@ FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') flags.DECLARE('fake_network', 'nova.network.manager') -FLAGS.network_size = 16 -FLAGS.num_networks = 5 +FLAGS.network_size = 8 +FLAGS.num_networks = 2 FLAGS.fake_network = True flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f3..ccb5298bd 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -42,15 +42,13 @@ class NetworkTestCase(test.TestCase): # flags in the corresponding section in nova-dhcpbridge self.flags(connection_type='fake', fake_call=True, - fake_network=True, - network_size=16, - num_networks=5) + fake_network=True) self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] self.network = utils.import_object(FLAGS.network_manager) self.context = context.RequestContext(project=None, user=self.user) - for i in range(5): + for i in range(FLAGS.num_networks): name = 'project%s' % i project = self.manager.create_project(name, 'netuser', name) self.projects.append(project) @@ -192,7 +190,7 @@ class NetworkTestCase(test.TestCase): first = self._create_address(0) lease_ip(first) instance_ids = [] - for i in range(1, 5): + for i in range(1, FLAGS.num_networks): instance_ref = self._create_instance(i, mac=utils.generate_mac()) instance_ids.append(instance_ref['id']) address = self._create_address(i, instance_ref['id']) -- cgit From a9075d4edc126b95910258face7f00073449073d Mon Sep 17 00:00:00 2001 From: Salvatore Orlando <salvatore.orlando@eu.citrix.com> Date: Wed, 23 Feb 2011 10:27:30 +0000 Subject: FlatManager.init_host now inhibits call to method in superclass. Floating IP methods have been redefined in FlatManager to raise NotImplementedError --- nova/network/manager.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nova/network/manager.py b/nova/network/manager.py index c6eba225e..a7f263daa 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -322,6 +322,17 @@ class FlatManager(NetworkManager): """ timeout_fixed_ips = False + def init_host(self): + """Do any initialization that needs to be run if this is a + standalone service. + """ + #Fix for bug 723298 - do not call init_host on superclass + #Following code has been copied for NetworkManager.init_host + ctxt = context.get_admin_context() + for network in self.db.host_get_networks(ctxt, self.host): + self._on_set_network_host(ctxt, network['id']) + + def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute @@ -406,6 +417,22 @@ class FlatManager(NetworkManager): net['dns'] = FLAGS.flat_network_dns self.db.network_update(context, network_id, net) + def allocate_floating_ip(self, context, project_id): + #Fix for bug 723298 + raise NotImplementedError() + + def associate_floating_ip(self, context, floating_address, fixed_address): + #Fix for bug 723298 + raise NotImplementedError() + + def disassociate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + + def deallocate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + class FlatDHCPManager(FlatManager): """Flat networking with dhcp. -- cgit From 79a4c527fbb75bc563721fa23be4ea4aa97b39ee Mon Sep 17 00:00:00 2001 From: Salvatore Orlando <salvatore.orlando@eu.citrix.com> Date: Wed, 23 Feb 2011 11:49:47 +0000 Subject: Fixed pep8 errors --- nova/network/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index a7f263daa..1df193be0 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -332,7 +332,6 @@ class FlatManager(NetworkManager): for network in self.db.host_get_networks(ctxt, self.host): self._on_set_network_host(ctxt, network['id']) - def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute -- cgit From d160455b77d7e180f252f4b412e3f65d7286b51f Mon Sep 17 00:00:00 2001 From: termie <github@anarkystic.com> Date: Wed, 23 Feb 2011 08:45:27 -0800 Subject: allow users to omit 'nova.tests' with run_tests --- run_tests.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 6d96454b9..47e3ee317 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,6 +17,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + import gettext import os import unittest @@ -62,6 +79,15 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': logging.setup() + # If any argument looks like a test name but doesn't have "nova.tests" in + # front of it, automatically add that so we don't have to type as much + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('nova.tests.%s' % x) + else: + argv.append(x) + c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, @@ -70,4 +96,4 @@ if __name__ == '__main__': runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, config=c) - sys.exit(not core.run(config=c, testRunner=runner)) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) -- cgit From 2bbbfc5af62db57158a8d6aa26912ba234d0296e Mon Sep 17 00:00:00 2001 From: termie <github@anarkystic.com> Date: Wed, 23 Feb 2011 08:46:11 -0800 Subject: dump error output directly on short import errors --- run_tests.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 70212cc6a..4e8159e7b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -40,7 +40,18 @@ done function run_tests { # Just run the test suites in current environment ${wrapper} rm -f nova.sqlite - ${wrapper} $NOSETESTS + ${wrapper} $NOSETESTS 2> run_tests.err.log + # If we get some short import error right away, print the error log directly + RESULT=$? + if [ "$RESULT" -ne "0" ]; + then + ERRSIZE=`wc -l run_tests.err.log | awk '{print \$1}'` + if [ "$ERRSIZE" -lt "40" ]; + then + cat run_tests.err.log + fi + fi + return $RESULT } NOSETESTS="python run_tests.py $noseargs" -- cgit From ef0dfb6809f31cfe8ca8056892fc9dcc2f00a0d7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 09:40:43 -0800 Subject: Changed unit test to refer to compute API, per Todd's suggestion. Avoids needing to extend our implementation of the EC2 API. --- nova/api/ec2/cloud.py | 6 +----- nova/tests/test_quota.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 5db865b02..882cdcfc9 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -783,9 +783,6 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) - # NOTE(justinsb): the EC2 API doesn't support metadata here, but this - # is needed for the unit tests. Maybe the unit tests shouldn't be - # calling the EC2 code instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), @@ -800,8 +797,7 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone'), - metadata=kwargs.get('metadata', [])) + 'AvailabilityZone')) return self._format_run_instances(context, instances[0]['reservation_id']) diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 36ccc273e..1e42fddf3 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -16,6 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nova import compute from nova import context from nova import db from nova import flags @@ -168,7 +169,7 @@ class QuotaTestCase(test.TestCase): metadata = {} for i in range(FLAGS.quota_metadata_items + 1): metadata['key%s' % i] = 'value%s' % i - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, -- cgit From 3c09d486f862de7069b848e8124787cfbf4247f8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 11:20:52 -0800 Subject: use flags for sqlite db names and fix flags in dhcpbridge --- bin/nova-dhcpbridge | 13 +------------ nova/flags.py | 3 ++- nova/test.py | 11 +++++++---- nova/tests/__init__.py | 7 ++++++- nova/tests/fake_flags.py | 2 +- run_tests.py | 11 ++++++++--- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 04a1771f0..3dd9de367 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -105,18 +105,7 @@ def main(): logging.setup() interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') if int(os.environ.get('TESTING', '0')): - FLAGS.fake_rabbit = True - FLAGS.network_size = 16 - FLAGS.connection_type = 'fake' - FLAGS.fake_network = True - FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' - FLAGS.num_networks = 5 - path = os.path.abspath(os.path.join(os.path.dirname(__file__), - '..', - 'nova', - 'tests', - 'tests.sqlite')) - FLAGS.sql_connection = 'sqlite:///%s' % path + from nova.tests import fake_flags action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/nova/flags.py b/nova/flags.py index f64a62da9..ab1adc6e3 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -285,8 +285,9 @@ DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), DEFINE_string('logdir', None, 'output to a per-service log file in named ' 'directory') +DEFINE_string('sqlite_db', 'nova.sqlite', 'file name for sqlite') DEFINE_string('sql_connection', - 'sqlite:///$state_path/nova.sqlite', + 'sqlite:///$state_path/$sqlite_db', 'connection string for sql database') DEFINE_integer('sql_idle_timeout', 3600, diff --git a/nova/test.py b/nova/test.py index 6cbbb9e8e..9c961a987 100644 --- a/nova/test.py +++ b/nova/test.py @@ -22,12 +22,14 @@ Allows overriding of flags for use of fakes, and some black magic for inline callbacks. """ + import datetime +import os +import shutil import uuid import unittest import mox -import shutil import stubout from nova import context @@ -39,8 +41,8 @@ from nova import service FLAGS = flags.FLAGS -flags.DEFINE_bool('flush_db', True, - 'Flush the database before running fake tests') +flags.DEFINE_string('sqlite_clean_db', 'clean.sqlite', + 'File name of clean sqlite db') flags.DEFINE_bool('fake_tests', True, 'should we use everything for testing') @@ -65,7 +67,8 @@ class TestCase(unittest.TestCase): # now that we have some required db setup for the system # to work properly. self.start = datetime.datetime.utcnow() - shutil.copyfile("clean.sqlite", "tests.sqlite") + shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db), + os.path.join(FLAGS.state_path, FLAGS.sqlite_db)) # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index 5afd9389d..dbd433054 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -37,13 +37,17 @@ setattr(__builtin__, '_', lambda x: x) def setup(): + import os import shutil + from nova import context from nova import flags from nova.db import migration from nova.network import manager as network_manager from nova.tests import fake_flags + FLAGS = flags.FLAGS + migration.db_sync() ctxt = context.get_admin_context() network_manager.VlanManager().create_networks(ctxt, @@ -54,4 +58,5 @@ def setup(): FLAGS.vlan_start, FLAGS.vpn_start, ) - shutil.copyfile("tests.sqlite", "clean.sqlite") + shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_db), + os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db)) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index a8291a968..dcc8a676d 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -39,6 +39,6 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///tests.sqlite' +FLAGS.sqlite_db = "tests.sqlite" FLAGS.use_ipv6 = True FLAGS.logfile = 'tests.log' diff --git a/run_tests.py b/run_tests.py index 01a1cf8bc..88c42bd31 100644 --- a/run_tests.py +++ b/run_tests.py @@ -46,13 +46,17 @@ import unittest import sys from nose import config -from nose import result from nose import core +from nose import result +from nova import flags from nova import log as logging from nova.tests import fake_flags +FLAGS = flags.FLAGS + + class _AnsiColorizer(object): """ A colorizer is an object that loosely wraps around a stream, allowing @@ -259,10 +263,11 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': logging.setup() - testdir = os.path.abspath(os.path.join("nova","tests")) - testdb = os.path.join(testdir, "tests.sqlite") + testdb = os.path.join(FLAGS.state_path, + FLAGS.sqlite_db) if os.path.exists(testdb): os.unlink(testdb) + testdir = os.path.abspath(os.path.join("nova","tests")) c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, -- cgit From 48d4054e093a2faccbd819de8e9e02c03d28cda0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 11:26:38 -0800 Subject: fix for failing describe_instances test --- nova/api/ec2/cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 1b96567eb..e219fb30c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -675,7 +675,8 @@ class CloudController(object): instances = [] for ec2_id in instance_id: internal_id = ec2_id_to_id(ec2_id) - instance = self.compute_api.get(context, internal_id) + instance = self.compute_api.get(context, + instance_id=internal_id) instances.append(instance) else: instances = self.compute_api.get_all(context, **kwargs) -- cgit From 1b2d67e769ff1a6fe68a933e8b966d72588ce8ac Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 11:44:09 -0800 Subject: merged trunk --- nova/test.py | 18 ++++-------------- nova/tests/__init__.py | 13 +++++++++++++ nova/tests/api/openstack/__init__.py | 4 ++-- nova/tests/api/openstack/test_adminapi.py | 8 ++++---- nova/tests/api/openstack/test_api.py | 4 ++-- nova/tests/api/openstack/test_auth.py | 14 +++++++------- nova/tests/api/openstack/test_common.py | 5 +++-- nova/tests/api/openstack/test_faults.py | 4 ++-- nova/tests/api/openstack/test_flavors.py | 10 ++++------ nova/tests/api/openstack/test_images.py | 14 ++++++++++---- nova/tests/api/openstack/test_ratelimiting.py | 15 +++++++-------- nova/tests/api/openstack/test_servers.py | 10 ++++------ nova/tests/api/openstack/test_shared_ip_groups.py | 7 ++++--- nova/tests/api/openstack/test_zones.py | 10 ++++------ nova/tests/api/test_wsgi.py | 6 +++--- nova/tests/objectstore_unittest.py | 1 + nova/tests/test_direct.py | 1 + nova/tests/test_scheduler.py | 1 + nova/tests/test_virt.py | 3 ++- 19 files changed, 78 insertions(+), 70 deletions(-) diff --git a/nova/test.py b/nova/test.py index bff43b6c7..826a4bd11 100644 --- a/nova/test.py +++ b/nova/test.py @@ -26,15 +26,14 @@ import datetime import unittest import mox +import shutil import stubout from nova import context from nova import db from nova import fakerabbit from nova import flags -from nova import log as logging from nova import rpc -from nova.network import manager as network_manager FLAGS = flags.FLAGS @@ -65,14 +64,8 @@ class TestCase(unittest.TestCase): # to work properly. self.start = datetime.datetime.utcnow() ctxt = context.get_admin_context() - if db.network_count(ctxt) != 5: - network_manager.VlanManager().create_networks(ctxt, - FLAGS.fixed_range, - 5, 16, - FLAGS.fixed_range_v6, - FLAGS.vlan_start, - FLAGS.vpn_start, - ) + shutil.copyfile("tests.sqlite", "clean.sqlite") + assert(db.security_group_get_all(ctxt) == []) # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators @@ -86,6 +79,7 @@ class TestCase(unittest.TestCase): def tearDown(self): """Runs after each test method to finalize/tear down test environment.""" + shutil.copyfile("clean.sqlite", "tests.sqlite") try: self.mox.UnsetStubs() self.stubs.UnsetAll() @@ -93,9 +87,6 @@ class TestCase(unittest.TestCase): self.mox.VerifyAll() # NOTE(vish): Clean up any ips associated during the test. ctxt = context.get_admin_context() - db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, - self.start) - db.network_disassociate_all(ctxt) rpc.Consumer.attach_to_eventlet = self.originalAttach for x in self.injected: try: @@ -106,7 +97,6 @@ class TestCase(unittest.TestCase): if FLAGS.fake_rabbit: fakerabbit.reset_all() - db.security_group_destroy_all(ctxt) super(TestCase, self).tearDown() finally: self.reset_flags() diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index 592d5bea9..d3ab02887 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -38,4 +38,17 @@ setattr(__builtin__, '_', lambda x: x) def setup(): from nova.db import migration + from nova.network import manager as network_manager + from nova import context + from nova import flags + from nova.tests import fake_flags + FLAGS = flags.FLAGS migration.db_sync() + ctxt = context.get_admin_context() + network_manager.VlanManager().create_networks(ctxt, + FLAGS.fixed_range, + 5, 16, + FLAGS.fixed_range_v6, + FLAGS.vlan_start, + FLAGS.vpn_start, + ) diff --git a/nova/tests/api/openstack/__init__.py b/nova/tests/api/openstack/__init__.py index 77b1dd37f..e18120285 100644 --- a/nova/tests/api/openstack/__init__.py +++ b/nova/tests/api/openstack/__init__.py @@ -16,7 +16,7 @@ # under the License. import webob.dec -import unittest +from nova import test from nova import context from nova import flags @@ -33,7 +33,7 @@ def simple_wsgi(req): return "" -class RateLimitingMiddlewareTest(unittest.TestCase): +class RateLimitingMiddlewareTest(test.TestCase): def test_get_action_name(self): middleware = RateLimitingMiddleware(simple_wsgi) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 73120c31d..125fbe973 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -15,13 +15,13 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import stubout import webob from paste import urlmap from nova import flags +from nova import test from nova.api import openstack from nova.api.openstack import ratelimiting from nova.api.openstack import auth @@ -30,9 +30,10 @@ from nova.tests.api.openstack import fakes FLAGS = flags.FLAGS -class AdminAPITest(unittest.TestCase): +class AdminAPITest(test.TestCase): def setUp(self): + super(AdminAPITest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -44,6 +45,7 @@ class AdminAPITest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + super(AdminAPITest, self).tearDown() def test_admin_enabled(self): FLAGS.allow_admin_api = True @@ -61,5 +63,3 @@ class AdminAPITest(unittest.TestCase): self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are unavailable. -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index db0fe1060..5112c486f 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -15,17 +15,17 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import webob.exc import webob.dec from webob import Request +from nova import test from nova.api import openstack from nova.api.openstack import faults -class APITest(unittest.TestCase): +class APITest(test.TestCase): def _wsgi_app(self, inner_app): # simpler version of the app than fakes.wsgi_app diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 0dd65d321..13f6c3a1c 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -16,7 +16,6 @@ # under the License. import datetime -import unittest import stubout import webob @@ -27,12 +26,14 @@ import nova.api.openstack.auth import nova.auth.manager from nova import auth from nova import context +from nova import test from nova.tests.api.openstack import fakes -class Test(unittest.TestCase): +class Test(test.TestCase): def setUp(self): + super(Test, self).setUp() self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) @@ -45,6 +46,7 @@ class Test(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() fakes.fake_data_store = {} + super(Test, self).tearDown() def test_authorize_user(self): f = fakes.FakeAuthManager() @@ -128,8 +130,9 @@ class Test(unittest.TestCase): self.assertEqual(result.status, '401 Unauthorized') -class TestLimiter(unittest.TestCase): +class TestLimiter(test.TestCase): def setUp(self): + super(TestLimiter, self).setUp() self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) @@ -141,6 +144,7 @@ class TestLimiter(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() fakes.fake_data_store = {} + super(TestLimiter, self).tearDown() def test_authorize_token(self): f = fakes.FakeAuthManager() @@ -161,7 +165,3 @@ class TestLimiter(unittest.TestCase): result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 9d9837cc9..59d850157 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -19,14 +19,14 @@ Test suites for 'common' code used throughout the OpenStack HTTP API. """ -import unittest from webob import Request +from nova import test from nova.api.openstack.common import limited -class LimiterTest(unittest.TestCase): +class LimiterTest(test.TestCase): """ Unit tests for the `nova.api.openstack.common.limited` method which takes in a list of items and, depending on the 'offset' and 'limit' GET params, @@ -37,6 +37,7 @@ class LimiterTest(unittest.TestCase): """ Run before each test. """ + super(LimiterTest, self).setUp() self.tiny = range(1) self.small = range(10) self.medium = range(1000) diff --git a/nova/tests/api/openstack/test_faults.py b/nova/tests/api/openstack/test_faults.py index fda2b5ede..7667753f4 100644 --- a/nova/tests/api/openstack/test_faults.py +++ b/nova/tests/api/openstack/test_faults.py @@ -15,15 +15,15 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import webob import webob.dec import webob.exc +from nova import test from nova.api.openstack import faults -class TestFaults(unittest.TestCase): +class TestFaults(test.TestCase): def test_fault_parts(self): req = webob.Request.blank('/.xml') diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 1bdaea161..761265965 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -15,18 +15,18 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest - import stubout import webob +from nova import test import nova.api from nova.api.openstack import flavors from nova.tests.api.openstack import fakes -class FlavorsTest(unittest.TestCase): +class FlavorsTest(test.TestCase): def setUp(self): + super(FlavorsTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -36,6 +36,7 @@ class FlavorsTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() + super(FlavorsTest, self).tearDown() def test_get_flavor_list(self): req = webob.Request.blank('/v1.0/flavors') @@ -43,6 +44,3 @@ class FlavorsTest(unittest.TestCase): def test_get_flavor_by_id(self): pass - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 8ab4d7569..e232bc3d5 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -22,7 +22,6 @@ and as a WSGI layer import json import datetime -import unittest import stubout import webob @@ -30,6 +29,7 @@ import webob from nova import context from nova import exception from nova import flags +from nova import test from nova import utils import nova.api.openstack from nova.api.openstack import images @@ -130,12 +130,13 @@ class BaseImageServiceTests(object): self.assertEquals(1, num_images) -class LocalImageServiceTest(unittest.TestCase, +class LocalImageServiceTest(test.TestCase, BaseImageServiceTests): """Tests the local image service""" def setUp(self): + super(LocalImageServiceTest, self).setUp() self.stubs = stubout.StubOutForTesting() service_class = 'nova.image.local.LocalImageService' self.service = utils.import_object(service_class) @@ -145,14 +146,16 @@ class LocalImageServiceTest(unittest.TestCase, self.service.delete_all() self.service.delete_imagedir() self.stubs.UnsetAll() + super(LocalImageServiceTest, self).tearDown() -class GlanceImageServiceTest(unittest.TestCase, +class GlanceImageServiceTest(test.TestCase, BaseImageServiceTests): """Tests the local image service""" def setUp(self): + super(GlanceImageServiceTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.stub_out_glance(self.stubs) fakes.stub_out_compute_api_snapshot(self.stubs) @@ -163,9 +166,10 @@ class GlanceImageServiceTest(unittest.TestCase, def tearDown(self): self.stubs.UnsetAll() + super(GlanceImageServiceTest, self).tearDown() -class ImageControllerWithGlanceServiceTest(unittest.TestCase): +class ImageControllerWithGlanceServiceTest(test.TestCase): """Test of the OpenStack API /images application controller""" @@ -194,6 +198,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): 'image_type': 'ramdisk'}] def setUp(self): + super(ImageControllerWithGlanceServiceTest, self).setUp() self.orig_image_service = FLAGS.image_service FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.stubs = stubout.StubOutForTesting() @@ -208,6 +213,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.image_service = self.orig_image_service + super(ImageControllerWithGlanceServiceTest, self).tearDown() def test_get_image_index(self): req = webob.Request.blank('/v1.0/images') diff --git a/nova/tests/api/openstack/test_ratelimiting.py b/nova/tests/api/openstack/test_ratelimiting.py index 4c9d6bc23..9ae90ee20 100644 --- a/nova/tests/api/openstack/test_ratelimiting.py +++ b/nova/tests/api/openstack/test_ratelimiting.py @@ -1,15 +1,16 @@ import httplib import StringIO import time -import unittest import webob +from nova import test import nova.api.openstack.ratelimiting as ratelimiting -class LimiterTest(unittest.TestCase): +class LimiterTest(test.TestCase): def setUp(self): + super(LimiterTest, self).setUp() self.limits = { 'a': (5, ratelimiting.PER_SECOND), 'b': (5, ratelimiting.PER_MINUTE), @@ -83,9 +84,10 @@ class FakeLimiter(object): return self._delay -class WSGIAppTest(unittest.TestCase): +class WSGIAppTest(test.TestCase): def setUp(self): + super(WSGIAppTest, self).setUp() self.limiter = FakeLimiter(self) self.app = ratelimiting.WSGIApp(self.limiter) @@ -206,7 +208,7 @@ def wire_HTTPConnection_to_WSGI(host, app): httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection) -class WSGIAppProxyTest(unittest.TestCase): +class WSGIAppProxyTest(test.TestCase): def setUp(self): """Our WSGIAppProxy is going to call across an HTTPConnection to a @@ -218,6 +220,7 @@ class WSGIAppProxyTest(unittest.TestCase): at the WSGIApp. And the limiter isn't real -- it's a fake that behaves the way we tell it to. """ + super(WSGIAppProxyTest, self).setUp() self.limiter = FakeLimiter(self) app = ratelimiting.WSGIApp(self.limiter) wire_HTTPConnection_to_WSGI('100.100.100.100:80', app) @@ -238,7 +241,3 @@ class WSGIAppProxyTest(unittest.TestCase): self.limiter.mock('murder', 'brutus', None) self.proxy.perform('stab', 'brutus') self.assertRaises(AssertionError, shouldRaise) - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..ea29dcf9b 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -17,13 +17,13 @@ import datetime import json -import unittest import stubout import webob from nova import db from nova import flags +from nova import test import nova.api.openstack from nova.api.openstack import servers import nova.db.api @@ -108,9 +108,10 @@ def fake_compute_api(cls, req, id): return True -class ServersTest(unittest.TestCase): +class ServersTest(test.TestCase): def setUp(self): + super(ServersTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -141,6 +142,7 @@ class ServersTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + super(ServersTest, self).tearDown() def test_get_server_by_id(self): req = webob.Request.blank('/v1.0/servers/1') @@ -410,7 +412,3 @@ class ServersTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) - - -if __name__ == "__main__": - unittest.main() diff --git a/nova/tests/api/openstack/test_shared_ip_groups.py b/nova/tests/api/openstack/test_shared_ip_groups.py index c2fc3a203..b4de2ef41 100644 --- a/nova/tests/api/openstack/test_shared_ip_groups.py +++ b/nova/tests/api/openstack/test_shared_ip_groups.py @@ -15,19 +15,20 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest - import stubout +from nova import test from nova.api.openstack import shared_ip_groups -class SharedIpGroupsTest(unittest.TestCase): +class SharedIpGroupsTest(test.TestCase): def setUp(self): + super(SharedIpGroupsTest, self).setUp() self.stubs = stubout.StubOutForTesting() def tearDown(self): self.stubs.UnsetAll() + super(SharedIpGroupsTest, self).tearDown() def test_get_shared_ip_groups(self): pass diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index df497ef1b..555b206b9 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import stubout import webob @@ -22,6 +21,7 @@ import json import nova.db from nova import context from nova import flags +from nova import test from nova.api.openstack import zones from nova.tests.api.openstack import fakes @@ -60,8 +60,9 @@ def zone_get_all(context): password='qwerty')] -class ZonesTest(unittest.TestCase): +class ZonesTest(test.TestCase): def setUp(self): + super(ZonesTest, self).setUp() self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} @@ -81,6 +82,7 @@ class ZonesTest(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + super(ZonesTest, self).tearDown() def test_get_zone_list(self): req = webob.Request.blank('/v1.0/zones') @@ -134,7 +136,3 @@ class ZonesTest(unittest.TestCase): self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('username' in res_dict['zone']) - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 44e2d615c..2c7852214 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -21,7 +21,7 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ -import unittest +from nova import test import routes import webob @@ -29,7 +29,7 @@ import webob from nova import wsgi -class Test(unittest.TestCase): +class Test(test.TestCase): def test_debug(self): @@ -92,7 +92,7 @@ class Test(unittest.TestCase): self.assertNotEqual(result.body, "123") -class SerializerTest(unittest.TestCase): +class SerializerTest(test.TestCase): def match(self, url, accept, expect): input_dict = dict(servers=dict(a=(2, 3))) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index da86e6e11..5a1be08eb 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -311,4 +311,5 @@ class S3APITestCase(test.TestCase): self.auth_manager.delete_user('admin') self.auth_manager.delete_project('admin') stop_listening = defer.maybeDeferred(self.listening_port.stopListening) + super(S3APITestCase, self).tearDown() return defer.DeferredList([stop_listening]) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 7656f5396..b6bfab534 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -52,6 +52,7 @@ class DirectTestCase(test.TestCase): def tearDown(self): direct.ROUTES = {} + super(DirectTestCase, self).tearDown() def test_delegated_auth(self): req = webob.Request.blank('/fake/context') diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 9d458244b..1bad364e5 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -150,6 +150,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..7aadd65d5 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -206,9 +206,9 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) def tearDown(self): - super(LibvirtConnTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + super(LibvirtConnTestCase, self).tearDown() class IptablesFirewallTestCase(test.TestCase): @@ -388,6 +388,7 @@ class NWFilterTestCase(test.TestCase): def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) + super(NWFilterTestCase, self).tearDown() def test_cidr_rule_nwfilter_xml(self): cloud_controller = cloud.CloudController() -- cgit From b09534dac05a3b4c127c633d8c050bb310a27166 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 11:52:10 -0800 Subject: put the redirection back in to run_tests.sh and fix terminal colors by using original stdout --- nova/tests/fake_flags.py | 1 - run_tests.py | 16 +++++++++------- run_tests.sh | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 575fefff6..2b1919407 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -41,4 +41,3 @@ FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True -FLAGS.logfile = 'tests.log' diff --git a/run_tests.py b/run_tests.py index 01a1cf8bc..c78f88831 100644 --- a/run_tests.py +++ b/run_tests.py @@ -82,7 +82,7 @@ class _AnsiColorizer(object): try: return curses.tigetnum("colors") > 2 except curses.error: - curses.setupterm(fd=stream.fileno()) + curses.setupterm() return curses.tigetnum("colors") > 2 except: raise @@ -107,13 +107,13 @@ class _Win32Colorizer(object): See _AnsiColorizer docstring. """ def __init__(self, stream): - from win32console import GetStdHandle, STD_ERROR_HANDLE, \ + from win32console import GetStdHandle, STD_OUT_HANDLE, \ FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ FOREGROUND_INTENSITY red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, FOREGROUND_BLUE, FOREGROUND_INTENSITY) self.stream = stream - self.screenBuffer = GetStdHandle(STD_ERROR_HANDLE) + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) self._colors = { 'normal': red | green | blue, 'red': red | bold, @@ -129,7 +129,7 @@ class _Win32Colorizer(object): try: import win32console screenBuffer = win32console.GetStdHandle( - win32console.STD_ERROR_HANDLE) + win32console.STD_OUT_HANDLE) except ImportError: return False import pywintypes @@ -170,12 +170,14 @@ class NovaTestResult(result.TextTestResult): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: - # NOTE(vish): nose does funky stuff with stdout, so use stderr - # to setup the colorizer - if colorizer.supported(sys.stderr): + if colorizer.supported(): self.colorizer = colorizer(self.stream) break + sys.stdout = stdout def getDescription(self, test): return str(test) diff --git a/run_tests.sh b/run_tests.sh index e8433bc06..ebe236baf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -39,7 +39,7 @@ done function run_tests { # Just run the test suites in current environment - ${wrapper} $NOSETESTS + ${wrapper} $NOSETESTS 2> run_tests.log } NOSETESTS="python run_tests.py $noseargs" -- cgit From f7751eedc0e895f90d48104e2110bc2b320735fc Mon Sep 17 00:00:00 2001 From: Dan Prince <dan.prince@rackspace.com> Date: Wed, 23 Feb 2011 13:53:02 -0600 Subject: Revert commit 709. This fixes issues with the Openstack API causing 'No user for access key admin' errors. --- nova/api/openstack/auth.py | 4 ++-- nova/tests/api/openstack/fakes.py | 8 ++------ nova/tests/api/openstack/test_auth.py | 6 +++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index c3fe0cc8c..1dfdd5318 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -120,8 +120,8 @@ class AuthMiddleware(wsgi.Middleware): req - webob.Request object """ ctxt = context.get_admin_context() - user = self.auth.get_user_from_access_key(username) - if user and user.secret == key: + user = self.auth.get_user_from_access_key(key) + if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, time.time())).hexdigest() token_dict = {} diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e0b7b8029..fb282f1c9 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -221,8 +221,7 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): auth_data = {} - def add_user(self, user): - key = user.id + def add_user(self, key, user): FakeAuthManager.auth_data[key] = user def get_user(self, uid): @@ -235,10 +234,7 @@ class FakeAuthManager(object): return None def get_user_from_access_key(self, key): - for k, v in FakeAuthManager.auth_data.iteritems(): - if v.access == key: - return v - return None + return FakeAuthManager.auth_data.get(key, None) class FakeRateLimiter(object): diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index eab78b50c..0dd65d321 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -48,7 +48,7 @@ class Test(unittest.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -62,7 +62,7 @@ class Test(unittest.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -144,7 +144,7 @@ class TestLimiter(unittest.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' -- cgit From 19dc13131b7fe512cb7897a888093b5c9a62e69d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 12:00:02 -0800 Subject: move the deletion of the db into fixtures --- nova/tests/__init__.py | 8 ++++++-- run_tests.py | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index dbd433054..7fba02a93 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -48,6 +48,9 @@ def setup(): FLAGS = flags.FLAGS + testdb = os.path.join(FLAGS.state_path, FLAGS.sqlite_db) + if os.path.exists(testdb): + os.unlink(testdb) migration.db_sync() ctxt = context.get_admin_context() network_manager.VlanManager().create_networks(ctxt, @@ -58,5 +61,6 @@ def setup(): FLAGS.vlan_start, FLAGS.vpn_start, ) - shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_db), - os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db)) + + cleandb = os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db) + shutil.copyfile(testdb, cleandb) diff --git a/run_tests.py b/run_tests.py index bb6f0dd1e..5fb5ba32f 100644 --- a/run_tests.py +++ b/run_tests.py @@ -265,10 +265,6 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': logging.setup() - testdb = os.path.join(FLAGS.state_path, - FLAGS.sqlite_db) - if os.path.exists(testdb): - os.unlink(testdb) testdir = os.path.abspath(os.path.join("nova","tests")) c = config.Config(stream=sys.stdout, env=os.environ, -- cgit From 5283e1c131a21ea4963c702a7137536f7b894bb6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 12:05:49 -0800 Subject: Created mini XPath implementation, to simplify mapping logic --- nova/api/openstack/servers.py | 21 ++----- nova/tests/test_minixpath.py | 141 ++++++++++++++++++++++++++++++++++++++++++ nova/utils.py | 45 ++++++++++++++ 3 files changed, 191 insertions(+), 16 deletions(-) create mode 100644 nova/tests/test_minixpath.py diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index b54e28c0c..794705306 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -30,7 +30,7 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack - +import types LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) @@ -63,22 +63,11 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - fixed_ip = inst['fixed_ip'] - if fixed_ip: - # grab single private fixed ip - try: - private_ip = fixed_ip['address'] - if private_ip: - inst_dict['addresses']['private'].append(private_ip) - except KeyError: - LOG.debug(_("Failed to read private ip")) + private_ips = utils.minixpath_select(inst, 'fixed_ip/address') + inst_dict['addresses']['private'] = private_ips - # grab all public floating ips - try: - for floating in fixed_ip['floating_ips']: - inst_dict['addresses']['public'].append(floating['address']) - except KeyError: - LOG.debug(_("Failed to read public ip(s)")) + public_ips = utils.minixpath_select(inst, 'fixed_ip/floating_ips/address') + inst_dict['addresses']['public'] = public_ips inst_dict['metadata'] = {} inst_dict['hostId'] = '' diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py new file mode 100644 index 000000000..7fddcf9e9 --- /dev/null +++ b/nova/tests/test_minixpath.py @@ -0,0 +1,141 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class MiniXPathTestCase(test.TestCase): + def test_tolerates_nones(self): + xp = utils.minixpath_select + + input = [] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [None] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + def test_does_select(self): + xp = utils.minixpath_select + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], + xp(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) + + self.assertEquals([], xp(input, "a/b/c/d")) + self.assertEquals([], xp(input, "c/a/b/d")) + self.assertEquals([], xp(input, "i/r/t")) + + def test_flattens_lists(self): + xp = utils.minixpath_select + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + def test_bad_xpath(self): + xp = utils.minixpath_select + + self.assertRaises(exception.Error, xp, [], None) + self.assertRaises(exception.Error, xp, [], "") + self.assertRaises(exception.Error, xp, [], "/") + self.assertRaises(exception.Error, xp, [], "/a") + self.assertRaises(exception.Error, xp, [], "/a/") + self.assertRaises(exception.Error, xp, [], "//") + self.assertRaises(exception.Error, xp, [], "//a") + self.assertRaises(exception.Error, xp, [], "a//a") + self.assertRaises(exception.Error, xp, [], "a//a/") + self.assertRaises(exception.Error, xp, [], "a/a/") diff --git a/nova/utils.py b/nova/utils.py index 42efa0008..2f926bd82 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -2,6 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -34,6 +35,7 @@ import time from xml.sax import saxutils import re import netaddr +import types from eventlet import event from eventlet import greenthread @@ -499,3 +501,46 @@ def ensure_b64_encoding(val): return val except TypeError: return base64.b64encode(val) + + +def minixpath_select(items, minixpath): + """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each + item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of + the intermediate results are lists it will treat each list item + individually. A 'None' in items or any child expressions will be ignored, + this function will not throw because of None (anywhere) in items""" + + if minixpath is None: + raise exception.Error("Invalid mini_xpath") + + (first_token, sep, remainder) = minixpath.partition("/") + + if first_token == "": + raise exception.Error("Invalid mini_xpath") + + results = [] + + if items is None: + return results + + for item in items: + if item is None: + continue + get_method = getattr(item, "get", None) + if get_method is None: + continue + child = get_method(first_token) + if child is None: + continue + if isinstance(child, types.ListType): + # Flatten intermediate lists + for x in child: + results.append(x) + else: + results.append(child) + + if not sep: + # No more tokens + return results + else: + return minixpath_select(results, remainder) -- cgit From b3b005f50de54b5ef6c62e387dcec5a123f93cf6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 12:36:09 -0800 Subject: Cope when we pass a non-list to xpath_select - wrap it in a list --- nova/tests/test_minixpath.py | 38 ++++++++++++++++++++++++++++++++++++++ nova/utils.py | 8 +++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 7fddcf9e9..3b1bdf40b 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -139,3 +139,41 @@ class MiniXPathTestCase(test.TestCase): self.assertRaises(exception.Error, xp, [], "a//a") self.assertRaises(exception.Error, xp, [], "a//a/") self.assertRaises(exception.Error, xp, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + xp = utils.minixpath_select + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = xp(inst, 'fixed_ip/address') + public_ips = xp(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + xp = utils.minixpath_select + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + diff --git a/nova/utils.py b/nova/utils.py index 2f926bd82..c2cbeb2a7 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -508,7 +508,8 @@ def minixpath_select(items, minixpath): item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of the intermediate results are lists it will treat each list item individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items""" + this function will not throw because of None (anywhere) in items. The + returned list will contain no None values.""" if minixpath is None: raise exception.Error("Invalid mini_xpath") @@ -523,6 +524,10 @@ def minixpath_select(items, minixpath): if items is None: return results + if not isinstance(items, types.ListType): + # Wrap single objects in a list + items = [items] + for item in items: if item is None: continue @@ -532,6 +537,7 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue + #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: -- cgit From 21ebea24b4b77f8bd1fd42152454f1b0189843d4 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 12:54:46 -0800 Subject: fix describe_availability_zones --- nova/api/ec2/cloud.py | 5 +++-- nova/db/api.py | 4 ++-- nova/db/sqlalchemy/api.py | 9 +++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 99b6d5cb6..9e8764836 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -198,8 +198,9 @@ class CloudController(object): return self._describe_availability_zones(context, **kwargs) def _describe_availability_zones(self, context, **kwargs): - enabled_services = db.service_get_all(context) - disabled_services = db.service_get_all(context, True) + ctxt = context.elevated() + enabled_services = db.service_get_all(ctxt) + disabled_services = db.service_get_all(ctxt, True) available_zones = [] for zone in [service.availability_zone for service in enabled_services]: diff --git a/nova/db/api.py b/nova/db/api.py index d7f3746d2..0a010e727 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -85,8 +85,8 @@ def service_get(context, service_id): def service_get_all(context, disabled=False): - """Get all service.""" - return IMPL.service_get_all(context, None, disabled) + """Get all services.""" + return IMPL.service_get_all(context, disabled) def service_get_all_by_topic(context, topic): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index be29fe2a0..d8751bef4 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -136,15 +136,12 @@ def service_get(context, service_id, session=None): @require_admin_context -def service_get_all(context, session=None, disabled=False): - if not session: - session = get_session() - - result = session.query(models.Service).\ +def service_get_all(context, disabled=False): + session = get_session() + return session.query(models.Service).\ filter_by(deleted=can_read_deleted(context)).\ filter_by(disabled=disabled).\ all() - return result @require_admin_context -- cgit From 89ade95d2eaabf77f9c81a8d50c7cc11aa175464 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 12:55:07 -0800 Subject: Fix pep8 violation (trailing whitespace) --- nova/tests/test_minixpath.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 3b1bdf40b..cc4a35ef3 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -175,5 +175,3 @@ class MiniXPathTestCase(test.TestCase): input = {'a': [1, 2, {'b': 'b_1'}]} self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) self.assertEquals(['b_1'], xp(input, "a/b")) - - -- cgit From 1183c9e11b12984b1f5007ace831864e80483712 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 14:07:08 -0800 Subject: Rename minixpath_select to get_from_path --- nova/api/openstack/servers.py | 5 +- nova/tests/test_minixpath.py | 177 ------------------------------------------ nova/tests/test_utils.py | 174 +++++++++++++++++++++++++++++++++++++++++ nova/utils.py | 24 +++--- 4 files changed, 188 insertions(+), 192 deletions(-) delete mode 100644 nova/tests/test_minixpath.py create mode 100644 nova/tests/test_utils.py diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 794705306..ce4a6256a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -30,7 +30,6 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack -import types LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) @@ -63,10 +62,10 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - private_ips = utils.minixpath_select(inst, 'fixed_ip/address') + private_ips = utils.get_from_path(inst, 'fixed_ip/address') inst_dict['addresses']['private'] = private_ips - public_ips = utils.minixpath_select(inst, 'fixed_ip/floating_ips/address') + public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') inst_dict['addresses']['public'] = public_ips inst_dict['metadata'] = {} diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py deleted file mode 100644 index cc4a35ef3..000000000 --- a/nova/tests/test_minixpath.py +++ /dev/null @@ -1,177 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from nova import test -from nova import utils -from nova import exception - - -class MiniXPathTestCase(test.TestCase): - def test_tolerates_nones(self): - xp = utils.minixpath_select - - input = [] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [None] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': None}] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': None}}] - self.assertEquals([{'b': None}], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': None}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] - self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - def test_does_select(self): - xp = utils.minixpath_select - - input = [{'a': 'a_1'}] - self.assertEquals(['a_1'], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': 'b_1'}}] - self.assertEquals([{'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] - self.assertEquals([{'b': {'c': 'c_1'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': None}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': {'c': 'c_2'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], - xp(input, "a/b")) - self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) - - self.assertEquals([], xp(input, "a/b/c/d")) - self.assertEquals([], xp(input, "c/a/b/d")) - self.assertEquals([], xp(input, "i/r/t")) - - def test_flattens_lists(self): - xp = utils.minixpath_select - - input = [{'a': [1, 2, 3]}] - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}] - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [1, 2, {'b': 'b_1'}]}] - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - - def test_bad_xpath(self): - xp = utils.minixpath_select - - self.assertRaises(exception.Error, xp, [], None) - self.assertRaises(exception.Error, xp, [], "") - self.assertRaises(exception.Error, xp, [], "/") - self.assertRaises(exception.Error, xp, [], "/a") - self.assertRaises(exception.Error, xp, [], "/a/") - self.assertRaises(exception.Error, xp, [], "//") - self.assertRaises(exception.Error, xp, [], "//a") - self.assertRaises(exception.Error, xp, [], "a//a") - self.assertRaises(exception.Error, xp, [], "a//a/") - self.assertRaises(exception.Error, xp, [], "a/a/") - - def test_real_failure1(self): - # Real world failure case... - # We weren't coping when the input was a Dictionary instead of a List - # This led to test_accepts_dictionaries - xp = utils.minixpath_select - - inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], - 'address': '192.168.0.3'}, - 'hostname': ''} - - private_ips = xp(inst, 'fixed_ip/address') - public_ips = xp(inst, 'fixed_ip/floating_ips/address') - self.assertEquals(['192.168.0.3'], private_ips) - self.assertEquals(['1.2.3.4'], public_ips) - - def test_accepts_dictionaries(self): - xp = utils.minixpath_select - - input = {'a': [1, 2, 3]} - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': {'b': [1, 2, 3]}} - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [1, 2, {'b': 'b_1'}]} - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py new file mode 100644 index 000000000..34a407f1a --- /dev/null +++ b/nova/tests/test_utils.py @@ -0,0 +1,174 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class GetFromPathTestCase(test.TestCase): + def test_tolerates_nones(self): + f = utils.get_from_path + + input = [] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [None] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + def test_does_select(self): + f = utils.get_from_path + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + f(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], f(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], f(input, "a/b/c")) + + self.assertEquals([], f(input, "a/b/c/d")) + self.assertEquals([], f(input, "c/a/b/d")) + self.assertEquals([], f(input, "i/r/t")) + + def test_flattens_lists(self): + f = utils.get_from_path + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + + def test_bad_xpath(self): + f = utils.get_from_path + + self.assertRaises(exception.Error, f, [], None) + self.assertRaises(exception.Error, f, [], "") + self.assertRaises(exception.Error, f, [], "/") + self.assertRaises(exception.Error, f, [], "/a") + self.assertRaises(exception.Error, f, [], "/a/") + self.assertRaises(exception.Error, f, [], "//") + self.assertRaises(exception.Error, f, [], "//a") + self.assertRaises(exception.Error, f, [], "a//a") + self.assertRaises(exception.Error, f, [], "a//a/") + self.assertRaises(exception.Error, f, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + f = utils.get_from_path + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = f(inst, 'fixed_ip/address') + public_ips = f(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + f = utils.get_from_path + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) diff --git a/nova/utils.py b/nova/utils.py index c2cbeb2a7..65e28c648 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -32,10 +32,10 @@ import string import struct import sys import time +import types from xml.sax import saxutils import re import netaddr -import types from eventlet import event from eventlet import greenthread @@ -503,18 +503,19 @@ def ensure_b64_encoding(val): return base64.b64encode(val) -def minixpath_select(items, minixpath): - """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each - item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of - the intermediate results are lists it will treat each list item - individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items. The - returned list will contain no None values.""" +def get_from_path(items, path): + """ Returns a list of items matching the specified path. Takes an + XPath-like expression e.g. prop1/prop2/prop3, and for each item in items, + looks up items[prop1][prop2][prop3]. Like XPath, if any of the + intermediate results are lists it will treat each list item individually. + A 'None' in items or any child expressions will be ignored, this function + will not throw because of None (anywhere) in items. The returned list + will contain no None values.""" - if minixpath is None: + if path is None: raise exception.Error("Invalid mini_xpath") - (first_token, sep, remainder) = minixpath.partition("/") + (first_token, sep, remainder) = path.partition("/") if first_token == "": raise exception.Error("Invalid mini_xpath") @@ -537,7 +538,6 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue - #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: @@ -549,4 +549,4 @@ def minixpath_select(items, minixpath): # No more tokens return results else: - return minixpath_select(results, remainder) + return get_from_path(results, remainder) -- cgit From a508e2dce781b98db5a719df75a451d9a2727fca Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 14:12:43 -0800 Subject: Make sure there are two blank links after the import --- nova/api/openstack/servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ce4a6256a..6c227d71a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -31,6 +31,7 @@ from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack + LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) -- cgit From 8c007b56b586257d048b6db4ecfbed8f502381fd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 14:16:31 -0800 Subject: Put back the comments I accidentally removed --- nova/api/openstack/servers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 6c227d71a..97323f66f 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -63,9 +63,11 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) + # grab single private fixed ip private_ips = utils.get_from_path(inst, 'fixed_ip/address') inst_dict['addresses']['private'] = private_ips + # grab all public floating ips public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') inst_dict['addresses']['public'] = public_ips -- cgit From 24090232272e0db163060e0ca32dbf97c05120c9 Mon Sep 17 00:00:00 2001 From: termie <github@anarkystic.com> Date: Wed, 23 Feb 2011 15:14:16 -0800 Subject: updates to nova.flags to get help working better Fixes some old bugs that were brought up on the mailing list. First step towards moving flags into the places where they belong. Also moves manager import into service's init so that we can get all the dynamically loaded flags shortly after loading. --- nova/flags.py | 49 +++++++++++++++++++++++++++++++++++++++++----- nova/service.py | 31 +++++++++++++++++------------ nova/tests/test_service.py | 7 ------- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index f64a62da9..24bca0caf 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -160,9 +160,45 @@ class StrWrapper(object): raise KeyError(name) -FLAGS = FlagValues() -gflags.FLAGS = FLAGS -gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) +# Copied from gflags with small mods to get the naming correct. +# Originally gflags checks for the first module that is not gflags that is +# in the call chain, we want to check for the first module that is not gflags +# and not this module. +def _GetCallingModule(): + """Returns the name of the module that's calling into this module. + + We generally use this function to get the name of the module calling a + DEFINE_foo... function. + """ + # Walk down the stack to find the first globals dict that's not ours. + for depth in range(1, sys.getrecursionlimit()): + if not sys._getframe(depth).f_globals is globals(): + module_name = __GetModuleName(sys._getframe(depth).f_globals) + if module_name == 'gflags': + continue + if module_name is not None: + return module_name + raise AssertionError("No module was found") + + +# Copied from gflags because it is a private function +def __GetModuleName(globals_dict): + """Given a globals dict, returns the name of the module that defines it. + + Args: + globals_dict: A dictionary that should correspond to an environment + providing the values of the globals. + + Returns: + A string (the name of the module) or None (if the module could not + be identified. + """ + for name, module in sys.modules.iteritems(): + if getattr(module, '__dict__', None) is globals_dict: + if name == '__main__': + return sys.argv[0] + return name + return None def _wrapper(func): @@ -173,6 +209,11 @@ def _wrapper(func): return _wrapped +FLAGS = FlagValues() +gflags.FLAGS = FLAGS +gflags._GetCallingModule = _GetCallingModule + + DEFINE = _wrapper(gflags.DEFINE) DEFINE_string = _wrapper(gflags.DEFINE_string) DEFINE_integer = _wrapper(gflags.DEFINE_integer) @@ -185,8 +226,6 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) DEFINE_flag = _wrapper(gflags.DEFINE_flag) - - HelpFlag = gflags.HelpFlag HelpshortFlag = gflags.HelpshortFlag HelpXMLFlag = gflags.HelpXMLFlag diff --git a/nova/service.py b/nova/service.py index cc88ac233..f47358089 100644 --- a/nova/service.py +++ b/nova/service.py @@ -45,15 +45,10 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('report_interval', 10, 'seconds between nodes reporting state to datastore', lower_bound=1) - flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) -flags.DEFINE_flag(flags.HelpFlag()) -flags.DEFINE_flag(flags.HelpshortFlag()) -flags.DEFINE_flag(flags.HelpXMLFlag()) - class Service(object): """Base class for workers that run on hosts.""" @@ -64,6 +59,8 @@ class Service(object): self.binary = binary self.topic = topic self.manager_class_name = manager + manager_class = utils.import_class(self.manager_class_name) + self.manager = manager_class(host=self.host, *args, **kwargs) self.report_interval = report_interval self.periodic_interval = periodic_interval super(Service, self).__init__(*args, **kwargs) @@ -71,9 +68,9 @@ class Service(object): self.timers = [] def start(self): - manager_class = utils.import_class(self.manager_class_name) - self.manager = manager_class(host=self.host, *self.saved_args, - **self.saved_kwargs) + vcs_string = version.version_string_with_vcs() + logging.audit(_("Starting %(topic)s node (version %(vcs_string)s)"), + {'topic': self.topic, 'vcs_string': vcs_string}) self.manager.init_host() self.model_disconnected = False ctxt = context.get_admin_context() @@ -153,9 +150,6 @@ class Service(object): report_interval = FLAGS.report_interval if not periodic_interval: periodic_interval = FLAGS.periodic_interval - vcs_string = version.version_string_with_vcs() - logging.audit(_("Starting %(topic)s node (version %(vcs_string)s)") - % locals()) service_obj = cls(host, binary, topic, manager, report_interval, periodic_interval) @@ -217,8 +211,19 @@ class Service(object): def serve(*services): - if not services: - services = [Service.create()] + try: + if not services: + services = [Service.create()] + except Exception: + logging.exception('in Service.create()') + raise + finally: + # After we've loaded up all our dynamic bits, check + # whether we should print help + flags.DEFINE_flag(flags.HelpFlag()) + flags.DEFINE_flag(flags.HelpshortFlag()) + flags.DEFINE_flag(flags.HelpXMLFlag()) + FLAGS.ParseNewFlags() name = '_'.join(x.binary for x in services) logging.debug(_("Serving %s"), name) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index a67c8d1e8..45d9afa6c 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -50,13 +50,6 @@ class ExtendedService(service.Service): class ServiceManagerTestCase(test.TestCase): """Test cases for Services""" - def test_attribute_error_for_no_manager(self): - serv = service.Service('test', - 'test', - 'test', - 'nova.tests.test_service.FakeManager') - self.assertRaises(AttributeError, getattr, serv, 'test_method') - def test_message_gets_to_manager(self): serv = service.Service('test', 'test', -- cgit From b7d89758af54b291492eecae74cee29461ca28b9 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 15:16:56 -0800 Subject: remove processName from debug output since we aren't using multiprocessing and it doesn't exist in python 2.6.1 --- nova/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/log.py b/nova/log.py index 10c14d74b..7866c34f7 100644 --- a/nova/log.py +++ b/nova/log.py @@ -54,7 +54,7 @@ flags.DEFINE_string('logging_default_format_string', 'format string to use for log messages without context') flags.DEFINE_string('logging_debug_format_suffix', - 'from %(processName)s (pid=%(process)d) %(funcName)s' + 'from (pid=%(process)d) %(funcName)s' ' %(pathname)s:%(lineno)d', 'data to append to log format when level is DEBUG') -- cgit From 3115a65f9981371ea8587a288b360c3c519de865 Mon Sep 17 00:00:00 2001 From: termie <github@anarkystic.com> Date: Wed, 23 Feb 2011 15:26:52 -0800 Subject: add help back to the scripts that don't use service.py --- bin/nova-ajax-console-proxy | 4 +++- bin/nova-api | 3 +++ bin/nova-direct-api | 4 ++++ bin/nova-manage | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 392b328b1..1e11c6d58 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -47,9 +47,11 @@ from nova import utils from nova import wsgi FLAGS = flags.FLAGS - flags.DEFINE_integer('ajax_console_idle_timeout', 300, 'Seconds before idle connection destroyed') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) LOG = logging.getLogger('nova.ajax_console_proxy') LOG.setLevel(logging.DEBUG) diff --git a/bin/nova-api b/bin/nova-api index d5efb4687..cf140570a 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -48,6 +48,9 @@ flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') flags.DEFINE_string('osapi_listen', "0.0.0.0", 'IP address for OpenStack API to listen') flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) API_ENDPOINTS = ['ec2', 'osapi'] diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 6c63bd26b..bf29d9a5e 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -45,6 +45,10 @@ from nova.compute import api as compute_api FLAGS = flags.FLAGS flags.DEFINE_integer('direct_port', 8001, 'Direct API port') flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) + if __name__ == '__main__': utils.default_flagfile() diff --git a/bin/nova-manage b/bin/nova-manage index 5189de0e1..b603c8b07 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -93,6 +93,9 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) def param2id(object_id): -- cgit From 5e2f82b1487b8f8e43539d0c71466fbbfed23121 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 15:29:02 -0800 Subject: revert logfile redirection and make colors work by temporarily switching stdout --- nova/tests/fake_flags.py | 1 - run_tests.py | 16 +++++++++------- run_tests.sh | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 575fefff6..2b1919407 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -41,4 +41,3 @@ FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True -FLAGS.logfile = 'tests.log' diff --git a/run_tests.py b/run_tests.py index 877849ab5..2c04a0641 100644 --- a/run_tests.py +++ b/run_tests.py @@ -97,7 +97,7 @@ class _AnsiColorizer(object): try: return curses.tigetnum("colors") > 2 except curses.error: - curses.setupterm(fd=stream.fileno()) + curses.setupterm() return curses.tigetnum("colors") > 2 except: raise @@ -122,13 +122,13 @@ class _Win32Colorizer(object): See _AnsiColorizer docstring. """ def __init__(self, stream): - from win32console import GetStdHandle, STD_ERROR_HANDLE, \ + from win32console import GetStdHandle, STD_OUT_HANDLE, \ FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ FOREGROUND_INTENSITY red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, FOREGROUND_BLUE, FOREGROUND_INTENSITY) self.stream = stream - self.screenBuffer = GetStdHandle(STD_ERROR_HANDLE) + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) self._colors = { 'normal': red | green | blue, 'red': red | bold, @@ -144,7 +144,7 @@ class _Win32Colorizer(object): try: import win32console screenBuffer = win32console.GetStdHandle( - win32console.STD_ERROR_HANDLE) + win32console.STD_OUT_HANDLE) except ImportError: return False import pywintypes @@ -185,12 +185,14 @@ class NovaTestResult(result.TextTestResult): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: - # NOTE(vish): nose does funky stuff with stdout, so use stderr - # to setup the colorizer - if colorizer.supported(sys.stderr): + if colorizer.supported(): self.colorizer = colorizer(self.stream) break + sys.stdout = stdout def getDescription(self, test): return str(test) diff --git a/run_tests.sh b/run_tests.sh index 7aab9ecd9..d4586a57e 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -39,12 +39,12 @@ done function run_tests { # Just run the test suites in current environment - ${wrapper} $NOSETESTS + ${wrapper} $NOSETESTS 2> run_tests.log # If we get some short import error right away, print the error log directly RESULT=$? if [ "$RESULT" -ne "0" ]; then - ERRSIZE=`wc -l run_tests.err.log | awk '{print \$1}'` + ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` if [ "$ERRSIZE" -lt "40" ]; then cat run_tests.err.log -- cgit From c27c19ea316b343f1623a7c1bf21c53cd426603b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 15:35:30 -0800 Subject: fix pep8 --- run_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run_tests.py b/run_tests.py index 2c04a0641..eef0120bc 100644 --- a/run_tests.py +++ b/run_tests.py @@ -87,7 +87,7 @@ class _AnsiColorizer(object): coloring terminal output using this method. Returns False otherwise. """ if not stream.isatty(): - return False # auto color only on TTYs + return False # auto color only on TTYs try: import curses except ImportError: @@ -180,6 +180,7 @@ class _NullColorizer(object): def write(self, text, color): self.stream.write(text) + class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): result.TextTestResult.__init__(self, *args, **kw) @@ -276,7 +277,7 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': logging.setup() - testdir = os.path.abspath(os.path.join("nova","tests")) + testdir = os.path.abspath(os.path.join("nova", "tests")) testdb = os.path.join(testdir, "tests.sqlite") if os.path.exists(testdb): os.unlink(testdb) -- cgit From 78eddce564cccf0d9be19b303cbc122966f5fa71 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 15:42:59 -0800 Subject: added comments about where code came from --- run_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_tests.py b/run_tests.py index eef0120bc..8025548e5 100644 --- a/run_tests.py +++ b/run_tests.py @@ -198,6 +198,7 @@ class NovaTestResult(result.TextTestResult): def getDescription(self, test): return str(test) + # NOTE(vish): copied from unittest with edit to add color def addSuccess(self, test): unittest.TestResult.addSuccess(self, test) if self.showAll: @@ -207,6 +208,7 @@ class NovaTestResult(result.TextTestResult): self.stream.write('.') self.stream.flush() + # NOTE(vish): copied from unittest with edit to add color def addFailure(self, test, err): unittest.TestResult.addFailure(self, test, err) if self.showAll: @@ -216,6 +218,7 @@ class NovaTestResult(result.TextTestResult): self.stream.write('F') self.stream.flush() + # NOTE(vish): copied from nose with edit to add color def addError(self, test, err): """Overrides normal addError to add support for errorClasses. If the exception is a registered class, the -- cgit From 3aa0183bca5585c706b4b02e0a7542422015c693 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 18:54:13 -0800 Subject: fix missed err.log --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index d4586a57e..7ac3ff33f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -47,7 +47,7 @@ function run_tests { ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` if [ "$ERRSIZE" -lt "40" ]; then - cat run_tests.err.log + cat run_tests.log fi fi return $RESULT -- cgit From f86c45764d6cf62b1ded928e807e93198ed6ffd1 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 22:02:06 -0800 Subject: PEP 257 fixes --- nova/volume/driver.py | 42 +++++++++++++++++++++++------------------- nova/volume/san.py | 22 +++++++++++++++------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 22c2c2fc3..4263a6a8d 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -227,14 +227,16 @@ class FakeAOEDriver(AOEDriver): class ISCSIDriver(VolumeDriver): - """Executes commands relating to ISCSI volumes. We make use of model - provider properties as follows: - provider_location - if present, contains the iSCSI target information - in the same format as an ietadm discovery - i.e. '<target ip/port>,<target portal> <target IQN>' - provider_auth - if present, contains a space-separated triple: - '<auth method> <auth username> <auth password>'. CHAP is the only - auth_method in use at the moment.""" + """Executes commands relating to ISCSI volumes. + + We make use of model provider properties as follows: + :provider_location: if present, contains the iSCSI target information + in the same format as an ietadm discovery + i.e. '<ip>:<port>,<portal> <target IQN>' + :provider_auth: if present, contains a space-separated triple: + '<auth method> <auth username> <auth password>'. + `CHAP` is the only auth_method in use at the moment. + """ def ensure_export(self, context, volume): """Synchronously recreates an export for a logical volume.""" @@ -321,17 +323,19 @@ class ISCSIDriver(VolumeDriver): return None def _get_iscsi_properties(self, volume): - """Gets iscsi configuration, ideally from saved information in the - volume entity, but falling back to discovery if need be. The - properties are: - target_discovered - boolean indicating whether discovery was used, - target_iqn - the IQN of the iSCSI target, - target_portal - the portal of the iSCSI target, - and auth_method, auth_username and auth_password - - the authentication details. Right now, either - auth_method is not present meaning no authentication, or - auth_method == 'CHAP' meaning use CHAP with the specified - credentials.""" + """Gets iscsi configuration + + We ideally get saved information in the volume entity, but fall back + to discovery if need be. Discovery may be completely removed in future + The properties are: + :target_discovered: boolean indicating whether discovery was used + :target_iqn: the IQN of the iSCSI target, + :target_portal: the portal of the iSCSI target + :auth_method:, :auth_username:, :auth_password: + the authentication details. Right now, either auth_method is not + present meaning no authentication, or auth_method == `CHAP` + meaning use CHAP with the specified credentials. + """ properties = {} diff --git a/nova/volume/san.py b/nova/volume/san.py index 09192bc9f..d6598d16e 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -16,8 +16,9 @@ # under the License. """ Drivers for san-stored volumes. + The unique thing about a SAN is that we don't expect that we can run the volume - controller on the SAN hardware. We expect to access it over SSH or some API. +controller on the SAN hardware. We expect to access it over SSH or some API. """ import os @@ -51,7 +52,11 @@ flags.DEFINE_integer('san_ssh_port', 22, class SanISCSIDriver(ISCSIDriver): """ Base class for SAN-style storage volumes - (storage providers we access over SSH)""" + + A SAN-style storage value is 'different' because the volume controller + probably won't run on it, so we need to access is over SSH or another + remote protocol. + """ def _build_iscsi_target_name(self, volume): return "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) @@ -137,6 +142,7 @@ def _get_prefixed_values(data, prefix): class SolarisISCSIDriver(SanISCSIDriver): """Executes commands relating to Solaris-hosted ISCSI volumes. + Basic setup for a Solaris iSCSI server: pkg install storage-server SUNWiscsit svcadm enable stmf @@ -330,13 +336,14 @@ class SolarisISCSIDriver(SanISCSIDriver): class HpSanISCSIDriver(SanISCSIDriver): """Executes commands relating to HP/Lefthand SAN ISCSI volumes. + We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: - CLIQ createVolume (creates the volume) - CLIQ getVolumeInfo (to discover the IQN etc) - CLIQ getClusterInfo (to discover the iSCSI target IP address) - CLIQ assignVolumeChap (exports it with CHAP security) + :createVolume: (creates the volume) + :getVolumeInfo: (to discover the IQN etc) + :getClusterInfo: (to discover the iSCSI target IP address) + :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so normally a volume mount would need both to configure the SAN in the volume @@ -344,7 +351,8 @@ class HpSanISCSIDriver(SanISCSIDriver): not catered for at the moment in the nova architecture, so instead we share the volume using CHAP at volume creation time. Then the mount need only use those CHAP credentials, so can take place exclusively in the - compute layer""" + compute layer. + """ def _cliq_run(self, verb, cliq_args): """Runs a CLIQ command over SSH, without doing any result parsing""" -- cgit From 0a649493d21a8766f416ceca2c0122c20945ca1b Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 22:47:57 -0800 Subject: Documentation fixes so that output looks better --- nova/volume/driver.py | 9 ++++++++- nova/volume/san.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 4263a6a8d..e3744c790 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -230,9 +230,11 @@ class ISCSIDriver(VolumeDriver): """Executes commands relating to ISCSI volumes. We make use of model provider properties as follows: + :provider_location: if present, contains the iSCSI target information in the same format as an ietadm discovery i.e. '<ip>:<port>,<portal> <target IQN>' + :provider_auth: if present, contains a space-separated triple: '<auth method> <auth username> <auth password>'. `CHAP` is the only auth_method in use at the moment. @@ -328,10 +330,15 @@ class ISCSIDriver(VolumeDriver): We ideally get saved information in the volume entity, but fall back to discovery if need be. Discovery may be completely removed in future The properties are: + :target_discovered: boolean indicating whether discovery was used - :target_iqn: the IQN of the iSCSI target, + + :target_iqn: the IQN of the iSCSI target + :target_portal: the portal of the iSCSI target + :auth_method:, :auth_username:, :auth_password: + the authentication details. Right now, either auth_method is not present meaning no authentication, or auth_method == `CHAP` meaning use CHAP with the specified credentials. diff --git a/nova/volume/san.py b/nova/volume/san.py index d6598d16e..9532c8116 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -144,16 +144,25 @@ class SolarisISCSIDriver(SanISCSIDriver): """Executes commands relating to Solaris-hosted ISCSI volumes. Basic setup for a Solaris iSCSI server: + pkg install storage-server SUNWiscsit + svcadm enable stmf + svcadm enable -r svc:/network/iscsi/target:default + pfexec itadm create-tpg e1000g0 ${MYIP} + pfexec itadm create-target -t e1000g0 + Then grant the user that will be logging on lots of permissions. I'm not sure exactly which though: + zfs allow justinsb create,mount,destroy rpool + usermod -P'File System Management' justinsb + usermod -P'Primary Administrator' justinsb Also make sure you can login using san_login & san_password/san_privatekey @@ -340,9 +349,13 @@ class HpSanISCSIDriver(SanISCSIDriver): We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: + :createVolume: (creates the volume) + :getVolumeInfo: (to discover the IQN etc) + :getClusterInfo: (to discover the iSCSI target IP address) + :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so -- cgit From 701e1c15944062f7d229e59f2ede06398226b165 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara <justin@fathomdb.com> Date: Wed, 23 Feb 2011 22:53:44 -0800 Subject: Hotfix to not require metadata --- nova/api/openstack/servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 63e047b39..841bab6d0 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -177,7 +177,7 @@ class Controller(wsgi.Controller): # However, the CloudServers API is not definitive on this front, # and we want to be compatible. metadata = [] - if env['server']['metadata']: + if env['server'].get('metadata'): for k, v in env['server']['metadata'].items(): metadata.append({'key': k, 'value': v}) -- cgit From cc8276655cd2424b71689776f24e2a5cc767e844 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 23:37:52 -0800 Subject: move relevant code to baseclass and make flatdhcp not inherit from flat --- nova/network/manager.py | 141 +++++++++++++++++++++--------------------------- 1 file changed, 62 insertions(+), 79 deletions(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index 1df193be0..e999adae5 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -163,11 +163,22 @@ class NetworkManager(manager.Manager): def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" - raise NotImplementedError() + # TODO(vish): when this is called by compute, we can associate compute + # with a network, or a cluster of computes with a network + # and use that network here with a method like + # network_get_by_compute_host + network_ref = self.db.network_get_by_bridge(context, + FLAGS.flat_network_bridge) + address = self.db.fixed_ip_associate_pool(context.elevated(), + network_ref['id'], + instance_id) + self.db.fixed_ip_update(context, address, {'allocated': True}) + return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): """Returns a fixed ip to the pool.""" - raise NotImplementedError() + self.db.fixed_ip_update(context, address, {'allocated': False}) + self.db.fixed_ip_disassociate(context.elevated(), address) def setup_fixed_ip(self, context, address): """Sets up rules for fixed ip.""" @@ -257,12 +268,57 @@ class NetworkManager(manager.Manager): def get_network_host(self, context): """Get the network host for the current context.""" - raise NotImplementedError() + network_ref = self.db.network_get_by_bridge(context, + FLAGS.flat_network_bridge) + # NOTE(vish): If the network has no host, use the network_host flag. + # This could eventually be a a db lookup of some sort, but + # a flag is easy to handle for now. + host = network_ref['host'] + if not host: + topic = self.db.queue_get_for(context, + FLAGS.network_topic, + FLAGS.network_host) + if FLAGS.fake_call: + return self.set_network_host(context, network_ref['id']) + host = rpc.call(context, + FLAGS.network_topic, + {"method": "set_network_host", + "args": {"network_id": network_ref['id']}}) + return host def create_networks(self, context, cidr, num_networks, network_size, - cidr_v6, *args, **kwargs): + cidr_v6, label, *args, **kwargs): """Create networks based on parameters.""" - raise NotImplementedError() + fixed_net = IPy.IP(cidr) + fixed_net_v6 = IPy.IP(cidr_v6) + significant_bits_v6 = 64 + count = 1 + for index in range(num_networks): + start = index * network_size + significant_bits = 32 - int(math.log(network_size, 2)) + cidr = "%s/%s" % (fixed_net[start], significant_bits) + project_net = IPy.IP(cidr) + net = {} + net['bridge'] = FLAGS.flat_network_bridge + net['cidr'] = cidr + net['netmask'] = str(project_net.netmask()) + net['gateway'] = str(project_net[1]) + net['broadcast'] = str(project_net.broadcast()) + net['dhcp_start'] = str(project_net[2]) + if num_networks > 1: + net['label'] = "%s_%d" % (label, count) + else: + net['label'] = label + count += 1 + + if(FLAGS.use_ipv6): + cidr_v6 = "%s/%s" % (fixed_net_v6[0], significant_bits_v6) + net['cidr_v6'] = cidr_v6 + + network_ref = self.db.network_create_safe(context, net) + + if network_ref: + self._create_fixed_ips(context, network_ref['id']) @property def _bottom_reserved_ips(self): # pylint: disable-msg=R0201 @@ -332,83 +388,10 @@ class FlatManager(NetworkManager): for network in self.db.host_get_networks(ctxt, self.host): self._on_set_network_host(ctxt, network['id']) - def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool.""" - # TODO(vish): when this is called by compute, we can associate compute - # with a network, or a cluster of computes with a network - # and use that network here with a method like - # network_get_by_compute_host - network_ref = self.db.network_get_by_bridge(context, - FLAGS.flat_network_bridge) - address = self.db.fixed_ip_associate_pool(context.elevated(), - network_ref['id'], - instance_id) - self.db.fixed_ip_update(context, address, {'allocated': True}) - return address - - def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool.""" - self.db.fixed_ip_update(context, address, {'allocated': False}) - self.db.fixed_ip_disassociate(context.elevated(), address) - def setup_compute_network(self, context, instance_id): """Network is created manually.""" pass - def create_networks(self, context, cidr, num_networks, network_size, - cidr_v6, label, *args, **kwargs): - """Create networks based on parameters.""" - fixed_net = IPy.IP(cidr) - fixed_net_v6 = IPy.IP(cidr_v6) - significant_bits_v6 = 64 - count = 1 - for index in range(num_networks): - start = index * network_size - significant_bits = 32 - int(math.log(network_size, 2)) - cidr = "%s/%s" % (fixed_net[start], significant_bits) - project_net = IPy.IP(cidr) - net = {} - net['bridge'] = FLAGS.flat_network_bridge - net['cidr'] = cidr - net['netmask'] = str(project_net.netmask()) - net['gateway'] = str(project_net[1]) - net['broadcast'] = str(project_net.broadcast()) - net['dhcp_start'] = str(project_net[2]) - if num_networks > 1: - net['label'] = "%s_%d" % (label, count) - else: - net['label'] = label - count += 1 - - if(FLAGS.use_ipv6): - cidr_v6 = "%s/%s" % (fixed_net_v6[0], significant_bits_v6) - net['cidr_v6'] = cidr_v6 - - network_ref = self.db.network_create_safe(context, net) - - if network_ref: - self._create_fixed_ips(context, network_ref['id']) - - def get_network_host(self, context): - """Get the network host for the current context.""" - network_ref = self.db.network_get_by_bridge(context, - FLAGS.flat_network_bridge) - # NOTE(vish): If the network has no host, use the network_host flag. - # This could eventually be a a db lookup of some sort, but - # a flag is easy to handle for now. - host = network_ref['host'] - if not host: - topic = self.db.queue_get_for(context, - FLAGS.network_topic, - FLAGS.network_host) - if FLAGS.fake_call: - return self.set_network_host(context, network_ref['id']) - host = rpc.call(context, - FLAGS.network_topic, - {"method": "set_network_host", - "args": {"network_id": network_ref['id']}}) - return host - def _on_set_network_host(self, context, network_id): """Called when this host becomes the host for a network.""" net = {} @@ -433,7 +416,7 @@ class FlatManager(NetworkManager): raise NotImplementedError() -class FlatDHCPManager(FlatManager): +class FlatDHCPManager(NetworkManager): """Flat networking with dhcp. FlatDHCPManager will start up one dhcp server to give out addresses. -- cgit From 7e463d262339acc7c611b86471b1422dea50d1ee Mon Sep 17 00:00:00 2001 From: Salvatore Orlando <salvatore.orlando@eu.citrix.com> Date: Thu, 24 Feb 2011 11:43:40 +0000 Subject: xenapi plugin function now checks whether /boot/guest already exists. If not, it creates the directory --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 8cb439259..61b947c25 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -73,6 +73,10 @@ def _copy_kernel_vdi(dest, copy_args): logging.debug("copying kernel/ramdisk file from %s to /boot/guest/%s", dest, vdi_uuid) filename = KERNEL_DIR + '/' + vdi_uuid + #make sure KERNEL_DIR exists, otherwise create it + if not os.path.isdir(KERNEL_DIR): + logging.debug("Creating directory %s", KERNEL_DIR) + os.makedirs(KERNEL_DIR) #read data from /dev/ and write into a file on /boot/guest of = open(filename, 'wb') f = open(dest, 'rb') -- cgit From f8640adff4beff1e7d77bbd67771a6d072d42140 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio <armando.migliaccio@citrix.com> Date: Thu, 24 Feb 2011 13:58:31 +0000 Subject: Get DNS value from Flag, when working in FlatNetworking mode. Passing the flag was ineffective previously. This is an easy fix. I think we would need nova-manage to accept dns also from command line --- nova/network/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/network/manager.py b/nova/network/manager.py index 1df193be0..12a0c5018 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -369,6 +369,7 @@ class FlatManager(NetworkManager): project_net = IPy.IP(cidr) net = {} net['bridge'] = FLAGS.flat_network_bridge + net['dns'] = FLAGS.flat_network_dns net['cidr'] = cidr net['netmask'] = str(project_net.netmask()) net['gateway'] = str(project_net[1]) -- cgit From a721d2208ed9d88884925d6f1e2c8e26d7d1ea27 Mon Sep 17 00:00:00 2001 From: Thierry Carrez <thierry@openstack.org> Date: Thu, 24 Feb 2011 15:19:29 +0100 Subject: Globally exclude .pyc files from tarball contents --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index f0a9cffb3..2ceed34f3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -38,3 +38,4 @@ include nova/tests/db/nova.austin.sqlite include plugins/xenapi/README include plugins/xenapi/etc/xapi.d/plugins/objectstore include plugins/xenapi/etc/xapi.d/plugins/pluginlib_nova.py +global-exclude *.pyc -- cgit From 8e2ebb1a963f58514d8fb6aab4a75627e72484b9 Mon Sep 17 00:00:00 2001 From: Salvatore Orlando <salvatore.orlando@eu.citrix.com> Date: Thu, 24 Feb 2011 16:04:13 +0000 Subject: stubbing out _is_vdi_pv for test purposes --- nova/tests/test_xenapi.py | 1 + nova/tests/xenapi/stubs.py | 6 ++++++ nova/virt/xenapi/vm_utils.py | 27 ++++++++++++++------------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 6b8efc9d8..2cbe58aab 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -167,6 +167,7 @@ class XenAPIVMTestCase(test.TestCase): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) stubs.stubout_get_this_vm_uuid(self.stubs) stubs.stubout_stream_disk(self.stubs) + stubs.stubout_is_vdi_pv(self.stubs) self.stubs.Set(VMOps, 'reset_network', reset_network) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 624995ada..4fec2bd75 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -130,6 +130,12 @@ def stubout_stream_disk(stubs): stubs.Set(vm_utils, '_stream_disk', f) +def stubout_is_vdi_pv(stubs): + def f(_1): + return False + stubs.Set(vm_utils, '_is_vdi_pv', f) + + class FakeSessionForVMTests(fake.SessionBase): """ Stubs out a XenAPISession for VM tests """ def __init__(self, uri): diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 80cc3035d..564a25057 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -400,19 +400,7 @@ class VMHelper(HelperBase): @classmethod def _lookup_image_glance(cls, session, vdi_ref): LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) - - def is_vdi_pv(dev): - LOG.debug(_("Running pygrub against %s"), dev) - output = os.popen('pygrub -qn /dev/%s' % dev) - for line in output.readlines(): - #try to find kernel string - m = re.search('(?<=kernel:)/.*(?:>)', line) - if m and m.group(0).find('xen') != -1: - LOG.debug(_("Found Xen kernel %s") % m.group(0)) - return True - LOG.debug(_("No Xen kernel found. Booting HVM.")) - return False - return with_vdi_attached_here(session, vdi_ref, True, is_vdi_pv) + return with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) @classmethod def lookup(cls, session, i): @@ -714,6 +702,19 @@ def get_this_vm_ref(session): return session.get_xenapi().VM.get_by_uuid(get_this_vm_uuid()) +def _is_vdi_pv(dev): + LOG.debug(_("Running pygrub against %s"), dev) + output = os.popen('pygrub -qn /dev/%s' % dev) + for line in output.readlines(): + #try to find kernel string + m = re.search('(?<=kernel:)/.*(?:>)', line) + if m and m.group(0).find('xen') != -1: + LOG.debug(_("Found Xen kernel %s") % m.group(0)) + return True + LOG.debug(_("No Xen kernel found. Booting HVM.")) + return False + + def _stream_disk(dev, type, virtual_size, image_file): offset = 0 if type == ImageType.DISK: -- cgit From 377b59ec5ae13d37cc375dd9b9cf7eb8d89be196 Mon Sep 17 00:00:00 2001 From: Todd Willey <todd@ansolabs.com> Date: Thu, 24 Feb 2011 14:36:15 -0500 Subject: Fix copypasta pep8 violation. --- nova/adminclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index fe2aca351..fc3c5c5fe 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -198,7 +198,7 @@ class HostInfo(object): class Vpn(object): """ Information about a Vpn, as parsed through SAX - + **Fields Include** * instance_id -- cgit From 8635f7a306e4cfa41ad09a18602efa7793f6da95 Mon Sep 17 00:00:00 2001 From: Trey Morris <trey.morris@rackspace.com> Date: Thu, 24 Feb 2011 15:04:07 -0600 Subject: moved network injection and vif creation to above vm start in vmops spawn --- nova/virt/xenapi/vmops.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 76b88a8bd..3aa5a23e5 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -104,6 +104,10 @@ class VMOps(object): instance, kernel, ramdisk, pv_kernel) VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) + # inject_network_info and create vifs + networks = self.inject_network_info(instance) + self.create_vifs(instance, networks) + LOG.debug(_('Starting VM %s...'), vm_ref) self._session.call_xenapi('VM.start', vm_ref, False, False) instance_name = instance.name @@ -134,9 +138,7 @@ class VMOps(object): timer.f = _wait_for_boot - # call to reset network to inject network info and configure - networks = self.inject_network_info(instance) - self.create_vifs(instance, networks) + # call to reset network to configure network from xenstore self.reset_network(instance) return timer.start(interval=0.5, now=True) -- cgit From f4221346418ef103635b104fc152a2507d60a8dc Mon Sep 17 00:00:00 2001 From: Trey Morris <trey.morris@rackspace.com> Date: Thu, 24 Feb 2011 17:25:00 -0600 Subject: forgot to get vm_opaque_ref --- nova/virt/xenapi/vmops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3aa5a23e5..8abaa14fa 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -450,6 +450,7 @@ class VMOps(object): Creates vifs for an instance """ + vm_opaque_ref = self._get_vm_opaque_ref(instance.id) logging.debug(_("creating vif(s) for vm: |%s|"), vm_opaque_ref) if networks is None: networks = db.network_get_all_by_instance(admin_context, -- cgit From 2218cb025adca1ded3e6596acc182b88742e3a51 Mon Sep 17 00:00:00 2001 From: Todd Willey <todd@ansolabs.com> Date: Thu, 24 Feb 2011 21:59:36 -0500 Subject: Rename auth_token db methods to follow standard. --- nova/api/openstack/auth.py | 6 +++--- nova/db/api.py | 12 ++++++------ nova/db/sqlalchemy/api.py | 6 +++--- nova/tests/api/openstack/fakes.py | 6 +++--- nova/tests/api/openstack/test_auth.py | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 1dfdd5318..c844c6231 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -103,11 +103,11 @@ class AuthMiddleware(wsgi.Middleware): 2 days ago. """ ctxt = context.get_admin_context() - token = self.db.auth_get_token(ctxt, token_hash) + token = self.db.auth_token_get(ctxt, token_hash) if token: delta = datetime.datetime.now() - token.created_at if delta.days >= 2: - self.db.auth_destroy_token(ctxt, token) + self.db.auth_token_destroy(ctxt, token) else: return self.auth.get_user(token.user_id) return None @@ -131,6 +131,6 @@ class AuthMiddleware(wsgi.Middleware): token_dict['server_management_url'] = req.url token_dict['storage_url'] = '' token_dict['user_id'] = user.id - token = self.db.auth_create_token(ctxt, token_dict) + token = self.db.auth_token_create(ctxt, token_dict) return token, user return None, None diff --git a/nova/db/api.py b/nova/db/api.py index 0a010e727..aeb9b7ebf 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -630,19 +630,19 @@ def iscsi_target_create_safe(context, values): ############### -def auth_destroy_token(context, token): +def auth_token_destroy(context, token): """Destroy an auth token.""" - return IMPL.auth_destroy_token(context, token) + return IMPL.auth_token_destroy(context, token) -def auth_get_token(context, token_hash): +def auth_token_get(context, token_hash): """Retrieves a token given the hash representing it.""" - return IMPL.auth_get_token(context, token_hash) + return IMPL.auth_token_get(context, token_hash) -def auth_create_token(context, token): +def auth_token_create(context, token): """Creates a new token.""" - return IMPL.auth_create_token(context, token) + return IMPL.auth_token_create(context, token) ################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index d8751bef4..0c11b2982 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1262,13 +1262,13 @@ def iscsi_target_create_safe(context, values): @require_admin_context -def auth_destroy_token(_context, token): +def auth_token_destroy(_context, token): session = get_session() session.delete(token) @require_admin_context -def auth_get_token(_context, token_hash): +def auth_token_get(_context, token_hash): session = get_session() tk = session.query(models.AuthToken).\ filter_by(token_hash=token_hash).\ @@ -1279,7 +1279,7 @@ def auth_get_token(_context, token_hash): @require_admin_context -def auth_create_token(_context, token): +def auth_token_create(_context, token): tk = models.AuthToken() tk.update(token) tk.save() diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index fb282f1c9..142626de9 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -203,17 +203,17 @@ class FakeAuthDatabase(object): data = {} @staticmethod - def auth_get_token(context, token_hash): + def auth_token_get(context, token_hash): return FakeAuthDatabase.data.get(token_hash, None) @staticmethod - def auth_create_token(context, token): + def auth_token_create(context, token): fake_token = FakeToken(created_at=datetime.datetime.now(), **token) FakeAuthDatabase.data[fake_token.token_hash] = fake_token return fake_token @staticmethod - def auth_destroy_token(context, token): + def auth_token_destroy(context, token): if token.token_hash in FakeAuthDatabase.data: del FakeAuthDatabase.data['token_hash'] diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 13f6c3a1c..86dfb110f 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -99,10 +99,10 @@ class Test(test.TestCase): token_hash=token_hash, created_at=datetime.datetime(1990, 1, 1)) - self.stubs.Set(fakes.FakeAuthDatabase, 'auth_destroy_token', + self.stubs.Set(fakes.FakeAuthDatabase, 'auth_token_destroy', destroy_token_mock) - self.stubs.Set(fakes.FakeAuthDatabase, 'auth_get_token', + self.stubs.Set(fakes.FakeAuthDatabase, 'auth_token_get', bad_token) req = webob.Request.blank('/v1.0/') -- cgit From 865c3d57f8b84dfcc493ecead12816874b160e35 Mon Sep 17 00:00:00 2001 From: Todd Willey <todd@ansolabs.com> Date: Thu, 24 Feb 2011 23:51:17 -0500 Subject: Pass id of token to be deleted to the db api, not the actual object. --- nova/api/openstack/auth.py | 2 +- nova/db/api.py | 4 ++-- nova/db/sqlalchemy/api.py | 9 ++++++--- nova/tests/api/openstack/fakes.py | 13 ++++++++++--- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index c844c6231..dff69a7f2 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -107,7 +107,7 @@ class AuthMiddleware(wsgi.Middleware): if token: delta = datetime.datetime.now() - token.created_at if delta.days >= 2: - self.db.auth_token_destroy(ctxt, token) + self.db.auth_token_destroy(ctxt, token.id) else: return self.auth.get_user(token.user_id) return None diff --git a/nova/db/api.py b/nova/db/api.py index aeb9b7ebf..4c7eb857f 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -630,9 +630,9 @@ def iscsi_target_create_safe(context, values): ############### -def auth_token_destroy(context, token): +def auth_token_destroy(context, token_id): """Destroy an auth token.""" - return IMPL.auth_token_destroy(context, token) + return IMPL.auth_token_destroy(context, token_id) def auth_token_get(context, token_hash): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 0c11b2982..0be08c4d1 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1262,16 +1262,19 @@ def iscsi_target_create_safe(context, values): @require_admin_context -def auth_token_destroy(_context, token): +def auth_token_destroy(context, token_id): session = get_session() - session.delete(token) + with session.begin(): + token_ref = auth_token_get(context, token_id, session=session) + token_ref.delete(session=session) @require_admin_context -def auth_token_get(_context, token_hash): +def auth_token_get(context, token_hash): session = get_session() tk = session.query(models.AuthToken).\ filter_by(token_hash=token_hash).\ + filter_by(deleted=can_read_deleted(context)).\ first() if not tk: raise exception.NotFound(_('Token %s does not exist') % token_hash) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 142626de9..49ce8c1b5 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -188,7 +188,11 @@ def stub_out_glance(stubs, initial_fixtures=None): class FakeToken(object): + id = 0 + def __init__(self, **kwargs): + FakeToken.id += 1 + self.id = FakeToken.id for k, v in kwargs.iteritems(): setattr(self, k, v) @@ -210,12 +214,15 @@ class FakeAuthDatabase(object): def auth_token_create(context, token): fake_token = FakeToken(created_at=datetime.datetime.now(), **token) FakeAuthDatabase.data[fake_token.token_hash] = fake_token + FakeAuthDatabase.data['id_%i' % fake_token.id] = fake_token return fake_token @staticmethod - def auth_token_destroy(context, token): - if token.token_hash in FakeAuthDatabase.data: - del FakeAuthDatabase.data['token_hash'] + def auth_token_destroy(context, token_id): + token = FakeAuthDatabase.data.get('id_%i' % token_id) + if token and token.token_hash in FakeAuthDatabase.data: + del FakeAuthDatabase.data[token.token_hash] + del FakeAuthDatabase.data['id_%i' % token_id] class FakeAuthManager(object): -- cgit From 26d55d0d24dd5363d3b0dd6da0351fe389b79503 Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Fri, 25 Feb 2011 19:59:26 +0100 Subject: check if QUERY_STRING is empty or not before building the request URL --- bin/nova-ajax-console-proxy | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 1e11c6d58..023e4023f 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -63,10 +63,16 @@ class AjaxConsoleProxy(object): def __call__(self, env, start_response): try: - req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], - env['HTTP_HOST'], - env['PATH_INFO'], - env['QUERY_STRING']) + if env.has_key('QUERY_STRING'): + req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], + env['HTTP_HOST'], + env['PATH_INFO'], + env['QUERY_STRING']) + else: + req_url = '%s://%s%s' % (env['wsgi.url_scheme'], + env['HTTP_HOST'], + env['PATH_INFO']) + if 'HTTP_REFERER' in env: auth_url = env['HTTP_REFERER'] else: -- cgit From 4453021476fac599c0cee126b6eaa426d4878145 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Fri, 18 Mar 2011 02:09:46 +0000 Subject: Copy over to current trunk my tests, the 401/500 fix, and a couple of fixes to the committed fix which was actually brittle around the edges... --- Authors | 1 + nova/api/openstack/auth.py | 8 ++++++-- nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 14 ++++++++++++-- nova/tests/api/openstack/test_auth.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/Authors b/Authors index 494e614a0..a3c06dcf5 100644 --- a/Authors +++ b/Authors @@ -36,6 +36,7 @@ Joshua McKenty <jmckenty@gmail.com> Justin Santa Barbara <justin@fathomdb.com> Kei Masumoto <masumotok@nttdata.co.jp> Ken Pepple <ken.pepple@gmail.com> +Kevin L. Mitchell <kevin.mitchell@rackspace.com> Koji Iida <iida.koji@lab.ntt.co.jp> Lorin Hochstein <lorin@isi.edu> Matt Dietz <matt.dietz@rackspace.com> diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index dff69a7f2..6011e6115 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -26,6 +26,7 @@ import webob.dec from nova import auth from nova import context from nova import db +from nova import exception from nova import flags from nova import manager from nova import utils @@ -103,11 +104,14 @@ class AuthMiddleware(wsgi.Middleware): 2 days ago. """ ctxt = context.get_admin_context() - token = self.db.auth_token_get(ctxt, token_hash) + try: + token = self.db.auth_token_get(ctxt, token_hash) + except exception.NotFound: + return None if token: delta = datetime.datetime.now() - token.created_at if delta.days >= 2: - self.db.auth_token_destroy(ctxt, token.id) + self.db.auth_token_destroy(ctxt, token.token_hash) else: return self.auth.get_user(token.user_id) return None diff --git a/nova/db/api.py b/nova/db/api.py index 4c7eb857f..dcaf55e8f 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -640,6 +640,11 @@ def auth_token_get(context, token_hash): return IMPL.auth_token_get(context, token_hash) +def auth_token_update(context, token_hash, values): + """Updates a token given the hash representing it.""" + return IMPL.auth_token_update(context, token_hash, values) + + def auth_token_create(context, token): """Creates a new token.""" return IMPL.auth_token_create(context, token) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 0be08c4d1..6df2a8843 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1270,8 +1270,9 @@ def auth_token_destroy(context, token_id): @require_admin_context -def auth_token_get(context, token_hash): - session = get_session() +def auth_token_get(context, token_hash, session=None): + if session is None: + session = get_session() tk = session.query(models.AuthToken).\ filter_by(token_hash=token_hash).\ filter_by(deleted=can_read_deleted(context)).\ @@ -1281,6 +1282,15 @@ def auth_token_get(context, token_hash): return tk +@require_admin_context +def auth_token_update(context, token_hash, values): + session = get_session() + with session.begin(): + token_ref = auth_token_get(context, token_hash, session=session) + token_ref.update(values) + token_ref.save(session=session) + + @require_admin_context def auth_token_create(_context, token): tk = models.AuthToken() diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 86dfb110f..c42c12f7b 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -26,6 +26,7 @@ import nova.api.openstack.auth import nova.auth.manager from nova import auth from nova import context +from nova import db from nova import test from nova.tests.api.openstack import fakes @@ -130,6 +131,33 @@ class Test(test.TestCase): self.assertEqual(result.status, '401 Unauthorized') +class TestFunctional(test.TestCase): + def test_token_lp718999(self): + ctx = context.get_admin_context() + tok = db.auth_token_create(ctx, dict( + token_hash='bacon', + cdn_management_url='', + server_management_url='', + storage_url='', + user_id='ham', + )) + + db.auth_token_update(ctx, tok.token_hash, dict( + created_at=datetime.datetime(2000, 1, 1, 12, 0, 0), + )) + + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-Token'] = 'bacon' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_token_doesnotexist(self): + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-Token'] = 'ham' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + class TestLimiter(test.TestCase): def setUp(self): super(TestLimiter, self).setUp() -- cgit From fa6778586ab303f9e65aa3c50b80d20a4f097c6f Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Mon, 21 Mar 2011 00:41:23 +0000 Subject: Rename test to describe what it actually does --- nova/tests/api/openstack/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index c42c12f7b..ff8d42a14 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -132,7 +132,7 @@ class Test(test.TestCase): class TestFunctional(test.TestCase): - def test_token_lp718999(self): + def test_token_expiry(self): ctx = context.get_admin_context() tok = db.auth_token_create(ctx, dict( token_hash='bacon', -- cgit From 3368b32338f58ac6b77d0c142722c241961f858e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Fri, 25 Feb 2011 12:16:58 -0800 Subject: use default flagfile in nova-api --- bin/nova-api | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/nova-api b/bin/nova-api index cf140570a..14be4b841 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -36,6 +36,7 @@ gettext.install('nova', unicode=1) from nova import flags from nova import log as logging +from nova import utils from nova import version from nova import wsgi @@ -80,6 +81,7 @@ def run_app(paste_config_file): if __name__ == '__main__': + utils.default_flagfile() FLAGS(sys.argv) logging.setup() LOG.audit(_("Starting nova-api node (version %s)"), -- cgit From e6ceb2987d1f30a1e972a4bf21a734f1a605d60e Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Fri, 25 Feb 2011 22:15:51 +0100 Subject: fixed: bin/nova-ajax-console-proxy:66:19: W601 .has_key() is deprecated, use 'in' --- bin/nova-ajax-console-proxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 023e4023f..bbd60bade 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -63,7 +63,7 @@ class AjaxConsoleProxy(object): def __call__(self, env, start_response): try: - if env.has_key('QUERY_STRING'): + if 'QUERY_STRING' in env: req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], env['HTTP_HOST'], env['PATH_INFO'], -- cgit From c7c3fe85186862d9d8989ebe01ad69e6cdaa4ee3 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Sun, 20 Mar 2011 22:58:49 +0000 Subject: No reason to dump a stack trace just because we can't reach the AMQP servire; it ends up being just noise --- nova/rpc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 205bb524a..089bf99c1 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -95,14 +95,14 @@ class Consumer(messaging.Consumer): fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = FLAGS.rabbit_retry_interval - LOG.exception(_("AMQP server on %(fl_host)s:%(fl_port)d is" + LOG.error(_("AMQP server on %(fl_host)s:%(fl_port)d is" " unreachable. Trying again in %(fl_intv)d seconds.") % locals()) self.failed_connection = True if self.failed_connection: - LOG.exception(_("Unable to connect to AMQP server " - "after %d tries. Shutting down."), - FLAGS.rabbit_max_retries) + LOG.error(_("Unable to connect to AMQP server " + "after %d tries. Shutting down."), + FLAGS.rabbit_max_retries) sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): -- cgit From 4a217b640c3e0f488c87572787c92b6808ab0a9a Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Fri, 25 Mar 2011 05:30:36 +0000 Subject: Add error message to the error report so we know why the AMQP server is unreachable --- nova/rpc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 089bf99c1..8fe4565dd 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -91,12 +91,13 @@ class Consumer(messaging.Consumer): super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False break - except: # Catching all because carrot sucks + except Exception as e: # Catching all because carrot sucks fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = FLAGS.rabbit_retry_interval LOG.error(_("AMQP server on %(fl_host)s:%(fl_port)d is" - " unreachable. Trying again in %(fl_intv)d seconds.") + " unreachable: %(e)s. Trying again in %(fl_intv)d" + " seconds.") % locals()) self.failed_connection = True if self.failed_connection: -- cgit From 4c83130464c88650df7dfc7974f2fd088a733fec Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Sat, 26 Feb 2011 17:26:38 +0100 Subject: introduced new flag "maximum_nbd_devices" to set the number of possible NBD devices --- nova/virt/disk.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/virt/disk.py b/nova/virt/disk.py index cb639a102..8789d8123 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -40,6 +40,8 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') flags.DEFINE_integer('timeout_nbd', 10, 'time to wait for a NBD device coming up') +flags.DEFINE_integer('maximum_nbd_devices', 16, + 'maximum number of possible nbd devices') def extend(image, size): @@ -141,7 +143,7 @@ def _unlink_device(device, nbd): utils.execute('sudo losetup --detach %s' % device) -_DEVICES = ['/dev/nbd%s' % i for i in xrange(16)] +_DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.maximum_nbd_devices)] def _allocate_device(): -- cgit From 2a3ea106d3aaac3b97295c14742f23b250aa6874 Mon Sep 17 00:00:00 2001 From: Anne Gentle <anne@openstack.org> Date: Sat, 26 Feb 2011 11:13:32 -0600 Subject: Deleting test dir from a pull from trunk --- test/.Python | 1 - test/bin/activate | 76 -- test/bin/activate.csh | 32 - test/bin/activate.fish | 79 -- test/bin/activate_this.py | 32 - test/bin/easy_install | 9 - test/bin/easy_install-2.6 | 9 - test/bin/pip | 9 - test/bin/pip-2.6 | 9 - test/bin/python | Bin 50720 -> 0 bytes test/bin/python2.6 | 1 - test/include/python2.6 | 1 - test/lib/python2.6/UserDict.py | 1 - test/lib/python2.6/_abcoll.py | 1 - test/lib/python2.6/abc.py | 1 - test/lib/python2.6/codecs.py | 1 - test/lib/python2.6/config | 1 - test/lib/python2.6/copy_reg.py | 1 - test/lib/python2.6/distutils/__init__.py | 91 -- test/lib/python2.6/distutils/distutils.cfg | 6 - test/lib/python2.6/encodings | 1 - test/lib/python2.6/fnmatch.py | 1 - test/lib/python2.6/genericpath.py | 1 - test/lib/python2.6/lib-dynload | 1 - test/lib/python2.6/linecache.py | 1 - test/lib/python2.6/locale.py | 1 - test/lib/python2.6/ntpath.py | 1 - test/lib/python2.6/orig-prefix.txt | 1 - test/lib/python2.6/os.py | 1 - test/lib/python2.6/posixpath.py | 1 - test/lib/python2.6/re.py | 1 - test/lib/python2.6/site-packages/easy-install.pth | 4 - .../pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO | 348 ----- .../pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt | 57 - .../EGG-INFO/dependency_links.txt | 1 - .../pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt | 4 - .../pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe | 1 - .../pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt | 1 - .../pip-0.8.1-py2.6.egg/pip/__init__.py | 261 ---- .../pip-0.8.1-py2.6.egg/pip/_pkgutil.py | 589 -------- .../pip-0.8.1-py2.6.egg/pip/backwardcompat.py | 55 - .../pip-0.8.1-py2.6.egg/pip/basecommand.py | 203 --- .../pip-0.8.1-py2.6.egg/pip/baseparser.py | 231 ---- .../pip-0.8.1-py2.6.egg/pip/commands/__init__.py | 1 - .../pip-0.8.1-py2.6.egg/pip/commands/bundle.py | 33 - .../pip-0.8.1-py2.6.egg/pip/commands/completion.py | 60 - .../pip-0.8.1-py2.6.egg/pip/commands/freeze.py | 109 -- .../pip-0.8.1-py2.6.egg/pip/commands/help.py | 32 - .../pip-0.8.1-py2.6.egg/pip/commands/install.py | 247 ---- .../pip-0.8.1-py2.6.egg/pip/commands/search.py | 116 -- .../pip-0.8.1-py2.6.egg/pip/commands/uninstall.py | 42 - .../pip-0.8.1-py2.6.egg/pip/commands/unzip.py | 9 - .../pip-0.8.1-py2.6.egg/pip/commands/zip.py | 346 ----- .../pip-0.8.1-py2.6.egg/pip/download.py | 470 ------- .../pip-0.8.1-py2.6.egg/pip/exceptions.py | 17 - .../site-packages/pip-0.8.1-py2.6.egg/pip/index.py | 686 ---------- .../pip-0.8.1-py2.6.egg/pip/locations.py | 45 - .../site-packages/pip-0.8.1-py2.6.egg/pip/log.py | 181 --- .../site-packages/pip-0.8.1-py2.6.egg/pip/req.py | 1432 -------------------- .../pip-0.8.1-py2.6.egg/pip/runner.py | 18 - .../site-packages/pip-0.8.1-py2.6.egg/pip/util.py | 479 ------- .../pip-0.8.1-py2.6.egg/pip/vcs/__init__.py | 238 ---- .../pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py | 138 -- .../pip-0.8.1-py2.6.egg/pip/vcs/git.py | 204 --- .../pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py | 162 --- .../pip-0.8.1-py2.6.egg/pip/vcs/subversion.py | 260 ---- .../site-packages/pip-0.8.1-py2.6.egg/pip/venv.py | 53 - .../site-packages/setuptools-0.6c11-py2.6.egg | Bin 333447 -> 0 bytes test/lib/python2.6/site-packages/setuptools.pth | 1 - test/lib/python2.6/site.py | 713 ---------- test/lib/python2.6/sre.py | 1 - test/lib/python2.6/sre_compile.py | 1 - test/lib/python2.6/sre_constants.py | 1 - test/lib/python2.6/sre_parse.py | 1 - test/lib/python2.6/stat.py | 1 - test/lib/python2.6/types.py | 1 - test/lib/python2.6/warnings.py | 1 - 77 files changed, 8226 deletions(-) delete mode 120000 test/.Python delete mode 100644 test/bin/activate delete mode 100644 test/bin/activate.csh delete mode 100644 test/bin/activate.fish delete mode 100644 test/bin/activate_this.py delete mode 100755 test/bin/easy_install delete mode 100755 test/bin/easy_install-2.6 delete mode 100755 test/bin/pip delete mode 100755 test/bin/pip-2.6 delete mode 100755 test/bin/python delete mode 120000 test/bin/python2.6 delete mode 120000 test/include/python2.6 delete mode 120000 test/lib/python2.6/UserDict.py delete mode 120000 test/lib/python2.6/_abcoll.py delete mode 120000 test/lib/python2.6/abc.py delete mode 120000 test/lib/python2.6/codecs.py delete mode 120000 test/lib/python2.6/config delete mode 120000 test/lib/python2.6/copy_reg.py delete mode 100644 test/lib/python2.6/distutils/__init__.py delete mode 100644 test/lib/python2.6/distutils/distutils.cfg delete mode 120000 test/lib/python2.6/encodings delete mode 120000 test/lib/python2.6/fnmatch.py delete mode 120000 test/lib/python2.6/genericpath.py delete mode 120000 test/lib/python2.6/lib-dynload delete mode 120000 test/lib/python2.6/linecache.py delete mode 120000 test/lib/python2.6/locale.py delete mode 120000 test/lib/python2.6/ntpath.py delete mode 100644 test/lib/python2.6/orig-prefix.txt delete mode 120000 test/lib/python2.6/os.py delete mode 120000 test/lib/python2.6/posixpath.py delete mode 120000 test/lib/python2.6/re.py delete mode 100644 test/lib/python2.6/site-packages/easy-install.pth delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py delete mode 100644 test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg delete mode 100644 test/lib/python2.6/site-packages/setuptools.pth delete mode 100644 test/lib/python2.6/site.py delete mode 120000 test/lib/python2.6/sre.py delete mode 120000 test/lib/python2.6/sre_compile.py delete mode 120000 test/lib/python2.6/sre_constants.py delete mode 120000 test/lib/python2.6/sre_parse.py delete mode 120000 test/lib/python2.6/stat.py delete mode 120000 test/lib/python2.6/types.py delete mode 120000 test/lib/python2.6/warnings.py diff --git a/test/.Python b/test/.Python deleted file mode 120000 index 6cce156bd..000000000 --- a/test/.Python +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/Python \ No newline at end of file diff --git a/test/bin/activate b/test/bin/activate deleted file mode 100644 index 588b54105..000000000 --- a/test/bin/activate +++ /dev/null @@ -1,76 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "$_OLD_VIRTUAL_PATH" ] ; then - PATH="$_OLD_VIRTUAL_PATH" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then - PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then - hash -r - fi - - if [ -n "$_OLD_VIRTUAL_PS1" ] ; then - PS1="$_OLD_VIRTUAL_PS1" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - if [ ! "$1" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelavent variables -deactivate nondestructive - -VIRTUAL_ENV="/Users/anne.gentle/src/nova/docslice/test" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "$PYTHONHOME" ] ; then - _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" - unset PYTHONHOME -fi - -if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then - _OLD_VIRTUAL_PS1="$PS1" - if [ "x" != x ] ; then - PS1="$PS1" - else - if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" - else - PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" - fi - fi - export PS1 -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then - hash -r -fi diff --git a/test/bin/activate.csh b/test/bin/activate.csh deleted file mode 100644 index d29ac339a..000000000 --- a/test/bin/activate.csh +++ /dev/null @@ -1,32 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi <davidedb@gmail.com>. - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelavent variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if ("" != "") then - set env_name = "" -else - if (`basename "$VIRTUAL_ENV"` == "__") then - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` - else - set env_name = `basename "$VIRTUAL_ENV"` - endif -endif -set prompt = "[$env_name] $prompt" -unset env_name - -rehash - diff --git a/test/bin/activate.fish b/test/bin/activate.fish deleted file mode 100644 index de3a8901e..000000000 --- a/test/bin/activate.fish +++ /dev/null @@ -1,79 +0,0 @@ -# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) -# you cannot run it directly - -function deactivate -d "Exit virtualenv and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - end - - set -e VIRTUAL_ENV - if test "$argv[1]" != "nondestructive" - # Self destruct! - functions -e deactivate - end -end - -# unset irrelavent variables -deactivate nondestructive - -set -gx VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# unset PYTHONHOME if set -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish shell uses a function, instead of env vars, - # to produce the prompt. Overriding the existing function is easy. - # However, adding to the current prompt, instead of clobbering it, - # is a little more work. - set -l oldpromptfile (tempfile) - if test $status - # save the current fish_prompt function... - echo "function _old_fish_prompt" >> $oldpromptfile - echo -n \# >> $oldpromptfile - functions fish_prompt >> $oldpromptfile - # we've made the "_old_fish_prompt" file, source it. - . $oldpromptfile - rm -f $oldpromptfile - - if test -n "" - # We've been given us a prompt override. - # - # FIXME: Unsure how to handle this *safely*. We could just eval() - # whatever is given, but the risk is a bit much. - echo "activate.fish: Alternative prompt prefix is not supported under fish-shell." 1>&2 - echo "activate.fish: Alter the fish_prompt in this file as needed." 1>&2 - end - - # with the original prompt function renamed, we can override with our own. - function fish_prompt - set -l _checkbase (basename "$VIRTUAL_ENV") - if test $_checkbase = "__" - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - printf "%s[%s]%s %s" (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) (_old_fish_prompt) - else - printf "%s(%s)%s%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) (_old_fish_prompt) - end - end - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - end -end - diff --git a/test/bin/activate_this.py b/test/bin/activate_this.py deleted file mode 100644 index aff6927d6..000000000 --- a/test/bin/activate_this.py +++ /dev/null @@ -1,32 +0,0 @@ -"""By using execfile(this_file, dict(__file__=this_file)) you will -activate this virtualenv environment. - -This can be used when you must use an existing Python interpreter, not -the virtualenv bin/python -""" - -try: - __file__ -except NameError: - raise AssertionError( - "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))") -import sys -import os - -base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if sys.platform == 'win32': - site_packages = os.path.join(base, 'Lib', 'site-packages') -else: - site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') -prev_sys_path = list(sys.path) -import site -site.addsitedir(site_packages) -sys.real_prefix = sys.prefix -sys.prefix = base -# Move the added items to the front of the path: -new_sys_path = [] -for item in list(sys.path): - if item not in prev_sys_path: - new_sys_path.append(item) - sys.path.remove(item) -sys.path[:0] = new_sys_path diff --git a/test/bin/easy_install b/test/bin/easy_install deleted file mode 100755 index 8544573fc..000000000 --- a/test/bin/easy_install +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install' -__requires__ = 'setuptools==0.6c11' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install')() -) diff --git a/test/bin/easy_install-2.6 b/test/bin/easy_install-2.6 deleted file mode 100755 index bad01d2d7..000000000 --- a/test/bin/easy_install-2.6 +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install-2.6' -__requires__ = 'setuptools==0.6c11' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install-2.6')() -) diff --git a/test/bin/pip b/test/bin/pip deleted file mode 100755 index a3d2ed311..000000000 --- a/test/bin/pip +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip' -__requires__ = 'pip==0.8.1' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('pip==0.8.1', 'console_scripts', 'pip')() -) diff --git a/test/bin/pip-2.6 b/test/bin/pip-2.6 deleted file mode 100755 index 4ad06add1..000000000 --- a/test/bin/pip-2.6 +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip-2.6' -__requires__ = 'pip==0.8.1' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('pip==0.8.1', 'console_scripts', 'pip-2.6')() -) diff --git a/test/bin/python b/test/bin/python deleted file mode 100755 index 271b6ca30..000000000 Binary files a/test/bin/python and /dev/null differ diff --git a/test/bin/python2.6 b/test/bin/python2.6 deleted file mode 120000 index d8654aa0e..000000000 --- a/test/bin/python2.6 +++ /dev/null @@ -1 +0,0 @@ -python \ No newline at end of file diff --git a/test/include/python2.6 b/test/include/python2.6 deleted file mode 120000 index 788ac55ba..000000000 --- a/test/include/python2.6 +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/include/python2.6 \ No newline at end of file diff --git a/test/lib/python2.6/UserDict.py b/test/lib/python2.6/UserDict.py deleted file mode 120000 index 93e0864b1..000000000 --- a/test/lib/python2.6/UserDict.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/UserDict.py \ No newline at end of file diff --git a/test/lib/python2.6/_abcoll.py b/test/lib/python2.6/_abcoll.py deleted file mode 120000 index ff1769246..000000000 --- a/test/lib/python2.6/_abcoll.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/_abcoll.py \ No newline at end of file diff --git a/test/lib/python2.6/abc.py b/test/lib/python2.6/abc.py deleted file mode 120000 index 79fa108b2..000000000 --- a/test/lib/python2.6/abc.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/abc.py \ No newline at end of file diff --git a/test/lib/python2.6/codecs.py b/test/lib/python2.6/codecs.py deleted file mode 120000 index 9e4bfb7fa..000000000 --- a/test/lib/python2.6/codecs.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/codecs.py \ No newline at end of file diff --git a/test/lib/python2.6/config b/test/lib/python2.6/config deleted file mode 120000 index eef498c62..000000000 --- a/test/lib/python2.6/config +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/config \ No newline at end of file diff --git a/test/lib/python2.6/copy_reg.py b/test/lib/python2.6/copy_reg.py deleted file mode 120000 index 7285fda0e..000000000 --- a/test/lib/python2.6/copy_reg.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/copy_reg.py \ No newline at end of file diff --git a/test/lib/python2.6/distutils/__init__.py b/test/lib/python2.6/distutils/__init__.py deleted file mode 100644 index 7ebb41c04..000000000 --- a/test/lib/python2.6/distutils/__init__.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -import warnings -import opcode # opcode is not a virtualenv module, so we can use it to find the stdlib - # Important! To work on pypy, this must be a module that resides in the - # lib-python/modified-x.y.z directory - -dirname = os.path.dirname - -distutils_path = os.path.join(os.path.dirname(opcode.__file__), 'distutils') -if os.path.normpath(distutils_path) == os.path.dirname(os.path.normpath(__file__)): - warnings.warn( - "The virtualenv distutils package at %s appears to be in the same location as the system distutils?") -else: - __path__.insert(0, distutils_path) - exec open(os.path.join(distutils_path, '__init__.py')).read() - -import dist -import sysconfig - - -## patch build_ext (distutils doesn't know how to get the libs directory -## path on windows - it hardcodes the paths around the patched sys.prefix) - -if sys.platform == 'win32': - from distutils.command.build_ext import build_ext as old_build_ext - class build_ext(old_build_ext): - def finalize_options (self): - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, basestring): - self.library_dirs = self.library_dirs.split(os.pathsep) - - self.library_dirs.insert(0, os.path.join(sys.real_prefix, "Libs")) - old_build_ext.finalize_options(self) - - from distutils.command import build_ext as build_ext_module - build_ext_module.build_ext = build_ext - -## distutils.dist patches: - -old_find_config_files = dist.Distribution.find_config_files -def find_config_files(self): - found = old_find_config_files(self) - system_distutils = os.path.join(distutils_path, 'distutils.cfg') - #if os.path.exists(system_distutils): - # found.insert(0, system_distutils) - # What to call the per-user config file - if os.name == 'posix': - user_filename = ".pydistutils.cfg" - else: - user_filename = "pydistutils.cfg" - user_filename = os.path.join(sys.prefix, user_filename) - if os.path.isfile(user_filename): - for item in list(found): - if item.endswith('pydistutils.cfg'): - found.remove(item) - found.append(user_filename) - return found -dist.Distribution.find_config_files = find_config_files - -## distutils.sysconfig patches: - -old_get_python_inc = sysconfig.get_python_inc -def sysconfig_get_python_inc(plat_specific=0, prefix=None): - if prefix is None: - prefix = sys.real_prefix - return old_get_python_inc(plat_specific, prefix) -sysconfig_get_python_inc.__doc__ = old_get_python_inc.__doc__ -sysconfig.get_python_inc = sysconfig_get_python_inc - -old_get_python_lib = sysconfig.get_python_lib -def sysconfig_get_python_lib(plat_specific=0, standard_lib=0, prefix=None): - if standard_lib and prefix is None: - prefix = sys.real_prefix - return old_get_python_lib(plat_specific, standard_lib, prefix) -sysconfig_get_python_lib.__doc__ = old_get_python_lib.__doc__ -sysconfig.get_python_lib = sysconfig_get_python_lib - -old_get_config_vars = sysconfig.get_config_vars -def sysconfig_get_config_vars(*args): - real_vars = old_get_config_vars(*args) - if sys.platform == 'win32': - lib_dir = os.path.join(sys.real_prefix, "libs") - if isinstance(real_vars, dict) and 'LIBDIR' not in real_vars: - real_vars['LIBDIR'] = lib_dir # asked for all - elif isinstance(real_vars, list) and 'LIBDIR' in args: - real_vars = real_vars + [lib_dir] # asked for list - return real_vars -sysconfig_get_config_vars.__doc__ = old_get_config_vars.__doc__ -sysconfig.get_config_vars = sysconfig_get_config_vars diff --git a/test/lib/python2.6/distutils/distutils.cfg b/test/lib/python2.6/distutils/distutils.cfg deleted file mode 100644 index 1af230ec9..000000000 --- a/test/lib/python2.6/distutils/distutils.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# This is a config file local to this virtualenv installation -# You may include options that will be used by all distutils commands, -# and by easy_install. For instance: -# -# [easy_install] -# find_links = http://mylocalsite diff --git a/test/lib/python2.6/encodings b/test/lib/python2.6/encodings deleted file mode 120000 index 89f28f824..000000000 --- a/test/lib/python2.6/encodings +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/encodings \ No newline at end of file diff --git a/test/lib/python2.6/fnmatch.py b/test/lib/python2.6/fnmatch.py deleted file mode 120000 index cce0594f2..000000000 --- a/test/lib/python2.6/fnmatch.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/fnmatch.py \ No newline at end of file diff --git a/test/lib/python2.6/genericpath.py b/test/lib/python2.6/genericpath.py deleted file mode 120000 index b14e1bc26..000000000 --- a/test/lib/python2.6/genericpath.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/genericpath.py \ No newline at end of file diff --git a/test/lib/python2.6/lib-dynload b/test/lib/python2.6/lib-dynload deleted file mode 120000 index 4644b7072..000000000 --- a/test/lib/python2.6/lib-dynload +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/lib-dynload \ No newline at end of file diff --git a/test/lib/python2.6/linecache.py b/test/lib/python2.6/linecache.py deleted file mode 120000 index 783624da8..000000000 --- a/test/lib/python2.6/linecache.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/linecache.py \ No newline at end of file diff --git a/test/lib/python2.6/locale.py b/test/lib/python2.6/locale.py deleted file mode 120000 index 4e674c7b6..000000000 --- a/test/lib/python2.6/locale.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/locale.py \ No newline at end of file diff --git a/test/lib/python2.6/ntpath.py b/test/lib/python2.6/ntpath.py deleted file mode 120000 index 9b6b40f48..000000000 --- a/test/lib/python2.6/ntpath.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/ntpath.py \ No newline at end of file diff --git a/test/lib/python2.6/orig-prefix.txt b/test/lib/python2.6/orig-prefix.txt deleted file mode 100644 index 535eb0f0e..000000000 --- a/test/lib/python2.6/orig-prefix.txt +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6 \ No newline at end of file diff --git a/test/lib/python2.6/os.py b/test/lib/python2.6/os.py deleted file mode 120000 index 92e6e9a7c..000000000 --- a/test/lib/python2.6/os.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/os.py \ No newline at end of file diff --git a/test/lib/python2.6/posixpath.py b/test/lib/python2.6/posixpath.py deleted file mode 120000 index c095d16a1..000000000 --- a/test/lib/python2.6/posixpath.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/posixpath.py \ No newline at end of file diff --git a/test/lib/python2.6/re.py b/test/lib/python2.6/re.py deleted file mode 120000 index b4710c5f7..000000000 --- a/test/lib/python2.6/re.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/re.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/easy-install.pth b/test/lib/python2.6/site-packages/easy-install.pth deleted file mode 100644 index 7a6ae2b6d..000000000 --- a/test/lib/python2.6/site-packages/easy-install.pth +++ /dev/null @@ -1,4 +0,0 @@ -import sys; sys.__plen = len(sys.path) -./setuptools-0.6c11-py2.6.egg -./pip-0.8.1-py2.6.egg -import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO deleted file mode 100644 index 29c30cb8c..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO +++ /dev/null @@ -1,348 +0,0 @@ -Metadata-Version: 1.0 -Name: pip -Version: 0.8.1 -Summary: pip installs packages. Python packages. An easy_install replacement -Home-page: http://pip.openplans.org -Author: Ian Bicking -Author-email: python-virtualenv@groups.google.com -License: MIT -Description: The main website for pip is `pip.openplans.org - <http://pip.openplans.org>`_. You can also install - the `in-development version <http://bitbucket.org/ianb/pip/get/tip.gz#egg=pip-dev>`_ - of pip with ``easy_install pip==dev``. - - - Introduction - ------------ - - pip installs packages. Python packages. - - If you use `virtualenv <http://virtualenv.openplans.org>`__ -- a tool - for installing libraries in a local and isolated manner -- you'll - automatically get a copy of pip. Free bonus! - - Once you have pip, you can use it like this:: - - $ pip install SomePackage - - SomePackage is some package you'll find on `PyPI - <http://pypi.python.org/pypi/>`_. This installs the package and all - its dependencies. - - pip does other stuff too, with packages, but install is the biggest - one. You can ``pip uninstall`` too. - - You can also install from a URL (that points to a tar or zip file), - install from some version control system (use URLs like - ``hg+http://domain/repo`` -- or prefix ``git+``, ``svn+`` etc). pip - knows a bunch of stuff about revisions and stuff, so if you need to do - things like install a very specific revision from a repository pip can - do that too. - - If you've ever used ``python setup.py develop``, you can do something - like that with ``pip install -e ./`` -- this works with packages that - use ``distutils`` too (usually this only works with Setuptools - projects). - - You can use ``pip install --upgrade SomePackage`` to upgrade to a - newer version, or ``pip install SomePackage==1.0.4`` to install a very - specific version. - - Pip Compared To easy_install - ---------------------------- - - pip is a replacement for `easy_install - <http://peak.telecommunity.com/DevCenter/EasyInstall>`_. It uses mostly the - same techniques for finding packages, so packages that were made - easy_installable should be pip-installable as well. - - pip is meant to improve on easy_install. Some of the improvements: - - * All packages are downloaded before installation. Partially-completed - installation doesn't occur as a result. - - * Care is taken to present useful output on the console. - - * The reasons for actions are kept track of. For instance, if a package is - being installed, pip keeps track of why that package was required. - - * Error messages should be useful. - - * The code is relatively concise and cohesive, making it easier to use - programmatically. - - * Packages don't have to be installed as egg archives, they can be installed - flat (while keeping the egg metadata). - - * Native support for other version control systems (Git, Mercurial and Bazaar) - - * Uninstallation of packages. - - * Simple to define fixed sets of requirements and reliably reproduce a - set of packages. - - pip doesn't do everything that easy_install does. Specifically: - - * It cannot install from eggs. It only installs from source. (In the - future it would be good if it could install binaries from Windows ``.exe`` - or ``.msi`` -- binary install on other platforms is not a priority.) - - * It doesn't understand Setuptools extras (like ``package[test]``). This should - be added eventually. - - * It is incompatible with some packages that extensively customize distutils - or setuptools in their ``setup.py`` files. - - pip is complementary with `virtualenv - <http://pypi.python.org/pypi/virtualenv>`__, and it is encouraged that you use - virtualenv to isolate your installation. - - Community - --------- - - The homepage for pip is temporarily located `on PyPI - <http://pypi.python.org/pypi/pip>`_ -- a more proper homepage will - follow. Bugs can go on the `pip issue tracker - <http://bitbucket.org/ianb/pip/issues/>`_. Discussion should happen on the - `virtualenv email group - <http://groups.google.com/group/python-virtualenv?hl=en>`_. - - Uninstall - --------- - - pip is able to uninstall most installed packages with ``pip uninstall - package-name``. - - Known exceptions include pure-distutils packages installed with - ``python setup.py install`` (such packages leave behind no metadata allowing - determination of what files were installed), and script wrappers installed - by develop-installs (``python setup.py develop``). - - pip also performs an automatic uninstall of an old version of a package - before upgrading to a newer version, so outdated files (and egg-info data) - from conflicting versions aren't left hanging around to cause trouble. The - old version of the package is automatically restored if the new version - fails to download or install. - - .. _`requirements file`: - - Requirements Files - ------------------ - - When installing software, and Python packages in particular, it's common that - you get a lot of libraries installed. You just did ``easy_install MyPackage`` - and you get a dozen packages. Each of these packages has its own version. - - Maybe you ran that installation and it works. Great! Will it keep working? - Did you have to provide special options to get it to find everything? Did you - have to install a bunch of other optional pieces? Most of all, will you be able - to do it again? Requirements files give you a way to create an *environment*: - a *set* of packages that work together. - - If you've ever tried to setup an application on a new system, or with slightly - updated pieces, and had it fail, pip requirements are for you. If you - haven't had this problem then you will eventually, so pip requirements are - for you too -- requirements make explicit, repeatable installation of packages. - - So what are requirements files? They are very simple: lists of packages to - install. Instead of running something like ``pip MyApp`` and getting - whatever libraries come along, you can create a requirements file something like:: - - MyApp - Framework==0.9.4 - Library>=0.2 - - Then, regardless of what MyApp lists in ``setup.py``, you'll get a - specific version of Framework (0.9.4) and at least the 0.2 version of - Library. (You might think you could list these specific versions in - MyApp's ``setup.py`` -- but if you do that you'll have to edit MyApp - if you want to try a new version of Framework, or release a new - version of MyApp if you determine that Library 0.3 doesn't work with - your application.) You can also add optional libraries and support - tools that MyApp doesn't strictly require, giving people a set of - recommended libraries. - - You can also include "editable" packages -- packages that are checked out from - Subversion, Git, Mercurial and Bazaar. These are just like using the ``-e`` - option to pip. They look like:: - - -e svn+http://myrepo/svn/MyApp#egg=MyApp - - You have to start the URL with ``svn+`` (``git+``, ``hg+`` or ``bzr+``), and - you have to include ``#egg=Package`` so pip knows what to expect at that URL. - You can also include ``@rev`` in the URL, e.g., ``@275`` to check out - revision 275. - - Requirement files are mostly *flat*. Maybe ``MyApp`` requires - ``Framework``, and ``Framework`` requires ``Library``. I encourage - you to still list all these in a single requirement file; it is the - nature of Python programs that there are implicit bindings *directly* - between MyApp and Library. For instance, Framework might expose one - of Library's objects, and so if Library is updated it might directly - break MyApp. If that happens you can update the requirements file to - force an earlier version of Library, and you can do that without - having to re-release MyApp at all. - - Read the `requirements file format <http://pip.openplans.org/requirement-format.html>`_ to - learn about other features. - - Freezing Requirements - --------------------- - - So you have a working set of packages, and you want to be able to install them - elsewhere. `Requirements files`_ let you install exact versions, but it won't - tell you what all the exact versions are. - - To create a new requirements file from a known working environment, use:: - - $ pip freeze > stable-req.txt - - This will write a listing of *all* installed libraries to ``stable-req.txt`` - with exact versions for every library. You may want to edit the file down after - generating (e.g., to eliminate unnecessary libraries), but it'll give you a - stable starting point for constructing your requirements file. - - You can also give it an existing requirements file, and it will use that as a - sort of template for the new file. So if you do:: - - $ pip freeze -r devel-req.txt > stable-req.txt - - it will keep the packages listed in ``devel-req.txt`` in order and preserve - comments. - - Bundles - ------- - - Another way to distribute a set of libraries is a bundle format (specific to - pip). This format is not stable at this time (there simply hasn't been - any feedback, nor a great deal of thought). A bundle file contains all the - source for your package, and you can have pip install them all together. - Once you have the bundle file further network access won't be necessary. To - build a bundle file, do:: - - $ pip bundle MyApp.pybundle MyApp - - (Using a `requirements file`_ would be wise.) Then someone else can get the - file ``MyApp.pybundle`` and run:: - - $ pip install MyApp.pybundle - - This is *not* a binary format. This only packages source. If you have binary - packages, then the person who installs the files will have to have a compiler, - any necessary headers installed, etc. Binary packages are hard, this is - relatively easy. - - Using pip with virtualenv - ------------------------- - - pip is most nutritious when used with `virtualenv - <http://pypi.python.org/pypi/virtualenv>`__. One of the reasons pip - doesn't install "multi-version" eggs is that virtualenv removes much of the need - for it. Because pip is installed by virtualenv, just use - ``path/to/my/environment/bin/pip`` to install things into that - specific environment. - - To tell pip to only run if there is a virtualenv currently activated, - and to bail if not, use:: - - export PIP_REQUIRE_VIRTUALENV=true - - To tell pip to automatically use the currently active virtualenv:: - - export PIP_RESPECT_VIRTUALENV=true - - Providing an environment with ``-E`` will be ignored. - - Using pip with virtualenvwrapper - --------------------------------- - - If you are using `virtualenvwrapper - <http://www.doughellmann.com/projects/virtualenvwrapper/>`_, you might - want pip to automatically create its virtualenvs in your - ``$WORKON_HOME``. - - You can tell pip to do so by defining ``PIP_VIRTUALENV_BASE`` in your - environment and setting it to the same value as that of - ``$WORKON_HOME``. - - Do so by adding the line:: - - export PIP_VIRTUALENV_BASE=$WORKON_HOME - - in your .bashrc under the line starting with ``export WORKON_HOME``. - - Using pip with buildout - ----------------------- - - If you are using `zc.buildout - <http://pypi.python.org/pypi/zc.buildout>`_ you should look at - `gp.recipe.pip <http://pypi.python.org/pypi/gp.recipe.pip>`_ as an - option to use pip and virtualenv in your buildouts. - - Command line completion - ----------------------- - - pip comes with support for command line completion in bash and zsh and - allows you tab complete commands and options. To enable it you simply - need copy the required shell script to the your shell startup file - (e.g. ``.profile`` or ``.zprofile``) by running the special ``completion`` - command, e.g. for bash:: - - $ pip completion --bash >> ~/.profile - - And for zsh:: - - $ pip completion --zsh >> ~/.zprofile - - Alternatively, you can use the result of the ``completion`` command - directly with the eval function of you shell, e.g. by adding:: - - eval "`pip completion --bash`" - - to your startup file. - - Searching for packages - ---------------------- - - pip can search the `Python Package Index <http://pypi.python.org/pypi>`_ (PyPI) - for packages using the ``pip search`` command. To search, run:: - - $ pip search "query" - - The query will be used to search the names and summaries of all packages - indexed. - - pip searches http://pypi.python.org/pypi by default but alternative indexes - can be searched by using the ``--index`` flag. - - Mirror support - -------------- - - The `PyPI mirroring infrastructure <http://pypi.python.org/mirrors>`_ as - described in `PEP 381 <http://www.python.org/dev/peps/pep-0381/>`_ can be - used by passing the ``--use-mirrors`` option to the install command. - Alternatively, you can use the other ways to configure pip, e.g.:: - - $ export PIP_USE_MIRRORS=true - - If enabled, pip will automatically query the DNS entry of the mirror index URL - to find the list of mirrors to use. In case you want to override this list, - please use the ``--mirrors`` option of the install command, or add to your pip - configuration file:: - - [install] - use-mirrors = true - mirrors = - http://d.pypi.python.org - http://b.pypi.python.org - -Keywords: easy_install distutils setuptools egg virtualenv -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Topic :: Software Development :: Build Tools -Classifier: Programming Language :: Python :: 2.4 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt deleted file mode 100644 index 3a068547e..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt +++ /dev/null @@ -1,57 +0,0 @@ -MANIFEST.in -setup.cfg -setup.py -docs/branches.txt -docs/ci-server-step-by-step.txt -docs/configuration.txt -docs/how-to-contribute.txt -docs/index.txt -docs/license.txt -docs/news.txt -docs/requirement-format.txt -docs/running-tests.txt -docs/_build/branches.html -docs/_build/ci-server-step-by-step.html -docs/_build/configuration.html -docs/_build/how-to-contribute.html -docs/_build/index.html -docs/_build/license.html -docs/_build/news.html -docs/_build/requirement-format.html -docs/_build/running-tests.html -docs/_build/search.html -pip/__init__.py -pip/_pkgutil.py -pip/backwardcompat.py -pip/basecommand.py -pip/baseparser.py -pip/download.py -pip/exceptions.py -pip/index.py -pip/locations.py -pip/log.py -pip/req.py -pip/runner.py -pip/util.py -pip/venv.py -pip.egg-info/PKG-INFO -pip.egg-info/SOURCES.txt -pip.egg-info/dependency_links.txt -pip.egg-info/entry_points.txt -pip.egg-info/not-zip-safe -pip.egg-info/top_level.txt -pip/commands/__init__.py -pip/commands/bundle.py -pip/commands/completion.py -pip/commands/freeze.py -pip/commands/help.py -pip/commands/install.py -pip/commands/search.py -pip/commands/uninstall.py -pip/commands/unzip.py -pip/commands/zip.py -pip/vcs/__init__.py -pip/vcs/bazaar.py -pip/vcs/git.py -pip/vcs/mercurial.py -pip/vcs/subversion.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt deleted file mode 100644 index 8b1378917..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt deleted file mode 100644 index 2b0afba73..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt +++ /dev/null @@ -1,4 +0,0 @@ -[console_scripts] -pip = pip:main -pip-2.6 = pip:main - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe deleted file mode 100644 index 8b1378917..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt deleted file mode 100644 index a1b589e38..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py deleted file mode 100644 index c5de5c9a8..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python -import os -import optparse - -import subprocess -import sys -import re -import difflib - -from pip.basecommand import command_dict, load_command, load_all_commands, command_names -from pip.baseparser import parser -from pip.exceptions import InstallationError -from pip.log import logger -from pip.util import get_installed_distributions -from pip.backwardcompat import walk_packages - - -def autocomplete(): - """Command and option completion for the main option parser (and options) - and its subcommands (and options). - - Enable by sourcing one of the completion shell scripts (bash or zsh). - """ - # Don't complete if user hasn't sourced bash_completion file. - if 'PIP_AUTO_COMPLETE' not in os.environ: - return - cwords = os.environ['COMP_WORDS'].split()[1:] - cword = int(os.environ['COMP_CWORD']) - try: - current = cwords[cword-1] - except IndexError: - current = '' - load_all_commands() - subcommands = [cmd for cmd, cls in command_dict.items() if not cls.hidden] - options = [] - # subcommand - try: - subcommand_name = [w for w in cwords if w in subcommands][0] - except IndexError: - subcommand_name = None - # subcommand options - if subcommand_name: - # special case: 'help' subcommand has no options - if subcommand_name == 'help': - sys.exit(1) - # special case: list locally installed dists for uninstall command - if subcommand_name == 'uninstall' and not current.startswith('-'): - installed = [] - lc = current.lower() - for dist in get_installed_distributions(local_only=True): - if dist.key.startswith(lc) and dist.key not in cwords[1:]: - installed.append(dist.key) - # if there are no dists installed, fall back to option completion - if installed: - for dist in installed: - print dist - sys.exit(1) - subcommand = command_dict.get(subcommand_name) - options += [(opt.get_opt_string(), opt.nargs) - for opt in subcommand.parser.option_list - if opt.help != optparse.SUPPRESS_HELP] - # filter out previously specified options from available options - prev_opts = [x.split('=')[0] for x in cwords[1:cword-1]] - options = filter(lambda (x, v): x not in prev_opts, options) - # filter options by current input - options = [(k, v) for k, v in options if k.startswith(current)] - for option in options: - opt_label = option[0] - # append '=' to options which require args - if option[1]: - opt_label += '=' - print opt_label - else: - # show options of main parser only when necessary - if current.startswith('-') or current.startswith('--'): - subcommands += [opt.get_opt_string() - for opt in parser.option_list - if opt.help != optparse.SUPPRESS_HELP] - print ' '.join(filter(lambda x: x.startswith(current), subcommands)) - sys.exit(1) - - -def version_control(): - # Import all the version control support modules: - from pip import vcs - for importer, modname, ispkg in \ - walk_packages(path=vcs.__path__, prefix=vcs.__name__+'.'): - __import__(modname) - - -def main(initial_args=None): - if initial_args is None: - initial_args = sys.argv[1:] - autocomplete() - version_control() - options, args = parser.parse_args(initial_args) - if options.help and not args: - args = ['help'] - if not args: - parser.error('You must give a command (use "pip help" to see a list of commands)') - command = args[0].lower() - load_command(command) - if command not in command_dict: - close_commands = difflib.get_close_matches(command, command_names()) - if close_commands: - guess = close_commands[0] - if args[1:]: - guess = "%s %s" % (guess, " ".join(args[1:])) - else: - guess = 'install %s' % command - error_dict = {'arg': command, 'guess': guess, - 'script': os.path.basename(sys.argv[0])} - parser.error('No command by the name %(script)s %(arg)s\n ' - '(maybe you meant "%(script)s %(guess)s")' % error_dict) - command = command_dict[command] - return command.main(initial_args, args[1:], options) - - -############################################################ -## Writing freeze files - - -class FrozenRequirement(object): - - def __init__(self, name, req, editable, comments=()): - self.name = name - self.req = req - self.editable = editable - self.comments = comments - - _rev_re = re.compile(r'-r(\d+)$') - _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') - - @classmethod - def from_dist(cls, dist, dependency_links, find_tags=False): - location = os.path.normcase(os.path.abspath(dist.location)) - comments = [] - from pip.vcs import vcs, get_src_requirement - if vcs.get_backend_name(location): - editable = True - req = get_src_requirement(dist, location, find_tags) - if req is None: - logger.warn('Could not determine repository location of %s' % location) - comments.append('## !! Could not determine repository location') - req = dist.as_requirement() - editable = False - else: - editable = False - req = dist.as_requirement() - specs = req.specs - assert len(specs) == 1 and specs[0][0] == '==' - version = specs[0][1] - ver_match = cls._rev_re.search(version) - date_match = cls._date_re.search(version) - if ver_match or date_match: - svn_backend = vcs.get_backend('svn') - if svn_backend: - svn_location = svn_backend( - ).get_location(dist, dependency_links) - if not svn_location: - logger.warn( - 'Warning: cannot find svn location for %s' % req) - comments.append('## FIXME: could not find svn URL in dependency_links for this package:') - else: - comments.append('# Installing as editable to satisfy requirement %s:' % req) - if ver_match: - rev = ver_match.group(1) - else: - rev = '{%s}' % date_match.group(1) - editable = True - req = '%s@%s#egg=%s' % (svn_location, rev, cls.egg_name(dist)) - return cls(dist.project_name, req, editable, comments) - - @staticmethod - def egg_name(dist): - name = dist.egg_name() - match = re.search(r'-py\d\.\d$', name) - if match: - name = name[:match.start()] - return name - - def __str__(self): - req = self.req - if self.editable: - req = '-e %s' % req - return '\n'.join(list(self.comments)+[str(req)])+'\n' - -############################################################ -## Requirement files - - -def call_subprocess(cmd, show_stdout=True, - filter_stdout=None, cwd=None, - raise_on_returncode=True, - command_level=logger.DEBUG, command_desc=None, - extra_environ=None): - if command_desc is None: - cmd_parts = [] - for part in cmd: - if ' ' in part or '\n' in part or '"' in part or "'" in part: - part = '"%s"' % part.replace('"', '\\"') - cmd_parts.append(part) - command_desc = ' '.join(cmd_parts) - if show_stdout: - stdout = None - else: - stdout = subprocess.PIPE - logger.log(command_level, "Running command %s" % command_desc) - env = os.environ.copy() - if extra_environ: - env.update(extra_environ) - try: - proc = subprocess.Popen( - cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, - cwd=cwd, env=env) - except Exception, e: - logger.fatal( - "Error %s while executing command %s" % (e, command_desc)) - raise - all_output = [] - if stdout is not None: - stdout = proc.stdout - while 1: - line = stdout.readline() - if not line: - break - line = line.rstrip() - all_output.append(line + '\n') - if filter_stdout: - level = filter_stdout(line) - if isinstance(level, tuple): - level, line = level - logger.log(level, line) - if not logger.stdout_level_matches(level): - logger.show_progress() - else: - logger.info(line) - else: - returned_stdout, returned_stderr = proc.communicate() - all_output = [returned_stdout or ''] - proc.wait() - if proc.returncode: - if raise_on_returncode: - if all_output: - logger.notify('Complete output from command %s:' % command_desc) - logger.notify('\n'.join(all_output) + '\n----------------------------------------') - raise InstallationError( - "Command %s failed with error code %s" - % (command_desc, proc.returncode)) - else: - logger.warn( - "Command %s had error code %s" - % (command_desc, proc.returncode)) - if stdout is not None: - return ''.join(all_output) - - -if __name__ == '__main__': - exit = main() - if exit: - sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py deleted file mode 100644 index f8fb8aa61..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py +++ /dev/null @@ -1,589 +0,0 @@ -"""Utilities to support packages.""" - -# NOTE: This module must remain compatible with Python 2.3, as it is shared -# by setuptools for distribution with Python 2.3 and up. - -import os -import sys -import imp -import os.path -from types import ModuleType - -__all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', - 'walk_packages', 'iter_modules', - 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', -] - - -def read_code(stream): - # This helper is needed in order for the PEP 302 emulation to - # correctly handle compiled files - import marshal - - magic = stream.read(4) - if magic != imp.get_magic(): - return None - - stream.read(4) # Skip timestamp - return marshal.load(stream) - - -def simplegeneric(func): - """Make a trivial single-dispatch generic function""" - registry = {} - - def wrapper(*args, **kw): - ob = args[0] - try: - cls = ob.__class__ - except AttributeError: - cls = type(ob) - try: - mro = cls.__mro__ - except AttributeError: - try: - - class cls(cls, object): - pass - - mro = cls.__mro__[1:] - except TypeError: - mro = object, # must be an ExtensionClass or some such :( - for t in mro: - if t in registry: - return registry[t](*args, **kw) - else: - return func(*args, **kw) - try: - wrapper.__name__ = func.__name__ - except (TypeError, AttributeError): - pass # Python 2.3 doesn't allow functions to be renamed - - def register(typ, func=None): - if func is None: - return lambda f: register(typ, f) - registry[typ] = func - return func - - wrapper.__dict__ = func.__dict__ - wrapper.__doc__ = func.__doc__ - wrapper.register = register - return wrapper - - -def walk_packages(path=None, prefix='', onerror=None): - """Yields (module_loader, name, ispkg) for all modules recursively - on path, or, if path is None, all accessible modules. - - 'path' should be either None or a list of paths to look for - modules in. - - 'prefix' is a string to output on the front of every module name - on output. - - Note that this function must import all *packages* (NOT all - modules!) on the given path, in order to access the __path__ - attribute to find submodules. - - 'onerror' is a function which gets called with one argument (the - name of the package which was being imported) if any exception - occurs while trying to import a package. If no onerror function is - supplied, ImportErrors are caught and ignored, while all other - exceptions are propagated, terminating the search. - - Examples: - - # list all modules python can access - walk_packages() - - # list all submodules of ctypes - walk_packages(ctypes.__path__, ctypes.__name__+'.') - """ - - def seen(p, m={}): - if p in m: - return True - m[p] = True - - for importer, name, ispkg in iter_modules(path, prefix): - yield importer, name, ispkg - - if ispkg: - try: - __import__(name) - except ImportError: - if onerror is not None: - onerror(name) - except Exception: - if onerror is not None: - onerror(name) - else: - raise - else: - path = getattr(sys.modules[name], '__path__', None) or [] - - # don't traverse path items we've seen before - path = [p for p in path if not seen(p)] - - for item in walk_packages(path, name+'.', onerror): - yield item - - -def iter_modules(path=None, prefix=''): - """Yields (module_loader, name, ispkg) for all submodules on path, - or, if path is None, all top-level modules on sys.path. - - 'path' should be either None or a list of paths to look for - modules in. - - 'prefix' is a string to output on the front of every module name - on output. - """ - - if path is None: - importers = iter_importers() - else: - importers = map(get_importer, path) - - yielded = {} - for i in importers: - for name, ispkg in iter_importer_modules(i, prefix): - if name not in yielded: - yielded[name] = 1 - yield i, name, ispkg - - -#@simplegeneric -def iter_importer_modules(importer, prefix=''): - if not hasattr(importer, 'iter_modules'): - return [] - return importer.iter_modules(prefix) - -iter_importer_modules = simplegeneric(iter_importer_modules) - - -class ImpImporter: - """PEP 302 Importer that wraps Python's "classic" import algorithm - - ImpImporter(dirname) produces a PEP 302 importer that searches that - directory. ImpImporter(None) produces a PEP 302 importer that searches - the current sys.path, plus any modules that are frozen or built-in. - - Note that ImpImporter does not currently support being used by placement - on sys.meta_path. - """ - - def __init__(self, path=None): - self.path = path - - def find_module(self, fullname, path=None): - # Note: we ignore 'path' argument since it is only used via meta_path - subname = fullname.split(".")[-1] - if subname != fullname and self.path is None: - return None - if self.path is None: - path = None - else: - path = [os.path.realpath(self.path)] - try: - file, filename, etc = imp.find_module(subname, path) - except ImportError: - return None - return ImpLoader(fullname, file, filename, etc) - - def iter_modules(self, prefix=''): - if self.path is None or not os.path.isdir(self.path): - return - - yielded = {} - import inspect - - filenames = os.listdir(self.path) - filenames.sort() # handle packages before same-named modules - - for fn in filenames: - modname = inspect.getmodulename(fn) - if modname=='__init__' or modname in yielded: - continue - - path = os.path.join(self.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - for fn in os.listdir(path): - subname = inspect.getmodulename(fn) - if subname=='__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - - -class ImpLoader: - """PEP 302 Loader that wraps Python's "classic" import algorithm - """ - code = source = None - - def __init__(self, fullname, file, filename, etc): - self.file = file - self.filename = filename - self.fullname = fullname - self.etc = etc - - def load_module(self, fullname): - self._reopen() - try: - mod = imp.load_module(fullname, self.file, self.filename, self.etc) - finally: - if self.file: - self.file.close() - # Note: we don't set __loader__ because we want the module to look - # normal; i.e. this is just a wrapper for standard import machinery - return mod - - def get_data(self, pathname): - return open(pathname, "rb").read() - - def _reopen(self): - if self.file and self.file.closed: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self.file = open(self.filename, 'rU') - elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): - self.file = open(self.filename, 'rb') - - def _fix_name(self, fullname): - if fullname is None: - fullname = self.fullname - elif fullname != self.fullname: - raise ImportError("Loader for module %s cannot handle " - "module %s" % (self.fullname, fullname)) - return fullname - - def is_package(self, fullname): - fullname = self._fix_name(fullname) - return self.etc[2]==imp.PKG_DIRECTORY - - def get_code(self, fullname=None): - fullname = self._fix_name(fullname) - if self.code is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - source = self.get_source(fullname) - self.code = compile(source, self.filename, 'exec') - elif mod_type==imp.PY_COMPILED: - self._reopen() - try: - self.code = read_code(self.file) - finally: - self.file.close() - elif mod_type==imp.PKG_DIRECTORY: - self.code = self._get_delegate().get_code() - return self.code - - def get_source(self, fullname=None): - fullname = self._fix_name(fullname) - if self.source is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self._reopen() - try: - self.source = self.file.read() - finally: - self.file.close() - elif mod_type==imp.PY_COMPILED: - if os.path.exists(self.filename[:-1]): - f = open(self.filename[:-1], 'rU') - self.source = f.read() - f.close() - elif mod_type==imp.PKG_DIRECTORY: - self.source = self._get_delegate().get_source() - return self.source - - def _get_delegate(self): - return ImpImporter(self.filename).find_module('__init__') - - def get_filename(self, fullname=None): - fullname = self._fix_name(fullname) - mod_type = self.etc[2] - if self.etc[2]==imp.PKG_DIRECTORY: - return self._get_delegate().get_filename() - elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): - return self.filename - return None - - -try: - import zipimport - from zipimport import zipimporter - - def iter_zipimport_modules(importer, prefix=''): - dirlist = zipimport._zip_directory_cache[importer.archive].keys() - dirlist.sort() - _prefix = importer.prefix - plen = len(_prefix) - yielded = {} - import inspect - for fn in dirlist: - if not fn.startswith(_prefix): - continue - - fn = fn[plen:].split(os.sep) - - if len(fn)==2 and fn[1].startswith('__init__.py'): - if fn[0] not in yielded: - yielded[fn[0]] = 1 - yield fn[0], True - - if len(fn)!=1: - continue - - modname = inspect.getmodulename(fn[0]) - if modname=='__init__': - continue - - if modname and '.' not in modname and modname not in yielded: - yielded[modname] = 1 - yield prefix + modname, False - - iter_importer_modules.register(zipimporter, iter_zipimport_modules) - -except ImportError: - pass - - -def get_importer(path_item): - """Retrieve a PEP 302 importer for the given path item - - The returned importer is cached in sys.path_importer_cache - if it was newly created by a path hook. - - If there is no importer, a wrapper around the basic import - machinery is returned. This wrapper is never inserted into - the importer cache (None is inserted instead). - - The cache (or part of it) can be cleared manually if a - rescan of sys.path_hooks is necessary. - """ - try: - importer = sys.path_importer_cache[path_item] - except KeyError: - for path_hook in sys.path_hooks: - try: - importer = path_hook(path_item) - break - except ImportError: - pass - else: - importer = None - sys.path_importer_cache.setdefault(path_item, importer) - - if importer is None: - try: - importer = ImpImporter(path_item) - except ImportError: - importer = None - return importer - - -def iter_importers(fullname=""): - """Yield PEP 302 importers for the given module name - - If fullname contains a '.', the importers will be for the package - containing fullname, otherwise they will be importers for sys.meta_path, - sys.path, and Python's "classic" import machinery, in that order. If - the named module is in a package, that package is imported as a side - effect of invoking this function. - - Non PEP 302 mechanisms (e.g. the Windows registry) used by the - standard import machinery to find files in alternative locations - are partially supported, but are searched AFTER sys.path. Normally, - these locations are searched BEFORE sys.path, preventing sys.path - entries from shadowing them. - - For this to cause a visible difference in behaviour, there must - be a module or package name that is accessible via both sys.path - and one of the non PEP 302 file system mechanisms. In this case, - the emulation will find the former version, while the builtin - import mechanism will find the latter. - - Items of the following types can be affected by this discrepancy: - imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY - """ - if fullname.startswith('.'): - raise ImportError("Relative module names not supported") - if '.' in fullname: - # Get the containing package's __path__ - pkg = '.'.join(fullname.split('.')[:-1]) - if pkg not in sys.modules: - __import__(pkg) - path = getattr(sys.modules[pkg], '__path__', None) or [] - else: - for importer in sys.meta_path: - yield importer - path = sys.path - for item in path: - yield get_importer(item) - if '.' not in fullname: - yield ImpImporter() - - -def get_loader(module_or_name): - """Get a PEP 302 "loader" object for module_or_name - - If the module or package is accessible via the normal import - mechanism, a wrapper around the relevant part of that machinery - is returned. Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - - This function uses iter_importers(), and is thus subject to the same - limitations regarding platform-specific special import locations such - as the Windows registry. - """ - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a PEP 302 "loader" object for fullname - - If fullname contains dots, path must be the containing package's __path__. - Returns None if the module cannot be found or imported. This function uses - iter_importers(), and is thus subject to the same limitations regarding - platform-specific special import locations such as the Windows registry. - """ - for importer in iter_importers(fullname): - loader = importer.find_module(fullname) - if loader is not None: - return loader - - return None - - -def extend_path(path, name): - """Extend a package's path. - - Intended use is to place the following code in a package's __init__.py: - - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) - - This will add to the package's __path__ all subdirectories of - directories on sys.path named after the package. This is useful - if one wants to distribute different parts of a single logical - package as multiple directories. - - It also looks for *.pkg files beginning where * matches the name - argument. This feature is similar to *.pth files (see site.py), - except that it doesn't special-case lines starting with 'import'. - A *.pkg file is trusted at face value: apart from checking for - duplicates, all entries found in a *.pkg file are added to the - path, regardless of whether they are exist the filesystem. (This - is a feature.) - - If the input path is not a list (as is the case for frozen - packages) it is returned unchanged. The input path is not - modified; an extended copy is returned. Items are only appended - to the copy at the end. - - It is assumed that sys.path is a sequence. Items of sys.path that - are not (unicode or 8-bit) strings referring to existing - directories are ignored. Unicode items of sys.path that cause - errors when used as filenames may cause this function to raise an - exception (in line with os.path.isdir() behavior). - """ - - if not isinstance(path, list): - # This could happen e.g. when this is called from inside a - # frozen package. Return the path unchanged in that case. - return path - - pname = os.path.join(*name.split('.')) # Reconstitute as relative path - # Just in case os.extsep != '.' - sname = os.extsep.join(name.split('.')) - sname_pkg = sname + os.extsep + "pkg" - init_py = "__init__" + os.extsep + "py" - - path = path[:] # Start with a copy of the existing path - - for dir in sys.path: - if not isinstance(dir, basestring) or not os.path.isdir(dir): - continue - subdir = os.path.join(dir, pname) - # XXX This may still add duplicate entries to path on - # case-insensitive filesystems - initfile = os.path.join(subdir, init_py) - if subdir not in path and os.path.isfile(initfile): - path.append(subdir) - # XXX Is this the right thing for subpackages like zope.app? - # It looks for a file named "zope.app.pkg" - pkgfile = os.path.join(dir, sname_pkg) - if os.path.isfile(pkgfile): - try: - f = open(pkgfile) - except IOError, msg: - sys.stderr.write("Can't open %s: %s\n" % - (pkgfile, msg)) - else: - for line in f: - line = line.rstrip('\n') - if not line or line.startswith('#'): - continue - path.append(line) # Don't check for existence! - f.close() - - return path - - -def get_data(package, resource): - """Get a resource from a package. - - This is a wrapper round the PEP 302 loader get_data API. The package - argument should be the name of a package, in standard module format - (foo.bar). The resource argument should be in the form of a relative - filename, using '/' as the path separator. The parent directory name '..' - is not allowed, and nor is a rooted name (starting with a '/'). - - The function returns a binary string, which is the contents of the - specified resource. - - For packages located in the filesystem, which have already been imported, - this is the rough equivalent of - - d = os.path.dirname(sys.modules[package].__file__) - data = open(os.path.join(d, resource), 'rb').read() - - If the package cannot be located or loaded, or it uses a PEP 302 loader - which does not support get_data(), then None is returned. - """ - - loader = get_loader(package) - if loader is None or not hasattr(loader, 'get_data'): - return None - mod = sys.modules.get(package) or loader.load_module(package) - if mod is None or not hasattr(mod, '__file__'): - return None - - # Modify the resource name to be compatible with the loader.get_data - # signature - an os.path format "filename" starting with the dirname of - # the package's __file__ - parts = resource.split('/') - parts.insert(0, os.path.dirname(mod.__file__)) - resource_name = os.path.join(*parts) - return loader.get_data(resource_name) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py deleted file mode 100644 index e7c11f1d3..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Stuff that isn't in some old versions of Python""" - -import sys -import os -import shutil - -__all__ = ['any', 'WindowsError', 'md5', 'copytree'] - -try: - WindowsError = WindowsError -except NameError: - WindowsError = None -try: - from hashlib import md5 -except ImportError: - import md5 as md5_module - md5 = md5_module.new - -try: - from pkgutil import walk_packages -except ImportError: - # let's fall back as long as we can - from _pkgutil import walk_packages - -try: - any = any -except NameError: - - def any(seq): - for item in seq: - if item: - return True - return False - - -def copytree(src, dst): - if sys.version_info < (2, 5): - before_last_dir = os.path.dirname(dst) - if not os.path.exists(before_last_dir): - os.makedirs(before_last_dir) - shutil.copytree(src, dst) - shutil.copymode(src, dst) - else: - shutil.copytree(src, dst) - - -def product(*args, **kwds): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = map(tuple, args) * kwds.get('repeat', 1) - result = [[]] - for pool in pools: - result = [x+[y] for x in result for y in pool] - for prod in result: - yield tuple(prod) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py deleted file mode 100644 index f450e8393..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Base Command class, and related routines""" - -from cStringIO import StringIO -import getpass -import os -import socket -import sys -import traceback -import time -import urllib -import urllib2 - -from pip import commands -from pip.log import logger -from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpFormatter -from pip.download import urlopen -from pip.exceptions import BadCommand, InstallationError, UninstallationError -from pip.venv import restart_in_venv -from pip.backwardcompat import walk_packages - -__all__ = ['command_dict', 'Command', 'load_all_commands', - 'load_command', 'command_names'] - -command_dict = {} - -# for backwards compatibiliy -get_proxy = urlopen.get_proxy - - -class Command(object): - name = None - usage = None - hidden = False - - def __init__(self): - assert self.name - self.parser = ConfigOptionParser( - usage=self.usage, - prog='%s %s' % (sys.argv[0], self.name), - version=parser.version, - formatter=UpdatingDefaultsHelpFormatter(), - name=self.name) - for option in parser.option_list: - if not option.dest or option.dest == 'help': - # -h, --version, etc - continue - self.parser.add_option(option) - command_dict[self.name] = self - - def merge_options(self, initial_options, options): - # Make sure we have all global options carried over - for attr in ['log', 'venv', 'proxy', 'venv_base', 'require_venv', - 'respect_venv', 'log_explicit_levels', 'log_file', - 'timeout', 'default_vcs', 'skip_requirements_regex', - 'no_input']: - setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr)) - options.quiet += initial_options.quiet - options.verbose += initial_options.verbose - - def setup_logging(self): - pass - - def main(self, complete_args, args, initial_options): - options, args = self.parser.parse_args(args) - self.merge_options(initial_options, options) - - level = 1 # Notify - level += options.verbose - level -= options.quiet - level = logger.level_for_integer(4-level) - complete_log = [] - logger.consumers.extend( - [(level, sys.stdout), - (logger.DEBUG, complete_log.append)]) - if options.log_explicit_levels: - logger.explicit_levels = True - - self.setup_logging() - - if options.require_venv and not options.venv: - # If a venv is required check if it can really be found - if not os.environ.get('VIRTUAL_ENV'): - logger.fatal('Could not find an activated virtualenv (required).') - sys.exit(3) - # Automatically install in currently activated venv if required - options.respect_venv = True - - if args and args[-1] == '___VENV_RESTART___': - ## FIXME: We don't do anything this this value yet: - args = args[:-2] - options.venv = None - else: - # If given the option to respect the activated environment - # check if no venv is given as a command line parameter - if options.respect_venv and os.environ.get('VIRTUAL_ENV'): - if options.venv and os.path.exists(options.venv): - # Make sure command line venv and environmental are the same - if (os.path.realpath(os.path.expanduser(options.venv)) != - os.path.realpath(os.environ.get('VIRTUAL_ENV'))): - logger.fatal("Given virtualenv (%s) doesn't match " - "currently activated virtualenv (%s)." - % (options.venv, os.environ.get('VIRTUAL_ENV'))) - sys.exit(3) - else: - options.venv = os.environ.get('VIRTUAL_ENV') - logger.info('Using already activated environment %s' % options.venv) - if options.venv: - logger.info('Running in environment %s' % options.venv) - site_packages=False - if options.site_packages: - site_packages=True - restart_in_venv(options.venv, options.venv_base, site_packages, - complete_args) - # restart_in_venv should actually never return, but for clarity... - return - - ## FIXME: not sure if this sure come before or after venv restart - if options.log: - log_fp = open_logfile(options.log, 'a') - logger.consumers.append((logger.DEBUG, log_fp)) - else: - log_fp = None - - socket.setdefaulttimeout(options.timeout or None) - - urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) - - exit = 0 - try: - self.run(options, args) - except (InstallationError, UninstallationError), e: - logger.fatal(str(e)) - logger.info('Exception information:\n%s' % format_exc()) - exit = 1 - except BadCommand, e: - logger.fatal(str(e)) - logger.info('Exception information:\n%s' % format_exc()) - exit = 1 - except: - logger.fatal('Exception:\n%s' % format_exc()) - exit = 2 - - if log_fp is not None: - log_fp.close() - if exit: - log_fn = options.log_file - text = '\n'.join(complete_log) - logger.fatal('Storing complete log in %s' % log_fn) - log_fp = open_logfile(log_fn, 'w') - log_fp.write(text) - log_fp.close() - return exit - - - - -def format_exc(exc_info=None): - if exc_info is None: - exc_info = sys.exc_info() - out = StringIO() - traceback.print_exception(*exc_info, **dict(file=out)) - return out.getvalue() - - -def open_logfile(filename, mode='a'): - """Open the named log file in append mode. - - If the file already exists, a separator will also be printed to - the file to separate past activity from current activity. - """ - filename = os.path.expanduser(filename) - filename = os.path.abspath(filename) - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - exists = os.path.exists(filename) - - log_fp = open(filename, mode) - if exists: - print >> log_fp, '-'*60 - print >> log_fp, '%s run on %s' % (sys.argv[0], time.strftime('%c')) - return log_fp - - -def load_command(name): - full_name = 'pip.commands.%s' % name - if full_name in sys.modules: - return - try: - __import__(full_name) - except ImportError: - pass - - -def load_all_commands(): - for name in command_names(): - load_command(name) - - -def command_names(): - names = set((pkg[1] for pkg in walk_packages(path=commands.__path__))) - return list(names) - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py deleted file mode 100644 index a8bd6ce4f..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py +++ /dev/null @@ -1,231 +0,0 @@ -"""Base option parser setup""" - -import sys -import optparse -import pkg_resources -import ConfigParser -import os -from distutils.util import strtobool -from pip.locations import default_config_file, default_log_file - - -class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter): - """Custom help formatter for use in ConfigOptionParser that updates - the defaults before expanding them, allowing them to show up correctly - in the help listing""" - - def expand_default(self, option): - if self.parser is not None: - self.parser.update_defaults(self.parser.defaults) - return optparse.IndentedHelpFormatter.expand_default(self, option) - - -class ConfigOptionParser(optparse.OptionParser): - """Custom option parser which updates its defaults by by checking the - configuration files and environmental variables""" - - def __init__(self, *args, **kwargs): - self.config = ConfigParser.RawConfigParser() - self.name = kwargs.pop('name') - self.files = self.get_config_files() - self.config.read(self.files) - assert self.name - optparse.OptionParser.__init__(self, *args, **kwargs) - - def get_config_files(self): - config_file = os.environ.get('PIP_CONFIG_FILE', False) - if config_file and os.path.exists(config_file): - return [config_file] - return [default_config_file] - - def update_defaults(self, defaults): - """Updates the given defaults with values from the config files and - the environ. Does a little special handling for certain types of - options (lists).""" - # Then go and look for the other sources of configuration: - config = {} - # 1. config files - for section in ('global', self.name): - config.update(dict(self.get_config_section(section))) - # 2. environmental variables - config.update(dict(self.get_environ_vars())) - # Then set the options with those values - for key, val in config.iteritems(): - key = key.replace('_', '-') - if not key.startswith('--'): - key = '--%s' % key # only prefer long opts - option = self.get_option(key) - if option is not None: - # ignore empty values - if not val: - continue - # handle multiline configs - if option.action == 'append': - val = val.split() - else: - option.nargs = 1 - if option.action in ('store_true', 'store_false', 'count'): - val = strtobool(val) - try: - val = option.convert_value(key, val) - except optparse.OptionValueError, e: - print ("An error occured during configuration: %s" % e) - sys.exit(3) - defaults[option.dest] = val - return defaults - - def get_config_section(self, name): - """Get a section of a configuration""" - if self.config.has_section(name): - return self.config.items(name) - return [] - - def get_environ_vars(self, prefix='PIP_'): - """Returns a generator with all environmental vars with prefix PIP_""" - for key, val in os.environ.iteritems(): - if key.startswith(prefix): - yield (key.replace(prefix, '').lower(), val) - - def get_default_values(self): - """Overridding to make updating the defaults after instantiation of - the option parser possible, update_defaults() does the dirty work.""" - if not self.process_default_values: - # Old, pre-Optik 1.5 behaviour. - return optparse.Values(self.defaults) - - defaults = self.update_defaults(self.defaults.copy()) # ours - for option in self._get_all_options(): - default = defaults.get(option.dest) - if isinstance(default, basestring): - opt_str = option.get_opt_string() - defaults[option.dest] = option.check_value(opt_str, default) - return optparse.Values(defaults) - -try: - pip_dist = pkg_resources.get_distribution('pip') - version = '%s from %s (python %s)' % ( - pip_dist, pip_dist.location, sys.version[:3]) -except pkg_resources.DistributionNotFound: - # when running pip.py without installing - version=None - -parser = ConfigOptionParser( - usage='%prog COMMAND [OPTIONS]', - version=version, - add_help_option=False, - formatter=UpdatingDefaultsHelpFormatter(), - name='global') - -parser.add_option( - '-h', '--help', - dest='help', - action='store_true', - help='Show help') -parser.add_option( - '-E', '--environment', - dest='venv', - metavar='DIR', - help='virtualenv environment to run pip in (either give the ' - 'interpreter or the environment base directory)') -parser.add_option( - '-s', '--enable-site-packages', - dest='site_packages', - action='store_true', - help='Include site-packages in virtualenv if one is to be ' - 'created. Ignored if --environment is not used or ' - 'the virtualenv already exists.') -parser.add_option( - # Defines a default root directory for virtualenvs, relative - # virtualenvs names/paths are considered relative to it. - '--virtualenv-base', - dest='venv_base', - type='str', - default='', - help=optparse.SUPPRESS_HELP) -parser.add_option( - # Run only if inside a virtualenv, bail if not. - '--require-virtualenv', '--require-venv', - dest='require_venv', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) -parser.add_option( - # Use automatically an activated virtualenv instead of installing - # globally. -E will be ignored if used. - '--respect-virtualenv', '--respect-venv', - dest='respect_venv', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) - -parser.add_option( - '-v', '--verbose', - dest='verbose', - action='count', - default=0, - help='Give more output') -parser.add_option( - '-q', '--quiet', - dest='quiet', - action='count', - default=0, - help='Give less output') -parser.add_option( - '--log', - dest='log', - metavar='FILENAME', - help='Log file where a complete (maximum verbosity) record will be kept') -parser.add_option( - # Writes the log levels explicitely to the log' - '--log-explicit-levels', - dest='log_explicit_levels', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) -parser.add_option( - # The default log file - '--local-log', '--log-file', - dest='log_file', - metavar='FILENAME', - default=default_log_file, - help=optparse.SUPPRESS_HELP) -parser.add_option( - # Don't ask for input - '--no-input', - dest='no_input', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) - -parser.add_option( - '--proxy', - dest='proxy', - type='str', - default='', - help="Specify a proxy in the form user:passwd@proxy.server:port. " - "Note that the user:password@ is optional and required only if you " - "are behind an authenticated proxy. If you provide " - "user@proxy.server:port then you will be prompted for a password.") -parser.add_option( - '--timeout', '--default-timeout', - metavar='SECONDS', - dest='timeout', - type='float', - default=15, - help='Set the socket timeout (default %default seconds)') -parser.add_option( - # The default version control system for editables, e.g. 'svn' - '--default-vcs', - dest='default_vcs', - type='str', - default='', - help=optparse.SUPPRESS_HELP) -parser.add_option( - # A regex to be used to skip requirements - '--skip-requirements-regex', - dest='skip_requirements_regex', - type='str', - default='', - help=optparse.SUPPRESS_HELP) - -parser.disable_interspersed_args() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py deleted file mode 100644 index 792d60054..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py deleted file mode 100644 index fb0f75704..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py +++ /dev/null @@ -1,33 +0,0 @@ -from pip.locations import build_prefix, src_prefix -from pip.util import display_path, backup_dir -from pip.log import logger -from pip.exceptions import InstallationError -from pip.commands.install import InstallCommand - - -class BundleCommand(InstallCommand): - name = 'bundle' - usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...' - summary = 'Create pybundles (archives containing multiple packages)' - bundle = True - - def __init__(self): - super(BundleCommand, self).__init__() - - def run(self, options, args): - if not args: - raise InstallationError('You must give a bundle filename') - if not options.build_dir: - options.build_dir = backup_dir(build_prefix, '-bundle') - if not options.src_dir: - options.src_dir = backup_dir(src_prefix, '-bundle') - # We have to get everything when creating a bundle: - options.ignore_installed = True - logger.notify('Putting temporary build files in %s and source/develop files in %s' - % (display_path(options.build_dir), display_path(options.src_dir))) - self.bundle_filename = args.pop(0) - requirement_set = super(BundleCommand, self).run(options, args) - return requirement_set - - -BundleCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py deleted file mode 100644 index d003b9ae3..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py +++ /dev/null @@ -1,60 +0,0 @@ -import sys -from pip.basecommand import Command - -BASE_COMPLETION = """ -# pip %(shell)s completion start%(script)s# pip %(shell)s completion end -""" - -COMPLETION_SCRIPTS = { - 'bash': """ -_pip_completion() -{ - COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ - COMP_CWORD=$COMP_CWORD \\ - PIP_AUTO_COMPLETE=1 $1 ) ) -} -complete -o default -F _pip_completion pip -""", 'zsh': """ -function _pip_completion { - local words cword - read -Ac words - read -cn cword - reply=( $( COMP_WORDS="$words[*]" \\ - COMP_CWORD=$(( cword-1 )) \\ - PIP_AUTO_COMPLETE=1 $words[1] ) ) -} -compctl -K _pip_completion pip -"""} - - -class CompletionCommand(Command): - name = 'completion' - summary = 'A helper command to be used for command completion' - hidden = True - - def __init__(self): - super(CompletionCommand, self).__init__() - self.parser.add_option( - '--bash', '-b', - action='store_const', - const='bash', - dest='shell', - help='Emit completion code for bash') - self.parser.add_option( - '--zsh', '-z', - action='store_const', - const='zsh', - dest='shell', - help='Emit completion code for zsh') - - def run(self, options, args): - """Prints the completion code of the given shell""" - shells = COMPLETION_SCRIPTS.keys() - shell_options = ['--'+shell for shell in sorted(shells)] - if options.shell in shells: - script = COMPLETION_SCRIPTS.get(options.shell, '') - print BASE_COMPLETION % {'script': script, 'shell': options.shell} - else: - sys.stderr.write('ERROR: You must pass %s\n' % ' or '.join(shell_options)) - -CompletionCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py deleted file mode 100644 index 01b5df934..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py +++ /dev/null @@ -1,109 +0,0 @@ -import re -import sys -import pkg_resources -import pip -from pip.req import InstallRequirement -from pip.log import logger -from pip.basecommand import Command -from pip.util import get_installed_distributions - - -class FreezeCommand(Command): - name = 'freeze' - usage = '%prog [OPTIONS]' - summary = 'Output all currently installed packages (exact versions) to stdout' - - def __init__(self): - super(FreezeCommand, self).__init__() - self.parser.add_option( - '-r', '--requirement', - dest='requirement', - action='store', - default=None, - metavar='FILENAME', - help='Use the given requirements file as a hint about how to generate the new frozen requirements') - self.parser.add_option( - '-f', '--find-links', - dest='find_links', - action='append', - default=[], - metavar='URL', - help='URL for finding packages, which will be added to the frozen requirements file') - self.parser.add_option( - '-l', '--local', - dest='local', - action='store_true', - default=False, - help='If in a virtualenv, do not report globally-installed packages') - - def setup_logging(self): - logger.move_stdout_to_stderr() - - def run(self, options, args): - requirement = options.requirement - find_links = options.find_links or [] - local_only = options.local - ## FIXME: Obviously this should be settable: - find_tags = False - skip_match = None - - skip_regex = options.skip_requirements_regex - if skip_regex: - skip_match = re.compile(skip_regex) - - dependency_links = [] - - f = sys.stdout - - for dist in pkg_resources.working_set: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) - for link in find_links: - if '#egg=' in link: - dependency_links.append(link) - for link in find_links: - f.write('-f %s\n' % link) - installations = {} - for dist in get_installed_distributions(local_only=local_only): - req = pip.FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags) - installations[req.name] = req - if requirement: - req_f = open(requirement) - for line in req_f: - if not line.strip() or line.strip().startswith('#'): - f.write(line) - continue - if skip_match and skip_match.search(line): - f.write(line) - continue - elif line.startswith('-e') or line.startswith('--editable'): - if line.startswith('-e'): - line = line[2:].strip() - else: - line = line[len('--editable'):].strip().lstrip('=') - line_req = InstallRequirement.from_editable(line, default_vcs=options.default_vcs) - elif (line.startswith('-r') or line.startswith('--requirement') - or line.startswith('-Z') or line.startswith('--always-unzip') - or line.startswith('-f') or line.startswith('-i') - or line.startswith('--extra-index-url')): - f.write(line) - continue - else: - line_req = InstallRequirement.from_line(line) - if not line_req.name: - logger.notify("Skipping line because it's not clear what it would install: %s" - % line.strip()) - logger.notify(" (add #egg=PackageName to the URL to avoid this warning)") - continue - if line_req.name not in installations: - logger.warn("Requirement file contains %s, but that package is not installed" - % line.strip()) - continue - f.write(str(installations[line_req.name])) - del installations[line_req.name] - f.write('## The following requirements were added by pip --freeze:\n') - for installation in sorted(installations.values(), key=lambda x: x.name): - f.write(str(installation)) - - -FreezeCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py deleted file mode 100644 index b0b366112..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py +++ /dev/null @@ -1,32 +0,0 @@ -from pip.basecommand import Command, command_dict, load_all_commands -from pip.exceptions import InstallationError -from pip.baseparser import parser - - -class HelpCommand(Command): - name = 'help' - usage = '%prog' - summary = 'Show available commands' - - def run(self, options, args): - load_all_commands() - if args: - ## FIXME: handle errors better here - command = args[0] - if command not in command_dict: - raise InstallationError('No command with the name: %s' % command) - command = command_dict[command] - command.parser.print_help() - return - parser.print_help() - print - print 'Commands available:' - commands = list(set(command_dict.values())) - commands.sort(key=lambda x: x.name) - for command in commands: - if command.hidden: - continue - print ' %s: %s' % (command.name, command.summary) - - -HelpCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py deleted file mode 100644 index 861c332bf..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py +++ /dev/null @@ -1,247 +0,0 @@ -import os, sys -from pip.req import InstallRequirement, RequirementSet -from pip.req import parse_requirements -from pip.log import logger -from pip.locations import build_prefix, src_prefix -from pip.basecommand import Command -from pip.index import PackageFinder -from pip.exceptions import InstallationError - - -class InstallCommand(Command): - name = 'install' - usage = '%prog [OPTIONS] PACKAGE_NAMES...' - summary = 'Install packages' - bundle = False - - def __init__(self): - super(InstallCommand, self).__init__() - self.parser.add_option( - '-e', '--editable', - dest='editables', - action='append', - default=[], - metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE', - help='Install a package directly from a checkout. Source will be checked ' - 'out into src/PACKAGE (lower-case) and installed in-place (using ' - 'setup.py develop). You can run this on an existing directory/checkout (like ' - 'pip install -e src/mycheckout). This option may be provided multiple times. ' - 'Possible values for VCS are: svn, git, hg and bzr.') - self.parser.add_option( - '-r', '--requirement', - dest='requirements', - action='append', - default=[], - metavar='FILENAME', - help='Install all the packages listed in the given requirements file. ' - 'This option can be used multiple times.') - self.parser.add_option( - '-f', '--find-links', - dest='find_links', - action='append', - default=[], - metavar='URL', - help='URL to look for packages at') - self.parser.add_option( - '-i', '--index-url', '--pypi-url', - dest='index_url', - metavar='URL', - default='http://pypi.python.org/simple/', - help='Base URL of Python Package Index (default %default)') - self.parser.add_option( - '--extra-index-url', - dest='extra_index_urls', - metavar='URL', - action='append', - default=[], - help='Extra URLs of package indexes to use in addition to --index-url') - self.parser.add_option( - '--no-index', - dest='no_index', - action='store_true', - default=False, - help='Ignore package index (only looking at --find-links URLs instead)') - self.parser.add_option( - '-M', '--use-mirrors', - dest='use_mirrors', - action='store_true', - default=False, - help='Use the PyPI mirrors as a fallback in case the main index is down.') - self.parser.add_option( - '--mirrors', - dest='mirrors', - metavar='URL', - action='append', - default=[], - help='Specific mirror URLs to query when --use-mirrors is used') - - self.parser.add_option( - '-b', '--build', '--build-dir', '--build-directory', - dest='build_dir', - metavar='DIR', - default=None, - help='Unpack packages into DIR (default %s) and build from there' % build_prefix) - self.parser.add_option( - '-d', '--download', '--download-dir', '--download-directory', - dest='download_dir', - metavar='DIR', - default=None, - help='Download packages into DIR instead of installing them') - self.parser.add_option( - '--download-cache', - dest='download_cache', - metavar='DIR', - default=None, - help='Cache downloaded packages in DIR') - self.parser.add_option( - '--src', '--source', '--source-dir', '--source-directory', - dest='src_dir', - metavar='DIR', - default=None, - help='Check out --editable packages into DIR (default %s)' % src_prefix) - - self.parser.add_option( - '-U', '--upgrade', - dest='upgrade', - action='store_true', - help='Upgrade all packages to the newest available version') - self.parser.add_option( - '-I', '--ignore-installed', - dest='ignore_installed', - action='store_true', - help='Ignore the installed packages (reinstalling instead)') - self.parser.add_option( - '--no-deps', '--no-dependencies', - dest='ignore_dependencies', - action='store_true', - default=False, - help='Ignore package dependencies') - self.parser.add_option( - '--no-install', - dest='no_install', - action='store_true', - help="Download and unpack all packages, but don't actually install them") - self.parser.add_option( - '--no-download', - dest='no_download', - action="store_true", - help="Don't download any packages, just install the ones already downloaded " - "(completes an install run with --no-install)") - - self.parser.add_option( - '--install-option', - dest='install_options', - action='append', - help="Extra arguments to be supplied to the setup.py install " - "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). " - "Use multiple --install-option options to pass multiple options to setup.py install. " - "If you are using an option with a directory path, be sure to use absolute path.") - - self.parser.add_option( - '--global-option', - dest='global_options', - action='append', - help="Extra global options to be supplied to the setup.py" - "call before the install command") - - self.parser.add_option( - '--user', - dest='use_user_site', - action='store_true', - help='Install to user-site') - - def _build_package_finder(self, options, index_urls): - """ - Create a package finder appropriate to this install command. - This method is meant to be overridden by subclasses, not - called directly. - """ - return PackageFinder(find_links=options.find_links, - index_urls=index_urls, - use_mirrors=options.use_mirrors, - mirrors=options.mirrors) - - def run(self, options, args): - if not options.build_dir: - options.build_dir = build_prefix - if not options.src_dir: - options.src_dir = src_prefix - if options.download_dir: - options.no_install = True - options.ignore_installed = True - options.build_dir = os.path.abspath(options.build_dir) - options.src_dir = os.path.abspath(options.src_dir) - install_options = options.install_options or [] - if options.use_user_site: - install_options.append('--user') - global_options = options.global_options or [] - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.notify('Ignoring indexes: %s' % ','.join(index_urls)) - index_urls = [] - - finder = self._build_package_finder(options, index_urls) - - requirement_set = RequirementSet( - build_dir=options.build_dir, - src_dir=options.src_dir, - download_dir=options.download_dir, - download_cache=options.download_cache, - upgrade=options.upgrade, - ignore_installed=options.ignore_installed, - ignore_dependencies=options.ignore_dependencies) - for name in args: - requirement_set.add_requirement( - InstallRequirement.from_line(name, None)) - for name in options.editables: - requirement_set.add_requirement( - InstallRequirement.from_editable(name, default_vcs=options.default_vcs)) - for filename in options.requirements: - for req in parse_requirements(filename, finder=finder, options=options): - requirement_set.add_requirement(req) - - if not requirement_set.has_requirements: - if options.find_links: - raise InstallationError('You must give at least one ' - 'requirement to %s (maybe you meant "pip install %s"?)' - % (self.name, " ".join(options.find_links))) - raise InstallationError('You must give at least one requirement ' - 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) - - if (options.use_user_site and - sys.version_info < (2, 6)): - raise InstallationError('--user is only supported in Python version 2.6 and newer') - - import setuptools - if (options.use_user_site and - requirement_set.has_editables and - not getattr(setuptools, '_distribute', False)): - - raise InstallationError('--user --editable not supported with setuptools, use distribute') - - if not options.no_download: - requirement_set.prepare_files(finder, force_root_egg_info=self.bundle, bundle=self.bundle) - else: - requirement_set.locate_files() - - if not options.no_install and not self.bundle: - requirement_set.install(install_options, global_options) - installed = ' '.join([req.name for req in - requirement_set.successfully_installed]) - if installed: - logger.notify('Successfully installed %s' % installed) - elif not self.bundle: - downloaded = ' '.join([req.name for req in - requirement_set.successfully_downloaded]) - if downloaded: - logger.notify('Successfully downloaded %s' % downloaded) - elif self.bundle: - requirement_set.create_bundle(self.bundle_filename) - logger.notify('Created bundle in %s' % self.bundle_filename) - # Clean up - if not options.no_install: - requirement_set.cleanup_files(bundle=self.bundle) - return requirement_set - - -InstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py deleted file mode 100644 index 73da58ac6..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py +++ /dev/null @@ -1,116 +0,0 @@ -import sys -import xmlrpclib -import textwrap -import pkg_resources -import pip.download -from pip.basecommand import Command -from pip.util import get_terminal_size -from pip.log import logger -from distutils.version import StrictVersion, LooseVersion - - -class SearchCommand(Command): - name = 'search' - usage = '%prog QUERY' - summary = 'Search PyPI' - - def __init__(self): - super(SearchCommand, self).__init__() - self.parser.add_option( - '--index', - dest='index', - metavar='URL', - default='http://pypi.python.org/pypi', - help='Base URL of Python Package Index (default %default)') - - def run(self, options, args): - if not args: - logger.warn('ERROR: Missing required argument (search query).') - return - query = ' '.join(args) - index_url = options.index - - pypi_hits = self.search(query, index_url) - hits = transform_hits(pypi_hits) - - terminal_width = None - if sys.stdout.isatty(): - terminal_width = get_terminal_size()[0] - - print_results(hits, terminal_width=terminal_width) - - def search(self, query, index_url): - pypi = xmlrpclib.ServerProxy(index_url, pip.download.xmlrpclib_transport) - hits = pypi.search({'name': query, 'summary': query}, 'or') - return hits - - -def transform_hits(hits): - """ - The list from pypi is really a list of versions. We want a list of - packages with the list of versions stored inline. This converts the - list from pypi into one we can use. - """ - packages = {} - for hit in hits: - name = hit['name'] - summary = hit['summary'] - version = hit['version'] - score = hit['_pypi_ordering'] - - if name not in packages.keys(): - packages[name] = {'name': name, 'summary': summary, 'versions': [version], 'score': score} - else: - packages[name]['versions'].append(version) - - # if this is the highest version, replace summary and score - if version == highest_version(packages[name]['versions']): - packages[name]['summary'] = summary - packages[name]['score'] = score - - # each record has a unique name now, so we will convert the dict into a list sorted by score - package_list = sorted(packages.values(), lambda x, y: cmp(y['score'], x['score'])) - return package_list - - -def print_results(hits, name_column_width=25, terminal_width=None): - installed_packages = [p.project_name for p in pkg_resources.working_set] - for hit in hits: - name = hit['name'] - summary = hit['summary'] or '' - if terminal_width is not None: - # wrap and indent summary to fit terminal - summary = textwrap.wrap(summary, terminal_width - name_column_width - 5) - summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) - line = '%s - %s' % (name.ljust(name_column_width), summary) - try: - logger.notify(line) - if name in installed_packages: - dist = pkg_resources.get_distribution(name) - logger.indent += 2 - try: - latest = highest_version(hit['versions']) - if dist.version == latest: - logger.notify('INSTALLED: %s (latest)' % dist.version) - else: - logger.notify('INSTALLED: %s' % dist.version) - logger.notify('LATEST: %s' % latest) - finally: - logger.indent -= 2 - except UnicodeEncodeError: - pass - - -def compare_versions(version1, version2): - try: - return cmp(StrictVersion(version1), StrictVersion(version2)) - # in case of abnormal version number, fall back to LooseVersion - except ValueError: - return cmp(LooseVersion(version1), LooseVersion(version2)) - - -def highest_version(versions): - return reduce((lambda v1, v2: compare_versions(v1, v2) == 1 and v1 or v2), versions) - - -SearchCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py deleted file mode 100644 index 7effd844e..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py +++ /dev/null @@ -1,42 +0,0 @@ -from pip.req import InstallRequirement, RequirementSet, parse_requirements -from pip.basecommand import Command -from pip.exceptions import InstallationError - -class UninstallCommand(Command): - name = 'uninstall' - usage = '%prog [OPTIONS] PACKAGE_NAMES ...' - summary = 'Uninstall packages' - - def __init__(self): - super(UninstallCommand, self).__init__() - self.parser.add_option( - '-r', '--requirement', - dest='requirements', - action='append', - default=[], - metavar='FILENAME', - help='Uninstall all the packages listed in the given requirements file. ' - 'This option can be used multiple times.') - self.parser.add_option( - '-y', '--yes', - dest='yes', - action='store_true', - help="Don't ask for confirmation of uninstall deletions.") - - def run(self, options, args): - requirement_set = RequirementSet( - build_dir=None, - src_dir=None, - download_dir=None) - for name in args: - requirement_set.add_requirement( - InstallRequirement.from_line(name)) - for filename in options.requirements: - for req in parse_requirements(filename, options=options): - requirement_set.add_requirement(req) - if not requirement_set.has_requirements: - raise InstallationError('You must give at least one requirement ' - 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) - requirement_set.uninstall(auto_confirm=options.yes) - -UninstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py deleted file mode 100644 index f83e18205..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py +++ /dev/null @@ -1,9 +0,0 @@ -from pip.commands.zip import ZipCommand - - -class UnzipCommand(ZipCommand): - name = 'unzip' - summary = 'Unzip individual packages' - - -UnzipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py deleted file mode 100644 index 346fc0519..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py +++ /dev/null @@ -1,346 +0,0 @@ -import sys -import re -import fnmatch -import os -import shutil -import zipfile -from pip.util import display_path, backup_dir -from pip.log import logger -from pip.exceptions import InstallationError -from pip.basecommand import Command - - -class ZipCommand(Command): - name = 'zip' - usage = '%prog [OPTIONS] PACKAGE_NAMES...' - summary = 'Zip individual packages' - - def __init__(self): - super(ZipCommand, self).__init__() - if self.name == 'zip': - self.parser.add_option( - '--unzip', - action='store_true', - dest='unzip', - help='Unzip (rather than zip) a package') - else: - self.parser.add_option( - '--zip', - action='store_false', - dest='unzip', - default=True, - help='Zip (rather than unzip) a package') - self.parser.add_option( - '--no-pyc', - action='store_true', - dest='no_pyc', - help='Do not include .pyc files in zip files (useful on Google App Engine)') - self.parser.add_option( - '-l', '--list', - action='store_true', - dest='list', - help='List the packages available, and their zip status') - self.parser.add_option( - '--sort-files', - action='store_true', - dest='sort_files', - help='With --list, sort packages according to how many files they contain') - self.parser.add_option( - '--path', - action='append', - dest='paths', - help='Restrict operations to the given paths (may include wildcards)') - self.parser.add_option( - '-n', '--simulate', - action='store_true', - help='Do not actually perform the zip/unzip operation') - - def paths(self): - """All the entries of sys.path, possibly restricted by --path""" - if not self.select_paths: - return sys.path - result = [] - match_any = set() - for path in sys.path: - path = os.path.normcase(os.path.abspath(path)) - for match in self.select_paths: - match = os.path.normcase(os.path.abspath(match)) - if '*' in match: - if re.search(fnmatch.translate(match+'*'), path): - result.append(path) - match_any.add(match) - break - else: - if path.startswith(match): - result.append(path) - match_any.add(match) - break - else: - logger.debug("Skipping path %s because it doesn't match %s" - % (path, ', '.join(self.select_paths))) - for match in self.select_paths: - if match not in match_any and '*' not in match: - result.append(match) - logger.debug("Adding path %s because it doesn't match anything already on sys.path" - % match) - return result - - def run(self, options, args): - self.select_paths = options.paths - self.simulate = options.simulate - if options.list: - return self.list(options, args) - if not args: - raise InstallationError( - 'You must give at least one package to zip or unzip') - packages = [] - for arg in args: - module_name, filename = self.find_package(arg) - if options.unzip and os.path.isdir(filename): - raise InstallationError( - 'The module %s (in %s) is not a zip file; cannot be unzipped' - % (module_name, filename)) - elif not options.unzip and not os.path.isdir(filename): - raise InstallationError( - 'The module %s (in %s) is not a directory; cannot be zipped' - % (module_name, filename)) - packages.append((module_name, filename)) - last_status = None - for module_name, filename in packages: - if options.unzip: - last_status = self.unzip_package(module_name, filename) - else: - last_status = self.zip_package(module_name, filename, options.no_pyc) - return last_status - - def unzip_package(self, module_name, filename): - zip_filename = os.path.dirname(filename) - if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename): - raise InstallationError( - 'Module %s (in %s) isn\'t located in a zip file in %s' - % (module_name, filename, zip_filename)) - package_path = os.path.dirname(zip_filename) - if not package_path in self.paths(): - logger.warn( - 'Unpacking %s into %s, but %s is not on sys.path' - % (display_path(zip_filename), display_path(package_path), - display_path(package_path))) - logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename))) - if self.simulate: - logger.notify('Skipping remaining operations because of --simulate') - return - logger.indent += 2 - try: - ## FIXME: this should be undoable: - zip = zipfile.ZipFile(zip_filename) - to_save = [] - for name in zip.namelist(): - if name.startswith(module_name + os.path.sep): - content = zip.read(name) - dest = os.path.join(package_path, name) - if not os.path.exists(os.path.dirname(dest)): - os.makedirs(os.path.dirname(dest)) - if not content and dest.endswith(os.path.sep): - if not os.path.exists(dest): - os.makedirs(dest) - else: - f = open(dest, 'wb') - f.write(content) - f.close() - else: - to_save.append((name, zip.read(name))) - zip.close() - if not to_save: - logger.info('Removing now-empty zip file %s' % display_path(zip_filename)) - os.unlink(zip_filename) - self.remove_filename_from_pth(zip_filename) - else: - logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename))) - zip = zipfile.ZipFile(zip_filename, 'w') - for name, content in to_save: - zip.writestr(name, content) - zip.close() - finally: - logger.indent -= 2 - - def zip_package(self, module_name, filename, no_pyc): - orig_filename = filename - logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) - logger.indent += 2 - if filename.endswith('.egg'): - dest_filename = filename - else: - dest_filename = filename + '.zip' - try: - ## FIXME: I think this needs to be undoable: - if filename == dest_filename: - filename = backup_dir(orig_filename) - logger.notify('Moving %s aside to %s' % (orig_filename, filename)) - if not self.simulate: - shutil.move(orig_filename, filename) - try: - logger.info('Creating zip file in %s' % display_path(dest_filename)) - if not self.simulate: - zip = zipfile.ZipFile(dest_filename, 'w') - zip.writestr(module_name + '/', '') - for dirpath, dirnames, filenames in os.walk(filename): - if no_pyc: - filenames = [f for f in filenames - if not f.lower().endswith('.pyc')] - for fns, is_dir in [(dirnames, True), (filenames, False)]: - for fn in fns: - full = os.path.join(dirpath, fn) - dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn) - if is_dir: - zip.writestr(dest+'/', '') - else: - zip.write(full, dest) - zip.close() - logger.info('Removing old directory %s' % display_path(filename)) - if not self.simulate: - shutil.rmtree(filename) - except: - ## FIXME: need to do an undo here - raise - ## FIXME: should also be undone: - self.add_filename_to_pth(dest_filename) - finally: - logger.indent -= 2 - - def remove_filename_from_pth(self, filename): - for pth in self.pth_files(): - f = open(pth, 'r') - lines = f.readlines() - f.close() - new_lines = [ - l for l in lines if l.strip() != filename] - if lines != new_lines: - logger.info('Removing reference to %s from .pth file %s' - % (display_path(filename), display_path(pth))) - if not filter(None, new_lines): - logger.info('%s file would be empty: deleting' % display_path(pth)) - if not self.simulate: - os.unlink(pth) - else: - if not self.simulate: - f = open(pth, 'wb') - f.writelines(new_lines) - f.close() - return - logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename)) - - def add_filename_to_pth(self, filename): - path = os.path.dirname(filename) - dest = os.path.join(path, filename + '.pth') - if path not in self.paths(): - logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest)) - if not self.simulate: - if os.path.exists(dest): - f = open(dest) - lines = f.readlines() - f.close() - if lines and not lines[-1].endswith('\n'): - lines[-1] += '\n' - lines.append(filename+'\n') - else: - lines = [filename + '\n'] - f = open(dest, 'wb') - f.writelines(lines) - f.close() - - def pth_files(self): - for path in self.paths(): - if not os.path.exists(path) or not os.path.isdir(path): - continue - for filename in os.listdir(path): - if filename.endswith('.pth'): - yield os.path.join(path, filename) - - def find_package(self, package): - for path in self.paths(): - full = os.path.join(path, package) - if os.path.exists(full): - return package, full - if not os.path.isdir(path) and zipfile.is_zipfile(path): - zip = zipfile.ZipFile(path, 'r') - try: - zip.read(os.path.join(package, '__init__.py')) - except KeyError: - pass - else: - zip.close() - return package, full - zip.close() - ## FIXME: need special error for package.py case: - raise InstallationError( - 'No package with the name %s found' % package) - - def list(self, options, args): - if args: - raise InstallationError( - 'You cannot give an argument with --list') - for path in sorted(self.paths()): - if not os.path.exists(path): - continue - basename = os.path.basename(path.rstrip(os.path.sep)) - if os.path.isfile(path) and zipfile.is_zipfile(path): - if os.path.dirname(path) not in self.paths(): - logger.notify('Zipped egg: %s' % display_path(path)) - continue - if (basename != 'site-packages' and basename != 'dist-packages' - and not path.replace('\\', '/').endswith('lib/python')): - continue - logger.notify('In %s:' % display_path(path)) - logger.indent += 2 - zipped = [] - unzipped = [] - try: - for filename in sorted(os.listdir(path)): - ext = os.path.splitext(filename)[1].lower() - if ext in ('.pth', '.egg-info', '.egg-link'): - continue - if ext == '.py': - logger.info('Not displaying %s: not a package' % display_path(filename)) - continue - full = os.path.join(path, filename) - if os.path.isdir(full): - unzipped.append((filename, self.count_package(full))) - elif zipfile.is_zipfile(full): - zipped.append(filename) - else: - logger.info('Unknown file: %s' % display_path(filename)) - if zipped: - logger.notify('Zipped packages:') - logger.indent += 2 - try: - for filename in zipped: - logger.notify(filename) - finally: - logger.indent -= 2 - else: - logger.notify('No zipped packages.') - if unzipped: - if options.sort_files: - unzipped.sort(key=lambda x: -x[1]) - logger.notify('Unzipped packages:') - logger.indent += 2 - try: - for filename, count in unzipped: - logger.notify('%s (%i files)' % (filename, count)) - finally: - logger.indent -= 2 - else: - logger.notify('No unzipped packages.') - finally: - logger.indent -= 2 - - def count_package(self, path): - total = 0 - for dirpath, dirnames, filenames in os.walk(path): - filenames = [f for f in filenames - if not f.lower().endswith('.pyc')] - total += len(filenames) - return total - - -ZipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py deleted file mode 100644 index f1b63936b..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py +++ /dev/null @@ -1,470 +0,0 @@ -import xmlrpclib -import re -import getpass -import urllib -import urllib2 -import urlparse -import os -import mimetypes -import shutil -import tempfile -from pip.backwardcompat import md5, copytree -from pip.exceptions import InstallationError -from pip.util import (splitext, - format_size, display_path, backup_dir, ask, - unpack_file, create_download_cache_folder, cache_download) -from pip.vcs import vcs -from pip.log import logger - - -__all__ = ['xmlrpclib_transport', 'get_file_content', 'urlopen', - 'is_url', 'url_to_path', 'path_to_url', 'path_to_url2', - 'geturl', 'is_archive_file', 'unpack_vcs_link', - 'unpack_file_url', 'is_vcs_url', 'is_file_url', 'unpack_http_url'] - - -xmlrpclib_transport = xmlrpclib.Transport() - - -def get_file_content(url, comes_from=None): - """Gets the content of a file; it may be a filename, file: URL, or - http: URL. Returns (location, content)""" - match = _scheme_re.search(url) - if match: - scheme = match.group(1).lower() - if (scheme == 'file' and comes_from - and comes_from.startswith('http')): - raise InstallationError( - 'Requirements file %s references URL %s, which is local' - % (comes_from, url)) - if scheme == 'file': - path = url.split(':', 1)[1] - path = path.replace('\\', '/') - match = _url_slash_drive_re.match(path) - if match: - path = match.group(1) + ':' + path.split('|', 1)[1] - path = urllib.unquote(path) - if path.startswith('/'): - path = '/' + path.lstrip('/') - url = path - else: - ## FIXME: catch some errors - resp = urlopen(url) - return geturl(resp), resp.read() - try: - f = open(url) - content = f.read() - except IOError, e: - raise InstallationError('Could not open requirements file: %s' % str(e)) - else: - f.close() - return url, content - - -_scheme_re = re.compile(r'^(http|https|file):', re.I) -_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) - -class URLOpener(object): - """ - pip's own URL helper that adds HTTP auth and proxy support - """ - def __init__(self): - self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - - def __call__(self, url): - """ - If the given url contains auth info or if a normal request gets a 401 - response, an attempt is made to fetch the resource using basic HTTP - auth. - - """ - url, username, password = self.extract_credentials(url) - if username is None: - try: - response = urllib2.urlopen(self.get_request(url)) - except urllib2.HTTPError, e: - if e.code != 401: - raise - response = self.get_response(url) - else: - response = self.get_response(url, username, password) - return response - - def get_request(self, url): - """ - Wraps the URL to retrieve to protects against "creative" - interpretation of the RFC: http://bugs.python.org/issue8732 - """ - if isinstance(url, basestring): - url = urllib2.Request(url, headers={'Accept-encoding': 'identity'}) - return url - - def get_response(self, url, username=None, password=None): - """ - does the dirty work of actually getting the rsponse object using urllib2 - and its HTTP auth builtins. - """ - scheme, netloc, path, query, frag = urlparse.urlsplit(url) - pass_url = urlparse.urlunsplit(('_none_', netloc, path, query, frag)).replace('_none_://', '', 1) - req = self.get_request(url) - - stored_username, stored_password = self.passman.find_user_password(None, netloc) - # see if we have a password stored - if stored_username is None: - if username is None and self.prompting: - username = urllib.quote(raw_input('User for %s: ' % netloc)) - password = urllib.quote(getpass.getpass('Password: ')) - if username and password: - self.passman.add_password(None, netloc, username, password) - stored_username, stored_password = self.passman.find_user_password(None, netloc) - authhandler = urllib2.HTTPBasicAuthHandler(self.passman) - opener = urllib2.build_opener(authhandler) - # FIXME: should catch a 401 and offer to let the user reenter credentials - return opener.open(req) - - def setup(self, proxystr='', prompting=True): - """ - Sets the proxy handler given the option passed on the command - line. If an empty string is passed it looks at the HTTP_PROXY - environment variable. - """ - self.prompting = prompting - proxy = self.get_proxy(proxystr) - if proxy: - proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy}) - opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler) - urllib2.install_opener(opener) - - def parse_credentials(self, netloc): - if "@" in netloc: - userinfo = netloc.rsplit("@", 1)[0] - if ":" in userinfo: - return userinfo.split(":", 1) - return userinfo, None - return None, None - - def extract_credentials(self, url): - """ - Extracts user/password from a url. - - Returns a tuple: - (url-without-auth, username, password) - """ - if isinstance(url, urllib2.Request): - result = urlparse.urlsplit(url.get_full_url()) - else: - result = urlparse.urlsplit(url) - scheme, netloc, path, query, frag = result - - username, password = self.parse_credentials(netloc) - if username is None: - return url, None, None - elif password is None and self.prompting: - # remove the auth credentials from the url part - netloc = netloc.replace('%s@' % username, '', 1) - # prompt for the password - prompt = 'Password for %s@%s: ' % (username, netloc) - password = urllib.quote(getpass.getpass(prompt)) - else: - # remove the auth credentials from the url part - netloc = netloc.replace('%s:%s@' % (username, password), '', 1) - - target_url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - return target_url, username, password - - def get_proxy(self, proxystr=''): - """ - Get the proxy given the option passed on the command line. - If an empty string is passed it looks at the HTTP_PROXY - environment variable. - """ - if not proxystr: - proxystr = os.environ.get('HTTP_PROXY', '') - if proxystr: - if '@' in proxystr: - user_password, server_port = proxystr.split('@', 1) - if ':' in user_password: - user, password = user_password.split(':', 1) - else: - user = user_password - prompt = 'Password for %s@%s: ' % (user, server_port) - password = urllib.quote(getpass.getpass(prompt)) - return '%s:%s@%s' % (user, password, server_port) - else: - return proxystr - else: - return None - -urlopen = URLOpener() - - -def is_url(name): - """Returns true if the name looks like a URL""" - if ':' not in name: - return False - scheme = name.split(':', 1)[0].lower() - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def url_to_path(url): - """ - Convert a file: URL to a path. - """ - assert url.startswith('file:'), ( - "You can only turn file: urls into filenames (not %r)" % url) - path = url[len('file:'):].lstrip('/') - path = urllib.unquote(path) - if _url_drive_re.match(path): - path = path[0] + ':' + path[2:] - else: - path = '/' + path - return path - - -_drive_re = re.compile('^([a-z]):', re.I) -_url_drive_re = re.compile('^([a-z])[:|]', re.I) - - -def path_to_url(path): - """ - Convert a path to a file: URL. The path will be made absolute. - """ - path = os.path.normcase(os.path.abspath(path)) - if _drive_re.match(path): - path = path[0] + '|' + path[2:] - url = urllib.quote(path) - url = url.replace(os.path.sep, '/') - url = url.lstrip('/') - return 'file:///' + url - - -def path_to_url2(path): - """ - Convert a path to a file: URL. The path will be made absolute and have - quoted path parts. - """ - path = os.path.normpath(os.path.abspath(path)) - drive, path = os.path.splitdrive(path) - filepath = path.split(os.path.sep) - url = '/'.join([urllib.quote(part) for part in filepath]) - if not drive: - url = url.lstrip('/') - return 'file:///' + drive + url - - -def geturl(urllib2_resp): - """ - Use instead of urllib.addinfourl.geturl(), which appears to have - some issues with dropping the double slash for certain schemes - (e.g. file://). This implementation is probably over-eager, as it - always restores '://' if it is missing, and it appears some url - schemata aren't always followed by '//' after the colon, but as - far as I know pip doesn't need any of those. - The URI RFC can be found at: http://tools.ietf.org/html/rfc1630 - - This function assumes that - scheme:/foo/bar - is the same as - scheme:///foo/bar - """ - url = urllib2_resp.geturl() - scheme, rest = url.split(':', 1) - if rest.startswith('//'): - return url - else: - # FIXME: write a good test to cover it - return '%s://%s' % (scheme, rest) - - -def is_archive_file(name): - """Return True if `name` is a considered as an archive file.""" - archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle') - ext = splitext(name)[1].lower() - if ext in archives: - return True - return False - - -def unpack_vcs_link(link, location, only_download=False): - vcs_backend = _get_used_vcs_backend(link) - if only_download: - vcs_backend.export(location) - else: - vcs_backend.unpack(location) - - -def unpack_file_url(link, location): - source = url_to_path(link.url) - content_type = mimetypes.guess_type(source)[0] - if os.path.isdir(source): - # delete the location since shutil will create it again :( - if os.path.isdir(location): - shutil.rmtree(location) - copytree(source, location) - else: - unpack_file(source, location, content_type, link) - - -def _get_used_vcs_backend(link): - for backend in vcs.backends: - if link.scheme in backend.schemes: - vcs_backend = backend(link.url) - return vcs_backend - - -def is_vcs_url(link): - return bool(_get_used_vcs_backend(link)) - - -def is_file_url(link): - return link.url.lower().startswith('file:') - - -def _check_md5(download_hash, link): - download_hash = download_hash.hexdigest() - if download_hash != link.md5_hash: - logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!" - % (link, download_hash, link.md5_hash)) - raise InstallationError('Bad MD5 hash for package %s' % link) - - -def _get_md5_from_file(target_file, link): - download_hash = md5() - fp = open(target_file, 'rb') - while 1: - chunk = fp.read(4096) - if not chunk: - break - download_hash.update(chunk) - fp.close() - return download_hash - - -def _download_url(resp, link, temp_location): - fp = open(temp_location, 'wb') - download_hash = None - if link.md5_hash: - download_hash = md5() - try: - total_length = int(resp.info()['content-length']) - except (ValueError, KeyError): - total_length = 0 - downloaded = 0 - show_progress = total_length > 40*1000 or not total_length - show_url = link.show_url - try: - if show_progress: - ## FIXME: the URL can get really long in this message: - if total_length: - logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) - else: - logger.start_progress('Downloading %s (unknown size): ' % show_url) - else: - logger.notify('Downloading %s' % show_url) - logger.debug('Downloading from URL %s' % link) - - while 1: - chunk = resp.read(4096) - if not chunk: - break - downloaded += len(chunk) - if show_progress: - if not total_length: - logger.show_progress('%s' % format_size(downloaded)) - else: - logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded))) - if link.md5_hash: - download_hash.update(chunk) - fp.write(chunk) - fp.close() - finally: - if show_progress: - logger.end_progress('%s downloaded' % format_size(downloaded)) - return download_hash - - -def _copy_file(filename, location, content_type, link): - copy = True - download_location = os.path.join(location, link.filename) - if os.path.exists(download_location): - response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' - % display_path(download_location), ('i', 'w', 'b')) - if response == 'i': - copy = False - elif response == 'w': - logger.warn('Deleting %s' % display_path(download_location)) - os.remove(download_location) - elif response == 'b': - dest_file = backup_dir(download_location) - logger.warn('Backing up %s to %s' - % (display_path(download_location), display_path(dest_file))) - shutil.move(download_location, dest_file) - if copy: - shutil.copy(filename, download_location) - logger.indent -= 2 - logger.notify('Saved %s' % display_path(download_location)) - - -def unpack_http_url(link, location, download_cache, only_download): - temp_dir = tempfile.mkdtemp('-unpack', 'pip-') - target_url = link.url.split('#', 1)[0] - target_file = None - download_hash = None - if download_cache: - target_file = os.path.join(download_cache, - urllib.quote(target_url, '')) - if not os.path.isdir(download_cache): - create_download_cache_folder(download_cache) - if (target_file - and os.path.exists(target_file) - and os.path.exists(target_file+'.content-type')): - fp = open(target_file+'.content-type') - content_type = fp.read().strip() - fp.close() - if link.md5_hash: - download_hash = _get_md5_from_file(target_file, link) - temp_location = target_file - logger.notify('Using download cache from %s' % target_file) - else: - resp = _get_response_from_url(target_url, link) - content_type = resp.info()['content-type'] - filename = link.filename - ext = splitext(filename)[1] - if not ext: - ext = mimetypes.guess_extension(content_type) - if ext: - filename += ext - if not ext and link.url != geturl(resp): - ext = os.path.splitext(geturl(resp))[1] - if ext: - filename += ext - temp_location = os.path.join(temp_dir, filename) - download_hash = _download_url(resp, link, temp_location) - if link.md5_hash: - _check_md5(download_hash, link) - if only_download: - _copy_file(temp_location, location, content_type, link) - else: - unpack_file(temp_location, location, content_type, link) - if target_file and target_file != temp_location: - cache_download(target_file, temp_location, content_type) - if target_file is None: - os.unlink(temp_location) - os.rmdir(temp_dir) - - -def _get_response_from_url(target_url, link): - try: - resp = urlopen(target_url) - except urllib2.HTTPError, e: - logger.fatal("HTTP error %s while getting %s" % (e.code, link)) - raise - except IOError, e: - # Typically an FTP error - logger.fatal("Error %s while getting %s" % (e, link)) - raise - return resp - -class Urllib2HeadRequest(urllib2.Request): - def get_method(self): - return "HEAD" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py deleted file mode 100644 index 1ad1a616d..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Exceptions used throughout package""" - - -class InstallationError(Exception): - """General exception during installation""" - - -class UninstallationError(Exception): - """General exception during uninstallation""" - - -class DistributionNotFound(InstallationError): - """Raised when a distribution cannot be found to satisfy a requirement""" - - -class BadCommand(Exception): - """Raised when virtualenv or a command is not found""" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py deleted file mode 100644 index e42d8c868..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py +++ /dev/null @@ -1,686 +0,0 @@ -"""Routines related to PyPI, indexes""" - -import sys -import os -import re -import mimetypes -import threading -import posixpath -import pkg_resources -import urllib -import urllib2 -import urlparse -import httplib -import random -import socket -import string -from Queue import Queue -from Queue import Empty as QueueEmpty -from pip.log import logger -from pip.util import Inf -from pip.util import normalize_name, splitext -from pip.exceptions import DistributionNotFound -from pip.backwardcompat import WindowsError, product -from pip.download import urlopen, path_to_url2, url_to_path, geturl, Urllib2HeadRequest - -__all__ = ['PackageFinder'] - - -DEFAULT_MIRROR_URL = "last.pypi.python.org" - - -class PackageFinder(object): - """This finds packages. - - This is meant to match easy_install's technique for looking for - packages, by reading pages and looking for appropriate links - """ - - def __init__(self, find_links, index_urls, - use_mirrors=False, mirrors=None, main_mirror_url=None): - self.find_links = find_links - self.index_urls = index_urls - self.dependency_links = [] - self.cache = PageCache() - # These are boring links that have already been logged somehow: - self.logged_links = set() - if use_mirrors: - self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) - logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) - else: - self.mirror_urls = [] - - def add_dependency_links(self, links): - ## FIXME: this shouldn't be global list this, it should only - ## apply to requirements of the package that specifies the - ## dependency_links value - ## FIXME: also, we should track comes_from (i.e., use Link) - self.dependency_links.extend(links) - - @staticmethod - def _sort_locations(locations): - """ - Sort locations into "files" (archives) and "urls", and return - a pair of lists (files,urls) - """ - files = [] - urls = [] - - # puts the url for the given file path into the appropriate - # list - def sort_path(path): - url = path_to_url2(path) - if mimetypes.guess_type(url, strict=False)[0] == 'text/html': - urls.append(url) - else: - files.append(url) - - for url in locations: - if url.startswith('file:'): - path = url_to_path(url) - if os.path.isdir(path): - path = os.path.realpath(path) - for item in os.listdir(path): - sort_path(os.path.join(path, item)) - elif os.path.isfile(path): - sort_path(path) - else: - urls.append(url) - return files, urls - - def find_requirement(self, req, upgrade): - url_name = req.url_name - # Only check main index if index URL is given: - main_index_url = None - if self.index_urls: - # Check that we have the url_name correctly spelled: - main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) - # This will also cache the page, so it's okay that we get it again later: - page = self._get_page(main_index_url, req) - if page is None: - url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name - - # Combine index URLs with mirror URLs here to allow - # adding more index URLs from requirements files - all_index_urls = self.index_urls + self.mirror_urls - - def mkurl_pypi_url(url): - loc = posixpath.join(url, url_name) - # For maximum compatibility with easy_install, ensure the path - # ends in a trailing slash. Although this isn't in the spec - # (and PyPI can handle it without the slash) some other index - # implementations might break if they relied on easy_install's behavior. - if not loc.endswith('/'): - loc = loc + '/' - return loc - if url_name is not None: - locations = [ - mkurl_pypi_url(url) - for url in all_index_urls] + self.find_links - else: - locations = list(self.find_links) - locations.extend(self.dependency_links) - for version in req.absolute_versions: - if url_name is not None and main_index_url is not None: - locations = [ - posixpath.join(main_index_url.url, version)] + locations - - file_locations, url_locations = self._sort_locations(locations) - - locations = [Link(url) for url in url_locations] - logger.debug('URLs to search for versions for %s:' % req) - for location in locations: - logger.debug('* %s' % location) - found_versions = [] - found_versions.extend( - self._package_versions( - [Link(url, '-f') for url in self.find_links], req.name.lower())) - page_versions = [] - for page in self._get_pages(locations, req): - logger.debug('Analyzing links from page %s' % page.url) - logger.indent += 2 - try: - page_versions.extend(self._package_versions(page.links, req.name.lower())) - finally: - logger.indent -= 2 - dependency_versions = list(self._package_versions( - [Link(url) for url in self.dependency_links], req.name.lower())) - if dependency_versions: - logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) - file_versions = list(self._package_versions( - [Link(url) for url in file_locations], req.name.lower())) - if not found_versions and not page_versions and not dependency_versions and not file_versions: - logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) - raise DistributionNotFound('No distributions at all found for %s' % req) - if req.satisfied_by is not None: - found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) - if file_versions: - file_versions.sort(reverse=True) - logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) - found_versions = file_versions + found_versions - all_versions = found_versions + page_versions + dependency_versions - applicable_versions = [] - for (parsed_version, link, version) in all_versions: - if version not in req.req: - logger.info("Ignoring link %s, version %s doesn't match %s" - % (link, version, ','.join([''.join(s) for s in req.req.specs]))) - continue - applicable_versions.append((link, version)) - applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) - existing_applicable = bool([link for link, version in applicable_versions if link is Inf]) - if not upgrade and existing_applicable: - if applicable_versions[0][1] is Inf: - logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' - % req.satisfied_by.version) - else: - logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' - % (req.satisfied_by.version, applicable_versions[0][1])) - return None - if not applicable_versions: - logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' - % (req, ', '.join([version for parsed_version, link, version in found_versions]))) - raise DistributionNotFound('No distributions matching the version for %s' % req) - if applicable_versions[0][0] is Inf: - # We have an existing version, and its the best version - logger.info('Installed version (%s) is most up-to-date (past versions: %s)' - % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none')) - return None - if len(applicable_versions) > 1: - logger.info('Using version %s (newest of versions: %s)' % - (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions]))) - return applicable_versions[0][0] - - def _find_url_name(self, index_url, url_name, req): - """Finds the true URL name of a package, when the given name isn't quite correct. - This is usually used to implement case-insensitivity.""" - if not index_url.url.endswith('/'): - # Vaguely part of the PyPI API... weird but true. - ## FIXME: bad to modify this? - index_url.url += '/' - page = self._get_page(index_url, req) - if page is None: - logger.fatal('Cannot fetch index base URL %s' % index_url) - return - norm_name = normalize_name(req.url_name) - for link in page.links: - base = posixpath.basename(link.path.rstrip('/')) - if norm_name == normalize_name(base): - logger.notify('Real name of requirement %s is %s' % (url_name, base)) - return base - return None - - def _get_pages(self, locations, req): - """Yields (page, page_url) from the given locations, skipping - locations that have errors, and adding download/homepage links""" - pending_queue = Queue() - for location in locations: - pending_queue.put(location) - done = [] - seen = set() - threads = [] - for i in range(min(10, len(locations))): - t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen)) - t.setDaemon(True) - threads.append(t) - t.start() - for t in threads: - t.join() - return done - - _log_lock = threading.Lock() - - def _get_queued_page(self, req, pending_queue, done, seen): - while 1: - try: - location = pending_queue.get(False) - except QueueEmpty: - return - if location in seen: - continue - seen.add(location) - page = self._get_page(location, req) - if page is None: - continue - done.append(page) - for link in page.rel_links(): - pending_queue.put(link) - - _egg_fragment_re = re.compile(r'#egg=([^&]*)') - _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I) - _py_version_re = re.compile(r'-py([123]\.[0-9])$') - - def _sort_links(self, links): - "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates" - eggs, no_eggs = [], [] - seen = set() - for link in links: - if link not in seen: - seen.add(link) - if link.egg_fragment: - eggs.append(link) - else: - no_eggs.append(link) - return no_eggs + eggs - - def _package_versions(self, links, search_name): - for link in self._sort_links(links): - for v in self._link_package_versions(link, search_name): - yield v - - def _link_package_versions(self, link, search_name): - """ - Return an iterable of triples (pkg_resources_version_key, - link, python_version) that can be extracted from the given - link. - - Meant to be overridden by subclasses, not called by clients. - """ - if link.egg_fragment: - egg_info = link.egg_fragment - else: - egg_info, ext = link.splitext() - if not ext: - if link not in self.logged_links: - logger.debug('Skipping link %s; not a file' % link) - self.logged_links.add(link) - return [] - if egg_info.endswith('.tar'): - # Special double-extension case: - egg_info = egg_info[:-4] - ext = '.tar' + ext - if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'): - if link not in self.logged_links: - logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext)) - self.logged_links.add(link) - return [] - version = self._egg_info_matches(egg_info, search_name, link) - if version is None: - logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name)) - return [] - match = self._py_version_re.search(version) - if match: - version = version[:match.start()] - py_version = match.group(1) - if py_version != sys.version[:3]: - logger.debug('Skipping %s because Python version is incorrect' % link) - return [] - logger.debug('Found link %s, version: %s' % (link, version)) - return [(pkg_resources.parse_version(version), - link, - version)] - - def _egg_info_matches(self, egg_info, search_name, link): - match = self._egg_info_re.search(egg_info) - if not match: - logger.debug('Could not parse version from link: %s' % link) - return None - name = match.group(0).lower() - # To match the "safe" name that pkg_resources creates: - name = name.replace('_', '-') - if name.startswith(search_name.lower()): - return match.group(0)[len(search_name):].lstrip('-') - else: - return None - - def _get_page(self, link, req): - return HTMLPage.get_page(link, req, cache=self.cache) - - def _get_mirror_urls(self, mirrors=None, main_mirror_url=None): - """Retrieves a list of URLs from the main mirror DNS entry - unless a list of mirror URLs are passed. - """ - if not mirrors: - mirrors = get_mirrors(main_mirror_url) - # Should this be made "less random"? E.g. netselect like? - random.shuffle(mirrors) - - mirror_urls = set() - for mirror_url in mirrors: - # Make sure we have a valid URL - if not ("http://" or "https://" or "file://") in mirror_url: - mirror_url = "http://%s" % mirror_url - if not mirror_url.endswith("/simple"): - mirror_url = "%s/simple/" % mirror_url - mirror_urls.add(mirror_url) - - return list(mirror_urls) - - -class PageCache(object): - """Cache of HTML pages""" - - failure_limit = 3 - - def __init__(self): - self._failures = {} - self._pages = {} - self._archives = {} - - def too_many_failures(self, url): - return self._failures.get(url, 0) >= self.failure_limit - - def get_page(self, url): - return self._pages.get(url) - - def is_archive(self, url): - return self._archives.get(url, False) - - def set_is_archive(self, url, value=True): - self._archives[url] = value - - def add_page_failure(self, url, level): - self._failures[url] = self._failures.get(url, 0)+level - - def add_page(self, urls, page): - for url in urls: - self._pages[url] = page - - -class HTMLPage(object): - """Represents one page, along with its URL""" - - ## FIXME: these regexes are horrible hacks: - _homepage_re = re.compile(r'<th>\s*home\s*page', re.I) - _download_re = re.compile(r'<th>\s*download\s+url', re.I) - ## These aren't so aweful: - _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I) - _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S) - _base_re = re.compile(r"""<base\s+href\s*=\s*['"]?([^'">]+)""", re.I) - - def __init__(self, content, url, headers=None): - self.content = content - self.url = url - self.headers = headers - - def __str__(self): - return self.url - - @classmethod - def get_page(cls, link, req, cache=None, skip_archives=True): - url = link.url - url = url.split('#', 1)[0] - if cache.too_many_failures(url): - return None - - # Check for VCS schemes that do not support lookup as web pages. - from pip.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals()) - return None - - if cache is not None: - inst = cache.get_page(url) - if inst is not None: - return inst - try: - if skip_archives: - if cache is not None: - if cache.is_archive(url): - return None - filename = link.filename - for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']: - if filename.endswith(bad_ext): - content_type = cls._get_content_type(url) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type)) - if cache is not None: - cache.set_is_archive(url) - return None - logger.debug('Getting page %s' % url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) - if scheme == 'file' and os.path.isdir(urllib.url2pathname(path)): - # add trailing slash if not present so urljoin doesn't trim final segment - if not url.endswith('/'): - url += '/' - url = urlparse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s' % url) - - resp = urlopen(url) - - real_url = geturl(resp) - headers = resp.info() - inst = cls(resp.read(), real_url, headers) - except (urllib2.HTTPError, urllib2.URLError, socket.timeout, socket.error, OSError, WindowsError), e: - desc = str(e) - if isinstance(e, socket.timeout): - log_meth = logger.info - level =1 - desc = 'timed out' - elif isinstance(e, urllib2.URLError): - log_meth = logger.info - if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout): - desc = 'timed out' - level = 1 - else: - level = 2 - elif isinstance(e, urllib2.HTTPError) and e.code == 404: - ## FIXME: notify? - log_meth = logger.info - level = 2 - else: - log_meth = logger.info - level = 1 - log_meth('Could not fetch URL %s: %s' % (link, desc)) - log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req)) - if cache is not None: - cache.add_page_failure(url, level) - return None - if cache is not None: - cache.add_page([url, real_url], inst) - return inst - - @staticmethod - def _get_content_type(url): - """Get the Content-Type of the given url, using a HEAD request""" - scheme, netloc, path, query, fragment = urlparse.urlsplit(url) - if not scheme in ('http', 'https', 'ftp', 'ftps'): - ## FIXME: some warning or something? - ## assertion error? - return '' - req = Urllib2HeadRequest(url, headers={'Host': netloc}) - resp = urlopen(req) - try: - if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'): - ## FIXME: doesn't handle redirects - return '' - return resp.info().get('content-type', '') - finally: - resp.close() - - @property - def base_url(self): - if not hasattr(self, "_base_url"): - match = self._base_re.search(self.content) - if match: - self._base_url = match.group(1) - else: - self._base_url = self.url - return self._base_url - - @property - def links(self): - """Yields all links in the page""" - for match in self._href_re.finditer(self.content): - url = match.group(1) or match.group(2) or match.group(3) - url = self.clean_link(urlparse.urljoin(self.base_url, url)) - yield Link(url, self) - - def rel_links(self): - for url in self.explicit_rel_links(): - yield url - for url in self.scraped_rel_links(): - yield url - - def explicit_rel_links(self, rels=('homepage', 'download')): - """Yields all links with the given relations""" - for match in self._rel_re.finditer(self.content): - found_rels = match.group(1).lower().split() - for rel in rels: - if rel in found_rels: - break - else: - continue - match = self._href_re.search(match.group(0)) - if not match: - continue - url = match.group(1) or match.group(2) or match.group(3) - url = self.clean_link(urlparse.urljoin(self.base_url, url)) - yield Link(url, self) - - def scraped_rel_links(self): - for regex in (self._homepage_re, self._download_re): - match = regex.search(self.content) - if not match: - continue - href_match = self._href_re.search(self.content, pos=match.end()) - if not href_match: - continue - url = match.group(1) or match.group(2) or match.group(3) - if not url: - continue - url = self.clean_link(urlparse.urljoin(self.base_url, url)) - yield Link(url, self) - - _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - - def clean_link(self, url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return self._clean_re.sub( - lambda match: '%%%2x' % ord(match.group(0)), url) - - -class Link(object): - - def __init__(self, url, comes_from=None): - self.url = url - self.comes_from = comes_from - - def __str__(self): - if self.comes_from: - return '%s (from %s)' % (self.url, self.comes_from) - else: - return self.url - - def __repr__(self): - return '<Link %s>' % self - - def __eq__(self, other): - return self.url == other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - url = self.url - url = url.split('#', 1)[0] - url = url.split('?', 1)[0] - url = url.rstrip('/') - name = posixpath.basename(url) - assert name, ( - 'URL %r produced no filename' % url) - return name - - @property - def scheme(self): - return urlparse.urlsplit(self.url)[0] - - @property - def path(self): - return urlparse.urlsplit(self.url)[2] - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - _egg_fragment_re = re.compile(r'#egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _md5_re = re.compile(r'md5=([a-f0-9]+)') - - @property - def md5_hash(self): - match = self._md5_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - -def get_requirement_from_url(url): - """Get a requirement from the URL, if possible. This looks for #egg - in the URL""" - link = Link(url) - egg_info = link.egg_fragment - if not egg_info: - egg_info = splitext(link.filename)[0] - return package_to_requirement(egg_info) - - -def package_to_requirement(package_name): - """Translate a name like Foo-1.2 to Foo==1.3""" - match = re.search(r'^(.*?)(-dev|-\d.*)', package_name) - if match: - name = match.group(1) - version = match.group(2) - else: - name = package_name - version = '' - if version: - return '%s==%s' % (name, version) - else: - return name - - -def get_mirrors(hostname=None): - """Return the list of mirrors from the last record found on the DNS - entry:: - - >>> from pip.index import get_mirrors - >>> get_mirrors() - ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', - 'd.pypi.python.org'] - - Originally written for the distutils2 project by Alexis Metaireau. - """ - if hostname is None: - hostname = DEFAULT_MIRROR_URL - - # return the last mirror registered on PyPI. - try: - hostname = socket.gethostbyname_ex(hostname)[0] - except socket.gaierror: - return [] - end_letter = hostname.split(".", 1) - - # determine the list from the last one. - return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] - - -def string_range(last): - """Compute the range of string between "a" and last. - - This works for simple "a to z" lists, but also for "a to zz" lists. - """ - for k in range(len(last)): - for x in product(string.ascii_lowercase, repeat=k+1): - result = ''.join(x) - yield result - if result == last: - return - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py deleted file mode 100644 index 4254ef2fb..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Locations where we look for configs, install stuff, etc""" - -import sys -import os -from distutils import sysconfig - - -def running_under_virtualenv(): - """ - Return True if we're running inside a virtualenv, False otherwise. - - """ - return hasattr(sys, 'real_prefix') - - -if running_under_virtualenv(): - ## FIXME: is build/ a good name? - build_prefix = os.path.join(sys.prefix, 'build') - src_prefix = os.path.join(sys.prefix, 'src') -else: - ## FIXME: this isn't a very good default - build_prefix = os.path.join(os.getcwd(), 'build') - src_prefix = os.path.join(os.getcwd(), 'src') - -# FIXME doesn't account for venv linked to global site-packages - -site_packages = sysconfig.get_python_lib() -user_dir = os.path.expanduser('~') -if sys.platform == 'win32': - bin_py = os.path.join(sys.prefix, 'Scripts') - # buildout uses 'bin' on Windows too? - if not os.path.exists(bin_py): - bin_py = os.path.join(sys.prefix, 'bin') - user_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming - default_storage_dir = os.path.join(user_dir, 'pip') - default_config_file = os.path.join(default_storage_dir, 'pip.ini') - default_log_file = os.path.join(default_storage_dir, 'pip.log') -else: - bin_py = os.path.join(sys.prefix, 'bin') - default_storage_dir = os.path.join(user_dir, '.pip') - default_config_file = os.path.join(default_storage_dir, 'pip.conf') - default_log_file = os.path.join(default_storage_dir, 'pip.log') - # Forcing to use /usr/local/bin for standard Mac OS X framework installs - if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': - bin_py = '/usr/local/bin' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py deleted file mode 100644 index 0218ab1ae..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Logging -""" - -import sys -import logging - - -class Logger(object): - - """ - Logging object for use in command-line script. Allows ranges of - levels, to avoid some redundancy of displayed information. - """ - - VERBOSE_DEBUG = logging.DEBUG-1 - DEBUG = logging.DEBUG - INFO = logging.INFO - NOTIFY = (logging.INFO+logging.WARN)/2 - WARN = WARNING = logging.WARN - ERROR = logging.ERROR - FATAL = logging.FATAL - - LEVELS = [VERBOSE_DEBUG, DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL] - - def __init__(self): - self.consumers = [] - self.indent = 0 - self.explicit_levels = False - self.in_progress = None - self.in_progress_hanging = False - - def debug(self, msg, *args, **kw): - self.log(self.DEBUG, msg, *args, **kw) - - def info(self, msg, *args, **kw): - self.log(self.INFO, msg, *args, **kw) - - def notify(self, msg, *args, **kw): - self.log(self.NOTIFY, msg, *args, **kw) - - def warn(self, msg, *args, **kw): - self.log(self.WARN, msg, *args, **kw) - - def error(self, msg, *args, **kw): - self.log(self.WARN, msg, *args, **kw) - - def fatal(self, msg, *args, **kw): - self.log(self.FATAL, msg, *args, **kw) - - def log(self, level, msg, *args, **kw): - if args: - if kw: - raise TypeError( - "You may give positional or keyword arguments, not both") - args = args or kw - rendered = None - for consumer_level, consumer in self.consumers: - if self.level_matches(level, consumer_level): - if (self.in_progress_hanging - and consumer in (sys.stdout, sys.stderr)): - self.in_progress_hanging = False - sys.stdout.write('\n') - sys.stdout.flush() - if rendered is None: - if args: - rendered = msg % args - else: - rendered = msg - rendered = ' '*self.indent + rendered - if self.explicit_levels: - ## FIXME: should this be a name, not a level number? - rendered = '%02i %s' % (level, rendered) - if hasattr(consumer, 'write'): - consumer.write(rendered+'\n') - else: - consumer(rendered) - - def start_progress(self, msg): - assert not self.in_progress, ( - "Tried to start_progress(%r) while in_progress %r" - % (msg, self.in_progress)) - if self.level_matches(self.NOTIFY, self._stdout_level()): - sys.stdout.write(' '*self.indent + msg) - sys.stdout.flush() - self.in_progress_hanging = True - else: - self.in_progress_hanging = False - self.in_progress = msg - self.last_message = None - - def end_progress(self, msg='done.'): - assert self.in_progress, ( - "Tried to end_progress without start_progress") - if self.stdout_level_matches(self.NOTIFY): - if not self.in_progress_hanging: - # Some message has been printed out since start_progress - sys.stdout.write('...' + self.in_progress + msg + '\n') - sys.stdout.flush() - else: - # These erase any messages shown with show_progress (besides .'s) - logger.show_progress('') - logger.show_progress('') - sys.stdout.write(msg + '\n') - sys.stdout.flush() - self.in_progress = None - self.in_progress_hanging = False - - def show_progress(self, message=None): - """If we are in a progress scope, and no log messages have been - shown, write out another '.'""" - if self.in_progress_hanging: - if message is None: - sys.stdout.write('.') - sys.stdout.flush() - else: - if self.last_message: - padding = ' ' * max(0, len(self.last_message)-len(message)) - else: - padding = '' - sys.stdout.write('\r%s%s%s%s' % (' '*self.indent, self.in_progress, message, padding)) - sys.stdout.flush() - self.last_message = message - - def stdout_level_matches(self, level): - """Returns true if a message at this level will go to stdout""" - return self.level_matches(level, self._stdout_level()) - - def _stdout_level(self): - """Returns the level that stdout runs at""" - for level, consumer in self.consumers: - if consumer is sys.stdout: - return level - return self.FATAL - - def level_matches(self, level, consumer_level): - """ - >>> l = Logger() - >>> l.level_matches(3, 4) - False - >>> l.level_matches(3, 2) - True - >>> l.level_matches(slice(None, 3), 3) - False - >>> l.level_matches(slice(None, 3), 2) - True - >>> l.level_matches(slice(1, 3), 1) - True - >>> l.level_matches(slice(2, 3), 1) - False - """ - if isinstance(level, slice): - start, stop = level.start, level.stop - if start is not None and start > consumer_level: - return False - if stop is not None or stop <= consumer_level: - return False - return True - else: - return level >= consumer_level - - @classmethod - def level_for_integer(cls, level): - levels = cls.LEVELS - if level < 0: - return levels[0] - if level >= len(levels): - return levels[-1] - return levels[level] - - def move_stdout_to_stderr(self): - to_remove = [] - to_add = [] - for consumer_level, consumer in self.consumers: - if consumer == sys.stdout: - to_remove.append((consumer_level, consumer)) - to_add.append((consumer_level, sys.stderr)) - for item in to_remove: - self.consumers.remove(item) - self.consumers.extend(to_add) - -logger = Logger() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py deleted file mode 100644 index 444e7252b..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py +++ /dev/null @@ -1,1432 +0,0 @@ -import sys -import os -import shutil -import re -import zipfile -import pkg_resources -import tempfile -import urlparse -import urllib2 -import urllib -import ConfigParser -from distutils.sysconfig import get_python_version -from email.FeedParser import FeedParser -from pip.locations import bin_py, running_under_virtualenv -from pip.exceptions import InstallationError, UninstallationError -from pip.vcs import vcs -from pip.log import logger -from pip.util import display_path, rmtree -from pip.util import ask, backup_dir -from pip.util import is_installable_dir, is_local, dist_is_local -from pip.util import renames, normalize_path, egg_link_path -from pip.util import make_path_relative -from pip import call_subprocess -from pip.backwardcompat import any, copytree -from pip.index import Link -from pip.locations import build_prefix -from pip.download import (get_file_content, is_url, url_to_path, - path_to_url, is_archive_file, - unpack_vcs_link, is_vcs_url, is_file_url, - unpack_file_url, unpack_http_url) - - -PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' - - -class InstallRequirement(object): - - def __init__(self, req, comes_from, source_dir=None, editable=False, - url=None, update=True): - if isinstance(req, basestring): - req = pkg_resources.Requirement.parse(req) - self.req = req - self.comes_from = comes_from - self.source_dir = source_dir - self.editable = editable - self.url = url - self._egg_info_path = None - # This holds the pkg_resources.Distribution object if this requirement - # is already available: - self.satisfied_by = None - # This hold the pkg_resources.Distribution object if this requirement - # conflicts with another installed distribution: - self.conflicts_with = None - self._temp_build_dir = None - self._is_bundle = None - # True if the editable should be updated: - self.update = update - # Set to True after successful installation - self.install_succeeded = None - # UninstallPathSet of uninstalled distribution (for possible rollback) - self.uninstalled = None - - @classmethod - def from_editable(cls, editable_req, comes_from=None, default_vcs=None): - name, url = parse_editable(editable_req, default_vcs) - if url.startswith('file:'): - source_dir = url_to_path(url) - else: - source_dir = None - return cls(name, comes_from, source_dir=source_dir, editable=True, url=url) - - @classmethod - def from_line(cls, name, comes_from=None): - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. - """ - url = None - name = name.strip() - req = name - path = os.path.normpath(os.path.abspath(name)) - - if is_url(name): - url = name - ## FIXME: I think getting the requirement here is a bad idea: - #req = get_requirement_from_url(url) - req = None - elif os.path.isdir(path) and (os.path.sep in name or name.startswith('.')): - if not is_installable_dir(path): - raise InstallationError("Directory %r is not installable. File 'setup.py' not found." - % name) - url = path_to_url(name) - #req = get_requirement_from_url(url) - req = None - elif is_archive_file(path): - if not os.path.isfile(path): - logger.warn('Requirement %r looks like a filename, but the file does not exist' - % name) - url = path_to_url(name) - #req = get_requirement_from_url(url) - req = None - return cls(req, comes_from, url=url) - - def __str__(self): - if self.req: - s = str(self.req) - if self.url: - s += ' from %s' % self.url - else: - s = self.url - if self.satisfied_by is not None: - s += ' in %s' % display_path(self.satisfied_by.location) - if self.comes_from: - if isinstance(self.comes_from, basestring): - comes_from = self.comes_from - else: - comes_from = self.comes_from.from_path() - if comes_from: - s += ' (from %s)' % comes_from - return s - - def from_path(self): - if self.req is None: - return None - s = str(self.req) - if self.comes_from: - if isinstance(self.comes_from, basestring): - comes_from = self.comes_from - else: - comes_from = self.comes_from.from_path() - if comes_from: - s += '->' + comes_from - return s - - def build_location(self, build_dir, unpack=True): - if self._temp_build_dir is not None: - return self._temp_build_dir - if self.req is None: - self._temp_build_dir = tempfile.mkdtemp('-build', 'pip-') - self._ideal_build_dir = build_dir - return self._temp_build_dir - if self.editable: - name = self.name.lower() - else: - name = self.name - # FIXME: Is there a better place to create the build_dir? (hg and bzr need this) - if not os.path.exists(build_dir): - _make_build_dir(build_dir) - return os.path.join(build_dir, name) - - def correct_build_location(self): - """If the build location was a temporary directory, this will move it - to a new more permanent location""" - if self.source_dir is not None: - return - assert self.req is not None - assert self._temp_build_dir - old_location = self._temp_build_dir - new_build_dir = self._ideal_build_dir - del self._ideal_build_dir - if self.editable: - name = self.name.lower() - else: - name = self.name - new_location = os.path.join(new_build_dir, name) - if not os.path.exists(new_build_dir): - logger.debug('Creating directory %s' % new_build_dir) - _make_build_dir(new_build_dir) - if os.path.exists(new_location): - raise InstallationError( - 'A package already exists in %s; please remove it to continue' - % display_path(new_location)) - logger.debug('Moving package %s from %s to new location %s' - % (self, display_path(old_location), display_path(new_location))) - shutil.move(old_location, new_location) - self._temp_build_dir = new_location - self.source_dir = new_location - self._egg_info_path = None - - @property - def name(self): - if self.req is None: - return None - return self.req.project_name - - @property - def url_name(self): - if self.req is None: - return None - return urllib.quote(self.req.unsafe_name) - - @property - def setup_py(self): - return os.path.join(self.source_dir, 'setup.py') - - def run_egg_info(self, force_root_egg_info=False): - assert self.source_dir - if self.name: - logger.notify('Running setup.py egg_info for package %s' % self.name) - else: - logger.notify('Running setup.py egg_info for package from %s' % self.url) - logger.indent += 2 - try: - script = self._run_setup_py - script = script.replace('__SETUP_PY__', repr(self.setup_py)) - script = script.replace('__PKG_NAME__', repr(self.name)) - # We can't put the .egg-info files at the root, because then the source code will be mistaken - # for an installed egg, causing problems - if self.editable or force_root_egg_info: - egg_base_option = [] - else: - egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') - if not os.path.exists(egg_info_dir): - os.makedirs(egg_info_dir) - egg_base_option = ['--egg-base', 'pip-egg-info'] - call_subprocess( - [sys.executable, '-c', script, 'egg_info'] + egg_base_option, - cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False, - command_level=logger.VERBOSE_DEBUG, - command_desc='python setup.py egg_info') - finally: - logger.indent -= 2 - if not self.req: - self.req = pkg_resources.Requirement.parse(self.pkg_info()['Name']) - self.correct_build_location() - - ## FIXME: This is a lame hack, entirely for PasteScript which has - ## a self-provided entry point that causes this awkwardness - _run_setup_py = """ -__file__ = __SETUP_PY__ -from setuptools.command import egg_info -def replacement_run(self): - self.mkpath(self.egg_info) - installer = self.distribution.fetch_build_egg - for ep in egg_info.iter_entry_points('egg_info.writers'): - # require=False is the change we're making: - writer = ep.load(require=False) - if writer: - writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name)) - self.find_sources() -egg_info.egg_info.run = replacement_run -execfile(__file__) -""" - - def egg_info_data(self, filename): - if self.satisfied_by is not None: - if not self.satisfied_by.has_metadata(filename): - return None - return self.satisfied_by.get_metadata(filename) - assert self.source_dir - filename = self.egg_info_path(filename) - if not os.path.exists(filename): - return None - fp = open(filename, 'r') - data = fp.read() - fp.close() - return data - - def egg_info_path(self, filename): - if self._egg_info_path is None: - if self.editable: - base = self.source_dir - else: - base = os.path.join(self.source_dir, 'pip-egg-info') - filenames = os.listdir(base) - if self.editable: - filenames = [] - for root, dirs, files in os.walk(base): - for dir in vcs.dirnames: - if dir in dirs: - dirs.remove(dir) - for dir in dirs: - # Don't search in anything that looks like a virtualenv environment - if (os.path.exists(os.path.join(root, dir, 'bin', 'python')) - or os.path.exists(os.path.join(root, dir, 'Scripts', 'Python.exe'))): - dirs.remove(dir) - # Also don't search through tests - if dir == 'test' or dir == 'tests': - dirs.remove(dir) - filenames.extend([os.path.join(root, dir) - for dir in dirs]) - filenames = [f for f in filenames if f.endswith('.egg-info')] - - if not filenames: - raise InstallationError('No files/directores in %s (from %s)' % (base, filename)) - assert filenames, "No files/directories in %s (from %s)" % (base, filename) - - # if we have more than one match, we pick the toplevel one. This can - # easily be the case if there is a dist folder which contains an - # extracted tarball for testing purposes. - if len(filenames) > 1: - filenames.sort(key=lambda x: x.count(os.path.sep) + - (os.path.altsep and - x.count(os.path.altsep) or 0)) - self._egg_info_path = os.path.join(base, filenames[0]) - return os.path.join(self._egg_info_path, filename) - - def egg_info_lines(self, filename): - data = self.egg_info_data(filename) - if not data: - return [] - result = [] - for line in data.splitlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - result.append(line) - return result - - def pkg_info(self): - p = FeedParser() - data = self.egg_info_data('PKG-INFO') - if not data: - logger.warn('No PKG-INFO file found in %s' % display_path(self.egg_info_path('PKG-INFO'))) - p.feed(data or '') - return p.close() - - @property - def dependency_links(self): - return self.egg_info_lines('dependency_links.txt') - - _requirements_section_re = re.compile(r'\[(.*?)\]') - - def requirements(self, extras=()): - in_extra = None - for line in self.egg_info_lines('requires.txt'): - match = self._requirements_section_re.match(line) - if match: - in_extra = match.group(1) - continue - if in_extra and in_extra not in extras: - # Skip requirement for an extra we aren't requiring - continue - yield line - - @property - def absolute_versions(self): - for qualifier, version in self.req.specs: - if qualifier == '==': - yield version - - @property - def installed_version(self): - return self.pkg_info()['version'] - - def assert_source_matches_version(self): - assert self.source_dir - if self.comes_from is None: - # We don't check the versions of things explicitly installed. - # This makes, e.g., "pip Package==dev" possible - return - version = self.installed_version - if version not in self.req: - logger.fatal( - 'Source in %s has the version %s, which does not match the requirement %s' - % (display_path(self.source_dir), version, self)) - raise InstallationError( - 'Source in %s has version %s that conflicts with %s' - % (display_path(self.source_dir), version, self)) - else: - logger.debug('Source in %s has version %s, which satisfies requirement %s' - % (display_path(self.source_dir), version, self)) - - def update_editable(self, obtain=True): - if not self.url: - logger.info("Cannot update repository at %s; repository location is unknown" % self.source_dir) - return - assert self.editable - assert self.source_dir - if self.url.startswith('file:'): - # Static paths don't get updated - return - assert '+' in self.url, "bad url: %r" % self.url - if not self.update: - return - vc_type, url = self.url.split('+', 1) - backend = vcs.get_backend(vc_type) - if backend: - vcs_backend = backend(self.url) - if obtain: - vcs_backend.obtain(self.source_dir) - else: - vcs_backend.export(self.source_dir) - else: - assert 0, ( - 'Unexpected version control type (in %s): %s' - % (self.url, vc_type)) - - def uninstall(self, auto_confirm=False): - """ - Uninstall the distribution currently satisfying this requirement. - - Prompts before removing or modifying files unless - ``auto_confirm`` is True. - - Refuses to delete or modify files outside of ``sys.prefix`` - - thus uninstallation within a virtual environment can only - modify that virtual environment, even if the virtualenv is - linked to global site-packages. - - """ - if not self.check_if_exists(): - raise UninstallationError("Cannot uninstall requirement %s, not installed" % (self.name,)) - dist = self.satisfied_by or self.conflicts_with - - paths_to_remove = UninstallPathSet(dist) - - pip_egg_info_path = os.path.join(dist.location, - dist.egg_name()) + '.egg-info' - easy_install_egg = dist.egg_name() + '.egg' - develop_egg_link = egg_link_path(dist) - if os.path.exists(pip_egg_info_path): - # package installed by pip - paths_to_remove.add(pip_egg_info_path) - if dist.has_metadata('installed-files.txt'): - for installed_file in dist.get_metadata('installed-files.txt').splitlines(): - path = os.path.normpath(os.path.join(pip_egg_info_path, installed_file)) - paths_to_remove.add(path) - if dist.has_metadata('top_level.txt'): - if dist.has_metadata('namespace_packages.txt'): - namespaces = dist.get_metadata('namespace_packages.txt') - else: - namespaces = [] - for top_level_pkg in [p for p - in dist.get_metadata('top_level.txt').splitlines() - if p and p not in namespaces]: - path = os.path.join(dist.location, top_level_pkg) - paths_to_remove.add(path) - paths_to_remove.add(path + '.py') - paths_to_remove.add(path + '.pyc') - - elif dist.location.endswith(easy_install_egg): - # package installed by easy_install - paths_to_remove.add(dist.location) - easy_install_pth = os.path.join(os.path.dirname(dist.location), - 'easy-install.pth') - paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) - - elif os.path.isfile(develop_egg_link): - # develop egg - fh = open(develop_egg_link, 'r') - link_pointer = os.path.normcase(fh.readline().strip()) - fh.close() - assert (link_pointer == dist.location), 'Egg-link %s does not match installed location of %s (at %s)' % (link_pointer, self.name, dist.location) - paths_to_remove.add(develop_egg_link) - easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), - 'easy-install.pth') - paths_to_remove.add_pth(easy_install_pth, dist.location) - - # find distutils scripts= scripts - if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): - for script in dist.metadata_listdir('scripts'): - paths_to_remove.add(os.path.join(bin_py, script)) - if sys.platform == 'win32': - paths_to_remove.add(os.path.join(bin_py, script) + '.bat') - - # find console_scripts - if dist.has_metadata('entry_points.txt'): - config = ConfigParser.SafeConfigParser() - config.readfp(FakeFile(dist.get_metadata_lines('entry_points.txt'))) - if config.has_section('console_scripts'): - for name, value in config.items('console_scripts'): - paths_to_remove.add(os.path.join(bin_py, name)) - if sys.platform == 'win32': - paths_to_remove.add(os.path.join(bin_py, name) + '.exe') - paths_to_remove.add(os.path.join(bin_py, name) + '.exe.manifest') - paths_to_remove.add(os.path.join(bin_py, name) + '-script.py') - - paths_to_remove.remove(auto_confirm) - self.uninstalled = paths_to_remove - - def rollback_uninstall(self): - if self.uninstalled: - self.uninstalled.rollback() - else: - logger.error("Can't rollback %s, nothing uninstalled." - % (self.project_name,)) - - def commit_uninstall(self): - if self.uninstalled: - self.uninstalled.commit() - else: - logger.error("Can't commit %s, nothing uninstalled." - % (self.project_name,)) - - def archive(self, build_dir): - assert self.source_dir - create_archive = True - archive_name = '%s-%s.zip' % (self.name, self.installed_version) - archive_path = os.path.join(build_dir, archive_name) - if os.path.exists(archive_path): - response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' - % display_path(archive_path), ('i', 'w', 'b')) - if response == 'i': - create_archive = False - elif response == 'w': - logger.warn('Deleting %s' % display_path(archive_path)) - os.remove(archive_path) - elif response == 'b': - dest_file = backup_dir(archive_path) - logger.warn('Backing up %s to %s' - % (display_path(archive_path), display_path(dest_file))) - shutil.move(archive_path, dest_file) - if create_archive: - zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) - dir = os.path.normcase(os.path.abspath(self.source_dir)) - for dirpath, dirnames, filenames in os.walk(dir): - if 'pip-egg-info' in dirnames: - dirnames.remove('pip-egg-info') - for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') - zipdir.external_attr = 0755 << 16L - zip.writestr(zipdir, '') - for filename in filenames: - if filename == PIP_DELETE_MARKER_FILENAME: - continue - filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, self.name + '/' + name) - zip.close() - logger.indent -= 2 - logger.notify('Saved %s' % display_path(archive_path)) - - def _clean_zip_name(self, name, prefix): - assert name.startswith(prefix+os.path.sep), ( - "name %r doesn't start with prefix %r" % (name, prefix)) - name = name[len(prefix)+1:] - name = name.replace(os.path.sep, '/') - return name - - def install(self, install_options, global_options=()): - if self.editable: - self.install_editable(install_options, global_options) - return - temp_location = tempfile.mkdtemp('-record', 'pip-') - record_filename = os.path.join(temp_location, 'install-record.txt') - try: - - install_args = [ - sys.executable, '-c', - "import setuptools;__file__=%r;"\ - "execfile(__file__)" % self.setup_py] +\ - list(global_options) + [ - 'install', - '--single-version-externally-managed', - '--record', record_filename] - - if running_under_virtualenv(): - ## FIXME: I'm not sure if this is a reasonable location; probably not - ## but we can't put it in the default location, as that is a virtualenv symlink that isn't writable - install_args += ['--install-headers', - os.path.join(sys.prefix, 'include', 'site', - 'python' + get_python_version())] - logger.notify('Running setup.py install for %s' % self.name) - logger.indent += 2 - try: - call_subprocess(install_args + install_options, - cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False) - finally: - logger.indent -= 2 - if not os.path.exists(record_filename): - logger.notify('Record file %s not found' % record_filename) - return - self.install_succeeded = True - f = open(record_filename) - for line in f: - line = line.strip() - if line.endswith('.egg-info'): - egg_info_dir = line - break - else: - logger.warn('Could not find .egg-info directory in install record for %s' % self) - ## FIXME: put the record somewhere - ## FIXME: should this be an error? - return - f.close() - new_lines = [] - f = open(record_filename) - for line in f: - filename = line.strip() - if os.path.isdir(filename): - filename += os.path.sep - new_lines.append(make_path_relative(filename, egg_info_dir)) - f.close() - f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') - f.write('\n'.join(new_lines)+'\n') - f.close() - finally: - if os.path.exists(record_filename): - os.remove(record_filename) - os.rmdir(temp_location) - - def remove_temporary_source(self): - """Remove the source files from this requirement, if they are marked - for deletion""" - if self.is_bundle or os.path.exists(self.delete_marker_filename): - logger.info('Removing source in %s' % self.source_dir) - if self.source_dir: - rmtree(self.source_dir) - self.source_dir = None - if self._temp_build_dir and os.path.exists(self._temp_build_dir): - rmtree(self._temp_build_dir) - self._temp_build_dir = None - - def install_editable(self, install_options, global_options=()): - logger.notify('Running setup.py develop for %s' % self.name) - logger.indent += 2 - try: - ## FIXME: should we do --install-headers here too? - call_subprocess( - [sys.executable, '-c', - "import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py)] - + list(global_options) + ['develop', '--no-deps'] + list(install_options), - - cwd=self.source_dir, filter_stdout=self._filter_install, - show_stdout=False) - finally: - logger.indent -= 2 - self.install_succeeded = True - - def _filter_install(self, line): - level = logger.NOTIFY - for regex in [r'^running .*', r'^writing .*', '^creating .*', '^[Cc]opying .*', - r'^reading .*', r"^removing .*\.egg-info' \(and everything under it\)$", - r'^byte-compiling ', - # Not sure what this warning is, but it seems harmless: - r"^warning: manifest_maker: standard file '-c' not found$"]: - if re.search(regex, line.strip()): - level = logger.INFO - break - return (level, line) - - def check_if_exists(self): - """Find an installed distribution that satisfies or conflicts - with this requirement, and set self.satisfied_by or - self.conflicts_with appropriately.""" - if self.req is None: - return False - try: - self.satisfied_by = pkg_resources.get_distribution(self.req) - except pkg_resources.DistributionNotFound: - return False - except pkg_resources.VersionConflict: - self.conflicts_with = pkg_resources.get_distribution(self.req.project_name) - return True - - @property - def is_bundle(self): - if self._is_bundle is not None: - return self._is_bundle - base = self._temp_build_dir - if not base: - ## FIXME: this doesn't seem right: - return False - self._is_bundle = (os.path.exists(os.path.join(base, 'pip-manifest.txt')) - or os.path.exists(os.path.join(base, 'pyinstall-manifest.txt'))) - return self._is_bundle - - def bundle_requirements(self): - for dest_dir in self._bundle_editable_dirs: - package = os.path.basename(dest_dir) - ## FIXME: svnism: - for vcs_backend in vcs.backends: - url = rev = None - vcs_bundle_file = os.path.join( - dest_dir, vcs_backend.bundle_file) - if os.path.exists(vcs_bundle_file): - vc_type = vcs_backend.name - fp = open(vcs_bundle_file) - content = fp.read() - fp.close() - url, rev = vcs_backend().parse_vcs_bundle_file(content) - break - if url: - url = '%s+%s@%s' % (vc_type, url, rev) - else: - url = None - yield InstallRequirement( - package, self, editable=True, url=url, - update=False, source_dir=dest_dir) - for dest_dir in self._bundle_build_dirs: - package = os.path.basename(dest_dir) - yield InstallRequirement( - package, self, - source_dir=dest_dir) - - def move_bundle_files(self, dest_build_dir, dest_src_dir): - base = self._temp_build_dir - assert base - src_dir = os.path.join(base, 'src') - build_dir = os.path.join(base, 'build') - bundle_build_dirs = [] - bundle_editable_dirs = [] - for source_dir, dest_dir, dir_collection in [ - (src_dir, dest_src_dir, bundle_editable_dirs), - (build_dir, dest_build_dir, bundle_build_dirs)]: - if os.path.exists(source_dir): - for dirname in os.listdir(source_dir): - dest = os.path.join(dest_dir, dirname) - dir_collection.append(dest) - if os.path.exists(dest): - logger.warn('The directory %s (containing package %s) already exists; cannot move source from bundle %s' - % (dest, dirname, self)) - continue - if not os.path.exists(dest_dir): - logger.info('Creating directory %s' % dest_dir) - os.makedirs(dest_dir) - shutil.move(os.path.join(source_dir, dirname), dest) - if not os.listdir(source_dir): - os.rmdir(source_dir) - self._temp_build_dir = None - self._bundle_build_dirs = bundle_build_dirs - self._bundle_editable_dirs = bundle_editable_dirs - - @property - def delete_marker_filename(self): - assert self.source_dir - return os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME) - - -DELETE_MARKER_MESSAGE = '''\ -This file is placed here by pip to indicate the source was put -here by pip. - -Once this package is successfully installed this source code will be -deleted (unless you remove this file). -''' - - -class RequirementSet(object): - - def __init__(self, build_dir, src_dir, download_dir, download_cache=None, - upgrade=False, ignore_installed=False, - ignore_dependencies=False): - self.build_dir = build_dir - self.src_dir = src_dir - self.download_dir = download_dir - self.download_cache = download_cache - self.upgrade = upgrade - self.ignore_installed = ignore_installed - self.requirements = {} - # Mapping of alias: real_name - self.requirement_aliases = {} - self.unnamed_requirements = [] - self.ignore_dependencies = ignore_dependencies - self.successfully_downloaded = [] - self.successfully_installed = [] - self.reqs_to_cleanup = [] - - def __str__(self): - reqs = [req for req in self.requirements.values() - if not req.comes_from] - reqs.sort(key=lambda req: req.name.lower()) - return ' '.join([str(req.req) for req in reqs]) - - def add_requirement(self, install_req): - name = install_req.name - if not name: - self.unnamed_requirements.append(install_req) - else: - if self.has_requirement(name): - raise InstallationError( - 'Double requirement given: %s (aready in %s, name=%r)' - % (install_req, self.get_requirement(name), name)) - self.requirements[name] = install_req - ## FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name - - def has_requirement(self, project_name): - for name in project_name, project_name.lower(): - if name in self.requirements or name in self.requirement_aliases: - return True - return False - - @property - def has_requirements(self): - return self.requirements.values() or self.unnamed_requirements - - @property - def has_editables(self): - if any(req.editable for req in self.requirements.values()): - return True - if any(req.editable for req in self.unnamed_requirements): - return True - return False - - @property - def is_download(self): - if self.download_dir: - self.download_dir = os.path.expanduser(self.download_dir) - if os.path.exists(self.download_dir): - return True - else: - logger.fatal('Could not find download directory') - raise InstallationError( - "Could not find or access download directory '%s'" - % display_path(self.download_dir)) - return False - - def get_requirement(self, project_name): - for name in project_name, project_name.lower(): - if name in self.requirements: - return self.requirements[name] - if name in self.requirement_aliases: - return self.requirements[self.requirement_aliases[name]] - raise KeyError("No project with the name %r" % project_name) - - def uninstall(self, auto_confirm=False): - for req in self.requirements.values(): - req.uninstall(auto_confirm=auto_confirm) - req.commit_uninstall() - - def locate_files(self): - ## FIXME: duplicates code from install_files; relevant code should - ## probably be factored out into a separate method - unnamed = list(self.unnamed_requirements) - reqs = self.requirements.values() - while reqs or unnamed: - if unnamed: - req_to_install = unnamed.pop(0) - else: - req_to_install = reqs.pop(0) - install_needed = True - if not self.ignore_installed and not req_to_install.editable: - req_to_install.check_if_exists() - if req_to_install.satisfied_by: - if self.upgrade: - req_to_install.conflicts_with = req_to_install.satisfied_by - req_to_install.satisfied_by = None - else: - install_needed = False - if req_to_install.satisfied_by: - logger.notify('Requirement already satisfied ' - '(use --upgrade to upgrade): %s' - % req_to_install) - - if req_to_install.editable: - if req_to_install.source_dir is None: - req_to_install.source_dir = req_to_install.build_location(self.src_dir) - elif install_needed: - req_to_install.source_dir = req_to_install.build_location(self.build_dir, not self.is_download) - - if req_to_install.source_dir is not None and not os.path.isdir(req_to_install.source_dir): - raise InstallationError('Could not install requirement %s ' - 'because source folder %s does not exist ' - '(perhaps --no-download was used without first running ' - 'an equivalent install with --no-install?)' - % (req_to_install, req_to_install.source_dir)) - - def prepare_files(self, finder, force_root_egg_info=False, bundle=False): - """Prepare process. Create temp directories, download and/or unpack files.""" - unnamed = list(self.unnamed_requirements) - reqs = self.requirements.values() - while reqs or unnamed: - if unnamed: - req_to_install = unnamed.pop(0) - else: - req_to_install = reqs.pop(0) - install = True - if not self.ignore_installed and not req_to_install.editable: - req_to_install.check_if_exists() - if req_to_install.satisfied_by: - if self.upgrade: - req_to_install.conflicts_with = req_to_install.satisfied_by - req_to_install.satisfied_by = None - else: - install = False - if req_to_install.satisfied_by: - logger.notify('Requirement already satisfied ' - '(use --upgrade to upgrade): %s' - % req_to_install) - if req_to_install.editable: - logger.notify('Obtaining %s' % req_to_install) - elif install: - if req_to_install.url and req_to_install.url.lower().startswith('file:'): - logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) - else: - logger.notify('Downloading/unpacking %s' % req_to_install) - logger.indent += 2 - try: - is_bundle = False - if req_to_install.editable: - if req_to_install.source_dir is None: - location = req_to_install.build_location(self.src_dir) - req_to_install.source_dir = location - else: - location = req_to_install.source_dir - if not os.path.exists(self.build_dir): - _make_build_dir(self.build_dir) - req_to_install.update_editable(not self.is_download) - if self.is_download: - req_to_install.run_egg_info() - req_to_install.archive(self.download_dir) - else: - req_to_install.run_egg_info() - elif install: - ##@@ if filesystem packages are not marked - ##editable in a req, a non deterministic error - ##occurs when the script attempts to unpack the - ##build directory - - location = req_to_install.build_location(self.build_dir, not self.is_download) - ## FIXME: is the existance of the checkout good enough to use it? I don't think so. - unpack = True - if not os.path.exists(os.path.join(location, 'setup.py')): - ## FIXME: this won't upgrade when there's an existing package unpacked in `location` - if req_to_install.url is None: - url = finder.find_requirement(req_to_install, upgrade=self.upgrade) - else: - ## FIXME: should req_to_install.url already be a link? - url = Link(req_to_install.url) - assert url - if url: - try: - self.unpack_url(url, location, self.is_download) - except urllib2.HTTPError, e: - logger.fatal('Could not install requirement %s because of error %s' - % (req_to_install, e)) - raise InstallationError( - 'Could not install requirement %s because of HTTP error %s for URL %s' - % (req_to_install, e, url)) - else: - unpack = False - if unpack: - is_bundle = req_to_install.is_bundle - url = None - if is_bundle: - req_to_install.move_bundle_files(self.build_dir, self.src_dir) - for subreq in req_to_install.bundle_requirements(): - reqs.append(subreq) - self.add_requirement(subreq) - elif self.is_download: - req_to_install.source_dir = location - if url and url.scheme in vcs.all_schemes: - req_to_install.run_egg_info() - req_to_install.archive(self.download_dir) - else: - req_to_install.source_dir = location - req_to_install.run_egg_info() - if force_root_egg_info: - # We need to run this to make sure that the .egg-info/ - # directory is created for packing in the bundle - req_to_install.run_egg_info(force_root_egg_info=True) - req_to_install.assert_source_matches_version() - #@@ sketchy way of identifying packages not grabbed from an index - if bundle and req_to_install.url: - self.copy_to_build_dir(req_to_install) - if not is_bundle and not self.is_download: - ## FIXME: shouldn't be globally added: - finder.add_dependency_links(req_to_install.dependency_links) - ## FIXME: add extras in here: - if not self.ignore_dependencies: - for req in req_to_install.requirements(): - try: - name = pkg_resources.Requirement.parse(req).project_name - except ValueError, e: - ## FIXME: proper warning - logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) - continue - if self.has_requirement(name): - ## FIXME: check for conflict - continue - subreq = InstallRequirement(req, req_to_install) - reqs.append(subreq) - self.add_requirement(subreq) - if req_to_install.name not in self.requirements: - self.requirements[req_to_install.name] = req_to_install - else: - self.reqs_to_cleanup.append(req_to_install) - if install: - self.successfully_downloaded.append(req_to_install) - if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): - self.copy_to_build_dir(req_to_install) - finally: - logger.indent -= 2 - - def cleanup_files(self, bundle=False): - """Clean up files, remove builds.""" - logger.notify('Cleaning up...') - logger.indent += 2 - for req in self.reqs_to_cleanup: - req.remove_temporary_source() - - remove_dir = [] - if self._pip_has_created_build_dir(): - remove_dir.append(self.build_dir) - - # The source dir of a bundle can always be removed. - if bundle: - remove_dir.append(self.src_dir) - - for dir in remove_dir: - if os.path.exists(dir): - logger.info('Removing temporary dir %s...' % dir) - rmtree(dir) - - logger.indent -= 2 - - def _pip_has_created_build_dir(self): - return (self.build_dir == build_prefix and - os.path.exists(os.path.join(self.build_dir, PIP_DELETE_MARKER_FILENAME))) - - def copy_to_build_dir(self, req_to_install): - target_dir = req_to_install.editable and self.src_dir or self.build_dir - logger.info("Copying %s to %s" %(req_to_install.name, target_dir)) - dest = os.path.join(target_dir, req_to_install.name) - copytree(req_to_install.source_dir, dest) - call_subprocess(["python", "%s/setup.py"%dest, "clean"]) - - def unpack_url(self, link, location, only_download=False): - if only_download: - location = self.download_dir - if is_vcs_url(link): - return unpack_vcs_link(link, location, only_download) - elif is_file_url(link): - return unpack_file_url(link, location) - else: - if self.download_cache: - self.download_cache = os.path.expanduser(self.download_cache) - return unpack_http_url(link, location, self.download_cache, only_download) - - def install(self, install_options, global_options=()): - """Install everything in this set (after having downloaded and unpacked the packages)""" - to_install = sorted([r for r in self.requirements.values() - if self.upgrade or not r.satisfied_by], - key=lambda p: p.name.lower()) - if to_install: - logger.notify('Installing collected packages: %s' % (', '.join([req.name for req in to_install]))) - logger.indent += 2 - try: - for requirement in to_install: - if requirement.conflicts_with: - logger.notify('Found existing installation: %s' - % requirement.conflicts_with) - logger.indent += 2 - try: - requirement.uninstall(auto_confirm=True) - finally: - logger.indent -= 2 - try: - requirement.install(install_options, global_options) - except: - # if install did not succeed, rollback previous uninstall - if requirement.conflicts_with and not requirement.install_succeeded: - requirement.rollback_uninstall() - raise - else: - if requirement.conflicts_with and requirement.install_succeeded: - requirement.commit_uninstall() - requirement.remove_temporary_source() - finally: - logger.indent -= 2 - self.successfully_installed = to_install - - def create_bundle(self, bundle_filename): - ## FIXME: can't decide which is better; zip is easier to read - ## random files from, but tar.bz2 is smaller and not as lame a - ## format. - - ## FIXME: this file should really include a manifest of the - ## packages, maybe some other metadata files. It would make - ## it easier to detect as well. - zip = zipfile.ZipFile(bundle_filename, 'w', zipfile.ZIP_DEFLATED) - vcs_dirs = [] - for dir, basename in (self.build_dir, 'build'), (self.src_dir, 'src'): - dir = os.path.normcase(os.path.abspath(dir)) - for dirpath, dirnames, filenames in os.walk(dir): - for backend in vcs.backends: - vcs_backend = backend() - vcs_url = vcs_rev = None - if vcs_backend.dirname in dirnames: - for vcs_dir in vcs_dirs: - if dirpath.startswith(vcs_dir): - # vcs bundle file already in parent directory - break - else: - vcs_url, vcs_rev = vcs_backend.get_info( - os.path.join(dir, dirpath)) - vcs_dirs.append(dirpath) - vcs_bundle_file = vcs_backend.bundle_file - vcs_guide = vcs_backend.guide % {'url': vcs_url, - 'rev': vcs_rev} - dirnames.remove(vcs_backend.dirname) - break - if 'pip-egg-info' in dirnames: - dirnames.remove('pip-egg-info') - for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zip.writestr(basename + '/' + name + '/', '') - for filename in filenames: - if filename == PIP_DELETE_MARKER_FILENAME: - continue - filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, basename + '/' + name) - if vcs_url: - name = os.path.join(dirpath, vcs_bundle_file) - name = self._clean_zip_name(name, dir) - zip.writestr(basename + '/' + name, vcs_guide) - - zip.writestr('pip-manifest.txt', self.bundle_requirements()) - zip.close() - - BUNDLE_HEADER = '''\ -# This is a pip bundle file, that contains many source packages -# that can be installed as a group. You can install this like: -# pip this_file.zip -# The rest of the file contains a list of all the packages included: -''' - - def bundle_requirements(self): - parts = [self.BUNDLE_HEADER] - for req in sorted( - [req for req in self.requirements.values() - if not req.comes_from], - key=lambda x: x.name): - parts.append('%s==%s\n' % (req.name, req.installed_version)) - parts.append('# These packages were installed to satisfy the above requirements:\n') - for req in sorted( - [req for req in self.requirements.values() - if req.comes_from], - key=lambda x: x.name): - parts.append('%s==%s\n' % (req.name, req.installed_version)) - ## FIXME: should we do something with self.unnamed_requirements? - return ''.join(parts) - - def _clean_zip_name(self, name, prefix): - assert name.startswith(prefix+os.path.sep), ( - "name %r doesn't start with prefix %r" % (name, prefix)) - name = name[len(prefix)+1:] - name = name.replace(os.path.sep, '/') - return name - - -def _make_build_dir(build_dir): - os.makedirs(build_dir) - _write_delete_marker_message(os.path.join(build_dir, PIP_DELETE_MARKER_FILENAME)) - - -def _write_delete_marker_message(filepath): - marker_fp = open(filepath, 'w') - marker_fp.write(DELETE_MARKER_MESSAGE) - marker_fp.close() - - -_scheme_re = re.compile(r'^(http|https|file):', re.I) - - -def parse_requirements(filename, finder=None, comes_from=None, options=None): - skip_match = None - skip_regex = options.skip_requirements_regex - if skip_regex: - skip_match = re.compile(skip_regex) - filename, content = get_file_content(filename, comes_from=comes_from) - for line_number, line in enumerate(content.splitlines()): - line_number += 1 - line = line.strip() - if not line or line.startswith('#'): - continue - if skip_match and skip_match.search(line): - continue - if line.startswith('-r') or line.startswith('--requirement'): - if line.startswith('-r'): - req_url = line[2:].strip() - else: - req_url = line[len('--requirement'):].strip().strip('=') - if _scheme_re.search(filename): - # Relative to a URL - req_url = urlparse.urljoin(req_url, filename) - elif not _scheme_re.search(req_url): - req_url = os.path.join(os.path.dirname(filename), req_url) - for item in parse_requirements(req_url, finder, comes_from=filename, options=options): - yield item - elif line.startswith('-Z') or line.startswith('--always-unzip'): - # No longer used, but previously these were used in - # requirement files, so we'll ignore. - pass - elif line.startswith('-f') or line.startswith('--find-links'): - if line.startswith('-f'): - line = line[2:].strip() - else: - line = line[len('--find-links'):].strip().lstrip('=') - ## FIXME: it would be nice to keep track of the source of - ## the find_links: - if finder: - finder.find_links.append(line) - elif line.startswith('-i') or line.startswith('--index-url'): - if line.startswith('-i'): - line = line[2:].strip() - else: - line = line[len('--index-url'):].strip().lstrip('=') - if finder: - finder.index_urls = [line] - elif line.startswith('--extra-index-url'): - line = line[len('--extra-index-url'):].strip().lstrip('=') - if finder: - finder.index_urls.append(line) - else: - comes_from = '-r %s (line %s)' % (filename, line_number) - if line.startswith('-e') or line.startswith('--editable'): - if line.startswith('-e'): - line = line[2:].strip() - else: - line = line[len('--editable'):].strip() - req = InstallRequirement.from_editable( - line, comes_from=comes_from, default_vcs=options.default_vcs) - else: - req = InstallRequirement.from_line(line, comes_from) - yield req - - -def parse_editable(editable_req, default_vcs=None): - """Parses svn+http://blahblah@rev#egg=Foobar into a requirement - (Foobar) and a URL""" - url = editable_req - if os.path.isdir(url) and os.path.exists(os.path.join(url, 'setup.py')): - # Treating it as code that has already been checked out - url = path_to_url(url) - if url.lower().startswith('file:'): - return None, url - for version_control in vcs: - if url.lower().startswith('%s:' % version_control): - url = '%s+%s' % (version_control, url) - if '+' not in url: - if default_vcs: - url = default_vcs + '+' + url - else: - raise InstallationError( - '--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req) - vc_type = url.split('+', 1)[0].lower() - if not vcs.get_backend(vc_type): - raise InstallationError( - 'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req) - match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req) - if (not match or not match.group(1)) and vcs.get_backend(vc_type): - parts = [p for p in editable_req.split('#', 1)[0].split('/') if p] - if parts[-2] in ('tags', 'branches', 'tag', 'branch'): - req = parts[-3] - elif parts[-1] == 'trunk': - req = parts[-2] - else: - raise InstallationError( - '--editable=%s is not the right format; it must have #egg=Package' - % editable_req) - else: - req = match.group(1) - ## FIXME: use package_to_requirement? - match = re.search(r'^(.*?)(?:-dev|-\d.*)', req) - if match: - # Strip off -dev, -0.2, etc. - req = match.group(1) - return req, url - - -class UninstallPathSet(object): - """A set of file paths to be removed in the uninstallation of a - requirement.""" - def __init__(self, dist): - self.paths = set() - self._refuse = set() - self.pth = {} - self.dist = dist - self.save_dir = None - self._moved_paths = [] - - def _permitted(self, path): - """ - Return True if the given path is one we are permitted to - remove/modify, False otherwise. - - """ - return is_local(path) - - def _can_uninstall(self): - if not dist_is_local(self.dist): - logger.notify("Not uninstalling %s at %s, outside environment %s" - % (self.dist.project_name, normalize_path(self.dist.location), sys.prefix)) - return False - return True - - def add(self, path): - path = normalize_path(path) - if not os.path.exists(path): - return - if self._permitted(path): - self.paths.add(path) - else: - self._refuse.add(path) - - def add_pth(self, pth_file, entry): - pth_file = normalize_path(pth_file) - if self._permitted(pth_file): - if pth_file not in self.pth: - self.pth[pth_file] = UninstallPthEntries(pth_file) - self.pth[pth_file].add(entry) - else: - self._refuse.add(pth_file) - - def compact(self, paths): - """Compact a path set to contain the minimal number of paths - necessary to contain all paths in the set. If /a/path/ and - /a/path/to/a/file.txt are both in the set, leave only the - shorter path.""" - short_paths = set() - for path in sorted(paths, key=len): - if not any([(path.startswith(shortpath) and - path[len(shortpath.rstrip(os.path.sep))] == os.path.sep) - for shortpath in short_paths]): - short_paths.add(path) - return short_paths - - def _stash(self, path): - return os.path.join( - self.save_dir, os.path.splitdrive(path)[1].lstrip(os.path.sep)) - - def remove(self, auto_confirm=False): - """Remove paths in ``self.paths`` with confirmation (unless - ``auto_confirm`` is True).""" - if not self._can_uninstall(): - return - logger.notify('Uninstalling %s:' % self.dist.project_name) - logger.indent += 2 - paths = sorted(self.compact(self.paths)) - try: - if auto_confirm: - response = 'y' - else: - for path in paths: - logger.notify(path) - response = ask('Proceed (y/n)? ', ('y', 'n')) - if self._refuse: - logger.notify('Not removing or modifying (outside of prefix):') - for path in self.compact(self._refuse): - logger.notify(path) - if response == 'y': - self.save_dir = tempfile.mkdtemp(suffix='-uninstall', - prefix='pip-') - for path in paths: - new_path = self._stash(path) - logger.info('Removing file or directory %s' % path) - self._moved_paths.append(path) - renames(path, new_path) - for pth in self.pth.values(): - pth.remove() - logger.notify('Successfully uninstalled %s' % self.dist.project_name) - - finally: - logger.indent -= 2 - - def rollback(self): - """Rollback the changes previously made by remove().""" - if self.save_dir is None: - logger.error("Can't roll back %s; was not uninstalled" % self.dist.project_name) - return False - logger.notify('Rolling back uninstall of %s' % self.dist.project_name) - for path in self._moved_paths: - tmp_path = self._stash(path) - logger.info('Replacing %s' % path) - renames(tmp_path, path) - for pth in self.pth: - pth.rollback() - - def commit(self): - """Remove temporary save dir: rollback will no longer be possible.""" - if self.save_dir is not None: - shutil.rmtree(self.save_dir) - self.save_dir = None - self._moved_paths = [] - - -class UninstallPthEntries(object): - def __init__(self, pth_file): - if not os.path.isfile(pth_file): - raise UninstallationError("Cannot remove entries from nonexistent file %s" % pth_file) - self.file = pth_file - self.entries = set() - self._saved_lines = None - - def add(self, entry): - entry = os.path.normcase(entry) - # On Windows, os.path.normcase converts the entry to use - # backslashes. This is correct for entries that describe absolute - # paths outside of site-packages, but all the others use forward - # slashes. - if sys.platform == 'win32' and not os.path.splitdrive(entry)[0]: - entry = entry.replace('\\', '/') - self.entries.add(entry) - - def remove(self): - logger.info('Removing pth entries from %s:' % self.file) - fh = open(self.file, 'r') - lines = fh.readlines() - self._saved_lines = lines - fh.close() - try: - for entry in self.entries: - logger.info('Removing entry: %s' % entry) - try: - lines.remove(entry + '\n') - except ValueError: - pass - finally: - pass - fh = open(self.file, 'wb') - fh.writelines(lines) - fh.close() - - def rollback(self): - if self._saved_lines is None: - logger.error('Cannot roll back changes to %s, none were made' % self.file) - return False - logger.info('Rolling %s back to previous state' % self.file) - fh = open(self.file, 'wb') - fh.writelines(self._saved_lines) - fh.close() - return True - - -class FakeFile(object): - """Wrap a list of lines in an object with readline() to make - ConfigParser happy.""" - def __init__(self, lines): - self._gen = (l for l in lines) - - def readline(self): - try: - return self._gen.next() - except StopIteration: - return '' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py deleted file mode 100644 index be830ad9a..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py +++ /dev/null @@ -1,18 +0,0 @@ -import sys -import os - - -def run(): - base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ## FIXME: this is kind of crude; if we could create a fake pip - ## module, then exec into it and update pip.__path__ properly, we - ## wouldn't have to update sys.path: - sys.path.insert(0, base) - import pip - return pip.main() - - -if __name__ == '__main__': - exit = run() - if exit: - sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py deleted file mode 100644 index 1eab34c06..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py +++ /dev/null @@ -1,479 +0,0 @@ -import sys -import shutil -import os -import stat -import re -import posixpath -import pkg_resources -import zipfile -import tarfile -from pip.exceptions import InstallationError -from pip.backwardcompat import WindowsError -from pip.locations import site_packages, running_under_virtualenv -from pip.log import logger - -__all__ = ['rmtree', 'display_path', 'backup_dir', - 'find_command', 'ask', 'Inf', - 'normalize_name', 'splitext', - 'format_size', 'is_installable_dir', - 'is_svn_page', 'file_contents', - 'split_leading_dir', 'has_leading_dir', - 'make_path_relative', 'normalize_path', - 'renames', 'get_terminal_size', - 'unzip_file', 'untar_file', 'create_download_cache_folder', - 'cache_download', 'unpack_file'] - - -def rmtree(dir): - shutil.rmtree(dir, ignore_errors=True, - onerror=rmtree_errorhandler) - - -def rmtree_errorhandler(func, path, exc_info): - """On Windows, the files in .svn are read-only, so when rmtree() tries to - remove them, an exception is thrown. We catch that here, remove the - read-only attribute, and hopefully continue without problems.""" - exctype, value = exc_info[:2] - # lookin for a windows error - if exctype is not WindowsError or 'Access is denied' not in str(value): - raise - # file type should currently be read only - if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD): - raise - # convert to read/write - os.chmod(path, stat.S_IWRITE) - # use the original function to repeat the operation - func(path) - - -def display_path(path): - """Gives the display value for a given path, making it relative to cwd - if possible.""" - path = os.path.normcase(os.path.abspath(path)) - if path.startswith(os.getcwd() + os.path.sep): - path = '.' + path[len(os.getcwd()):] - return path - - -def backup_dir(dir, ext='.bak'): - """Figure out the name of a directory to back up the given dir to - (adding .bak, .bak2, etc)""" - n = 1 - extension = ext - while os.path.exists(dir + extension): - n += 1 - extension = ext + str(n) - return dir + extension - - -def find_command(cmd, paths=None, pathext=None): - """Searches the PATH for the given command and returns its path""" - if paths is None: - paths = os.environ.get('PATH', []).split(os.pathsep) - if isinstance(paths, basestring): - paths = [paths] - # check if there are funny path extensions for executables, e.g. Windows - if pathext is None: - pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD') - pathext = [ext for ext in pathext.lower().split(os.pathsep)] - # don't use extensions if the command ends with one of them - if os.path.splitext(cmd)[1].lower() in pathext: - pathext = [''] - # check if we find the command on PATH - for path in paths: - # try without extension first - cmd_path = os.path.join(path, cmd) - for ext in pathext: - # then including the extension - cmd_path_ext = cmd_path + ext - if os.path.exists(cmd_path_ext): - return cmd_path_ext - if os.path.exists(cmd_path): - return cmd_path - return None - - -def ask(message, options): - """Ask the message interactively, with the given possible responses""" - while 1: - if os.environ.get('PIP_NO_INPUT'): - raise Exception('No input was expected ($PIP_NO_INPUT set); question: %s' % message) - response = raw_input(message) - response = response.strip().lower() - if response not in options: - print 'Your response (%r) was not one of the expected responses: %s' % ( - response, ', '.join(options)) - else: - return response - - -class _Inf(object): - """I am bigger than everything!""" - def __cmp__(self, a): - if self is a: - return 0 - return 1 - - def __repr__(self): - return 'Inf' - -Inf = _Inf() -del _Inf - - -_normalize_re = re.compile(r'[^a-z]', re.I) - - -def normalize_name(name): - return _normalize_re.sub('-', name.lower()) - - -def format_size(bytes): - if bytes > 1000*1000: - return '%.1fMb' % (bytes/1000.0/1000) - elif bytes > 10*1000: - return '%iKb' % (bytes/1000) - elif bytes > 1000: - return '%.1fKb' % (bytes/1000.0) - else: - return '%ibytes' % bytes - - -def is_installable_dir(path): - """Return True if `path` is a directory containing a setup.py file.""" - if not os.path.isdir(path): - return False - setup_py = os.path.join(path, 'setup.py') - if os.path.isfile(setup_py): - return True - return False - - -def is_svn_page(html): - """Returns true if the page appears to be the index page of an svn repository""" - return (re.search(r'<title>[^<]*Revision \d+:', html) - and re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I)) - - -def file_contents(filename): - fp = open(filename, 'rb') - try: - return fp.read() - finally: - fp.close() - - -def split_leading_dir(path): - path = str(path) - path = path.lstrip('/').lstrip('\\') - if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) - or '\\' not in path): - return path.split('/', 1) - elif '\\' in path: - return path.split('\\', 1) - else: - return path, '' - - -def has_leading_dir(paths): - """Returns true if all the paths have the same leading path name - (i.e., everything is in one subdirectory in an archive)""" - common_prefix = None - for path in paths: - prefix, rest = split_leading_dir(path) - if not prefix: - return False - elif common_prefix is None: - common_prefix = prefix - elif prefix != common_prefix: - return False - return True - - -def make_path_relative(path, rel_to): - """ - Make a filename relative, where the filename path, and it is - relative to rel_to - - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/usr/share/another-place/src/Directory') - '../../../something/a-file.pth' - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/home/user/src/Directory') - '../../../usr/share/something/a-file.pth' - >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/') - 'a-file.pth' - """ - path_filename = os.path.basename(path) - path = os.path.dirname(path) - path = os.path.normpath(os.path.abspath(path)) - rel_to = os.path.normpath(os.path.abspath(rel_to)) - path_parts = path.strip(os.path.sep).split(os.path.sep) - rel_to_parts = rel_to.strip(os.path.sep).split(os.path.sep) - while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]: - path_parts.pop(0) - rel_to_parts.pop(0) - full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename] - if full_parts == ['']: - return '.' + os.path.sep - return os.path.sep.join(full_parts) - - -def normalize_path(path): - """ - Convert a path to its canonical, case-normalized, absolute version. - - """ - return os.path.normcase(os.path.realpath(path)) - - -def splitext(path): - """Like os.path.splitext, but take off .tar too""" - base, ext = posixpath.splitext(path) - if base.lower().endswith('.tar'): - ext = base[-4:] + ext - base = base[:-4] - return base, ext - - -def renames(old, new): - """Like os.renames(), but handles renaming across devices.""" - # Implementation borrowed from os.renames(). - head, tail = os.path.split(new) - if head and tail and not os.path.exists(head): - os.makedirs(head) - - shutil.move(old, new) - - head, tail = os.path.split(old) - if head and tail: - try: - os.removedirs(head) - except OSError: - pass - - -def is_local(path): - """ - Return True if path is within sys.prefix, if we're running in a virtualenv. - - If we're not in a virtualenv, all paths are considered "local." - - """ - if not running_under_virtualenv(): - return True - return normalize_path(path).startswith(normalize_path(sys.prefix)) - - -def dist_is_local(dist): - """ - Return True if given Distribution object is installed locally - (i.e. within current virtualenv). - - Always True if we're not in a virtualenv. - - """ - return is_local(dist_location(dist)) - - -def get_installed_distributions(local_only=True, skip=('setuptools', 'pip', 'python')): - """ - Return a list of installed Distribution objects. - - If ``local_only`` is True (default), only return installations - local to the current virtualenv, if in a virtualenv. - - ``skip`` argument is an iterable of lower-case project names to - ignore; defaults to ('setuptools', 'pip', 'python'). [FIXME also - skip virtualenv?] - - """ - if local_only: - local_test = dist_is_local - else: - local_test = lambda d: True - return [d for d in pkg_resources.working_set if local_test(d) and d.key not in skip] - - -def egg_link_path(dist): - """ - Return the path where we'd expect to find a .egg-link file for - this distribution. (There doesn't seem to be any metadata in the - Distribution object for a develop egg that points back to its - .egg-link and easy-install.pth files). - - This won't find a globally-installed develop egg if we're in a - virtualenv. - - """ - return os.path.join(site_packages, dist.project_name) + '.egg-link' - - -def dist_location(dist): - """ - Get the site-packages location of this distribution. Generally - this is dist.location, except in the case of develop-installed - packages, where dist.location is the source code location, and we - want to know where the egg-link file is. - - """ - egg_link = egg_link_path(dist) - if os.path.exists(egg_link): - return egg_link - return dist.location - - -def get_terminal_size(): - """Returns a tuple (x, y) representing the width(x) and the height(x) - in characters of the terminal window.""" - def ioctl_GWINSZ(fd): - try: - import fcntl - import termios - import struct - cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, - '1234')) - except: - return None - if cr == (0, 0): - return None - if cr == (0, 0): - return None - return cr - cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - cr = ioctl_GWINSZ(fd) - os.close(fd) - except: - pass - if not cr: - cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) - return int(cr[1]), int(cr[0]) - - -def unzip_file(filename, location, flatten=True): - """Unzip the file (zip file located at filename) to the destination - location""" - if not os.path.exists(location): - os.makedirs(location) - zipfp = open(filename, 'rb') - try: - zip = zipfile.ZipFile(zipfp) - leading = has_leading_dir(zip.namelist()) and flatten - for name in zip.namelist(): - data = zip.read(name) - fn = name - if leading: - fn = split_leading_dir(name)[1] - fn = os.path.join(location, fn) - dir = os.path.dirname(fn) - if not os.path.exists(dir): - os.makedirs(dir) - if fn.endswith('/') or fn.endswith('\\'): - # A directory - if not os.path.exists(fn): - os.makedirs(fn) - else: - fp = open(fn, 'wb') - try: - fp.write(data) - finally: - fp.close() - finally: - zipfp.close() - - -def untar_file(filename, location): - """Untar the file (tar file located at filename) to the destination location""" - if not os.path.exists(location): - os.makedirs(location) - if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): - mode = 'r:gz' - elif filename.lower().endswith('.bz2') or filename.lower().endswith('.tbz'): - mode = 'r:bz2' - elif filename.lower().endswith('.tar'): - mode = 'r' - else: - logger.warn('Cannot determine compression type for file %s' % filename) - mode = 'r:*' - tar = tarfile.open(filename, mode) - try: - # note: python<=2.5 doesnt seem to know about pax headers, filter them - leading = has_leading_dir([ - member.name for member in tar.getmembers() - if member.name != 'pax_global_header' - ]) - for member in tar.getmembers(): - fn = member.name - if fn == 'pax_global_header': - continue - if leading: - fn = split_leading_dir(fn)[1] - path = os.path.join(location, fn) - if member.isdir(): - if not os.path.exists(path): - os.makedirs(path) - else: - try: - fp = tar.extractfile(member) - except (KeyError, AttributeError), e: - # Some corrupt tar files seem to produce this - # (specifically bad symlinks) - logger.warn( - 'In the tar file %s the member %s is invalid: %s' - % (filename, member.name, e)) - continue - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - destfp = open(path, 'wb') - try: - shutil.copyfileobj(fp, destfp) - finally: - destfp.close() - fp.close() - finally: - tar.close() - - -def create_download_cache_folder(folder): - logger.indent -= 2 - logger.notify('Creating supposed download cache at %s' % folder) - logger.indent += 2 - os.makedirs(folder) - - -def cache_download(target_file, temp_location, content_type): - logger.notify('Storing download in cache at %s' % display_path(target_file)) - shutil.copyfile(temp_location, target_file) - fp = open(target_file+'.content-type', 'w') - fp.write(content_type) - fp.close() - os.unlink(temp_location) - - -def unpack_file(filename, location, content_type, link): - if (content_type == 'application/zip' - or filename.endswith('.zip') - or filename.endswith('.pybundle') - or zipfile.is_zipfile(filename)): - unzip_file(filename, location, flatten=not filename.endswith('.pybundle')) - elif (content_type == 'application/x-gzip' - or tarfile.is_tarfile(filename) - or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')): - untar_file(filename, location) - elif (content_type and content_type.startswith('text/html') - and is_svn_page(file_contents(filename))): - # We don't really care about this - from pip.vcs.subversion import Subversion - Subversion('svn+' + link.url).unpack(location) - else: - ## FIXME: handle? - ## FIXME: magic signatures? - logger.fatal('Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format' - % (filename, location, content_type)) - raise InstallationError('Cannot determine archive format of %s' % location) - - - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py deleted file mode 100644 index e110440c7..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Handles all VCS (version control) support""" - -import os -import shutil -import urlparse -import urllib - -from pip.exceptions import BadCommand -from pip.log import logger -from pip.util import display_path, backup_dir, find_command, ask - - -__all__ = ['vcs', 'get_src_requirement', 'import_vcs_support'] - - -class VcsSupport(object): - _registry = {} - schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp'] - - def __init__(self): - # Register more schemes with urlparse for various version control systems - urlparse.uses_netloc.extend(self.schemes) - urlparse.uses_fragment.extend(self.schemes) - super(VcsSupport, self).__init__() - - def __iter__(self): - return self._registry.__iter__() - - @property - def backends(self): - return self._registry.values() - - @property - def dirnames(self): - return [backend.dirname for backend in self.backends] - - @property - def all_schemes(self): - schemes = [] - for backend in self.backends: - schemes.extend(backend.schemes) - return schemes - - def register(self, cls): - if not hasattr(cls, 'name'): - logger.warn('Cannot register VCS %s' % cls.__name__) - return - if cls.name not in self._registry: - self._registry[cls.name] = cls - - def unregister(self, cls=None, name=None): - if name in self._registry: - del self._registry[name] - elif cls in self._registry.values(): - del self._registry[cls.name] - else: - logger.warn('Cannot unregister because no class or name given') - - def get_backend_name(self, location): - """ - Return the name of the version control backend if found at given - location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') - """ - for vc_type in self._registry.values(): - path = os.path.join(location, vc_type.dirname) - if os.path.exists(path): - return vc_type.name - return None - - def get_backend(self, name): - name = name.lower() - if name in self._registry: - return self._registry[name] - - def get_backend_from_location(self, location): - vc_type = self.get_backend_name(location) - if vc_type: - return self.get_backend(vc_type) - return None - - -vcs = VcsSupport() - - -class VersionControl(object): - name = '' - dirname = '' - - def __init__(self, url=None, *args, **kwargs): - self.url = url - self._cmd = None - super(VersionControl, self).__init__(*args, **kwargs) - - def _filter(self, line): - return (logger.INFO, line) - - def _is_local_repository(self, repo): - """ - posix absolute paths start with os.path.sep, - win32 ones ones start with drive (like c:\\folder) - """ - drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or drive - - @property - def cmd(self): - if self._cmd is not None: - return self._cmd - command = find_command(self.name) - if command is None: - raise BadCommand('Cannot find command %r' % self.name) - logger.info('Found command %r at %r' % (self.name, command)) - self._cmd = command - return command - - def get_url_rev(self): - """ - Returns the correct repository URL and revision by parsing the given - repository URL - """ - url = self.url.split('+', 1)[1] - scheme, netloc, path, query, frag = urlparse.urlsplit(url) - rev = None - if '@' in path: - path, rev = path.rsplit('@', 1) - url = urlparse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev - - def get_info(self, location): - """ - Returns (url, revision), where both are strings - """ - assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) - - def normalize_url(self, url): - """ - Normalize a URL for comparison by unquoting it and removing any trailing slash. - """ - return urllib.unquote(url).rstrip('/') - - def compare_urls(self, url1, url2): - """ - Compare two repo URLs for identity, ignoring incidental differences. - """ - return (self.normalize_url(url1) == self.normalize_url(url2)) - - def parse_vcs_bundle_file(self, content): - """ - Takes the contents of the bundled text file that explains how to revert - the stripped off version control data of the given package and returns - the URL and revision of it. - """ - raise NotImplementedError - - def obtain(self, dest): - """ - Called when installing or updating an editable package, takes the - source path of the checkout. - """ - raise NotImplementedError - - def switch(self, dest, url, rev_options): - """ - Switch the repo at ``dest`` to point to ``URL``. - """ - raise NotImplemented - - def update(self, dest, rev_options): - """ - Update an already-existing repo to the given ``rev_options``. - """ - raise NotImplementedError - - def check_destination(self, dest, url, rev_options, rev_display): - """ - Prepare a location to receive a checkout/clone. - - Return True if the location is ready for (and requires) a - checkout/clone, False otherwise. - """ - checkout = True - prompt = False - if os.path.exists(dest): - checkout = False - if os.path.exists(os.path.join(dest, self.dirname)): - existing_url = self.get_url(dest) - if self.compare_urls(existing_url, url): - logger.info('%s in %s exists, and has correct URL (%s)' - % (self.repo_name.title(), display_path(dest), url)) - logger.notify('Updating %s %s%s' - % (display_path(dest), self.repo_name, rev_display)) - self.update(dest, rev_options) - else: - logger.warn('%s %s in %s exists with URL %s' - % (self.name, self.repo_name, display_path(dest), existing_url)) - prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) - else: - logger.warn('Directory %s already exists, and is not a %s %s.' - % (dest, self.name, self.repo_name)) - prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) - if prompt: - logger.warn('The plan is to install the %s repository %s' - % (self.name, url)) - response = ask('What to do? %s' % prompt[0], prompt[1]) - - if response == 's': - logger.notify('Switching %s %s to %s%s' - % (self.repo_name, display_path(dest), url, rev_display)) - self.switch(dest, url, rev_options) - elif response == 'i': - # do nothing - pass - elif response == 'w': - logger.warn('Deleting %s' % display_path(dest)) - shutil.rmtree(dest) - checkout = True - elif response == 'b': - dest_dir = backup_dir(dest) - logger.warn('Backing up %s to %s' - % (display_path(dest), dest_dir)) - shutil.move(dest, dest_dir) - checkout = True - return checkout - - def unpack(self, location): - raise NotImplementedError - - def get_src_requirement(self, dist, location, find_tags=False): - raise NotImplementedError - - -def get_src_requirement(dist, location, find_tags): - version_control = vcs.get_backend_from_location(location) - if version_control: - return version_control().get_src_requirement(dist, location, find_tags) - logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location) - return dist.as_requirement() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py deleted file mode 100644 index 3b6ea8f04..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py +++ /dev/null @@ -1,138 +0,0 @@ -import os -import shutil -import tempfile -import re -from pip import call_subprocess -from pip.log import logger -from pip.util import rmtree, display_path -from pip.vcs import vcs, VersionControl -from pip.download import path_to_url2 - - -class Bazaar(VersionControl): - name = 'bzr' - dirname = '.bzr' - repo_name = 'branch' - bundle_file = 'bzr-branch.txt' - schemes = ('bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp') - guide = ('# This was a Bazaar branch; to make it a branch again run:\n' - 'bzr branch -r %(rev)s %(url)s .\n') - - def parse_vcs_bundle_file(self, content): - url = rev = None - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - match = re.search(r'^bzr\s*branch\s*-r\s*(\d*)', line) - if match: - rev = match.group(1).strip() - url = line[match.end():].strip().split(None, 1)[0] - if url and rev: - return url, rev - return None, None - - def unpack(self, location): - """Get the bzr branch at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Checking out bzr repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - os.rmdir(location) - call_subprocess( - [self.cmd, 'branch', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the Bazaar repository at the url to the destination location""" - temp_dir = tempfile.mkdtemp('-export', 'pip-') - self.unpack(temp_dir) - if os.path.exists(location): - # Remove the location to make sure Bazaar can export it correctly - rmtree(location) - try: - call_subprocess([self.cmd, 'export', location], cwd=temp_dir, - filter_stdout=self._filter, show_stdout=False) - finally: - shutil.rmtree(temp_dir) - - def switch(self, dest, url, rev_options): - call_subprocess([self.cmd, 'switch', url], cwd=dest) - - def update(self, dest, rev_options): - call_subprocess( - [self.cmd, 'pull', '-q'] + rev_options, cwd=dest) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = ['-r', rev] - rev_display = ' (to revision %s)' % rev - else: - rev_options = [] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Checking out %s%s to %s' - % (url, rev_display, display_path(dest))) - call_subprocess( - [self.cmd, 'branch', '-q'] + rev_options + [url, dest]) - - def get_url_rev(self): - # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev() - if url.startswith('ssh://'): - url = 'bzr+' + url - return url, rev - - def get_url(self, location): - urls = call_subprocess( - [self.cmd, 'info'], show_stdout=False, cwd=location) - for line in urls.splitlines(): - line = line.strip() - for x in ('checkout of branch: ', - 'parent branch: '): - if line.startswith(x): - repo = line.split(x)[1] - if self._is_local_repository(repo): - return path_to_url2(repo) - return repo - return None - - def get_revision(self, location): - revision = call_subprocess( - [self.cmd, 'revno'], show_stdout=False, cwd=location) - return revision.splitlines()[-1] - - def get_tag_revs(self, location): - tags = call_subprocess( - [self.cmd, 'tags'], show_stdout=False, cwd=location) - tag_revs = [] - for line in tags.splitlines(): - tags_match = re.search(r'([.\w-]+)\s*(.*)$', line) - if tags_match: - tag = tags_match.group(1) - rev = tags_match.group(2) - tag_revs.append((rev.strip(), tag.strip())) - return dict(tag_revs) - - def get_src_requirement(self, dist, location, find_tags): - repo = self.get_url(location) - if not repo.lower().startswith('bzr:'): - repo = 'bzr+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None - current_rev = self.get_revision(location) - tag_revs = self.get_tag_revs(location) - - if current_rev in tag_revs: - # It's a tag - full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) - else: - full_egg_name = '%s-dev_r%s' % (dist.egg_name(), current_rev) - return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) - - -vcs.register(Bazaar) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py deleted file mode 100644 index 0701e49ec..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py +++ /dev/null @@ -1,204 +0,0 @@ -import os -import shutil -import tempfile -import re -from pip import call_subprocess -from pip.util import display_path -from pip.vcs import vcs, VersionControl -from pip.log import logger -from urllib import url2pathname -from urlparse import urlsplit, urlunsplit - - -class Git(VersionControl): - name = 'git' - dirname = '.git' - repo_name = 'clone' - schemes = ('git', 'git+http', 'git+ssh', 'git+git', 'git+file') - bundle_file = 'git-clone.txt' - guide = ('# This was a Git repo; to make it a repo again run:\n' - 'git init\ngit remote add origin %(url)s -f\ngit checkout %(rev)s\n') - - def __init__(self, url=None, *args, **kwargs): - - # Works around an apparent Git bug - # (see http://article.gmane.org/gmane.comp.version-control.git/146500) - if url: - scheme, netloc, path, query, fragment = urlsplit(url) - if scheme.endswith('file'): - initial_slashes = path[:-len(path.lstrip('/'))] - newpath = initial_slashes + url2pathname(path).replace('\\', '/').lstrip('/') - url = urlunsplit((scheme, netloc, newpath, query, fragment)) - after_plus = scheme.find('+')+1 - url = scheme[:after_plus]+ urlunsplit((scheme[after_plus:], netloc, newpath, query, fragment)) - - super(Git, self).__init__(url, *args, **kwargs) - - def parse_vcs_bundle_file(self, content): - url = rev = None - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - url_match = re.search(r'git\s*remote\s*add\s*origin(.*)\s*-f', line) - if url_match: - url = url_match.group(1).strip() - rev_match = re.search(r'^git\s*checkout\s*-q\s*(.*)\s*', line) - if rev_match: - rev = rev_match.group(1).strip() - if url and rev: - return url, rev - return None, None - - def unpack(self, location): - """Clone the Git repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Cloning Git repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - os.rmdir(location) - call_subprocess( - [self.cmd, 'clone', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the Git repository at the url to the destination location""" - temp_dir = tempfile.mkdtemp('-export', 'pip-') - self.unpack(temp_dir) - try: - if not location.endswith('/'): - location = location + '/' - call_subprocess( - [self.cmd, 'checkout-index', '-a', '-f', '--prefix', location], - filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) - finally: - shutil.rmtree(temp_dir) - - def check_rev_options(self, rev, dest, rev_options): - """Check the revision options before checkout to compensate that tags - and branches may need origin/ as a prefix. - Returns the SHA1 of the branch or tag if found. - """ - revisions = self.get_tag_revs(dest) - revisions.update(self.get_branch_revs(dest)) - inverse_revisions = dict((v, k) for k, v in revisions.iteritems()) - # Check if rev is a branch name - origin_rev = 'origin/%s' % rev - if origin_rev in inverse_revisions: - return [inverse_revisions[origin_rev]] - elif rev in inverse_revisions: - return [inverse_revisions[rev]] - else: - logger.warn("Could not find a tag or branch '%s', assuming commit." % rev) - return rev_options - - def switch(self, dest, url, rev_options): - call_subprocess( - [self.cmd, 'config', 'remote.origin.url', url], cwd=dest) - call_subprocess( - [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) - - def update(self, dest, rev_options): - call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) - call_subprocess( - [self.cmd, 'checkout', '-q', '-f'] + rev_options, cwd=dest) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = [rev] - rev_display = ' (to %s)' % rev - else: - rev_options = ['master'] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest))) - call_subprocess([self.cmd, 'clone', '-q', url, dest]) - if rev: - rev_options = self.check_rev_options(rev, dest, rev_options) - # Only do a checkout if rev_options differs from HEAD - if not self.get_revision(dest).startswith(rev_options[0]): - call_subprocess([self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) - - def get_url(self, location): - url = call_subprocess( - [self.cmd, 'config', 'remote.origin.url'], - show_stdout=False, cwd=location) - return url.strip() - - def get_revision(self, location): - current_rev = call_subprocess( - [self.cmd, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location) - return current_rev.strip() - - def get_tag_revs(self, location): - tags = call_subprocess( - [self.cmd, 'tag', '-l'], - show_stdout=False, raise_on_returncode=False, cwd=location) - tag_revs = [] - for line in tags.splitlines(): - tag = line.strip() - rev = call_subprocess( - [self.cmd, 'rev-parse', tag], show_stdout=False, cwd=location) - tag_revs.append((rev.strip(), tag)) - tag_revs = dict(tag_revs) - return tag_revs - - def get_branch_revs(self, location): - branches = call_subprocess( - [self.cmd, 'branch', '-r'], show_stdout=False, cwd=location) - branch_revs = [] - for line in branches.splitlines(): - line = line.split('->')[0].strip() - branch = "".join([b for b in line.split() if b != '*']) - rev = call_subprocess( - [self.cmd, 'rev-parse', branch], show_stdout=False, cwd=location) - branch_revs.append((rev.strip(), branch)) - branch_revs = dict(branch_revs) - return branch_revs - - def get_src_requirement(self, dist, location, find_tags): - repo = self.get_url(location) - if not repo.lower().startswith('git:'): - repo = 'git+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None - current_rev = self.get_revision(location) - tag_revs = self.get_tag_revs(location) - branch_revs = self.get_branch_revs(location) - - if current_rev in tag_revs: - # It's a tag - full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) - elif (current_rev in branch_revs and - branch_revs[current_rev] != 'origin/master'): - # It's the head of a branch - full_egg_name = '%s-%s' % (dist.egg_name(), - branch_revs[current_rev].replace('origin/', '')) - else: - full_egg_name = '%s-dev' % dist.egg_name() - - return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) - - def get_url_rev(self): - """ - Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. - That's required because although they use SSH they sometimes doesn't - work with a ssh:// scheme (e.g. Github). But we need a scheme for - parsing. Hence we remove it again afterwards and return it as a stub. - """ - if not '://' in self.url: - assert not 'file:' in self.url - self.url = self.url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev() - url = url.replace('ssh://', '') - else: - url, rev = super(Git, self).get_url_rev() - - return url, rev - - -vcs.register(Git) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py deleted file mode 100644 index 70c8c833d..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -import shutil -import tempfile -import re -import ConfigParser -from pip import call_subprocess -from pip.util import display_path -from pip.log import logger -from pip.vcs import vcs, VersionControl -from pip.download import path_to_url2 - - -class Mercurial(VersionControl): - name = 'hg' - dirname = '.hg' - repo_name = 'clone' - schemes = ('hg', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http') - bundle_file = 'hg-clone.txt' - guide = ('# This was a Mercurial repo; to make it a repo again run:\n' - 'hg init\nhg pull %(url)s\nhg update -r %(rev)s\n') - - def parse_vcs_bundle_file(self, content): - url = rev = None - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - url_match = re.search(r'hg\s*pull\s*(.*)\s*', line) - if url_match: - url = url_match.group(1).strip() - rev_match = re.search(r'^hg\s*update\s*-r\s*(.*)\s*', line) - if rev_match: - rev = rev_match.group(1).strip() - if url and rev: - return url, rev - return None, None - - def unpack(self, location): - """Clone the Hg repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Cloning Mercurial repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - os.rmdir(location) - call_subprocess( - [self.cmd, 'clone', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the Hg repository at the url to the destination location""" - temp_dir = tempfile.mkdtemp('-export', 'pip-') - self.unpack(temp_dir) - try: - call_subprocess( - [self.cmd, 'archive', location], - filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) - finally: - shutil.rmtree(temp_dir) - - def switch(self, dest, url, rev_options): - repo_config = os.path.join(dest, self.dirname, 'hgrc') - config = ConfigParser.SafeConfigParser() - try: - config.read(repo_config) - config.set('paths', 'default', url) - config_file = open(repo_config, 'w') - config.write(config_file) - config_file.close() - except (OSError, ConfigParser.NoSectionError), e: - logger.warn( - 'Could not switch Mercurial repository to %s: %s' - % (url, e)) - else: - call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) - - def update(self, dest, rev_options): - call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) - call_subprocess( - [self.cmd, 'update', '-q'] + rev_options, cwd=dest) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = [rev] - rev_display = ' (to revision %s)' % rev - else: - rev_options = [] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Cloning hg %s%s to %s' - % (url, rev_display, display_path(dest))) - call_subprocess([self.cmd, 'clone', '--noupdate', '-q', url, dest]) - call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) - - def get_url(self, location): - url = call_subprocess( - [self.cmd, 'showconfig', 'paths.default'], - show_stdout=False, cwd=location).strip() - if self._is_local_repository(url): - url = path_to_url2(url) - return url.strip() - - def get_tag_revs(self, location): - tags = call_subprocess( - [self.cmd, 'tags'], show_stdout=False, cwd=location) - tag_revs = [] - for line in tags.splitlines(): - tags_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) - if tags_match: - tag = tags_match.group(1) - rev = tags_match.group(2) - tag_revs.append((rev.strip(), tag.strip())) - return dict(tag_revs) - - def get_branch_revs(self, location): - branches = call_subprocess( - [self.cmd, 'branches'], show_stdout=False, cwd=location) - branch_revs = [] - for line in branches.splitlines(): - branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) - if branches_match: - branch = branches_match.group(1) - rev = branches_match.group(2) - branch_revs.append((rev.strip(), branch.strip())) - return dict(branch_revs) - - def get_revision(self, location): - current_revision = call_subprocess( - [self.cmd, 'parents', '--template={rev}'], - show_stdout=False, cwd=location).strip() - return current_revision - - def get_revision_hash(self, location): - current_rev_hash = call_subprocess( - [self.cmd, 'parents', '--template={node}'], - show_stdout=False, cwd=location).strip() - return current_rev_hash - - def get_src_requirement(self, dist, location, find_tags): - repo = self.get_url(location) - if not repo.lower().startswith('hg:'): - repo = 'hg+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None - current_rev = self.get_revision(location) - current_rev_hash = self.get_revision_hash(location) - tag_revs = self.get_tag_revs(location) - branch_revs = self.get_branch_revs(location) - if current_rev in tag_revs: - # It's a tag - full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) - elif current_rev in branch_revs: - # It's the tip of a branch - full_egg_name = '%s-%s' % (dist.egg_name(), branch_revs[current_rev]) - else: - full_egg_name = '%s-dev' % dist.egg_name() - return '%s@%s#egg=%s' % (repo, current_rev_hash, full_egg_name) - -vcs.register(Mercurial) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py deleted file mode 100644 index 85715d97a..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py +++ /dev/null @@ -1,260 +0,0 @@ -import os -import re -from pip import call_subprocess -from pip.index import Link -from pip.util import rmtree, display_path -from pip.log import logger -from pip.vcs import vcs, VersionControl - -_svn_xml_url_re = re.compile('url="([^"]+)"') -_svn_rev_re = re.compile('committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') - - -class Subversion(VersionControl): - name = 'svn' - dirname = '.svn' - repo_name = 'checkout' - schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https') - bundle_file = 'svn-checkout.txt' - guide = ('# This was an svn checkout; to make it a checkout again run:\n' - 'svn checkout --force -r %(rev)s %(url)s .\n') - - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location - output = call_subprocess( - [self.cmd, 'info', location], show_stdout=False, extra_environ={'LANG': 'C'}) - match = _svn_url_re.search(output) - if not match: - logger.warn('Cannot determine URL of svn checkout %s' % display_path(location)) - logger.info('Output that cannot be parsed: \n%s' % output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warn('Cannot determine revision of svn checkout %s' % display_path(location)) - logger.info('Output that cannot be parsed: \n%s' % output) - return url, None - return url, match.group(1) - - def parse_vcs_bundle_file(self, content): - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - match = re.search(r'^-r\s*([^ ])?', line) - if not match: - return None, None - rev = match.group(1) - rest = line[match.end():].strip().split(None, 1)[0] - return rest, rev - return None, None - - def unpack(self, location): - """Check out the svn repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Checking out svn repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - # Subversion doesn't like to check out over an existing directory - # --force fixes this, but was only added in svn 1.5 - rmtree(location) - call_subprocess( - [self.cmd, 'checkout', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the svn repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Exporting svn repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - # Subversion doesn't like to check out over an existing directory - # --force fixes this, but was only added in svn 1.5 - rmtree(location) - call_subprocess( - [self.cmd, 'export', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def switch(self, dest, url, rev_options): - call_subprocess( - [self.cmd, 'switch'] + rev_options + [url, dest]) - - def update(self, dest, rev_options): - call_subprocess( - [self.cmd, 'update'] + rev_options + [dest]) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = ['-r', rev] - rev_display = ' (to revision %s)' % rev - else: - rev_options = [] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Checking out %s%s to %s' - % (url, rev_display, display_path(dest))) - call_subprocess( - [self.cmd, 'checkout', '-q'] + rev_options + [url, dest]) - - def get_location(self, dist, dependency_links): - for url in dependency_links: - egg_fragment = Link(url).egg_fragment - if not egg_fragment: - continue - if '-' in egg_fragment: - ## FIXME: will this work when a package has - in the name? - key = '-'.join(egg_fragment.split('-')[:-1]).lower() - else: - key = egg_fragment - if key == dist.key: - return url.split('#', 1)[0] - return None - - def get_revision(self, location): - """ - Return the maximum revision for all files under a given location - """ - # Note: taken from setuptools.command.egg_info - revision = 0 - - for base, dirs, files in os.walk(location): - if self.dirname not in dirs: - dirs[:] = [] - continue # no sense walking uncontrolled subdirs - dirs.remove(self.dirname) - entries_fn = os.path.join(base, self.dirname, 'entries') - if not os.path.exists(entries_fn): - ## FIXME: should we warn? - continue - f = open(entries_fn) - data = f.read() - f.close() - - if data.startswith('8') or data.startswith('9') or data.startswith('10'): - data = map(str.splitlines, data.split('\n\x0c\n')) - del data[0][0] # get rid of the '8' - dirurl = data[0][3] - revs = [int(d[9]) for d in data if len(d)>9 and d[9]]+[0] - if revs: - localrev = max(revs) - else: - localrev = 0 - elif data.startswith('<?xml'): - dirurl = _svn_xml_url_re.search(data).group(1) # get repository URL - revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)]+[0] - if revs: - localrev = max(revs) - else: - localrev = 0 - else: - logger.warn("Unrecognized .svn/entries format; skipping %s", base) - dirs[:] = [] - continue - if base == location: - base_url = dirurl+'/' # save the root url - elif not dirurl.startswith(base_url): - dirs[:] = [] - continue # not part of the same svn tree, skip it - revision = max(revision, localrev) - return revision - - def get_url_rev(self): - # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev() - if url.startswith('ssh://'): - url = 'svn+' + url - return url, rev - - def get_url(self, location): - # In cases where the source is in a subdirectory, not alongside setup.py - # we have to look up in the location until we find a real setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without finding setup.py - logger.warn("Could not find setup.py for directory %s (tried all parent directories)" - % orig_location) - return None - f = open(os.path.join(location, self.dirname, 'entries')) - data = f.read() - f.close() - if data.startswith('8') or data.startswith('9') or data.startswith('10'): - data = map(str.splitlines, data.split('\n\x0c\n')) - del data[0][0] # get rid of the '8' - return data[0][3] - elif data.startswith('<?xml'): - match = _svn_xml_url_re.search(data) - if not match: - raise ValueError('Badly formatted data: %r' % data) - return match.group(1) # get repository URL - else: - logger.warn("Unrecognized .svn/entries format in %s" % location) - # Or raise exception? - return None - - def get_tag_revs(self, svn_tag_url): - stdout = call_subprocess( - [self.cmd, 'ls', '-v', svn_tag_url], show_stdout=False) - results = [] - for line in stdout.splitlines(): - parts = line.split() - rev = int(parts[0]) - tag = parts[-1].strip('/') - results.append((tag, rev)) - return results - - def find_tag_match(self, rev, tag_revs): - best_match_rev = None - best_tag = None - for tag, tag_rev in tag_revs: - if (tag_rev > rev and - (best_match_rev is None or best_match_rev > tag_rev)): - # FIXME: Is best_match > tag_rev really possible? - # or is it a sign something is wacky? - best_match_rev = tag_rev - best_tag = tag - return best_tag - - def get_src_requirement(self, dist, location, find_tags=False): - repo = self.get_url(location) - if repo is None: - return None - parts = repo.split('/') - ## FIXME: why not project name? - egg_project_name = dist.egg_name().split('-', 1)[0] - rev = self.get_revision(location) - if parts[-2] in ('tags', 'tag'): - # It's a tag, perfect! - full_egg_name = '%s-%s' % (egg_project_name, parts[-1]) - elif parts[-2] in ('branches', 'branch'): - # It's a branch :( - full_egg_name = '%s-%s-r%s' % (dist.egg_name(), parts[-1], rev) - elif parts[-1] == 'trunk': - # Trunk :-/ - full_egg_name = '%s-dev_r%s' % (dist.egg_name(), rev) - if find_tags: - tag_url = '/'.join(parts[:-1]) + '/tags' - tag_revs = self.get_tag_revs(tag_url) - match = self.find_tag_match(rev, tag_revs) - if match: - logger.notify('trunk checkout %s seems to be equivalent to tag %s' % match) - repo = '%s/%s' % (tag_url, match) - full_egg_name = '%s-%s' % (egg_project_name, match) - else: - # Don't know what it is - logger.warn('svn URL does not fit normal structure (tags/branches/trunk): %s' % repo) - full_egg_name = '%s-dev_r%s' % (egg_project_name, rev) - return 'svn+%s@%s#egg=%s' % (repo, rev, full_egg_name) - -vcs.register(Subversion) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py deleted file mode 100644 index 708abb05b..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Tools for working with virtualenv environments""" - -import os -import sys -import subprocess -from pip.exceptions import BadCommand -from pip.log import logger - - -def restart_in_venv(venv, base, site_packages, args): - """ - Restart this script using the interpreter in the given virtual environment - """ - if base and not os.path.isabs(venv) and not venv.startswith('~'): - base = os.path.expanduser(base) - # ensure we have an abs basepath at this point: - # a relative one makes no sense (or does it?) - if os.path.isabs(base): - venv = os.path.join(base, venv) - - if venv.startswith('~'): - venv = os.path.expanduser(venv) - - if not os.path.exists(venv): - try: - import virtualenv - except ImportError: - print 'The virtual environment does not exist: %s' % venv - print 'and virtualenv is not installed, so a new environment cannot be created' - sys.exit(3) - print 'Creating new virtualenv environment in %s' % venv - virtualenv.logger = logger - logger.indent += 2 - virtualenv.create_environment(venv, site_packages=site_packages) - if sys.platform == 'win32': - python = os.path.join(venv, 'Scripts', 'python.exe') - # check for bin directory which is used in buildouts - if not os.path.exists(python): - python = os.path.join(venv, 'bin', 'python.exe') - else: - python = os.path.join(venv, 'bin', 'python') - if not os.path.exists(python): - python = venv - if not os.path.exists(python): - raise BadCommand('Cannot find virtual environment interpreter at %s' % python) - base = os.path.dirname(os.path.dirname(python)) - file = os.path.join(os.path.dirname(__file__), 'runner.py') - if file.endswith('.pyc'): - file = file[:-1] - proc = subprocess.Popen( - [python, file] + args + [base, '___VENV_RESTART___']) - proc.wait() - sys.exit(proc.returncode) diff --git a/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg b/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg deleted file mode 100644 index 3c72d15b5..000000000 Binary files a/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg and /dev/null differ diff --git a/test/lib/python2.6/site-packages/setuptools.pth b/test/lib/python2.6/site-packages/setuptools.pth deleted file mode 100644 index bf7602056..000000000 --- a/test/lib/python2.6/site-packages/setuptools.pth +++ /dev/null @@ -1 +0,0 @@ -./setuptools-0.6c11-py2.6.egg diff --git a/test/lib/python2.6/site.py b/test/lib/python2.6/site.py deleted file mode 100644 index a49cfc349..000000000 --- a/test/lib/python2.6/site.py +++ /dev/null @@ -1,713 +0,0 @@ -"""Append module search paths for third-party packages to sys.path. - -**************************************************************** -* This module is automatically imported during initialization. * -**************************************************************** - -In earlier versions of Python (up to 1.5a3), scripts or modules that -needed to use site-specific modules would place ``import site'' -somewhere near the top of their code. Because of the automatic -import, this is no longer necessary (but code that does it still -works). - -This will append site-specific paths to the module search path. On -Unix, it starts with sys.prefix and sys.exec_prefix (if different) and -appends lib/python<version>/site-packages as well as lib/site-python. -It also supports the Debian convention of -lib/python<version>/dist-packages. On other platforms (mainly Mac and -Windows), it uses just sys.prefix (and sys.exec_prefix, if different, -but this is unlikely). The resulting directories, if they exist, are -appended to sys.path, and also inspected for path configuration files. - -FOR DEBIAN, this sys.path is augmented with directories in /usr/local. -Local addons go into /usr/local/lib/python<version>/site-packages -(resp. /usr/local/lib/site-python), Debian addons install into -/usr/{lib,share}/python<version>/dist-packages. - -A path configuration file is a file whose name has the form -<package>.pth; its contents are additional directories (one per line) -to be added to sys.path. Non-existing directories (or -non-directories) are never added to sys.path; no directory is added to -sys.path more than once. Blank lines and lines beginning with -'#' are skipped. Lines starting with 'import' are executed. - -For example, suppose sys.prefix and sys.exec_prefix are set to -/usr/local and there is a directory /usr/local/lib/python2.X/site-packages -with three subdirectories, foo, bar and spam, and two path -configuration files, foo.pth and bar.pth. Assume foo.pth contains the -following: - - # foo package configuration - foo - bar - bletch - -and bar.pth contains: - - # bar package configuration - bar - -Then the following directories are added to sys.path, in this order: - - /usr/local/lib/python2.X/site-packages/bar - /usr/local/lib/python2.X/site-packages/foo - -Note that bletch is omitted because it doesn't exist; bar precedes foo -because bar.pth comes alphabetically before foo.pth; and spam is -omitted because it is not mentioned in either path configuration file. - -After these path manipulations, an attempt is made to import a module -named sitecustomize, which can perform arbitrary additional -site-specific customizations. If this import fails with an -ImportError exception, it is silently ignored. - -""" - -import sys -import os -import __builtin__ -try: - set -except NameError: - from sets import Set as set - -# Prefixes for site-packages; add additional prefixes like /usr/local here -PREFIXES = [sys.prefix, sys.exec_prefix] -# Enable per user site-packages directory -# set it to False to disable the feature or True to force the feature -ENABLE_USER_SITE = None -# for distutils.commands.install -USER_SITE = None -USER_BASE = None - -_is_pypy = hasattr(sys, 'pypy_version_info') -_is_jython = sys.platform[:4] == 'java' -if _is_jython: - ModuleType = type(os) - -def makepath(*paths): - dir = os.path.join(*paths) - if _is_jython and (dir == '__classpath__' or - dir.startswith('__pyclasspath__')): - return dir, dir - dir = os.path.abspath(dir) - return dir, os.path.normcase(dir) - -def abs__file__(): - """Set all module' __file__ attribute to an absolute path""" - for m in sys.modules.values(): - if ((_is_jython and not isinstance(m, ModuleType)) or - hasattr(m, '__loader__')): - # only modules need the abspath in Jython. and don't mess - # with a PEP 302-supplied __file__ - continue - f = getattr(m, '__file__', None) - if f is None: - continue - m.__file__ = os.path.abspath(f) - -def removeduppaths(): - """ Remove duplicate entries from sys.path along with making them - absolute""" - # This ensures that the initial path provided by the interpreter contains - # only absolute pathnames, even if we're running from the build directory. - L = [] - known_paths = set() - for dir in sys.path: - # Filter out duplicate paths (on case-insensitive file systems also - # if they only differ in case); turn relative paths into absolute - # paths. - dir, dircase = makepath(dir) - if not dircase in known_paths: - L.append(dir) - known_paths.add(dircase) - sys.path[:] = L - return known_paths - -# XXX This should not be part of site.py, since it is needed even when -# using the -S option for Python. See http://www.python.org/sf/586680 -def addbuilddir(): - """Append ./build/lib.<platform> in case we're running in the build dir - (especially for Guido :-)""" - from distutils.util import get_platform - s = "build/lib.%s-%.3s" % (get_platform(), sys.version) - if hasattr(sys, 'gettotalrefcount'): - s += '-pydebug' - s = os.path.join(os.path.dirname(sys.path[-1]), s) - sys.path.append(s) - -def _init_pathinfo(): - """Return a set containing all existing directory entries from sys.path""" - d = set() - for dir in sys.path: - try: - if os.path.isdir(dir): - dir, dircase = makepath(dir) - d.add(dircase) - except TypeError: - continue - return d - -def addpackage(sitedir, name, known_paths): - """Add a new path to known_paths by combining sitedir and 'name' or execute - sitedir if it starts with 'import'""" - if known_paths is None: - _init_pathinfo() - reset = 1 - else: - reset = 0 - fullname = os.path.join(sitedir, name) - try: - f = open(fullname, "rU") - except IOError: - return - try: - for line in f: - if line.startswith("#"): - continue - if line.startswith("import"): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - finally: - f.close() - if reset: - known_paths = None - return known_paths - -def addsitedir(sitedir, known_paths=None): - """Add 'sitedir' argument to sys.path if missing and handle .pth files in - 'sitedir'""" - if known_paths is None: - known_paths = _init_pathinfo() - reset = 1 - else: - reset = 0 - sitedir, sitedircase = makepath(sitedir) - if not sitedircase in known_paths: - sys.path.append(sitedir) # Add path component - try: - names = os.listdir(sitedir) - except os.error: - return - names.sort() - for name in names: - if name.endswith(os.extsep + "pth"): - addpackage(sitedir, name, known_paths) - if reset: - known_paths = None - return known_paths - -def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix): - """Add site-packages (and possibly site-python) to sys.path""" - prefixes = [os.path.join(sys_prefix, "local"), sys_prefix] - if exec_prefix != sys_prefix: - prefixes.append(os.path.join(exec_prefix, "local")) - - for prefix in prefixes: - if prefix: - if sys.platform in ('os2emx', 'riscos') or _is_jython: - sitedirs = [os.path.join(prefix, "Lib", "site-packages")] - elif _is_pypy: - sitedirs = [os.path.join(prefix, 'site-packages')] - elif sys.platform == 'darwin' and prefix == sys_prefix: - - if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python - - sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"), - os.path.join(prefix, "Extras", "lib", "python")] - - else: # any other Python distros on OSX work this way - sitedirs = [os.path.join(prefix, "lib", - "python" + sys.version[:3], "site-packages")] - - elif os.sep == '/': - sitedirs = [os.path.join(prefix, - "lib", - "python" + sys.version[:3], - "site-packages"), - os.path.join(prefix, "lib", "site-python"), - os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")] - lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages") - if (os.path.exists(lib64_dir) and - os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]): - sitedirs.append(lib64_dir) - try: - # sys.getobjects only available in --with-pydebug build - sys.getobjects - sitedirs.insert(0, os.path.join(sitedirs[0], 'debug')) - except AttributeError: - pass - # Debian-specific dist-packages directories: - sitedirs.append(os.path.join(prefix, "lib", - "python" + sys.version[:3], - "dist-packages")) - sitedirs.append(os.path.join(prefix, "local/lib", - "python" + sys.version[:3], - "dist-packages")) - sitedirs.append(os.path.join(prefix, "lib", "dist-python")) - else: - sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")] - if sys.platform == 'darwin': - # for framework builds *only* we add the standard Apple - # locations. Currently only per-user, but /Library and - # /Network/Library could be added too - if 'Python.framework' in prefix: - home = os.environ.get('HOME') - if home: - sitedirs.append( - os.path.join(home, - 'Library', - 'Python', - sys.version[:3], - 'site-packages')) - for sitedir in sitedirs: - if os.path.isdir(sitedir): - addsitedir(sitedir, known_paths) - return None - -def check_enableusersite(): - """Check if user site directory is safe for inclusion - - The function tests for the command line flag (including environment var), - process uid/gid equal to effective uid/gid. - - None: Disabled for security reasons - False: Disabled by user (command line option) - True: Safe and enabled - """ - if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False): - return False - - if hasattr(os, "getuid") and hasattr(os, "geteuid"): - # check process uid == effective uid - if os.geteuid() != os.getuid(): - return None - if hasattr(os, "getgid") and hasattr(os, "getegid"): - # check process gid == effective gid - if os.getegid() != os.getgid(): - return None - - return True - -def addusersitepackages(known_paths): - """Add a per user site-package to sys.path - - Each user has its own python directory with site-packages in the - home directory. - - USER_BASE is the root directory for all Python versions - - USER_SITE is the user specific site-packages directory - - USER_SITE/.. can be used for data. - """ - global USER_BASE, USER_SITE, ENABLE_USER_SITE - env_base = os.environ.get("PYTHONUSERBASE", None) - - def joinuser(*args): - return os.path.expanduser(os.path.join(*args)) - - #if sys.platform in ('os2emx', 'riscos'): - # # Don't know what to put here - # USER_BASE = '' - # USER_SITE = '' - if os.name == "nt": - base = os.environ.get("APPDATA") or "~" - if env_base: - USER_BASE = env_base - else: - USER_BASE = joinuser(base, "Python") - USER_SITE = os.path.join(USER_BASE, - "Python" + sys.version[0] + sys.version[2], - "site-packages") - else: - if env_base: - USER_BASE = env_base - else: - USER_BASE = joinuser("~", ".local") - USER_SITE = os.path.join(USER_BASE, "lib", - "python" + sys.version[:3], - "site-packages") - - if ENABLE_USER_SITE and os.path.isdir(USER_SITE): - addsitedir(USER_SITE, known_paths) - if ENABLE_USER_SITE: - for dist_libdir in ("lib", "local/lib"): - user_site = os.path.join(USER_BASE, dist_libdir, - "python" + sys.version[:3], - "dist-packages") - if os.path.isdir(user_site): - addsitedir(user_site, known_paths) - return known_paths - - - -def setBEGINLIBPATH(): - """The OS/2 EMX port has optional extension modules that do double duty - as DLLs (and must use the .DLL file extension) for other extensions. - The library search path needs to be amended so these will be found - during module import. Use BEGINLIBPATH so that these are at the start - of the library search path. - - """ - dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload") - libpath = os.environ['BEGINLIBPATH'].split(';') - if libpath[-1]: - libpath.append(dllpath) - else: - libpath[-1] = dllpath - os.environ['BEGINLIBPATH'] = ';'.join(libpath) - - -def setquit(): - """Define new built-ins 'quit' and 'exit'. - These are simply strings that display a hint on how to exit. - - """ - if os.sep == ':': - eof = 'Cmd-Q' - elif os.sep == '\\': - eof = 'Ctrl-Z plus Return' - else: - eof = 'Ctrl-D (i.e. EOF)' - - class Quitter(object): - def __init__(self, name): - self.name = name - def __repr__(self): - return 'Use %s() or %s to exit' % (self.name, eof) - def __call__(self, code=None): - # Shells like IDLE catch the SystemExit, but listen when their - # stdin wrapper is closed. - try: - sys.stdin.close() - except: - pass - raise SystemExit(code) - __builtin__.quit = Quitter('quit') - __builtin__.exit = Quitter('exit') - - -class _Printer(object): - """interactive prompt objects for printing the license text, a list of - contributors and the copyright notice.""" - - MAXLINES = 23 - - def __init__(self, name, data, files=(), dirs=()): - self.__name = name - self.__data = data - self.__files = files - self.__dirs = dirs - self.__lines = None - - def __setup(self): - if self.__lines: - return - data = None - for dir in self.__dirs: - for filename in self.__files: - filename = os.path.join(dir, filename) - try: - fp = file(filename, "rU") - data = fp.read() - fp.close() - break - except IOError: - pass - if data: - break - if not data: - data = self.__data - self.__lines = data.split('\n') - self.__linecnt = len(self.__lines) - - def __repr__(self): - self.__setup() - if len(self.__lines) <= self.MAXLINES: - return "\n".join(self.__lines) - else: - return "Type %s() to see the full %s text" % ((self.__name,)*2) - - def __call__(self): - self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: - try: - for i in range(lineno, lineno + self.MAXLINES): - print self.__lines[i] - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = raw_input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break - -def setcopyright(): - """Set 'copyright' and 'credits' in __builtin__""" - __builtin__.copyright = _Printer("copyright", sys.copyright) - if _is_jython: - __builtin__.credits = _Printer( - "credits", - "Jython is maintained by the Jython developers (www.jython.org).") - elif _is_pypy: - __builtin__.credits = _Printer( - "credits", - "PyPy is maintained by the PyPy developers: http://codespeak.net/pypy") - else: - __builtin__.credits = _Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") - here = os.path.dirname(os.__file__) - __builtin__.license = _Printer( - "license", "See http://www.python.org/%.3s/license.html" % sys.version, - ["LICENSE.txt", "LICENSE"], - [os.path.join(here, os.pardir), here, os.curdir]) - - -class _Helper(object): - """Define the built-in 'help'. - This is a wrapper around pydoc.help (with a twist). - - """ - - def __repr__(self): - return "Type help() for interactive help, " \ - "or help(object) for help about object." - def __call__(self, *args, **kwds): - import pydoc - return pydoc.help(*args, **kwds) - -def sethelper(): - __builtin__.help = _Helper() - -def aliasmbcs(): - """On Windows, some default encodings are not provided by Python, - while they are always available as "mbcs" in each locale. Make - them usable by aliasing to "mbcs" in such a case.""" - if sys.platform == 'win32': - import locale, codecs - enc = locale.getdefaultlocale()[1] - if enc.startswith('cp'): # "cp***" ? - try: - codecs.lookup(enc) - except LookupError: - import encodings - encodings._cache[enc] = encodings._unknown - encodings.aliases.aliases[enc] = 'mbcs' - -def setencoding(): - """Set the string encoding used by the Unicode implementation. The - default is 'ascii', but if you're willing to experiment, you can - change this.""" - encoding = "ascii" # Default value set by _PyUnicode_Init() - if 0: - # Enable to support locale aware default string encodings. - import locale - loc = locale.getdefaultlocale() - if loc[1]: - encoding = loc[1] - if 0: - # Enable to switch off string to Unicode coercion and implicit - # Unicode to string conversion. - encoding = "undefined" - if encoding != "ascii": - # On Non-Unicode builds this will raise an AttributeError... - sys.setdefaultencoding(encoding) # Needs Python Unicode build ! - - -def execsitecustomize(): - """Run custom site specific code, if available.""" - try: - import sitecustomize - except ImportError: - pass - -def virtual_install_main_packages(): - f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt')) - sys.real_prefix = f.read().strip() - f.close() - pos = 2 - if sys.path[0] == '': - pos += 1 - if sys.platform == 'win32': - paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')] - elif _is_jython: - paths = [os.path.join(sys.real_prefix, 'Lib')] - elif _is_pypy: - cpyver = '%d.%d.%d' % sys.version_info[:3] - paths = [os.path.join(sys.real_prefix, 'lib_pypy'), - os.path.join(sys.real_prefix, 'lib-python', 'modified-%s' % cpyver), - os.path.join(sys.real_prefix, 'lib-python', cpyver)] - else: - paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])] - lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3]) - if os.path.exists(lib64_path): - paths.append(lib64_path) - # This is hardcoded in the Python executable, but relative to sys.prefix: - plat_path = os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], - 'plat-%s' % sys.platform) - if os.path.exists(plat_path): - paths.append(plat_path) - # This is hardcoded in the Python executable, but - # relative to sys.prefix, so we have to fix up: - for path in list(paths): - tk_dir = os.path.join(path, 'lib-tk') - if os.path.exists(tk_dir): - paths.append(tk_dir) - - # These are hardcoded in the Apple's Python executable, - # but relative to sys.prefix, so we have to fix them up: - if sys.platform == 'darwin': - hardcoded_paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], module) - for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')] - - for path in hardcoded_paths: - if os.path.exists(path): - paths.append(path) - - sys.path.extend(paths) - -def force_global_eggs_after_local_site_packages(): - """ - Force easy_installed eggs in the global environment to get placed - in sys.path after all packages inside the virtualenv. This - maintains the "least surprise" result that packages in the - virtualenv always mask global packages, never the other way - around. - - """ - egginsert = getattr(sys, '__egginsert', 0) - for i, path in enumerate(sys.path): - if i > egginsert and path.startswith(sys.prefix): - egginsert = i - sys.__egginsert = egginsert + 1 - -def virtual_addsitepackages(known_paths): - force_global_eggs_after_local_site_packages() - return addsitepackages(known_paths, sys_prefix=sys.real_prefix) - -def fixclasspath(): - """Adjust the special classpath sys.path entries for Jython. These - entries should follow the base virtualenv lib directories. - """ - paths = [] - classpaths = [] - for path in sys.path: - if path == '__classpath__' or path.startswith('__pyclasspath__'): - classpaths.append(path) - else: - paths.append(path) - sys.path = paths - sys.path.extend(classpaths) - -def execusercustomize(): - """Run custom user specific code, if available.""" - try: - import usercustomize - except ImportError: - pass - - -def main(): - global ENABLE_USER_SITE - virtual_install_main_packages() - abs__file__() - paths_in_sys = removeduppaths() - if (os.name == "posix" and sys.path and - os.path.basename(sys.path[-1]) == "Modules"): - addbuilddir() - if _is_jython: - fixclasspath() - GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt')) - if not GLOBAL_SITE_PACKAGES: - ENABLE_USER_SITE = False - if ENABLE_USER_SITE is None: - ENABLE_USER_SITE = check_enableusersite() - paths_in_sys = addsitepackages(paths_in_sys) - paths_in_sys = addusersitepackages(paths_in_sys) - if GLOBAL_SITE_PACKAGES: - paths_in_sys = virtual_addsitepackages(paths_in_sys) - if sys.platform == 'os2emx': - setBEGINLIBPATH() - setquit() - setcopyright() - sethelper() - aliasmbcs() - setencoding() - execsitecustomize() - if ENABLE_USER_SITE: - execusercustomize() - # Remove sys.setdefaultencoding() so that users cannot change the - # encoding after initialization. The test for presence is needed when - # this module is run as a script, because this code is executed twice. - if hasattr(sys, "setdefaultencoding"): - del sys.setdefaultencoding - -main() - -def _script(): - help = """\ - %s [--user-base] [--user-site] - - Without arguments print some useful information - With arguments print the value of USER_BASE and/or USER_SITE separated - by '%s'. - - Exit codes with --user-base or --user-site: - 0 - user site directory is enabled - 1 - user site directory is disabled by user - 2 - uses site directory is disabled by super user - or for security reasons - >2 - unknown error - """ - args = sys.argv[1:] - if not args: - print "sys.path = [" - for dir in sys.path: - print " %r," % (dir,) - print "]" - def exists(path): - if os.path.isdir(path): - return "exists" - else: - return "doesn't exist" - print "USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE)) - print "USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE)) - print "ENABLE_USER_SITE: %r" % ENABLE_USER_SITE - sys.exit(0) - - buffer = [] - if '--user-base' in args: - buffer.append(USER_BASE) - if '--user-site' in args: - buffer.append(USER_SITE) - - if buffer: - print os.pathsep.join(buffer) - if ENABLE_USER_SITE: - sys.exit(0) - elif ENABLE_USER_SITE is False: - sys.exit(1) - elif ENABLE_USER_SITE is None: - sys.exit(2) - else: - sys.exit(3) - else: - import textwrap - print textwrap.dedent(help % (sys.argv[0], os.pathsep)) - sys.exit(10) - -if __name__ == '__main__': - _script() diff --git a/test/lib/python2.6/sre.py b/test/lib/python2.6/sre.py deleted file mode 120000 index 3ade4e80e..000000000 --- a/test/lib/python2.6/sre.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_compile.py b/test/lib/python2.6/sre_compile.py deleted file mode 120000 index fc13a37d5..000000000 --- a/test/lib/python2.6/sre_compile.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_compile.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_constants.py b/test/lib/python2.6/sre_constants.py deleted file mode 120000 index 98b0d5543..000000000 --- a/test/lib/python2.6/sre_constants.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_constants.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_parse.py b/test/lib/python2.6/sre_parse.py deleted file mode 120000 index 76b24c687..000000000 --- a/test/lib/python2.6/sre_parse.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_parse.py \ No newline at end of file diff --git a/test/lib/python2.6/stat.py b/test/lib/python2.6/stat.py deleted file mode 120000 index 68d61e662..000000000 --- a/test/lib/python2.6/stat.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/stat.py \ No newline at end of file diff --git a/test/lib/python2.6/types.py b/test/lib/python2.6/types.py deleted file mode 120000 index 72b00ba35..000000000 --- a/test/lib/python2.6/types.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/types.py \ No newline at end of file diff --git a/test/lib/python2.6/warnings.py b/test/lib/python2.6/warnings.py deleted file mode 120000 index 1af4e2833..000000000 --- a/test/lib/python2.6/warnings.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/warnings.py \ No newline at end of file -- cgit From dcfa9670a669ff881865750b2f96f62195dce2df Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Sat, 26 Feb 2011 19:19:02 +0100 Subject: renamed flag from maximum_... to max_... --- nova/virt/disk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 8789d8123..2bded07a4 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -40,7 +40,7 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') flags.DEFINE_integer('timeout_nbd', 10, 'time to wait for a NBD device coming up') -flags.DEFINE_integer('maximum_nbd_devices', 16, +flags.DEFINE_integer('max_nbd_devices', 16, 'maximum number of possible nbd devices') @@ -143,7 +143,7 @@ def _unlink_device(device, nbd): utils.execute('sudo losetup --detach %s' % device) -_DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.maximum_nbd_devices)] +_DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.max_nbd_devices)] def _allocate_device(): -- cgit From 8b3e9ad11c2f5c425701f1eb4abb7b3f577ae1cc Mon Sep 17 00:00:00 2001 From: Soren Hansen <soren@linux2go.dk> Date: Mon, 28 Feb 2011 12:37:02 +0100 Subject: Add utils.synchronized decorator to allow for synchronising method entrance across multiple workers on the same host. --- nova/tests/test_misc.py | 37 ++++++++++++++++++++++++++++++++++++- nova/utils.py | 11 +++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index e6da6112a..154b6fae6 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,10 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. +from datetime import datetime +import errno import os +import select +import time from nova import test -from nova.utils import parse_mailmap, str_dict_replace +from nova.utils import parse_mailmap, str_dict_replace, synchronized class ProjectTestCase(test.TestCase): @@ -55,3 +59,34 @@ class ProjectTestCase(test.TestCase): '%r not listed in Authors' % missing) finally: tree.unlock() + + +class LockTestCase(test.TestCase): + def test_synchronized(self): + rpipe, wpipe = os.pipe() + pid = os.fork() + if pid > 0: + os.close(wpipe) + + @synchronized('testlock') + def f(): + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") + os.close(rpipe) + + f() + else: + os.close(rpipe) + + @synchronized('testlock') + def g(): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return + g() + os._exit(0) diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..cb1ea5a7d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -25,6 +25,7 @@ import base64 import datetime import inspect import json +import lockfile import os import random import socket @@ -491,6 +492,16 @@ def loads(s): return json.loads(s) +def synchronized(name): + def wrap(f): + def inner(*args, **kwargs): + lock = lockfile.FileLock('nova-%s.lock' % name) + with lock: + return f(*args, **kwargs) + return inner + return wrap + + def ensure_b64_encoding(val): """Safety method to ensure that values expected to be base64-encoded actually are. If they are, the value is returned unchanged. Otherwise, -- cgit