From 41e4e18a0324593c0076c3936d63bb6dcca487cb Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Mon, 14 Feb 2011 23:12:34 +0000 Subject: First cut on XenServer unified-images --- nova/api/openstack/servers.py | 6 +- nova/virt/xenapi/vm_utils.py | 47 ++++- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 204 +++++++++++++++++---- 3 files changed, 217 insertions(+), 40 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 17c5519a1..1c9dcd9a6 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -140,7 +140,11 @@ class Controller(wsgi.Controller): image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) - return lookup('kernel_id'), lookup('ramdisk_id') + disk_format = image['properties'].get('disk_format', None) + if disk_format == "vhd": + return None, None + else: + return lookup('kernel_id'), lookup('ramdisk_id') def create(self, req): """ Creates a new server for a given user """ diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 4bbd522c1..6d6067da5 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -266,7 +266,9 @@ class VMHelper(HelperBase): session, instance_id, sr_ref, vm_vdi_ref, original_parent_uuid) #TODO(sirp): we need to assert only one parent, not parents two deep - return template_vm_ref, [template_vdi_uuid, parent_uuid] + template_vdi_uuids = {'image': parent_uuid, + 'snap': template_vdi_uuid} + return template_vm_ref, template_vdi_uuids @classmethod def upload_image(cls, session, instance_id, vdi_uuids, image_id): @@ -303,15 +305,36 @@ class VMHelper(HelperBase): return cls._fetch_image_objectstore(session, instance_id, image, access, user.secret, type) + @classmethod - def _fetch_image_glance(cls, session, instance_id, image, access, type): + def _fetch_image_glance_vhd(cls, session, instance_id, image, access, type): + LOG.debug(_("Asking xapi to fetch vhd image %(image)s") + % locals()) + + sr_ref = find_sr(session) + if sr_ref is None: + raise exception.NotFound('Cannot find SR to write VDI to') + + params = {'image_id': image, + 'glance_host': FLAGS.glance_host, + 'glance_port': FLAGS.glance_port} + + kwargs = {'params': pickle.dumps(params)} + task = session.async_call_plugin('glance', 'get_vdi', kwargs) + vdi_uuid = session.wait_for_task(instance_id, task) + scan_sr(session, instance_id, sr_ref) + LOG.debug(_("Xapi 'get_vdi' returned VDI UUID %(vdi_uuid)s") % locals()) + return vdi_uuid + + @classmethod + def _fetch_image_glance_disk(cls, session, instance_id, image, access, type): sr = find_sr(session) if sr is None: raise exception.NotFound('Cannot find SR to write VDI to') - c = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) + client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) - meta, image_file = c.get_image(image) + meta, image_file = client.get_image(image) virtual_size = int(meta['size']) vdi_size = virtual_size LOG.debug(_("Size for image %(image)s:%(virtual_size)d") % locals()) @@ -344,6 +367,22 @@ class VMHelper(HelperBase): else: return session.get_xenapi().VDI.get_uuid(vdi) + @classmethod + def _fetch_image_glance(cls, session, instance_id, image, access, type): + client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) + meta = client.get_image_meta(image) + properties = meta['properties'] + disk_format = properties.get('disk_format', None) + + # TODO(sirp): When Glance treats disk_format as a first class + # attribute, we should start using that rather than an image-property + if disk_format == 'vhd': + return cls._fetch_image_glance_vhd( + session, instance_id, image, access, type) + else: + return cls._fetch_image_glance_disk( + session, instance_id, image, access, type) + @classmethod def _fetch_image_objectstore(cls, session, instance_id, image, access, secret, type): diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index aadacce57..8352d7ee6 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -29,7 +29,10 @@ import os import os.path import pickle import sha +import shlex +import shutil import subprocess +import tempfile import time import urlparse @@ -66,65 +69,190 @@ def _copy_kernel_vdi(dest,copy_args): data=f.read(vdi_size) of.write(data) f.close() - of.close() + of.close() logging.debug("Done. Filename: %s",filename) - return filename + return filename + + +def execute(cmd): + args = shlex.split(cmd) + proc = subprocess.Popen( + args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) + return proc + + +def get_vdi(session, args): + """ + """ + params = pickle.loads(exists(args, 'params')) + image_id = params["image_id"] + glance_host = params["glance_host"] + glance_port = params["glance_port"] + + def unbundle_xfer(sr_path, staging_path): + """ + + """ + conn = httplib.HTTPConnection(glance_host, glance_port) + conn.request('GET', '/images/%s' % image_id) + resp = conn.getresponse() + if resp.status == httplib.NOT_FOUND: + raise Exception("Image '%s' not found in Glance" % image_id) + elif resp.status != httplib.OK: + raise Exception("Unexpected response from Glance %i" % res.status) + + tar_proc = execute("tar -zx --directory=%(staging_path)s" % locals()) + chunk = resp.read(CHUNK_SIZE) + while chunk: + tar_proc.stdin.write(chunk) + chunk = resp.read(CHUNK_SIZE) + out, err = tar_proc.communicate() + # TODO(sirp): write assert_process_success + ret = tar_proc.returncode + if ret != 0: + raise Exception( + "tar returned non-zero exit code (%i): '%s'" % (ret, err)) + conn.close() + + def fixup_vhds(sr_path, staging_path): + """ + """ + def rename_with_uuid(orig_path): + """Generate a uuid and rename the file with the uuid""" + orig_dirname = os.path.dirname(orig_path) + uuid = generate_uuid() + new_path = os.path.join(orig_dirname, "%s.vhd" % uuid) + os.rename(orig_path, new_path) + return new_path, uuid + + def move_into_sr(orig_path): + """Move a file into the SR""" + filename = os.path.basename(orig_path) + new_path = os.path.join(sr_path, filename) + #os.rename(orig_path, new_path) + # FIXME(sirp) : testing + shutil.copyfile(orig_path, new_path) + return new_path + + def link_vhds(child_path, parent_path): + proc = execute("vhd-util modify -n %(child_path)s -p %(parent_path)s" + % locals()) + out, err = proc.communicate() + if proc.returncode != 0: + raise Exception("Failed to link vhds: '%s'" % err) + + image_path = os.path.join(staging_path, 'image') + + orig_base_copy_path = os.path.join(image_path, 'image.vhd') + if not os.path.exists(orig_base_copy_path): + raise Exception("Invalid image: image.vhd not present") + + base_copy_path, base_copy_uuid = rename_with_uuid(orig_base_copy_path) + + vdi_uuid = base_copy_uuid + orig_snap_path = os.path.join(image_path, 'snap.vhd') + if os.path.exists(orig_snap_path): + snap_path, snap_uuid = rename_with_uuid(orig_snap_path) + vdi_uuid = snap_uuid + # NOTE(sirp): this step is necessary so that an SR scan won't + # delete the base_copy out from under us (since it would be + # orphaned) + link_vhds(snap_path, base_copy_path) + move_into_sr(snap_path) + else: + raise Exception("path '%s' not found!!!" % orig_snap_path) + move_into_sr(base_copy_path) + return vdi_uuid + + sr_path = get_sr_path(session) + staging_path = make_staging_area(sr_path) + try: + unbundle_xfer(sr_path, staging_path) + vdi_uuid = fixup_vhds(sr_path, staging_path) + return vdi_uuid + finally: + # FIXME(sirp) : testing + pass + #cleanup_staging_area(staging_path) + def put_vdis(session, args): + """ + """ params = pickle.loads(exists(args, 'params')) vdi_uuids = params["vdi_uuids"] image_id = params["image_id"] glance_host = params["glance_host"] glance_port = params["glance_port"] - - sr_path = get_sr_path(session) - #FIXME(sirp): writing to a temp file until Glance supports chunked-PUTs - tmp_file = "%s.tar.gz" % os.path.join('/tmp', str(image_id)) - tar_cmd = ['tar', '-zcf', tmp_file, '--directory=%s' % sr_path] - paths = [ "%s.vhd" % vdi_uuid for vdi_uuid in vdi_uuids ] - tar_cmd.extend(paths) - logging.debug("Bundling image with cmd: %s", tar_cmd) - subprocess.call(tar_cmd) - logging.debug("Writing to test file %s", tmp_file) - put_bundle_in_glance(tmp_file, image_id, glance_host, glance_port) - return "" # FIXME(sirp): return anything useful here? + def prepare_staging_area(sr_path, staging_path): + """ + Explain preparing staging area here... + """ + image_path = os.path.join(staging_path, 'image') + os.mkdir(image_path) + for name, uuid in vdi_uuids.items(): + source = os.path.join(sr_path, "%s.vhd" % uuid) + link_name = os.path.join(image_path, "%s.vhd" % name) + os.link(source, link_name) -def put_bundle_in_glance(tmp_file, image_id, glance_host, glance_port): - size = os.path.getsize(tmp_file) - basename = os.path.basename(tmp_file) + def bundle_xfer(staging_path): + conn = httplib.HTTPConnection(glance_host, glance_port) + #NOTE(sirp): httplib under python2.4 won't accept a file-like object + # to request + conn.putrequest('PUT', '/images/%s' % image_id) - bundle = open(tmp_file, 'r') - try: headers = { 'x-image-meta-store': 'file', 'x-image-meta-is_public': 'True', 'x-image-meta-type': 'raw', - 'x-image-meta-size': size, - 'content-length': size, + 'x-image-meta-property-disk-format': 'vhd', + 'x-image-meta-property-container-format': 'tarball', + 'transfer-encoding': "chunked", 'content-type': 'application/octet-stream', } - conn = httplib.HTTPConnection(glance_host, glance_port) - #NOTE(sirp): httplib under python2.4 won't accept a file-like object - # to request - conn.putrequest('PUT', '/images/%s' % image_id) - for header, value in headers.iteritems(): conn.putheader(header, value) conn.endheaders() - - chunk = bundle.read(CHUNK_SIZE) + + tar_proc = execute("tar -zc --directory=%(staging_path)s ." % locals()) + + chunk = tar_proc.stdout.read(CHUNK_SIZE) while chunk: - conn.send(chunk) - chunk = bundle.read(CHUNK_SIZE) - + conn.send("%x\r\n%s\r\n" % (len(chunk), chunk)) + chunk = tar_proc.stdout.read(CHUNK_SIZE) + conn.send("0\r\n\r\n") - res = conn.getresponse() + resp = conn.getresponse() #FIXME(sirp): should this be 201 Created? - if res.status != httplib.OK: + if resp.status != httplib.OK: raise Exception("Unexpected response from Glance %i" % res.status) + + conn.close() + + sr_path = get_sr_path(session) + staging_path = make_staging_area(sr_path) + try: + prepare_staging_area(sr_path, staging_path) + bundle_xfer(staging_path) finally: - bundle.close() + cleanup_staging_area(staging_path) + + return "" # FIXME(sirp): return anything useful here? + + +def make_staging_area(sr_path): + """ + Explain staging area here... + """ + # NOTE(sirp): staging area is created in SR to allow hard-linking + staging_path = tempfile.mkdtemp(dir=sr_path) + return staging_path + + +def cleanup_staging_area(staging_path): + shutil.rmtree(staging_path) + def get_sr_path(session): sr_ref = find_sr(session) @@ -154,7 +282,13 @@ def find_sr(session): return sr return None +def generate_uuid(): + # NOTE(sirp): Python2.4 does not include the uuid module + proc = execute("uuidgen") + uuid = proc.stdout.read().strip() + return uuid if __name__ == '__main__': - XenAPIPlugin.dispatch({'put_vdis': put_vdis, + XenAPIPlugin.dispatch({'put_vdis': put_vdis, + 'get_vdi': get_vdi, 'copy_kernel_vdi': copy_kernel_vdi}) -- cgit From 238ae2b5a8c559acc362a3b44160404771f1259f Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 00:00:56 +0000 Subject: Removing testing statements --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 8352d7ee6..0cb3b52db 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -129,9 +129,7 @@ def get_vdi(session, args): """Move a file into the SR""" filename = os.path.basename(orig_path) new_path = os.path.join(sr_path, filename) - #os.rename(orig_path, new_path) - # FIXME(sirp) : testing - shutil.copyfile(orig_path, new_path) + os.rename(orig_path, new_path) return new_path def link_vhds(child_path, parent_path): @@ -159,8 +157,7 @@ def get_vdi(session, args): # orphaned) link_vhds(snap_path, base_copy_path) move_into_sr(snap_path) - else: - raise Exception("path '%s' not found!!!" % orig_snap_path) + move_into_sr(base_copy_path) return vdi_uuid @@ -171,9 +168,7 @@ def get_vdi(session, args): vdi_uuid = fixup_vhds(sr_path, staging_path) return vdi_uuid finally: - # FIXME(sirp) : testing - pass - #cleanup_staging_area(staging_path) + cleanup_staging_area(staging_path) def put_vdis(session, args): -- cgit From fe6efb38ee30c5a3e532cd19faef0fec063b7446 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 01:37:54 +0000 Subject: Adding DISK_VHD to ImageTypes --- nova/virt/xenapi/vm_utils.py | 38 ++++++++++++++++++++++++++++++-------- nova/virt/xenapi/vmops.py | 22 +++++++++++++++------- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 6d6067da5..a6312d00c 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -63,11 +63,14 @@ class ImageType: 0 - kernel/ramdisk image (goes on dom0's filesystem) 1 - disk image (local SR, partitioned by objectstore plugin) 2 - raw disk image (local SR, NOT partitioned by plugin) + 3 - vhd disk image (local SR, NOT inspected by XS, PV assumed for + linux, HVM assumed for Windows) """ KERNEL_RAMDISK = 0 DISK = 1 DISK_RAW = 2 + DISK_VHD = 3 class VMHelper(HelperBase): @@ -368,15 +371,34 @@ class VMHelper(HelperBase): return session.get_xenapi().VDI.get_uuid(vdi) @classmethod - def _fetch_image_glance(cls, session, instance_id, image, access, type): - client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) - meta = client.get_image_meta(image) - properties = meta['properties'] - disk_format = properties.get('disk_format', None) + def determine_disk_image_type(cls, instance): + instance_id = instance.id + if instance.kernel_id: + #if kernel is not present we must download a raw disk + LOG.debug(_("Instance %(instance_id)s will use DISK format") % + locals()) + return ImageType.DISK - # TODO(sirp): When Glance treats disk_format as a first class - # attribute, we should start using that rather than an image-property - if disk_format == 'vhd': + if FLAGS.xenapi_image_service == 'glance': + # if using glance, then we could be VHD format + client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) + meta = client.get_image_meta(instance.image_id) + properties = meta['properties'] + disk_format = properties.get('disk_format', None) + # TODO(sirp): When Glance treats disk_format as a first class + # attribute, we should start using that rather than an image-property + if disk_format == 'vhd': + LOG.debug(_("Instance %(instance_id)s will use DISK_VHD format") % + locals()) + return ImageType.DISK_VHD + + LOG.debug(_("Instance %(instance_id)s will use DISK_RAW format") % + locals()) + return ImageType.DISK_RAW + + @classmethod + def _fetch_image_glance(cls, session, instance_id, image, access, type): + if type == ImageType.DISK_VHD: return cls._fetch_image_glance_vhd( session, instance_id, image, access, type) else: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index e84ce20c4..34ceea358 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -74,27 +74,34 @@ class VMOps(object): user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) - #if kernel is not present we must download a raw disk - if instance.kernel_id: - disk_image_type = ImageType.DISK - else: - disk_image_type = ImageType.DISK_RAW + + disk_image_type = VMHelper.determine_disk_image_type(instance) + vdi_uuid = VMHelper.fetch_image(self._session, instance.id, instance.image_id, user, project, disk_image_type) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - #Have a look at the VDI and see if it has a PV kernel + pv_kernel = False - if not instance.kernel_id: + if disk_image_type == ImageType.DISK_RAW: + #Have a look at the VDI and see if it has a PV kernel pv_kernel = VMHelper.lookup_image(self._session, instance.id, vdi_ref) + elif disk_image_type == ImageType.DISK_VHD: + # TODO(sirp): Assuming PV for now; this will need to be + # configurable as Windows will use HVM. + pv_kernel = True + kernel = None if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK) + ramdisk = None if instance.ramdisk_id: ramdisk = VMHelper.fetch_image(self._session, instance.id, instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) + vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, pv_kernel) VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) @@ -102,6 +109,7 @@ class VMOps(object): if network_ref: VMHelper.create_vif(self._session, vm_ref, network_ref, instance.mac_address) + LOG.debug(_('Starting VM %s...'), vm_ref) self._session.call_xenapi('VM.start', vm_ref, False, False) instance_name = instance.name -- cgit From 15cdeef7820aacd0b1ff95da48816cba9f2544ba Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 18:01:13 +0000 Subject: Adding more documentation, code-cleanup --- nova/virt/xenapi/vm_utils.py | 52 ++++++++++++++----- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 60 +++++++++++----------- 2 files changed, 70 insertions(+), 42 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index a6312d00c..5eab2a38f 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -24,6 +24,7 @@ import pickle import re import time import urllib +import uuid from xml.dom import minidom from eventlet import event @@ -318,9 +319,15 @@ class VMHelper(HelperBase): if sr_ref is None: raise exception.NotFound('Cannot find SR to write VDI to') + # NOTE(sirp): The Glance plugin runs under Python 2.4 which does not + # have the `uuid` module. To work around this, we generate the uuids + # here (under Python 2.6+) and pass them as arguments + uuid_stack = [str(uuid.uuid4()) for i in xrange(2)] + params = {'image_id': image, 'glance_host': FLAGS.glance_host, - 'glance_port': FLAGS.glance_port} + 'glance_port': FLAGS.glance_port, + 'uuid_stack': uuid_stack} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'get_vdi', kwargs) @@ -372,14 +379,35 @@ class VMHelper(HelperBase): @classmethod def determine_disk_image_type(cls, instance): - instance_id = instance.id + """Disk Image Types are used to determine where the kernel will reside + within an image. To figure out which type we're dealing with, we use + the following rules: + + 1. If the instance is specifying a kernel explicitly, we must be using + a 'disk' image (kernel outside of the image) + + 2. If the kernel isn't specified, then we have two different + scenarios: + + a) If the image is in Glance, then we can use the 'disk_format' + property to determine if the image is really a VHD-style image + or if it's a RAW image + + b) If the image is not in Glance, then it must be a RAW image + (since we don't have a way of identifying VHD images...yet) + """ + def log_disk_format(disk_format): + disk_format = disk_format.upper() + image_id = instance.image_id + instance_id = instance.id + LOG.debug(_("Detected %(disk_format)s format for image " + "%(image_id)s, instance %(instance_id)s") % locals()) + if instance.kernel_id: - #if kernel is not present we must download a raw disk - LOG.debug(_("Instance %(instance_id)s will use DISK format") % - locals()) + # 1. DISK + log_disk_format('disk') return ImageType.DISK - - if FLAGS.xenapi_image_service == 'glance': + elif FLAGS.xenapi_image_service == 'glance': # if using glance, then we could be VHD format client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) meta = client.get_image_meta(instance.image_id) @@ -388,12 +416,12 @@ class VMHelper(HelperBase): # TODO(sirp): When Glance treats disk_format as a first class # attribute, we should start using that rather than an image-property if disk_format == 'vhd': - LOG.debug(_("Instance %(instance_id)s will use DISK_VHD format") % - locals()) + # 2a. DISK_VHD + log_disk_format('disk_vhd') return ImageType.DISK_VHD - - LOG.debug(_("Instance %(instance_id)s will use DISK_RAW format") % - locals()) + + # 2b. DISK_RAW + log_disk_format('disk_raw') return ImageType.DISK_RAW @classmethod diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 0cb3b52db..2018dca5f 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -21,20 +21,14 @@ # XenAPI plugin for managing glance images # -import base64 -import errno -import hmac import httplib import os import os.path import pickle -import sha import shlex import shutil import subprocess import tempfile -import time -import urlparse import XenAPIPlugin @@ -74,11 +68,14 @@ def _copy_kernel_vdi(dest,copy_args): return filename -def execute(cmd): - args = shlex.split(cmd) - proc = subprocess.Popen( - args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) - return proc +def assert_process_success(proc, cmd): + """Ensure that the process returned a zero exit code indicating success + """ + out, err = proc.communicate() + ret = proc.returncode + if ret != 0: + msg = "%(cmd)s returned non-zero exit code (%i): '%s'" % (ret, err) + raise Exception(msg) def get_vdi(session, args): @@ -88,6 +85,7 @@ def get_vdi(session, args): image_id = params["image_id"] glance_host = params["glance_host"] glance_port = params["glance_port"] + uuid_stack = params["uuid_stack"] def unbundle_xfer(sr_path, staging_path): """ @@ -101,17 +99,17 @@ def get_vdi(session, args): elif resp.status != httplib.OK: raise Exception("Unexpected response from Glance %i" % res.status) - tar_proc = execute("tar -zx --directory=%(staging_path)s" % locals()) + tar_args = shlex.split( + "tar -zx --directory=%(staging_path)s" % locals()) + tar_proc = subprocess.Popen( + tar_args, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + chunk = resp.read(CHUNK_SIZE) while chunk: tar_proc.stdin.write(chunk) chunk = resp.read(CHUNK_SIZE) - out, err = tar_proc.communicate() - # TODO(sirp): write assert_process_success - ret = tar_proc.returncode - if ret != 0: - raise Exception( - "tar returned non-zero exit code (%i): '%s'" % (ret, err)) + + assert_process_success(tar_proc, "tar") conn.close() def fixup_vhds(sr_path, staging_path): @@ -120,7 +118,7 @@ def get_vdi(session, args): def rename_with_uuid(orig_path): """Generate a uuid and rename the file with the uuid""" orig_dirname = os.path.dirname(orig_path) - uuid = generate_uuid() + uuid = uuid_stack.pop() new_path = os.path.join(orig_dirname, "%s.vhd" % uuid) os.rename(orig_path, new_path) return new_path, uuid @@ -133,11 +131,13 @@ def get_vdi(session, args): return new_path def link_vhds(child_path, parent_path): - proc = execute("vhd-util modify -n %(child_path)s -p %(parent_path)s" - % locals()) - out, err = proc.communicate() - if proc.returncode != 0: - raise Exception("Failed to link vhds: '%s'" % err) + modify_args = shlex.split( + "vhd-util modify -n %(child_path)s -p %(parent_path)s" + % locals()) + modify_proc = subprocess.Popen( + modify_args, stderr=subprocess.PIPE) + assert_process_success(modify_proc, "vhd-util") + image_path = os.path.join(staging_path, 'image') @@ -210,7 +210,10 @@ def put_vdis(session, args): conn.putheader(header, value) conn.endheaders() - tar_proc = execute("tar -zc --directory=%(staging_path)s ." % locals()) + tar_args = shlex.split( + "tar -zc --directory=%(staging_path)s ." % locals()) + tar_proc = subprocess.Popen( + tar_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) chunk = tar_proc.stdout.read(CHUNK_SIZE) while chunk: @@ -218,6 +221,8 @@ def put_vdis(session, args): chunk = tar_proc.stdout.read(CHUNK_SIZE) conn.send("0\r\n\r\n") + assert_process_success(tar_proc, "tar") + resp = conn.getresponse() #FIXME(sirp): should this be 201 Created? if resp.status != httplib.OK: @@ -277,11 +282,6 @@ def find_sr(session): return sr return None -def generate_uuid(): - # NOTE(sirp): Python2.4 does not include the uuid module - proc = execute("uuidgen") - uuid = proc.stdout.read().strip() - return uuid if __name__ == '__main__': XenAPIPlugin.dispatch({'put_vdis': put_vdis, -- cgit From 0fc3a184230c479254b9f713ea61de2f24f680ab Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 18:23:14 +0000 Subject: Moving SR path code outside of glance plugin --- nova/virt/xenapi/vm_utils.py | 21 +++++++++++-- nova/virt/xenapi_conn.py | 2 ++ plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 34 ++-------------------- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 5eab2a38f..e73bc7f2b 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -285,7 +285,8 @@ class VMHelper(HelperBase): params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, 'glance_host': FLAGS.glance_host, - 'glance_port': FLAGS.glance_port} + 'glance_port': FLAGS.glance_port, + 'sr_path': get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'put_vdis', kwargs) @@ -327,7 +328,8 @@ class VMHelper(HelperBase): params = {'image_id': image, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, - 'uuid_stack': uuid_stack} + 'uuid_stack': uuid_stack, + 'sr_path': get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'get_vdi', kwargs) @@ -685,6 +687,21 @@ def find_sr(session): return None +def get_sr_path(session): + """Return the path to our Storage Repository + + This is used when we're dealing with VHDs directly, either by taking + snapshots or by restoring an image in the DISK_VHD format. + """ + # TODO(sirp): add safe_find_sr + sr_ref = find_sr(session) + if sr_ref is None: + raise Exception('Cannot find SR to read VDI from') + sr_rec = session.get_xenapi().SR.get_record(sr_ref) + sr_uuid = sr_rec["uuid"] + return os.path.join(FLAGS.xenapi_sr_base_path, sr_uuid) + + def remap_vbd_dev(dev): """Return the appropriate location for a plugged-in VBD device diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index a0b0499b8..8ae5684b0 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -100,6 +100,8 @@ flags.DEFINE_integer('xenapi_vhd_coalesce_max_attempts', 5, 'Max number of times to poll for VHD to coalesce.' ' Used only if connection_type=xenapi.') +flags.DEFINE_string('xenapi_sr_base_path', '/var/run/sr-mount', + 'Base path to the storage repository') flags.DEFINE_string('target_host', None, 'iSCSI Target Host') diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 2018dca5f..0b270f5f9 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -38,7 +38,6 @@ configure_logging('glance') CHUNK_SIZE = 8192 KERNEL_DIR = '/boot/guest' -FILE_SR_PATH = '/var/run/sr-mount' def copy_kernel_vdi(session,args): vdi = exists(args, 'vdi-ref') @@ -86,6 +85,7 @@ def get_vdi(session, args): glance_host = params["glance_host"] glance_port = params["glance_port"] uuid_stack = params["uuid_stack"] + sr_path = params["sr_path"] def unbundle_xfer(sr_path, staging_path): """ @@ -161,7 +161,6 @@ def get_vdi(session, args): move_into_sr(base_copy_path) return vdi_uuid - sr_path = get_sr_path(session) staging_path = make_staging_area(sr_path) try: unbundle_xfer(sr_path, staging_path) @@ -179,6 +178,7 @@ def put_vdis(session, args): image_id = params["image_id"] glance_host = params["glance_host"] glance_port = params["glance_port"] + sr_path = params["sr_path"] def prepare_staging_area(sr_path, staging_path): """ @@ -230,7 +230,6 @@ def put_vdis(session, args): conn.close() - sr_path = get_sr_path(session) staging_path = make_staging_area(sr_path) try: prepare_staging_area(sr_path, staging_path) @@ -254,35 +253,6 @@ def cleanup_staging_area(staging_path): shutil.rmtree(staging_path) -def get_sr_path(session): - sr_ref = find_sr(session) - - if sr_ref is None: - raise Exception('Cannot find SR to read VDI from') - - sr_rec = session.xenapi.SR.get_record(sr_ref) - sr_uuid = sr_rec["uuid"] - sr_path = os.path.join(FILE_SR_PATH, sr_uuid) - return sr_path - - -#TODO(sirp): both objectstore and glance need this, should this be refactored -#into common lib -def find_sr(session): - host = get_this_host(session) - srs = session.xenapi.SR.get_all() - for sr in srs: - sr_rec = session.xenapi.SR.get_record(sr) - if not ('i18n-key' in sr_rec['other_config'] and - sr_rec['other_config']['i18n-key'] == 'local-storage'): - continue - for pbd in sr_rec['PBDs']: - pbd_rec = session.xenapi.PBD.get_record(pbd) - if pbd_rec['host'] == host: - return sr - return None - - if __name__ == '__main__': XenAPIPlugin.dispatch({'put_vdis': put_vdis, 'get_vdi': get_vdi, -- cgit From b4b1a7fbd55784157b3084016d4dfe2bd0120e51 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 18:36:17 +0000 Subject: Adding safe_find_sr --- nova/virt/xenapi/vm_utils.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index e73bc7f2b..37ce86885 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -316,9 +316,7 @@ class VMHelper(HelperBase): LOG.debug(_("Asking xapi to fetch vhd image %(image)s") % locals()) - sr_ref = find_sr(session) - if sr_ref is None: - raise exception.NotFound('Cannot find SR to write VDI to') + sr_ref = safe_find_sr(session) # NOTE(sirp): The Glance plugin runs under Python 2.4 which does not # have the `uuid` module. To work around this, we generate the uuids @@ -340,16 +338,14 @@ class VMHelper(HelperBase): @classmethod def _fetch_image_glance_disk(cls, session, instance_id, image, access, type): - sr = find_sr(session) - if sr is None: - raise exception.NotFound('Cannot find SR to write VDI to') + sr_ref = safe_find_sr(session) client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) - meta, image_file = client.get_image(image) virtual_size = int(meta['size']) vdi_size = virtual_size LOG.debug(_("Size for image %(image)s:%(virtual_size)d") % locals()) + if type == ImageType.DISK: # Make room for MBR. vdi_size += MBR_SIZE_BYTES @@ -672,7 +668,18 @@ def get_vdi_for_vm_safely(session, vm_ref): return vdi_ref, vdi_rec +def safe_find_sr(session): + """Same as find_sr except raises a NotFound exception if SR cannot be + determined + """ + sr_ref = find_sr(session) + if sr_ref is None: + raise exception.NotFound(_('Cannot find SR to read/write VDI')) + return sr_ref + + def find_sr(session): + """Return the storage repository to hold VM images""" host = session.get_xenapi_host() srs = session.get_xenapi().SR.get_all() for sr in srs: @@ -688,15 +695,12 @@ def find_sr(session): def get_sr_path(session): - """Return the path to our Storage Repository + """Return the path to our storage repository This is used when we're dealing with VHDs directly, either by taking snapshots or by restoring an image in the DISK_VHD format. """ - # TODO(sirp): add safe_find_sr - sr_ref = find_sr(session) - if sr_ref is None: - raise Exception('Cannot find SR to read VDI from') + sr_ref = safe_find_sr(session) sr_rec = session.get_xenapi().SR.get_record(sr_ref) sr_uuid = sr_rec["uuid"] return os.path.join(FLAGS.xenapi_sr_base_path, sr_uuid) -- cgit From eb603b5ec3d54b2b6c893f8d41e7d12bbaa49e57 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 18:50:40 +0000 Subject: Refactoring put_vdis --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 136 +++++++++++---------- 1 file changed, 69 insertions(+), 67 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 0b270f5f9..3fe2d4059 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -67,16 +67,6 @@ def _copy_kernel_vdi(dest,copy_args): return filename -def assert_process_success(proc, cmd): - """Ensure that the process returned a zero exit code indicating success - """ - out, err = proc.communicate() - ret = proc.returncode - if ret != 0: - msg = "%(cmd)s returned non-zero exit code (%i): '%s'" % (ret, err) - raise Exception(msg) - - def get_vdi(session, args): """ """ @@ -109,7 +99,7 @@ def get_vdi(session, args): tar_proc.stdin.write(chunk) chunk = resp.read(CHUNK_SIZE) - assert_process_success(tar_proc, "tar") + _assert_process_success(tar_proc, "tar") conn.close() def fixup_vhds(sr_path, staging_path): @@ -136,7 +126,7 @@ def get_vdi(session, args): % locals()) modify_proc = subprocess.Popen( modify_args, stderr=subprocess.PIPE) - assert_process_success(modify_proc, "vhd-util") + _assert_process_success(modify_proc, "vhd-util") image_path = os.path.join(staging_path, 'image') @@ -161,13 +151,13 @@ def get_vdi(session, args): move_into_sr(base_copy_path) return vdi_uuid - staging_path = make_staging_area(sr_path) + staging_path = _make_staging_area(sr_path) try: unbundle_xfer(sr_path, staging_path) vdi_uuid = fixup_vhds(sr_path, staging_path) return vdi_uuid finally: - cleanup_staging_area(staging_path) + _cleanup_staging_area(staging_path) def put_vdis(session, args): @@ -180,67 +170,69 @@ def put_vdis(session, args): glance_port = params["glance_port"] sr_path = params["sr_path"] - def prepare_staging_area(sr_path, staging_path): - """ - Explain preparing staging area here... - """ - image_path = os.path.join(staging_path, 'image') - os.mkdir(image_path) - for name, uuid in vdi_uuids.items(): - source = os.path.join(sr_path, "%s.vhd" % uuid) - link_name = os.path.join(image_path, "%s.vhd" % name) - os.link(source, link_name) - - def bundle_xfer(staging_path): - conn = httplib.HTTPConnection(glance_host, glance_port) - #NOTE(sirp): httplib under python2.4 won't accept a file-like object - # to request - conn.putrequest('PUT', '/images/%s' % image_id) - - headers = { - 'x-image-meta-store': 'file', - 'x-image-meta-is_public': 'True', - 'x-image-meta-type': 'raw', - 'x-image-meta-property-disk-format': 'vhd', - 'x-image-meta-property-container-format': 'tarball', - 'transfer-encoding': "chunked", - 'content-type': 'application/octet-stream', - } - for header, value in headers.iteritems(): - conn.putheader(header, value) - conn.endheaders() + staging_path = _make_staging_area(sr_path) + try: + _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids) + _bundle_xfer(staging_path, image_id, glance_host, glance_port) + finally: + _cleanup_staging_area(staging_path) - tar_args = shlex.split( - "tar -zc --directory=%(staging_path)s ." % locals()) - tar_proc = subprocess.Popen( - tar_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return "" - chunk = tar_proc.stdout.read(CHUNK_SIZE) - while chunk: - conn.send("%x\r\n%s\r\n" % (len(chunk), chunk)) - chunk = tar_proc.stdout.read(CHUNK_SIZE) - conn.send("0\r\n\r\n") - assert_process_success(tar_proc, "tar") +def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): + """ + Explain preparing staging area here... + """ + image_path = os.path.join(staging_path, 'image') + os.mkdir(image_path) + for name, uuid in vdi_uuids.items(): + source = os.path.join(sr_path, "%s.vhd" % uuid) + link_name = os.path.join(image_path, "%s.vhd" % name) + os.link(source, link_name) - resp = conn.getresponse() - #FIXME(sirp): should this be 201 Created? - if resp.status != httplib.OK: - raise Exception("Unexpected response from Glance %i" % res.status) - conn.close() +def _bundle_xfer(staging_path, image_id, glance_host, glance_port): + """ + """ + conn = httplib.HTTPConnection(glance_host, glance_port) + #NOTE(sirp): httplib under python2.4 won't accept a file-like object + # to request + conn.putrequest('PUT', '/images/%s' % image_id) + + headers = { + 'x-image-meta-store': 'file', + 'x-image-meta-is_public': 'True', + 'x-image-meta-type': 'raw', + 'x-image-meta-property-disk-format': 'vhd', + 'x-image-meta-property-container-format': 'tarball', + 'transfer-encoding': "chunked", + 'content-type': 'application/octet-stream', + } + for header, value in headers.iteritems(): + conn.putheader(header, value) + conn.endheaders() + + tar_args = shlex.split( + "tar -zc --directory=%(staging_path)s ." % locals()) + tar_proc = subprocess.Popen( + tar_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + chunk = tar_proc.stdout.read(CHUNK_SIZE) + while chunk: + conn.send("%x\r\n%s\r\n" % (len(chunk), chunk)) + chunk = tar_proc.stdout.read(CHUNK_SIZE) + conn.send("0\r\n\r\n") - staging_path = make_staging_area(sr_path) - try: - prepare_staging_area(sr_path, staging_path) - bundle_xfer(staging_path) - finally: - cleanup_staging_area(staging_path) + _assert_process_success(tar_proc, "tar") - return "" # FIXME(sirp): return anything useful here? + resp = conn.getresponse() + if resp.status != httplib.OK: + raise Exception("Unexpected response from Glance %i" % res.status) + conn.close() -def make_staging_area(sr_path): +def _make_staging_area(sr_path): """ Explain staging area here... """ @@ -249,10 +241,20 @@ def make_staging_area(sr_path): return staging_path -def cleanup_staging_area(staging_path): +def _cleanup_staging_area(staging_path): shutil.rmtree(staging_path) +def _assert_process_success(proc, cmd): + """Ensure that the process returned a zero exit code indicating success + """ + out, err = proc.communicate() + ret = proc.returncode + if ret != 0: + msg = "%(cmd)s returned non-zero exit code (%i): '%s'" % (ret, err) + raise Exception(msg) + + if __name__ == '__main__': XenAPIPlugin.dispatch({'put_vdis': put_vdis, 'get_vdi': get_vdi, -- cgit From acf95a640cfeb0812a55577b6a08bff972ad523b Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 19:17:27 +0000 Subject: Regrouping methods so they make sense --- nova/virt/xenapi/vm_utils.py | 7 +- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 233 +++++++++++---------- 2 files changed, 123 insertions(+), 117 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 37ce86885..ba3e5e423 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -289,7 +289,7 @@ class VMHelper(HelperBase): 'sr_path': get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} - task = session.async_call_plugin('glance', 'put_vdis', kwargs) + task = session.async_call_plugin('glance', 'upload_image', kwargs) session.wait_for_task(instance_id, task) @classmethod @@ -310,7 +310,6 @@ class VMHelper(HelperBase): return cls._fetch_image_objectstore(session, instance_id, image, access, user.secret, type) - @classmethod def _fetch_image_glance_vhd(cls, session, instance_id, image, access, type): LOG.debug(_("Asking xapi to fetch vhd image %(image)s") @@ -330,10 +329,10 @@ class VMHelper(HelperBase): 'sr_path': get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} - task = session.async_call_plugin('glance', 'get_vdi', kwargs) + task = session.async_call_plugin('glance', 'download_image', kwargs) vdi_uuid = session.wait_for_task(instance_id, task) scan_sr(session, instance_id, sr_ref) - LOG.debug(_("Xapi 'get_vdi' returned VDI UUID %(vdi_uuid)s") % locals()) + LOG.debug(_("xapi 'download_image' returned VDI UUID %(vdi_uuid)s") % locals()) return vdi_uuid @classmethod diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 3fe2d4059..7bbab4f52 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -39,17 +39,6 @@ configure_logging('glance') CHUNK_SIZE = 8192 KERNEL_DIR = '/boot/guest' -def copy_kernel_vdi(session,args): - vdi = exists(args, 'vdi-ref') - size = exists(args,'image-size') - #Use the uuid as a filename - vdi_uuid=session.xenapi.VDI.get_uuid(vdi) - copy_args={'vdi_uuid':vdi_uuid,'vdi_size':int(size)} - filename=with_vdi_in_dom0(session, vdi, False, - lambda dev: - _copy_kernel_vdi('/dev/%s' % dev,copy_args)) - return filename - def _copy_kernel_vdi(dest,copy_args): vdi_uuid=copy_args['vdi_uuid'] vdi_size=copy_args['vdi_size'] @@ -67,117 +56,83 @@ def _copy_kernel_vdi(dest,copy_args): return filename -def get_vdi(session, args): - """ +def _download_tarball(sr_path, staging_path, image_id, glance_host, + glance_port): """ - params = pickle.loads(exists(args, 'params')) - image_id = params["image_id"] - glance_host = params["glance_host"] - glance_port = params["glance_port"] - uuid_stack = params["uuid_stack"] - sr_path = params["sr_path"] - - def unbundle_xfer(sr_path, staging_path): - """ - """ - conn = httplib.HTTPConnection(glance_host, glance_port) - conn.request('GET', '/images/%s' % image_id) - resp = conn.getresponse() - if resp.status == httplib.NOT_FOUND: - raise Exception("Image '%s' not found in Glance" % image_id) - elif resp.status != httplib.OK: - raise Exception("Unexpected response from Glance %i" % res.status) + """ + conn = httplib.HTTPConnection(glance_host, glance_port) + conn.request('GET', '/images/%s' % image_id) + resp = conn.getresponse() + if resp.status == httplib.NOT_FOUND: + raise Exception("Image '%s' not found in Glance" % image_id) + elif resp.status != httplib.OK: + raise Exception("Unexpected response from Glance %i" % res.status) - tar_args = shlex.split( - "tar -zx --directory=%(staging_path)s" % locals()) - tar_proc = subprocess.Popen( - tar_args, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + tar_args = shlex.split( + "tar -zx --directory=%(staging_path)s" % locals()) + tar_proc = subprocess.Popen( + tar_args, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + chunk = resp.read(CHUNK_SIZE) + while chunk: + tar_proc.stdin.write(chunk) chunk = resp.read(CHUNK_SIZE) - while chunk: - tar_proc.stdin.write(chunk) - chunk = resp.read(CHUNK_SIZE) - - _assert_process_success(tar_proc, "tar") - conn.close() - - def fixup_vhds(sr_path, staging_path): - """ - """ - def rename_with_uuid(orig_path): - """Generate a uuid and rename the file with the uuid""" - orig_dirname = os.path.dirname(orig_path) - uuid = uuid_stack.pop() - new_path = os.path.join(orig_dirname, "%s.vhd" % uuid) - os.rename(orig_path, new_path) - return new_path, uuid - - def move_into_sr(orig_path): - """Move a file into the SR""" - filename = os.path.basename(orig_path) - new_path = os.path.join(sr_path, filename) - os.rename(orig_path, new_path) - return new_path - - def link_vhds(child_path, parent_path): - modify_args = shlex.split( - "vhd-util modify -n %(child_path)s -p %(parent_path)s" - % locals()) - modify_proc = subprocess.Popen( - modify_args, stderr=subprocess.PIPE) - _assert_process_success(modify_proc, "vhd-util") - - - image_path = os.path.join(staging_path, 'image') - - orig_base_copy_path = os.path.join(image_path, 'image.vhd') - if not os.path.exists(orig_base_copy_path): - raise Exception("Invalid image: image.vhd not present") - - base_copy_path, base_copy_uuid = rename_with_uuid(orig_base_copy_path) - - vdi_uuid = base_copy_uuid - orig_snap_path = os.path.join(image_path, 'snap.vhd') - if os.path.exists(orig_snap_path): - snap_path, snap_uuid = rename_with_uuid(orig_snap_path) - vdi_uuid = snap_uuid - # NOTE(sirp): this step is necessary so that an SR scan won't - # delete the base_copy out from under us (since it would be - # orphaned) - link_vhds(snap_path, base_copy_path) - move_into_sr(snap_path) - - move_into_sr(base_copy_path) - return vdi_uuid - staging_path = _make_staging_area(sr_path) - try: - unbundle_xfer(sr_path, staging_path) - vdi_uuid = fixup_vhds(sr_path, staging_path) - return vdi_uuid - finally: - _cleanup_staging_area(staging_path) + _assert_process_success(tar_proc, "tar") + conn.close() -def put_vdis(session, args): +def fixup_vhds(sr_path, staging_path, uuid_stack): """ """ - params = pickle.loads(exists(args, 'params')) - vdi_uuids = params["vdi_uuids"] - image_id = params["image_id"] - glance_host = params["glance_host"] - glance_port = params["glance_port"] - sr_path = params["sr_path"] + def rename_with_uuid(orig_path): + """Generate a uuid and rename the file with the uuid""" + orig_dirname = os.path.dirname(orig_path) + uuid = uuid_stack.pop() + new_path = os.path.join(orig_dirname, "%s.vhd" % uuid) + os.rename(orig_path, new_path) + return new_path, uuid - staging_path = _make_staging_area(sr_path) - try: - _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids) - _bundle_xfer(staging_path, image_id, glance_host, glance_port) - finally: - _cleanup_staging_area(staging_path) - return "" + def link_vhds(child_path, parent_path): + modify_args = shlex.split( + "vhd-util modify -n %(child_path)s -p %(parent_path)s" + % locals()) + modify_proc = subprocess.Popen( + modify_args, stderr=subprocess.PIPE) + _assert_process_success(modify_proc, "vhd-util") + + + def move_into_sr(orig_path): + """Move a file into the SR""" + filename = os.path.basename(orig_path) + new_path = os.path.join(sr_path, filename) + os.rename(orig_path, new_path) + return new_path + + + image_path = os.path.join(staging_path, 'image') + + orig_base_copy_path = os.path.join(image_path, 'image.vhd') + if not os.path.exists(orig_base_copy_path): + raise Exception("Invalid image: image.vhd not present") + + base_copy_path, base_copy_uuid = rename_with_uuid(orig_base_copy_path) + + vdi_uuid = base_copy_uuid + orig_snap_path = os.path.join(image_path, 'snap.vhd') + if os.path.exists(orig_snap_path): + snap_path, snap_uuid = rename_with_uuid(orig_snap_path) + vdi_uuid = snap_uuid + # NOTE(sirp): this step is necessary so that an SR scan won't + # delete the base_copy out from under us (since it would be + # orphaned) + link_vhds(snap_path, base_copy_path) + move_into_sr(snap_path) + + move_into_sr(base_copy_path) + return vdi_uuid def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): @@ -192,7 +147,7 @@ def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): os.link(source, link_name) -def _bundle_xfer(staging_path, image_id, glance_host, glance_port): +def _upload_tarball(staging_path, image_id, glance_host, glance_port): """ """ conn = httplib.HTTPConnection(glance_host, glance_port) @@ -255,7 +210,59 @@ def _assert_process_success(proc, cmd): raise Exception(msg) +def download_image(session, args): + """ + """ + params = pickle.loads(exists(args, 'params')) + image_id = params["image_id"] + glance_host = params["glance_host"] + glance_port = params["glance_port"] + uuid_stack = params["uuid_stack"] + sr_path = params["sr_path"] + + staging_path = _make_staging_area(sr_path) + try: + _download_tarball(sr_path, staging_path, image_id, glance_host, + glance_port) + vdi_uuid = fixup_vhds(sr_path, staging_path, uuid_stack) + return vdi_uuid + finally: + _cleanup_staging_area(staging_path) + + +def upload_image(session, args): + """ + """ + params = pickle.loads(exists(args, 'params')) + vdi_uuids = params["vdi_uuids"] + image_id = params["image_id"] + glance_host = params["glance_host"] + glance_port = params["glance_port"] + sr_path = params["sr_path"] + + staging_path = _make_staging_area(sr_path) + try: + _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids) + _upload_tarball(staging_path, image_id, glance_host, glance_port) + finally: + _cleanup_staging_area(staging_path) + + return "" + + +def copy_kernel_vdi(session,args): + vdi = exists(args, 'vdi-ref') + size = exists(args,'image-size') + #Use the uuid as a filename + vdi_uuid=session.xenapi.VDI.get_uuid(vdi) + copy_args={'vdi_uuid':vdi_uuid,'vdi_size':int(size)} + filename=with_vdi_in_dom0(session, vdi, False, + lambda dev: + _copy_kernel_vdi('/dev/%s' % dev,copy_args)) + return filename + + if __name__ == '__main__': - XenAPIPlugin.dispatch({'put_vdis': put_vdis, - 'get_vdi': get_vdi, + XenAPIPlugin.dispatch({'upload_image': upload_image, + 'download_image': download_image, 'copy_kernel_vdi': copy_kernel_vdi}) -- cgit From 300657f298fbecf9a08792b6d15e462560a6cdf5 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 19:51:23 +0000 Subject: Adding documentation --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 85 +++++++++++++++++++--- 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 7bbab4f52..dac773ff1 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -58,8 +58,8 @@ def _copy_kernel_vdi(dest,copy_args): def _download_tarball(sr_path, staging_path, image_id, glance_host, glance_port): - """ - + """Download the tarball image from Glance and extract it into the staging + area. """ conn = httplib.HTTPConnection(glance_host, glance_port) conn.request('GET', '/images/%s' % image_id) @@ -84,10 +84,34 @@ def _download_tarball(sr_path, staging_path, image_id, glance_host, def fixup_vhds(sr_path, staging_path, uuid_stack): - """ + """Fixup the downloaded VHDs before we move them into the SR. + + We cannot extract VHDs directly into the SR since they don't yet have + UUIDs, aren't properly associated with each other, and would be subject to + a race-condition of one-file being present and the other not being + downloaded yet. + + To avoid these we problems, we use a staging area to fixup the VHDs before + moving them into the SR. The steps involved are: + + 1. Extracting tarball into staging area + + 2. Renaming VHDs to use UUIDs ('snap.vhd' -> 'ffff-aaaa-...vhd') + + 3. Linking the two VHDs together + + 4. Pseudo-atomically moving the images into the SR. (It's not really + atomic because it takes place as two os.rename operations; however, the + chances of an SR.scan occuring between the two rename() invocations is so + small that we can safely ignore it) """ def rename_with_uuid(orig_path): - """Generate a uuid and rename the file with the uuid""" + """Rename VHD using UUID so that it will be recognized by SR on a + subsequent scan. + + Since Python2.4 doesn't have the `uuid` module, we pass a stack of + pre-computed UUIDs from the compute worker. + """ orig_dirname = os.path.dirname(orig_path) uuid = uuid_stack.pop() new_path = os.path.join(orig_dirname, "%s.vhd" % uuid) @@ -96,6 +120,11 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): def link_vhds(child_path, parent_path): + """Use vhd-util to associate the snapshot VHD with its base_copy. + + This needs to be done before we move both VHDs into the SR to prevent + the base_copy from being DOA (deleted-on-arrival). + """ modify_args = shlex.split( "vhd-util modify -n %(child_path)s -p %(parent_path)s" % locals()) @@ -136,8 +165,8 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): - """ - Explain preparing staging area here... + """Hard-link VHDs into staging area with appropriate filename + ('snap' or 'image.vhd') """ image_path = os.path.join(staging_path, 'image') os.mkdir(image_path) @@ -149,6 +178,8 @@ def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): def _upload_tarball(staging_path, image_id, glance_host, glance_port): """ + Create a tarball of the image and then stream that into Glance + using chunked-transfer-encoded HTTP. """ conn = httplib.HTTPConnection(glance_host, glance_port) #NOTE(sirp): httplib under python2.4 won't accept a file-like object @@ -189,7 +220,36 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): def _make_staging_area(sr_path): """ - Explain staging area here... + The staging area is a place where we can temporarily store and + manipulate VHDs. The use of the staging area is different for upload and + download: + + Download + ======== + + When we download the tarball, the VHDs contained within will have names + like "snap.vhd" and "image.vhd". We need to assign UUIDs to them before + moving them into the SR. However, since 'image.vhd' may be a base_copy, we + need to link it to 'snap.vhd' (using vhd-util modify) before moving both + into the SR (otherwise the SR.scan will cause 'image.vhd' to be deleted). + The staging area gives us a place to perform these operations before they + are moved to the SR, scanned, and then registered with XenServer. + + Upload + ====== + + On upload, we want to rename the VHDs to reflect what they are, 'snap.vhd' + in the case of the snapshot VHD, and 'image.vhd' in the case of the + base_copy. The staging area provides a directory in which we can create + hard-links to rename the VHDs without affecting what's in the SR. + + + NOTE + ==== + + The staging area is created as a subdirectory within the SR in order to + guarantee that it resides within the same filesystem and therefore permit + hard-linking and cheap file moves. """ # NOTE(sirp): staging area is created in SR to allow hard-linking staging_path = tempfile.mkdtemp(dir=sr_path) @@ -197,6 +257,12 @@ def _make_staging_area(sr_path): def _cleanup_staging_area(staging_path): + """Remove staging area directory + + On upload, the staging area contains hard-links to the VHDs in the SR; + it's safe to remove the staging-area because the SR will keep the link + count > 0 (so the VHDs in the SR will not be deleted). + """ shutil.rmtree(staging_path) @@ -211,7 +277,8 @@ def _assert_process_success(proc, cmd): def download_image(session, args): - """ + """Download an image from Glance, unbundle it, and then deposit the VHDs + into the storage repository """ params = pickle.loads(exists(args, 'params')) image_id = params["image_id"] @@ -231,7 +298,7 @@ def download_image(session, args): def upload_image(session, args): - """ + """Bundle the VHDs comprising an image and then stream them into Glance. """ params = pickle.loads(exists(args, 'params')) vdi_uuids = params["vdi_uuids"] -- cgit From 1d72b9d3ddc835d788ba1fec1a937c2788e94b38 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 21:36:13 +0000 Subject: Using Nova style nokernel --- nova/api/openstack/servers.py | 8 +------- nova/compute/api.py | 2 ++ plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 9 ++++++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 1c9dcd9a6..cd0cea235 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -138,13 +138,7 @@ class Controller(wsgi.Controller): _("%(param)s property not found for image %(_image_id)s") % locals()) - image_id = str(image_id) - image = self._image_service.show(req.environ['nova.context'], image_id) - disk_format = image['properties'].get('disk_format', None) - if disk_format == "vhd": - return None, None - else: - return lookup('kernel_id'), lookup('ramdisk_id') + return lookup('kernel_id'), lookup('ramdisk_id') def create(self, req): """ Creates a new server for a given user """ diff --git a/nova/compute/api.py b/nova/compute/api.py index ac02dbcfa..6d28e376c 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -106,6 +106,8 @@ class API(base.Base): kernel_id = image.get('kernelId', None) if ramdisk_id is None: ramdisk_id = image.get('ramdiskId', None) + + # FIXME(sirp): is there a way we can remove null_kernel? # No kernel and ramdisk for raw images if kernel_id == str(FLAGS.null_kernel): kernel_id = None diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index dac773ff1..75bdda2c6 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -182,15 +182,19 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): using chunked-transfer-encoded HTTP. """ conn = httplib.HTTPConnection(glance_host, glance_port) - #NOTE(sirp): httplib under python2.4 won't accept a file-like object + # NOTE(sirp): httplib under python2.4 won't accept a file-like object # to request conn.putrequest('PUT', '/images/%s' % image_id) + # FIXME(sirp): nokernel signals Nova to use a raw image. This is defined by + # FLAGS.null_kernel. Is there a way to get rid of this? headers = { 'x-image-meta-store': 'file', 'x-image-meta-is_public': 'True', 'x-image-meta-type': 'raw', 'x-image-meta-property-disk-format': 'vhd', + 'x-image-meta-property-kernel-id': 'nokernel', + 'x-image-meta-property-ramdisk-id': 'noramdisk', 'x-image-meta-property-container-format': 'tarball', 'transfer-encoding': "chunked", 'content-type': 'application/octet-stream', @@ -251,7 +255,6 @@ def _make_staging_area(sr_path): guarantee that it resides within the same filesystem and therefore permit hard-linking and cheap file moves. """ - # NOTE(sirp): staging area is created in SR to allow hard-linking staging_path = tempfile.mkdtemp(dir=sr_path) return staging_path @@ -314,7 +317,7 @@ def upload_image(session, args): finally: _cleanup_staging_area(staging_path) - return "" + return "" # Nothing useful to return on an upload def copy_kernel_vdi(session,args): -- cgit From c33378fbbe0fd76e807530522715ba4175af18d8 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 21:54:42 +0000 Subject: Fixing typo --- nova/api/openstack/servers.py | 11 ++++++++--- nova/virt/xenapi/vm_utils.py | 14 +++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index cd0cea235..c15e499a0 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -129,8 +129,12 @@ class Controller(wsgi.Controller): Machine images are associated with Kernels and Ramdisk images via metadata stored in Glance as 'image_properties' """ - def lookup(param): - _image_id = image_id + # FIXME(sirp): Currently Nova requires us to specify the `null_kernel` + # identifier ('nokernel') to indicate a RAW (or VHD) image. It would + # be better if we could omit the kernel_id and ramdisk_id properties + # on the image + def lookup(image, param): + _image_id = image.id try: return image['properties'][param] except KeyError: @@ -138,7 +142,8 @@ class Controller(wsgi.Controller): _("%(param)s property not found for image %(_image_id)s") % locals()) - return lookup('kernel_id'), lookup('ramdisk_id') + image = self._image_service.show(req.environ['nova.context'], image_id) + return lookup(image, 'kernel_id'), lookup(image, 'ramdisk_id') def create(self, req): """ Creates a new server for a given user """ diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index ba3e5e423..6fa03ee68 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -331,12 +331,24 @@ class VMHelper(HelperBase): kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'download_image', kwargs) vdi_uuid = session.wait_for_task(instance_id, task) + # TODO(sirp): set name-label on VDI scan_sr(session, instance_id, sr_ref) LOG.debug(_("xapi 'download_image' returned VDI UUID %(vdi_uuid)s") % locals()) return vdi_uuid @classmethod def _fetch_image_glance_disk(cls, session, instance_id, image, access, type): + """Fetch the image from Glance + + NOTE: + Unlike _fetch_image_glance_vhd, this method does not use the Glance + plugin; instead, it streams the disks through domU to the VDI + directly. + + """ + # FIXME(sirp): Since the Glance plugin seems to be required for the VHD disk, + # it may be worth using the plugin for both VHD and RAW and DISK + # restores sr_ref = safe_find_sr(session) client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) @@ -349,7 +361,7 @@ class VMHelper(HelperBase): # Make room for MBR. vdi_size += MBR_SIZE_BYTES - vdi = cls.create_vdi(session, sr, _('Glance image %s') % image, + vdi = cls.create_vdi(session, sr_ref, _('Glance image %s') % image, vdi_size, False) with_vdi_attached_here(session, vdi, False, -- cgit From 24d988263324c9136a1cd9aa5a3a3c4fdf229651 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 22:19:58 +0000 Subject: Set name-label on VDI --- nova/virt/xenapi/vm_utils.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 6fa03ee68..738372da0 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -331,8 +331,14 @@ class VMHelper(HelperBase): kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'download_image', kwargs) vdi_uuid = session.wait_for_task(instance_id, task) - # TODO(sirp): set name-label on VDI + scan_sr(session, instance_id, sr_ref) + + # Set the name-label to ease debugging + vdi_ref = session.get_xenapi().VDI.get_by_uuid(vdi_uuid) + name_label = get_name_label_for_image(image) + session.get_xenapi().VDI.set_name_label(vdi_ref, name_label) + LOG.debug(_("xapi 'download_image' returned VDI UUID %(vdi_uuid)s") % locals()) return vdi_uuid @@ -361,8 +367,8 @@ class VMHelper(HelperBase): # Make room for MBR. vdi_size += MBR_SIZE_BYTES - vdi = cls.create_vdi(session, sr_ref, _('Glance image %s') % image, - vdi_size, False) + name_label = get_name_label_for_image(image) + vdi = cls.create_vdi(session, sr_ref, name_label, vdi_size, False) with_vdi_attached_here(session, vdi, False, lambda dev: @@ -848,3 +854,8 @@ def _write_partition(virtual_size, dev): (dest, primary_first, primary_last)) LOG.debug(_('Writing partition table %s done.'), dest) + + +def get_name_label_for_image(image): + # TODO(sirp): This should eventually be the URI for the Glance image + return _('Glance image %s') % image -- cgit From 28ba475d9c32a384570ce6eb0e2f9cfc3dc79a08 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 22:24:48 +0000 Subject: Pep8 fixes --- nova/virt/xenapi/vm_utils.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 738372da0..93009b408 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -311,7 +311,8 @@ class VMHelper(HelperBase): access, user.secret, type) @classmethod - def _fetch_image_glance_vhd(cls, session, instance_id, image, access, type): + def _fetch_image_glance_vhd(cls, session, instance_id, image, access, + type): LOG.debug(_("Asking xapi to fetch vhd image %(image)s") % locals()) @@ -339,11 +340,13 @@ class VMHelper(HelperBase): name_label = get_name_label_for_image(image) session.get_xenapi().VDI.set_name_label(vdi_ref, name_label) - LOG.debug(_("xapi 'download_image' returned VDI UUID %(vdi_uuid)s") % locals()) + LOG.debug(_("xapi 'download_image' returned VDI UUID %(vdi_uuid)s") + % locals()) return vdi_uuid @classmethod - def _fetch_image_glance_disk(cls, session, instance_id, image, access, type): + def _fetch_image_glance_disk(cls, session, instance_id, image, access, + type): """Fetch the image from Glance NOTE: @@ -352,9 +355,9 @@ class VMHelper(HelperBase): directly. """ - # FIXME(sirp): Since the Glance plugin seems to be required for the VHD disk, - # it may be worth using the plugin for both VHD and RAW and DISK - # restores + # FIXME(sirp): Since the Glance plugin seems to be required for the + # VHD disk, it may be worth using the plugin for both VHD and RAW and + # DISK restores sr_ref = safe_find_sr(session) client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) @@ -429,12 +432,13 @@ class VMHelper(HelperBase): properties = meta['properties'] disk_format = properties.get('disk_format', None) # TODO(sirp): When Glance treats disk_format as a first class - # attribute, we should start using that rather than an image-property + # attribute, we should start using that rather than an + # image-property if disk_format == 'vhd': # 2a. DISK_VHD log_disk_format('disk_vhd') return ImageType.DISK_VHD - + # 2b. DISK_RAW log_disk_format('disk_raw') return ImageType.DISK_RAW -- cgit From c7cd7b755c86bd15e2b19f70a09f88f62361596c Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 22:31:51 +0000 Subject: More pep8 fixes --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 51 +++++++++++----------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 75bdda2c6..43c58dcff 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -39,20 +39,22 @@ configure_logging('glance') CHUNK_SIZE = 8192 KERNEL_DIR = '/boot/guest' -def _copy_kernel_vdi(dest,copy_args): - vdi_uuid=copy_args['vdi_uuid'] - vdi_size=copy_args['vdi_size'] - logging.debug("copying kernel/ramdisk file from %s to /boot/guest/%s",dest,vdi_uuid) - filename=KERNEL_DIR + '/' + vdi_uuid + +def _copy_kernel_vdi(dest, copy_args): + vdi_uuid = copy_args['vdi_uuid'] + vdi_size = copy_args['vdi_size'] + logging.debug("copying kernel/ramdisk file from %s to /boot/guest/%s", + dest, vdi_uuid) + filename = KERNEL_DIR + '/' + vdi_uuid #read data from /dev/ and write into a file on /boot/guest - of=open(filename,'wb') - f=open(dest,'rb') + of = open(filename, 'wb') + f = open(dest, 'rb') #copy only vdi_size bytes - data=f.read(vdi_size) + data = f.read(vdi_size) of.write(data) f.close() of.close() - logging.debug("Done. Filename: %s",filename) + logging.debug("Done. Filename: %s", filename) return filename @@ -101,9 +103,9 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): 3. Linking the two VHDs together 4. Pseudo-atomically moving the images into the SR. (It's not really - atomic because it takes place as two os.rename operations; however, the - chances of an SR.scan occuring between the two rename() invocations is so - small that we can safely ignore it) + atomic because it takes place as two os.rename operations; however, + the chances of an SR.scan occuring between the two rename() + invocations is so small that we can safely ignore it) """ def rename_with_uuid(orig_path): """Rename VHD using UUID so that it will be recognized by SR on a @@ -118,7 +120,6 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): os.rename(orig_path, new_path) return new_path, uuid - def link_vhds(child_path, parent_path): """Use vhd-util to associate the snapshot VHD with its base_copy. @@ -132,7 +133,6 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): modify_args, stderr=subprocess.PIPE) _assert_process_success(modify_proc, "vhd-util") - def move_into_sr(orig_path): """Move a file into the SR""" filename = os.path.basename(orig_path) @@ -140,7 +140,6 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): os.rename(orig_path, new_path) return new_path - image_path = os.path.join(staging_path, 'image') orig_base_copy_path = os.path.join(image_path, 'image.vhd') @@ -157,7 +156,7 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): # NOTE(sirp): this step is necessary so that an SR scan won't # delete the base_copy out from under us (since it would be # orphaned) - link_vhds(snap_path, base_copy_path) + link_vhds(snap_path, base_copy_path) move_into_sr(snap_path) move_into_sr(base_copy_path) @@ -227,7 +226,7 @@ def _make_staging_area(sr_path): The staging area is a place where we can temporarily store and manipulate VHDs. The use of the staging area is different for upload and download: - + Download ======== @@ -241,7 +240,7 @@ def _make_staging_area(sr_path): Upload ====== - + On upload, we want to rename the VHDs to reflect what they are, 'snap.vhd' in the case of the snapshot VHD, and 'image.vhd' in the case of the base_copy. The staging area provides a directory in which we can create @@ -264,7 +263,7 @@ def _cleanup_staging_area(staging_path): On upload, the staging area contains hard-links to the VHDs in the SR; it's safe to remove the staging-area because the SR will keep the link - count > 0 (so the VHDs in the SR will not be deleted). + count > 0 (so the VHDs in the SR will not be deleted). """ shutil.rmtree(staging_path) @@ -320,15 +319,15 @@ def upload_image(session, args): return "" # Nothing useful to return on an upload -def copy_kernel_vdi(session,args): +def copy_kernel_vdi(session, args): vdi = exists(args, 'vdi-ref') - size = exists(args,'image-size') + size = exists(args, 'image-size') #Use the uuid as a filename - vdi_uuid=session.xenapi.VDI.get_uuid(vdi) - copy_args={'vdi_uuid':vdi_uuid,'vdi_size':int(size)} - filename=with_vdi_in_dom0(session, vdi, False, - lambda dev: - _copy_kernel_vdi('/dev/%s' % dev,copy_args)) + vdi_uuid = session.xenapi.VDI.get_uuid(vdi) + copy_args = {'vdi_uuid': vdi_uuid, 'vdi_size': int(size)} + filename = with_vdi_in_dom0(session, vdi, False, + lambda dev: + _copy_kernel_vdi('/dev/%s' % dev, copy_args)) return filename -- cgit From 21a3d77fee681d05c465c74e40177ae022bc24af Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 22:41:27 +0000 Subject: Fixing test by adding stub for get_image_meta --- nova/tests/glance/stubs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py index f182b857a..4cd5c357f 100644 --- a/nova/tests/glance/stubs.py +++ b/nova/tests/glance/stubs.py @@ -29,9 +29,10 @@ class FakeGlance(object): def __init__(self, host, port=None, use_ssl=False): pass - def get_image(self, image): - meta = { - 'size': 0, - } + def get_image_meta(self, image_id): + return {'size': 0, 'properties': {}} + + def get_image(self, image_id): + meta = self.get_image_meta(image_id) image_file = StringIO.StringIO('') return meta, image_file -- cgit From af920572f42b07c3ea491015d30eb5001d1f735d Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 15 Feb 2011 23:36:23 +0000 Subject: Adding vhd hidden sanity check --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 23 +++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 43c58dcff..ceb9a2185 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -140,6 +140,25 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): os.rename(orig_path, new_path) return new_path + def assert_vhd_not_hidden(path): + """ + This is a sanity check on the image; if a snap.vhd isn't + present, then the image.vhd better not be marked 'hidden' or it will + be deleted when moved into the SR. + """ + vhd_query_args = shlex.split( + "vhd-util query -n %(path)s -f" % locals()) + vhd_query_proc = subprocess.Popen( + vhd_query_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = _assert_process_success(vhd_query_proc, "vhd-util") + for line in out.splitlines(): + if line.startswith('hidden'): + value = line.split(':')[1].strip() + if value == "1": + raise Exception( + "VHD %(path)s is marked as hidden without child" % + locals()) + image_path = os.path.join(staging_path, 'image') orig_base_copy_path = os.path.join(image_path, 'image.vhd') @@ -158,6 +177,8 @@ def fixup_vhds(sr_path, staging_path, uuid_stack): # orphaned) link_vhds(snap_path, base_copy_path) move_into_sr(snap_path) + else: + assert_vhd_not_hidden(base_copy_path) move_into_sr(base_copy_path) return vdi_uuid @@ -276,7 +297,7 @@ def _assert_process_success(proc, cmd): if ret != 0: msg = "%(cmd)s returned non-zero exit code (%i): '%s'" % (ret, err) raise Exception(msg) - + return out, err def download_image(session, args): """Download an image from Glance, unbundle it, and then deposit the VHDs -- cgit From 163e81ac2bc2f9945273b0659ceb473767e5b19f Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Wed, 16 Feb 2011 11:53:50 -0500 Subject: This implements the blueprint 'Openstack API support for hostId': https://blueprints.launchpad.net/nova/+spec/openstack-api-hostid Now instances will have a unique hostId which for now is just a hash of the host. If the instance does not have a host yet, the hostId will be ''. --- nova/api/openstack/servers.py | 7 +++++- nova/tests/api/openstack/test_servers.py | 41 +++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 17c5519a1..58eda53b9 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib import json import traceback @@ -65,7 +66,11 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) inst_dict['metadata'] = {} - inst_dict['hostId'] = '' + + if inst['host']: + inst_dict['hostId'] = hashlib.sha224(inst['host']).hexdigest() + else: + inst_dict['hostId'] = '' return dict(server=inst_dict) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 724f14f19..e615141ff 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -43,6 +43,10 @@ def return_servers(context, user_id=1): return [stub_instance(i, user_id) for i in xrange(5)] +def return_servers_with_host(context, user_id=1): + return [stub_instance(i, user_id, 1) for i in xrange(5)] + + def return_security_group(context, instance_id, security_group_id): pass @@ -55,9 +59,13 @@ def instance_address(context, instance_id): return None -def stub_instance(id, user_id=1): - return Instance(id=id, state=0, image_id=10, user_id=user_id, - display_name='server%s' % id) +def stub_instance(id, user_id=1, with_hosts=False): + if with_hosts: + return Instance(id=id, state=0, image_id=10, user_id=user_id, + display_name='server%s' % id, host='host%s' % (id % 2)) + else: + return Instance(id=id, state=0, image_id=10, user_id=user_id, + display_name='server%s' % id) def fake_compute_api(cls, req, id): @@ -229,6 +237,33 @@ class ServersTest(unittest.TestCase): i = 0 for s in res_dict['servers']: self.assertEqual(s['id'], i) + self.assertEqual(s['hostId'], '') + self.assertEqual(s['name'], 'server%d' % i) + self.assertEqual(s['imageId'], 10) + i += 1 + + def test_get_all_server_details_with_host(self): + ''' + We want to make sure that if two instances are on the same host, then + they return the same hostId. If two instances are on different hosts, + they should return different hostId's. In this test, we get 5 instances + back where 2 are on one host and 3 are on another. + ''' + self.stubs.Set(nova.db.api, 'instance_get_all_by_user', + return_servers_with_host) + req = webob.Request.blank('/v1.0/servers/detail') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + server_list = res_dict['servers'] + host_ids = [server_list[0]['hostId'], server_list[1]['hostId']] + self.assertTrue(host_ids[0]) + self.assertTrue(host_ids[1]) + self.assertTrue(host_ids[0] != host_ids[1]) + i = 0 + for s in res_dict['servers']: + self.assertEqual(s['id'], i) + self.assertEqual(s['hostId'], host_ids[i % 2]) self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s['imageId'], 10) i += 1 -- cgit From 585ba4d6cf25eabf83b1b33a6de794ce671c0c98 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 16 Feb 2011 18:43:55 +0000 Subject: Putting glance plugin under pep8 control --- nova/compute/api.py | 3 --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 1 + run_tests.sh | 6 +++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 8a16baf45..ea81c7ff0 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -104,11 +104,8 @@ class API(base.Base): image = self.image_service.show(context, image_id) if kernel_id is None: kernel_id = image.get('kernel_id', None) - # FIXME(sirp): which one to use? - #kernel_id = image.get('kernelId', None) if ramdisk_id is None: ramdisk_id = image.get('ramdisk_id', None) - #ramdisk_id = image.get('ramdiskId', None) # FIXME(sirp): is there a way we can remove null_kernel? # No kernel and ramdisk for raw images diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 47e094f02..f737c6c41 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -299,6 +299,7 @@ def _assert_process_success(proc, cmd): raise Exception(msg) return out, err + def download_image(session, args): """Download an image from Glance, unbundle it, and then deposit the VHDs into the storage repository diff --git a/run_tests.sh b/run_tests.sh index 4e21fe945..6c7dd5f46 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -73,7 +73,11 @@ fi if [ -z "$noseargs" ]; then - run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py bin/* nova setup.py || exit 1 + PEP8_EXCLUDE=vcsversion.py + PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat --show-pep8 --show-source" + # TODO(sirp): Put tests/ run_tests.py and all plugins/ under pep8 control + PEP8_INCLUDE="bin/* nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" + run_tests && pep8 $PEP8_OPTIONS $PEP8_INCLUDE || exit 1 else run_tests fi -- cgit From a5ec2be709d28267075ddc9616c5c29b62622af5 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 16 Feb 2011 23:07:43 +0000 Subject: Adding basic test --- nova/tests/glance/stubs.py | 22 ++++++++++++++++++---- nova/tests/test_xenapi.py | 4 ++++ nova/tests/xenapi/stubs.py | 6 ++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py index 4cd5c357f..fc120e523 100644 --- a/nova/tests/glance/stubs.py +++ b/nova/tests/glance/stubs.py @@ -26,13 +26,27 @@ def stubout_glance_client(stubs, cls): class FakeGlance(object): + IMAGE_FIXTURES = { + 1: {'image_meta': {'name': 'fakemachine', 'size': 0, + 'properties': {}}, + 'image_data': StringIO.StringIO('') }, + 2: {'image_meta': {'name': 'fakekernel', 'size': 0, + 'properties': {}}, + 'image_data': StringIO.StringIO('') }, + 3: {'image_meta': {'name': 'fakekernel', 'size': 0, + 'properties': {}}, + 'image_data': StringIO.StringIO('') }, + 4: {'image_meta': {'name': 'fakekernel', 'size': 0, + 'properties': {'disk_format': 'vhd'}}, + 'image_data': StringIO.StringIO('') }, + } + def __init__(self, host, port=None, use_ssl=False): pass def get_image_meta(self, image_id): - return {'size': 0, 'properties': {}} + return self.IMAGE_FIXTURES[image_id]['image_meta'] def get_image(self, image_id): - meta = self.get_image_meta(image_id) - image_file = StringIO.StringIO('') - return meta, image_file + image = self.IMAGE_FIXTURES[image_id] + return image['image_meta'], image['image_data'] diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index d5660c5d1..75387e7f5 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -279,6 +279,10 @@ class XenAPIVMTestCase(test.TestCase): FLAGS.xenapi_image_service = 'glance' self._test_spawn(1, None, None) + def test_spawn_vhd_glance(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(4, None, None) + def test_spawn_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(1, 2, 3) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 624995ada..2e3b62a77 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -171,6 +171,12 @@ class FakeSessionForVMTests(fake.SessionBase): def VM_destroy(self, session_ref, vm_ref): fake.destroy_vm(vm_ref) + def SR_scan(self, session_ref, sr_ref): + pass + + def VDI_set_name_label(self, session_ref, vdi_ref, name_label): + pass + class FakeSessionForVolumeTests(fake.SessionBase): """ Stubs out a XenAPISession for Volume tests """ -- cgit From c56b1814cfae7a9c814b2d37388aff5e772771b6 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 16 Feb 2011 23:39:12 +0000 Subject: Pep8 fixes --- nova/tests/glance/stubs.py | 8 ++++---- nova/virt/xenapi/vm_utils.py | 8 +++++--- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 8 ++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py index fc120e523..c58357962 100644 --- a/nova/tests/glance/stubs.py +++ b/nova/tests/glance/stubs.py @@ -29,16 +29,16 @@ class FakeGlance(object): IMAGE_FIXTURES = { 1: {'image_meta': {'name': 'fakemachine', 'size': 0, 'properties': {}}, - 'image_data': StringIO.StringIO('') }, + 'image_data': StringIO.StringIO('')}, 2: {'image_meta': {'name': 'fakekernel', 'size': 0, 'properties': {}}, - 'image_data': StringIO.StringIO('') }, + 'image_data': StringIO.StringIO('')}, 3: {'image_meta': {'name': 'fakekernel', 'size': 0, 'properties': {}}, - 'image_data': StringIO.StringIO('') }, + 'image_data': StringIO.StringIO('')}, 4: {'image_meta': {'name': 'fakekernel', 'size': 0, 'properties': {'disk_format': 'vhd'}}, - 'image_data': StringIO.StringIO('') }, + 'image_data': StringIO.StringIO('')}, } def __init__(self, host, port=None, use_ssl=False): diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index d77db2ddb..33945aca3 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -289,6 +289,8 @@ class VMHelper(HelperBase): """ Requests that the Glance plugin bundle the specified VDIs and push them into Glance using the specified human-friendly name. """ + # NOTE(sirp): Currently we only support uploading images as VHD, there + # is no RAW equivalent (yet) logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) @@ -299,7 +301,7 @@ class VMHelper(HelperBase): 'sr_path': get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} - task = session.async_call_plugin('glance', 'upload_image', kwargs) + task = session.async_call_plugin('glance', 'upload_vhd', kwargs) session.wait_for_task(instance_id, task) @classmethod @@ -340,7 +342,7 @@ class VMHelper(HelperBase): 'sr_path': get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} - task = session.async_call_plugin('glance', 'download_image', kwargs) + task = session.async_call_plugin('glance', 'download_vhd', kwargs) vdi_uuid = session.wait_for_task(instance_id, task) scan_sr(session, instance_id, sr_ref) @@ -350,7 +352,7 @@ class VMHelper(HelperBase): name_label = get_name_label_for_image(image) session.get_xenapi().VDI.set_name_label(vdi_ref, name_label) - LOG.debug(_("xapi 'download_image' returned VDI UUID %(vdi_uuid)s") + LOG.debug(_("xapi 'download_vhd' returned VDI UUID %(vdi_uuid)s") % locals()) return vdi_uuid diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index f737c6c41..c3f793ddd 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -300,7 +300,7 @@ def _assert_process_success(proc, cmd): return out, err -def download_image(session, args): +def download_vhd(session, args): """Download an image from Glance, unbundle it, and then deposit the VHDs into the storage repository """ @@ -321,7 +321,7 @@ def download_image(session, args): _cleanup_staging_area(staging_path) -def upload_image(session, args): +def upload_vhd(session, args): """Bundle the VHDs comprising an image and then stream them into Glance. """ params = pickle.loads(exists(args, 'params')) @@ -365,7 +365,7 @@ def remove_kernel_ramdisk(session, args): if __name__ == '__main__': - XenAPIPlugin.dispatch({'upload_image': upload_image, - 'download_image': download_image, + XenAPIPlugin.dispatch({'upload_vhd': upload_vhd, + 'download_vhd': download_vhd, 'copy_kernel_vdi': copy_kernel_vdi, 'remove_kernel_ramdisk': remove_kernel_ramdisk}) -- cgit From ce847afcc1e24463d7aa522f227a08193c72fcc0 Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Wed, 16 Feb 2011 19:12:44 -0500 Subject: Moved definition of return_servers_with_host stub to inside the test_get_all_server_details_with_host test. --- nova/api/openstack/servers.py | 3 +-- nova/tests/api/openstack/test_servers.py | 29 ++++++++++++++--------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 58eda53b9..323e6fda6 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -67,10 +67,9 @@ def _translate_detail_keys(inst): inst_dict['addresses'] = dict(public=[], private=[]) inst_dict['metadata'] = {} + inst_dict['hostId'] = '' if inst['host']: inst_dict['hostId'] = hashlib.sha224(inst['host']).hexdigest() - else: - inst_dict['hostId'] = '' return dict(server=inst_dict) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index e615141ff..6c91b3f5b 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -43,10 +43,6 @@ def return_servers(context, user_id=1): return [stub_instance(i, user_id) for i in xrange(5)] -def return_servers_with_host(context, user_id=1): - return [stub_instance(i, user_id, 1) for i in xrange(5)] - - def return_security_group(context, instance_id, security_group_id): pass @@ -60,12 +56,8 @@ def instance_address(context, instance_id): def stub_instance(id, user_id=1, with_hosts=False): - if with_hosts: - return Instance(id=id, state=0, image_id=10, user_id=user_id, - display_name='server%s' % id, host='host%s' % (id % 2)) - else: - return Instance(id=id, state=0, image_id=10, user_id=user_id, - display_name='server%s' % id) + return Instance(id=id, state=0, image_id=10, user_id=user_id, + display_name='server%s' % id) def fake_compute_api(cls, req, id): @@ -246,20 +238,27 @@ class ServersTest(unittest.TestCase): ''' We want to make sure that if two instances are on the same host, then they return the same hostId. If two instances are on different hosts, - they should return different hostId's. In this test, we get 5 instances - back where 2 are on one host and 3 are on another. + they should return different hostId's. In this test, there are 5 + instances - 2 on one host and 3 on another. ''' + + def return_servers_with_host(context, user_id=1): + return [ + Instance(id=i, state=0, image_id=10, user_id=user_id, + display_name='server%s' % i, host='host%s' % (i % 2)) + for i in xrange(5)] self.stubs.Set(nova.db.api, 'instance_get_all_by_user', - return_servers_with_host) + return_servers_with_host) + req = webob.Request.blank('/v1.0/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) server_list = res_dict['servers'] host_ids = [server_list[0]['hostId'], server_list[1]['hostId']] - self.assertTrue(host_ids[0]) - self.assertTrue(host_ids[1]) + self.assertTrue(host_ids[0] and host_ids[1]) self.assertTrue(host_ids[0] != host_ids[1]) + i = 0 for s in res_dict['servers']: self.assertEqual(s['id'], i) -- cgit From 56ad2a63f1dcf4a900fa4464671015dbaac05fdc Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Wed, 16 Feb 2011 19:37:28 -0500 Subject: Minor change. Adding a helper function stub_instance() inside the test test_get_all_server_details_with_host for readability. --- nova/tests/api/openstack/test_servers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 6c91b3f5b..630e1e5eb 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -55,7 +55,7 @@ def instance_address(context, instance_id): return None -def stub_instance(id, user_id=1, with_hosts=False): +def stub_instance(id, user_id=1): return Instance(id=id, state=0, image_id=10, user_id=user_id, display_name='server%s' % id) @@ -242,11 +242,13 @@ class ServersTest(unittest.TestCase): instances - 2 on one host and 3 on another. ''' + def stub_instance(id, user_id=1): + return Instance(id=id, state=0, image_id=10, user_id=user_id, + display_name='server%s' % id, host='host%s' % (id % 2)) + def return_servers_with_host(context, user_id=1): - return [ - Instance(id=i, state=0, image_id=10, user_id=user_id, - display_name='server%s' % i, host='host%s' % (i % 2)) - for i in xrange(5)] + return [stub_instance(i) for i in xrange(5)] + self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers_with_host) -- cgit From 04e29f6dc4b13b6fd0cbe5013cf241a727eb56ac Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Thu, 17 Feb 2011 01:24:31 +0000 Subject: Use glance image type to determine disk type --- nova/tests/glance/stubs.py | 12 ++-- nova/virt/xenapi/vm_utils.py | 68 ++++++++++++---------- nova/virt/xenapi/vmops.py | 9 ++- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 3 +- 4 files changed, 49 insertions(+), 43 deletions(-) diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py index c58357962..1a5fb7ffb 100644 --- a/nova/tests/glance/stubs.py +++ b/nova/tests/glance/stubs.py @@ -28,16 +28,16 @@ def stubout_glance_client(stubs, cls): class FakeGlance(object): IMAGE_FIXTURES = { 1: {'image_meta': {'name': 'fakemachine', 'size': 0, - 'properties': {}}, + 'type': 'machine'}, 'image_data': StringIO.StringIO('')}, 2: {'image_meta': {'name': 'fakekernel', 'size': 0, - 'properties': {}}, + 'type': 'kernel'}, 'image_data': StringIO.StringIO('')}, - 3: {'image_meta': {'name': 'fakekernel', 'size': 0, - 'properties': {}}, + 3: {'image_meta': {'name': 'fakeramdisk', 'size': 0, + 'type': 'ramdisk'}, 'image_data': StringIO.StringIO('')}, - 4: {'image_meta': {'name': 'fakekernel', 'size': 0, - 'properties': {'disk_format': 'vhd'}}, + 4: {'image_meta': {'name': 'fakevhd', 'size': 0, + 'type': 'vhd'}, 'image_data': StringIO.StringIO('')}, } diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 33945aca3..278e52211 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -413,47 +413,51 @@ class VMHelper(HelperBase): within an image. To figure out which type we're dealing with, we use the following rules: - 1. If the instance is specifying a kernel explicitly, we must be using - a 'disk' image (kernel outside of the image) + 1. If we're using Glance, we can use the image_type field to + determine the image_type - 2. If the kernel isn't specified, then we have two different - scenarios: - - a) If the image is in Glance, then we can use the 'disk_format' - property to determine if the image is really a VHD-style image - or if it's a RAW image - - b) If the image is not in Glance, then it must be a RAW image - (since we don't have a way of identifying VHD images...yet) + 2. If we're not using Glance, then we need to deduce this based on + whether a kernel_id is specified. """ - def log_disk_format(disk_format): - disk_format = disk_format.upper() + def log_disk_format(image_type): + pretty_format = {ImageType.KERNEL_RAMDISK: 'KERNEL_RAMDISK', + ImageType.DISK: 'DISK', + ImageType.DISK_RAW: 'DISK_RAW', + ImageType.DISK_VHD: 'DISK_VHD'} + disk_format = pretty_format[image_type] image_id = instance.image_id instance_id = instance.id LOG.debug(_("Detected %(disk_format)s format for image " "%(image_id)s, instance %(instance_id)s") % locals()) - if instance.kernel_id: - # 1. DISK - log_disk_format('disk') - return ImageType.DISK - elif FLAGS.xenapi_image_service == 'glance': - # if using glance, then we could be VHD format + def determine_from_glance(): + glance_type2nova_type = {'machine': ImageType.DISK, + 'raw': ImageType.DISK_RAW, + 'vhd': ImageType.DISK_VHD, + 'kernel': ImageType.KERNEL_RAMDISK, + 'ramdisk': ImageType.KERNEL_RAMDISK} client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) meta = client.get_image_meta(instance.image_id) - properties = meta['properties'] - disk_format = properties.get('disk_format', None) - # TODO(sirp): When Glance treats disk_format as a first class - # attribute, we should start using that rather than an - # image-property - if disk_format == 'vhd': - # 2a. DISK_VHD - log_disk_format('disk_vhd') - return ImageType.DISK_VHD - - # 2b. DISK_RAW - log_disk_format('disk_raw') - return ImageType.DISK_RAW + type_ = meta['type'] + try: + return glance_type2nova_type[type_] + except KeyError: + raise exception.NotFound( + _("Unrecognized image type '%(type_)s'") % locals()) + + def determine_from_instance(): + if instance.kernel_id: + return ImageType.DISK + else: + return ImageType.DISK_RAW + + if FLAGS.xenapi_image_service == 'glance': + image_type = determine_from_glance() + else: + image_type = determine_from_instance() + + log_disk_format(image_type) + return image_type @classmethod def _fetch_image_glance(cls, session, instance_id, image, access, type): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 943d74da3..961d589d5 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -224,7 +224,8 @@ class VMOps(object): VMHelper.upload_image( self._session, instance.id, template_vdi_uuids, image_id) finally: - self._destroy(instance, template_vm_ref, shutdown=False) + self._destroy(instance, template_vm_ref, shutdown=False, + destroy_kernel_ramdisk=False) logging.debug(_("Finished snapshot and upload for VM %s"), instance) @@ -368,7 +369,8 @@ class VMOps(object): vm = VMHelper.lookup(self._session, instance.name) return self._destroy(instance, vm, shutdown=True) - def _destroy(self, instance, vm, shutdown=True): + def _destroy(self, instance, vm, shutdown=True, + destroy_kernel_ramdisk=True): """ Destroys VM instance by performing: @@ -385,7 +387,8 @@ class VMOps(object): self._shutdown(instance, vm) self._destroy_vdis(instance, vm) - self._destroy_kernel_ramdisk(instance, vm) + if destroy_kernel_ramdisk: + self._destroy_kernel_ramdisk(instance, vm) self._destroy_vm(instance, vm) def _wait_with_callback(self, instance_id, task, callback): diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index c3f793ddd..a91f8a7c1 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -211,8 +211,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): headers = { 'x-image-meta-store': 'file', 'x-image-meta-is_public': 'True', - 'x-image-meta-type': 'raw', - 'x-image-meta-property-disk-format': 'vhd', + 'x-image-meta-type': 'vhd', 'x-image-meta-property-kernel-id': 'nokernel', 'x-image-meta-property-ramdisk-id': 'noramdisk', 'x-image-meta-property-container-format': 'tarball', -- cgit From 923a4938b73b84aa8a31f08a7c7b983cc82959fe Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Thu, 17 Feb 2011 07:29:50 +0000 Subject: Adding tests --- nova/tests/glance/stubs.py | 25 ++++++++++++---- nova/tests/test_xenapi.py | 70 ++++++++++++++++++++++++++++++++++++++++++-- nova/tests/xenapi/stubs.py | 10 +++++++ nova/virt/xenapi/vm_utils.py | 4 +++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py index 1a5fb7ffb..3ff8d7ce5 100644 --- a/nova/tests/glance/stubs.py +++ b/nova/tests/glance/stubs.py @@ -26,20 +26,33 @@ def stubout_glance_client(stubs, cls): class FakeGlance(object): + IMAGE_MACHINE = 1 + IMAGE_KERNEL = 2 + IMAGE_RAMDISK = 3 + IMAGE_RAW = 4 + IMAGE_VHD = 5 + IMAGE_FIXTURES = { - 1: {'image_meta': {'name': 'fakemachine', 'size': 0, + IMAGE_MACHINE: { + 'image_meta': {'name': 'fakemachine', 'size': 0, 'type': 'machine'}, 'image_data': StringIO.StringIO('')}, - 2: {'image_meta': {'name': 'fakekernel', 'size': 0, + IMAGE_KERNEL: { + 'image_meta': {'name': 'fakekernel', 'size': 0, 'type': 'kernel'}, 'image_data': StringIO.StringIO('')}, - 3: {'image_meta': {'name': 'fakeramdisk', 'size': 0, + IMAGE_RAMDISK: { + 'image_meta': {'name': 'fakeramdisk', 'size': 0, 'type': 'ramdisk'}, 'image_data': StringIO.StringIO('')}, - 4: {'image_meta': {'name': 'fakevhd', 'size': 0, - 'type': 'vhd'}, + IMAGE_RAW: { + 'image_meta': {'name': 'fakeraw', 'size': 0, + 'type': 'raw'}, 'image_data': StringIO.StringIO('')}, - } + IMAGE_VHD: { + 'image_meta': {'name': 'fakevhd', 'size': 0, + 'type': 'vhd'}, + 'image_data': StringIO.StringIO('')}} def __init__(self, host, port=None, use_ssl=False): pass diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 75387e7f5..f8a3d72c4 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -31,6 +31,7 @@ from nova.compute import power_state from nova.virt import xenapi_conn from nova.virt.xenapi import fake as xenapi_fake from nova.virt.xenapi import volume_utils +from nova.virt.xenapi import vm_utils from nova.virt.xenapi.vmops import SimpleDH from nova.tests.db import fakes as db_fakes from nova.tests.xenapi import stubs @@ -162,6 +163,7 @@ class XenAPIVMTestCase(test.TestCase): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) stubs.stubout_get_this_vm_uuid(self.stubs) stubs.stubout_stream_disk(self.stubs) + stubs.stubout_lookup_image(self.stubs) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) self.conn = xenapi_conn.get_connection(False) @@ -277,15 +279,17 @@ class XenAPIVMTestCase(test.TestCase): def test_spawn_raw_glance(self): FLAGS.xenapi_image_service = 'glance' - self._test_spawn(1, None, None) + self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None) def test_spawn_vhd_glance(self): FLAGS.xenapi_image_service = 'glance' - self._test_spawn(4, None, None) + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None) def test_spawn_glance(self): FLAGS.xenapi_image_service = 'glance' - self._test_spawn(1, 2, 3) + self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, + glance_stubs.FakeGlance.IMAGE_KERNEL, + glance_stubs.FakeGlance.IMAGE_RAMDISK) def tearDown(self): super(XenAPIVMTestCase, self).tearDown() @@ -334,3 +338,63 @@ class XenAPIDiffieHellmanTestCase(test.TestCase): def tearDown(self): super(XenAPIDiffieHellmanTestCase, self).tearDown() + + +class XenAPIDetermineDiskImageTestCase(test.TestCase): + """ + Unit tests for code that detects the ImageType + """ + def setUp(self): + super(XenAPIDetermineDiskImageTestCase, self).setUp() + glance_stubs.stubout_glance_client(self.stubs, + glance_stubs.FakeGlance) + + class FakeInstance(object): + pass + + self.fake_instance = FakeInstance() + self.fake_instance.id = 42 + + def assert_disk_type(self, disk_type): + dt = vm_utils.VMHelper.determine_disk_image_type( + self.fake_instance) + self.assertEqual(disk_type, dt) + + def test_instance_disk(self): + """ + If a kernel is specified then the image type is DISK (aka machine) + """ + FLAGS.xenapi_image_service = 'objectstore' + self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_MACHINE + self.fake_instance.kernel_id = glance_stubs.FakeGlance.IMAGE_KERNEL + self.assert_disk_type(vm_utils.ImageType.DISK) + + def test_instance_disk_raw(self): + """ + If the kernel isn't specified, and we're not using Glance, then + DISK_RAW is assumed. + """ + FLAGS.xenapi_image_service = 'objectstore' + self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_RAW + self.fake_instance.kernel_id = None + self.assert_disk_type(vm_utils.ImageType.DISK_RAW) + + def test_glance_disk_raw(self): + """ + If we're using Glance, then defer to the image_type field, which in + this case will be 'raw'. + """ + FLAGS.xenapi_image_service = 'glance' + self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_RAW + self.fake_instance.kernel_id = None + self.assert_disk_type(vm_utils.ImageType.DISK_RAW) + + def test_glance_disk_vhd(self): + """ + If we're using Glance, then defer to the image_type field, which in + this case will be 'vhd'. + """ + FLAGS.xenapi_image_service = 'glance' + self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_VHD + self.fake_instance.kernel_id = None + self.assert_disk_type(vm_utils.ImageType.DISK_VHD) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 2e3b62a77..1e6758a33 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -130,6 +130,16 @@ def stubout_stream_disk(stubs): stubs.Set(vm_utils, '_stream_disk', f) +def stubout_lookup_image(stubs): + @classmethod + def fake_lookup_image(cls, session, instance_id, vdi_ref): + # NOTE(sirp): pretending each image is paravirtualized for now + is_pv = True + return is_pv + + stubs.Set(vm_utils.VMHelper, 'lookup_image', fake_lookup_image) + + class FakeSessionForVMTests(fake.SessionBase): """ Stubs out a XenAPISession for VM tests """ def __init__(self, uri): diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 278e52211..9027d58c4 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -490,6 +490,9 @@ class VMHelper(HelperBase): @classmethod def lookup_image(cls, session, instance_id, vdi_ref): + """ + Determine if VDI is using a PV kernel + """ if FLAGS.xenapi_image_service == 'glance': return cls._lookup_image_glance(session, vdi_ref) else: @@ -517,6 +520,7 @@ class VMHelper(HelperBase): def is_vdi_pv(dev): LOG.debug(_("Running pygrub against %s"), dev) + # TODO(sirp): use subprocess here output = os.popen('pygrub -qn /dev/%s' % dev) for line in output.readlines(): #try to find kernel string -- cgit From 8dceaccb81e95b55fac2156df4f04ef0a7469112 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Thu, 17 Feb 2011 07:58:42 +0000 Subject: Typo fixes --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 4 ++-- run_tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index a91f8a7c1..3b5cedda7 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -85,7 +85,7 @@ def _download_tarball(sr_path, staging_path, image_id, glance_host, conn.close() -def fixup_vhds(sr_path, staging_path, uuid_stack): +def _fixup_vhds(sr_path, staging_path, uuid_stack): """Fixup the downloaded VHDs before we move them into the SR. We cannot extract VHDs directly into the SR since they don't yet have @@ -314,7 +314,7 @@ def download_vhd(session, args): try: _download_tarball(sr_path, staging_path, image_id, glance_host, glance_port) - vdi_uuid = fixup_vhds(sr_path, staging_path, uuid_stack) + vdi_uuid = _fixup_vhds(sr_path, staging_path, uuid_stack) return vdi_uuid finally: _cleanup_staging_area(staging_path) diff --git a/run_tests.sh b/run_tests.sh index 6c7dd5f46..7ac5ae071 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -75,7 +75,7 @@ if [ -z "$noseargs" ]; then PEP8_EXCLUDE=vcsversion.py PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat --show-pep8 --show-source" - # TODO(sirp): Put tests/ run_tests.py and all plugins/ under pep8 control + # TODO(sirp): Put run_tests.py and all plugins/ under pep8 control PEP8_INCLUDE="bin/* nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" run_tests && pep8 $PEP8_OPTIONS $PEP8_INCLUDE || exit 1 else -- cgit From 841b02230866cc163c26a264e86bba94c4b0335d Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Thu, 17 Feb 2011 13:15:28 -0500 Subject: Adding myself to Authors and .mailmap files. --- .mailmap | 1 + Authors | 1 + 2 files changed, 2 insertions(+) diff --git a/.mailmap b/.mailmap index c6f6c9a8b..95bf1a1bc 100644 --- a/.mailmap +++ b/.mailmap @@ -34,3 +34,4 @@ + diff --git a/Authors b/Authors index b359fec22..168a3cf9a 100644 --- a/Authors +++ b/Authors @@ -42,6 +42,7 @@ Monty Taylor MORITA Kazutaka Muneyuki Noguchi Nachi Ueno +Naveed Massjouni Paul Voccio Ricardo Carrillo Cruz Rick Clark -- cgit From 94f5a5748158e61bde43327970dd8e513ca36575 Mon Sep 17 00:00:00 2001 From: Nirmal Ranganathan Date: Thu, 17 Feb 2011 17:13:59 -0600 Subject: Always compare incoming flavor_id as an int --- nova/compute/instance_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/compute/instance_types.py b/nova/compute/instance_types.py index 309313fd0..7a2a5baa3 100644 --- a/nova/compute/instance_types.py +++ b/nova/compute/instance_types.py @@ -45,6 +45,6 @@ def get_by_type(instance_type): def get_by_flavor_id(flavor_id): for instance_type, details in INSTANCE_TYPES.iteritems(): - if details['flavorid'] == flavor_id: + if details['flavorid'] == int(flavor_id): return instance_type return FLAGS.default_instance_type -- cgit From 4673afddcb5a1069f75fb3493e43498ed1de11f9 Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Fri, 18 Feb 2011 02:23:30 -0500 Subject: Incorporating minor cleanups suggested by Rick Harris: * Use assertNotEqual instead of assertTrue * Use enumerate function instead of maintaining a counter --- nova/tests/api/openstack/test_servers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 630e1e5eb..4ed13022b 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -259,15 +259,13 @@ class ServersTest(unittest.TestCase): server_list = res_dict['servers'] host_ids = [server_list[0]['hostId'], server_list[1]['hostId']] self.assertTrue(host_ids[0] and host_ids[1]) - self.assertTrue(host_ids[0] != host_ids[1]) + self.assertNotEqual(host_ids[0], host_ids[1]) - i = 0 - for s in res_dict['servers']: + for i, s in enumerate(res_dict['servers']): self.assertEqual(s['id'], i) self.assertEqual(s['hostId'], host_ids[i % 2]) self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s['imageId'], 10) - i += 1 def test_server_pause(self): FLAGS.allow_admin_api = True -- cgit From cd533e160e9c98a0c14b4e0bc32a6e94c7ab8657 Mon Sep 17 00:00:00 2001 From: Nirmal Ranganathan Date: Fri, 18 Feb 2011 11:44:38 -0600 Subject: Added Author and tests --- Authors | 1 + nova/tests/test_compute.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/Authors b/Authors index 494e614a0..1d2316cf5 100644 --- a/Authors +++ b/Authors @@ -46,6 +46,7 @@ MORITA Kazutaka Muneyuki Noguchi Nachi Ueno Naveed Massjouni +Nirmal Ranganathan Paul Voccio Ricardo Carrillo Cruz Rick Clark diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index b049ac943..e338a7cfa 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -30,6 +30,7 @@ from nova import log as logging from nova import test from nova import utils from nova.auth import manager +from nova.compute import instance_types LOG = logging.getLogger('nova.tests.compute') @@ -266,3 +267,11 @@ class ComputeTestCase(test.TestCase): self.assertEqual(ret_val, None) self.compute.terminate_instance(self.context, instance_id) + + def test_get_by_flavor_id(self): + type = instance_types.get_by_flavor_id(1) + self.assertEqual(type, 'm1.tiny') + + type = instance_types.get_by_flavor_id("1") + self.assertEqual(type, 'm1.tiny') + -- cgit From 8916442e7f2920938a317777de71f75faf463005 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Fri, 18 Feb 2011 21:42:04 +0000 Subject: Typo fix --- nova/virt/xenapi/vm_utils.py | 2 ++ nova/virt/xenapi/vmops.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 9027d58c4..88a205d2f 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -451,6 +451,8 @@ class VMHelper(HelperBase): else: return ImageType.DISK_RAW + # FIXME(sirp): can we unify the ImageService and xenapi_image_service + # abstractions? if FLAGS.xenapi_image_service == 'glance': image_type = determine_from_glance() else: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 8d1c79c0b..09abd5861 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -394,7 +394,7 @@ class VMOps(object): """ Three situations can occur: - 1. We have netiher a ramdisk or a kernel, in which case we are a + 1. We have neither a ramdisk nor a kernel, in which case we are a RAW image and can omit this step 2. We have one or the other, in which case, we should flag as an -- cgit From eefc8938d8a8010052affab9a5c0d010778d9780 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Fri, 18 Feb 2011 22:25:19 +0000 Subject: Changing type -> image_type --- nova/virt/xenapi/vm_utils.py | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 88a205d2f..2114bfa42 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -305,9 +305,10 @@ class VMHelper(HelperBase): session.wait_for_task(instance_id, task) @classmethod - def fetch_image(cls, session, instance_id, image, user, project, type): + def fetch_image(cls, session, instance_id, image, user, project, + image_type): """ - type is interpreted as an ImageType instance + image_type is interpreted as an ImageType instance Related flags: xenapi_image_service = ['glance', 'objectstore'] glance_address = 'address for glance services' @@ -317,14 +318,15 @@ class VMHelper(HelperBase): if FLAGS.xenapi_image_service == 'glance': return cls._fetch_image_glance(session, instance_id, image, - access, type) + access, image_type) else: return cls._fetch_image_objectstore(session, instance_id, image, - access, user.secret, type) + access, user.secret, + image_type) @classmethod def _fetch_image_glance_vhd(cls, session, instance_id, image, access, - type): + image_type): LOG.debug(_("Asking xapi to fetch vhd image %(image)s") % locals()) @@ -358,7 +360,7 @@ class VMHelper(HelperBase): @classmethod def _fetch_image_glance_disk(cls, session, instance_id, image, access, - type): + image_type): """Fetch the image from Glance NOTE: @@ -378,7 +380,7 @@ class VMHelper(HelperBase): vdi_size = virtual_size LOG.debug(_("Size for image %(image)s:%(virtual_size)d") % locals()) - if type == ImageType.DISK: + if image_type == ImageType.DISK: # Make room for MBR. vdi_size += MBR_SIZE_BYTES @@ -387,9 +389,9 @@ class VMHelper(HelperBase): with_vdi_attached_here(session, vdi, False, lambda dev: - _stream_disk(dev, type, + _stream_disk(dev, image_type, virtual_size, image_file)) - if (type == ImageType.KERNEL_RAMDISK): + if image_type == ImageType.KERNEL_RAMDISK: #we need to invoke a plugin for copying VDI's #content into proper path LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi) @@ -462,29 +464,33 @@ class VMHelper(HelperBase): return image_type @classmethod - def _fetch_image_glance(cls, session, instance_id, image, access, type): - if type == ImageType.DISK_VHD: + def _fetch_image_glance(cls, session, instance_id, image, access, + image_type): + if image_type == ImageType.DISK_VHD: return cls._fetch_image_glance_vhd( - session, instance_id, image, access, type) + session, instance_id, image, access, image_type) else: return cls._fetch_image_glance_disk( - session, instance_id, image, access, type) + session, instance_id, image, access, image_type) @classmethod def _fetch_image_objectstore(cls, session, instance_id, image, access, - secret, type): + secret, image_type): url = images.image_url(image) LOG.debug(_("Asking xapi to fetch %(url)s as %(access)s") % locals()) - fn = (type != ImageType.KERNEL_RAMDISK) and 'get_vdi' or 'get_kernel' + if image_type == ImageType.KERNEL_RAMDISK: + fn = 'get_kernel' + else: + fn = 'get_vdi' args = {} args['src_url'] = url args['username'] = access args['password'] = secret args['add_partition'] = 'false' args['raw'] = 'false' - if type != ImageType.KERNEL_RAMDISK: + if image_type != ImageType.KERNEL_RAMDISK: args['add_partition'] = 'true' - if type == ImageType.DISK_RAW: + if image_type == ImageType.DISK_RAW: args['raw'] = 'true' task = session.async_call_plugin('objectstore', fn, args) uuid = session.wait_for_task(instance_id, task) @@ -857,9 +863,9 @@ def get_this_vm_ref(session): return session.get_xenapi().VM.get_by_uuid(get_this_vm_uuid()) -def _stream_disk(dev, type, virtual_size, image_file): +def _stream_disk(dev, image_type, virtual_size, image_file): offset = 0 - if type == ImageType.DISK: + if image_type == ImageType.DISK: offset = MBR_SIZE_BYTES _write_partition(virtual_size, dev) -- cgit From 05d135be0c0d8a90a97d62005a101345964800cf Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Fri, 18 Feb 2011 22:50:13 +0000 Subject: Typo fix --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 3b5cedda7..097670b58 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -215,7 +215,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): 'x-image-meta-property-kernel-id': 'nokernel', 'x-image-meta-property-ramdisk-id': 'noramdisk', 'x-image-meta-property-container-format': 'tarball', - 'transfer-encoding': "chunked", + 'transfer-encoding': 'chunked', 'content-type': 'application/octet-stream', } for header, value in headers.iteritems(): -- cgit From 3392f6b4b060402c8d9a442f1a89a24fa31c9342 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 21 Feb 2011 14:27:37 -0600 Subject: Removing duplicate installation docs and adding flag file information, plus pointing to docs.openstack.org for Admin-audience docs --- doc/source/adminguide/binaries.rst | 57 ---- doc/source/adminguide/distros/others.rst | 88 ------ doc/source/adminguide/distros/ubuntu.10.04.rst | 40 --- doc/source/adminguide/distros/ubuntu.10.10.rst | 41 --- doc/source/adminguide/euca2ools.rst | 49 ---- doc/source/adminguide/flags.rst | 23 -- doc/source/adminguide/getting.started.rst | 167 ----------- doc/source/adminguide/index.rst | 91 ------ doc/source/adminguide/managing.images.rst | 21 -- doc/source/adminguide/managing.instances.rst | 59 ---- doc/source/adminguide/managing.networks.rst | 70 ----- doc/source/adminguide/managing.projects.rst | 68 ----- doc/source/adminguide/managing.users.rst | 82 ------ doc/source/adminguide/managingsecurity.rst | 39 --- doc/source/adminguide/monitoring.rst | 27 -- doc/source/adminguide/multi.node.install.rst | 392 ------------------------- doc/source/adminguide/network.flat.rst | 60 ---- doc/source/adminguide/network.vlan.rst | 179 ----------- doc/source/adminguide/nova.manage.rst | 239 --------------- doc/source/adminguide/single.node.install.rst | 362 ----------------------- doc/source/community.rst | 12 +- doc/source/index.rst | 20 +- doc/source/object.model.rst | 14 +- doc/source/quickstart.rst | 2 +- 24 files changed, 24 insertions(+), 2178 deletions(-) delete mode 100644 doc/source/adminguide/binaries.rst delete mode 100644 doc/source/adminguide/distros/others.rst delete mode 100644 doc/source/adminguide/distros/ubuntu.10.04.rst delete mode 100644 doc/source/adminguide/distros/ubuntu.10.10.rst delete mode 100644 doc/source/adminguide/euca2ools.rst delete mode 100644 doc/source/adminguide/flags.rst delete mode 100644 doc/source/adminguide/getting.started.rst delete mode 100644 doc/source/adminguide/index.rst delete mode 100644 doc/source/adminguide/managing.images.rst delete mode 100644 doc/source/adminguide/managing.instances.rst delete mode 100644 doc/source/adminguide/managing.networks.rst delete mode 100644 doc/source/adminguide/managing.projects.rst delete mode 100644 doc/source/adminguide/managing.users.rst delete mode 100644 doc/source/adminguide/managingsecurity.rst delete mode 100644 doc/source/adminguide/monitoring.rst delete mode 100644 doc/source/adminguide/multi.node.install.rst delete mode 100644 doc/source/adminguide/network.flat.rst delete mode 100644 doc/source/adminguide/network.vlan.rst delete mode 100644 doc/source/adminguide/nova.manage.rst delete mode 100644 doc/source/adminguide/single.node.install.rst diff --git a/doc/source/adminguide/binaries.rst b/doc/source/adminguide/binaries.rst deleted file mode 100644 index 5c50a51f1..000000000 --- a/doc/source/adminguide/binaries.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _binaries: - -Nova Daemons -============= - -The configuration of these binaries relies on "flagfiles" using the google -gflags package:: - - $ nova-xxxxx --flagfile flagfile - -The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. - -nova-api --------- - -Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. - -nova-objectstore ----------------- - -Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with glance and a simple image manager. - -nova-compute ------------- - -Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. - -nova-volume ------------ - -Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. - -nova-network ------------- - -Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: - - $ nova-network --network_manager=nova.network.manager.FlatManager - -IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/adminguide/distros/others.rst b/doc/source/adminguide/distros/others.rst deleted file mode 100644 index ec14a9abb..000000000 --- a/doc/source/adminguide/distros/others.rst +++ /dev/null @@ -1,88 +0,0 @@ -Installation on other distros (like Debian, Fedora or CentOS ) -============================================================== - -Feel free to add additional notes for additional distributions. - -Nova installation on CentOS 5.5 -------------------------------- - -These are notes for installing OpenStack Compute on CentOS 5.5 and will be updated but are NOT final. Please test for accuracy and edit as you see fit. - -The principle botleneck for running nova on centos in python 2.6. Nova is written in python 2.6 and CentOS 5.5. comes with python 2.4. We can not update python system wide as some core utilities (like yum) is dependent on python 2.4. Also very few python 2.6 modules are available in centos/epel repos. - -Pre-reqs --------- - -Add euca2ools and EPEL repo first.:: - - cat >/etc/yum.repos.d/euca2ools.repo << EUCA_REPO_CONF_EOF - [eucalyptus] - name=euca2ools - baseurl=http://www.eucalyptussoftware.com/downloads/repo/euca2ools/1.3.1/yum/centos/ - enabled=1 - gpgcheck=0 - - EUCA_REPO_CONF_EOF - -:: - - rpm -Uvh 'http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-4.noarch.rpm' - -Now install python2.6, kvm and few other libraries through yum:: - - yum -y install dnsmasq vblade kpartx kvm gawk iptables ebtables bzr screen euca2ools curl rabbitmq-server gcc gcc-c++ autoconf automake swig openldap openldap-servers nginx python26 python26-devel python26-distribute git openssl-devel python26-tools mysql-server qemu kmod-kvm libxml2 libxslt libxslt-devel mysql-devel - -Then download the latest aoetools and then build(and install) it, check for the latest version on sourceforge, exact url will change if theres a new release:: - - wget -c http://sourceforge.net/projects/aoetools/files/aoetools/32/aoetools-32.tar.gz/download - tar -zxvf aoetools-32.tar.gz - cd aoetools-32 - make - make install - -Add the udev rules for aoetools:: - - cat > /etc/udev/rules.d/60-aoe.rules << AOE_RULES_EOF - SUBSYSTEM=="aoe", KERNEL=="discover", NAME="etherd/%k", GROUP="disk", MODE="0220" - SUBSYSTEM=="aoe", KERNEL=="err", NAME="etherd/%k", GROUP="disk", MODE="0440" - SUBSYSTEM=="aoe", KERNEL=="interfaces", NAME="etherd/%k", GROUP="disk", MODE="0220" - SUBSYSTEM=="aoe", KERNEL=="revalidate", NAME="etherd/%k", GROUP="disk", MODE="0220" - # aoe block devices - KERNEL=="etherd*", NAME="%k", GROUP="disk" - AOE_RULES_EOF - -Load the kernel modules:: - - modprobe aoe - -:: - - modprobe kvm - -Now, install the python modules using easy_install-2.6, this ensures the installation are done against python 2.6 - - -easy_install-2.6 twisted sqlalchemy mox greenlet carrot daemon eventlet tornado IPy routes lxml MySQL-python -python-gflags need to be downloaded and installed manually, use these commands (check the exact url for newer releases ): - -:: - - wget -c "http://python-gflags.googlecode.com/files/python-gflags-1.4.tar.gz" - tar -zxvf python-gflags-1.4.tar.gz - cd python-gflags-1.4 - python2.6 setup.py install - cd .. - -Same for python2.6-libxml2 module, notice the --with-python and --prefix flags. --with-python ensures we are building it against python2.6 (otherwise it will build against python2.4, which is default):: - - wget -c "ftp://xmlsoft.org/libxml2/libxml2-2.7.3.tar.gz" - tar -zxvf libxml2-2.7.3.tar.gz - cd libxml2-2.7.3 - ./configure --with-python=/usr/bin/python26 --prefix=/usr - make all - make install - cd python - python2.6 setup.py install - cd .. - -Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/distros/ubuntu.10.04.rst b/doc/source/adminguide/distros/ubuntu.10.04.rst deleted file mode 100644 index bd0693c46..000000000 --- a/doc/source/adminguide/distros/ubuntu.10.04.rst +++ /dev/null @@ -1,40 +0,0 @@ -Installing on Ubuntu 10.04 (Lucid) -================================== - -Step 1: Install dependencies ----------------------------- -Grab the latest code from launchpad: - -:: - - bzr clone lp:nova - -Here's a script you can use to install (and then run) Nova on Ubuntu or Debian (when using Debian, edit nova.sh to have USE_PPA=0): - -.. todo:: give a link to a stable releases page - -Step 2: Install dependencies ----------------------------- - -Nova requires rabbitmq for messaging, so install that first. - -*Note:* You must have sudo installed to run these commands as shown here. - -:: - - sudo apt-get install rabbitmq-server - - -You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. - -If you're running on Ubuntu 10.04, you'll need to install Twisted and python-gflags which is included in the OpenStack PPA. - -:: - - sudo apt-get install python-software-properties - sudo add-apt-repository ppa:nova-core/trunk - sudo apt-get update - sudo apt-get install python-twisted python-gflags - - -Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/distros/ubuntu.10.10.rst b/doc/source/adminguide/distros/ubuntu.10.10.rst deleted file mode 100644 index a3fa2def1..000000000 --- a/doc/source/adminguide/distros/ubuntu.10.10.rst +++ /dev/null @@ -1,41 +0,0 @@ -Installing on Ubuntu 10.10 (Maverick) -===================================== -Single Machine Installation (Ubuntu 10.10) - -While we wouldn't expect you to put OpenStack Compute into production on a non-LTS version of Ubuntu, these instructions are up-to-date with the latest version of Ubuntu. - -Make sure you are running Ubuntu 10.10 so that the packages will be available. This install requires more than 70 MB of free disk space. - -These instructions are based on Soren Hansen's blog entry, Openstack on Maverick. A script is in progress as well. - -Step 1: Install required prerequisites --------------------------------------- -Nova requires rabbitmq for messaging and redis for storing state (for now), so we'll install these first.:: - - sudo apt-get install rabbitmq-server redis-server - -You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. - -Step 2: Install Nova packages available in Maverick Meerkat ------------------------------------------------------------ -Type or copy/paste in the following line to get the packages that you use to run OpenStack Compute.:: - - sudo apt-get install python-nova - sudo apt-get install nova-api nova-objectstore nova-compute nova-scheduler nova-network euca2ools unzip - -You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. This operation may take a while as many dependent packages will be installed. Note: there is a dependency problem with python-nova which can be worked around by installing first. - -When the installation is complete, you'll see the following lines confirming::: - - Adding system user `nova' (UID 106) ... - Adding new user `nova' (UID 106) with group `nogroup' ... - Not creating home directory `/var/lib/nova'. - Setting up nova-scheduler (0.9.1~bzr331-0ubuntu2) ... - * Starting nova scheduler nova-scheduler - WARNING:root:Starting scheduler node - ...done. - Processing triggers for libc-bin ... - ldconfig deferred processing now taking place - Processing triggers for python-support ... - -Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/euca2ools.rst b/doc/source/adminguide/euca2ools.rst deleted file mode 100644 index 6f0c57358..000000000 --- a/doc/source/adminguide/euca2ools.rst +++ /dev/null @@ -1,49 +0,0 @@ -Euca2ools -========= - -Nova is compatible with most of the euca2ools command line utilities. Both Administrators and Users will find these tools helpful for day-to-day administration. - -* euca-add-group -* euca-delete-bundle -* euca-describe-instances -* euca-register -* euca-add-keypair -* euca-delete-group -* euca-describe-keypairs -* euca-release-address -* euca-allocate-address -* euca-delete-keypair -* euca-describe-regions -* euca-reset-image-attribute -* euca-associate-address -* euca-delete-snapshot -* euca-describe-snapshots -* euca-revoke -* euca-attach-volume -* euca-delete-volume -* euca-describe-volumes -* euca-run-instances -* euca-authorize -* euca-deregister -* euca-detach-volume -* euca-terminate-instances -* euca-bundle-image -* euca-describe-addresses -* euca-disassociate-address -* euca-unbundle -* euca-bundle-vol -* euca-describe-availability-zones -* euca-download-bundle -* euca-upload-bundle -* euca-confirm-product-instance -* euca-describe-groups -* euca-get-console-output -* euca-version -* euca-create-snapshot -* euca-describe-image-attribute -* euca-modify-image-attribute -* euca-create-volume -* euca-describe-images -* euca-reboot-instances - - diff --git a/doc/source/adminguide/flags.rst b/doc/source/adminguide/flags.rst deleted file mode 100644 index 072f0a1a5..000000000 --- a/doc/source/adminguide/flags.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Flags and Flagfiles -=================== - -* python-gflags -* flagfiles -* list of flags by component (see concepts list) diff --git a/doc/source/adminguide/getting.started.rst b/doc/source/adminguide/getting.started.rst deleted file mode 100644 index 675d8e664..000000000 --- a/doc/source/adminguide/getting.started.rst +++ /dev/null @@ -1,167 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Getting Started with Nova -========================= - -This code base is continually changing, so dependencies also change. If you -encounter any problems, see the :doc:`../community` page. -The `contrib/nova.sh` script should be kept up to date, and may be a good -resource to review when debugging. - -The purpose of this document is to get a system installed that you can use to -test your setup assumptions. Working from this base installtion you can -tweak configurations and work with different flags to monitor interaction with -your hardware, network, and other factors that will allow you to determine -suitability for your deployment. After following this setup method, you should -be able to experiment with different managers, drivers, and flags to get the -best performance. - -Dependencies ------------- - -Related servers we rely on - -* **RabbitMQ**: messaging queue, used for all communication between components - -Optional servers - -* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by - setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP - (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. - There is a script in the sources (`nova/auth/slap.sh`) to install a very basic - openldap server on ubuntu. -* **ReDIS**: There is a fake ldap auth driver - `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was - created for testing ldap implementation on systems that don't have an easy - means to install ldap. -* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to - be avilable. Currently, only sqlite3 an mysql have been tested. - -Python libraries that we use (from pip-requires): - -.. literalinclude:: ../../../tools/pip-requires - -Other libraries: - -* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available - from http://wiki.xensource.com/xenwiki/XCP_SDK or - http://community.citrix.com/cdn/xs/sdks. - -External unix tools that are required: - -* iptables -* ebtables -* gawk -* curl -* kvm -* libvirt -* dnsmasq -* vlan -* open-iscsi and iscsitarget (if you use iscsi volumes) -* aoetools and vblade-persist (if you use aoe-volumes) - -Nova uses cutting-edge versions of many packages. There are ubuntu packages in -the nova-core trunk ppa. You can use add this ppa to your sources list on an -ubuntu machine with the following commands:: - - sudo apt-get install -y python-software-properties - sudo add-apt-repository ppa:nova-core/trunk - -Recommended ------------ - -* euca2ools: python implementation of aws ec2-tools and ami tools -* build tornado to use C module for evented section - - -Installation --------------- - -You can install from packages for your particular Linux distribution if they are -available. Otherwise you can install from source by checking out the source -files from the `Nova Source Code Repository `_ -and running:: - - python setup.py install - -Configuration ---------------- - -Configuring the host system -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As you read through the Administration Guide you will notice configuration hints -inline with documentation on the subsystem you are configuring. Presented in -this "Getting Started with Nova" document, we only provide what you need to -get started as quickly as possible. For a more detailed description of system -configuration, start reading through :doc:`multi.node.install`. - -* Create a volume group (you can use an actual disk for the volume group as - well):: - - # This creates a 1GB file to create volumes out of - dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 - losetup --show -f MY_FILE_PATH - # replace /dev/loop0 below with whatever losetup returns - # nova-volumes is the default for the --volume_group flag - vgcreate nova-volumes /dev/loop0 - - -Configuring Nova -~~~~~~~~~~~~~~~~ - -Configuration of the entire system is performed through python-gflags. The -best way to track configuration is through the use of a flagfile. - -A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary -when you launch it. Flagfiles for nova are typically stored in -``/etc/nova/nova.conf``, and flags specific to a certain program are stored in -``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another -flagfile, so typically a file like ``nova-manage.conf`` would have as its first -line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before -specifying overrides or additional options. - -A sample configuration to test the system follows:: - - --verbose - --nodaemon - --auth_driver=nova.auth.dbdriver.DbDriver - -Running ---------- - -There are many parts to the nova system, each with a specific function. They -are built to be highly-available, so there are may configurations they can be -run in (ie: on many machines, many listeners per machine, etc). This part -of the guide only gets you started quickly, to learn about HA options, see -:doc:`multi.node.install`. - -Launch supporting services - -* rabbitmq -* redis (optional) -* mysql (optional) -* openldap (optional) - -Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` - -* nova-api -* nova-compute -* nova-objectstore -* nova-volume -* nova-scheduler diff --git a/doc/source/adminguide/index.rst b/doc/source/adminguide/index.rst deleted file mode 100644 index 3bd72cfdc..000000000 --- a/doc/source/adminguide/index.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Administration Guide -==================== - -This guide describes the basics of running and managing Nova. - -Running the Cloud ------------------ - -The fastest way to get a test cloud running is by following the directions in the :doc:`../quickstart`. - -Nova's cloud works via the interaction of a series of daemon processes that reside persistently on the host machine(s). Fortunately, the :doc:`../quickstart` process launches sample versions of all these daemons for you. Once you are familiar with basic Nova usage, you can learn more about daemons by reading :doc:`../service.architecture` and :doc:`binaries`. - -Administration Utilities ------------------------- - -There are two main tools that a system administrator will find useful to manage their Nova cloud: - -.. toctree:: - :maxdepth: 1 - - nova.manage - euca2ools - -The nova-manage command may only be run by users with admin priviledges. Commands for euca2ools can be used by all users, though specific commands may be restricted by Role Based Access Control. You can read more about creating and managing users in :doc:`managing.users` - -User and Resource Management ----------------------------- - -The nova-manage and euca2ools commands provide the basic interface to perform a broad range of administration functions. In this section, you can read more about how to accomplish specific administration tasks. - -For background on the core objects referenced in this section, see :doc:`../object.model` - -.. toctree:: - :maxdepth: 1 - - managing.users - managing.projects - managing.instances - managing.images - managing.volumes - managing.networks - -Deployment ----------- - -For a starting multi-node architecture, you would start with two nodes - a cloud controller node and a compute node. The cloud controller node contains the nova- services plus the Nova database. The compute node installs all the nova-services but then refers to the database installation, which is hosted by the cloud controller node. Ensure that the nova.conf file is identical on each node. If you find performance issues not related to database reads or writes, but due to the messaging queue backing up, you could add additional messaging services (rabbitmq). - -.. toctree:: - :maxdepth: 1 - - multi.node.install - dbsync - - -Networking -^^^^^^^^^^ - -.. toctree:: - :maxdepth: 1 - - multi.node.install - network.vlan.rst - network.flat.rst - - -Advanced Topics ---------------- - -.. toctree:: - :maxdepth: 1 - - flags - monitoring - diff --git a/doc/source/adminguide/managing.images.rst b/doc/source/adminguide/managing.images.rst deleted file mode 100644 index c5d93a6e8..000000000 --- a/doc/source/adminguide/managing.images.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Managing Images -=============== - -.. todo:: Put info on managing images here! diff --git a/doc/source/adminguide/managing.instances.rst b/doc/source/adminguide/managing.instances.rst deleted file mode 100644 index e62352017..000000000 --- a/doc/source/adminguide/managing.instances.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Managing Instances -================== - -Keypairs --------- - -Images can be shared by many users, so it is dangerous to put passwords into the images. Nova therefore supports injecting ssh keys into instances before they are booted. This allows a user to login to the instances that he or she creates securely. Generally the first thing that a user does when using the system is create a keypair. Nova generates a public and private key pair, and sends the private key to the user. The public key is stored so that it can be injected into instances. - -Keypairs are created through the api. They can be created on the command line using the euca2ools script euca-add-keypair. Refer to the man page for the available options. Example usage:: - - euca-add-keypair test > test.pem - chmod 600 test.pem - euca-run-instances -k test -t m1.tiny ami-tiny - # wait for boot - ssh -i test.pem root@ip.of.instance - - -Basic Management ----------------- -Instance management can be accomplished with euca commands: - - -To run an instance: - -:: - - euca-run-instances - - -To terminate an instance: - -:: - - euca-terminate-instances - -To reboot an instance: - -:: - - euca-reboot-instances - -See the euca2ools documentation for more information diff --git a/doc/source/adminguide/managing.networks.rst b/doc/source/adminguide/managing.networks.rst deleted file mode 100644 index 9eea46d70..000000000 --- a/doc/source/adminguide/managing.networks.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - Overview Sections Copyright 2010-2011 Citrix - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Networking Overview -=================== -In Nova, users organize their cloud resources in projects. A Nova project consists of a number of VM instances created by a user. For each VM instance, Nova assigns to it a private IP address. (Currently, Nova only supports Linux bridge networking that allows the virtual interfaces to connect to the outside network through the physical interface. Other virtual network technologies, such as Open vSwitch, could be supported in the future.) The Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. - -Nova Network Strategies ------------------------ - -Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can co-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. - -Read more about Nova network strategies here: - -.. toctree:: - :maxdepth: 1 - - network.flat.rst - network.vlan.rst - - -Network Management Commands ---------------------------- - -Admins and Network Administrators can use the 'nova-manage' command to manage network resources: - -VPN Management -~~~~~~~~~~~~~~ - -* vpn list: Print a listing of the VPNs for all projects. - * arguments: none -* vpn run: Start the VPN for a given project. - * arguments: project -* vpn spawn: Run all VPNs. - * arguments: none - - -Floating IP Management -~~~~~~~~~~~~~~~~~~~~~~ - -* floating create: Creates floating ips for host by range - * arguments: host ip_range -* floating delete: Deletes floating ips by range - * arguments: range -* floating list: Prints a listing of all floating ips - * arguments: none - -Network Management -~~~~~~~~~~~~~~~~~~ - -* network create: Creates fixed ips for host by range - * arguments: [fixed_range=FLAG], [num_networks=FLAG], - [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG] - diff --git a/doc/source/adminguide/managing.projects.rst b/doc/source/adminguide/managing.projects.rst deleted file mode 100644 index 5dd7f2de9..000000000 --- a/doc/source/adminguide/managing.projects.rst +++ /dev/null @@ -1,68 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Managing Projects -================= - -Projects are isolated resource containers forming the principal organizational structure within Nova. They consist of a separate vlan, volumes, instances, images, keys, and users. - -Although the original ec2 api only supports users, nova adds the concept of projects. A user can specify which project he or she wishes to use by appending `:project_id` to his or her access key. If no project is specified in the api request, nova will attempt to use a project with the same id as the user. - -The api will return NotAuthorized if a normal user attempts to make requests for a project that he or she is not a member of. Note that admins or users with special admin roles skip this check and can make requests for any project. - -To create a project, use the `project create` command of nova-manage. The syntax is nova-manage project create projectname manager_id [description] You must specify a projectname and a manager_id. For example:: - nova-manage project create john_project john "This is a sample project" - -You can add and remove users from projects with `project add` and `project remove`:: - nova-manage project add john_project john - nova-manage project remove john_project john - -Project Commands ----------------- - -Admins and Project Managers can use the 'nova-manage project' command to manage project resources: - -* project add: Adds user to project - * arguments: project user -* project create: Creates a new project - * arguments: name project_manager [description] -* project delete: Deletes an existing project - * arguments: project_id -* project environment: Exports environment variables to an sourcable file - * arguments: project_id user_id [filename='novarc] -* project list: lists all projects - * arguments: none -* project remove: Removes user from project - * arguments: project user -* project scrub: Deletes data associated with project - * arguments: project -* project zipfile: Exports credentials for project to a zip file - * arguments: project_id user_id [filename='nova.zip] - -Setting Quotas --------------- -Nova utilizes a quota system at the project level to control resource consumption across available hardware resources. Current quota controls are available to limit the: - -* Number of volumes which may be created -* Total size of all volumes within a project as measured in GB -* Number of instances which may be launched -* Number of processor cores which may be allocated -* Publicly accessible IP addresses - -Use the following command to set quotas for a project -* project quota: Set or display quotas for project - * arguments: project_id [key] [value] diff --git a/doc/source/adminguide/managing.users.rst b/doc/source/adminguide/managing.users.rst deleted file mode 100644 index 392142e86..000000000 --- a/doc/source/adminguide/managing.users.rst +++ /dev/null @@ -1,82 +0,0 @@ -Managing Users -============== - - -Users and Access Keys ---------------------- - -Access to the ec2 api is controlled by an access and secret key. The user's access key needs to be included in the request, and the request must be signed with the secret key. Upon receipt of api requests, nova will verify the signature and execute commands on behalf of the user. - -In order to begin using nova, you will need a to create a user. This can be easily accomplished using the user create or user admin commands in nova-manage. `user create` will create a regular user, whereas `user admin` will create an admin user. The syntax of the command is nova-manage user create username [access] [secret]. For example:: - - nova-manage user create john my-access-key a-super-secret-key - -If you do not specify an access or secret key, a random uuid will be created automatically. - -Credentials ------------ - -Nova can generate a handy set of credentials for a user. These credentials include a CA for bundling images and a file for setting environment variables to be used by euca2ools. If you don't need to bundle images, just the environment script is required. You can export one with the `project environment` command. The syntax of the command is nova-manage project environment project_id user_id [filename]. If you don't specify a filename, it will be exported as novarc. After generating the file, you can simply source it in bash to add the variables to your environment:: - - nova-manage project environment john_project john - . novarc - -If you do need to bundle images, you will need to get all of the credentials using `project zipfile`. Note that zipfile will give you an error message if networks haven't been created yet. Otherwise zipfile has the same syntax as environment, only the default file name is nova.zip. Example usage:: - - nova-manage project zipfile john_project john - unzip nova.zip - . novarc - -Role Based Access Control -------------------------- -Roles control the api actions that a user is allowed to perform. For example, a user cannot allocate a public ip without the `netadmin` role. It is important to remember that a users de facto permissions in a project is the intersection of user (global) roles and project (local) roles. So for john to have netadmin permissions in his project, he needs to separate roles specified. You can add roles with `role add`. The syntax is nova-manage role add user_id role [project_id]. Let's give john the netadmin role for his project:: - - nova-manage role add john netadmin - nova-manage role add john netadmin john_project - -Role-based access control (RBAC) is an approach to restricting system access to authorized users based on an individual’s role within an organization. Various employee functions require certain levels of system access in order to be successful. These functions are mapped to defined roles and individuals are categorized accordingly. Since users are not assigned permissions directly, but only acquire them through their role (or roles), management of individual user rights becomes a matter of assigning appropriate roles to the user. This simplifies common operations, such as adding a user, or changing a user's department. - -Nova’s rights management system employs the RBAC model and currently supports the following five roles: - -* **Cloud Administrator.** (admin) Users of this class enjoy complete system access. -* **IT Security.** (itsec) This role is limited to IT security personnel. It permits role holders to quarantine instances. -* **Project Manager.** (projectmanager)The default for project owners, this role affords users the ability to add other users to a project, interact with project images, and launch and terminate instances. -* **Network Administrator.** (netadmin) Users with this role are permitted to allocate and assign publicly accessible IP addresses as well as create and modify firewall rules. -* **Developer.** This is a general purpose role that is assigned to users by default. - -RBAC management is exposed through the dashboard for simplified user management. - - -User Commands -~~~~~~~~~~~~ - -Users, including admins, are created through the ``user`` commands. - -* user admin: creates a new admin and prints exports - * arguments: name [access] [secret] -* user create: creates a new user and prints exports - * arguments: name [access] [secret] -* user delete: deletes an existing user - * arguments: name -* user exports: prints access and secrets for user in export format - * arguments: name -* user list: lists all users - * arguments: none -* user modify: update a users keys & admin flag - * arguments: accesskey secretkey admin - * leave any field blank to ignore it, admin should be 'T', 'F', or blank - - -User Role Management -~~~~~~~~~~~~~~~~~~~~ - -* role add: adds role to user - * if project is specified, adds project specific role - * arguments: user, role [project] -* role has: checks to see if user has role - * if project is specified, returns True if user has - the global role and the project role - * arguments: user, role [project] -* role remove: removes role from user - * if project is specified, removes project specific role - * arguments: user, role [project] diff --git a/doc/source/adminguide/managingsecurity.rst b/doc/source/adminguide/managingsecurity.rst deleted file mode 100644 index 7893925e7..000000000 --- a/doc/source/adminguide/managingsecurity.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Security Considerations -======================= - -.. todo:: This doc is vague and just high-level right now. Describe architecture that enables security. - -The goal of securing a cloud computing system involves both protecting the instances, data on the instances, and -ensuring users are authenticated for actions and that borders are understood by the users and the system. -Protecting the system from intrusion or attack involves authentication, network protections, and -compromise detection. - -Key Concepts ------------- - -Authentication - Each instance is authenticated with a key pair. - -Network - Instances can communicate with each other but you can configure the boundaries through firewall -configuration. - -Monitoring - Log all API commands and audit those logs. - -Encryption - Data transfer between instances is not encrypted. - diff --git a/doc/source/adminguide/monitoring.rst b/doc/source/adminguide/monitoring.rst deleted file mode 100644 index 2c93c71b5..000000000 --- a/doc/source/adminguide/monitoring.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Monitoring -========== - -* components -* throughput -* exceptions -* hardware - -* ganglia -* syslog diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst deleted file mode 100644 index c53455e3e..000000000 --- a/doc/source/adminguide/multi.node.install.rst +++ /dev/null @@ -1,392 +0,0 @@ - -Installing Nova on Multiple Servers -=================================== - -When you move beyond evaluating the technology and into building an actual -production environment, you will need to know how to configure your datacenter -and how to deploy components across your clusters. This guide should help you -through that process. - -You can install multiple nodes to increase performance and availability of the OpenStack Compute installation. - -This setup is based on an Ubuntu Lucid 10.04 installation with the latest updates. Most of this works around issues that need to be resolved either in packaging or bug-fixing. It also needs to eventually be generalized, but the intent here is to get the multi-node configuration bootstrapped so folks can move forward. - -For a starting architecture, these instructions describing installing a cloud controller node and a compute node. The cloud controller node contains the nova- services plus the database. The compute node installs all the nova-services but then refers to the database installation, which is hosted by the cloud controller node. - -Requirements for a multi-node installation ------------------------------------------- - -* You need a real database, compatible with SQLAlchemy (mysql, postgresql) There's not a specific reason to choose one over another, it basically depends what you know. MySQL is easier to do High Availability (HA) with, but people may already know PostgreSQL. We should document both configurations, though. -* For a recommended HA setup, consider a MySQL master/slave replication, with as many slaves as you like, and probably a heartbeat to kick one of the slaves into being a master if it dies. -* For performance optimization, split reads and writes to the database. MySQL proxy is the easiest way to make this work if running MySQL. - -Assumptions ------------ - -* Networking is configured between/through the physical machines on a single subnet. -* Installation and execution are both performed by ROOT user. - - -Scripted Installation ---------------------- -A script available to get your OpenStack cloud running quickly. You can copy the file to the server where you want to install OpenStack Compute services - typically you would install a compute node and a cloud controller node. - -You must run these scripts with root permissions. - -From a server you intend to use as a cloud controller node, use this command to get the cloud controller script. This script is a work-in-progress and the maintainer plans to keep it up, but it is offered "as-is." Feel free to collaborate on it in GitHub - https://github.com/dubsquared/OpenStack-NOVA-Installer-Script/. - -:: - - wget --no-check-certificate https://github.com/dubsquared/OpenStack-NOVA-Installer-Script/raw/master/nova-CC-install-v1.1.sh - -Ensure you can execute the script by modifying the permissions on the script file. - -:: - - sudo chmod 755 nova-CC-install-v1.1.sh - - -:: - - sudo ./nova-CC-install-v1.1.sh - -Next, from a server you intend to use as a compute node (doesn't contain the database), install the nova services. You can use the nova-NODE-installer.sh script from the above github-hosted project for the compute node installation. - -Copy the nova.conf from the cloud controller node to the compute node. - -Restart related services:: - - libvirtd restart; service nova-network restart; service nova-compute restart; service nova-api restart; service nova-objectstore restart; service nova-scheduler restart - -You can go to the `Configuration section`_ for next steps. - -Manual Installation - Step-by-Step ----------------------------------- -The following sections show you how to install Nova manually with a cloud controller node and a separate compute node. The cloud controller node contains the database plus all nova- services, and the compute node runs nova- services only. - -Cloud Controller Installation -````````````````````````````` -On the cloud controller node, you install nova services and the related helper applications, and then configure with the nova.conf file. You will then copy the nova.conf file to the compute node, which you install as a second node in the `Compute Installation`_. - -Step 1 - Use apt-get to get the latest code -------------------------------------------- - -1. Setup Nova PPA with https://launchpad.net/~nova-core/+archive/trunk. The ‘python-software-properties’ package is a pre-requisite for setting up the nova package repo: - -:: - - sudo apt-get install python-software-properties - sudo add-apt-repository ppa:nova-core/trunk - -2. Run update. - -:: - - sudo apt-get update - -3. Install python required packages, nova-packages, and helper apps. - -:: - - sudo apt-get install python-greenlet python-mysqldb python-nova nova-common nova-doc nova-api nova-network nova-objectstore nova-scheduler nova-compute euca2ools unzip - -It is highly likely that there will be errors when the nova services come up since they are not yet configured. Don't worry, you're only at step 1! - -Step 2 Set up configuration file (installed in /etc/nova) ---------------------------------------------------------- - -1. Nova development has consolidated all config files to nova.conf as of November 2010. There is a default set of options that are already configured in nova.conf: - -:: - ---daemonize=1 ---dhcpbridge_flagfile=/etc/nova/nova.conf ---dhcpbridge=/usr/bin/nova-dhcpbridge ---logdir=/var/log/nova ---state_path=/var/lib/nova - -The following items ALSO need to be defined in /etc/nova/nova.conf. I’ve added some explanation of the variables, as comments CANNOT be in nova.conf. There seems to be an issue with nova-manage not processing the comments/whitespace correctly: - ---sql_connection ### Location of Nova SQL DB - ---s3_host ### This is where Nova is hosting the objectstore service, which will contain the VM images and buckets - ---rabbit_host ### This is where the rabbit AMQP messaging service is hosted - ---cc_host ### This is where the the nova-api service lives - ---verbose ### Optional but very helpful during initial setup - ---ec2_url ### The location to interface nova-api - ---network_manager ### Many options here, discussed below. This is how your controller will communicate with additional Nova nodes and VMs: - -nova.network.manager.FlatManager # Simple, no-vlan networking type -nova.network.manager. FlatDHCPManager # Flat networking with DHCP -nova.network.manager.VlanManager # Vlan networking with DHCP – /DEFAULT/ if no network manager is defined in nova.conf - ---fixed_range= ### This will be the IP network that ALL the projects for future VM guests will reside on. E.g. 192.168.0.0/12 - ---network_size=<# of addrs> ### This is the total number of IP Addrs to use for VM guests, of all projects. E.g. 5000 - -The following code can be cut and paste, and edited to your setup: - -Note: CC_ADDR= - -Detailed explanation of the following example is available above. - -:: - ---sql_connection=mysql://root:nova@/nova ---s3_host= ---rabbit_host= ---cc_host= ---verbose ---ec2_url=http://:8773/services/Cloud ---network_manager=nova.network.manager.VlanManager ---fixed_range= ---network_size=<# of addrs> - -2. Create a “nova” group, and set permissions:: - - addgroup nova - -The Nova config file should have its owner set to root:nova, and mode set to 0644, since they contain your MySQL server's root password. :: - - chown -R root:nova /etc/nova - chmod 644 /etc/nova/nova.conf - -Step 3 - Setup the SQL DB (MySQL for this setup) ------------------------------------------------- - -1. First you 'preseed' to bypass all the installation prompts:: - - bash - MYSQL_PASS=nova - cat < - # The loopback network interface - auto lo - iface lo inet loopback - - # Networking for NOVA - auto br100 - - iface br100 inet dhcp - bridge_ports eth0 - bridge_stp off - bridge_maxwait 0 - bridge_fd 0 - < end /etc/network/interfaces > - -Next, restart networking to apply the changes:: - - sudo /etc/init.d/networking restart - -Configuration -````````````` - -On the Compute node, you should continue with these configuration steps. - -Step 1 - Set up the Nova environment ------------------------------------- - -These are the commands you run to update the database if needed, and then set up a user and project:: - - /usr/bin/python /usr/bin/nova-manage db sync - /usr/bin/python /usr/bin/nova-manage user admin - /usr/bin/python /usr/bin/nova-manage project create - /usr/bin/python /usr/bin/nova-manage network create - -Here is an example of what this looks like with real data:: - - /usr/bin/python /usr/bin/nova-manage db sync - /usr/bin/python /usr/bin/nova-manage user admin dub - /usr/bin/python /usr/bin/nova-manage project create dubproject dub - /usr/bin/python /usr/bin/nova-manage network create 192.168.0.0/24 1 255 - -(I chose a /24 since that falls inside my /12 range I set in ‘fixed-range’ in nova.conf. Currently, there can only be one network, and I am using the max IP’s available in a /24. You can choose to use any valid amount that you would like.) - -Note: The nova-manage service assumes that the first IP address is your network (like 192.168.0.0), that the 2nd IP is your gateway (192.168.0.1), and that the broadcast is the very last IP in the range you defined (192.168.0.255). If this is not the case you will need to manually edit the sql db 'networks' table.o. - -On running the "nova-manage network create" command, entries are made in the 'networks' and 'fixed_ips' table. However, one of the networks listed in the 'networks' table needs to be marked as bridge in order for the code to know that a bridge exists. The Network is marked as bridged automatically based on the type of network manager selected. You only need to mark the network as a bridge if you chose FlatManager as your network type. More information can be found at the end of this document discussing setting up the bridge device. - - -Step 2 - Create Nova certifications ------------------------------------ - -1. Generate the certs as a zip file. These are the certs you will use to launch instances, bundle images, and all the other assorted api functions. - -:: - - mkdir –p /root/creds - /usr/bin/python /usr/bin/nova-manage project zipfile $NOVA_PROJECT $NOVA_PROJECT_USER /root/creds/novacreds.zip - -2. Unzip them in your home directory, and add them to your environment. - -:: - - unzip /root/creds/novacreds.zip -d /root/creds/ - cat /root/creds/novarc >> ~/.bashrc - source ~/.bashrc - -Step 3 - Restart all relevant services --------------------------------------- - -Restart all six services in total, just to cover the entire spectrum:: - - libvirtd restart; service nova-network restart; service nova-compute restart; service nova-api restart; service nova-objectstore restart; service nova-scheduler restart - -Step 4 - Closing steps, and cleaning up ---------------------------------------- - -One of the most commonly missed configuration areas is not allowing the proper access to VMs. Use the 'euca-authorize' command to enable access. Below, you will find the commands to allow 'ping' and 'ssh' to your VMs:: - - euca-authorize -P icmp -t -1:-1 default - euca-authorize -P tcp -p 22 default - -Another common issue is you cannot ping or SSH your instances after issusing the 'euca-authorize' commands. Something to look at is the amount of 'dnsmasq' processes that are running. If you have a running instance, check to see that TWO 'dnsmasq' processes are running. If not, perform the following:: - - killall dnsmasq - service nova-network restart - -To avoid issues with KVM and permissions with Nova, run the following commands to ensure we have VM's that are running optimally:: - - chgrp kvm /dev/kvm - chmod g+rwx /dev/kvm - -If you want to use the 10.04 Ubuntu Enterprise Cloud images that are readily available at http://uec-images.ubuntu.com/releases/10.04/release/, you may run into delays with booting. Any server that does not have nova-api running on it needs this iptables entry so that UEC images can get metadata info. On compute nodes, configure the iptables with this next step:: - - # iptables -t nat -A PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $NOVA_API_IP:8773 - -Testing the Installation -```````````````````````` - -You can confirm that your compute node is talking to your cloud controller. From the cloud controller, run this database query:: - - mysql -u$MYSQL_USER -p$MYSQL_PASS nova -e 'select * from services;' - -In return, you should see something similar to this:: - +---------------------+---------------------+------------+---------+----+----------+----------------+-----------+--------------+----------+-------------------+ - | created_at | updated_at | deleted_at | deleted | id | host | binary | topic | report_count | disabled | availability_zone | - +---------------------+---------------------+------------+---------+----+----------+----------------+-----------+--------------+----------+-------------------+ - | 2011-01-28 22:52:46 | 2011-02-03 06:55:48 | NULL | 0 | 1 | osdemo02 | nova-network | network | 46064 | 0 | nova | - | 2011-01-28 22:52:48 | 2011-02-03 06:55:57 | NULL | 0 | 2 | osdemo02 | nova-compute | compute | 46056 | 0 | nova | - | 2011-01-28 22:52:52 | 2011-02-03 06:55:50 | NULL | 0 | 3 | osdemo02 | nova-scheduler | scheduler | 46065 | 0 | nova | - | 2011-01-29 23:49:29 | 2011-02-03 06:54:26 | NULL | 0 | 4 | osdemo01 | nova-compute | compute | 37050 | 0 | nova | - | 2011-01-30 23:42:24 | 2011-02-03 06:55:44 | NULL | 0 | 9 | osdemo04 | nova-compute | compute | 28484 | 0 | nova | - | 2011-01-30 21:27:28 | 2011-02-03 06:54:23 | NULL | 0 | 8 | osdemo05 | nova-compute | compute | 29284 | 0 | nova | - +---------------------+---------------------+------------+---------+----+----------+----------------+-----------+--------------+----------+-------------------+ -You can see that 'osdemo0{1,2,4,5} are all running 'nova-compute.' When you start spinning up instances, they will allocate on any node that is running nova-compute from this list. - -You can then use `euca2ools` to test some items:: - - euca-describe-images - euca-describe-instances - -If you have issues with the API key, you may need to re-source your creds file:: - - . /root/creds/novarc - -If you don’t get any immediate errors, you’re successfully making calls to your cloud! - -Spinning up a VM for Testing -```````````````````````````` - -(This excerpt is from Thierry Carrez's blog, with reference to http://wiki.openstack.org/GettingImages.) - -The image that you will use here will be a ttylinux image, so this is a limited function server. You will be able to ping and SSH to this instance, but it is in no way a full production VM. - -UPDATE: Due to `bug 661159 `_, we can’t use images without ramdisks yet, so we can’t use the classic Ubuntu cloud images from http://uec-images.ubuntu.com/releases/ yet. For the sake of this tutorial, we’ll use the `ttylinux images from Scott Moser instead `_. - -Download the image, and publish to your bucket: - -:: - - image="ttylinux-uec-amd64-12.1_2.6.35-22_1.tar.gz" - wget http://smoser.brickies.net/ubuntu/ttylinux-uec/$image - uec-publish-tarball $image mybucket - -This will output three references, an "emi", an "eri" and an "eki." (Image, ramdisk, and kernel) The emi is the one we use to launch instances, so take note of this. - -Create a keypair to SSH to the server: - -:: - - euca-add-keypair mykey > mykey.priv - - chmod 0600 mykey.priv - -Boot your instance: - -:: - - euca-run-instances $emi -k mykey -t m1.tiny - -($emi is replaced with the output from the previous command) - -Checking status, and confirming communication: - -Once you have booted the instance, you can check the status the the `euca-describe-instances` command. Here you can view the instance ID, IP, and current status of the VM. - -:: - - euca-describe-instances - -Once in a "running" state, you can use your SSH key connect: - -:: - - ssh -i mykey.priv root@$ipaddress - -When you are ready to terminate the instance, you may do so with the `euca-terminate-instances` command: - -:: - - euca-terminate-instances $instance-id - -You can determine the instance-id with `euca-describe-instances`, and the format is "i-" with a series of letter and numbers following: e.g. i-a4g9d. - -For more information in creating you own custom (production ready) instance images, please visit http://wiki.openstack.org/GettingImages for more information! - -Enjoy your new private cloud, and play responsibly! diff --git a/doc/source/adminguide/network.flat.rst b/doc/source/adminguide/network.flat.rst deleted file mode 100644 index 3d8680c6f..000000000 --- a/doc/source/adminguide/network.flat.rst +++ /dev/null @@ -1,60 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -Flat Network Mode (Original and Flat) -===================================== - -Flat network mode removes most of the complexity of VLAN mode by simply -bridging all instance interfaces onto a single network. - -There are two variations of flat mode that differ mostly in how IP addresses -are given to instances. - - -Original Flat Mode ------------------- -IP addresses for VM instances are grabbed from a subnet specified by the network administrator, and injected into the image on launch. All instances of the system are attached to the same Linux networking bridge, configured manually by the network administrator both on the network controller hosting the network and on the computer controllers hosting the instances. To recap: - -* Each compute host creates a single bridge for all instances to use to attach to the external network. -* The networking configuration is injected into the instance before it is booted or it is obtained by a guest agent installed in the instance. - -Note that the configuration injection currently only works on linux-style systems that keep networking -configuration in /etc/network/interfaces. - - -Flat DHCP Mode --------------- -IP addresses for VM instances are grabbed from a subnet specified by the network administrator. Similar to the flat network, a single Linux networking bridge is created and configured manually by the network administrator and used for all instances. A DHCP server is started to pass out IP addresses to VM instances from the specified subnet. To recap: - -* Like flat mode, all instances are attached to a single bridge on the compute node. -* In addition a DHCP server is running to configure instances. - -Implementation --------------- - -The network nodes do not act as a default gateway in flat mode. Instances -are given public IP addresses. - -Compute nodes have iptables/ebtables entries created per project and -instance to protect against IP/MAC address spoofing and ARP poisoning. - - -Examples --------- - -.. todo:: add flat network mode configuration examples diff --git a/doc/source/adminguide/network.vlan.rst b/doc/source/adminguide/network.vlan.rst deleted file mode 100644 index c06ce8e8b..000000000 --- a/doc/source/adminguide/network.vlan.rst +++ /dev/null @@ -1,179 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -VLAN Network Mode -================= -VLAN Network Mode is the default mode for Nova. It provides a private network -segment for each project's instances that can be accessed via a dedicated -VPN connection from the Internet. - -In this mode, each project gets its own VLAN, Linux networking bridge, and subnet. The subnets are specified by the network administrator, and are assigned dynamically to a project when required. A DHCP Server is started for each VLAN to pass out IP addresses to VM instances from the subnet assigned to the project. All instances belonging to one project are bridged into the same VLAN for that project. The Linux networking bridges and VLANs are created by Nova when required, described in more detail in Nova VLAN Network Management Implementation. - -.. - (this text revised above) - Because the flat network and flat DhCP network are simple to understand and yet do not scale well enough for real-world cloud systems, this section focuses on the VLAN network implementation by the VLAN Network Manager. - - - In the VLAN network mode, all the VM instances of a project are connected together in a VLAN with the specified private subnet. Each running VM instance is assigned an IP address within the given private subnet. - -.. image:: /images/Novadiagram.png - :width: 790 - -While network traffic between VM instances belonging to the same VLAN is always open, Nova can enforce isolation of network traffic between different projects by enforcing one VLAN per project. - -In addition, the network administrator can specify a pool of public IP addresses that users may allocate and then assign to VMs, either at boot or dynamically at run-time. This capability is similar to Amazon's 'elastic IPs'. A public IP address may be associated with a running instances, allowing the VM instance to be accessed from the public network. The public IP addresses are accessible from the network host and NATed to the private IP address of the project. - -.. todo:: Describe how a public IP address could be associated with a project (a VLAN) - -This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code named :ref:`cloudpipe `) needs to be created. Nova generates a certificate and key for the user to access the vpn and starts the vpn automatically. More information on cloudpipe can be found :ref:`here `. - -The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) - -.. image:: /images/cloudpipe.png - :width: 100% - -Goals ------ - -For our implementation of Nova, our goal is that each project is in a protected network segment. Here are the specifications we keep in mind for meeting this goal. - - * RFC-1918 IP space - * public IP via NAT - * no default inbound Internet access without public NAT - * limited (project-admin controllable) outbound Internet access - * limited (project-admin controllable) access to other project segments - * all connectivity to instance and cloud API is via VPN into the project segment - -We also keep as a goal a common DMZ segment for support services, meaning these items are only visible from project segment: - - * metadata - * dashboard - -Limitations ------------ - -We kept in mind some of these limitations: - -* Projects / cluster limited to available VLANs in switching infrastructure -* Requires VPN for access to project segment - -Implementation --------------- -Currently Nova segregates project VLANs using 802.1q VLAN tagging in the -switching layer. Compute hosts create VLAN-specific interfaces and bridges -as required. - -The network nodes act as default gateway for project networks and contain -all of the routing and firewall rules implementing security groups. The -network node also handles DHCP to provide instance IPs for each project. - -VPN access is provided by running a small instance called CloudPipe -on the IP immediately following the gateway IP for each project. The -network node maps a dedicated public IP/port to the CloudPipe instance. - -Compute nodes have per-VLAN interfaces and bridges created as required. -These do NOT have IP addresses in the host to protect host access. -Compute nodes have iptables/ebtables entries created per project and -instance to protect against IP/MAC address spoofing and ARP poisoning. - -The network assignment to a project, and IP address assignment to a VM instance, are triggered when a user starts to run a VM instance. When running a VM instance, a user needs to specify a project for the instances, and the security groups (described in Security Groups) when the instance wants to join. If this is the first instance to be created for the project, then Nova (the cloud controller) needs to find a network controller to be the network host for the project; it then sets up a private network by finding an unused VLAN id, an unused subnet, and then the controller assigns them to the project, it also assigns a name to the project's Linux bridge (br100 stored in the Nova database), and allocating a private IP within the project's subnet for the new instance. - -If the instance the user wants to start is not the project's first, a subnet and a VLAN must have already been assigned to the project; therefore the system needs only to find an available IP address within the subnet and assign it to the new starting instance. If there is no private IP available within the subnet, an exception will be raised to the cloud controller, and the VM creation cannot proceed. - - -External Infrastructure ------------------------ - -Nova assumes the following is available: - -* DNS -* NTP -* Internet connectivity - - -Example -------- - -This example network configuration demonstrates most of the capabilities -of VLAN Mode. It splits administrative access to the nodes onto a dedicated -management network and uses dedicated network nodes to handle all -routing and gateway functions. - -It uses a 10GB network for instance traffic and a 1GB network for management. - - -Hardware -~~~~~~~~ - -* All nodes have a minimum of two NICs for management and production. - - * management is 1GB - * production is 10GB - * add additional NICs for bonding or HA/performance - -* network nodes should have an additional NIC dedicated to public Internet traffic -* switch needs to support enough simultaneous VLANs for number of projects -* production network configured as 802.1q trunk on switch - - -Operation -~~~~~~~~~ - -The network node controls the project network configuration: - -* assigns each project a VLAN and private IP range -* starts dnsmasq on project VLAN to serve private IP range -* configures iptables on network node for default project access -* launches CloudPipe instance and configures iptables access - -When starting an instance the network node: - -* sets up a VLAN interface and bridge on each host as required when an - instance is started on that host -* assigns private IP to instance -* generates MAC address for instance -* update dnsmasq with IP/MAC for instance - -When starting an instance the compute node: - -* sets up a VLAN interface and bridge on each host as required when an - instance is started on that host - - -Setup -~~~~~ - -* Assign VLANs in the switch: - - * public Internet segment - * production network - * management network - * cluster DMZ - -* Assign a contiguous range of VLANs to Nova for project use. -* Configure management NIC ports as management VLAN access ports. -* Configure management VLAN with Internet access as required -* Configure production NIC ports as 802.1q trunk ports. -* Configure Nova (need to add specifics here) - - * public IPs - * instance IPs - * project network size - * DMZ network - -.. todo:: need specific Nova configuration added diff --git a/doc/source/adminguide/nova.manage.rst b/doc/source/adminguide/nova.manage.rst deleted file mode 100644 index 0e9a29b6b..000000000 --- a/doc/source/adminguide/nova.manage.rst +++ /dev/null @@ -1,239 +0,0 @@ -.. - Copyright 2010-2011 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -The nova-manage command -======================= - -Introduction -~~~~~~~~~~~~ - -The nova-manage command is used to perform many essential functions for -administration and ongoing maintenance of nova, such as user creation, -vpn management, and much more. - -The standard pattern for executing a nova-manage command is: -``nova-manage []`` - -For example, to obtain a list of all projects: -``nova-manage project list`` - -Run without arguments to see a list of available command categories: -``nova-manage`` - -Categories are user, project, role, shell, vpn, and floating. Detailed descriptions are below. - -You can also run with a category argument such as user to see a list of all commands in that category: -``nova-manage user`` - -These sections describe the available categories and arguments for nova-manage. - -Nova Db -~~~~~~~ - -``nova-manage db version`` - - Print the current database version. - -``nova-manage db sync`` - - Sync the database up to the most recent version. This is the standard way to create the db as well. - -Nova User -~~~~~~~~~ - -``nova-manage user admin `` - - Create an admin user with the name . - -``nova-manage user create `` - - Create a normal user with the name . - -``nova-manage user delete `` - - Delete the user with the name . - -``nova-manage user exports `` - - Outputs a list of access key and secret keys for user to the screen - -``nova-manage user list`` - - Outputs a list of all the user names to the screen. - -``nova-manage user modify `` - - Updates the indicated user keys, indicating with T or F if the user is an admin user. Leave any argument blank if you do not want to update it. - -Nova Project -~~~~~~~~~~~~ - -``nova-manage project add `` - - Add a nova project with the name to the database. - -``nova-manage project create `` - - Create a new nova project with the name (you still need to do nova-manage project add to add it to the database). - -``nova-manage project delete `` - - Delete a nova project with the name . - -``nova-manage project environment `` - - Exports environment variables for the named project to a file named novarc. - -``nova-manage project list`` - - Outputs a list of all the projects to the screen. - -``nova-manage project quota `` - - Outputs the size and specs of the project's instances including gigabytes, instances, floating IPs, volumes, and cores. - -``nova-manage project remove `` - - Deletes the project with the name . - -``nova-manage project zipfile`` - - Compresses all related files for a created project into a zip file nova.zip. - -Nova Role -~~~~~~~~~ - -nova-manage role [] -``nova-manage role add <(optional) projectname>`` - - Add a user to either a global or project-based role with the indicated assigned to the named user. Role names can be one of the following five roles: admin, itsec, projectmanager, netadmin, developer. If you add the project name as the last argument then the role is assigned just for that project, otherwise the user is assigned the named role for all projects. - -``nova-manage role has `` - Checks the user or project and responds with True if the user has a global role with a particular project. - -``nova-manage role remove `` - Remove the indicated role from the user. - -Nova Shell -~~~~~~~~~~ - -``nova-manage shell bpython`` - - Starts a new bpython shell. - -``nova-manage shell ipython`` - - Starts a new ipython shell. - -``nova-manage shell python`` - - Starts a new python shell. - -``nova-manage shell run`` - - Starts a new shell using python. - -``nova-manage shell script `` - - Runs the named script from the specified path with flags set. - -Nova VPN -~~~~~~~~ - -``nova-manage vpn list`` - - Displays a list of projects, their IP prot numbers, and what state they're in. - -``nova-manage vpn run `` - - Starts the VPN for the named project. - -``nova-manage vpn spawn`` - - Runs all VPNs. - -Nova Floating IPs -~~~~~~~~~~~~~~~~~ - -``nova-manage floating create `` - - Creates floating IP addresses for the named host by the given range. - -``nova-manage floating delete `` - - Deletes floating IP addresses in the range given. - -``nova-manage floating list`` - - Displays a list of all floating IP addresses. - -Concept: Flags --------------- - -python-gflags - - -Concept: Plugins ----------------- - -* Managers/Drivers: utils.import_object from string flag -* virt/connections: conditional loading from string flag -* db: LazyPluggable via string flag -* auth_manager: utils.import_class based on string flag -* Volumes: moving to pluggable driver instead of manager -* Network: pluggable managers -* Compute: same driver used, but pluggable at connection - - -Concept: IPC/RPC ----------------- - -Rabbit! - - -Concept: Fakes --------------- - -* auth -* ldap - - -Concept: Scheduler ------------------- - -* simple -* random - - -Concept: Security Groups ------------------------- - -Security groups - - -Concept: Certificate Authority ------------------------------- - -Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns <../cloudpipe>` and decrypting bundled images. - - -Concept: Images ---------------- - -* launching -* bundling diff --git a/doc/source/adminguide/single.node.install.rst b/doc/source/adminguide/single.node.install.rst deleted file mode 100644 index ff43aa90b..000000000 --- a/doc/source/adminguide/single.node.install.rst +++ /dev/null @@ -1,362 +0,0 @@ -Installing Nova on a Single Host -================================ - -Nova can be run on a single machine, and it is recommended that new users practice managing this type of installation before graduating to multi node systems. - -The fastest way to get a test cloud running is through our :doc:`../quickstart`. But for more detail on installing the system read this doc. - - -Step 1 and 2: Get the latest Nova code system software ------------------------------------------------------- - -Depending on your system, the method for accomplishing this varies - -.. toctree:: - :maxdepth: 1 - - distros/ubuntu.10.04 - distros/ubuntu.10.10 - distros/others - - -Step 3: Build and install Nova services ---------------------------------------- - -Switch to the base nova source directory. - -Then type or copy/paste in the following line to compile the Python code for OpenStack Compute. - -:: - - sudo python setup.py build - sudo python setup.py install - - -When the installation is complete, you'll see the following lines: - -:: - - Installing nova-network script to /usr/local/bin - Installing nova-volume script to /usr/local/bin - Installing nova-objectstore script to /usr/local/bin - Installing nova-manage script to /usr/local/bin - Installing nova-scheduler script to /usr/local/bin - Installing nova-dhcpbridge script to /usr/local/bin - Installing nova-compute script to /usr/local/bin - Installing nova-instancemonitor script to /usr/local/bin - Installing nova-api script to /usr/local/bin - Installing nova-import-canonical-imagestore script to /usr/local/bin - - Installed /usr/local/lib/python2.6/dist-packages/nova-2010.1-py2.6.egg - Processing dependencies for nova==2010.1 - Finished processing dependencies for nova==2010.1 - - -Step 4: Create the Nova Database --------------------------------- -Type or copy/paste in the following line to create your nova db:: - - sudo nova-manage db sync - -Step 5: Create a Nova administrator ------------------------------------ -Type or copy/paste in the following line to create a user named "anne.":: - - sudo nova-manage user admin anne - -You see an access key and a secret key export, such as these made-up ones::: - - export EC2_ACCESS_KEY=4e6498a2-blah-blah-blah-17d1333t97fd - export EC2_SECRET_KEY=0a520304-blah-blah-blah-340sp34k05bbe9a7 - -Step 6: Create the network --------------------------- - -Type or copy/paste in the following line to create a network prior to creating a project. - -:: - - sudo nova-manage network create 10.0.0.0/8 1 64 - -For this command, the IP address is the cidr notation for your netmask, such as 192.168.1.0/24. The value 1 is the total number of networks you want made, and the 64 value is the total number of ips in all networks. - -After running this command, entries are made in the 'networks' and 'fixed_ips' table in the database. - -Step 7: Create a project with the user you created --------------------------------------------------- -Type or copy/paste in the following line to create a project named IRT (for Ice Road Truckers, of course) with the newly-created user named anne. - -:: - - sudo nova-manage project create IRT anne - -:: - - Generating RSA private key, 1024 bit long modulus - .....++++++ - ..++++++ - e is 65537 (0x10001) - Using configuration from ./openssl.cnf - Check that the request matches the signature - Signature ok - The Subject's Distinguished Name is as follows - countryName :PRINTABLE:'US' - stateOrProvinceName :PRINTABLE:'California' - localityName :PRINTABLE:'MountainView' - organizationName :PRINTABLE:'AnsoLabs' - organizationalUnitName:PRINTABLE:'NovaDev' - commonName :PRINTABLE:'anne-2010-10-12T21:12:35Z' - Certificate is to be certified until Oct 12 21:12:35 2011 GMT (365 days) - - Write out database with 1 new entries - Data Base Updated - - -Step 8: Unzip the nova.zip --------------------------- - -You should have a nova.zip file in your current working directory. Unzip it with this command: - -:: - - unzip nova.zip - - -You'll see these files extract. - -:: - - Archive: nova.zip - extracting: novarc - extracting: pk.pem - extracting: cert.pem - extracting: nova-vpn.conf - extracting: cacert.pem - - -Step 9: Source the rc file --------------------------- -Type or copy/paste the following to source the novarc file in your current working directory. - -:: - - . novarc - - -Step 10: Pat yourself on the back :) ------------------------------------ -Congratulations, your cloud is up and running, you’ve created an admin user, created a network, retrieved the user's credentials and put them in your environment. - -Now you need an image. - - -Step 11: Get an image --------------------- -To make things easier, we've provided a small image on the Rackspace CDN. Use this command to get it on your server. - -:: - - wget http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz - - -:: - - --2010-10-12 21:40:55-- http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz - Resolving cblah2.cdn.cloudfiles.rackspacecloud.com... 208.111.196.6, 208.111.196.7 - Connecting to cblah2.cdn.cloudfiles.rackspacecloud.com|208.111.196.6|:80... connected. - HTTP request sent, awaiting response... 200 OK - Length: 58520278 (56M) [application/x-gzip] - Saving to: `images.tgz' - - 100%[======================================>] 58,520,278 14.1M/s in 3.9s - - 2010-10-12 21:40:59 (14.1 MB/s) - `images.tgz' saved [58520278/58520278] - - - -Step 12: Decompress the image file ----------------------------------- -Use this command to extract the image files::: - - tar xvzf images.tgz - -You get a directory listing like so::: - - images - |-- aki-lucid - | |-- image - | `-- info.json - |-- ami-tiny - | |-- image - | `-- info.json - `-- ari-lucid - |-- image - `-- info.json - -Step 13: Send commands to upload sample image to the cloud ----------------------------------------------------------- - -Type or copy/paste the following commands to create a manifest for the kernel.:: - - euca-bundle-image -i images/aki-lucid/image -p kernel --kernel true - -You should see this in response::: - - Checking image - Tarring image - Encrypting image - Splitting image... - Part: kernel.part.0 - Generating manifest /tmp/kernel.manifest.xml - -Type or copy/paste the following commands to create a manifest for the ramdisk.:: - - euca-bundle-image -i images/ari-lucid/image -p ramdisk --ramdisk true - -You should see this in response::: - - Checking image - Tarring image - Encrypting image - Splitting image... - Part: ramdisk.part.0 - Generating manifest /tmp/ramdisk.manifest.xml - -Type or copy/paste the following commands to upload the kernel bundle.:: - - euca-upload-bundle -m /tmp/kernel.manifest.xml -b mybucket - -You should see this in response::: - - Checking bucket: mybucket - Creating bucket: mybucket - Uploading manifest file - Uploading part: kernel.part.0 - Uploaded image as mybucket/kernel.manifest.xml - -Type or copy/paste the following commands to upload the ramdisk bundle.:: - - euca-upload-bundle -m /tmp/ramdisk.manifest.xml -b mybucket - -You should see this in response::: - - Checking bucket: mybucket - Uploading manifest file - Uploading part: ramdisk.part.0 - Uploaded image as mybucket/ramdisk.manifest.xml - -Type or copy/paste the following commands to register the kernel and get its ID.:: - - euca-register mybucket/kernel.manifest.xml - -You should see this in response::: - - IMAGE ami-fcbj2non - -Type or copy/paste the following commands to register the ramdisk and get its ID.:: - - euca-register mybucket/ramdisk.manifest.xml - -You should see this in response::: - - IMAGE ami-orukptrc - -Type or copy/paste the following commands to create a manifest for the machine image associated with the ramdisk and kernel IDs that you got from the previous commands.:: - - euca-bundle-image -i images/ami-tiny/image -p machine --kernel ami-fcbj2non --ramdisk ami-orukptrc - -You should see this in response::: - - Checking image - Tarring image - Encrypting image - Splitting image... - Part: machine.part.0 - Part: machine.part.1 - Part: machine.part.2 - Part: machine.part.3 - Part: machine.part.4 - Generating manifest /tmp/machine.manifest.xml - -Type or copy/paste the following commands to upload the machine image bundle.:: - - euca-upload-bundle -m /tmp/machine.manifest.xml -b mybucket - -You should see this in response::: - - Checking bucket: mybucket - Uploading manifest file - Uploading part: machine.part.0 - Uploading part: machine.part.1 - Uploading part: machine.part.2 - Uploading part: machine.part.3 - Uploading part: machine.part.4 - Uploaded image as mybucket/machine.manifest.xml - -Type or copy/paste the following commands to register the machine image and get its ID.:: - - euca-register mybucket/machine.manifest.xml - -You should see this in response::: - - IMAGE ami-g06qbntt - -Type or copy/paste the following commands to register a SSH keypair for use in starting and accessing the instances.:: - - euca-add-keypair mykey > mykey.priv - chmod 600 mykey.priv - -Type or copy/paste the following commands to run an instance using the keypair and IDs that we previously created.:: - - euca-run-instances ami-g06qbntt --kernel ami-fcbj2non --ramdisk ami-orukptrc -k mykey - -You should see this in response::: - - RESERVATION r-0at28z12 IRT - INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 scheduling mykey (IRT, None) m1.small 2010-10-18 19:02:10.443599 - -Type or copy/paste the following commands to watch as the scheduler launches, and completes booting your instance.:: - - euca-describe-instances - -You should see this in response::: - - RESERVATION r-0at28z12 IRT - INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 launching mykey (IRT, cloud02) m1.small 2010-10-18 19:02:10.443599 - -Type or copy/paste the following commands to see when loading is completed and the instance is running.:: - - euca-describe-instances - -You should see this in response::: - - RESERVATION r-0at28z12 IRT - INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 running mykey (IRT, cloud02) 0 m1.small 2010-10-18 19:02:10.443599 - -Type or copy/paste the following commands to check that the virtual machine is running.:: - - virsh list - -You should see this in response::: - - Id Name State - ---------------------------------- - 1 2842445831 running - -Type or copy/paste the following commands to ssh to the instance using your private key.:: - - ssh -i mykey.priv root@10.0.0.3 - - -Troubleshooting Installation ----------------------------- - -If you see an "error loading the config file './openssl.cnf'" it means you can copy the openssl.cnf file to the location where Nova expects it and reboot, then try the command again. - -:: - - cp /etc/ssl/openssl.cnf ~ - sudo reboot - - - diff --git a/doc/source/community.rst b/doc/source/community.rst index 4ae32f1eb..e925a47bd 100644 --- a/doc/source/community.rst +++ b/doc/source/community.rst @@ -18,7 +18,7 @@ Getting Involved ================ -The Nova community is a very friendly group and there are places online to join in with the +The OpenStack community for Nova is a very friendly group and there are places online to join in with the community. Feel free to ask questions. This document points you to some of the places where you can communicate with people. @@ -83,3 +83,13 @@ Twitter Because all the cool kids do it: `@openstack `_. Also follow the `#openstack `_ tag for relevant tweets. + +OpenStack Docs Site +------------------- + +The `nova.openstack.org `_ site is geared towards developer documentation, +and the `docs.openstack.org `_ site is intended for cloud administrators +who are standing up and running OpenStack Compute in production. You can contribute to the Docs Site +by using bzr and Launchpad and contributing to the openstack-manuals project at http://launchpad.net/openstack-manuals. + + diff --git a/doc/source/index.rst b/doc/source/index.rst index d337fb69f..846d3cfcd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,11 +32,13 @@ Nova is written with the following design guidelines in mind: * **API Compatibility**: Nova strives to provide API-compatible with popular systems like Amazon EC2 This documentation is generated by the Sphinx toolkit and lives in the source -tree. Additional documentation on Nova and other components of OpenStack can -be found on the `OpenStack wiki`_. Also see the :doc:`community` page for -other ways to interact with the community. +tree. Additional draft and project documentation on Nova and other components of OpenStack can +be found on the `OpenStack wiki`_. Cloud administrators, refer to `docs.openstack.org`_. + +Also see the :doc:`community` page for other ways to interact with the community. .. _`OpenStack wiki`: http://wiki.openstack.org +.. _`docs.openstack.org`: http://docs.openstack.org Key Concepts @@ -50,17 +52,7 @@ Key Concepts service.architecture nova.object.model swift.object.model - -Administrator's Documentation -============================= - -.. toctree:: - :maxdepth: 1 - - livecd - adminguide/index - adminguide/single.node.install - adminguide/multi.node.install + runnova/index Developer Docs ============== diff --git a/doc/source/object.model.rst b/doc/source/object.model.rst index d02f151fd..419e89b0c 100644 --- a/doc/source/object.model.rst +++ b/doc/source/object.model.rst @@ -18,8 +18,6 @@ Object Model ============ -.. todo:: Add brief description for core models - .. graphviz:: digraph foo { @@ -42,27 +40,27 @@ Object Model Users ----- -Each Nova User is authorized based on their access key and secret key, assigned per-user. Read more at :doc:`/adminguide/managing.users`. +Each Nova User is authorized based on their access key and secret key, assigned per-user. Read more at :doc:`/runnova/managing.users`. Projects -------- -For Nova, access to images is based on the project. Read more at :doc:`/adminguide/managing.projects`. +For Nova, access to images is based on the project. Read more at :doc:`/runnova/managing.projects`. Images ------ -Images are binary files that run the operating system. Read more at :doc:`/adminguide/managing.images`. +Images are binary files that run the operating system. Read more at :doc:`/runnova/managing.images`. Instances --------- -Instances are running virtual servers. Read more at :doc:`/adminguide/managing.instances`. +Instances are running virtual servers. Read more at :doc:`/runnova/managing.instances`. Volumes ------- -.. todo:: Write doc about volumes +Volumes offer extra block level storage to instances. Read more at `Managing Volumes `_. Security Groups --------------- @@ -72,7 +70,7 @@ In Nova, a security group is a named collection of network access rules, like fi VLANs ----- -VLAN is the default network mode for Nova. Read more at :doc:`/adminguide/network.vlan`. +VLAN is the default network mode for Nova. Read more at :doc:`/runnova/network.vlan`. IP Addresses ------------ diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 17c9e10a8..84ed3fe01 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -54,7 +54,7 @@ Environment Variables By tweaking the environment that nova.sh run in, you can build slightly different configurations (though for more complex setups you should see -:doc:`/adminguide/getting.started` and :doc:`/adminguide/multi.node.install`). +`Installing and Configuring OpenStack Compute `_). * HOST_IP * Default: address of first interface from the ifconfig command -- cgit From df1213b8091c9a63a25b15c6eb82272255f96cb6 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 21 Feb 2011 14:30:20 -0600 Subject: Updated to remove built docs --- doc/.autogenerated | 406 ++++++ doc/build/.DS_Store | Bin 0 -> 6148 bytes doc/build/html/.DS_Store | Bin 0 -> 6148 bytes doc/build/html/.buildinfo | 4 + doc/source/api/autoindex.rst | 138 ++ doc/source/api/nova..adminclient.rst | 6 + doc/source/api/nova..api.direct.rst | 6 + doc/source/api/nova..api.ec2.admin.rst | 6 + doc/source/api/nova..api.ec2.apirequest.rst | 6 + doc/source/api/nova..api.ec2.cloud.rst | 6 + .../api/nova..api.ec2.metadatarequesthandler.rst | 6 + doc/source/api/nova..api.openstack.auth.rst | 6 + .../api/nova..api.openstack.backup_schedules.rst | 6 + doc/source/api/nova..api.openstack.common.rst | 6 + doc/source/api/nova..api.openstack.consoles.rst | 6 + doc/source/api/nova..api.openstack.faults.rst | 6 + doc/source/api/nova..api.openstack.flavors.rst | 6 + doc/source/api/nova..api.openstack.images.rst | 6 + doc/source/api/nova..api.openstack.servers.rst | 6 + .../api/nova..api.openstack.shared_ip_groups.rst | 6 + doc/source/api/nova..api.openstack.zones.rst | 6 + doc/source/api/nova..auth.dbdriver.rst | 6 + doc/source/api/nova..auth.fakeldap.rst | 6 + doc/source/api/nova..auth.ldapdriver.rst | 6 + doc/source/api/nova..auth.manager.rst | 6 + doc/source/api/nova..auth.signer.rst | 6 + doc/source/api/nova..cloudpipe.pipelib.rst | 6 + doc/source/api/nova..compute.api.rst | 6 + doc/source/api/nova..compute.instance_types.rst | 6 + doc/source/api/nova..compute.manager.rst | 6 + doc/source/api/nova..compute.monitor.rst | 6 + doc/source/api/nova..compute.power_state.rst | 6 + doc/source/api/nova..console.api.rst | 6 + doc/source/api/nova..console.fake.rst | 6 + doc/source/api/nova..console.manager.rst | 6 + doc/source/api/nova..console.xvp.rst | 6 + doc/source/api/nova..context.rst | 6 + doc/source/api/nova..crypto.rst | 6 + doc/source/api/nova..db.api.rst | 6 + doc/source/api/nova..db.base.rst | 6 + doc/source/api/nova..db.migration.rst | 6 + doc/source/api/nova..db.sqlalchemy.api.rst | 6 + .../nova..db.sqlalchemy.migrate_repo.manage.rst | 6 + ...sqlalchemy.migrate_repo.versions.001_austin.rst | 6 + ....sqlalchemy.migrate_repo.versions.002_bexar.rst | 6 + ...ate_repo.versions.003_add_label_to_networks.rst | 6 + ...y.migrate_repo.versions.004_add_zone_tables.rst | 6 + doc/source/api/nova..db.sqlalchemy.migration.rst | 6 + doc/source/api/nova..db.sqlalchemy.models.rst | 6 + doc/source/api/nova..db.sqlalchemy.session.rst | 6 + doc/source/api/nova..exception.rst | 6 + doc/source/api/nova..fakememcache.rst | 6 + doc/source/api/nova..fakerabbit.rst | 6 + doc/source/api/nova..flags.rst | 6 + doc/source/api/nova..image.glance.rst | 6 + doc/source/api/nova..image.local.rst | 6 + doc/source/api/nova..image.s3.rst | 6 + doc/source/api/nova..image.service.rst | 6 + doc/source/api/nova..log.rst | 6 + doc/source/api/nova..manager.rst | 6 + doc/source/api/nova..network.api.rst | 6 + doc/source/api/nova..network.linux_net.rst | 6 + doc/source/api/nova..network.manager.rst | 6 + doc/source/api/nova..objectstore.bucket.rst | 6 + doc/source/api/nova..objectstore.handler.rst | 6 + doc/source/api/nova..objectstore.image.rst | 6 + doc/source/api/nova..objectstore.stored.rst | 6 + doc/source/api/nova..quota.rst | 6 + doc/source/api/nova..rpc.rst | 6 + doc/source/api/nova..scheduler.chance.rst | 6 + doc/source/api/nova..scheduler.driver.rst | 6 + doc/source/api/nova..scheduler.manager.rst | 6 + doc/source/api/nova..scheduler.simple.rst | 6 + doc/source/api/nova..scheduler.zone.rst | 6 + doc/source/api/nova..service.rst | 6 + doc/source/api/nova..test.rst | 6 + doc/source/api/nova..tests.api.openstack.fakes.rst | 6 + .../nova..tests.api.openstack.test_adminapi.rst | 6 + .../api/nova..tests.api.openstack.test_api.rst | 6 + .../api/nova..tests.api.openstack.test_auth.rst | 6 + .../api/nova..tests.api.openstack.test_common.rst | 6 + .../api/nova..tests.api.openstack.test_faults.rst | 6 + .../api/nova..tests.api.openstack.test_flavors.rst | 6 + .../api/nova..tests.api.openstack.test_images.rst | 6 + ...nova..tests.api.openstack.test_ratelimiting.rst | 6 + .../api/nova..tests.api.openstack.test_servers.rst | 6 + .....tests.api.openstack.test_shared_ip_groups.rst | 6 + .../api/nova..tests.api.openstack.test_zones.rst | 6 + doc/source/api/nova..tests.api.test_wsgi.rst | 6 + doc/source/api/nova..tests.db.fakes.rst | 6 + doc/source/api/nova..tests.declare_flags.rst | 6 + doc/source/api/nova..tests.fake_flags.rst | 6 + doc/source/api/nova..tests.glance.stubs.rst | 6 + doc/source/api/nova..tests.hyperv_unittest.rst | 6 + .../api/nova..tests.objectstore_unittest.rst | 6 + doc/source/api/nova..tests.real_flags.rst | 6 + doc/source/api/nova..tests.runtime_flags.rst | 6 + doc/source/api/nova..tests.test_access.rst | 6 + doc/source/api/nova..tests.test_api.rst | 6 + doc/source/api/nova..tests.test_auth.rst | 6 + doc/source/api/nova..tests.test_cloud.rst | 6 + doc/source/api/nova..tests.test_compute.rst | 6 + doc/source/api/nova..tests.test_console.rst | 6 + doc/source/api/nova..tests.test_direct.rst | 6 + doc/source/api/nova..tests.test_flags.rst | 6 + doc/source/api/nova..tests.test_localization.rst | 6 + doc/source/api/nova..tests.test_log.rst | 6 + doc/source/api/nova..tests.test_middleware.rst | 6 + doc/source/api/nova..tests.test_misc.rst | 6 + doc/source/api/nova..tests.test_network.rst | 6 + doc/source/api/nova..tests.test_quota.rst | 6 + doc/source/api/nova..tests.test_rpc.rst | 6 + doc/source/api/nova..tests.test_scheduler.rst | 6 + doc/source/api/nova..tests.test_service.rst | 6 + doc/source/api/nova..tests.test_twistd.rst | 6 + doc/source/api/nova..tests.test_virt.rst | 6 + doc/source/api/nova..tests.test_volume.rst | 6 + doc/source/api/nova..tests.test_xenapi.rst | 6 + doc/source/api/nova..tests.xenapi.stubs.rst | 6 + doc/source/api/nova..twistd.rst | 6 + doc/source/api/nova..utils.rst | 6 + doc/source/api/nova..version.rst | 6 + doc/source/api/nova..virt.connection.rst | 6 + doc/source/api/nova..virt.disk.rst | 6 + doc/source/api/nova..virt.fake.rst | 6 + doc/source/api/nova..virt.hyperv.rst | 6 + doc/source/api/nova..virt.images.rst | 6 + doc/source/api/nova..virt.libvirt_conn.rst | 6 + doc/source/api/nova..virt.xenapi.fake.rst | 6 + doc/source/api/nova..virt.xenapi.network_utils.rst | 6 + doc/source/api/nova..virt.xenapi.vm_utils.rst | 6 + doc/source/api/nova..virt.xenapi.vmops.rst | 6 + doc/source/api/nova..virt.xenapi.volume_utils.rst | 6 + doc/source/api/nova..virt.xenapi.volumeops.rst | 6 + doc/source/api/nova..virt.xenapi_conn.rst | 6 + doc/source/api/nova..volume.api.rst | 6 + doc/source/api/nova..volume.driver.rst | 6 + doc/source/api/nova..volume.manager.rst | 6 + doc/source/api/nova..volume.san.rst | 6 + doc/source/api/nova..wsgi.rst | 6 + doc/source/runnova/binaries.rst | 57 + doc/source/runnova/euca2ools.rst | 49 + doc/source/runnova/flags.rst | 193 +++ doc/source/runnova/getting.started.rst | 168 +++ doc/source/runnova/index.rst | 90 ++ doc/source/runnova/managing.images.rst | 21 + doc/source/runnova/managing.instances.rst | 59 + doc/source/runnova/managing.networks.rst | 70 + doc/source/runnova/managing.projects.rst | 68 + doc/source/runnova/managing.users.rst | 82 ++ doc/source/runnova/managingsecurity.rst | 39 + doc/source/runnova/monitoring.rst | 27 + doc/source/runnova/network.flat.rst | 60 + doc/source/runnova/network.vlan.rst | 179 +++ doc/source/runnova/nova.manage.rst | 239 ++++ test/.Python | 1 + test/bin/activate | 76 ++ test/bin/activate.csh | 32 + test/bin/activate.fish | 79 ++ test/bin/activate_this.py | 32 + test/bin/easy_install | 9 + test/bin/easy_install-2.6 | 9 + test/bin/pip | 9 + test/bin/pip-2.6 | 9 + test/bin/python | Bin 0 -> 50720 bytes test/bin/python2.6 | 1 + test/include/python2.6 | 1 + test/lib/python2.6/UserDict.py | 1 + test/lib/python2.6/_abcoll.py | 1 + test/lib/python2.6/abc.py | 1 + test/lib/python2.6/codecs.py | 1 + test/lib/python2.6/config | 1 + test/lib/python2.6/copy_reg.py | 1 + test/lib/python2.6/distutils/__init__.py | 91 ++ test/lib/python2.6/distutils/distutils.cfg | 6 + test/lib/python2.6/encodings | 1 + test/lib/python2.6/fnmatch.py | 1 + test/lib/python2.6/genericpath.py | 1 + test/lib/python2.6/lib-dynload | 1 + test/lib/python2.6/linecache.py | 1 + test/lib/python2.6/locale.py | 1 + test/lib/python2.6/ntpath.py | 1 + test/lib/python2.6/orig-prefix.txt | 1 + test/lib/python2.6/os.py | 1 + test/lib/python2.6/posixpath.py | 1 + test/lib/python2.6/re.py | 1 + test/lib/python2.6/site-packages/easy-install.pth | 4 + .../pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO | 348 +++++ .../pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt | 57 + .../EGG-INFO/dependency_links.txt | 1 + .../pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt | 4 + .../pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe | 1 + .../pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt | 1 + .../pip-0.8.1-py2.6.egg/pip/__init__.py | 261 ++++ .../pip-0.8.1-py2.6.egg/pip/_pkgutil.py | 589 ++++++++ .../pip-0.8.1-py2.6.egg/pip/backwardcompat.py | 55 + .../pip-0.8.1-py2.6.egg/pip/basecommand.py | 203 +++ .../pip-0.8.1-py2.6.egg/pip/baseparser.py | 231 ++++ .../pip-0.8.1-py2.6.egg/pip/commands/__init__.py | 1 + .../pip-0.8.1-py2.6.egg/pip/commands/bundle.py | 33 + .../pip-0.8.1-py2.6.egg/pip/commands/completion.py | 60 + .../pip-0.8.1-py2.6.egg/pip/commands/freeze.py | 109 ++ .../pip-0.8.1-py2.6.egg/pip/commands/help.py | 32 + .../pip-0.8.1-py2.6.egg/pip/commands/install.py | 247 ++++ .../pip-0.8.1-py2.6.egg/pip/commands/search.py | 116 ++ .../pip-0.8.1-py2.6.egg/pip/commands/uninstall.py | 42 + .../pip-0.8.1-py2.6.egg/pip/commands/unzip.py | 9 + .../pip-0.8.1-py2.6.egg/pip/commands/zip.py | 346 +++++ .../pip-0.8.1-py2.6.egg/pip/download.py | 470 +++++++ .../pip-0.8.1-py2.6.egg/pip/exceptions.py | 17 + .../site-packages/pip-0.8.1-py2.6.egg/pip/index.py | 686 ++++++++++ .../pip-0.8.1-py2.6.egg/pip/locations.py | 45 + .../site-packages/pip-0.8.1-py2.6.egg/pip/log.py | 181 +++ .../site-packages/pip-0.8.1-py2.6.egg/pip/req.py | 1432 ++++++++++++++++++++ .../pip-0.8.1-py2.6.egg/pip/runner.py | 18 + .../site-packages/pip-0.8.1-py2.6.egg/pip/util.py | 479 +++++++ .../pip-0.8.1-py2.6.egg/pip/vcs/__init__.py | 238 ++++ .../pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py | 138 ++ .../pip-0.8.1-py2.6.egg/pip/vcs/git.py | 204 +++ .../pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py | 162 +++ .../pip-0.8.1-py2.6.egg/pip/vcs/subversion.py | 260 ++++ .../site-packages/pip-0.8.1-py2.6.egg/pip/venv.py | 53 + .../site-packages/setuptools-0.6c11-py2.6.egg | Bin 0 -> 333447 bytes test/lib/python2.6/site-packages/setuptools.pth | 1 + test/lib/python2.6/site.py | 713 ++++++++++ test/lib/python2.6/sre.py | 1 + test/lib/python2.6/sre_compile.py | 1 + test/lib/python2.6/sre_constants.py | 1 + test/lib/python2.6/sre_parse.py | 1 + test/lib/python2.6/stat.py | 1 + test/lib/python2.6/types.py | 1 + test/lib/python2.6/warnings.py | 1 + 232 files changed, 10985 insertions(+) create mode 100644 doc/.autogenerated create mode 100644 doc/build/.DS_Store create mode 100644 doc/build/html/.DS_Store create mode 100644 doc/build/html/.buildinfo create mode 100644 doc/source/api/autoindex.rst create mode 100644 doc/source/api/nova..adminclient.rst create mode 100644 doc/source/api/nova..api.direct.rst create mode 100644 doc/source/api/nova..api.ec2.admin.rst create mode 100644 doc/source/api/nova..api.ec2.apirequest.rst create mode 100644 doc/source/api/nova..api.ec2.cloud.rst create mode 100644 doc/source/api/nova..api.ec2.metadatarequesthandler.rst create mode 100644 doc/source/api/nova..api.openstack.auth.rst create mode 100644 doc/source/api/nova..api.openstack.backup_schedules.rst create mode 100644 doc/source/api/nova..api.openstack.common.rst create mode 100644 doc/source/api/nova..api.openstack.consoles.rst create mode 100644 doc/source/api/nova..api.openstack.faults.rst create mode 100644 doc/source/api/nova..api.openstack.flavors.rst create mode 100644 doc/source/api/nova..api.openstack.images.rst create mode 100644 doc/source/api/nova..api.openstack.servers.rst create mode 100644 doc/source/api/nova..api.openstack.shared_ip_groups.rst create mode 100644 doc/source/api/nova..api.openstack.zones.rst create mode 100644 doc/source/api/nova..auth.dbdriver.rst create mode 100644 doc/source/api/nova..auth.fakeldap.rst create mode 100644 doc/source/api/nova..auth.ldapdriver.rst create mode 100644 doc/source/api/nova..auth.manager.rst create mode 100644 doc/source/api/nova..auth.signer.rst create mode 100644 doc/source/api/nova..cloudpipe.pipelib.rst create mode 100644 doc/source/api/nova..compute.api.rst create mode 100644 doc/source/api/nova..compute.instance_types.rst create mode 100644 doc/source/api/nova..compute.manager.rst create mode 100644 doc/source/api/nova..compute.monitor.rst create mode 100644 doc/source/api/nova..compute.power_state.rst create mode 100644 doc/source/api/nova..console.api.rst create mode 100644 doc/source/api/nova..console.fake.rst create mode 100644 doc/source/api/nova..console.manager.rst create mode 100644 doc/source/api/nova..console.xvp.rst create mode 100644 doc/source/api/nova..context.rst create mode 100644 doc/source/api/nova..crypto.rst create mode 100644 doc/source/api/nova..db.api.rst create mode 100644 doc/source/api/nova..db.base.rst create mode 100644 doc/source/api/nova..db.migration.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.api.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.migration.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.models.rst create mode 100644 doc/source/api/nova..db.sqlalchemy.session.rst create mode 100644 doc/source/api/nova..exception.rst create mode 100644 doc/source/api/nova..fakememcache.rst create mode 100644 doc/source/api/nova..fakerabbit.rst create mode 100644 doc/source/api/nova..flags.rst create mode 100644 doc/source/api/nova..image.glance.rst create mode 100644 doc/source/api/nova..image.local.rst create mode 100644 doc/source/api/nova..image.s3.rst create mode 100644 doc/source/api/nova..image.service.rst create mode 100644 doc/source/api/nova..log.rst create mode 100644 doc/source/api/nova..manager.rst create mode 100644 doc/source/api/nova..network.api.rst create mode 100644 doc/source/api/nova..network.linux_net.rst create mode 100644 doc/source/api/nova..network.manager.rst create mode 100644 doc/source/api/nova..objectstore.bucket.rst create mode 100644 doc/source/api/nova..objectstore.handler.rst create mode 100644 doc/source/api/nova..objectstore.image.rst create mode 100644 doc/source/api/nova..objectstore.stored.rst create mode 100644 doc/source/api/nova..quota.rst create mode 100644 doc/source/api/nova..rpc.rst create mode 100644 doc/source/api/nova..scheduler.chance.rst create mode 100644 doc/source/api/nova..scheduler.driver.rst create mode 100644 doc/source/api/nova..scheduler.manager.rst create mode 100644 doc/source/api/nova..scheduler.simple.rst create mode 100644 doc/source/api/nova..scheduler.zone.rst create mode 100644 doc/source/api/nova..service.rst create mode 100644 doc/source/api/nova..test.rst create mode 100644 doc/source/api/nova..tests.api.openstack.fakes.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_adminapi.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_api.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_auth.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_common.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_faults.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_flavors.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_images.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_servers.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst create mode 100644 doc/source/api/nova..tests.api.openstack.test_zones.rst create mode 100644 doc/source/api/nova..tests.api.test_wsgi.rst create mode 100644 doc/source/api/nova..tests.db.fakes.rst create mode 100644 doc/source/api/nova..tests.declare_flags.rst create mode 100644 doc/source/api/nova..tests.fake_flags.rst create mode 100644 doc/source/api/nova..tests.glance.stubs.rst create mode 100644 doc/source/api/nova..tests.hyperv_unittest.rst create mode 100644 doc/source/api/nova..tests.objectstore_unittest.rst create mode 100644 doc/source/api/nova..tests.real_flags.rst create mode 100644 doc/source/api/nova..tests.runtime_flags.rst create mode 100644 doc/source/api/nova..tests.test_access.rst create mode 100644 doc/source/api/nova..tests.test_api.rst create mode 100644 doc/source/api/nova..tests.test_auth.rst create mode 100644 doc/source/api/nova..tests.test_cloud.rst create mode 100644 doc/source/api/nova..tests.test_compute.rst create mode 100644 doc/source/api/nova..tests.test_console.rst create mode 100644 doc/source/api/nova..tests.test_direct.rst create mode 100644 doc/source/api/nova..tests.test_flags.rst create mode 100644 doc/source/api/nova..tests.test_localization.rst create mode 100644 doc/source/api/nova..tests.test_log.rst create mode 100644 doc/source/api/nova..tests.test_middleware.rst create mode 100644 doc/source/api/nova..tests.test_misc.rst create mode 100644 doc/source/api/nova..tests.test_network.rst create mode 100644 doc/source/api/nova..tests.test_quota.rst create mode 100644 doc/source/api/nova..tests.test_rpc.rst create mode 100644 doc/source/api/nova..tests.test_scheduler.rst create mode 100644 doc/source/api/nova..tests.test_service.rst create mode 100644 doc/source/api/nova..tests.test_twistd.rst create mode 100644 doc/source/api/nova..tests.test_virt.rst create mode 100644 doc/source/api/nova..tests.test_volume.rst create mode 100644 doc/source/api/nova..tests.test_xenapi.rst create mode 100644 doc/source/api/nova..tests.xenapi.stubs.rst create mode 100644 doc/source/api/nova..twistd.rst create mode 100644 doc/source/api/nova..utils.rst create mode 100644 doc/source/api/nova..version.rst create mode 100644 doc/source/api/nova..virt.connection.rst create mode 100644 doc/source/api/nova..virt.disk.rst create mode 100644 doc/source/api/nova..virt.fake.rst create mode 100644 doc/source/api/nova..virt.hyperv.rst create mode 100644 doc/source/api/nova..virt.images.rst create mode 100644 doc/source/api/nova..virt.libvirt_conn.rst create mode 100644 doc/source/api/nova..virt.xenapi.fake.rst create mode 100644 doc/source/api/nova..virt.xenapi.network_utils.rst create mode 100644 doc/source/api/nova..virt.xenapi.vm_utils.rst create mode 100644 doc/source/api/nova..virt.xenapi.vmops.rst create mode 100644 doc/source/api/nova..virt.xenapi.volume_utils.rst create mode 100644 doc/source/api/nova..virt.xenapi.volumeops.rst create mode 100644 doc/source/api/nova..virt.xenapi_conn.rst create mode 100644 doc/source/api/nova..volume.api.rst create mode 100644 doc/source/api/nova..volume.driver.rst create mode 100644 doc/source/api/nova..volume.manager.rst create mode 100644 doc/source/api/nova..volume.san.rst create mode 100644 doc/source/api/nova..wsgi.rst create mode 100644 doc/source/runnova/binaries.rst create mode 100644 doc/source/runnova/euca2ools.rst create mode 100644 doc/source/runnova/flags.rst create mode 100644 doc/source/runnova/getting.started.rst create mode 100644 doc/source/runnova/index.rst create mode 100644 doc/source/runnova/managing.images.rst create mode 100644 doc/source/runnova/managing.instances.rst create mode 100644 doc/source/runnova/managing.networks.rst create mode 100644 doc/source/runnova/managing.projects.rst create mode 100644 doc/source/runnova/managing.users.rst create mode 100644 doc/source/runnova/managingsecurity.rst create mode 100644 doc/source/runnova/monitoring.rst create mode 100644 doc/source/runnova/network.flat.rst create mode 100644 doc/source/runnova/network.vlan.rst create mode 100644 doc/source/runnova/nova.manage.rst create mode 120000 test/.Python create mode 100644 test/bin/activate create mode 100644 test/bin/activate.csh create mode 100644 test/bin/activate.fish create mode 100644 test/bin/activate_this.py create mode 100755 test/bin/easy_install create mode 100755 test/bin/easy_install-2.6 create mode 100755 test/bin/pip create mode 100755 test/bin/pip-2.6 create mode 100755 test/bin/python create mode 120000 test/bin/python2.6 create mode 120000 test/include/python2.6 create mode 120000 test/lib/python2.6/UserDict.py create mode 120000 test/lib/python2.6/_abcoll.py create mode 120000 test/lib/python2.6/abc.py create mode 120000 test/lib/python2.6/codecs.py create mode 120000 test/lib/python2.6/config create mode 120000 test/lib/python2.6/copy_reg.py create mode 100644 test/lib/python2.6/distutils/__init__.py create mode 100644 test/lib/python2.6/distutils/distutils.cfg create mode 120000 test/lib/python2.6/encodings create mode 120000 test/lib/python2.6/fnmatch.py create mode 120000 test/lib/python2.6/genericpath.py create mode 120000 test/lib/python2.6/lib-dynload create mode 120000 test/lib/python2.6/linecache.py create mode 120000 test/lib/python2.6/locale.py create mode 120000 test/lib/python2.6/ntpath.py create mode 100644 test/lib/python2.6/orig-prefix.txt create mode 120000 test/lib/python2.6/os.py create mode 120000 test/lib/python2.6/posixpath.py create mode 120000 test/lib/python2.6/re.py create mode 100644 test/lib/python2.6/site-packages/easy-install.pth create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py create mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py create mode 100644 test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg create mode 100644 test/lib/python2.6/site-packages/setuptools.pth create mode 100644 test/lib/python2.6/site.py create mode 120000 test/lib/python2.6/sre.py create mode 120000 test/lib/python2.6/sre_compile.py create mode 120000 test/lib/python2.6/sre_constants.py create mode 120000 test/lib/python2.6/sre_parse.py create mode 120000 test/lib/python2.6/stat.py create mode 120000 test/lib/python2.6/types.py create mode 120000 test/lib/python2.6/warnings.py diff --git a/doc/.autogenerated b/doc/.autogenerated new file mode 100644 index 000000000..e4c98ec9b --- /dev/null +++ b/doc/.autogenerated @@ -0,0 +1,406 @@ +source/api/nova..adminclient.rst +source/api/nova..api.direct.rst +source/api/nova..api.ec2.admin.rst +source/api/nova..api.ec2.apirequest.rst +source/api/nova..api.ec2.cloud.rst +source/api/nova..api.ec2.metadatarequesthandler.rst +source/api/nova..api.openstack.auth.rst +source/api/nova..api.openstack.backup_schedules.rst +source/api/nova..api.openstack.common.rst +source/api/nova..api.openstack.consoles.rst +source/api/nova..api.openstack.faults.rst +source/api/nova..api.openstack.flavors.rst +source/api/nova..api.openstack.images.rst +source/api/nova..api.openstack.servers.rst +source/api/nova..api.openstack.shared_ip_groups.rst +source/api/nova..api.openstack.zones.rst +source/api/nova..auth.dbdriver.rst +source/api/nova..auth.fakeldap.rst +source/api/nova..auth.ldapdriver.rst +source/api/nova..auth.manager.rst +source/api/nova..auth.signer.rst +source/api/nova..cloudpipe.pipelib.rst +source/api/nova..compute.api.rst +source/api/nova..compute.instance_types.rst +source/api/nova..compute.manager.rst +source/api/nova..compute.monitor.rst +source/api/nova..compute.power_state.rst +source/api/nova..console.api.rst +source/api/nova..console.fake.rst +source/api/nova..console.manager.rst +source/api/nova..console.xvp.rst +source/api/nova..context.rst +source/api/nova..crypto.rst +source/api/nova..db.api.rst +source/api/nova..db.base.rst +source/api/nova..db.migration.rst +source/api/nova..db.sqlalchemy.api.rst +source/api/nova..db.sqlalchemy.migrate_repo.manage.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst +source/api/nova..db.sqlalchemy.migration.rst +source/api/nova..db.sqlalchemy.models.rst +source/api/nova..db.sqlalchemy.session.rst +source/api/nova..exception.rst +source/api/nova..fakememcache.rst +source/api/nova..fakerabbit.rst +source/api/nova..flags.rst +source/api/nova..image.glance.rst +source/api/nova..image.local.rst +source/api/nova..image.s3.rst +source/api/nova..image.service.rst +source/api/nova..log.rst +source/api/nova..manager.rst +source/api/nova..network.api.rst +source/api/nova..network.linux_net.rst +source/api/nova..network.manager.rst +source/api/nova..objectstore.bucket.rst +source/api/nova..objectstore.handler.rst +source/api/nova..objectstore.image.rst +source/api/nova..objectstore.stored.rst +source/api/nova..quota.rst +source/api/nova..rpc.rst +source/api/nova..scheduler.chance.rst +source/api/nova..scheduler.driver.rst +source/api/nova..scheduler.manager.rst +source/api/nova..scheduler.simple.rst +source/api/nova..scheduler.zone.rst +source/api/nova..service.rst +source/api/nova..test.rst +source/api/nova..tests.api.openstack.fakes.rst +source/api/nova..tests.api.openstack.test_adminapi.rst +source/api/nova..tests.api.openstack.test_api.rst +source/api/nova..tests.api.openstack.test_auth.rst +source/api/nova..tests.api.openstack.test_common.rst +source/api/nova..tests.api.openstack.test_faults.rst +source/api/nova..tests.api.openstack.test_flavors.rst +source/api/nova..tests.api.openstack.test_images.rst +source/api/nova..tests.api.openstack.test_ratelimiting.rst +source/api/nova..tests.api.openstack.test_servers.rst +source/api/nova..tests.api.openstack.test_shared_ip_groups.rst +source/api/nova..tests.api.openstack.test_zones.rst +source/api/nova..tests.api.test_wsgi.rst +source/api/nova..tests.db.fakes.rst +source/api/nova..tests.declare_flags.rst +source/api/nova..tests.fake_flags.rst +source/api/nova..tests.glance.stubs.rst +source/api/nova..tests.hyperv_unittest.rst +source/api/nova..tests.objectstore_unittest.rst +source/api/nova..tests.real_flags.rst +source/api/nova..tests.runtime_flags.rst +source/api/nova..tests.test_access.rst +source/api/nova..tests.test_api.rst +source/api/nova..tests.test_auth.rst +source/api/nova..tests.test_cloud.rst +source/api/nova..tests.test_compute.rst +source/api/nova..tests.test_console.rst +source/api/nova..tests.test_direct.rst +source/api/nova..tests.test_flags.rst +source/api/nova..tests.test_localization.rst +source/api/nova..tests.test_log.rst +source/api/nova..tests.test_middleware.rst +source/api/nova..tests.test_misc.rst +source/api/nova..tests.test_network.rst +source/api/nova..tests.test_quota.rst +source/api/nova..tests.test_rpc.rst +source/api/nova..tests.test_scheduler.rst +source/api/nova..tests.test_service.rst +source/api/nova..tests.test_twistd.rst +source/api/nova..tests.test_virt.rst +source/api/nova..tests.test_volume.rst +source/api/nova..tests.test_xenapi.rst +source/api/nova..tests.xenapi.stubs.rst +source/api/nova..twistd.rst +source/api/nova..utils.rst +source/api/nova..version.rst +source/api/nova..virt.connection.rst +source/api/nova..virt.disk.rst +source/api/nova..virt.fake.rst +source/api/nova..virt.hyperv.rst +source/api/nova..virt.images.rst +source/api/nova..virt.libvirt_conn.rst +source/api/nova..virt.xenapi.fake.rst +source/api/nova..virt.xenapi.network_utils.rst +source/api/nova..virt.xenapi.vm_utils.rst +source/api/nova..virt.xenapi.vmops.rst +source/api/nova..virt.xenapi.volume_utils.rst +source/api/nova..virt.xenapi.volumeops.rst +source/api/nova..virt.xenapi_conn.rst +source/api/nova..volume.api.rst +source/api/nova..volume.driver.rst +source/api/nova..volume.manager.rst +source/api/nova..volume.san.rst +source/api/nova..wsgi.rst +source/api/autoindex.rst +source/api/nova..adminclient.rst +source/api/nova..api.direct.rst +source/api/nova..api.ec2.admin.rst +source/api/nova..api.ec2.apirequest.rst +source/api/nova..api.ec2.cloud.rst +source/api/nova..api.ec2.metadatarequesthandler.rst +source/api/nova..api.openstack.auth.rst +source/api/nova..api.openstack.backup_schedules.rst +source/api/nova..api.openstack.common.rst +source/api/nova..api.openstack.consoles.rst +source/api/nova..api.openstack.faults.rst +source/api/nova..api.openstack.flavors.rst +source/api/nova..api.openstack.images.rst +source/api/nova..api.openstack.servers.rst +source/api/nova..api.openstack.shared_ip_groups.rst +source/api/nova..api.openstack.zones.rst +source/api/nova..auth.dbdriver.rst +source/api/nova..auth.fakeldap.rst +source/api/nova..auth.ldapdriver.rst +source/api/nova..auth.manager.rst +source/api/nova..auth.signer.rst +source/api/nova..cloudpipe.pipelib.rst +source/api/nova..compute.api.rst +source/api/nova..compute.instance_types.rst +source/api/nova..compute.manager.rst +source/api/nova..compute.monitor.rst +source/api/nova..compute.power_state.rst +source/api/nova..console.api.rst +source/api/nova..console.fake.rst +source/api/nova..console.manager.rst +source/api/nova..console.xvp.rst +source/api/nova..context.rst +source/api/nova..crypto.rst +source/api/nova..db.api.rst +source/api/nova..db.base.rst +source/api/nova..db.migration.rst +source/api/nova..db.sqlalchemy.api.rst +source/api/nova..db.sqlalchemy.migrate_repo.manage.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst +source/api/nova..db.sqlalchemy.migration.rst +source/api/nova..db.sqlalchemy.models.rst +source/api/nova..db.sqlalchemy.session.rst +source/api/nova..exception.rst +source/api/nova..fakememcache.rst +source/api/nova..fakerabbit.rst +source/api/nova..flags.rst +source/api/nova..image.glance.rst +source/api/nova..image.local.rst +source/api/nova..image.s3.rst +source/api/nova..image.service.rst +source/api/nova..log.rst +source/api/nova..manager.rst +source/api/nova..network.api.rst +source/api/nova..network.linux_net.rst +source/api/nova..network.manager.rst +source/api/nova..objectstore.bucket.rst +source/api/nova..objectstore.handler.rst +source/api/nova..objectstore.image.rst +source/api/nova..objectstore.stored.rst +source/api/nova..quota.rst +source/api/nova..rpc.rst +source/api/nova..scheduler.chance.rst +source/api/nova..scheduler.driver.rst +source/api/nova..scheduler.manager.rst +source/api/nova..scheduler.simple.rst +source/api/nova..scheduler.zone.rst +source/api/nova..service.rst +source/api/nova..test.rst +source/api/nova..tests.api.openstack.fakes.rst +source/api/nova..tests.api.openstack.test_adminapi.rst +source/api/nova..tests.api.openstack.test_api.rst +source/api/nova..tests.api.openstack.test_auth.rst +source/api/nova..tests.api.openstack.test_common.rst +source/api/nova..tests.api.openstack.test_faults.rst +source/api/nova..tests.api.openstack.test_flavors.rst +source/api/nova..tests.api.openstack.test_images.rst +source/api/nova..tests.api.openstack.test_ratelimiting.rst +source/api/nova..tests.api.openstack.test_servers.rst +source/api/nova..tests.api.openstack.test_shared_ip_groups.rst +source/api/nova..tests.api.openstack.test_zones.rst +source/api/nova..tests.api.test_wsgi.rst +source/api/nova..tests.db.fakes.rst +source/api/nova..tests.declare_flags.rst +source/api/nova..tests.fake_flags.rst +source/api/nova..tests.glance.stubs.rst +source/api/nova..tests.hyperv_unittest.rst +source/api/nova..tests.objectstore_unittest.rst +source/api/nova..tests.real_flags.rst +source/api/nova..tests.runtime_flags.rst +source/api/nova..tests.test_access.rst +source/api/nova..tests.test_api.rst +source/api/nova..tests.test_auth.rst +source/api/nova..tests.test_cloud.rst +source/api/nova..tests.test_compute.rst +source/api/nova..tests.test_console.rst +source/api/nova..tests.test_direct.rst +source/api/nova..tests.test_flags.rst +source/api/nova..tests.test_localization.rst +source/api/nova..tests.test_log.rst +source/api/nova..tests.test_middleware.rst +source/api/nova..tests.test_misc.rst +source/api/nova..tests.test_network.rst +source/api/nova..tests.test_quota.rst +source/api/nova..tests.test_rpc.rst +source/api/nova..tests.test_scheduler.rst +source/api/nova..tests.test_service.rst +source/api/nova..tests.test_twistd.rst +source/api/nova..tests.test_virt.rst +source/api/nova..tests.test_volume.rst +source/api/nova..tests.test_xenapi.rst +source/api/nova..tests.xenapi.stubs.rst +source/api/nova..twistd.rst +source/api/nova..utils.rst +source/api/nova..version.rst +source/api/nova..virt.connection.rst +source/api/nova..virt.disk.rst +source/api/nova..virt.fake.rst +source/api/nova..virt.hyperv.rst +source/api/nova..virt.images.rst +source/api/nova..virt.libvirt_conn.rst +source/api/nova..virt.xenapi.fake.rst +source/api/nova..virt.xenapi.network_utils.rst +source/api/nova..virt.xenapi.vm_utils.rst +source/api/nova..virt.xenapi.vmops.rst +source/api/nova..virt.xenapi.volume_utils.rst +source/api/nova..virt.xenapi.volumeops.rst +source/api/nova..virt.xenapi_conn.rst +source/api/nova..volume.api.rst +source/api/nova..volume.driver.rst +source/api/nova..volume.manager.rst +source/api/nova..volume.san.rst +source/api/nova..wsgi.rst +source/api/nova..adminclient.rst +source/api/nova..api.direct.rst +source/api/nova..api.ec2.admin.rst +source/api/nova..api.ec2.apirequest.rst +source/api/nova..api.ec2.cloud.rst +source/api/nova..api.ec2.metadatarequesthandler.rst +source/api/nova..api.openstack.auth.rst +source/api/nova..api.openstack.backup_schedules.rst +source/api/nova..api.openstack.common.rst +source/api/nova..api.openstack.consoles.rst +source/api/nova..api.openstack.faults.rst +source/api/nova..api.openstack.flavors.rst +source/api/nova..api.openstack.images.rst +source/api/nova..api.openstack.servers.rst +source/api/nova..api.openstack.shared_ip_groups.rst +source/api/nova..api.openstack.zones.rst +source/api/nova..auth.dbdriver.rst +source/api/nova..auth.fakeldap.rst +source/api/nova..auth.ldapdriver.rst +source/api/nova..auth.manager.rst +source/api/nova..auth.signer.rst +source/api/nova..cloudpipe.pipelib.rst +source/api/nova..compute.api.rst +source/api/nova..compute.instance_types.rst +source/api/nova..compute.manager.rst +source/api/nova..compute.monitor.rst +source/api/nova..compute.power_state.rst +source/api/nova..console.api.rst +source/api/nova..console.fake.rst +source/api/nova..console.manager.rst +source/api/nova..console.xvp.rst +source/api/nova..context.rst +source/api/nova..crypto.rst +source/api/nova..db.api.rst +source/api/nova..db.base.rst +source/api/nova..db.migration.rst +source/api/nova..db.sqlalchemy.api.rst +source/api/nova..db.sqlalchemy.migrate_repo.manage.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst +source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst +source/api/nova..db.sqlalchemy.migration.rst +source/api/nova..db.sqlalchemy.models.rst +source/api/nova..db.sqlalchemy.session.rst +source/api/nova..exception.rst +source/api/nova..fakememcache.rst +source/api/nova..fakerabbit.rst +source/api/nova..flags.rst +source/api/nova..image.glance.rst +source/api/nova..image.local.rst +source/api/nova..image.s3.rst +source/api/nova..image.service.rst +source/api/nova..log.rst +source/api/nova..manager.rst +source/api/nova..network.api.rst +source/api/nova..network.linux_net.rst +source/api/nova..network.manager.rst +source/api/nova..objectstore.bucket.rst +source/api/nova..objectstore.handler.rst +source/api/nova..objectstore.image.rst +source/api/nova..objectstore.stored.rst +source/api/nova..quota.rst +source/api/nova..rpc.rst +source/api/nova..scheduler.chance.rst +source/api/nova..scheduler.driver.rst +source/api/nova..scheduler.manager.rst +source/api/nova..scheduler.simple.rst +source/api/nova..scheduler.zone.rst +source/api/nova..service.rst +source/api/nova..test.rst +source/api/nova..tests.api.openstack.fakes.rst +source/api/nova..tests.api.openstack.test_adminapi.rst +source/api/nova..tests.api.openstack.test_api.rst +source/api/nova..tests.api.openstack.test_auth.rst +source/api/nova..tests.api.openstack.test_common.rst +source/api/nova..tests.api.openstack.test_faults.rst +source/api/nova..tests.api.openstack.test_flavors.rst +source/api/nova..tests.api.openstack.test_images.rst +source/api/nova..tests.api.openstack.test_ratelimiting.rst +source/api/nova..tests.api.openstack.test_servers.rst +source/api/nova..tests.api.openstack.test_shared_ip_groups.rst +source/api/nova..tests.api.openstack.test_zones.rst +source/api/nova..tests.api.test_wsgi.rst +source/api/nova..tests.db.fakes.rst +source/api/nova..tests.declare_flags.rst +source/api/nova..tests.fake_flags.rst +source/api/nova..tests.glance.stubs.rst +source/api/nova..tests.hyperv_unittest.rst +source/api/nova..tests.objectstore_unittest.rst +source/api/nova..tests.real_flags.rst +source/api/nova..tests.runtime_flags.rst +source/api/nova..tests.test_access.rst +source/api/nova..tests.test_api.rst +source/api/nova..tests.test_auth.rst +source/api/nova..tests.test_cloud.rst +source/api/nova..tests.test_compute.rst +source/api/nova..tests.test_console.rst +source/api/nova..tests.test_direct.rst +source/api/nova..tests.test_flags.rst +source/api/nova..tests.test_localization.rst +source/api/nova..tests.test_log.rst +source/api/nova..tests.test_middleware.rst +source/api/nova..tests.test_misc.rst +source/api/nova..tests.test_network.rst +source/api/nova..tests.test_quota.rst +source/api/nova..tests.test_rpc.rst +source/api/nova..tests.test_scheduler.rst +source/api/nova..tests.test_service.rst +source/api/nova..tests.test_twistd.rst +source/api/nova..tests.test_virt.rst +source/api/nova..tests.test_volume.rst +source/api/nova..tests.test_xenapi.rst +source/api/nova..tests.xenapi.stubs.rst +source/api/nova..twistd.rst +source/api/nova..utils.rst +source/api/nova..version.rst +source/api/nova..virt.connection.rst +source/api/nova..virt.disk.rst +source/api/nova..virt.fake.rst +source/api/nova..virt.hyperv.rst +source/api/nova..virt.images.rst +source/api/nova..virt.libvirt_conn.rst +source/api/nova..virt.xenapi.fake.rst +source/api/nova..virt.xenapi.network_utils.rst +source/api/nova..virt.xenapi.vm_utils.rst +source/api/nova..virt.xenapi.vmops.rst +source/api/nova..virt.xenapi.volume_utils.rst +source/api/nova..virt.xenapi.volumeops.rst +source/api/nova..virt.xenapi_conn.rst +source/api/nova..volume.api.rst +source/api/nova..volume.driver.rst +source/api/nova..volume.manager.rst +source/api/nova..volume.san.rst +source/api/nova..wsgi.rst diff --git a/doc/build/.DS_Store b/doc/build/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/doc/build/.DS_Store differ diff --git a/doc/build/html/.DS_Store b/doc/build/html/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/doc/build/html/.DS_Store differ diff --git a/doc/build/html/.buildinfo b/doc/build/html/.buildinfo new file mode 100644 index 000000000..091736d4f --- /dev/null +++ b/doc/build/html/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. +config: 2a2fe6198f4be4a4d6f289b09d16d74a +tags: fbb0d17656682115ca4d033fb2f83ba1 diff --git a/doc/source/api/autoindex.rst b/doc/source/api/autoindex.rst new file mode 100644 index 000000000..41fc1f4a9 --- /dev/null +++ b/doc/source/api/autoindex.rst @@ -0,0 +1,138 @@ +.. toctree:: + :maxdepth: 1 + + nova..adminclient.rst + nova..api.direct.rst + nova..api.ec2.admin.rst + nova..api.ec2.apirequest.rst + nova..api.ec2.cloud.rst + nova..api.ec2.metadatarequesthandler.rst + nova..api.openstack.auth.rst + nova..api.openstack.backup_schedules.rst + nova..api.openstack.common.rst + nova..api.openstack.consoles.rst + nova..api.openstack.faults.rst + nova..api.openstack.flavors.rst + nova..api.openstack.images.rst + nova..api.openstack.servers.rst + nova..api.openstack.shared_ip_groups.rst + nova..api.openstack.zones.rst + nova..auth.dbdriver.rst + nova..auth.fakeldap.rst + nova..auth.ldapdriver.rst + nova..auth.manager.rst + nova..auth.signer.rst + nova..cloudpipe.pipelib.rst + nova..compute.api.rst + nova..compute.instance_types.rst + nova..compute.manager.rst + nova..compute.monitor.rst + nova..compute.power_state.rst + nova..console.api.rst + nova..console.fake.rst + nova..console.manager.rst + nova..console.xvp.rst + nova..context.rst + nova..crypto.rst + nova..db.api.rst + nova..db.base.rst + nova..db.migration.rst + nova..db.sqlalchemy.api.rst + nova..db.sqlalchemy.migrate_repo.manage.rst + nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst + nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst + nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst + nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst + nova..db.sqlalchemy.migration.rst + nova..db.sqlalchemy.models.rst + nova..db.sqlalchemy.session.rst + nova..exception.rst + nova..fakememcache.rst + nova..fakerabbit.rst + nova..flags.rst + nova..image.glance.rst + nova..image.local.rst + nova..image.s3.rst + nova..image.service.rst + nova..log.rst + nova..manager.rst + nova..network.api.rst + nova..network.linux_net.rst + nova..network.manager.rst + nova..objectstore.bucket.rst + nova..objectstore.handler.rst + nova..objectstore.image.rst + nova..objectstore.stored.rst + nova..quota.rst + nova..rpc.rst + nova..scheduler.chance.rst + nova..scheduler.driver.rst + nova..scheduler.manager.rst + nova..scheduler.simple.rst + nova..scheduler.zone.rst + nova..service.rst + nova..test.rst + nova..tests.api.openstack.fakes.rst + nova..tests.api.openstack.test_adminapi.rst + nova..tests.api.openstack.test_api.rst + nova..tests.api.openstack.test_auth.rst + nova..tests.api.openstack.test_common.rst + nova..tests.api.openstack.test_faults.rst + nova..tests.api.openstack.test_flavors.rst + nova..tests.api.openstack.test_images.rst + nova..tests.api.openstack.test_ratelimiting.rst + nova..tests.api.openstack.test_servers.rst + nova..tests.api.openstack.test_shared_ip_groups.rst + nova..tests.api.openstack.test_zones.rst + nova..tests.api.test_wsgi.rst + nova..tests.db.fakes.rst + nova..tests.declare_flags.rst + nova..tests.fake_flags.rst + nova..tests.glance.stubs.rst + nova..tests.hyperv_unittest.rst + nova..tests.objectstore_unittest.rst + nova..tests.real_flags.rst + nova..tests.runtime_flags.rst + nova..tests.test_access.rst + nova..tests.test_api.rst + nova..tests.test_auth.rst + nova..tests.test_cloud.rst + nova..tests.test_compute.rst + nova..tests.test_console.rst + nova..tests.test_direct.rst + nova..tests.test_flags.rst + nova..tests.test_localization.rst + nova..tests.test_log.rst + nova..tests.test_middleware.rst + nova..tests.test_misc.rst + nova..tests.test_network.rst + nova..tests.test_quota.rst + nova..tests.test_rpc.rst + nova..tests.test_scheduler.rst + nova..tests.test_service.rst + nova..tests.test_twistd.rst + nova..tests.test_virt.rst + nova..tests.test_volume.rst + nova..tests.test_xenapi.rst + nova..tests.xenapi.stubs.rst + nova..twistd.rst + nova..utils.rst + nova..version.rst + nova..virt.connection.rst + nova..virt.disk.rst + nova..virt.fake.rst + nova..virt.hyperv.rst + nova..virt.images.rst + nova..virt.libvirt_conn.rst + nova..virt.xenapi.fake.rst + nova..virt.xenapi.network_utils.rst + nova..virt.xenapi.vm_utils.rst + nova..virt.xenapi.vmops.rst + nova..virt.xenapi.volume_utils.rst + nova..virt.xenapi.volumeops.rst + nova..virt.xenapi_conn.rst + nova..volume.api.rst + nova..volume.driver.rst + nova..volume.manager.rst + nova..volume.san.rst + nova..wsgi.rst diff --git a/doc/source/api/nova..adminclient.rst b/doc/source/api/nova..adminclient.rst new file mode 100644 index 000000000..35fa839e1 --- /dev/null +++ b/doc/source/api/nova..adminclient.rst @@ -0,0 +1,6 @@ +The :mod:`nova..adminclient` Module +============================================================================== +.. automodule:: nova..adminclient + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.direct.rst b/doc/source/api/nova..api.direct.rst new file mode 100644 index 000000000..a1705c707 --- /dev/null +++ b/doc/source/api/nova..api.direct.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.direct` Module +============================================================================== +.. automodule:: nova..api.direct + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.admin.rst b/doc/source/api/nova..api.ec2.admin.rst new file mode 100644 index 000000000..4e9ab308b --- /dev/null +++ b/doc/source/api/nova..api.ec2.admin.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.admin` Module +============================================================================== +.. automodule:: nova..api.ec2.admin + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.apirequest.rst b/doc/source/api/nova..api.ec2.apirequest.rst new file mode 100644 index 000000000..c17a2ff3a --- /dev/null +++ b/doc/source/api/nova..api.ec2.apirequest.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.apirequest` Module +============================================================================== +.. automodule:: nova..api.ec2.apirequest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.cloud.rst b/doc/source/api/nova..api.ec2.cloud.rst new file mode 100644 index 000000000..f6145c217 --- /dev/null +++ b/doc/source/api/nova..api.ec2.cloud.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.cloud` Module +============================================================================== +.. automodule:: nova..api.ec2.cloud + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.ec2.metadatarequesthandler.rst b/doc/source/api/nova..api.ec2.metadatarequesthandler.rst new file mode 100644 index 000000000..75f5169e5 --- /dev/null +++ b/doc/source/api/nova..api.ec2.metadatarequesthandler.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.ec2.metadatarequesthandler` Module +============================================================================== +.. automodule:: nova..api.ec2.metadatarequesthandler + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.auth.rst b/doc/source/api/nova..api.openstack.auth.rst new file mode 100644 index 000000000..8c3f8f2da --- /dev/null +++ b/doc/source/api/nova..api.openstack.auth.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.auth` Module +============================================================================== +.. automodule:: nova..api.openstack.auth + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.backup_schedules.rst b/doc/source/api/nova..api.openstack.backup_schedules.rst new file mode 100644 index 000000000..6b406f12d --- /dev/null +++ b/doc/source/api/nova..api.openstack.backup_schedules.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.backup_schedules` Module +============================================================================== +.. automodule:: nova..api.openstack.backup_schedules + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.common.rst b/doc/source/api/nova..api.openstack.common.rst new file mode 100644 index 000000000..4fd734790 --- /dev/null +++ b/doc/source/api/nova..api.openstack.common.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.common` Module +============================================================================== +.. automodule:: nova..api.openstack.common + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.consoles.rst b/doc/source/api/nova..api.openstack.consoles.rst new file mode 100644 index 000000000..1e3e09599 --- /dev/null +++ b/doc/source/api/nova..api.openstack.consoles.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.consoles` Module +============================================================================== +.. automodule:: nova..api.openstack.consoles + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.faults.rst b/doc/source/api/nova..api.openstack.faults.rst new file mode 100644 index 000000000..7b25561f7 --- /dev/null +++ b/doc/source/api/nova..api.openstack.faults.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.faults` Module +============================================================================== +.. automodule:: nova..api.openstack.faults + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.flavors.rst b/doc/source/api/nova..api.openstack.flavors.rst new file mode 100644 index 000000000..0deb724de --- /dev/null +++ b/doc/source/api/nova..api.openstack.flavors.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.flavors` Module +============================================================================== +.. automodule:: nova..api.openstack.flavors + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.images.rst b/doc/source/api/nova..api.openstack.images.rst new file mode 100644 index 000000000..82bd5f1e8 --- /dev/null +++ b/doc/source/api/nova..api.openstack.images.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.images` Module +============================================================================== +.. automodule:: nova..api.openstack.images + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.servers.rst b/doc/source/api/nova..api.openstack.servers.rst new file mode 100644 index 000000000..c36856ea2 --- /dev/null +++ b/doc/source/api/nova..api.openstack.servers.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.servers` Module +============================================================================== +.. automodule:: nova..api.openstack.servers + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.shared_ip_groups.rst b/doc/source/api/nova..api.openstack.shared_ip_groups.rst new file mode 100644 index 000000000..4b1f44efe --- /dev/null +++ b/doc/source/api/nova..api.openstack.shared_ip_groups.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.shared_ip_groups` Module +============================================================================== +.. automodule:: nova..api.openstack.shared_ip_groups + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..api.openstack.zones.rst b/doc/source/api/nova..api.openstack.zones.rst new file mode 100644 index 000000000..ebe4569c5 --- /dev/null +++ b/doc/source/api/nova..api.openstack.zones.rst @@ -0,0 +1,6 @@ +The :mod:`nova..api.openstack.zones` Module +============================================================================== +.. automodule:: nova..api.openstack.zones + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.dbdriver.rst b/doc/source/api/nova..auth.dbdriver.rst new file mode 100644 index 000000000..7de68b6e0 --- /dev/null +++ b/doc/source/api/nova..auth.dbdriver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.dbdriver` Module +============================================================================== +.. automodule:: nova..auth.dbdriver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.fakeldap.rst b/doc/source/api/nova..auth.fakeldap.rst new file mode 100644 index 000000000..ca8a3ad4d --- /dev/null +++ b/doc/source/api/nova..auth.fakeldap.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.fakeldap` Module +============================================================================== +.. automodule:: nova..auth.fakeldap + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.ldapdriver.rst b/doc/source/api/nova..auth.ldapdriver.rst new file mode 100644 index 000000000..c44463522 --- /dev/null +++ b/doc/source/api/nova..auth.ldapdriver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.ldapdriver` Module +============================================================================== +.. automodule:: nova..auth.ldapdriver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.manager.rst b/doc/source/api/nova..auth.manager.rst new file mode 100644 index 000000000..bc5ce2ec3 --- /dev/null +++ b/doc/source/api/nova..auth.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.manager` Module +============================================================================== +.. automodule:: nova..auth.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..auth.signer.rst b/doc/source/api/nova..auth.signer.rst new file mode 100644 index 000000000..aad824ead --- /dev/null +++ b/doc/source/api/nova..auth.signer.rst @@ -0,0 +1,6 @@ +The :mod:`nova..auth.signer` Module +============================================================================== +.. automodule:: nova..auth.signer + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..cloudpipe.pipelib.rst b/doc/source/api/nova..cloudpipe.pipelib.rst new file mode 100644 index 000000000..054aaf484 --- /dev/null +++ b/doc/source/api/nova..cloudpipe.pipelib.rst @@ -0,0 +1,6 @@ +The :mod:`nova..cloudpipe.pipelib` Module +============================================================================== +.. automodule:: nova..cloudpipe.pipelib + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.api.rst b/doc/source/api/nova..compute.api.rst new file mode 100644 index 000000000..caa66313a --- /dev/null +++ b/doc/source/api/nova..compute.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.api` Module +============================================================================== +.. automodule:: nova..compute.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.instance_types.rst b/doc/source/api/nova..compute.instance_types.rst new file mode 100644 index 000000000..d206ff3a4 --- /dev/null +++ b/doc/source/api/nova..compute.instance_types.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.instance_types` Module +============================================================================== +.. automodule:: nova..compute.instance_types + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.manager.rst b/doc/source/api/nova..compute.manager.rst new file mode 100644 index 000000000..33a337c39 --- /dev/null +++ b/doc/source/api/nova..compute.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.manager` Module +============================================================================== +.. automodule:: nova..compute.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.monitor.rst b/doc/source/api/nova..compute.monitor.rst new file mode 100644 index 000000000..a91169ecd --- /dev/null +++ b/doc/source/api/nova..compute.monitor.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.monitor` Module +============================================================================== +.. automodule:: nova..compute.monitor + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..compute.power_state.rst b/doc/source/api/nova..compute.power_state.rst new file mode 100644 index 000000000..41b1080e5 --- /dev/null +++ b/doc/source/api/nova..compute.power_state.rst @@ -0,0 +1,6 @@ +The :mod:`nova..compute.power_state` Module +============================================================================== +.. automodule:: nova..compute.power_state + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.api.rst b/doc/source/api/nova..console.api.rst new file mode 100644 index 000000000..82a51d4c7 --- /dev/null +++ b/doc/source/api/nova..console.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.api` Module +============================================================================== +.. automodule:: nova..console.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.fake.rst b/doc/source/api/nova..console.fake.rst new file mode 100644 index 000000000..f053f85d6 --- /dev/null +++ b/doc/source/api/nova..console.fake.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.fake` Module +============================================================================== +.. automodule:: nova..console.fake + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.manager.rst b/doc/source/api/nova..console.manager.rst new file mode 100644 index 000000000..f9283a6c3 --- /dev/null +++ b/doc/source/api/nova..console.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.manager` Module +============================================================================== +.. automodule:: nova..console.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..console.xvp.rst b/doc/source/api/nova..console.xvp.rst new file mode 100644 index 000000000..a0887009e --- /dev/null +++ b/doc/source/api/nova..console.xvp.rst @@ -0,0 +1,6 @@ +The :mod:`nova..console.xvp` Module +============================================================================== +.. automodule:: nova..console.xvp + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..context.rst b/doc/source/api/nova..context.rst new file mode 100644 index 000000000..9de1adb24 --- /dev/null +++ b/doc/source/api/nova..context.rst @@ -0,0 +1,6 @@ +The :mod:`nova..context` Module +============================================================================== +.. automodule:: nova..context + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..crypto.rst b/doc/source/api/nova..crypto.rst new file mode 100644 index 000000000..af9f63634 --- /dev/null +++ b/doc/source/api/nova..crypto.rst @@ -0,0 +1,6 @@ +The :mod:`nova..crypto` Module +============================================================================== +.. automodule:: nova..crypto + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.api.rst b/doc/source/api/nova..db.api.rst new file mode 100644 index 000000000..6d998fbb2 --- /dev/null +++ b/doc/source/api/nova..db.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.api` Module +============================================================================== +.. automodule:: nova..db.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.base.rst b/doc/source/api/nova..db.base.rst new file mode 100644 index 000000000..29fb417d6 --- /dev/null +++ b/doc/source/api/nova..db.base.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.base` Module +============================================================================== +.. automodule:: nova..db.base + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.migration.rst b/doc/source/api/nova..db.migration.rst new file mode 100644 index 000000000..71dfea301 --- /dev/null +++ b/doc/source/api/nova..db.migration.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.migration` Module +============================================================================== +.. automodule:: nova..db.migration + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.api.rst b/doc/source/api/nova..db.sqlalchemy.api.rst new file mode 100644 index 000000000..76d0c1bd3 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.api` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst new file mode 100644 index 000000000..93decfb27 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.manage.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.manage` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.manage + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst new file mode 100644 index 000000000..4b1219edb --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.001_austin` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.001_austin + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst new file mode 100644 index 000000000..82f1f4680 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.002_bexar` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.002_bexar + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst new file mode 100644 index 000000000..98f3e8da7 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst new file mode 100644 index 000000000..5cbb81191 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.migration.rst b/doc/source/api/nova..db.sqlalchemy.migration.rst new file mode 100644 index 000000000..3a9b01b9a --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.migration.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.migration` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.migration + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.models.rst b/doc/source/api/nova..db.sqlalchemy.models.rst new file mode 100644 index 000000000..9c795d7f5 --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.models.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.models` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.models + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..db.sqlalchemy.session.rst b/doc/source/api/nova..db.sqlalchemy.session.rst new file mode 100644 index 000000000..cbfd6416a --- /dev/null +++ b/doc/source/api/nova..db.sqlalchemy.session.rst @@ -0,0 +1,6 @@ +The :mod:`nova..db.sqlalchemy.session` Module +============================================================================== +.. automodule:: nova..db.sqlalchemy.session + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..exception.rst b/doc/source/api/nova..exception.rst new file mode 100644 index 000000000..97ac6b752 --- /dev/null +++ b/doc/source/api/nova..exception.rst @@ -0,0 +1,6 @@ +The :mod:`nova..exception` Module +============================================================================== +.. automodule:: nova..exception + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..fakememcache.rst b/doc/source/api/nova..fakememcache.rst new file mode 100644 index 000000000..7e7ffb98b --- /dev/null +++ b/doc/source/api/nova..fakememcache.rst @@ -0,0 +1,6 @@ +The :mod:`nova..fakememcache` Module +============================================================================== +.. automodule:: nova..fakememcache + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..fakerabbit.rst b/doc/source/api/nova..fakerabbit.rst new file mode 100644 index 000000000..f1e27c266 --- /dev/null +++ b/doc/source/api/nova..fakerabbit.rst @@ -0,0 +1,6 @@ +The :mod:`nova..fakerabbit` Module +============================================================================== +.. automodule:: nova..fakerabbit + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..flags.rst b/doc/source/api/nova..flags.rst new file mode 100644 index 000000000..08165be44 --- /dev/null +++ b/doc/source/api/nova..flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..flags` Module +============================================================================== +.. automodule:: nova..flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.glance.rst b/doc/source/api/nova..image.glance.rst new file mode 100644 index 000000000..b0882d5ec --- /dev/null +++ b/doc/source/api/nova..image.glance.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.glance` Module +============================================================================== +.. automodule:: nova..image.glance + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.local.rst b/doc/source/api/nova..image.local.rst new file mode 100644 index 000000000..b6ad5470b --- /dev/null +++ b/doc/source/api/nova..image.local.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.local` Module +============================================================================== +.. automodule:: nova..image.local + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.s3.rst b/doc/source/api/nova..image.s3.rst new file mode 100644 index 000000000..e5b236127 --- /dev/null +++ b/doc/source/api/nova..image.s3.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.s3` Module +============================================================================== +.. automodule:: nova..image.s3 + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..image.service.rst b/doc/source/api/nova..image.service.rst new file mode 100644 index 000000000..78ef1ecca --- /dev/null +++ b/doc/source/api/nova..image.service.rst @@ -0,0 +1,6 @@ +The :mod:`nova..image.service` Module +============================================================================== +.. automodule:: nova..image.service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..log.rst b/doc/source/api/nova..log.rst new file mode 100644 index 000000000..ff209709f --- /dev/null +++ b/doc/source/api/nova..log.rst @@ -0,0 +1,6 @@ +The :mod:`nova..log` Module +============================================================================== +.. automodule:: nova..log + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..manager.rst b/doc/source/api/nova..manager.rst new file mode 100644 index 000000000..576902491 --- /dev/null +++ b/doc/source/api/nova..manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..manager` Module +============================================================================== +.. automodule:: nova..manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..network.api.rst b/doc/source/api/nova..network.api.rst new file mode 100644 index 000000000..b63be2ba3 --- /dev/null +++ b/doc/source/api/nova..network.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..network.api` Module +============================================================================== +.. automodule:: nova..network.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..network.linux_net.rst b/doc/source/api/nova..network.linux_net.rst new file mode 100644 index 000000000..7af78d5ad --- /dev/null +++ b/doc/source/api/nova..network.linux_net.rst @@ -0,0 +1,6 @@ +The :mod:`nova..network.linux_net` Module +============================================================================== +.. automodule:: nova..network.linux_net + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..network.manager.rst b/doc/source/api/nova..network.manager.rst new file mode 100644 index 000000000..0ea705533 --- /dev/null +++ b/doc/source/api/nova..network.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..network.manager` Module +============================================================================== +.. automodule:: nova..network.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.bucket.rst b/doc/source/api/nova..objectstore.bucket.rst new file mode 100644 index 000000000..3bfdf639c --- /dev/null +++ b/doc/source/api/nova..objectstore.bucket.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.bucket` Module +============================================================================== +.. automodule:: nova..objectstore.bucket + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.handler.rst b/doc/source/api/nova..objectstore.handler.rst new file mode 100644 index 000000000..0eb8c4efb --- /dev/null +++ b/doc/source/api/nova..objectstore.handler.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.handler` Module +============================================================================== +.. automodule:: nova..objectstore.handler + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.image.rst b/doc/source/api/nova..objectstore.image.rst new file mode 100644 index 000000000..fa4c971f1 --- /dev/null +++ b/doc/source/api/nova..objectstore.image.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.image` Module +============================================================================== +.. automodule:: nova..objectstore.image + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..objectstore.stored.rst b/doc/source/api/nova..objectstore.stored.rst new file mode 100644 index 000000000..2b1d997a3 --- /dev/null +++ b/doc/source/api/nova..objectstore.stored.rst @@ -0,0 +1,6 @@ +The :mod:`nova..objectstore.stored` Module +============================================================================== +.. automodule:: nova..objectstore.stored + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..quota.rst b/doc/source/api/nova..quota.rst new file mode 100644 index 000000000..4140d95d6 --- /dev/null +++ b/doc/source/api/nova..quota.rst @@ -0,0 +1,6 @@ +The :mod:`nova..quota` Module +============================================================================== +.. automodule:: nova..quota + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..rpc.rst b/doc/source/api/nova..rpc.rst new file mode 100644 index 000000000..5b2a9b8e2 --- /dev/null +++ b/doc/source/api/nova..rpc.rst @@ -0,0 +1,6 @@ +The :mod:`nova..rpc` Module +============================================================================== +.. automodule:: nova..rpc + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.chance.rst b/doc/source/api/nova..scheduler.chance.rst new file mode 100644 index 000000000..89c074c8f --- /dev/null +++ b/doc/source/api/nova..scheduler.chance.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.chance` Module +============================================================================== +.. automodule:: nova..scheduler.chance + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.driver.rst b/doc/source/api/nova..scheduler.driver.rst new file mode 100644 index 000000000..793ed9c7b --- /dev/null +++ b/doc/source/api/nova..scheduler.driver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.driver` Module +============================================================================== +.. automodule:: nova..scheduler.driver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.manager.rst b/doc/source/api/nova..scheduler.manager.rst new file mode 100644 index 000000000..d0fc7c423 --- /dev/null +++ b/doc/source/api/nova..scheduler.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.manager` Module +============================================================================== +.. automodule:: nova..scheduler.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.simple.rst b/doc/source/api/nova..scheduler.simple.rst new file mode 100644 index 000000000..dacc2cf30 --- /dev/null +++ b/doc/source/api/nova..scheduler.simple.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.simple` Module +============================================================================== +.. automodule:: nova..scheduler.simple + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..scheduler.zone.rst b/doc/source/api/nova..scheduler.zone.rst new file mode 100644 index 000000000..54c4bf201 --- /dev/null +++ b/doc/source/api/nova..scheduler.zone.rst @@ -0,0 +1,6 @@ +The :mod:`nova..scheduler.zone` Module +============================================================================== +.. automodule:: nova..scheduler.zone + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..service.rst b/doc/source/api/nova..service.rst new file mode 100644 index 000000000..2d2dfcf2e --- /dev/null +++ b/doc/source/api/nova..service.rst @@ -0,0 +1,6 @@ +The :mod:`nova..service` Module +============================================================================== +.. automodule:: nova..service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..test.rst b/doc/source/api/nova..test.rst new file mode 100644 index 000000000..a6bdb6f1f --- /dev/null +++ b/doc/source/api/nova..test.rst @@ -0,0 +1,6 @@ +The :mod:`nova..test` Module +============================================================================== +.. automodule:: nova..test + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.fakes.rst b/doc/source/api/nova..tests.api.openstack.fakes.rst new file mode 100644 index 000000000..4a9ff5938 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.fakes.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.fakes` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.fakes + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_adminapi.rst b/doc/source/api/nova..tests.api.openstack.test_adminapi.rst new file mode 100644 index 000000000..19a85ca0f --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_adminapi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_adminapi` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_adminapi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_api.rst b/doc/source/api/nova..tests.api.openstack.test_api.rst new file mode 100644 index 000000000..68106d221 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_api` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_auth.rst b/doc/source/api/nova..tests.api.openstack.test_auth.rst new file mode 100644 index 000000000..9f0011669 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_auth.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_auth` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_auth + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_common.rst b/doc/source/api/nova..tests.api.openstack.test_common.rst new file mode 100644 index 000000000..82f40ecb8 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_common.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_common` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_common + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_faults.rst b/doc/source/api/nova..tests.api.openstack.test_faults.rst new file mode 100644 index 000000000..b839ae8a3 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_faults.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_faults` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_faults + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_flavors.rst b/doc/source/api/nova..tests.api.openstack.test_flavors.rst new file mode 100644 index 000000000..471fac56e --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_flavors.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_flavors` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_flavors + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_images.rst b/doc/source/api/nova..tests.api.openstack.test_images.rst new file mode 100644 index 000000000..57ae93c8c --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_images.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_images` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_images + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst b/doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst new file mode 100644 index 000000000..9a857f795 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_ratelimiting.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_ratelimiting` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_ratelimiting + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_servers.rst b/doc/source/api/nova..tests.api.openstack.test_servers.rst new file mode 100644 index 000000000..ea602e6ab --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_servers.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_servers` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_servers + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst b/doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst new file mode 100644 index 000000000..48814af00 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_shared_ip_groups.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_shared_ip_groups` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_shared_ip_groups + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.openstack.test_zones.rst b/doc/source/api/nova..tests.api.openstack.test_zones.rst new file mode 100644 index 000000000..ba7078e63 --- /dev/null +++ b/doc/source/api/nova..tests.api.openstack.test_zones.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.openstack.test_zones` Module +============================================================================== +.. automodule:: nova..tests.api.openstack.test_zones + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.api.test_wsgi.rst b/doc/source/api/nova..tests.api.test_wsgi.rst new file mode 100644 index 000000000..8e79caa4d --- /dev/null +++ b/doc/source/api/nova..tests.api.test_wsgi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.api.test_wsgi` Module +============================================================================== +.. automodule:: nova..tests.api.test_wsgi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.db.fakes.rst b/doc/source/api/nova..tests.db.fakes.rst new file mode 100644 index 000000000..cc79e55e2 --- /dev/null +++ b/doc/source/api/nova..tests.db.fakes.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.db.fakes` Module +============================================================================== +.. automodule:: nova..tests.db.fakes + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.declare_flags.rst b/doc/source/api/nova..tests.declare_flags.rst new file mode 100644 index 000000000..524e72e91 --- /dev/null +++ b/doc/source/api/nova..tests.declare_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.declare_flags` Module +============================================================================== +.. automodule:: nova..tests.declare_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.fake_flags.rst b/doc/source/api/nova..tests.fake_flags.rst new file mode 100644 index 000000000..a8dc3df36 --- /dev/null +++ b/doc/source/api/nova..tests.fake_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.fake_flags` Module +============================================================================== +.. automodule:: nova..tests.fake_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.glance.stubs.rst b/doc/source/api/nova..tests.glance.stubs.rst new file mode 100644 index 000000000..7ef5fccbe --- /dev/null +++ b/doc/source/api/nova..tests.glance.stubs.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.glance.stubs` Module +============================================================================== +.. automodule:: nova..tests.glance.stubs + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.hyperv_unittest.rst b/doc/source/api/nova..tests.hyperv_unittest.rst new file mode 100644 index 000000000..c08443121 --- /dev/null +++ b/doc/source/api/nova..tests.hyperv_unittest.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.hyperv_unittest` Module +============================================================================== +.. automodule:: nova..tests.hyperv_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.objectstore_unittest.rst b/doc/source/api/nova..tests.objectstore_unittest.rst new file mode 100644 index 000000000..0ae252f04 --- /dev/null +++ b/doc/source/api/nova..tests.objectstore_unittest.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.objectstore_unittest` Module +============================================================================== +.. automodule:: nova..tests.objectstore_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.real_flags.rst b/doc/source/api/nova..tests.real_flags.rst new file mode 100644 index 000000000..e9c0d1abd --- /dev/null +++ b/doc/source/api/nova..tests.real_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.real_flags` Module +============================================================================== +.. automodule:: nova..tests.real_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.runtime_flags.rst b/doc/source/api/nova..tests.runtime_flags.rst new file mode 100644 index 000000000..984e21199 --- /dev/null +++ b/doc/source/api/nova..tests.runtime_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.runtime_flags` Module +============================================================================== +.. automodule:: nova..tests.runtime_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_access.rst b/doc/source/api/nova..tests.test_access.rst new file mode 100644 index 000000000..300d8109e --- /dev/null +++ b/doc/source/api/nova..tests.test_access.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_access` Module +============================================================================== +.. automodule:: nova..tests.test_access + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_api.rst b/doc/source/api/nova..tests.test_api.rst new file mode 100644 index 000000000..f9473062e --- /dev/null +++ b/doc/source/api/nova..tests.test_api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_api` Module +============================================================================== +.. automodule:: nova..tests.test_api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_auth.rst b/doc/source/api/nova..tests.test_auth.rst new file mode 100644 index 000000000..ff4445ae4 --- /dev/null +++ b/doc/source/api/nova..tests.test_auth.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_auth` Module +============================================================================== +.. automodule:: nova..tests.test_auth + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_cloud.rst b/doc/source/api/nova..tests.test_cloud.rst new file mode 100644 index 000000000..7bd03db9a --- /dev/null +++ b/doc/source/api/nova..tests.test_cloud.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_cloud` Module +============================================================================== +.. automodule:: nova..tests.test_cloud + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_compute.rst b/doc/source/api/nova..tests.test_compute.rst new file mode 100644 index 000000000..90fd6e9d1 --- /dev/null +++ b/doc/source/api/nova..tests.test_compute.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_compute` Module +============================================================================== +.. automodule:: nova..tests.test_compute + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_console.rst b/doc/source/api/nova..tests.test_console.rst new file mode 100644 index 000000000..f695f5d17 --- /dev/null +++ b/doc/source/api/nova..tests.test_console.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_console` Module +============================================================================== +.. automodule:: nova..tests.test_console + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_direct.rst b/doc/source/api/nova..tests.test_direct.rst new file mode 100644 index 000000000..4f7adef19 --- /dev/null +++ b/doc/source/api/nova..tests.test_direct.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_direct` Module +============================================================================== +.. automodule:: nova..tests.test_direct + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_flags.rst b/doc/source/api/nova..tests.test_flags.rst new file mode 100644 index 000000000..2ec35d6c2 --- /dev/null +++ b/doc/source/api/nova..tests.test_flags.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_flags` Module +============================================================================== +.. automodule:: nova..tests.test_flags + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_localization.rst b/doc/source/api/nova..tests.test_localization.rst new file mode 100644 index 000000000..d93c83ba7 --- /dev/null +++ b/doc/source/api/nova..tests.test_localization.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_localization` Module +============================================================================== +.. automodule:: nova..tests.test_localization + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_log.rst b/doc/source/api/nova..tests.test_log.rst new file mode 100644 index 000000000..04ff5ead1 --- /dev/null +++ b/doc/source/api/nova..tests.test_log.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_log` Module +============================================================================== +.. automodule:: nova..tests.test_log + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_middleware.rst b/doc/source/api/nova..tests.test_middleware.rst new file mode 100644 index 000000000..2f9df5832 --- /dev/null +++ b/doc/source/api/nova..tests.test_middleware.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_middleware` Module +============================================================================== +.. automodule:: nova..tests.test_middleware + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_misc.rst b/doc/source/api/nova..tests.test_misc.rst new file mode 100644 index 000000000..4975f89d7 --- /dev/null +++ b/doc/source/api/nova..tests.test_misc.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_misc` Module +============================================================================== +.. automodule:: nova..tests.test_misc + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_network.rst b/doc/source/api/nova..tests.test_network.rst new file mode 100644 index 000000000..3a4b04ea4 --- /dev/null +++ b/doc/source/api/nova..tests.test_network.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_network` Module +============================================================================== +.. automodule:: nova..tests.test_network + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_quota.rst b/doc/source/api/nova..tests.test_quota.rst new file mode 100644 index 000000000..24ebf9ca3 --- /dev/null +++ b/doc/source/api/nova..tests.test_quota.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_quota` Module +============================================================================== +.. automodule:: nova..tests.test_quota + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_rpc.rst b/doc/source/api/nova..tests.test_rpc.rst new file mode 100644 index 000000000..c141d6889 --- /dev/null +++ b/doc/source/api/nova..tests.test_rpc.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_rpc` Module +============================================================================== +.. automodule:: nova..tests.test_rpc + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_scheduler.rst b/doc/source/api/nova..tests.test_scheduler.rst new file mode 100644 index 000000000..1cd9991db --- /dev/null +++ b/doc/source/api/nova..tests.test_scheduler.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_scheduler` Module +============================================================================== +.. automodule:: nova..tests.test_scheduler + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_service.rst b/doc/source/api/nova..tests.test_service.rst new file mode 100644 index 000000000..a264fbb55 --- /dev/null +++ b/doc/source/api/nova..tests.test_service.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_service` Module +============================================================================== +.. automodule:: nova..tests.test_service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_twistd.rst b/doc/source/api/nova..tests.test_twistd.rst new file mode 100644 index 000000000..cae0c0a28 --- /dev/null +++ b/doc/source/api/nova..tests.test_twistd.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_twistd` Module +============================================================================== +.. automodule:: nova..tests.test_twistd + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_virt.rst b/doc/source/api/nova..tests.test_virt.rst new file mode 100644 index 000000000..9b0dc1e46 --- /dev/null +++ b/doc/source/api/nova..tests.test_virt.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_virt` Module +============================================================================== +.. automodule:: nova..tests.test_virt + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_volume.rst b/doc/source/api/nova..tests.test_volume.rst new file mode 100644 index 000000000..b5affe53c --- /dev/null +++ b/doc/source/api/nova..tests.test_volume.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_volume` Module +============================================================================== +.. automodule:: nova..tests.test_volume + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.test_xenapi.rst b/doc/source/api/nova..tests.test_xenapi.rst new file mode 100644 index 000000000..7128baee4 --- /dev/null +++ b/doc/source/api/nova..tests.test_xenapi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.test_xenapi` Module +============================================================================== +.. automodule:: nova..tests.test_xenapi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..tests.xenapi.stubs.rst b/doc/source/api/nova..tests.xenapi.stubs.rst new file mode 100644 index 000000000..356eed9a7 --- /dev/null +++ b/doc/source/api/nova..tests.xenapi.stubs.rst @@ -0,0 +1,6 @@ +The :mod:`nova..tests.xenapi.stubs` Module +============================================================================== +.. automodule:: nova..tests.xenapi.stubs + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..twistd.rst b/doc/source/api/nova..twistd.rst new file mode 100644 index 000000000..d4145396d --- /dev/null +++ b/doc/source/api/nova..twistd.rst @@ -0,0 +1,6 @@ +The :mod:`nova..twistd` Module +============================================================================== +.. automodule:: nova..twistd + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..utils.rst b/doc/source/api/nova..utils.rst new file mode 100644 index 000000000..1131d1080 --- /dev/null +++ b/doc/source/api/nova..utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..utils` Module +============================================================================== +.. automodule:: nova..utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..version.rst b/doc/source/api/nova..version.rst new file mode 100644 index 000000000..4b0fc078f --- /dev/null +++ b/doc/source/api/nova..version.rst @@ -0,0 +1,6 @@ +The :mod:`nova..version` Module +============================================================================== +.. automodule:: nova..version + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.connection.rst b/doc/source/api/nova..virt.connection.rst new file mode 100644 index 000000000..caf766765 --- /dev/null +++ b/doc/source/api/nova..virt.connection.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.connection` Module +============================================================================== +.. automodule:: nova..virt.connection + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.disk.rst b/doc/source/api/nova..virt.disk.rst new file mode 100644 index 000000000..4a6c0f406 --- /dev/null +++ b/doc/source/api/nova..virt.disk.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.disk` Module +============================================================================== +.. automodule:: nova..virt.disk + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.fake.rst b/doc/source/api/nova..virt.fake.rst new file mode 100644 index 000000000..06ecdbf7d --- /dev/null +++ b/doc/source/api/nova..virt.fake.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.fake` Module +============================================================================== +.. automodule:: nova..virt.fake + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.hyperv.rst b/doc/source/api/nova..virt.hyperv.rst new file mode 100644 index 000000000..48d89378e --- /dev/null +++ b/doc/source/api/nova..virt.hyperv.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.hyperv` Module +============================================================================== +.. automodule:: nova..virt.hyperv + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.images.rst b/doc/source/api/nova..virt.images.rst new file mode 100644 index 000000000..4fdeb7af8 --- /dev/null +++ b/doc/source/api/nova..virt.images.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.images` Module +============================================================================== +.. automodule:: nova..virt.images + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.libvirt_conn.rst b/doc/source/api/nova..virt.libvirt_conn.rst new file mode 100644 index 000000000..7fb8aed5f --- /dev/null +++ b/doc/source/api/nova..virt.libvirt_conn.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.libvirt_conn` Module +============================================================================== +.. automodule:: nova..virt.libvirt_conn + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.fake.rst b/doc/source/api/nova..virt.xenapi.fake.rst new file mode 100644 index 000000000..752dabb14 --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.fake.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.fake` Module +============================================================================== +.. automodule:: nova..virt.xenapi.fake + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.network_utils.rst b/doc/source/api/nova..virt.xenapi.network_utils.rst new file mode 100644 index 000000000..15f52973e --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.network_utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.network_utils` Module +============================================================================== +.. automodule:: nova..virt.xenapi.network_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.vm_utils.rst b/doc/source/api/nova..virt.xenapi.vm_utils.rst new file mode 100644 index 000000000..18745dc71 --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.vm_utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.vm_utils` Module +============================================================================== +.. automodule:: nova..virt.xenapi.vm_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.vmops.rst b/doc/source/api/nova..virt.xenapi.vmops.rst new file mode 100644 index 000000000..30662c58d --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.vmops.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.vmops` Module +============================================================================== +.. automodule:: nova..virt.xenapi.vmops + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.volume_utils.rst b/doc/source/api/nova..virt.xenapi.volume_utils.rst new file mode 100644 index 000000000..413e4dc4b --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.volume_utils.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.volume_utils` Module +============================================================================== +.. automodule:: nova..virt.xenapi.volume_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi.volumeops.rst b/doc/source/api/nova..virt.xenapi.volumeops.rst new file mode 100644 index 000000000..626f164df --- /dev/null +++ b/doc/source/api/nova..virt.xenapi.volumeops.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi.volumeops` Module +============================================================================== +.. automodule:: nova..virt.xenapi.volumeops + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..virt.xenapi_conn.rst b/doc/source/api/nova..virt.xenapi_conn.rst new file mode 100644 index 000000000..14ac5147f --- /dev/null +++ b/doc/source/api/nova..virt.xenapi_conn.rst @@ -0,0 +1,6 @@ +The :mod:`nova..virt.xenapi_conn` Module +============================================================================== +.. automodule:: nova..virt.xenapi_conn + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.api.rst b/doc/source/api/nova..volume.api.rst new file mode 100644 index 000000000..8ad36e049 --- /dev/null +++ b/doc/source/api/nova..volume.api.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.api` Module +============================================================================== +.. automodule:: nova..volume.api + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.driver.rst b/doc/source/api/nova..volume.driver.rst new file mode 100644 index 000000000..51f5c0729 --- /dev/null +++ b/doc/source/api/nova..volume.driver.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.driver` Module +============================================================================== +.. automodule:: nova..volume.driver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.manager.rst b/doc/source/api/nova..volume.manager.rst new file mode 100644 index 000000000..91a192a8f --- /dev/null +++ b/doc/source/api/nova..volume.manager.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.manager` Module +============================================================================== +.. automodule:: nova..volume.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..volume.san.rst b/doc/source/api/nova..volume.san.rst new file mode 100644 index 000000000..1de068928 --- /dev/null +++ b/doc/source/api/nova..volume.san.rst @@ -0,0 +1,6 @@ +The :mod:`nova..volume.san` Module +============================================================================== +.. automodule:: nova..volume.san + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/nova..wsgi.rst b/doc/source/api/nova..wsgi.rst new file mode 100644 index 000000000..0bff1c332 --- /dev/null +++ b/doc/source/api/nova..wsgi.rst @@ -0,0 +1,6 @@ +The :mod:`nova..wsgi` Module +============================================================================== +.. automodule:: nova..wsgi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/runnova/binaries.rst b/doc/source/runnova/binaries.rst new file mode 100644 index 000000000..023831021 --- /dev/null +++ b/doc/source/runnova/binaries.rst @@ -0,0 +1,57 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _binaries: + +Nova Daemons +============= + +The configuration of these binaries relies on "flagfiles" using the google +gflags package:: + + $ nova-xxxxx --flagfile flagfile + +The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. + +nova-api +-------- + +Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. + +nova-objectstore +---------------- + +Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with Glance (http://glance.openstack.org) and a simple image manager. + +nova-compute +------------ + +Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. + +nova-volume +----------- + +Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. + +nova-network +------------ + +Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: + + $ nova-network --network_manager=nova.network.manager.FlatManager + +IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/runnova/euca2ools.rst b/doc/source/runnova/euca2ools.rst new file mode 100644 index 000000000..6f0c57358 --- /dev/null +++ b/doc/source/runnova/euca2ools.rst @@ -0,0 +1,49 @@ +Euca2ools +========= + +Nova is compatible with most of the euca2ools command line utilities. Both Administrators and Users will find these tools helpful for day-to-day administration. + +* euca-add-group +* euca-delete-bundle +* euca-describe-instances +* euca-register +* euca-add-keypair +* euca-delete-group +* euca-describe-keypairs +* euca-release-address +* euca-allocate-address +* euca-delete-keypair +* euca-describe-regions +* euca-reset-image-attribute +* euca-associate-address +* euca-delete-snapshot +* euca-describe-snapshots +* euca-revoke +* euca-attach-volume +* euca-delete-volume +* euca-describe-volumes +* euca-run-instances +* euca-authorize +* euca-deregister +* euca-detach-volume +* euca-terminate-instances +* euca-bundle-image +* euca-describe-addresses +* euca-disassociate-address +* euca-unbundle +* euca-bundle-vol +* euca-describe-availability-zones +* euca-download-bundle +* euca-upload-bundle +* euca-confirm-product-instance +* euca-describe-groups +* euca-get-console-output +* euca-version +* euca-create-snapshot +* euca-describe-image-attribute +* euca-modify-image-attribute +* euca-create-volume +* euca-describe-images +* euca-reboot-instances + + diff --git a/doc/source/runnova/flags.rst b/doc/source/runnova/flags.rst new file mode 100644 index 000000000..1bfa022d9 --- /dev/null +++ b/doc/source/runnova/flags.rst @@ -0,0 +1,193 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Flags and Flagfiles +=================== + +Nova uses a configuration file containing flags located in /etc/nova/nova.conf. You can get the most recent listing of avaialble flags by running nova-(servicename) --help, for example, nova-api --help. + +Here's a list of available flags and their default settings. + + --ajax_console_proxy_port: port that ajax_console_proxy binds + (default: '8000') + --ajax_console_proxy_topic: the topic ajax proxy nodes listen on + (default: 'ajax_proxy') + --ajax_console_proxy_url: location of ajax console proxy, in the form + "http://127.0.0.1:8000" + (default: 'http://127.0.0.1:8000') + --auth_token_ttl: Seconds for auth tokens to linger + (default: '3600') + (an integer) + --aws_access_key_id: AWS Access ID + (default: 'admin') + --aws_secret_access_key: AWS Access Key + (default: 'admin') + --compute_manager: Manager for compute + (default: 'nova.compute.manager.ComputeManager') + --compute_topic: the topic compute nodes listen on + (default: 'compute') + --connection_type: libvirt, xenapi or fake + (default: 'libvirt') + --console_manager: Manager for console proxy + (default: 'nova.console.manager.ConsoleProxyManager') + --console_topic: the topic console proxy nodes listen on + (default: 'console') + --control_exchange: the main exchange to connect to + (default: 'nova') + --db_backend: The backend to use for db + (default: 'sqlalchemy') + --default_image: default image to use, testing only + (default: 'ami-11111') + --default_instance_type: default instance type to use, testing only + (default: 'm1.small') + --default_log_levels: list of logger=LEVEL pairs + (default: 'amqplib=WARN,sqlalchemy=WARN,eventlet.wsgi.server=WARN') + (a comma separated list) + --default_project: default project for openstack + (default: 'openstack') + --ec2_dmz_host: internal ip of api server + (default: '$my_ip') + --ec2_host: ip of api server + (default: '$my_ip') + --ec2_path: suffix for ec2 + (default: '/services/Cloud') + --ec2_port: cloud controller port + (default: '8773') + (an integer) + --ec2_scheme: prefix for ec2 + (default: 'http') + --[no]enable_new_services: Services to be added to the available pool on + create + (default: 'true') + --[no]fake_network: should we use fake network devices and addresses + (default: 'false') + --[no]fake_rabbit: use a fake rabbit + (default: 'false') + --glance_host: glance host + (default: '$my_ip') + --glance_port: glance port + (default: '9292') + (an integer) + -?,--[no]help: show this help + --[no]helpshort: show usage only for this module + --[no]helpxml: like --help, but generates XML output + --host: name of this node + (default: 'osdemo03') + --image_service: The service to use for retrieving and searching for images. + (default: 'nova.image.s3.S3ImageService') + --instance_name_template: Template string to be used to generate instance + names + (default: 'instance-%08x') + --logfile: output to named file + --logging_context_format_string: format string to use for log messages with + context + (default: '%(asctime)s %(levelname)s %(name)s [%(request_id)s %(user)s + %(project)s] %(message)s') + --logging_debug_format_suffix: data to append to log format when level is + DEBUG + (default: 'from %(processName)s (pid=%(process)d) %(funcName)s + %(pathname)s:%(lineno)d') + --logging_default_format_string: format string to use for log messages without + context + (default: '%(asctime)s %(levelname)s %(name)s [-] %(message)s') + --logging_exception_prefix: prefix each line of exception output with this + format + (default: '(%(name)s): TRACE: ') + --my_ip: host ip address + (default: '184.106.73.68') + --network_manager: Manager for network + (default: 'nova.network.manager.VlanManager') + --network_topic: the topic network nodes listen on + (default: 'network') + --node_availability_zone: availability zone of this node + (default: 'nova') + --null_kernel: kernel image that indicates not to use a kernel, but to use a + raw disk image instead + (default: 'nokernel') + --osapi_host: ip of api server + (default: '$my_ip') + --osapi_path: suffix for openstack + (default: '/v1.0/') + --osapi_port: OpenStack API port + (default: '8774') + (an integer) + --osapi_scheme: prefix for openstack + (default: 'http') + --periodic_interval: seconds between running periodic tasks + (default: '60') + (a positive integer) + --pidfile: pidfile to use for this service + --rabbit_host: rabbit host + (default: 'localhost') + --rabbit_max_retries: rabbit connection attempts + (default: '12') + (an integer) + --rabbit_password: rabbit password + (default: 'guest') + --rabbit_port: rabbit port + (default: '5672') + (an integer) + --rabbit_retry_interval: rabbit connection retry interval + (default: '10') + (an integer) + --rabbit_userid: rabbit userid + (default: 'guest') + --rabbit_virtual_host: rabbit virtual host + (default: '/') + --region_list: list of region=fqdn pairs separated by commas + (default: '') + (a comma separated list) + --report_interval: seconds between nodes reporting state to datastore + (default: '10') + (a positive integer) + --s3_dmz: s3 dmz ip (for instances) + (default: '$my_ip') + --s3_host: s3 host (for infrastructure) + (default: '$my_ip') + --s3_port: s3 port + (default: '3333') + (an integer) + --scheduler_manager: Manager for scheduler + (default: 'nova.scheduler.manager.SchedulerManager') + --scheduler_topic: the topic scheduler nodes listen on + (default: 'scheduler') + --sql_connection: connection string for sql database + (default: 'sqlite:///$state_path/nova.sqlite') + --sql_idle_timeout: timeout for idle sql database connections + (default: '3600') + --sql_max_retries: sql connection attempts + (default: '12') + (an integer) + --sql_retry_interval: sql connection retry interval + (default: '10') + (an integer) + --state_path: Top-level directory for maintaining nova's state + (default: '/usr/lib/pymodules/python2.6/nova/../') + --[no]use_syslog: output to syslog + (default: 'false') + --[no]verbose: show debug output + (default: 'false') + --volume_manager: Manager for volume + (default: 'nova.volume.manager.VolumeManager') + --volume_name_template: Template string to be used to generate instance names + (default: 'volume-%08x') + --volume_topic: the topic volume nodes listen on + (default: 'volume') + --vpn_image_id: AMI for cloudpipe vpn server + (default: 'ami-cloudpipe') + --vpn_key_suffix: Suffix to add to project name for vpn key and secgroups + (default: '-vpn') \ No newline at end of file diff --git a/doc/source/runnova/getting.started.rst b/doc/source/runnova/getting.started.rst new file mode 100644 index 000000000..4cc7307b0 --- /dev/null +++ b/doc/source/runnova/getting.started.rst @@ -0,0 +1,168 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Getting Started with Nova +========================= + +This code base is continually changing, so dependencies also change. If you +encounter any problems, see the :doc:`../community` page. +The `contrib/nova.sh` script should be kept up to date, and may be a good +resource to review when debugging. + +The purpose of this document is to get a system installed that you can use to +test your setup assumptions. Working from this base installtion you can +tweak configurations and work with different flags to monitor interaction with +your hardware, network, and other factors that will allow you to determine +suitability for your deployment. After following this setup method, you should +be able to experiment with different managers, drivers, and flags to get the +best performance. + +Dependencies +------------ + +Related servers we rely on + +* **RabbitMQ**: messaging queue, used for all communication between components + +Optional servers + +* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by + setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP + (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. + There is a script in the sources (`nova/auth/slap.sh`) to install a very basic + openldap server on ubuntu. +* **ReDIS**: There is a fake ldap auth driver + `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was + created for testing ldap implementation on systems that don't have an easy + means to install ldap. +* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to + be avilable. Currently, only sqlite3 an mysql have been tested. + +Python libraries that we use (from pip-requires): + +.. literalinclude:: ../../../tools/pip-requires + +Other libraries: + +* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available + from http://wiki.xensource.com/xenwiki/XCP_SDK or + http://community.citrix.com/cdn/xs/sdks. + +External unix tools that are required: + +* iptables +* ebtables +* gawk +* curl +* kvm +* libvirt +* dnsmasq +* vlan +* open-iscsi and iscsitarget (if you use iscsi volumes) +* aoetools and vblade-persist (if you use aoe-volumes) + +Nova uses cutting-edge versions of many packages. There are ubuntu packages in +the nova-core trunk ppa. You can use add this ppa to your sources list on an +ubuntu machine with the following commands:: + + sudo apt-get install -y python-software-properties + sudo add-apt-repository ppa:nova-core/trunk + +Recommended +----------- + +* euca2ools: python implementation of aws ec2-tools and ami tools +* build tornado to use C module for evented section + + +Installation +-------------- + +You can install from packages for your particular Linux distribution if they are +available. Otherwise you can install from source by checking out the source +files from the `Nova Source Code Repository `_ +and running:: + + python setup.py install + +Configuration +--------------- + +Configuring the host system +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Nova can be configured in many different ways. In this "Getting Started with Nova" document, we only provide what you need to get started as quickly as possible. For a more detailed description of system +configuration, start reading through `Installing and Configuring OpenStack Compute `_. + +`Detailed instructions for creating a volume group are available `_, or use these quick instructions. + +* Create a volume group (you can use an actual disk for the volume group as + well):: + + # This creates a 1GB file to create volumes out of + dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 + losetup --show -f MY_FILE_PATH + # replace /dev/loop0 below with whatever losetup returns + # nova-volumes is the default for the --volume_group flag + vgcreate nova-volumes /dev/loop0 + + +Configuring Nova +~~~~~~~~~~~~~~~~ + +Configuration of the entire system is performed through python-gflags. The +best way to track configuration is through the use of a flagfile. + +A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary +when you launch it. Flagfiles for nova are typically stored in +``/etc/nova/nova.conf``, and flags specific to a certain program are stored in +``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another +flagfile, so typically a file like ``nova-manage.conf`` would have as its first +line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before +specifying overrides or additional options. + +To get a current comprehensive list of flag file options, run bin/nova- --help, or refer to a static list at `Reference for Flags in nova.conf `_. + +A sample configuration to test the system follows:: + + --verbose + --nodaemon + --auth_driver=nova.auth.dbdriver.DbDriver + +Running +------- + +There are many parts to the nova system, each with a specific function. They +are built to be highly-available, so there are may configurations they can be +run in (ie: on many machines, many listeners per machine, etc). This part +of the guide only gets you started quickly, to learn about HA options, see +`Installing and Configuring OpenStack Compute `_. + +Launch supporting services + +* rabbitmq +* redis (optional) +* mysql (optional) +* openldap (optional) + +Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` + +* nova-api +* nova-compute +* nova-objectstore +* nova-volume +* nova-scheduler diff --git a/doc/source/runnova/index.rst b/doc/source/runnova/index.rst new file mode 100644 index 000000000..283d268ce --- /dev/null +++ b/doc/source/runnova/index.rst @@ -0,0 +1,90 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Running Nova +============ + +This guide describes the basics of running and managing Nova. For more administrator's documentation, refer to `docs.openstack.org `_. + +Running the Cloud +----------------- + +The fastest way to get a test cloud running is by following the directions in the :doc:`../quickstart`. It relies on a nova.sh script to run on a single machine. + +Nova's cloud works via the interaction of a series of daemon processes that reside persistently on the host machine(s). Fortunately, the :doc:`../quickstart` process launches sample versions of all these daemons for you. Once you are familiar with basic Nova usage, you can learn more about daemons by reading :doc:`../service.architecture` and :doc:`binaries`. + +Administration Utilities +------------------------ + +There are two main tools that a system administrator will find useful to manage their Nova cloud: + +.. toctree:: + :maxdepth: 1 + + nova.manage + euca2ools + +The nova-manage command may only be run by users with admin priviledges. Commands for euca2ools can be used by all users, though specific commands may be restricted by Role Based Access Control. You can read more about creating and managing users in :doc:`managing.users` + +User and Resource Management +---------------------------- + +The nova-manage and euca2ools commands provide the basic interface to perform a broad range of administration functions. In this section, you can read more about how to accomplish specific administration tasks. + +For background on the core objects referenced in this section, see :doc:`../object.model` + +.. toctree:: + :maxdepth: 1 + + managing.users + managing.projects + managing.instances + managing.images + managing.volumes + managing.networks + +Deployment +---------- + +For a starting multi-node architecture, you would start with two nodes - a cloud controller node and a compute node. The cloud controller node contains the nova- services plus the Nova database. The compute node installs all the nova-services but then refers to the database installation, which is hosted by the cloud controller node. Ensure that the nova.conf file is identical on each node. If you find performance issues not related to database reads or writes, but due to the messaging queue backing up, you could add additional messaging services (rabbitmq). For instructions on multi-server installations, refer to `Installing and Configuring OpenStack Compute `_. + + +.. toctree:: + :maxdepth: 1 + + dbsync + + +Networking +^^^^^^^^^^ + +.. toctree:: + :maxdepth: 1 + + network.vlan.rst + network.flat.rst + + +Advanced Topics +--------------- + +.. toctree:: + :maxdepth: 1 + + flags + monitoring + diff --git a/doc/source/runnova/managing.images.rst b/doc/source/runnova/managing.images.rst new file mode 100644 index 000000000..c5d93a6e8 --- /dev/null +++ b/doc/source/runnova/managing.images.rst @@ -0,0 +1,21 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Images +=============== + +.. todo:: Put info on managing images here! diff --git a/doc/source/runnova/managing.instances.rst b/doc/source/runnova/managing.instances.rst new file mode 100644 index 000000000..e62352017 --- /dev/null +++ b/doc/source/runnova/managing.instances.rst @@ -0,0 +1,59 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Instances +================== + +Keypairs +-------- + +Images can be shared by many users, so it is dangerous to put passwords into the images. Nova therefore supports injecting ssh keys into instances before they are booted. This allows a user to login to the instances that he or she creates securely. Generally the first thing that a user does when using the system is create a keypair. Nova generates a public and private key pair, and sends the private key to the user. The public key is stored so that it can be injected into instances. + +Keypairs are created through the api. They can be created on the command line using the euca2ools script euca-add-keypair. Refer to the man page for the available options. Example usage:: + + euca-add-keypair test > test.pem + chmod 600 test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + # wait for boot + ssh -i test.pem root@ip.of.instance + + +Basic Management +---------------- +Instance management can be accomplished with euca commands: + + +To run an instance: + +:: + + euca-run-instances + + +To terminate an instance: + +:: + + euca-terminate-instances + +To reboot an instance: + +:: + + euca-reboot-instances + +See the euca2ools documentation for more information diff --git a/doc/source/runnova/managing.networks.rst b/doc/source/runnova/managing.networks.rst new file mode 100644 index 000000000..9eea46d70 --- /dev/null +++ b/doc/source/runnova/managing.networks.rst @@ -0,0 +1,70 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + Overview Sections Copyright 2010-2011 Citrix + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Networking Overview +=================== +In Nova, users organize their cloud resources in projects. A Nova project consists of a number of VM instances created by a user. For each VM instance, Nova assigns to it a private IP address. (Currently, Nova only supports Linux bridge networking that allows the virtual interfaces to connect to the outside network through the physical interface. Other virtual network technologies, such as Open vSwitch, could be supported in the future.) The Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. + +Nova Network Strategies +----------------------- + +Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can co-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. + +Read more about Nova network strategies here: + +.. toctree:: + :maxdepth: 1 + + network.flat.rst + network.vlan.rst + + +Network Management Commands +--------------------------- + +Admins and Network Administrators can use the 'nova-manage' command to manage network resources: + +VPN Management +~~~~~~~~~~~~~~ + +* vpn list: Print a listing of the VPNs for all projects. + * arguments: none +* vpn run: Start the VPN for a given project. + * arguments: project +* vpn spawn: Run all VPNs. + * arguments: none + + +Floating IP Management +~~~~~~~~~~~~~~~~~~~~~~ + +* floating create: Creates floating ips for host by range + * arguments: host ip_range +* floating delete: Deletes floating ips by range + * arguments: range +* floating list: Prints a listing of all floating ips + * arguments: none + +Network Management +~~~~~~~~~~~~~~~~~~ + +* network create: Creates fixed ips for host by range + * arguments: [fixed_range=FLAG], [num_networks=FLAG], + [network_size=FLAG], [vlan_start=FLAG], + [vpn_start=FLAG] + diff --git a/doc/source/runnova/managing.projects.rst b/doc/source/runnova/managing.projects.rst new file mode 100644 index 000000000..5dd7f2de9 --- /dev/null +++ b/doc/source/runnova/managing.projects.rst @@ -0,0 +1,68 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Projects +================= + +Projects are isolated resource containers forming the principal organizational structure within Nova. They consist of a separate vlan, volumes, instances, images, keys, and users. + +Although the original ec2 api only supports users, nova adds the concept of projects. A user can specify which project he or she wishes to use by appending `:project_id` to his or her access key. If no project is specified in the api request, nova will attempt to use a project with the same id as the user. + +The api will return NotAuthorized if a normal user attempts to make requests for a project that he or she is not a member of. Note that admins or users with special admin roles skip this check and can make requests for any project. + +To create a project, use the `project create` command of nova-manage. The syntax is nova-manage project create projectname manager_id [description] You must specify a projectname and a manager_id. For example:: + nova-manage project create john_project john "This is a sample project" + +You can add and remove users from projects with `project add` and `project remove`:: + nova-manage project add john_project john + nova-manage project remove john_project john + +Project Commands +---------------- + +Admins and Project Managers can use the 'nova-manage project' command to manage project resources: + +* project add: Adds user to project + * arguments: project user +* project create: Creates a new project + * arguments: name project_manager [description] +* project delete: Deletes an existing project + * arguments: project_id +* project environment: Exports environment variables to an sourcable file + * arguments: project_id user_id [filename='novarc] +* project list: lists all projects + * arguments: none +* project remove: Removes user from project + * arguments: project user +* project scrub: Deletes data associated with project + * arguments: project +* project zipfile: Exports credentials for project to a zip file + * arguments: project_id user_id [filename='nova.zip] + +Setting Quotas +-------------- +Nova utilizes a quota system at the project level to control resource consumption across available hardware resources. Current quota controls are available to limit the: + +* Number of volumes which may be created +* Total size of all volumes within a project as measured in GB +* Number of instances which may be launched +* Number of processor cores which may be allocated +* Publicly accessible IP addresses + +Use the following command to set quotas for a project +* project quota: Set or display quotas for project + * arguments: project_id [key] [value] diff --git a/doc/source/runnova/managing.users.rst b/doc/source/runnova/managing.users.rst new file mode 100644 index 000000000..392142e86 --- /dev/null +++ b/doc/source/runnova/managing.users.rst @@ -0,0 +1,82 @@ +Managing Users +============== + + +Users and Access Keys +--------------------- + +Access to the ec2 api is controlled by an access and secret key. The user's access key needs to be included in the request, and the request must be signed with the secret key. Upon receipt of api requests, nova will verify the signature and execute commands on behalf of the user. + +In order to begin using nova, you will need a to create a user. This can be easily accomplished using the user create or user admin commands in nova-manage. `user create` will create a regular user, whereas `user admin` will create an admin user. The syntax of the command is nova-manage user create username [access] [secret]. For example:: + + nova-manage user create john my-access-key a-super-secret-key + +If you do not specify an access or secret key, a random uuid will be created automatically. + +Credentials +----------- + +Nova can generate a handy set of credentials for a user. These credentials include a CA for bundling images and a file for setting environment variables to be used by euca2ools. If you don't need to bundle images, just the environment script is required. You can export one with the `project environment` command. The syntax of the command is nova-manage project environment project_id user_id [filename]. If you don't specify a filename, it will be exported as novarc. After generating the file, you can simply source it in bash to add the variables to your environment:: + + nova-manage project environment john_project john + . novarc + +If you do need to bundle images, you will need to get all of the credentials using `project zipfile`. Note that zipfile will give you an error message if networks haven't been created yet. Otherwise zipfile has the same syntax as environment, only the default file name is nova.zip. Example usage:: + + nova-manage project zipfile john_project john + unzip nova.zip + . novarc + +Role Based Access Control +------------------------- +Roles control the api actions that a user is allowed to perform. For example, a user cannot allocate a public ip without the `netadmin` role. It is important to remember that a users de facto permissions in a project is the intersection of user (global) roles and project (local) roles. So for john to have netadmin permissions in his project, he needs to separate roles specified. You can add roles with `role add`. The syntax is nova-manage role add user_id role [project_id]. Let's give john the netadmin role for his project:: + + nova-manage role add john netadmin + nova-manage role add john netadmin john_project + +Role-based access control (RBAC) is an approach to restricting system access to authorized users based on an individual’s role within an organization. Various employee functions require certain levels of system access in order to be successful. These functions are mapped to defined roles and individuals are categorized accordingly. Since users are not assigned permissions directly, but only acquire them through their role (or roles), management of individual user rights becomes a matter of assigning appropriate roles to the user. This simplifies common operations, such as adding a user, or changing a user's department. + +Nova’s rights management system employs the RBAC model and currently supports the following five roles: + +* **Cloud Administrator.** (admin) Users of this class enjoy complete system access. +* **IT Security.** (itsec) This role is limited to IT security personnel. It permits role holders to quarantine instances. +* **Project Manager.** (projectmanager)The default for project owners, this role affords users the ability to add other users to a project, interact with project images, and launch and terminate instances. +* **Network Administrator.** (netadmin) Users with this role are permitted to allocate and assign publicly accessible IP addresses as well as create and modify firewall rules. +* **Developer.** This is a general purpose role that is assigned to users by default. + +RBAC management is exposed through the dashboard for simplified user management. + + +User Commands +~~~~~~~~~~~~ + +Users, including admins, are created through the ``user`` commands. + +* user admin: creates a new admin and prints exports + * arguments: name [access] [secret] +* user create: creates a new user and prints exports + * arguments: name [access] [secret] +* user delete: deletes an existing user + * arguments: name +* user exports: prints access and secrets for user in export format + * arguments: name +* user list: lists all users + * arguments: none +* user modify: update a users keys & admin flag + * arguments: accesskey secretkey admin + * leave any field blank to ignore it, admin should be 'T', 'F', or blank + + +User Role Management +~~~~~~~~~~~~~~~~~~~~ + +* role add: adds role to user + * if project is specified, adds project specific role + * arguments: user, role [project] +* role has: checks to see if user has role + * if project is specified, returns True if user has + the global role and the project role + * arguments: user, role [project] +* role remove: removes role from user + * if project is specified, removes project specific role + * arguments: user, role [project] diff --git a/doc/source/runnova/managingsecurity.rst b/doc/source/runnova/managingsecurity.rst new file mode 100644 index 000000000..7893925e7 --- /dev/null +++ b/doc/source/runnova/managingsecurity.rst @@ -0,0 +1,39 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Security Considerations +======================= + +.. todo:: This doc is vague and just high-level right now. Describe architecture that enables security. + +The goal of securing a cloud computing system involves both protecting the instances, data on the instances, and +ensuring users are authenticated for actions and that borders are understood by the users and the system. +Protecting the system from intrusion or attack involves authentication, network protections, and +compromise detection. + +Key Concepts +------------ + +Authentication - Each instance is authenticated with a key pair. + +Network - Instances can communicate with each other but you can configure the boundaries through firewall +configuration. + +Monitoring - Log all API commands and audit those logs. + +Encryption - Data transfer between instances is not encrypted. + diff --git a/doc/source/runnova/monitoring.rst b/doc/source/runnova/monitoring.rst new file mode 100644 index 000000000..2c93c71b5 --- /dev/null +++ b/doc/source/runnova/monitoring.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Monitoring +========== + +* components +* throughput +* exceptions +* hardware + +* ganglia +* syslog diff --git a/doc/source/runnova/network.flat.rst b/doc/source/runnova/network.flat.rst new file mode 100644 index 000000000..3d8680c6f --- /dev/null +++ b/doc/source/runnova/network.flat.rst @@ -0,0 +1,60 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +Flat Network Mode (Original and Flat) +===================================== + +Flat network mode removes most of the complexity of VLAN mode by simply +bridging all instance interfaces onto a single network. + +There are two variations of flat mode that differ mostly in how IP addresses +are given to instances. + + +Original Flat Mode +------------------ +IP addresses for VM instances are grabbed from a subnet specified by the network administrator, and injected into the image on launch. All instances of the system are attached to the same Linux networking bridge, configured manually by the network administrator both on the network controller hosting the network and on the computer controllers hosting the instances. To recap: + +* Each compute host creates a single bridge for all instances to use to attach to the external network. +* The networking configuration is injected into the instance before it is booted or it is obtained by a guest agent installed in the instance. + +Note that the configuration injection currently only works on linux-style systems that keep networking +configuration in /etc/network/interfaces. + + +Flat DHCP Mode +-------------- +IP addresses for VM instances are grabbed from a subnet specified by the network administrator. Similar to the flat network, a single Linux networking bridge is created and configured manually by the network administrator and used for all instances. A DHCP server is started to pass out IP addresses to VM instances from the specified subnet. To recap: + +* Like flat mode, all instances are attached to a single bridge on the compute node. +* In addition a DHCP server is running to configure instances. + +Implementation +-------------- + +The network nodes do not act as a default gateway in flat mode. Instances +are given public IP addresses. + +Compute nodes have iptables/ebtables entries created per project and +instance to protect against IP/MAC address spoofing and ARP poisoning. + + +Examples +-------- + +.. todo:: add flat network mode configuration examples diff --git a/doc/source/runnova/network.vlan.rst b/doc/source/runnova/network.vlan.rst new file mode 100644 index 000000000..c06ce8e8b --- /dev/null +++ b/doc/source/runnova/network.vlan.rst @@ -0,0 +1,179 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +VLAN Network Mode +================= +VLAN Network Mode is the default mode for Nova. It provides a private network +segment for each project's instances that can be accessed via a dedicated +VPN connection from the Internet. + +In this mode, each project gets its own VLAN, Linux networking bridge, and subnet. The subnets are specified by the network administrator, and are assigned dynamically to a project when required. A DHCP Server is started for each VLAN to pass out IP addresses to VM instances from the subnet assigned to the project. All instances belonging to one project are bridged into the same VLAN for that project. The Linux networking bridges and VLANs are created by Nova when required, described in more detail in Nova VLAN Network Management Implementation. + +.. + (this text revised above) + Because the flat network and flat DhCP network are simple to understand and yet do not scale well enough for real-world cloud systems, this section focuses on the VLAN network implementation by the VLAN Network Manager. + + + In the VLAN network mode, all the VM instances of a project are connected together in a VLAN with the specified private subnet. Each running VM instance is assigned an IP address within the given private subnet. + +.. image:: /images/Novadiagram.png + :width: 790 + +While network traffic between VM instances belonging to the same VLAN is always open, Nova can enforce isolation of network traffic between different projects by enforcing one VLAN per project. + +In addition, the network administrator can specify a pool of public IP addresses that users may allocate and then assign to VMs, either at boot or dynamically at run-time. This capability is similar to Amazon's 'elastic IPs'. A public IP address may be associated with a running instances, allowing the VM instance to be accessed from the public network. The public IP addresses are accessible from the network host and NATed to the private IP address of the project. + +.. todo:: Describe how a public IP address could be associated with a project (a VLAN) + +This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code named :ref:`cloudpipe `) needs to be created. Nova generates a certificate and key for the user to access the vpn and starts the vpn automatically. More information on cloudpipe can be found :ref:`here `. + +The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) + +.. image:: /images/cloudpipe.png + :width: 100% + +Goals +----- + +For our implementation of Nova, our goal is that each project is in a protected network segment. Here are the specifications we keep in mind for meeting this goal. + + * RFC-1918 IP space + * public IP via NAT + * no default inbound Internet access without public NAT + * limited (project-admin controllable) outbound Internet access + * limited (project-admin controllable) access to other project segments + * all connectivity to instance and cloud API is via VPN into the project segment + +We also keep as a goal a common DMZ segment for support services, meaning these items are only visible from project segment: + + * metadata + * dashboard + +Limitations +----------- + +We kept in mind some of these limitations: + +* Projects / cluster limited to available VLANs in switching infrastructure +* Requires VPN for access to project segment + +Implementation +-------------- +Currently Nova segregates project VLANs using 802.1q VLAN tagging in the +switching layer. Compute hosts create VLAN-specific interfaces and bridges +as required. + +The network nodes act as default gateway for project networks and contain +all of the routing and firewall rules implementing security groups. The +network node also handles DHCP to provide instance IPs for each project. + +VPN access is provided by running a small instance called CloudPipe +on the IP immediately following the gateway IP for each project. The +network node maps a dedicated public IP/port to the CloudPipe instance. + +Compute nodes have per-VLAN interfaces and bridges created as required. +These do NOT have IP addresses in the host to protect host access. +Compute nodes have iptables/ebtables entries created per project and +instance to protect against IP/MAC address spoofing and ARP poisoning. + +The network assignment to a project, and IP address assignment to a VM instance, are triggered when a user starts to run a VM instance. When running a VM instance, a user needs to specify a project for the instances, and the security groups (described in Security Groups) when the instance wants to join. If this is the first instance to be created for the project, then Nova (the cloud controller) needs to find a network controller to be the network host for the project; it then sets up a private network by finding an unused VLAN id, an unused subnet, and then the controller assigns them to the project, it also assigns a name to the project's Linux bridge (br100 stored in the Nova database), and allocating a private IP within the project's subnet for the new instance. + +If the instance the user wants to start is not the project's first, a subnet and a VLAN must have already been assigned to the project; therefore the system needs only to find an available IP address within the subnet and assign it to the new starting instance. If there is no private IP available within the subnet, an exception will be raised to the cloud controller, and the VM creation cannot proceed. + + +External Infrastructure +----------------------- + +Nova assumes the following is available: + +* DNS +* NTP +* Internet connectivity + + +Example +------- + +This example network configuration demonstrates most of the capabilities +of VLAN Mode. It splits administrative access to the nodes onto a dedicated +management network and uses dedicated network nodes to handle all +routing and gateway functions. + +It uses a 10GB network for instance traffic and a 1GB network for management. + + +Hardware +~~~~~~~~ + +* All nodes have a minimum of two NICs for management and production. + + * management is 1GB + * production is 10GB + * add additional NICs for bonding or HA/performance + +* network nodes should have an additional NIC dedicated to public Internet traffic +* switch needs to support enough simultaneous VLANs for number of projects +* production network configured as 802.1q trunk on switch + + +Operation +~~~~~~~~~ + +The network node controls the project network configuration: + +* assigns each project a VLAN and private IP range +* starts dnsmasq on project VLAN to serve private IP range +* configures iptables on network node for default project access +* launches CloudPipe instance and configures iptables access + +When starting an instance the network node: + +* sets up a VLAN interface and bridge on each host as required when an + instance is started on that host +* assigns private IP to instance +* generates MAC address for instance +* update dnsmasq with IP/MAC for instance + +When starting an instance the compute node: + +* sets up a VLAN interface and bridge on each host as required when an + instance is started on that host + + +Setup +~~~~~ + +* Assign VLANs in the switch: + + * public Internet segment + * production network + * management network + * cluster DMZ + +* Assign a contiguous range of VLANs to Nova for project use. +* Configure management NIC ports as management VLAN access ports. +* Configure management VLAN with Internet access as required +* Configure production NIC ports as 802.1q trunk ports. +* Configure Nova (need to add specifics here) + + * public IPs + * instance IPs + * project network size + * DMZ network + +.. todo:: need specific Nova configuration added diff --git a/doc/source/runnova/nova.manage.rst b/doc/source/runnova/nova.manage.rst new file mode 100644 index 000000000..0e9a29b6b --- /dev/null +++ b/doc/source/runnova/nova.manage.rst @@ -0,0 +1,239 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +The nova-manage command +======================= + +Introduction +~~~~~~~~~~~~ + +The nova-manage command is used to perform many essential functions for +administration and ongoing maintenance of nova, such as user creation, +vpn management, and much more. + +The standard pattern for executing a nova-manage command is: +``nova-manage []`` + +For example, to obtain a list of all projects: +``nova-manage project list`` + +Run without arguments to see a list of available command categories: +``nova-manage`` + +Categories are user, project, role, shell, vpn, and floating. Detailed descriptions are below. + +You can also run with a category argument such as user to see a list of all commands in that category: +``nova-manage user`` + +These sections describe the available categories and arguments for nova-manage. + +Nova Db +~~~~~~~ + +``nova-manage db version`` + + Print the current database version. + +``nova-manage db sync`` + + Sync the database up to the most recent version. This is the standard way to create the db as well. + +Nova User +~~~~~~~~~ + +``nova-manage user admin `` + + Create an admin user with the name . + +``nova-manage user create `` + + Create a normal user with the name . + +``nova-manage user delete `` + + Delete the user with the name . + +``nova-manage user exports `` + + Outputs a list of access key and secret keys for user to the screen + +``nova-manage user list`` + + Outputs a list of all the user names to the screen. + +``nova-manage user modify `` + + Updates the indicated user keys, indicating with T or F if the user is an admin user. Leave any argument blank if you do not want to update it. + +Nova Project +~~~~~~~~~~~~ + +``nova-manage project add `` + + Add a nova project with the name to the database. + +``nova-manage project create `` + + Create a new nova project with the name (you still need to do nova-manage project add to add it to the database). + +``nova-manage project delete `` + + Delete a nova project with the name . + +``nova-manage project environment `` + + Exports environment variables for the named project to a file named novarc. + +``nova-manage project list`` + + Outputs a list of all the projects to the screen. + +``nova-manage project quota `` + + Outputs the size and specs of the project's instances including gigabytes, instances, floating IPs, volumes, and cores. + +``nova-manage project remove `` + + Deletes the project with the name . + +``nova-manage project zipfile`` + + Compresses all related files for a created project into a zip file nova.zip. + +Nova Role +~~~~~~~~~ + +nova-manage role [] +``nova-manage role add <(optional) projectname>`` + + Add a user to either a global or project-based role with the indicated assigned to the named user. Role names can be one of the following five roles: admin, itsec, projectmanager, netadmin, developer. If you add the project name as the last argument then the role is assigned just for that project, otherwise the user is assigned the named role for all projects. + +``nova-manage role has `` + Checks the user or project and responds with True if the user has a global role with a particular project. + +``nova-manage role remove `` + Remove the indicated role from the user. + +Nova Shell +~~~~~~~~~~ + +``nova-manage shell bpython`` + + Starts a new bpython shell. + +``nova-manage shell ipython`` + + Starts a new ipython shell. + +``nova-manage shell python`` + + Starts a new python shell. + +``nova-manage shell run`` + + Starts a new shell using python. + +``nova-manage shell script `` + + Runs the named script from the specified path with flags set. + +Nova VPN +~~~~~~~~ + +``nova-manage vpn list`` + + Displays a list of projects, their IP prot numbers, and what state they're in. + +``nova-manage vpn run `` + + Starts the VPN for the named project. + +``nova-manage vpn spawn`` + + Runs all VPNs. + +Nova Floating IPs +~~~~~~~~~~~~~~~~~ + +``nova-manage floating create `` + + Creates floating IP addresses for the named host by the given range. + +``nova-manage floating delete `` + + Deletes floating IP addresses in the range given. + +``nova-manage floating list`` + + Displays a list of all floating IP addresses. + +Concept: Flags +-------------- + +python-gflags + + +Concept: Plugins +---------------- + +* Managers/Drivers: utils.import_object from string flag +* virt/connections: conditional loading from string flag +* db: LazyPluggable via string flag +* auth_manager: utils.import_class based on string flag +* Volumes: moving to pluggable driver instead of manager +* Network: pluggable managers +* Compute: same driver used, but pluggable at connection + + +Concept: IPC/RPC +---------------- + +Rabbit! + + +Concept: Fakes +-------------- + +* auth +* ldap + + +Concept: Scheduler +------------------ + +* simple +* random + + +Concept: Security Groups +------------------------ + +Security groups + + +Concept: Certificate Authority +------------------------------ + +Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns <../cloudpipe>` and decrypting bundled images. + + +Concept: Images +--------------- + +* launching +* bundling diff --git a/test/.Python b/test/.Python new file mode 120000 index 000000000..6cce156bd --- /dev/null +++ b/test/.Python @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/Python \ No newline at end of file diff --git a/test/bin/activate b/test/bin/activate new file mode 100644 index 000000000..588b54105 --- /dev/null +++ b/test/bin/activate @@ -0,0 +1,76 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "$_OLD_VIRTUAL_PATH" ] ; then + PATH="$_OLD_VIRTUAL_PATH" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then + PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then + hash -r + fi + + if [ -n "$_OLD_VIRTUAL_PS1" ] ; then + PS1="$_OLD_VIRTUAL_PS1" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "$1" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelavent variables +deactivate nondestructive + +VIRTUAL_ENV="/Users/anne.gentle/src/nova/docslice/test" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "$PYTHONHOME" ] ; then + _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" + unset PYTHONHOME +fi + +if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then + _OLD_VIRTUAL_PS1="$PS1" + if [ "x" != x ] ; then + PS1="$PS1" + else + if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" + else + PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" + fi + fi + export PS1 +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then + hash -r +fi diff --git a/test/bin/activate.csh b/test/bin/activate.csh new file mode 100644 index 000000000..d29ac339a --- /dev/null +++ b/test/bin/activate.csh @@ -0,0 +1,32 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelavent variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if ("" != "") then + set env_name = "" +else + if (`basename "$VIRTUAL_ENV"` == "__") then + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` + else + set env_name = `basename "$VIRTUAL_ENV"` + endif +endif +set prompt = "[$env_name] $prompt" +unset env_name + +rehash + diff --git a/test/bin/activate.fish b/test/bin/activate.fish new file mode 100644 index 000000000..de3a8901e --- /dev/null +++ b/test/bin/activate.fish @@ -0,0 +1,79 @@ +# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) +# you cannot run it directly + +function deactivate -d "Exit virtualenv and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + end + + set -e VIRTUAL_ENV + if test "$argv[1]" != "nondestructive" + # Self destruct! + functions -e deactivate + end +end + +# unset irrelavent variables +deactivate nondestructive + +set -gx VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# unset PYTHONHOME if set +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish shell uses a function, instead of env vars, + # to produce the prompt. Overriding the existing function is easy. + # However, adding to the current prompt, instead of clobbering it, + # is a little more work. + set -l oldpromptfile (tempfile) + if test $status + # save the current fish_prompt function... + echo "function _old_fish_prompt" >> $oldpromptfile + echo -n \# >> $oldpromptfile + functions fish_prompt >> $oldpromptfile + # we've made the "_old_fish_prompt" file, source it. + . $oldpromptfile + rm -f $oldpromptfile + + if test -n "" + # We've been given us a prompt override. + # + # FIXME: Unsure how to handle this *safely*. We could just eval() + # whatever is given, but the risk is a bit much. + echo "activate.fish: Alternative prompt prefix is not supported under fish-shell." 1>&2 + echo "activate.fish: Alter the fish_prompt in this file as needed." 1>&2 + end + + # with the original prompt function renamed, we can override with our own. + function fish_prompt + set -l _checkbase (basename "$VIRTUAL_ENV") + if test $_checkbase = "__" + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + printf "%s[%s]%s %s" (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) (_old_fish_prompt) + else + printf "%s(%s)%s%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) (_old_fish_prompt) + end + end + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" + end +end + diff --git a/test/bin/activate_this.py b/test/bin/activate_this.py new file mode 100644 index 000000000..aff6927d6 --- /dev/null +++ b/test/bin/activate_this.py @@ -0,0 +1,32 @@ +"""By using execfile(this_file, dict(__file__=this_file)) you will +activate this virtualenv environment. + +This can be used when you must use an existing Python interpreter, not +the virtualenv bin/python +""" + +try: + __file__ +except NameError: + raise AssertionError( + "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))") +import sys +import os + +base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if sys.platform == 'win32': + site_packages = os.path.join(base, 'Lib', 'site-packages') +else: + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') +prev_sys_path = list(sys.path) +import site +site.addsitedir(site_packages) +sys.real_prefix = sys.prefix +sys.prefix = base +# Move the added items to the front of the path: +new_sys_path = [] +for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) +sys.path[:0] = new_sys_path diff --git a/test/bin/easy_install b/test/bin/easy_install new file mode 100755 index 000000000..8544573fc --- /dev/null +++ b/test/bin/easy_install @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install' +__requires__ = 'setuptools==0.6c11' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install')() +) diff --git a/test/bin/easy_install-2.6 b/test/bin/easy_install-2.6 new file mode 100755 index 000000000..bad01d2d7 --- /dev/null +++ b/test/bin/easy_install-2.6 @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install-2.6' +__requires__ = 'setuptools==0.6c11' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install-2.6')() +) diff --git a/test/bin/pip b/test/bin/pip new file mode 100755 index 000000000..a3d2ed311 --- /dev/null +++ b/test/bin/pip @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip' +__requires__ = 'pip==0.8.1' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('pip==0.8.1', 'console_scripts', 'pip')() +) diff --git a/test/bin/pip-2.6 b/test/bin/pip-2.6 new file mode 100755 index 000000000..4ad06add1 --- /dev/null +++ b/test/bin/pip-2.6 @@ -0,0 +1,9 @@ +#!/Users/anne.gentle/src/nova/docslice/test/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip-2.6' +__requires__ = 'pip==0.8.1' +import sys +from pkg_resources import load_entry_point + +sys.exit( + load_entry_point('pip==0.8.1', 'console_scripts', 'pip-2.6')() +) diff --git a/test/bin/python b/test/bin/python new file mode 100755 index 000000000..271b6ca30 Binary files /dev/null and b/test/bin/python differ diff --git a/test/bin/python2.6 b/test/bin/python2.6 new file mode 120000 index 000000000..d8654aa0e --- /dev/null +++ b/test/bin/python2.6 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/test/include/python2.6 b/test/include/python2.6 new file mode 120000 index 000000000..788ac55ba --- /dev/null +++ b/test/include/python2.6 @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/include/python2.6 \ No newline at end of file diff --git a/test/lib/python2.6/UserDict.py b/test/lib/python2.6/UserDict.py new file mode 120000 index 000000000..93e0864b1 --- /dev/null +++ b/test/lib/python2.6/UserDict.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/UserDict.py \ No newline at end of file diff --git a/test/lib/python2.6/_abcoll.py b/test/lib/python2.6/_abcoll.py new file mode 120000 index 000000000..ff1769246 --- /dev/null +++ b/test/lib/python2.6/_abcoll.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/_abcoll.py \ No newline at end of file diff --git a/test/lib/python2.6/abc.py b/test/lib/python2.6/abc.py new file mode 120000 index 000000000..79fa108b2 --- /dev/null +++ b/test/lib/python2.6/abc.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/abc.py \ No newline at end of file diff --git a/test/lib/python2.6/codecs.py b/test/lib/python2.6/codecs.py new file mode 120000 index 000000000..9e4bfb7fa --- /dev/null +++ b/test/lib/python2.6/codecs.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/codecs.py \ No newline at end of file diff --git a/test/lib/python2.6/config b/test/lib/python2.6/config new file mode 120000 index 000000000..eef498c62 --- /dev/null +++ b/test/lib/python2.6/config @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/config \ No newline at end of file diff --git a/test/lib/python2.6/copy_reg.py b/test/lib/python2.6/copy_reg.py new file mode 120000 index 000000000..7285fda0e --- /dev/null +++ b/test/lib/python2.6/copy_reg.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/copy_reg.py \ No newline at end of file diff --git a/test/lib/python2.6/distutils/__init__.py b/test/lib/python2.6/distutils/__init__.py new file mode 100644 index 000000000..7ebb41c04 --- /dev/null +++ b/test/lib/python2.6/distutils/__init__.py @@ -0,0 +1,91 @@ +import os +import sys +import warnings +import opcode # opcode is not a virtualenv module, so we can use it to find the stdlib + # Important! To work on pypy, this must be a module that resides in the + # lib-python/modified-x.y.z directory + +dirname = os.path.dirname + +distutils_path = os.path.join(os.path.dirname(opcode.__file__), 'distutils') +if os.path.normpath(distutils_path) == os.path.dirname(os.path.normpath(__file__)): + warnings.warn( + "The virtualenv distutils package at %s appears to be in the same location as the system distutils?") +else: + __path__.insert(0, distutils_path) + exec open(os.path.join(distutils_path, '__init__.py')).read() + +import dist +import sysconfig + + +## patch build_ext (distutils doesn't know how to get the libs directory +## path on windows - it hardcodes the paths around the patched sys.prefix) + +if sys.platform == 'win32': + from distutils.command.build_ext import build_ext as old_build_ext + class build_ext(old_build_ext): + def finalize_options (self): + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, basestring): + self.library_dirs = self.library_dirs.split(os.pathsep) + + self.library_dirs.insert(0, os.path.join(sys.real_prefix, "Libs")) + old_build_ext.finalize_options(self) + + from distutils.command import build_ext as build_ext_module + build_ext_module.build_ext = build_ext + +## distutils.dist patches: + +old_find_config_files = dist.Distribution.find_config_files +def find_config_files(self): + found = old_find_config_files(self) + system_distutils = os.path.join(distutils_path, 'distutils.cfg') + #if os.path.exists(system_distutils): + # found.insert(0, system_distutils) + # What to call the per-user config file + if os.name == 'posix': + user_filename = ".pydistutils.cfg" + else: + user_filename = "pydistutils.cfg" + user_filename = os.path.join(sys.prefix, user_filename) + if os.path.isfile(user_filename): + for item in list(found): + if item.endswith('pydistutils.cfg'): + found.remove(item) + found.append(user_filename) + return found +dist.Distribution.find_config_files = find_config_files + +## distutils.sysconfig patches: + +old_get_python_inc = sysconfig.get_python_inc +def sysconfig_get_python_inc(plat_specific=0, prefix=None): + if prefix is None: + prefix = sys.real_prefix + return old_get_python_inc(plat_specific, prefix) +sysconfig_get_python_inc.__doc__ = old_get_python_inc.__doc__ +sysconfig.get_python_inc = sysconfig_get_python_inc + +old_get_python_lib = sysconfig.get_python_lib +def sysconfig_get_python_lib(plat_specific=0, standard_lib=0, prefix=None): + if standard_lib and prefix is None: + prefix = sys.real_prefix + return old_get_python_lib(plat_specific, standard_lib, prefix) +sysconfig_get_python_lib.__doc__ = old_get_python_lib.__doc__ +sysconfig.get_python_lib = sysconfig_get_python_lib + +old_get_config_vars = sysconfig.get_config_vars +def sysconfig_get_config_vars(*args): + real_vars = old_get_config_vars(*args) + if sys.platform == 'win32': + lib_dir = os.path.join(sys.real_prefix, "libs") + if isinstance(real_vars, dict) and 'LIBDIR' not in real_vars: + real_vars['LIBDIR'] = lib_dir # asked for all + elif isinstance(real_vars, list) and 'LIBDIR' in args: + real_vars = real_vars + [lib_dir] # asked for list + return real_vars +sysconfig_get_config_vars.__doc__ = old_get_config_vars.__doc__ +sysconfig.get_config_vars = sysconfig_get_config_vars diff --git a/test/lib/python2.6/distutils/distutils.cfg b/test/lib/python2.6/distutils/distutils.cfg new file mode 100644 index 000000000..1af230ec9 --- /dev/null +++ b/test/lib/python2.6/distutils/distutils.cfg @@ -0,0 +1,6 @@ +# This is a config file local to this virtualenv installation +# You may include options that will be used by all distutils commands, +# and by easy_install. For instance: +# +# [easy_install] +# find_links = http://mylocalsite diff --git a/test/lib/python2.6/encodings b/test/lib/python2.6/encodings new file mode 120000 index 000000000..89f28f824 --- /dev/null +++ b/test/lib/python2.6/encodings @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/encodings \ No newline at end of file diff --git a/test/lib/python2.6/fnmatch.py b/test/lib/python2.6/fnmatch.py new file mode 120000 index 000000000..cce0594f2 --- /dev/null +++ b/test/lib/python2.6/fnmatch.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/fnmatch.py \ No newline at end of file diff --git a/test/lib/python2.6/genericpath.py b/test/lib/python2.6/genericpath.py new file mode 120000 index 000000000..b14e1bc26 --- /dev/null +++ b/test/lib/python2.6/genericpath.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/genericpath.py \ No newline at end of file diff --git a/test/lib/python2.6/lib-dynload b/test/lib/python2.6/lib-dynload new file mode 120000 index 000000000..4644b7072 --- /dev/null +++ b/test/lib/python2.6/lib-dynload @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/lib-dynload \ No newline at end of file diff --git a/test/lib/python2.6/linecache.py b/test/lib/python2.6/linecache.py new file mode 120000 index 000000000..783624da8 --- /dev/null +++ b/test/lib/python2.6/linecache.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/linecache.py \ No newline at end of file diff --git a/test/lib/python2.6/locale.py b/test/lib/python2.6/locale.py new file mode 120000 index 000000000..4e674c7b6 --- /dev/null +++ b/test/lib/python2.6/locale.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/locale.py \ No newline at end of file diff --git a/test/lib/python2.6/ntpath.py b/test/lib/python2.6/ntpath.py new file mode 120000 index 000000000..9b6b40f48 --- /dev/null +++ b/test/lib/python2.6/ntpath.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/ntpath.py \ No newline at end of file diff --git a/test/lib/python2.6/orig-prefix.txt b/test/lib/python2.6/orig-prefix.txt new file mode 100644 index 000000000..535eb0f0e --- /dev/null +++ b/test/lib/python2.6/orig-prefix.txt @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6 \ No newline at end of file diff --git a/test/lib/python2.6/os.py b/test/lib/python2.6/os.py new file mode 120000 index 000000000..92e6e9a7c --- /dev/null +++ b/test/lib/python2.6/os.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/os.py \ No newline at end of file diff --git a/test/lib/python2.6/posixpath.py b/test/lib/python2.6/posixpath.py new file mode 120000 index 000000000..c095d16a1 --- /dev/null +++ b/test/lib/python2.6/posixpath.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/posixpath.py \ No newline at end of file diff --git a/test/lib/python2.6/re.py b/test/lib/python2.6/re.py new file mode 120000 index 000000000..b4710c5f7 --- /dev/null +++ b/test/lib/python2.6/re.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/re.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/easy-install.pth b/test/lib/python2.6/site-packages/easy-install.pth new file mode 100644 index 000000000..7a6ae2b6d --- /dev/null +++ b/test/lib/python2.6/site-packages/easy-install.pth @@ -0,0 +1,4 @@ +import sys; sys.__plen = len(sys.path) +./setuptools-0.6c11-py2.6.egg +./pip-0.8.1-py2.6.egg +import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO new file mode 100644 index 000000000..29c30cb8c --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO @@ -0,0 +1,348 @@ +Metadata-Version: 1.0 +Name: pip +Version: 0.8.1 +Summary: pip installs packages. Python packages. An easy_install replacement +Home-page: http://pip.openplans.org +Author: Ian Bicking +Author-email: python-virtualenv@groups.google.com +License: MIT +Description: The main website for pip is `pip.openplans.org + `_. You can also install + the `in-development version `_ + of pip with ``easy_install pip==dev``. + + + Introduction + ------------ + + pip installs packages. Python packages. + + If you use `virtualenv `__ -- a tool + for installing libraries in a local and isolated manner -- you'll + automatically get a copy of pip. Free bonus! + + Once you have pip, you can use it like this:: + + $ pip install SomePackage + + SomePackage is some package you'll find on `PyPI + `_. This installs the package and all + its dependencies. + + pip does other stuff too, with packages, but install is the biggest + one. You can ``pip uninstall`` too. + + You can also install from a URL (that points to a tar or zip file), + install from some version control system (use URLs like + ``hg+http://domain/repo`` -- or prefix ``git+``, ``svn+`` etc). pip + knows a bunch of stuff about revisions and stuff, so if you need to do + things like install a very specific revision from a repository pip can + do that too. + + If you've ever used ``python setup.py develop``, you can do something + like that with ``pip install -e ./`` -- this works with packages that + use ``distutils`` too (usually this only works with Setuptools + projects). + + You can use ``pip install --upgrade SomePackage`` to upgrade to a + newer version, or ``pip install SomePackage==1.0.4`` to install a very + specific version. + + Pip Compared To easy_install + ---------------------------- + + pip is a replacement for `easy_install + `_. It uses mostly the + same techniques for finding packages, so packages that were made + easy_installable should be pip-installable as well. + + pip is meant to improve on easy_install. Some of the improvements: + + * All packages are downloaded before installation. Partially-completed + installation doesn't occur as a result. + + * Care is taken to present useful output on the console. + + * The reasons for actions are kept track of. For instance, if a package is + being installed, pip keeps track of why that package was required. + + * Error messages should be useful. + + * The code is relatively concise and cohesive, making it easier to use + programmatically. + + * Packages don't have to be installed as egg archives, they can be installed + flat (while keeping the egg metadata). + + * Native support for other version control systems (Git, Mercurial and Bazaar) + + * Uninstallation of packages. + + * Simple to define fixed sets of requirements and reliably reproduce a + set of packages. + + pip doesn't do everything that easy_install does. Specifically: + + * It cannot install from eggs. It only installs from source. (In the + future it would be good if it could install binaries from Windows ``.exe`` + or ``.msi`` -- binary install on other platforms is not a priority.) + + * It doesn't understand Setuptools extras (like ``package[test]``). This should + be added eventually. + + * It is incompatible with some packages that extensively customize distutils + or setuptools in their ``setup.py`` files. + + pip is complementary with `virtualenv + `__, and it is encouraged that you use + virtualenv to isolate your installation. + + Community + --------- + + The homepage for pip is temporarily located `on PyPI + `_ -- a more proper homepage will + follow. Bugs can go on the `pip issue tracker + `_. Discussion should happen on the + `virtualenv email group + `_. + + Uninstall + --------- + + pip is able to uninstall most installed packages with ``pip uninstall + package-name``. + + Known exceptions include pure-distutils packages installed with + ``python setup.py install`` (such packages leave behind no metadata allowing + determination of what files were installed), and script wrappers installed + by develop-installs (``python setup.py develop``). + + pip also performs an automatic uninstall of an old version of a package + before upgrading to a newer version, so outdated files (and egg-info data) + from conflicting versions aren't left hanging around to cause trouble. The + old version of the package is automatically restored if the new version + fails to download or install. + + .. _`requirements file`: + + Requirements Files + ------------------ + + When installing software, and Python packages in particular, it's common that + you get a lot of libraries installed. You just did ``easy_install MyPackage`` + and you get a dozen packages. Each of these packages has its own version. + + Maybe you ran that installation and it works. Great! Will it keep working? + Did you have to provide special options to get it to find everything? Did you + have to install a bunch of other optional pieces? Most of all, will you be able + to do it again? Requirements files give you a way to create an *environment*: + a *set* of packages that work together. + + If you've ever tried to setup an application on a new system, or with slightly + updated pieces, and had it fail, pip requirements are for you. If you + haven't had this problem then you will eventually, so pip requirements are + for you too -- requirements make explicit, repeatable installation of packages. + + So what are requirements files? They are very simple: lists of packages to + install. Instead of running something like ``pip MyApp`` and getting + whatever libraries come along, you can create a requirements file something like:: + + MyApp + Framework==0.9.4 + Library>=0.2 + + Then, regardless of what MyApp lists in ``setup.py``, you'll get a + specific version of Framework (0.9.4) and at least the 0.2 version of + Library. (You might think you could list these specific versions in + MyApp's ``setup.py`` -- but if you do that you'll have to edit MyApp + if you want to try a new version of Framework, or release a new + version of MyApp if you determine that Library 0.3 doesn't work with + your application.) You can also add optional libraries and support + tools that MyApp doesn't strictly require, giving people a set of + recommended libraries. + + You can also include "editable" packages -- packages that are checked out from + Subversion, Git, Mercurial and Bazaar. These are just like using the ``-e`` + option to pip. They look like:: + + -e svn+http://myrepo/svn/MyApp#egg=MyApp + + You have to start the URL with ``svn+`` (``git+``, ``hg+`` or ``bzr+``), and + you have to include ``#egg=Package`` so pip knows what to expect at that URL. + You can also include ``@rev`` in the URL, e.g., ``@275`` to check out + revision 275. + + Requirement files are mostly *flat*. Maybe ``MyApp`` requires + ``Framework``, and ``Framework`` requires ``Library``. I encourage + you to still list all these in a single requirement file; it is the + nature of Python programs that there are implicit bindings *directly* + between MyApp and Library. For instance, Framework might expose one + of Library's objects, and so if Library is updated it might directly + break MyApp. If that happens you can update the requirements file to + force an earlier version of Library, and you can do that without + having to re-release MyApp at all. + + Read the `requirements file format `_ to + learn about other features. + + Freezing Requirements + --------------------- + + So you have a working set of packages, and you want to be able to install them + elsewhere. `Requirements files`_ let you install exact versions, but it won't + tell you what all the exact versions are. + + To create a new requirements file from a known working environment, use:: + + $ pip freeze > stable-req.txt + + This will write a listing of *all* installed libraries to ``stable-req.txt`` + with exact versions for every library. You may want to edit the file down after + generating (e.g., to eliminate unnecessary libraries), but it'll give you a + stable starting point for constructing your requirements file. + + You can also give it an existing requirements file, and it will use that as a + sort of template for the new file. So if you do:: + + $ pip freeze -r devel-req.txt > stable-req.txt + + it will keep the packages listed in ``devel-req.txt`` in order and preserve + comments. + + Bundles + ------- + + Another way to distribute a set of libraries is a bundle format (specific to + pip). This format is not stable at this time (there simply hasn't been + any feedback, nor a great deal of thought). A bundle file contains all the + source for your package, and you can have pip install them all together. + Once you have the bundle file further network access won't be necessary. To + build a bundle file, do:: + + $ pip bundle MyApp.pybundle MyApp + + (Using a `requirements file`_ would be wise.) Then someone else can get the + file ``MyApp.pybundle`` and run:: + + $ pip install MyApp.pybundle + + This is *not* a binary format. This only packages source. If you have binary + packages, then the person who installs the files will have to have a compiler, + any necessary headers installed, etc. Binary packages are hard, this is + relatively easy. + + Using pip with virtualenv + ------------------------- + + pip is most nutritious when used with `virtualenv + `__. One of the reasons pip + doesn't install "multi-version" eggs is that virtualenv removes much of the need + for it. Because pip is installed by virtualenv, just use + ``path/to/my/environment/bin/pip`` to install things into that + specific environment. + + To tell pip to only run if there is a virtualenv currently activated, + and to bail if not, use:: + + export PIP_REQUIRE_VIRTUALENV=true + + To tell pip to automatically use the currently active virtualenv:: + + export PIP_RESPECT_VIRTUALENV=true + + Providing an environment with ``-E`` will be ignored. + + Using pip with virtualenvwrapper + --------------------------------- + + If you are using `virtualenvwrapper + `_, you might + want pip to automatically create its virtualenvs in your + ``$WORKON_HOME``. + + You can tell pip to do so by defining ``PIP_VIRTUALENV_BASE`` in your + environment and setting it to the same value as that of + ``$WORKON_HOME``. + + Do so by adding the line:: + + export PIP_VIRTUALENV_BASE=$WORKON_HOME + + in your .bashrc under the line starting with ``export WORKON_HOME``. + + Using pip with buildout + ----------------------- + + If you are using `zc.buildout + `_ you should look at + `gp.recipe.pip `_ as an + option to use pip and virtualenv in your buildouts. + + Command line completion + ----------------------- + + pip comes with support for command line completion in bash and zsh and + allows you tab complete commands and options. To enable it you simply + need copy the required shell script to the your shell startup file + (e.g. ``.profile`` or ``.zprofile``) by running the special ``completion`` + command, e.g. for bash:: + + $ pip completion --bash >> ~/.profile + + And for zsh:: + + $ pip completion --zsh >> ~/.zprofile + + Alternatively, you can use the result of the ``completion`` command + directly with the eval function of you shell, e.g. by adding:: + + eval "`pip completion --bash`" + + to your startup file. + + Searching for packages + ---------------------- + + pip can search the `Python Package Index `_ (PyPI) + for packages using the ``pip search`` command. To search, run:: + + $ pip search "query" + + The query will be used to search the names and summaries of all packages + indexed. + + pip searches http://pypi.python.org/pypi by default but alternative indexes + can be searched by using the ``--index`` flag. + + Mirror support + -------------- + + The `PyPI mirroring infrastructure `_ as + described in `PEP 381 `_ can be + used by passing the ``--use-mirrors`` option to the install command. + Alternatively, you can use the other ways to configure pip, e.g.:: + + $ export PIP_USE_MIRRORS=true + + If enabled, pip will automatically query the DNS entry of the mirror index URL + to find the list of mirrors to use. In case you want to override this list, + please use the ``--mirrors`` option of the install command, or add to your pip + configuration file:: + + [install] + use-mirrors = true + mirrors = + http://d.pypi.python.org + http://b.pypi.python.org + +Keywords: easy_install distutils setuptools egg virtualenv +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Topic :: Software Development :: Build Tools +Classifier: Programming Language :: Python :: 2.4 +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt new file mode 100644 index 000000000..3a068547e --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt @@ -0,0 +1,57 @@ +MANIFEST.in +setup.cfg +setup.py +docs/branches.txt +docs/ci-server-step-by-step.txt +docs/configuration.txt +docs/how-to-contribute.txt +docs/index.txt +docs/license.txt +docs/news.txt +docs/requirement-format.txt +docs/running-tests.txt +docs/_build/branches.html +docs/_build/ci-server-step-by-step.html +docs/_build/configuration.html +docs/_build/how-to-contribute.html +docs/_build/index.html +docs/_build/license.html +docs/_build/news.html +docs/_build/requirement-format.html +docs/_build/running-tests.html +docs/_build/search.html +pip/__init__.py +pip/_pkgutil.py +pip/backwardcompat.py +pip/basecommand.py +pip/baseparser.py +pip/download.py +pip/exceptions.py +pip/index.py +pip/locations.py +pip/log.py +pip/req.py +pip/runner.py +pip/util.py +pip/venv.py +pip.egg-info/PKG-INFO +pip.egg-info/SOURCES.txt +pip.egg-info/dependency_links.txt +pip.egg-info/entry_points.txt +pip.egg-info/not-zip-safe +pip.egg-info/top_level.txt +pip/commands/__init__.py +pip/commands/bundle.py +pip/commands/completion.py +pip/commands/freeze.py +pip/commands/help.py +pip/commands/install.py +pip/commands/search.py +pip/commands/uninstall.py +pip/commands/unzip.py +pip/commands/zip.py +pip/vcs/__init__.py +pip/vcs/bazaar.py +pip/vcs/git.py +pip/vcs/mercurial.py +pip/vcs/subversion.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt new file mode 100644 index 000000000..2b0afba73 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt @@ -0,0 +1,4 @@ +[console_scripts] +pip = pip:main +pip-2.6 = pip:main + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe @@ -0,0 +1 @@ + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt new file mode 100644 index 000000000..a1b589e38 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt @@ -0,0 +1 @@ +pip diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py new file mode 100644 index 000000000..c5de5c9a8 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +import os +import optparse + +import subprocess +import sys +import re +import difflib + +from pip.basecommand import command_dict, load_command, load_all_commands, command_names +from pip.baseparser import parser +from pip.exceptions import InstallationError +from pip.log import logger +from pip.util import get_installed_distributions +from pip.backwardcompat import walk_packages + + +def autocomplete(): + """Command and option completion for the main option parser (and options) + and its subcommands (and options). + + Enable by sourcing one of the completion shell scripts (bash or zsh). + """ + # Don't complete if user hasn't sourced bash_completion file. + if 'PIP_AUTO_COMPLETE' not in os.environ: + return + cwords = os.environ['COMP_WORDS'].split()[1:] + cword = int(os.environ['COMP_CWORD']) + try: + current = cwords[cword-1] + except IndexError: + current = '' + load_all_commands() + subcommands = [cmd for cmd, cls in command_dict.items() if not cls.hidden] + options = [] + # subcommand + try: + subcommand_name = [w for w in cwords if w in subcommands][0] + except IndexError: + subcommand_name = None + # subcommand options + if subcommand_name: + # special case: 'help' subcommand has no options + if subcommand_name == 'help': + sys.exit(1) + # special case: list locally installed dists for uninstall command + if subcommand_name == 'uninstall' and not current.startswith('-'): + installed = [] + lc = current.lower() + for dist in get_installed_distributions(local_only=True): + if dist.key.startswith(lc) and dist.key not in cwords[1:]: + installed.append(dist.key) + # if there are no dists installed, fall back to option completion + if installed: + for dist in installed: + print dist + sys.exit(1) + subcommand = command_dict.get(subcommand_name) + options += [(opt.get_opt_string(), opt.nargs) + for opt in subcommand.parser.option_list + if opt.help != optparse.SUPPRESS_HELP] + # filter out previously specified options from available options + prev_opts = [x.split('=')[0] for x in cwords[1:cword-1]] + options = filter(lambda (x, v): x not in prev_opts, options) + # filter options by current input + options = [(k, v) for k, v in options if k.startswith(current)] + for option in options: + opt_label = option[0] + # append '=' to options which require args + if option[1]: + opt_label += '=' + print opt_label + else: + # show options of main parser only when necessary + if current.startswith('-') or current.startswith('--'): + subcommands += [opt.get_opt_string() + for opt in parser.option_list + if opt.help != optparse.SUPPRESS_HELP] + print ' '.join(filter(lambda x: x.startswith(current), subcommands)) + sys.exit(1) + + +def version_control(): + # Import all the version control support modules: + from pip import vcs + for importer, modname, ispkg in \ + walk_packages(path=vcs.__path__, prefix=vcs.__name__+'.'): + __import__(modname) + + +def main(initial_args=None): + if initial_args is None: + initial_args = sys.argv[1:] + autocomplete() + version_control() + options, args = parser.parse_args(initial_args) + if options.help and not args: + args = ['help'] + if not args: + parser.error('You must give a command (use "pip help" to see a list of commands)') + command = args[0].lower() + load_command(command) + if command not in command_dict: + close_commands = difflib.get_close_matches(command, command_names()) + if close_commands: + guess = close_commands[0] + if args[1:]: + guess = "%s %s" % (guess, " ".join(args[1:])) + else: + guess = 'install %s' % command + error_dict = {'arg': command, 'guess': guess, + 'script': os.path.basename(sys.argv[0])} + parser.error('No command by the name %(script)s %(arg)s\n ' + '(maybe you meant "%(script)s %(guess)s")' % error_dict) + command = command_dict[command] + return command.main(initial_args, args[1:], options) + + +############################################################ +## Writing freeze files + + +class FrozenRequirement(object): + + def __init__(self, name, req, editable, comments=()): + self.name = name + self.req = req + self.editable = editable + self.comments = comments + + _rev_re = re.compile(r'-r(\d+)$') + _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') + + @classmethod + def from_dist(cls, dist, dependency_links, find_tags=False): + location = os.path.normcase(os.path.abspath(dist.location)) + comments = [] + from pip.vcs import vcs, get_src_requirement + if vcs.get_backend_name(location): + editable = True + req = get_src_requirement(dist, location, find_tags) + if req is None: + logger.warn('Could not determine repository location of %s' % location) + comments.append('## !! Could not determine repository location') + req = dist.as_requirement() + editable = False + else: + editable = False + req = dist.as_requirement() + specs = req.specs + assert len(specs) == 1 and specs[0][0] == '==' + version = specs[0][1] + ver_match = cls._rev_re.search(version) + date_match = cls._date_re.search(version) + if ver_match or date_match: + svn_backend = vcs.get_backend('svn') + if svn_backend: + svn_location = svn_backend( + ).get_location(dist, dependency_links) + if not svn_location: + logger.warn( + 'Warning: cannot find svn location for %s' % req) + comments.append('## FIXME: could not find svn URL in dependency_links for this package:') + else: + comments.append('# Installing as editable to satisfy requirement %s:' % req) + if ver_match: + rev = ver_match.group(1) + else: + rev = '{%s}' % date_match.group(1) + editable = True + req = '%s@%s#egg=%s' % (svn_location, rev, cls.egg_name(dist)) + return cls(dist.project_name, req, editable, comments) + + @staticmethod + def egg_name(dist): + name = dist.egg_name() + match = re.search(r'-py\d\.\d$', name) + if match: + name = name[:match.start()] + return name + + def __str__(self): + req = self.req + if self.editable: + req = '-e %s' % req + return '\n'.join(list(self.comments)+[str(req)])+'\n' + +############################################################ +## Requirement files + + +def call_subprocess(cmd, show_stdout=True, + filter_stdout=None, cwd=None, + raise_on_returncode=True, + command_level=logger.DEBUG, command_desc=None, + extra_environ=None): + if command_desc is None: + cmd_parts = [] + for part in cmd: + if ' ' in part or '\n' in part or '"' in part or "'" in part: + part = '"%s"' % part.replace('"', '\\"') + cmd_parts.append(part) + command_desc = ' '.join(cmd_parts) + if show_stdout: + stdout = None + else: + stdout = subprocess.PIPE + logger.log(command_level, "Running command %s" % command_desc) + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + try: + proc = subprocess.Popen( + cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, + cwd=cwd, env=env) + except Exception, e: + logger.fatal( + "Error %s while executing command %s" % (e, command_desc)) + raise + all_output = [] + if stdout is not None: + stdout = proc.stdout + while 1: + line = stdout.readline() + if not line: + break + line = line.rstrip() + all_output.append(line + '\n') + if filter_stdout: + level = filter_stdout(line) + if isinstance(level, tuple): + level, line = level + logger.log(level, line) + if not logger.stdout_level_matches(level): + logger.show_progress() + else: + logger.info(line) + else: + returned_stdout, returned_stderr = proc.communicate() + all_output = [returned_stdout or ''] + proc.wait() + if proc.returncode: + if raise_on_returncode: + if all_output: + logger.notify('Complete output from command %s:' % command_desc) + logger.notify('\n'.join(all_output) + '\n----------------------------------------') + raise InstallationError( + "Command %s failed with error code %s" + % (command_desc, proc.returncode)) + else: + logger.warn( + "Command %s had error code %s" + % (command_desc, proc.returncode)) + if stdout is not None: + return ''.join(all_output) + + +if __name__ == '__main__': + exit = main() + if exit: + sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py new file mode 100644 index 000000000..f8fb8aa61 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py @@ -0,0 +1,589 @@ +"""Utilities to support packages.""" + +# NOTE: This module must remain compatible with Python 2.3, as it is shared +# by setuptools for distribution with Python 2.3 and up. + +import os +import sys +import imp +import os.path +from types import ModuleType + +__all__ = [ + 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'walk_packages', 'iter_modules', + 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', +] + + +def read_code(stream): + # This helper is needed in order for the PEP 302 emulation to + # correctly handle compiled files + import marshal + + magic = stream.read(4) + if magic != imp.get_magic(): + return None + + stream.read(4) # Skip timestamp + return marshal.load(stream) + + +def simplegeneric(func): + """Make a trivial single-dispatch generic function""" + registry = {} + + def wrapper(*args, **kw): + ob = args[0] + try: + cls = ob.__class__ + except AttributeError: + cls = type(ob) + try: + mro = cls.__mro__ + except AttributeError: + try: + + class cls(cls, object): + pass + + mro = cls.__mro__[1:] + except TypeError: + mro = object, # must be an ExtensionClass or some such :( + for t in mro: + if t in registry: + return registry[t](*args, **kw) + else: + return func(*args, **kw) + try: + wrapper.__name__ = func.__name__ + except (TypeError, AttributeError): + pass # Python 2.3 doesn't allow functions to be renamed + + def register(typ, func=None): + if func is None: + return lambda f: register(typ, f) + registry[typ] = func + return func + + wrapper.__dict__ = func.__dict__ + wrapper.__doc__ = func.__doc__ + wrapper.register = register + return wrapper + + +def walk_packages(path=None, prefix='', onerror=None): + """Yields (module_loader, name, ispkg) for all modules recursively + on path, or, if path is None, all accessible modules. + + 'path' should be either None or a list of paths to look for + modules in. + + 'prefix' is a string to output on the front of every module name + on output. + + Note that this function must import all *packages* (NOT all + modules!) on the given path, in order to access the __path__ + attribute to find submodules. + + 'onerror' is a function which gets called with one argument (the + name of the package which was being imported) if any exception + occurs while trying to import a package. If no onerror function is + supplied, ImportErrors are caught and ignored, while all other + exceptions are propagated, terminating the search. + + Examples: + + # list all modules python can access + walk_packages() + + # list all submodules of ctypes + walk_packages(ctypes.__path__, ctypes.__name__+'.') + """ + + def seen(p, m={}): + if p in m: + return True + m[p] = True + + for importer, name, ispkg in iter_modules(path, prefix): + yield importer, name, ispkg + + if ispkg: + try: + __import__(name) + except ImportError: + if onerror is not None: + onerror(name) + except Exception: + if onerror is not None: + onerror(name) + else: + raise + else: + path = getattr(sys.modules[name], '__path__', None) or [] + + # don't traverse path items we've seen before + path = [p for p in path if not seen(p)] + + for item in walk_packages(path, name+'.', onerror): + yield item + + +def iter_modules(path=None, prefix=''): + """Yields (module_loader, name, ispkg) for all submodules on path, + or, if path is None, all top-level modules on sys.path. + + 'path' should be either None or a list of paths to look for + modules in. + + 'prefix' is a string to output on the front of every module name + on output. + """ + + if path is None: + importers = iter_importers() + else: + importers = map(get_importer, path) + + yielded = {} + for i in importers: + for name, ispkg in iter_importer_modules(i, prefix): + if name not in yielded: + yielded[name] = 1 + yield i, name, ispkg + + +#@simplegeneric +def iter_importer_modules(importer, prefix=''): + if not hasattr(importer, 'iter_modules'): + return [] + return importer.iter_modules(prefix) + +iter_importer_modules = simplegeneric(iter_importer_modules) + + +class ImpImporter: + """PEP 302 Importer that wraps Python's "classic" import algorithm + + ImpImporter(dirname) produces a PEP 302 importer that searches that + directory. ImpImporter(None) produces a PEP 302 importer that searches + the current sys.path, plus any modules that are frozen or built-in. + + Note that ImpImporter does not currently support being used by placement + on sys.meta_path. + """ + + def __init__(self, path=None): + self.path = path + + def find_module(self, fullname, path=None): + # Note: we ignore 'path' argument since it is only used via meta_path + subname = fullname.split(".")[-1] + if subname != fullname and self.path is None: + return None + if self.path is None: + path = None + else: + path = [os.path.realpath(self.path)] + try: + file, filename, etc = imp.find_module(subname, path) + except ImportError: + return None + return ImpLoader(fullname, file, filename, etc) + + def iter_modules(self, prefix=''): + if self.path is None or not os.path.isdir(self.path): + return + + yielded = {} + import inspect + + filenames = os.listdir(self.path) + filenames.sort() # handle packages before same-named modules + + for fn in filenames: + modname = inspect.getmodulename(fn) + if modname=='__init__' or modname in yielded: + continue + + path = os.path.join(self.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + for fn in os.listdir(path): + subname = inspect.getmodulename(fn) + if subname=='__init__': + ispkg = True + break + else: + continue # not a package + + if modname and '.' not in modname: + yielded[modname] = 1 + yield prefix + modname, ispkg + + +class ImpLoader: + """PEP 302 Loader that wraps Python's "classic" import algorithm + """ + code = source = None + + def __init__(self, fullname, file, filename, etc): + self.file = file + self.filename = filename + self.fullname = fullname + self.etc = etc + + def load_module(self, fullname): + self._reopen() + try: + mod = imp.load_module(fullname, self.file, self.filename, self.etc) + finally: + if self.file: + self.file.close() + # Note: we don't set __loader__ because we want the module to look + # normal; i.e. this is just a wrapper for standard import machinery + return mod + + def get_data(self, pathname): + return open(pathname, "rb").read() + + def _reopen(self): + if self.file and self.file.closed: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + self.file = open(self.filename, 'rU') + elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): + self.file = open(self.filename, 'rb') + + def _fix_name(self, fullname): + if fullname is None: + fullname = self.fullname + elif fullname != self.fullname: + raise ImportError("Loader for module %s cannot handle " + "module %s" % (self.fullname, fullname)) + return fullname + + def is_package(self, fullname): + fullname = self._fix_name(fullname) + return self.etc[2]==imp.PKG_DIRECTORY + + def get_code(self, fullname=None): + fullname = self._fix_name(fullname) + if self.code is None: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + source = self.get_source(fullname) + self.code = compile(source, self.filename, 'exec') + elif mod_type==imp.PY_COMPILED: + self._reopen() + try: + self.code = read_code(self.file) + finally: + self.file.close() + elif mod_type==imp.PKG_DIRECTORY: + self.code = self._get_delegate().get_code() + return self.code + + def get_source(self, fullname=None): + fullname = self._fix_name(fullname) + if self.source is None: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + self._reopen() + try: + self.source = self.file.read() + finally: + self.file.close() + elif mod_type==imp.PY_COMPILED: + if os.path.exists(self.filename[:-1]): + f = open(self.filename[:-1], 'rU') + self.source = f.read() + f.close() + elif mod_type==imp.PKG_DIRECTORY: + self.source = self._get_delegate().get_source() + return self.source + + def _get_delegate(self): + return ImpImporter(self.filename).find_module('__init__') + + def get_filename(self, fullname=None): + fullname = self._fix_name(fullname) + mod_type = self.etc[2] + if self.etc[2]==imp.PKG_DIRECTORY: + return self._get_delegate().get_filename() + elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): + return self.filename + return None + + +try: + import zipimport + from zipimport import zipimporter + + def iter_zipimport_modules(importer, prefix=''): + dirlist = zipimport._zip_directory_cache[importer.archive].keys() + dirlist.sort() + _prefix = importer.prefix + plen = len(_prefix) + yielded = {} + import inspect + for fn in dirlist: + if not fn.startswith(_prefix): + continue + + fn = fn[plen:].split(os.sep) + + if len(fn)==2 and fn[1].startswith('__init__.py'): + if fn[0] not in yielded: + yielded[fn[0]] = 1 + yield fn[0], True + + if len(fn)!=1: + continue + + modname = inspect.getmodulename(fn[0]) + if modname=='__init__': + continue + + if modname and '.' not in modname and modname not in yielded: + yielded[modname] = 1 + yield prefix + modname, False + + iter_importer_modules.register(zipimporter, iter_zipimport_modules) + +except ImportError: + pass + + +def get_importer(path_item): + """Retrieve a PEP 302 importer for the given path item + + The returned importer is cached in sys.path_importer_cache + if it was newly created by a path hook. + + If there is no importer, a wrapper around the basic import + machinery is returned. This wrapper is never inserted into + the importer cache (None is inserted instead). + + The cache (or part of it) can be cleared manually if a + rescan of sys.path_hooks is necessary. + """ + try: + importer = sys.path_importer_cache[path_item] + except KeyError: + for path_hook in sys.path_hooks: + try: + importer = path_hook(path_item) + break + except ImportError: + pass + else: + importer = None + sys.path_importer_cache.setdefault(path_item, importer) + + if importer is None: + try: + importer = ImpImporter(path_item) + except ImportError: + importer = None + return importer + + +def iter_importers(fullname=""): + """Yield PEP 302 importers for the given module name + + If fullname contains a '.', the importers will be for the package + containing fullname, otherwise they will be importers for sys.meta_path, + sys.path, and Python's "classic" import machinery, in that order. If + the named module is in a package, that package is imported as a side + effect of invoking this function. + + Non PEP 302 mechanisms (e.g. the Windows registry) used by the + standard import machinery to find files in alternative locations + are partially supported, but are searched AFTER sys.path. Normally, + these locations are searched BEFORE sys.path, preventing sys.path + entries from shadowing them. + + For this to cause a visible difference in behaviour, there must + be a module or package name that is accessible via both sys.path + and one of the non PEP 302 file system mechanisms. In this case, + the emulation will find the former version, while the builtin + import mechanism will find the latter. + + Items of the following types can be affected by this discrepancy: + imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY + """ + if fullname.startswith('.'): + raise ImportError("Relative module names not supported") + if '.' in fullname: + # Get the containing package's __path__ + pkg = '.'.join(fullname.split('.')[:-1]) + if pkg not in sys.modules: + __import__(pkg) + path = getattr(sys.modules[pkg], '__path__', None) or [] + else: + for importer in sys.meta_path: + yield importer + path = sys.path + for item in path: + yield get_importer(item) + if '.' not in fullname: + yield ImpImporter() + + +def get_loader(module_or_name): + """Get a PEP 302 "loader" object for module_or_name + + If the module or package is accessible via the normal import + mechanism, a wrapper around the relevant part of that machinery + is returned. Returns None if the module cannot be found or imported. + If the named module is not already imported, its containing package + (if any) is imported, in order to establish the package __path__. + + This function uses iter_importers(), and is thus subject to the same + limitations regarding platform-specific special import locations such + as the Windows registry. + """ + if module_or_name in sys.modules: + module_or_name = sys.modules[module_or_name] + if isinstance(module_or_name, ModuleType): + module = module_or_name + loader = getattr(module, '__loader__', None) + if loader is not None: + return loader + fullname = module.__name__ + else: + fullname = module_or_name + return find_loader(fullname) + + +def find_loader(fullname): + """Find a PEP 302 "loader" object for fullname + + If fullname contains dots, path must be the containing package's __path__. + Returns None if the module cannot be found or imported. This function uses + iter_importers(), and is thus subject to the same limitations regarding + platform-specific special import locations such as the Windows registry. + """ + for importer in iter_importers(fullname): + loader = importer.find_module(fullname) + if loader is not None: + return loader + + return None + + +def extend_path(path, name): + """Extend a package's path. + + Intended use is to place the following code in a package's __init__.py: + + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) + + This will add to the package's __path__ all subdirectories of + directories on sys.path named after the package. This is useful + if one wants to distribute different parts of a single logical + package as multiple directories. + + It also looks for *.pkg files beginning where * matches the name + argument. This feature is similar to *.pth files (see site.py), + except that it doesn't special-case lines starting with 'import'. + A *.pkg file is trusted at face value: apart from checking for + duplicates, all entries found in a *.pkg file are added to the + path, regardless of whether they are exist the filesystem. (This + is a feature.) + + If the input path is not a list (as is the case for frozen + packages) it is returned unchanged. The input path is not + modified; an extended copy is returned. Items are only appended + to the copy at the end. + + It is assumed that sys.path is a sequence. Items of sys.path that + are not (unicode or 8-bit) strings referring to existing + directories are ignored. Unicode items of sys.path that cause + errors when used as filenames may cause this function to raise an + exception (in line with os.path.isdir() behavior). + """ + + if not isinstance(path, list): + # This could happen e.g. when this is called from inside a + # frozen package. Return the path unchanged in that case. + return path + + pname = os.path.join(*name.split('.')) # Reconstitute as relative path + # Just in case os.extsep != '.' + sname = os.extsep.join(name.split('.')) + sname_pkg = sname + os.extsep + "pkg" + init_py = "__init__" + os.extsep + "py" + + path = path[:] # Start with a copy of the existing path + + for dir in sys.path: + if not isinstance(dir, basestring) or not os.path.isdir(dir): + continue + subdir = os.path.join(dir, pname) + # XXX This may still add duplicate entries to path on + # case-insensitive filesystems + initfile = os.path.join(subdir, init_py) + if subdir not in path and os.path.isfile(initfile): + path.append(subdir) + # XXX Is this the right thing for subpackages like zope.app? + # It looks for a file named "zope.app.pkg" + pkgfile = os.path.join(dir, sname_pkg) + if os.path.isfile(pkgfile): + try: + f = open(pkgfile) + except IOError, msg: + sys.stderr.write("Can't open %s: %s\n" % + (pkgfile, msg)) + else: + for line in f: + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + path.append(line) # Don't check for existence! + f.close() + + return path + + +def get_data(package, resource): + """Get a resource from a package. + + This is a wrapper round the PEP 302 loader get_data API. The package + argument should be the name of a package, in standard module format + (foo.bar). The resource argument should be in the form of a relative + filename, using '/' as the path separator. The parent directory name '..' + is not allowed, and nor is a rooted name (starting with a '/'). + + The function returns a binary string, which is the contents of the + specified resource. + + For packages located in the filesystem, which have already been imported, + this is the rough equivalent of + + d = os.path.dirname(sys.modules[package].__file__) + data = open(os.path.join(d, resource), 'rb').read() + + If the package cannot be located or loaded, or it uses a PEP 302 loader + which does not support get_data(), then None is returned. + """ + + loader = get_loader(package) + if loader is None or not hasattr(loader, 'get_data'): + return None + mod = sys.modules.get(package) or loader.load_module(package) + if mod is None or not hasattr(mod, '__file__'): + return None + + # Modify the resource name to be compatible with the loader.get_data + # signature - an os.path format "filename" starting with the dirname of + # the package's __file__ + parts = resource.split('/') + parts.insert(0, os.path.dirname(mod.__file__)) + resource_name = os.path.join(*parts) + return loader.get_data(resource_name) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py new file mode 100644 index 000000000..e7c11f1d3 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py @@ -0,0 +1,55 @@ +"""Stuff that isn't in some old versions of Python""" + +import sys +import os +import shutil + +__all__ = ['any', 'WindowsError', 'md5', 'copytree'] + +try: + WindowsError = WindowsError +except NameError: + WindowsError = None +try: + from hashlib import md5 +except ImportError: + import md5 as md5_module + md5 = md5_module.new + +try: + from pkgutil import walk_packages +except ImportError: + # let's fall back as long as we can + from _pkgutil import walk_packages + +try: + any = any +except NameError: + + def any(seq): + for item in seq: + if item: + return True + return False + + +def copytree(src, dst): + if sys.version_info < (2, 5): + before_last_dir = os.path.dirname(dst) + if not os.path.exists(before_last_dir): + os.makedirs(before_last_dir) + shutil.copytree(src, dst) + shutil.copymode(src, dst) + else: + shutil.copytree(src, dst) + + +def product(*args, **kwds): + # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy + # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py new file mode 100644 index 000000000..f450e8393 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py @@ -0,0 +1,203 @@ +"""Base Command class, and related routines""" + +from cStringIO import StringIO +import getpass +import os +import socket +import sys +import traceback +import time +import urllib +import urllib2 + +from pip import commands +from pip.log import logger +from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpFormatter +from pip.download import urlopen +from pip.exceptions import BadCommand, InstallationError, UninstallationError +from pip.venv import restart_in_venv +from pip.backwardcompat import walk_packages + +__all__ = ['command_dict', 'Command', 'load_all_commands', + 'load_command', 'command_names'] + +command_dict = {} + +# for backwards compatibiliy +get_proxy = urlopen.get_proxy + + +class Command(object): + name = None + usage = None + hidden = False + + def __init__(self): + assert self.name + self.parser = ConfigOptionParser( + usage=self.usage, + prog='%s %s' % (sys.argv[0], self.name), + version=parser.version, + formatter=UpdatingDefaultsHelpFormatter(), + name=self.name) + for option in parser.option_list: + if not option.dest or option.dest == 'help': + # -h, --version, etc + continue + self.parser.add_option(option) + command_dict[self.name] = self + + def merge_options(self, initial_options, options): + # Make sure we have all global options carried over + for attr in ['log', 'venv', 'proxy', 'venv_base', 'require_venv', + 'respect_venv', 'log_explicit_levels', 'log_file', + 'timeout', 'default_vcs', 'skip_requirements_regex', + 'no_input']: + setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr)) + options.quiet += initial_options.quiet + options.verbose += initial_options.verbose + + def setup_logging(self): + pass + + def main(self, complete_args, args, initial_options): + options, args = self.parser.parse_args(args) + self.merge_options(initial_options, options) + + level = 1 # Notify + level += options.verbose + level -= options.quiet + level = logger.level_for_integer(4-level) + complete_log = [] + logger.consumers.extend( + [(level, sys.stdout), + (logger.DEBUG, complete_log.append)]) + if options.log_explicit_levels: + logger.explicit_levels = True + + self.setup_logging() + + if options.require_venv and not options.venv: + # If a venv is required check if it can really be found + if not os.environ.get('VIRTUAL_ENV'): + logger.fatal('Could not find an activated virtualenv (required).') + sys.exit(3) + # Automatically install in currently activated venv if required + options.respect_venv = True + + if args and args[-1] == '___VENV_RESTART___': + ## FIXME: We don't do anything this this value yet: + args = args[:-2] + options.venv = None + else: + # If given the option to respect the activated environment + # check if no venv is given as a command line parameter + if options.respect_venv and os.environ.get('VIRTUAL_ENV'): + if options.venv and os.path.exists(options.venv): + # Make sure command line venv and environmental are the same + if (os.path.realpath(os.path.expanduser(options.venv)) != + os.path.realpath(os.environ.get('VIRTUAL_ENV'))): + logger.fatal("Given virtualenv (%s) doesn't match " + "currently activated virtualenv (%s)." + % (options.venv, os.environ.get('VIRTUAL_ENV'))) + sys.exit(3) + else: + options.venv = os.environ.get('VIRTUAL_ENV') + logger.info('Using already activated environment %s' % options.venv) + if options.venv: + logger.info('Running in environment %s' % options.venv) + site_packages=False + if options.site_packages: + site_packages=True + restart_in_venv(options.venv, options.venv_base, site_packages, + complete_args) + # restart_in_venv should actually never return, but for clarity... + return + + ## FIXME: not sure if this sure come before or after venv restart + if options.log: + log_fp = open_logfile(options.log, 'a') + logger.consumers.append((logger.DEBUG, log_fp)) + else: + log_fp = None + + socket.setdefaulttimeout(options.timeout or None) + + urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) + + exit = 0 + try: + self.run(options, args) + except (InstallationError, UninstallationError), e: + logger.fatal(str(e)) + logger.info('Exception information:\n%s' % format_exc()) + exit = 1 + except BadCommand, e: + logger.fatal(str(e)) + logger.info('Exception information:\n%s' % format_exc()) + exit = 1 + except: + logger.fatal('Exception:\n%s' % format_exc()) + exit = 2 + + if log_fp is not None: + log_fp.close() + if exit: + log_fn = options.log_file + text = '\n'.join(complete_log) + logger.fatal('Storing complete log in %s' % log_fn) + log_fp = open_logfile(log_fn, 'w') + log_fp.write(text) + log_fp.close() + return exit + + + + +def format_exc(exc_info=None): + if exc_info is None: + exc_info = sys.exc_info() + out = StringIO() + traceback.print_exception(*exc_info, **dict(file=out)) + return out.getvalue() + + +def open_logfile(filename, mode='a'): + """Open the named log file in append mode. + + If the file already exists, a separator will also be printed to + the file to separate past activity from current activity. + """ + filename = os.path.expanduser(filename) + filename = os.path.abspath(filename) + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + exists = os.path.exists(filename) + + log_fp = open(filename, mode) + if exists: + print >> log_fp, '-'*60 + print >> log_fp, '%s run on %s' % (sys.argv[0], time.strftime('%c')) + return log_fp + + +def load_command(name): + full_name = 'pip.commands.%s' % name + if full_name in sys.modules: + return + try: + __import__(full_name) + except ImportError: + pass + + +def load_all_commands(): + for name in command_names(): + load_command(name) + + +def command_names(): + names = set((pkg[1] for pkg in walk_packages(path=commands.__path__))) + return list(names) + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py new file mode 100644 index 000000000..a8bd6ce4f --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py @@ -0,0 +1,231 @@ +"""Base option parser setup""" + +import sys +import optparse +import pkg_resources +import ConfigParser +import os +from distutils.util import strtobool +from pip.locations import default_config_file, default_log_file + + +class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter): + """Custom help formatter for use in ConfigOptionParser that updates + the defaults before expanding them, allowing them to show up correctly + in the help listing""" + + def expand_default(self, option): + if self.parser is not None: + self.parser.update_defaults(self.parser.defaults) + return optparse.IndentedHelpFormatter.expand_default(self, option) + + +class ConfigOptionParser(optparse.OptionParser): + """Custom option parser which updates its defaults by by checking the + configuration files and environmental variables""" + + def __init__(self, *args, **kwargs): + self.config = ConfigParser.RawConfigParser() + self.name = kwargs.pop('name') + self.files = self.get_config_files() + self.config.read(self.files) + assert self.name + optparse.OptionParser.__init__(self, *args, **kwargs) + + def get_config_files(self): + config_file = os.environ.get('PIP_CONFIG_FILE', False) + if config_file and os.path.exists(config_file): + return [config_file] + return [default_config_file] + + def update_defaults(self, defaults): + """Updates the given defaults with values from the config files and + the environ. Does a little special handling for certain types of + options (lists).""" + # Then go and look for the other sources of configuration: + config = {} + # 1. config files + for section in ('global', self.name): + config.update(dict(self.get_config_section(section))) + # 2. environmental variables + config.update(dict(self.get_environ_vars())) + # Then set the options with those values + for key, val in config.iteritems(): + key = key.replace('_', '-') + if not key.startswith('--'): + key = '--%s' % key # only prefer long opts + option = self.get_option(key) + if option is not None: + # ignore empty values + if not val: + continue + # handle multiline configs + if option.action == 'append': + val = val.split() + else: + option.nargs = 1 + if option.action in ('store_true', 'store_false', 'count'): + val = strtobool(val) + try: + val = option.convert_value(key, val) + except optparse.OptionValueError, e: + print ("An error occured during configuration: %s" % e) + sys.exit(3) + defaults[option.dest] = val + return defaults + + def get_config_section(self, name): + """Get a section of a configuration""" + if self.config.has_section(name): + return self.config.items(name) + return [] + + def get_environ_vars(self, prefix='PIP_'): + """Returns a generator with all environmental vars with prefix PIP_""" + for key, val in os.environ.iteritems(): + if key.startswith(prefix): + yield (key.replace(prefix, '').lower(), val) + + def get_default_values(self): + """Overridding to make updating the defaults after instantiation of + the option parser possible, update_defaults() does the dirty work.""" + if not self.process_default_values: + # Old, pre-Optik 1.5 behaviour. + return optparse.Values(self.defaults) + + defaults = self.update_defaults(self.defaults.copy()) # ours + for option in self._get_all_options(): + default = defaults.get(option.dest) + if isinstance(default, basestring): + opt_str = option.get_opt_string() + defaults[option.dest] = option.check_value(opt_str, default) + return optparse.Values(defaults) + +try: + pip_dist = pkg_resources.get_distribution('pip') + version = '%s from %s (python %s)' % ( + pip_dist, pip_dist.location, sys.version[:3]) +except pkg_resources.DistributionNotFound: + # when running pip.py without installing + version=None + +parser = ConfigOptionParser( + usage='%prog COMMAND [OPTIONS]', + version=version, + add_help_option=False, + formatter=UpdatingDefaultsHelpFormatter(), + name='global') + +parser.add_option( + '-h', '--help', + dest='help', + action='store_true', + help='Show help') +parser.add_option( + '-E', '--environment', + dest='venv', + metavar='DIR', + help='virtualenv environment to run pip in (either give the ' + 'interpreter or the environment base directory)') +parser.add_option( + '-s', '--enable-site-packages', + dest='site_packages', + action='store_true', + help='Include site-packages in virtualenv if one is to be ' + 'created. Ignored if --environment is not used or ' + 'the virtualenv already exists.') +parser.add_option( + # Defines a default root directory for virtualenvs, relative + # virtualenvs names/paths are considered relative to it. + '--virtualenv-base', + dest='venv_base', + type='str', + default='', + help=optparse.SUPPRESS_HELP) +parser.add_option( + # Run only if inside a virtualenv, bail if not. + '--require-virtualenv', '--require-venv', + dest='require_venv', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) +parser.add_option( + # Use automatically an activated virtualenv instead of installing + # globally. -E will be ignored if used. + '--respect-virtualenv', '--respect-venv', + dest='respect_venv', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) + +parser.add_option( + '-v', '--verbose', + dest='verbose', + action='count', + default=0, + help='Give more output') +parser.add_option( + '-q', '--quiet', + dest='quiet', + action='count', + default=0, + help='Give less output') +parser.add_option( + '--log', + dest='log', + metavar='FILENAME', + help='Log file where a complete (maximum verbosity) record will be kept') +parser.add_option( + # Writes the log levels explicitely to the log' + '--log-explicit-levels', + dest='log_explicit_levels', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) +parser.add_option( + # The default log file + '--local-log', '--log-file', + dest='log_file', + metavar='FILENAME', + default=default_log_file, + help=optparse.SUPPRESS_HELP) +parser.add_option( + # Don't ask for input + '--no-input', + dest='no_input', + action='store_true', + default=False, + help=optparse.SUPPRESS_HELP) + +parser.add_option( + '--proxy', + dest='proxy', + type='str', + default='', + help="Specify a proxy in the form user:passwd@proxy.server:port. " + "Note that the user:password@ is optional and required only if you " + "are behind an authenticated proxy. If you provide " + "user@proxy.server:port then you will be prompted for a password.") +parser.add_option( + '--timeout', '--default-timeout', + metavar='SECONDS', + dest='timeout', + type='float', + default=15, + help='Set the socket timeout (default %default seconds)') +parser.add_option( + # The default version control system for editables, e.g. 'svn' + '--default-vcs', + dest='default_vcs', + type='str', + default='', + help=optparse.SUPPRESS_HELP) +parser.add_option( + # A regex to be used to skip requirements + '--skip-requirements-regex', + dest='skip_requirements_regex', + type='str', + default='', + help=optparse.SUPPRESS_HELP) + +parser.disable_interspersed_args() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py new file mode 100644 index 000000000..792d60054 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py @@ -0,0 +1 @@ +# diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py new file mode 100644 index 000000000..fb0f75704 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py @@ -0,0 +1,33 @@ +from pip.locations import build_prefix, src_prefix +from pip.util import display_path, backup_dir +from pip.log import logger +from pip.exceptions import InstallationError +from pip.commands.install import InstallCommand + + +class BundleCommand(InstallCommand): + name = 'bundle' + usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...' + summary = 'Create pybundles (archives containing multiple packages)' + bundle = True + + def __init__(self): + super(BundleCommand, self).__init__() + + def run(self, options, args): + if not args: + raise InstallationError('You must give a bundle filename') + if not options.build_dir: + options.build_dir = backup_dir(build_prefix, '-bundle') + if not options.src_dir: + options.src_dir = backup_dir(src_prefix, '-bundle') + # We have to get everything when creating a bundle: + options.ignore_installed = True + logger.notify('Putting temporary build files in %s and source/develop files in %s' + % (display_path(options.build_dir), display_path(options.src_dir))) + self.bundle_filename = args.pop(0) + requirement_set = super(BundleCommand, self).run(options, args) + return requirement_set + + +BundleCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py new file mode 100644 index 000000000..d003b9ae3 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py @@ -0,0 +1,60 @@ +import sys +from pip.basecommand import Command + +BASE_COMPLETION = """ +# pip %(shell)s completion start%(script)s# pip %(shell)s completion end +""" + +COMPLETION_SCRIPTS = { + 'bash': """ +_pip_completion() +{ + COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ + COMP_CWORD=$COMP_CWORD \\ + PIP_AUTO_COMPLETE=1 $1 ) ) +} +complete -o default -F _pip_completion pip +""", 'zsh': """ +function _pip_completion { + local words cword + read -Ac words + read -cn cword + reply=( $( COMP_WORDS="$words[*]" \\ + COMP_CWORD=$(( cword-1 )) \\ + PIP_AUTO_COMPLETE=1 $words[1] ) ) +} +compctl -K _pip_completion pip +"""} + + +class CompletionCommand(Command): + name = 'completion' + summary = 'A helper command to be used for command completion' + hidden = True + + def __init__(self): + super(CompletionCommand, self).__init__() + self.parser.add_option( + '--bash', '-b', + action='store_const', + const='bash', + dest='shell', + help='Emit completion code for bash') + self.parser.add_option( + '--zsh', '-z', + action='store_const', + const='zsh', + dest='shell', + help='Emit completion code for zsh') + + def run(self, options, args): + """Prints the completion code of the given shell""" + shells = COMPLETION_SCRIPTS.keys() + shell_options = ['--'+shell for shell in sorted(shells)] + if options.shell in shells: + script = COMPLETION_SCRIPTS.get(options.shell, '') + print BASE_COMPLETION % {'script': script, 'shell': options.shell} + else: + sys.stderr.write('ERROR: You must pass %s\n' % ' or '.join(shell_options)) + +CompletionCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py new file mode 100644 index 000000000..01b5df934 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py @@ -0,0 +1,109 @@ +import re +import sys +import pkg_resources +import pip +from pip.req import InstallRequirement +from pip.log import logger +from pip.basecommand import Command +from pip.util import get_installed_distributions + + +class FreezeCommand(Command): + name = 'freeze' + usage = '%prog [OPTIONS]' + summary = 'Output all currently installed packages (exact versions) to stdout' + + def __init__(self): + super(FreezeCommand, self).__init__() + self.parser.add_option( + '-r', '--requirement', + dest='requirement', + action='store', + default=None, + metavar='FILENAME', + help='Use the given requirements file as a hint about how to generate the new frozen requirements') + self.parser.add_option( + '-f', '--find-links', + dest='find_links', + action='append', + default=[], + metavar='URL', + help='URL for finding packages, which will be added to the frozen requirements file') + self.parser.add_option( + '-l', '--local', + dest='local', + action='store_true', + default=False, + help='If in a virtualenv, do not report globally-installed packages') + + def setup_logging(self): + logger.move_stdout_to_stderr() + + def run(self, options, args): + requirement = options.requirement + find_links = options.find_links or [] + local_only = options.local + ## FIXME: Obviously this should be settable: + find_tags = False + skip_match = None + + skip_regex = options.skip_requirements_regex + if skip_regex: + skip_match = re.compile(skip_regex) + + dependency_links = [] + + f = sys.stdout + + for dist in pkg_resources.working_set: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + for link in find_links: + if '#egg=' in link: + dependency_links.append(link) + for link in find_links: + f.write('-f %s\n' % link) + installations = {} + for dist in get_installed_distributions(local_only=local_only): + req = pip.FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags) + installations[req.name] = req + if requirement: + req_f = open(requirement) + for line in req_f: + if not line.strip() or line.strip().startswith('#'): + f.write(line) + continue + if skip_match and skip_match.search(line): + f.write(line) + continue + elif line.startswith('-e') or line.startswith('--editable'): + if line.startswith('-e'): + line = line[2:].strip() + else: + line = line[len('--editable'):].strip().lstrip('=') + line_req = InstallRequirement.from_editable(line, default_vcs=options.default_vcs) + elif (line.startswith('-r') or line.startswith('--requirement') + or line.startswith('-Z') or line.startswith('--always-unzip') + or line.startswith('-f') or line.startswith('-i') + or line.startswith('--extra-index-url')): + f.write(line) + continue + else: + line_req = InstallRequirement.from_line(line) + if not line_req.name: + logger.notify("Skipping line because it's not clear what it would install: %s" + % line.strip()) + logger.notify(" (add #egg=PackageName to the URL to avoid this warning)") + continue + if line_req.name not in installations: + logger.warn("Requirement file contains %s, but that package is not installed" + % line.strip()) + continue + f.write(str(installations[line_req.name])) + del installations[line_req.name] + f.write('## The following requirements were added by pip --freeze:\n') + for installation in sorted(installations.values(), key=lambda x: x.name): + f.write(str(installation)) + + +FreezeCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py new file mode 100644 index 000000000..b0b366112 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py @@ -0,0 +1,32 @@ +from pip.basecommand import Command, command_dict, load_all_commands +from pip.exceptions import InstallationError +from pip.baseparser import parser + + +class HelpCommand(Command): + name = 'help' + usage = '%prog' + summary = 'Show available commands' + + def run(self, options, args): + load_all_commands() + if args: + ## FIXME: handle errors better here + command = args[0] + if command not in command_dict: + raise InstallationError('No command with the name: %s' % command) + command = command_dict[command] + command.parser.print_help() + return + parser.print_help() + print + print 'Commands available:' + commands = list(set(command_dict.values())) + commands.sort(key=lambda x: x.name) + for command in commands: + if command.hidden: + continue + print ' %s: %s' % (command.name, command.summary) + + +HelpCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py new file mode 100644 index 000000000..861c332bf --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py @@ -0,0 +1,247 @@ +import os, sys +from pip.req import InstallRequirement, RequirementSet +from pip.req import parse_requirements +from pip.log import logger +from pip.locations import build_prefix, src_prefix +from pip.basecommand import Command +from pip.index import PackageFinder +from pip.exceptions import InstallationError + + +class InstallCommand(Command): + name = 'install' + usage = '%prog [OPTIONS] PACKAGE_NAMES...' + summary = 'Install packages' + bundle = False + + def __init__(self): + super(InstallCommand, self).__init__() + self.parser.add_option( + '-e', '--editable', + dest='editables', + action='append', + default=[], + metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE', + help='Install a package directly from a checkout. Source will be checked ' + 'out into src/PACKAGE (lower-case) and installed in-place (using ' + 'setup.py develop). You can run this on an existing directory/checkout (like ' + 'pip install -e src/mycheckout). This option may be provided multiple times. ' + 'Possible values for VCS are: svn, git, hg and bzr.') + self.parser.add_option( + '-r', '--requirement', + dest='requirements', + action='append', + default=[], + metavar='FILENAME', + help='Install all the packages listed in the given requirements file. ' + 'This option can be used multiple times.') + self.parser.add_option( + '-f', '--find-links', + dest='find_links', + action='append', + default=[], + metavar='URL', + help='URL to look for packages at') + self.parser.add_option( + '-i', '--index-url', '--pypi-url', + dest='index_url', + metavar='URL', + default='http://pypi.python.org/simple/', + help='Base URL of Python Package Index (default %default)') + self.parser.add_option( + '--extra-index-url', + dest='extra_index_urls', + metavar='URL', + action='append', + default=[], + help='Extra URLs of package indexes to use in addition to --index-url') + self.parser.add_option( + '--no-index', + dest='no_index', + action='store_true', + default=False, + help='Ignore package index (only looking at --find-links URLs instead)') + self.parser.add_option( + '-M', '--use-mirrors', + dest='use_mirrors', + action='store_true', + default=False, + help='Use the PyPI mirrors as a fallback in case the main index is down.') + self.parser.add_option( + '--mirrors', + dest='mirrors', + metavar='URL', + action='append', + default=[], + help='Specific mirror URLs to query when --use-mirrors is used') + + self.parser.add_option( + '-b', '--build', '--build-dir', '--build-directory', + dest='build_dir', + metavar='DIR', + default=None, + help='Unpack packages into DIR (default %s) and build from there' % build_prefix) + self.parser.add_option( + '-d', '--download', '--download-dir', '--download-directory', + dest='download_dir', + metavar='DIR', + default=None, + help='Download packages into DIR instead of installing them') + self.parser.add_option( + '--download-cache', + dest='download_cache', + metavar='DIR', + default=None, + help='Cache downloaded packages in DIR') + self.parser.add_option( + '--src', '--source', '--source-dir', '--source-directory', + dest='src_dir', + metavar='DIR', + default=None, + help='Check out --editable packages into DIR (default %s)' % src_prefix) + + self.parser.add_option( + '-U', '--upgrade', + dest='upgrade', + action='store_true', + help='Upgrade all packages to the newest available version') + self.parser.add_option( + '-I', '--ignore-installed', + dest='ignore_installed', + action='store_true', + help='Ignore the installed packages (reinstalling instead)') + self.parser.add_option( + '--no-deps', '--no-dependencies', + dest='ignore_dependencies', + action='store_true', + default=False, + help='Ignore package dependencies') + self.parser.add_option( + '--no-install', + dest='no_install', + action='store_true', + help="Download and unpack all packages, but don't actually install them") + self.parser.add_option( + '--no-download', + dest='no_download', + action="store_true", + help="Don't download any packages, just install the ones already downloaded " + "(completes an install run with --no-install)") + + self.parser.add_option( + '--install-option', + dest='install_options', + action='append', + help="Extra arguments to be supplied to the setup.py install " + "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). " + "Use multiple --install-option options to pass multiple options to setup.py install. " + "If you are using an option with a directory path, be sure to use absolute path.") + + self.parser.add_option( + '--global-option', + dest='global_options', + action='append', + help="Extra global options to be supplied to the setup.py" + "call before the install command") + + self.parser.add_option( + '--user', + dest='use_user_site', + action='store_true', + help='Install to user-site') + + def _build_package_finder(self, options, index_urls): + """ + Create a package finder appropriate to this install command. + This method is meant to be overridden by subclasses, not + called directly. + """ + return PackageFinder(find_links=options.find_links, + index_urls=index_urls, + use_mirrors=options.use_mirrors, + mirrors=options.mirrors) + + def run(self, options, args): + if not options.build_dir: + options.build_dir = build_prefix + if not options.src_dir: + options.src_dir = src_prefix + if options.download_dir: + options.no_install = True + options.ignore_installed = True + options.build_dir = os.path.abspath(options.build_dir) + options.src_dir = os.path.abspath(options.src_dir) + install_options = options.install_options or [] + if options.use_user_site: + install_options.append('--user') + global_options = options.global_options or [] + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index: + logger.notify('Ignoring indexes: %s' % ','.join(index_urls)) + index_urls = [] + + finder = self._build_package_finder(options, index_urls) + + requirement_set = RequirementSet( + build_dir=options.build_dir, + src_dir=options.src_dir, + download_dir=options.download_dir, + download_cache=options.download_cache, + upgrade=options.upgrade, + ignore_installed=options.ignore_installed, + ignore_dependencies=options.ignore_dependencies) + for name in args: + requirement_set.add_requirement( + InstallRequirement.from_line(name, None)) + for name in options.editables: + requirement_set.add_requirement( + InstallRequirement.from_editable(name, default_vcs=options.default_vcs)) + for filename in options.requirements: + for req in parse_requirements(filename, finder=finder, options=options): + requirement_set.add_requirement(req) + + if not requirement_set.has_requirements: + if options.find_links: + raise InstallationError('You must give at least one ' + 'requirement to %s (maybe you meant "pip install %s"?)' + % (self.name, " ".join(options.find_links))) + raise InstallationError('You must give at least one requirement ' + 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) + + if (options.use_user_site and + sys.version_info < (2, 6)): + raise InstallationError('--user is only supported in Python version 2.6 and newer') + + import setuptools + if (options.use_user_site and + requirement_set.has_editables and + not getattr(setuptools, '_distribute', False)): + + raise InstallationError('--user --editable not supported with setuptools, use distribute') + + if not options.no_download: + requirement_set.prepare_files(finder, force_root_egg_info=self.bundle, bundle=self.bundle) + else: + requirement_set.locate_files() + + if not options.no_install and not self.bundle: + requirement_set.install(install_options, global_options) + installed = ' '.join([req.name for req in + requirement_set.successfully_installed]) + if installed: + logger.notify('Successfully installed %s' % installed) + elif not self.bundle: + downloaded = ' '.join([req.name for req in + requirement_set.successfully_downloaded]) + if downloaded: + logger.notify('Successfully downloaded %s' % downloaded) + elif self.bundle: + requirement_set.create_bundle(self.bundle_filename) + logger.notify('Created bundle in %s' % self.bundle_filename) + # Clean up + if not options.no_install: + requirement_set.cleanup_files(bundle=self.bundle) + return requirement_set + + +InstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py new file mode 100644 index 000000000..73da58ac6 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py @@ -0,0 +1,116 @@ +import sys +import xmlrpclib +import textwrap +import pkg_resources +import pip.download +from pip.basecommand import Command +from pip.util import get_terminal_size +from pip.log import logger +from distutils.version import StrictVersion, LooseVersion + + +class SearchCommand(Command): + name = 'search' + usage = '%prog QUERY' + summary = 'Search PyPI' + + def __init__(self): + super(SearchCommand, self).__init__() + self.parser.add_option( + '--index', + dest='index', + metavar='URL', + default='http://pypi.python.org/pypi', + help='Base URL of Python Package Index (default %default)') + + def run(self, options, args): + if not args: + logger.warn('ERROR: Missing required argument (search query).') + return + query = ' '.join(args) + index_url = options.index + + pypi_hits = self.search(query, index_url) + hits = transform_hits(pypi_hits) + + terminal_width = None + if sys.stdout.isatty(): + terminal_width = get_terminal_size()[0] + + print_results(hits, terminal_width=terminal_width) + + def search(self, query, index_url): + pypi = xmlrpclib.ServerProxy(index_url, pip.download.xmlrpclib_transport) + hits = pypi.search({'name': query, 'summary': query}, 'or') + return hits + + +def transform_hits(hits): + """ + The list from pypi is really a list of versions. We want a list of + packages with the list of versions stored inline. This converts the + list from pypi into one we can use. + """ + packages = {} + for hit in hits: + name = hit['name'] + summary = hit['summary'] + version = hit['version'] + score = hit['_pypi_ordering'] + + if name not in packages.keys(): + packages[name] = {'name': name, 'summary': summary, 'versions': [version], 'score': score} + else: + packages[name]['versions'].append(version) + + # if this is the highest version, replace summary and score + if version == highest_version(packages[name]['versions']): + packages[name]['summary'] = summary + packages[name]['score'] = score + + # each record has a unique name now, so we will convert the dict into a list sorted by score + package_list = sorted(packages.values(), lambda x, y: cmp(y['score'], x['score'])) + return package_list + + +def print_results(hits, name_column_width=25, terminal_width=None): + installed_packages = [p.project_name for p in pkg_resources.working_set] + for hit in hits: + name = hit['name'] + summary = hit['summary'] or '' + if terminal_width is not None: + # wrap and indent summary to fit terminal + summary = textwrap.wrap(summary, terminal_width - name_column_width - 5) + summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) + line = '%s - %s' % (name.ljust(name_column_width), summary) + try: + logger.notify(line) + if name in installed_packages: + dist = pkg_resources.get_distribution(name) + logger.indent += 2 + try: + latest = highest_version(hit['versions']) + if dist.version == latest: + logger.notify('INSTALLED: %s (latest)' % dist.version) + else: + logger.notify('INSTALLED: %s' % dist.version) + logger.notify('LATEST: %s' % latest) + finally: + logger.indent -= 2 + except UnicodeEncodeError: + pass + + +def compare_versions(version1, version2): + try: + return cmp(StrictVersion(version1), StrictVersion(version2)) + # in case of abnormal version number, fall back to LooseVersion + except ValueError: + return cmp(LooseVersion(version1), LooseVersion(version2)) + + +def highest_version(versions): + return reduce((lambda v1, v2: compare_versions(v1, v2) == 1 and v1 or v2), versions) + + +SearchCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py new file mode 100644 index 000000000..7effd844e --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py @@ -0,0 +1,42 @@ +from pip.req import InstallRequirement, RequirementSet, parse_requirements +from pip.basecommand import Command +from pip.exceptions import InstallationError + +class UninstallCommand(Command): + name = 'uninstall' + usage = '%prog [OPTIONS] PACKAGE_NAMES ...' + summary = 'Uninstall packages' + + def __init__(self): + super(UninstallCommand, self).__init__() + self.parser.add_option( + '-r', '--requirement', + dest='requirements', + action='append', + default=[], + metavar='FILENAME', + help='Uninstall all the packages listed in the given requirements file. ' + 'This option can be used multiple times.') + self.parser.add_option( + '-y', '--yes', + dest='yes', + action='store_true', + help="Don't ask for confirmation of uninstall deletions.") + + def run(self, options, args): + requirement_set = RequirementSet( + build_dir=None, + src_dir=None, + download_dir=None) + for name in args: + requirement_set.add_requirement( + InstallRequirement.from_line(name)) + for filename in options.requirements: + for req in parse_requirements(filename, options=options): + requirement_set.add_requirement(req) + if not requirement_set.has_requirements: + raise InstallationError('You must give at least one requirement ' + 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) + requirement_set.uninstall(auto_confirm=options.yes) + +UninstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py new file mode 100644 index 000000000..f83e18205 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py @@ -0,0 +1,9 @@ +from pip.commands.zip import ZipCommand + + +class UnzipCommand(ZipCommand): + name = 'unzip' + summary = 'Unzip individual packages' + + +UnzipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py new file mode 100644 index 000000000..346fc0519 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py @@ -0,0 +1,346 @@ +import sys +import re +import fnmatch +import os +import shutil +import zipfile +from pip.util import display_path, backup_dir +from pip.log import logger +from pip.exceptions import InstallationError +from pip.basecommand import Command + + +class ZipCommand(Command): + name = 'zip' + usage = '%prog [OPTIONS] PACKAGE_NAMES...' + summary = 'Zip individual packages' + + def __init__(self): + super(ZipCommand, self).__init__() + if self.name == 'zip': + self.parser.add_option( + '--unzip', + action='store_true', + dest='unzip', + help='Unzip (rather than zip) a package') + else: + self.parser.add_option( + '--zip', + action='store_false', + dest='unzip', + default=True, + help='Zip (rather than unzip) a package') + self.parser.add_option( + '--no-pyc', + action='store_true', + dest='no_pyc', + help='Do not include .pyc files in zip files (useful on Google App Engine)') + self.parser.add_option( + '-l', '--list', + action='store_true', + dest='list', + help='List the packages available, and their zip status') + self.parser.add_option( + '--sort-files', + action='store_true', + dest='sort_files', + help='With --list, sort packages according to how many files they contain') + self.parser.add_option( + '--path', + action='append', + dest='paths', + help='Restrict operations to the given paths (may include wildcards)') + self.parser.add_option( + '-n', '--simulate', + action='store_true', + help='Do not actually perform the zip/unzip operation') + + def paths(self): + """All the entries of sys.path, possibly restricted by --path""" + if not self.select_paths: + return sys.path + result = [] + match_any = set() + for path in sys.path: + path = os.path.normcase(os.path.abspath(path)) + for match in self.select_paths: + match = os.path.normcase(os.path.abspath(match)) + if '*' in match: + if re.search(fnmatch.translate(match+'*'), path): + result.append(path) + match_any.add(match) + break + else: + if path.startswith(match): + result.append(path) + match_any.add(match) + break + else: + logger.debug("Skipping path %s because it doesn't match %s" + % (path, ', '.join(self.select_paths))) + for match in self.select_paths: + if match not in match_any and '*' not in match: + result.append(match) + logger.debug("Adding path %s because it doesn't match anything already on sys.path" + % match) + return result + + def run(self, options, args): + self.select_paths = options.paths + self.simulate = options.simulate + if options.list: + return self.list(options, args) + if not args: + raise InstallationError( + 'You must give at least one package to zip or unzip') + packages = [] + for arg in args: + module_name, filename = self.find_package(arg) + if options.unzip and os.path.isdir(filename): + raise InstallationError( + 'The module %s (in %s) is not a zip file; cannot be unzipped' + % (module_name, filename)) + elif not options.unzip and not os.path.isdir(filename): + raise InstallationError( + 'The module %s (in %s) is not a directory; cannot be zipped' + % (module_name, filename)) + packages.append((module_name, filename)) + last_status = None + for module_name, filename in packages: + if options.unzip: + last_status = self.unzip_package(module_name, filename) + else: + last_status = self.zip_package(module_name, filename, options.no_pyc) + return last_status + + def unzip_package(self, module_name, filename): + zip_filename = os.path.dirname(filename) + if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename): + raise InstallationError( + 'Module %s (in %s) isn\'t located in a zip file in %s' + % (module_name, filename, zip_filename)) + package_path = os.path.dirname(zip_filename) + if not package_path in self.paths(): + logger.warn( + 'Unpacking %s into %s, but %s is not on sys.path' + % (display_path(zip_filename), display_path(package_path), + display_path(package_path))) + logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename))) + if self.simulate: + logger.notify('Skipping remaining operations because of --simulate') + return + logger.indent += 2 + try: + ## FIXME: this should be undoable: + zip = zipfile.ZipFile(zip_filename) + to_save = [] + for name in zip.namelist(): + if name.startswith(module_name + os.path.sep): + content = zip.read(name) + dest = os.path.join(package_path, name) + if not os.path.exists(os.path.dirname(dest)): + os.makedirs(os.path.dirname(dest)) + if not content and dest.endswith(os.path.sep): + if not os.path.exists(dest): + os.makedirs(dest) + else: + f = open(dest, 'wb') + f.write(content) + f.close() + else: + to_save.append((name, zip.read(name))) + zip.close() + if not to_save: + logger.info('Removing now-empty zip file %s' % display_path(zip_filename)) + os.unlink(zip_filename) + self.remove_filename_from_pth(zip_filename) + else: + logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename))) + zip = zipfile.ZipFile(zip_filename, 'w') + for name, content in to_save: + zip.writestr(name, content) + zip.close() + finally: + logger.indent -= 2 + + def zip_package(self, module_name, filename, no_pyc): + orig_filename = filename + logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) + logger.indent += 2 + if filename.endswith('.egg'): + dest_filename = filename + else: + dest_filename = filename + '.zip' + try: + ## FIXME: I think this needs to be undoable: + if filename == dest_filename: + filename = backup_dir(orig_filename) + logger.notify('Moving %s aside to %s' % (orig_filename, filename)) + if not self.simulate: + shutil.move(orig_filename, filename) + try: + logger.info('Creating zip file in %s' % display_path(dest_filename)) + if not self.simulate: + zip = zipfile.ZipFile(dest_filename, 'w') + zip.writestr(module_name + '/', '') + for dirpath, dirnames, filenames in os.walk(filename): + if no_pyc: + filenames = [f for f in filenames + if not f.lower().endswith('.pyc')] + for fns, is_dir in [(dirnames, True), (filenames, False)]: + for fn in fns: + full = os.path.join(dirpath, fn) + dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn) + if is_dir: + zip.writestr(dest+'/', '') + else: + zip.write(full, dest) + zip.close() + logger.info('Removing old directory %s' % display_path(filename)) + if not self.simulate: + shutil.rmtree(filename) + except: + ## FIXME: need to do an undo here + raise + ## FIXME: should also be undone: + self.add_filename_to_pth(dest_filename) + finally: + logger.indent -= 2 + + def remove_filename_from_pth(self, filename): + for pth in self.pth_files(): + f = open(pth, 'r') + lines = f.readlines() + f.close() + new_lines = [ + l for l in lines if l.strip() != filename] + if lines != new_lines: + logger.info('Removing reference to %s from .pth file %s' + % (display_path(filename), display_path(pth))) + if not filter(None, new_lines): + logger.info('%s file would be empty: deleting' % display_path(pth)) + if not self.simulate: + os.unlink(pth) + else: + if not self.simulate: + f = open(pth, 'wb') + f.writelines(new_lines) + f.close() + return + logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename)) + + def add_filename_to_pth(self, filename): + path = os.path.dirname(filename) + dest = os.path.join(path, filename + '.pth') + if path not in self.paths(): + logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest)) + if not self.simulate: + if os.path.exists(dest): + f = open(dest) + lines = f.readlines() + f.close() + if lines and not lines[-1].endswith('\n'): + lines[-1] += '\n' + lines.append(filename+'\n') + else: + lines = [filename + '\n'] + f = open(dest, 'wb') + f.writelines(lines) + f.close() + + def pth_files(self): + for path in self.paths(): + if not os.path.exists(path) or not os.path.isdir(path): + continue + for filename in os.listdir(path): + if filename.endswith('.pth'): + yield os.path.join(path, filename) + + def find_package(self, package): + for path in self.paths(): + full = os.path.join(path, package) + if os.path.exists(full): + return package, full + if not os.path.isdir(path) and zipfile.is_zipfile(path): + zip = zipfile.ZipFile(path, 'r') + try: + zip.read(os.path.join(package, '__init__.py')) + except KeyError: + pass + else: + zip.close() + return package, full + zip.close() + ## FIXME: need special error for package.py case: + raise InstallationError( + 'No package with the name %s found' % package) + + def list(self, options, args): + if args: + raise InstallationError( + 'You cannot give an argument with --list') + for path in sorted(self.paths()): + if not os.path.exists(path): + continue + basename = os.path.basename(path.rstrip(os.path.sep)) + if os.path.isfile(path) and zipfile.is_zipfile(path): + if os.path.dirname(path) not in self.paths(): + logger.notify('Zipped egg: %s' % display_path(path)) + continue + if (basename != 'site-packages' and basename != 'dist-packages' + and not path.replace('\\', '/').endswith('lib/python')): + continue + logger.notify('In %s:' % display_path(path)) + logger.indent += 2 + zipped = [] + unzipped = [] + try: + for filename in sorted(os.listdir(path)): + ext = os.path.splitext(filename)[1].lower() + if ext in ('.pth', '.egg-info', '.egg-link'): + continue + if ext == '.py': + logger.info('Not displaying %s: not a package' % display_path(filename)) + continue + full = os.path.join(path, filename) + if os.path.isdir(full): + unzipped.append((filename, self.count_package(full))) + elif zipfile.is_zipfile(full): + zipped.append(filename) + else: + logger.info('Unknown file: %s' % display_path(filename)) + if zipped: + logger.notify('Zipped packages:') + logger.indent += 2 + try: + for filename in zipped: + logger.notify(filename) + finally: + logger.indent -= 2 + else: + logger.notify('No zipped packages.') + if unzipped: + if options.sort_files: + unzipped.sort(key=lambda x: -x[1]) + logger.notify('Unzipped packages:') + logger.indent += 2 + try: + for filename, count in unzipped: + logger.notify('%s (%i files)' % (filename, count)) + finally: + logger.indent -= 2 + else: + logger.notify('No unzipped packages.') + finally: + logger.indent -= 2 + + def count_package(self, path): + total = 0 + for dirpath, dirnames, filenames in os.walk(path): + filenames = [f for f in filenames + if not f.lower().endswith('.pyc')] + total += len(filenames) + return total + + +ZipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py new file mode 100644 index 000000000..f1b63936b --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py @@ -0,0 +1,470 @@ +import xmlrpclib +import re +import getpass +import urllib +import urllib2 +import urlparse +import os +import mimetypes +import shutil +import tempfile +from pip.backwardcompat import md5, copytree +from pip.exceptions import InstallationError +from pip.util import (splitext, + format_size, display_path, backup_dir, ask, + unpack_file, create_download_cache_folder, cache_download) +from pip.vcs import vcs +from pip.log import logger + + +__all__ = ['xmlrpclib_transport', 'get_file_content', 'urlopen', + 'is_url', 'url_to_path', 'path_to_url', 'path_to_url2', + 'geturl', 'is_archive_file', 'unpack_vcs_link', + 'unpack_file_url', 'is_vcs_url', 'is_file_url', 'unpack_http_url'] + + +xmlrpclib_transport = xmlrpclib.Transport() + + +def get_file_content(url, comes_from=None): + """Gets the content of a file; it may be a filename, file: URL, or + http: URL. Returns (location, content)""" + match = _scheme_re.search(url) + if match: + scheme = match.group(1).lower() + if (scheme == 'file' and comes_from + and comes_from.startswith('http')): + raise InstallationError( + 'Requirements file %s references URL %s, which is local' + % (comes_from, url)) + if scheme == 'file': + path = url.split(':', 1)[1] + path = path.replace('\\', '/') + match = _url_slash_drive_re.match(path) + if match: + path = match.group(1) + ':' + path.split('|', 1)[1] + path = urllib.unquote(path) + if path.startswith('/'): + path = '/' + path.lstrip('/') + url = path + else: + ## FIXME: catch some errors + resp = urlopen(url) + return geturl(resp), resp.read() + try: + f = open(url) + content = f.read() + except IOError, e: + raise InstallationError('Could not open requirements file: %s' % str(e)) + else: + f.close() + return url, content + + +_scheme_re = re.compile(r'^(http|https|file):', re.I) +_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) + +class URLOpener(object): + """ + pip's own URL helper that adds HTTP auth and proxy support + """ + def __init__(self): + self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + + def __call__(self, url): + """ + If the given url contains auth info or if a normal request gets a 401 + response, an attempt is made to fetch the resource using basic HTTP + auth. + + """ + url, username, password = self.extract_credentials(url) + if username is None: + try: + response = urllib2.urlopen(self.get_request(url)) + except urllib2.HTTPError, e: + if e.code != 401: + raise + response = self.get_response(url) + else: + response = self.get_response(url, username, password) + return response + + def get_request(self, url): + """ + Wraps the URL to retrieve to protects against "creative" + interpretation of the RFC: http://bugs.python.org/issue8732 + """ + if isinstance(url, basestring): + url = urllib2.Request(url, headers={'Accept-encoding': 'identity'}) + return url + + def get_response(self, url, username=None, password=None): + """ + does the dirty work of actually getting the rsponse object using urllib2 + and its HTTP auth builtins. + """ + scheme, netloc, path, query, frag = urlparse.urlsplit(url) + pass_url = urlparse.urlunsplit(('_none_', netloc, path, query, frag)).replace('_none_://', '', 1) + req = self.get_request(url) + + stored_username, stored_password = self.passman.find_user_password(None, netloc) + # see if we have a password stored + if stored_username is None: + if username is None and self.prompting: + username = urllib.quote(raw_input('User for %s: ' % netloc)) + password = urllib.quote(getpass.getpass('Password: ')) + if username and password: + self.passman.add_password(None, netloc, username, password) + stored_username, stored_password = self.passman.find_user_password(None, netloc) + authhandler = urllib2.HTTPBasicAuthHandler(self.passman) + opener = urllib2.build_opener(authhandler) + # FIXME: should catch a 401 and offer to let the user reenter credentials + return opener.open(req) + + def setup(self, proxystr='', prompting=True): + """ + Sets the proxy handler given the option passed on the command + line. If an empty string is passed it looks at the HTTP_PROXY + environment variable. + """ + self.prompting = prompting + proxy = self.get_proxy(proxystr) + if proxy: + proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy}) + opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler) + urllib2.install_opener(opener) + + def parse_credentials(self, netloc): + if "@" in netloc: + userinfo = netloc.rsplit("@", 1)[0] + if ":" in userinfo: + return userinfo.split(":", 1) + return userinfo, None + return None, None + + def extract_credentials(self, url): + """ + Extracts user/password from a url. + + Returns a tuple: + (url-without-auth, username, password) + """ + if isinstance(url, urllib2.Request): + result = urlparse.urlsplit(url.get_full_url()) + else: + result = urlparse.urlsplit(url) + scheme, netloc, path, query, frag = result + + username, password = self.parse_credentials(netloc) + if username is None: + return url, None, None + elif password is None and self.prompting: + # remove the auth credentials from the url part + netloc = netloc.replace('%s@' % username, '', 1) + # prompt for the password + prompt = 'Password for %s@%s: ' % (username, netloc) + password = urllib.quote(getpass.getpass(prompt)) + else: + # remove the auth credentials from the url part + netloc = netloc.replace('%s:%s@' % (username, password), '', 1) + + target_url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + return target_url, username, password + + def get_proxy(self, proxystr=''): + """ + Get the proxy given the option passed on the command line. + If an empty string is passed it looks at the HTTP_PROXY + environment variable. + """ + if not proxystr: + proxystr = os.environ.get('HTTP_PROXY', '') + if proxystr: + if '@' in proxystr: + user_password, server_port = proxystr.split('@', 1) + if ':' in user_password: + user, password = user_password.split(':', 1) + else: + user = user_password + prompt = 'Password for %s@%s: ' % (user, server_port) + password = urllib.quote(getpass.getpass(prompt)) + return '%s:%s@%s' % (user, password, server_port) + else: + return proxystr + else: + return None + +urlopen = URLOpener() + + +def is_url(name): + """Returns true if the name looks like a URL""" + if ':' not in name: + return False + scheme = name.split(':', 1)[0].lower() + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + +def url_to_path(url): + """ + Convert a file: URL to a path. + """ + assert url.startswith('file:'), ( + "You can only turn file: urls into filenames (not %r)" % url) + path = url[len('file:'):].lstrip('/') + path = urllib.unquote(path) + if _url_drive_re.match(path): + path = path[0] + ':' + path[2:] + else: + path = '/' + path + return path + + +_drive_re = re.compile('^([a-z]):', re.I) +_url_drive_re = re.compile('^([a-z])[:|]', re.I) + + +def path_to_url(path): + """ + Convert a path to a file: URL. The path will be made absolute. + """ + path = os.path.normcase(os.path.abspath(path)) + if _drive_re.match(path): + path = path[0] + '|' + path[2:] + url = urllib.quote(path) + url = url.replace(os.path.sep, '/') + url = url.lstrip('/') + return 'file:///' + url + + +def path_to_url2(path): + """ + Convert a path to a file: URL. The path will be made absolute and have + quoted path parts. + """ + path = os.path.normpath(os.path.abspath(path)) + drive, path = os.path.splitdrive(path) + filepath = path.split(os.path.sep) + url = '/'.join([urllib.quote(part) for part in filepath]) + if not drive: + url = url.lstrip('/') + return 'file:///' + drive + url + + +def geturl(urllib2_resp): + """ + Use instead of urllib.addinfourl.geturl(), which appears to have + some issues with dropping the double slash for certain schemes + (e.g. file://). This implementation is probably over-eager, as it + always restores '://' if it is missing, and it appears some url + schemata aren't always followed by '//' after the colon, but as + far as I know pip doesn't need any of those. + The URI RFC can be found at: http://tools.ietf.org/html/rfc1630 + + This function assumes that + scheme:/foo/bar + is the same as + scheme:///foo/bar + """ + url = urllib2_resp.geturl() + scheme, rest = url.split(':', 1) + if rest.startswith('//'): + return url + else: + # FIXME: write a good test to cover it + return '%s://%s' % (scheme, rest) + + +def is_archive_file(name): + """Return True if `name` is a considered as an archive file.""" + archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle') + ext = splitext(name)[1].lower() + if ext in archives: + return True + return False + + +def unpack_vcs_link(link, location, only_download=False): + vcs_backend = _get_used_vcs_backend(link) + if only_download: + vcs_backend.export(location) + else: + vcs_backend.unpack(location) + + +def unpack_file_url(link, location): + source = url_to_path(link.url) + content_type = mimetypes.guess_type(source)[0] + if os.path.isdir(source): + # delete the location since shutil will create it again :( + if os.path.isdir(location): + shutil.rmtree(location) + copytree(source, location) + else: + unpack_file(source, location, content_type, link) + + +def _get_used_vcs_backend(link): + for backend in vcs.backends: + if link.scheme in backend.schemes: + vcs_backend = backend(link.url) + return vcs_backend + + +def is_vcs_url(link): + return bool(_get_used_vcs_backend(link)) + + +def is_file_url(link): + return link.url.lower().startswith('file:') + + +def _check_md5(download_hash, link): + download_hash = download_hash.hexdigest() + if download_hash != link.md5_hash: + logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!" + % (link, download_hash, link.md5_hash)) + raise InstallationError('Bad MD5 hash for package %s' % link) + + +def _get_md5_from_file(target_file, link): + download_hash = md5() + fp = open(target_file, 'rb') + while 1: + chunk = fp.read(4096) + if not chunk: + break + download_hash.update(chunk) + fp.close() + return download_hash + + +def _download_url(resp, link, temp_location): + fp = open(temp_location, 'wb') + download_hash = None + if link.md5_hash: + download_hash = md5() + try: + total_length = int(resp.info()['content-length']) + except (ValueError, KeyError): + total_length = 0 + downloaded = 0 + show_progress = total_length > 40*1000 or not total_length + show_url = link.show_url + try: + if show_progress: + ## FIXME: the URL can get really long in this message: + if total_length: + logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) + else: + logger.start_progress('Downloading %s (unknown size): ' % show_url) + else: + logger.notify('Downloading %s' % show_url) + logger.debug('Downloading from URL %s' % link) + + while 1: + chunk = resp.read(4096) + if not chunk: + break + downloaded += len(chunk) + if show_progress: + if not total_length: + logger.show_progress('%s' % format_size(downloaded)) + else: + logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded))) + if link.md5_hash: + download_hash.update(chunk) + fp.write(chunk) + fp.close() + finally: + if show_progress: + logger.end_progress('%s downloaded' % format_size(downloaded)) + return download_hash + + +def _copy_file(filename, location, content_type, link): + copy = True + download_location = os.path.join(location, link.filename) + if os.path.exists(download_location): + response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' + % display_path(download_location), ('i', 'w', 'b')) + if response == 'i': + copy = False + elif response == 'w': + logger.warn('Deleting %s' % display_path(download_location)) + os.remove(download_location) + elif response == 'b': + dest_file = backup_dir(download_location) + logger.warn('Backing up %s to %s' + % (display_path(download_location), display_path(dest_file))) + shutil.move(download_location, dest_file) + if copy: + shutil.copy(filename, download_location) + logger.indent -= 2 + logger.notify('Saved %s' % display_path(download_location)) + + +def unpack_http_url(link, location, download_cache, only_download): + temp_dir = tempfile.mkdtemp('-unpack', 'pip-') + target_url = link.url.split('#', 1)[0] + target_file = None + download_hash = None + if download_cache: + target_file = os.path.join(download_cache, + urllib.quote(target_url, '')) + if not os.path.isdir(download_cache): + create_download_cache_folder(download_cache) + if (target_file + and os.path.exists(target_file) + and os.path.exists(target_file+'.content-type')): + fp = open(target_file+'.content-type') + content_type = fp.read().strip() + fp.close() + if link.md5_hash: + download_hash = _get_md5_from_file(target_file, link) + temp_location = target_file + logger.notify('Using download cache from %s' % target_file) + else: + resp = _get_response_from_url(target_url, link) + content_type = resp.info()['content-type'] + filename = link.filename + ext = splitext(filename)[1] + if not ext: + ext = mimetypes.guess_extension(content_type) + if ext: + filename += ext + if not ext and link.url != geturl(resp): + ext = os.path.splitext(geturl(resp))[1] + if ext: + filename += ext + temp_location = os.path.join(temp_dir, filename) + download_hash = _download_url(resp, link, temp_location) + if link.md5_hash: + _check_md5(download_hash, link) + if only_download: + _copy_file(temp_location, location, content_type, link) + else: + unpack_file(temp_location, location, content_type, link) + if target_file and target_file != temp_location: + cache_download(target_file, temp_location, content_type) + if target_file is None: + os.unlink(temp_location) + os.rmdir(temp_dir) + + +def _get_response_from_url(target_url, link): + try: + resp = urlopen(target_url) + except urllib2.HTTPError, e: + logger.fatal("HTTP error %s while getting %s" % (e.code, link)) + raise + except IOError, e: + # Typically an FTP error + logger.fatal("Error %s while getting %s" % (e, link)) + raise + return resp + +class Urllib2HeadRequest(urllib2.Request): + def get_method(self): + return "HEAD" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py new file mode 100644 index 000000000..1ad1a616d --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py @@ -0,0 +1,17 @@ +"""Exceptions used throughout package""" + + +class InstallationError(Exception): + """General exception during installation""" + + +class UninstallationError(Exception): + """General exception during uninstallation""" + + +class DistributionNotFound(InstallationError): + """Raised when a distribution cannot be found to satisfy a requirement""" + + +class BadCommand(Exception): + """Raised when virtualenv or a command is not found""" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py new file mode 100644 index 000000000..e42d8c868 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py @@ -0,0 +1,686 @@ +"""Routines related to PyPI, indexes""" + +import sys +import os +import re +import mimetypes +import threading +import posixpath +import pkg_resources +import urllib +import urllib2 +import urlparse +import httplib +import random +import socket +import string +from Queue import Queue +from Queue import Empty as QueueEmpty +from pip.log import logger +from pip.util import Inf +from pip.util import normalize_name, splitext +from pip.exceptions import DistributionNotFound +from pip.backwardcompat import WindowsError, product +from pip.download import urlopen, path_to_url2, url_to_path, geturl, Urllib2HeadRequest + +__all__ = ['PackageFinder'] + + +DEFAULT_MIRROR_URL = "last.pypi.python.org" + + +class PackageFinder(object): + """This finds packages. + + This is meant to match easy_install's technique for looking for + packages, by reading pages and looking for appropriate links + """ + + def __init__(self, find_links, index_urls, + use_mirrors=False, mirrors=None, main_mirror_url=None): + self.find_links = find_links + self.index_urls = index_urls + self.dependency_links = [] + self.cache = PageCache() + # These are boring links that have already been logged somehow: + self.logged_links = set() + if use_mirrors: + self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) + logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) + else: + self.mirror_urls = [] + + def add_dependency_links(self, links): + ## FIXME: this shouldn't be global list this, it should only + ## apply to requirements of the package that specifies the + ## dependency_links value + ## FIXME: also, we should track comes_from (i.e., use Link) + self.dependency_links.extend(links) + + @staticmethod + def _sort_locations(locations): + """ + Sort locations into "files" (archives) and "urls", and return + a pair of lists (files,urls) + """ + files = [] + urls = [] + + # puts the url for the given file path into the appropriate + # list + def sort_path(path): + url = path_to_url2(path) + if mimetypes.guess_type(url, strict=False)[0] == 'text/html': + urls.append(url) + else: + files.append(url) + + for url in locations: + if url.startswith('file:'): + path = url_to_path(url) + if os.path.isdir(path): + path = os.path.realpath(path) + for item in os.listdir(path): + sort_path(os.path.join(path, item)) + elif os.path.isfile(path): + sort_path(path) + else: + urls.append(url) + return files, urls + + def find_requirement(self, req, upgrade): + url_name = req.url_name + # Only check main index if index URL is given: + main_index_url = None + if self.index_urls: + # Check that we have the url_name correctly spelled: + main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) + # This will also cache the page, so it's okay that we get it again later: + page = self._get_page(main_index_url, req) + if page is None: + url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name + + # Combine index URLs with mirror URLs here to allow + # adding more index URLs from requirements files + all_index_urls = self.index_urls + self.mirror_urls + + def mkurl_pypi_url(url): + loc = posixpath.join(url, url_name) + # For maximum compatibility with easy_install, ensure the path + # ends in a trailing slash. Although this isn't in the spec + # (and PyPI can handle it without the slash) some other index + # implementations might break if they relied on easy_install's behavior. + if not loc.endswith('/'): + loc = loc + '/' + return loc + if url_name is not None: + locations = [ + mkurl_pypi_url(url) + for url in all_index_urls] + self.find_links + else: + locations = list(self.find_links) + locations.extend(self.dependency_links) + for version in req.absolute_versions: + if url_name is not None and main_index_url is not None: + locations = [ + posixpath.join(main_index_url.url, version)] + locations + + file_locations, url_locations = self._sort_locations(locations) + + locations = [Link(url) for url in url_locations] + logger.debug('URLs to search for versions for %s:' % req) + for location in locations: + logger.debug('* %s' % location) + found_versions = [] + found_versions.extend( + self._package_versions( + [Link(url, '-f') for url in self.find_links], req.name.lower())) + page_versions = [] + for page in self._get_pages(locations, req): + logger.debug('Analyzing links from page %s' % page.url) + logger.indent += 2 + try: + page_versions.extend(self._package_versions(page.links, req.name.lower())) + finally: + logger.indent -= 2 + dependency_versions = list(self._package_versions( + [Link(url) for url in self.dependency_links], req.name.lower())) + if dependency_versions: + logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) + file_versions = list(self._package_versions( + [Link(url) for url in file_locations], req.name.lower())) + if not found_versions and not page_versions and not dependency_versions and not file_versions: + logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) + raise DistributionNotFound('No distributions at all found for %s' % req) + if req.satisfied_by is not None: + found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) + if file_versions: + file_versions.sort(reverse=True) + logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) + found_versions = file_versions + found_versions + all_versions = found_versions + page_versions + dependency_versions + applicable_versions = [] + for (parsed_version, link, version) in all_versions: + if version not in req.req: + logger.info("Ignoring link %s, version %s doesn't match %s" + % (link, version, ','.join([''.join(s) for s in req.req.specs]))) + continue + applicable_versions.append((link, version)) + applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) + existing_applicable = bool([link for link, version in applicable_versions if link is Inf]) + if not upgrade and existing_applicable: + if applicable_versions[0][1] is Inf: + logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' + % req.satisfied_by.version) + else: + logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' + % (req.satisfied_by.version, applicable_versions[0][1])) + return None + if not applicable_versions: + logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' + % (req, ', '.join([version for parsed_version, link, version in found_versions]))) + raise DistributionNotFound('No distributions matching the version for %s' % req) + if applicable_versions[0][0] is Inf: + # We have an existing version, and its the best version + logger.info('Installed version (%s) is most up-to-date (past versions: %s)' + % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none')) + return None + if len(applicable_versions) > 1: + logger.info('Using version %s (newest of versions: %s)' % + (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions]))) + return applicable_versions[0][0] + + def _find_url_name(self, index_url, url_name, req): + """Finds the true URL name of a package, when the given name isn't quite correct. + This is usually used to implement case-insensitivity.""" + if not index_url.url.endswith('/'): + # Vaguely part of the PyPI API... weird but true. + ## FIXME: bad to modify this? + index_url.url += '/' + page = self._get_page(index_url, req) + if page is None: + logger.fatal('Cannot fetch index base URL %s' % index_url) + return + norm_name = normalize_name(req.url_name) + for link in page.links: + base = posixpath.basename(link.path.rstrip('/')) + if norm_name == normalize_name(base): + logger.notify('Real name of requirement %s is %s' % (url_name, base)) + return base + return None + + def _get_pages(self, locations, req): + """Yields (page, page_url) from the given locations, skipping + locations that have errors, and adding download/homepage links""" + pending_queue = Queue() + for location in locations: + pending_queue.put(location) + done = [] + seen = set() + threads = [] + for i in range(min(10, len(locations))): + t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen)) + t.setDaemon(True) + threads.append(t) + t.start() + for t in threads: + t.join() + return done + + _log_lock = threading.Lock() + + def _get_queued_page(self, req, pending_queue, done, seen): + while 1: + try: + location = pending_queue.get(False) + except QueueEmpty: + return + if location in seen: + continue + seen.add(location) + page = self._get_page(location, req) + if page is None: + continue + done.append(page) + for link in page.rel_links(): + pending_queue.put(link) + + _egg_fragment_re = re.compile(r'#egg=([^&]*)') + _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I) + _py_version_re = re.compile(r'-py([123]\.[0-9])$') + + def _sort_links(self, links): + "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates" + eggs, no_eggs = [], [] + seen = set() + for link in links: + if link not in seen: + seen.add(link) + if link.egg_fragment: + eggs.append(link) + else: + no_eggs.append(link) + return no_eggs + eggs + + def _package_versions(self, links, search_name): + for link in self._sort_links(links): + for v in self._link_package_versions(link, search_name): + yield v + + def _link_package_versions(self, link, search_name): + """ + Return an iterable of triples (pkg_resources_version_key, + link, python_version) that can be extracted from the given + link. + + Meant to be overridden by subclasses, not called by clients. + """ + if link.egg_fragment: + egg_info = link.egg_fragment + else: + egg_info, ext = link.splitext() + if not ext: + if link not in self.logged_links: + logger.debug('Skipping link %s; not a file' % link) + self.logged_links.add(link) + return [] + if egg_info.endswith('.tar'): + # Special double-extension case: + egg_info = egg_info[:-4] + ext = '.tar' + ext + if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'): + if link not in self.logged_links: + logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext)) + self.logged_links.add(link) + return [] + version = self._egg_info_matches(egg_info, search_name, link) + if version is None: + logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name)) + return [] + match = self._py_version_re.search(version) + if match: + version = version[:match.start()] + py_version = match.group(1) + if py_version != sys.version[:3]: + logger.debug('Skipping %s because Python version is incorrect' % link) + return [] + logger.debug('Found link %s, version: %s' % (link, version)) + return [(pkg_resources.parse_version(version), + link, + version)] + + def _egg_info_matches(self, egg_info, search_name, link): + match = self._egg_info_re.search(egg_info) + if not match: + logger.debug('Could not parse version from link: %s' % link) + return None + name = match.group(0).lower() + # To match the "safe" name that pkg_resources creates: + name = name.replace('_', '-') + if name.startswith(search_name.lower()): + return match.group(0)[len(search_name):].lstrip('-') + else: + return None + + def _get_page(self, link, req): + return HTMLPage.get_page(link, req, cache=self.cache) + + def _get_mirror_urls(self, mirrors=None, main_mirror_url=None): + """Retrieves a list of URLs from the main mirror DNS entry + unless a list of mirror URLs are passed. + """ + if not mirrors: + mirrors = get_mirrors(main_mirror_url) + # Should this be made "less random"? E.g. netselect like? + random.shuffle(mirrors) + + mirror_urls = set() + for mirror_url in mirrors: + # Make sure we have a valid URL + if not ("http://" or "https://" or "file://") in mirror_url: + mirror_url = "http://%s" % mirror_url + if not mirror_url.endswith("/simple"): + mirror_url = "%s/simple/" % mirror_url + mirror_urls.add(mirror_url) + + return list(mirror_urls) + + +class PageCache(object): + """Cache of HTML pages""" + + failure_limit = 3 + + def __init__(self): + self._failures = {} + self._pages = {} + self._archives = {} + + def too_many_failures(self, url): + return self._failures.get(url, 0) >= self.failure_limit + + def get_page(self, url): + return self._pages.get(url) + + def is_archive(self, url): + return self._archives.get(url, False) + + def set_is_archive(self, url, value=True): + self._archives[url] = value + + def add_page_failure(self, url, level): + self._failures[url] = self._failures.get(url, 0)+level + + def add_page(self, urls, page): + for url in urls: + self._pages[url] = page + + +class HTMLPage(object): + """Represents one page, along with its URL""" + + ## FIXME: these regexes are horrible hacks: + _homepage_re = re.compile(r'\s*home\s*page', re.I) + _download_re = re.compile(r'\s*download\s+url', re.I) + ## These aren't so aweful: + _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I) + _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S) + _base_re = re.compile(r"""]+)""", re.I) + + def __init__(self, content, url, headers=None): + self.content = content + self.url = url + self.headers = headers + + def __str__(self): + return self.url + + @classmethod + def get_page(cls, link, req, cache=None, skip_archives=True): + url = link.url + url = url.split('#', 1)[0] + if cache.too_many_failures(url): + return None + + # Check for VCS schemes that do not support lookup as web pages. + from pip.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals()) + return None + + if cache is not None: + inst = cache.get_page(url) + if inst is not None: + return inst + try: + if skip_archives: + if cache is not None: + if cache.is_archive(url): + return None + filename = link.filename + for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']: + if filename.endswith(bad_ext): + content_type = cls._get_content_type(url) + if content_type.lower().startswith('text/html'): + break + else: + logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type)) + if cache is not None: + cache.set_is_archive(url) + return None + logger.debug('Getting page %s' % url) + + # Tack index.html onto file:// URLs that point to directories + (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) + if scheme == 'file' and os.path.isdir(urllib.url2pathname(path)): + # add trailing slash if not present so urljoin doesn't trim final segment + if not url.endswith('/'): + url += '/' + url = urlparse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s' % url) + + resp = urlopen(url) + + real_url = geturl(resp) + headers = resp.info() + inst = cls(resp.read(), real_url, headers) + except (urllib2.HTTPError, urllib2.URLError, socket.timeout, socket.error, OSError, WindowsError), e: + desc = str(e) + if isinstance(e, socket.timeout): + log_meth = logger.info + level =1 + desc = 'timed out' + elif isinstance(e, urllib2.URLError): + log_meth = logger.info + if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout): + desc = 'timed out' + level = 1 + else: + level = 2 + elif isinstance(e, urllib2.HTTPError) and e.code == 404: + ## FIXME: notify? + log_meth = logger.info + level = 2 + else: + log_meth = logger.info + level = 1 + log_meth('Could not fetch URL %s: %s' % (link, desc)) + log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req)) + if cache is not None: + cache.add_page_failure(url, level) + return None + if cache is not None: + cache.add_page([url, real_url], inst) + return inst + + @staticmethod + def _get_content_type(url): + """Get the Content-Type of the given url, using a HEAD request""" + scheme, netloc, path, query, fragment = urlparse.urlsplit(url) + if not scheme in ('http', 'https', 'ftp', 'ftps'): + ## FIXME: some warning or something? + ## assertion error? + return '' + req = Urllib2HeadRequest(url, headers={'Host': netloc}) + resp = urlopen(req) + try: + if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'): + ## FIXME: doesn't handle redirects + return '' + return resp.info().get('content-type', '') + finally: + resp.close() + + @property + def base_url(self): + if not hasattr(self, "_base_url"): + match = self._base_re.search(self.content) + if match: + self._base_url = match.group(1) + else: + self._base_url = self.url + return self._base_url + + @property + def links(self): + """Yields all links in the page""" + for match in self._href_re.finditer(self.content): + url = match.group(1) or match.group(2) or match.group(3) + url = self.clean_link(urlparse.urljoin(self.base_url, url)) + yield Link(url, self) + + def rel_links(self): + for url in self.explicit_rel_links(): + yield url + for url in self.scraped_rel_links(): + yield url + + def explicit_rel_links(self, rels=('homepage', 'download')): + """Yields all links with the given relations""" + for match in self._rel_re.finditer(self.content): + found_rels = match.group(1).lower().split() + for rel in rels: + if rel in found_rels: + break + else: + continue + match = self._href_re.search(match.group(0)) + if not match: + continue + url = match.group(1) or match.group(2) or match.group(3) + url = self.clean_link(urlparse.urljoin(self.base_url, url)) + yield Link(url, self) + + def scraped_rel_links(self): + for regex in (self._homepage_re, self._download_re): + match = regex.search(self.content) + if not match: + continue + href_match = self._href_re.search(self.content, pos=match.end()) + if not href_match: + continue + url = match.group(1) or match.group(2) or match.group(3) + if not url: + continue + url = self.clean_link(urlparse.urljoin(self.base_url, url)) + yield Link(url, self) + + _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) + + def clean_link(self, url): + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + return self._clean_re.sub( + lambda match: '%%%2x' % ord(match.group(0)), url) + + +class Link(object): + + def __init__(self, url, comes_from=None): + self.url = url + self.comes_from = comes_from + + def __str__(self): + if self.comes_from: + return '%s (from %s)' % (self.url, self.comes_from) + else: + return self.url + + def __repr__(self): + return '' % self + + def __eq__(self, other): + return self.url == other.url + + def __hash__(self): + return hash(self.url) + + @property + def filename(self): + url = self.url + url = url.split('#', 1)[0] + url = url.split('?', 1)[0] + url = url.rstrip('/') + name = posixpath.basename(url) + assert name, ( + 'URL %r produced no filename' % url) + return name + + @property + def scheme(self): + return urlparse.urlsplit(self.url)[0] + + @property + def path(self): + return urlparse.urlsplit(self.url)[2] + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + _egg_fragment_re = re.compile(r'#egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _md5_re = re.compile(r'md5=([a-f0-9]+)') + + @property + def md5_hash(self): + match = self._md5_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + +def get_requirement_from_url(url): + """Get a requirement from the URL, if possible. This looks for #egg + in the URL""" + link = Link(url) + egg_info = link.egg_fragment + if not egg_info: + egg_info = splitext(link.filename)[0] + return package_to_requirement(egg_info) + + +def package_to_requirement(package_name): + """Translate a name like Foo-1.2 to Foo==1.3""" + match = re.search(r'^(.*?)(-dev|-\d.*)', package_name) + if match: + name = match.group(1) + version = match.group(2) + else: + name = package_name + version = '' + if version: + return '%s==%s' % (name, version) + else: + return name + + +def get_mirrors(hostname=None): + """Return the list of mirrors from the last record found on the DNS + entry:: + + >>> from pip.index import get_mirrors + >>> get_mirrors() + ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', + 'd.pypi.python.org'] + + Originally written for the distutils2 project by Alexis Metaireau. + """ + if hostname is None: + hostname = DEFAULT_MIRROR_URL + + # return the last mirror registered on PyPI. + try: + hostname = socket.gethostbyname_ex(hostname)[0] + except socket.gaierror: + return [] + end_letter = hostname.split(".", 1) + + # determine the list from the last one. + return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] + + +def string_range(last): + """Compute the range of string between "a" and last. + + This works for simple "a to z" lists, but also for "a to zz" lists. + """ + for k in range(len(last)): + for x in product(string.ascii_lowercase, repeat=k+1): + result = ''.join(x) + yield result + if result == last: + return + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py new file mode 100644 index 000000000..4254ef2fb --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py @@ -0,0 +1,45 @@ +"""Locations where we look for configs, install stuff, etc""" + +import sys +import os +from distutils import sysconfig + + +def running_under_virtualenv(): + """ + Return True if we're running inside a virtualenv, False otherwise. + + """ + return hasattr(sys, 'real_prefix') + + +if running_under_virtualenv(): + ## FIXME: is build/ a good name? + build_prefix = os.path.join(sys.prefix, 'build') + src_prefix = os.path.join(sys.prefix, 'src') +else: + ## FIXME: this isn't a very good default + build_prefix = os.path.join(os.getcwd(), 'build') + src_prefix = os.path.join(os.getcwd(), 'src') + +# FIXME doesn't account for venv linked to global site-packages + +site_packages = sysconfig.get_python_lib() +user_dir = os.path.expanduser('~') +if sys.platform == 'win32': + bin_py = os.path.join(sys.prefix, 'Scripts') + # buildout uses 'bin' on Windows too? + if not os.path.exists(bin_py): + bin_py = os.path.join(sys.prefix, 'bin') + user_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming + default_storage_dir = os.path.join(user_dir, 'pip') + default_config_file = os.path.join(default_storage_dir, 'pip.ini') + default_log_file = os.path.join(default_storage_dir, 'pip.log') +else: + bin_py = os.path.join(sys.prefix, 'bin') + default_storage_dir = os.path.join(user_dir, '.pip') + default_config_file = os.path.join(default_storage_dir, 'pip.conf') + default_log_file = os.path.join(default_storage_dir, 'pip.log') + # Forcing to use /usr/local/bin for standard Mac OS X framework installs + if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': + bin_py = '/usr/local/bin' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py new file mode 100644 index 000000000..0218ab1ae --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py @@ -0,0 +1,181 @@ +"""Logging +""" + +import sys +import logging + + +class Logger(object): + + """ + Logging object for use in command-line script. Allows ranges of + levels, to avoid some redundancy of displayed information. + """ + + VERBOSE_DEBUG = logging.DEBUG-1 + DEBUG = logging.DEBUG + INFO = logging.INFO + NOTIFY = (logging.INFO+logging.WARN)/2 + WARN = WARNING = logging.WARN + ERROR = logging.ERROR + FATAL = logging.FATAL + + LEVELS = [VERBOSE_DEBUG, DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL] + + def __init__(self): + self.consumers = [] + self.indent = 0 + self.explicit_levels = False + self.in_progress = None + self.in_progress_hanging = False + + def debug(self, msg, *args, **kw): + self.log(self.DEBUG, msg, *args, **kw) + + def info(self, msg, *args, **kw): + self.log(self.INFO, msg, *args, **kw) + + def notify(self, msg, *args, **kw): + self.log(self.NOTIFY, msg, *args, **kw) + + def warn(self, msg, *args, **kw): + self.log(self.WARN, msg, *args, **kw) + + def error(self, msg, *args, **kw): + self.log(self.WARN, msg, *args, **kw) + + def fatal(self, msg, *args, **kw): + self.log(self.FATAL, msg, *args, **kw) + + def log(self, level, msg, *args, **kw): + if args: + if kw: + raise TypeError( + "You may give positional or keyword arguments, not both") + args = args or kw + rendered = None + for consumer_level, consumer in self.consumers: + if self.level_matches(level, consumer_level): + if (self.in_progress_hanging + and consumer in (sys.stdout, sys.stderr)): + self.in_progress_hanging = False + sys.stdout.write('\n') + sys.stdout.flush() + if rendered is None: + if args: + rendered = msg % args + else: + rendered = msg + rendered = ' '*self.indent + rendered + if self.explicit_levels: + ## FIXME: should this be a name, not a level number? + rendered = '%02i %s' % (level, rendered) + if hasattr(consumer, 'write'): + consumer.write(rendered+'\n') + else: + consumer(rendered) + + def start_progress(self, msg): + assert not self.in_progress, ( + "Tried to start_progress(%r) while in_progress %r" + % (msg, self.in_progress)) + if self.level_matches(self.NOTIFY, self._stdout_level()): + sys.stdout.write(' '*self.indent + msg) + sys.stdout.flush() + self.in_progress_hanging = True + else: + self.in_progress_hanging = False + self.in_progress = msg + self.last_message = None + + def end_progress(self, msg='done.'): + assert self.in_progress, ( + "Tried to end_progress without start_progress") + if self.stdout_level_matches(self.NOTIFY): + if not self.in_progress_hanging: + # Some message has been printed out since start_progress + sys.stdout.write('...' + self.in_progress + msg + '\n') + sys.stdout.flush() + else: + # These erase any messages shown with show_progress (besides .'s) + logger.show_progress('') + logger.show_progress('') + sys.stdout.write(msg + '\n') + sys.stdout.flush() + self.in_progress = None + self.in_progress_hanging = False + + def show_progress(self, message=None): + """If we are in a progress scope, and no log messages have been + shown, write out another '.'""" + if self.in_progress_hanging: + if message is None: + sys.stdout.write('.') + sys.stdout.flush() + else: + if self.last_message: + padding = ' ' * max(0, len(self.last_message)-len(message)) + else: + padding = '' + sys.stdout.write('\r%s%s%s%s' % (' '*self.indent, self.in_progress, message, padding)) + sys.stdout.flush() + self.last_message = message + + def stdout_level_matches(self, level): + """Returns true if a message at this level will go to stdout""" + return self.level_matches(level, self._stdout_level()) + + def _stdout_level(self): + """Returns the level that stdout runs at""" + for level, consumer in self.consumers: + if consumer is sys.stdout: + return level + return self.FATAL + + def level_matches(self, level, consumer_level): + """ + >>> l = Logger() + >>> l.level_matches(3, 4) + False + >>> l.level_matches(3, 2) + True + >>> l.level_matches(slice(None, 3), 3) + False + >>> l.level_matches(slice(None, 3), 2) + True + >>> l.level_matches(slice(1, 3), 1) + True + >>> l.level_matches(slice(2, 3), 1) + False + """ + if isinstance(level, slice): + start, stop = level.start, level.stop + if start is not None and start > consumer_level: + return False + if stop is not None or stop <= consumer_level: + return False + return True + else: + return level >= consumer_level + + @classmethod + def level_for_integer(cls, level): + levels = cls.LEVELS + if level < 0: + return levels[0] + if level >= len(levels): + return levels[-1] + return levels[level] + + def move_stdout_to_stderr(self): + to_remove = [] + to_add = [] + for consumer_level, consumer in self.consumers: + if consumer == sys.stdout: + to_remove.append((consumer_level, consumer)) + to_add.append((consumer_level, sys.stderr)) + for item in to_remove: + self.consumers.remove(item) + self.consumers.extend(to_add) + +logger = Logger() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py new file mode 100644 index 000000000..444e7252b --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py @@ -0,0 +1,1432 @@ +import sys +import os +import shutil +import re +import zipfile +import pkg_resources +import tempfile +import urlparse +import urllib2 +import urllib +import ConfigParser +from distutils.sysconfig import get_python_version +from email.FeedParser import FeedParser +from pip.locations import bin_py, running_under_virtualenv +from pip.exceptions import InstallationError, UninstallationError +from pip.vcs import vcs +from pip.log import logger +from pip.util import display_path, rmtree +from pip.util import ask, backup_dir +from pip.util import is_installable_dir, is_local, dist_is_local +from pip.util import renames, normalize_path, egg_link_path +from pip.util import make_path_relative +from pip import call_subprocess +from pip.backwardcompat import any, copytree +from pip.index import Link +from pip.locations import build_prefix +from pip.download import (get_file_content, is_url, url_to_path, + path_to_url, is_archive_file, + unpack_vcs_link, is_vcs_url, is_file_url, + unpack_file_url, unpack_http_url) + + +PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' + + +class InstallRequirement(object): + + def __init__(self, req, comes_from, source_dir=None, editable=False, + url=None, update=True): + if isinstance(req, basestring): + req = pkg_resources.Requirement.parse(req) + self.req = req + self.comes_from = comes_from + self.source_dir = source_dir + self.editable = editable + self.url = url + self._egg_info_path = None + # This holds the pkg_resources.Distribution object if this requirement + # is already available: + self.satisfied_by = None + # This hold the pkg_resources.Distribution object if this requirement + # conflicts with another installed distribution: + self.conflicts_with = None + self._temp_build_dir = None + self._is_bundle = None + # True if the editable should be updated: + self.update = update + # Set to True after successful installation + self.install_succeeded = None + # UninstallPathSet of uninstalled distribution (for possible rollback) + self.uninstalled = None + + @classmethod + def from_editable(cls, editable_req, comes_from=None, default_vcs=None): + name, url = parse_editable(editable_req, default_vcs) + if url.startswith('file:'): + source_dir = url_to_path(url) + else: + source_dir = None + return cls(name, comes_from, source_dir=source_dir, editable=True, url=url) + + @classmethod + def from_line(cls, name, comes_from=None): + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + """ + url = None + name = name.strip() + req = name + path = os.path.normpath(os.path.abspath(name)) + + if is_url(name): + url = name + ## FIXME: I think getting the requirement here is a bad idea: + #req = get_requirement_from_url(url) + req = None + elif os.path.isdir(path) and (os.path.sep in name or name.startswith('.')): + if not is_installable_dir(path): + raise InstallationError("Directory %r is not installable. File 'setup.py' not found." + % name) + url = path_to_url(name) + #req = get_requirement_from_url(url) + req = None + elif is_archive_file(path): + if not os.path.isfile(path): + logger.warn('Requirement %r looks like a filename, but the file does not exist' + % name) + url = path_to_url(name) + #req = get_requirement_from_url(url) + req = None + return cls(req, comes_from, url=url) + + def __str__(self): + if self.req: + s = str(self.req) + if self.url: + s += ' from %s' % self.url + else: + s = self.url + if self.satisfied_by is not None: + s += ' in %s' % display_path(self.satisfied_by.location) + if self.comes_from: + if isinstance(self.comes_from, basestring): + comes_from = self.comes_from + else: + comes_from = self.comes_from.from_path() + if comes_from: + s += ' (from %s)' % comes_from + return s + + def from_path(self): + if self.req is None: + return None + s = str(self.req) + if self.comes_from: + if isinstance(self.comes_from, basestring): + comes_from = self.comes_from + else: + comes_from = self.comes_from.from_path() + if comes_from: + s += '->' + comes_from + return s + + def build_location(self, build_dir, unpack=True): + if self._temp_build_dir is not None: + return self._temp_build_dir + if self.req is None: + self._temp_build_dir = tempfile.mkdtemp('-build', 'pip-') + self._ideal_build_dir = build_dir + return self._temp_build_dir + if self.editable: + name = self.name.lower() + else: + name = self.name + # FIXME: Is there a better place to create the build_dir? (hg and bzr need this) + if not os.path.exists(build_dir): + _make_build_dir(build_dir) + return os.path.join(build_dir, name) + + def correct_build_location(self): + """If the build location was a temporary directory, this will move it + to a new more permanent location""" + if self.source_dir is not None: + return + assert self.req is not None + assert self._temp_build_dir + old_location = self._temp_build_dir + new_build_dir = self._ideal_build_dir + del self._ideal_build_dir + if self.editable: + name = self.name.lower() + else: + name = self.name + new_location = os.path.join(new_build_dir, name) + if not os.path.exists(new_build_dir): + logger.debug('Creating directory %s' % new_build_dir) + _make_build_dir(new_build_dir) + if os.path.exists(new_location): + raise InstallationError( + 'A package already exists in %s; please remove it to continue' + % display_path(new_location)) + logger.debug('Moving package %s from %s to new location %s' + % (self, display_path(old_location), display_path(new_location))) + shutil.move(old_location, new_location) + self._temp_build_dir = new_location + self.source_dir = new_location + self._egg_info_path = None + + @property + def name(self): + if self.req is None: + return None + return self.req.project_name + + @property + def url_name(self): + if self.req is None: + return None + return urllib.quote(self.req.unsafe_name) + + @property + def setup_py(self): + return os.path.join(self.source_dir, 'setup.py') + + def run_egg_info(self, force_root_egg_info=False): + assert self.source_dir + if self.name: + logger.notify('Running setup.py egg_info for package %s' % self.name) + else: + logger.notify('Running setup.py egg_info for package from %s' % self.url) + logger.indent += 2 + try: + script = self._run_setup_py + script = script.replace('__SETUP_PY__', repr(self.setup_py)) + script = script.replace('__PKG_NAME__', repr(self.name)) + # We can't put the .egg-info files at the root, because then the source code will be mistaken + # for an installed egg, causing problems + if self.editable or force_root_egg_info: + egg_base_option = [] + else: + egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') + if not os.path.exists(egg_info_dir): + os.makedirs(egg_info_dir) + egg_base_option = ['--egg-base', 'pip-egg-info'] + call_subprocess( + [sys.executable, '-c', script, 'egg_info'] + egg_base_option, + cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False, + command_level=logger.VERBOSE_DEBUG, + command_desc='python setup.py egg_info') + finally: + logger.indent -= 2 + if not self.req: + self.req = pkg_resources.Requirement.parse(self.pkg_info()['Name']) + self.correct_build_location() + + ## FIXME: This is a lame hack, entirely for PasteScript which has + ## a self-provided entry point that causes this awkwardness + _run_setup_py = """ +__file__ = __SETUP_PY__ +from setuptools.command import egg_info +def replacement_run(self): + self.mkpath(self.egg_info) + installer = self.distribution.fetch_build_egg + for ep in egg_info.iter_entry_points('egg_info.writers'): + # require=False is the change we're making: + writer = ep.load(require=False) + if writer: + writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name)) + self.find_sources() +egg_info.egg_info.run = replacement_run +execfile(__file__) +""" + + def egg_info_data(self, filename): + if self.satisfied_by is not None: + if not self.satisfied_by.has_metadata(filename): + return None + return self.satisfied_by.get_metadata(filename) + assert self.source_dir + filename = self.egg_info_path(filename) + if not os.path.exists(filename): + return None + fp = open(filename, 'r') + data = fp.read() + fp.close() + return data + + def egg_info_path(self, filename): + if self._egg_info_path is None: + if self.editable: + base = self.source_dir + else: + base = os.path.join(self.source_dir, 'pip-egg-info') + filenames = os.listdir(base) + if self.editable: + filenames = [] + for root, dirs, files in os.walk(base): + for dir in vcs.dirnames: + if dir in dirs: + dirs.remove(dir) + for dir in dirs: + # Don't search in anything that looks like a virtualenv environment + if (os.path.exists(os.path.join(root, dir, 'bin', 'python')) + or os.path.exists(os.path.join(root, dir, 'Scripts', 'Python.exe'))): + dirs.remove(dir) + # Also don't search through tests + if dir == 'test' or dir == 'tests': + dirs.remove(dir) + filenames.extend([os.path.join(root, dir) + for dir in dirs]) + filenames = [f for f in filenames if f.endswith('.egg-info')] + + if not filenames: + raise InstallationError('No files/directores in %s (from %s)' % (base, filename)) + assert filenames, "No files/directories in %s (from %s)" % (base, filename) + + # if we have more than one match, we pick the toplevel one. This can + # easily be the case if there is a dist folder which contains an + # extracted tarball for testing purposes. + if len(filenames) > 1: + filenames.sort(key=lambda x: x.count(os.path.sep) + + (os.path.altsep and + x.count(os.path.altsep) or 0)) + self._egg_info_path = os.path.join(base, filenames[0]) + return os.path.join(self._egg_info_path, filename) + + def egg_info_lines(self, filename): + data = self.egg_info_data(filename) + if not data: + return [] + result = [] + for line in data.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + result.append(line) + return result + + def pkg_info(self): + p = FeedParser() + data = self.egg_info_data('PKG-INFO') + if not data: + logger.warn('No PKG-INFO file found in %s' % display_path(self.egg_info_path('PKG-INFO'))) + p.feed(data or '') + return p.close() + + @property + def dependency_links(self): + return self.egg_info_lines('dependency_links.txt') + + _requirements_section_re = re.compile(r'\[(.*?)\]') + + def requirements(self, extras=()): + in_extra = None + for line in self.egg_info_lines('requires.txt'): + match = self._requirements_section_re.match(line) + if match: + in_extra = match.group(1) + continue + if in_extra and in_extra not in extras: + # Skip requirement for an extra we aren't requiring + continue + yield line + + @property + def absolute_versions(self): + for qualifier, version in self.req.specs: + if qualifier == '==': + yield version + + @property + def installed_version(self): + return self.pkg_info()['version'] + + def assert_source_matches_version(self): + assert self.source_dir + if self.comes_from is None: + # We don't check the versions of things explicitly installed. + # This makes, e.g., "pip Package==dev" possible + return + version = self.installed_version + if version not in self.req: + logger.fatal( + 'Source in %s has the version %s, which does not match the requirement %s' + % (display_path(self.source_dir), version, self)) + raise InstallationError( + 'Source in %s has version %s that conflicts with %s' + % (display_path(self.source_dir), version, self)) + else: + logger.debug('Source in %s has version %s, which satisfies requirement %s' + % (display_path(self.source_dir), version, self)) + + def update_editable(self, obtain=True): + if not self.url: + logger.info("Cannot update repository at %s; repository location is unknown" % self.source_dir) + return + assert self.editable + assert self.source_dir + if self.url.startswith('file:'): + # Static paths don't get updated + return + assert '+' in self.url, "bad url: %r" % self.url + if not self.update: + return + vc_type, url = self.url.split('+', 1) + backend = vcs.get_backend(vc_type) + if backend: + vcs_backend = backend(self.url) + if obtain: + vcs_backend.obtain(self.source_dir) + else: + vcs_backend.export(self.source_dir) + else: + assert 0, ( + 'Unexpected version control type (in %s): %s' + % (self.url, vc_type)) + + def uninstall(self, auto_confirm=False): + """ + Uninstall the distribution currently satisfying this requirement. + + Prompts before removing or modifying files unless + ``auto_confirm`` is True. + + Refuses to delete or modify files outside of ``sys.prefix`` - + thus uninstallation within a virtual environment can only + modify that virtual environment, even if the virtualenv is + linked to global site-packages. + + """ + if not self.check_if_exists(): + raise UninstallationError("Cannot uninstall requirement %s, not installed" % (self.name,)) + dist = self.satisfied_by or self.conflicts_with + + paths_to_remove = UninstallPathSet(dist) + + pip_egg_info_path = os.path.join(dist.location, + dist.egg_name()) + '.egg-info' + easy_install_egg = dist.egg_name() + '.egg' + develop_egg_link = egg_link_path(dist) + if os.path.exists(pip_egg_info_path): + # package installed by pip + paths_to_remove.add(pip_egg_info_path) + if dist.has_metadata('installed-files.txt'): + for installed_file in dist.get_metadata('installed-files.txt').splitlines(): + path = os.path.normpath(os.path.join(pip_egg_info_path, installed_file)) + paths_to_remove.add(path) + if dist.has_metadata('top_level.txt'): + if dist.has_metadata('namespace_packages.txt'): + namespaces = dist.get_metadata('namespace_packages.txt') + else: + namespaces = [] + for top_level_pkg in [p for p + in dist.get_metadata('top_level.txt').splitlines() + if p and p not in namespaces]: + path = os.path.join(dist.location, top_level_pkg) + paths_to_remove.add(path) + paths_to_remove.add(path + '.py') + paths_to_remove.add(path + '.pyc') + + elif dist.location.endswith(easy_install_egg): + # package installed by easy_install + paths_to_remove.add(dist.location) + easy_install_pth = os.path.join(os.path.dirname(dist.location), + 'easy-install.pth') + paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) + + elif os.path.isfile(develop_egg_link): + # develop egg + fh = open(develop_egg_link, 'r') + link_pointer = os.path.normcase(fh.readline().strip()) + fh.close() + assert (link_pointer == dist.location), 'Egg-link %s does not match installed location of %s (at %s)' % (link_pointer, self.name, dist.location) + paths_to_remove.add(develop_egg_link) + easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), + 'easy-install.pth') + paths_to_remove.add_pth(easy_install_pth, dist.location) + + # find distutils scripts= scripts + if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): + for script in dist.metadata_listdir('scripts'): + paths_to_remove.add(os.path.join(bin_py, script)) + if sys.platform == 'win32': + paths_to_remove.add(os.path.join(bin_py, script) + '.bat') + + # find console_scripts + if dist.has_metadata('entry_points.txt'): + config = ConfigParser.SafeConfigParser() + config.readfp(FakeFile(dist.get_metadata_lines('entry_points.txt'))) + if config.has_section('console_scripts'): + for name, value in config.items('console_scripts'): + paths_to_remove.add(os.path.join(bin_py, name)) + if sys.platform == 'win32': + paths_to_remove.add(os.path.join(bin_py, name) + '.exe') + paths_to_remove.add(os.path.join(bin_py, name) + '.exe.manifest') + paths_to_remove.add(os.path.join(bin_py, name) + '-script.py') + + paths_to_remove.remove(auto_confirm) + self.uninstalled = paths_to_remove + + def rollback_uninstall(self): + if self.uninstalled: + self.uninstalled.rollback() + else: + logger.error("Can't rollback %s, nothing uninstalled." + % (self.project_name,)) + + def commit_uninstall(self): + if self.uninstalled: + self.uninstalled.commit() + else: + logger.error("Can't commit %s, nothing uninstalled." + % (self.project_name,)) + + def archive(self, build_dir): + assert self.source_dir + create_archive = True + archive_name = '%s-%s.zip' % (self.name, self.installed_version) + archive_path = os.path.join(build_dir, archive_name) + if os.path.exists(archive_path): + response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' + % display_path(archive_path), ('i', 'w', 'b')) + if response == 'i': + create_archive = False + elif response == 'w': + logger.warn('Deleting %s' % display_path(archive_path)) + os.remove(archive_path) + elif response == 'b': + dest_file = backup_dir(archive_path) + logger.warn('Backing up %s to %s' + % (display_path(archive_path), display_path(dest_file))) + shutil.move(archive_path, dest_file) + if create_archive: + zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) + dir = os.path.normcase(os.path.abspath(self.source_dir)) + for dirpath, dirnames, filenames in os.walk(dir): + if 'pip-egg-info' in dirnames: + dirnames.remove('pip-egg-info') + for dirname in dirnames: + dirname = os.path.join(dirpath, dirname) + name = self._clean_zip_name(dirname, dir) + zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') + zipdir.external_attr = 0755 << 16L + zip.writestr(zipdir, '') + for filename in filenames: + if filename == PIP_DELETE_MARKER_FILENAME: + continue + filename = os.path.join(dirpath, filename) + name = self._clean_zip_name(filename, dir) + zip.write(filename, self.name + '/' + name) + zip.close() + logger.indent -= 2 + logger.notify('Saved %s' % display_path(archive_path)) + + def _clean_zip_name(self, name, prefix): + assert name.startswith(prefix+os.path.sep), ( + "name %r doesn't start with prefix %r" % (name, prefix)) + name = name[len(prefix)+1:] + name = name.replace(os.path.sep, '/') + return name + + def install(self, install_options, global_options=()): + if self.editable: + self.install_editable(install_options, global_options) + return + temp_location = tempfile.mkdtemp('-record', 'pip-') + record_filename = os.path.join(temp_location, 'install-record.txt') + try: + + install_args = [ + sys.executable, '-c', + "import setuptools;__file__=%r;"\ + "execfile(__file__)" % self.setup_py] +\ + list(global_options) + [ + 'install', + '--single-version-externally-managed', + '--record', record_filename] + + if running_under_virtualenv(): + ## FIXME: I'm not sure if this is a reasonable location; probably not + ## but we can't put it in the default location, as that is a virtualenv symlink that isn't writable + install_args += ['--install-headers', + os.path.join(sys.prefix, 'include', 'site', + 'python' + get_python_version())] + logger.notify('Running setup.py install for %s' % self.name) + logger.indent += 2 + try: + call_subprocess(install_args + install_options, + cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False) + finally: + logger.indent -= 2 + if not os.path.exists(record_filename): + logger.notify('Record file %s not found' % record_filename) + return + self.install_succeeded = True + f = open(record_filename) + for line in f: + line = line.strip() + if line.endswith('.egg-info'): + egg_info_dir = line + break + else: + logger.warn('Could not find .egg-info directory in install record for %s' % self) + ## FIXME: put the record somewhere + ## FIXME: should this be an error? + return + f.close() + new_lines = [] + f = open(record_filename) + for line in f: + filename = line.strip() + if os.path.isdir(filename): + filename += os.path.sep + new_lines.append(make_path_relative(filename, egg_info_dir)) + f.close() + f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') + f.write('\n'.join(new_lines)+'\n') + f.close() + finally: + if os.path.exists(record_filename): + os.remove(record_filename) + os.rmdir(temp_location) + + def remove_temporary_source(self): + """Remove the source files from this requirement, if they are marked + for deletion""" + if self.is_bundle or os.path.exists(self.delete_marker_filename): + logger.info('Removing source in %s' % self.source_dir) + if self.source_dir: + rmtree(self.source_dir) + self.source_dir = None + if self._temp_build_dir and os.path.exists(self._temp_build_dir): + rmtree(self._temp_build_dir) + self._temp_build_dir = None + + def install_editable(self, install_options, global_options=()): + logger.notify('Running setup.py develop for %s' % self.name) + logger.indent += 2 + try: + ## FIXME: should we do --install-headers here too? + call_subprocess( + [sys.executable, '-c', + "import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py)] + + list(global_options) + ['develop', '--no-deps'] + list(install_options), + + cwd=self.source_dir, filter_stdout=self._filter_install, + show_stdout=False) + finally: + logger.indent -= 2 + self.install_succeeded = True + + def _filter_install(self, line): + level = logger.NOTIFY + for regex in [r'^running .*', r'^writing .*', '^creating .*', '^[Cc]opying .*', + r'^reading .*', r"^removing .*\.egg-info' \(and everything under it\)$", + r'^byte-compiling ', + # Not sure what this warning is, but it seems harmless: + r"^warning: manifest_maker: standard file '-c' not found$"]: + if re.search(regex, line.strip()): + level = logger.INFO + break + return (level, line) + + def check_if_exists(self): + """Find an installed distribution that satisfies or conflicts + with this requirement, and set self.satisfied_by or + self.conflicts_with appropriately.""" + if self.req is None: + return False + try: + self.satisfied_by = pkg_resources.get_distribution(self.req) + except pkg_resources.DistributionNotFound: + return False + except pkg_resources.VersionConflict: + self.conflicts_with = pkg_resources.get_distribution(self.req.project_name) + return True + + @property + def is_bundle(self): + if self._is_bundle is not None: + return self._is_bundle + base = self._temp_build_dir + if not base: + ## FIXME: this doesn't seem right: + return False + self._is_bundle = (os.path.exists(os.path.join(base, 'pip-manifest.txt')) + or os.path.exists(os.path.join(base, 'pyinstall-manifest.txt'))) + return self._is_bundle + + def bundle_requirements(self): + for dest_dir in self._bundle_editable_dirs: + package = os.path.basename(dest_dir) + ## FIXME: svnism: + for vcs_backend in vcs.backends: + url = rev = None + vcs_bundle_file = os.path.join( + dest_dir, vcs_backend.bundle_file) + if os.path.exists(vcs_bundle_file): + vc_type = vcs_backend.name + fp = open(vcs_bundle_file) + content = fp.read() + fp.close() + url, rev = vcs_backend().parse_vcs_bundle_file(content) + break + if url: + url = '%s+%s@%s' % (vc_type, url, rev) + else: + url = None + yield InstallRequirement( + package, self, editable=True, url=url, + update=False, source_dir=dest_dir) + for dest_dir in self._bundle_build_dirs: + package = os.path.basename(dest_dir) + yield InstallRequirement( + package, self, + source_dir=dest_dir) + + def move_bundle_files(self, dest_build_dir, dest_src_dir): + base = self._temp_build_dir + assert base + src_dir = os.path.join(base, 'src') + build_dir = os.path.join(base, 'build') + bundle_build_dirs = [] + bundle_editable_dirs = [] + for source_dir, dest_dir, dir_collection in [ + (src_dir, dest_src_dir, bundle_editable_dirs), + (build_dir, dest_build_dir, bundle_build_dirs)]: + if os.path.exists(source_dir): + for dirname in os.listdir(source_dir): + dest = os.path.join(dest_dir, dirname) + dir_collection.append(dest) + if os.path.exists(dest): + logger.warn('The directory %s (containing package %s) already exists; cannot move source from bundle %s' + % (dest, dirname, self)) + continue + if not os.path.exists(dest_dir): + logger.info('Creating directory %s' % dest_dir) + os.makedirs(dest_dir) + shutil.move(os.path.join(source_dir, dirname), dest) + if not os.listdir(source_dir): + os.rmdir(source_dir) + self._temp_build_dir = None + self._bundle_build_dirs = bundle_build_dirs + self._bundle_editable_dirs = bundle_editable_dirs + + @property + def delete_marker_filename(self): + assert self.source_dir + return os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME) + + +DELETE_MARKER_MESSAGE = '''\ +This file is placed here by pip to indicate the source was put +here by pip. + +Once this package is successfully installed this source code will be +deleted (unless you remove this file). +''' + + +class RequirementSet(object): + + def __init__(self, build_dir, src_dir, download_dir, download_cache=None, + upgrade=False, ignore_installed=False, + ignore_dependencies=False): + self.build_dir = build_dir + self.src_dir = src_dir + self.download_dir = download_dir + self.download_cache = download_cache + self.upgrade = upgrade + self.ignore_installed = ignore_installed + self.requirements = {} + # Mapping of alias: real_name + self.requirement_aliases = {} + self.unnamed_requirements = [] + self.ignore_dependencies = ignore_dependencies + self.successfully_downloaded = [] + self.successfully_installed = [] + self.reqs_to_cleanup = [] + + def __str__(self): + reqs = [req for req in self.requirements.values() + if not req.comes_from] + reqs.sort(key=lambda req: req.name.lower()) + return ' '.join([str(req.req) for req in reqs]) + + def add_requirement(self, install_req): + name = install_req.name + if not name: + self.unnamed_requirements.append(install_req) + else: + if self.has_requirement(name): + raise InstallationError( + 'Double requirement given: %s (aready in %s, name=%r)' + % (install_req, self.get_requirement(name), name)) + self.requirements[name] = install_req + ## FIXME: what about other normalizations? E.g., _ vs. -? + if name.lower() != name: + self.requirement_aliases[name.lower()] = name + + def has_requirement(self, project_name): + for name in project_name, project_name.lower(): + if name in self.requirements or name in self.requirement_aliases: + return True + return False + + @property + def has_requirements(self): + return self.requirements.values() or self.unnamed_requirements + + @property + def has_editables(self): + if any(req.editable for req in self.requirements.values()): + return True + if any(req.editable for req in self.unnamed_requirements): + return True + return False + + @property + def is_download(self): + if self.download_dir: + self.download_dir = os.path.expanduser(self.download_dir) + if os.path.exists(self.download_dir): + return True + else: + logger.fatal('Could not find download directory') + raise InstallationError( + "Could not find or access download directory '%s'" + % display_path(self.download_dir)) + return False + + def get_requirement(self, project_name): + for name in project_name, project_name.lower(): + if name in self.requirements: + return self.requirements[name] + if name in self.requirement_aliases: + return self.requirements[self.requirement_aliases[name]] + raise KeyError("No project with the name %r" % project_name) + + def uninstall(self, auto_confirm=False): + for req in self.requirements.values(): + req.uninstall(auto_confirm=auto_confirm) + req.commit_uninstall() + + def locate_files(self): + ## FIXME: duplicates code from install_files; relevant code should + ## probably be factored out into a separate method + unnamed = list(self.unnamed_requirements) + reqs = self.requirements.values() + while reqs or unnamed: + if unnamed: + req_to_install = unnamed.pop(0) + else: + req_to_install = reqs.pop(0) + install_needed = True + if not self.ignore_installed and not req_to_install.editable: + req_to_install.check_if_exists() + if req_to_install.satisfied_by: + if self.upgrade: + req_to_install.conflicts_with = req_to_install.satisfied_by + req_to_install.satisfied_by = None + else: + install_needed = False + if req_to_install.satisfied_by: + logger.notify('Requirement already satisfied ' + '(use --upgrade to upgrade): %s' + % req_to_install) + + if req_to_install.editable: + if req_to_install.source_dir is None: + req_to_install.source_dir = req_to_install.build_location(self.src_dir) + elif install_needed: + req_to_install.source_dir = req_to_install.build_location(self.build_dir, not self.is_download) + + if req_to_install.source_dir is not None and not os.path.isdir(req_to_install.source_dir): + raise InstallationError('Could not install requirement %s ' + 'because source folder %s does not exist ' + '(perhaps --no-download was used without first running ' + 'an equivalent install with --no-install?)' + % (req_to_install, req_to_install.source_dir)) + + def prepare_files(self, finder, force_root_egg_info=False, bundle=False): + """Prepare process. Create temp directories, download and/or unpack files.""" + unnamed = list(self.unnamed_requirements) + reqs = self.requirements.values() + while reqs or unnamed: + if unnamed: + req_to_install = unnamed.pop(0) + else: + req_to_install = reqs.pop(0) + install = True + if not self.ignore_installed and not req_to_install.editable: + req_to_install.check_if_exists() + if req_to_install.satisfied_by: + if self.upgrade: + req_to_install.conflicts_with = req_to_install.satisfied_by + req_to_install.satisfied_by = None + else: + install = False + if req_to_install.satisfied_by: + logger.notify('Requirement already satisfied ' + '(use --upgrade to upgrade): %s' + % req_to_install) + if req_to_install.editable: + logger.notify('Obtaining %s' % req_to_install) + elif install: + if req_to_install.url and req_to_install.url.lower().startswith('file:'): + logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) + else: + logger.notify('Downloading/unpacking %s' % req_to_install) + logger.indent += 2 + try: + is_bundle = False + if req_to_install.editable: + if req_to_install.source_dir is None: + location = req_to_install.build_location(self.src_dir) + req_to_install.source_dir = location + else: + location = req_to_install.source_dir + if not os.path.exists(self.build_dir): + _make_build_dir(self.build_dir) + req_to_install.update_editable(not self.is_download) + if self.is_download: + req_to_install.run_egg_info() + req_to_install.archive(self.download_dir) + else: + req_to_install.run_egg_info() + elif install: + ##@@ if filesystem packages are not marked + ##editable in a req, a non deterministic error + ##occurs when the script attempts to unpack the + ##build directory + + location = req_to_install.build_location(self.build_dir, not self.is_download) + ## FIXME: is the existance of the checkout good enough to use it? I don't think so. + unpack = True + if not os.path.exists(os.path.join(location, 'setup.py')): + ## FIXME: this won't upgrade when there's an existing package unpacked in `location` + if req_to_install.url is None: + url = finder.find_requirement(req_to_install, upgrade=self.upgrade) + else: + ## FIXME: should req_to_install.url already be a link? + url = Link(req_to_install.url) + assert url + if url: + try: + self.unpack_url(url, location, self.is_download) + except urllib2.HTTPError, e: + logger.fatal('Could not install requirement %s because of error %s' + % (req_to_install, e)) + raise InstallationError( + 'Could not install requirement %s because of HTTP error %s for URL %s' + % (req_to_install, e, url)) + else: + unpack = False + if unpack: + is_bundle = req_to_install.is_bundle + url = None + if is_bundle: + req_to_install.move_bundle_files(self.build_dir, self.src_dir) + for subreq in req_to_install.bundle_requirements(): + reqs.append(subreq) + self.add_requirement(subreq) + elif self.is_download: + req_to_install.source_dir = location + if url and url.scheme in vcs.all_schemes: + req_to_install.run_egg_info() + req_to_install.archive(self.download_dir) + else: + req_to_install.source_dir = location + req_to_install.run_egg_info() + if force_root_egg_info: + # We need to run this to make sure that the .egg-info/ + # directory is created for packing in the bundle + req_to_install.run_egg_info(force_root_egg_info=True) + req_to_install.assert_source_matches_version() + #@@ sketchy way of identifying packages not grabbed from an index + if bundle and req_to_install.url: + self.copy_to_build_dir(req_to_install) + if not is_bundle and not self.is_download: + ## FIXME: shouldn't be globally added: + finder.add_dependency_links(req_to_install.dependency_links) + ## FIXME: add extras in here: + if not self.ignore_dependencies: + for req in req_to_install.requirements(): + try: + name = pkg_resources.Requirement.parse(req).project_name + except ValueError, e: + ## FIXME: proper warning + logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) + continue + if self.has_requirement(name): + ## FIXME: check for conflict + continue + subreq = InstallRequirement(req, req_to_install) + reqs.append(subreq) + self.add_requirement(subreq) + if req_to_install.name not in self.requirements: + self.requirements[req_to_install.name] = req_to_install + else: + self.reqs_to_cleanup.append(req_to_install) + if install: + self.successfully_downloaded.append(req_to_install) + if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): + self.copy_to_build_dir(req_to_install) + finally: + logger.indent -= 2 + + def cleanup_files(self, bundle=False): + """Clean up files, remove builds.""" + logger.notify('Cleaning up...') + logger.indent += 2 + for req in self.reqs_to_cleanup: + req.remove_temporary_source() + + remove_dir = [] + if self._pip_has_created_build_dir(): + remove_dir.append(self.build_dir) + + # The source dir of a bundle can always be removed. + if bundle: + remove_dir.append(self.src_dir) + + for dir in remove_dir: + if os.path.exists(dir): + logger.info('Removing temporary dir %s...' % dir) + rmtree(dir) + + logger.indent -= 2 + + def _pip_has_created_build_dir(self): + return (self.build_dir == build_prefix and + os.path.exists(os.path.join(self.build_dir, PIP_DELETE_MARKER_FILENAME))) + + def copy_to_build_dir(self, req_to_install): + target_dir = req_to_install.editable and self.src_dir or self.build_dir + logger.info("Copying %s to %s" %(req_to_install.name, target_dir)) + dest = os.path.join(target_dir, req_to_install.name) + copytree(req_to_install.source_dir, dest) + call_subprocess(["python", "%s/setup.py"%dest, "clean"]) + + def unpack_url(self, link, location, only_download=False): + if only_download: + location = self.download_dir + if is_vcs_url(link): + return unpack_vcs_link(link, location, only_download) + elif is_file_url(link): + return unpack_file_url(link, location) + else: + if self.download_cache: + self.download_cache = os.path.expanduser(self.download_cache) + return unpack_http_url(link, location, self.download_cache, only_download) + + def install(self, install_options, global_options=()): + """Install everything in this set (after having downloaded and unpacked the packages)""" + to_install = sorted([r for r in self.requirements.values() + if self.upgrade or not r.satisfied_by], + key=lambda p: p.name.lower()) + if to_install: + logger.notify('Installing collected packages: %s' % (', '.join([req.name for req in to_install]))) + logger.indent += 2 + try: + for requirement in to_install: + if requirement.conflicts_with: + logger.notify('Found existing installation: %s' + % requirement.conflicts_with) + logger.indent += 2 + try: + requirement.uninstall(auto_confirm=True) + finally: + logger.indent -= 2 + try: + requirement.install(install_options, global_options) + except: + # if install did not succeed, rollback previous uninstall + if requirement.conflicts_with and not requirement.install_succeeded: + requirement.rollback_uninstall() + raise + else: + if requirement.conflicts_with and requirement.install_succeeded: + requirement.commit_uninstall() + requirement.remove_temporary_source() + finally: + logger.indent -= 2 + self.successfully_installed = to_install + + def create_bundle(self, bundle_filename): + ## FIXME: can't decide which is better; zip is easier to read + ## random files from, but tar.bz2 is smaller and not as lame a + ## format. + + ## FIXME: this file should really include a manifest of the + ## packages, maybe some other metadata files. It would make + ## it easier to detect as well. + zip = zipfile.ZipFile(bundle_filename, 'w', zipfile.ZIP_DEFLATED) + vcs_dirs = [] + for dir, basename in (self.build_dir, 'build'), (self.src_dir, 'src'): + dir = os.path.normcase(os.path.abspath(dir)) + for dirpath, dirnames, filenames in os.walk(dir): + for backend in vcs.backends: + vcs_backend = backend() + vcs_url = vcs_rev = None + if vcs_backend.dirname in dirnames: + for vcs_dir in vcs_dirs: + if dirpath.startswith(vcs_dir): + # vcs bundle file already in parent directory + break + else: + vcs_url, vcs_rev = vcs_backend.get_info( + os.path.join(dir, dirpath)) + vcs_dirs.append(dirpath) + vcs_bundle_file = vcs_backend.bundle_file + vcs_guide = vcs_backend.guide % {'url': vcs_url, + 'rev': vcs_rev} + dirnames.remove(vcs_backend.dirname) + break + if 'pip-egg-info' in dirnames: + dirnames.remove('pip-egg-info') + for dirname in dirnames: + dirname = os.path.join(dirpath, dirname) + name = self._clean_zip_name(dirname, dir) + zip.writestr(basename + '/' + name + '/', '') + for filename in filenames: + if filename == PIP_DELETE_MARKER_FILENAME: + continue + filename = os.path.join(dirpath, filename) + name = self._clean_zip_name(filename, dir) + zip.write(filename, basename + '/' + name) + if vcs_url: + name = os.path.join(dirpath, vcs_bundle_file) + name = self._clean_zip_name(name, dir) + zip.writestr(basename + '/' + name, vcs_guide) + + zip.writestr('pip-manifest.txt', self.bundle_requirements()) + zip.close() + + BUNDLE_HEADER = '''\ +# This is a pip bundle file, that contains many source packages +# that can be installed as a group. You can install this like: +# pip this_file.zip +# The rest of the file contains a list of all the packages included: +''' + + def bundle_requirements(self): + parts = [self.BUNDLE_HEADER] + for req in sorted( + [req for req in self.requirements.values() + if not req.comes_from], + key=lambda x: x.name): + parts.append('%s==%s\n' % (req.name, req.installed_version)) + parts.append('# These packages were installed to satisfy the above requirements:\n') + for req in sorted( + [req for req in self.requirements.values() + if req.comes_from], + key=lambda x: x.name): + parts.append('%s==%s\n' % (req.name, req.installed_version)) + ## FIXME: should we do something with self.unnamed_requirements? + return ''.join(parts) + + def _clean_zip_name(self, name, prefix): + assert name.startswith(prefix+os.path.sep), ( + "name %r doesn't start with prefix %r" % (name, prefix)) + name = name[len(prefix)+1:] + name = name.replace(os.path.sep, '/') + return name + + +def _make_build_dir(build_dir): + os.makedirs(build_dir) + _write_delete_marker_message(os.path.join(build_dir, PIP_DELETE_MARKER_FILENAME)) + + +def _write_delete_marker_message(filepath): + marker_fp = open(filepath, 'w') + marker_fp.write(DELETE_MARKER_MESSAGE) + marker_fp.close() + + +_scheme_re = re.compile(r'^(http|https|file):', re.I) + + +def parse_requirements(filename, finder=None, comes_from=None, options=None): + skip_match = None + skip_regex = options.skip_requirements_regex + if skip_regex: + skip_match = re.compile(skip_regex) + filename, content = get_file_content(filename, comes_from=comes_from) + for line_number, line in enumerate(content.splitlines()): + line_number += 1 + line = line.strip() + if not line or line.startswith('#'): + continue + if skip_match and skip_match.search(line): + continue + if line.startswith('-r') or line.startswith('--requirement'): + if line.startswith('-r'): + req_url = line[2:].strip() + else: + req_url = line[len('--requirement'):].strip().strip('=') + if _scheme_re.search(filename): + # Relative to a URL + req_url = urlparse.urljoin(req_url, filename) + elif not _scheme_re.search(req_url): + req_url = os.path.join(os.path.dirname(filename), req_url) + for item in parse_requirements(req_url, finder, comes_from=filename, options=options): + yield item + elif line.startswith('-Z') or line.startswith('--always-unzip'): + # No longer used, but previously these were used in + # requirement files, so we'll ignore. + pass + elif line.startswith('-f') or line.startswith('--find-links'): + if line.startswith('-f'): + line = line[2:].strip() + else: + line = line[len('--find-links'):].strip().lstrip('=') + ## FIXME: it would be nice to keep track of the source of + ## the find_links: + if finder: + finder.find_links.append(line) + elif line.startswith('-i') or line.startswith('--index-url'): + if line.startswith('-i'): + line = line[2:].strip() + else: + line = line[len('--index-url'):].strip().lstrip('=') + if finder: + finder.index_urls = [line] + elif line.startswith('--extra-index-url'): + line = line[len('--extra-index-url'):].strip().lstrip('=') + if finder: + finder.index_urls.append(line) + else: + comes_from = '-r %s (line %s)' % (filename, line_number) + if line.startswith('-e') or line.startswith('--editable'): + if line.startswith('-e'): + line = line[2:].strip() + else: + line = line[len('--editable'):].strip() + req = InstallRequirement.from_editable( + line, comes_from=comes_from, default_vcs=options.default_vcs) + else: + req = InstallRequirement.from_line(line, comes_from) + yield req + + +def parse_editable(editable_req, default_vcs=None): + """Parses svn+http://blahblah@rev#egg=Foobar into a requirement + (Foobar) and a URL""" + url = editable_req + if os.path.isdir(url) and os.path.exists(os.path.join(url, 'setup.py')): + # Treating it as code that has already been checked out + url = path_to_url(url) + if url.lower().startswith('file:'): + return None, url + for version_control in vcs: + if url.lower().startswith('%s:' % version_control): + url = '%s+%s' % (version_control, url) + if '+' not in url: + if default_vcs: + url = default_vcs + '+' + url + else: + raise InstallationError( + '--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req) + vc_type = url.split('+', 1)[0].lower() + if not vcs.get_backend(vc_type): + raise InstallationError( + 'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req) + match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req) + if (not match or not match.group(1)) and vcs.get_backend(vc_type): + parts = [p for p in editable_req.split('#', 1)[0].split('/') if p] + if parts[-2] in ('tags', 'branches', 'tag', 'branch'): + req = parts[-3] + elif parts[-1] == 'trunk': + req = parts[-2] + else: + raise InstallationError( + '--editable=%s is not the right format; it must have #egg=Package' + % editable_req) + else: + req = match.group(1) + ## FIXME: use package_to_requirement? + match = re.search(r'^(.*?)(?:-dev|-\d.*)', req) + if match: + # Strip off -dev, -0.2, etc. + req = match.group(1) + return req, url + + +class UninstallPathSet(object): + """A set of file paths to be removed in the uninstallation of a + requirement.""" + def __init__(self, dist): + self.paths = set() + self._refuse = set() + self.pth = {} + self.dist = dist + self.save_dir = None + self._moved_paths = [] + + def _permitted(self, path): + """ + Return True if the given path is one we are permitted to + remove/modify, False otherwise. + + """ + return is_local(path) + + def _can_uninstall(self): + if not dist_is_local(self.dist): + logger.notify("Not uninstalling %s at %s, outside environment %s" + % (self.dist.project_name, normalize_path(self.dist.location), sys.prefix)) + return False + return True + + def add(self, path): + path = normalize_path(path) + if not os.path.exists(path): + return + if self._permitted(path): + self.paths.add(path) + else: + self._refuse.add(path) + + def add_pth(self, pth_file, entry): + pth_file = normalize_path(pth_file) + if self._permitted(pth_file): + if pth_file not in self.pth: + self.pth[pth_file] = UninstallPthEntries(pth_file) + self.pth[pth_file].add(entry) + else: + self._refuse.add(pth_file) + + def compact(self, paths): + """Compact a path set to contain the minimal number of paths + necessary to contain all paths in the set. If /a/path/ and + /a/path/to/a/file.txt are both in the set, leave only the + shorter path.""" + short_paths = set() + for path in sorted(paths, key=len): + if not any([(path.startswith(shortpath) and + path[len(shortpath.rstrip(os.path.sep))] == os.path.sep) + for shortpath in short_paths]): + short_paths.add(path) + return short_paths + + def _stash(self, path): + return os.path.join( + self.save_dir, os.path.splitdrive(path)[1].lstrip(os.path.sep)) + + def remove(self, auto_confirm=False): + """Remove paths in ``self.paths`` with confirmation (unless + ``auto_confirm`` is True).""" + if not self._can_uninstall(): + return + logger.notify('Uninstalling %s:' % self.dist.project_name) + logger.indent += 2 + paths = sorted(self.compact(self.paths)) + try: + if auto_confirm: + response = 'y' + else: + for path in paths: + logger.notify(path) + response = ask('Proceed (y/n)? ', ('y', 'n')) + if self._refuse: + logger.notify('Not removing or modifying (outside of prefix):') + for path in self.compact(self._refuse): + logger.notify(path) + if response == 'y': + self.save_dir = tempfile.mkdtemp(suffix='-uninstall', + prefix='pip-') + for path in paths: + new_path = self._stash(path) + logger.info('Removing file or directory %s' % path) + self._moved_paths.append(path) + renames(path, new_path) + for pth in self.pth.values(): + pth.remove() + logger.notify('Successfully uninstalled %s' % self.dist.project_name) + + finally: + logger.indent -= 2 + + def rollback(self): + """Rollback the changes previously made by remove().""" + if self.save_dir is None: + logger.error("Can't roll back %s; was not uninstalled" % self.dist.project_name) + return False + logger.notify('Rolling back uninstall of %s' % self.dist.project_name) + for path in self._moved_paths: + tmp_path = self._stash(path) + logger.info('Replacing %s' % path) + renames(tmp_path, path) + for pth in self.pth: + pth.rollback() + + def commit(self): + """Remove temporary save dir: rollback will no longer be possible.""" + if self.save_dir is not None: + shutil.rmtree(self.save_dir) + self.save_dir = None + self._moved_paths = [] + + +class UninstallPthEntries(object): + def __init__(self, pth_file): + if not os.path.isfile(pth_file): + raise UninstallationError("Cannot remove entries from nonexistent file %s" % pth_file) + self.file = pth_file + self.entries = set() + self._saved_lines = None + + def add(self, entry): + entry = os.path.normcase(entry) + # On Windows, os.path.normcase converts the entry to use + # backslashes. This is correct for entries that describe absolute + # paths outside of site-packages, but all the others use forward + # slashes. + if sys.platform == 'win32' and not os.path.splitdrive(entry)[0]: + entry = entry.replace('\\', '/') + self.entries.add(entry) + + def remove(self): + logger.info('Removing pth entries from %s:' % self.file) + fh = open(self.file, 'r') + lines = fh.readlines() + self._saved_lines = lines + fh.close() + try: + for entry in self.entries: + logger.info('Removing entry: %s' % entry) + try: + lines.remove(entry + '\n') + except ValueError: + pass + finally: + pass + fh = open(self.file, 'wb') + fh.writelines(lines) + fh.close() + + def rollback(self): + if self._saved_lines is None: + logger.error('Cannot roll back changes to %s, none were made' % self.file) + return False + logger.info('Rolling %s back to previous state' % self.file) + fh = open(self.file, 'wb') + fh.writelines(self._saved_lines) + fh.close() + return True + + +class FakeFile(object): + """Wrap a list of lines in an object with readline() to make + ConfigParser happy.""" + def __init__(self, lines): + self._gen = (l for l in lines) + + def readline(self): + try: + return self._gen.next() + except StopIteration: + return '' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py new file mode 100644 index 000000000..be830ad9a --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py @@ -0,0 +1,18 @@ +import sys +import os + + +def run(): + base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ## FIXME: this is kind of crude; if we could create a fake pip + ## module, then exec into it and update pip.__path__ properly, we + ## wouldn't have to update sys.path: + sys.path.insert(0, base) + import pip + return pip.main() + + +if __name__ == '__main__': + exit = run() + if exit: + sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py new file mode 100644 index 000000000..1eab34c06 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py @@ -0,0 +1,479 @@ +import sys +import shutil +import os +import stat +import re +import posixpath +import pkg_resources +import zipfile +import tarfile +from pip.exceptions import InstallationError +from pip.backwardcompat import WindowsError +from pip.locations import site_packages, running_under_virtualenv +from pip.log import logger + +__all__ = ['rmtree', 'display_path', 'backup_dir', + 'find_command', 'ask', 'Inf', + 'normalize_name', 'splitext', + 'format_size', 'is_installable_dir', + 'is_svn_page', 'file_contents', + 'split_leading_dir', 'has_leading_dir', + 'make_path_relative', 'normalize_path', + 'renames', 'get_terminal_size', + 'unzip_file', 'untar_file', 'create_download_cache_folder', + 'cache_download', 'unpack_file'] + + +def rmtree(dir): + shutil.rmtree(dir, ignore_errors=True, + onerror=rmtree_errorhandler) + + +def rmtree_errorhandler(func, path, exc_info): + """On Windows, the files in .svn are read-only, so when rmtree() tries to + remove them, an exception is thrown. We catch that here, remove the + read-only attribute, and hopefully continue without problems.""" + exctype, value = exc_info[:2] + # lookin for a windows error + if exctype is not WindowsError or 'Access is denied' not in str(value): + raise + # file type should currently be read only + if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD): + raise + # convert to read/write + os.chmod(path, stat.S_IWRITE) + # use the original function to repeat the operation + func(path) + + +def display_path(path): + """Gives the display value for a given path, making it relative to cwd + if possible.""" + path = os.path.normcase(os.path.abspath(path)) + if path.startswith(os.getcwd() + os.path.sep): + path = '.' + path[len(os.getcwd()):] + return path + + +def backup_dir(dir, ext='.bak'): + """Figure out the name of a directory to back up the given dir to + (adding .bak, .bak2, etc)""" + n = 1 + extension = ext + while os.path.exists(dir + extension): + n += 1 + extension = ext + str(n) + return dir + extension + + +def find_command(cmd, paths=None, pathext=None): + """Searches the PATH for the given command and returns its path""" + if paths is None: + paths = os.environ.get('PATH', []).split(os.pathsep) + if isinstance(paths, basestring): + paths = [paths] + # check if there are funny path extensions for executables, e.g. Windows + if pathext is None: + pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD') + pathext = [ext for ext in pathext.lower().split(os.pathsep)] + # don't use extensions if the command ends with one of them + if os.path.splitext(cmd)[1].lower() in pathext: + pathext = [''] + # check if we find the command on PATH + for path in paths: + # try without extension first + cmd_path = os.path.join(path, cmd) + for ext in pathext: + # then including the extension + cmd_path_ext = cmd_path + ext + if os.path.exists(cmd_path_ext): + return cmd_path_ext + if os.path.exists(cmd_path): + return cmd_path + return None + + +def ask(message, options): + """Ask the message interactively, with the given possible responses""" + while 1: + if os.environ.get('PIP_NO_INPUT'): + raise Exception('No input was expected ($PIP_NO_INPUT set); question: %s' % message) + response = raw_input(message) + response = response.strip().lower() + if response not in options: + print 'Your response (%r) was not one of the expected responses: %s' % ( + response, ', '.join(options)) + else: + return response + + +class _Inf(object): + """I am bigger than everything!""" + def __cmp__(self, a): + if self is a: + return 0 + return 1 + + def __repr__(self): + return 'Inf' + +Inf = _Inf() +del _Inf + + +_normalize_re = re.compile(r'[^a-z]', re.I) + + +def normalize_name(name): + return _normalize_re.sub('-', name.lower()) + + +def format_size(bytes): + if bytes > 1000*1000: + return '%.1fMb' % (bytes/1000.0/1000) + elif bytes > 10*1000: + return '%iKb' % (bytes/1000) + elif bytes > 1000: + return '%.1fKb' % (bytes/1000.0) + else: + return '%ibytes' % bytes + + +def is_installable_dir(path): + """Return True if `path` is a directory containing a setup.py file.""" + if not os.path.isdir(path): + return False + setup_py = os.path.join(path, 'setup.py') + if os.path.isfile(setup_py): + return True + return False + + +def is_svn_page(html): + """Returns true if the page appears to be the index page of an svn repository""" + return (re.search(r'[^<]*Revision \d+:', html) + and re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I)) + + +def file_contents(filename): + fp = open(filename, 'rb') + try: + return fp.read() + finally: + fp.close() + + +def split_leading_dir(path): + path = str(path) + path = path.lstrip('/').lstrip('\\') + if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) + or '\\' not in path): + return path.split('/', 1) + elif '\\' in path: + return path.split('\\', 1) + else: + return path, '' + + +def has_leading_dir(paths): + """Returns true if all the paths have the same leading path name + (i.e., everything is in one subdirectory in an archive)""" + common_prefix = None + for path in paths: + prefix, rest = split_leading_dir(path) + if not prefix: + return False + elif common_prefix is None: + common_prefix = prefix + elif prefix != common_prefix: + return False + return True + + +def make_path_relative(path, rel_to): + """ + Make a filename relative, where the filename path, and it is + relative to rel_to + + >>> make_relative_path('/usr/share/something/a-file.pth', + ... '/usr/share/another-place/src/Directory') + '../../../something/a-file.pth' + >>> make_relative_path('/usr/share/something/a-file.pth', + ... '/home/user/src/Directory') + '../../../usr/share/something/a-file.pth' + >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/') + 'a-file.pth' + """ + path_filename = os.path.basename(path) + path = os.path.dirname(path) + path = os.path.normpath(os.path.abspath(path)) + rel_to = os.path.normpath(os.path.abspath(rel_to)) + path_parts = path.strip(os.path.sep).split(os.path.sep) + rel_to_parts = rel_to.strip(os.path.sep).split(os.path.sep) + while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]: + path_parts.pop(0) + rel_to_parts.pop(0) + full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename] + if full_parts == ['']: + return '.' + os.path.sep + return os.path.sep.join(full_parts) + + +def normalize_path(path): + """ + Convert a path to its canonical, case-normalized, absolute version. + + """ + return os.path.normcase(os.path.realpath(path)) + + +def splitext(path): + """Like os.path.splitext, but take off .tar too""" + base, ext = posixpath.splitext(path) + if base.lower().endswith('.tar'): + ext = base[-4:] + ext + base = base[:-4] + return base, ext + + +def renames(old, new): + """Like os.renames(), but handles renaming across devices.""" + # Implementation borrowed from os.renames(). + head, tail = os.path.split(new) + if head and tail and not os.path.exists(head): + os.makedirs(head) + + shutil.move(old, new) + + head, tail = os.path.split(old) + if head and tail: + try: + os.removedirs(head) + except OSError: + pass + + +def is_local(path): + """ + Return True if path is within sys.prefix, if we're running in a virtualenv. + + If we're not in a virtualenv, all paths are considered "local." + + """ + if not running_under_virtualenv(): + return True + return normalize_path(path).startswith(normalize_path(sys.prefix)) + + +def dist_is_local(dist): + """ + Return True if given Distribution object is installed locally + (i.e. within current virtualenv). + + Always True if we're not in a virtualenv. + + """ + return is_local(dist_location(dist)) + + +def get_installed_distributions(local_only=True, skip=('setuptools', 'pip', 'python')): + """ + Return a list of installed Distribution objects. + + If ``local_only`` is True (default), only return installations + local to the current virtualenv, if in a virtualenv. + + ``skip`` argument is an iterable of lower-case project names to + ignore; defaults to ('setuptools', 'pip', 'python'). [FIXME also + skip virtualenv?] + + """ + if local_only: + local_test = dist_is_local + else: + local_test = lambda d: True + return [d for d in pkg_resources.working_set if local_test(d) and d.key not in skip] + + +def egg_link_path(dist): + """ + Return the path where we'd expect to find a .egg-link file for + this distribution. (There doesn't seem to be any metadata in the + Distribution object for a develop egg that points back to its + .egg-link and easy-install.pth files). + + This won't find a globally-installed develop egg if we're in a + virtualenv. + + """ + return os.path.join(site_packages, dist.project_name) + '.egg-link' + + +def dist_location(dist): + """ + Get the site-packages location of this distribution. Generally + this is dist.location, except in the case of develop-installed + packages, where dist.location is the source code location, and we + want to know where the egg-link file is. + + """ + egg_link = egg_link_path(dist) + if os.path.exists(egg_link): + return egg_link + return dist.location + + +def get_terminal_size(): + """Returns a tuple (x, y) representing the width(x) and the height(x) + in characters of the terminal window.""" + def ioctl_GWINSZ(fd): + try: + import fcntl + import termios + import struct + cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, + '1234')) + except: + return None + if cr == (0, 0): + return None + if cr == (0, 0): + return None + return cr + cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + if not cr: + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + cr = ioctl_GWINSZ(fd) + os.close(fd) + except: + pass + if not cr: + cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) + return int(cr[1]), int(cr[0]) + + +def unzip_file(filename, location, flatten=True): + """Unzip the file (zip file located at filename) to the destination + location""" + if not os.path.exists(location): + os.makedirs(location) + zipfp = open(filename, 'rb') + try: + zip = zipfile.ZipFile(zipfp) + leading = has_leading_dir(zip.namelist()) and flatten + for name in zip.namelist(): + data = zip.read(name) + fn = name + if leading: + fn = split_leading_dir(name)[1] + fn = os.path.join(location, fn) + dir = os.path.dirname(fn) + if not os.path.exists(dir): + os.makedirs(dir) + if fn.endswith('/') or fn.endswith('\\'): + # A directory + if not os.path.exists(fn): + os.makedirs(fn) + else: + fp = open(fn, 'wb') + try: + fp.write(data) + finally: + fp.close() + finally: + zipfp.close() + + +def untar_file(filename, location): + """Untar the file (tar file located at filename) to the destination location""" + if not os.path.exists(location): + os.makedirs(location) + if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): + mode = 'r:gz' + elif filename.lower().endswith('.bz2') or filename.lower().endswith('.tbz'): + mode = 'r:bz2' + elif filename.lower().endswith('.tar'): + mode = 'r' + else: + logger.warn('Cannot determine compression type for file %s' % filename) + mode = 'r:*' + tar = tarfile.open(filename, mode) + try: + # note: python<=2.5 doesnt seem to know about pax headers, filter them + leading = has_leading_dir([ + member.name for member in tar.getmembers() + if member.name != 'pax_global_header' + ]) + for member in tar.getmembers(): + fn = member.name + if fn == 'pax_global_header': + continue + if leading: + fn = split_leading_dir(fn)[1] + path = os.path.join(location, fn) + if member.isdir(): + if not os.path.exists(path): + os.makedirs(path) + else: + try: + fp = tar.extractfile(member) + except (KeyError, AttributeError), e: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warn( + 'In the tar file %s the member %s is invalid: %s' + % (filename, member.name, e)) + continue + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + destfp = open(path, 'wb') + try: + shutil.copyfileobj(fp, destfp) + finally: + destfp.close() + fp.close() + finally: + tar.close() + + +def create_download_cache_folder(folder): + logger.indent -= 2 + logger.notify('Creating supposed download cache at %s' % folder) + logger.indent += 2 + os.makedirs(folder) + + +def cache_download(target_file, temp_location, content_type): + logger.notify('Storing download in cache at %s' % display_path(target_file)) + shutil.copyfile(temp_location, target_file) + fp = open(target_file+'.content-type', 'w') + fp.write(content_type) + fp.close() + os.unlink(temp_location) + + +def unpack_file(filename, location, content_type, link): + if (content_type == 'application/zip' + or filename.endswith('.zip') + or filename.endswith('.pybundle') + or zipfile.is_zipfile(filename)): + unzip_file(filename, location, flatten=not filename.endswith('.pybundle')) + elif (content_type == 'application/x-gzip' + or tarfile.is_tarfile(filename) + or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')): + untar_file(filename, location) + elif (content_type and content_type.startswith('text/html') + and is_svn_page(file_contents(filename))): + # We don't really care about this + from pip.vcs.subversion import Subversion + Subversion('svn+' + link.url).unpack(location) + else: + ## FIXME: handle? + ## FIXME: magic signatures? + logger.fatal('Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format' + % (filename, location, content_type)) + raise InstallationError('Cannot determine archive format of %s' % location) + + + diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py new file mode 100644 index 000000000..e110440c7 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py @@ -0,0 +1,238 @@ +"""Handles all VCS (version control) support""" + +import os +import shutil +import urlparse +import urllib + +from pip.exceptions import BadCommand +from pip.log import logger +from pip.util import display_path, backup_dir, find_command, ask + + +__all__ = ['vcs', 'get_src_requirement', 'import_vcs_support'] + + +class VcsSupport(object): + _registry = {} + schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp'] + + def __init__(self): + # Register more schemes with urlparse for various version control systems + urlparse.uses_netloc.extend(self.schemes) + urlparse.uses_fragment.extend(self.schemes) + super(VcsSupport, self).__init__() + + def __iter__(self): + return self._registry.__iter__() + + @property + def backends(self): + return self._registry.values() + + @property + def dirnames(self): + return [backend.dirname for backend in self.backends] + + @property + def all_schemes(self): + schemes = [] + for backend in self.backends: + schemes.extend(backend.schemes) + return schemes + + def register(self, cls): + if not hasattr(cls, 'name'): + logger.warn('Cannot register VCS %s' % cls.__name__) + return + if cls.name not in self._registry: + self._registry[cls.name] = cls + + def unregister(self, cls=None, name=None): + if name in self._registry: + del self._registry[name] + elif cls in self._registry.values(): + del self._registry[cls.name] + else: + logger.warn('Cannot unregister because no class or name given') + + def get_backend_name(self, location): + """ + Return the name of the version control backend if found at given + location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') + """ + for vc_type in self._registry.values(): + path = os.path.join(location, vc_type.dirname) + if os.path.exists(path): + return vc_type.name + return None + + def get_backend(self, name): + name = name.lower() + if name in self._registry: + return self._registry[name] + + def get_backend_from_location(self, location): + vc_type = self.get_backend_name(location) + if vc_type: + return self.get_backend(vc_type) + return None + + +vcs = VcsSupport() + + +class VersionControl(object): + name = '' + dirname = '' + + def __init__(self, url=None, *args, **kwargs): + self.url = url + self._cmd = None + super(VersionControl, self).__init__(*args, **kwargs) + + def _filter(self, line): + return (logger.INFO, line) + + def _is_local_repository(self, repo): + """ + posix absolute paths start with os.path.sep, + win32 ones ones start with drive (like c:\\folder) + """ + drive, tail = os.path.splitdrive(repo) + return repo.startswith(os.path.sep) or drive + + @property + def cmd(self): + if self._cmd is not None: + return self._cmd + command = find_command(self.name) + if command is None: + raise BadCommand('Cannot find command %r' % self.name) + logger.info('Found command %r at %r' % (self.name, command)) + self._cmd = command + return command + + def get_url_rev(self): + """ + Returns the correct repository URL and revision by parsing the given + repository URL + """ + url = self.url.split('+', 1)[1] + scheme, netloc, path, query, frag = urlparse.urlsplit(url) + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + url = urlparse.urlunsplit((scheme, netloc, path, query, '')) + return url, rev + + def get_info(self, location): + """ + Returns (url, revision), where both are strings + """ + assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location + return self.get_url(location), self.get_revision(location) + + def normalize_url(self, url): + """ + Normalize a URL for comparison by unquoting it and removing any trailing slash. + """ + return urllib.unquote(url).rstrip('/') + + def compare_urls(self, url1, url2): + """ + Compare two repo URLs for identity, ignoring incidental differences. + """ + return (self.normalize_url(url1) == self.normalize_url(url2)) + + def parse_vcs_bundle_file(self, content): + """ + Takes the contents of the bundled text file that explains how to revert + the stripped off version control data of the given package and returns + the URL and revision of it. + """ + raise NotImplementedError + + def obtain(self, dest): + """ + Called when installing or updating an editable package, takes the + source path of the checkout. + """ + raise NotImplementedError + + def switch(self, dest, url, rev_options): + """ + Switch the repo at ``dest`` to point to ``URL``. + """ + raise NotImplemented + + def update(self, dest, rev_options): + """ + Update an already-existing repo to the given ``rev_options``. + """ + raise NotImplementedError + + def check_destination(self, dest, url, rev_options, rev_display): + """ + Prepare a location to receive a checkout/clone. + + Return True if the location is ready for (and requires) a + checkout/clone, False otherwise. + """ + checkout = True + prompt = False + if os.path.exists(dest): + checkout = False + if os.path.exists(os.path.join(dest, self.dirname)): + existing_url = self.get_url(dest) + if self.compare_urls(existing_url, url): + logger.info('%s in %s exists, and has correct URL (%s)' + % (self.repo_name.title(), display_path(dest), url)) + logger.notify('Updating %s %s%s' + % (display_path(dest), self.repo_name, rev_display)) + self.update(dest, rev_options) + else: + logger.warn('%s %s in %s exists with URL %s' + % (self.name, self.repo_name, display_path(dest), existing_url)) + prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) + else: + logger.warn('Directory %s already exists, and is not a %s %s.' + % (dest, self.name, self.repo_name)) + prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) + if prompt: + logger.warn('The plan is to install the %s repository %s' + % (self.name, url)) + response = ask('What to do? %s' % prompt[0], prompt[1]) + + if response == 's': + logger.notify('Switching %s %s to %s%s' + % (self.repo_name, display_path(dest), url, rev_display)) + self.switch(dest, url, rev_options) + elif response == 'i': + # do nothing + pass + elif response == 'w': + logger.warn('Deleting %s' % display_path(dest)) + shutil.rmtree(dest) + checkout = True + elif response == 'b': + dest_dir = backup_dir(dest) + logger.warn('Backing up %s to %s' + % (display_path(dest), dest_dir)) + shutil.move(dest, dest_dir) + checkout = True + return checkout + + def unpack(self, location): + raise NotImplementedError + + def get_src_requirement(self, dist, location, find_tags=False): + raise NotImplementedError + + +def get_src_requirement(dist, location, find_tags): + version_control = vcs.get_backend_from_location(location) + if version_control: + return version_control().get_src_requirement(dist, location, find_tags) + logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location) + return dist.as_requirement() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py new file mode 100644 index 000000000..3b6ea8f04 --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py @@ -0,0 +1,138 @@ +import os +import shutil +import tempfile +import re +from pip import call_subprocess +from pip.log import logger +from pip.util import rmtree, display_path +from pip.vcs import vcs, VersionControl +from pip.download import path_to_url2 + + +class Bazaar(VersionControl): + name = 'bzr' + dirname = '.bzr' + repo_name = 'branch' + bundle_file = 'bzr-branch.txt' + schemes = ('bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp') + guide = ('# This was a Bazaar branch; to make it a branch again run:\n' + 'bzr branch -r %(rev)s %(url)s .\n') + + def parse_vcs_bundle_file(self, content): + url = rev = None + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + match = re.search(r'^bzr\s*branch\s*-r\s*(\d*)', line) + if match: + rev = match.group(1).strip() + url = line[match.end():].strip().split(None, 1)[0] + if url and rev: + return url, rev + return None, None + + def unpack(self, location): + """Get the bzr branch at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Checking out bzr repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + os.rmdir(location) + call_subprocess( + [self.cmd, 'branch', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the Bazaar repository at the url to the destination location""" + temp_dir = tempfile.mkdtemp('-export', 'pip-') + self.unpack(temp_dir) + if os.path.exists(location): + # Remove the location to make sure Bazaar can export it correctly + rmtree(location) + try: + call_subprocess([self.cmd, 'export', location], cwd=temp_dir, + filter_stdout=self._filter, show_stdout=False) + finally: + shutil.rmtree(temp_dir) + + def switch(self, dest, url, rev_options): + call_subprocess([self.cmd, 'switch', url], cwd=dest) + + def update(self, dest, rev_options): + call_subprocess( + [self.cmd, 'pull', '-q'] + rev_options, cwd=dest) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = ['-r', rev] + rev_display = ' (to revision %s)' % rev + else: + rev_options = [] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Checking out %s%s to %s' + % (url, rev_display, display_path(dest))) + call_subprocess( + [self.cmd, 'branch', '-q'] + rev_options + [url, dest]) + + def get_url_rev(self): + # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it + url, rev = super(Bazaar, self).get_url_rev() + if url.startswith('ssh://'): + url = 'bzr+' + url + return url, rev + + def get_url(self, location): + urls = call_subprocess( + [self.cmd, 'info'], show_stdout=False, cwd=location) + for line in urls.splitlines(): + line = line.strip() + for x in ('checkout of branch: ', + 'parent branch: '): + if line.startswith(x): + repo = line.split(x)[1] + if self._is_local_repository(repo): + return path_to_url2(repo) + return repo + return None + + def get_revision(self, location): + revision = call_subprocess( + [self.cmd, 'revno'], show_stdout=False, cwd=location) + return revision.splitlines()[-1] + + def get_tag_revs(self, location): + tags = call_subprocess( + [self.cmd, 'tags'], show_stdout=False, cwd=location) + tag_revs = [] + for line in tags.splitlines(): + tags_match = re.search(r'([.\w-]+)\s*(.*)$', line) + if tags_match: + tag = tags_match.group(1) + rev = tags_match.group(2) + tag_revs.append((rev.strip(), tag.strip())) + return dict(tag_revs) + + def get_src_requirement(self, dist, location, find_tags): + repo = self.get_url(location) + if not repo.lower().startswith('bzr:'): + repo = 'bzr+' + repo + egg_project_name = dist.egg_name().split('-', 1)[0] + if not repo: + return None + current_rev = self.get_revision(location) + tag_revs = self.get_tag_revs(location) + + if current_rev in tag_revs: + # It's a tag + full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) + else: + full_egg_name = '%s-dev_r%s' % (dist.egg_name(), current_rev) + return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) + + +vcs.register(Bazaar) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py new file mode 100644 index 000000000..0701e49ec --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py @@ -0,0 +1,204 @@ +import os +import shutil +import tempfile +import re +from pip import call_subprocess +from pip.util import display_path +from pip.vcs import vcs, VersionControl +from pip.log import logger +from urllib import url2pathname +from urlparse import urlsplit, urlunsplit + + +class Git(VersionControl): + name = 'git' + dirname = '.git' + repo_name = 'clone' + schemes = ('git', 'git+http', 'git+ssh', 'git+git', 'git+file') + bundle_file = 'git-clone.txt' + guide = ('# This was a Git repo; to make it a repo again run:\n' + 'git init\ngit remote add origin %(url)s -f\ngit checkout %(rev)s\n') + + def __init__(self, url=None, *args, **kwargs): + + # Works around an apparent Git bug + # (see http://article.gmane.org/gmane.comp.version-control.git/146500) + if url: + scheme, netloc, path, query, fragment = urlsplit(url) + if scheme.endswith('file'): + initial_slashes = path[:-len(path.lstrip('/'))] + newpath = initial_slashes + url2pathname(path).replace('\\', '/').lstrip('/') + url = urlunsplit((scheme, netloc, newpath, query, fragment)) + after_plus = scheme.find('+')+1 + url = scheme[:after_plus]+ urlunsplit((scheme[after_plus:], netloc, newpath, query, fragment)) + + super(Git, self).__init__(url, *args, **kwargs) + + def parse_vcs_bundle_file(self, content): + url = rev = None + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + url_match = re.search(r'git\s*remote\s*add\s*origin(.*)\s*-f', line) + if url_match: + url = url_match.group(1).strip() + rev_match = re.search(r'^git\s*checkout\s*-q\s*(.*)\s*', line) + if rev_match: + rev = rev_match.group(1).strip() + if url and rev: + return url, rev + return None, None + + def unpack(self, location): + """Clone the Git repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Cloning Git repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + os.rmdir(location) + call_subprocess( + [self.cmd, 'clone', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the Git repository at the url to the destination location""" + temp_dir = tempfile.mkdtemp('-export', 'pip-') + self.unpack(temp_dir) + try: + if not location.endswith('/'): + location = location + '/' + call_subprocess( + [self.cmd, 'checkout-index', '-a', '-f', '--prefix', location], + filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) + finally: + shutil.rmtree(temp_dir) + + def check_rev_options(self, rev, dest, rev_options): + """Check the revision options before checkout to compensate that tags + and branches may need origin/ as a prefix. + Returns the SHA1 of the branch or tag if found. + """ + revisions = self.get_tag_revs(dest) + revisions.update(self.get_branch_revs(dest)) + inverse_revisions = dict((v, k) for k, v in revisions.iteritems()) + # Check if rev is a branch name + origin_rev = 'origin/%s' % rev + if origin_rev in inverse_revisions: + return [inverse_revisions[origin_rev]] + elif rev in inverse_revisions: + return [inverse_revisions[rev]] + else: + logger.warn("Could not find a tag or branch '%s', assuming commit." % rev) + return rev_options + + def switch(self, dest, url, rev_options): + call_subprocess( + [self.cmd, 'config', 'remote.origin.url', url], cwd=dest) + call_subprocess( + [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) + + def update(self, dest, rev_options): + call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) + call_subprocess( + [self.cmd, 'checkout', '-q', '-f'] + rev_options, cwd=dest) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = [rev] + rev_display = ' (to %s)' % rev + else: + rev_options = ['master'] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest))) + call_subprocess([self.cmd, 'clone', '-q', url, dest]) + if rev: + rev_options = self.check_rev_options(rev, dest, rev_options) + # Only do a checkout if rev_options differs from HEAD + if not self.get_revision(dest).startswith(rev_options[0]): + call_subprocess([self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) + + def get_url(self, location): + url = call_subprocess( + [self.cmd, 'config', 'remote.origin.url'], + show_stdout=False, cwd=location) + return url.strip() + + def get_revision(self, location): + current_rev = call_subprocess( + [self.cmd, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location) + return current_rev.strip() + + def get_tag_revs(self, location): + tags = call_subprocess( + [self.cmd, 'tag', '-l'], + show_stdout=False, raise_on_returncode=False, cwd=location) + tag_revs = [] + for line in tags.splitlines(): + tag = line.strip() + rev = call_subprocess( + [self.cmd, 'rev-parse', tag], show_stdout=False, cwd=location) + tag_revs.append((rev.strip(), tag)) + tag_revs = dict(tag_revs) + return tag_revs + + def get_branch_revs(self, location): + branches = call_subprocess( + [self.cmd, 'branch', '-r'], show_stdout=False, cwd=location) + branch_revs = [] + for line in branches.splitlines(): + line = line.split('->')[0].strip() + branch = "".join([b for b in line.split() if b != '*']) + rev = call_subprocess( + [self.cmd, 'rev-parse', branch], show_stdout=False, cwd=location) + branch_revs.append((rev.strip(), branch)) + branch_revs = dict(branch_revs) + return branch_revs + + def get_src_requirement(self, dist, location, find_tags): + repo = self.get_url(location) + if not repo.lower().startswith('git:'): + repo = 'git+' + repo + egg_project_name = dist.egg_name().split('-', 1)[0] + if not repo: + return None + current_rev = self.get_revision(location) + tag_revs = self.get_tag_revs(location) + branch_revs = self.get_branch_revs(location) + + if current_rev in tag_revs: + # It's a tag + full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) + elif (current_rev in branch_revs and + branch_revs[current_rev] != 'origin/master'): + # It's the head of a branch + full_egg_name = '%s-%s' % (dist.egg_name(), + branch_revs[current_rev].replace('origin/', '')) + else: + full_egg_name = '%s-dev' % dist.egg_name() + + return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) + + def get_url_rev(self): + """ + Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. + That's required because although they use SSH they sometimes doesn't + work with a ssh:// scheme (e.g. Github). But we need a scheme for + parsing. Hence we remove it again afterwards and return it as a stub. + """ + if not '://' in self.url: + assert not 'file:' in self.url + self.url = self.url.replace('git+', 'git+ssh://') + url, rev = super(Git, self).get_url_rev() + url = url.replace('ssh://', '') + else: + url, rev = super(Git, self).get_url_rev() + + return url, rev + + +vcs.register(Git) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py new file mode 100644 index 000000000..70c8c833d --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py @@ -0,0 +1,162 @@ +import os +import shutil +import tempfile +import re +import ConfigParser +from pip import call_subprocess +from pip.util import display_path +from pip.log import logger +from pip.vcs import vcs, VersionControl +from pip.download import path_to_url2 + + +class Mercurial(VersionControl): + name = 'hg' + dirname = '.hg' + repo_name = 'clone' + schemes = ('hg', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http') + bundle_file = 'hg-clone.txt' + guide = ('# This was a Mercurial repo; to make it a repo again run:\n' + 'hg init\nhg pull %(url)s\nhg update -r %(rev)s\n') + + def parse_vcs_bundle_file(self, content): + url = rev = None + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + url_match = re.search(r'hg\s*pull\s*(.*)\s*', line) + if url_match: + url = url_match.group(1).strip() + rev_match = re.search(r'^hg\s*update\s*-r\s*(.*)\s*', line) + if rev_match: + rev = rev_match.group(1).strip() + if url and rev: + return url, rev + return None, None + + def unpack(self, location): + """Clone the Hg repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Cloning Mercurial repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + os.rmdir(location) + call_subprocess( + [self.cmd, 'clone', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the Hg repository at the url to the destination location""" + temp_dir = tempfile.mkdtemp('-export', 'pip-') + self.unpack(temp_dir) + try: + call_subprocess( + [self.cmd, 'archive', location], + filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) + finally: + shutil.rmtree(temp_dir) + + def switch(self, dest, url, rev_options): + repo_config = os.path.join(dest, self.dirname, 'hgrc') + config = ConfigParser.SafeConfigParser() + try: + config.read(repo_config) + config.set('paths', 'default', url) + config_file = open(repo_config, 'w') + config.write(config_file) + config_file.close() + except (OSError, ConfigParser.NoSectionError), e: + logger.warn( + 'Could not switch Mercurial repository to %s: %s' + % (url, e)) + else: + call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) + + def update(self, dest, rev_options): + call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) + call_subprocess( + [self.cmd, 'update', '-q'] + rev_options, cwd=dest) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = [rev] + rev_display = ' (to revision %s)' % rev + else: + rev_options = [] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Cloning hg %s%s to %s' + % (url, rev_display, display_path(dest))) + call_subprocess([self.cmd, 'clone', '--noupdate', '-q', url, dest]) + call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) + + def get_url(self, location): + url = call_subprocess( + [self.cmd, 'showconfig', 'paths.default'], + show_stdout=False, cwd=location).strip() + if self._is_local_repository(url): + url = path_to_url2(url) + return url.strip() + + def get_tag_revs(self, location): + tags = call_subprocess( + [self.cmd, 'tags'], show_stdout=False, cwd=location) + tag_revs = [] + for line in tags.splitlines(): + tags_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) + if tags_match: + tag = tags_match.group(1) + rev = tags_match.group(2) + tag_revs.append((rev.strip(), tag.strip())) + return dict(tag_revs) + + def get_branch_revs(self, location): + branches = call_subprocess( + [self.cmd, 'branches'], show_stdout=False, cwd=location) + branch_revs = [] + for line in branches.splitlines(): + branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) + if branches_match: + branch = branches_match.group(1) + rev = branches_match.group(2) + branch_revs.append((rev.strip(), branch.strip())) + return dict(branch_revs) + + def get_revision(self, location): + current_revision = call_subprocess( + [self.cmd, 'parents', '--template={rev}'], + show_stdout=False, cwd=location).strip() + return current_revision + + def get_revision_hash(self, location): + current_rev_hash = call_subprocess( + [self.cmd, 'parents', '--template={node}'], + show_stdout=False, cwd=location).strip() + return current_rev_hash + + def get_src_requirement(self, dist, location, find_tags): + repo = self.get_url(location) + if not repo.lower().startswith('hg:'): + repo = 'hg+' + repo + egg_project_name = dist.egg_name().split('-', 1)[0] + if not repo: + return None + current_rev = self.get_revision(location) + current_rev_hash = self.get_revision_hash(location) + tag_revs = self.get_tag_revs(location) + branch_revs = self.get_branch_revs(location) + if current_rev in tag_revs: + # It's a tag + full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) + elif current_rev in branch_revs: + # It's the tip of a branch + full_egg_name = '%s-%s' % (dist.egg_name(), branch_revs[current_rev]) + else: + full_egg_name = '%s-dev' % dist.egg_name() + return '%s@%s#egg=%s' % (repo, current_rev_hash, full_egg_name) + +vcs.register(Mercurial) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py new file mode 100644 index 000000000..85715d97a --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py @@ -0,0 +1,260 @@ +import os +import re +from pip import call_subprocess +from pip.index import Link +from pip.util import rmtree, display_path +from pip.log import logger +from pip.vcs import vcs, VersionControl + +_svn_xml_url_re = re.compile('url="([^"]+)"') +_svn_rev_re = re.compile('committed-rev="(\d+)"') +_svn_url_re = re.compile(r'URL: (.+)') +_svn_revision_re = re.compile(r'Revision: (.+)') + + +class Subversion(VersionControl): + name = 'svn' + dirname = '.svn' + repo_name = 'checkout' + schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https') + bundle_file = 'svn-checkout.txt' + guide = ('# This was an svn checkout; to make it a checkout again run:\n' + 'svn checkout --force -r %(rev)s %(url)s .\n') + + def get_info(self, location): + """Returns (url, revision), where both are strings""" + assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location + output = call_subprocess( + [self.cmd, 'info', location], show_stdout=False, extra_environ={'LANG': 'C'}) + match = _svn_url_re.search(output) + if not match: + logger.warn('Cannot determine URL of svn checkout %s' % display_path(location)) + logger.info('Output that cannot be parsed: \n%s' % output) + return None, None + url = match.group(1).strip() + match = _svn_revision_re.search(output) + if not match: + logger.warn('Cannot determine revision of svn checkout %s' % display_path(location)) + logger.info('Output that cannot be parsed: \n%s' % output) + return url, None + return url, match.group(1) + + def parse_vcs_bundle_file(self, content): + for line in content.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + match = re.search(r'^-r\s*([^ ])?', line) + if not match: + return None, None + rev = match.group(1) + rest = line[match.end():].strip().split(None, 1)[0] + return rest, rev + return None, None + + def unpack(self, location): + """Check out the svn repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Checking out svn repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + # Subversion doesn't like to check out over an existing directory + # --force fixes this, but was only added in svn 1.5 + rmtree(location) + call_subprocess( + [self.cmd, 'checkout', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def export(self, location): + """Export the svn repository at the url to the destination location""" + url, rev = self.get_url_rev() + logger.notify('Exporting svn repository %s to %s' % (url, location)) + logger.indent += 2 + try: + if os.path.exists(location): + # Subversion doesn't like to check out over an existing directory + # --force fixes this, but was only added in svn 1.5 + rmtree(location) + call_subprocess( + [self.cmd, 'export', url, location], + filter_stdout=self._filter, show_stdout=False) + finally: + logger.indent -= 2 + + def switch(self, dest, url, rev_options): + call_subprocess( + [self.cmd, 'switch'] + rev_options + [url, dest]) + + def update(self, dest, rev_options): + call_subprocess( + [self.cmd, 'update'] + rev_options + [dest]) + + def obtain(self, dest): + url, rev = self.get_url_rev() + if rev: + rev_options = ['-r', rev] + rev_display = ' (to revision %s)' % rev + else: + rev_options = [] + rev_display = '' + if self.check_destination(dest, url, rev_options, rev_display): + logger.notify('Checking out %s%s to %s' + % (url, rev_display, display_path(dest))) + call_subprocess( + [self.cmd, 'checkout', '-q'] + rev_options + [url, dest]) + + def get_location(self, dist, dependency_links): + for url in dependency_links: + egg_fragment = Link(url).egg_fragment + if not egg_fragment: + continue + if '-' in egg_fragment: + ## FIXME: will this work when a package has - in the name? + key = '-'.join(egg_fragment.split('-')[:-1]).lower() + else: + key = egg_fragment + if key == dist.key: + return url.split('#', 1)[0] + return None + + def get_revision(self, location): + """ + Return the maximum revision for all files under a given location + """ + # Note: taken from setuptools.command.egg_info + revision = 0 + + for base, dirs, files in os.walk(location): + if self.dirname not in dirs: + dirs[:] = [] + continue # no sense walking uncontrolled subdirs + dirs.remove(self.dirname) + entries_fn = os.path.join(base, self.dirname, 'entries') + if not os.path.exists(entries_fn): + ## FIXME: should we warn? + continue + f = open(entries_fn) + data = f.read() + f.close() + + if data.startswith('8') or data.startswith('9') or data.startswith('10'): + data = map(str.splitlines, data.split('\n\x0c\n')) + del data[0][0] # get rid of the '8' + dirurl = data[0][3] + revs = [int(d[9]) for d in data if len(d)>9 and d[9]]+[0] + if revs: + localrev = max(revs) + else: + localrev = 0 + elif data.startswith('<?xml'): + dirurl = _svn_xml_url_re.search(data).group(1) # get repository URL + revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)]+[0] + if revs: + localrev = max(revs) + else: + localrev = 0 + else: + logger.warn("Unrecognized .svn/entries format; skipping %s", base) + dirs[:] = [] + continue + if base == location: + base_url = dirurl+'/' # save the root url + elif not dirurl.startswith(base_url): + dirs[:] = [] + continue # not part of the same svn tree, skip it + revision = max(revision, localrev) + return revision + + def get_url_rev(self): + # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it + url, rev = super(Subversion, self).get_url_rev() + if url.startswith('ssh://'): + url = 'svn+' + url + return url, rev + + def get_url(self, location): + # In cases where the source is in a subdirectory, not alongside setup.py + # we have to look up in the location until we find a real setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without finding setup.py + logger.warn("Could not find setup.py for directory %s (tried all parent directories)" + % orig_location) + return None + f = open(os.path.join(location, self.dirname, 'entries')) + data = f.read() + f.close() + if data.startswith('8') or data.startswith('9') or data.startswith('10'): + data = map(str.splitlines, data.split('\n\x0c\n')) + del data[0][0] # get rid of the '8' + return data[0][3] + elif data.startswith('<?xml'): + match = _svn_xml_url_re.search(data) + if not match: + raise ValueError('Badly formatted data: %r' % data) + return match.group(1) # get repository URL + else: + logger.warn("Unrecognized .svn/entries format in %s" % location) + # Or raise exception? + return None + + def get_tag_revs(self, svn_tag_url): + stdout = call_subprocess( + [self.cmd, 'ls', '-v', svn_tag_url], show_stdout=False) + results = [] + for line in stdout.splitlines(): + parts = line.split() + rev = int(parts[0]) + tag = parts[-1].strip('/') + results.append((tag, rev)) + return results + + def find_tag_match(self, rev, tag_revs): + best_match_rev = None + best_tag = None + for tag, tag_rev in tag_revs: + if (tag_rev > rev and + (best_match_rev is None or best_match_rev > tag_rev)): + # FIXME: Is best_match > tag_rev really possible? + # or is it a sign something is wacky? + best_match_rev = tag_rev + best_tag = tag + return best_tag + + def get_src_requirement(self, dist, location, find_tags=False): + repo = self.get_url(location) + if repo is None: + return None + parts = repo.split('/') + ## FIXME: why not project name? + egg_project_name = dist.egg_name().split('-', 1)[0] + rev = self.get_revision(location) + if parts[-2] in ('tags', 'tag'): + # It's a tag, perfect! + full_egg_name = '%s-%s' % (egg_project_name, parts[-1]) + elif parts[-2] in ('branches', 'branch'): + # It's a branch :( + full_egg_name = '%s-%s-r%s' % (dist.egg_name(), parts[-1], rev) + elif parts[-1] == 'trunk': + # Trunk :-/ + full_egg_name = '%s-dev_r%s' % (dist.egg_name(), rev) + if find_tags: + tag_url = '/'.join(parts[:-1]) + '/tags' + tag_revs = self.get_tag_revs(tag_url) + match = self.find_tag_match(rev, tag_revs) + if match: + logger.notify('trunk checkout %s seems to be equivalent to tag %s' % match) + repo = '%s/%s' % (tag_url, match) + full_egg_name = '%s-%s' % (egg_project_name, match) + else: + # Don't know what it is + logger.warn('svn URL does not fit normal structure (tags/branches/trunk): %s' % repo) + full_egg_name = '%s-dev_r%s' % (egg_project_name, rev) + return 'svn+%s@%s#egg=%s' % (repo, rev, full_egg_name) + +vcs.register(Subversion) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py new file mode 100644 index 000000000..708abb05b --- /dev/null +++ b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py @@ -0,0 +1,53 @@ +"""Tools for working with virtualenv environments""" + +import os +import sys +import subprocess +from pip.exceptions import BadCommand +from pip.log import logger + + +def restart_in_venv(venv, base, site_packages, args): + """ + Restart this script using the interpreter in the given virtual environment + """ + if base and not os.path.isabs(venv) and not venv.startswith('~'): + base = os.path.expanduser(base) + # ensure we have an abs basepath at this point: + # a relative one makes no sense (or does it?) + if os.path.isabs(base): + venv = os.path.join(base, venv) + + if venv.startswith('~'): + venv = os.path.expanduser(venv) + + if not os.path.exists(venv): + try: + import virtualenv + except ImportError: + print 'The virtual environment does not exist: %s' % venv + print 'and virtualenv is not installed, so a new environment cannot be created' + sys.exit(3) + print 'Creating new virtualenv environment in %s' % venv + virtualenv.logger = logger + logger.indent += 2 + virtualenv.create_environment(venv, site_packages=site_packages) + if sys.platform == 'win32': + python = os.path.join(venv, 'Scripts', 'python.exe') + # check for bin directory which is used in buildouts + if not os.path.exists(python): + python = os.path.join(venv, 'bin', 'python.exe') + else: + python = os.path.join(venv, 'bin', 'python') + if not os.path.exists(python): + python = venv + if not os.path.exists(python): + raise BadCommand('Cannot find virtual environment interpreter at %s' % python) + base = os.path.dirname(os.path.dirname(python)) + file = os.path.join(os.path.dirname(__file__), 'runner.py') + if file.endswith('.pyc'): + file = file[:-1] + proc = subprocess.Popen( + [python, file] + args + [base, '___VENV_RESTART___']) + proc.wait() + sys.exit(proc.returncode) diff --git a/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg b/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg new file mode 100644 index 000000000..3c72d15b5 Binary files /dev/null and b/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg differ diff --git a/test/lib/python2.6/site-packages/setuptools.pth b/test/lib/python2.6/site-packages/setuptools.pth new file mode 100644 index 000000000..bf7602056 --- /dev/null +++ b/test/lib/python2.6/site-packages/setuptools.pth @@ -0,0 +1 @@ +./setuptools-0.6c11-py2.6.egg diff --git a/test/lib/python2.6/site.py b/test/lib/python2.6/site.py new file mode 100644 index 000000000..a49cfc349 --- /dev/null +++ b/test/lib/python2.6/site.py @@ -0,0 +1,713 @@ +"""Append module search paths for third-party packages to sys.path. + +**************************************************************** +* This module is automatically imported during initialization. * +**************************************************************** + +In earlier versions of Python (up to 1.5a3), scripts or modules that +needed to use site-specific modules would place ``import site'' +somewhere near the top of their code. Because of the automatic +import, this is no longer necessary (but code that does it still +works). + +This will append site-specific paths to the module search path. On +Unix, it starts with sys.prefix and sys.exec_prefix (if different) and +appends lib/python<version>/site-packages as well as lib/site-python. +It also supports the Debian convention of +lib/python<version>/dist-packages. On other platforms (mainly Mac and +Windows), it uses just sys.prefix (and sys.exec_prefix, if different, +but this is unlikely). The resulting directories, if they exist, are +appended to sys.path, and also inspected for path configuration files. + +FOR DEBIAN, this sys.path is augmented with directories in /usr/local. +Local addons go into /usr/local/lib/python<version>/site-packages +(resp. /usr/local/lib/site-python), Debian addons install into +/usr/{lib,share}/python<version>/dist-packages. + +A path configuration file is a file whose name has the form +<package>.pth; its contents are additional directories (one per line) +to be added to sys.path. Non-existing directories (or +non-directories) are never added to sys.path; no directory is added to +sys.path more than once. Blank lines and lines beginning with +'#' are skipped. Lines starting with 'import' are executed. + +For example, suppose sys.prefix and sys.exec_prefix are set to +/usr/local and there is a directory /usr/local/lib/python2.X/site-packages +with three subdirectories, foo, bar and spam, and two path +configuration files, foo.pth and bar.pth. Assume foo.pth contains the +following: + + # foo package configuration + foo + bar + bletch + +and bar.pth contains: + + # bar package configuration + bar + +Then the following directories are added to sys.path, in this order: + + /usr/local/lib/python2.X/site-packages/bar + /usr/local/lib/python2.X/site-packages/foo + +Note that bletch is omitted because it doesn't exist; bar precedes foo +because bar.pth comes alphabetically before foo.pth; and spam is +omitted because it is not mentioned in either path configuration file. + +After these path manipulations, an attempt is made to import a module +named sitecustomize, which can perform arbitrary additional +site-specific customizations. If this import fails with an +ImportError exception, it is silently ignored. + +""" + +import sys +import os +import __builtin__ +try: + set +except NameError: + from sets import Set as set + +# Prefixes for site-packages; add additional prefixes like /usr/local here +PREFIXES = [sys.prefix, sys.exec_prefix] +# Enable per user site-packages directory +# set it to False to disable the feature or True to force the feature +ENABLE_USER_SITE = None +# for distutils.commands.install +USER_SITE = None +USER_BASE = None + +_is_pypy = hasattr(sys, 'pypy_version_info') +_is_jython = sys.platform[:4] == 'java' +if _is_jython: + ModuleType = type(os) + +def makepath(*paths): + dir = os.path.join(*paths) + if _is_jython and (dir == '__classpath__' or + dir.startswith('__pyclasspath__')): + return dir, dir + dir = os.path.abspath(dir) + return dir, os.path.normcase(dir) + +def abs__file__(): + """Set all module' __file__ attribute to an absolute path""" + for m in sys.modules.values(): + if ((_is_jython and not isinstance(m, ModuleType)) or + hasattr(m, '__loader__')): + # only modules need the abspath in Jython. and don't mess + # with a PEP 302-supplied __file__ + continue + f = getattr(m, '__file__', None) + if f is None: + continue + m.__file__ = os.path.abspath(f) + +def removeduppaths(): + """ Remove duplicate entries from sys.path along with making them + absolute""" + # This ensures that the initial path provided by the interpreter contains + # only absolute pathnames, even if we're running from the build directory. + L = [] + known_paths = set() + for dir in sys.path: + # Filter out duplicate paths (on case-insensitive file systems also + # if they only differ in case); turn relative paths into absolute + # paths. + dir, dircase = makepath(dir) + if not dircase in known_paths: + L.append(dir) + known_paths.add(dircase) + sys.path[:] = L + return known_paths + +# XXX This should not be part of site.py, since it is needed even when +# using the -S option for Python. See http://www.python.org/sf/586680 +def addbuilddir(): + """Append ./build/lib.<platform> in case we're running in the build dir + (especially for Guido :-)""" + from distutils.util import get_platform + s = "build/lib.%s-%.3s" % (get_platform(), sys.version) + if hasattr(sys, 'gettotalrefcount'): + s += '-pydebug' + s = os.path.join(os.path.dirname(sys.path[-1]), s) + sys.path.append(s) + +def _init_pathinfo(): + """Return a set containing all existing directory entries from sys.path""" + d = set() + for dir in sys.path: + try: + if os.path.isdir(dir): + dir, dircase = makepath(dir) + d.add(dircase) + except TypeError: + continue + return d + +def addpackage(sitedir, name, known_paths): + """Add a new path to known_paths by combining sitedir and 'name' or execute + sitedir if it starts with 'import'""" + if known_paths is None: + _init_pathinfo() + reset = 1 + else: + reset = 0 + fullname = os.path.join(sitedir, name) + try: + f = open(fullname, "rU") + except IOError: + return + try: + for line in f: + if line.startswith("#"): + continue + if line.startswith("import"): + exec line + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + finally: + f.close() + if reset: + known_paths = None + return known_paths + +def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" + if known_paths is None: + known_paths = _init_pathinfo() + reset = 1 + else: + reset = 0 + sitedir, sitedircase = makepath(sitedir) + if not sitedircase in known_paths: + sys.path.append(sitedir) # Add path component + try: + names = os.listdir(sitedir) + except os.error: + return + names.sort() + for name in names: + if name.endswith(os.extsep + "pth"): + addpackage(sitedir, name, known_paths) + if reset: + known_paths = None + return known_paths + +def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix): + """Add site-packages (and possibly site-python) to sys.path""" + prefixes = [os.path.join(sys_prefix, "local"), sys_prefix] + if exec_prefix != sys_prefix: + prefixes.append(os.path.join(exec_prefix, "local")) + + for prefix in prefixes: + if prefix: + if sys.platform in ('os2emx', 'riscos') or _is_jython: + sitedirs = [os.path.join(prefix, "Lib", "site-packages")] + elif _is_pypy: + sitedirs = [os.path.join(prefix, 'site-packages')] + elif sys.platform == 'darwin' and prefix == sys_prefix: + + if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python + + sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"), + os.path.join(prefix, "Extras", "lib", "python")] + + else: # any other Python distros on OSX work this way + sitedirs = [os.path.join(prefix, "lib", + "python" + sys.version[:3], "site-packages")] + + elif os.sep == '/': + sitedirs = [os.path.join(prefix, + "lib", + "python" + sys.version[:3], + "site-packages"), + os.path.join(prefix, "lib", "site-python"), + os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")] + lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages") + if (os.path.exists(lib64_dir) and + os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]): + sitedirs.append(lib64_dir) + try: + # sys.getobjects only available in --with-pydebug build + sys.getobjects + sitedirs.insert(0, os.path.join(sitedirs[0], 'debug')) + except AttributeError: + pass + # Debian-specific dist-packages directories: + sitedirs.append(os.path.join(prefix, "lib", + "python" + sys.version[:3], + "dist-packages")) + sitedirs.append(os.path.join(prefix, "local/lib", + "python" + sys.version[:3], + "dist-packages")) + sitedirs.append(os.path.join(prefix, "lib", "dist-python")) + else: + sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")] + if sys.platform == 'darwin': + # for framework builds *only* we add the standard Apple + # locations. Currently only per-user, but /Library and + # /Network/Library could be added too + if 'Python.framework' in prefix: + home = os.environ.get('HOME') + if home: + sitedirs.append( + os.path.join(home, + 'Library', + 'Python', + sys.version[:3], + 'site-packages')) + for sitedir in sitedirs: + if os.path.isdir(sitedir): + addsitedir(sitedir, known_paths) + return None + +def check_enableusersite(): + """Check if user site directory is safe for inclusion + + The function tests for the command line flag (including environment var), + process uid/gid equal to effective uid/gid. + + None: Disabled for security reasons + False: Disabled by user (command line option) + True: Safe and enabled + """ + if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False): + return False + + if hasattr(os, "getuid") and hasattr(os, "geteuid"): + # check process uid == effective uid + if os.geteuid() != os.getuid(): + return None + if hasattr(os, "getgid") and hasattr(os, "getegid"): + # check process gid == effective gid + if os.getegid() != os.getgid(): + return None + + return True + +def addusersitepackages(known_paths): + """Add a per user site-package to sys.path + + Each user has its own python directory with site-packages in the + home directory. + + USER_BASE is the root directory for all Python versions + + USER_SITE is the user specific site-packages directory + + USER_SITE/.. can be used for data. + """ + global USER_BASE, USER_SITE, ENABLE_USER_SITE + env_base = os.environ.get("PYTHONUSERBASE", None) + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + #if sys.platform in ('os2emx', 'riscos'): + # # Don't know what to put here + # USER_BASE = '' + # USER_SITE = '' + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser(base, "Python") + USER_SITE = os.path.join(USER_BASE, + "Python" + sys.version[0] + sys.version[2], + "site-packages") + else: + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser("~", ".local") + USER_SITE = os.path.join(USER_BASE, "lib", + "python" + sys.version[:3], + "site-packages") + + if ENABLE_USER_SITE and os.path.isdir(USER_SITE): + addsitedir(USER_SITE, known_paths) + if ENABLE_USER_SITE: + for dist_libdir in ("lib", "local/lib"): + user_site = os.path.join(USER_BASE, dist_libdir, + "python" + sys.version[:3], + "dist-packages") + if os.path.isdir(user_site): + addsitedir(user_site, known_paths) + return known_paths + + + +def setBEGINLIBPATH(): + """The OS/2 EMX port has optional extension modules that do double duty + as DLLs (and must use the .DLL file extension) for other extensions. + The library search path needs to be amended so these will be found + during module import. Use BEGINLIBPATH so that these are at the start + of the library search path. + + """ + dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload") + libpath = os.environ['BEGINLIBPATH'].split(';') + if libpath[-1]: + libpath.append(dllpath) + else: + libpath[-1] = dllpath + os.environ['BEGINLIBPATH'] = ';'.join(libpath) + + +def setquit(): + """Define new built-ins 'quit' and 'exit'. + These are simply strings that display a hint on how to exit. + + """ + if os.sep == ':': + eof = 'Cmd-Q' + elif os.sep == '\\': + eof = 'Ctrl-Z plus Return' + else: + eof = 'Ctrl-D (i.e. EOF)' + + class Quitter(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return 'Use %s() or %s to exit' % (self.name, eof) + def __call__(self, code=None): + # Shells like IDLE catch the SystemExit, but listen when their + # stdin wrapper is closed. + try: + sys.stdin.close() + except: + pass + raise SystemExit(code) + __builtin__.quit = Quitter('quit') + __builtin__.exit = Quitter('exit') + + +class _Printer(object): + """interactive prompt objects for printing the license text, a list of + contributors and the copyright notice.""" + + MAXLINES = 23 + + def __init__(self, name, data, files=(), dirs=()): + self.__name = name + self.__data = data + self.__files = files + self.__dirs = dirs + self.__lines = None + + def __setup(self): + if self.__lines: + return + data = None + for dir in self.__dirs: + for filename in self.__files: + filename = os.path.join(dir, filename) + try: + fp = file(filename, "rU") + data = fp.read() + fp.close() + break + except IOError: + pass + if data: + break + if not data: + data = self.__data + self.__lines = data.split('\n') + self.__linecnt = len(self.__lines) + + def __repr__(self): + self.__setup() + if len(self.__lines) <= self.MAXLINES: + return "\n".join(self.__lines) + else: + return "Type %s() to see the full %s text" % ((self.__name,)*2) + + def __call__(self): + self.__setup() + prompt = 'Hit Return for more, or q (and Return) to quit: ' + lineno = 0 + while 1: + try: + for i in range(lineno, lineno + self.MAXLINES): + print self.__lines[i] + except IndexError: + break + else: + lineno += self.MAXLINES + key = None + while key is None: + key = raw_input(prompt) + if key not in ('', 'q'): + key = None + if key == 'q': + break + +def setcopyright(): + """Set 'copyright' and 'credits' in __builtin__""" + __builtin__.copyright = _Printer("copyright", sys.copyright) + if _is_jython: + __builtin__.credits = _Printer( + "credits", + "Jython is maintained by the Jython developers (www.jython.org).") + elif _is_pypy: + __builtin__.credits = _Printer( + "credits", + "PyPy is maintained by the PyPy developers: http://codespeak.net/pypy") + else: + __builtin__.credits = _Printer("credits", """\ + Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands + for supporting Python development. See www.python.org for more information.""") + here = os.path.dirname(os.__file__) + __builtin__.license = _Printer( + "license", "See http://www.python.org/%.3s/license.html" % sys.version, + ["LICENSE.txt", "LICENSE"], + [os.path.join(here, os.pardir), here, os.curdir]) + + +class _Helper(object): + """Define the built-in 'help'. + This is a wrapper around pydoc.help (with a twist). + + """ + + def __repr__(self): + return "Type help() for interactive help, " \ + "or help(object) for help about object." + def __call__(self, *args, **kwds): + import pydoc + return pydoc.help(*args, **kwds) + +def sethelper(): + __builtin__.help = _Helper() + +def aliasmbcs(): + """On Windows, some default encodings are not provided by Python, + while they are always available as "mbcs" in each locale. Make + them usable by aliasing to "mbcs" in such a case.""" + if sys.platform == 'win32': + import locale, codecs + enc = locale.getdefaultlocale()[1] + if enc.startswith('cp'): # "cp***" ? + try: + codecs.lookup(enc) + except LookupError: + import encodings + encodings._cache[enc] = encodings._unknown + encodings.aliases.aliases[enc] = 'mbcs' + +def setencoding(): + """Set the string encoding used by the Unicode implementation. The + default is 'ascii', but if you're willing to experiment, you can + change this.""" + encoding = "ascii" # Default value set by _PyUnicode_Init() + if 0: + # Enable to support locale aware default string encodings. + import locale + loc = locale.getdefaultlocale() + if loc[1]: + encoding = loc[1] + if 0: + # Enable to switch off string to Unicode coercion and implicit + # Unicode to string conversion. + encoding = "undefined" + if encoding != "ascii": + # On Non-Unicode builds this will raise an AttributeError... + sys.setdefaultencoding(encoding) # Needs Python Unicode build ! + + +def execsitecustomize(): + """Run custom site specific code, if available.""" + try: + import sitecustomize + except ImportError: + pass + +def virtual_install_main_packages(): + f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt')) + sys.real_prefix = f.read().strip() + f.close() + pos = 2 + if sys.path[0] == '': + pos += 1 + if sys.platform == 'win32': + paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')] + elif _is_jython: + paths = [os.path.join(sys.real_prefix, 'Lib')] + elif _is_pypy: + cpyver = '%d.%d.%d' % sys.version_info[:3] + paths = [os.path.join(sys.real_prefix, 'lib_pypy'), + os.path.join(sys.real_prefix, 'lib-python', 'modified-%s' % cpyver), + os.path.join(sys.real_prefix, 'lib-python', cpyver)] + else: + paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])] + lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3]) + if os.path.exists(lib64_path): + paths.append(lib64_path) + # This is hardcoded in the Python executable, but relative to sys.prefix: + plat_path = os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], + 'plat-%s' % sys.platform) + if os.path.exists(plat_path): + paths.append(plat_path) + # This is hardcoded in the Python executable, but + # relative to sys.prefix, so we have to fix up: + for path in list(paths): + tk_dir = os.path.join(path, 'lib-tk') + if os.path.exists(tk_dir): + paths.append(tk_dir) + + # These are hardcoded in the Apple's Python executable, + # but relative to sys.prefix, so we have to fix them up: + if sys.platform == 'darwin': + hardcoded_paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], module) + for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')] + + for path in hardcoded_paths: + if os.path.exists(path): + paths.append(path) + + sys.path.extend(paths) + +def force_global_eggs_after_local_site_packages(): + """ + Force easy_installed eggs in the global environment to get placed + in sys.path after all packages inside the virtualenv. This + maintains the "least surprise" result that packages in the + virtualenv always mask global packages, never the other way + around. + + """ + egginsert = getattr(sys, '__egginsert', 0) + for i, path in enumerate(sys.path): + if i > egginsert and path.startswith(sys.prefix): + egginsert = i + sys.__egginsert = egginsert + 1 + +def virtual_addsitepackages(known_paths): + force_global_eggs_after_local_site_packages() + return addsitepackages(known_paths, sys_prefix=sys.real_prefix) + +def fixclasspath(): + """Adjust the special classpath sys.path entries for Jython. These + entries should follow the base virtualenv lib directories. + """ + paths = [] + classpaths = [] + for path in sys.path: + if path == '__classpath__' or path.startswith('__pyclasspath__'): + classpaths.append(path) + else: + paths.append(path) + sys.path = paths + sys.path.extend(classpaths) + +def execusercustomize(): + """Run custom user specific code, if available.""" + try: + import usercustomize + except ImportError: + pass + + +def main(): + global ENABLE_USER_SITE + virtual_install_main_packages() + abs__file__() + paths_in_sys = removeduppaths() + if (os.name == "posix" and sys.path and + os.path.basename(sys.path[-1]) == "Modules"): + addbuilddir() + if _is_jython: + fixclasspath() + GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt')) + if not GLOBAL_SITE_PACKAGES: + ENABLE_USER_SITE = False + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + paths_in_sys = addsitepackages(paths_in_sys) + paths_in_sys = addusersitepackages(paths_in_sys) + if GLOBAL_SITE_PACKAGES: + paths_in_sys = virtual_addsitepackages(paths_in_sys) + if sys.platform == 'os2emx': + setBEGINLIBPATH() + setquit() + setcopyright() + sethelper() + aliasmbcs() + setencoding() + execsitecustomize() + if ENABLE_USER_SITE: + execusercustomize() + # Remove sys.setdefaultencoding() so that users cannot change the + # encoding after initialization. The test for presence is needed when + # this module is run as a script, because this code is executed twice. + if hasattr(sys, "setdefaultencoding"): + del sys.setdefaultencoding + +main() + +def _script(): + help = """\ + %s [--user-base] [--user-site] + + Without arguments print some useful information + With arguments print the value of USER_BASE and/or USER_SITE separated + by '%s'. + + Exit codes with --user-base or --user-site: + 0 - user site directory is enabled + 1 - user site directory is disabled by user + 2 - uses site directory is disabled by super user + or for security reasons + >2 - unknown error + """ + args = sys.argv[1:] + if not args: + print "sys.path = [" + for dir in sys.path: + print " %r," % (dir,) + print "]" + def exists(path): + if os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + print "USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE)) + print "USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE)) + print "ENABLE_USER_SITE: %r" % ENABLE_USER_SITE + sys.exit(0) + + buffer = [] + if '--user-base' in args: + buffer.append(USER_BASE) + if '--user-site' in args: + buffer.append(USER_SITE) + + if buffer: + print os.pathsep.join(buffer) + if ENABLE_USER_SITE: + sys.exit(0) + elif ENABLE_USER_SITE is False: + sys.exit(1) + elif ENABLE_USER_SITE is None: + sys.exit(2) + else: + sys.exit(3) + else: + import textwrap + print textwrap.dedent(help % (sys.argv[0], os.pathsep)) + sys.exit(10) + +if __name__ == '__main__': + _script() diff --git a/test/lib/python2.6/sre.py b/test/lib/python2.6/sre.py new file mode 120000 index 000000000..3ade4e80e --- /dev/null +++ b/test/lib/python2.6/sre.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_compile.py b/test/lib/python2.6/sre_compile.py new file mode 120000 index 000000000..fc13a37d5 --- /dev/null +++ b/test/lib/python2.6/sre_compile.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_compile.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_constants.py b/test/lib/python2.6/sre_constants.py new file mode 120000 index 000000000..98b0d5543 --- /dev/null +++ b/test/lib/python2.6/sre_constants.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_constants.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_parse.py b/test/lib/python2.6/sre_parse.py new file mode 120000 index 000000000..76b24c687 --- /dev/null +++ b/test/lib/python2.6/sre_parse.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_parse.py \ No newline at end of file diff --git a/test/lib/python2.6/stat.py b/test/lib/python2.6/stat.py new file mode 120000 index 000000000..68d61e662 --- /dev/null +++ b/test/lib/python2.6/stat.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/stat.py \ No newline at end of file diff --git a/test/lib/python2.6/types.py b/test/lib/python2.6/types.py new file mode 120000 index 000000000..72b00ba35 --- /dev/null +++ b/test/lib/python2.6/types.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/types.py \ No newline at end of file diff --git a/test/lib/python2.6/warnings.py b/test/lib/python2.6/warnings.py new file mode 120000 index 000000000..1af4e2833 --- /dev/null +++ b/test/lib/python2.6/warnings.py @@ -0,0 +1 @@ +/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/warnings.py \ No newline at end of file -- cgit From ec7ab22b8b08820f7ffa3b05e40b09a925f1ef95 Mon Sep 17 00:00:00 2001 From: Anne Gentle <anne@openstack.org> Date: Mon, 21 Feb 2011 14:35:30 -0600 Subject: Removing pesky DS_Store files too. Begone. --- .bzrignore | 1 + doc/build/.DS_Store | Bin 6148 -> 0 bytes doc/build/html/.DS_Store | Bin 6148 -> 0 bytes 3 files changed, 1 insertion(+) delete mode 100644 doc/build/.DS_Store delete mode 100644 doc/build/html/.DS_Store diff --git a/.bzrignore b/.bzrignore index b271561a3..d22b62629 100644 --- a/.bzrignore +++ b/.bzrignore @@ -13,3 +13,4 @@ CA/serial* CA/newcerts/*.pem CA/private/cakey.pem nova/vcsversion.py +*.DS_Store diff --git a/doc/build/.DS_Store b/doc/build/.DS_Store deleted file mode 100644 index 5008ddfcf..000000000 Binary files a/doc/build/.DS_Store and /dev/null differ diff --git a/doc/build/html/.DS_Store b/doc/build/html/.DS_Store deleted file mode 100644 index 5008ddfcf..000000000 Binary files a/doc/build/html/.DS_Store and /dev/null differ -- cgit From 6e6c3fcfe97d60b6262bf90d423a77fa250bd383 Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Tue, 22 Feb 2011 08:22:42 +0100 Subject: added disabled services to the list of displayed services in bin/nova-manage brontes:~ # nova-manage service list ares nova-scheduler enabled :-) 2011-02-22 07:21:29 ares nova-network enabled :-) 2011-02-22 07:21:29 ares nova-volume enabled XXX 2011-02-16 19:04:29 brontes nova-volume enabled XXX 2011-02-12 18:31:43 brontes nova-network enabled :-) 2011-02-22 07:21:29 ares nova-compute disabled :-) 2011-02-22 07:21:25 brontes nova-compute disabled :-) 2011-02-22 07:21:24 --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 6d67252b8..af8432441 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -552,7 +552,7 @@ class ServiceCommands(object): args: [host] [service]""" ctxt = context.get_admin_context() now = datetime.datetime.utcnow() - services = db.service_get_all(ctxt) + services = db.service_get_all(ctxt) + db.service_get_all(ctxt, True) if host: services = [s for s in services if s['host'] == host] if service: -- cgit From cc8276655cd2424b71689776f24e2a5cc767e844 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Wed, 23 Feb 2011 23:37:52 -0800 Subject: move relevant code to baseclass and make flatdhcp not inherit from flat --- nova/network/manager.py | 141 +++++++++++++++++++++--------------------------- 1 file changed, 62 insertions(+), 79 deletions(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index 1df193be0..e999adae5 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -163,11 +163,22 @@ class NetworkManager(manager.Manager): def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" - raise NotImplementedError() + # TODO(vish): when this is called by compute, we can associate compute + # with a network, or a cluster of computes with a network + # and use that network here with a method like + # network_get_by_compute_host + network_ref = self.db.network_get_by_bridge(context, + FLAGS.flat_network_bridge) + address = self.db.fixed_ip_associate_pool(context.elevated(), + network_ref['id'], + instance_id) + self.db.fixed_ip_update(context, address, {'allocated': True}) + return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): """Returns a fixed ip to the pool.""" - raise NotImplementedError() + self.db.fixed_ip_update(context, address, {'allocated': False}) + self.db.fixed_ip_disassociate(context.elevated(), address) def setup_fixed_ip(self, context, address): """Sets up rules for fixed ip.""" @@ -257,12 +268,57 @@ class NetworkManager(manager.Manager): def get_network_host(self, context): """Get the network host for the current context.""" - raise NotImplementedError() + network_ref = self.db.network_get_by_bridge(context, + FLAGS.flat_network_bridge) + # NOTE(vish): If the network has no host, use the network_host flag. + # This could eventually be a a db lookup of some sort, but + # a flag is easy to handle for now. + host = network_ref['host'] + if not host: + topic = self.db.queue_get_for(context, + FLAGS.network_topic, + FLAGS.network_host) + if FLAGS.fake_call: + return self.set_network_host(context, network_ref['id']) + host = rpc.call(context, + FLAGS.network_topic, + {"method": "set_network_host", + "args": {"network_id": network_ref['id']}}) + return host def create_networks(self, context, cidr, num_networks, network_size, - cidr_v6, *args, **kwargs): + cidr_v6, label, *args, **kwargs): """Create networks based on parameters.""" - raise NotImplementedError() + fixed_net = IPy.IP(cidr) + fixed_net_v6 = IPy.IP(cidr_v6) + significant_bits_v6 = 64 + count = 1 + for index in range(num_networks): + start = index * network_size + significant_bits = 32 - int(math.log(network_size, 2)) + cidr = "%s/%s" % (fixed_net[start], significant_bits) + project_net = IPy.IP(cidr) + net = {} + net['bridge'] = FLAGS.flat_network_bridge + net['cidr'] = cidr + net['netmask'] = str(project_net.netmask()) + net['gateway'] = str(project_net[1]) + net['broadcast'] = str(project_net.broadcast()) + net['dhcp_start'] = str(project_net[2]) + if num_networks > 1: + net['label'] = "%s_%d" % (label, count) + else: + net['label'] = label + count += 1 + + if(FLAGS.use_ipv6): + cidr_v6 = "%s/%s" % (fixed_net_v6[0], significant_bits_v6) + net['cidr_v6'] = cidr_v6 + + network_ref = self.db.network_create_safe(context, net) + + if network_ref: + self._create_fixed_ips(context, network_ref['id']) @property def _bottom_reserved_ips(self): # pylint: disable-msg=R0201 @@ -332,83 +388,10 @@ class FlatManager(NetworkManager): for network in self.db.host_get_networks(ctxt, self.host): self._on_set_network_host(ctxt, network['id']) - def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool.""" - # TODO(vish): when this is called by compute, we can associate compute - # with a network, or a cluster of computes with a network - # and use that network here with a method like - # network_get_by_compute_host - network_ref = self.db.network_get_by_bridge(context, - FLAGS.flat_network_bridge) - address = self.db.fixed_ip_associate_pool(context.elevated(), - network_ref['id'], - instance_id) - self.db.fixed_ip_update(context, address, {'allocated': True}) - return address - - def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool.""" - self.db.fixed_ip_update(context, address, {'allocated': False}) - self.db.fixed_ip_disassociate(context.elevated(), address) - def setup_compute_network(self, context, instance_id): """Network is created manually.""" pass - def create_networks(self, context, cidr, num_networks, network_size, - cidr_v6, label, *args, **kwargs): - """Create networks based on parameters.""" - fixed_net = IPy.IP(cidr) - fixed_net_v6 = IPy.IP(cidr_v6) - significant_bits_v6 = 64 - count = 1 - for index in range(num_networks): - start = index * network_size - significant_bits = 32 - int(math.log(network_size, 2)) - cidr = "%s/%s" % (fixed_net[start], significant_bits) - project_net = IPy.IP(cidr) - net = {} - net['bridge'] = FLAGS.flat_network_bridge - net['cidr'] = cidr - net['netmask'] = str(project_net.netmask()) - net['gateway'] = str(project_net[1]) - net['broadcast'] = str(project_net.broadcast()) - net['dhcp_start'] = str(project_net[2]) - if num_networks > 1: - net['label'] = "%s_%d" % (label, count) - else: - net['label'] = label - count += 1 - - if(FLAGS.use_ipv6): - cidr_v6 = "%s/%s" % (fixed_net_v6[0], significant_bits_v6) - net['cidr_v6'] = cidr_v6 - - network_ref = self.db.network_create_safe(context, net) - - if network_ref: - self._create_fixed_ips(context, network_ref['id']) - - def get_network_host(self, context): - """Get the network host for the current context.""" - network_ref = self.db.network_get_by_bridge(context, - FLAGS.flat_network_bridge) - # NOTE(vish): If the network has no host, use the network_host flag. - # This could eventually be a a db lookup of some sort, but - # a flag is easy to handle for now. - host = network_ref['host'] - if not host: - topic = self.db.queue_get_for(context, - FLAGS.network_topic, - FLAGS.network_host) - if FLAGS.fake_call: - return self.set_network_host(context, network_ref['id']) - host = rpc.call(context, - FLAGS.network_topic, - {"method": "set_network_host", - "args": {"network_id": network_ref['id']}}) - return host - def _on_set_network_host(self, context, network_id): """Called when this host becomes the host for a network.""" net = {} @@ -433,7 +416,7 @@ class FlatManager(NetworkManager): raise NotImplementedError() -class FlatDHCPManager(FlatManager): +class FlatDHCPManager(NetworkManager): """Flat networking with dhcp. FlatDHCPManager will start up one dhcp server to give out addresses. -- cgit From 8cf6a0c01ee39066f17a11d5e9313c2828a59634 Mon Sep 17 00:00:00 2001 From: Rick Harris <rick.harris@rackspace.com> Date: Fri, 25 Feb 2011 01:50:18 +0000 Subject: No longer users image/ directory in tarball --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index e229a9358..bcdf34413 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -163,16 +163,14 @@ def _fixup_vhds(sr_path, staging_path, uuid_stack): "VHD %(path)s is marked as hidden without child" % locals()) - image_path = os.path.join(staging_path, 'image') - - orig_base_copy_path = os.path.join(image_path, 'image.vhd') + orig_base_copy_path = os.path.join(staging_path, 'image.vhd') if not os.path.exists(orig_base_copy_path): raise Exception("Invalid image: image.vhd not present") base_copy_path, base_copy_uuid = rename_with_uuid(orig_base_copy_path) vdi_uuid = base_copy_uuid - orig_snap_path = os.path.join(image_path, 'snap.vhd') + orig_snap_path = os.path.join(staging_path, 'snap.vhd') if os.path.exists(orig_snap_path): snap_path, snap_uuid = rename_with_uuid(orig_snap_path) vdi_uuid = snap_uuid @@ -192,11 +190,9 @@ def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): """Hard-link VHDs into staging area with appropriate filename ('snap' or 'image.vhd') """ - image_path = os.path.join(staging_path, 'image') - os.mkdir(image_path) for name, uuid in vdi_uuids.items(): source = os.path.join(sr_path, "%s.vhd" % uuid) - link_name = os.path.join(image_path, "%s.vhd" % name) + link_name = os.path.join(staging_path, "%s.vhd" % name) os.link(source, link_name) -- cgit From ec9ede003c839248ca9593c03160a23ff8ec0db1 Mon Sep 17 00:00:00 2001 From: Rick Harris <rick.harris@rackspace.com> Date: Fri, 25 Feb 2011 02:26:46 +0000 Subject: Adding _make_subprocess function --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 57 ++++++++++++---------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index bcdf34413..869d46e70 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -75,17 +75,15 @@ def _download_tarball(sr_path, staging_path, image_id, glance_host, elif resp.status != httplib.OK: raise Exception("Unexpected response from Glance %i" % res.status) - tar_args = shlex.split( - "tar -zx --directory=%(staging_path)s" % locals()) - tar_proc = subprocess.Popen( - tar_args, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + tar_cmd = "tar -zx --directory=%(staging_path)s" % locals() + tar_proc = _make_subprocess(tar_cmd, stderr=True, stdin=True) chunk = resp.read(CHUNK_SIZE) while chunk: tar_proc.stdin.write(chunk) chunk = resp.read(CHUNK_SIZE) - _assert_process_success(tar_proc, "tar") + _finish_subprocess(tar_proc, "tar") conn.close() @@ -130,12 +128,10 @@ def _fixup_vhds(sr_path, staging_path, uuid_stack): This needs to be done before we move both VHDs into the SR to prevent the base_copy from being DOA (deleted-on-arrival). """ - modify_args = shlex.split( - "vhd-util modify -n %(child_path)s -p %(parent_path)s" - % locals()) - modify_proc = subprocess.Popen( - modify_args, stderr=subprocess.PIPE) - _assert_process_success(modify_proc, "vhd-util") + modify_cmd = ("vhd-util modify -n %(child_path)s -p %(parent_path)s" + % locals()) + modify_proc = _make_subprocess(modify_cmd, stderr=True) + _finish_subprocess(modify_proc, "vhd-util") def move_into_sr(orig_path): """Move a file into the SR""" @@ -150,11 +146,9 @@ def _fixup_vhds(sr_path, staging_path, uuid_stack): present, then the image.vhd better not be marked 'hidden' or it will be deleted when moved into the SR. """ - vhd_query_args = shlex.split( - "vhd-util query -n %(path)s -f" % locals()) - vhd_query_proc = subprocess.Popen( - vhd_query_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = _assert_process_success(vhd_query_proc, "vhd-util") + query_cmd = "vhd-util query -n %(path)s -f" % locals() + query_proc = _make_subprocess(query_cmd, stdout=True, stderr=True) + out, err = _finish_subprocess(query_proc, "vhd-util") for line in out.splitlines(): if line.startswith('hidden'): value = line.split(':')[1].strip() @@ -208,25 +202,24 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): # FIXME(sirp): nokernel signals Nova to use a raw image. This is defined by # FLAGS.null_kernel. Is there a way to get rid of this? + # TODO(sirp): make `store` configurable headers = { - 'x-image-meta-store': 'file', + 'content-type': 'application/octet-stream', + 'transfer-encoding': 'chunked', 'x-image-meta-is_public': 'True', - 'x-image-meta-type': 'vhd', 'x-image-meta-status': 'queued', + 'x-image-meta-store': 'file', + 'x-image-meta-type': 'vhd', 'x-image-meta-property-kernel-id': 'nokernel', 'x-image-meta-property-ramdisk-id': 'noramdisk', 'x-image-meta-property-container-format': 'tarball', - 'transfer-encoding': 'chunked', - 'content-type': 'application/octet-stream', } for header, value in headers.iteritems(): conn.putheader(header, value) conn.endheaders() - tar_args = shlex.split( - "tar -zc --directory=%(staging_path)s ." % locals()) - tar_proc = subprocess.Popen( - tar_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + tar_cmd = "tar -zc --directory=%(staging_path)s ." % locals() + tar_proc = _make_subprocess(tar_cmd, stdout=True, stderr=True) chunk = tar_proc.stdout.read(CHUNK_SIZE) while chunk: @@ -234,7 +227,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): chunk = tar_proc.stdout.read(CHUNK_SIZE) conn.send("0\r\n\r\n") - _assert_process_success(tar_proc, "tar") + _finish_subprocess(tar_proc, "tar") resp = conn.getresponse() if resp.status != httplib.OK: @@ -289,7 +282,19 @@ def _cleanup_staging_area(staging_path): shutil.rmtree(staging_path) -def _assert_process_success(proc, cmd): +def _make_subprocess(cmdline, stdout=False, stderr=False, stdin=False): + """Make a subprocess according to the given command-line string + """ + kwargs = {} + kwargs['stdout'] = stdout and subprocess.PIPE or None + kwargs['stderr'] = stderr and subprocess.PIPE or None + kwargs['stdin'] = stdin and subprocess.PIPE or None + args = shlex.split(cmdline) + proc = subprocess.Popen(args, **kwargs) + return proc + + +def _finish_subprocess(proc, cmd): """Ensure that the process returned a zero exit code indicating success """ out, err = proc.communicate() -- cgit From e3d6dc70a6b77d80afcf87473bc79549540ac4ce Mon Sep 17 00:00:00 2001 From: Rick Harris <rick.harris@rackspace.com> Date: Fri, 25 Feb 2011 02:51:14 +0000 Subject: Removing unecessary nokernel stuff --- nova/api/openstack/servers.py | 51 ++++++++++++---------- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 4 -- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index c8c94f1fd..f51da0cdd 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -136,28 +136,6 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def _get_kernel_ramdisk_from_image(self, req, image_id): - """ - Machine images are associated with Kernels and Ramdisk images via - metadata stored in Glance as 'image_properties' - """ - # FIXME(sirp): Currently Nova requires us to specify the `null_kernel` - # identifier ('nokernel') to indicate a RAW (or VHD) image. It would - # be better if we could omit the kernel_id and ramdisk_id properties - # on the image - def lookup(image, param): - _image_id = image['id'] - try: - return image['properties'][param] - except KeyError: - LOG.debug( - _("%(param)s property not found for image %(_image_id)s") % - locals()) - return None - - image = self._image_service.show(req.environ['nova.context'], image_id) - return lookup(image, 'kernel_id'), lookup(image, 'ramdisk_id') - def create(self, req): """ Creates a new server for a given user """ env = self._deserialize(req.body, req) @@ -367,3 +345,32 @@ class Controller(wsgi.Controller): action=item.action, error=item.error)) return dict(actions=actions) + + def _get_kernel_ramdisk_from_image(self, req, image_id): + """Retrevies kernel and ramdisk IDs from Glance + + Only 'machine' (ami) type use kernel and ramdisk outside of the + image. + """ + # FIXME(sirp): Since we're retrieving the kernel_id from an + # image_property, this means only Glance is supported. + # The BaseImageService needs to expose a consistent way of accessing + # kernel_id and ramdisk_id + image = self._image_service.show(req.environ['nova.context'], image_id) + + if image['type'] != 'machine': + return None, None + + try: + kernel_id = image['properties']['kernel_id'] + except KeyError: + raise exception.NotFound( + _("Kernel not found for image %(image_id)s") % locals()) + + try: + ramdisk_id = image['properties']['ramdisk_id'] + except KeyError: + raise exception.NotFound( + _("Ramdisk not found for image %(image_id)s") % locals()) + + return kernel_id, ramdisk_id diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 869d46e70..18e4866f7 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -200,8 +200,6 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): # to request conn.putrequest('PUT', '/images/%s' % image_id) - # FIXME(sirp): nokernel signals Nova to use a raw image. This is defined by - # FLAGS.null_kernel. Is there a way to get rid of this? # TODO(sirp): make `store` configurable headers = { 'content-type': 'application/octet-stream', @@ -210,8 +208,6 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): 'x-image-meta-status': 'queued', 'x-image-meta-store': 'file', 'x-image-meta-type': 'vhd', - 'x-image-meta-property-kernel-id': 'nokernel', - 'x-image-meta-property-ramdisk-id': 'noramdisk', 'x-image-meta-property-container-format': 'tarball', } for header, value in headers.iteritems(): -- cgit From d27197ae53f3282280198d9bfe7f37a059fa8a35 Mon Sep 17 00:00:00 2001 From: Rick Harris <rick.harris@rackspace.com> Date: Fri, 25 Feb 2011 03:16:04 +0000 Subject: Removing unecessary headers --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 18e4866f7..7531af4ec 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -83,7 +83,7 @@ def _download_tarball(sr_path, staging_path, image_id, glance_host, tar_proc.stdin.write(chunk) chunk = resp.read(CHUNK_SIZE) - _finish_subprocess(tar_proc, "tar") + _finish_subprocess(tar_proc, tar_cmd) conn.close() @@ -131,7 +131,7 @@ def _fixup_vhds(sr_path, staging_path, uuid_stack): modify_cmd = ("vhd-util modify -n %(child_path)s -p %(parent_path)s" % locals()) modify_proc = _make_subprocess(modify_cmd, stderr=True) - _finish_subprocess(modify_proc, "vhd-util") + _finish_subprocess(modify_proc, modify_cmd) def move_into_sr(orig_path): """Move a file into the SR""" @@ -148,7 +148,8 @@ def _fixup_vhds(sr_path, staging_path, uuid_stack): """ query_cmd = "vhd-util query -n %(path)s -f" % locals() query_proc = _make_subprocess(query_cmd, stdout=True, stderr=True) - out, err = _finish_subprocess(query_proc, "vhd-util") + out, err = _finish_subprocess(query_proc, query_cmd) + for line in out.splitlines(): if line.startswith('hidden'): value = line.split(':')[1].strip() @@ -206,9 +207,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): 'transfer-encoding': 'chunked', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'queued', - 'x-image-meta-store': 'file', - 'x-image-meta-type': 'vhd', - 'x-image-meta-property-container-format': 'tarball', + 'x-image-meta-type': 'vhd' } for header, value in headers.iteritems(): conn.putheader(header, value) @@ -223,7 +222,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): chunk = tar_proc.stdout.read(CHUNK_SIZE) conn.send("0\r\n\r\n") - _finish_subprocess(tar_proc, "tar") + _finish_subprocess(tar_proc, tar_cmd) resp = conn.getresponse() if resp.status != httplib.OK: @@ -290,14 +289,14 @@ def _make_subprocess(cmdline, stdout=False, stderr=False, stdin=False): return proc -def _finish_subprocess(proc, cmd): +def _finish_subprocess(proc, cmdline): """Ensure that the process returned a zero exit code indicating success """ out, err = proc.communicate() ret = proc.returncode if ret != 0: - msg = "%(cmd)s returned non-zero exit code (%i): '%s'" % (ret, err) - raise Exception(msg) + raise Exception("'%(cmdline)s' returned non-zero exit code: " + "retcode=%(ret)i, stderr='%(err)s'" % locals()) return out, err -- cgit From 079b532a1080da9fe5d99e90fa9c60d16506de06 Mon Sep 17 00:00:00 2001 From: Rick Harris <rick.harris@rackspace.com> Date: Fri, 25 Feb 2011 16:24:51 +0000 Subject: Verify status of image is active --- nova/api/openstack/servers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index f51da0cdd..bf5663f60 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -358,6 +358,11 @@ class Controller(wsgi.Controller): # kernel_id and ramdisk_id image = self._image_service.show(req.environ['nova.context'], image_id) + if image['status'] != 'active': + raise exception.Invalid( + _("Cannot build from image %(image_id)s, status not active") % + locals()) + if image['type'] != 'machine': return None, None -- cgit From 26d55d0d24dd5363d3b0dd6da0351fe389b79503 Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Fri, 25 Feb 2011 19:59:26 +0100 Subject: check if QUERY_STRING is empty or not before building the request URL --- bin/nova-ajax-console-proxy | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 1e11c6d58..023e4023f 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -63,10 +63,16 @@ class AjaxConsoleProxy(object): def __call__(self, env, start_response): try: - req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], - env['HTTP_HOST'], - env['PATH_INFO'], - env['QUERY_STRING']) + if env.has_key('QUERY_STRING'): + req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], + env['HTTP_HOST'], + env['PATH_INFO'], + env['QUERY_STRING']) + else: + req_url = '%s://%s%s' % (env['wsgi.url_scheme'], + env['HTTP_HOST'], + env['PATH_INFO']) + if 'HTTP_REFERER' in env: auth_url = env['HTTP_REFERER'] else: -- cgit From 4453021476fac599c0cee126b6eaa426d4878145 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Fri, 18 Mar 2011 02:09:46 +0000 Subject: Copy over to current trunk my tests, the 401/500 fix, and a couple of fixes to the committed fix which was actually brittle around the edges... --- Authors | 1 + nova/api/openstack/auth.py | 8 ++++++-- nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 14 ++++++++++++-- nova/tests/api/openstack/test_auth.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/Authors b/Authors index 494e614a0..a3c06dcf5 100644 --- a/Authors +++ b/Authors @@ -36,6 +36,7 @@ Joshua McKenty <jmckenty@gmail.com> Justin Santa Barbara <justin@fathomdb.com> Kei Masumoto <masumotok@nttdata.co.jp> Ken Pepple <ken.pepple@gmail.com> +Kevin L. Mitchell <kevin.mitchell@rackspace.com> Koji Iida <iida.koji@lab.ntt.co.jp> Lorin Hochstein <lorin@isi.edu> Matt Dietz <matt.dietz@rackspace.com> diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index dff69a7f2..6011e6115 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -26,6 +26,7 @@ import webob.dec from nova import auth from nova import context from nova import db +from nova import exception from nova import flags from nova import manager from nova import utils @@ -103,11 +104,14 @@ class AuthMiddleware(wsgi.Middleware): 2 days ago. """ ctxt = context.get_admin_context() - token = self.db.auth_token_get(ctxt, token_hash) + try: + token = self.db.auth_token_get(ctxt, token_hash) + except exception.NotFound: + return None if token: delta = datetime.datetime.now() - token.created_at if delta.days >= 2: - self.db.auth_token_destroy(ctxt, token.id) + self.db.auth_token_destroy(ctxt, token.token_hash) else: return self.auth.get_user(token.user_id) return None diff --git a/nova/db/api.py b/nova/db/api.py index 4c7eb857f..dcaf55e8f 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -640,6 +640,11 @@ def auth_token_get(context, token_hash): return IMPL.auth_token_get(context, token_hash) +def auth_token_update(context, token_hash, values): + """Updates a token given the hash representing it.""" + return IMPL.auth_token_update(context, token_hash, values) + + def auth_token_create(context, token): """Creates a new token.""" return IMPL.auth_token_create(context, token) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 0be08c4d1..6df2a8843 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1270,8 +1270,9 @@ def auth_token_destroy(context, token_id): @require_admin_context -def auth_token_get(context, token_hash): - session = get_session() +def auth_token_get(context, token_hash, session=None): + if session is None: + session = get_session() tk = session.query(models.AuthToken).\ filter_by(token_hash=token_hash).\ filter_by(deleted=can_read_deleted(context)).\ @@ -1281,6 +1282,15 @@ def auth_token_get(context, token_hash): return tk +@require_admin_context +def auth_token_update(context, token_hash, values): + session = get_session() + with session.begin(): + token_ref = auth_token_get(context, token_hash, session=session) + token_ref.update(values) + token_ref.save(session=session) + + @require_admin_context def auth_token_create(_context, token): tk = models.AuthToken() diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 86dfb110f..c42c12f7b 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -26,6 +26,7 @@ import nova.api.openstack.auth import nova.auth.manager from nova import auth from nova import context +from nova import db from nova import test from nova.tests.api.openstack import fakes @@ -130,6 +131,33 @@ class Test(test.TestCase): self.assertEqual(result.status, '401 Unauthorized') +class TestFunctional(test.TestCase): + def test_token_lp718999(self): + ctx = context.get_admin_context() + tok = db.auth_token_create(ctx, dict( + token_hash='bacon', + cdn_management_url='', + server_management_url='', + storage_url='', + user_id='ham', + )) + + db.auth_token_update(ctx, tok.token_hash, dict( + created_at=datetime.datetime(2000, 1, 1, 12, 0, 0), + )) + + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-Token'] = 'bacon' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_token_doesnotexist(self): + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-Token'] = 'ham' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + class TestLimiter(test.TestCase): def setUp(self): super(TestLimiter, self).setUp() -- cgit From fa6778586ab303f9e65aa3c50b80d20a4f097c6f Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Mon, 21 Mar 2011 00:41:23 +0000 Subject: Rename test to describe what it actually does --- nova/tests/api/openstack/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index c42c12f7b..ff8d42a14 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -132,7 +132,7 @@ class Test(test.TestCase): class TestFunctional(test.TestCase): - def test_token_lp718999(self): + def test_token_expiry(self): ctx = context.get_admin_context() tok = db.auth_token_create(ctx, dict( token_hash='bacon', -- cgit From 3368b32338f58ac6b77d0c142722c241961f858e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya <vishvananda@gmail.com> Date: Fri, 25 Feb 2011 12:16:58 -0800 Subject: use default flagfile in nova-api --- bin/nova-api | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/nova-api b/bin/nova-api index cf140570a..14be4b841 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -36,6 +36,7 @@ gettext.install('nova', unicode=1) from nova import flags from nova import log as logging +from nova import utils from nova import version from nova import wsgi @@ -80,6 +81,7 @@ def run_app(paste_config_file): if __name__ == '__main__': + utils.default_flagfile() FLAGS(sys.argv) logging.setup() LOG.audit(_("Starting nova-api node (version %s)"), -- cgit From e6ceb2987d1f30a1e972a4bf21a734f1a605d60e Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Fri, 25 Feb 2011 22:15:51 +0100 Subject: fixed: bin/nova-ajax-console-proxy:66:19: W601 .has_key() is deprecated, use 'in' --- bin/nova-ajax-console-proxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 023e4023f..bbd60bade 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -63,7 +63,7 @@ class AjaxConsoleProxy(object): def __call__(self, env, start_response): try: - if env.has_key('QUERY_STRING'): + if 'QUERY_STRING' in env: req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], env['HTTP_HOST'], env['PATH_INFO'], -- cgit From c7c3fe85186862d9d8989ebe01ad69e6cdaa4ee3 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Sun, 20 Mar 2011 22:58:49 +0000 Subject: No reason to dump a stack trace just because we can't reach the AMQP servire; it ends up being just noise --- nova/rpc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 205bb524a..089bf99c1 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -95,14 +95,14 @@ class Consumer(messaging.Consumer): fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = FLAGS.rabbit_retry_interval - LOG.exception(_("AMQP server on %(fl_host)s:%(fl_port)d is" + LOG.error(_("AMQP server on %(fl_host)s:%(fl_port)d is" " unreachable. Trying again in %(fl_intv)d seconds.") % locals()) self.failed_connection = True if self.failed_connection: - LOG.exception(_("Unable to connect to AMQP server " - "after %d tries. Shutting down."), - FLAGS.rabbit_max_retries) + LOG.error(_("Unable to connect to AMQP server " + "after %d tries. Shutting down."), + FLAGS.rabbit_max_retries) sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): -- cgit From 4a217b640c3e0f488c87572787c92b6808ab0a9a Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" <kevin.mitchell@rackspace.com> Date: Fri, 25 Mar 2011 05:30:36 +0000 Subject: Add error message to the error report so we know why the AMQP server is unreachable --- nova/rpc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 089bf99c1..8fe4565dd 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -91,12 +91,13 @@ class Consumer(messaging.Consumer): super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False break - except: # Catching all because carrot sucks + except Exception as e: # Catching all because carrot sucks fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = FLAGS.rabbit_retry_interval LOG.error(_("AMQP server on %(fl_host)s:%(fl_port)d is" - " unreachable. Trying again in %(fl_intv)d seconds.") + " unreachable: %(e)s. Trying again in %(fl_intv)d" + " seconds.") % locals()) self.failed_connection = True if self.failed_connection: -- cgit From 4c83130464c88650df7dfc7974f2fd088a733fec Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Sat, 26 Feb 2011 17:26:38 +0100 Subject: introduced new flag "maximum_nbd_devices" to set the number of possible NBD devices --- nova/virt/disk.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/virt/disk.py b/nova/virt/disk.py index cb639a102..8789d8123 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -40,6 +40,8 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') flags.DEFINE_integer('timeout_nbd', 10, 'time to wait for a NBD device coming up') +flags.DEFINE_integer('maximum_nbd_devices', 16, + 'maximum number of possible nbd devices') def extend(image, size): @@ -141,7 +143,7 @@ def _unlink_device(device, nbd): utils.execute('sudo losetup --detach %s' % device) -_DEVICES = ['/dev/nbd%s' % i for i in xrange(16)] +_DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.maximum_nbd_devices)] def _allocate_device(): -- cgit From 2a3ea106d3aaac3b97295c14742f23b250aa6874 Mon Sep 17 00:00:00 2001 From: Anne Gentle <anne@openstack.org> Date: Sat, 26 Feb 2011 11:13:32 -0600 Subject: Deleting test dir from a pull from trunk --- test/.Python | 1 - test/bin/activate | 76 -- test/bin/activate.csh | 32 - test/bin/activate.fish | 79 -- test/bin/activate_this.py | 32 - test/bin/easy_install | 9 - test/bin/easy_install-2.6 | 9 - test/bin/pip | 9 - test/bin/pip-2.6 | 9 - test/bin/python | Bin 50720 -> 0 bytes test/bin/python2.6 | 1 - test/include/python2.6 | 1 - test/lib/python2.6/UserDict.py | 1 - test/lib/python2.6/_abcoll.py | 1 - test/lib/python2.6/abc.py | 1 - test/lib/python2.6/codecs.py | 1 - test/lib/python2.6/config | 1 - test/lib/python2.6/copy_reg.py | 1 - test/lib/python2.6/distutils/__init__.py | 91 -- test/lib/python2.6/distutils/distutils.cfg | 6 - test/lib/python2.6/encodings | 1 - test/lib/python2.6/fnmatch.py | 1 - test/lib/python2.6/genericpath.py | 1 - test/lib/python2.6/lib-dynload | 1 - test/lib/python2.6/linecache.py | 1 - test/lib/python2.6/locale.py | 1 - test/lib/python2.6/ntpath.py | 1 - test/lib/python2.6/orig-prefix.txt | 1 - test/lib/python2.6/os.py | 1 - test/lib/python2.6/posixpath.py | 1 - test/lib/python2.6/re.py | 1 - test/lib/python2.6/site-packages/easy-install.pth | 4 - .../pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO | 348 ----- .../pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt | 57 - .../EGG-INFO/dependency_links.txt | 1 - .../pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt | 4 - .../pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe | 1 - .../pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt | 1 - .../pip-0.8.1-py2.6.egg/pip/__init__.py | 261 ---- .../pip-0.8.1-py2.6.egg/pip/_pkgutil.py | 589 -------- .../pip-0.8.1-py2.6.egg/pip/backwardcompat.py | 55 - .../pip-0.8.1-py2.6.egg/pip/basecommand.py | 203 --- .../pip-0.8.1-py2.6.egg/pip/baseparser.py | 231 ---- .../pip-0.8.1-py2.6.egg/pip/commands/__init__.py | 1 - .../pip-0.8.1-py2.6.egg/pip/commands/bundle.py | 33 - .../pip-0.8.1-py2.6.egg/pip/commands/completion.py | 60 - .../pip-0.8.1-py2.6.egg/pip/commands/freeze.py | 109 -- .../pip-0.8.1-py2.6.egg/pip/commands/help.py | 32 - .../pip-0.8.1-py2.6.egg/pip/commands/install.py | 247 ---- .../pip-0.8.1-py2.6.egg/pip/commands/search.py | 116 -- .../pip-0.8.1-py2.6.egg/pip/commands/uninstall.py | 42 - .../pip-0.8.1-py2.6.egg/pip/commands/unzip.py | 9 - .../pip-0.8.1-py2.6.egg/pip/commands/zip.py | 346 ----- .../pip-0.8.1-py2.6.egg/pip/download.py | 470 ------- .../pip-0.8.1-py2.6.egg/pip/exceptions.py | 17 - .../site-packages/pip-0.8.1-py2.6.egg/pip/index.py | 686 ---------- .../pip-0.8.1-py2.6.egg/pip/locations.py | 45 - .../site-packages/pip-0.8.1-py2.6.egg/pip/log.py | 181 --- .../site-packages/pip-0.8.1-py2.6.egg/pip/req.py | 1432 -------------------- .../pip-0.8.1-py2.6.egg/pip/runner.py | 18 - .../site-packages/pip-0.8.1-py2.6.egg/pip/util.py | 479 ------- .../pip-0.8.1-py2.6.egg/pip/vcs/__init__.py | 238 ---- .../pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py | 138 -- .../pip-0.8.1-py2.6.egg/pip/vcs/git.py | 204 --- .../pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py | 162 --- .../pip-0.8.1-py2.6.egg/pip/vcs/subversion.py | 260 ---- .../site-packages/pip-0.8.1-py2.6.egg/pip/venv.py | 53 - .../site-packages/setuptools-0.6c11-py2.6.egg | Bin 333447 -> 0 bytes test/lib/python2.6/site-packages/setuptools.pth | 1 - test/lib/python2.6/site.py | 713 ---------- test/lib/python2.6/sre.py | 1 - test/lib/python2.6/sre_compile.py | 1 - test/lib/python2.6/sre_constants.py | 1 - test/lib/python2.6/sre_parse.py | 1 - test/lib/python2.6/stat.py | 1 - test/lib/python2.6/types.py | 1 - test/lib/python2.6/warnings.py | 1 - 77 files changed, 8226 deletions(-) delete mode 120000 test/.Python delete mode 100644 test/bin/activate delete mode 100644 test/bin/activate.csh delete mode 100644 test/bin/activate.fish delete mode 100644 test/bin/activate_this.py delete mode 100755 test/bin/easy_install delete mode 100755 test/bin/easy_install-2.6 delete mode 100755 test/bin/pip delete mode 100755 test/bin/pip-2.6 delete mode 100755 test/bin/python delete mode 120000 test/bin/python2.6 delete mode 120000 test/include/python2.6 delete mode 120000 test/lib/python2.6/UserDict.py delete mode 120000 test/lib/python2.6/_abcoll.py delete mode 120000 test/lib/python2.6/abc.py delete mode 120000 test/lib/python2.6/codecs.py delete mode 120000 test/lib/python2.6/config delete mode 120000 test/lib/python2.6/copy_reg.py delete mode 100644 test/lib/python2.6/distutils/__init__.py delete mode 100644 test/lib/python2.6/distutils/distutils.cfg delete mode 120000 test/lib/python2.6/encodings delete mode 120000 test/lib/python2.6/fnmatch.py delete mode 120000 test/lib/python2.6/genericpath.py delete mode 120000 test/lib/python2.6/lib-dynload delete mode 120000 test/lib/python2.6/linecache.py delete mode 120000 test/lib/python2.6/locale.py delete mode 120000 test/lib/python2.6/ntpath.py delete mode 100644 test/lib/python2.6/orig-prefix.txt delete mode 120000 test/lib/python2.6/os.py delete mode 120000 test/lib/python2.6/posixpath.py delete mode 120000 test/lib/python2.6/re.py delete mode 100644 test/lib/python2.6/site-packages/easy-install.pth delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py delete mode 100644 test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py delete mode 100644 test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg delete mode 100644 test/lib/python2.6/site-packages/setuptools.pth delete mode 100644 test/lib/python2.6/site.py delete mode 120000 test/lib/python2.6/sre.py delete mode 120000 test/lib/python2.6/sre_compile.py delete mode 120000 test/lib/python2.6/sre_constants.py delete mode 120000 test/lib/python2.6/sre_parse.py delete mode 120000 test/lib/python2.6/stat.py delete mode 120000 test/lib/python2.6/types.py delete mode 120000 test/lib/python2.6/warnings.py diff --git a/test/.Python b/test/.Python deleted file mode 120000 index 6cce156bd..000000000 --- a/test/.Python +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/Python \ No newline at end of file diff --git a/test/bin/activate b/test/bin/activate deleted file mode 100644 index 588b54105..000000000 --- a/test/bin/activate +++ /dev/null @@ -1,76 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "$_OLD_VIRTUAL_PATH" ] ; then - PATH="$_OLD_VIRTUAL_PATH" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then - PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then - hash -r - fi - - if [ -n "$_OLD_VIRTUAL_PS1" ] ; then - PS1="$_OLD_VIRTUAL_PS1" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - if [ ! "$1" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelavent variables -deactivate nondestructive - -VIRTUAL_ENV="/Users/anne.gentle/src/nova/docslice/test" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "$PYTHONHOME" ] ; then - _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" - unset PYTHONHOME -fi - -if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then - _OLD_VIRTUAL_PS1="$PS1" - if [ "x" != x ] ; then - PS1="$PS1" - else - if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" - else - PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" - fi - fi - export PS1 -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then - hash -r -fi diff --git a/test/bin/activate.csh b/test/bin/activate.csh deleted file mode 100644 index d29ac339a..000000000 --- a/test/bin/activate.csh +++ /dev/null @@ -1,32 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi <davidedb@gmail.com>. - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelavent variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if ("" != "") then - set env_name = "" -else - if (`basename "$VIRTUAL_ENV"` == "__") then - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` - else - set env_name = `basename "$VIRTUAL_ENV"` - endif -endif -set prompt = "[$env_name] $prompt" -unset env_name - -rehash - diff --git a/test/bin/activate.fish b/test/bin/activate.fish deleted file mode 100644 index de3a8901e..000000000 --- a/test/bin/activate.fish +++ /dev/null @@ -1,79 +0,0 @@ -# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) -# you cannot run it directly - -function deactivate -d "Exit virtualenv and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - end - - set -e VIRTUAL_ENV - if test "$argv[1]" != "nondestructive" - # Self destruct! - functions -e deactivate - end -end - -# unset irrelavent variables -deactivate nondestructive - -set -gx VIRTUAL_ENV "/Users/anne.gentle/src/nova/docslice/test" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# unset PYTHONHOME if set -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish shell uses a function, instead of env vars, - # to produce the prompt. Overriding the existing function is easy. - # However, adding to the current prompt, instead of clobbering it, - # is a little more work. - set -l oldpromptfile (tempfile) - if test $status - # save the current fish_prompt function... - echo "function _old_fish_prompt" >> $oldpromptfile - echo -n \# >> $oldpromptfile - functions fish_prompt >> $oldpromptfile - # we've made the "_old_fish_prompt" file, source it. - . $oldpromptfile - rm -f $oldpromptfile - - if test -n "" - # We've been given us a prompt override. - # - # FIXME: Unsure how to handle this *safely*. We could just eval() - # whatever is given, but the risk is a bit much. - echo "activate.fish: Alternative prompt prefix is not supported under fish-shell." 1>&2 - echo "activate.fish: Alter the fish_prompt in this file as needed." 1>&2 - end - - # with the original prompt function renamed, we can override with our own. - function fish_prompt - set -l _checkbase (basename "$VIRTUAL_ENV") - if test $_checkbase = "__" - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - printf "%s[%s]%s %s" (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) (_old_fish_prompt) - else - printf "%s(%s)%s%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) (_old_fish_prompt) - end - end - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - end -end - diff --git a/test/bin/activate_this.py b/test/bin/activate_this.py deleted file mode 100644 index aff6927d6..000000000 --- a/test/bin/activate_this.py +++ /dev/null @@ -1,32 +0,0 @@ -"""By using execfile(this_file, dict(__file__=this_file)) you will -activate this virtualenv environment. - -This can be used when you must use an existing Python interpreter, not -the virtualenv bin/python -""" - -try: - __file__ -except NameError: - raise AssertionError( - "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))") -import sys -import os - -base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if sys.platform == 'win32': - site_packages = os.path.join(base, 'Lib', 'site-packages') -else: - site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') -prev_sys_path = list(sys.path) -import site -site.addsitedir(site_packages) -sys.real_prefix = sys.prefix -sys.prefix = base -# Move the added items to the front of the path: -new_sys_path = [] -for item in list(sys.path): - if item not in prev_sys_path: - new_sys_path.append(item) - sys.path.remove(item) -sys.path[:0] = new_sys_path diff --git a/test/bin/easy_install b/test/bin/easy_install deleted file mode 100755 index 8544573fc..000000000 --- a/test/bin/easy_install +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install' -__requires__ = 'setuptools==0.6c11' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install')() -) diff --git a/test/bin/easy_install-2.6 b/test/bin/easy_install-2.6 deleted file mode 100755 index bad01d2d7..000000000 --- a/test/bin/easy_install-2.6 +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==0.6c11','console_scripts','easy_install-2.6' -__requires__ = 'setuptools==0.6c11' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install-2.6')() -) diff --git a/test/bin/pip b/test/bin/pip deleted file mode 100755 index a3d2ed311..000000000 --- a/test/bin/pip +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip' -__requires__ = 'pip==0.8.1' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('pip==0.8.1', 'console_scripts', 'pip')() -) diff --git a/test/bin/pip-2.6 b/test/bin/pip-2.6 deleted file mode 100755 index 4ad06add1..000000000 --- a/test/bin/pip-2.6 +++ /dev/null @@ -1,9 +0,0 @@ -#!/Users/anne.gentle/src/nova/docslice/test/bin/python -# EASY-INSTALL-ENTRY-SCRIPT: 'pip==0.8.1','console_scripts','pip-2.6' -__requires__ = 'pip==0.8.1' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('pip==0.8.1', 'console_scripts', 'pip-2.6')() -) diff --git a/test/bin/python b/test/bin/python deleted file mode 100755 index 271b6ca30..000000000 Binary files a/test/bin/python and /dev/null differ diff --git a/test/bin/python2.6 b/test/bin/python2.6 deleted file mode 120000 index d8654aa0e..000000000 --- a/test/bin/python2.6 +++ /dev/null @@ -1 +0,0 @@ -python \ No newline at end of file diff --git a/test/include/python2.6 b/test/include/python2.6 deleted file mode 120000 index 788ac55ba..000000000 --- a/test/include/python2.6 +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/include/python2.6 \ No newline at end of file diff --git a/test/lib/python2.6/UserDict.py b/test/lib/python2.6/UserDict.py deleted file mode 120000 index 93e0864b1..000000000 --- a/test/lib/python2.6/UserDict.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/UserDict.py \ No newline at end of file diff --git a/test/lib/python2.6/_abcoll.py b/test/lib/python2.6/_abcoll.py deleted file mode 120000 index ff1769246..000000000 --- a/test/lib/python2.6/_abcoll.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/_abcoll.py \ No newline at end of file diff --git a/test/lib/python2.6/abc.py b/test/lib/python2.6/abc.py deleted file mode 120000 index 79fa108b2..000000000 --- a/test/lib/python2.6/abc.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/abc.py \ No newline at end of file diff --git a/test/lib/python2.6/codecs.py b/test/lib/python2.6/codecs.py deleted file mode 120000 index 9e4bfb7fa..000000000 --- a/test/lib/python2.6/codecs.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/codecs.py \ No newline at end of file diff --git a/test/lib/python2.6/config b/test/lib/python2.6/config deleted file mode 120000 index eef498c62..000000000 --- a/test/lib/python2.6/config +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/config \ No newline at end of file diff --git a/test/lib/python2.6/copy_reg.py b/test/lib/python2.6/copy_reg.py deleted file mode 120000 index 7285fda0e..000000000 --- a/test/lib/python2.6/copy_reg.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/copy_reg.py \ No newline at end of file diff --git a/test/lib/python2.6/distutils/__init__.py b/test/lib/python2.6/distutils/__init__.py deleted file mode 100644 index 7ebb41c04..000000000 --- a/test/lib/python2.6/distutils/__init__.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -import warnings -import opcode # opcode is not a virtualenv module, so we can use it to find the stdlib - # Important! To work on pypy, this must be a module that resides in the - # lib-python/modified-x.y.z directory - -dirname = os.path.dirname - -distutils_path = os.path.join(os.path.dirname(opcode.__file__), 'distutils') -if os.path.normpath(distutils_path) == os.path.dirname(os.path.normpath(__file__)): - warnings.warn( - "The virtualenv distutils package at %s appears to be in the same location as the system distutils?") -else: - __path__.insert(0, distutils_path) - exec open(os.path.join(distutils_path, '__init__.py')).read() - -import dist -import sysconfig - - -## patch build_ext (distutils doesn't know how to get the libs directory -## path on windows - it hardcodes the paths around the patched sys.prefix) - -if sys.platform == 'win32': - from distutils.command.build_ext import build_ext as old_build_ext - class build_ext(old_build_ext): - def finalize_options (self): - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, basestring): - self.library_dirs = self.library_dirs.split(os.pathsep) - - self.library_dirs.insert(0, os.path.join(sys.real_prefix, "Libs")) - old_build_ext.finalize_options(self) - - from distutils.command import build_ext as build_ext_module - build_ext_module.build_ext = build_ext - -## distutils.dist patches: - -old_find_config_files = dist.Distribution.find_config_files -def find_config_files(self): - found = old_find_config_files(self) - system_distutils = os.path.join(distutils_path, 'distutils.cfg') - #if os.path.exists(system_distutils): - # found.insert(0, system_distutils) - # What to call the per-user config file - if os.name == 'posix': - user_filename = ".pydistutils.cfg" - else: - user_filename = "pydistutils.cfg" - user_filename = os.path.join(sys.prefix, user_filename) - if os.path.isfile(user_filename): - for item in list(found): - if item.endswith('pydistutils.cfg'): - found.remove(item) - found.append(user_filename) - return found -dist.Distribution.find_config_files = find_config_files - -## distutils.sysconfig patches: - -old_get_python_inc = sysconfig.get_python_inc -def sysconfig_get_python_inc(plat_specific=0, prefix=None): - if prefix is None: - prefix = sys.real_prefix - return old_get_python_inc(plat_specific, prefix) -sysconfig_get_python_inc.__doc__ = old_get_python_inc.__doc__ -sysconfig.get_python_inc = sysconfig_get_python_inc - -old_get_python_lib = sysconfig.get_python_lib -def sysconfig_get_python_lib(plat_specific=0, standard_lib=0, prefix=None): - if standard_lib and prefix is None: - prefix = sys.real_prefix - return old_get_python_lib(plat_specific, standard_lib, prefix) -sysconfig_get_python_lib.__doc__ = old_get_python_lib.__doc__ -sysconfig.get_python_lib = sysconfig_get_python_lib - -old_get_config_vars = sysconfig.get_config_vars -def sysconfig_get_config_vars(*args): - real_vars = old_get_config_vars(*args) - if sys.platform == 'win32': - lib_dir = os.path.join(sys.real_prefix, "libs") - if isinstance(real_vars, dict) and 'LIBDIR' not in real_vars: - real_vars['LIBDIR'] = lib_dir # asked for all - elif isinstance(real_vars, list) and 'LIBDIR' in args: - real_vars = real_vars + [lib_dir] # asked for list - return real_vars -sysconfig_get_config_vars.__doc__ = old_get_config_vars.__doc__ -sysconfig.get_config_vars = sysconfig_get_config_vars diff --git a/test/lib/python2.6/distutils/distutils.cfg b/test/lib/python2.6/distutils/distutils.cfg deleted file mode 100644 index 1af230ec9..000000000 --- a/test/lib/python2.6/distutils/distutils.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# This is a config file local to this virtualenv installation -# You may include options that will be used by all distutils commands, -# and by easy_install. For instance: -# -# [easy_install] -# find_links = http://mylocalsite diff --git a/test/lib/python2.6/encodings b/test/lib/python2.6/encodings deleted file mode 120000 index 89f28f824..000000000 --- a/test/lib/python2.6/encodings +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/encodings \ No newline at end of file diff --git a/test/lib/python2.6/fnmatch.py b/test/lib/python2.6/fnmatch.py deleted file mode 120000 index cce0594f2..000000000 --- a/test/lib/python2.6/fnmatch.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/fnmatch.py \ No newline at end of file diff --git a/test/lib/python2.6/genericpath.py b/test/lib/python2.6/genericpath.py deleted file mode 120000 index b14e1bc26..000000000 --- a/test/lib/python2.6/genericpath.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/genericpath.py \ No newline at end of file diff --git a/test/lib/python2.6/lib-dynload b/test/lib/python2.6/lib-dynload deleted file mode 120000 index 4644b7072..000000000 --- a/test/lib/python2.6/lib-dynload +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/lib-dynload \ No newline at end of file diff --git a/test/lib/python2.6/linecache.py b/test/lib/python2.6/linecache.py deleted file mode 120000 index 783624da8..000000000 --- a/test/lib/python2.6/linecache.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/linecache.py \ No newline at end of file diff --git a/test/lib/python2.6/locale.py b/test/lib/python2.6/locale.py deleted file mode 120000 index 4e674c7b6..000000000 --- a/test/lib/python2.6/locale.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/locale.py \ No newline at end of file diff --git a/test/lib/python2.6/ntpath.py b/test/lib/python2.6/ntpath.py deleted file mode 120000 index 9b6b40f48..000000000 --- a/test/lib/python2.6/ntpath.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/ntpath.py \ No newline at end of file diff --git a/test/lib/python2.6/orig-prefix.txt b/test/lib/python2.6/orig-prefix.txt deleted file mode 100644 index 535eb0f0e..000000000 --- a/test/lib/python2.6/orig-prefix.txt +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6 \ No newline at end of file diff --git a/test/lib/python2.6/os.py b/test/lib/python2.6/os.py deleted file mode 120000 index 92e6e9a7c..000000000 --- a/test/lib/python2.6/os.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/os.py \ No newline at end of file diff --git a/test/lib/python2.6/posixpath.py b/test/lib/python2.6/posixpath.py deleted file mode 120000 index c095d16a1..000000000 --- a/test/lib/python2.6/posixpath.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/posixpath.py \ No newline at end of file diff --git a/test/lib/python2.6/re.py b/test/lib/python2.6/re.py deleted file mode 120000 index b4710c5f7..000000000 --- a/test/lib/python2.6/re.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/re.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/easy-install.pth b/test/lib/python2.6/site-packages/easy-install.pth deleted file mode 100644 index 7a6ae2b6d..000000000 --- a/test/lib/python2.6/site-packages/easy-install.pth +++ /dev/null @@ -1,4 +0,0 @@ -import sys; sys.__plen = len(sys.path) -./setuptools-0.6c11-py2.6.egg -./pip-0.8.1-py2.6.egg -import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO deleted file mode 100644 index 29c30cb8c..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/PKG-INFO +++ /dev/null @@ -1,348 +0,0 @@ -Metadata-Version: 1.0 -Name: pip -Version: 0.8.1 -Summary: pip installs packages. Python packages. An easy_install replacement -Home-page: http://pip.openplans.org -Author: Ian Bicking -Author-email: python-virtualenv@groups.google.com -License: MIT -Description: The main website for pip is `pip.openplans.org - <http://pip.openplans.org>`_. You can also install - the `in-development version <http://bitbucket.org/ianb/pip/get/tip.gz#egg=pip-dev>`_ - of pip with ``easy_install pip==dev``. - - - Introduction - ------------ - - pip installs packages. Python packages. - - If you use `virtualenv <http://virtualenv.openplans.org>`__ -- a tool - for installing libraries in a local and isolated manner -- you'll - automatically get a copy of pip. Free bonus! - - Once you have pip, you can use it like this:: - - $ pip install SomePackage - - SomePackage is some package you'll find on `PyPI - <http://pypi.python.org/pypi/>`_. This installs the package and all - its dependencies. - - pip does other stuff too, with packages, but install is the biggest - one. You can ``pip uninstall`` too. - - You can also install from a URL (that points to a tar or zip file), - install from some version control system (use URLs like - ``hg+http://domain/repo`` -- or prefix ``git+``, ``svn+`` etc). pip - knows a bunch of stuff about revisions and stuff, so if you need to do - things like install a very specific revision from a repository pip can - do that too. - - If you've ever used ``python setup.py develop``, you can do something - like that with ``pip install -e ./`` -- this works with packages that - use ``distutils`` too (usually this only works with Setuptools - projects). - - You can use ``pip install --upgrade SomePackage`` to upgrade to a - newer version, or ``pip install SomePackage==1.0.4`` to install a very - specific version. - - Pip Compared To easy_install - ---------------------------- - - pip is a replacement for `easy_install - <http://peak.telecommunity.com/DevCenter/EasyInstall>`_. It uses mostly the - same techniques for finding packages, so packages that were made - easy_installable should be pip-installable as well. - - pip is meant to improve on easy_install. Some of the improvements: - - * All packages are downloaded before installation. Partially-completed - installation doesn't occur as a result. - - * Care is taken to present useful output on the console. - - * The reasons for actions are kept track of. For instance, if a package is - being installed, pip keeps track of why that package was required. - - * Error messages should be useful. - - * The code is relatively concise and cohesive, making it easier to use - programmatically. - - * Packages don't have to be installed as egg archives, they can be installed - flat (while keeping the egg metadata). - - * Native support for other version control systems (Git, Mercurial and Bazaar) - - * Uninstallation of packages. - - * Simple to define fixed sets of requirements and reliably reproduce a - set of packages. - - pip doesn't do everything that easy_install does. Specifically: - - * It cannot install from eggs. It only installs from source. (In the - future it would be good if it could install binaries from Windows ``.exe`` - or ``.msi`` -- binary install on other platforms is not a priority.) - - * It doesn't understand Setuptools extras (like ``package[test]``). This should - be added eventually. - - * It is incompatible with some packages that extensively customize distutils - or setuptools in their ``setup.py`` files. - - pip is complementary with `virtualenv - <http://pypi.python.org/pypi/virtualenv>`__, and it is encouraged that you use - virtualenv to isolate your installation. - - Community - --------- - - The homepage for pip is temporarily located `on PyPI - <http://pypi.python.org/pypi/pip>`_ -- a more proper homepage will - follow. Bugs can go on the `pip issue tracker - <http://bitbucket.org/ianb/pip/issues/>`_. Discussion should happen on the - `virtualenv email group - <http://groups.google.com/group/python-virtualenv?hl=en>`_. - - Uninstall - --------- - - pip is able to uninstall most installed packages with ``pip uninstall - package-name``. - - Known exceptions include pure-distutils packages installed with - ``python setup.py install`` (such packages leave behind no metadata allowing - determination of what files were installed), and script wrappers installed - by develop-installs (``python setup.py develop``). - - pip also performs an automatic uninstall of an old version of a package - before upgrading to a newer version, so outdated files (and egg-info data) - from conflicting versions aren't left hanging around to cause trouble. The - old version of the package is automatically restored if the new version - fails to download or install. - - .. _`requirements file`: - - Requirements Files - ------------------ - - When installing software, and Python packages in particular, it's common that - you get a lot of libraries installed. You just did ``easy_install MyPackage`` - and you get a dozen packages. Each of these packages has its own version. - - Maybe you ran that installation and it works. Great! Will it keep working? - Did you have to provide special options to get it to find everything? Did you - have to install a bunch of other optional pieces? Most of all, will you be able - to do it again? Requirements files give you a way to create an *environment*: - a *set* of packages that work together. - - If you've ever tried to setup an application on a new system, or with slightly - updated pieces, and had it fail, pip requirements are for you. If you - haven't had this problem then you will eventually, so pip requirements are - for you too -- requirements make explicit, repeatable installation of packages. - - So what are requirements files? They are very simple: lists of packages to - install. Instead of running something like ``pip MyApp`` and getting - whatever libraries come along, you can create a requirements file something like:: - - MyApp - Framework==0.9.4 - Library>=0.2 - - Then, regardless of what MyApp lists in ``setup.py``, you'll get a - specific version of Framework (0.9.4) and at least the 0.2 version of - Library. (You might think you could list these specific versions in - MyApp's ``setup.py`` -- but if you do that you'll have to edit MyApp - if you want to try a new version of Framework, or release a new - version of MyApp if you determine that Library 0.3 doesn't work with - your application.) You can also add optional libraries and support - tools that MyApp doesn't strictly require, giving people a set of - recommended libraries. - - You can also include "editable" packages -- packages that are checked out from - Subversion, Git, Mercurial and Bazaar. These are just like using the ``-e`` - option to pip. They look like:: - - -e svn+http://myrepo/svn/MyApp#egg=MyApp - - You have to start the URL with ``svn+`` (``git+``, ``hg+`` or ``bzr+``), and - you have to include ``#egg=Package`` so pip knows what to expect at that URL. - You can also include ``@rev`` in the URL, e.g., ``@275`` to check out - revision 275. - - Requirement files are mostly *flat*. Maybe ``MyApp`` requires - ``Framework``, and ``Framework`` requires ``Library``. I encourage - you to still list all these in a single requirement file; it is the - nature of Python programs that there are implicit bindings *directly* - between MyApp and Library. For instance, Framework might expose one - of Library's objects, and so if Library is updated it might directly - break MyApp. If that happens you can update the requirements file to - force an earlier version of Library, and you can do that without - having to re-release MyApp at all. - - Read the `requirements file format <http://pip.openplans.org/requirement-format.html>`_ to - learn about other features. - - Freezing Requirements - --------------------- - - So you have a working set of packages, and you want to be able to install them - elsewhere. `Requirements files`_ let you install exact versions, but it won't - tell you what all the exact versions are. - - To create a new requirements file from a known working environment, use:: - - $ pip freeze > stable-req.txt - - This will write a listing of *all* installed libraries to ``stable-req.txt`` - with exact versions for every library. You may want to edit the file down after - generating (e.g., to eliminate unnecessary libraries), but it'll give you a - stable starting point for constructing your requirements file. - - You can also give it an existing requirements file, and it will use that as a - sort of template for the new file. So if you do:: - - $ pip freeze -r devel-req.txt > stable-req.txt - - it will keep the packages listed in ``devel-req.txt`` in order and preserve - comments. - - Bundles - ------- - - Another way to distribute a set of libraries is a bundle format (specific to - pip). This format is not stable at this time (there simply hasn't been - any feedback, nor a great deal of thought). A bundle file contains all the - source for your package, and you can have pip install them all together. - Once you have the bundle file further network access won't be necessary. To - build a bundle file, do:: - - $ pip bundle MyApp.pybundle MyApp - - (Using a `requirements file`_ would be wise.) Then someone else can get the - file ``MyApp.pybundle`` and run:: - - $ pip install MyApp.pybundle - - This is *not* a binary format. This only packages source. If you have binary - packages, then the person who installs the files will have to have a compiler, - any necessary headers installed, etc. Binary packages are hard, this is - relatively easy. - - Using pip with virtualenv - ------------------------- - - pip is most nutritious when used with `virtualenv - <http://pypi.python.org/pypi/virtualenv>`__. One of the reasons pip - doesn't install "multi-version" eggs is that virtualenv removes much of the need - for it. Because pip is installed by virtualenv, just use - ``path/to/my/environment/bin/pip`` to install things into that - specific environment. - - To tell pip to only run if there is a virtualenv currently activated, - and to bail if not, use:: - - export PIP_REQUIRE_VIRTUALENV=true - - To tell pip to automatically use the currently active virtualenv:: - - export PIP_RESPECT_VIRTUALENV=true - - Providing an environment with ``-E`` will be ignored. - - Using pip with virtualenvwrapper - --------------------------------- - - If you are using `virtualenvwrapper - <http://www.doughellmann.com/projects/virtualenvwrapper/>`_, you might - want pip to automatically create its virtualenvs in your - ``$WORKON_HOME``. - - You can tell pip to do so by defining ``PIP_VIRTUALENV_BASE`` in your - environment and setting it to the same value as that of - ``$WORKON_HOME``. - - Do so by adding the line:: - - export PIP_VIRTUALENV_BASE=$WORKON_HOME - - in your .bashrc under the line starting with ``export WORKON_HOME``. - - Using pip with buildout - ----------------------- - - If you are using `zc.buildout - <http://pypi.python.org/pypi/zc.buildout>`_ you should look at - `gp.recipe.pip <http://pypi.python.org/pypi/gp.recipe.pip>`_ as an - option to use pip and virtualenv in your buildouts. - - Command line completion - ----------------------- - - pip comes with support for command line completion in bash and zsh and - allows you tab complete commands and options. To enable it you simply - need copy the required shell script to the your shell startup file - (e.g. ``.profile`` or ``.zprofile``) by running the special ``completion`` - command, e.g. for bash:: - - $ pip completion --bash >> ~/.profile - - And for zsh:: - - $ pip completion --zsh >> ~/.zprofile - - Alternatively, you can use the result of the ``completion`` command - directly with the eval function of you shell, e.g. by adding:: - - eval "`pip completion --bash`" - - to your startup file. - - Searching for packages - ---------------------- - - pip can search the `Python Package Index <http://pypi.python.org/pypi>`_ (PyPI) - for packages using the ``pip search`` command. To search, run:: - - $ pip search "query" - - The query will be used to search the names and summaries of all packages - indexed. - - pip searches http://pypi.python.org/pypi by default but alternative indexes - can be searched by using the ``--index`` flag. - - Mirror support - -------------- - - The `PyPI mirroring infrastructure <http://pypi.python.org/mirrors>`_ as - described in `PEP 381 <http://www.python.org/dev/peps/pep-0381/>`_ can be - used by passing the ``--use-mirrors`` option to the install command. - Alternatively, you can use the other ways to configure pip, e.g.:: - - $ export PIP_USE_MIRRORS=true - - If enabled, pip will automatically query the DNS entry of the mirror index URL - to find the list of mirrors to use. In case you want to override this list, - please use the ``--mirrors`` option of the install command, or add to your pip - configuration file:: - - [install] - use-mirrors = true - mirrors = - http://d.pypi.python.org - http://b.pypi.python.org - -Keywords: easy_install distutils setuptools egg virtualenv -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Topic :: Software Development :: Build Tools -Classifier: Programming Language :: Python :: 2.4 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt deleted file mode 100644 index 3a068547e..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/SOURCES.txt +++ /dev/null @@ -1,57 +0,0 @@ -MANIFEST.in -setup.cfg -setup.py -docs/branches.txt -docs/ci-server-step-by-step.txt -docs/configuration.txt -docs/how-to-contribute.txt -docs/index.txt -docs/license.txt -docs/news.txt -docs/requirement-format.txt -docs/running-tests.txt -docs/_build/branches.html -docs/_build/ci-server-step-by-step.html -docs/_build/configuration.html -docs/_build/how-to-contribute.html -docs/_build/index.html -docs/_build/license.html -docs/_build/news.html -docs/_build/requirement-format.html -docs/_build/running-tests.html -docs/_build/search.html -pip/__init__.py -pip/_pkgutil.py -pip/backwardcompat.py -pip/basecommand.py -pip/baseparser.py -pip/download.py -pip/exceptions.py -pip/index.py -pip/locations.py -pip/log.py -pip/req.py -pip/runner.py -pip/util.py -pip/venv.py -pip.egg-info/PKG-INFO -pip.egg-info/SOURCES.txt -pip.egg-info/dependency_links.txt -pip.egg-info/entry_points.txt -pip.egg-info/not-zip-safe -pip.egg-info/top_level.txt -pip/commands/__init__.py -pip/commands/bundle.py -pip/commands/completion.py -pip/commands/freeze.py -pip/commands/help.py -pip/commands/install.py -pip/commands/search.py -pip/commands/uninstall.py -pip/commands/unzip.py -pip/commands/zip.py -pip/vcs/__init__.py -pip/vcs/bazaar.py -pip/vcs/git.py -pip/vcs/mercurial.py -pip/vcs/subversion.py \ No newline at end of file diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt deleted file mode 100644 index 8b1378917..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt deleted file mode 100644 index 2b0afba73..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/entry_points.txt +++ /dev/null @@ -1,4 +0,0 @@ -[console_scripts] -pip = pip:main -pip-2.6 = pip:main - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe deleted file mode 100644 index 8b1378917..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt deleted file mode 100644 index a1b589e38..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/EGG-INFO/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py deleted file mode 100644 index c5de5c9a8..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/__init__.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python -import os -import optparse - -import subprocess -import sys -import re -import difflib - -from pip.basecommand import command_dict, load_command, load_all_commands, command_names -from pip.baseparser import parser -from pip.exceptions import InstallationError -from pip.log import logger -from pip.util import get_installed_distributions -from pip.backwardcompat import walk_packages - - -def autocomplete(): - """Command and option completion for the main option parser (and options) - and its subcommands (and options). - - Enable by sourcing one of the completion shell scripts (bash or zsh). - """ - # Don't complete if user hasn't sourced bash_completion file. - if 'PIP_AUTO_COMPLETE' not in os.environ: - return - cwords = os.environ['COMP_WORDS'].split()[1:] - cword = int(os.environ['COMP_CWORD']) - try: - current = cwords[cword-1] - except IndexError: - current = '' - load_all_commands() - subcommands = [cmd for cmd, cls in command_dict.items() if not cls.hidden] - options = [] - # subcommand - try: - subcommand_name = [w for w in cwords if w in subcommands][0] - except IndexError: - subcommand_name = None - # subcommand options - if subcommand_name: - # special case: 'help' subcommand has no options - if subcommand_name == 'help': - sys.exit(1) - # special case: list locally installed dists for uninstall command - if subcommand_name == 'uninstall' and not current.startswith('-'): - installed = [] - lc = current.lower() - for dist in get_installed_distributions(local_only=True): - if dist.key.startswith(lc) and dist.key not in cwords[1:]: - installed.append(dist.key) - # if there are no dists installed, fall back to option completion - if installed: - for dist in installed: - print dist - sys.exit(1) - subcommand = command_dict.get(subcommand_name) - options += [(opt.get_opt_string(), opt.nargs) - for opt in subcommand.parser.option_list - if opt.help != optparse.SUPPRESS_HELP] - # filter out previously specified options from available options - prev_opts = [x.split('=')[0] for x in cwords[1:cword-1]] - options = filter(lambda (x, v): x not in prev_opts, options) - # filter options by current input - options = [(k, v) for k, v in options if k.startswith(current)] - for option in options: - opt_label = option[0] - # append '=' to options which require args - if option[1]: - opt_label += '=' - print opt_label - else: - # show options of main parser only when necessary - if current.startswith('-') or current.startswith('--'): - subcommands += [opt.get_opt_string() - for opt in parser.option_list - if opt.help != optparse.SUPPRESS_HELP] - print ' '.join(filter(lambda x: x.startswith(current), subcommands)) - sys.exit(1) - - -def version_control(): - # Import all the version control support modules: - from pip import vcs - for importer, modname, ispkg in \ - walk_packages(path=vcs.__path__, prefix=vcs.__name__+'.'): - __import__(modname) - - -def main(initial_args=None): - if initial_args is None: - initial_args = sys.argv[1:] - autocomplete() - version_control() - options, args = parser.parse_args(initial_args) - if options.help and not args: - args = ['help'] - if not args: - parser.error('You must give a command (use "pip help" to see a list of commands)') - command = args[0].lower() - load_command(command) - if command not in command_dict: - close_commands = difflib.get_close_matches(command, command_names()) - if close_commands: - guess = close_commands[0] - if args[1:]: - guess = "%s %s" % (guess, " ".join(args[1:])) - else: - guess = 'install %s' % command - error_dict = {'arg': command, 'guess': guess, - 'script': os.path.basename(sys.argv[0])} - parser.error('No command by the name %(script)s %(arg)s\n ' - '(maybe you meant "%(script)s %(guess)s")' % error_dict) - command = command_dict[command] - return command.main(initial_args, args[1:], options) - - -############################################################ -## Writing freeze files - - -class FrozenRequirement(object): - - def __init__(self, name, req, editable, comments=()): - self.name = name - self.req = req - self.editable = editable - self.comments = comments - - _rev_re = re.compile(r'-r(\d+)$') - _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') - - @classmethod - def from_dist(cls, dist, dependency_links, find_tags=False): - location = os.path.normcase(os.path.abspath(dist.location)) - comments = [] - from pip.vcs import vcs, get_src_requirement - if vcs.get_backend_name(location): - editable = True - req = get_src_requirement(dist, location, find_tags) - if req is None: - logger.warn('Could not determine repository location of %s' % location) - comments.append('## !! Could not determine repository location') - req = dist.as_requirement() - editable = False - else: - editable = False - req = dist.as_requirement() - specs = req.specs - assert len(specs) == 1 and specs[0][0] == '==' - version = specs[0][1] - ver_match = cls._rev_re.search(version) - date_match = cls._date_re.search(version) - if ver_match or date_match: - svn_backend = vcs.get_backend('svn') - if svn_backend: - svn_location = svn_backend( - ).get_location(dist, dependency_links) - if not svn_location: - logger.warn( - 'Warning: cannot find svn location for %s' % req) - comments.append('## FIXME: could not find svn URL in dependency_links for this package:') - else: - comments.append('# Installing as editable to satisfy requirement %s:' % req) - if ver_match: - rev = ver_match.group(1) - else: - rev = '{%s}' % date_match.group(1) - editable = True - req = '%s@%s#egg=%s' % (svn_location, rev, cls.egg_name(dist)) - return cls(dist.project_name, req, editable, comments) - - @staticmethod - def egg_name(dist): - name = dist.egg_name() - match = re.search(r'-py\d\.\d$', name) - if match: - name = name[:match.start()] - return name - - def __str__(self): - req = self.req - if self.editable: - req = '-e %s' % req - return '\n'.join(list(self.comments)+[str(req)])+'\n' - -############################################################ -## Requirement files - - -def call_subprocess(cmd, show_stdout=True, - filter_stdout=None, cwd=None, - raise_on_returncode=True, - command_level=logger.DEBUG, command_desc=None, - extra_environ=None): - if command_desc is None: - cmd_parts = [] - for part in cmd: - if ' ' in part or '\n' in part or '"' in part or "'" in part: - part = '"%s"' % part.replace('"', '\\"') - cmd_parts.append(part) - command_desc = ' '.join(cmd_parts) - if show_stdout: - stdout = None - else: - stdout = subprocess.PIPE - logger.log(command_level, "Running command %s" % command_desc) - env = os.environ.copy() - if extra_environ: - env.update(extra_environ) - try: - proc = subprocess.Popen( - cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, - cwd=cwd, env=env) - except Exception, e: - logger.fatal( - "Error %s while executing command %s" % (e, command_desc)) - raise - all_output = [] - if stdout is not None: - stdout = proc.stdout - while 1: - line = stdout.readline() - if not line: - break - line = line.rstrip() - all_output.append(line + '\n') - if filter_stdout: - level = filter_stdout(line) - if isinstance(level, tuple): - level, line = level - logger.log(level, line) - if not logger.stdout_level_matches(level): - logger.show_progress() - else: - logger.info(line) - else: - returned_stdout, returned_stderr = proc.communicate() - all_output = [returned_stdout or ''] - proc.wait() - if proc.returncode: - if raise_on_returncode: - if all_output: - logger.notify('Complete output from command %s:' % command_desc) - logger.notify('\n'.join(all_output) + '\n----------------------------------------') - raise InstallationError( - "Command %s failed with error code %s" - % (command_desc, proc.returncode)) - else: - logger.warn( - "Command %s had error code %s" - % (command_desc, proc.returncode)) - if stdout is not None: - return ''.join(all_output) - - -if __name__ == '__main__': - exit = main() - if exit: - sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py deleted file mode 100644 index f8fb8aa61..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/_pkgutil.py +++ /dev/null @@ -1,589 +0,0 @@ -"""Utilities to support packages.""" - -# NOTE: This module must remain compatible with Python 2.3, as it is shared -# by setuptools for distribution with Python 2.3 and up. - -import os -import sys -import imp -import os.path -from types import ModuleType - -__all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', - 'walk_packages', 'iter_modules', - 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', -] - - -def read_code(stream): - # This helper is needed in order for the PEP 302 emulation to - # correctly handle compiled files - import marshal - - magic = stream.read(4) - if magic != imp.get_magic(): - return None - - stream.read(4) # Skip timestamp - return marshal.load(stream) - - -def simplegeneric(func): - """Make a trivial single-dispatch generic function""" - registry = {} - - def wrapper(*args, **kw): - ob = args[0] - try: - cls = ob.__class__ - except AttributeError: - cls = type(ob) - try: - mro = cls.__mro__ - except AttributeError: - try: - - class cls(cls, object): - pass - - mro = cls.__mro__[1:] - except TypeError: - mro = object, # must be an ExtensionClass or some such :( - for t in mro: - if t in registry: - return registry[t](*args, **kw) - else: - return func(*args, **kw) - try: - wrapper.__name__ = func.__name__ - except (TypeError, AttributeError): - pass # Python 2.3 doesn't allow functions to be renamed - - def register(typ, func=None): - if func is None: - return lambda f: register(typ, f) - registry[typ] = func - return func - - wrapper.__dict__ = func.__dict__ - wrapper.__doc__ = func.__doc__ - wrapper.register = register - return wrapper - - -def walk_packages(path=None, prefix='', onerror=None): - """Yields (module_loader, name, ispkg) for all modules recursively - on path, or, if path is None, all accessible modules. - - 'path' should be either None or a list of paths to look for - modules in. - - 'prefix' is a string to output on the front of every module name - on output. - - Note that this function must import all *packages* (NOT all - modules!) on the given path, in order to access the __path__ - attribute to find submodules. - - 'onerror' is a function which gets called with one argument (the - name of the package which was being imported) if any exception - occurs while trying to import a package. If no onerror function is - supplied, ImportErrors are caught and ignored, while all other - exceptions are propagated, terminating the search. - - Examples: - - # list all modules python can access - walk_packages() - - # list all submodules of ctypes - walk_packages(ctypes.__path__, ctypes.__name__+'.') - """ - - def seen(p, m={}): - if p in m: - return True - m[p] = True - - for importer, name, ispkg in iter_modules(path, prefix): - yield importer, name, ispkg - - if ispkg: - try: - __import__(name) - except ImportError: - if onerror is not None: - onerror(name) - except Exception: - if onerror is not None: - onerror(name) - else: - raise - else: - path = getattr(sys.modules[name], '__path__', None) or [] - - # don't traverse path items we've seen before - path = [p for p in path if not seen(p)] - - for item in walk_packages(path, name+'.', onerror): - yield item - - -def iter_modules(path=None, prefix=''): - """Yields (module_loader, name, ispkg) for all submodules on path, - or, if path is None, all top-level modules on sys.path. - - 'path' should be either None or a list of paths to look for - modules in. - - 'prefix' is a string to output on the front of every module name - on output. - """ - - if path is None: - importers = iter_importers() - else: - importers = map(get_importer, path) - - yielded = {} - for i in importers: - for name, ispkg in iter_importer_modules(i, prefix): - if name not in yielded: - yielded[name] = 1 - yield i, name, ispkg - - -#@simplegeneric -def iter_importer_modules(importer, prefix=''): - if not hasattr(importer, 'iter_modules'): - return [] - return importer.iter_modules(prefix) - -iter_importer_modules = simplegeneric(iter_importer_modules) - - -class ImpImporter: - """PEP 302 Importer that wraps Python's "classic" import algorithm - - ImpImporter(dirname) produces a PEP 302 importer that searches that - directory. ImpImporter(None) produces a PEP 302 importer that searches - the current sys.path, plus any modules that are frozen or built-in. - - Note that ImpImporter does not currently support being used by placement - on sys.meta_path. - """ - - def __init__(self, path=None): - self.path = path - - def find_module(self, fullname, path=None): - # Note: we ignore 'path' argument since it is only used via meta_path - subname = fullname.split(".")[-1] - if subname != fullname and self.path is None: - return None - if self.path is None: - path = None - else: - path = [os.path.realpath(self.path)] - try: - file, filename, etc = imp.find_module(subname, path) - except ImportError: - return None - return ImpLoader(fullname, file, filename, etc) - - def iter_modules(self, prefix=''): - if self.path is None or not os.path.isdir(self.path): - return - - yielded = {} - import inspect - - filenames = os.listdir(self.path) - filenames.sort() # handle packages before same-named modules - - for fn in filenames: - modname = inspect.getmodulename(fn) - if modname=='__init__' or modname in yielded: - continue - - path = os.path.join(self.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - for fn in os.listdir(path): - subname = inspect.getmodulename(fn) - if subname=='__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - - -class ImpLoader: - """PEP 302 Loader that wraps Python's "classic" import algorithm - """ - code = source = None - - def __init__(self, fullname, file, filename, etc): - self.file = file - self.filename = filename - self.fullname = fullname - self.etc = etc - - def load_module(self, fullname): - self._reopen() - try: - mod = imp.load_module(fullname, self.file, self.filename, self.etc) - finally: - if self.file: - self.file.close() - # Note: we don't set __loader__ because we want the module to look - # normal; i.e. this is just a wrapper for standard import machinery - return mod - - def get_data(self, pathname): - return open(pathname, "rb").read() - - def _reopen(self): - if self.file and self.file.closed: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self.file = open(self.filename, 'rU') - elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): - self.file = open(self.filename, 'rb') - - def _fix_name(self, fullname): - if fullname is None: - fullname = self.fullname - elif fullname != self.fullname: - raise ImportError("Loader for module %s cannot handle " - "module %s" % (self.fullname, fullname)) - return fullname - - def is_package(self, fullname): - fullname = self._fix_name(fullname) - return self.etc[2]==imp.PKG_DIRECTORY - - def get_code(self, fullname=None): - fullname = self._fix_name(fullname) - if self.code is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - source = self.get_source(fullname) - self.code = compile(source, self.filename, 'exec') - elif mod_type==imp.PY_COMPILED: - self._reopen() - try: - self.code = read_code(self.file) - finally: - self.file.close() - elif mod_type==imp.PKG_DIRECTORY: - self.code = self._get_delegate().get_code() - return self.code - - def get_source(self, fullname=None): - fullname = self._fix_name(fullname) - if self.source is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self._reopen() - try: - self.source = self.file.read() - finally: - self.file.close() - elif mod_type==imp.PY_COMPILED: - if os.path.exists(self.filename[:-1]): - f = open(self.filename[:-1], 'rU') - self.source = f.read() - f.close() - elif mod_type==imp.PKG_DIRECTORY: - self.source = self._get_delegate().get_source() - return self.source - - def _get_delegate(self): - return ImpImporter(self.filename).find_module('__init__') - - def get_filename(self, fullname=None): - fullname = self._fix_name(fullname) - mod_type = self.etc[2] - if self.etc[2]==imp.PKG_DIRECTORY: - return self._get_delegate().get_filename() - elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): - return self.filename - return None - - -try: - import zipimport - from zipimport import zipimporter - - def iter_zipimport_modules(importer, prefix=''): - dirlist = zipimport._zip_directory_cache[importer.archive].keys() - dirlist.sort() - _prefix = importer.prefix - plen = len(_prefix) - yielded = {} - import inspect - for fn in dirlist: - if not fn.startswith(_prefix): - continue - - fn = fn[plen:].split(os.sep) - - if len(fn)==2 and fn[1].startswith('__init__.py'): - if fn[0] not in yielded: - yielded[fn[0]] = 1 - yield fn[0], True - - if len(fn)!=1: - continue - - modname = inspect.getmodulename(fn[0]) - if modname=='__init__': - continue - - if modname and '.' not in modname and modname not in yielded: - yielded[modname] = 1 - yield prefix + modname, False - - iter_importer_modules.register(zipimporter, iter_zipimport_modules) - -except ImportError: - pass - - -def get_importer(path_item): - """Retrieve a PEP 302 importer for the given path item - - The returned importer is cached in sys.path_importer_cache - if it was newly created by a path hook. - - If there is no importer, a wrapper around the basic import - machinery is returned. This wrapper is never inserted into - the importer cache (None is inserted instead). - - The cache (or part of it) can be cleared manually if a - rescan of sys.path_hooks is necessary. - """ - try: - importer = sys.path_importer_cache[path_item] - except KeyError: - for path_hook in sys.path_hooks: - try: - importer = path_hook(path_item) - break - except ImportError: - pass - else: - importer = None - sys.path_importer_cache.setdefault(path_item, importer) - - if importer is None: - try: - importer = ImpImporter(path_item) - except ImportError: - importer = None - return importer - - -def iter_importers(fullname=""): - """Yield PEP 302 importers for the given module name - - If fullname contains a '.', the importers will be for the package - containing fullname, otherwise they will be importers for sys.meta_path, - sys.path, and Python's "classic" import machinery, in that order. If - the named module is in a package, that package is imported as a side - effect of invoking this function. - - Non PEP 302 mechanisms (e.g. the Windows registry) used by the - standard import machinery to find files in alternative locations - are partially supported, but are searched AFTER sys.path. Normally, - these locations are searched BEFORE sys.path, preventing sys.path - entries from shadowing them. - - For this to cause a visible difference in behaviour, there must - be a module or package name that is accessible via both sys.path - and one of the non PEP 302 file system mechanisms. In this case, - the emulation will find the former version, while the builtin - import mechanism will find the latter. - - Items of the following types can be affected by this discrepancy: - imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY - """ - if fullname.startswith('.'): - raise ImportError("Relative module names not supported") - if '.' in fullname: - # Get the containing package's __path__ - pkg = '.'.join(fullname.split('.')[:-1]) - if pkg not in sys.modules: - __import__(pkg) - path = getattr(sys.modules[pkg], '__path__', None) or [] - else: - for importer in sys.meta_path: - yield importer - path = sys.path - for item in path: - yield get_importer(item) - if '.' not in fullname: - yield ImpImporter() - - -def get_loader(module_or_name): - """Get a PEP 302 "loader" object for module_or_name - - If the module or package is accessible via the normal import - mechanism, a wrapper around the relevant part of that machinery - is returned. Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - - This function uses iter_importers(), and is thus subject to the same - limitations regarding platform-specific special import locations such - as the Windows registry. - """ - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a PEP 302 "loader" object for fullname - - If fullname contains dots, path must be the containing package's __path__. - Returns None if the module cannot be found or imported. This function uses - iter_importers(), and is thus subject to the same limitations regarding - platform-specific special import locations such as the Windows registry. - """ - for importer in iter_importers(fullname): - loader = importer.find_module(fullname) - if loader is not None: - return loader - - return None - - -def extend_path(path, name): - """Extend a package's path. - - Intended use is to place the following code in a package's __init__.py: - - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) - - This will add to the package's __path__ all subdirectories of - directories on sys.path named after the package. This is useful - if one wants to distribute different parts of a single logical - package as multiple directories. - - It also looks for *.pkg files beginning where * matches the name - argument. This feature is similar to *.pth files (see site.py), - except that it doesn't special-case lines starting with 'import'. - A *.pkg file is trusted at face value: apart from checking for - duplicates, all entries found in a *.pkg file are added to the - path, regardless of whether they are exist the filesystem. (This - is a feature.) - - If the input path is not a list (as is the case for frozen - packages) it is returned unchanged. The input path is not - modified; an extended copy is returned. Items are only appended - to the copy at the end. - - It is assumed that sys.path is a sequence. Items of sys.path that - are not (unicode or 8-bit) strings referring to existing - directories are ignored. Unicode items of sys.path that cause - errors when used as filenames may cause this function to raise an - exception (in line with os.path.isdir() behavior). - """ - - if not isinstance(path, list): - # This could happen e.g. when this is called from inside a - # frozen package. Return the path unchanged in that case. - return path - - pname = os.path.join(*name.split('.')) # Reconstitute as relative path - # Just in case os.extsep != '.' - sname = os.extsep.join(name.split('.')) - sname_pkg = sname + os.extsep + "pkg" - init_py = "__init__" + os.extsep + "py" - - path = path[:] # Start with a copy of the existing path - - for dir in sys.path: - if not isinstance(dir, basestring) or not os.path.isdir(dir): - continue - subdir = os.path.join(dir, pname) - # XXX This may still add duplicate entries to path on - # case-insensitive filesystems - initfile = os.path.join(subdir, init_py) - if subdir not in path and os.path.isfile(initfile): - path.append(subdir) - # XXX Is this the right thing for subpackages like zope.app? - # It looks for a file named "zope.app.pkg" - pkgfile = os.path.join(dir, sname_pkg) - if os.path.isfile(pkgfile): - try: - f = open(pkgfile) - except IOError, msg: - sys.stderr.write("Can't open %s: %s\n" % - (pkgfile, msg)) - else: - for line in f: - line = line.rstrip('\n') - if not line or line.startswith('#'): - continue - path.append(line) # Don't check for existence! - f.close() - - return path - - -def get_data(package, resource): - """Get a resource from a package. - - This is a wrapper round the PEP 302 loader get_data API. The package - argument should be the name of a package, in standard module format - (foo.bar). The resource argument should be in the form of a relative - filename, using '/' as the path separator. The parent directory name '..' - is not allowed, and nor is a rooted name (starting with a '/'). - - The function returns a binary string, which is the contents of the - specified resource. - - For packages located in the filesystem, which have already been imported, - this is the rough equivalent of - - d = os.path.dirname(sys.modules[package].__file__) - data = open(os.path.join(d, resource), 'rb').read() - - If the package cannot be located or loaded, or it uses a PEP 302 loader - which does not support get_data(), then None is returned. - """ - - loader = get_loader(package) - if loader is None or not hasattr(loader, 'get_data'): - return None - mod = sys.modules.get(package) or loader.load_module(package) - if mod is None or not hasattr(mod, '__file__'): - return None - - # Modify the resource name to be compatible with the loader.get_data - # signature - an os.path format "filename" starting with the dirname of - # the package's __file__ - parts = resource.split('/') - parts.insert(0, os.path.dirname(mod.__file__)) - resource_name = os.path.join(*parts) - return loader.get_data(resource_name) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py deleted file mode 100644 index e7c11f1d3..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/backwardcompat.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Stuff that isn't in some old versions of Python""" - -import sys -import os -import shutil - -__all__ = ['any', 'WindowsError', 'md5', 'copytree'] - -try: - WindowsError = WindowsError -except NameError: - WindowsError = None -try: - from hashlib import md5 -except ImportError: - import md5 as md5_module - md5 = md5_module.new - -try: - from pkgutil import walk_packages -except ImportError: - # let's fall back as long as we can - from _pkgutil import walk_packages - -try: - any = any -except NameError: - - def any(seq): - for item in seq: - if item: - return True - return False - - -def copytree(src, dst): - if sys.version_info < (2, 5): - before_last_dir = os.path.dirname(dst) - if not os.path.exists(before_last_dir): - os.makedirs(before_last_dir) - shutil.copytree(src, dst) - shutil.copymode(src, dst) - else: - shutil.copytree(src, dst) - - -def product(*args, **kwds): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = map(tuple, args) * kwds.get('repeat', 1) - result = [[]] - for pool in pools: - result = [x+[y] for x in result for y in pool] - for prod in result: - yield tuple(prod) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py deleted file mode 100644 index f450e8393..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/basecommand.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Base Command class, and related routines""" - -from cStringIO import StringIO -import getpass -import os -import socket -import sys -import traceback -import time -import urllib -import urllib2 - -from pip import commands -from pip.log import logger -from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpFormatter -from pip.download import urlopen -from pip.exceptions import BadCommand, InstallationError, UninstallationError -from pip.venv import restart_in_venv -from pip.backwardcompat import walk_packages - -__all__ = ['command_dict', 'Command', 'load_all_commands', - 'load_command', 'command_names'] - -command_dict = {} - -# for backwards compatibiliy -get_proxy = urlopen.get_proxy - - -class Command(object): - name = None - usage = None - hidden = False - - def __init__(self): - assert self.name - self.parser = ConfigOptionParser( - usage=self.usage, - prog='%s %s' % (sys.argv[0], self.name), - version=parser.version, - formatter=UpdatingDefaultsHelpFormatter(), - name=self.name) - for option in parser.option_list: - if not option.dest or option.dest == 'help': - # -h, --version, etc - continue - self.parser.add_option(option) - command_dict[self.name] = self - - def merge_options(self, initial_options, options): - # Make sure we have all global options carried over - for attr in ['log', 'venv', 'proxy', 'venv_base', 'require_venv', - 'respect_venv', 'log_explicit_levels', 'log_file', - 'timeout', 'default_vcs', 'skip_requirements_regex', - 'no_input']: - setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr)) - options.quiet += initial_options.quiet - options.verbose += initial_options.verbose - - def setup_logging(self): - pass - - def main(self, complete_args, args, initial_options): - options, args = self.parser.parse_args(args) - self.merge_options(initial_options, options) - - level = 1 # Notify - level += options.verbose - level -= options.quiet - level = logger.level_for_integer(4-level) - complete_log = [] - logger.consumers.extend( - [(level, sys.stdout), - (logger.DEBUG, complete_log.append)]) - if options.log_explicit_levels: - logger.explicit_levels = True - - self.setup_logging() - - if options.require_venv and not options.venv: - # If a venv is required check if it can really be found - if not os.environ.get('VIRTUAL_ENV'): - logger.fatal('Could not find an activated virtualenv (required).') - sys.exit(3) - # Automatically install in currently activated venv if required - options.respect_venv = True - - if args and args[-1] == '___VENV_RESTART___': - ## FIXME: We don't do anything this this value yet: - args = args[:-2] - options.venv = None - else: - # If given the option to respect the activated environment - # check if no venv is given as a command line parameter - if options.respect_venv and os.environ.get('VIRTUAL_ENV'): - if options.venv and os.path.exists(options.venv): - # Make sure command line venv and environmental are the same - if (os.path.realpath(os.path.expanduser(options.venv)) != - os.path.realpath(os.environ.get('VIRTUAL_ENV'))): - logger.fatal("Given virtualenv (%s) doesn't match " - "currently activated virtualenv (%s)." - % (options.venv, os.environ.get('VIRTUAL_ENV'))) - sys.exit(3) - else: - options.venv = os.environ.get('VIRTUAL_ENV') - logger.info('Using already activated environment %s' % options.venv) - if options.venv: - logger.info('Running in environment %s' % options.venv) - site_packages=False - if options.site_packages: - site_packages=True - restart_in_venv(options.venv, options.venv_base, site_packages, - complete_args) - # restart_in_venv should actually never return, but for clarity... - return - - ## FIXME: not sure if this sure come before or after venv restart - if options.log: - log_fp = open_logfile(options.log, 'a') - logger.consumers.append((logger.DEBUG, log_fp)) - else: - log_fp = None - - socket.setdefaulttimeout(options.timeout or None) - - urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) - - exit = 0 - try: - self.run(options, args) - except (InstallationError, UninstallationError), e: - logger.fatal(str(e)) - logger.info('Exception information:\n%s' % format_exc()) - exit = 1 - except BadCommand, e: - logger.fatal(str(e)) - logger.info('Exception information:\n%s' % format_exc()) - exit = 1 - except: - logger.fatal('Exception:\n%s' % format_exc()) - exit = 2 - - if log_fp is not None: - log_fp.close() - if exit: - log_fn = options.log_file - text = '\n'.join(complete_log) - logger.fatal('Storing complete log in %s' % log_fn) - log_fp = open_logfile(log_fn, 'w') - log_fp.write(text) - log_fp.close() - return exit - - - - -def format_exc(exc_info=None): - if exc_info is None: - exc_info = sys.exc_info() - out = StringIO() - traceback.print_exception(*exc_info, **dict(file=out)) - return out.getvalue() - - -def open_logfile(filename, mode='a'): - """Open the named log file in append mode. - - If the file already exists, a separator will also be printed to - the file to separate past activity from current activity. - """ - filename = os.path.expanduser(filename) - filename = os.path.abspath(filename) - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - exists = os.path.exists(filename) - - log_fp = open(filename, mode) - if exists: - print >> log_fp, '-'*60 - print >> log_fp, '%s run on %s' % (sys.argv[0], time.strftime('%c')) - return log_fp - - -def load_command(name): - full_name = 'pip.commands.%s' % name - if full_name in sys.modules: - return - try: - __import__(full_name) - except ImportError: - pass - - -def load_all_commands(): - for name in command_names(): - load_command(name) - - -def command_names(): - names = set((pkg[1] for pkg in walk_packages(path=commands.__path__))) - return list(names) - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py deleted file mode 100644 index a8bd6ce4f..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/baseparser.py +++ /dev/null @@ -1,231 +0,0 @@ -"""Base option parser setup""" - -import sys -import optparse -import pkg_resources -import ConfigParser -import os -from distutils.util import strtobool -from pip.locations import default_config_file, default_log_file - - -class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter): - """Custom help formatter for use in ConfigOptionParser that updates - the defaults before expanding them, allowing them to show up correctly - in the help listing""" - - def expand_default(self, option): - if self.parser is not None: - self.parser.update_defaults(self.parser.defaults) - return optparse.IndentedHelpFormatter.expand_default(self, option) - - -class ConfigOptionParser(optparse.OptionParser): - """Custom option parser which updates its defaults by by checking the - configuration files and environmental variables""" - - def __init__(self, *args, **kwargs): - self.config = ConfigParser.RawConfigParser() - self.name = kwargs.pop('name') - self.files = self.get_config_files() - self.config.read(self.files) - assert self.name - optparse.OptionParser.__init__(self, *args, **kwargs) - - def get_config_files(self): - config_file = os.environ.get('PIP_CONFIG_FILE', False) - if config_file and os.path.exists(config_file): - return [config_file] - return [default_config_file] - - def update_defaults(self, defaults): - """Updates the given defaults with values from the config files and - the environ. Does a little special handling for certain types of - options (lists).""" - # Then go and look for the other sources of configuration: - config = {} - # 1. config files - for section in ('global', self.name): - config.update(dict(self.get_config_section(section))) - # 2. environmental variables - config.update(dict(self.get_environ_vars())) - # Then set the options with those values - for key, val in config.iteritems(): - key = key.replace('_', '-') - if not key.startswith('--'): - key = '--%s' % key # only prefer long opts - option = self.get_option(key) - if option is not None: - # ignore empty values - if not val: - continue - # handle multiline configs - if option.action == 'append': - val = val.split() - else: - option.nargs = 1 - if option.action in ('store_true', 'store_false', 'count'): - val = strtobool(val) - try: - val = option.convert_value(key, val) - except optparse.OptionValueError, e: - print ("An error occured during configuration: %s" % e) - sys.exit(3) - defaults[option.dest] = val - return defaults - - def get_config_section(self, name): - """Get a section of a configuration""" - if self.config.has_section(name): - return self.config.items(name) - return [] - - def get_environ_vars(self, prefix='PIP_'): - """Returns a generator with all environmental vars with prefix PIP_""" - for key, val in os.environ.iteritems(): - if key.startswith(prefix): - yield (key.replace(prefix, '').lower(), val) - - def get_default_values(self): - """Overridding to make updating the defaults after instantiation of - the option parser possible, update_defaults() does the dirty work.""" - if not self.process_default_values: - # Old, pre-Optik 1.5 behaviour. - return optparse.Values(self.defaults) - - defaults = self.update_defaults(self.defaults.copy()) # ours - for option in self._get_all_options(): - default = defaults.get(option.dest) - if isinstance(default, basestring): - opt_str = option.get_opt_string() - defaults[option.dest] = option.check_value(opt_str, default) - return optparse.Values(defaults) - -try: - pip_dist = pkg_resources.get_distribution('pip') - version = '%s from %s (python %s)' % ( - pip_dist, pip_dist.location, sys.version[:3]) -except pkg_resources.DistributionNotFound: - # when running pip.py without installing - version=None - -parser = ConfigOptionParser( - usage='%prog COMMAND [OPTIONS]', - version=version, - add_help_option=False, - formatter=UpdatingDefaultsHelpFormatter(), - name='global') - -parser.add_option( - '-h', '--help', - dest='help', - action='store_true', - help='Show help') -parser.add_option( - '-E', '--environment', - dest='venv', - metavar='DIR', - help='virtualenv environment to run pip in (either give the ' - 'interpreter or the environment base directory)') -parser.add_option( - '-s', '--enable-site-packages', - dest='site_packages', - action='store_true', - help='Include site-packages in virtualenv if one is to be ' - 'created. Ignored if --environment is not used or ' - 'the virtualenv already exists.') -parser.add_option( - # Defines a default root directory for virtualenvs, relative - # virtualenvs names/paths are considered relative to it. - '--virtualenv-base', - dest='venv_base', - type='str', - default='', - help=optparse.SUPPRESS_HELP) -parser.add_option( - # Run only if inside a virtualenv, bail if not. - '--require-virtualenv', '--require-venv', - dest='require_venv', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) -parser.add_option( - # Use automatically an activated virtualenv instead of installing - # globally. -E will be ignored if used. - '--respect-virtualenv', '--respect-venv', - dest='respect_venv', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) - -parser.add_option( - '-v', '--verbose', - dest='verbose', - action='count', - default=0, - help='Give more output') -parser.add_option( - '-q', '--quiet', - dest='quiet', - action='count', - default=0, - help='Give less output') -parser.add_option( - '--log', - dest='log', - metavar='FILENAME', - help='Log file where a complete (maximum verbosity) record will be kept') -parser.add_option( - # Writes the log levels explicitely to the log' - '--log-explicit-levels', - dest='log_explicit_levels', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) -parser.add_option( - # The default log file - '--local-log', '--log-file', - dest='log_file', - metavar='FILENAME', - default=default_log_file, - help=optparse.SUPPRESS_HELP) -parser.add_option( - # Don't ask for input - '--no-input', - dest='no_input', - action='store_true', - default=False, - help=optparse.SUPPRESS_HELP) - -parser.add_option( - '--proxy', - dest='proxy', - type='str', - default='', - help="Specify a proxy in the form user:passwd@proxy.server:port. " - "Note that the user:password@ is optional and required only if you " - "are behind an authenticated proxy. If you provide " - "user@proxy.server:port then you will be prompted for a password.") -parser.add_option( - '--timeout', '--default-timeout', - metavar='SECONDS', - dest='timeout', - type='float', - default=15, - help='Set the socket timeout (default %default seconds)') -parser.add_option( - # The default version control system for editables, e.g. 'svn' - '--default-vcs', - dest='default_vcs', - type='str', - default='', - help=optparse.SUPPRESS_HELP) -parser.add_option( - # A regex to be used to skip requirements - '--skip-requirements-regex', - dest='skip_requirements_regex', - type='str', - default='', - help=optparse.SUPPRESS_HELP) - -parser.disable_interspersed_args() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py deleted file mode 100644 index 792d60054..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py deleted file mode 100644 index fb0f75704..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/bundle.py +++ /dev/null @@ -1,33 +0,0 @@ -from pip.locations import build_prefix, src_prefix -from pip.util import display_path, backup_dir -from pip.log import logger -from pip.exceptions import InstallationError -from pip.commands.install import InstallCommand - - -class BundleCommand(InstallCommand): - name = 'bundle' - usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...' - summary = 'Create pybundles (archives containing multiple packages)' - bundle = True - - def __init__(self): - super(BundleCommand, self).__init__() - - def run(self, options, args): - if not args: - raise InstallationError('You must give a bundle filename') - if not options.build_dir: - options.build_dir = backup_dir(build_prefix, '-bundle') - if not options.src_dir: - options.src_dir = backup_dir(src_prefix, '-bundle') - # We have to get everything when creating a bundle: - options.ignore_installed = True - logger.notify('Putting temporary build files in %s and source/develop files in %s' - % (display_path(options.build_dir), display_path(options.src_dir))) - self.bundle_filename = args.pop(0) - requirement_set = super(BundleCommand, self).run(options, args) - return requirement_set - - -BundleCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py deleted file mode 100644 index d003b9ae3..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/completion.py +++ /dev/null @@ -1,60 +0,0 @@ -import sys -from pip.basecommand import Command - -BASE_COMPLETION = """ -# pip %(shell)s completion start%(script)s# pip %(shell)s completion end -""" - -COMPLETION_SCRIPTS = { - 'bash': """ -_pip_completion() -{ - COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ - COMP_CWORD=$COMP_CWORD \\ - PIP_AUTO_COMPLETE=1 $1 ) ) -} -complete -o default -F _pip_completion pip -""", 'zsh': """ -function _pip_completion { - local words cword - read -Ac words - read -cn cword - reply=( $( COMP_WORDS="$words[*]" \\ - COMP_CWORD=$(( cword-1 )) \\ - PIP_AUTO_COMPLETE=1 $words[1] ) ) -} -compctl -K _pip_completion pip -"""} - - -class CompletionCommand(Command): - name = 'completion' - summary = 'A helper command to be used for command completion' - hidden = True - - def __init__(self): - super(CompletionCommand, self).__init__() - self.parser.add_option( - '--bash', '-b', - action='store_const', - const='bash', - dest='shell', - help='Emit completion code for bash') - self.parser.add_option( - '--zsh', '-z', - action='store_const', - const='zsh', - dest='shell', - help='Emit completion code for zsh') - - def run(self, options, args): - """Prints the completion code of the given shell""" - shells = COMPLETION_SCRIPTS.keys() - shell_options = ['--'+shell for shell in sorted(shells)] - if options.shell in shells: - script = COMPLETION_SCRIPTS.get(options.shell, '') - print BASE_COMPLETION % {'script': script, 'shell': options.shell} - else: - sys.stderr.write('ERROR: You must pass %s\n' % ' or '.join(shell_options)) - -CompletionCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py deleted file mode 100644 index 01b5df934..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/freeze.py +++ /dev/null @@ -1,109 +0,0 @@ -import re -import sys -import pkg_resources -import pip -from pip.req import InstallRequirement -from pip.log import logger -from pip.basecommand import Command -from pip.util import get_installed_distributions - - -class FreezeCommand(Command): - name = 'freeze' - usage = '%prog [OPTIONS]' - summary = 'Output all currently installed packages (exact versions) to stdout' - - def __init__(self): - super(FreezeCommand, self).__init__() - self.parser.add_option( - '-r', '--requirement', - dest='requirement', - action='store', - default=None, - metavar='FILENAME', - help='Use the given requirements file as a hint about how to generate the new frozen requirements') - self.parser.add_option( - '-f', '--find-links', - dest='find_links', - action='append', - default=[], - metavar='URL', - help='URL for finding packages, which will be added to the frozen requirements file') - self.parser.add_option( - '-l', '--local', - dest='local', - action='store_true', - default=False, - help='If in a virtualenv, do not report globally-installed packages') - - def setup_logging(self): - logger.move_stdout_to_stderr() - - def run(self, options, args): - requirement = options.requirement - find_links = options.find_links or [] - local_only = options.local - ## FIXME: Obviously this should be settable: - find_tags = False - skip_match = None - - skip_regex = options.skip_requirements_regex - if skip_regex: - skip_match = re.compile(skip_regex) - - dependency_links = [] - - f = sys.stdout - - for dist in pkg_resources.working_set: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) - for link in find_links: - if '#egg=' in link: - dependency_links.append(link) - for link in find_links: - f.write('-f %s\n' % link) - installations = {} - for dist in get_installed_distributions(local_only=local_only): - req = pip.FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags) - installations[req.name] = req - if requirement: - req_f = open(requirement) - for line in req_f: - if not line.strip() or line.strip().startswith('#'): - f.write(line) - continue - if skip_match and skip_match.search(line): - f.write(line) - continue - elif line.startswith('-e') or line.startswith('--editable'): - if line.startswith('-e'): - line = line[2:].strip() - else: - line = line[len('--editable'):].strip().lstrip('=') - line_req = InstallRequirement.from_editable(line, default_vcs=options.default_vcs) - elif (line.startswith('-r') or line.startswith('--requirement') - or line.startswith('-Z') or line.startswith('--always-unzip') - or line.startswith('-f') or line.startswith('-i') - or line.startswith('--extra-index-url')): - f.write(line) - continue - else: - line_req = InstallRequirement.from_line(line) - if not line_req.name: - logger.notify("Skipping line because it's not clear what it would install: %s" - % line.strip()) - logger.notify(" (add #egg=PackageName to the URL to avoid this warning)") - continue - if line_req.name not in installations: - logger.warn("Requirement file contains %s, but that package is not installed" - % line.strip()) - continue - f.write(str(installations[line_req.name])) - del installations[line_req.name] - f.write('## The following requirements were added by pip --freeze:\n') - for installation in sorted(installations.values(), key=lambda x: x.name): - f.write(str(installation)) - - -FreezeCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py deleted file mode 100644 index b0b366112..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/help.py +++ /dev/null @@ -1,32 +0,0 @@ -from pip.basecommand import Command, command_dict, load_all_commands -from pip.exceptions import InstallationError -from pip.baseparser import parser - - -class HelpCommand(Command): - name = 'help' - usage = '%prog' - summary = 'Show available commands' - - def run(self, options, args): - load_all_commands() - if args: - ## FIXME: handle errors better here - command = args[0] - if command not in command_dict: - raise InstallationError('No command with the name: %s' % command) - command = command_dict[command] - command.parser.print_help() - return - parser.print_help() - print - print 'Commands available:' - commands = list(set(command_dict.values())) - commands.sort(key=lambda x: x.name) - for command in commands: - if command.hidden: - continue - print ' %s: %s' % (command.name, command.summary) - - -HelpCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py deleted file mode 100644 index 861c332bf..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/install.py +++ /dev/null @@ -1,247 +0,0 @@ -import os, sys -from pip.req import InstallRequirement, RequirementSet -from pip.req import parse_requirements -from pip.log import logger -from pip.locations import build_prefix, src_prefix -from pip.basecommand import Command -from pip.index import PackageFinder -from pip.exceptions import InstallationError - - -class InstallCommand(Command): - name = 'install' - usage = '%prog [OPTIONS] PACKAGE_NAMES...' - summary = 'Install packages' - bundle = False - - def __init__(self): - super(InstallCommand, self).__init__() - self.parser.add_option( - '-e', '--editable', - dest='editables', - action='append', - default=[], - metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE', - help='Install a package directly from a checkout. Source will be checked ' - 'out into src/PACKAGE (lower-case) and installed in-place (using ' - 'setup.py develop). You can run this on an existing directory/checkout (like ' - 'pip install -e src/mycheckout). This option may be provided multiple times. ' - 'Possible values for VCS are: svn, git, hg and bzr.') - self.parser.add_option( - '-r', '--requirement', - dest='requirements', - action='append', - default=[], - metavar='FILENAME', - help='Install all the packages listed in the given requirements file. ' - 'This option can be used multiple times.') - self.parser.add_option( - '-f', '--find-links', - dest='find_links', - action='append', - default=[], - metavar='URL', - help='URL to look for packages at') - self.parser.add_option( - '-i', '--index-url', '--pypi-url', - dest='index_url', - metavar='URL', - default='http://pypi.python.org/simple/', - help='Base URL of Python Package Index (default %default)') - self.parser.add_option( - '--extra-index-url', - dest='extra_index_urls', - metavar='URL', - action='append', - default=[], - help='Extra URLs of package indexes to use in addition to --index-url') - self.parser.add_option( - '--no-index', - dest='no_index', - action='store_true', - default=False, - help='Ignore package index (only looking at --find-links URLs instead)') - self.parser.add_option( - '-M', '--use-mirrors', - dest='use_mirrors', - action='store_true', - default=False, - help='Use the PyPI mirrors as a fallback in case the main index is down.') - self.parser.add_option( - '--mirrors', - dest='mirrors', - metavar='URL', - action='append', - default=[], - help='Specific mirror URLs to query when --use-mirrors is used') - - self.parser.add_option( - '-b', '--build', '--build-dir', '--build-directory', - dest='build_dir', - metavar='DIR', - default=None, - help='Unpack packages into DIR (default %s) and build from there' % build_prefix) - self.parser.add_option( - '-d', '--download', '--download-dir', '--download-directory', - dest='download_dir', - metavar='DIR', - default=None, - help='Download packages into DIR instead of installing them') - self.parser.add_option( - '--download-cache', - dest='download_cache', - metavar='DIR', - default=None, - help='Cache downloaded packages in DIR') - self.parser.add_option( - '--src', '--source', '--source-dir', '--source-directory', - dest='src_dir', - metavar='DIR', - default=None, - help='Check out --editable packages into DIR (default %s)' % src_prefix) - - self.parser.add_option( - '-U', '--upgrade', - dest='upgrade', - action='store_true', - help='Upgrade all packages to the newest available version') - self.parser.add_option( - '-I', '--ignore-installed', - dest='ignore_installed', - action='store_true', - help='Ignore the installed packages (reinstalling instead)') - self.parser.add_option( - '--no-deps', '--no-dependencies', - dest='ignore_dependencies', - action='store_true', - default=False, - help='Ignore package dependencies') - self.parser.add_option( - '--no-install', - dest='no_install', - action='store_true', - help="Download and unpack all packages, but don't actually install them") - self.parser.add_option( - '--no-download', - dest='no_download', - action="store_true", - help="Don't download any packages, just install the ones already downloaded " - "(completes an install run with --no-install)") - - self.parser.add_option( - '--install-option', - dest='install_options', - action='append', - help="Extra arguments to be supplied to the setup.py install " - "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). " - "Use multiple --install-option options to pass multiple options to setup.py install. " - "If you are using an option with a directory path, be sure to use absolute path.") - - self.parser.add_option( - '--global-option', - dest='global_options', - action='append', - help="Extra global options to be supplied to the setup.py" - "call before the install command") - - self.parser.add_option( - '--user', - dest='use_user_site', - action='store_true', - help='Install to user-site') - - def _build_package_finder(self, options, index_urls): - """ - Create a package finder appropriate to this install command. - This method is meant to be overridden by subclasses, not - called directly. - """ - return PackageFinder(find_links=options.find_links, - index_urls=index_urls, - use_mirrors=options.use_mirrors, - mirrors=options.mirrors) - - def run(self, options, args): - if not options.build_dir: - options.build_dir = build_prefix - if not options.src_dir: - options.src_dir = src_prefix - if options.download_dir: - options.no_install = True - options.ignore_installed = True - options.build_dir = os.path.abspath(options.build_dir) - options.src_dir = os.path.abspath(options.src_dir) - install_options = options.install_options or [] - if options.use_user_site: - install_options.append('--user') - global_options = options.global_options or [] - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.notify('Ignoring indexes: %s' % ','.join(index_urls)) - index_urls = [] - - finder = self._build_package_finder(options, index_urls) - - requirement_set = RequirementSet( - build_dir=options.build_dir, - src_dir=options.src_dir, - download_dir=options.download_dir, - download_cache=options.download_cache, - upgrade=options.upgrade, - ignore_installed=options.ignore_installed, - ignore_dependencies=options.ignore_dependencies) - for name in args: - requirement_set.add_requirement( - InstallRequirement.from_line(name, None)) - for name in options.editables: - requirement_set.add_requirement( - InstallRequirement.from_editable(name, default_vcs=options.default_vcs)) - for filename in options.requirements: - for req in parse_requirements(filename, finder=finder, options=options): - requirement_set.add_requirement(req) - - if not requirement_set.has_requirements: - if options.find_links: - raise InstallationError('You must give at least one ' - 'requirement to %s (maybe you meant "pip install %s"?)' - % (self.name, " ".join(options.find_links))) - raise InstallationError('You must give at least one requirement ' - 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) - - if (options.use_user_site and - sys.version_info < (2, 6)): - raise InstallationError('--user is only supported in Python version 2.6 and newer') - - import setuptools - if (options.use_user_site and - requirement_set.has_editables and - not getattr(setuptools, '_distribute', False)): - - raise InstallationError('--user --editable not supported with setuptools, use distribute') - - if not options.no_download: - requirement_set.prepare_files(finder, force_root_egg_info=self.bundle, bundle=self.bundle) - else: - requirement_set.locate_files() - - if not options.no_install and not self.bundle: - requirement_set.install(install_options, global_options) - installed = ' '.join([req.name for req in - requirement_set.successfully_installed]) - if installed: - logger.notify('Successfully installed %s' % installed) - elif not self.bundle: - downloaded = ' '.join([req.name for req in - requirement_set.successfully_downloaded]) - if downloaded: - logger.notify('Successfully downloaded %s' % downloaded) - elif self.bundle: - requirement_set.create_bundle(self.bundle_filename) - logger.notify('Created bundle in %s' % self.bundle_filename) - # Clean up - if not options.no_install: - requirement_set.cleanup_files(bundle=self.bundle) - return requirement_set - - -InstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py deleted file mode 100644 index 73da58ac6..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/search.py +++ /dev/null @@ -1,116 +0,0 @@ -import sys -import xmlrpclib -import textwrap -import pkg_resources -import pip.download -from pip.basecommand import Command -from pip.util import get_terminal_size -from pip.log import logger -from distutils.version import StrictVersion, LooseVersion - - -class SearchCommand(Command): - name = 'search' - usage = '%prog QUERY' - summary = 'Search PyPI' - - def __init__(self): - super(SearchCommand, self).__init__() - self.parser.add_option( - '--index', - dest='index', - metavar='URL', - default='http://pypi.python.org/pypi', - help='Base URL of Python Package Index (default %default)') - - def run(self, options, args): - if not args: - logger.warn('ERROR: Missing required argument (search query).') - return - query = ' '.join(args) - index_url = options.index - - pypi_hits = self.search(query, index_url) - hits = transform_hits(pypi_hits) - - terminal_width = None - if sys.stdout.isatty(): - terminal_width = get_terminal_size()[0] - - print_results(hits, terminal_width=terminal_width) - - def search(self, query, index_url): - pypi = xmlrpclib.ServerProxy(index_url, pip.download.xmlrpclib_transport) - hits = pypi.search({'name': query, 'summary': query}, 'or') - return hits - - -def transform_hits(hits): - """ - The list from pypi is really a list of versions. We want a list of - packages with the list of versions stored inline. This converts the - list from pypi into one we can use. - """ - packages = {} - for hit in hits: - name = hit['name'] - summary = hit['summary'] - version = hit['version'] - score = hit['_pypi_ordering'] - - if name not in packages.keys(): - packages[name] = {'name': name, 'summary': summary, 'versions': [version], 'score': score} - else: - packages[name]['versions'].append(version) - - # if this is the highest version, replace summary and score - if version == highest_version(packages[name]['versions']): - packages[name]['summary'] = summary - packages[name]['score'] = score - - # each record has a unique name now, so we will convert the dict into a list sorted by score - package_list = sorted(packages.values(), lambda x, y: cmp(y['score'], x['score'])) - return package_list - - -def print_results(hits, name_column_width=25, terminal_width=None): - installed_packages = [p.project_name for p in pkg_resources.working_set] - for hit in hits: - name = hit['name'] - summary = hit['summary'] or '' - if terminal_width is not None: - # wrap and indent summary to fit terminal - summary = textwrap.wrap(summary, terminal_width - name_column_width - 5) - summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) - line = '%s - %s' % (name.ljust(name_column_width), summary) - try: - logger.notify(line) - if name in installed_packages: - dist = pkg_resources.get_distribution(name) - logger.indent += 2 - try: - latest = highest_version(hit['versions']) - if dist.version == latest: - logger.notify('INSTALLED: %s (latest)' % dist.version) - else: - logger.notify('INSTALLED: %s' % dist.version) - logger.notify('LATEST: %s' % latest) - finally: - logger.indent -= 2 - except UnicodeEncodeError: - pass - - -def compare_versions(version1, version2): - try: - return cmp(StrictVersion(version1), StrictVersion(version2)) - # in case of abnormal version number, fall back to LooseVersion - except ValueError: - return cmp(LooseVersion(version1), LooseVersion(version2)) - - -def highest_version(versions): - return reduce((lambda v1, v2: compare_versions(v1, v2) == 1 and v1 or v2), versions) - - -SearchCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py deleted file mode 100644 index 7effd844e..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/uninstall.py +++ /dev/null @@ -1,42 +0,0 @@ -from pip.req import InstallRequirement, RequirementSet, parse_requirements -from pip.basecommand import Command -from pip.exceptions import InstallationError - -class UninstallCommand(Command): - name = 'uninstall' - usage = '%prog [OPTIONS] PACKAGE_NAMES ...' - summary = 'Uninstall packages' - - def __init__(self): - super(UninstallCommand, self).__init__() - self.parser.add_option( - '-r', '--requirement', - dest='requirements', - action='append', - default=[], - metavar='FILENAME', - help='Uninstall all the packages listed in the given requirements file. ' - 'This option can be used multiple times.') - self.parser.add_option( - '-y', '--yes', - dest='yes', - action='store_true', - help="Don't ask for confirmation of uninstall deletions.") - - def run(self, options, args): - requirement_set = RequirementSet( - build_dir=None, - src_dir=None, - download_dir=None) - for name in args: - requirement_set.add_requirement( - InstallRequirement.from_line(name)) - for filename in options.requirements: - for req in parse_requirements(filename, options=options): - requirement_set.add_requirement(req) - if not requirement_set.has_requirements: - raise InstallationError('You must give at least one requirement ' - 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) - requirement_set.uninstall(auto_confirm=options.yes) - -UninstallCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py deleted file mode 100644 index f83e18205..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/unzip.py +++ /dev/null @@ -1,9 +0,0 @@ -from pip.commands.zip import ZipCommand - - -class UnzipCommand(ZipCommand): - name = 'unzip' - summary = 'Unzip individual packages' - - -UnzipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py deleted file mode 100644 index 346fc0519..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/commands/zip.py +++ /dev/null @@ -1,346 +0,0 @@ -import sys -import re -import fnmatch -import os -import shutil -import zipfile -from pip.util import display_path, backup_dir -from pip.log import logger -from pip.exceptions import InstallationError -from pip.basecommand import Command - - -class ZipCommand(Command): - name = 'zip' - usage = '%prog [OPTIONS] PACKAGE_NAMES...' - summary = 'Zip individual packages' - - def __init__(self): - super(ZipCommand, self).__init__() - if self.name == 'zip': - self.parser.add_option( - '--unzip', - action='store_true', - dest='unzip', - help='Unzip (rather than zip) a package') - else: - self.parser.add_option( - '--zip', - action='store_false', - dest='unzip', - default=True, - help='Zip (rather than unzip) a package') - self.parser.add_option( - '--no-pyc', - action='store_true', - dest='no_pyc', - help='Do not include .pyc files in zip files (useful on Google App Engine)') - self.parser.add_option( - '-l', '--list', - action='store_true', - dest='list', - help='List the packages available, and their zip status') - self.parser.add_option( - '--sort-files', - action='store_true', - dest='sort_files', - help='With --list, sort packages according to how many files they contain') - self.parser.add_option( - '--path', - action='append', - dest='paths', - help='Restrict operations to the given paths (may include wildcards)') - self.parser.add_option( - '-n', '--simulate', - action='store_true', - help='Do not actually perform the zip/unzip operation') - - def paths(self): - """All the entries of sys.path, possibly restricted by --path""" - if not self.select_paths: - return sys.path - result = [] - match_any = set() - for path in sys.path: - path = os.path.normcase(os.path.abspath(path)) - for match in self.select_paths: - match = os.path.normcase(os.path.abspath(match)) - if '*' in match: - if re.search(fnmatch.translate(match+'*'), path): - result.append(path) - match_any.add(match) - break - else: - if path.startswith(match): - result.append(path) - match_any.add(match) - break - else: - logger.debug("Skipping path %s because it doesn't match %s" - % (path, ', '.join(self.select_paths))) - for match in self.select_paths: - if match not in match_any and '*' not in match: - result.append(match) - logger.debug("Adding path %s because it doesn't match anything already on sys.path" - % match) - return result - - def run(self, options, args): - self.select_paths = options.paths - self.simulate = options.simulate - if options.list: - return self.list(options, args) - if not args: - raise InstallationError( - 'You must give at least one package to zip or unzip') - packages = [] - for arg in args: - module_name, filename = self.find_package(arg) - if options.unzip and os.path.isdir(filename): - raise InstallationError( - 'The module %s (in %s) is not a zip file; cannot be unzipped' - % (module_name, filename)) - elif not options.unzip and not os.path.isdir(filename): - raise InstallationError( - 'The module %s (in %s) is not a directory; cannot be zipped' - % (module_name, filename)) - packages.append((module_name, filename)) - last_status = None - for module_name, filename in packages: - if options.unzip: - last_status = self.unzip_package(module_name, filename) - else: - last_status = self.zip_package(module_name, filename, options.no_pyc) - return last_status - - def unzip_package(self, module_name, filename): - zip_filename = os.path.dirname(filename) - if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename): - raise InstallationError( - 'Module %s (in %s) isn\'t located in a zip file in %s' - % (module_name, filename, zip_filename)) - package_path = os.path.dirname(zip_filename) - if not package_path in self.paths(): - logger.warn( - 'Unpacking %s into %s, but %s is not on sys.path' - % (display_path(zip_filename), display_path(package_path), - display_path(package_path))) - logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename))) - if self.simulate: - logger.notify('Skipping remaining operations because of --simulate') - return - logger.indent += 2 - try: - ## FIXME: this should be undoable: - zip = zipfile.ZipFile(zip_filename) - to_save = [] - for name in zip.namelist(): - if name.startswith(module_name + os.path.sep): - content = zip.read(name) - dest = os.path.join(package_path, name) - if not os.path.exists(os.path.dirname(dest)): - os.makedirs(os.path.dirname(dest)) - if not content and dest.endswith(os.path.sep): - if not os.path.exists(dest): - os.makedirs(dest) - else: - f = open(dest, 'wb') - f.write(content) - f.close() - else: - to_save.append((name, zip.read(name))) - zip.close() - if not to_save: - logger.info('Removing now-empty zip file %s' % display_path(zip_filename)) - os.unlink(zip_filename) - self.remove_filename_from_pth(zip_filename) - else: - logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename))) - zip = zipfile.ZipFile(zip_filename, 'w') - for name, content in to_save: - zip.writestr(name, content) - zip.close() - finally: - logger.indent -= 2 - - def zip_package(self, module_name, filename, no_pyc): - orig_filename = filename - logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) - logger.indent += 2 - if filename.endswith('.egg'): - dest_filename = filename - else: - dest_filename = filename + '.zip' - try: - ## FIXME: I think this needs to be undoable: - if filename == dest_filename: - filename = backup_dir(orig_filename) - logger.notify('Moving %s aside to %s' % (orig_filename, filename)) - if not self.simulate: - shutil.move(orig_filename, filename) - try: - logger.info('Creating zip file in %s' % display_path(dest_filename)) - if not self.simulate: - zip = zipfile.ZipFile(dest_filename, 'w') - zip.writestr(module_name + '/', '') - for dirpath, dirnames, filenames in os.walk(filename): - if no_pyc: - filenames = [f for f in filenames - if not f.lower().endswith('.pyc')] - for fns, is_dir in [(dirnames, True), (filenames, False)]: - for fn in fns: - full = os.path.join(dirpath, fn) - dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn) - if is_dir: - zip.writestr(dest+'/', '') - else: - zip.write(full, dest) - zip.close() - logger.info('Removing old directory %s' % display_path(filename)) - if not self.simulate: - shutil.rmtree(filename) - except: - ## FIXME: need to do an undo here - raise - ## FIXME: should also be undone: - self.add_filename_to_pth(dest_filename) - finally: - logger.indent -= 2 - - def remove_filename_from_pth(self, filename): - for pth in self.pth_files(): - f = open(pth, 'r') - lines = f.readlines() - f.close() - new_lines = [ - l for l in lines if l.strip() != filename] - if lines != new_lines: - logger.info('Removing reference to %s from .pth file %s' - % (display_path(filename), display_path(pth))) - if not filter(None, new_lines): - logger.info('%s file would be empty: deleting' % display_path(pth)) - if not self.simulate: - os.unlink(pth) - else: - if not self.simulate: - f = open(pth, 'wb') - f.writelines(new_lines) - f.close() - return - logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename)) - - def add_filename_to_pth(self, filename): - path = os.path.dirname(filename) - dest = os.path.join(path, filename + '.pth') - if path not in self.paths(): - logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest)) - if not self.simulate: - if os.path.exists(dest): - f = open(dest) - lines = f.readlines() - f.close() - if lines and not lines[-1].endswith('\n'): - lines[-1] += '\n' - lines.append(filename+'\n') - else: - lines = [filename + '\n'] - f = open(dest, 'wb') - f.writelines(lines) - f.close() - - def pth_files(self): - for path in self.paths(): - if not os.path.exists(path) or not os.path.isdir(path): - continue - for filename in os.listdir(path): - if filename.endswith('.pth'): - yield os.path.join(path, filename) - - def find_package(self, package): - for path in self.paths(): - full = os.path.join(path, package) - if os.path.exists(full): - return package, full - if not os.path.isdir(path) and zipfile.is_zipfile(path): - zip = zipfile.ZipFile(path, 'r') - try: - zip.read(os.path.join(package, '__init__.py')) - except KeyError: - pass - else: - zip.close() - return package, full - zip.close() - ## FIXME: need special error for package.py case: - raise InstallationError( - 'No package with the name %s found' % package) - - def list(self, options, args): - if args: - raise InstallationError( - 'You cannot give an argument with --list') - for path in sorted(self.paths()): - if not os.path.exists(path): - continue - basename = os.path.basename(path.rstrip(os.path.sep)) - if os.path.isfile(path) and zipfile.is_zipfile(path): - if os.path.dirname(path) not in self.paths(): - logger.notify('Zipped egg: %s' % display_path(path)) - continue - if (basename != 'site-packages' and basename != 'dist-packages' - and not path.replace('\\', '/').endswith('lib/python')): - continue - logger.notify('In %s:' % display_path(path)) - logger.indent += 2 - zipped = [] - unzipped = [] - try: - for filename in sorted(os.listdir(path)): - ext = os.path.splitext(filename)[1].lower() - if ext in ('.pth', '.egg-info', '.egg-link'): - continue - if ext == '.py': - logger.info('Not displaying %s: not a package' % display_path(filename)) - continue - full = os.path.join(path, filename) - if os.path.isdir(full): - unzipped.append((filename, self.count_package(full))) - elif zipfile.is_zipfile(full): - zipped.append(filename) - else: - logger.info('Unknown file: %s' % display_path(filename)) - if zipped: - logger.notify('Zipped packages:') - logger.indent += 2 - try: - for filename in zipped: - logger.notify(filename) - finally: - logger.indent -= 2 - else: - logger.notify('No zipped packages.') - if unzipped: - if options.sort_files: - unzipped.sort(key=lambda x: -x[1]) - logger.notify('Unzipped packages:') - logger.indent += 2 - try: - for filename, count in unzipped: - logger.notify('%s (%i files)' % (filename, count)) - finally: - logger.indent -= 2 - else: - logger.notify('No unzipped packages.') - finally: - logger.indent -= 2 - - def count_package(self, path): - total = 0 - for dirpath, dirnames, filenames in os.walk(path): - filenames = [f for f in filenames - if not f.lower().endswith('.pyc')] - total += len(filenames) - return total - - -ZipCommand() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py deleted file mode 100644 index f1b63936b..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/download.py +++ /dev/null @@ -1,470 +0,0 @@ -import xmlrpclib -import re -import getpass -import urllib -import urllib2 -import urlparse -import os -import mimetypes -import shutil -import tempfile -from pip.backwardcompat import md5, copytree -from pip.exceptions import InstallationError -from pip.util import (splitext, - format_size, display_path, backup_dir, ask, - unpack_file, create_download_cache_folder, cache_download) -from pip.vcs import vcs -from pip.log import logger - - -__all__ = ['xmlrpclib_transport', 'get_file_content', 'urlopen', - 'is_url', 'url_to_path', 'path_to_url', 'path_to_url2', - 'geturl', 'is_archive_file', 'unpack_vcs_link', - 'unpack_file_url', 'is_vcs_url', 'is_file_url', 'unpack_http_url'] - - -xmlrpclib_transport = xmlrpclib.Transport() - - -def get_file_content(url, comes_from=None): - """Gets the content of a file; it may be a filename, file: URL, or - http: URL. Returns (location, content)""" - match = _scheme_re.search(url) - if match: - scheme = match.group(1).lower() - if (scheme == 'file' and comes_from - and comes_from.startswith('http')): - raise InstallationError( - 'Requirements file %s references URL %s, which is local' - % (comes_from, url)) - if scheme == 'file': - path = url.split(':', 1)[1] - path = path.replace('\\', '/') - match = _url_slash_drive_re.match(path) - if match: - path = match.group(1) + ':' + path.split('|', 1)[1] - path = urllib.unquote(path) - if path.startswith('/'): - path = '/' + path.lstrip('/') - url = path - else: - ## FIXME: catch some errors - resp = urlopen(url) - return geturl(resp), resp.read() - try: - f = open(url) - content = f.read() - except IOError, e: - raise InstallationError('Could not open requirements file: %s' % str(e)) - else: - f.close() - return url, content - - -_scheme_re = re.compile(r'^(http|https|file):', re.I) -_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) - -class URLOpener(object): - """ - pip's own URL helper that adds HTTP auth and proxy support - """ - def __init__(self): - self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - - def __call__(self, url): - """ - If the given url contains auth info or if a normal request gets a 401 - response, an attempt is made to fetch the resource using basic HTTP - auth. - - """ - url, username, password = self.extract_credentials(url) - if username is None: - try: - response = urllib2.urlopen(self.get_request(url)) - except urllib2.HTTPError, e: - if e.code != 401: - raise - response = self.get_response(url) - else: - response = self.get_response(url, username, password) - return response - - def get_request(self, url): - """ - Wraps the URL to retrieve to protects against "creative" - interpretation of the RFC: http://bugs.python.org/issue8732 - """ - if isinstance(url, basestring): - url = urllib2.Request(url, headers={'Accept-encoding': 'identity'}) - return url - - def get_response(self, url, username=None, password=None): - """ - does the dirty work of actually getting the rsponse object using urllib2 - and its HTTP auth builtins. - """ - scheme, netloc, path, query, frag = urlparse.urlsplit(url) - pass_url = urlparse.urlunsplit(('_none_', netloc, path, query, frag)).replace('_none_://', '', 1) - req = self.get_request(url) - - stored_username, stored_password = self.passman.find_user_password(None, netloc) - # see if we have a password stored - if stored_username is None: - if username is None and self.prompting: - username = urllib.quote(raw_input('User for %s: ' % netloc)) - password = urllib.quote(getpass.getpass('Password: ')) - if username and password: - self.passman.add_password(None, netloc, username, password) - stored_username, stored_password = self.passman.find_user_password(None, netloc) - authhandler = urllib2.HTTPBasicAuthHandler(self.passman) - opener = urllib2.build_opener(authhandler) - # FIXME: should catch a 401 and offer to let the user reenter credentials - return opener.open(req) - - def setup(self, proxystr='', prompting=True): - """ - Sets the proxy handler given the option passed on the command - line. If an empty string is passed it looks at the HTTP_PROXY - environment variable. - """ - self.prompting = prompting - proxy = self.get_proxy(proxystr) - if proxy: - proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy}) - opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler) - urllib2.install_opener(opener) - - def parse_credentials(self, netloc): - if "@" in netloc: - userinfo = netloc.rsplit("@", 1)[0] - if ":" in userinfo: - return userinfo.split(":", 1) - return userinfo, None - return None, None - - def extract_credentials(self, url): - """ - Extracts user/password from a url. - - Returns a tuple: - (url-without-auth, username, password) - """ - if isinstance(url, urllib2.Request): - result = urlparse.urlsplit(url.get_full_url()) - else: - result = urlparse.urlsplit(url) - scheme, netloc, path, query, frag = result - - username, password = self.parse_credentials(netloc) - if username is None: - return url, None, None - elif password is None and self.prompting: - # remove the auth credentials from the url part - netloc = netloc.replace('%s@' % username, '', 1) - # prompt for the password - prompt = 'Password for %s@%s: ' % (username, netloc) - password = urllib.quote(getpass.getpass(prompt)) - else: - # remove the auth credentials from the url part - netloc = netloc.replace('%s:%s@' % (username, password), '', 1) - - target_url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - return target_url, username, password - - def get_proxy(self, proxystr=''): - """ - Get the proxy given the option passed on the command line. - If an empty string is passed it looks at the HTTP_PROXY - environment variable. - """ - if not proxystr: - proxystr = os.environ.get('HTTP_PROXY', '') - if proxystr: - if '@' in proxystr: - user_password, server_port = proxystr.split('@', 1) - if ':' in user_password: - user, password = user_password.split(':', 1) - else: - user = user_password - prompt = 'Password for %s@%s: ' % (user, server_port) - password = urllib.quote(getpass.getpass(prompt)) - return '%s:%s@%s' % (user, password, server_port) - else: - return proxystr - else: - return None - -urlopen = URLOpener() - - -def is_url(name): - """Returns true if the name looks like a URL""" - if ':' not in name: - return False - scheme = name.split(':', 1)[0].lower() - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def url_to_path(url): - """ - Convert a file: URL to a path. - """ - assert url.startswith('file:'), ( - "You can only turn file: urls into filenames (not %r)" % url) - path = url[len('file:'):].lstrip('/') - path = urllib.unquote(path) - if _url_drive_re.match(path): - path = path[0] + ':' + path[2:] - else: - path = '/' + path - return path - - -_drive_re = re.compile('^([a-z]):', re.I) -_url_drive_re = re.compile('^([a-z])[:|]', re.I) - - -def path_to_url(path): - """ - Convert a path to a file: URL. The path will be made absolute. - """ - path = os.path.normcase(os.path.abspath(path)) - if _drive_re.match(path): - path = path[0] + '|' + path[2:] - url = urllib.quote(path) - url = url.replace(os.path.sep, '/') - url = url.lstrip('/') - return 'file:///' + url - - -def path_to_url2(path): - """ - Convert a path to a file: URL. The path will be made absolute and have - quoted path parts. - """ - path = os.path.normpath(os.path.abspath(path)) - drive, path = os.path.splitdrive(path) - filepath = path.split(os.path.sep) - url = '/'.join([urllib.quote(part) for part in filepath]) - if not drive: - url = url.lstrip('/') - return 'file:///' + drive + url - - -def geturl(urllib2_resp): - """ - Use instead of urllib.addinfourl.geturl(), which appears to have - some issues with dropping the double slash for certain schemes - (e.g. file://). This implementation is probably over-eager, as it - always restores '://' if it is missing, and it appears some url - schemata aren't always followed by '//' after the colon, but as - far as I know pip doesn't need any of those. - The URI RFC can be found at: http://tools.ietf.org/html/rfc1630 - - This function assumes that - scheme:/foo/bar - is the same as - scheme:///foo/bar - """ - url = urllib2_resp.geturl() - scheme, rest = url.split(':', 1) - if rest.startswith('//'): - return url - else: - # FIXME: write a good test to cover it - return '%s://%s' % (scheme, rest) - - -def is_archive_file(name): - """Return True if `name` is a considered as an archive file.""" - archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle') - ext = splitext(name)[1].lower() - if ext in archives: - return True - return False - - -def unpack_vcs_link(link, location, only_download=False): - vcs_backend = _get_used_vcs_backend(link) - if only_download: - vcs_backend.export(location) - else: - vcs_backend.unpack(location) - - -def unpack_file_url(link, location): - source = url_to_path(link.url) - content_type = mimetypes.guess_type(source)[0] - if os.path.isdir(source): - # delete the location since shutil will create it again :( - if os.path.isdir(location): - shutil.rmtree(location) - copytree(source, location) - else: - unpack_file(source, location, content_type, link) - - -def _get_used_vcs_backend(link): - for backend in vcs.backends: - if link.scheme in backend.schemes: - vcs_backend = backend(link.url) - return vcs_backend - - -def is_vcs_url(link): - return bool(_get_used_vcs_backend(link)) - - -def is_file_url(link): - return link.url.lower().startswith('file:') - - -def _check_md5(download_hash, link): - download_hash = download_hash.hexdigest() - if download_hash != link.md5_hash: - logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!" - % (link, download_hash, link.md5_hash)) - raise InstallationError('Bad MD5 hash for package %s' % link) - - -def _get_md5_from_file(target_file, link): - download_hash = md5() - fp = open(target_file, 'rb') - while 1: - chunk = fp.read(4096) - if not chunk: - break - download_hash.update(chunk) - fp.close() - return download_hash - - -def _download_url(resp, link, temp_location): - fp = open(temp_location, 'wb') - download_hash = None - if link.md5_hash: - download_hash = md5() - try: - total_length = int(resp.info()['content-length']) - except (ValueError, KeyError): - total_length = 0 - downloaded = 0 - show_progress = total_length > 40*1000 or not total_length - show_url = link.show_url - try: - if show_progress: - ## FIXME: the URL can get really long in this message: - if total_length: - logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) - else: - logger.start_progress('Downloading %s (unknown size): ' % show_url) - else: - logger.notify('Downloading %s' % show_url) - logger.debug('Downloading from URL %s' % link) - - while 1: - chunk = resp.read(4096) - if not chunk: - break - downloaded += len(chunk) - if show_progress: - if not total_length: - logger.show_progress('%s' % format_size(downloaded)) - else: - logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded))) - if link.md5_hash: - download_hash.update(chunk) - fp.write(chunk) - fp.close() - finally: - if show_progress: - logger.end_progress('%s downloaded' % format_size(downloaded)) - return download_hash - - -def _copy_file(filename, location, content_type, link): - copy = True - download_location = os.path.join(location, link.filename) - if os.path.exists(download_location): - response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' - % display_path(download_location), ('i', 'w', 'b')) - if response == 'i': - copy = False - elif response == 'w': - logger.warn('Deleting %s' % display_path(download_location)) - os.remove(download_location) - elif response == 'b': - dest_file = backup_dir(download_location) - logger.warn('Backing up %s to %s' - % (display_path(download_location), display_path(dest_file))) - shutil.move(download_location, dest_file) - if copy: - shutil.copy(filename, download_location) - logger.indent -= 2 - logger.notify('Saved %s' % display_path(download_location)) - - -def unpack_http_url(link, location, download_cache, only_download): - temp_dir = tempfile.mkdtemp('-unpack', 'pip-') - target_url = link.url.split('#', 1)[0] - target_file = None - download_hash = None - if download_cache: - target_file = os.path.join(download_cache, - urllib.quote(target_url, '')) - if not os.path.isdir(download_cache): - create_download_cache_folder(download_cache) - if (target_file - and os.path.exists(target_file) - and os.path.exists(target_file+'.content-type')): - fp = open(target_file+'.content-type') - content_type = fp.read().strip() - fp.close() - if link.md5_hash: - download_hash = _get_md5_from_file(target_file, link) - temp_location = target_file - logger.notify('Using download cache from %s' % target_file) - else: - resp = _get_response_from_url(target_url, link) - content_type = resp.info()['content-type'] - filename = link.filename - ext = splitext(filename)[1] - if not ext: - ext = mimetypes.guess_extension(content_type) - if ext: - filename += ext - if not ext and link.url != geturl(resp): - ext = os.path.splitext(geturl(resp))[1] - if ext: - filename += ext - temp_location = os.path.join(temp_dir, filename) - download_hash = _download_url(resp, link, temp_location) - if link.md5_hash: - _check_md5(download_hash, link) - if only_download: - _copy_file(temp_location, location, content_type, link) - else: - unpack_file(temp_location, location, content_type, link) - if target_file and target_file != temp_location: - cache_download(target_file, temp_location, content_type) - if target_file is None: - os.unlink(temp_location) - os.rmdir(temp_dir) - - -def _get_response_from_url(target_url, link): - try: - resp = urlopen(target_url) - except urllib2.HTTPError, e: - logger.fatal("HTTP error %s while getting %s" % (e.code, link)) - raise - except IOError, e: - # Typically an FTP error - logger.fatal("Error %s while getting %s" % (e, link)) - raise - return resp - -class Urllib2HeadRequest(urllib2.Request): - def get_method(self): - return "HEAD" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py deleted file mode 100644 index 1ad1a616d..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/exceptions.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Exceptions used throughout package""" - - -class InstallationError(Exception): - """General exception during installation""" - - -class UninstallationError(Exception): - """General exception during uninstallation""" - - -class DistributionNotFound(InstallationError): - """Raised when a distribution cannot be found to satisfy a requirement""" - - -class BadCommand(Exception): - """Raised when virtualenv or a command is not found""" diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py deleted file mode 100644 index e42d8c868..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/index.py +++ /dev/null @@ -1,686 +0,0 @@ -"""Routines related to PyPI, indexes""" - -import sys -import os -import re -import mimetypes -import threading -import posixpath -import pkg_resources -import urllib -import urllib2 -import urlparse -import httplib -import random -import socket -import string -from Queue import Queue -from Queue import Empty as QueueEmpty -from pip.log import logger -from pip.util import Inf -from pip.util import normalize_name, splitext -from pip.exceptions import DistributionNotFound -from pip.backwardcompat import WindowsError, product -from pip.download import urlopen, path_to_url2, url_to_path, geturl, Urllib2HeadRequest - -__all__ = ['PackageFinder'] - - -DEFAULT_MIRROR_URL = "last.pypi.python.org" - - -class PackageFinder(object): - """This finds packages. - - This is meant to match easy_install's technique for looking for - packages, by reading pages and looking for appropriate links - """ - - def __init__(self, find_links, index_urls, - use_mirrors=False, mirrors=None, main_mirror_url=None): - self.find_links = find_links - self.index_urls = index_urls - self.dependency_links = [] - self.cache = PageCache() - # These are boring links that have already been logged somehow: - self.logged_links = set() - if use_mirrors: - self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) - logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) - else: - self.mirror_urls = [] - - def add_dependency_links(self, links): - ## FIXME: this shouldn't be global list this, it should only - ## apply to requirements of the package that specifies the - ## dependency_links value - ## FIXME: also, we should track comes_from (i.e., use Link) - self.dependency_links.extend(links) - - @staticmethod - def _sort_locations(locations): - """ - Sort locations into "files" (archives) and "urls", and return - a pair of lists (files,urls) - """ - files = [] - urls = [] - - # puts the url for the given file path into the appropriate - # list - def sort_path(path): - url = path_to_url2(path) - if mimetypes.guess_type(url, strict=False)[0] == 'text/html': - urls.append(url) - else: - files.append(url) - - for url in locations: - if url.startswith('file:'): - path = url_to_path(url) - if os.path.isdir(path): - path = os.path.realpath(path) - for item in os.listdir(path): - sort_path(os.path.join(path, item)) - elif os.path.isfile(path): - sort_path(path) - else: - urls.append(url) - return files, urls - - def find_requirement(self, req, upgrade): - url_name = req.url_name - # Only check main index if index URL is given: - main_index_url = None - if self.index_urls: - # Check that we have the url_name correctly spelled: - main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) - # This will also cache the page, so it's okay that we get it again later: - page = self._get_page(main_index_url, req) - if page is None: - url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name - - # Combine index URLs with mirror URLs here to allow - # adding more index URLs from requirements files - all_index_urls = self.index_urls + self.mirror_urls - - def mkurl_pypi_url(url): - loc = posixpath.join(url, url_name) - # For maximum compatibility with easy_install, ensure the path - # ends in a trailing slash. Although this isn't in the spec - # (and PyPI can handle it without the slash) some other index - # implementations might break if they relied on easy_install's behavior. - if not loc.endswith('/'): - loc = loc + '/' - return loc - if url_name is not None: - locations = [ - mkurl_pypi_url(url) - for url in all_index_urls] + self.find_links - else: - locations = list(self.find_links) - locations.extend(self.dependency_links) - for version in req.absolute_versions: - if url_name is not None and main_index_url is not None: - locations = [ - posixpath.join(main_index_url.url, version)] + locations - - file_locations, url_locations = self._sort_locations(locations) - - locations = [Link(url) for url in url_locations] - logger.debug('URLs to search for versions for %s:' % req) - for location in locations: - logger.debug('* %s' % location) - found_versions = [] - found_versions.extend( - self._package_versions( - [Link(url, '-f') for url in self.find_links], req.name.lower())) - page_versions = [] - for page in self._get_pages(locations, req): - logger.debug('Analyzing links from page %s' % page.url) - logger.indent += 2 - try: - page_versions.extend(self._package_versions(page.links, req.name.lower())) - finally: - logger.indent -= 2 - dependency_versions = list(self._package_versions( - [Link(url) for url in self.dependency_links], req.name.lower())) - if dependency_versions: - logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) - file_versions = list(self._package_versions( - [Link(url) for url in file_locations], req.name.lower())) - if not found_versions and not page_versions and not dependency_versions and not file_versions: - logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) - raise DistributionNotFound('No distributions at all found for %s' % req) - if req.satisfied_by is not None: - found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) - if file_versions: - file_versions.sort(reverse=True) - logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) - found_versions = file_versions + found_versions - all_versions = found_versions + page_versions + dependency_versions - applicable_versions = [] - for (parsed_version, link, version) in all_versions: - if version not in req.req: - logger.info("Ignoring link %s, version %s doesn't match %s" - % (link, version, ','.join([''.join(s) for s in req.req.specs]))) - continue - applicable_versions.append((link, version)) - applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) - existing_applicable = bool([link for link, version in applicable_versions if link is Inf]) - if not upgrade and existing_applicable: - if applicable_versions[0][1] is Inf: - logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' - % req.satisfied_by.version) - else: - logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' - % (req.satisfied_by.version, applicable_versions[0][1])) - return None - if not applicable_versions: - logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' - % (req, ', '.join([version for parsed_version, link, version in found_versions]))) - raise DistributionNotFound('No distributions matching the version for %s' % req) - if applicable_versions[0][0] is Inf: - # We have an existing version, and its the best version - logger.info('Installed version (%s) is most up-to-date (past versions: %s)' - % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none')) - return None - if len(applicable_versions) > 1: - logger.info('Using version %s (newest of versions: %s)' % - (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions]))) - return applicable_versions[0][0] - - def _find_url_name(self, index_url, url_name, req): - """Finds the true URL name of a package, when the given name isn't quite correct. - This is usually used to implement case-insensitivity.""" - if not index_url.url.endswith('/'): - # Vaguely part of the PyPI API... weird but true. - ## FIXME: bad to modify this? - index_url.url += '/' - page = self._get_page(index_url, req) - if page is None: - logger.fatal('Cannot fetch index base URL %s' % index_url) - return - norm_name = normalize_name(req.url_name) - for link in page.links: - base = posixpath.basename(link.path.rstrip('/')) - if norm_name == normalize_name(base): - logger.notify('Real name of requirement %s is %s' % (url_name, base)) - return base - return None - - def _get_pages(self, locations, req): - """Yields (page, page_url) from the given locations, skipping - locations that have errors, and adding download/homepage links""" - pending_queue = Queue() - for location in locations: - pending_queue.put(location) - done = [] - seen = set() - threads = [] - for i in range(min(10, len(locations))): - t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen)) - t.setDaemon(True) - threads.append(t) - t.start() - for t in threads: - t.join() - return done - - _log_lock = threading.Lock() - - def _get_queued_page(self, req, pending_queue, done, seen): - while 1: - try: - location = pending_queue.get(False) - except QueueEmpty: - return - if location in seen: - continue - seen.add(location) - page = self._get_page(location, req) - if page is None: - continue - done.append(page) - for link in page.rel_links(): - pending_queue.put(link) - - _egg_fragment_re = re.compile(r'#egg=([^&]*)') - _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I) - _py_version_re = re.compile(r'-py([123]\.[0-9])$') - - def _sort_links(self, links): - "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates" - eggs, no_eggs = [], [] - seen = set() - for link in links: - if link not in seen: - seen.add(link) - if link.egg_fragment: - eggs.append(link) - else: - no_eggs.append(link) - return no_eggs + eggs - - def _package_versions(self, links, search_name): - for link in self._sort_links(links): - for v in self._link_package_versions(link, search_name): - yield v - - def _link_package_versions(self, link, search_name): - """ - Return an iterable of triples (pkg_resources_version_key, - link, python_version) that can be extracted from the given - link. - - Meant to be overridden by subclasses, not called by clients. - """ - if link.egg_fragment: - egg_info = link.egg_fragment - else: - egg_info, ext = link.splitext() - if not ext: - if link not in self.logged_links: - logger.debug('Skipping link %s; not a file' % link) - self.logged_links.add(link) - return [] - if egg_info.endswith('.tar'): - # Special double-extension case: - egg_info = egg_info[:-4] - ext = '.tar' + ext - if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'): - if link not in self.logged_links: - logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext)) - self.logged_links.add(link) - return [] - version = self._egg_info_matches(egg_info, search_name, link) - if version is None: - logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name)) - return [] - match = self._py_version_re.search(version) - if match: - version = version[:match.start()] - py_version = match.group(1) - if py_version != sys.version[:3]: - logger.debug('Skipping %s because Python version is incorrect' % link) - return [] - logger.debug('Found link %s, version: %s' % (link, version)) - return [(pkg_resources.parse_version(version), - link, - version)] - - def _egg_info_matches(self, egg_info, search_name, link): - match = self._egg_info_re.search(egg_info) - if not match: - logger.debug('Could not parse version from link: %s' % link) - return None - name = match.group(0).lower() - # To match the "safe" name that pkg_resources creates: - name = name.replace('_', '-') - if name.startswith(search_name.lower()): - return match.group(0)[len(search_name):].lstrip('-') - else: - return None - - def _get_page(self, link, req): - return HTMLPage.get_page(link, req, cache=self.cache) - - def _get_mirror_urls(self, mirrors=None, main_mirror_url=None): - """Retrieves a list of URLs from the main mirror DNS entry - unless a list of mirror URLs are passed. - """ - if not mirrors: - mirrors = get_mirrors(main_mirror_url) - # Should this be made "less random"? E.g. netselect like? - random.shuffle(mirrors) - - mirror_urls = set() - for mirror_url in mirrors: - # Make sure we have a valid URL - if not ("http://" or "https://" or "file://") in mirror_url: - mirror_url = "http://%s" % mirror_url - if not mirror_url.endswith("/simple"): - mirror_url = "%s/simple/" % mirror_url - mirror_urls.add(mirror_url) - - return list(mirror_urls) - - -class PageCache(object): - """Cache of HTML pages""" - - failure_limit = 3 - - def __init__(self): - self._failures = {} - self._pages = {} - self._archives = {} - - def too_many_failures(self, url): - return self._failures.get(url, 0) >= self.failure_limit - - def get_page(self, url): - return self._pages.get(url) - - def is_archive(self, url): - return self._archives.get(url, False) - - def set_is_archive(self, url, value=True): - self._archives[url] = value - - def add_page_failure(self, url, level): - self._failures[url] = self._failures.get(url, 0)+level - - def add_page(self, urls, page): - for url in urls: - self._pages[url] = page - - -class HTMLPage(object): - """Represents one page, along with its URL""" - - ## FIXME: these regexes are horrible hacks: - _homepage_re = re.compile(r'<th>\s*home\s*page', re.I) - _download_re = re.compile(r'<th>\s*download\s+url', re.I) - ## These aren't so aweful: - _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I) - _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S) - _base_re = re.compile(r"""<base\s+href\s*=\s*['"]?([^'">]+)""", re.I) - - def __init__(self, content, url, headers=None): - self.content = content - self.url = url - self.headers = headers - - def __str__(self): - return self.url - - @classmethod - def get_page(cls, link, req, cache=None, skip_archives=True): - url = link.url - url = url.split('#', 1)[0] - if cache.too_many_failures(url): - return None - - # Check for VCS schemes that do not support lookup as web pages. - from pip.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals()) - return None - - if cache is not None: - inst = cache.get_page(url) - if inst is not None: - return inst - try: - if skip_archives: - if cache is not None: - if cache.is_archive(url): - return None - filename = link.filename - for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']: - if filename.endswith(bad_ext): - content_type = cls._get_content_type(url) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type)) - if cache is not None: - cache.set_is_archive(url) - return None - logger.debug('Getting page %s' % url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) - if scheme == 'file' and os.path.isdir(urllib.url2pathname(path)): - # add trailing slash if not present so urljoin doesn't trim final segment - if not url.endswith('/'): - url += '/' - url = urlparse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s' % url) - - resp = urlopen(url) - - real_url = geturl(resp) - headers = resp.info() - inst = cls(resp.read(), real_url, headers) - except (urllib2.HTTPError, urllib2.URLError, socket.timeout, socket.error, OSError, WindowsError), e: - desc = str(e) - if isinstance(e, socket.timeout): - log_meth = logger.info - level =1 - desc = 'timed out' - elif isinstance(e, urllib2.URLError): - log_meth = logger.info - if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout): - desc = 'timed out' - level = 1 - else: - level = 2 - elif isinstance(e, urllib2.HTTPError) and e.code == 404: - ## FIXME: notify? - log_meth = logger.info - level = 2 - else: - log_meth = logger.info - level = 1 - log_meth('Could not fetch URL %s: %s' % (link, desc)) - log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req)) - if cache is not None: - cache.add_page_failure(url, level) - return None - if cache is not None: - cache.add_page([url, real_url], inst) - return inst - - @staticmethod - def _get_content_type(url): - """Get the Content-Type of the given url, using a HEAD request""" - scheme, netloc, path, query, fragment = urlparse.urlsplit(url) - if not scheme in ('http', 'https', 'ftp', 'ftps'): - ## FIXME: some warning or something? - ## assertion error? - return '' - req = Urllib2HeadRequest(url, headers={'Host': netloc}) - resp = urlopen(req) - try: - if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'): - ## FIXME: doesn't handle redirects - return '' - return resp.info().get('content-type', '') - finally: - resp.close() - - @property - def base_url(self): - if not hasattr(self, "_base_url"): - match = self._base_re.search(self.content) - if match: - self._base_url = match.group(1) - else: - self._base_url = self.url - return self._base_url - - @property - def links(self): - """Yields all links in the page""" - for match in self._href_re.finditer(self.content): - url = match.group(1) or match.group(2) or match.group(3) - url = self.clean_link(urlparse.urljoin(self.base_url, url)) - yield Link(url, self) - - def rel_links(self): - for url in self.explicit_rel_links(): - yield url - for url in self.scraped_rel_links(): - yield url - - def explicit_rel_links(self, rels=('homepage', 'download')): - """Yields all links with the given relations""" - for match in self._rel_re.finditer(self.content): - found_rels = match.group(1).lower().split() - for rel in rels: - if rel in found_rels: - break - else: - continue - match = self._href_re.search(match.group(0)) - if not match: - continue - url = match.group(1) or match.group(2) or match.group(3) - url = self.clean_link(urlparse.urljoin(self.base_url, url)) - yield Link(url, self) - - def scraped_rel_links(self): - for regex in (self._homepage_re, self._download_re): - match = regex.search(self.content) - if not match: - continue - href_match = self._href_re.search(self.content, pos=match.end()) - if not href_match: - continue - url = match.group(1) or match.group(2) or match.group(3) - if not url: - continue - url = self.clean_link(urlparse.urljoin(self.base_url, url)) - yield Link(url, self) - - _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - - def clean_link(self, url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return self._clean_re.sub( - lambda match: '%%%2x' % ord(match.group(0)), url) - - -class Link(object): - - def __init__(self, url, comes_from=None): - self.url = url - self.comes_from = comes_from - - def __str__(self): - if self.comes_from: - return '%s (from %s)' % (self.url, self.comes_from) - else: - return self.url - - def __repr__(self): - return '<Link %s>' % self - - def __eq__(self, other): - return self.url == other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - url = self.url - url = url.split('#', 1)[0] - url = url.split('?', 1)[0] - url = url.rstrip('/') - name = posixpath.basename(url) - assert name, ( - 'URL %r produced no filename' % url) - return name - - @property - def scheme(self): - return urlparse.urlsplit(self.url)[0] - - @property - def path(self): - return urlparse.urlsplit(self.url)[2] - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - _egg_fragment_re = re.compile(r'#egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _md5_re = re.compile(r'md5=([a-f0-9]+)') - - @property - def md5_hash(self): - match = self._md5_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - -def get_requirement_from_url(url): - """Get a requirement from the URL, if possible. This looks for #egg - in the URL""" - link = Link(url) - egg_info = link.egg_fragment - if not egg_info: - egg_info = splitext(link.filename)[0] - return package_to_requirement(egg_info) - - -def package_to_requirement(package_name): - """Translate a name like Foo-1.2 to Foo==1.3""" - match = re.search(r'^(.*?)(-dev|-\d.*)', package_name) - if match: - name = match.group(1) - version = match.group(2) - else: - name = package_name - version = '' - if version: - return '%s==%s' % (name, version) - else: - return name - - -def get_mirrors(hostname=None): - """Return the list of mirrors from the last record found on the DNS - entry:: - - >>> from pip.index import get_mirrors - >>> get_mirrors() - ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', - 'd.pypi.python.org'] - - Originally written for the distutils2 project by Alexis Metaireau. - """ - if hostname is None: - hostname = DEFAULT_MIRROR_URL - - # return the last mirror registered on PyPI. - try: - hostname = socket.gethostbyname_ex(hostname)[0] - except socket.gaierror: - return [] - end_letter = hostname.split(".", 1) - - # determine the list from the last one. - return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] - - -def string_range(last): - """Compute the range of string between "a" and last. - - This works for simple "a to z" lists, but also for "a to zz" lists. - """ - for k in range(len(last)): - for x in product(string.ascii_lowercase, repeat=k+1): - result = ''.join(x) - yield result - if result == last: - return - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py deleted file mode 100644 index 4254ef2fb..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/locations.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Locations where we look for configs, install stuff, etc""" - -import sys -import os -from distutils import sysconfig - - -def running_under_virtualenv(): - """ - Return True if we're running inside a virtualenv, False otherwise. - - """ - return hasattr(sys, 'real_prefix') - - -if running_under_virtualenv(): - ## FIXME: is build/ a good name? - build_prefix = os.path.join(sys.prefix, 'build') - src_prefix = os.path.join(sys.prefix, 'src') -else: - ## FIXME: this isn't a very good default - build_prefix = os.path.join(os.getcwd(), 'build') - src_prefix = os.path.join(os.getcwd(), 'src') - -# FIXME doesn't account for venv linked to global site-packages - -site_packages = sysconfig.get_python_lib() -user_dir = os.path.expanduser('~') -if sys.platform == 'win32': - bin_py = os.path.join(sys.prefix, 'Scripts') - # buildout uses 'bin' on Windows too? - if not os.path.exists(bin_py): - bin_py = os.path.join(sys.prefix, 'bin') - user_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming - default_storage_dir = os.path.join(user_dir, 'pip') - default_config_file = os.path.join(default_storage_dir, 'pip.ini') - default_log_file = os.path.join(default_storage_dir, 'pip.log') -else: - bin_py = os.path.join(sys.prefix, 'bin') - default_storage_dir = os.path.join(user_dir, '.pip') - default_config_file = os.path.join(default_storage_dir, 'pip.conf') - default_log_file = os.path.join(default_storage_dir, 'pip.log') - # Forcing to use /usr/local/bin for standard Mac OS X framework installs - if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': - bin_py = '/usr/local/bin' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py deleted file mode 100644 index 0218ab1ae..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/log.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Logging -""" - -import sys -import logging - - -class Logger(object): - - """ - Logging object for use in command-line script. Allows ranges of - levels, to avoid some redundancy of displayed information. - """ - - VERBOSE_DEBUG = logging.DEBUG-1 - DEBUG = logging.DEBUG - INFO = logging.INFO - NOTIFY = (logging.INFO+logging.WARN)/2 - WARN = WARNING = logging.WARN - ERROR = logging.ERROR - FATAL = logging.FATAL - - LEVELS = [VERBOSE_DEBUG, DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL] - - def __init__(self): - self.consumers = [] - self.indent = 0 - self.explicit_levels = False - self.in_progress = None - self.in_progress_hanging = False - - def debug(self, msg, *args, **kw): - self.log(self.DEBUG, msg, *args, **kw) - - def info(self, msg, *args, **kw): - self.log(self.INFO, msg, *args, **kw) - - def notify(self, msg, *args, **kw): - self.log(self.NOTIFY, msg, *args, **kw) - - def warn(self, msg, *args, **kw): - self.log(self.WARN, msg, *args, **kw) - - def error(self, msg, *args, **kw): - self.log(self.WARN, msg, *args, **kw) - - def fatal(self, msg, *args, **kw): - self.log(self.FATAL, msg, *args, **kw) - - def log(self, level, msg, *args, **kw): - if args: - if kw: - raise TypeError( - "You may give positional or keyword arguments, not both") - args = args or kw - rendered = None - for consumer_level, consumer in self.consumers: - if self.level_matches(level, consumer_level): - if (self.in_progress_hanging - and consumer in (sys.stdout, sys.stderr)): - self.in_progress_hanging = False - sys.stdout.write('\n') - sys.stdout.flush() - if rendered is None: - if args: - rendered = msg % args - else: - rendered = msg - rendered = ' '*self.indent + rendered - if self.explicit_levels: - ## FIXME: should this be a name, not a level number? - rendered = '%02i %s' % (level, rendered) - if hasattr(consumer, 'write'): - consumer.write(rendered+'\n') - else: - consumer(rendered) - - def start_progress(self, msg): - assert not self.in_progress, ( - "Tried to start_progress(%r) while in_progress %r" - % (msg, self.in_progress)) - if self.level_matches(self.NOTIFY, self._stdout_level()): - sys.stdout.write(' '*self.indent + msg) - sys.stdout.flush() - self.in_progress_hanging = True - else: - self.in_progress_hanging = False - self.in_progress = msg - self.last_message = None - - def end_progress(self, msg='done.'): - assert self.in_progress, ( - "Tried to end_progress without start_progress") - if self.stdout_level_matches(self.NOTIFY): - if not self.in_progress_hanging: - # Some message has been printed out since start_progress - sys.stdout.write('...' + self.in_progress + msg + '\n') - sys.stdout.flush() - else: - # These erase any messages shown with show_progress (besides .'s) - logger.show_progress('') - logger.show_progress('') - sys.stdout.write(msg + '\n') - sys.stdout.flush() - self.in_progress = None - self.in_progress_hanging = False - - def show_progress(self, message=None): - """If we are in a progress scope, and no log messages have been - shown, write out another '.'""" - if self.in_progress_hanging: - if message is None: - sys.stdout.write('.') - sys.stdout.flush() - else: - if self.last_message: - padding = ' ' * max(0, len(self.last_message)-len(message)) - else: - padding = '' - sys.stdout.write('\r%s%s%s%s' % (' '*self.indent, self.in_progress, message, padding)) - sys.stdout.flush() - self.last_message = message - - def stdout_level_matches(self, level): - """Returns true if a message at this level will go to stdout""" - return self.level_matches(level, self._stdout_level()) - - def _stdout_level(self): - """Returns the level that stdout runs at""" - for level, consumer in self.consumers: - if consumer is sys.stdout: - return level - return self.FATAL - - def level_matches(self, level, consumer_level): - """ - >>> l = Logger() - >>> l.level_matches(3, 4) - False - >>> l.level_matches(3, 2) - True - >>> l.level_matches(slice(None, 3), 3) - False - >>> l.level_matches(slice(None, 3), 2) - True - >>> l.level_matches(slice(1, 3), 1) - True - >>> l.level_matches(slice(2, 3), 1) - False - """ - if isinstance(level, slice): - start, stop = level.start, level.stop - if start is not None and start > consumer_level: - return False - if stop is not None or stop <= consumer_level: - return False - return True - else: - return level >= consumer_level - - @classmethod - def level_for_integer(cls, level): - levels = cls.LEVELS - if level < 0: - return levels[0] - if level >= len(levels): - return levels[-1] - return levels[level] - - def move_stdout_to_stderr(self): - to_remove = [] - to_add = [] - for consumer_level, consumer in self.consumers: - if consumer == sys.stdout: - to_remove.append((consumer_level, consumer)) - to_add.append((consumer_level, sys.stderr)) - for item in to_remove: - self.consumers.remove(item) - self.consumers.extend(to_add) - -logger = Logger() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py deleted file mode 100644 index 444e7252b..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/req.py +++ /dev/null @@ -1,1432 +0,0 @@ -import sys -import os -import shutil -import re -import zipfile -import pkg_resources -import tempfile -import urlparse -import urllib2 -import urllib -import ConfigParser -from distutils.sysconfig import get_python_version -from email.FeedParser import FeedParser -from pip.locations import bin_py, running_under_virtualenv -from pip.exceptions import InstallationError, UninstallationError -from pip.vcs import vcs -from pip.log import logger -from pip.util import display_path, rmtree -from pip.util import ask, backup_dir -from pip.util import is_installable_dir, is_local, dist_is_local -from pip.util import renames, normalize_path, egg_link_path -from pip.util import make_path_relative -from pip import call_subprocess -from pip.backwardcompat import any, copytree -from pip.index import Link -from pip.locations import build_prefix -from pip.download import (get_file_content, is_url, url_to_path, - path_to_url, is_archive_file, - unpack_vcs_link, is_vcs_url, is_file_url, - unpack_file_url, unpack_http_url) - - -PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' - - -class InstallRequirement(object): - - def __init__(self, req, comes_from, source_dir=None, editable=False, - url=None, update=True): - if isinstance(req, basestring): - req = pkg_resources.Requirement.parse(req) - self.req = req - self.comes_from = comes_from - self.source_dir = source_dir - self.editable = editable - self.url = url - self._egg_info_path = None - # This holds the pkg_resources.Distribution object if this requirement - # is already available: - self.satisfied_by = None - # This hold the pkg_resources.Distribution object if this requirement - # conflicts with another installed distribution: - self.conflicts_with = None - self._temp_build_dir = None - self._is_bundle = None - # True if the editable should be updated: - self.update = update - # Set to True after successful installation - self.install_succeeded = None - # UninstallPathSet of uninstalled distribution (for possible rollback) - self.uninstalled = None - - @classmethod - def from_editable(cls, editable_req, comes_from=None, default_vcs=None): - name, url = parse_editable(editable_req, default_vcs) - if url.startswith('file:'): - source_dir = url_to_path(url) - else: - source_dir = None - return cls(name, comes_from, source_dir=source_dir, editable=True, url=url) - - @classmethod - def from_line(cls, name, comes_from=None): - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. - """ - url = None - name = name.strip() - req = name - path = os.path.normpath(os.path.abspath(name)) - - if is_url(name): - url = name - ## FIXME: I think getting the requirement here is a bad idea: - #req = get_requirement_from_url(url) - req = None - elif os.path.isdir(path) and (os.path.sep in name or name.startswith('.')): - if not is_installable_dir(path): - raise InstallationError("Directory %r is not installable. File 'setup.py' not found." - % name) - url = path_to_url(name) - #req = get_requirement_from_url(url) - req = None - elif is_archive_file(path): - if not os.path.isfile(path): - logger.warn('Requirement %r looks like a filename, but the file does not exist' - % name) - url = path_to_url(name) - #req = get_requirement_from_url(url) - req = None - return cls(req, comes_from, url=url) - - def __str__(self): - if self.req: - s = str(self.req) - if self.url: - s += ' from %s' % self.url - else: - s = self.url - if self.satisfied_by is not None: - s += ' in %s' % display_path(self.satisfied_by.location) - if self.comes_from: - if isinstance(self.comes_from, basestring): - comes_from = self.comes_from - else: - comes_from = self.comes_from.from_path() - if comes_from: - s += ' (from %s)' % comes_from - return s - - def from_path(self): - if self.req is None: - return None - s = str(self.req) - if self.comes_from: - if isinstance(self.comes_from, basestring): - comes_from = self.comes_from - else: - comes_from = self.comes_from.from_path() - if comes_from: - s += '->' + comes_from - return s - - def build_location(self, build_dir, unpack=True): - if self._temp_build_dir is not None: - return self._temp_build_dir - if self.req is None: - self._temp_build_dir = tempfile.mkdtemp('-build', 'pip-') - self._ideal_build_dir = build_dir - return self._temp_build_dir - if self.editable: - name = self.name.lower() - else: - name = self.name - # FIXME: Is there a better place to create the build_dir? (hg and bzr need this) - if not os.path.exists(build_dir): - _make_build_dir(build_dir) - return os.path.join(build_dir, name) - - def correct_build_location(self): - """If the build location was a temporary directory, this will move it - to a new more permanent location""" - if self.source_dir is not None: - return - assert self.req is not None - assert self._temp_build_dir - old_location = self._temp_build_dir - new_build_dir = self._ideal_build_dir - del self._ideal_build_dir - if self.editable: - name = self.name.lower() - else: - name = self.name - new_location = os.path.join(new_build_dir, name) - if not os.path.exists(new_build_dir): - logger.debug('Creating directory %s' % new_build_dir) - _make_build_dir(new_build_dir) - if os.path.exists(new_location): - raise InstallationError( - 'A package already exists in %s; please remove it to continue' - % display_path(new_location)) - logger.debug('Moving package %s from %s to new location %s' - % (self, display_path(old_location), display_path(new_location))) - shutil.move(old_location, new_location) - self._temp_build_dir = new_location - self.source_dir = new_location - self._egg_info_path = None - - @property - def name(self): - if self.req is None: - return None - return self.req.project_name - - @property - def url_name(self): - if self.req is None: - return None - return urllib.quote(self.req.unsafe_name) - - @property - def setup_py(self): - return os.path.join(self.source_dir, 'setup.py') - - def run_egg_info(self, force_root_egg_info=False): - assert self.source_dir - if self.name: - logger.notify('Running setup.py egg_info for package %s' % self.name) - else: - logger.notify('Running setup.py egg_info for package from %s' % self.url) - logger.indent += 2 - try: - script = self._run_setup_py - script = script.replace('__SETUP_PY__', repr(self.setup_py)) - script = script.replace('__PKG_NAME__', repr(self.name)) - # We can't put the .egg-info files at the root, because then the source code will be mistaken - # for an installed egg, causing problems - if self.editable or force_root_egg_info: - egg_base_option = [] - else: - egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') - if not os.path.exists(egg_info_dir): - os.makedirs(egg_info_dir) - egg_base_option = ['--egg-base', 'pip-egg-info'] - call_subprocess( - [sys.executable, '-c', script, 'egg_info'] + egg_base_option, - cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False, - command_level=logger.VERBOSE_DEBUG, - command_desc='python setup.py egg_info') - finally: - logger.indent -= 2 - if not self.req: - self.req = pkg_resources.Requirement.parse(self.pkg_info()['Name']) - self.correct_build_location() - - ## FIXME: This is a lame hack, entirely for PasteScript which has - ## a self-provided entry point that causes this awkwardness - _run_setup_py = """ -__file__ = __SETUP_PY__ -from setuptools.command import egg_info -def replacement_run(self): - self.mkpath(self.egg_info) - installer = self.distribution.fetch_build_egg - for ep in egg_info.iter_entry_points('egg_info.writers'): - # require=False is the change we're making: - writer = ep.load(require=False) - if writer: - writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name)) - self.find_sources() -egg_info.egg_info.run = replacement_run -execfile(__file__) -""" - - def egg_info_data(self, filename): - if self.satisfied_by is not None: - if not self.satisfied_by.has_metadata(filename): - return None - return self.satisfied_by.get_metadata(filename) - assert self.source_dir - filename = self.egg_info_path(filename) - if not os.path.exists(filename): - return None - fp = open(filename, 'r') - data = fp.read() - fp.close() - return data - - def egg_info_path(self, filename): - if self._egg_info_path is None: - if self.editable: - base = self.source_dir - else: - base = os.path.join(self.source_dir, 'pip-egg-info') - filenames = os.listdir(base) - if self.editable: - filenames = [] - for root, dirs, files in os.walk(base): - for dir in vcs.dirnames: - if dir in dirs: - dirs.remove(dir) - for dir in dirs: - # Don't search in anything that looks like a virtualenv environment - if (os.path.exists(os.path.join(root, dir, 'bin', 'python')) - or os.path.exists(os.path.join(root, dir, 'Scripts', 'Python.exe'))): - dirs.remove(dir) - # Also don't search through tests - if dir == 'test' or dir == 'tests': - dirs.remove(dir) - filenames.extend([os.path.join(root, dir) - for dir in dirs]) - filenames = [f for f in filenames if f.endswith('.egg-info')] - - if not filenames: - raise InstallationError('No files/directores in %s (from %s)' % (base, filename)) - assert filenames, "No files/directories in %s (from %s)" % (base, filename) - - # if we have more than one match, we pick the toplevel one. This can - # easily be the case if there is a dist folder which contains an - # extracted tarball for testing purposes. - if len(filenames) > 1: - filenames.sort(key=lambda x: x.count(os.path.sep) + - (os.path.altsep and - x.count(os.path.altsep) or 0)) - self._egg_info_path = os.path.join(base, filenames[0]) - return os.path.join(self._egg_info_path, filename) - - def egg_info_lines(self, filename): - data = self.egg_info_data(filename) - if not data: - return [] - result = [] - for line in data.splitlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - result.append(line) - return result - - def pkg_info(self): - p = FeedParser() - data = self.egg_info_data('PKG-INFO') - if not data: - logger.warn('No PKG-INFO file found in %s' % display_path(self.egg_info_path('PKG-INFO'))) - p.feed(data or '') - return p.close() - - @property - def dependency_links(self): - return self.egg_info_lines('dependency_links.txt') - - _requirements_section_re = re.compile(r'\[(.*?)\]') - - def requirements(self, extras=()): - in_extra = None - for line in self.egg_info_lines('requires.txt'): - match = self._requirements_section_re.match(line) - if match: - in_extra = match.group(1) - continue - if in_extra and in_extra not in extras: - # Skip requirement for an extra we aren't requiring - continue - yield line - - @property - def absolute_versions(self): - for qualifier, version in self.req.specs: - if qualifier == '==': - yield version - - @property - def installed_version(self): - return self.pkg_info()['version'] - - def assert_source_matches_version(self): - assert self.source_dir - if self.comes_from is None: - # We don't check the versions of things explicitly installed. - # This makes, e.g., "pip Package==dev" possible - return - version = self.installed_version - if version not in self.req: - logger.fatal( - 'Source in %s has the version %s, which does not match the requirement %s' - % (display_path(self.source_dir), version, self)) - raise InstallationError( - 'Source in %s has version %s that conflicts with %s' - % (display_path(self.source_dir), version, self)) - else: - logger.debug('Source in %s has version %s, which satisfies requirement %s' - % (display_path(self.source_dir), version, self)) - - def update_editable(self, obtain=True): - if not self.url: - logger.info("Cannot update repository at %s; repository location is unknown" % self.source_dir) - return - assert self.editable - assert self.source_dir - if self.url.startswith('file:'): - # Static paths don't get updated - return - assert '+' in self.url, "bad url: %r" % self.url - if not self.update: - return - vc_type, url = self.url.split('+', 1) - backend = vcs.get_backend(vc_type) - if backend: - vcs_backend = backend(self.url) - if obtain: - vcs_backend.obtain(self.source_dir) - else: - vcs_backend.export(self.source_dir) - else: - assert 0, ( - 'Unexpected version control type (in %s): %s' - % (self.url, vc_type)) - - def uninstall(self, auto_confirm=False): - """ - Uninstall the distribution currently satisfying this requirement. - - Prompts before removing or modifying files unless - ``auto_confirm`` is True. - - Refuses to delete or modify files outside of ``sys.prefix`` - - thus uninstallation within a virtual environment can only - modify that virtual environment, even if the virtualenv is - linked to global site-packages. - - """ - if not self.check_if_exists(): - raise UninstallationError("Cannot uninstall requirement %s, not installed" % (self.name,)) - dist = self.satisfied_by or self.conflicts_with - - paths_to_remove = UninstallPathSet(dist) - - pip_egg_info_path = os.path.join(dist.location, - dist.egg_name()) + '.egg-info' - easy_install_egg = dist.egg_name() + '.egg' - develop_egg_link = egg_link_path(dist) - if os.path.exists(pip_egg_info_path): - # package installed by pip - paths_to_remove.add(pip_egg_info_path) - if dist.has_metadata('installed-files.txt'): - for installed_file in dist.get_metadata('installed-files.txt').splitlines(): - path = os.path.normpath(os.path.join(pip_egg_info_path, installed_file)) - paths_to_remove.add(path) - if dist.has_metadata('top_level.txt'): - if dist.has_metadata('namespace_packages.txt'): - namespaces = dist.get_metadata('namespace_packages.txt') - else: - namespaces = [] - for top_level_pkg in [p for p - in dist.get_metadata('top_level.txt').splitlines() - if p and p not in namespaces]: - path = os.path.join(dist.location, top_level_pkg) - paths_to_remove.add(path) - paths_to_remove.add(path + '.py') - paths_to_remove.add(path + '.pyc') - - elif dist.location.endswith(easy_install_egg): - # package installed by easy_install - paths_to_remove.add(dist.location) - easy_install_pth = os.path.join(os.path.dirname(dist.location), - 'easy-install.pth') - paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) - - elif os.path.isfile(develop_egg_link): - # develop egg - fh = open(develop_egg_link, 'r') - link_pointer = os.path.normcase(fh.readline().strip()) - fh.close() - assert (link_pointer == dist.location), 'Egg-link %s does not match installed location of %s (at %s)' % (link_pointer, self.name, dist.location) - paths_to_remove.add(develop_egg_link) - easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), - 'easy-install.pth') - paths_to_remove.add_pth(easy_install_pth, dist.location) - - # find distutils scripts= scripts - if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): - for script in dist.metadata_listdir('scripts'): - paths_to_remove.add(os.path.join(bin_py, script)) - if sys.platform == 'win32': - paths_to_remove.add(os.path.join(bin_py, script) + '.bat') - - # find console_scripts - if dist.has_metadata('entry_points.txt'): - config = ConfigParser.SafeConfigParser() - config.readfp(FakeFile(dist.get_metadata_lines('entry_points.txt'))) - if config.has_section('console_scripts'): - for name, value in config.items('console_scripts'): - paths_to_remove.add(os.path.join(bin_py, name)) - if sys.platform == 'win32': - paths_to_remove.add(os.path.join(bin_py, name) + '.exe') - paths_to_remove.add(os.path.join(bin_py, name) + '.exe.manifest') - paths_to_remove.add(os.path.join(bin_py, name) + '-script.py') - - paths_to_remove.remove(auto_confirm) - self.uninstalled = paths_to_remove - - def rollback_uninstall(self): - if self.uninstalled: - self.uninstalled.rollback() - else: - logger.error("Can't rollback %s, nothing uninstalled." - % (self.project_name,)) - - def commit_uninstall(self): - if self.uninstalled: - self.uninstalled.commit() - else: - logger.error("Can't commit %s, nothing uninstalled." - % (self.project_name,)) - - def archive(self, build_dir): - assert self.source_dir - create_archive = True - archive_name = '%s-%s.zip' % (self.name, self.installed_version) - archive_path = os.path.join(build_dir, archive_name) - if os.path.exists(archive_path): - response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' - % display_path(archive_path), ('i', 'w', 'b')) - if response == 'i': - create_archive = False - elif response == 'w': - logger.warn('Deleting %s' % display_path(archive_path)) - os.remove(archive_path) - elif response == 'b': - dest_file = backup_dir(archive_path) - logger.warn('Backing up %s to %s' - % (display_path(archive_path), display_path(dest_file))) - shutil.move(archive_path, dest_file) - if create_archive: - zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) - dir = os.path.normcase(os.path.abspath(self.source_dir)) - for dirpath, dirnames, filenames in os.walk(dir): - if 'pip-egg-info' in dirnames: - dirnames.remove('pip-egg-info') - for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') - zipdir.external_attr = 0755 << 16L - zip.writestr(zipdir, '') - for filename in filenames: - if filename == PIP_DELETE_MARKER_FILENAME: - continue - filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, self.name + '/' + name) - zip.close() - logger.indent -= 2 - logger.notify('Saved %s' % display_path(archive_path)) - - def _clean_zip_name(self, name, prefix): - assert name.startswith(prefix+os.path.sep), ( - "name %r doesn't start with prefix %r" % (name, prefix)) - name = name[len(prefix)+1:] - name = name.replace(os.path.sep, '/') - return name - - def install(self, install_options, global_options=()): - if self.editable: - self.install_editable(install_options, global_options) - return - temp_location = tempfile.mkdtemp('-record', 'pip-') - record_filename = os.path.join(temp_location, 'install-record.txt') - try: - - install_args = [ - sys.executable, '-c', - "import setuptools;__file__=%r;"\ - "execfile(__file__)" % self.setup_py] +\ - list(global_options) + [ - 'install', - '--single-version-externally-managed', - '--record', record_filename] - - if running_under_virtualenv(): - ## FIXME: I'm not sure if this is a reasonable location; probably not - ## but we can't put it in the default location, as that is a virtualenv symlink that isn't writable - install_args += ['--install-headers', - os.path.join(sys.prefix, 'include', 'site', - 'python' + get_python_version())] - logger.notify('Running setup.py install for %s' % self.name) - logger.indent += 2 - try: - call_subprocess(install_args + install_options, - cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False) - finally: - logger.indent -= 2 - if not os.path.exists(record_filename): - logger.notify('Record file %s not found' % record_filename) - return - self.install_succeeded = True - f = open(record_filename) - for line in f: - line = line.strip() - if line.endswith('.egg-info'): - egg_info_dir = line - break - else: - logger.warn('Could not find .egg-info directory in install record for %s' % self) - ## FIXME: put the record somewhere - ## FIXME: should this be an error? - return - f.close() - new_lines = [] - f = open(record_filename) - for line in f: - filename = line.strip() - if os.path.isdir(filename): - filename += os.path.sep - new_lines.append(make_path_relative(filename, egg_info_dir)) - f.close() - f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') - f.write('\n'.join(new_lines)+'\n') - f.close() - finally: - if os.path.exists(record_filename): - os.remove(record_filename) - os.rmdir(temp_location) - - def remove_temporary_source(self): - """Remove the source files from this requirement, if they are marked - for deletion""" - if self.is_bundle or os.path.exists(self.delete_marker_filename): - logger.info('Removing source in %s' % self.source_dir) - if self.source_dir: - rmtree(self.source_dir) - self.source_dir = None - if self._temp_build_dir and os.path.exists(self._temp_build_dir): - rmtree(self._temp_build_dir) - self._temp_build_dir = None - - def install_editable(self, install_options, global_options=()): - logger.notify('Running setup.py develop for %s' % self.name) - logger.indent += 2 - try: - ## FIXME: should we do --install-headers here too? - call_subprocess( - [sys.executable, '-c', - "import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py)] - + list(global_options) + ['develop', '--no-deps'] + list(install_options), - - cwd=self.source_dir, filter_stdout=self._filter_install, - show_stdout=False) - finally: - logger.indent -= 2 - self.install_succeeded = True - - def _filter_install(self, line): - level = logger.NOTIFY - for regex in [r'^running .*', r'^writing .*', '^creating .*', '^[Cc]opying .*', - r'^reading .*', r"^removing .*\.egg-info' \(and everything under it\)$", - r'^byte-compiling ', - # Not sure what this warning is, but it seems harmless: - r"^warning: manifest_maker: standard file '-c' not found$"]: - if re.search(regex, line.strip()): - level = logger.INFO - break - return (level, line) - - def check_if_exists(self): - """Find an installed distribution that satisfies or conflicts - with this requirement, and set self.satisfied_by or - self.conflicts_with appropriately.""" - if self.req is None: - return False - try: - self.satisfied_by = pkg_resources.get_distribution(self.req) - except pkg_resources.DistributionNotFound: - return False - except pkg_resources.VersionConflict: - self.conflicts_with = pkg_resources.get_distribution(self.req.project_name) - return True - - @property - def is_bundle(self): - if self._is_bundle is not None: - return self._is_bundle - base = self._temp_build_dir - if not base: - ## FIXME: this doesn't seem right: - return False - self._is_bundle = (os.path.exists(os.path.join(base, 'pip-manifest.txt')) - or os.path.exists(os.path.join(base, 'pyinstall-manifest.txt'))) - return self._is_bundle - - def bundle_requirements(self): - for dest_dir in self._bundle_editable_dirs: - package = os.path.basename(dest_dir) - ## FIXME: svnism: - for vcs_backend in vcs.backends: - url = rev = None - vcs_bundle_file = os.path.join( - dest_dir, vcs_backend.bundle_file) - if os.path.exists(vcs_bundle_file): - vc_type = vcs_backend.name - fp = open(vcs_bundle_file) - content = fp.read() - fp.close() - url, rev = vcs_backend().parse_vcs_bundle_file(content) - break - if url: - url = '%s+%s@%s' % (vc_type, url, rev) - else: - url = None - yield InstallRequirement( - package, self, editable=True, url=url, - update=False, source_dir=dest_dir) - for dest_dir in self._bundle_build_dirs: - package = os.path.basename(dest_dir) - yield InstallRequirement( - package, self, - source_dir=dest_dir) - - def move_bundle_files(self, dest_build_dir, dest_src_dir): - base = self._temp_build_dir - assert base - src_dir = os.path.join(base, 'src') - build_dir = os.path.join(base, 'build') - bundle_build_dirs = [] - bundle_editable_dirs = [] - for source_dir, dest_dir, dir_collection in [ - (src_dir, dest_src_dir, bundle_editable_dirs), - (build_dir, dest_build_dir, bundle_build_dirs)]: - if os.path.exists(source_dir): - for dirname in os.listdir(source_dir): - dest = os.path.join(dest_dir, dirname) - dir_collection.append(dest) - if os.path.exists(dest): - logger.warn('The directory %s (containing package %s) already exists; cannot move source from bundle %s' - % (dest, dirname, self)) - continue - if not os.path.exists(dest_dir): - logger.info('Creating directory %s' % dest_dir) - os.makedirs(dest_dir) - shutil.move(os.path.join(source_dir, dirname), dest) - if not os.listdir(source_dir): - os.rmdir(source_dir) - self._temp_build_dir = None - self._bundle_build_dirs = bundle_build_dirs - self._bundle_editable_dirs = bundle_editable_dirs - - @property - def delete_marker_filename(self): - assert self.source_dir - return os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME) - - -DELETE_MARKER_MESSAGE = '''\ -This file is placed here by pip to indicate the source was put -here by pip. - -Once this package is successfully installed this source code will be -deleted (unless you remove this file). -''' - - -class RequirementSet(object): - - def __init__(self, build_dir, src_dir, download_dir, download_cache=None, - upgrade=False, ignore_installed=False, - ignore_dependencies=False): - self.build_dir = build_dir - self.src_dir = src_dir - self.download_dir = download_dir - self.download_cache = download_cache - self.upgrade = upgrade - self.ignore_installed = ignore_installed - self.requirements = {} - # Mapping of alias: real_name - self.requirement_aliases = {} - self.unnamed_requirements = [] - self.ignore_dependencies = ignore_dependencies - self.successfully_downloaded = [] - self.successfully_installed = [] - self.reqs_to_cleanup = [] - - def __str__(self): - reqs = [req for req in self.requirements.values() - if not req.comes_from] - reqs.sort(key=lambda req: req.name.lower()) - return ' '.join([str(req.req) for req in reqs]) - - def add_requirement(self, install_req): - name = install_req.name - if not name: - self.unnamed_requirements.append(install_req) - else: - if self.has_requirement(name): - raise InstallationError( - 'Double requirement given: %s (aready in %s, name=%r)' - % (install_req, self.get_requirement(name), name)) - self.requirements[name] = install_req - ## FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name - - def has_requirement(self, project_name): - for name in project_name, project_name.lower(): - if name in self.requirements or name in self.requirement_aliases: - return True - return False - - @property - def has_requirements(self): - return self.requirements.values() or self.unnamed_requirements - - @property - def has_editables(self): - if any(req.editable for req in self.requirements.values()): - return True - if any(req.editable for req in self.unnamed_requirements): - return True - return False - - @property - def is_download(self): - if self.download_dir: - self.download_dir = os.path.expanduser(self.download_dir) - if os.path.exists(self.download_dir): - return True - else: - logger.fatal('Could not find download directory') - raise InstallationError( - "Could not find or access download directory '%s'" - % display_path(self.download_dir)) - return False - - def get_requirement(self, project_name): - for name in project_name, project_name.lower(): - if name in self.requirements: - return self.requirements[name] - if name in self.requirement_aliases: - return self.requirements[self.requirement_aliases[name]] - raise KeyError("No project with the name %r" % project_name) - - def uninstall(self, auto_confirm=False): - for req in self.requirements.values(): - req.uninstall(auto_confirm=auto_confirm) - req.commit_uninstall() - - def locate_files(self): - ## FIXME: duplicates code from install_files; relevant code should - ## probably be factored out into a separate method - unnamed = list(self.unnamed_requirements) - reqs = self.requirements.values() - while reqs or unnamed: - if unnamed: - req_to_install = unnamed.pop(0) - else: - req_to_install = reqs.pop(0) - install_needed = True - if not self.ignore_installed and not req_to_install.editable: - req_to_install.check_if_exists() - if req_to_install.satisfied_by: - if self.upgrade: - req_to_install.conflicts_with = req_to_install.satisfied_by - req_to_install.satisfied_by = None - else: - install_needed = False - if req_to_install.satisfied_by: - logger.notify('Requirement already satisfied ' - '(use --upgrade to upgrade): %s' - % req_to_install) - - if req_to_install.editable: - if req_to_install.source_dir is None: - req_to_install.source_dir = req_to_install.build_location(self.src_dir) - elif install_needed: - req_to_install.source_dir = req_to_install.build_location(self.build_dir, not self.is_download) - - if req_to_install.source_dir is not None and not os.path.isdir(req_to_install.source_dir): - raise InstallationError('Could not install requirement %s ' - 'because source folder %s does not exist ' - '(perhaps --no-download was used without first running ' - 'an equivalent install with --no-install?)' - % (req_to_install, req_to_install.source_dir)) - - def prepare_files(self, finder, force_root_egg_info=False, bundle=False): - """Prepare process. Create temp directories, download and/or unpack files.""" - unnamed = list(self.unnamed_requirements) - reqs = self.requirements.values() - while reqs or unnamed: - if unnamed: - req_to_install = unnamed.pop(0) - else: - req_to_install = reqs.pop(0) - install = True - if not self.ignore_installed and not req_to_install.editable: - req_to_install.check_if_exists() - if req_to_install.satisfied_by: - if self.upgrade: - req_to_install.conflicts_with = req_to_install.satisfied_by - req_to_install.satisfied_by = None - else: - install = False - if req_to_install.satisfied_by: - logger.notify('Requirement already satisfied ' - '(use --upgrade to upgrade): %s' - % req_to_install) - if req_to_install.editable: - logger.notify('Obtaining %s' % req_to_install) - elif install: - if req_to_install.url and req_to_install.url.lower().startswith('file:'): - logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) - else: - logger.notify('Downloading/unpacking %s' % req_to_install) - logger.indent += 2 - try: - is_bundle = False - if req_to_install.editable: - if req_to_install.source_dir is None: - location = req_to_install.build_location(self.src_dir) - req_to_install.source_dir = location - else: - location = req_to_install.source_dir - if not os.path.exists(self.build_dir): - _make_build_dir(self.build_dir) - req_to_install.update_editable(not self.is_download) - if self.is_download: - req_to_install.run_egg_info() - req_to_install.archive(self.download_dir) - else: - req_to_install.run_egg_info() - elif install: - ##@@ if filesystem packages are not marked - ##editable in a req, a non deterministic error - ##occurs when the script attempts to unpack the - ##build directory - - location = req_to_install.build_location(self.build_dir, not self.is_download) - ## FIXME: is the existance of the checkout good enough to use it? I don't think so. - unpack = True - if not os.path.exists(os.path.join(location, 'setup.py')): - ## FIXME: this won't upgrade when there's an existing package unpacked in `location` - if req_to_install.url is None: - url = finder.find_requirement(req_to_install, upgrade=self.upgrade) - else: - ## FIXME: should req_to_install.url already be a link? - url = Link(req_to_install.url) - assert url - if url: - try: - self.unpack_url(url, location, self.is_download) - except urllib2.HTTPError, e: - logger.fatal('Could not install requirement %s because of error %s' - % (req_to_install, e)) - raise InstallationError( - 'Could not install requirement %s because of HTTP error %s for URL %s' - % (req_to_install, e, url)) - else: - unpack = False - if unpack: - is_bundle = req_to_install.is_bundle - url = None - if is_bundle: - req_to_install.move_bundle_files(self.build_dir, self.src_dir) - for subreq in req_to_install.bundle_requirements(): - reqs.append(subreq) - self.add_requirement(subreq) - elif self.is_download: - req_to_install.source_dir = location - if url and url.scheme in vcs.all_schemes: - req_to_install.run_egg_info() - req_to_install.archive(self.download_dir) - else: - req_to_install.source_dir = location - req_to_install.run_egg_info() - if force_root_egg_info: - # We need to run this to make sure that the .egg-info/ - # directory is created for packing in the bundle - req_to_install.run_egg_info(force_root_egg_info=True) - req_to_install.assert_source_matches_version() - #@@ sketchy way of identifying packages not grabbed from an index - if bundle and req_to_install.url: - self.copy_to_build_dir(req_to_install) - if not is_bundle and not self.is_download: - ## FIXME: shouldn't be globally added: - finder.add_dependency_links(req_to_install.dependency_links) - ## FIXME: add extras in here: - if not self.ignore_dependencies: - for req in req_to_install.requirements(): - try: - name = pkg_resources.Requirement.parse(req).project_name - except ValueError, e: - ## FIXME: proper warning - logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) - continue - if self.has_requirement(name): - ## FIXME: check for conflict - continue - subreq = InstallRequirement(req, req_to_install) - reqs.append(subreq) - self.add_requirement(subreq) - if req_to_install.name not in self.requirements: - self.requirements[req_to_install.name] = req_to_install - else: - self.reqs_to_cleanup.append(req_to_install) - if install: - self.successfully_downloaded.append(req_to_install) - if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): - self.copy_to_build_dir(req_to_install) - finally: - logger.indent -= 2 - - def cleanup_files(self, bundle=False): - """Clean up files, remove builds.""" - logger.notify('Cleaning up...') - logger.indent += 2 - for req in self.reqs_to_cleanup: - req.remove_temporary_source() - - remove_dir = [] - if self._pip_has_created_build_dir(): - remove_dir.append(self.build_dir) - - # The source dir of a bundle can always be removed. - if bundle: - remove_dir.append(self.src_dir) - - for dir in remove_dir: - if os.path.exists(dir): - logger.info('Removing temporary dir %s...' % dir) - rmtree(dir) - - logger.indent -= 2 - - def _pip_has_created_build_dir(self): - return (self.build_dir == build_prefix and - os.path.exists(os.path.join(self.build_dir, PIP_DELETE_MARKER_FILENAME))) - - def copy_to_build_dir(self, req_to_install): - target_dir = req_to_install.editable and self.src_dir or self.build_dir - logger.info("Copying %s to %s" %(req_to_install.name, target_dir)) - dest = os.path.join(target_dir, req_to_install.name) - copytree(req_to_install.source_dir, dest) - call_subprocess(["python", "%s/setup.py"%dest, "clean"]) - - def unpack_url(self, link, location, only_download=False): - if only_download: - location = self.download_dir - if is_vcs_url(link): - return unpack_vcs_link(link, location, only_download) - elif is_file_url(link): - return unpack_file_url(link, location) - else: - if self.download_cache: - self.download_cache = os.path.expanduser(self.download_cache) - return unpack_http_url(link, location, self.download_cache, only_download) - - def install(self, install_options, global_options=()): - """Install everything in this set (after having downloaded and unpacked the packages)""" - to_install = sorted([r for r in self.requirements.values() - if self.upgrade or not r.satisfied_by], - key=lambda p: p.name.lower()) - if to_install: - logger.notify('Installing collected packages: %s' % (', '.join([req.name for req in to_install]))) - logger.indent += 2 - try: - for requirement in to_install: - if requirement.conflicts_with: - logger.notify('Found existing installation: %s' - % requirement.conflicts_with) - logger.indent += 2 - try: - requirement.uninstall(auto_confirm=True) - finally: - logger.indent -= 2 - try: - requirement.install(install_options, global_options) - except: - # if install did not succeed, rollback previous uninstall - if requirement.conflicts_with and not requirement.install_succeeded: - requirement.rollback_uninstall() - raise - else: - if requirement.conflicts_with and requirement.install_succeeded: - requirement.commit_uninstall() - requirement.remove_temporary_source() - finally: - logger.indent -= 2 - self.successfully_installed = to_install - - def create_bundle(self, bundle_filename): - ## FIXME: can't decide which is better; zip is easier to read - ## random files from, but tar.bz2 is smaller and not as lame a - ## format. - - ## FIXME: this file should really include a manifest of the - ## packages, maybe some other metadata files. It would make - ## it easier to detect as well. - zip = zipfile.ZipFile(bundle_filename, 'w', zipfile.ZIP_DEFLATED) - vcs_dirs = [] - for dir, basename in (self.build_dir, 'build'), (self.src_dir, 'src'): - dir = os.path.normcase(os.path.abspath(dir)) - for dirpath, dirnames, filenames in os.walk(dir): - for backend in vcs.backends: - vcs_backend = backend() - vcs_url = vcs_rev = None - if vcs_backend.dirname in dirnames: - for vcs_dir in vcs_dirs: - if dirpath.startswith(vcs_dir): - # vcs bundle file already in parent directory - break - else: - vcs_url, vcs_rev = vcs_backend.get_info( - os.path.join(dir, dirpath)) - vcs_dirs.append(dirpath) - vcs_bundle_file = vcs_backend.bundle_file - vcs_guide = vcs_backend.guide % {'url': vcs_url, - 'rev': vcs_rev} - dirnames.remove(vcs_backend.dirname) - break - if 'pip-egg-info' in dirnames: - dirnames.remove('pip-egg-info') - for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zip.writestr(basename + '/' + name + '/', '') - for filename in filenames: - if filename == PIP_DELETE_MARKER_FILENAME: - continue - filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, basename + '/' + name) - if vcs_url: - name = os.path.join(dirpath, vcs_bundle_file) - name = self._clean_zip_name(name, dir) - zip.writestr(basename + '/' + name, vcs_guide) - - zip.writestr('pip-manifest.txt', self.bundle_requirements()) - zip.close() - - BUNDLE_HEADER = '''\ -# This is a pip bundle file, that contains many source packages -# that can be installed as a group. You can install this like: -# pip this_file.zip -# The rest of the file contains a list of all the packages included: -''' - - def bundle_requirements(self): - parts = [self.BUNDLE_HEADER] - for req in sorted( - [req for req in self.requirements.values() - if not req.comes_from], - key=lambda x: x.name): - parts.append('%s==%s\n' % (req.name, req.installed_version)) - parts.append('# These packages were installed to satisfy the above requirements:\n') - for req in sorted( - [req for req in self.requirements.values() - if req.comes_from], - key=lambda x: x.name): - parts.append('%s==%s\n' % (req.name, req.installed_version)) - ## FIXME: should we do something with self.unnamed_requirements? - return ''.join(parts) - - def _clean_zip_name(self, name, prefix): - assert name.startswith(prefix+os.path.sep), ( - "name %r doesn't start with prefix %r" % (name, prefix)) - name = name[len(prefix)+1:] - name = name.replace(os.path.sep, '/') - return name - - -def _make_build_dir(build_dir): - os.makedirs(build_dir) - _write_delete_marker_message(os.path.join(build_dir, PIP_DELETE_MARKER_FILENAME)) - - -def _write_delete_marker_message(filepath): - marker_fp = open(filepath, 'w') - marker_fp.write(DELETE_MARKER_MESSAGE) - marker_fp.close() - - -_scheme_re = re.compile(r'^(http|https|file):', re.I) - - -def parse_requirements(filename, finder=None, comes_from=None, options=None): - skip_match = None - skip_regex = options.skip_requirements_regex - if skip_regex: - skip_match = re.compile(skip_regex) - filename, content = get_file_content(filename, comes_from=comes_from) - for line_number, line in enumerate(content.splitlines()): - line_number += 1 - line = line.strip() - if not line or line.startswith('#'): - continue - if skip_match and skip_match.search(line): - continue - if line.startswith('-r') or line.startswith('--requirement'): - if line.startswith('-r'): - req_url = line[2:].strip() - else: - req_url = line[len('--requirement'):].strip().strip('=') - if _scheme_re.search(filename): - # Relative to a URL - req_url = urlparse.urljoin(req_url, filename) - elif not _scheme_re.search(req_url): - req_url = os.path.join(os.path.dirname(filename), req_url) - for item in parse_requirements(req_url, finder, comes_from=filename, options=options): - yield item - elif line.startswith('-Z') or line.startswith('--always-unzip'): - # No longer used, but previously these were used in - # requirement files, so we'll ignore. - pass - elif line.startswith('-f') or line.startswith('--find-links'): - if line.startswith('-f'): - line = line[2:].strip() - else: - line = line[len('--find-links'):].strip().lstrip('=') - ## FIXME: it would be nice to keep track of the source of - ## the find_links: - if finder: - finder.find_links.append(line) - elif line.startswith('-i') or line.startswith('--index-url'): - if line.startswith('-i'): - line = line[2:].strip() - else: - line = line[len('--index-url'):].strip().lstrip('=') - if finder: - finder.index_urls = [line] - elif line.startswith('--extra-index-url'): - line = line[len('--extra-index-url'):].strip().lstrip('=') - if finder: - finder.index_urls.append(line) - else: - comes_from = '-r %s (line %s)' % (filename, line_number) - if line.startswith('-e') or line.startswith('--editable'): - if line.startswith('-e'): - line = line[2:].strip() - else: - line = line[len('--editable'):].strip() - req = InstallRequirement.from_editable( - line, comes_from=comes_from, default_vcs=options.default_vcs) - else: - req = InstallRequirement.from_line(line, comes_from) - yield req - - -def parse_editable(editable_req, default_vcs=None): - """Parses svn+http://blahblah@rev#egg=Foobar into a requirement - (Foobar) and a URL""" - url = editable_req - if os.path.isdir(url) and os.path.exists(os.path.join(url, 'setup.py')): - # Treating it as code that has already been checked out - url = path_to_url(url) - if url.lower().startswith('file:'): - return None, url - for version_control in vcs: - if url.lower().startswith('%s:' % version_control): - url = '%s+%s' % (version_control, url) - if '+' not in url: - if default_vcs: - url = default_vcs + '+' + url - else: - raise InstallationError( - '--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req) - vc_type = url.split('+', 1)[0].lower() - if not vcs.get_backend(vc_type): - raise InstallationError( - 'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req) - match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req) - if (not match or not match.group(1)) and vcs.get_backend(vc_type): - parts = [p for p in editable_req.split('#', 1)[0].split('/') if p] - if parts[-2] in ('tags', 'branches', 'tag', 'branch'): - req = parts[-3] - elif parts[-1] == 'trunk': - req = parts[-2] - else: - raise InstallationError( - '--editable=%s is not the right format; it must have #egg=Package' - % editable_req) - else: - req = match.group(1) - ## FIXME: use package_to_requirement? - match = re.search(r'^(.*?)(?:-dev|-\d.*)', req) - if match: - # Strip off -dev, -0.2, etc. - req = match.group(1) - return req, url - - -class UninstallPathSet(object): - """A set of file paths to be removed in the uninstallation of a - requirement.""" - def __init__(self, dist): - self.paths = set() - self._refuse = set() - self.pth = {} - self.dist = dist - self.save_dir = None - self._moved_paths = [] - - def _permitted(self, path): - """ - Return True if the given path is one we are permitted to - remove/modify, False otherwise. - - """ - return is_local(path) - - def _can_uninstall(self): - if not dist_is_local(self.dist): - logger.notify("Not uninstalling %s at %s, outside environment %s" - % (self.dist.project_name, normalize_path(self.dist.location), sys.prefix)) - return False - return True - - def add(self, path): - path = normalize_path(path) - if not os.path.exists(path): - return - if self._permitted(path): - self.paths.add(path) - else: - self._refuse.add(path) - - def add_pth(self, pth_file, entry): - pth_file = normalize_path(pth_file) - if self._permitted(pth_file): - if pth_file not in self.pth: - self.pth[pth_file] = UninstallPthEntries(pth_file) - self.pth[pth_file].add(entry) - else: - self._refuse.add(pth_file) - - def compact(self, paths): - """Compact a path set to contain the minimal number of paths - necessary to contain all paths in the set. If /a/path/ and - /a/path/to/a/file.txt are both in the set, leave only the - shorter path.""" - short_paths = set() - for path in sorted(paths, key=len): - if not any([(path.startswith(shortpath) and - path[len(shortpath.rstrip(os.path.sep))] == os.path.sep) - for shortpath in short_paths]): - short_paths.add(path) - return short_paths - - def _stash(self, path): - return os.path.join( - self.save_dir, os.path.splitdrive(path)[1].lstrip(os.path.sep)) - - def remove(self, auto_confirm=False): - """Remove paths in ``self.paths`` with confirmation (unless - ``auto_confirm`` is True).""" - if not self._can_uninstall(): - return - logger.notify('Uninstalling %s:' % self.dist.project_name) - logger.indent += 2 - paths = sorted(self.compact(self.paths)) - try: - if auto_confirm: - response = 'y' - else: - for path in paths: - logger.notify(path) - response = ask('Proceed (y/n)? ', ('y', 'n')) - if self._refuse: - logger.notify('Not removing or modifying (outside of prefix):') - for path in self.compact(self._refuse): - logger.notify(path) - if response == 'y': - self.save_dir = tempfile.mkdtemp(suffix='-uninstall', - prefix='pip-') - for path in paths: - new_path = self._stash(path) - logger.info('Removing file or directory %s' % path) - self._moved_paths.append(path) - renames(path, new_path) - for pth in self.pth.values(): - pth.remove() - logger.notify('Successfully uninstalled %s' % self.dist.project_name) - - finally: - logger.indent -= 2 - - def rollback(self): - """Rollback the changes previously made by remove().""" - if self.save_dir is None: - logger.error("Can't roll back %s; was not uninstalled" % self.dist.project_name) - return False - logger.notify('Rolling back uninstall of %s' % self.dist.project_name) - for path in self._moved_paths: - tmp_path = self._stash(path) - logger.info('Replacing %s' % path) - renames(tmp_path, path) - for pth in self.pth: - pth.rollback() - - def commit(self): - """Remove temporary save dir: rollback will no longer be possible.""" - if self.save_dir is not None: - shutil.rmtree(self.save_dir) - self.save_dir = None - self._moved_paths = [] - - -class UninstallPthEntries(object): - def __init__(self, pth_file): - if not os.path.isfile(pth_file): - raise UninstallationError("Cannot remove entries from nonexistent file %s" % pth_file) - self.file = pth_file - self.entries = set() - self._saved_lines = None - - def add(self, entry): - entry = os.path.normcase(entry) - # On Windows, os.path.normcase converts the entry to use - # backslashes. This is correct for entries that describe absolute - # paths outside of site-packages, but all the others use forward - # slashes. - if sys.platform == 'win32' and not os.path.splitdrive(entry)[0]: - entry = entry.replace('\\', '/') - self.entries.add(entry) - - def remove(self): - logger.info('Removing pth entries from %s:' % self.file) - fh = open(self.file, 'r') - lines = fh.readlines() - self._saved_lines = lines - fh.close() - try: - for entry in self.entries: - logger.info('Removing entry: %s' % entry) - try: - lines.remove(entry + '\n') - except ValueError: - pass - finally: - pass - fh = open(self.file, 'wb') - fh.writelines(lines) - fh.close() - - def rollback(self): - if self._saved_lines is None: - logger.error('Cannot roll back changes to %s, none were made' % self.file) - return False - logger.info('Rolling %s back to previous state' % self.file) - fh = open(self.file, 'wb') - fh.writelines(self._saved_lines) - fh.close() - return True - - -class FakeFile(object): - """Wrap a list of lines in an object with readline() to make - ConfigParser happy.""" - def __init__(self, lines): - self._gen = (l for l in lines) - - def readline(self): - try: - return self._gen.next() - except StopIteration: - return '' diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py deleted file mode 100644 index be830ad9a..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/runner.py +++ /dev/null @@ -1,18 +0,0 @@ -import sys -import os - - -def run(): - base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ## FIXME: this is kind of crude; if we could create a fake pip - ## module, then exec into it and update pip.__path__ properly, we - ## wouldn't have to update sys.path: - sys.path.insert(0, base) - import pip - return pip.main() - - -if __name__ == '__main__': - exit = run() - if exit: - sys.exit(exit) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py deleted file mode 100644 index 1eab34c06..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/util.py +++ /dev/null @@ -1,479 +0,0 @@ -import sys -import shutil -import os -import stat -import re -import posixpath -import pkg_resources -import zipfile -import tarfile -from pip.exceptions import InstallationError -from pip.backwardcompat import WindowsError -from pip.locations import site_packages, running_under_virtualenv -from pip.log import logger - -__all__ = ['rmtree', 'display_path', 'backup_dir', - 'find_command', 'ask', 'Inf', - 'normalize_name', 'splitext', - 'format_size', 'is_installable_dir', - 'is_svn_page', 'file_contents', - 'split_leading_dir', 'has_leading_dir', - 'make_path_relative', 'normalize_path', - 'renames', 'get_terminal_size', - 'unzip_file', 'untar_file', 'create_download_cache_folder', - 'cache_download', 'unpack_file'] - - -def rmtree(dir): - shutil.rmtree(dir, ignore_errors=True, - onerror=rmtree_errorhandler) - - -def rmtree_errorhandler(func, path, exc_info): - """On Windows, the files in .svn are read-only, so when rmtree() tries to - remove them, an exception is thrown. We catch that here, remove the - read-only attribute, and hopefully continue without problems.""" - exctype, value = exc_info[:2] - # lookin for a windows error - if exctype is not WindowsError or 'Access is denied' not in str(value): - raise - # file type should currently be read only - if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD): - raise - # convert to read/write - os.chmod(path, stat.S_IWRITE) - # use the original function to repeat the operation - func(path) - - -def display_path(path): - """Gives the display value for a given path, making it relative to cwd - if possible.""" - path = os.path.normcase(os.path.abspath(path)) - if path.startswith(os.getcwd() + os.path.sep): - path = '.' + path[len(os.getcwd()):] - return path - - -def backup_dir(dir, ext='.bak'): - """Figure out the name of a directory to back up the given dir to - (adding .bak, .bak2, etc)""" - n = 1 - extension = ext - while os.path.exists(dir + extension): - n += 1 - extension = ext + str(n) - return dir + extension - - -def find_command(cmd, paths=None, pathext=None): - """Searches the PATH for the given command and returns its path""" - if paths is None: - paths = os.environ.get('PATH', []).split(os.pathsep) - if isinstance(paths, basestring): - paths = [paths] - # check if there are funny path extensions for executables, e.g. Windows - if pathext is None: - pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD') - pathext = [ext for ext in pathext.lower().split(os.pathsep)] - # don't use extensions if the command ends with one of them - if os.path.splitext(cmd)[1].lower() in pathext: - pathext = [''] - # check if we find the command on PATH - for path in paths: - # try without extension first - cmd_path = os.path.join(path, cmd) - for ext in pathext: - # then including the extension - cmd_path_ext = cmd_path + ext - if os.path.exists(cmd_path_ext): - return cmd_path_ext - if os.path.exists(cmd_path): - return cmd_path - return None - - -def ask(message, options): - """Ask the message interactively, with the given possible responses""" - while 1: - if os.environ.get('PIP_NO_INPUT'): - raise Exception('No input was expected ($PIP_NO_INPUT set); question: %s' % message) - response = raw_input(message) - response = response.strip().lower() - if response not in options: - print 'Your response (%r) was not one of the expected responses: %s' % ( - response, ', '.join(options)) - else: - return response - - -class _Inf(object): - """I am bigger than everything!""" - def __cmp__(self, a): - if self is a: - return 0 - return 1 - - def __repr__(self): - return 'Inf' - -Inf = _Inf() -del _Inf - - -_normalize_re = re.compile(r'[^a-z]', re.I) - - -def normalize_name(name): - return _normalize_re.sub('-', name.lower()) - - -def format_size(bytes): - if bytes > 1000*1000: - return '%.1fMb' % (bytes/1000.0/1000) - elif bytes > 10*1000: - return '%iKb' % (bytes/1000) - elif bytes > 1000: - return '%.1fKb' % (bytes/1000.0) - else: - return '%ibytes' % bytes - - -def is_installable_dir(path): - """Return True if `path` is a directory containing a setup.py file.""" - if not os.path.isdir(path): - return False - setup_py = os.path.join(path, 'setup.py') - if os.path.isfile(setup_py): - return True - return False - - -def is_svn_page(html): - """Returns true if the page appears to be the index page of an svn repository""" - return (re.search(r'<title>[^<]*Revision \d+:', html) - and re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I)) - - -def file_contents(filename): - fp = open(filename, 'rb') - try: - return fp.read() - finally: - fp.close() - - -def split_leading_dir(path): - path = str(path) - path = path.lstrip('/').lstrip('\\') - if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) - or '\\' not in path): - return path.split('/', 1) - elif '\\' in path: - return path.split('\\', 1) - else: - return path, '' - - -def has_leading_dir(paths): - """Returns true if all the paths have the same leading path name - (i.e., everything is in one subdirectory in an archive)""" - common_prefix = None - for path in paths: - prefix, rest = split_leading_dir(path) - if not prefix: - return False - elif common_prefix is None: - common_prefix = prefix - elif prefix != common_prefix: - return False - return True - - -def make_path_relative(path, rel_to): - """ - Make a filename relative, where the filename path, and it is - relative to rel_to - - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/usr/share/another-place/src/Directory') - '../../../something/a-file.pth' - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/home/user/src/Directory') - '../../../usr/share/something/a-file.pth' - >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/') - 'a-file.pth' - """ - path_filename = os.path.basename(path) - path = os.path.dirname(path) - path = os.path.normpath(os.path.abspath(path)) - rel_to = os.path.normpath(os.path.abspath(rel_to)) - path_parts = path.strip(os.path.sep).split(os.path.sep) - rel_to_parts = rel_to.strip(os.path.sep).split(os.path.sep) - while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]: - path_parts.pop(0) - rel_to_parts.pop(0) - full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename] - if full_parts == ['']: - return '.' + os.path.sep - return os.path.sep.join(full_parts) - - -def normalize_path(path): - """ - Convert a path to its canonical, case-normalized, absolute version. - - """ - return os.path.normcase(os.path.realpath(path)) - - -def splitext(path): - """Like os.path.splitext, but take off .tar too""" - base, ext = posixpath.splitext(path) - if base.lower().endswith('.tar'): - ext = base[-4:] + ext - base = base[:-4] - return base, ext - - -def renames(old, new): - """Like os.renames(), but handles renaming across devices.""" - # Implementation borrowed from os.renames(). - head, tail = os.path.split(new) - if head and tail and not os.path.exists(head): - os.makedirs(head) - - shutil.move(old, new) - - head, tail = os.path.split(old) - if head and tail: - try: - os.removedirs(head) - except OSError: - pass - - -def is_local(path): - """ - Return True if path is within sys.prefix, if we're running in a virtualenv. - - If we're not in a virtualenv, all paths are considered "local." - - """ - if not running_under_virtualenv(): - return True - return normalize_path(path).startswith(normalize_path(sys.prefix)) - - -def dist_is_local(dist): - """ - Return True if given Distribution object is installed locally - (i.e. within current virtualenv). - - Always True if we're not in a virtualenv. - - """ - return is_local(dist_location(dist)) - - -def get_installed_distributions(local_only=True, skip=('setuptools', 'pip', 'python')): - """ - Return a list of installed Distribution objects. - - If ``local_only`` is True (default), only return installations - local to the current virtualenv, if in a virtualenv. - - ``skip`` argument is an iterable of lower-case project names to - ignore; defaults to ('setuptools', 'pip', 'python'). [FIXME also - skip virtualenv?] - - """ - if local_only: - local_test = dist_is_local - else: - local_test = lambda d: True - return [d for d in pkg_resources.working_set if local_test(d) and d.key not in skip] - - -def egg_link_path(dist): - """ - Return the path where we'd expect to find a .egg-link file for - this distribution. (There doesn't seem to be any metadata in the - Distribution object for a develop egg that points back to its - .egg-link and easy-install.pth files). - - This won't find a globally-installed develop egg if we're in a - virtualenv. - - """ - return os.path.join(site_packages, dist.project_name) + '.egg-link' - - -def dist_location(dist): - """ - Get the site-packages location of this distribution. Generally - this is dist.location, except in the case of develop-installed - packages, where dist.location is the source code location, and we - want to know where the egg-link file is. - - """ - egg_link = egg_link_path(dist) - if os.path.exists(egg_link): - return egg_link - return dist.location - - -def get_terminal_size(): - """Returns a tuple (x, y) representing the width(x) and the height(x) - in characters of the terminal window.""" - def ioctl_GWINSZ(fd): - try: - import fcntl - import termios - import struct - cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, - '1234')) - except: - return None - if cr == (0, 0): - return None - if cr == (0, 0): - return None - return cr - cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - cr = ioctl_GWINSZ(fd) - os.close(fd) - except: - pass - if not cr: - cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) - return int(cr[1]), int(cr[0]) - - -def unzip_file(filename, location, flatten=True): - """Unzip the file (zip file located at filename) to the destination - location""" - if not os.path.exists(location): - os.makedirs(location) - zipfp = open(filename, 'rb') - try: - zip = zipfile.ZipFile(zipfp) - leading = has_leading_dir(zip.namelist()) and flatten - for name in zip.namelist(): - data = zip.read(name) - fn = name - if leading: - fn = split_leading_dir(name)[1] - fn = os.path.join(location, fn) - dir = os.path.dirname(fn) - if not os.path.exists(dir): - os.makedirs(dir) - if fn.endswith('/') or fn.endswith('\\'): - # A directory - if not os.path.exists(fn): - os.makedirs(fn) - else: - fp = open(fn, 'wb') - try: - fp.write(data) - finally: - fp.close() - finally: - zipfp.close() - - -def untar_file(filename, location): - """Untar the file (tar file located at filename) to the destination location""" - if not os.path.exists(location): - os.makedirs(location) - if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): - mode = 'r:gz' - elif filename.lower().endswith('.bz2') or filename.lower().endswith('.tbz'): - mode = 'r:bz2' - elif filename.lower().endswith('.tar'): - mode = 'r' - else: - logger.warn('Cannot determine compression type for file %s' % filename) - mode = 'r:*' - tar = tarfile.open(filename, mode) - try: - # note: python<=2.5 doesnt seem to know about pax headers, filter them - leading = has_leading_dir([ - member.name for member in tar.getmembers() - if member.name != 'pax_global_header' - ]) - for member in tar.getmembers(): - fn = member.name - if fn == 'pax_global_header': - continue - if leading: - fn = split_leading_dir(fn)[1] - path = os.path.join(location, fn) - if member.isdir(): - if not os.path.exists(path): - os.makedirs(path) - else: - try: - fp = tar.extractfile(member) - except (KeyError, AttributeError), e: - # Some corrupt tar files seem to produce this - # (specifically bad symlinks) - logger.warn( - 'In the tar file %s the member %s is invalid: %s' - % (filename, member.name, e)) - continue - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - destfp = open(path, 'wb') - try: - shutil.copyfileobj(fp, destfp) - finally: - destfp.close() - fp.close() - finally: - tar.close() - - -def create_download_cache_folder(folder): - logger.indent -= 2 - logger.notify('Creating supposed download cache at %s' % folder) - logger.indent += 2 - os.makedirs(folder) - - -def cache_download(target_file, temp_location, content_type): - logger.notify('Storing download in cache at %s' % display_path(target_file)) - shutil.copyfile(temp_location, target_file) - fp = open(target_file+'.content-type', 'w') - fp.write(content_type) - fp.close() - os.unlink(temp_location) - - -def unpack_file(filename, location, content_type, link): - if (content_type == 'application/zip' - or filename.endswith('.zip') - or filename.endswith('.pybundle') - or zipfile.is_zipfile(filename)): - unzip_file(filename, location, flatten=not filename.endswith('.pybundle')) - elif (content_type == 'application/x-gzip' - or tarfile.is_tarfile(filename) - or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')): - untar_file(filename, location) - elif (content_type and content_type.startswith('text/html') - and is_svn_page(file_contents(filename))): - # We don't really care about this - from pip.vcs.subversion import Subversion - Subversion('svn+' + link.url).unpack(location) - else: - ## FIXME: handle? - ## FIXME: magic signatures? - logger.fatal('Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format' - % (filename, location, content_type)) - raise InstallationError('Cannot determine archive format of %s' % location) - - - diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py deleted file mode 100644 index e110440c7..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/__init__.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Handles all VCS (version control) support""" - -import os -import shutil -import urlparse -import urllib - -from pip.exceptions import BadCommand -from pip.log import logger -from pip.util import display_path, backup_dir, find_command, ask - - -__all__ = ['vcs', 'get_src_requirement', 'import_vcs_support'] - - -class VcsSupport(object): - _registry = {} - schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp'] - - def __init__(self): - # Register more schemes with urlparse for various version control systems - urlparse.uses_netloc.extend(self.schemes) - urlparse.uses_fragment.extend(self.schemes) - super(VcsSupport, self).__init__() - - def __iter__(self): - return self._registry.__iter__() - - @property - def backends(self): - return self._registry.values() - - @property - def dirnames(self): - return [backend.dirname for backend in self.backends] - - @property - def all_schemes(self): - schemes = [] - for backend in self.backends: - schemes.extend(backend.schemes) - return schemes - - def register(self, cls): - if not hasattr(cls, 'name'): - logger.warn('Cannot register VCS %s' % cls.__name__) - return - if cls.name not in self._registry: - self._registry[cls.name] = cls - - def unregister(self, cls=None, name=None): - if name in self._registry: - del self._registry[name] - elif cls in self._registry.values(): - del self._registry[cls.name] - else: - logger.warn('Cannot unregister because no class or name given') - - def get_backend_name(self, location): - """ - Return the name of the version control backend if found at given - location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') - """ - for vc_type in self._registry.values(): - path = os.path.join(location, vc_type.dirname) - if os.path.exists(path): - return vc_type.name - return None - - def get_backend(self, name): - name = name.lower() - if name in self._registry: - return self._registry[name] - - def get_backend_from_location(self, location): - vc_type = self.get_backend_name(location) - if vc_type: - return self.get_backend(vc_type) - return None - - -vcs = VcsSupport() - - -class VersionControl(object): - name = '' - dirname = '' - - def __init__(self, url=None, *args, **kwargs): - self.url = url - self._cmd = None - super(VersionControl, self).__init__(*args, **kwargs) - - def _filter(self, line): - return (logger.INFO, line) - - def _is_local_repository(self, repo): - """ - posix absolute paths start with os.path.sep, - win32 ones ones start with drive (like c:\\folder) - """ - drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or drive - - @property - def cmd(self): - if self._cmd is not None: - return self._cmd - command = find_command(self.name) - if command is None: - raise BadCommand('Cannot find command %r' % self.name) - logger.info('Found command %r at %r' % (self.name, command)) - self._cmd = command - return command - - def get_url_rev(self): - """ - Returns the correct repository URL and revision by parsing the given - repository URL - """ - url = self.url.split('+', 1)[1] - scheme, netloc, path, query, frag = urlparse.urlsplit(url) - rev = None - if '@' in path: - path, rev = path.rsplit('@', 1) - url = urlparse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev - - def get_info(self, location): - """ - Returns (url, revision), where both are strings - """ - assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) - - def normalize_url(self, url): - """ - Normalize a URL for comparison by unquoting it and removing any trailing slash. - """ - return urllib.unquote(url).rstrip('/') - - def compare_urls(self, url1, url2): - """ - Compare two repo URLs for identity, ignoring incidental differences. - """ - return (self.normalize_url(url1) == self.normalize_url(url2)) - - def parse_vcs_bundle_file(self, content): - """ - Takes the contents of the bundled text file that explains how to revert - the stripped off version control data of the given package and returns - the URL and revision of it. - """ - raise NotImplementedError - - def obtain(self, dest): - """ - Called when installing or updating an editable package, takes the - source path of the checkout. - """ - raise NotImplementedError - - def switch(self, dest, url, rev_options): - """ - Switch the repo at ``dest`` to point to ``URL``. - """ - raise NotImplemented - - def update(self, dest, rev_options): - """ - Update an already-existing repo to the given ``rev_options``. - """ - raise NotImplementedError - - def check_destination(self, dest, url, rev_options, rev_display): - """ - Prepare a location to receive a checkout/clone. - - Return True if the location is ready for (and requires) a - checkout/clone, False otherwise. - """ - checkout = True - prompt = False - if os.path.exists(dest): - checkout = False - if os.path.exists(os.path.join(dest, self.dirname)): - existing_url = self.get_url(dest) - if self.compare_urls(existing_url, url): - logger.info('%s in %s exists, and has correct URL (%s)' - % (self.repo_name.title(), display_path(dest), url)) - logger.notify('Updating %s %s%s' - % (display_path(dest), self.repo_name, rev_display)) - self.update(dest, rev_options) - else: - logger.warn('%s %s in %s exists with URL %s' - % (self.name, self.repo_name, display_path(dest), existing_url)) - prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) - else: - logger.warn('Directory %s already exists, and is not a %s %s.' - % (dest, self.name, self.repo_name)) - prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) - if prompt: - logger.warn('The plan is to install the %s repository %s' - % (self.name, url)) - response = ask('What to do? %s' % prompt[0], prompt[1]) - - if response == 's': - logger.notify('Switching %s %s to %s%s' - % (self.repo_name, display_path(dest), url, rev_display)) - self.switch(dest, url, rev_options) - elif response == 'i': - # do nothing - pass - elif response == 'w': - logger.warn('Deleting %s' % display_path(dest)) - shutil.rmtree(dest) - checkout = True - elif response == 'b': - dest_dir = backup_dir(dest) - logger.warn('Backing up %s to %s' - % (display_path(dest), dest_dir)) - shutil.move(dest, dest_dir) - checkout = True - return checkout - - def unpack(self, location): - raise NotImplementedError - - def get_src_requirement(self, dist, location, find_tags=False): - raise NotImplementedError - - -def get_src_requirement(dist, location, find_tags): - version_control = vcs.get_backend_from_location(location) - if version_control: - return version_control().get_src_requirement(dist, location, find_tags) - logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location) - return dist.as_requirement() diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py deleted file mode 100644 index 3b6ea8f04..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/bazaar.py +++ /dev/null @@ -1,138 +0,0 @@ -import os -import shutil -import tempfile -import re -from pip import call_subprocess -from pip.log import logger -from pip.util import rmtree, display_path -from pip.vcs import vcs, VersionControl -from pip.download import path_to_url2 - - -class Bazaar(VersionControl): - name = 'bzr' - dirname = '.bzr' - repo_name = 'branch' - bundle_file = 'bzr-branch.txt' - schemes = ('bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp') - guide = ('# This was a Bazaar branch; to make it a branch again run:\n' - 'bzr branch -r %(rev)s %(url)s .\n') - - def parse_vcs_bundle_file(self, content): - url = rev = None - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - match = re.search(r'^bzr\s*branch\s*-r\s*(\d*)', line) - if match: - rev = match.group(1).strip() - url = line[match.end():].strip().split(None, 1)[0] - if url and rev: - return url, rev - return None, None - - def unpack(self, location): - """Get the bzr branch at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Checking out bzr repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - os.rmdir(location) - call_subprocess( - [self.cmd, 'branch', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the Bazaar repository at the url to the destination location""" - temp_dir = tempfile.mkdtemp('-export', 'pip-') - self.unpack(temp_dir) - if os.path.exists(location): - # Remove the location to make sure Bazaar can export it correctly - rmtree(location) - try: - call_subprocess([self.cmd, 'export', location], cwd=temp_dir, - filter_stdout=self._filter, show_stdout=False) - finally: - shutil.rmtree(temp_dir) - - def switch(self, dest, url, rev_options): - call_subprocess([self.cmd, 'switch', url], cwd=dest) - - def update(self, dest, rev_options): - call_subprocess( - [self.cmd, 'pull', '-q'] + rev_options, cwd=dest) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = ['-r', rev] - rev_display = ' (to revision %s)' % rev - else: - rev_options = [] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Checking out %s%s to %s' - % (url, rev_display, display_path(dest))) - call_subprocess( - [self.cmd, 'branch', '-q'] + rev_options + [url, dest]) - - def get_url_rev(self): - # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev() - if url.startswith('ssh://'): - url = 'bzr+' + url - return url, rev - - def get_url(self, location): - urls = call_subprocess( - [self.cmd, 'info'], show_stdout=False, cwd=location) - for line in urls.splitlines(): - line = line.strip() - for x in ('checkout of branch: ', - 'parent branch: '): - if line.startswith(x): - repo = line.split(x)[1] - if self._is_local_repository(repo): - return path_to_url2(repo) - return repo - return None - - def get_revision(self, location): - revision = call_subprocess( - [self.cmd, 'revno'], show_stdout=False, cwd=location) - return revision.splitlines()[-1] - - def get_tag_revs(self, location): - tags = call_subprocess( - [self.cmd, 'tags'], show_stdout=False, cwd=location) - tag_revs = [] - for line in tags.splitlines(): - tags_match = re.search(r'([.\w-]+)\s*(.*)$', line) - if tags_match: - tag = tags_match.group(1) - rev = tags_match.group(2) - tag_revs.append((rev.strip(), tag.strip())) - return dict(tag_revs) - - def get_src_requirement(self, dist, location, find_tags): - repo = self.get_url(location) - if not repo.lower().startswith('bzr:'): - repo = 'bzr+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None - current_rev = self.get_revision(location) - tag_revs = self.get_tag_revs(location) - - if current_rev in tag_revs: - # It's a tag - full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) - else: - full_egg_name = '%s-dev_r%s' % (dist.egg_name(), current_rev) - return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) - - -vcs.register(Bazaar) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py deleted file mode 100644 index 0701e49ec..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/git.py +++ /dev/null @@ -1,204 +0,0 @@ -import os -import shutil -import tempfile -import re -from pip import call_subprocess -from pip.util import display_path -from pip.vcs import vcs, VersionControl -from pip.log import logger -from urllib import url2pathname -from urlparse import urlsplit, urlunsplit - - -class Git(VersionControl): - name = 'git' - dirname = '.git' - repo_name = 'clone' - schemes = ('git', 'git+http', 'git+ssh', 'git+git', 'git+file') - bundle_file = 'git-clone.txt' - guide = ('# This was a Git repo; to make it a repo again run:\n' - 'git init\ngit remote add origin %(url)s -f\ngit checkout %(rev)s\n') - - def __init__(self, url=None, *args, **kwargs): - - # Works around an apparent Git bug - # (see http://article.gmane.org/gmane.comp.version-control.git/146500) - if url: - scheme, netloc, path, query, fragment = urlsplit(url) - if scheme.endswith('file'): - initial_slashes = path[:-len(path.lstrip('/'))] - newpath = initial_slashes + url2pathname(path).replace('\\', '/').lstrip('/') - url = urlunsplit((scheme, netloc, newpath, query, fragment)) - after_plus = scheme.find('+')+1 - url = scheme[:after_plus]+ urlunsplit((scheme[after_plus:], netloc, newpath, query, fragment)) - - super(Git, self).__init__(url, *args, **kwargs) - - def parse_vcs_bundle_file(self, content): - url = rev = None - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - url_match = re.search(r'git\s*remote\s*add\s*origin(.*)\s*-f', line) - if url_match: - url = url_match.group(1).strip() - rev_match = re.search(r'^git\s*checkout\s*-q\s*(.*)\s*', line) - if rev_match: - rev = rev_match.group(1).strip() - if url and rev: - return url, rev - return None, None - - def unpack(self, location): - """Clone the Git repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Cloning Git repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - os.rmdir(location) - call_subprocess( - [self.cmd, 'clone', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the Git repository at the url to the destination location""" - temp_dir = tempfile.mkdtemp('-export', 'pip-') - self.unpack(temp_dir) - try: - if not location.endswith('/'): - location = location + '/' - call_subprocess( - [self.cmd, 'checkout-index', '-a', '-f', '--prefix', location], - filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) - finally: - shutil.rmtree(temp_dir) - - def check_rev_options(self, rev, dest, rev_options): - """Check the revision options before checkout to compensate that tags - and branches may need origin/ as a prefix. - Returns the SHA1 of the branch or tag if found. - """ - revisions = self.get_tag_revs(dest) - revisions.update(self.get_branch_revs(dest)) - inverse_revisions = dict((v, k) for k, v in revisions.iteritems()) - # Check if rev is a branch name - origin_rev = 'origin/%s' % rev - if origin_rev in inverse_revisions: - return [inverse_revisions[origin_rev]] - elif rev in inverse_revisions: - return [inverse_revisions[rev]] - else: - logger.warn("Could not find a tag or branch '%s', assuming commit." % rev) - return rev_options - - def switch(self, dest, url, rev_options): - call_subprocess( - [self.cmd, 'config', 'remote.origin.url', url], cwd=dest) - call_subprocess( - [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) - - def update(self, dest, rev_options): - call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) - call_subprocess( - [self.cmd, 'checkout', '-q', '-f'] + rev_options, cwd=dest) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = [rev] - rev_display = ' (to %s)' % rev - else: - rev_options = ['master'] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest))) - call_subprocess([self.cmd, 'clone', '-q', url, dest]) - if rev: - rev_options = self.check_rev_options(rev, dest, rev_options) - # Only do a checkout if rev_options differs from HEAD - if not self.get_revision(dest).startswith(rev_options[0]): - call_subprocess([self.cmd, 'checkout', '-q'] + rev_options, cwd=dest) - - def get_url(self, location): - url = call_subprocess( - [self.cmd, 'config', 'remote.origin.url'], - show_stdout=False, cwd=location) - return url.strip() - - def get_revision(self, location): - current_rev = call_subprocess( - [self.cmd, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location) - return current_rev.strip() - - def get_tag_revs(self, location): - tags = call_subprocess( - [self.cmd, 'tag', '-l'], - show_stdout=False, raise_on_returncode=False, cwd=location) - tag_revs = [] - for line in tags.splitlines(): - tag = line.strip() - rev = call_subprocess( - [self.cmd, 'rev-parse', tag], show_stdout=False, cwd=location) - tag_revs.append((rev.strip(), tag)) - tag_revs = dict(tag_revs) - return tag_revs - - def get_branch_revs(self, location): - branches = call_subprocess( - [self.cmd, 'branch', '-r'], show_stdout=False, cwd=location) - branch_revs = [] - for line in branches.splitlines(): - line = line.split('->')[0].strip() - branch = "".join([b for b in line.split() if b != '*']) - rev = call_subprocess( - [self.cmd, 'rev-parse', branch], show_stdout=False, cwd=location) - branch_revs.append((rev.strip(), branch)) - branch_revs = dict(branch_revs) - return branch_revs - - def get_src_requirement(self, dist, location, find_tags): - repo = self.get_url(location) - if not repo.lower().startswith('git:'): - repo = 'git+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None - current_rev = self.get_revision(location) - tag_revs = self.get_tag_revs(location) - branch_revs = self.get_branch_revs(location) - - if current_rev in tag_revs: - # It's a tag - full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) - elif (current_rev in branch_revs and - branch_revs[current_rev] != 'origin/master'): - # It's the head of a branch - full_egg_name = '%s-%s' % (dist.egg_name(), - branch_revs[current_rev].replace('origin/', '')) - else: - full_egg_name = '%s-dev' % dist.egg_name() - - return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name) - - def get_url_rev(self): - """ - Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. - That's required because although they use SSH they sometimes doesn't - work with a ssh:// scheme (e.g. Github). But we need a scheme for - parsing. Hence we remove it again afterwards and return it as a stub. - """ - if not '://' in self.url: - assert not 'file:' in self.url - self.url = self.url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev() - url = url.replace('ssh://', '') - else: - url, rev = super(Git, self).get_url_rev() - - return url, rev - - -vcs.register(Git) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py deleted file mode 100644 index 70c8c833d..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/mercurial.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -import shutil -import tempfile -import re -import ConfigParser -from pip import call_subprocess -from pip.util import display_path -from pip.log import logger -from pip.vcs import vcs, VersionControl -from pip.download import path_to_url2 - - -class Mercurial(VersionControl): - name = 'hg' - dirname = '.hg' - repo_name = 'clone' - schemes = ('hg', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http') - bundle_file = 'hg-clone.txt' - guide = ('# This was a Mercurial repo; to make it a repo again run:\n' - 'hg init\nhg pull %(url)s\nhg update -r %(rev)s\n') - - def parse_vcs_bundle_file(self, content): - url = rev = None - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - url_match = re.search(r'hg\s*pull\s*(.*)\s*', line) - if url_match: - url = url_match.group(1).strip() - rev_match = re.search(r'^hg\s*update\s*-r\s*(.*)\s*', line) - if rev_match: - rev = rev_match.group(1).strip() - if url and rev: - return url, rev - return None, None - - def unpack(self, location): - """Clone the Hg repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Cloning Mercurial repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - os.rmdir(location) - call_subprocess( - [self.cmd, 'clone', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the Hg repository at the url to the destination location""" - temp_dir = tempfile.mkdtemp('-export', 'pip-') - self.unpack(temp_dir) - try: - call_subprocess( - [self.cmd, 'archive', location], - filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) - finally: - shutil.rmtree(temp_dir) - - def switch(self, dest, url, rev_options): - repo_config = os.path.join(dest, self.dirname, 'hgrc') - config = ConfigParser.SafeConfigParser() - try: - config.read(repo_config) - config.set('paths', 'default', url) - config_file = open(repo_config, 'w') - config.write(config_file) - config_file.close() - except (OSError, ConfigParser.NoSectionError), e: - logger.warn( - 'Could not switch Mercurial repository to %s: %s' - % (url, e)) - else: - call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) - - def update(self, dest, rev_options): - call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) - call_subprocess( - [self.cmd, 'update', '-q'] + rev_options, cwd=dest) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = [rev] - rev_display = ' (to revision %s)' % rev - else: - rev_options = [] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Cloning hg %s%s to %s' - % (url, rev_display, display_path(dest))) - call_subprocess([self.cmd, 'clone', '--noupdate', '-q', url, dest]) - call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) - - def get_url(self, location): - url = call_subprocess( - [self.cmd, 'showconfig', 'paths.default'], - show_stdout=False, cwd=location).strip() - if self._is_local_repository(url): - url = path_to_url2(url) - return url.strip() - - def get_tag_revs(self, location): - tags = call_subprocess( - [self.cmd, 'tags'], show_stdout=False, cwd=location) - tag_revs = [] - for line in tags.splitlines(): - tags_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) - if tags_match: - tag = tags_match.group(1) - rev = tags_match.group(2) - tag_revs.append((rev.strip(), tag.strip())) - return dict(tag_revs) - - def get_branch_revs(self, location): - branches = call_subprocess( - [self.cmd, 'branches'], show_stdout=False, cwd=location) - branch_revs = [] - for line in branches.splitlines(): - branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) - if branches_match: - branch = branches_match.group(1) - rev = branches_match.group(2) - branch_revs.append((rev.strip(), branch.strip())) - return dict(branch_revs) - - def get_revision(self, location): - current_revision = call_subprocess( - [self.cmd, 'parents', '--template={rev}'], - show_stdout=False, cwd=location).strip() - return current_revision - - def get_revision_hash(self, location): - current_rev_hash = call_subprocess( - [self.cmd, 'parents', '--template={node}'], - show_stdout=False, cwd=location).strip() - return current_rev_hash - - def get_src_requirement(self, dist, location, find_tags): - repo = self.get_url(location) - if not repo.lower().startswith('hg:'): - repo = 'hg+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None - current_rev = self.get_revision(location) - current_rev_hash = self.get_revision_hash(location) - tag_revs = self.get_tag_revs(location) - branch_revs = self.get_branch_revs(location) - if current_rev in tag_revs: - # It's a tag - full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev]) - elif current_rev in branch_revs: - # It's the tip of a branch - full_egg_name = '%s-%s' % (dist.egg_name(), branch_revs[current_rev]) - else: - full_egg_name = '%s-dev' % dist.egg_name() - return '%s@%s#egg=%s' % (repo, current_rev_hash, full_egg_name) - -vcs.register(Mercurial) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py deleted file mode 100644 index 85715d97a..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/vcs/subversion.py +++ /dev/null @@ -1,260 +0,0 @@ -import os -import re -from pip import call_subprocess -from pip.index import Link -from pip.util import rmtree, display_path -from pip.log import logger -from pip.vcs import vcs, VersionControl - -_svn_xml_url_re = re.compile('url="([^"]+)"') -_svn_rev_re = re.compile('committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') - - -class Subversion(VersionControl): - name = 'svn' - dirname = '.svn' - repo_name = 'checkout' - schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https') - bundle_file = 'svn-checkout.txt' - guide = ('# This was an svn checkout; to make it a checkout again run:\n' - 'svn checkout --force -r %(rev)s %(url)s .\n') - - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location - output = call_subprocess( - [self.cmd, 'info', location], show_stdout=False, extra_environ={'LANG': 'C'}) - match = _svn_url_re.search(output) - if not match: - logger.warn('Cannot determine URL of svn checkout %s' % display_path(location)) - logger.info('Output that cannot be parsed: \n%s' % output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warn('Cannot determine revision of svn checkout %s' % display_path(location)) - logger.info('Output that cannot be parsed: \n%s' % output) - return url, None - return url, match.group(1) - - def parse_vcs_bundle_file(self, content): - for line in content.splitlines(): - if not line.strip() or line.strip().startswith('#'): - continue - match = re.search(r'^-r\s*([^ ])?', line) - if not match: - return None, None - rev = match.group(1) - rest = line[match.end():].strip().split(None, 1)[0] - return rest, rev - return None, None - - def unpack(self, location): - """Check out the svn repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Checking out svn repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - # Subversion doesn't like to check out over an existing directory - # --force fixes this, but was only added in svn 1.5 - rmtree(location) - call_subprocess( - [self.cmd, 'checkout', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def export(self, location): - """Export the svn repository at the url to the destination location""" - url, rev = self.get_url_rev() - logger.notify('Exporting svn repository %s to %s' % (url, location)) - logger.indent += 2 - try: - if os.path.exists(location): - # Subversion doesn't like to check out over an existing directory - # --force fixes this, but was only added in svn 1.5 - rmtree(location) - call_subprocess( - [self.cmd, 'export', url, location], - filter_stdout=self._filter, show_stdout=False) - finally: - logger.indent -= 2 - - def switch(self, dest, url, rev_options): - call_subprocess( - [self.cmd, 'switch'] + rev_options + [url, dest]) - - def update(self, dest, rev_options): - call_subprocess( - [self.cmd, 'update'] + rev_options + [dest]) - - def obtain(self, dest): - url, rev = self.get_url_rev() - if rev: - rev_options = ['-r', rev] - rev_display = ' (to revision %s)' % rev - else: - rev_options = [] - rev_display = '' - if self.check_destination(dest, url, rev_options, rev_display): - logger.notify('Checking out %s%s to %s' - % (url, rev_display, display_path(dest))) - call_subprocess( - [self.cmd, 'checkout', '-q'] + rev_options + [url, dest]) - - def get_location(self, dist, dependency_links): - for url in dependency_links: - egg_fragment = Link(url).egg_fragment - if not egg_fragment: - continue - if '-' in egg_fragment: - ## FIXME: will this work when a package has - in the name? - key = '-'.join(egg_fragment.split('-')[:-1]).lower() - else: - key = egg_fragment - if key == dist.key: - return url.split('#', 1)[0] - return None - - def get_revision(self, location): - """ - Return the maximum revision for all files under a given location - """ - # Note: taken from setuptools.command.egg_info - revision = 0 - - for base, dirs, files in os.walk(location): - if self.dirname not in dirs: - dirs[:] = [] - continue # no sense walking uncontrolled subdirs - dirs.remove(self.dirname) - entries_fn = os.path.join(base, self.dirname, 'entries') - if not os.path.exists(entries_fn): - ## FIXME: should we warn? - continue - f = open(entries_fn) - data = f.read() - f.close() - - if data.startswith('8') or data.startswith('9') or data.startswith('10'): - data = map(str.splitlines, data.split('\n\x0c\n')) - del data[0][0] # get rid of the '8' - dirurl = data[0][3] - revs = [int(d[9]) for d in data if len(d)>9 and d[9]]+[0] - if revs: - localrev = max(revs) - else: - localrev = 0 - elif data.startswith('<?xml'): - dirurl = _svn_xml_url_re.search(data).group(1) # get repository URL - revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)]+[0] - if revs: - localrev = max(revs) - else: - localrev = 0 - else: - logger.warn("Unrecognized .svn/entries format; skipping %s", base) - dirs[:] = [] - continue - if base == location: - base_url = dirurl+'/' # save the root url - elif not dirurl.startswith(base_url): - dirs[:] = [] - continue # not part of the same svn tree, skip it - revision = max(revision, localrev) - return revision - - def get_url_rev(self): - # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev() - if url.startswith('ssh://'): - url = 'svn+' + url - return url, rev - - def get_url(self, location): - # In cases where the source is in a subdirectory, not alongside setup.py - # we have to look up in the location until we find a real setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without finding setup.py - logger.warn("Could not find setup.py for directory %s (tried all parent directories)" - % orig_location) - return None - f = open(os.path.join(location, self.dirname, 'entries')) - data = f.read() - f.close() - if data.startswith('8') or data.startswith('9') or data.startswith('10'): - data = map(str.splitlines, data.split('\n\x0c\n')) - del data[0][0] # get rid of the '8' - return data[0][3] - elif data.startswith('<?xml'): - match = _svn_xml_url_re.search(data) - if not match: - raise ValueError('Badly formatted data: %r' % data) - return match.group(1) # get repository URL - else: - logger.warn("Unrecognized .svn/entries format in %s" % location) - # Or raise exception? - return None - - def get_tag_revs(self, svn_tag_url): - stdout = call_subprocess( - [self.cmd, 'ls', '-v', svn_tag_url], show_stdout=False) - results = [] - for line in stdout.splitlines(): - parts = line.split() - rev = int(parts[0]) - tag = parts[-1].strip('/') - results.append((tag, rev)) - return results - - def find_tag_match(self, rev, tag_revs): - best_match_rev = None - best_tag = None - for tag, tag_rev in tag_revs: - if (tag_rev > rev and - (best_match_rev is None or best_match_rev > tag_rev)): - # FIXME: Is best_match > tag_rev really possible? - # or is it a sign something is wacky? - best_match_rev = tag_rev - best_tag = tag - return best_tag - - def get_src_requirement(self, dist, location, find_tags=False): - repo = self.get_url(location) - if repo is None: - return None - parts = repo.split('/') - ## FIXME: why not project name? - egg_project_name = dist.egg_name().split('-', 1)[0] - rev = self.get_revision(location) - if parts[-2] in ('tags', 'tag'): - # It's a tag, perfect! - full_egg_name = '%s-%s' % (egg_project_name, parts[-1]) - elif parts[-2] in ('branches', 'branch'): - # It's a branch :( - full_egg_name = '%s-%s-r%s' % (dist.egg_name(), parts[-1], rev) - elif parts[-1] == 'trunk': - # Trunk :-/ - full_egg_name = '%s-dev_r%s' % (dist.egg_name(), rev) - if find_tags: - tag_url = '/'.join(parts[:-1]) + '/tags' - tag_revs = self.get_tag_revs(tag_url) - match = self.find_tag_match(rev, tag_revs) - if match: - logger.notify('trunk checkout %s seems to be equivalent to tag %s' % match) - repo = '%s/%s' % (tag_url, match) - full_egg_name = '%s-%s' % (egg_project_name, match) - else: - # Don't know what it is - logger.warn('svn URL does not fit normal structure (tags/branches/trunk): %s' % repo) - full_egg_name = '%s-dev_r%s' % (egg_project_name, rev) - return 'svn+%s@%s#egg=%s' % (repo, rev, full_egg_name) - -vcs.register(Subversion) diff --git a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py b/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py deleted file mode 100644 index 708abb05b..000000000 --- a/test/lib/python2.6/site-packages/pip-0.8.1-py2.6.egg/pip/venv.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Tools for working with virtualenv environments""" - -import os -import sys -import subprocess -from pip.exceptions import BadCommand -from pip.log import logger - - -def restart_in_venv(venv, base, site_packages, args): - """ - Restart this script using the interpreter in the given virtual environment - """ - if base and not os.path.isabs(venv) and not venv.startswith('~'): - base = os.path.expanduser(base) - # ensure we have an abs basepath at this point: - # a relative one makes no sense (or does it?) - if os.path.isabs(base): - venv = os.path.join(base, venv) - - if venv.startswith('~'): - venv = os.path.expanduser(venv) - - if not os.path.exists(venv): - try: - import virtualenv - except ImportError: - print 'The virtual environment does not exist: %s' % venv - print 'and virtualenv is not installed, so a new environment cannot be created' - sys.exit(3) - print 'Creating new virtualenv environment in %s' % venv - virtualenv.logger = logger - logger.indent += 2 - virtualenv.create_environment(venv, site_packages=site_packages) - if sys.platform == 'win32': - python = os.path.join(venv, 'Scripts', 'python.exe') - # check for bin directory which is used in buildouts - if not os.path.exists(python): - python = os.path.join(venv, 'bin', 'python.exe') - else: - python = os.path.join(venv, 'bin', 'python') - if not os.path.exists(python): - python = venv - if not os.path.exists(python): - raise BadCommand('Cannot find virtual environment interpreter at %s' % python) - base = os.path.dirname(os.path.dirname(python)) - file = os.path.join(os.path.dirname(__file__), 'runner.py') - if file.endswith('.pyc'): - file = file[:-1] - proc = subprocess.Popen( - [python, file] + args + [base, '___VENV_RESTART___']) - proc.wait() - sys.exit(proc.returncode) diff --git a/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg b/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg deleted file mode 100644 index 3c72d15b5..000000000 Binary files a/test/lib/python2.6/site-packages/setuptools-0.6c11-py2.6.egg and /dev/null differ diff --git a/test/lib/python2.6/site-packages/setuptools.pth b/test/lib/python2.6/site-packages/setuptools.pth deleted file mode 100644 index bf7602056..000000000 --- a/test/lib/python2.6/site-packages/setuptools.pth +++ /dev/null @@ -1 +0,0 @@ -./setuptools-0.6c11-py2.6.egg diff --git a/test/lib/python2.6/site.py b/test/lib/python2.6/site.py deleted file mode 100644 index a49cfc349..000000000 --- a/test/lib/python2.6/site.py +++ /dev/null @@ -1,713 +0,0 @@ -"""Append module search paths for third-party packages to sys.path. - -**************************************************************** -* This module is automatically imported during initialization. * -**************************************************************** - -In earlier versions of Python (up to 1.5a3), scripts or modules that -needed to use site-specific modules would place ``import site'' -somewhere near the top of their code. Because of the automatic -import, this is no longer necessary (but code that does it still -works). - -This will append site-specific paths to the module search path. On -Unix, it starts with sys.prefix and sys.exec_prefix (if different) and -appends lib/python<version>/site-packages as well as lib/site-python. -It also supports the Debian convention of -lib/python<version>/dist-packages. On other platforms (mainly Mac and -Windows), it uses just sys.prefix (and sys.exec_prefix, if different, -but this is unlikely). The resulting directories, if they exist, are -appended to sys.path, and also inspected for path configuration files. - -FOR DEBIAN, this sys.path is augmented with directories in /usr/local. -Local addons go into /usr/local/lib/python<version>/site-packages -(resp. /usr/local/lib/site-python), Debian addons install into -/usr/{lib,share}/python<version>/dist-packages. - -A path configuration file is a file whose name has the form -<package>.pth; its contents are additional directories (one per line) -to be added to sys.path. Non-existing directories (or -non-directories) are never added to sys.path; no directory is added to -sys.path more than once. Blank lines and lines beginning with -'#' are skipped. Lines starting with 'import' are executed. - -For example, suppose sys.prefix and sys.exec_prefix are set to -/usr/local and there is a directory /usr/local/lib/python2.X/site-packages -with three subdirectories, foo, bar and spam, and two path -configuration files, foo.pth and bar.pth. Assume foo.pth contains the -following: - - # foo package configuration - foo - bar - bletch - -and bar.pth contains: - - # bar package configuration - bar - -Then the following directories are added to sys.path, in this order: - - /usr/local/lib/python2.X/site-packages/bar - /usr/local/lib/python2.X/site-packages/foo - -Note that bletch is omitted because it doesn't exist; bar precedes foo -because bar.pth comes alphabetically before foo.pth; and spam is -omitted because it is not mentioned in either path configuration file. - -After these path manipulations, an attempt is made to import a module -named sitecustomize, which can perform arbitrary additional -site-specific customizations. If this import fails with an -ImportError exception, it is silently ignored. - -""" - -import sys -import os -import __builtin__ -try: - set -except NameError: - from sets import Set as set - -# Prefixes for site-packages; add additional prefixes like /usr/local here -PREFIXES = [sys.prefix, sys.exec_prefix] -# Enable per user site-packages directory -# set it to False to disable the feature or True to force the feature -ENABLE_USER_SITE = None -# for distutils.commands.install -USER_SITE = None -USER_BASE = None - -_is_pypy = hasattr(sys, 'pypy_version_info') -_is_jython = sys.platform[:4] == 'java' -if _is_jython: - ModuleType = type(os) - -def makepath(*paths): - dir = os.path.join(*paths) - if _is_jython and (dir == '__classpath__' or - dir.startswith('__pyclasspath__')): - return dir, dir - dir = os.path.abspath(dir) - return dir, os.path.normcase(dir) - -def abs__file__(): - """Set all module' __file__ attribute to an absolute path""" - for m in sys.modules.values(): - if ((_is_jython and not isinstance(m, ModuleType)) or - hasattr(m, '__loader__')): - # only modules need the abspath in Jython. and don't mess - # with a PEP 302-supplied __file__ - continue - f = getattr(m, '__file__', None) - if f is None: - continue - m.__file__ = os.path.abspath(f) - -def removeduppaths(): - """ Remove duplicate entries from sys.path along with making them - absolute""" - # This ensures that the initial path provided by the interpreter contains - # only absolute pathnames, even if we're running from the build directory. - L = [] - known_paths = set() - for dir in sys.path: - # Filter out duplicate paths (on case-insensitive file systems also - # if they only differ in case); turn relative paths into absolute - # paths. - dir, dircase = makepath(dir) - if not dircase in known_paths: - L.append(dir) - known_paths.add(dircase) - sys.path[:] = L - return known_paths - -# XXX This should not be part of site.py, since it is needed even when -# using the -S option for Python. See http://www.python.org/sf/586680 -def addbuilddir(): - """Append ./build/lib.<platform> in case we're running in the build dir - (especially for Guido :-)""" - from distutils.util import get_platform - s = "build/lib.%s-%.3s" % (get_platform(), sys.version) - if hasattr(sys, 'gettotalrefcount'): - s += '-pydebug' - s = os.path.join(os.path.dirname(sys.path[-1]), s) - sys.path.append(s) - -def _init_pathinfo(): - """Return a set containing all existing directory entries from sys.path""" - d = set() - for dir in sys.path: - try: - if os.path.isdir(dir): - dir, dircase = makepath(dir) - d.add(dircase) - except TypeError: - continue - return d - -def addpackage(sitedir, name, known_paths): - """Add a new path to known_paths by combining sitedir and 'name' or execute - sitedir if it starts with 'import'""" - if known_paths is None: - _init_pathinfo() - reset = 1 - else: - reset = 0 - fullname = os.path.join(sitedir, name) - try: - f = open(fullname, "rU") - except IOError: - return - try: - for line in f: - if line.startswith("#"): - continue - if line.startswith("import"): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - finally: - f.close() - if reset: - known_paths = None - return known_paths - -def addsitedir(sitedir, known_paths=None): - """Add 'sitedir' argument to sys.path if missing and handle .pth files in - 'sitedir'""" - if known_paths is None: - known_paths = _init_pathinfo() - reset = 1 - else: - reset = 0 - sitedir, sitedircase = makepath(sitedir) - if not sitedircase in known_paths: - sys.path.append(sitedir) # Add path component - try: - names = os.listdir(sitedir) - except os.error: - return - names.sort() - for name in names: - if name.endswith(os.extsep + "pth"): - addpackage(sitedir, name, known_paths) - if reset: - known_paths = None - return known_paths - -def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix): - """Add site-packages (and possibly site-python) to sys.path""" - prefixes = [os.path.join(sys_prefix, "local"), sys_prefix] - if exec_prefix != sys_prefix: - prefixes.append(os.path.join(exec_prefix, "local")) - - for prefix in prefixes: - if prefix: - if sys.platform in ('os2emx', 'riscos') or _is_jython: - sitedirs = [os.path.join(prefix, "Lib", "site-packages")] - elif _is_pypy: - sitedirs = [os.path.join(prefix, 'site-packages')] - elif sys.platform == 'darwin' and prefix == sys_prefix: - - if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python - - sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"), - os.path.join(prefix, "Extras", "lib", "python")] - - else: # any other Python distros on OSX work this way - sitedirs = [os.path.join(prefix, "lib", - "python" + sys.version[:3], "site-packages")] - - elif os.sep == '/': - sitedirs = [os.path.join(prefix, - "lib", - "python" + sys.version[:3], - "site-packages"), - os.path.join(prefix, "lib", "site-python"), - os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")] - lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages") - if (os.path.exists(lib64_dir) and - os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]): - sitedirs.append(lib64_dir) - try: - # sys.getobjects only available in --with-pydebug build - sys.getobjects - sitedirs.insert(0, os.path.join(sitedirs[0], 'debug')) - except AttributeError: - pass - # Debian-specific dist-packages directories: - sitedirs.append(os.path.join(prefix, "lib", - "python" + sys.version[:3], - "dist-packages")) - sitedirs.append(os.path.join(prefix, "local/lib", - "python" + sys.version[:3], - "dist-packages")) - sitedirs.append(os.path.join(prefix, "lib", "dist-python")) - else: - sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")] - if sys.platform == 'darwin': - # for framework builds *only* we add the standard Apple - # locations. Currently only per-user, but /Library and - # /Network/Library could be added too - if 'Python.framework' in prefix: - home = os.environ.get('HOME') - if home: - sitedirs.append( - os.path.join(home, - 'Library', - 'Python', - sys.version[:3], - 'site-packages')) - for sitedir in sitedirs: - if os.path.isdir(sitedir): - addsitedir(sitedir, known_paths) - return None - -def check_enableusersite(): - """Check if user site directory is safe for inclusion - - The function tests for the command line flag (including environment var), - process uid/gid equal to effective uid/gid. - - None: Disabled for security reasons - False: Disabled by user (command line option) - True: Safe and enabled - """ - if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False): - return False - - if hasattr(os, "getuid") and hasattr(os, "geteuid"): - # check process uid == effective uid - if os.geteuid() != os.getuid(): - return None - if hasattr(os, "getgid") and hasattr(os, "getegid"): - # check process gid == effective gid - if os.getegid() != os.getgid(): - return None - - return True - -def addusersitepackages(known_paths): - """Add a per user site-package to sys.path - - Each user has its own python directory with site-packages in the - home directory. - - USER_BASE is the root directory for all Python versions - - USER_SITE is the user specific site-packages directory - - USER_SITE/.. can be used for data. - """ - global USER_BASE, USER_SITE, ENABLE_USER_SITE - env_base = os.environ.get("PYTHONUSERBASE", None) - - def joinuser(*args): - return os.path.expanduser(os.path.join(*args)) - - #if sys.platform in ('os2emx', 'riscos'): - # # Don't know what to put here - # USER_BASE = '' - # USER_SITE = '' - if os.name == "nt": - base = os.environ.get("APPDATA") or "~" - if env_base: - USER_BASE = env_base - else: - USER_BASE = joinuser(base, "Python") - USER_SITE = os.path.join(USER_BASE, - "Python" + sys.version[0] + sys.version[2], - "site-packages") - else: - if env_base: - USER_BASE = env_base - else: - USER_BASE = joinuser("~", ".local") - USER_SITE = os.path.join(USER_BASE, "lib", - "python" + sys.version[:3], - "site-packages") - - if ENABLE_USER_SITE and os.path.isdir(USER_SITE): - addsitedir(USER_SITE, known_paths) - if ENABLE_USER_SITE: - for dist_libdir in ("lib", "local/lib"): - user_site = os.path.join(USER_BASE, dist_libdir, - "python" + sys.version[:3], - "dist-packages") - if os.path.isdir(user_site): - addsitedir(user_site, known_paths) - return known_paths - - - -def setBEGINLIBPATH(): - """The OS/2 EMX port has optional extension modules that do double duty - as DLLs (and must use the .DLL file extension) for other extensions. - The library search path needs to be amended so these will be found - during module import. Use BEGINLIBPATH so that these are at the start - of the library search path. - - """ - dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload") - libpath = os.environ['BEGINLIBPATH'].split(';') - if libpath[-1]: - libpath.append(dllpath) - else: - libpath[-1] = dllpath - os.environ['BEGINLIBPATH'] = ';'.join(libpath) - - -def setquit(): - """Define new built-ins 'quit' and 'exit'. - These are simply strings that display a hint on how to exit. - - """ - if os.sep == ':': - eof = 'Cmd-Q' - elif os.sep == '\\': - eof = 'Ctrl-Z plus Return' - else: - eof = 'Ctrl-D (i.e. EOF)' - - class Quitter(object): - def __init__(self, name): - self.name = name - def __repr__(self): - return 'Use %s() or %s to exit' % (self.name, eof) - def __call__(self, code=None): - # Shells like IDLE catch the SystemExit, but listen when their - # stdin wrapper is closed. - try: - sys.stdin.close() - except: - pass - raise SystemExit(code) - __builtin__.quit = Quitter('quit') - __builtin__.exit = Quitter('exit') - - -class _Printer(object): - """interactive prompt objects for printing the license text, a list of - contributors and the copyright notice.""" - - MAXLINES = 23 - - def __init__(self, name, data, files=(), dirs=()): - self.__name = name - self.__data = data - self.__files = files - self.__dirs = dirs - self.__lines = None - - def __setup(self): - if self.__lines: - return - data = None - for dir in self.__dirs: - for filename in self.__files: - filename = os.path.join(dir, filename) - try: - fp = file(filename, "rU") - data = fp.read() - fp.close() - break - except IOError: - pass - if data: - break - if not data: - data = self.__data - self.__lines = data.split('\n') - self.__linecnt = len(self.__lines) - - def __repr__(self): - self.__setup() - if len(self.__lines) <= self.MAXLINES: - return "\n".join(self.__lines) - else: - return "Type %s() to see the full %s text" % ((self.__name,)*2) - - def __call__(self): - self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: - try: - for i in range(lineno, lineno + self.MAXLINES): - print self.__lines[i] - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = raw_input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break - -def setcopyright(): - """Set 'copyright' and 'credits' in __builtin__""" - __builtin__.copyright = _Printer("copyright", sys.copyright) - if _is_jython: - __builtin__.credits = _Printer( - "credits", - "Jython is maintained by the Jython developers (www.jython.org).") - elif _is_pypy: - __builtin__.credits = _Printer( - "credits", - "PyPy is maintained by the PyPy developers: http://codespeak.net/pypy") - else: - __builtin__.credits = _Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") - here = os.path.dirname(os.__file__) - __builtin__.license = _Printer( - "license", "See http://www.python.org/%.3s/license.html" % sys.version, - ["LICENSE.txt", "LICENSE"], - [os.path.join(here, os.pardir), here, os.curdir]) - - -class _Helper(object): - """Define the built-in 'help'. - This is a wrapper around pydoc.help (with a twist). - - """ - - def __repr__(self): - return "Type help() for interactive help, " \ - "or help(object) for help about object." - def __call__(self, *args, **kwds): - import pydoc - return pydoc.help(*args, **kwds) - -def sethelper(): - __builtin__.help = _Helper() - -def aliasmbcs(): - """On Windows, some default encodings are not provided by Python, - while they are always available as "mbcs" in each locale. Make - them usable by aliasing to "mbcs" in such a case.""" - if sys.platform == 'win32': - import locale, codecs - enc = locale.getdefaultlocale()[1] - if enc.startswith('cp'): # "cp***" ? - try: - codecs.lookup(enc) - except LookupError: - import encodings - encodings._cache[enc] = encodings._unknown - encodings.aliases.aliases[enc] = 'mbcs' - -def setencoding(): - """Set the string encoding used by the Unicode implementation. The - default is 'ascii', but if you're willing to experiment, you can - change this.""" - encoding = "ascii" # Default value set by _PyUnicode_Init() - if 0: - # Enable to support locale aware default string encodings. - import locale - loc = locale.getdefaultlocale() - if loc[1]: - encoding = loc[1] - if 0: - # Enable to switch off string to Unicode coercion and implicit - # Unicode to string conversion. - encoding = "undefined" - if encoding != "ascii": - # On Non-Unicode builds this will raise an AttributeError... - sys.setdefaultencoding(encoding) # Needs Python Unicode build ! - - -def execsitecustomize(): - """Run custom site specific code, if available.""" - try: - import sitecustomize - except ImportError: - pass - -def virtual_install_main_packages(): - f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt')) - sys.real_prefix = f.read().strip() - f.close() - pos = 2 - if sys.path[0] == '': - pos += 1 - if sys.platform == 'win32': - paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')] - elif _is_jython: - paths = [os.path.join(sys.real_prefix, 'Lib')] - elif _is_pypy: - cpyver = '%d.%d.%d' % sys.version_info[:3] - paths = [os.path.join(sys.real_prefix, 'lib_pypy'), - os.path.join(sys.real_prefix, 'lib-python', 'modified-%s' % cpyver), - os.path.join(sys.real_prefix, 'lib-python', cpyver)] - else: - paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])] - lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3]) - if os.path.exists(lib64_path): - paths.append(lib64_path) - # This is hardcoded in the Python executable, but relative to sys.prefix: - plat_path = os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], - 'plat-%s' % sys.platform) - if os.path.exists(plat_path): - paths.append(plat_path) - # This is hardcoded in the Python executable, but - # relative to sys.prefix, so we have to fix up: - for path in list(paths): - tk_dir = os.path.join(path, 'lib-tk') - if os.path.exists(tk_dir): - paths.append(tk_dir) - - # These are hardcoded in the Apple's Python executable, - # but relative to sys.prefix, so we have to fix them up: - if sys.platform == 'darwin': - hardcoded_paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3], module) - for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')] - - for path in hardcoded_paths: - if os.path.exists(path): - paths.append(path) - - sys.path.extend(paths) - -def force_global_eggs_after_local_site_packages(): - """ - Force easy_installed eggs in the global environment to get placed - in sys.path after all packages inside the virtualenv. This - maintains the "least surprise" result that packages in the - virtualenv always mask global packages, never the other way - around. - - """ - egginsert = getattr(sys, '__egginsert', 0) - for i, path in enumerate(sys.path): - if i > egginsert and path.startswith(sys.prefix): - egginsert = i - sys.__egginsert = egginsert + 1 - -def virtual_addsitepackages(known_paths): - force_global_eggs_after_local_site_packages() - return addsitepackages(known_paths, sys_prefix=sys.real_prefix) - -def fixclasspath(): - """Adjust the special classpath sys.path entries for Jython. These - entries should follow the base virtualenv lib directories. - """ - paths = [] - classpaths = [] - for path in sys.path: - if path == '__classpath__' or path.startswith('__pyclasspath__'): - classpaths.append(path) - else: - paths.append(path) - sys.path = paths - sys.path.extend(classpaths) - -def execusercustomize(): - """Run custom user specific code, if available.""" - try: - import usercustomize - except ImportError: - pass - - -def main(): - global ENABLE_USER_SITE - virtual_install_main_packages() - abs__file__() - paths_in_sys = removeduppaths() - if (os.name == "posix" and sys.path and - os.path.basename(sys.path[-1]) == "Modules"): - addbuilddir() - if _is_jython: - fixclasspath() - GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt')) - if not GLOBAL_SITE_PACKAGES: - ENABLE_USER_SITE = False - if ENABLE_USER_SITE is None: - ENABLE_USER_SITE = check_enableusersite() - paths_in_sys = addsitepackages(paths_in_sys) - paths_in_sys = addusersitepackages(paths_in_sys) - if GLOBAL_SITE_PACKAGES: - paths_in_sys = virtual_addsitepackages(paths_in_sys) - if sys.platform == 'os2emx': - setBEGINLIBPATH() - setquit() - setcopyright() - sethelper() - aliasmbcs() - setencoding() - execsitecustomize() - if ENABLE_USER_SITE: - execusercustomize() - # Remove sys.setdefaultencoding() so that users cannot change the - # encoding after initialization. The test for presence is needed when - # this module is run as a script, because this code is executed twice. - if hasattr(sys, "setdefaultencoding"): - del sys.setdefaultencoding - -main() - -def _script(): - help = """\ - %s [--user-base] [--user-site] - - Without arguments print some useful information - With arguments print the value of USER_BASE and/or USER_SITE separated - by '%s'. - - Exit codes with --user-base or --user-site: - 0 - user site directory is enabled - 1 - user site directory is disabled by user - 2 - uses site directory is disabled by super user - or for security reasons - >2 - unknown error - """ - args = sys.argv[1:] - if not args: - print "sys.path = [" - for dir in sys.path: - print " %r," % (dir,) - print "]" - def exists(path): - if os.path.isdir(path): - return "exists" - else: - return "doesn't exist" - print "USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE)) - print "USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE)) - print "ENABLE_USER_SITE: %r" % ENABLE_USER_SITE - sys.exit(0) - - buffer = [] - if '--user-base' in args: - buffer.append(USER_BASE) - if '--user-site' in args: - buffer.append(USER_SITE) - - if buffer: - print os.pathsep.join(buffer) - if ENABLE_USER_SITE: - sys.exit(0) - elif ENABLE_USER_SITE is False: - sys.exit(1) - elif ENABLE_USER_SITE is None: - sys.exit(2) - else: - sys.exit(3) - else: - import textwrap - print textwrap.dedent(help % (sys.argv[0], os.pathsep)) - sys.exit(10) - -if __name__ == '__main__': - _script() diff --git a/test/lib/python2.6/sre.py b/test/lib/python2.6/sre.py deleted file mode 120000 index 3ade4e80e..000000000 --- a/test/lib/python2.6/sre.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_compile.py b/test/lib/python2.6/sre_compile.py deleted file mode 120000 index fc13a37d5..000000000 --- a/test/lib/python2.6/sre_compile.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_compile.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_constants.py b/test/lib/python2.6/sre_constants.py deleted file mode 120000 index 98b0d5543..000000000 --- a/test/lib/python2.6/sre_constants.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_constants.py \ No newline at end of file diff --git a/test/lib/python2.6/sre_parse.py b/test/lib/python2.6/sre_parse.py deleted file mode 120000 index 76b24c687..000000000 --- a/test/lib/python2.6/sre_parse.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/sre_parse.py \ No newline at end of file diff --git a/test/lib/python2.6/stat.py b/test/lib/python2.6/stat.py deleted file mode 120000 index 68d61e662..000000000 --- a/test/lib/python2.6/stat.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/stat.py \ No newline at end of file diff --git a/test/lib/python2.6/types.py b/test/lib/python2.6/types.py deleted file mode 120000 index 72b00ba35..000000000 --- a/test/lib/python2.6/types.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/types.py \ No newline at end of file diff --git a/test/lib/python2.6/warnings.py b/test/lib/python2.6/warnings.py deleted file mode 120000 index 1af4e2833..000000000 --- a/test/lib/python2.6/warnings.py +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/warnings.py \ No newline at end of file -- cgit From dcfa9670a669ff881865750b2f96f62195dce2df Mon Sep 17 00:00:00 2001 From: Christian Berendt <berendt@b1-systems.de> Date: Sat, 26 Feb 2011 19:19:02 +0100 Subject: renamed flag from maximum_... to max_... --- nova/virt/disk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 8789d8123..2bded07a4 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -40,7 +40,7 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') flags.DEFINE_integer('timeout_nbd', 10, 'time to wait for a NBD device coming up') -flags.DEFINE_integer('maximum_nbd_devices', 16, +flags.DEFINE_integer('max_nbd_devices', 16, 'maximum number of possible nbd devices') @@ -143,7 +143,7 @@ def _unlink_device(device, nbd): utils.execute('sudo losetup --detach %s' % device) -_DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.maximum_nbd_devices)] +_DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.max_nbd_devices)] def _allocate_device(): -- cgit From a457595224a5ca5cdb0191ba2f6fa542d16e18f5 Mon Sep 17 00:00:00 2001 From: Nirmal Ranganathan <nirmal.ranganathan@rackspace.com> Date: Mon, 28 Feb 2011 11:25:14 -0600 Subject: Removed extraneous newline --- nova/tests/test_compute.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index e338a7cfa..949b5e6eb 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -274,4 +274,3 @@ class ComputeTestCase(test.TestCase): type = instance_types.get_by_flavor_id("1") self.assertEqual(type, 'm1.tiny') - -- cgit From 4572ffcf734b734870b90497063fc27e7642f67c Mon Sep 17 00:00:00 2001 From: Naveed Massjouni <naveedm9@gmail.com> Date: Mon, 28 Feb 2011 19:56:46 -0500 Subject: No reason to initialize metadata twice. --- nova/api/openstack/servers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 7d20f681b..69273ad7b 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -72,8 +72,6 @@ def _translate_detail_keys(inst): public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') inst_dict['addresses']['public'] = public_ips - inst_dict['metadata'] = {} - # Return the metadata as a dictionary metadata = {} for item in inst['metadata']: -- cgit