From c524508bc58aa561b81c3133526c981cce835a59 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 20 Sep 2010 01:50:08 -0700 Subject: added rescue mode support and made reboot work from any state --- nova/compute/manager.py | 37 +++++++++--- nova/virt/fake.py | 12 ++++ nova/virt/libvirt.rescue.qemu.xml.template | 33 +++++++++++ nova/virt/libvirt.rescue.uml.xml.template | 26 +++++++++ nova/virt/libvirt_conn.py | 94 +++++++++++++++++++++++++----- 5 files changed, 179 insertions(+), 23 deletions(-) create mode 100644 nova/virt/libvirt.rescue.qemu.xml.template create mode 100644 nova/virt/libvirt.rescue.uml.xml.template diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 954227b42..56e08f881 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -122,17 +122,8 @@ class ComputeManager(manager.Manager): @exception.wrap_exception def reboot_instance(self, context, instance_id): """Reboot an instance on this server.""" - self._update_state(context, instance_id) instance_ref = self.db.instance_get(context, instance_id) - if instance_ref['state'] != power_state.RUNNING: - raise exception.Error( - 'trying to reboot a non-running' - 'instance: %s (state: %s excepted: %s)' % - (instance_ref['str_id'], - instance_ref['state'], - power_state.RUNNING)) - logging.debug('instance %s: rebooting', instance_ref['name']) self.db.instance_set_state(context, instance_id, @@ -141,6 +132,34 @@ class ComputeManager(manager.Manager): yield self.driver.reboot(instance_ref) self._update_state(context, instance_id) + @defer.inlineCallbacks + @exception.wrap_exception + def rescue_instance(self, context, instance_id): + """Rescue an instance on this server.""" + instance_ref = self.db.instance_get(context, instance_id) + + logging.debug('instance %s: rescuing', instance_ref['name']) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'rescuing') + yield self.driver.rescue(instance_ref) + self._update_state(context, instance_id) + + @defer.inlineCallbacks + @exception.wrap_exception + def unrescue_instance(self, context, instance_id): + """Rescue an instance on this server.""" + instance_ref = self.db.instance_get(context, instance_id) + + logging.debug('instance %s: unrescuing', instance_ref['name']) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'unrescuing') + yield self.driver.unrescue(instance_ref) + self._update_state(context, instance_id) + @exception.wrap_exception def get_console_output(self, context, instance_id): """Send the console output for an instance.""" diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 4ae6afcc4..3e88060f6 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -119,6 +119,18 @@ class FakeConnection(object): """ return defer.succeed(None) + def rescue(self, instance): + """ + Rescue the specified instance. + """ + return defer.succeed(None) + + def unrescue(self, instance): + """ + Unrescue the specified instance. + """ + return defer.succeed(None) + def destroy(self, instance): """ Destroy (shutdown and delete) the specified instance. diff --git a/nova/virt/libvirt.rescue.qemu.xml.template b/nova/virt/libvirt.rescue.qemu.xml.template new file mode 100644 index 000000000..164326452 --- /dev/null +++ b/nova/virt/libvirt.rescue.qemu.xml.template @@ -0,0 +1,33 @@ + + %(name)s + + hvm + %(basepath)s/rescue-kernel + %(basepath)s/rescue-ramdisk + root=/dev/vda1 console=ttyS0 + + + + + %(memory_kb)s + %(vcpus)s + + + + + + + + + + + + + + + + + + + + diff --git a/nova/virt/libvirt.rescue.uml.xml.template b/nova/virt/libvirt.rescue.uml.xml.template new file mode 100644 index 000000000..836f47532 --- /dev/null +++ b/nova/virt/libvirt.rescue.uml.xml.template @@ -0,0 +1,26 @@ + + %(name)s + %(memory_kb)s + + %(type)s + /usr/bin/linux + /dev/ubda1 + + + + + + + + + + + + + + + + + + + diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index d868e083c..b9edc8e85 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -44,6 +44,16 @@ libxml2 = None FLAGS = flags.FLAGS +flags.DEFINE_string('libvirt_rescue_xml_template', + utils.abspath('virt/libvirt.rescue.qemu.xml.template'), + 'Libvirt RESCUE XML Template for QEmu/KVM') +flags.DEFINE_string('libvirt_rescue_uml_xml_template', + utils.abspath('virt/libvirt.rescue.uml.xml.template'), + 'Libvirt RESCUE XML Template for user-mode-linux') +# TODO(vish): These flags should probably go into a shared location +flags.DEFINE_string('rescue_image_id', 'ami-rescue', 'Rescue ami image') +flags.DEFINE_string('rescue_kernel_id', 'aki-rescue', 'Rescue aki image') +flags.DEFINE_string('rescue_ramdisk_id', 'ari-rescue', 'Rescue ari image') flags.DEFINE_string('libvirt_xml_template', utils.abspath('virt/libvirt.qemu.xml.template'), 'Libvirt XML Template for QEmu/KVM') @@ -76,9 +86,12 @@ def get_connection(read_only): class LibvirtConnection(object): def __init__(self, read_only): - self.libvirt_uri, template_file = self.get_uri_and_template() + (self.libvirt_uri, + template_file, + rescue_file)= self.get_uri_and_templates() self.libvirt_xml = open(template_file).read() + self.rescue_xml = open(rescue_file).read() self._wrapped_conn = None self.read_only = read_only @@ -100,14 +113,16 @@ class LibvirtConnection(object): return False raise - def get_uri_and_template(self): + def get_uri_and_templates(self): if FLAGS.libvirt_type == 'uml': uri = FLAGS.libvirt_uri or 'uml:///system' template_file = FLAGS.libvirt_uml_xml_template + rescue_file = FLAGS.libvirt_rescue_uml_xml_template else: uri = FLAGS.libvirt_uri or 'qemu:///system' template_file = FLAGS.libvirt_xml_template - return uri, template_file + rescue_file = FLAGS.libvirt_rescue_xml_template + return uri, template_file, rescue_file def _connect(self, uri, read_only): auth = [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_NOECHOPROMPT], @@ -123,7 +138,7 @@ class LibvirtConnection(object): return [self._conn.lookupByID(x).name() for x in self._conn.listDomainsID()] - def destroy(self, instance): + def destroy(self, instance, cleanup=True): try: virt_dom = self._conn.lookupByName(instance['name']) virt_dom.destroy() @@ -131,7 +146,8 @@ class LibvirtConnection(object): pass # If the instance is already terminated, we're still happy d = defer.Deferred() - d.addCallback(lambda _: self._cleanup(instance)) + if cleanup: + d.addCallback(lambda _: self._cleanup(instance)) # FIXME: What does this comment mean? # TODO(termie): short-circuit me for tests # WE'LL save this for when we do shutdown, @@ -181,8 +197,8 @@ class LibvirtConnection(object): @defer.inlineCallbacks @exception.wrap_exception def reboot(self, instance): + yield self.destroy(instance, False) xml = self.to_xml(instance) - yield self._conn.lookupByName(instance['name']).destroy() yield self._conn.createXML(xml, 0) d = defer.Deferred() @@ -206,6 +222,46 @@ class LibvirtConnection(object): timer.start(interval=0.5, now=True) yield d + @defer.inlineCallbacks + @exception.wrap_exception + def rescue(self, instance): + yield self.destroy(instance, False) + + xml = self.to_xml(instance, rescue=True) + rescue_images = {'image_id': FLAGS.rescue_image_id, + 'kernel_id': FLAGS.rescue_kernel_id, + 'ramdisk_id': FLAGS.rescue_ramdisk_id} + yield self._create_image(instance, xml, 'rescue-', rescue_images) + yield self._conn.createXML(xml, 0) + + d = defer.Deferred() + timer = task.LoopingCall(f=None) + def _wait_for_rescue(): + try: + state = self.get_info(instance['name'])['state'] + db.instance_set_state(None, instance['id'], state) + if state == power_state.RUNNING: + logging.debug('instance %s: rescued', instance['name']) + timer.stop() + d.callback(None) + except Exception, exn: + logging.error('_wait_for_rescue failed: %s', exn) + db.instance_set_state(None, + instance['id'], + power_state.SHUTDOWN) + timer.stop() + d.callback(None) + timer.f = _wait_for_rescue + timer.start(interval=0.5, now=True) + yield d + + @defer.inlineCallbacks + @exception.wrap_exception + def unrescue(self, instance): + # NOTE(vish): Because reboot destroys and recreates an instance using + # the normal xml file, we can just call reboot here + yield self.reboot(instance) + @defer.inlineCallbacks @exception.wrap_exception def spawn(self, instance): @@ -243,15 +299,16 @@ class LibvirtConnection(object): yield local_d @defer.inlineCallbacks - def _create_image(self, inst, libvirt_xml): + def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): # syntactic nicety - basepath = lambda fname='': os.path.join(FLAGS.instances_path, + basepath = lambda fname='', prefix=prefix: os.path.join( + FLAGS.instances_path, inst['name'], - fname) + prefix + fname) # ensure directories exist and are writable - yield process.simple_execute('mkdir -p %s' % basepath()) - yield process.simple_execute('chmod 0777 %s' % basepath()) + yield process.simple_execute('mkdir -p %s' % basepath(prefix='')) + yield process.simple_execute('chmod 0777 %s' % basepath(prefix='')) # TODO(termie): these are blocking calls, it would be great @@ -261,11 +318,17 @@ class LibvirtConnection(object): f.write(libvirt_xml) f.close() - os.close(os.open(basepath('console.log'), os.O_CREAT | os.O_WRONLY, 0660)) + # NOTE(vish): No need add the prefix to console.log + os.close(os.open(basepath('console.log', ''), + os.O_CREAT | os.O_WRONLY, 0660)) user = manager.AuthManager().get_user(inst['user_id']) project = manager.AuthManager().get_project(inst['project_id']) + if not disk_images: + disk_images = {'image_id': inst['image_id'], + 'kernel_id': inst['kernel_id'], + 'ramdisk_id': inst['ramdisk_id']} if not os.path.exists(basepath('disk')): yield images.fetch(inst.image_id, basepath('disk-raw'), user, project) if not os.path.exists(basepath('kernel')): @@ -311,7 +374,7 @@ class LibvirtConnection(object): yield process.simple_execute('sudo chown root %s' % basepath('disk')) - def to_xml(self, instance): + def to_xml(self, instance, rescue=False): # TODO(termie): cache? logging.debug('instance %s: starting toXML method', instance['name']) network = db.project_get_network(None, instance['project_id']) @@ -325,7 +388,10 @@ class LibvirtConnection(object): 'vcpus': instance_type['vcpus'], 'bridge_name': network['bridge'], 'mac_address': instance['mac_address']} - libvirt_xml = self.libvirt_xml % xml_info + if rescue: + libvirt_xml = self.rescue_xml % xml_info + else: + libvirt_xml = self.libvirt_xml % xml_info logging.debug('instance %s: finished toXML method', instance['name']) return libvirt_xml -- cgit From 198af0ef9e65bc4c2efe74b9d93cf40210eb77bc Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Thu, 21 Oct 2010 14:29:34 -0400 Subject: Moves db writes into compute manager class. Cleans up sqlalchemy model/api to remove redundant calls for updating what is really a dict. --- nova/api/ec2/cloud.py | 43 ++++++++++++--------- nova/api/openstack/servers.py | 25 ++++++------ nova/compute/manager.py | 35 +++++++++++++++++ nova/db/sqlalchemy/api.py | 66 +++++++++++--------------------- nova/db/sqlalchemy/models.py | 10 +++++ nova/tests/api/openstack/fakes.py | 1 + nova/tests/api/openstack/test_servers.py | 4 +- 7 files changed, 104 insertions(+), 80 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6d4f58499..096ddf668 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -97,6 +97,7 @@ class CloudController(object): """ def __init__(self): self.network_manager = utils.import_object(FLAGS.network_manager) + self.compute_manager = utils.import_object(FLAGS.compute_manager) self.setup() def __str__(self): @@ -846,27 +847,29 @@ class CloudController(object): elevated = context.elevated() for num in range(num_instances): - instance_ref = db.instance_create(context, base_options) - inst_id = instance_ref['id'] + + instance_data = base_options + instance_data['mac_address'] = utils.generate_mac() + instance_data['launch_index'] = num - for security_group_id in security_groups: - db.instance_add_security_group(elevated, - inst_id, - security_group_id) + instance_ref = self.compute_manager.create_instance(context, + instance_data, + security_groups) - inst = {} - inst['mac_address'] = utils.generate_mac() - inst['launch_index'] = num internal_id = instance_ref['internal_id'] ec2_id = internal_id_to_ec2_id(internal_id) - inst['hostname'] = ec2_id - db.instance_update(context, inst_id, inst) + instance_ref['hostname'] = ec2_id + + self.compute_manager.update_instance(context, + instance_ref['id'], + instance_ref) + # TODO(vish): This probably should be done in the scheduler # or in compute as a call. The network should be # allocated after the host is assigned and setup # can happen at the same time. address = self.network_manager.allocate_fixed_ip(context, - inst_id, + instance_ref['id'], vpn) network_topic = self._get_network_topic(context) rpc.cast(elevated, @@ -878,9 +881,9 @@ class CloudController(object): FLAGS.scheduler_topic, {"method": "run_instance", "args": {"topic": FLAGS.compute_topic, - "instance_id": inst_id}}) + "instance_id": instance_ref['id']}}) logging.debug("Casting to scheduler for %s/%s's instance %s" % - (context.project.name, context.user.name, inst_id)) + (context.project.name, context.user.name, instance_ref['id'])) return self._format_run_instances(context, reservation_id) @@ -907,11 +910,13 @@ class CloudController(object): id_str) continue now = datetime.datetime.utcnow() - db.instance_update(context, - instance_ref['id'], - {'state_description': 'terminating', - 'state': 0, - 'terminated_at': now}) + updated_data = {'state_description': 'terminating', + 'state': 0, + 'terminated_at': now} + self.compute_manager.update_instance(context, + instance_ref['id'], + updated_data) + # FIXME(ja): where should network deallocate occur? address = db.instance_get_floating_address(context, instance_ref['id']) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index a73591ccc..6ce364eb7 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -94,6 +94,7 @@ class Controller(wsgi.Controller): db_driver = FLAGS.db_driver self.db_driver = utils.import_object(db_driver) self.network_manager = utils.import_object(FLAGS.network_manager) + self.compute_manager = utils.import_object(FLAGS.compute_manager) super(Controller, self).__init__() def index(self, req): @@ -241,34 +242,30 @@ class Controller(wsgi.Controller): inst['memory_mb'] = flavor['memory_mb'] inst['vcpus'] = flavor['vcpus'] inst['local_gb'] = flavor['local_gb'] - - ref = self.db_driver.instance_create(ctxt, inst) - inst['id'] = ref.internal_id - inst['mac_address'] = utils.generate_mac() - - #TODO(dietz) is this necessary? inst['launch_index'] = 0 - inst['hostname'] = str(ref.internal_id) - self.db_driver.instance_update(ctxt, inst['id'], inst) + ref = self.compute_manager.create_instance(ctxt, inst) + inst['id'] = ref['internal_id'] + + inst['hostname'] = str(ref['internal_id']) + self.compute_manager.update_instance(ctxt, inst['id'], inst) - network_manager = utils.import_object(FLAGS.network_manager) - address = network_manager.allocate_fixed_ip(ctxt, - inst['id']) + address = self.network_manager.allocate_fixed_ip(ctxt, + inst['id']) # TODO(vish): This probably should be done in the scheduler # network is setup when host is assigned - network_topic = self._get_network_topic(ctxt, network_manager) + network_topic = self._get_network_topic(ctxt) rpc.call(ctxt, network_topic, {"method": "setup_fixed_ip", "args": {"address": address}}) return inst - def _get_network_topic(self, context, network_manager): + def _get_network_topic(self, context): """Retrieves the network host for a project""" - network_ref = network_manager.get_network(context) + network_ref = self.network_manager.get_network(context) host = network_ref['host'] if not host: host = rpc.call(context, diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 523bb8893..c752d954b 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -67,6 +67,41 @@ class ComputeManager(manager.Manager): def refresh_security_group(self, context, security_group_id, **_kwargs): yield self.driver.refresh_security_group(security_group_id) + + def create_instance(self, context, instance_data, security_groups=[]): + """Creates the instance in the datastore and returns the + new instance as a mapping + + :param context: The security context + :param instance_data: mapping of instance options + :param security_groups: list of security group ids to + attach to the instance + + :retval Returns a mapping of the instance information + that has just been created + + """ + instance_ref = self.db.instance_create(context, instance_data) + inst_id = instance_ref['id'] + + elevated = context.elevated() + for security_group_id in security_groups: + self.db.instance_add_security_group(elevated, + inst_id, + security_group_id) + return instance_ref + + def update_instance(self, context, instance_id, instance_data): + """Updates the instance in the datastore + + :param context: The security context + :param instance_data: mapping of instance options + + :retval None + + """ + self.db.instance_update(context, instance_id, instance_data) + @defer.inlineCallbacks @exception.wrap_exception def run_instance(self, context, instance_id, **_kwargs): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 209d6e51f..74fd0fdc8 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -235,8 +235,7 @@ def service_get_by_args(context, host, binary): @require_admin_context def service_create(context, values): service_ref = models.Service() - for (key, value) in values.iteritems(): - service_ref[key] = value + service_ref.update(values) service_ref.save() return service_ref @@ -246,8 +245,7 @@ def service_update(context, service_id, values): session = get_session() with session.begin(): service_ref = service_get(context, service_id, session=session) - for (key, value) in values.iteritems(): - service_ref[key] = value + service_ref.update(values) service_ref.save(session=session) @@ -278,8 +276,7 @@ def floating_ip_allocate_address(context, host, project_id): @require_context def floating_ip_create(context, values): floating_ip_ref = models.FloatingIp() - for (key, value) in values.iteritems(): - floating_ip_ref[key] = value + floating_ip_ref.update(values) floating_ip_ref.save() return floating_ip_ref['address'] @@ -450,8 +447,7 @@ def fixed_ip_associate_pool(context, network_id, instance_id): @require_context def fixed_ip_create(_context, values): fixed_ip_ref = models.FixedIp() - for (key, value) in values.iteritems(): - fixed_ip_ref[key] = value + fixed_ip_ref.update(values) fixed_ip_ref.save() return fixed_ip_ref['address'] @@ -520,8 +516,7 @@ def fixed_ip_update(context, address, values): fixed_ip_ref = fixed_ip_get_by_address(context, address, session=session) - for (key, value) in values.iteritems(): - fixed_ip_ref[key] = value + fixed_ip_ref.update(values) fixed_ip_ref.save(session=session) @@ -534,8 +529,7 @@ def fixed_ip_update(context, address, values): @require_context def instance_create(context, values): instance_ref = models.Instance() - for (key, value) in values.iteritems(): - instance_ref[key] = value + instance_ref.update(values) session = get_session() with session.begin(): @@ -727,8 +721,7 @@ def instance_update(context, instance_id, values): session = get_session() with session.begin(): instance_ref = instance_get(context, instance_id, session=session) - for (key, value) in values.iteritems(): - instance_ref[key] = value + instance_ref.update(values) instance_ref.save(session=session) @@ -750,8 +743,7 @@ def instance_add_security_group(context, instance_id, security_group_id): @require_context def key_pair_create(context, values): key_pair_ref = models.KeyPair() - for (key, value) in values.iteritems(): - key_pair_ref[key] = value + key_pair_ref.update(values) key_pair_ref.save() return key_pair_ref @@ -866,8 +858,7 @@ def network_count_reserved_ips(context, network_id): @require_admin_context def network_create_safe(context, values): network_ref = models.Network() - for (key, value) in values.iteritems(): - network_ref[key] = value + network_ref.update(values) try: network_ref.save() return network_ref @@ -976,8 +967,7 @@ def network_update(context, network_id, values): session = get_session() with session.begin(): network_ref = network_get(context, network_id, session=session) - for (key, value) in values.iteritems(): - network_ref[key] = value + network_ref.update(values) network_ref.save(session=session) @@ -1027,8 +1017,7 @@ def export_device_count(context): @require_admin_context def export_device_create_safe(context, values): export_device_ref = models.ExportDevice() - for (key, value) in values.iteritems(): - export_device_ref[key] = value + export_device_ref.update(values) try: export_device_ref.save() return export_device_ref @@ -1054,8 +1043,7 @@ def auth_get_token(_context, token_hash): def auth_create_token(_context, token): tk = models.AuthToken() - for k,v in token.iteritems(): - tk[k] = v + tk.update(token) tk.save() return tk @@ -1081,8 +1069,7 @@ def quota_get(context, project_id, session=None): @require_admin_context def quota_create(context, values): quota_ref = models.Quota() - for (key, value) in values.iteritems(): - quota_ref[key] = value + quota_ref.update(values) quota_ref.save() return quota_ref @@ -1092,8 +1079,7 @@ def quota_update(context, project_id, values): session = get_session() with session.begin(): quota_ref = quota_get(context, project_id, session=session) - for (key, value) in values.iteritems(): - quota_ref[key] = value + quota_ref.update(values) quota_ref.save(session=session) @@ -1141,8 +1127,7 @@ def volume_attached(context, volume_id, instance_id, mountpoint): @require_context def volume_create(context, values): volume_ref = models.Volume() - for (key, value) in values.iteritems(): - volume_ref[key] = value + volume_ref.update(values) session = get_session() with session.begin(): @@ -1298,8 +1283,7 @@ def volume_update(context, volume_id, values): session = get_session() with session.begin(): volume_ref = volume_get(context, volume_id, session=session) - for (key, value) in values.iteritems(): - volume_ref[key] = value + volume_ref.update(values) volume_ref.save(session=session) @@ -1392,8 +1376,7 @@ def security_group_create(context, values): # FIXME(devcamcar): Unless I do this, rules fails with lazy load exception # once save() is called. This will get cleaned up in next orm pass. security_group_ref.rules - for (key, value) in values.iteritems(): - security_group_ref[key] = value + security_group_ref.update(values) security_group_ref.save() return security_group_ref @@ -1446,8 +1429,7 @@ def security_group_rule_get(context, security_group_rule_id, session=None): @require_context def security_group_rule_create(context, values): security_group_rule_ref = models.SecurityGroupIngressRule() - for (key, value) in values.iteritems(): - security_group_rule_ref[key] = value + security_group_rule_ref.update(values) security_group_rule_ref.save() return security_group_rule_ref @@ -1498,8 +1480,7 @@ def user_get_by_access_key(context, access_key, session=None): @require_admin_context def user_create(_context, values): user_ref = models.User() - for (key, value) in values.iteritems(): - user_ref[key] = value + user_ref.update(values) user_ref.save() return user_ref @@ -1527,8 +1508,7 @@ def user_get_all(context): def project_create(_context, values): project_ref = models.Project() - for (key, value) in values.iteritems(): - project_ref[key] = value + project_ref.update(values) project_ref.save() return project_ref @@ -1590,8 +1570,7 @@ def user_update(context, user_id, values): session = get_session() with session.begin(): user_ref = user_get(context, user_id, session=session) - for (key, value) in values.iteritems(): - user_ref[key] = value + user_ref.update(values) user_ref.save(session=session) @@ -1599,8 +1578,7 @@ def project_update(context, project_id, values): session = get_session() with session.begin(): project_ref = project_get(context, project_id, session=session) - for (key, value) in values.iteritems(): - project_ref[key] = value + project_ref.update(values) project_ref.save(session=session) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index a63bca2b0..853c320e4 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -90,6 +90,16 @@ class NovaBase(object): n = self._i.next().name return n, getattr(self, n) + def update(self, values): + """Make the model object behave like a dict""" + for k, v in values.iteritems(): + setattr(self, k, v) + + def iteritems(self): + """Make the model object behave like a dict""" + return iter(self) + + # TODO(vish): Store images in the database instead of file system #class Image(BASE, NovaBase): # """Represents an image in the datastore""" diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 14170fbb2..f12c7b610 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -30,6 +30,7 @@ from nova import exception as exc import nova.api.openstack.auth from nova.image import service from nova.image.services import glance +from nova.tests import fake_flags from nova.wsgi import Router diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index d1ee533b6..f4a09fd97 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -92,9 +92,7 @@ class ServersTest(unittest.TestCase): pass def instance_create(context, inst): - class Foo(object): - internal_id = 1 - return Foo() + return {'id': 1, 'internal_id': 1} def fake_method(*args, **kwargs): pass -- cgit From 5fdcbd6c831cb3ab2cb04c0eecc68e4b0b9d5a66 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 15:06:42 -0700 Subject: update tests --- nova/tests/virt_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index ce78d450c..d49383fb7 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -91,7 +91,7 @@ class LibvirtConnTestCase(test.TrialTestCase): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, template = conn.get_uri_and_template() + uri, _template, _rescue = conn.get_uri_and_templates() self.assertEquals(uri, expected_uri) xml = conn.to_xml(instance_ref) @@ -114,7 +114,7 @@ class LibvirtConnTestCase(test.TrialTestCase): for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, template = conn.get_uri_and_template() + uri, _template, _rescue = conn.get_uri_and_templates() self.assertEquals(uri, testuri) def tearDown(self): -- cgit From eecef70fcdd173cc54ad8ac7edba9e9b31855134 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 15:37:55 -0700 Subject: add methods to cloud for rescue and unrescue --- nova/api/cloud.py | 18 ++++++++++++++++++ nova/api/ec2/cloud.py | 17 +++++++++++++++-- nova/virt/libvirt_conn.py | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/nova/api/cloud.py b/nova/api/cloud.py index aa84075dc..46b342d32 100644 --- a/nova/api/cloud.py +++ b/nova/api/cloud.py @@ -36,3 +36,21 @@ def reboot(instance_id, context=None): db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "reboot_instance", "args": {"instance_id": instance_ref['id']}}) + +def rescue(instance_id, context): + """Rescue the given instance.""" + instance_ref = db.instance_get_by_internal_id(context, instance_id) + host = instance_ref['host'] + rpc.cast(context, + db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "rescue_instance", + "args": {"instance_id": instance_ref['id']}}) + +def unrescue(instance_id, context): + """Unrescue the given instance.""" + instance_ref = db.instance_get_by_internal_id(context, instance_id) + host = instance_ref['host'] + rpc.cast(context, + db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "unrescue_instance", + "args": {"instance_id": instance_ref['id']}}) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 784697b01..73f0dcc16 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -932,8 +932,21 @@ class CloudController(object): def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" - for id_str in instance_id: - cloud.reboot(id_str, context=context) + for ec2_id in instance_id: + internal_id = ec2_id_to_internal_id(ec2_id) + cloud.reboot(internal_id, context=context) + return True + + def rescue_instance(self, context, instance_id, **kwargs): + """This is an extension to the normal ec2_api""" + internal_id = ec2_id_to_internal_id(instance_id) + cloud.rescue(internal_id, context=context) + return True + + def unrescue_instance(self, context, instance_id, **kwargs): + """This is an extension to the normal ec2_api""" + internal_id = ec2_id_to_internal_id(instance_id) + cloud.unrescue(internal_id, context=context) return True def update_instance(self, context, ec2_id, **kwargs): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 1c75150ea..7d66d8454 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -484,7 +484,7 @@ class LibvirtConnection(object): try: virt_dom = self._conn.lookupByName(instance_name) except: - raise NotFound("Instance %s not found" % instance_name) + raise exception.NotFound("Instance %s not found" % instance_name) (state, max_mem, mem, num_cpu, cpu_time) = virt_dom.info() return {'state': state, 'max_mem': max_mem, -- cgit From 4948fed24d5e16d95f237ec95c1cd305fcc4df95 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 16:04:35 -0700 Subject: pep 8 cleanup and typo in resize --- nova/api/cloud.py | 2 ++ nova/virt/libvirt_conn.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/api/cloud.py b/nova/api/cloud.py index 46b342d32..b8f15019f 100644 --- a/nova/api/cloud.py +++ b/nova/api/cloud.py @@ -37,6 +37,7 @@ def reboot(instance_id, context=None): {"method": "reboot_instance", "args": {"instance_id": instance_ref['id']}}) + def rescue(instance_id, context): """Rescue the given instance.""" instance_ref = db.instance_get_by_internal_id(context, instance_id) @@ -46,6 +47,7 @@ def rescue(instance_id, context): {"method": "rescue_instance", "args": {"instance_id": instance_ref['id']}}) + def unrescue(instance_id, context): """Unrescue the given instance.""" instance_ref = db.instance_get_by_internal_id(context, instance_id) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7d66d8454..0096b1400 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -99,7 +99,7 @@ class LibvirtConnection(object): def __init__(self, read_only): (self.libvirt_uri, template_file, - rescue_file)= self.get_uri_and_templates() + rescue_file) = self.get_uri_and_templates() self.libvirt_xml = open(template_file).read() self.rescue_xml = open(rescue_file).read() @@ -258,6 +258,7 @@ class LibvirtConnection(object): d = defer.Deferred() timer = task.LoopingCall(f=None) + def _wait_for_rescue(): try: state = self.get_info(instance['name'])['state'] @@ -273,6 +274,7 @@ class LibvirtConnection(object): power_state.SHUTDOWN) timer.stop() d.callback(None) + timer.f = _wait_for_rescue timer.start(interval=0.5, now=True) yield d @@ -441,7 +443,7 @@ class LibvirtConnection(object): * 1024 * 1024 * 1024) resize = True - if inst['instance_type'] == 'm1.tiny' or prefix=='rescue': + if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': resize = False yield disk.partition(basepath('disk-raw'), basepath('disk'), local_bytes, resize, execute=execute) -- cgit From a80b06285d993696ccb90eff00bb2963df49ecc6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 17:18:24 -0700 Subject: add in the xen rescue template --- nova/virt/libvirt.rescue.xen.xml.template | 34 +++++++++++++++++++++++++++++++ nova/virt/libvirt.xen.xml.template | 6 +----- nova/virt/libvirt_conn.py | 3 +++ 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 nova/virt/libvirt.rescue.xen.xml.template diff --git a/nova/virt/libvirt.rescue.xen.xml.template b/nova/virt/libvirt.rescue.xen.xml.template new file mode 100644 index 000000000..3b8d27237 --- /dev/null +++ b/nova/virt/libvirt.rescue.xen.xml.template @@ -0,0 +1,34 @@ + + %(name)s + + linux + %(basepath)s/kernel + %(basepath)s/ramdisk + /dev/xvda1 + ro + + + + + %(memory_kb)s + %(vcpus)s + + + + + + + + + + + + + + + + + + + + diff --git a/nova/virt/libvirt.xen.xml.template b/nova/virt/libvirt.xen.xml.template index 3b8d27237..9677902c6 100644 --- a/nova/virt/libvirt.xen.xml.template +++ b/nova/virt/libvirt.xen.xml.template @@ -13,13 +13,9 @@ %(memory_kb)s %(vcpus)s - - - - - + diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0096b1400..e32945fa5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -51,6 +51,9 @@ FLAGS = flags.FLAGS flags.DEFINE_string('libvirt_rescue_xml_template', utils.abspath('virt/libvirt.rescue.qemu.xml.template'), 'Libvirt RESCUE XML Template for QEmu/KVM') +flags.DEFINE_string('libvirt_rescue_xen_xml_template', + utils.abspath('virt/libvirt.rescue.xen.xml.template'), + 'Libvirt RESCUE XML Template for xen') flags.DEFINE_string('libvirt_rescue_uml_xml_template', utils.abspath('virt/libvirt.rescue.uml.xml.template'), 'Libvirt RESCUE XML Template for user-mode-linux') -- cgit From 9ee74816c0c2a28f7d056d524111cd940513766a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 17:22:29 -0700 Subject: add NotFound to fake.py and document it --- nova/virt/fake.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index dae2a2410..66eff4c66 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -22,10 +22,9 @@ A fake (in-memory) hypervisor+api. Allows nova testing w/o a hypervisor. This module also documents the semantics of real hypervisor connections. """ -import logging - from twisted.internet import defer +from nova import exception from nova.compute import power_state @@ -160,7 +159,12 @@ class FakeConnection(object): current memory the instance has, in KiB, 'num_cpu': The current number of virtual CPUs the instance has, 'cpu_time': The total CPU time used by the instance, in nanoseconds. + + This method should raise exception.NotFound if the hypervisor has no + knowledge of the instance """ + if instance_name not in self.instances: + raise exception.NotFound("Instance %s Not Found" % instance_name) i = self.instances[instance_name] return {'state': i._state, 'max_mem': 0, -- cgit From 0c7b1ea7972defe67d8bebf4f23d189cc7b0422c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 19:52:02 -0700 Subject: logging.warn not raise logging.Warn --- nova/compute/manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index fb9a4cb39..574feec7c 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -132,11 +132,11 @@ class ComputeManager(manager.Manager): self._update_state(context, instance_id) if instance_ref['state'] != power_state.RUNNING: - raise logging.Warn('trying to reboot a non-running ' - 'instance: %s (state: %s excepted: %s)', - instance_ref['internal_id'], - instance_ref['state'], - power_state.RUNNING) + logging.warn('trying to reboot a non-running ' + 'instance: %s (state: %s excepted: %s)', + instance_ref['internal_id'], + instance_ref['state'], + power_state.RUNNING) logging.debug('instance %s: rebooting', instance_ref['name']) self.db.instance_set_state(context, -- cgit From 0e98d027d1deb8cd46ddb9a1df4558a5c8b2edfc Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 23:26:03 -0700 Subject: Removed unused imports and left over references to str_id --- nova/db/sqlalchemy/models.py | 25 +------------------------ nova/network/manager.py | 2 +- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 38c96bdec..2a3cfa94c 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -20,11 +20,9 @@ SQLAlchemy models for nova data """ -import sys import datetime -# TODO(vish): clean up these imports -from sqlalchemy.orm import relationship, backref, exc, object_mapper +from sqlalchemy.orm import relationship, backref, object_mapper from sqlalchemy import Column, Integer, String, schema from sqlalchemy import ForeignKey, DateTime, Boolean, Text from sqlalchemy.exc import IntegrityError @@ -46,17 +44,11 @@ class NovaBase(object): """Base class for Nova Models""" __table_args__ = {'mysql_engine': 'InnoDB'} __table_initialized__ = False - __prefix__ = 'none' created_at = Column(DateTime, default=datetime.datetime.utcnow) updated_at = Column(DateTime, onupdate=datetime.datetime.utcnow) deleted_at = Column(DateTime) deleted = Column(Boolean, default=False) - @property - def str_id(self): - """Get string id of object (generally prefix + '-' + id)""" - return "%s-%s" % (self.__prefix__, self.id) - def save(self, session=None): """Save this object""" if not session: @@ -94,7 +86,6 @@ class NovaBase(object): #class Image(BASE, NovaBase): # """Represents an image in the datastore""" # __tablename__ = 'images' -# __prefix__ = 'ami' # id = Column(Integer, primary_key=True) # ec2_id = Column(String(12), unique=True) # user_id = Column(String(255)) @@ -150,7 +141,6 @@ class Service(BASE, NovaBase): class Instance(BASE, NovaBase): """Represents a guest vm""" __tablename__ = 'instances' - __prefix__ = 'i' id = Column(Integer, primary_key=True) internal_id = Column(Integer, unique=True) @@ -227,7 +217,6 @@ class Instance(BASE, NovaBase): class Volume(BASE, NovaBase): """Represents a block storage device that can be attached to a vm""" __tablename__ = 'volumes' - __prefix__ = 'vol' id = Column(Integer, primary_key=True) ec2_id = Column(String(12), unique=True) @@ -269,10 +258,6 @@ class Quota(BASE, NovaBase): gigabytes = Column(Integer) floating_ips = Column(Integer) - @property - def str_id(self): - return self.project_id - class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on""" @@ -361,10 +346,6 @@ class KeyPair(BASE, NovaBase): fingerprint = Column(String(255)) public_key = Column(Text) - @property - def str_id(self): - return '%s.%s' % (self.user_id, self.name) - class Network(BASE, NovaBase): """Represents a network""" @@ -426,10 +407,6 @@ class FixedIp(BASE, NovaBase): leased = Column(Boolean, default=False) reserved = Column(Boolean, default=False) - @property - def str_id(self): - return self.address - class User(BASE, NovaBase): """Represents a user""" diff --git a/nova/network/manager.py b/nova/network/manager.py index fddb77663..8a20cb491 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -171,7 +171,7 @@ class NetworkManager(manager.Manager): if not fixed_ip_ref['leased']: logging.warn("IP %s released that was not leased", address) self.db.fixed_ip_update(context, - fixed_ip_ref['str_id'], + fixed_ip_ref['address'], {'leased': False}) if not fixed_ip_ref['allocated']: self.db.fixed_ip_disassociate(context, address) -- cgit From 3e2715b21a51c10451e6275e649385f0a846b033 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 00:45:33 -0700 Subject: ISCSI Volume support * Rewrite of Volume code to make VolumeManager more generic * AoE vs. iscsi moved to driver layer * Added db support for target ids * Added code to re-export volumes on restart of VolumeManager * Includes a script to create /dev/iscsi volumes on remote hosts --- nova/compute/manager.py | 25 ++++- nova/db/api.py | 43 +++++++- nova/db/sqlalchemy/api.py | 71 ++++++++++++ nova/db/sqlalchemy/models.py | 23 +++- nova/flags.py | 2 +- nova/tests/fake_flags.py | 8 +- nova/tests/volume_unittest.py | 26 ++--- nova/volume/driver.py | 250 ++++++++++++++++++++++++++++++++++++------ nova/volume/manager.py | 103 +++++++++-------- tools/iscsidev.sh | 41 +++++++ 10 files changed, 480 insertions(+), 112 deletions(-) create mode 100755 tools/iscsidev.sh diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 523bb8893..81b568f80 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -164,10 +164,18 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) dev_path = yield self.volume_manager.setup_compute_volume(context, volume_id) - yield self.driver.attach_volume(instance_ref['ec2_id'], - dev_path, - mountpoint) - self.db.volume_attached(context, volume_id, instance_id, mountpoint) + try: + yield self.driver.attach_volume(instance_ref['name'], + dev_path, + mountpoint) + self.db.volume_attached(context, + volume_id, + instance_id, + mountpoint) + except Exception: + yield self.volume_manager.remove_compute_volume(context, + volume_id) + raise defer.returnValue(True) @defer.inlineCallbacks @@ -180,7 +188,12 @@ class ComputeManager(manager.Manager): volume_id) instance_ref = self.db.instance_get(context, instance_id) volume_ref = self.db.volume_get(context, volume_id) - yield self.driver.detach_volume(instance_ref['ec2_id'], - volume_ref['mountpoint']) + if instance_ref['name'] not in self.driver.list_instances(): + logging.warn("Detaching volume from instance %s that isn't running", + instance_ref['name']) + else: + yield self.driver.detach_volume(instance_ref['name'], + volume_ref['mountpoint']) + yield self.volume_manager.remove_compute_volume(context, volume_id) self.db.volume_detached(context, volume_id) defer.returnValue(True) diff --git a/nova/db/api.py b/nova/db/api.py index 0731e2e05..5967b8ded 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -48,6 +48,11 @@ class NoMoreNetworks(exception.Error): pass +class NoMoreTargets(exception.Error): + """No more available blades""" + pass + + ################### @@ -481,6 +486,23 @@ def export_device_create_safe(context, values): ################### +def target_id_count_by_host(context, host): + """Return count of export devices.""" + return IMPL.target_id_count_by_host(context, host) + + +def target_id_create_safe(context, values): + """Create an target_id from the values dictionary. + + The device is not returned. If the create violates the unique + constraints because the target_id and host already exist, + no exception is raised.""" + return IMPL.target_id_create_safe(context, values) + + +############### + + def auth_destroy_token(context, token): """Destroy an auth token""" return IMPL.auth_destroy_token(context, token) @@ -527,6 +549,11 @@ def volume_allocate_shelf_and_blade(context, volume_id): return IMPL.volume_allocate_shelf_and_blade(context, volume_id) +def volume_allocate_target_id(context, volume_id, host): + """Atomically allocate a free target_id from the pool.""" + return IMPL.volume_allocate_target_id(context, volume_id, host) + + def volume_attached(context, volume_id, instance_id, mountpoint): """Ensure that a volume is set as attached.""" return IMPL.volume_attached(context, volume_id, instance_id, mountpoint) @@ -562,9 +589,9 @@ def volume_get_all(context): return IMPL.volume_get_all(context) -def volume_get_instance(context, volume_id): - """Get the instance that a volume is attached to.""" - return IMPL.volume_get_instance(context, volume_id) +def volume_get_all_by_host(context, host): + """Get all volumes belonging to a host.""" + return IMPL.volume_get_all_by_host(context, host) def volume_get_all_by_project(context, project_id): @@ -577,11 +604,21 @@ def volume_get_by_ec2_id(context, ec2_id): return IMPL.volume_get_by_ec2_id(context, ec2_id) +def volume_get_instance(context, volume_id): + """Get the instance that a volume is attached to.""" + return IMPL.volume_get_instance(context, volume_id) + + def volume_get_shelf_and_blade(context, volume_id): """Get the shelf and blade allocated to the volume.""" return IMPL.volume_get_shelf_and_blade(context, volume_id) +def volume_get_target_id(context, volume_id): + """Get the target id allocated to the volume.""" + return IMPL.volume_get_target_id(context, volume_id) + + def volume_update(context, volume_id, values): """Set the given properties on an volume and update it. diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 0cbe56499..538a4d94b 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1041,6 +1041,30 @@ def export_device_create_safe(context, values): ################### +@require_admin_context +def target_id_count_by_host(context, host): + session = get_session() + return session.query(models.TargetId).\ + filter_by(deleted=can_read_deleted(context)).\ + filter_by(host=host).\ + count() + + +@require_admin_context +def target_id_create_safe(context, values): + target_id_ref = models.TargetId() + for (key, value) in values.iteritems(): + target_id_ref[key] = value + try: + target_id_ref.save() + return target_id_ref + except IntegrityError: + return None + + +################### + + def auth_destroy_token(_context, token): session = get_session() session.delete(token) @@ -1130,6 +1154,25 @@ def volume_allocate_shelf_and_blade(context, volume_id): return (export_device.shelf_id, export_device.blade_id) +@require_admin_context +def volume_allocate_target_id(context, volume_id, host): + session = get_session() + with session.begin(): + target_id_ref = session.query(models.TargetId).\ + filter_by(volume=None).\ + filter_by(host=host).\ + filter_by(deleted=False).\ + with_lockmode('update').\ + first() + # NOTE(vish): if with_lockmode isn't supported, as in sqlite, + # then this has concurrency issues + if not target_id_ref: + raise db.NoMoreTargets() + target_id_ref.volume_id = volume_id + session.add(target_id_ref) + return target_id_ref.target_id + + @require_admin_context def volume_attached(context, volume_id, instance_id, mountpoint): session = get_session() @@ -1181,6 +1224,9 @@ def volume_destroy(context, volume_id): session.execute('update export_devices set volume_id=NULL ' 'where volume_id=:id', {'id': volume_id}) + session.execute('update target_ids set volume_id=NULL ' + 'where volume_id=:id', + {'id': volume_id}) @require_admin_context @@ -1222,6 +1268,17 @@ def volume_get(context, volume_id, session=None): def volume_get_all(context): session = get_session() return session.query(models.Volume).\ + options(joinedload('instance')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + + +@require_admin_context +def volume_get_all_by_host(context, host): + session = get_session() + return session.query(models.Volume).\ + options(joinedload('instance')).\ + filter_by(host=host).\ filter_by(deleted=can_read_deleted(context)).\ all() @@ -1232,6 +1289,7 @@ def volume_get_all_by_project(context, project_id): session = get_session() return session.query(models.Volume).\ + options(joinedload('instance')).\ filter_by(project_id=project_id).\ filter_by(deleted=can_read_deleted(context)).\ all() @@ -1299,6 +1357,19 @@ def volume_get_shelf_and_blade(context, volume_id): return (result.shelf_id, result.blade_id) +@require_admin_context +def volume_get_target_id(context, volume_id): + session = get_session() + result = session.query(models.TargetId).\ + filter_by(volume_id=volume_id).\ + first() + if not result: + raise exception.NotFound('No target id found for volume %s' % + volume_id) + + return result.target_id + + @require_context def volume_update(context, volume_id, values): session = get_session() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 38c96bdec..18d837e6b 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -255,6 +255,11 @@ class Volume(BASE, NovaBase): display_name = Column(String(255)) display_description = Column(String(255)) + @property + def name(self): + return self.ec2_id + + class Quota(BASE, NovaBase): """Represents quota overrides for a project""" @@ -290,6 +295,22 @@ class ExportDevice(BASE, NovaBase): 'ExportDevice.deleted==False)') +class TargetId(BASE, NovaBase): + """Represates an iscsi target_id for a given host""" + __tablename__ = 'target_ids' + __table_args__ = (schema.UniqueConstraint("target_id", "host"), + {'mysql_engine': 'InnoDB'}) + id = Column(Integer, primary_key=True) + target_id = Column(Integer) + host = Column(String(255)) + volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + volume = relationship(Volume, + backref=backref('target_id', uselist=False), + foreign_keys=volume_id, + primaryjoin='and_(TargetId.volume_id==Volume.id,' + 'TargetId.deleted==False)') + + class SecurityGroupInstanceAssociation(BASE, NovaBase): __tablename__ = 'security_group_instance_association' id = Column(Integer, primary_key=True) @@ -510,7 +531,7 @@ class FloatingIp(BASE, NovaBase): def register_models(): """Register Models and create metadata""" from sqlalchemy import create_engine - models = (Service, Instance, Volume, ExportDevice, FixedIp, + models = (Service, Instance, Volume, ExportDevice, TargetId, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, Project) # , Image, Host diff --git a/nova/flags.py b/nova/flags.py index f3b0384ad..380382a7a 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -224,7 +224,7 @@ DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager', 'Manager for compute') DEFINE_string('network_manager', 'nova.network.manager.VlanManager', 'Manager for network') -DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager', +DEFINE_string('volume_manager', 'nova.volume.manager.VolumeManager', 'Manager for volume') DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 4bbef8832..d695d68a6 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -21,7 +21,7 @@ from nova import flags FLAGS = flags.FLAGS flags.DECLARE('volume_driver', 'nova.volume.manager') -FLAGS.volume_driver = 'nova.volume.driver.FakeAOEDriver' +FLAGS.volume_driver = 'nova.volume.driver.FakeISCSIDriver' FLAGS.connection_type = 'fake' FLAGS.fake_rabbit = True FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' @@ -31,9 +31,11 @@ flags.DECLARE('fake_network', 'nova.network.manager') FLAGS.network_size = 16 FLAGS.num_networks = 5 FLAGS.fake_network = True -flags.DECLARE('num_shelves', 'nova.volume.manager') -flags.DECLARE('blades_per_shelf', 'nova.volume.manager') +flags.DECLARE('num_shelves', 'nova.volume.driver') +flags.DECLARE('blades_per_shelf', 'nova.volume.driver') +flags.DECLARE('iscsi_target_ids', 'nova.volume.driver') FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 +FLAGS.iscsi_target_ids = 8 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index fdee30b48..34e04c8b8 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -83,9 +83,9 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_many_volumes(self): - """Ensure that NoMoreBlades is raised when we run out of volumes""" + """Ensure that NoMoreTargets is raised when we run out of volumes""" vols = [] - total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf + total_slots = FLAGS.iscsi_target_ids for _index in xrange(total_slots): volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) @@ -93,7 +93,7 @@ class VolumeTestCase(test.TrialTestCase): volume_id = self._create_volume() self.assertFailure(self.volume.create_volume(self.context, volume_id), - db.NoMoreBlades) + db.NoMoreTargets) db.volume_destroy(context.get_admin_context(), volume_id) for volume_id in vols: yield self.volume.delete_volume(self.context, volume_id) @@ -148,23 +148,21 @@ class VolumeTestCase(test.TrialTestCase): db.instance_destroy(self.context, instance_id) @defer.inlineCallbacks - def test_concurrent_volumes_get_different_blades(self): - """Ensure multiple concurrent volumes get different blades""" + def test_concurrent_volumes_get_different_targets(self): + """Ensure multiple concurrent volumes get different targets""" volume_ids = [] - shelf_blades = [] + targets = [] def _check(volume_id): - """Make sure blades aren't duplicated""" + """Make sure targets aren't duplicated""" volume_ids.append(volume_id) admin_context = context.get_admin_context() - (shelf_id, blade_id) = db.volume_get_shelf_and_blade(admin_context, - volume_id) - shelf_blade = '%s.%s' % (shelf_id, blade_id) - self.assert_(shelf_blade not in shelf_blades) - shelf_blades.append(shelf_blade) - logging.debug("Blade %s allocated", shelf_blade) + target_id = db.volume_get_target_id(admin_context, volume_id) + self.assert_(target_id not in targets) + targets.append(target_id) + logging.debug("Target %s allocated", target_id) deferreds = [] - total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf + total_slots = FLAGS.iscsi_target_ids for _index in xrange(total_slots): volume_id = self._create_volume() d = self.volume.create_volume(self.context, volume_id) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 3fa29ba37..b69076244 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -27,6 +27,7 @@ from twisted.internet import defer from nova import exception from nova import flags from nova import process +from nova import utils FLAGS = flags.FLAGS @@ -36,12 +37,29 @@ flags.DEFINE_string('aoe_eth_dev', 'eth0', 'Which device to export the volumes on') flags.DEFINE_string('num_shell_tries', 3, 'number of times to attempt to run flakey shell commands') +flags.DEFINE_integer('num_shelves', + 100, + 'Number of vblade shelves') +flags.DEFINE_integer('blades_per_shelf', + 16, + 'Number of vblade blades per shelf') +flags.DEFINE_integer('iscsi_target_ids', + 100, + 'Number of iscsi target ids per host') +flags.DEFINE_string('iscsi_target_prefix', 'iqn.2010-10.org.openstack:', + 'prefix for iscsi volumes') +flags.DEFINE_string('iscsi_ip_prefix', '127.0.0', + 'only connect to the specified ip') -class AOEDriver(object): - """Executes commands relating to AOE volumes""" - def __init__(self, execute=process.simple_execute, *args, **kwargs): +class VolumeDriver(object): + """Executes commands relating to Volumes""" + def __init__(self, execute=process.simple_execute, + sync_exec=utils.execute, *args, **kwargs): + # NOTE(vish): db is set by Manager + self.db = None self._execute = execute + self._sync_exec = sync_exec @defer.inlineCallbacks def _try_execute(self, command): @@ -61,55 +79,93 @@ class AOEDriver(object): "Try number %s", tries) yield self._execute("sleep %s" % tries ** 2) + def check_for_setup_error(self): + """Returns an error if prerequesits aren't met""" + # NOTE(vish): makes sure that the volume group exists + (_out, err) = self._sync_exec("vgs %s" % FLAGS.volume_group, + check_exit_code=False) + if err: + raise exception.Error(err) + @defer.inlineCallbacks - def create_volume(self, volume_name, size): + def create_volume(self, volume): """Creates a logical volume""" - # NOTE(vish): makes sure that the volume group exists - yield self._execute("vgs %s" % FLAGS.volume_group) - if int(size) == 0: + if int(volume['size']) == 0: sizestr = '100M' else: - sizestr = '%sG' % size + sizestr = '%sG' % volume['size'] yield self._try_execute("sudo lvcreate -L %s -n %s %s" % (sizestr, - volume_name, + volume['name'], FLAGS.volume_group)) @defer.inlineCallbacks - def delete_volume(self, volume_name): + def delete_volume(self, volume): """Deletes a logical volume""" yield self._try_execute("sudo lvremove -f %s/%s" % (FLAGS.volume_group, - volume_name)) + volume['name'])) @defer.inlineCallbacks - def create_export(self, volume_name, shelf_id, blade_id): - """Creates an export for a logical volume""" - yield self._try_execute( - "sudo vblade-persist setup %s %s %s /dev/%s/%s" % - (shelf_id, - blade_id, - FLAGS.aoe_eth_dev, - FLAGS.volume_group, - volume_name)) + def local_path(self, volume): + defer.returnValue("/dev/%s/%s" % (FLAGS.volume_group, volume['name'])) + + def ensure_export(self, context, volume): + """Safely and synchronously recreates an export for a logical volume""" + raise NotImplementedError() @defer.inlineCallbacks - def discover_volume(self, _volume_name): - """Discover volume on a remote host""" - yield self._execute("sudo aoe-discover") - yield self._execute("sudo aoe-stat") + def create_export(self, context, volume): + """Exports the volume""" + raise NotImplementedError() @defer.inlineCallbacks - def remove_export(self, _volume_name, shelf_id, blade_id): + def remove_export(self, context, volume): """Removes an export for a logical volume""" - yield self._try_execute("sudo vblade-persist stop %s %s" % - (shelf_id, blade_id)) - yield self._try_execute("sudo vblade-persist destroy %s %s" % - (shelf_id, blade_id)) + raise NotImplementedError() + + @defer.inlineCallbacks + def discover_volume(self, volume): + """Discover volume on a remote host""" + raise NotImplementedError() @defer.inlineCallbacks - def ensure_exports(self): - """Runs all existing exports""" + def undiscover_volume(self, volume): + """Undiscover volume on a remote host""" + raise NotImplementedError() + + +class AOEDriver(VolumeDriver): + """Implements AOE specific volume commands""" + + def ensure_export(self, context, volume): + # NOTE(vish): we depend on vblade-persist for recreating exports + pass + + def _ensure_blades(self, context): + """Ensure that blades have been created in datastore""" + total_blades = FLAGS.num_shelves * FLAGS.blades_per_shelf + if self.db.export_device_count(context) >= total_blades: + return + for shelf_id in xrange(FLAGS.num_shelves): + for blade_id in xrange(FLAGS.blades_per_shelf): + dev = {'shelf_id': shelf_id, 'blade_id': blade_id} + self.db.export_device_create_safe(context, dev) + + @defer.inlineCallbacks + def create_export(self, context, volume): + """Creates an export for a logical volume""" + self._ensure_blades(context) + (shelf_id, + blade_id) = self.db.volume_allocate_shelf_and_blade(context, + volume['id']) + yield self._try_execute( + "sudo vblade-persist setup %s %s %s /dev/%s/%s" % + (shelf_id, + blade_id, + FLAGS.aoe_eth_dev, + FLAGS.volume_group, + volume['name'])) # NOTE(vish): The standard _try_execute does not work here # because these methods throw errors if other # volumes on this host are in the process of @@ -123,13 +179,143 @@ class AOEDriver(object): yield self._execute("sudo vblade-persist start all", check_exit_code=False) + @defer.inlineCallbacks + def remove_export(self, context, volume): + """Removes an export for a logical volume""" + (shelf_id, + blade_id) = self.db.volume_get_shelf_and_blade(context, + volume['id']) + yield self._try_execute("sudo vblade-persist stop %s %s" % + (shelf_id, blade_id)) + yield self._try_execute("sudo vblade-persist destroy %s %s" % + (shelf_id, blade_id)) + + @defer.inlineCallbacks + def discover_volume(self, _volume): + """Discover volume on a remote host""" + yield self._execute("sudo aoe-discover") + yield self._execute("sudo aoe-stat", check_exit_code=False) + + @defer.inlineCallbacks + def undiscover_volume(self, _volume): + """Undiscover volume on a remote host""" + yield + class FakeAOEDriver(AOEDriver): """Logs calls instead of executing""" def __init__(self, *args, **kwargs): - super(FakeAOEDriver, self).__init__(self.fake_execute) + super(FakeAOEDriver, self).__init__(execute=self.fake_execute, + sync_exec=self.fake_execute, + *args, **kwargs) @staticmethod def fake_execute(cmd, *_args, **_kwargs): """Execute that simply logs the command""" logging.debug("FAKE AOE: %s", cmd) + return (None, None) + + +class ISCSIDriver(VolumeDriver): + """Executes commands relating to ISCSI volumes""" + + def ensure_export(self, context, volume): + """Safely and synchronously recreates an export for a logical volume""" + target_id = self.db.volume_get_target_id(context, volume['id']) + iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) + volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) + self._sync_exec("sudo ietadm --op new " + "--tid=%s --params Name=%s" % + (target_id, iscsi_name), + check_exit_code=False) + self._sync_exec("sudo ietadm --op new --tid=%s " + "--lun=0 --params Path=%s,Type=fileio" % + (target_id, volume_path), + check_exit_code=False) + + def _ensure_target_ids(self, context, host): + """Ensure that target ids have been created in datastore""" + host_target_ids = self.db.target_id_count_by_host(context, host) + if host_target_ids >= FLAGS.iscsi_target_ids: + return + # NOTE(vish): Target ids start at 1, not 0. + for target_id in xrange(1, FLAGS.iscsi_target_ids + 1): + target = {'host': host, 'target_id': target_id} + self.db.target_id_create_safe(context, target) + + @defer.inlineCallbacks + def create_export(self, context, volume): + """Creates an export for a logical volume""" + self._ensure_target_ids(context, volume['host']) + target_id = self.db.volume_allocate_target_id(context, + volume['id'], + volume['host']) + iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) + volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) + yield self._execute("sudo ietadm --op new " + "--tid=%s --params Name=%s" % + (target_id, iscsi_name)) + yield self._execute("sudo ietadm --op new --tid=%s " + "--lun=0 --params Path=%s,Type=fileio" % + (target_id, volume_path)) + + @defer.inlineCallbacks + def remove_export(self, context, volume): + """Removes an export for a logical volume""" + target_id = self.db.volume_get_target_id(context, volume['name']) + yield self._execute("sudo ietadm --op delete --tid=%s " + "--lun=0" % target_id) + yield self._execute("sudo ietadm --op delete --tid=%s" % + target_id) + + @defer.inlineCallbacks + def _get_name_and_portal(self, volume_name, host): + (out, _err) = yield self._execute("sudo iscsiadm -m discovery -t " + "sendtargets -p %s" % host) + for target in out.splitlines(): + if FLAGS.iscsi_ip_prefix in target and volume_name in target: + (location, _sep, iscsi_name) = target.partition(" ") + break + iscsi_portal = location.split(",")[0] + defer.returnValue((iscsi_name, iscsi_portal)) + + @defer.inlineCallbacks + def discover_volume(self, volume): + """Discover volume on a remote host""" + (iscsi_name, + iscsi_portal) = yield self._get_name_and_portal(volume['id'], + volume['host']) + yield self._execute("sudo iscsiadm -m node -T %s -p %s --login" % + (iscsi_name, iscsi_portal)) + yield self._execute("sudo iscsiadm -m node -T %s -p %s --op update " + "-n node.startup -v automatic" % + (iscsi_name, iscsi_portal)) + defer.returnValue("/dev/iscsi/%s" % volume['name']) + + @defer.inlineCallbacks + def undiscover_volume(self, volume): + """Undiscover volume on a remote host""" + (iscsi_name, + iscsi_portal) = yield self._get_name_and_portal(volume['name'], + volume['host']) + yield self._execute("sudo iscsiadm -m node -T %s -p %s --op update " + "-n node.startup -v manual" % + (iscsi_name, iscsi_portal)) + yield self._execute("sudo iscsiadm -m node -T %s -p %s --logout " % + (iscsi_name, iscsi_portal)) + yield self._execute("sudo iscsiadm -m node --op delete " + "--targetname %s" % iscsi_name) + + +class FakeISCSIDriver(ISCSIDriver): + """Logs calls instead of executing""" + def __init__(self, *args, **kwargs): + super(FakeISCSIDriver, self).__init__(execute=self.fake_execute, + sync_exec=self.fake_execute, + *args, **kwargs) + + @staticmethod + def fake_execute(cmd, *_args, **_kwargs): + """Execute that simply logs the command""" + logging.debug("FAKE ISCSI: %s", cmd) + return (None, None) diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 2874459f9..910e71c9e 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -26,6 +26,7 @@ import datetime from twisted.internet import defer +from nova import context from nova import exception from nova import flags from nova import manager @@ -36,70 +37,58 @@ FLAGS = flags.FLAGS flags.DEFINE_string('storage_availability_zone', 'nova', 'availability zone of this service') -flags.DEFINE_string('volume_driver', 'nova.volume.driver.AOEDriver', +flags.DEFINE_string('volume_driver', 'nova.volume.driver.ISCSIDriver', 'Driver to use for volume creation') -flags.DEFINE_integer('num_shelves', - 100, - 'Number of vblade shelves') -flags.DEFINE_integer('blades_per_shelf', - 16, - 'Number of vblade blades per shelf') -class AOEManager(manager.Manager): - """Manages Ata-Over_Ethernet volumes""" +class VolumeManager(manager.Manager): + """Manages attachable block storage devices""" def __init__(self, volume_driver=None, *args, **kwargs): if not volume_driver: volume_driver = FLAGS.volume_driver self.driver = utils.import_object(volume_driver) - super(AOEManager, self).__init__(*args, **kwargs) - - def _ensure_blades(self, context): - """Ensure that blades have been created in datastore""" - total_blades = FLAGS.num_shelves * FLAGS.blades_per_shelf - if self.db.export_device_count(context) >= total_blades: - return - for shelf_id in xrange(FLAGS.num_shelves): - for blade_id in xrange(FLAGS.blades_per_shelf): - dev = {'shelf_id': shelf_id, 'blade_id': blade_id} - self.db.export_device_create_safe(context, dev) + super(VolumeManager, self).__init__(*args, **kwargs) + # NOTE(vish): Implementation specific db handling is done + # by the driver. + self.driver.db = self.db + + def init_host(self): + """Do any initialization that needs to be run if this is a + standalone service. + """ + self.driver.check_for_setup_error() + ctxt = context.get_admin_context() + volumes = self.db.volume_get_all_by_host(ctxt, self.host) + logging.debug("Re-exporting %s volumes", len(volumes)) + for volume in volumes: + self.driver.ensure_export(context, volume) @defer.inlineCallbacks def create_volume(self, context, volume_id): """Creates and exports the volume""" context = context.elevated() - logging.info("volume %s: creating", volume_id) - volume_ref = self.db.volume_get(context, volume_id) + logging.info("volume %s: creating", volume_ref['name']) self.db.volume_update(context, volume_id, {'host': self.host}) + # NOTE(vish): so we don't have to get volume from db again + # before passing it to the driver. + volume_ref['host'] = self.host - size = volume_ref['size'] - logging.debug("volume %s: creating lv of size %sG", volume_id, size) - yield self.driver.create_volume(volume_ref['ec2_id'], size) - - logging.debug("volume %s: allocating shelf & blade", volume_id) - self._ensure_blades(context) - rval = self.db.volume_allocate_shelf_and_blade(context, volume_id) - (shelf_id, blade_id) = rval - - logging.debug("volume %s: exporting shelf %s & blade %s", volume_id, - shelf_id, blade_id) - - yield self.driver.create_export(volume_ref['ec2_id'], - shelf_id, - blade_id) + logging.debug("volume %s: creating lv of size %sG", + volume_ref['name'], volume_ref['size']) + yield self.driver.create_volume(volume_ref) - logging.debug("volume %s: re-exporting all values", volume_id) - yield self.driver.ensure_exports() + logging.debug("volume %s: creating export", volume_ref['name']) + yield self.driver.create_export(context, volume_ref) now = datetime.datetime.utcnow() self.db.volume_update(context, volume_ref['id'], {'status': 'available', 'launched_at': now}) - logging.debug("volume %s: created successfully", volume_id) + logging.debug("volume %s: created successfully", volume_ref['name']) defer.returnValue(volume_id) @defer.inlineCallbacks @@ -111,14 +100,10 @@ class AOEManager(manager.Manager): raise exception.Error("Volume is still attached") if volume_ref['host'] != self.host: raise exception.Error("Volume is not local to this node") - logging.debug("Deleting volume with id of: %s", volume_id) - shelf_id, blade_id = self.db.volume_get_shelf_and_blade(context, - volume_id) - yield self.driver.remove_export(volume_ref['ec2_id'], - shelf_id, - blade_id) - yield self.driver.delete_volume(volume_ref['ec2_id']) + logging.debug("volume %s: deleting", volume_ref['name']) + yield self.driver.delete_volume(volume_ref) self.db.volume_destroy(context, volume_id) + logging.debug("volume %s: deleted successfully", volume_ref['name']) defer.returnValue(True) @defer.inlineCallbacks @@ -127,9 +112,23 @@ class AOEManager(manager.Manager): Returns path to device. """ - context = context.elevated() + context = context.admin() volume_ref = self.db.volume_get(context, volume_id) - yield self.driver.discover_volume(volume_ref['ec2_id']) - shelf_id, blade_id = self.db.volume_get_shelf_and_blade(context, - volume_id) - defer.returnValue("/dev/etherd/e%s.%s" % (shelf_id, blade_id)) + if volume_ref['host'] == self.host: + # NOTE(vish): No need to discover local volumes. + path = yield self.driver.local_path(volume_ref) + else: + path = yield self.driver.discover_volume(volume_ref) + defer.returnValue(path) + + @defer.inlineCallbacks + def remove_compute_volume(self, context, volume_id): + """Remove remote volume on compute host """ + context = context.admin() + volume_ref = self.db.volume_get(context, volume_id) + if volume_ref['host'] == self.host: + # NOTE(vish): No need to undiscover local volumes. + defer.returnValue(True) + else: + yield self.driver.undiscover_volume(volume_ref) + diff --git a/tools/iscsidev.sh b/tools/iscsidev.sh new file mode 100755 index 000000000..6f5b572df --- /dev/null +++ b/tools/iscsidev.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(vish): This script helps udev create common names for discovered iscsi +# volumes under /dev/iscsi. To use it, create /dev/iscsi and add +# a file to /etc/udev/rules.d like so: +# mkdir /dev/iscsi +# echo 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/path/to/iscsidev.sh +# %b",SYMLINK+="iscsi/%c%n"' > /etc/udev/rules.d/55-openiscsi.rules + +BUS=${1} +HOST=${BUS%%:*} + +if [ ! -e /sys/class/iscsi_host ]; then + exit 1 +fi + +file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" + +target_name=$(cat ${file}) + +if [ -z "${target_name}" ]; then + exit 1 +fi + +echo "${target_name##*:}" -- cgit From bde0d8d0f0e864d5b5d0f87e55ab23839846f71e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 01:37:01 -0700 Subject: fix bugs, describe volumes, detach on terminate --- nova/api/ec2/cloud.py | 13 ++++++++++--- nova/compute/manager.py | 3 +++ nova/volume/driver.py | 11 +++++------ nova/volume/manager.py | 4 ++-- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 784697b01..a1899c47f 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -463,24 +463,31 @@ class CloudController(object): return {'volumeSet': volumes} def _format_volume(self, context, volume): + instance_ec2_id = None + instance_data = None + if volume.get('instance', None): + internal_id = volume['instance']['internal_id'] + ec2_id = internal_id_to_ec2_id(internal_id) + instance_data = '%s[%s]' % (instance_ec2_id, + volume['instance']['host']) v = {} v['volumeId'] = volume['ec2_id'] v['status'] = volume['status'] v['size'] = volume['size'] v['availabilityZone'] = volume['availability_zone'] v['createTime'] = volume['created_at'] - if context.user.is_admin(): + if context.is_admin: v['status'] = '%s (%s, %s, %s, %s)' % ( volume['status'], volume['user_id'], volume['host'], - volume['instance_id'], + instance_data, volume['mountpoint']) if volume['attach_status'] == 'attached': v['attachmentSet'] = [{'attachTime': volume['attach_time'], 'deleteOnTermination': False, 'device': volume['mountpoint'], - 'instanceId': volume['instance_id'], + 'instanceId': instance_ec2_id, 'status': 'attached', 'volume_id': volume['ec2_id']}] else: diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 81b568f80..3b3208fea 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -111,6 +111,9 @@ class ComputeManager(manager.Manager): logging.debug("instance %s: terminating", instance_id) instance_ref = self.db.instance_get(context, instance_id) + volumes = instance_ref.get('volumes', []) or [] + for volume in volumes: + self.detach_volume(instance_id, volume['id']) if instance_ref['state'] == power_state.SHUTOFF: self.db.instance_destroy(context, instance_id) raise exception.Error('trying to destroy already destroyed' diff --git a/nova/volume/driver.py b/nova/volume/driver.py index b69076244..eff56d9c6 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -21,6 +21,7 @@ Drivers for volumes """ import logging +import os from twisted.internet import defer @@ -80,12 +81,10 @@ class VolumeDriver(object): yield self._execute("sleep %s" % tries ** 2) def check_for_setup_error(self): - """Returns an error if prerequesits aren't met""" - # NOTE(vish): makes sure that the volume group exists - (_out, err) = self._sync_exec("vgs %s" % FLAGS.volume_group, - check_exit_code=False) - if err: - raise exception.Error(err) + """Returns an error if prerequisites aren't met""" + if not os.path.isdir("/dev/%s" % FLAGS.volume_group): + raise exception.Error("volume group %s doesn't exist" + % FLAGS.volume_group) @defer.inlineCallbacks def create_volume(self, volume): diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 910e71c9e..f6146efe9 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -112,7 +112,7 @@ class VolumeManager(manager.Manager): Returns path to device. """ - context = context.admin() + context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host: # NOTE(vish): No need to discover local volumes. @@ -124,7 +124,7 @@ class VolumeManager(manager.Manager): @defer.inlineCallbacks def remove_compute_volume(self, context, volume_id): """Remove remote volume on compute host """ - context = context.admin() + context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host: # NOTE(vish): No need to undiscover local volumes. -- cgit From 43a545a8bd8f763eba7741a240c29da447aef61e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 03:11:00 -0700 Subject: more bugfixes, flag for local volumes --- nova/api/ec2/__init__.py | 1 + nova/api/ec2/cloud.py | 7 +++++-- nova/compute/manager.py | 4 +++- nova/db/sqlalchemy/models.py | 6 ++++++ nova/volume/driver.py | 9 +++++---- nova/volume/manager.py | 12 +++++++----- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 0df4d3710..c53ce6f5e 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -238,6 +238,7 @@ class Executor(wsgi.Application): return self._error(req, type(ex).__name__, str(ex)) def _error(self, req, code, message): + logging.error("%s: %s", code, message) resp = webob.Response() resp.status = 400 resp.headers['Content-Type'] = 'text/xml' diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index a1899c47f..7a057396c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -467,7 +467,7 @@ class CloudController(object): instance_data = None if volume.get('instance', None): internal_id = volume['instance']['internal_id'] - ec2_id = internal_id_to_ec2_id(internal_id) + instance_ec2_id = internal_id_to_ec2_id(internal_id) instance_data = '%s[%s]' % (instance_ec2_id, volume['instance']['host']) v = {} @@ -522,7 +522,10 @@ class CloudController(object): "args": {"topic": FLAGS.volume_topic, "volume_id": volume_ref['id']}}) - return {'volumeSet': [self._format_volume(context, volume_ref)]} + # TODO(vish): Instance should be None at db layer instead of + # trying to lazy load, but for now we turn it into + # a dict to avoid an error. + return {'volumeSet': [self._format_volume(context, dict(volume_ref))]} def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_ref = db.volume_get_by_ec2_id(context, volume_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3b3208fea..116bf11cc 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -113,7 +113,7 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) volumes = instance_ref.get('volumes', []) or [] for volume in volumes: - self.detach_volume(instance_id, volume['id']) + self.detach_volume(context, instance_id, volume['id']) if instance_ref['state'] == power_state.SHUTOFF: self.db.instance_destroy(context, instance_id) raise exception.Error('trying to destroy already destroyed' @@ -176,6 +176,8 @@ class ComputeManager(manager.Manager): instance_id, mountpoint) except Exception: + logging.debug("instance %s: attach failed to %s, removing export", + instance_id, mountpoint) yield self.volume_manager.remove_compute_volume(context, volume_id) raise diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 18d837e6b..b0adc3a2a 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -82,6 +82,12 @@ class NovaBase(object): def __getitem__(self, key): return getattr(self, key) + def get(self, key, default=None): + try: + return getattr(self, key) + except AttributeError, KeyError: + return default + def __iter__(self): self._i = iter(object_mapper(self).columns) return self diff --git a/nova/volume/driver.py b/nova/volume/driver.py index eff56d9c6..bffe4d6b5 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -49,8 +49,8 @@ flags.DEFINE_integer('iscsi_target_ids', 'Number of iscsi target ids per host') flags.DEFINE_string('iscsi_target_prefix', 'iqn.2010-10.org.openstack:', 'prefix for iscsi volumes') -flags.DEFINE_string('iscsi_ip_prefix', '127.0.0', - 'only connect to the specified ip') +flags.DEFINE_string('iscsi_ip_prefix', '127.0', + 'discover volumes on the ip that starts with this prefix') class VolumeDriver(object): @@ -107,6 +107,7 @@ class VolumeDriver(object): @defer.inlineCallbacks def local_path(self, volume): + yield # NOTE(vish): stops deprecation warning defer.returnValue("/dev/%s/%s" % (FLAGS.volume_group, volume['name'])) def ensure_export(self, context, volume): @@ -261,7 +262,7 @@ class ISCSIDriver(VolumeDriver): @defer.inlineCallbacks def remove_export(self, context, volume): """Removes an export for a logical volume""" - target_id = self.db.volume_get_target_id(context, volume['name']) + target_id = self.db.volume_get_target_id(context, volume['id']) yield self._execute("sudo ietadm --op delete --tid=%s " "--lun=0" % target_id) yield self._execute("sudo ietadm --op delete --tid=%s" % @@ -282,7 +283,7 @@ class ISCSIDriver(VolumeDriver): def discover_volume(self, volume): """Discover volume on a remote host""" (iscsi_name, - iscsi_portal) = yield self._get_name_and_portal(volume['id'], + iscsi_portal) = yield self._get_name_and_portal(volume['name'], volume['host']) yield self._execute("sudo iscsiadm -m node -T %s -p %s --login" % (iscsi_name, iscsi_portal)) diff --git a/nova/volume/manager.py b/nova/volume/manager.py index f6146efe9..bc49e28ee 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -39,6 +39,8 @@ flags.DEFINE_string('storage_availability_zone', 'availability zone of this service') flags.DEFINE_string('volume_driver', 'nova.volume.driver.ISCSIDriver', 'Driver to use for volume creation') +flags.DEFINE_boolean('use_local_volumes', True, + 'if True, will not discover local volumes') class VolumeManager(manager.Manager): @@ -61,7 +63,7 @@ class VolumeManager(manager.Manager): volumes = self.db.volume_get_all_by_host(ctxt, self.host) logging.debug("Re-exporting %s volumes", len(volumes)) for volume in volumes: - self.driver.ensure_export(context, volume) + self.driver.ensure_export(ctxt, volume) @defer.inlineCallbacks def create_volume(self, context, volume_id): @@ -100,6 +102,8 @@ class VolumeManager(manager.Manager): raise exception.Error("Volume is still attached") if volume_ref['host'] != self.host: raise exception.Error("Volume is not local to this node") + logging.debug("volume %s: removing export", volume_ref['name']) + yield self.driver.remove_export(context, volume_ref) logging.debug("volume %s: deleting", volume_ref['name']) yield self.driver.delete_volume(volume_ref) self.db.volume_destroy(context, volume_id) @@ -114,8 +118,7 @@ class VolumeManager(manager.Manager): """ context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) - if volume_ref['host'] == self.host: - # NOTE(vish): No need to discover local volumes. + if volume_ref['host'] == self.host and FLAGS.use_local_volumes: path = yield self.driver.local_path(volume_ref) else: path = yield self.driver.discover_volume(volume_ref) @@ -126,8 +129,7 @@ class VolumeManager(manager.Manager): """Remove remote volume on compute host """ context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) - if volume_ref['host'] == self.host: - # NOTE(vish): No need to undiscover local volumes. + if volume_ref['host'] == self.host and FLAGS.use_local_volumes: defer.returnValue(True) else: yield self.driver.undiscover_volume(volume_ref) -- cgit From 3508fe6e6fe56d86119158d1631d624c76087bf6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 11:12:56 -0700 Subject: use libvirt connection for attaching disks and avoid the symlink --- nova/virt/libvirt_conn.py | 45 ++++++++++++++++++++++++++++++++++++--------- nova/volume/driver.py | 5 ++++- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 509ed97a0..85edfff08 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -182,19 +182,46 @@ class LibvirtConnection(object): @defer.inlineCallbacks @exception.wrap_exception def attach_volume(self, instance_name, device_path, mountpoint): - yield process.simple_execute("sudo virsh attach-disk %s %s %s" % - (instance_name, - device_path, - mountpoint.rpartition('/dev/')[2])) + virt_dom = self._conn.lookupByName(instance_name) + mount_device = mountpoint.rpartition("/")[2] + xml = """ + + + + """ % (device_path, mount_device) + virt_dom.attachDevice(xml) + yield + + def _get_disk_xml(self, xml, device): + """Returns the xml for the disk mounted at device""" + try: + doc = libxml2.parseDoc(xml) + except: + return None + ctx = doc.xpathNewContext() + try: + ret = ctx.xpathEval('/domain/devices/disk') + for node in ret: + for child in node.children: + if child.name == 'target': + if child.prop('dev') == device: + return str(node) + finally: + if ctx != None: + ctx.xpathFreeContext() + if doc != None: + doc.freeDoc() @defer.inlineCallbacks @exception.wrap_exception def detach_volume(self, instance_name, mountpoint): - # NOTE(vish): despite the documentation, virsh detach-disk just - # wants the device name without the leading /dev/ - yield process.simple_execute("sudo virsh detach-disk %s %s" % - (instance_name, - mountpoint.rpartition('/dev/')[2])) + virt_dom = self._conn.lookupByName(instance_name) + mount_device = mountpoint.rpartition("/")[2] + xml = self._get_disk_xml(virt_dom.XMLDesc(0), mount_device) + if not xml: + raise exception.NotFound("No disk at %s" % mount_device) + virt_dom.detachDevice(xml) + yield @defer.inlineCallbacks @exception.wrap_exception diff --git a/nova/volume/driver.py b/nova/volume/driver.py index bffe4d6b5..8d98d2491 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -108,7 +108,10 @@ class VolumeDriver(object): @defer.inlineCallbacks def local_path(self, volume): yield # NOTE(vish): stops deprecation warning - defer.returnValue("/dev/%s/%s" % (FLAGS.volume_group, volume['name'])) + escaped_group = FLAGS.volume_group.replace('-', '--') + escaped_name = volume['name'].replace('-', '--') + defer.returnValue("/dev/mapper/%s-%s" % (escaped_group, + escaped_name)) def ensure_export(self, context, volume): """Safely and synchronously recreates an export for a logical volume""" -- cgit From 2738a380816b73f35e73f111bd9b4f3ef3101012 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 12:17:12 -0700 Subject: print the exception on fail, because it doesn't seem to reraise it --- nova/compute/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 116bf11cc..80931a309 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -176,8 +176,8 @@ class ComputeManager(manager.Manager): instance_id, mountpoint) except Exception: - logging.debug("instance %s: attach failed to %s, removing export", - instance_id, mountpoint) + logging.exception("instance %s: attach failed %s, removing", + instance_id, mountpoint) yield self.volume_manager.remove_compute_volume(context, volume_id) raise -- cgit From 60f3b009f3f846539dfeb2101eec73259553f8ea Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 12:54:22 -0700 Subject: pep8 cleanup --- nova/compute/manager.py | 4 ++-- nova/db/sqlalchemy/models.py | 1 - nova/volume/manager.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 80931a309..f2e80bff3 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -194,8 +194,8 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) volume_ref = self.db.volume_get(context, volume_id) if instance_ref['name'] not in self.driver.list_instances(): - logging.warn("Detaching volume from instance %s that isn't running", - instance_ref['name']) + logging.warn("Detaching volume from unknown instance %s", + instance_ref['name']) else: yield self.driver.detach_volume(instance_ref['name'], volume_ref['mountpoint']) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index b0adc3a2a..f0424cc64 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -266,7 +266,6 @@ class Volume(BASE, NovaBase): return self.ec2_id - class Quota(BASE, NovaBase): """Represents quota overrides for a project""" __tablename__ = 'quotas' diff --git a/nova/volume/manager.py b/nova/volume/manager.py index bc49e28ee..ee1c019ad 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -133,4 +133,3 @@ class VolumeManager(manager.Manager): defer.returnValue(True) else: yield self.driver.undiscover_volume(volume_ref) - -- cgit From 5318bf110019d820e6f000662194d6e29f3e315f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 17:15:56 -0700 Subject: fix tests by removing missed reference to prefix and unnecessary conditional in generate_uid --- nova/db/sqlalchemy/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 0cbe56499..a3d8dde2f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -19,6 +19,7 @@ Implementation of SQLAlchemy backend """ +import random import warnings from nova import db @@ -542,7 +543,8 @@ def instance_create(context, values): session = get_session() with session.begin(): while instance_ref.internal_id == None: - internal_id = utils.generate_uid(instance_ref.__prefix__) + # Instances have integer internal ids. + internal_id = random.randint(0, 2 ** 32 - 1) if not instance_internal_id_exists(context, internal_id, session=session): instance_ref.internal_id = internal_id @@ -1152,7 +1154,7 @@ def volume_create(context, values): session = get_session() with session.begin(): while volume_ref.ec2_id == None: - ec2_id = utils.generate_uid(volume_ref.__prefix__) + ec2_id = utils.generate_uid('vol') if not volume_ec2_id_exists(context, ec2_id, session=session): volume_ref.ec2_id = ec2_id volume_ref.save(session=session) -- cgit From 8ccdae97558d9660a9a0fac8dad809a09cbd3c71 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 17:20:10 -0700 Subject: actually remove the conditional --- nova/utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 7683fc9f4..7f6311209 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -131,13 +131,9 @@ def runthis(prompt, cmd, check_exit_code=True): def generate_uid(topic, size=8): - if topic == "i": - # Instances have integer internal ids. - return random.randint(0, 2 ** 32 - 1) - else: - characters = '01234567890abcdefghijklmnopqrstuvwxyz' - choices = [random.choice(characters) for x in xrange(size)] - return '%s-%s' % (topic, ''.join(choices)) + characters = '01234567890abcdefghijklmnopqrstuvwxyz' + choices = [random.choice(characters) for x in xrange(size)] + return '%s-%s' % (topic, ''.join(choices)) def generate_mac(): -- cgit From 627a968e79ed21d970225e5ece332d9100abe022 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 23:04:49 -0700 Subject: fix completely broken ServiceTestCase --- nova/tests/service_unittest.py | 125 +++++++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 55 deletions(-) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index e74e0f726..a268bc4fe 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -23,8 +23,8 @@ Unit Tests for remote procedure calls using queue import mox from twisted.application.app import startApplication +from twisted.internet import defer -from nova import context from nova import exception from nova import flags from nova import rpc @@ -48,7 +48,7 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.BaseTestCase): +class ServiceManagerTestCase(test.TrialTestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): @@ -75,13 +75,12 @@ class ServiceManagerTestCase(test.BaseTestCase): self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.BaseTestCase): +class ServiceTestCase(test.TrialTestCase): """Test cases for Services""" def setUp(self): super(ServiceTestCase, self).setUp() self.mox.StubOutWithMock(service, 'db') - self.context = context.get_admin_context() def test_create(self): host = 'foo' @@ -144,87 +143,103 @@ class ServiceTestCase(test.BaseTestCase): # whether it is disconnected, it looks for a variable on itself called # 'model_disconnected' and report_state doesn't really do much so this # these are mostly just for coverage - def test_report_state(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - + @defer.inlineCallbacks def test_report_state_no_service(self): host = 'foo' binary = 'bar' + topic = 'test' service_create = {'host': host, 'binary': binary, + 'topic': topic, 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, + service.db.service_get_by_args(mox.IgnoreArg(), host, binary).AndRaise(exception.NotFound()) - service.db.service_create(self.context, + service.db.service_create(mox.IgnoreArg(), service_create).AndReturn(service_ref) - service.db.service_get(self.context, + service.db.service_get(mox.IgnoreArg(), service_ref['id']).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], + service.db.service_update(mox.IgnoreArg(), service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + yield serv.report_state() + @defer.inlineCallbacks def test_report_state_newly_disconnected(self): host = 'foo' binary = 'bar' + topic = 'test' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(Exception()) + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + service.db.service_get(mox.IgnoreArg(), + mox.IgnoreArg()).AndRaise(Exception()) self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - self.assert_(s.model_disconnected) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + yield serv.report_state() + self.assert_(serv.model_disconnected) + @defer.inlineCallbacks def test_report_state_newly_connected(self): host = 'foo' binary = 'bar' + topic = 'test' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + service.db.service_get(mox.IgnoreArg(), + service_ref['id']).AndReturn(service_ref) + service.db.service_update(mox.IgnoreArg(), service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() - s = service.Service() - s.model_disconnected = True - rv = yield s.report_state(host, binary) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + serv.model_disconnected = True + yield serv.report_state() - self.assert_(not s.model_disconnected) + self.assert_(not serv.model_disconnected) -- cgit From ba6d9293204284a7c74b5b0cffe63767941fd25c Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 26 Oct 2010 11:48:20 -0400 Subject: Delete BaseTestCase and with it the last reference to tornado. Requires commenting out some service_unittest tests which were silently failing under BaseTestCase and which now fail under TrialTestCase. vishy says he wrote the code and thinks he knows what is going wrong. --- nova/test.py | 156 ---------------------------------- nova/tests/api_unittest.py | 4 +- nova/tests/service_unittest.py | 184 +++++++++++++++++++++-------------------- run_tests.py | 16 ---- 4 files changed, 96 insertions(+), 264 deletions(-) diff --git a/nova/test.py b/nova/test.py index 8ef7eca1a..5c2a72819 100644 --- a/nova/test.py +++ b/nova/test.py @@ -28,7 +28,6 @@ import time import mox import stubout -from tornado import ioloop from twisted.internet import defer from twisted.trial import unittest @@ -159,158 +158,3 @@ class TrialTestCase(unittest.TestCase): _wrapped.func_name = self.originalAttach.func_name rpc.Consumer.attach_to_twisted = _wrapped - - -class BaseTestCase(TrialTestCase): - # TODO(jaypipes): Can this be moved into the TrialTestCase class? - """Base test case class for all unit tests. - - DEPRECATED: This is being removed once Tornado is gone, use TrialTestCase. - """ - def setUp(self): - """Run before each test method to initialize test environment""" - super(BaseTestCase, self).setUp() - # TODO(termie): we could possibly keep a more global registry of - # the injected listeners... this is fine for now though - self.ioloop = ioloop.IOLoop.instance() - - self._waiting = None - self._done_waiting = False - self._timed_out = False - - def _wait_for_test(self, timeout=60): - """ Push the ioloop along to wait for our test to complete. """ - self._waiting = self.ioloop.add_timeout(time.time() + timeout, - self._timeout) - - def _wait(): - - """Wrapped wait function. Called on timeout.""" - if self._timed_out: - self.fail('test timed out') - self._done() - if self._done_waiting: - self.ioloop.stop() - return - # we can use add_callback here but this uses less cpu when testing - self.ioloop.add_timeout(time.time() + 0.01, _wait) - - self.ioloop.add_callback(_wait) - self.ioloop.start() - - def _done(self): - """Callback used for cleaning up deferred test methods.""" - if self._waiting: - try: - self.ioloop.remove_timeout(self._waiting) - except Exception: # pylint: disable-msg=W0703 - # TODO(jaypipes): This produces a pylint warning. Should - # we really be catching Exception and then passing here? - pass - self._waiting = None - self._done_waiting = True - - def _maybe_inline_callbacks(self, func): - """ If we're doing async calls in our tests, wait on them. - - This is probably the most complicated hunk of code we have so far. - - First up, if the function is normal (not async) we just act normal - and return. - - Async tests will use the "Inline Callbacks" pattern, which means - you yield Deferreds at every "waiting" step of your code instead - of making epic callback chains. - - Example (callback chain, ugly): - - # A deferred instance - d = self.compute.terminate_instance(instance_id) - def _describe(_): - # Another deferred instance - d_desc = self.compute.describe_instances() - return d_desc - def _checkDescribe(rv): - self.assertEqual(rv, []) - d.addCallback(_describe) - d.addCallback(_checkDescribe) - d.addCallback(lambda x: self._done()) - self._wait_for_test() - - Example (inline callbacks! yay!): - - yield self.compute.terminate_instance(instance_id) - rv = yield self.compute.describe_instances() - self.assertEqual(rv, []) - - If the test fits the Inline Callbacks pattern we will automatically - handle calling wait and done. - """ - # TODO(termie): this can be a wrapper function instead and - # and we can make a metaclass so that we don't - # have to copy all that "run" code below. - g = func() - if not hasattr(g, 'send'): - self._done() - return defer.succeed(g) - - inlined = defer.inlineCallbacks(func) - d = inlined() - return d - - def _catch_exceptions(self, result, failure): - """Catches all exceptions and handles keyboard interrupts.""" - exc = (failure.type, failure.value, failure.getTracebackObject()) - if isinstance(failure.value, self.failureException): - result.addFailure(self, exc) - elif isinstance(failure.value, KeyboardInterrupt): - raise - else: - result.addError(self, exc) - - self._done() - - def _timeout(self): - """Helper method which trips the timeouts""" - self._waiting = False - self._timed_out = True - - def run(self, result=None): - """Runs the test case""" - - result.startTest(self) - test_method = getattr(self, self._testMethodName) - try: - try: - self.setUp() - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - return - - ok = False - try: - d = self._maybe_inline_callbacks(test_method) - d.addErrback(lambda x: self._catch_exceptions(result, x)) - d.addBoth(lambda x: self._done() and x) - self._wait_for_test() - ok = True - except self.failureException: - result.addFailure(self, sys.exc_info()) - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - - try: - self.tearDown() - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - ok = False - if ok: - result.addSuccess(self) - finally: - result.stopTest(self) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 0b1c3e353..0a81c575b 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -83,7 +83,7 @@ class FakeHttplibConnection(object): pass -class XmlConversionTestCase(test.BaseTestCase): +class XmlConversionTestCase(test.TrialTestCase): """Unit test api xml conversion""" def test_number_conversion(self): conv = apirequest._try_convert @@ -100,7 +100,7 @@ class XmlConversionTestCase(test.BaseTestCase): self.assertEqual(conv('-0'), 0) -class ApiEc2TestCase(test.BaseTestCase): +class ApiEc2TestCase(test.TrialTestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): super(ApiEc2TestCase, self).setUp() diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index e74e0f726..142c2ebea 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -48,7 +48,7 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.BaseTestCase): +class ServiceManagerTestCase(test.TrialTestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): @@ -75,7 +75,7 @@ class ServiceManagerTestCase(test.BaseTestCase): self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.BaseTestCase): +class ServiceTestCase(test.TrialTestCase): """Test cases for Services""" def setUp(self): @@ -140,91 +140,95 @@ class ServiceTestCase(test.BaseTestCase): startApplication(app, False) self.assert_(app) - # We're testing sort of weird behavior in how report_state decides - # whether it is disconnected, it looks for a variable on itself called - # 'model_disconnected' and report_state doesn't really do much so this - # these are mostly just for coverage - def test_report_state(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - def test_report_state_no_service(self): - host = 'foo' - binary = 'bar' - service_create = {'host': host, - 'binary': binary, - 'report_count': 0} - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(self.context, - service_create).AndReturn(service_ref) - service.db.service_get(self.context, - service_ref['id']).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - def test_report_state_newly_disconnected(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(Exception()) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - self.assert_(s.model_disconnected) - - def test_report_state_newly_connected(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - s.model_disconnected = True - rv = yield s.report_state(host, binary) - - self.assert_(not s.model_disconnected) +# TODO(gundlach): These tests were "passing" when this class inherited from +# BaseTestCase. In reality, they were failing, but BaseTestCase was +# swallowing the error. Now that we inherit from TrialTestCase, these tests +# are failing, and need to get fixed. +# # We're testing sort of weird behavior in how report_state decides +# # whether it is disconnected, it looks for a variable on itself called +# # 'model_disconnected' and report_state doesn't really do much so this +# # these are mostly just for coverage +# def test_report_state(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# def test_report_state_no_service(self): +# host = 'foo' +# binary = 'bar' +# service_create = {'host': host, +# 'binary': binary, +# 'report_count': 0} +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndRaise(exception.NotFound()) +# service.db.service_create(self.context, +# service_create).AndReturn(service_ref) +# service.db.service_get(self.context, +# service_ref['id']).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# def test_report_state_newly_disconnected(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndRaise(Exception()) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# self.assert_(s.model_disconnected) +# +# def test_report_state_newly_connected(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# s.model_disconnected = True +# rv = yield s.report_state(host, binary) +# +# self.assert_(not s.model_disconnected) diff --git a/run_tests.py b/run_tests.py index 9a2f40dc9..c16c63249 100644 --- a/run_tests.py +++ b/run_tests.py @@ -48,24 +48,8 @@ from twisted.scripts import trial as trial_script from nova import flags from nova import twistd -from nova.tests.access_unittest import * from nova.tests.auth_unittest import * -from nova.tests.api_unittest import * -from nova.tests.cloud_unittest import * -from nova.tests.compute_unittest import * -from nova.tests.flags_unittest import * -from nova.tests.network_unittest import * -from nova.tests.objectstore_unittest import * -from nova.tests.process_unittest import * -from nova.tests.quota_unittest import * -from nova.tests.rpc_unittest import * -from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * -from nova.tests.twistd_unittest import * -from nova.tests.validator_unittest import * -from nova.tests.virt_unittest import * -from nova.tests.volume_unittest import * -from nova.tests.virt_unittest import * FLAGS = flags.FLAGS -- cgit From f0d79d7d602a31fff03d8d934203128a2cd8940d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 26 Oct 2010 11:58:46 -0400 Subject: Oops, didn't mean to check this one in. Ninja-patch --- run_tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/run_tests.py b/run_tests.py index c16c63249..9a2f40dc9 100644 --- a/run_tests.py +++ b/run_tests.py @@ -48,8 +48,24 @@ from twisted.scripts import trial as trial_script from nova import flags from nova import twistd +from nova.tests.access_unittest import * from nova.tests.auth_unittest import * +from nova.tests.api_unittest import * +from nova.tests.cloud_unittest import * +from nova.tests.compute_unittest import * +from nova.tests.flags_unittest import * +from nova.tests.network_unittest import * +from nova.tests.objectstore_unittest import * +from nova.tests.process_unittest import * +from nova.tests.quota_unittest import * +from nova.tests.rpc_unittest import * +from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * +from nova.tests.twistd_unittest import * +from nova.tests.validator_unittest import * +from nova.tests.virt_unittest import * +from nova.tests.volume_unittest import * +from nova.tests.virt_unittest import * FLAGS = flags.FLAGS -- cgit From cce61a2d29fac66cdbef74517bf1ab025df33d1f Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Tue, 26 Oct 2010 09:06:37 -0500 Subject: Added Google Analytics code --- doc/source/_templates/layout.html | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/source/_templates/layout.html diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html new file mode 100644 index 000000000..0b72a77ac --- /dev/null +++ b/doc/source/_templates/layout.html @@ -0,0 +1,17 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + + +{% endblock %} + -- cgit From d8d12549a5e47c7c44f449f12d6b556e2c56483d Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 26 Oct 2010 15:37:32 -0700 Subject: More PEP8 fixes that were introduced in the last couple commits. --- nova/tests/api/openstack/test_api.py | 6 ++++++ nova/utils.py | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index a8c0ff9f8..dd83991b9 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -24,22 +24,28 @@ from nova.api.openstack import API from nova.api.openstack import faults from webob import Request + class APITest(unittest.TestCase): def test_exceptions_are_converted_to_faults(self): + @webob.dec.wsgify def succeed(req): return 'Succeeded' + @webob.dec.wsgify def raise_webob_exc(req): raise webob.exc.HTTPNotFound(explanation='Raised a webob.exc') + @webob.dec.wsgify def fail(req): raise Exception("Threw an exception") + @webob.dec.wsgify def raise_api_fault(req): exc = webob.exc.HTTPNotFound(explanation='Raised a webob.exc') return faults.Fault(exc) + api = API() api.application = succeed diff --git a/nova/utils.py b/nova/utils.py index 2c53b027e..bc495a691 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -213,10 +213,10 @@ def deferredToThread(f): def xhtml_escape(value): """Escapes a string so it is valid within XML or XHTML. - + Code is directly from the utf8 function in http://github.com/facebook/tornado/blob/master/tornado/escape.py - + """ return saxutils.escape(value, {'"': """}) @@ -232,4 +232,3 @@ def utf8(value): return value.encode("utf-8") assert isinstance(value, str) return value - -- cgit From 8e12d65af48e69f6ad211c6b880a3162e1d03e78 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 27 Oct 2010 00:05:42 -0400 Subject: New structure for documentation. --- doc/source/administration.guide.rst | 31 ++++++++ doc/source/concepts.and.introduction.rst | 123 +++++++++++++++++++++++++++++++ doc/source/development.guide.rst | 49 ++++++++++++ doc/source/index.rst | 52 ++++++------- 4 files changed, 226 insertions(+), 29 deletions(-) create mode 100644 doc/source/administration.guide.rst create mode 100644 doc/source/concepts.and.introduction.rst create mode 100644 doc/source/development.guide.rst diff --git a/doc/source/administration.guide.rst b/doc/source/administration.guide.rst new file mode 100644 index 000000000..9bea5e2dd --- /dev/null +++ b/doc/source/administration.guide.rst @@ -0,0 +1,31 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Administration Guide +==================== + +How to deploy, monitor, and debug Nova. + + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + getting.started + diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst new file mode 100644 index 000000000..fc4ebed21 --- /dev/null +++ b/doc/source/concepts.and.introduction.rst @@ -0,0 +1,123 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Concepts and Introduction +========================= + + +Introduction +------------ + +Nova is the software that controls your Infrastructure as as Service (IaaS) +cloud computing platform. It is similar in scope to Amazon EC2 and Rackspace +CloudServers. Nova does not include any virtualization software, rather it +defines drivers that interact with underlying virtualization mechanisms that +run on your host operating system, and exposes functionality over a web API. + +This document does not attempt to explain fundamental concepts of cloud +computing, IaaS, virtualization, or other related technologies. Instead, it +focues on describing how Nova's implementation of those concepts is achieved. + + +Concept: Projects +----------------- + +Projects + + +Concept: Virtualization +----------------------- + +* KVM +* UML +* XEN +* HyperV + + +Concept: Volumes +---------------- + +Volumes + + +Concept: Quotas +--------------- + +* Defaults +* Override for project + + +Concept: API +------------ + +* EC2 +* OpenStack / Rackspace + + +Concept: Networking +------------------- + +* Simple networking +* NASA-Style Networking + + +Concept: Services +----------------- + +Services + + +Concept: Flags +-------------- + +python-gflags + + +Concept: Plugins +---------------- + +LazyPluggable + + +Concept: IPC +------------ + +Rabbit! + + +Concept: Fakes +-------------- + +Fakes + + +Concept: Scheduler +------------------ + +Scheduler + + +Concept: ORM +------------ + +ORM + + +Concept: Manager Pattern +------------------------ + +Managers diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst new file mode 100644 index 000000000..18b08d6fe --- /dev/null +++ b/doc/source/development.guide.rst @@ -0,0 +1,49 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Development Guide +====================== + +Nova is written in python + + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + architecture + network + storage + auth + compute + endpoint + nova + fakes + binaries + modules + packages + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/index.rst b/doc/source/index.rst index 1109e9011..d4cdd1408 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -18,37 +18,31 @@ Welcome to Nova's documentation! ================================ -Nova is a cloud computing fabric controller (the main part of an IaaS system). -It is written in Python and relies on the standard AMQP messaging protocol, uses the Twisted framework, -and optionally uses the Redis distributed key value store for authorization. - -Nova is intended to be easy to extend and adapt. For example, authentication and authorization -requests by default use an RDBMS-backed datastore driver. However, there is already support -for using LDAP backing authentication (slapd) and if you wish to "fake" LDAP, there is a module -available that uses ReDIS to store authentication information in an LDAP-like backing datastore. -It has extensive test coverage, and uses the Sphinx toolkit (the same as Python itself) for code -and developer documentation. Additional documentation is available on the -'OpenStack wiki '_. -While Nova is currently in Beta use within several organizations, the codebase -is very much under active development - please test it and log bugs! - -Contents: +Nova is a cloud computing fabric controller, the main part of an IaaS system. + +Nova is written with the following design guidelines in mind: + +* Component based architecture: Quickly add new behaviors +* Highly available: Scale to very serious workloads +* Fault-Tollerant: Isloated processes avoid cascading failures +* Recoverable: Failures should be easy to diagnose, debug, and rectify +* Open Standards: Be a reference implementation for a community-driven api + +This documentation is generated by the Sphinx toolkit and lives in the source +tree. Additional documentation on Nova and other components of OpenStack can +be found on the 'OpenStack wiki '. + + +Contents +-------- .. toctree:: - :maxdepth: 2 - - getting.started - architecture - network - storage - auth - compute - endpoint - nova - fakes - binaries - modules - packages + :maxdepth: 1 + + concepts.and.introduction + administration.guide + development.guide + Indices and tables ================== -- cgit From 79acdcca7d37e81d626be7a3369394ef9dface1b Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 27 Oct 2010 11:10:50 -0400 Subject: Style cleanups and review from Eric. --- nova/api/ec2/cloud.py | 23 ++++++++++++----------- nova/compute/manager.py | 15 ++++++++++++--- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 51e972aa7..9084958a1 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -836,29 +836,30 @@ class CloudController(object): elevated = context.elevated() for num in range(num_instances): - + instance_data = base_options - instance_data['mac_address'] = utils.generate_mac() - instance_data['launch_index'] = num instance_ref = self.compute_manager.create_instance(context, - instance_data, - security_groups) + instance_data, + security_groups, + mac_address=utils.generate_mac(), + launch_index=num) + inst_id = instance_ref['id'] internal_id = instance_ref['internal_id'] ec2_id = internal_id_to_ec2_id(internal_id) - instance_ref['hostname'] = ec2_id self.compute_manager.update_instance(context, - instance_ref['id'], - instance_ref) + inst_id, + instance_ref, + hostname=ec2_id) # TODO(vish): This probably should be done in the scheduler # or in compute as a call. The network should be # allocated after the host is assigned and setup # can happen at the same time. address = self.network_manager.allocate_fixed_ip(context, - instance_ref['id'], + inst_id, vpn) network_topic = self._get_network_topic(context) rpc.cast(elevated, @@ -870,9 +871,9 @@ class CloudController(object): FLAGS.scheduler_topic, {"method": "run_instance", "args": {"topic": FLAGS.compute_topic, - "instance_id": instance_ref['id']}}) + "instance_id": inst_id}}) logging.debug("Casting to scheduler for %s/%s's instance %s" % - (context.project.name, context.user.name, instance_ref['id'])) + (context.project.name, context.user.name, inst_id)) return self._format_run_instances(context, reservation_id) def terminate_instances(self, context, instance_id, **kwargs): diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d99d938af..c04dd213a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -69,8 +69,8 @@ class ComputeManager(manager.Manager): def refresh_security_group(self, context, security_group_id, **_kwargs): yield self.driver.refresh_security_group(security_group_id) - - def create_instance(self, context, instance_data, security_groups=[]): + def create_instance(self, context, instance_data, security_groups=[], + **kwargs): """Creates the instance in the datastore and returns the new instance as a mapping @@ -78,11 +78,15 @@ class ComputeManager(manager.Manager): :param instance_data: mapping of instance options :param security_groups: list of security group ids to attach to the instance + :param **kwargs: All additional keyword args are treated + as data fields of the instance to be + created :retval Returns a mapping of the instance information that has just been created """ + instance_data.update(kwargs) instance_ref = self.db.instance_create(context, instance_data) inst_id = instance_ref['id'] @@ -93,15 +97,20 @@ class ComputeManager(manager.Manager): security_group_id) return instance_ref - def update_instance(self, context, instance_id, instance_data): + def update_instance(self, context, instance_id, instance_data, + **kwargs): """Updates the instance in the datastore :param context: The security context :param instance_data: mapping of instance options + :param **kwargs: All additional keyword args are treated + as data fields of the instance to be + updated :retval None """ + instance_data.update(kwargs) self.db.instance_update(context, instance_id, instance_data) @defer.inlineCallbacks -- cgit From 7c74613eb801679c67f551e307265b4af1dc12a6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 27 Oct 2010 10:28:52 -0700 Subject: updates from review, fix models.get and note about exception raising --- nova/compute/manager.py | 7 +++++-- nova/db/sqlalchemy/models.py | 5 +---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 69ac398c0..b9ba6852a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -208,12 +208,15 @@ class ComputeManager(manager.Manager): volume_id, instance_id, mountpoint) - except Exception: + except Exception as exc: # pylint: disable-msg=W0702 + # NOTE(vish): The inline callback eats the exception info so we + # log the traceback here and reraise the same + # ecxception below. logging.exception("instance %s: attach failed %s, removing", instance_id, mountpoint) yield self.volume_manager.remove_compute_volume(context, volume_id) - raise + raise exc defer.returnValue(True) @defer.inlineCallbacks diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index ed1bf6c85..7d65cd371 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -75,10 +75,7 @@ class NovaBase(object): return getattr(self, key) def get(self, key, default=None): - try: - return getattr(self, key) - except AttributeError, KeyError: - return default + return getattr(self, key, default) def __iter__(self): self._i = iter(object_mapper(self).columns) -- cgit From 32a75c0181524a0d369419495f9a6e3d67d0e1e2 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 27 Oct 2010 13:49:27 -0400 Subject: Doc updates. --- doc/source/administration.guide.rst | 6 ++++-- doc/source/concepts.and.introduction.rst | 25 ++++++++++++++++++++----- doc/source/flags.rst | 23 +++++++++++++++++++++++ doc/source/multi.node.install.rst | 21 +++++++++++++++++++++ doc/source/quickstart.rst | 21 +++++++++++++++++++++ 5 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 doc/source/flags.rst create mode 100644 doc/source/multi.node.install.rst create mode 100644 doc/source/quickstart.rst diff --git a/doc/source/administration.guide.rst b/doc/source/administration.guide.rst index 9bea5e2dd..4ff1afc9a 100644 --- a/doc/source/administration.guide.rst +++ b/doc/source/administration.guide.rst @@ -27,5 +27,7 @@ Contents .. toctree:: :maxdepth: 1 - getting.started - + quickstart + getting.started + multi.node.install + flags diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index fc4ebed21..701e43dfb 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -15,6 +15,7 @@ License for the specific language governing permissions and limitations under the License. + Concepts and Introduction ========================= @@ -32,9 +33,17 @@ This document does not attempt to explain fundamental concepts of cloud computing, IaaS, virtualization, or other related technologies. Instead, it focues on describing how Nova's implementation of those concepts is achieved. +This page outlines concepts that you will need to understand as a user or +administrator of an OpenStack installation. Each section links to more more +detailed information in the `Administration Guide`_, but you'll probably want +to read this section straight-through before tackling the specifics presented +in the administration guide. + +.. _`Administration Guide`: administration.guide.html -Concept: Projects ------------------ + +Concept: Users and Projects +--------------------------- Projects @@ -72,7 +81,7 @@ Concept: Networking ------------------- * Simple networking -* NASA-Style Networking +* VLAN Networking Concept: Services @@ -93,8 +102,8 @@ Concept: Plugins LazyPluggable -Concept: IPC ------------- +Concept: IPC/RPC +---------------- Rabbit! @@ -121,3 +130,9 @@ Concept: Manager Pattern ------------------------ Managers + + +Concept: Security Groups +------------------------ + +Security groups diff --git a/doc/source/flags.rst b/doc/source/flags.rst new file mode 100644 index 000000000..4c950aa88 --- /dev/null +++ b/doc/source/flags.rst @@ -0,0 +1,23 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Flags and Flagfiles +=================== + +* python-gflags +* flagfiles +* list of flags by component (see concepts list) diff --git a/doc/source/multi.node.install.rst b/doc/source/multi.node.install.rst new file mode 100644 index 000000000..443e2c4c3 --- /dev/null +++ b/doc/source/multi.node.install.rst @@ -0,0 +1,21 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Multi-Node Nova +=============== + +http://etherpad.openstack.org/NovaMultinodeInstall diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst new file mode 100644 index 000000000..7a41d4be6 --- /dev/null +++ b/doc/source/quickstart.rst @@ -0,0 +1,21 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Quickstart +========================= + +http://github.com/vishvananda/novascript -- cgit From 213b9987365c4b336b63e08e1ca187a43d00fa3d Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 27 Oct 2010 14:55:01 -0400 Subject: OK, let's try this one more time. --- nova/api/ec2/cloud.py | 13 ++++++------- nova/api/openstack/servers.py | 4 ++-- nova/compute/manager.py | 16 ++++++---------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 9084958a1..7b6144ba5 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -837,13 +837,11 @@ class CloudController(object): for num in range(num_instances): - instance_data = base_options - instance_ref = self.compute_manager.create_instance(context, - instance_data, security_groups, mac_address=utils.generate_mac(), - launch_index=num) + launch_index=num, + **base_options) inst_id = instance_ref['id'] internal_id = instance_ref['internal_id'] @@ -851,7 +849,6 @@ class CloudController(object): self.compute_manager.update_instance(context, inst_id, - instance_ref, hostname=ec2_id) # TODO(vish): This probably should be done in the scheduler @@ -903,8 +900,10 @@ class CloudController(object): 'state': 0, 'terminated_at': now} self.compute_manager.update_instance(context, - instance_ref['id'], - updated_data) + instance_ref['id'], + state_description='terminating', + state=0, + terminated_at=now) # FIXME(ja): where should network deallocate occur? address = db.instance_get_floating_address(context, diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index e1a254d4e..1d8aa2fa4 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -246,11 +246,11 @@ class Controller(wsgi.Controller): inst['mac_address'] = utils.generate_mac() inst['launch_index'] = 0 - ref = self.compute_manager.create_instance(ctxt, inst) + ref = self.compute_manager.create_instance(ctxt, **inst) inst['id'] = ref['internal_id'] inst['hostname'] = str(ref['internal_id']) - self.compute_manager.update_instance(ctxt, inst['id'], inst) + self.compute_manager.update_instance(ctxt, inst['id'], **inst) address = self.network_manager.allocate_fixed_ip(ctxt, inst['id']) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index c04dd213a..7cdd6b110 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -69,13 +69,11 @@ class ComputeManager(manager.Manager): def refresh_security_group(self, context, security_group_id, **_kwargs): yield self.driver.refresh_security_group(security_group_id) - def create_instance(self, context, instance_data, security_groups=[], - **kwargs): + def create_instance(self, context, security_groups=[], **kwargs): """Creates the instance in the datastore and returns the new instance as a mapping :param context: The security context - :param instance_data: mapping of instance options :param security_groups: list of security group ids to attach to the instance :param **kwargs: All additional keyword args are treated @@ -86,23 +84,22 @@ class ComputeManager(manager.Manager): that has just been created """ - instance_data.update(kwargs) - instance_ref = self.db.instance_create(context, instance_data) + instance_ref = self.db.instance_create(context, kwargs) inst_id = instance_ref['id'] elevated = context.elevated() + security_groups = kwargs.get('security_groups', []) for security_group_id in security_groups: self.db.instance_add_security_group(elevated, inst_id, security_group_id) return instance_ref - def update_instance(self, context, instance_id, instance_data, - **kwargs): + def update_instance(self, context, instance_id, **kwargs): """Updates the instance in the datastore :param context: The security context - :param instance_data: mapping of instance options + :param instance_id: ID of the instance to update :param **kwargs: All additional keyword args are treated as data fields of the instance to be updated @@ -110,8 +107,7 @@ class ComputeManager(manager.Manager): :retval None """ - instance_data.update(kwargs) - self.db.instance_update(context, instance_id, instance_data) + self.db.instance_update(context, instance_id, kwargs) @defer.inlineCallbacks @exception.wrap_exception -- cgit From 55a0cfc96b92bd050115244c850f4ad58791c83f Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 27 Oct 2010 15:52:06 -0400 Subject: Doc skeleton from collaborative etherpad hack session. --- doc/source/administration.guide.rst | 2 + doc/source/auth.rst | 15 +++++++ doc/source/compute.rst | 21 ++++++++++ doc/source/concepts.and.introduction.rst | 70 ++++++++++++++++++++++++-------- doc/source/development.guide.rst | 15 +++++-- doc/source/monitoring.rst | 27 ++++++++++++ doc/source/multi.node.install.rst | 4 ++ doc/source/network.rst | 19 +++++++++ doc/source/services.rst | 31 ++++++++++++++ doc/source/storage.rst | 16 ++++++++ 10 files changed, 201 insertions(+), 19 deletions(-) create mode 100644 doc/source/monitoring.rst create mode 100644 doc/source/services.rst diff --git a/doc/source/administration.guide.rst b/doc/source/administration.guide.rst index 4ff1afc9a..789e97381 100644 --- a/doc/source/administration.guide.rst +++ b/doc/source/administration.guide.rst @@ -30,4 +30,6 @@ Contents quickstart getting.started multi.node.install + services flags + monitoring diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 3fcb309cd..2b8ad7924 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -18,6 +18,21 @@ Auth Documentation ================== + +from etherpad todo +------------------ + +:: + * FIX RST IMPORT ERRORS + * Manager + * Driver + * ldap + * fakeldap + * redisectomy + +legacy docs +----------- + Nova provides RBAC (Role-based access control) of the AWS-type APIs. We define the following roles: Roles-Based Access Control of AWS-style APIs using SAML Assertions diff --git a/doc/source/compute.rst b/doc/source/compute.rst index 5b08dbd5b..76957f316 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -18,6 +18,27 @@ Compute Documentation ===================== + +from etherpad todo +------------------ + +:: + * FIX RST IMPORT ERRORS + * Compute Manager + * public methods + * responsibilities + * Compute Driver + * public methods + * libvirt implementation + - kvm/qemu + - uml + * xen implementation + * hyperv implementation + + +legacy docs +----------- + This page contains the Compute Package documentation. diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 701e43dfb..5c006454d 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -45,7 +45,10 @@ in the administration guide. Concept: Users and Projects --------------------------- -Projects +* access to images is limited by project +* access/secret are per user +* keypairs are per user +* quotas are per project Concept: Virtualization @@ -55,6 +58,7 @@ Concept: Virtualization * UML * XEN * HyperV +* qemu Concept: Volumes @@ -70,6 +74,13 @@ Concept: Quotas * Override for project +Concept: RBAC +------------- + +* Intersecting Roles +* cloudadmin vs. user admin flag + + Concept: API ------------ @@ -80,14 +91,30 @@ Concept: API Concept: Networking ------------------- -* Simple networking -* VLAN Networking +* VLAN + * Cloudpipe + * Certificates (See also: CA) +* Flat Networking +* Flat with DHCP +* How to generate addresses +* Floating Addresses Concept: Services ----------------- -Services +* nova-api +* nova-scheduler +* nova-compute +* nova-volume +* nova-network +* nova-instancemonitor + + +Concept: nova-manage +-------------------- + +nova manage Concept: Flags @@ -99,7 +126,13 @@ python-gflags Concept: Plugins ---------------- -LazyPluggable +* Managers/Drivers: utils.import_object from string flag +* virt/connections: conditional loading from string flag +* db: LazyPluggable via string flag +* auth_manager: utils.import_class based on string flag +* Volumes: moving to pluggable driver instead of manager +* Network: pluggable managers +* Compute: same driver used, but pluggable at connection Concept: IPC/RPC @@ -111,28 +144,33 @@ Rabbit! Concept: Fakes -------------- -Fakes +* auth +* ldap Concept: Scheduler ------------------ -Scheduler +* simple +* random -Concept: ORM ------------- +Concept: Security Groups +------------------------ -ORM +Security groups -Concept: Manager Pattern ------------------------- +Concept: Certificate Authority +------------------------------ -Managers +Per-project CA +* Images +* VPNs -Concept: Security Groups ------------------------- +Concept: Images +--------------- -Security groups +* launching +* bundling diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 18b08d6fe..2c8fa86f9 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -27,11 +27,20 @@ Contents .. toctree:: :maxdepth: 1 - architecture - network + database storage - auth compute + network + auth + + +Older Contents +-------------- + +.. toctree:: + :maxdepth: 1 + + architecture endpoint nova fakes diff --git a/doc/source/monitoring.rst b/doc/source/monitoring.rst new file mode 100644 index 000000000..e7766a6e7 --- /dev/null +++ b/doc/source/monitoring.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Monitoring +========== + +* components +* throughput +* exceptions +* hardware + +* ganglia +* syslog diff --git a/doc/source/multi.node.install.rst b/doc/source/multi.node.install.rst index 443e2c4c3..dad9dcd01 100644 --- a/doc/source/multi.node.install.rst +++ b/doc/source/multi.node.install.rst @@ -19,3 +19,7 @@ Multi-Node Nova =============== http://etherpad.openstack.org/NovaMultinodeInstall + +* Puppet +* Chef +* PXE diff --git a/doc/source/network.rst b/doc/source/network.rst index 357a0517f..ef374a726 100644 --- a/doc/source/network.rst +++ b/doc/source/network.rst @@ -18,6 +18,25 @@ nova Networking ================ + +from etherpad todo +------------------ +:: + * Network Manager + * overview (use vish email) + * public methods + * responsibilities + * Vlan + DHCP implementation + * Flat + DHCP implementation + * Flat impleplementation + * Network Driver + * public methods + * hardware specific commands + + +legacy docs +----------- + The nova networking components manage private networks, public IP addressing, VPN connectivity, and firewall rules. Components diff --git a/doc/source/services.rst b/doc/source/services.rst new file mode 100644 index 000000000..7ff8fa6c3 --- /dev/null +++ b/doc/source/services.rst @@ -0,0 +1,31 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Services +======== + +Admin guide should go beyoned concepts & introduction by talking about how +each service interacts with each other, what requirements for deployment are, +monotoring, logging, etc + + +* nova-api +* nova-scheduler +* nova-compute +* nova-volume +* nova-network +* nova-instancemonitor diff --git a/doc/source/storage.rst b/doc/source/storage.rst index f77e5f0e5..72880a4a1 100644 --- a/doc/source/storage.rst +++ b/doc/source/storage.rst @@ -18,6 +18,22 @@ Storage in the Nova Cloud ========================= + +from etherpad todo +------------------- +:: + * Volume Manager + * public methods + * responsibilities + * Volume Driver + * public methods + * AoE implementation + * ISCSI implementation + + +legacy docs +----------- + There are three primary classes of storage in a nova cloud environment: * Ephemeral Storage (local disk within an instance) -- cgit From a1287cf4e15d469163ec6465ea5f6ce200c20543 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 27 Oct 2010 17:31:46 -0400 Subject: cleanup rrd doc generation. --- doc/source/auth.rst | 26 +++++++++++++++++++----- doc/source/compute.rst | 54 ++++++++++++++++++++++---------------------------- doc/source/conf.py | 2 +- nova/auth/fakeldap.py | 1 - nova/auth/manager.py | 3 +-- 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 2b8ad7924..d739e4b8e 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -195,18 +195,34 @@ The :mod:`signer` Module :undoc-members: :show-inheritance: -The :mod:`users` Module ------------------------ +The :mod:`manager` Module +------------------------- -.. automodule:: nova.auth.users +.. automodule:: nova.auth.manager :members: :undoc-members: :show-inheritance: -The :mod:`users_unittest` Module +The :mod:`ldapdriver` Module +---------------------------- + +.. automodule:: nova.auth.ldapdriver + :members: + :undoc-members: + :show-inheritance: + +The :mod:`dbdriver` Module +---------------------------- + +.. automodule:: nova.auth.dbdriver + :members: + :undoc-members: + :show-inheritance: + +The :mod:`auth_unittest` Module -------------------------------- -.. automodule:: nova.tests.users_unittest +.. automodule:: nova.tests.auth_unittest :members: :undoc-members: :show-inheritance: diff --git a/doc/source/compute.rst b/doc/source/compute.rst index 76957f316..93070fa88 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -15,80 +15,74 @@ License for the specific language governing permissions and limitations under the License. -Compute Documentation -===================== - from etherpad todo ------------------ -:: * FIX RST IMPORT ERRORS * Compute Manager * public methods * responsibilities * Compute Driver * public methods - * libvirt implementation - - kvm/qemu - - uml + * libvirt implementation (kvm/qemu vs uml) * xen implementation * hyperv implementation -legacy docs ------------ +Compute Documentation +===================== This page contains the Compute Package documentation. -The :mod:`disk` Module ----------------------- +The :mod:`manager` Module +------------------------- -.. automodule:: nova.compute.disk +.. automodule:: nova.compute.manager :members: :undoc-members: :show-inheritance: -The :mod:`exception` Module ---------------------------- +The :mod:`disk` Module +---------------------- -.. automodule:: nova.compute.exception +.. automodule:: nova.compute.disk :members: :undoc-members: :show-inheritance: -The :mod:`model` Module -------------------------- +The :mod:`instance_types` Module +-------------------------------- -.. automodule:: nova.compute.model +.. automodule:: nova.compute.instance_types :members: :undoc-members: :show-inheritance: - -The :mod:`network` Module -------------------------- -.. automodule:: nova.compute.network +The :mod:`power_state` Module +----------------------------- + +.. automodule:: nova.compute.power_state :members: :undoc-members: :show-inheritance: -The :mod:`node` Module ----------------------- +The :mod:`monitor` Module +------------------------- -.. automodule:: nova.compute.node +.. automodule:: nova.compute.monitor :members: :undoc-members: :show-inheritance: - + RELATED TESTS ---------------- +------------- -The :mod:`node_unittest` Module -------------------------------- +The :mod:`compute_unittest` Module +---------------------------------- -.. automodule:: nova.tests.node_unittest +.. automodule:: nova.tests.compute_unittest :members: :undoc-members: :show-inheritance: diff --git a/doc/source/conf.py b/doc/source/conf.py index 505771ff8..3cf79503a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append([os.path.abspath('../nova'), os.path.abspath('..'), os.path.abspath('../bin')]) +sys.path.insert(0, '../..') # -- General configuration ----------------------------------------------------- diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index cf3a84a5d..1a49b73fe 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -79,7 +79,6 @@ def _match_query(query, attrs): &, |, and ! are supported in the query. No syntax checking is performed, so malformed querys will not work correctly. - """ # cut off the parentheses inner = query[1:-1] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 001a96875..7b2b68161 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -84,12 +84,11 @@ class AuthBase(object): @classmethod def safe_id(cls, obj): - """Safe get object id + """Safely get object id. This method will return the id of the object if the object is of this class, otherwise it will return the original object. This allows methods to accept objects or ids as paramaters. - """ if isinstance(obj, cls): return obj.id -- cgit From 0211ec7d61bea7e818418b6f17fdd45770ca9337 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 27 Oct 2010 16:33:25 -0700 Subject: updates to auth, concepts, and network, fix of docstring --- doc/source/auth.rst | 46 ++++++++++++++++----------- doc/source/concepts.and.introduction.rst | 53 +++++++++++++++++++++++-------- doc/source/conf.py | 2 +- doc/source/images/cloudpipe.png | Bin 0 -> 89812 bytes doc/source/network.rst | 34 ++++++++++---------- nova/auth/manager.py | 1 - 6 files changed, 85 insertions(+), 51 deletions(-) create mode 100644 doc/source/images/cloudpipe.png diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 2b8ad7924..0e3303949 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,9 +15,10 @@ License for the specific language governing permissions and limitations under the License. -Auth Documentation -================== +.. _auth: +Auth Documentation +================== from etherpad todo ------------------ @@ -39,7 +40,7 @@ Roles-Based Access Control of AWS-style APIs using SAML Assertions “Achieving FIPS 199 Moderate certification of a hybrid cloud environment using CloudAudit and declarative C.I.A. classifications” Introduction --------------- +------------ We will investigate one method for integrating an AWS-style API with US eAuthentication-compatible federated authentication systems, to achieve access controls and limits based on traditional operational roles. Additionally, we will look at how combining this approach, with an implementation of the CloudAudit APIs, will allow us to achieve a certification under FIPS 199 Moderate classification for a hybrid cloud environment. @@ -54,24 +55,26 @@ Typical implementations of US eAuth authentication systems are structured as fol [ SUN Identity Manager or other SAML Policy Controller ] --> maps URLs to groups… [ Apache Policy Agent in front of eAuth-secured Web Application ] - + In more ideal implementations, the remainder of the application-specific account information is stored either in extended schema on the LDAP server itself, via the use of a translucent LDAP proxy, or in an independent datastore keyed off of the UID provided via SAML assertion. -Basic AWS API call structure ----------------------------- +.. _auth_roles: + +Roles +----- -AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: +AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: * System User * System Administrator * Network Administrator * Project Manager * Cloud Administrator -* (IT-Sec?) +* IT-Security -There is an additional, conceptual end-user that may or may not have API access: +There is an additional, conceptual end-user that may or may not have API access: -* (EXTERNAL) End-user / Third-party User +* (EXTERNAL) End-user / Third-party User Basic operations are available to any System User: @@ -106,7 +109,7 @@ Cloud Administrator: Enhancements ------------ -* SAML Token passing +* SAML Token passing * REST interfaces * SOAP interfaces @@ -151,7 +154,7 @@ Dirty Cloud – Hybrid Data Centers * CloudAudit bridge interfaces * Anything in the ARP table -A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. +A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. This interconnect is typically a bridged VPN connection. Any machines that can be bridged into a hybrid environment in this fashion (at Layer 2) must implement a minimum version of the CloudAudit spec, such that they can be queried to provide a complete picture of the IT-sec runtime environment. @@ -167,7 +170,7 @@ The Details System limits ------------- -The following limits need to be defined and enforced: +The following limits need to be defined and enforced: * Total number of instances allowed (user / project) * Total number of instances, per instance type (user / project) @@ -195,18 +198,18 @@ The :mod:`signer` Module :undoc-members: :show-inheritance: -The :mod:`users` Module +The :mod:`auth` Module ----------------------- -.. automodule:: nova.auth.users +.. automodule:: nova.auth.manager :members: :undoc-members: :show-inheritance: -The :mod:`users_unittest` Module +The :mod:`auth_unittest` Module -------------------------------- -.. automodule:: nova.tests.users_unittest +.. automodule:: nova.tests.auth_unittest :members: :undoc-members: :show-inheritance: @@ -219,4 +222,11 @@ The :mod:`access_unittest` Module :undoc-members: :show-inheritance: +The :mod:`ec2` Module +------------------------ + +.. automodule:: nova.api.ec2 + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 5c006454d..54a9c7ccd 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -61,25 +61,33 @@ Concept: Virtualization * qemu +Concept: Instances +------------------ + +An 'instance' is a word for a virtual machine that runs inside the cloud. + Concept: Volumes ---------------- -Volumes +A 'volume' is a detachable block storage device. You can think of it as a usb hard drive. It can only be attached to one instance at a time, and it behaves Concept: Quotas --------------- -* Defaults -* Override for project +Nova supports per-project quotas. There are currently quotas for number of instances, total number of cores, number of volumes, total number of gigabytes, and number of floating ips. Concept: RBAC ------------- -* Intersecting Roles -* cloudadmin vs. user admin flag +Nova provides roles based access control (RBAC) for access to api commands. A user can have a number of different :ref:`roles `. Roles define which api_commands a user can perform. + +It is important to know that there are user-specific (sometimes called global) roles and project-specific roles. A user's actual permissions in a particular project are the INTERSECTION of his user-specific roles and is project-specific roles. +For example: A user can access api commands allowed to the netadmin role (like allocate_address) only if he has the user-specific netadmin role AND the project-specific netadmin role. + +More information about RBAC can be found in the :ref:`auth`. Concept: API ------------ @@ -91,14 +99,31 @@ Concept: API Concept: Networking ------------------- -* VLAN - * Cloudpipe - * Certificates (See also: CA) -* Flat Networking -* Flat with DHCP -* How to generate addresses -* Floating Addresses +Nova has a concept of Fixed Ips and Floating ips. Fixed ips are assigned to an instance on creation and stay the same until the instance is explicitly terminated. Floating ips are ip addresses that can be dynamically associated with an instance. This address can be disassociated and associated with another instance at any time. + +There are multiple strategies available for implementing fixed ips: + +Flat Mode +^^^^^^^^^ + +The simplest networking mode. Each instance receives a fixed ip from the pool. All instances are attached to the same bridge (br100) by default. The bridge must be configured manually. The networking configuration is injected into the instance before it is booted. Note that this currently only works on linux-style systems that keep networking configuration in /etc/network/interfaces. +Flat DHCP Mode +^^^^^^^^^^^^^^ + +This is similar to the flat mode, in that all instances are attached to the same bridge. In this mode nova does a bit more configuration, it will attempt to bridge into an ethernet device (eth0 by default). It will also run dnsmasq as a dhcpserver listening on this bridge. Instances receive their fixed ips by doing a dhcpdiscover. + +VLAN DHCP Mode +^^^^^^^^^^^^^^ + +This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code name cloudpipe) needs to be created. Nova generates a certificate and key for the userto access the vpn and starts the vpn automatically. + +The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) + +.. image:: /images/cloudpipe.png + :width: 100% + +.. Concept: Services ----------------- @@ -114,7 +139,7 @@ Concept: Services Concept: nova-manage -------------------- -nova manage +nova-manage is a command line utility for performing administrative tasks and checking on the health of the system. Concept: Flags diff --git a/doc/source/conf.py b/doc/source/conf.py index 505771ff8..34f049362 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append([os.path.abspath('../nova'), os.path.abspath('..'), os.path.abspath('../bin')]) +sys.path.append([os.path.abspath('../../'), os.path.abspath('../../bin')]) # -- General configuration ----------------------------------------------------- diff --git a/doc/source/images/cloudpipe.png b/doc/source/images/cloudpipe.png new file mode 100644 index 000000000..ffdd181f2 Binary files /dev/null and b/doc/source/images/cloudpipe.png differ diff --git a/doc/source/network.rst b/doc/source/network.rst index ef374a726..b776ae40e 100644 --- a/doc/source/network.rst +++ b/doc/source/network.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -43,65 +43,65 @@ Components ---------- There are several key components: -* NetworkController (Manages address and vlan allocation) +* NetworkController (Manages address and vlan allocation) * RoutingNode (NATs public IPs to private IPs, and enforces firewall rules) * AddressingNode (runs DHCP services for private networks) * BridgingNode (a subclass of the basic nova ComputeNode) * TunnelingNode (provides VPN connectivity) - + Component Diagram ----------------- Overview:: - (PUBLIC INTERNET) + (PUBLIC INTERNET) | \ / \ / \ [RoutingNode] ... [RN] [TunnelingNode] ... [TN] | \ / | | | < AMQP > | | - [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] + [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] \ | \ / / \ / \ / \ / \ [BridgingNode] ... [BridgingNode] - - + + [NetworkController] ... [NetworkController] \ / < AMQP > | / \ - [CloudController]...[CloudController] + [CloudController]...[CloudController] -While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). +While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). State Model ----------- Network State consists of the following facts: * VLAN assignment (to a project) -* Private Subnet assignment (to a security group) in a VLAN +* Private Subnet assignment (to a security group) in a VLAN * Private IP assignments (to running instances) * Public IP allocations (to a project) * Public IP associations (to a private IP / running instance) -While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. +While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. The Public Traffic Path ----------------------- - + Public Traffic:: (PUBLIC INTERNET) | - <-- [RoutingNode] + <-- [RoutingNode] | [AddressingNode] --> | - ( VLAN ) + ( VLAN ) | <-- [BridgingNode] | - + -The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. +The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. -Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). +Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 001a96875..8388d5a5d 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -89,7 +89,6 @@ class AuthBase(object): This method will return the id of the object if the object is of this class, otherwise it will return the original object. This allows methods to accept objects or ids as paramaters. - """ if isinstance(obj, cls): return obj.id -- cgit From 938caf0eb6620599a48696e9ca14cd4013e5cffb Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Wed, 27 Oct 2010 17:14:24 -0700 Subject: added nova-manage docs --- doc/source/concepts.and.introduction.rst | 113 ++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 54a9c7ccd..76db9e9f2 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -135,11 +135,120 @@ Concept: Services * nova-network * nova-instancemonitor - Concept: nova-manage -------------------- -nova-manage is a command line utility for performing administrative tasks and checking on the health of the system. +Introduction +++++++++++++ + +The nova-manage command is used to perform many essential functions for +administration and ongoing maintenance of nova, such as user creation, +vpn management, and much more. + +The standard pattern for executing a nova-manage command is: + +``nova-manage []`` + +For example, to obtain a list of all projects: + +``nova-manage project list`` + +User Maintenance +++++++++++++++++ + +* user admin: creates a new admin and prints exports + * arguments: name [access] [secret] +* user create: creates a new user and prints exports + * arguments: name [access] [secret] +* user delete: deletes an existing user + * arguments: name +* user exports: prints access and secrets for user in export format + * arguments: name +* user list: lists all users + * arguments: none +* user modify: update a users keys & admin flag + * arguments: accesskey secretkey admin + * leave any field blank to ignore it, admin should be 'T', 'F', or blank + +Project Maintenance ++++++++++++++++++++ + +* project add: Adds user to project + * arguments: project user +* project create: Creates a new project + * arguments: name project_manager [description] +* project delete: Deletes an existing project + * arguments: project_id +* project environment: Exports environment variables to an sourcable file + * arguments: project_id user_id [filename='novarc] +* project list: lists all projects + * arguments: none +* project quota: Set or display quotas for project + * arguments: project_id [key] [value] +* project remove: Removes user from project + * arguments: project user +* project scrub: Deletes data associated with project + * arguments: project +* project zipfile: Exports credentials for project to a zip file + * arguments: project_id user_id [filename='nova.zip] + +User Role Management +++++++++++++++++++++ + +* role add: adds role to user + * if project is specified, adds project specific role + * arguments: user, role [project] +* role has: checks to see if user has role + * if project is specified, returns True if user has + the global role and the project role + * arguments: user, role [project] +* role remove: removes role from user + * if project is specified, removes project specific role + * arguments: user, role [project] + + +Nova Shell +++++++++++ + +* shell bpython + * start a new bpython shell +* shell ipython + * start a new ipython shell +* shell python + * start a new python shell +* shell run + * ??? +* shell script: Runs the script from the specifed path with flags set properly. + * arguments: path + +VPN Management +++++++++++++++ + +* vpn list: Print a listing of the VPNs for all projects. + * arguments: none +* vpn run: Start the VPN for a given project. + * arguments: project +* vpn spawn: Run all VPNs. + * arguments: none + + +Floating IP Management +++++++++++++++++++++++ + +* floating create: Creates floating ips for host by range + * arguments: host ip_range +* floating delete: Deletes floating ips by range + * arguments: range +* floating list: Prints a listing of all floating ips + * arguments: none + +Network Management +++++++++++++++++++ + +* network create: Creates fixed ips for host by range + * arguments: [fixed_range=FLAG], [num_networks=FLAG], + [network_size=FLAG], [vlan_start=FLAG], + [vpn_start=FLAG] Concept: Flags -- cgit From 196ab5b15cb87ea5bfe70050bfa0f14c724e2676 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 10:39:51 -0400 Subject: Fix wiki link. --- doc/source/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index d4cdd1408..54cfea05a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -30,7 +30,9 @@ Nova is written with the following design guidelines in mind: This documentation is generated by the Sphinx toolkit and lives in the source tree. Additional documentation on Nova and other components of OpenStack can -be found on the 'OpenStack wiki '. +be found on the `OpenStack wiki`_. + +.. _`OpenStack wiki`: http://wiki.openstack.org Contents -- cgit From 4012860b57593632d1f0061099e0d211dba58a59 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Thu, 28 Oct 2010 11:43:08 -0400 Subject: Remove unused updated_data variable --- nova/api/ec2/cloud.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 7b6144ba5..f2a6dc3b0 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -896,9 +896,6 @@ class CloudController(object): id_str) continue now = datetime.datetime.utcnow() - updated_data = {'state_description': 'terminating', - 'state': 0, - 'terminated_at': now} self.compute_manager.update_instance(context, instance_ref['id'], state_description='terminating', -- cgit From e85ba051c27ab7d50914c7bf91db74d7cf7faa97 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:00:25 -0400 Subject: clean up the compute documentation a bit. --- doc/source/compute.rst | 78 ++++++++++++++++++++++++++++++++++++++++++------- nova/compute/manager.py | 24 +++++++++++---- nova/virt/connection.py | 11 ++++++- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/doc/source/compute.rst b/doc/source/compute.rst index 93070fa88..737b2de98 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -16,9 +16,17 @@ under the License. +Compute +======= + +This page contains the Compute Package documentation. + + from etherpad todo ------------------ +:: + * FIX RST IMPORT ERRORS * Compute Manager * public methods @@ -30,57 +38,105 @@ from etherpad todo * hyperv implementation -Compute Documentation -===================== +Manager +------- -This page contains the Compute Package documentation. +Documentation for the compute manager and related files. For reading about +a specific virtualization backend, read Drivers_. The :mod:`manager` Module -------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.compute.manager :members: :undoc-members: :show-inheritance: + The :mod:`disk` Module ----------------------- +~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.compute.disk :members: :undoc-members: :show-inheritance: + The :mod:`instance_types` Module --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.compute.instance_types :members: :undoc-members: :show-inheritance: + The :mod:`power_state` Module ------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.compute.power_state :members: :undoc-members: :show-inheritance: + +Drivers +------- + + +Libvirt Implementations +~~~~~~~~~~~~~~~~~~~~~~~ + + +Libvirt: KVM +^^^^^^^^^^^^ + +KVM Driver + + +Libvirt: QEMU +^^^^^^^^^^^^^ + +QEMU Driver + + +Libvirt: UML +^^^^^^^^^^^^ + +User Mode Linux Driver + + +XEN +~~~ + +Xen Driver + + +Hyper-V +~~~~~~~ + +Hyper-V Driver + + +Monitoring +---------- + The :mod:`monitor` Module -------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.compute.monitor :members: :undoc-members: :show-inheritance: -RELATED TESTS -------------- + +Tests +----- + The :mod:`compute_unittest` Module ----------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.tests.compute_unittest :members: diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 574feec7c..c5102c35a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -17,7 +17,20 @@ # under the License. """ -Handles all code relating to instances (guest vms) +Handles all processes relating to instances (guest vms). + +The :py:class:`ComputeManager` class is a :py:class:`nova.manager.Manager` that +handles RPC calls relating to creating instances. It is responsible for +building a disk image, launching it via the underlying virtualization driver, +responding to calls to check it state, attaching persistent as well as termination. + +Related Flags +------------- +:instances_path: Where instances are kept on disk +:compute_driver: Name of class that is used to handle virtualization, loaded + by `nova.utils.import_object` +:volume_manager: Name of class that handles persistent storage, loaded by + `nova.utils.import_object` """ import datetime @@ -40,12 +53,12 @@ flags.DEFINE_string('compute_driver', 'nova.virt.connection.get_connection', class ComputeManager(manager.Manager): - """ - Manages the running instances. - """ + """Manages the running instances from creation to destruction.""" + def __init__(self, compute_driver=None, *args, **kwargs): """Load configuration options and connect to the hypervisor.""" # TODO(vish): sync driver creation logic with the rest of the system + # and redocument the module docstring if not compute_driver: compute_driver = FLAGS.compute_driver self.driver = utils.import_object(compute_driver) @@ -54,7 +67,7 @@ class ComputeManager(manager.Manager): super(ComputeManager, self).__init__(*args, **kwargs) def _update_state(self, context, instance_id): - """Update the state of an instance from the driver info""" + """Update the state of an instance from the driver info.""" # FIXME(ja): include other fields from state? instance_ref = self.db.instance_get(context, instance_id) try: @@ -67,6 +80,7 @@ class ComputeManager(manager.Manager): @defer.inlineCallbacks @exception.wrap_exception def refresh_security_group(self, context, security_group_id, **_kwargs): + """This call passes stright through to the virtualization driver.""" yield self.driver.refresh_security_group(security_group_id) @defer.inlineCallbacks diff --git a/nova/virt/connection.py b/nova/virt/connection.py index 34e37adf7..ceb7f1e4b 100644 --- a/nova/virt/connection.py +++ b/nova/virt/connection.py @@ -17,7 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Abstraction of the underlying virtualization API""" +"""Abstraction of the underlying virtualization API.""" import logging import sys @@ -39,6 +39,15 @@ def get_connection(read_only=False): Any object returned here must conform to the interface documented by FakeConnection. + + Related flags + ------------- + :connection_type: A string literal that falls through a if/elif structure + to determine what virtualization mechanism to use. + Values may be: + * fake + * libvirt + * xenapi """ # TODO(termie): maybe lazy load after initial check for permissions # TODO(termie): check whether we can be disconnected -- cgit From 44885b02a6c4db9ca2cbfc4067cd37fa3c504aee Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:09:59 -0400 Subject: Add missing rst file. --- doc/source/database.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/source/database.rst diff --git a/doc/source/database.rst b/doc/source/database.rst new file mode 100644 index 000000000..657e4399f --- /dev/null +++ b/doc/source/database.rst @@ -0,0 +1,24 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Database Programming Guide +=============================== + +:: + + * general interface + * sqlalchemy implementation -- cgit From 8fa3db896ca73bb8f72228afc387c9db500b9ae9 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:10:08 -0400 Subject: Clean up indentation error by preformatting. --- doc/source/concepts.and.introduction.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 5c006454d..ee03bbdd5 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -91,13 +91,15 @@ Concept: API Concept: Networking ------------------- -* VLAN - * Cloudpipe - * Certificates (See also: CA) -* Flat Networking -* Flat with DHCP -* How to generate addresses -* Floating Addresses +:: + + * VLAN + * Cloudpipe + * Certificates (See also: CA) + * Flat Networking + * Flat with DHCP + * How to generate addresses + * Floating Addresses Concept: Services -- cgit From 6cbc507cd821d91a9ce21e78b50748fa59ddf356 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:14:50 -0400 Subject: Remove outdated endpoint documentation. --- doc/source/development.guide.rst | 6 ++- doc/source/endpoint.rst | 91 ---------------------------------------- doc/source/modules.rst | 6 ++- 3 files changed, 10 insertions(+), 93 deletions(-) delete mode 100644 doc/source/endpoint.rst diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 2c8fa86f9..1581b8baf 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -41,7 +41,6 @@ Older Contents :maxdepth: 1 architecture - endpoint nova fakes binaries @@ -49,6 +48,11 @@ Older Contents packages +Removed +------- + +* endpoint + Indices and tables ------------------ diff --git a/doc/source/endpoint.rst b/doc/source/endpoint.rst deleted file mode 100644 index 399df4161..000000000 --- a/doc/source/endpoint.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Endpoint Documentation -====================== - -This page contains the Endpoint Package documentation. - -The :mod:`admin` Module ------------------------ - -.. automodule:: nova.endpoint.admin - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api` Module ---------------------- - -.. automodule:: nova.endpoint.api - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud` Module ------------------------ - -.. automodule:: nova.endpoint.cloud - :members: - :undoc-members: - :show-inheritance: - -The :mod:`images` Module ------------------------- - -.. automodule:: nova.endpoint.images - :members: - :undoc-members: - :show-inheritance: - - -RELATED TESTS --------------- - -The :mod:`api_unittest` Module ------------------------------- - -.. automodule:: nova.tests.api_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api_integration` Module ---------------------------------- - -.. automodule:: nova.tests.api_integration - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud_unittest` Module --------------------------------- - -.. automodule:: nova.tests.cloud_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`network_unittest` Module ----------------------------------- - -.. automodule:: nova.tests.network_unittest - :members: - :undoc-members: - :show-inheritance: - - diff --git a/doc/source/modules.rst b/doc/source/modules.rst index 82c61f008..9b31664b6 100644 --- a/doc/source/modules.rst +++ b/doc/source/modules.rst @@ -28,7 +28,11 @@ Modules: auth compute - endpoint fakes nova volume + +Removed: +-------- + +* endpoint -- cgit From 208da85e85131a9b60a1fadea3e4242fa70dcde2 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:25:39 -0400 Subject: Whitespace and docstring cleanups --- nova/auth/fakeldap.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 1a49b73fe..176a00f06 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -15,12 +15,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + """ -Fake LDAP server for test harnesses. +Fake LDAP server for test harness, backs to ReDIS. This class does very little error checking, and knows nothing about ldap -class definitions. It implements the minimum emulation of the python ldap +class definitions. It implements the minimum emulation of the python ldap library to work with nova. + """ import json @@ -77,8 +79,8 @@ def initialize(_uri): def _match_query(query, attrs): """Match an ldap query to an attribute dictionary. - &, |, and ! are supported in the query. No syntax checking is performed, - so malformed querys will not work correctly. + The characters &, |, and ! are supported in the query. No syntax checking + is performed, so malformed querys will not work correctly. """ # cut off the parentheses inner = query[1:-1] -- cgit From 89c881d651efcbba56406048112c3cf2cb676a8e Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:26:17 -0400 Subject: Fix underlining -> heading in rst file. --- doc/source/fakes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/fakes.rst b/doc/source/fakes.rst index a993fb4c8..eccd4ab43 100644 --- a/doc/source/fakes.rst +++ b/doc/source/fakes.rst @@ -19,7 +19,7 @@ Nova Fakes ========== The :mod:`virt.fake` Module --------------------------- +--------------------------- .. automodule:: nova.virt.fake :members: -- cgit From 4bd42d5ee9eadb9affb40ee6ed0f98b13609c895 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:26:29 -0400 Subject: Another heading was too distracting, use instead. --- nova/compute/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index c5102c35a..174fb0aca 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -24,8 +24,8 @@ handles RPC calls relating to creating instances. It is responsible for building a disk image, launching it via the underlying virtualization driver, responding to calls to check it state, attaching persistent as well as termination. -Related Flags -------------- +**Related Flags** + :instances_path: Where instances are kept on disk :compute_driver: Name of class that is used to handle virtualization, loaded by `nova.utils.import_object` -- cgit From 7d367e04c3e0dcdf7ce2121c464884c0825a552c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 13:23:43 -0400 Subject: Clean up volumes / storage info. --- doc/source/concepts.and.introduction.rst | 6 ++-- doc/source/database.rst | 32 ++++++++++++++++++---- doc/source/development.guide.rst | 2 +- doc/source/storage.rst | 47 -------------------------------- doc/source/volume.rst | 17 ++++++++++-- 5 files changed, 46 insertions(+), 58 deletions(-) delete mode 100644 doc/source/storage.rst diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index ee03bbdd5..c3057c958 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -61,10 +61,12 @@ Concept: Virtualization * qemu -Concept: Volumes +Concept: Storage ---------------- -Volumes +* Ephemeral +* Volumes +* Swift Concept: Quotas diff --git a/doc/source/database.rst b/doc/source/database.rst index 657e4399f..f0672f608 100644 --- a/doc/source/database.rst +++ b/doc/source/database.rst @@ -15,10 +15,32 @@ License for the specific language governing permissions and limitations under the License. -Nova Database Programming Guide -=============================== +Database Programming Guide +========================== -:: +The :mod:`api` Module +--------------------- + +.. automodule:: nova.db.api + :members: + :undoc-members: + :show-inheritance: + + +Drivers +------- + +The :mod:sqlalchemy Driver +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.db.sqlalchemy.api + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: nova.db.sqlalchemy.models + :members: + :undoc-members: + :show-inheritance: - * general interface - * sqlalchemy implementation diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 1581b8baf..d9a7c46f9 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -28,7 +28,7 @@ Contents :maxdepth: 1 database - storage + volume compute network auth diff --git a/doc/source/storage.rst b/doc/source/storage.rst deleted file mode 100644 index 72880a4a1..000000000 --- a/doc/source/storage.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Storage in the Nova Cloud -========================= - - -from etherpad todo -------------------- -:: - * Volume Manager - * public methods - * responsibilities - * Volume Driver - * public methods - * AoE implementation - * ISCSI implementation - - -legacy docs ------------ - -There are three primary classes of storage in a nova cloud environment: - -* Ephemeral Storage (local disk within an instance) -* Volume Storage (network-attached FS) -* Object Storage (redundant KVS with locality and MR) - -.. toctree:: - :maxdepth: 2 - - volume - objectstore diff --git a/doc/source/volume.rst b/doc/source/volume.rst index 619968458..c55961a46 100644 --- a/doc/source/volume.rst +++ b/doc/source/volume.rst @@ -15,9 +15,20 @@ License for the specific language governing permissions and limitations under the License. -Volume Documentation -==================== - +Volume Programming Guide +========================= + +The :mod:`nova.volume.manager` Module +------------------------------------- + +.. automodule:: nova.volume.manager + :members: + :undoc-members: + :show-inheritance: + +OLDHAT +------ + Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. -- cgit From b6539d86c7217290d46682e214a9e82cfc810447 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 13:58:20 -0400 Subject: Remove objectstore, not referenced anywhere. --- doc/source/objectstore.rst | 66 ---------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 doc/source/objectstore.rst diff --git a/doc/source/objectstore.rst b/doc/source/objectstore.rst deleted file mode 100644 index 6b8d293f4..000000000 --- a/doc/source/objectstore.rst +++ /dev/null @@ -1,66 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Objectstore Documentation -========================= - -This page contains the Objectstore Package documentation. - - -The :mod:`bucket` Module ------------------------- - -.. automodule:: nova.objectstore.bucket - :members: - :undoc-members: - :show-inheritance: - -The :mod:`handler` Module -------------------------- - -.. automodule:: nova.objectstore.handler - :members: - :undoc-members: - :show-inheritance: - -The :mod:`image` Module ------------------------ - -.. automodule:: nova.objectstore.image - :members: - :undoc-members: - :show-inheritance: - -The :mod:`stored` Module ------------------------- - -.. automodule:: nova.objectstore.stored - :members: - :undoc-members: - :show-inheritance: - -RELATED TESTS -------------- - -The :mod:`objectstore_unittest` Module --------------------------------------- - -.. automodule:: nova.tests.objectstore_unittest - :members: - :undoc-members: - :show-inheritance: - -- cgit From 9321ae23a1741267bc911e70b6755ac5c3251fbc Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 14:04:11 -0400 Subject: volume cleanups --- doc/source/volume.rst | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/doc/source/volume.rst b/doc/source/volume.rst index c55961a46..f83556795 100644 --- a/doc/source/volume.rst +++ b/doc/source/volume.rst @@ -26,31 +26,35 @@ The :mod:`nova.volume.manager` Module :undoc-members: :show-inheritance: -OLDHAT ------- - -Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. - -Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. - -AoE exports are numbered according to a "shelf and blade" syntax. In order to avoid collisions, we currently perform an AoE-discover of existing exports, and then grab the next unused number. (This obviously has race condition problems, and should be replaced by allocating a shelf-id to each storage node.) - -The underlying volumes are LVM logical volumes, created on demand within a single large volume group. - - -The :mod:`storage` Module -------------------------- +The :mod:`nova.volume.driver` Module +------------------------------------- -.. automodule:: nova.volume.storage +.. automodule:: nova.volume.driver :members: :undoc-members: :show-inheritance: -The :mod:`storage_unittest` Module ----------------------------------- +Tests +----- -.. automodule:: nova.tests.storage_unittest +.. automodule:: nova.tests.volume_unittest :members: :undoc-members: :show-inheritance: +Old Docs +-------- + +:: + + TODO(todd): change for iSCSI & AoE and move to top of document + +Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. + +Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. + +AoE exports are numbered according to a "shelf and blade" syntax. In order to avoid collisions, we currently perform an AoE-discover of existing exports, and then grab the next unused number. (This obviously has race condition problems, and should be replaced by allocating a shelf-id to each storage node.) + +The underlying volumes are LVM logical volumes, created on demand within a single large volume group. + + -- cgit From 39fd6f43873e6d314504f53dc4cb1c8a3a3cfbc3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 28 Oct 2010 11:33:29 -0700 Subject: wip architecture, a few auth formatting fixes, binaries, and overview --- doc/source/architecture.rst | 20 ++++++------ doc/source/auth.rst | 45 ++++++++++++++------------- doc/source/binaries.rst | 53 ++++++++++++++++++++++++-------- doc/source/concepts.and.introduction.rst | 3 ++ doc/source/services.rst | 25 +++++++-------- 5 files changed, 91 insertions(+), 55 deletions(-) diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst index 11813d2c8..eba6cbfb8 100644 --- a/doc/source/architecture.rst +++ b/doc/source/architecture.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -30,14 +30,16 @@ Below you will find a helpful explanation. :: - [ User Manager ] ---- ( LDAP ) - | - | / [ Storage ] - ( ATAoE ) - [ API server ] -> [ Cloud ] < AMQP > - | \ [ Nodes ] - ( libvirt/kvm ) - < HTTP > - | - [ S3 ] + [ Auth Manager ] ---- ( LDAP ) + | + | + | + | / [ Storage ] - ( ATAoE/iSCSI ) + [ Web Dashboard ] -> [ nova-api ] < AMQP > - + | \ [ Nodes ] - ( libvirt/kvm ) + < HTTP > + | + [ nova-objectstore ] * API: receives http requests from boto, converts commands to/from API format, and sending requests to cloud controller diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 4d3037253..adcb759a2 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -24,7 +24,7 @@ from etherpad todo ------------------ :: - * FIX RST IMPORT ERRORS + * Manager * Driver * ldap @@ -65,46 +65,47 @@ Roles AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: -* System User -* System Administrator +* Base User +* System Administrator/Developer (currently have the same permissions) * Network Administrator * Project Manager -* Cloud Administrator -* IT-Security +* Cloud Administrator/IT-Security (currently have the same permissions) There is an additional, conceptual end-user that may or may not have API access: * (EXTERNAL) End-user / Third-party User -Basic operations are available to any System User: +Basic operations are available to any : -* Launch Instance -* Terminate Instance (their own) -* Create keypair -* Delete keypair -* Create, Upload, Delete: Buckets and Keys (Object Store) – their own -* Create, Attach, Delete Volume (Block Store) – their own +* Describe Instances +* Describe Images +* Describe Volumes +* Describe Keypairs +* Create Keypair +* Delete Keypair +* Create, Upload, Delete: Buckets and Keys (Object Store) -System Administrators: +System Administrators/Developers/Project Manager: +* Create, Attach, Delete Volume (Block Store) +* Launch, Reboot, Terminate Instance * Register/Unregister Machine Image (project-wide) -* Change Machine Image properties (public / private) * Request / Review CloudAudit Scans +Project Manager: + +* Add and remove other users (currently no api) +* Set roles (currently no api) + Network Administrator: +* Change Machine Image properties (public / private) * Change Firewall Rules, define Security Groups * Allocate, Associate, Deassociate Public IP addresses -Project Manager: - -* Launch and Terminate Instances (project-wide) -* CRUD of Object and Block store (project-wide) - -Cloud Administrator: +Cloud Administrator/IT-Security: -* Register / Unregister Kernel and Ramdisk Images -* Register / Unregister Machine Image (any) +* All permissions Enhancements ------------ diff --git a/doc/source/binaries.rst b/doc/source/binaries.rst index 90a9581f7..6771f345e 100644 --- a/doc/source/binaries.rst +++ b/doc/source/binaries.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -18,14 +18,43 @@ Nova Binaries =============== -* nova-api -* nova-compute -* nova-manage -* nova-objectstore -* nova-volume - -The configuration of these binaries relies on "flagfiles" using the google -gflags package. If present, the nova.conf file will be used as the flagfile -- otherwise, it must be specified on the command line:: - - $ python node_worker.py --flagfile flagfile +The configuration of these binaries relies on "flagfiles" using the google +gflags package:: + + $ nova-xxxxx --flagfile flagfile + +The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. + +nova-manage +----------- + +Nova manage is a command line utility to administer the system. It will autmatically try to load a flagfile from /etc/nova/nova-manage.conf to save you having to type it. Info on the commands can be found :ref:`here `. + +nova-api +-------- + +Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. + +nova-objectstore +---------------- + +Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with glance and a simple image manager. + +nova-compute +------------ + +Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. + +nova-volume +----------- + +Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. + +nova-network +------------ + +Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: + + $ nova-network --network_manager=nova.network.manager.FlatManager + +IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 76db9e9f2..7f4aa153c 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -135,6 +135,9 @@ Concept: Services * nova-network * nova-instancemonitor + +.. _manage_usage: + Concept: nova-manage -------------------- diff --git a/doc/source/services.rst b/doc/source/services.rst index 7ff8fa6c3..607f5974b 100644 --- a/doc/source/services.rst +++ b/doc/source/services.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,17 +15,18 @@ License for the specific language governing permissions and limitations under the License. -Services -======== +.. _service_manager_driver: -Admin guide should go beyoned concepts & introduction by talking about how -each service interacts with each other, what requirements for deployment are, -monotoring, logging, etc +Services Managers and Drivers +============================= +Describe division of responsibilities for each component. -* nova-api -* nova-scheduler -* nova-compute -* nova-volume -* nova-network -* nova-instancemonitor +Service +------- + +Manager +------- + +Driver +------ -- cgit From 73773c2897894a4b7c40268f04fa100425411824 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 28 Oct 2010 11:38:43 -0700 Subject: fix title levels --- doc/source/concepts.and.introduction.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 7f4aa153c..f36949835 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -142,7 +142,7 @@ Concept: nova-manage -------------------- Introduction -++++++++++++ +^^^^^^^^^^^^ The nova-manage command is used to perform many essential functions for administration and ongoing maintenance of nova, such as user creation, @@ -157,7 +157,7 @@ For example, to obtain a list of all projects: ``nova-manage project list`` User Maintenance -++++++++++++++++ +^^^^^^^^^^^^^^^^ * user admin: creates a new admin and prints exports * arguments: name [access] [secret] @@ -174,7 +174,7 @@ User Maintenance * leave any field blank to ignore it, admin should be 'T', 'F', or blank Project Maintenance -+++++++++++++++++++ +^^^^^^^^^^^^^^^^^^^ * project add: Adds user to project * arguments: project user @@ -196,7 +196,7 @@ Project Maintenance * arguments: project_id user_id [filename='nova.zip] User Role Management -++++++++++++++++++++ +^^^^^^^^^^^^^^^^^^^^ * role add: adds role to user * if project is specified, adds project specific role @@ -211,7 +211,7 @@ User Role Management Nova Shell -++++++++++ +^^^^^^^^^^ * shell bpython * start a new bpython shell @@ -225,7 +225,7 @@ Nova Shell * arguments: path VPN Management -++++++++++++++ +^^^^^^^^^^^^^^ * vpn list: Print a listing of the VPNs for all projects. * arguments: none @@ -236,7 +236,7 @@ VPN Management Floating IP Management -++++++++++++++++++++++ +^^^^^^^^^^^^^^^^^^^^^^ * floating create: Creates floating ips for host by range * arguments: host ip_range @@ -246,7 +246,7 @@ Floating IP Management * arguments: none Network Management -++++++++++++++++++ +^^^^^^^^^^^^^^^^^^ * network create: Creates fixed ips for host by range * arguments: [fixed_range=FLAG], [num_networks=FLAG], -- cgit From 828b299a5660537062b11e6e58cfdf4c840053e1 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 14:39:25 -0400 Subject: cleanup todos --- doc/source/compute.rst | 13 +------------ doc/source/database.rst | 10 ++++++++-- doc/source/volume.rst | 10 ++++++---- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/doc/source/compute.rst b/doc/source/compute.rst index 737b2de98..f3173122a 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -22,20 +22,9 @@ Compute This page contains the Compute Package documentation. -from etherpad todo ------------------- - :: - * FIX RST IMPORT ERRORS - * Compute Manager - * public methods - * responsibilities - * Compute Driver - * public methods - * libvirt implementation (kvm/qemu vs uml) - * xen implementation - * hyperv implementation + TODO(todd): Document drivers Manager diff --git a/doc/source/database.rst b/doc/source/database.rst index f0672f608..41b362d19 100644 --- a/doc/source/database.rst +++ b/doc/source/database.rst @@ -18,6 +18,12 @@ Database Programming Guide ========================== +:: + + TODO(todd): should sqlalchemy.api be here? + pep-256 on db/api.py and models.py (periods) + document register_models (where should it be called from?) + The :mod:`api` Module --------------------- @@ -30,8 +36,8 @@ The :mod:`api` Module Drivers ------- -The :mod:sqlalchemy Driver -~~~~~~~~~~~~~~~~~~~~~~~~~~ +The :mod:`sqlalchemy` Driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.db.sqlalchemy.api :members: diff --git a/doc/source/volume.rst b/doc/source/volume.rst index f83556795..f4162560c 100644 --- a/doc/source/volume.rst +++ b/doc/source/volume.rst @@ -18,6 +18,12 @@ Volume Programming Guide ========================= +:: + + TODO(todd): document for iSCSI & AoE (see 'Old Docs') + pep-257 + + The :mod:`nova.volume.manager` Module ------------------------------------- @@ -45,10 +51,6 @@ Tests Old Docs -------- -:: - - TODO(todd): change for iSCSI & AoE and move to top of document - Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. -- cgit From 1915bb227d1eb847a681173772f4b9a769d64331 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 15:10:15 -0400 Subject: Todo cleanups in docs. --- doc/source/auth.rst | 24 +++++++++++++----------- doc/source/compute.rst | 4 ++-- doc/source/network.rst | 39 ++++++++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/doc/source/auth.rst b/doc/source/auth.rst index d739e4b8e..f4a7e0147 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -15,20 +15,22 @@ License for the specific language governing permissions and limitations under the License. -Auth Documentation -================== +Authentication and Authorization Development Guide +================================================== +:: -from etherpad todo ------------------- + TOOD(todd); + * Authentication + * Manager + * Driver + * ldap + * fakeldap + * redisectomy + * Authorizztion + * RBAC + * Quotas -:: - * FIX RST IMPORT ERRORS - * Manager - * Driver - * ldap - * fakeldap - * redisectomy legacy docs ----------- diff --git a/doc/source/compute.rst b/doc/source/compute.rst index f3173122a..9b4d10227 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -16,8 +16,8 @@ under the License. -Compute -======= +Virtualization Programming Guide +================================ This page contains the Compute Package documentation. diff --git a/doc/source/network.rst b/doc/source/network.rst index ef374a726..f9434b453 100644 --- a/doc/source/network.rst +++ b/doc/source/network.rst @@ -15,23 +15,32 @@ License for the specific language governing permissions and limitations under the License. -nova Networking -================ +Networking Programming Guide +============================ - -from etherpad todo ------------------- :: - * Network Manager - * overview (use vish email) - * public methods - * responsibilities - * Vlan + DHCP implementation - * Flat + DHCP implementation - * Flat impleplementation - * Network Driver - * public methods - * hardware specific commands + + TODO(todd): + * document how managers work + * document backend (linux-net) + * document hardware specific commands + * pep 257 + +The :mod:`Manager` module +------------------------- + +.. automodule:: nova.network.manager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`LinuxNet` driver +-------------------------- + +.. automodule:: nova.network.linux_net + :members: + :undoc-members: + :show-inheritance: legacy docs -- cgit From a592636054511382105dc81d4a6b2a44df0dad9a Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 17:08:13 -0400 Subject: :func: links to python functions in the documentation. --- nova/compute/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 174fb0aca..3346d1299 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -28,9 +28,9 @@ responding to calls to check it state, attaching persistent as well as terminati :instances_path: Where instances are kept on disk :compute_driver: Name of class that is used to handle virtualization, loaded - by `nova.utils.import_object` + by :func:`nova.utils.import_object` :volume_manager: Name of class that handles persistent storage, loaded by - `nova.utils.import_object` + :func:`nova.utils.import_object` """ import datetime -- cgit From 7dedbca140c7e923c7b2fcaa8679a19249fb8350 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 28 Oct 2010 14:27:24 -0700 Subject: link binaries section to concepts --- doc/source/binaries.rst | 2 ++ doc/source/concepts.and.introduction.rst | 10 ++-------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/doc/source/binaries.rst b/doc/source/binaries.rst index 6771f345e..3ea07d9c2 100644 --- a/doc/source/binaries.rst +++ b/doc/source/binaries.rst @@ -15,6 +15,8 @@ License for the specific language governing permissions and limitations under the License. +.. _binaries: + Nova Binaries =============== diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index f36949835..7f59c0c92 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -125,16 +125,10 @@ The following diagram illustrates how the communication that occurs between the .. -Concept: Services +Concept: Binaries ----------------- -* nova-api -* nova-scheduler -* nova-compute -* nova-volume -* nova-network -* nova-instancemonitor - +Nova is implemented by a number of related binaries. These binaries can run on the same machine or many machines. A detailed description of each binary is given in the :ref:`binaries section ` of the developer guide. .. _manage_usage: -- cgit From 1106d7622e9327d4db417ae8bc7a18bb5d64fa1c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 18:59:03 -0400 Subject: A shell script for showing modules that aren't documented in .rst files. --- doc/find_undocumented_classes.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 doc/find_undocumented_classes.sh diff --git a/doc/find_undocumented_classes.sh b/doc/find_undocumented_classes.sh new file mode 100755 index 000000000..845e0b5e0 --- /dev/null +++ b/doc/find_undocumented_classes.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +NOVA_DIR='../nova/' # include trailing slash +DOCS_DIR='source' + +modules='' +for x in `find ${NOVA_DIR} -name '*.py'`; do + if [ `basename ${x} .py` == "__init__" ] ; then + continue + fi + relative=nova.`echo ${x} | sed -e 's$^'${NOVA_DIR}'$$' -e 's/.py$//' -e 's$/$.$g'` + modules="${modules} ${relative}" +done + +for mod in ${modules} ; do + op=`grep ${mod} ${DOCS_DIR}/*.rst` + if [ -z "${op}" ] ; then + echo ${mod} + fi +done -- cgit From 654a61c26d94a719e0ea665699b8075084ad79e3 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 18:59:25 -0400 Subject: Clean up todos and the like for docs. --- doc/source/auth.rst | 131 +++++++++++++++++++-------------------- doc/source/compute.rst | 5 +- doc/source/database.rst | 2 + doc/source/development.guide.rst | 9 ++- doc/source/network.rst | 1 + doc/source/volume.rst | 1 + 6 files changed, 81 insertions(+), 68 deletions(-) diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 0230ec5c4..203efe1a5 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -22,16 +22,71 @@ Authentication and Authorization Development Guide :: - TOOD(todd); - * Authentication - * Manager - * Driver - * ldap - * fakeldap - * redisectomy - * Authorizztion - * RBAC - * Quotas + TOOD(todd): * Authorizztion + * RBAC + * Quotas + + +Auth Manager +------------ + +The :mod:`manager` Module +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.manager + :members: + :undoc-members: + :show-inheritance: + +Drivers +------- + +The :mod:`ldapdriver` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.ldapdriver + :members: + :undoc-members: + :show-inheritance: + +The :mod:`dbdriver` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.dbdriver + :members: + :undoc-members: + :show-inheritance: + +API Integration +--------------- + +The :mod:`signer` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.signer + :members: + :undoc-members: + :show-inheritance: + + +Related Tests +------------- + +The :mod:`auth_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.auth_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`access_unittest` Module +--------------------------------- + +.. automodule:: nova.tests.access_unittest + :members: + :undoc-members: + :show-inheritance: legacy docs @@ -194,59 +249,3 @@ Further Challenges -The :mod:`signer` Module ------------------------- - -.. automodule:: nova.auth.signer - :members: - :undoc-members: - :show-inheritance: - -The :mod:`manager` Module -------------------------- - -.. automodule:: nova.auth.manager - :members: - :undoc-members: - :show-inheritance: - -The :mod:`ldapdriver` Module ----------------------------- - -.. automodule:: nova.auth.ldapdriver - :members: - :undoc-members: - :show-inheritance: - -The :mod:`dbdriver` Module ----------------------------- - -.. automodule:: nova.auth.dbdriver - :members: - :undoc-members: - :show-inheritance: - -The :mod:`auth_unittest` Module --------------------------------- - -.. automodule:: nova.tests.auth_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`access_unittest` Module ---------------------------------- - -.. automodule:: nova.tests.access_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`ec2` Module ---------------------- - -.. automodule:: nova.api.ec2 - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/source/compute.rst b/doc/source/compute.rst index 9b4d10227..d29b96781 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -24,7 +24,10 @@ This page contains the Compute Package documentation. :: - TODO(todd): Document drivers + TODO(todd): * Document drivers + * get docstrings to come in for ComputeManager + * fix formatting of ascii art in disk module + * document instance_types and power_states Manager diff --git a/doc/source/database.rst b/doc/source/database.rst index 41b362d19..bab6b472d 100644 --- a/doc/source/database.rst +++ b/doc/source/database.rst @@ -23,6 +23,8 @@ Database Programming Guide TODO(todd): should sqlalchemy.api be here? pep-256 on db/api.py and models.py (periods) document register_models (where should it be called from?) + document any relevant test cases + document flags The :mod:`api` Module --------------------- diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index d9a7c46f9..04fee329a 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -18,7 +18,14 @@ Nova Development Guide ====================== -Nova is written in python +Nova is written in python. + +:: + + TODO(todd): * API + * Exceptions + * Nova libraries (utils, etc) + * Building packages Contents diff --git a/doc/source/network.rst b/doc/source/network.rst index f53360bd7..b044c6413 100644 --- a/doc/source/network.rst +++ b/doc/source/network.rst @@ -25,6 +25,7 @@ Networking Programming Guide * document backend (linux-net) * document hardware specific commands * pep 257 + * document flags The :mod:`Manager` module ------------------------- diff --git a/doc/source/volume.rst b/doc/source/volume.rst index f4162560c..27977761a 100644 --- a/doc/source/volume.rst +++ b/doc/source/volume.rst @@ -22,6 +22,7 @@ Volume Programming Guide TODO(todd): document for iSCSI & AoE (see 'Old Docs') pep-257 + document flags The :mod:`nova.volume.manager` Module -- cgit From 7cc4bcd344221d517054641171f759b88112a459 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 20:13:40 -0400 Subject: Pep-257 cleanups. --- nova/db/api.py | 128 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 70 insertions(+), 58 deletions(-) diff --git a/nova/db/api.py b/nova/db/api.py index 0731e2e05..7cce591ad 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. """ -Defines interface for DB access +Defines interface for DB access. """ from nova import exception @@ -34,17 +34,17 @@ IMPL = utils.LazyPluggable(FLAGS['db_backend'], class NoMoreAddresses(exception.Error): - """No more available addresses""" + """No more available addresses.""" pass class NoMoreBlades(exception.Error): - """No more available blades""" + """No more available blades.""" pass class NoMoreNetworks(exception.Error): - """No more available networks""" + """No more available networks.""" pass @@ -62,30 +62,33 @@ def service_get(context, service_id): def service_get_all_by_topic(context, topic): - """Get all compute services for a given topic """ + """Get all compute services for a given topic.""" return IMPL.service_get_all_by_topic(context, topic) def service_get_all_compute_sorted(context): - """Get all compute services sorted by instance count + """Get all compute services sorted by instance count. + + Returns a list of (Service, instance_count) tuples. - Returns a list of (Service, instance_count) tuples """ return IMPL.service_get_all_compute_sorted(context) def service_get_all_network_sorted(context): - """Get all network services sorted by network count + """Get all network services sorted by network count. + + Returns a list of (Service, network_count) tuples. - Returns a list of (Service, network_count) tuples """ return IMPL.service_get_all_network_sorted(context) def service_get_all_volume_sorted(context): - """Get all volume services sorted by volume count + """Get all volume services sorted by volume count. + + Returns a list of (Service, volume_count) tuples. - Returns a list of (Service, volume_count) tuples """ return IMPL.service_get_all_volume_sorted(context) @@ -116,6 +119,7 @@ def floating_ip_allocate_address(context, host, project_id): """Allocate free floating ip and return the address. Raises if one is not available. + """ return IMPL.floating_ip_allocate_address(context, host, project_id) @@ -144,6 +148,7 @@ def floating_ip_disassociate(context, address): """Disassociate an floating ip from a fixed ip by address. Returns the address of the existing fixed ip. + """ return IMPL.floating_ip_disassociate(context, address) @@ -182,6 +187,7 @@ def fixed_ip_associate(context, address, instance_id): """Associate fixed ip to instance. Raises if fixed ip is not available. + """ return IMPL.fixed_ip_associate(context, address, instance_id) @@ -190,6 +196,7 @@ def fixed_ip_associate_pool(context, network_id, instance_id): """Find free ip in network and associate it to instance. Raises if one is not available. + """ return IMPL.fixed_ip_associate_pool(context, network_id, instance_id) @@ -205,7 +212,7 @@ def fixed_ip_disassociate(context, address): def fixed_ip_disassociate_all_by_timeout(context, host, time): - """Disassociate old fixed ips from host""" + """Disassociate old fixed ips from host.""" return IMPL.fixed_ip_disassociate_all_by_timeout(context, host, time) @@ -283,7 +290,7 @@ def instance_get_floating_address(context, instance_id): def instance_get_by_internal_id(context, internal_id): - """Get an instance by ec2 id.""" + """Get an instance by internal id.""" return IMPL.instance_get_by_internal_id(context, internal_id) @@ -307,7 +314,7 @@ def instance_update(context, instance_id, values): def instance_add_security_group(context, instance_id, security_group_id): - """Associate the given security group with the given instance""" + """Associate the given security group with the given instance.""" return IMPL.instance_add_security_group(context, instance_id, security_group_id) @@ -369,10 +376,12 @@ def network_count_reserved_ips(context, network_id): def network_create_safe(context, values): - """Create a network from the values dict + """Create a network from the values dict. The network is only returned if the create succeeds. If the create violates - constraints because the network already exists, no exception is raised.""" + constraints because the network already exists, no exception is raised. + + """ return IMPL.network_create_safe(context, values) @@ -413,22 +422,22 @@ def network_get_by_instance(context, instance_id): def network_get_index(context, network_id): - """Get non-conflicting index for network""" + """Get non-conflicting index for network.""" return IMPL.network_get_index(context, network_id) def network_get_vpn_ip(context, network_id): - """Get non-conflicting index for network""" + """Get non-conflicting index for network.""" return IMPL.network_get_vpn_ip(context, network_id) def network_set_cidr(context, network_id, cidr): - """Set the Classless Inner Domain Routing for the network""" + """Set the Classless Inner Domain Routing for the network.""" return IMPL.network_set_cidr(context, network_id, cidr) def network_set_host(context, network_id, host_id): - """Safely set the host for network""" + """Safely set the host for network.""" return IMPL.network_set_host(context, network_id, host_id) @@ -474,7 +483,9 @@ def export_device_create_safe(context, values): The device is not returned. If the create violates the unique constraints because the shelf_id and blade_id already exist, - no exception is raised.""" + no exception is raised. + + """ return IMPL.export_device_create_safe(context, values) @@ -482,17 +493,17 @@ def export_device_create_safe(context, values): def auth_destroy_token(context, token): - """Destroy an auth token""" + """Destroy an auth token.""" return IMPL.auth_destroy_token(context, token) def auth_get_token(context, token_hash): - """Retrieves a token given the hash representing it""" + """Retrieves a token given the hash representing it.""" return IMPL.auth_get_token(context, token_hash) def auth_create_token(context, token): - """Creates a new token""" + """Creates a new token.""" return IMPL.auth_create_token(context, token) @@ -595,47 +606,47 @@ def volume_update(context, volume_id, values): def security_group_get_all(context): - """Get all security groups""" + """Get all security groups.""" return IMPL.security_group_get_all(context) def security_group_get(context, security_group_id): - """Get security group by its internal id""" + """Get security group by its internal id.""" return IMPL.security_group_get(context, security_group_id) def security_group_get_by_name(context, project_id, group_name): - """Returns a security group with the specified name from a project""" + """Returns a security group with the specified name from a project.""" return IMPL.security_group_get_by_name(context, project_id, group_name) def security_group_get_by_project(context, project_id): - """Get all security groups belonging to a project""" + """Get all security groups belonging to a project.""" return IMPL.security_group_get_by_project(context, project_id) def security_group_get_by_instance(context, instance_id): - """Get security groups to which the instance is assigned""" + """Get security groups to which the instance is assigned.""" return IMPL.security_group_get_by_instance(context, instance_id) def security_group_exists(context, project_id, group_name): - """Indicates if a group name exists in a project""" + """Indicates if a group name exists in a project.""" return IMPL.security_group_exists(context, project_id, group_name) def security_group_create(context, values): - """Create a new security group""" + """Create a new security group.""" return IMPL.security_group_create(context, values) def security_group_destroy(context, security_group_id): - """Deletes a security group""" + """Deletes a security group.""" return IMPL.security_group_destroy(context, security_group_id) def security_group_destroy_all(context): - """Deletes a security group""" + """Deletes a security group.""" return IMPL.security_group_destroy_all(context) @@ -643,18 +654,18 @@ def security_group_destroy_all(context): def security_group_rule_create(context, values): - """Create a new security group""" + """Create a new security group.""" return IMPL.security_group_rule_create(context, values) def security_group_rule_get_by_security_group(context, security_group_id): - """Get all rules for a a given security group""" + """Get all rules for a a given security group.""" return IMPL.security_group_rule_get_by_security_group(context, security_group_id) def security_group_rule_destroy(context, security_group_rule_id): - """Deletes a security group rule""" + """Deletes a security group rule.""" return IMPL.security_group_rule_destroy(context, security_group_rule_id) @@ -662,107 +673,107 @@ def security_group_rule_destroy(context, security_group_rule_id): def user_get(context, id): - """Get user by id""" + """Get user by id.""" return IMPL.user_get(context, id) def user_get_by_uid(context, uid): - """Get user by uid""" + """Get user by uid.""" return IMPL.user_get_by_uid(context, uid) def user_get_by_access_key(context, access_key): - """Get user by access key""" + """Get user by access key.""" return IMPL.user_get_by_access_key(context, access_key) def user_create(context, values): - """Create a new user""" + """Create a new user.""" return IMPL.user_create(context, values) def user_delete(context, id): - """Delete a user""" + """Delete a user.""" return IMPL.user_delete(context, id) def user_get_all(context): - """Create a new user""" + """Create a new user.""" return IMPL.user_get_all(context) def user_add_role(context, user_id, role): - """Add another global role for user""" + """Add another global role for user.""" return IMPL.user_add_role(context, user_id, role) def user_remove_role(context, user_id, role): - """Remove global role from user""" + """Remove global role from user.""" return IMPL.user_remove_role(context, user_id, role) def user_get_roles(context, user_id): - """Get global roles for user""" + """Get global roles for user.""" return IMPL.user_get_roles(context, user_id) def user_add_project_role(context, user_id, project_id, role): - """Add project role for user""" + """Add project role for user.""" return IMPL.user_add_project_role(context, user_id, project_id, role) def user_remove_project_role(context, user_id, project_id, role): - """Remove project role from user""" + """Remove project role from user.""" return IMPL.user_remove_project_role(context, user_id, project_id, role) def user_get_roles_for_project(context, user_id, project_id): - """Return list of roles a user holds on project""" + """Return list of roles a user holds on project.""" return IMPL.user_get_roles_for_project(context, user_id, project_id) def user_update(context, user_id, values): - """Update user""" + """Update user.""" return IMPL.user_update(context, user_id, values) def project_get(context, id): - """Get project by id""" + """Get project by id.""" return IMPL.project_get(context, id) def project_create(context, values): - """Create a new project""" + """Create a new project.""" return IMPL.project_create(context, values) def project_add_member(context, project_id, user_id): - """Add user to project""" + """Add user to project.""" return IMPL.project_add_member(context, project_id, user_id) def project_get_all(context): - """Get all projects""" + """Get all projects.""" return IMPL.project_get_all(context) def project_get_by_user(context, user_id): - """Get all projects of which the given user is a member""" + """Get all projects of which the given user is a member.""" return IMPL.project_get_by_user(context, user_id) def project_remove_member(context, project_id, user_id): - """Remove the given user from the given project""" + """Remove the given user from the given project.""" return IMPL.project_remove_member(context, project_id, user_id) def project_update(context, project_id, values): - """Update Remove the given user from the given project""" + """Update Remove the given user from the given project.""" return IMPL.project_update(context, project_id, values) def project_delete(context, project_id): - """Delete project""" + """Delete project.""" return IMPL.project_delete(context, project_id) @@ -771,6 +782,7 @@ def project_delete(context, project_id): def host_get_networks(context, host): """Return all networks for which the given host is the designated - network host + network host. + """ return IMPL.host_get_networks(context, host) -- cgit From 2132c0de46fd3f1b938e4b3b01b73fb2efaf6a38 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 20:28:13 -0400 Subject: Pep-257 --- nova/db/sqlalchemy/models.py | 47 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 2a3cfa94c..894ebcddd 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -15,7 +15,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ SQLAlchemy models for nova data """ @@ -35,13 +34,13 @@ from nova import auth from nova import exception from nova import flags -FLAGS = flags.FLAGS +FLAGS = flags.FLAGS BASE = declarative_base() class NovaBase(object): - """Base class for Nova Models""" + """Base class for Nova Models.""" __table_args__ = {'mysql_engine': 'InnoDB'} __table_initialized__ = False created_at = Column(DateTime, default=datetime.datetime.utcnow) @@ -50,7 +49,7 @@ class NovaBase(object): deleted = Column(Boolean, default=False) def save(self, session=None): - """Save this object""" + """Save this object.""" if not session: session = get_session() session.add(self) @@ -63,7 +62,7 @@ class NovaBase(object): raise def delete(self, session=None): - """Delete this object""" + """Delete this object.""" self.deleted = True self.deleted_at = datetime.datetime.utcnow() self.save(session=session) @@ -128,7 +127,8 @@ class NovaBase(object): class Service(BASE, NovaBase): - """Represents a running service on a host""" + """Represents a running service on a host.""" + __tablename__ = 'services' id = Column(Integer, primary_key=True) host = Column(String(255)) # , ForeignKey('hosts.id')) @@ -139,7 +139,7 @@ class Service(BASE, NovaBase): class Instance(BASE, NovaBase): - """Represents a guest vm""" + """Represents a guest vm.""" __tablename__ = 'instances' id = Column(Integer, primary_key=True) internal_id = Column(Integer, unique=True) @@ -215,7 +215,7 @@ class Instance(BASE, NovaBase): class Volume(BASE, NovaBase): - """Represents a block storage device that can be attached to a vm""" + """Represents a block storage device that can be attached to a vm.""" __tablename__ = 'volumes' id = Column(Integer, primary_key=True) ec2_id = Column(String(12), unique=True) @@ -246,7 +246,7 @@ class Volume(BASE, NovaBase): class Quota(BASE, NovaBase): - """Represents quota overrides for a project""" + """Represents quota overrides for a project.""" __tablename__ = 'quotas' id = Column(Integer, primary_key=True) @@ -260,7 +260,7 @@ class Quota(BASE, NovaBase): class ExportDevice(BASE, NovaBase): - """Represates a shelf and blade that a volume can be exported on""" + """Represates a shelf and blade that a volume can be exported on.""" __tablename__ = 'export_devices' __table_args__ = (schema.UniqueConstraint("shelf_id", "blade_id"), {'mysql_engine': 'InnoDB'}) @@ -283,7 +283,7 @@ class SecurityGroupInstanceAssociation(BASE, NovaBase): class SecurityGroup(BASE, NovaBase): - """Represents a security group""" + """Represents a security group.""" __tablename__ = 'security_groups' id = Column(Integer, primary_key=True) @@ -313,7 +313,7 @@ class SecurityGroup(BASE, NovaBase): class SecurityGroupIngressRule(BASE, NovaBase): - """Represents a rule in a security group""" + """Represents a rule in a security group.""" __tablename__ = 'security_group_rules' id = Column(Integer, primary_key=True) @@ -335,7 +335,7 @@ class SecurityGroupIngressRule(BASE, NovaBase): class KeyPair(BASE, NovaBase): - """Represents a public key pair for ssh""" + """Represents a public key pair for ssh.""" __tablename__ = 'key_pairs' id = Column(Integer, primary_key=True) @@ -348,7 +348,7 @@ class KeyPair(BASE, NovaBase): class Network(BASE, NovaBase): - """Represents a network""" + """Represents a network.""" __tablename__ = 'networks' __table_args__ = (schema.UniqueConstraint("vpn_public_address", "vpn_public_port"), @@ -377,9 +377,12 @@ class Network(BASE, NovaBase): class AuthToken(BASE, NovaBase): - """Represents an authorization token for all API transactions. Fields - are a string representing the actual token and a user id for mapping - to the actual user""" + """Represents an authorization token for all API transactions. + + Fields are a string representing the actual token and a user id for + mapping to the actual user + + """ __tablename__ = 'auth_tokens' token_hash = Column(String(255), primary_key=True) user_id = Column(Integer) @@ -390,7 +393,7 @@ class AuthToken(BASE, NovaBase): # TODO(vish): can these both come from the same baseclass? class FixedIp(BASE, NovaBase): - """Represents a fixed ip for an instance""" + """Represents a fixed ip for an instance.""" __tablename__ = 'fixed_ips' id = Column(Integer, primary_key=True) address = Column(String(255)) @@ -409,7 +412,7 @@ class FixedIp(BASE, NovaBase): class User(BASE, NovaBase): - """Represents a user""" + """Represents a user.""" __tablename__ = 'users' id = Column(String(255), primary_key=True) @@ -421,7 +424,7 @@ class User(BASE, NovaBase): class Project(BASE, NovaBase): - """Represents a project""" + """Represents a project.""" __tablename__ = 'projects' id = Column(String(255), primary_key=True) name = Column(String(255)) @@ -469,7 +472,7 @@ class UserProjectAssociation(BASE, NovaBase): class FloatingIp(BASE, NovaBase): - """Represents a floating ip that dynamically forwards to a fixed ip""" + """Represents a floating ip that dynamically forwards to a fixed ip.""" __tablename__ = 'floating_ips' id = Column(Integer, primary_key=True) address = Column(String(255)) @@ -485,7 +488,7 @@ class FloatingIp(BASE, NovaBase): def register_models(): - """Register Models and create metadata""" + """Register Models and create metadata.""" from sqlalchemy import create_engine models = (Service, Instance, Volume, ExportDevice, FixedIp, FloatingIp, Network, SecurityGroup, -- cgit From 05ada3f47a4250fb278ecc84c16f51922106b83d Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 20:32:32 -0400 Subject: Finished TODO item --- doc/source/database.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/database.rst b/doc/source/database.rst index bab6b472d..926f9be26 100644 --- a/doc/source/database.rst +++ b/doc/source/database.rst @@ -21,7 +21,6 @@ Database Programming Guide :: TODO(todd): should sqlalchemy.api be here? - pep-256 on db/api.py and models.py (periods) document register_models (where should it be called from?) document any relevant test cases document flags -- cgit From ee4ce8a55194be3fcb1e861e4206451cc7812d46 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 11:23:49 -0400 Subject: Remove "nova Packages and Dependencies" --- doc/source/development.guide.rst | 6 ------ doc/source/packages.rst | 29 ----------------------------- 2 files changed, 35 deletions(-) delete mode 100644 doc/source/packages.rst diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 04fee329a..61a664171 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -52,14 +52,8 @@ Older Contents fakes binaries modules - packages -Removed -------- - -* endpoint - Indices and tables ------------------ diff --git a/doc/source/packages.rst b/doc/source/packages.rst deleted file mode 100644 index 6029ad7d7..000000000 --- a/doc/source/packages.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -nova Packages & Dependencies -============================ - -Nova is being built on Ubuntu Lucid. - -The following packages are required: - - apt-get install python-ipy, python-libvirt, python-boto, python-pycurl, python-twisted, python-daemon, python-redis, python-carrot, python-lockfile - -In addition you need to install python: - - * python-gflags - http://code.google.com/p/python-gflags/ -- cgit From 133cd9973e17458bea3594490e70ccd3c524cf12 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 11:58:57 -0400 Subject: Document Fakes --- doc/source/development.guide.rst | 1 + doc/source/fakes.rst | 34 ++++++++++++++++++++++++++++++++-- nova/auth/fakeldap.py | 4 +--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 61a664171..0d852a098 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -39,6 +39,7 @@ Contents compute network auth + fakes Older Contents diff --git a/doc/source/fakes.rst b/doc/source/fakes.rst index eccd4ab43..72e5c287b 100644 --- a/doc/source/fakes.rst +++ b/doc/source/fakes.rst @@ -15,8 +15,13 @@ License for the specific language governing permissions and limitations under the License. -Nova Fakes -========== +Fakes Programming Guide +======================= + +:: + + TODO(todd): * document general info about fakes + The :mod:`virt.fake` Module --------------------------- @@ -41,3 +46,28 @@ The :mod:`fakerabbit` Module :members: :undoc-members: :show-inheritance: + +The :class:`FakeAOEDriver` Class +-------------------------------- + +.. autoclass:: nova.volume.driver.FakeAOEDriver + :members: + :undoc-members: + :show-inheritance: + +The :class:`service_unittest.FakeManager` Class +----------------------------------------------- + +.. autoclass:: nova.tests.service_unittest.FakeManager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.openstack.fakes` Module +------------------------------------------------ + +.. automodule:: nova.tests.api.openstack.fakes + :members: + :undoc-members: + :show-inheritance: + diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 176a00f06..46e0135b4 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -15,9 +15,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -""" -Fake LDAP server for test harness, backs to ReDIS. +"""Fake LDAP server for test harness, backs to ReDIS. This class does very little error checking, and knows nothing about ldap class definitions. It implements the minimum emulation of the python ldap -- cgit From 5feb2edcc322a8d44d3d698e2d3c27d81d16fe3f Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 12:16:49 -0400 Subject: Fakes cleanup (stop duplicate autodoc of FakeAOEDriver). --- doc/source/volume.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/volume.rst b/doc/source/volume.rst index 27977761a..91678d22a 100644 --- a/doc/source/volume.rst +++ b/doc/source/volume.rst @@ -40,6 +40,7 @@ The :mod:`nova.volume.driver` Module :members: :undoc-members: :show-inheritance: + :exclude-members: FakeAOEDriver Tests ----- -- cgit From 3ec095bed60490c844067c8d58ed43dbedee5f0a Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 12:35:46 -0400 Subject: Update database page a bit. --- doc/source/database.rst | 20 ++++++++++++++------ nova/db/sqlalchemy/api.py | 2 +- nova/db/sqlalchemy/models.py | 8 ++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/doc/source/database.rst b/doc/source/database.rst index 926f9be26..67f940fe1 100644 --- a/doc/source/database.rst +++ b/doc/source/database.rst @@ -20,8 +20,7 @@ Database Programming Guide :: - TODO(todd): should sqlalchemy.api be here? - document register_models (where should it be called from?) + TODO(todd): document register_models (where should it be called from?) document any relevant test cases document flags @@ -37,17 +36,26 @@ The :mod:`api` Module Drivers ------- -The :mod:`sqlalchemy` Driver -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Driver: Sqlalchemy +~~~~~~~~~~~~~~~~~~ + +The :mod:`sqlalchemy.api` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. automodule:: nova.db.sqlalchemy.api + +The :mod:`sqlalchemy.models` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: nova.db.sqlalchemy.models :members: :undoc-members: :show-inheritance: +The :mod:`sqlalchemy.session` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. automodule:: nova.db.sqlalchemy.models +.. automodule:: nova.db.sqlalchemy.session :members: :undoc-members: :show-inheritance: - diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index a3d8dde2f..d9b98655e 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. """ -Implementation of SQLAlchemy backend +Implementation of SQLAlchemy backend. """ import random diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 894ebcddd..29c3b74da 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. """ -SQLAlchemy models for nova data +SQLAlchemy models for nova data. """ import datetime @@ -488,7 +488,11 @@ class FloatingIp(BASE, NovaBase): def register_models(): - """Register Models and create metadata.""" + """Register Models and create metadata. + + Called from nova.db.sqlalchemy.__init__ as part of loading the driver, + it will never need to be called explicitly elsewhere. + """ from sqlalchemy import create_engine models = (Service, Instance, Volume, ExportDevice, FixedIp, FloatingIp, Network, SecurityGroup, -- cgit From f42967bdc7029e5c892811d84c7dfeb39a9b9f47 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Fri, 29 Oct 2010 11:53:09 -0500 Subject: Updated location of layout.html and change conf.py to use a build variable. --- doc/source/_ga/layout.html | 17 +++++++++++++++++ doc/source/_templates/.DS_Store | Bin 0 -> 6148 bytes doc/source/_templates/layout.html | 17 ----------------- 3 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 doc/source/_ga/layout.html create mode 100644 doc/source/_templates/.DS_Store delete mode 100644 doc/source/_templates/layout.html diff --git a/doc/source/_ga/layout.html b/doc/source/_ga/layout.html new file mode 100644 index 000000000..0b72a77ac --- /dev/null +++ b/doc/source/_ga/layout.html @@ -0,0 +1,17 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + + +{% endblock %} + diff --git a/doc/source/_templates/.DS_Store b/doc/source/_templates/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/doc/source/_templates/.DS_Store differ diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html deleted file mode 100644 index 0b72a77ac..000000000 --- a/doc/source/_templates/layout.html +++ /dev/null @@ -1,17 +0,0 @@ -{% extends "!layout.html" %} - -{% block footer %} -{{ super() }} - - -{% endblock %} - -- cgit From 75a0182e9a9a3cb6732e68eb3c9965b8033e1ce1 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Fri, 29 Oct 2010 12:27:30 -0500 Subject: Changes to conf.py --- doc/source/conf.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 505771ff8..e137e728a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -27,7 +27,13 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', todo_include_todos = True # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +# Changing the path so that the Hudson build output contains GA code and the source +# docs do not contain the code so local, offline sphinx builds are "clean." +templates_path = [] +if os.getenv('HUDSON_PUBLISH_DOCS'): + templates_path = ['_ga', '_templates'] +else: + templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' -- cgit From 489ddea1668c742f62acd6fd3e9af78f2f782912 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 15:30:39 -0400 Subject: Update database docs. --- doc/source/database.rst | 12 ++++++------ nova/db/api.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/doc/source/database.rst b/doc/source/database.rst index 67f940fe1..b58ea147d 100644 --- a/doc/source/database.rst +++ b/doc/source/database.rst @@ -18,12 +18,6 @@ Database Programming Guide ========================== -:: - - TODO(todd): document register_models (where should it be called from?) - document any relevant test cases - document flags - The :mod:`api` Module --------------------- @@ -59,3 +53,9 @@ The :mod:`sqlalchemy.session` Module :members: :undoc-members: :show-inheritance: + +Tests +----- + +Tests are lacking for the db api layer and for the sqlalchemy driver. +Failures in the drivers would be dectected in other test cases, though. diff --git a/nova/db/api.py b/nova/db/api.py index 7cce591ad..659bfd6b8 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -17,6 +17,16 @@ # under the License. """ Defines interface for DB access. + +The underlying driver is loaded as a :class:`LazyPluggable`. + +**Related Flags** + +:db_backend: string to lookup in the list of LazyPluggable backends. + `sqlalchemy` is the only supported backend right now. + +:sql_connection: string specifying the sqlalchemy connection to use, like: + `sqlite:///var/lib/nova/nova.sqlite`. """ from nova import exception -- cgit From 5ffbfdd32bda3de071f994760ab9539bed40172a Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 15:40:58 -0400 Subject: Remove fakes duplication. --- doc/source/development.guide.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 0d852a098..df8eb341a 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -50,7 +50,6 @@ Older Contents architecture nova - fakes binaries modules -- cgit From 7ca505a729ee3caae968cf25059721ab51d1327c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 16:18:00 -0400 Subject: Volume documentation. --- nova/tests/volume_unittest.py | 21 ++++++++++++--------- nova/volume/driver.py | 24 +++++++++++++----------- nova/volume/manager.py | 39 +++++++++++++++++++++++++++++++-------- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index fdee30b48..896800cea 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -16,7 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. """ -Tests for Volume Code +Tests for Volume Code. + """ import logging @@ -33,7 +34,8 @@ FLAGS = flags.FLAGS class VolumeTestCase(test.TrialTestCase): - """Test Case for volumes""" + """Test Case for volumes.""" + def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() @@ -44,7 +46,7 @@ class VolumeTestCase(test.TrialTestCase): @staticmethod def _create_volume(size='0'): - """Create a volume object""" + """Create a volume object.""" vol = {} vol['size'] = size vol['user_id'] = 'fake' @@ -56,7 +58,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_create_delete_volume(self): - """Test volume can be created and deleted""" + """Test volume can be created and deleted.""" volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) self.assertEqual(volume_id, db.volume_get(context.get_admin_context(), @@ -70,7 +72,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_big_volume(self): - """Ensure failure if a too large of a volume is requested""" + """Ensure failure if a too large of a volume is requested.""" # FIXME(vish): validation needs to move into the data layer in # volume_create defer.returnValue(True) @@ -83,7 +85,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_many_volumes(self): - """Ensure that NoMoreBlades is raised when we run out of volumes""" + """Ensure that NoMoreBlades is raised when we run out of volumes.""" vols = [] total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf for _index in xrange(total_slots): @@ -100,7 +102,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_attach_detach_volume(self): - """Make sure volume can be attached and detached from instance""" + """Make sure volume can be attached and detached from instance.""" inst = {} inst['image_id'] = 'ami-test' inst['reservation_id'] = 'r-fakeres' @@ -149,12 +151,13 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_concurrent_volumes_get_different_blades(self): - """Ensure multiple concurrent volumes get different blades""" + """Ensure multiple concurrent volumes get different blades.""" + volume_ids = [] shelf_blades = [] def _check(volume_id): - """Make sure blades aren't duplicated""" + """Make sure blades aren't duplicated.""" volume_ids.append(volume_id) admin_context = context.get_admin_context() (shelf_id, blade_id) = db.volume_get_shelf_and_blade(admin_context, diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 3fa29ba37..b99089374 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -15,9 +15,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ -Drivers for volumes +Drivers for volumes. + """ import logging @@ -39,7 +39,8 @@ flags.DEFINE_string('num_shell_tries', 3, class AOEDriver(object): - """Executes commands relating to AOE volumes""" + """Executes commands relating to AOE volumes.""" + def __init__(self, execute=process.simple_execute, *args, **kwargs): self._execute = execute @@ -63,7 +64,7 @@ class AOEDriver(object): @defer.inlineCallbacks def create_volume(self, volume_name, size): - """Creates a logical volume""" + """Creates a logical volume.""" # NOTE(vish): makes sure that the volume group exists yield self._execute("vgs %s" % FLAGS.volume_group) if int(size) == 0: @@ -77,14 +78,14 @@ class AOEDriver(object): @defer.inlineCallbacks def delete_volume(self, volume_name): - """Deletes a logical volume""" + """Deletes a logical volume.""" yield self._try_execute("sudo lvremove -f %s/%s" % (FLAGS.volume_group, volume_name)) @defer.inlineCallbacks def create_export(self, volume_name, shelf_id, blade_id): - """Creates an export for a logical volume""" + """Creates an export for a logical volume.""" yield self._try_execute( "sudo vblade-persist setup %s %s %s /dev/%s/%s" % (shelf_id, @@ -95,13 +96,13 @@ class AOEDriver(object): @defer.inlineCallbacks def discover_volume(self, _volume_name): - """Discover volume on a remote host""" + """Discover volume on a remote host.""" yield self._execute("sudo aoe-discover") yield self._execute("sudo aoe-stat") @defer.inlineCallbacks def remove_export(self, _volume_name, shelf_id, blade_id): - """Removes an export for a logical volume""" + """Removes an export for a logical volume.""" yield self._try_execute("sudo vblade-persist stop %s %s" % (shelf_id, blade_id)) yield self._try_execute("sudo vblade-persist destroy %s %s" % @@ -109,7 +110,7 @@ class AOEDriver(object): @defer.inlineCallbacks def ensure_exports(self): - """Runs all existing exports""" + """Runs all existing exports.""" # NOTE(vish): The standard _try_execute does not work here # because these methods throw errors if other # volumes on this host are in the process of @@ -125,11 +126,12 @@ class AOEDriver(object): class FakeAOEDriver(AOEDriver): - """Logs calls instead of executing""" + """Logs calls instead of executing.""" + def __init__(self, *args, **kwargs): super(FakeAOEDriver, self).__init__(self.fake_execute) @staticmethod def fake_execute(cmd, *_args, **_kwargs): - """Execute that simply logs the command""" + """Execute that simply logs the command.""" logging.debug("FAKE AOE: %s", cmd) diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 2874459f9..f6e220c5f 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -15,10 +15,31 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ -Volume manager manages creating, attaching, detaching, and -destroying persistent storage volumes, ala EBS. +Volume manager manages creating, attaching, detaching, and persistent storage. + +Persistant storage volumes keep their state independent of instances. You can +attach to an instance, terminate the instance, spawn a new instance (even +one from a different image) and re-attach the volume with the same data +intact. + +**Related Flags** + +:volume_topic: What :mod:`rpc` topic to listen to (default: `volume`). +:volume_manager: The module name of a class derived from + :class:`manager.Manager` (default: + :class:`nova.volume.manager.AOEManager`). +:storage_availability_zone: Defaults to `nova`. +:volume_driver: Used by :class:`AOEManager`. Defaults to + :class:`nova.volume.driver.AOEDriver`. +:num_shelves: Number of shelves for AoE (default: 100). +:num_blades: Number of vblades per shelf to allocate AoE storage from + (default: 16). +:volume_group: Name of the group that will contain exported volumes (default: + `nova-volumes`) +:aoe_eth_dev: Device name the volumes will be exported on (default: `eth0`). +:num_shell_tries: Number of times to attempt to run AoE commands (default: 3) + """ import logging @@ -47,15 +68,17 @@ flags.DEFINE_integer('blades_per_shelf', class AOEManager(manager.Manager): - """Manages Ata-Over_Ethernet volumes""" + """Manages Ata-Over_Ethernet volumes.""" + def __init__(self, volume_driver=None, *args, **kwargs): + """Load the driver from the one specified in args, or from flags.""" if not volume_driver: volume_driver = FLAGS.volume_driver self.driver = utils.import_object(volume_driver) super(AOEManager, self).__init__(*args, **kwargs) def _ensure_blades(self, context): - """Ensure that blades have been created in datastore""" + """Ensure that blades have been created in datastore.""" total_blades = FLAGS.num_shelves * FLAGS.blades_per_shelf if self.db.export_device_count(context) >= total_blades: return @@ -66,7 +89,7 @@ class AOEManager(manager.Manager): @defer.inlineCallbacks def create_volume(self, context, volume_id): - """Creates and exports the volume""" + """Creates and exports the volume.""" context = context.elevated() logging.info("volume %s: creating", volume_id) @@ -104,7 +127,7 @@ class AOEManager(manager.Manager): @defer.inlineCallbacks def delete_volume(self, context, volume_id): - """Deletes and unexports volume""" + """Deletes and unexports volume.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['attach_status'] == "attached": @@ -123,7 +146,7 @@ class AOEManager(manager.Manager): @defer.inlineCallbacks def setup_compute_volume(self, context, volume_id): - """Setup remote volume on compute host + """Setup remote volume on compute host. Returns path to device. """ -- cgit From 3bc84d66d35976794b559ad305dd10eec450216f Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 16:19:57 -0400 Subject: Change volume TODO list. --- doc/source/volume.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/source/volume.rst b/doc/source/volume.rst index 91678d22a..39b33a500 100644 --- a/doc/source/volume.rst +++ b/doc/source/volume.rst @@ -20,9 +20,7 @@ Volume Programming Guide :: - TODO(todd): document for iSCSI & AoE (see 'Old Docs') - pep-257 - document flags + TODO(todd): rework after iSCSI merge (see 'Old Docs') The :mod:`nova.volume.manager` Module -- cgit From bf15a6eb3de8c688dc1364959dd3e00d3e26a563 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Sat, 30 Oct 2010 20:05:31 -0400 Subject: Update compute/disk.py docs. --- nova/compute/disk.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index e362b4507..0b8568d33 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -15,10 +15,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ Utility methods to resize, repartition, and modify disk images. + Includes injection of SSH PGP keys into authorized_keys file. + """ import logging @@ -41,20 +42,23 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, @defer.inlineCallbacks def partition(infile, outfile, local_bytes=0, resize=True, local_type='ext2', execute=None): - """Takes a single partition represented by infile and writes a bootable - drive image into outfile. + """ + Turns a partition (infile) into a bootable drive image (outfile). The first 63 sectors (0-62) of the resulting image is a master boot record. Infile becomes the first primary partition. If local bytes is specified, a second primary partition is created and formatted as ext2. - In the diagram below, dashes represent drive sectors. - +-----+------. . .-------+------. . .------+ - | 0 a| b c|d e| - +-----+------. . .-------+------. . .------+ - | mbr | primary partiton | local partition | - +-----+------. . .-------+------. . .------+ + :: + + In the diagram below, dashes represent drive sectors. + +-----+------. . .-------+------. . .------+ + | 0 a| b c|d e| + +-----+------. . .-------+------. . .------+ + | mbr | primary partiton | local partition | + +-----+------. . .-------+------. . .------+ + """ sector_size = 512 file_size = os.path.getsize(infile) -- cgit From fad337b648ea887bb713aab73335aa4602746b62 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 30 Oct 2010 19:58:15 -0700 Subject: don't check for vgroup in fake mode --- nova/volume/driver.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 1fb0386eb..6b0510704 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -212,6 +212,10 @@ class FakeAOEDriver(AOEDriver): sync_exec=self.fake_execute, *args, **kwargs) + def check_for_setup_error(self): + """Returns an error if prerequisites aren't met""" + pass + @staticmethod def fake_execute(cmd, *_args, **_kwargs): """Execute that simply logs the command""" @@ -319,6 +323,10 @@ class FakeISCSIDriver(ISCSIDriver): sync_exec=self.fake_execute, *args, **kwargs) + def check_for_setup_error(self): + """Returns an error if prerequisites aren't met""" + pass + @staticmethod def fake_execute(cmd, *_args, **_kwargs): """Execute that simply logs the command""" -- cgit From 3bc28df8a1369dd9a717a5986000226c2c1d8c02 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 30 Oct 2010 20:57:18 -0700 Subject: Change retrieval of security groups from kwargs so they are associated properly and add test to verify --- nova/compute/manager.py | 5 +++-- nova/tests/compute_unittest.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d50607aca..850cded8a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -69,7 +69,7 @@ class ComputeManager(manager.Manager): def refresh_security_group(self, context, security_group_id, **_kwargs): yield self.driver.refresh_security_group(security_group_id) - def create_instance(self, context, security_groups=[], **kwargs): + def create_instance(self, context, security_groups=None, **kwargs): """Creates the instance in the datastore and returns the new instance as a mapping @@ -88,7 +88,8 @@ class ComputeManager(manager.Manager): inst_id = instance_ref['id'] elevated = context.elevated() - security_groups = kwargs.get('security_groups', []) + if not security_groups: + security_groups = [] for security_group_id in security_groups: self.db.instance_add_security_group(elevated, inst_id, diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 01b5651df..71a1a4457 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -66,6 +66,27 @@ class ComputeTestCase(test.TrialTestCase): inst['ami_launch_index'] = 0 return db.instance_create(self.context, inst)['id'] + def test_create_instance_associates_security_groups(self): + """Make sure create_instance associates security groups""" + inst = {} + inst['user_id'] = self.user.id + inst['project_id'] = self.project.id + values = {'name': 'default', + 'description': 'default', + 'user_id': self.user.id, + 'project_id': self.project.id} + group = db.security_group_create(self.context, values) + ref = self.compute.create_instance(self.context, + security_groups=[group['id']], + **inst) + # reload to get groups + instance_ref = db.instance_get(self.context, ref['id']) + try: + self.assertEqual(len(instance_ref['security_groups']), 1) + finally: + db.security_group_destroy(self.context, group['id']) + db.instance_destroy(self.context, instance_ref['id']) + @defer.inlineCallbacks def test_run_terminate(self): """Make sure it is possible to run and terminate instance""" -- cgit From 878eb4d25075f8d78f24ad9f78eb5d43702192ca Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 1 Nov 2010 16:13:18 -0400 Subject: Virt documentation. --- doc/source/compute.rst | 53 ++++++++++++++++++++++++----------------------- nova/virt/connection.py | 20 +++++++++++------- nova/virt/fake.py | 7 +++++-- nova/virt/libvirt_conn.py | 22 +++++++++++++++++++- nova/virt/xenapi.py | 12 +++++++++++ 5 files changed, 77 insertions(+), 37 deletions(-) diff --git a/doc/source/compute.rst b/doc/source/compute.rst index d29b96781..ba5f2917c 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -24,10 +24,7 @@ This page contains the Compute Package documentation. :: - TODO(todd): * Document drivers - * get docstrings to come in for ComputeManager - * fix formatting of ascii art in disk module - * document instance_types and power_states + TODO(todd): * document instance_types and power_states Manager @@ -45,6 +42,13 @@ The :mod:`manager` Module :undoc-members: :show-inheritance: +The :mod:`connection` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.connection + :members: + :undoc-members: + :show-inheritance: The :mod:`disk` Module ~~~~~~~~~~~~~~~~~~~~~~ @@ -76,40 +80,37 @@ The :mod:`power_state` Module Drivers ------- - Libvirt Implementations ~~~~~~~~~~~~~~~~~~~~~~~ +The libvirt driver is capable of supporting KVM, QEMU, and UML. -Libvirt: KVM -^^^^^^^^^^^^ - -KVM Driver - - -Libvirt: QEMU -^^^^^^^^^^^^^ - -QEMU Driver - - -Libvirt: UML -^^^^^^^^^^^^ - -User Mode Linux Driver +The :mod:`libvirt_conn` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automodule:: nova.virt.libvirt_conn + :members: + :undoc-members: + :show-inheritance: XEN ~~~ -Xen Driver - +The :mod:`xenapi` Module +^^^^^^^^^^^^^^^^^^^^^^^^ -Hyper-V -~~~~~~~ +.. automodule:: nova.virt.xenapi + :members: + :undoc-members: + :show-inheritance: -Hyper-V Driver +FAKE +~~~~ +.. automodule:: nova.virt.fake + :members: + :undoc-members: + :show-inheritance: Monitoring ---------- diff --git a/nova/virt/connection.py b/nova/virt/connection.py index ceb7f1e4b..11f0fa8ce 100644 --- a/nova/virt/connection.py +++ b/nova/virt/connection.py @@ -32,19 +32,23 @@ FLAGS = flags.FLAGS def get_connection(read_only=False): - """Returns an object representing the connection to a virtualization - platform. This could be nova.virt.fake.FakeConnection in test mode, - a connection to KVM or QEMU via libvirt, or a connection to XenServer - or Xen Cloud Platform via XenAPI. + """ + Returns an object representing the connection to a virtualization + platform. + + This could be :mod:`nova.virt.fake.FakeConnection` in test mode, + a connection to KVM, QEMU, or UML via :mod:`libvirt_conn`, or a connection + to XenServer or Xen Cloud Platform via :mod:`xenapi`. Any object returned here must conform to the interface documented by - FakeConnection. + :mod:`FakeConnection`. + + **Related flags** - Related flags - ------------- :connection_type: A string literal that falls through a if/elif structure to determine what virtualization mechanism to use. - Values may be: + Values may be + * fake * libvirt * xenapi diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 66eff4c66..f855523d3 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -18,8 +18,11 @@ # under the License. """ -A fake (in-memory) hypervisor+api. Allows nova testing w/o a hypervisor. -This module also documents the semantics of real hypervisor connections. +A fake (in-memory) hypervisor+api. + +Allows nova testing w/o a hypervisor. This module also documents the +semantics of real hypervisor connections. + """ from twisted.internet import defer diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e32945fa5..0170fc6d1 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -18,7 +18,27 @@ # under the License. """ -A connection to a hypervisor (e.g. KVM) through libvirt. +A connection to a hypervisor through libvirt. + +Supports KVM, QEMU, UML, and XEN. + +**Related Flags** + +:libvirt_type: Libvirt domain type. Can be kvm, qemu, uml, xen + (default: kvm). +:libvirt_uri: Override for the default libvirt URI (depends on libvirt_type). +:libvirt_xml_template: Libvirt XML Template (QEmu/KVM). +:libvirt_xen_xml_template: Libvirt XML Template (Xen). +:libvirt_uml_xml_template: Libvirt XML Template (User Mode Linux). +:libvirt_rescue_xml_template: XML template for rescue mode (KVM & QEMU). +:libvirt_rescue_xen_xml_template: XML templage for rescue mode (XEN). +:libvirt_rescue_uml_xml_template: XML template for rescue mode (UML). +:rescue_image_id: Rescue ami image (default: ami-rescue). +:rescue_kernel_id: Rescue aki image (default: aki-rescue). +:rescue_ramdisk_id: Rescue ari image (default: ari-rescue). +:injected_network_template: Template file for injected network +:allow_project_net_traffic: Whether to allow in project network traffic + """ import logging diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index a17e405ab..0f563aa41 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -33,6 +33,18 @@ long-running operations. FIXME: get_info currently doesn't conform to these rules, and will block the reactor thread if the VM.get_by_name_label or VM.get_record calls block. + +**Related Flags** + +:xenapi_connection_url: URL for connection to XenServer/Xen Cloud Platform. +:xenapi_connection_username: Username for connection to XenServer/Xen Cloud + Platform (default: root). +:xenapi_connection_password: Password for connection to XenServer/Xen Cloud + Platform. +:xenapi_task_poll_interval: The interval (seconds) used for polling of + remote tasks (Async.VM.start, etc) + (default: 0.5). + """ import logging -- cgit From 501850f3d470da646378c8e7de7657024411d2e0 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 1 Nov 2010 16:26:35 -0400 Subject: :noindex: on the fakes page for virt.fakes which is included in compute.rst --- doc/source/fakes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/fakes.rst b/doc/source/fakes.rst index 72e5c287b..5988e2ab8 100644 --- a/doc/source/fakes.rst +++ b/doc/source/fakes.rst @@ -30,6 +30,7 @@ The :mod:`virt.fake` Module :members: :undoc-members: :show-inheritance: + :noindex: The :mod:`fakeldap` Module -------------------------- -- cgit From c8e2341c98ffacfafffbadb7d204f10ff87cf89c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 1 Nov 2010 20:33:03 -0400 Subject: API endpoint documentation. --- doc/source/api.rst | 112 +++++++++++++++++++++++++++++++++++++++ doc/source/development.guide.rst | 13 +---- doc/source/index.rst | 10 ++-- nova/api/__init__.py | 8 ++- 4 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 doc/source/api.rst diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..03ec6195d --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,112 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +API Endpoints +============= + +:: + + TODO(todd): get actual docstrings from ec2/osapi_verions instead of @wsgify + +Nova has a system for managing multiple APIs on different subdomains. +Currently there is support for the OpenStack API, as well as the Amazon EC2 +API. + +Common Components +----------------- + +The :mod:`nova.api` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.cloud + :members: + :undoc-members: + :show-inheritance: + +OpenStack API +------------- + +The :mod:`openstack` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack + :members: + :undoc-members: + :show-inheritance: + +The :mod:`auth` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.auth + :members: + :undoc-members: + :show-inheritance: + +The :mod:`backup_schedules` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.backup_schedules + :members: + :undoc-members: + :show-inheritance: + +The :mod:`faults` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.faults + :members: + :undoc-members: + :show-inheritance: + +The :mod:`flavors` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.flavors + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.images + :members: + :undoc-members: + :show-inheritance: + +The :mod:`ratelimiting` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.ratelimiting + :members: + :undoc-members: + :show-inheritance: + +The :mod:`servers` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.servers + :members: + :undoc-members: + :show-inheritance: + +The :mod:`sharedipgroups` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.sharedipgroups + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index df8eb341a..8addc7813 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -39,21 +39,10 @@ Contents compute network auth + api fakes -Older Contents --------------- - -.. toctree:: - :maxdepth: 1 - - architecture - nova - binaries - modules - - Indices and tables ------------------ diff --git a/doc/source/index.rst b/doc/source/index.rst index 54cfea05a..261919629 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -22,11 +22,11 @@ Nova is a cloud computing fabric controller, the main part of an IaaS system. Nova is written with the following design guidelines in mind: -* Component based architecture: Quickly add new behaviors -* Highly available: Scale to very serious workloads -* Fault-Tollerant: Isloated processes avoid cascading failures -* Recoverable: Failures should be easy to diagnose, debug, and rectify -* Open Standards: Be a reference implementation for a community-driven api +* **Component based architecture**: Quickly add new behaviors +* **Highly available**: Scale to very serious workloads +* **Fault-Tollerant**: Isloated processes avoid cascading failures +* **Recoverable**: Failures should be easy to diagnose, debug, and rectify +* **Open Standards**: Be a reference implementation for a community-driven api This documentation is generated by the Sphinx toolkit and lives in the source tree. Additional documentation on Nova and other components of OpenStack can diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 8a1d9fe32..27b8199db 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -15,9 +15,15 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ Root WSGI middleware for all API controllers. + +**Related Flags** + +:osapi_subdomain: subdomain running the OpenStack API (default: api) +:ec2api_subdomain: subdomain running the EC2 API (default: ec2) +:FAKE_subdomain: set to 'api' or 'ec2', requests default to that endpoint + """ import routes -- cgit From ff588ab5baf400c243daeff82e7ca2fd27d87143 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 1 Nov 2010 21:13:51 -0400 Subject: Exceptions docs. --- doc/source/development.guide.rst | 4 ++-- doc/source/exceptions.rst | 27 +++++++++++++++++++++++++++ doc/source/nova.rst | 9 +-------- 3 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 doc/source/exceptions.rst diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 8addc7813..7322545de 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -22,8 +22,7 @@ Nova is written in python. :: - TODO(todd): * API - * Exceptions + TODO(todd): * Nova libraries (utils, etc) * Building packages @@ -40,6 +39,7 @@ Contents network auth api + exceptions fakes diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst new file mode 100644 index 000000000..aaf5b2c1a --- /dev/null +++ b/doc/source/exceptions.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Exceptions Programming Guide +============================ + +The :mod:`nova.excepton` Module +------------------------------- + +.. automodule:: nova.exception + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/nova.rst b/doc/source/nova.rst index 4b9c44a5f..a5573cb41 100644 --- a/doc/source/nova.rst +++ b/doc/source/nova.rst @@ -34,14 +34,6 @@ The :mod:`adminclient` Module :undoc-members: :show-inheritance: -The :mod:`datastore` Module ---------------------------- - -.. automodule:: nova.datastore - :members: - :undoc-members: - :show-inheritance: - The :mod:`exception` Module --------------------------- @@ -49,6 +41,7 @@ The :mod:`exception` Module :members: :undoc-members: :show-inheritance: + :noindex: The :mod:`flags` Module --------------------------- -- cgit From e0f889443f5c0732db28871f350c45e7c8e8d031 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 1 Nov 2010 21:47:16 -0400 Subject: Add ec2 api docs. --- doc/source/api.rst | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ nova/api/ec2/__init__.py | 4 +++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index 03ec6195d..75905d8b3 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -110,3 +110,54 @@ The :mod:`sharedipgroups` Module :members: :undoc-members: :show-inheritance: + +EC2 API +------- + +The :mod:`nova.api.ec2` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2 + :members: + :undoc-members: + :show-inheritance: + +The :mod:`admin` Module +~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.admin + :members: + :undoc-members: + :show-inheritance: + +The :mod:`apirequest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.apirequest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud` Module +~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.cloud + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.images + :members: + :undoc-members: + :show-inheritance: + +The :mod:`metadatarequesthandler` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.metadatarequesthandler + :members: + :undoc-members: + :show-inheritance: diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 0df4d3710..816314901 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -15,8 +15,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +""" +Starting point for routing EC2 requests. -"""Starting point for routing EC2 requests""" +""" import logging import routes -- cgit From 500f101c64a8e5db91111a7afd7c95ac360b67fb Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 1 Nov 2010 22:32:41 -0400 Subject: Language change for conformity. --- doc/source/api.rst | 4 ++-- doc/source/auth.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index 75905d8b3..3f1c69a15 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -15,8 +15,8 @@ License for the specific language governing permissions and limitations under the License. -API Endpoints -============= +API Endpoint Programming Guide +============================== :: diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 203efe1a5..61db39ec3 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -17,7 +17,7 @@ .. _auth: -Authentication and Authorization Development Guide +Authentication and Authorization Programming Guide ================================================== :: -- cgit From 56c22eab57bc1096c0cd7e6756b42d163649fae1 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 1 Nov 2010 23:32:56 -0400 Subject: More docs. --- doc/source/compute.rst | 8 ++++ doc/source/development.guide.rst | 7 ++- doc/source/exceptions.rst | 27 ----------- doc/source/nova.rst | 101 +++++++++++++++++++++++++++++++-------- doc/source/scheduler.rst | 51 ++++++++++++++++++++ 5 files changed, 144 insertions(+), 50 deletions(-) delete mode 100644 doc/source/exceptions.rst create mode 100644 doc/source/scheduler.rst diff --git a/doc/source/compute.rst b/doc/source/compute.rst index ba5f2917c..e9e37ebf8 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -58,6 +58,14 @@ The :mod:`disk` Module :undoc-members: :show-inheritance: +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.images + :members: + :undoc-members: + :show-inheritance: + The :mod:`instance_types` Module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 7322545de..261059d9b 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -22,9 +22,7 @@ Nova is written in python. :: - TODO(todd): - * Nova libraries (utils, etc) - * Building packages + TODO(todd): * Building packages Contents @@ -39,8 +37,9 @@ Contents network auth api - exceptions + scheduler fakes + nova Indices and tables diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst deleted file mode 100644 index aaf5b2c1a..000000000 --- a/doc/source/exceptions.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Exceptions Programming Guide -============================ - -The :mod:`nova.excepton` Module -------------------------------- - -.. automodule:: nova.exception - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/nova.rst b/doc/source/nova.rst index a5573cb41..59fd56c85 100644 --- a/doc/source/nova.rst +++ b/doc/source/nova.rst @@ -15,21 +15,29 @@ License for the specific language governing permissions and limitations under the License. -NOVA Libraries -=============== +Nova Libraries Programming Guide +================================ -The :mod:`crypto` Module ------------------------- +The :mod:`adminclient` Module +----------------------------- -.. automodule:: nova.crypto +.. automodule:: nova.adminclient :members: :undoc-members: :show-inheritance: - -The :mod:`adminclient` Module ------------------------------ -.. automodule:: nova.adminclient +The :mod:`context` Module +------------------------- + +.. automodule:: nova.context + :members: + :undoc-members: + :show-inheritance: + +The :mod:`crypto` Module +------------------------ + +.. automodule:: nova.crypto :members: :undoc-members: :show-inheritance: @@ -41,44 +49,99 @@ The :mod:`exception` Module :members: :undoc-members: :show-inheritance: - :noindex: - + The :mod:`flags` Module ---------------------------- +----------------------- .. automodule:: nova.flags :members: :undoc-members: :show-inheritance: +The :mod:`manager` Module +------------------------- + +.. automodule:: nova.manager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`process` Module +------------------------- + +.. automodule:: nova.process + :members: + :undoc-members: + :show-inheritance: + +The :mod:`quota` Module +----------------------- + +.. automodule:: nova.quota + :members: + :undoc-members: + :show-inheritance: + The :mod:`rpc` Module ---------------------------- +--------------------- .. automodule:: nova.rpc :members: :undoc-members: :show-inheritance: - + The :mod:`server` Module ---------------------------- +------------------------ .. automodule:: nova.server :members: :undoc-members: :show-inheritance: - + +The :mod:`service` Module +------------------------- + +.. automodule:: nova.service + :members: + :undoc-members: + :show-inheritance: + The :mod:`test` Module ---------------------------- +---------------------- .. automodule:: nova.test :members: :undoc-members: :show-inheritance: - + +The :mod:`twistd` Module +------------------------ + +.. automodule:: nova.twistd + :members: + :undoc-members: + :show-inheritance: + The :mod:`utils` Module ---------------------------- +----------------------- .. automodule:: nova.utils :members: :undoc-members: :show-inheritance: + +The :mod:`validate` Module +-------------------------- + +.. automodule:: nova.validate + :members: + :undoc-members: + :show-inheritance: + +The :mod:`wsgi` Module +---------------------- + +.. automodule:: nova.wsgi + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/scheduler.rst b/doc/source/scheduler.rst new file mode 100644 index 000000000..e809b0c54 --- /dev/null +++ b/doc/source/scheduler.rst @@ -0,0 +1,51 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Scheduler Programming Guide +=========================== + +The :mod:`manager` Module +------------------------- + +.. automodule:: nova.scheduler.manager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`driver` Module +------------------------ + +.. automodule:: nova.scheduler.driver + :members: + :undoc-members: + :show-inheritance: + +The :mod:`chance` Module +------------------------ + +.. automodule:: nova.scheduler.chance + :members: + :undoc-members: + :show-inheritance: + +The :mod:`simple` Module +------------------------ + +.. automodule:: nova.scheduler.simple + :members: + :undoc-members: + :show-inheritance: -- cgit From 2e2dce7ebf478258f67a9122c6b158ba5e89c1ed Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 2 Nov 2010 11:28:14 -0700 Subject: Added support for OpenStack and EC2 APIs to run on different ports. --- bin/nova-api | 9 +++++++-- nova/api/__init__.py | 34 +++++++++++++++------------------- nova/wsgi.py | 25 +++++++++++++++++++++---- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 20f1bd74f..a9002ae2d 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -37,13 +37,18 @@ from nova import utils from nova import server FLAGS = flags.FLAGS -flags.DEFINE_integer('api_port', 8773, 'API port') +flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') +flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') def main(_args): from nova import api from nova import wsgi - wsgi.run_server(api.API(), FLAGS.api_port) + server = wsgi.Server() + server.start(api.API('os'), FLAGS.osapi_port) + server.start(api.API('ec2'), FLAGS.ec2api_port) + server.wait() + if __name__ == '__main__': utils.default_flagfile() diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 8a1d9fe32..707c1623e 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -35,37 +35,31 @@ flags.DEFINE_string('osapi_subdomain', 'api', 'subdomain running the OpenStack API') flags.DEFINE_string('ec2api_subdomain', 'ec2', 'subdomain running the EC2 API') -flags.DEFINE_string('FAKE_subdomain', None, - 'set to api or ec2 to fake the subdomain of the host ' - 'for testing') FLAGS = flags.FLAGS class API(wsgi.Router): """Routes top-level requests to the appropriate controller.""" - def __init__(self): - osapidomain = {'sub_domain': [FLAGS.osapi_subdomain]} - ec2domain = {'sub_domain': [FLAGS.ec2api_subdomain]} - # If someone wants to pretend they're hitting the OSAPI subdomain - # on their local box, they can set FAKE_subdomain to 'api', which - # removes subdomain restrictions from the OpenStack API routes below. - if FLAGS.FAKE_subdomain == 'api': - osapidomain = {} - elif FLAGS.FAKE_subdomain == 'ec2': - ec2domain = {} + def __init__(self, default_api): + osapi_subdomain = {'sub_domain': [FLAGS.osapi_subdomain]} + ec2api_subdomain = {'sub_domain': [FLAGS.ec2api_subdomain]} + if default_api == 'os': + osapi_subdomain = {} + elif default_api == 'ec2': + ec2api_subdomain = {} mapper = routes.Mapper() mapper.sub_domains = True + mapper.connect("/", controller=self.osapi_versions, - conditions=osapidomain) + conditions=osapi_subdomain) mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), - conditions=osapidomain) + conditions=osapi_subdomain) mapper.connect("/", controller=self.ec2api_versions, - conditions=ec2domain) + conditions=ec2api_subdomain) mapper.connect("/services/{path_info:.*}", controller=ec2.API(), - conditions=ec2domain) - mapper.connect("/cloudpipe/{path_info:.*}", controller=cloudpipe.API()) + conditions=ec2api_subdomain) mrh = metadatarequesthandler.MetadataRequestHandler() for s in ['/latest', '/2009-04-04', @@ -78,7 +72,9 @@ class API(wsgi.Router): '/2007-01-19', '/1.0']: mapper.connect('%s/{path_info:.*}' % s, controller=mrh, - conditions=ec2domain) + conditions=ec2api_subdomain) + + mapper.connect("/cloudpipe/{path_info:.*}", controller=cloudpipe.API()) super(API, self).__init__(mapper) @webob.dec.wsgify diff --git a/nova/wsgi.py b/nova/wsgi.py index eb305a3d3..b04b487ea 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -39,10 +39,27 @@ import webob.exc logging.getLogger("routes.middleware").addHandler(logging.StreamHandler()) -def run_server(application, port): - """Run a WSGI server with the given application.""" - sock = eventlet.listen(('0.0.0.0', port)) - eventlet.wsgi.server(sock, application) +class Server(object): + """Server class to manage multiple WSGI sockets and applications.""" + + def __init__(self, threads=1000): + self.pool = eventlet.GreenPool(threads) + + def start(self, application, port, host='0.0.0.0', backlog=128): + """Run a WSGI server with the given application.""" + socket = eventlet.listen((host, port), backlog=backlog) + self.pool.spawn_n(self._run, application, socket) + + def wait(self): + """Wait until all servers have completed running.""" + try: + self.pool.waitall() + except KeyboardInterrupt: + pass + + def _run(self, application, socket): + """Start a WSGI server in a new green thread.""" + eventlet.wsgi.server(socket, application, custom_pool=self.pool) class Application(object): -- cgit From 785d60c9492a8d4583eb27b214abefda6c1fbcfc Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 2 Nov 2010 12:02:42 -0700 Subject: Fixed tests to work with new default API argument. --- nova/tests/api/__init__.py | 2 +- nova/tests/api/openstack/fakes.py | 4 ---- nova/tests/api/openstack/test_auth.py | 18 +++++++++--------- nova/tests/api/openstack/test_flavors.py | 2 +- nova/tests/api/openstack/test_images.py | 4 ++-- nova/tests/api/openstack/test_servers.py | 28 ++++++++++++++-------------- nova/tests/api_unittest.py | 6 +----- 7 files changed, 28 insertions(+), 36 deletions(-) diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index 46f09e906..9caa8c9d0 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -42,7 +42,7 @@ class Test(unittest.TestCase): environ_keys = {'HTTP_HOST': '%s.example.com' % subdomain} environ_keys.update(kwargs) req = webob.Request.blank(url, environ_keys) - return req.get_response(api.API()) + return req.get_response(api.API('ec2')) def test_openstack(self): self.stubs.Set(api.openstack, 'API', APIStub) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 1b8c18974..52b392601 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -34,9 +34,6 @@ from nova.tests import fake_flags from nova.wsgi import Router -FLAGS = flags.FLAGS - - class Context(object): pass @@ -108,7 +105,6 @@ def stub_out_networking(stubs): def get_my_ip(): return '127.0.0.1' stubs.Set(nova.utils, 'get_my_ip', get_my_ip) - FLAGS.FAKE_subdomain = 'api' def stub_out_glance(stubs, initial_fixtures=[]): diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index b63da187f..29f4b8874 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,7 +51,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-CDN-Management-Url'], @@ -65,7 +65,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], @@ -79,7 +79,7 @@ class Test(unittest.TestCase): fakes.FakeRouter) req = webob.Request.blank('/v1.0/fake') req.headers['X-Auth-Token'] = token - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') @@ -103,7 +103,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-Token'] = 'bacon' - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '401 Unauthorized') self.assertEqual(self.destroy_called, True) @@ -111,18 +111,18 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '401 Unauthorized') def test_no_user(self): req = webob.Request.blank('/v1.0/') - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '401 Unauthorized') def test_bad_token(self): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-Token'] = 'baconbaconbacon' - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '401 Unauthorized') @@ -146,7 +146,7 @@ class TestLimiter(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(len(result.headers['X-Auth-Token']), 40) token = result.headers['X-Auth-Token'] @@ -155,7 +155,7 @@ class TestLimiter(unittest.TestCase): req = webob.Request.blank('/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token - result = req.get_response(nova.api.API()) + result = req.get_response(nova.api.API('os')) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 8dd4d1f29..41018afdf 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -39,7 +39,7 @@ class FlavorsTest(unittest.TestCase): def test_get_flavor_list(self): req = webob.Request.blank('/v1.0/flavors') - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) def test_get_flavor_by_id(self): pass diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index d61c3a99b..0f3941c29 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -203,7 +203,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def test_get_image_index(self): req = webob.Request.blank('/v1.0/images') - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) res_dict = json.loads(res.body) fixture_index = [dict(id=f['id'], name=f['name']) for f @@ -215,7 +215,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def test_get_image_details(self): req = webob.Request.blank('/v1.0/images/detail') - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) res_dict = json.loads(res.body) for image in res_dict['images']: diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 785fb6f3a..8cfc6c45a 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -69,14 +69,14 @@ class ServersTest(unittest.TestCase): def test_get_server_by_id(self): req = webob.Request.blank('/v1.0/servers/1') - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], 1) self.assertEqual(res_dict['server']['name'], 'server1') def test_get_server_list(self): req = webob.Request.blank('/v1.0/servers') - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) res_dict = json.loads(res.body) i = 0 @@ -119,14 +119,14 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.body = json.dumps(body) - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) self.assertEqual(res.status_int, 200) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) self.assertEqual(res.status_int, 422) def test_update_bad_params(self): @@ -145,7 +145,7 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body - req.get_response(nova.api.API()) + req.get_response(nova.api.API('os')) def test_update_server(self): inst_dict = dict(name='server_test', adminPass='bacon') @@ -161,28 +161,28 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body - req.get_response(nova.api.API()) + req.get_response(nova.api.API('os')) def test_create_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'POST' - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) self.assertEqual(res.status, '404 Not Found') def test_delete_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'DELETE' - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) self.assertEqual(res.status, '404 Not Found') def test_get_server_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) self.assertEqual(res.status, '404 Not Found') def test_get_all_server_details(self): req = webob.Request.blank('/v1.0/servers/detail') - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) res_dict = json.loads(res.body) i = 0 @@ -200,7 +200,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) def test_server_rebuild(self): body = dict(server=dict( @@ -210,7 +210,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) def test_server_resize(self): body = dict(server=dict( @@ -220,7 +220,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) def test_delete_server_instance(self): req = webob.Request.blank('/v1.0/servers/1') @@ -234,7 +234,7 @@ class ServersTest(unittest.TestCase): self.stubs.Set(nova.db.api, 'instance_destroy', instance_destroy_mock) - res = req.get_response(nova.api.API()) + res = req.get_response(nova.api.API('os')) self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 0a81c575b..33d4cb294 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -34,10 +34,6 @@ from nova.api.ec2 import apirequest from nova.auth import manager -FLAGS = flags.FLAGS -FLAGS.FAKE_subdomain = 'ec2' - - class FakeHttplibSocket(object): """a fake socket implementation for httplib.HTTPResponse, trivial""" def __init__(self, response_string): @@ -109,7 +105,7 @@ class ApiEc2TestCase(test.TrialTestCase): self.host = '127.0.0.1' - self.app = api.API() + self.app = api.API('ec2') def expect_http(self, host=None, is_secure=False): """Returns a new EC2 connection""" -- cgit From 67d2d35e944b10f6f1e2e6eeb0a8b33496d4d39b Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 2 Nov 2010 13:51:09 -0700 Subject: Fixed --help display for non-twisted bin/* commands. --- nova/flags.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index f3b0384ad..e51f286ad 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -138,6 +138,8 @@ class FlagValues(gflags.FlagValues): FLAGS = FlagValues() +gflags.FLAGS = FLAGS +gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) def _wrapper(func): -- cgit From e50b8627ccac7f051d4ebea8879bd8d5c083837c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 2 Nov 2010 13:57:50 -0700 Subject: cloudpipe docs --- doc/source/cloudpipe.rst | 77 ++++++++++++++++++++++++++++++++ doc/source/concepts.and.introduction.rst | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 doc/source/cloudpipe.rst diff --git a/doc/source/cloudpipe.rst b/doc/source/cloudpipe.rst new file mode 100644 index 000000000..952a75351 --- /dev/null +++ b/doc/source/cloudpipe.rst @@ -0,0 +1,77 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +.. _cloudpipe: + +Cloudpipe +========= + +Cloudpipe is a method for connecting end users to their project insnances in vlan mode. + +Overview +-------- + +The support code for cloudpipe implements admin commands (via nova-manage) to automatically create a vm for a project that allows users to vpn into the private network of their project. Access to this vpn is provided through a public port on the network host for the project. This allows users to have free access to the virtual machines in their project without exposing those machines to the public internet. + +Cloudpipe Image +--------------- + +The cloudpipe image is basically just a linux instance with openvpn installed. It needs a simple script to grab user data from the metadata server, b64 decode it into a zip file, and run the autorun.sh script from inside the zip. The autorun script will configure and run openvpn to run using the data from nova. + +It is also useful to have a cron script that will periodically redownload the metadata and copy the new crl. This will keep revoked users from connecting and will disconnect any users that are connected with revoked certificates when their connection is renegotiated (every hour). + +Cloudpipe Launch +---------------- + +When you use nova-manage to launch a cloudpipe for a user, it goes through the following process: + +#. creates a keypair called -vpn and saves it in the keys directory +#. creates a security group -vpn and opens up 1194 and icmp +#. creates a cert and private key for the vpn instance and saves it in the CA/projects// directory +#. zips up the info and puts it b64 encoded as user data +#. launches an m1.tiny instance with the above settings using the flag-specified vpn image + +Vpn Access +---------- + +In vlan networking mode, the second ip in each private network is reserved for the cloudpipe instance. This gives a consistent ip to the instance so that nova-network can create forwarding rules for access from the outside world. The network for each project is given a specific high-numbered port on the public ip of the network host. This port is automatically forwarded to 1194 on the vpn instance. + +If specific high numbered ports do not work for your users, you can always allocate and associate a public ip to the instance, and then change the vpn_public_ip and vpn_public_port in the database. This will be turned into a nova-manage command or a flag soon. + + +Certificates and Revocation +--------------------------- + +If the use_project_ca flag is set (required to for cloudpipes to work securely), then each project has its own ca. This ca is used to sign the certificate for the vpn, and is also passed to the user for bundling images. When a certificate is revoked using nova-manage, a new Certificate Revocation List (crl) is generated. As long as cloudpipe has an updated crl, it will block revoked users from connecting to the vpn. + +The :mod:`cloudpipe` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.cloudpipe.pipelib + :members: + :undoc-members: + :show-inheritance: + +The :mod:`crypto` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.crypto + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 29b8f064a..d2ab6f14b 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -129,7 +129,7 @@ This is similar to the flat mode, in that all instances are attached to the same VLAN DHCP Mode ~~~~~~~~~~~~~~ -This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code name cloudpipe) needs to be created. Nova generates a certificate and key for the userto access the vpn and starts the vpn automatically. +This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code named :ref:`cloudpipe `) needs to be created. Nova generates a certificate and key for the userto access the vpn and starts the vpn automatically. More information on cloudpipe can be found :ref:`here `. The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) -- cgit From e493e324eb9a9fe31e72551b34bab768b507bc1d Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 2 Nov 2010 18:05:47 -0400 Subject: Document final undocumented python modules. --- doc/source/api.rst | 108 +++++++++++++++++++++++++++++++++++++++ doc/source/cloudpipe.rst | 27 ++++++++++ doc/source/compute.rst | 8 ++- doc/source/development.guide.rst | 3 ++ doc/source/glance.rst | 27 ++++++++++ doc/source/network.rst | 11 ++++ doc/source/nova.rst | 83 ++++++++++++++++++++++++++++++ doc/source/objectstore.rst | 62 ++++++++++++++++++++++ doc/source/scheduler.rst | 11 ++++ nova/image/service.py | 34 +++++------- 10 files changed, 353 insertions(+), 21 deletions(-) create mode 100644 doc/source/cloudpipe.rst create mode 100644 doc/source/glance.rst create mode 100644 doc/source/objectstore.rst diff --git a/doc/source/api.rst b/doc/source/api.rst index 3f1c69a15..0908d9e57 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -161,3 +161,111 @@ The :mod:`metadatarequesthandler` Module :members: :undoc-members: :show-inheritance: + +Tests +----- + +The :mod:`api_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api_integration` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api_integration + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.cloud_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.fakes` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.fakes + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.test_wsgi` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.test_wsgi + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_api` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_api + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_auth` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_auth + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_faults` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_faults + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_flavors` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_flavors + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_images` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_images + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_ratelimiting` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_ratelimiting + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_servers` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_servers + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_sharedipgroups` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_sharedipgroups + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/cloudpipe.rst b/doc/source/cloudpipe.rst new file mode 100644 index 000000000..f2e7cb85a --- /dev/null +++ b/doc/source/cloudpipe.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Cloudpipe Programmer Guide +========================== + +The :mod:`pipelib` Module +------------------------- + +.. automodule:: nova.cloudpipe.pipelib + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/compute.rst b/doc/source/compute.rst index e9e37ebf8..ff6aeab92 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -135,7 +135,6 @@ The :mod:`monitor` Module Tests ----- - The :mod:`compute_unittest` Module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -144,3 +143,10 @@ The :mod:`compute_unittest` Module :undoc-members: :show-inheritance: +The :mod:`virt_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.virt_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 261059d9b..306decccb 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -40,6 +40,9 @@ Contents scheduler fakes nova + cloudpipe + objectstore + glance Indices and tables diff --git a/doc/source/glance.rst b/doc/source/glance.rst new file mode 100644 index 000000000..3d0f4ebec --- /dev/null +++ b/doc/source/glance.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Glance Integration Programming Guide +==================================== + +The :mod:`image.service` Module +------------------------------- + +.. automodule:: nova.image.service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/network.rst b/doc/source/network.rst index b044c6413..cbff22d3f 100644 --- a/doc/source/network.rst +++ b/doc/source/network.rst @@ -43,6 +43,17 @@ The :mod:`LinuxNet` driver :undoc-members: :show-inheritance: +Tests +----- + +The :mod:`network_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.network_unittest + :members: + :undoc-members: + :show-inheritance: + legacy docs ----------- diff --git a/doc/source/nova.rst b/doc/source/nova.rst index 59fd56c85..17d32c010 100644 --- a/doc/source/nova.rst +++ b/doc/source/nova.rst @@ -145,3 +145,86 @@ The :mod:`wsgi` Module :members: :undoc-members: :show-inheritance: + +Tests +----- + +The :mod:`declare_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.declare_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`fake_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.fake_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`flags_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.flags_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`process_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.process_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`quota_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.quota_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`real_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.real_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`rpc_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.rpc_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`runtime_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.runtime_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`twistd_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.twistd_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`validator_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.validator_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/objectstore.rst b/doc/source/objectstore.rst new file mode 100644 index 000000000..4087b5dd0 --- /dev/null +++ b/doc/source/objectstore.rst @@ -0,0 +1,62 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Objectstore Programming Guide +============================= + +The :mod:`handler` Module +------------------------- + +.. automodule:: nova.objectstore.handler + :members: + :undoc-members: + :show-inheritance: + +The :mod:`bucket` Module +------------------------ + +.. automodule:: nova.objectstore.bucket + :members: + :undoc-members: + :show-inheritance: + +The :mod:`stored` Module +------------------------ + +.. automodule:: nova.objectstore.stored + :members: + :undoc-members: + :show-inheritance: + +The :mod:`image` Module +----------------------- + +.. automodule:: nova.objectstore.image + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`objectstore_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.objectstore_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/scheduler.rst b/doc/source/scheduler.rst index e809b0c54..df820c2f2 100644 --- a/doc/source/scheduler.rst +++ b/doc/source/scheduler.rst @@ -49,3 +49,14 @@ The :mod:`simple` Module :members: :undoc-members: :show-inheritance: + +Tests +----- + +The :mod:`scheduler_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.scheduler_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/nova/image/service.py b/nova/image/service.py index 37cadddcc..52ddd4e0f 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -45,13 +45,9 @@ class BaseImageService(object): Returns a sequence of mappings of id and name information about images. - :retval a sequence of mappings with the following signature: - - [ - {'id': opaque id of image, - 'name': name of image - }, ... - ] + :rtype: array + :retval: a sequence of mappings with the following signature + {'id': opaque id of image, 'name': name of image} """ raise NotImplementedError @@ -60,19 +56,17 @@ class BaseImageService(object): """ Returns a sequence of mappings of detailed information about images. - :retval a sequence of mappings with the following signature: - - [ - {'id': opaque id of image, - 'name': name of image, - 'created_at': creation timestamp, - 'updated_at': modification timestamp, - 'deleted_at': deletion timestamp or None, - 'deleted': boolean indicating if image has been deleted, - 'status': string description of image status, - 'is_public': boolean indicating if image is public - }, ... - ] + :rtype: array + :retval: a sequence of mappings with the following signature + {'id': opaque id of image, + 'name': name of image, + 'created_at': creation timestamp, + 'updated_at': modification timestamp, + 'deleted_at': deletion timestamp or None, + 'deleted': boolean indicating if image has been deleted, + 'status': string description of image status, + 'is_public': boolean indicating if image is public + } If the service does not implement a method that provides a detailed set of information about images, then the method should raise -- cgit From a6f867eb72599bb0bb1fe43f99d8c4b07972e5dd Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 2 Nov 2010 15:16:41 -0700 Subject: Documentation on Services, Managers, and Drivers --- doc/source/services.rst | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/doc/source/services.rst b/doc/source/services.rst index 607f5974b..517a51901 100644 --- a/doc/source/services.rst +++ b/doc/source/services.rst @@ -20,13 +20,53 @@ Services Managers and Drivers ============================= -Describe division of responsibilities for each component. +The responsibilities of Services, Managers, and Drivers, can be a bit confusing to people that are new to nova. This document attempts to outline the division of responsibilities to make understanding the system a little bit easier. + +Currently, Managers and Drivers are specified by flags and loaded using utils.load_object(). This method allows for them to be implemented as singletons, classes, modules or objects. As long as the path specified by the flag leads to an object (or a callable that returns an object) that responds to getattr, it should work as a manager or driver. Service ------- +A service is a very thin wrapper around a Manager object. It exposes the manager's public methods to other components of the system via rpc. It will report state periodically to the database and is responsible for initiating any periodic tasts that need to be executed on a given host. + +The :mod:`service` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.service + :members: + :undoc-members: + :show-inheritance: + Manager ------- +Managers are responsible for a certain aspect of the sytem. It is a logical grouping of code relating to a portion of the system. In general other components should be using the manager to make changes to the components that it is responsible for. + +For example, other components that need to deal with volumes in some way, should do so by calling methods on the VolumeManager instead of directly changing fields in the database. This allows us to keep all of the code relating to volumes in the same place. + +We have adopted a basic strategy of Smart managers and dumb data, which means rather than attaching methods to data objects, components should call manager methods that act on the data. + +Methods on managers that can be executed locally should be called directly. If a particular method must execute on a remote host, this should be done via rpc to the service that wraps the manager + +Managers should be responsible for most of the db access, and non-implementation specific data. Anything implementation specific that can't be generalized should be done by the Driver. + +In general, we prefer to have one manager with multiple drivers for different implementations, but sometimes it makes sense to have multiple managers. You can think of it this way: Abstract different overall strategies at the manager level(FlatNetwork vs VlanNetwork), and different implementations at the driver level(LinuxNetDriver vs CiscoNetDriver). + +Managers will often provide methods for initial setup of a host or periodic tasksto a wrapping service. + +The :mod:`manager` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.manager + :members: + :undoc-members: + :show-inheritance: + Driver ------ + +A manager will generally load a driver for some of its tasks. The driver is responsible for specific implementation details. Anything running shell commands on a host, or dealing with other non-python code should probably be happening in a driver. + +Drivers should minimize touching the database, although it is currently acceptable for implementation specific data. This may be reconsidered at some point. + +It usually makes sense to define an Abstract Base Class for the specific driver (i.e. VolumeDriver), to define the methods that a different driver would need to implement. -- cgit From 7898451cd91821e01451ff8f65de1169eeef8ae2 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 2 Nov 2010 16:49:34 -0700 Subject: removed some old instructions and updated concepts --- doc/source/concepts.and.introduction.rst | 18 ++++--------- doc/source/getting.started.rst | 44 +++++--------------------------- 2 files changed, 11 insertions(+), 51 deletions(-) diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index d2ab6f14b..d73447ba9 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -71,19 +71,13 @@ Concept: Storage Volumes ~~~~~~~ -A 'volume' is a detachable block storage device. You can think of it as a usb hard drive. It can only be attached to one instance at a time, and it behaves -Ephemeral -~~~~~~~~~ -:: - - TODO(vish): document +A 'volume' is a detachable block storage device. You can think of it as a usb hard drive. It can only be attached to one instance at a time, so it does not work like a SAN. If you wish to expose the same volume to multiple instances, you will have to use an NFS or SAMBA share from an existing instance. -Swift -~~~~~ -:: +Local Storage +~~~~~~~~~~~~~ - TODO(vish): document +Every instance larger than m1.tiny starts with some local storage (up to 160GB for m1.xlarge). This storage is currently the second partition on the root drive. Concept: Quotas --------------- @@ -308,9 +302,7 @@ Security groups Concept: Certificate Authority ------------------------------ -Per-project CA -* Images -* VPNs +Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns ` and decrypting bundled images. Concept: Images diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst index 2df4a45ea..70f26038f 100644 --- a/doc/source/getting.started.rst +++ b/doc/source/getting.started.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -18,7 +18,7 @@ Getting Started with Nova ========================= -This code base is continually changing so dependencies also change. +This code base is continually changing so dependencies also change. Dependencies ------------ @@ -30,7 +30,7 @@ Related servers we rely on Optional servers * OpenLDAP: By default, the auth server uses the RDBMS-backed datastore by setting FLAGS.auth_driver to 'nova.auth.dbdriver.DbDriver'. But OpenLDAP (or LDAP) could be configured. -* ReDIS: By default, this is not enabled as the auth driver. +* ReDIS: By default, this is not enabled as the auth driver. Python libraries we don't vendor @@ -62,49 +62,17 @@ Configuration These instructions are incomplete, but we are actively updating the `OpenStack wiki `_ with more configuration information. -On the cloud controller - -* Add yourself to the libvirtd group, log out, and log back in -* Fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances - -:: - - iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773 - iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE - - -* Configure NginX proxy (/etc/nginx/sites-enabled/default) - -:: - - server { - listen 3333 default; - server-name localhost; - client_max_body_size 10m; - - access_log /var/log/nginx/localhost.access.log; - - location ~ /_images/.+ { - root NOVA_PATH/images; - rewrite ^/_images/(.*)$ /$1 break; - } - - location / { - proxy_pass http://localhost:3334/; - } - } - On the volume node -* Create a filesystem (you can use an actual disk if you have one spare, default is /dev/sdb) +* Create a volume group (you can use an actual disk for the volume group as well) :: # This creates a 1GB file to create volumes out of dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 losetup --show -f MY_FILE_PATH - # replace loop0 below with whatever losetup returns - echo "--storage_dev=/dev/loop0" >> NOVA_PATH/bin/nova.conf + # replace /dev/loop0 below with whatever losetup returns + vgcreate nova-volumes /dev/loop0 Running --------- -- cgit From 583d1b1c4d039f1f9751c8a2cc0cf59bb77551e0 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 2 Nov 2010 20:31:17 -0400 Subject: Fixes after trunk merge. --- doc/source/nova.rst | 24 ------------------------ nova/tests/fake_flags.py | 1 + 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/doc/source/nova.rst b/doc/source/nova.rst index 17d32c010..3963992ae 100644 --- a/doc/source/nova.rst +++ b/doc/source/nova.rst @@ -34,14 +34,6 @@ The :mod:`context` Module :undoc-members: :show-inheritance: -The :mod:`crypto` Module ------------------------- - -.. automodule:: nova.crypto - :members: - :undoc-members: - :show-inheritance: - The :mod:`exception` Module --------------------------- @@ -58,14 +50,6 @@ The :mod:`flags` Module :undoc-members: :show-inheritance: -The :mod:`manager` Module -------------------------- - -.. automodule:: nova.manager - :members: - :undoc-members: - :show-inheritance: - The :mod:`process` Module ------------------------- @@ -98,14 +82,6 @@ The :mod:`server` Module :undoc-members: :show-inheritance: -The :mod:`service` Module -------------------------- - -.. automodule:: nova.service - :members: - :undoc-members: - :show-inheritance: - The :mod:`test` Module ---------------------- diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 4bbef8832..bc377f0da 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -24,6 +24,7 @@ flags.DECLARE('volume_driver', 'nova.volume.manager') FLAGS.volume_driver = 'nova.volume.driver.FakeAOEDriver' FLAGS.connection_type = 'fake' FLAGS.fake_rabbit = True +flags.DECLARE('auth_driver', 'nova.auth.manager') FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') -- cgit From ea79ca7e8855cf130cadb63fc42780c90a51fb15 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 2 Nov 2010 17:36:03 -0700 Subject: update of the architecture and fix some links --- doc/source/administration.guide.rst | 3 +-- doc/source/architecture.rst | 46 +++++++++++++++++++------------------ doc/source/development.guide.rst | 8 ++++--- doc/source/getting.started.rst | 38 ++++++++++++++++++++---------- 4 files changed, 56 insertions(+), 39 deletions(-) diff --git a/doc/source/administration.guide.rst b/doc/source/administration.guide.rst index 789e97381..8c36d3d44 100644 --- a/doc/source/administration.guide.rst +++ b/doc/source/administration.guide.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -30,6 +30,5 @@ Contents quickstart getting.started multi.node.install - services flags monitoring diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst index eba6cbfb8..1e23e1361 100644 --- a/doc/source/architecture.rst +++ b/doc/source/architecture.rst @@ -15,36 +15,38 @@ License for the specific language governing permissions and limitations under the License. -nova System Architecture +Nova System Architecture ======================== Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received. -In order to achieve shared-nothing with multiple copies of the same component (especially when the component is an API server that needs to reply with state information in a timely fashion), we need to keep all of our system state in a distributed data system. Updates to system state are written into this system, using atomic transactions when necessary. Requests for state are read out of this system. In limited cases, these read calls are memoized within controllers for short periods of time. (Such a limited case would be, for instance, the current list of system users.) - +Nova recently moved to using a sql-based central database that is shared by all components in the system. The amount and depth of the data fits into a sql database quite well. For small deployments this seems like an optimal solution. For larger deployments, and especially if security is a concern, nova will be moving towards multiple data stores with some kind of aggregation system. Components ---------- -Below you will find a helpful explanation. +Below you will find a helpful explanation of the different components. :: - [ Auth Manager ] ---- ( LDAP ) - | - | - | - | / [ Storage ] - ( ATAoE/iSCSI ) - [ Web Dashboard ] -> [ nova-api ] < AMQP > - - | \ [ Nodes ] - ( libvirt/kvm ) - < HTTP > - | - [ nova-objectstore ] - - -* API: receives http requests from boto, converts commands to/from API format, and sending requests to cloud controller -* Cloud Controller: global state of system, talks to ldap, s3, and node/storage workers through a queue -* Nodes: worker that spawns instances -* S3: tornado based http/s3 server -* User Manager: create/manage users, which are stored in ldap -* Network Controller: allocate and deallocate IPs and VLANs + /- ( LDAP ) + [ Auth Manager ] --- + | \- ( DB ) + | + | [ scheduler ] - [ volume ] - ( ATAoE/iSCSI ) + | / + [ Web Dashboard ] -> [ api ] -- < AMQP > ------ [ network ] - ( Flat/Vlan ) + | \ + < HTTP > [ scheduler ] - [ compute ] - ( libvirt/xen ) + | | + [ objectstore ] < - retrieves images + +* DB: sql database for data storage. Used by all components (LINKS NOT SHOWN) +* Web Dashboard: potential external component that talks to the api +* api: component that receives http requests, converts commands and communicates with other components via the queue or http (in the case of objectstore) +* Auth Manager: component responsible for users/projects/and roles. Can backend to DB or LDAP. This is not a separate binary, but rather a python class that is used by most components in the system. +* objectstore: twisted http server that replicates s3 api and allows storage and retrieval of images +* scheduler: decides which host gets each vm and volume +* volume: manages dynamically attachable block devices. +* network: manages ip forwarding, bridges, and vlans +* compute: manages communication with hypervisor and virtual machines. diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index 261059d9b..45ebe2f10 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -31,11 +31,13 @@ Contents .. toctree:: :maxdepth: 1 + architecture + services database volume compute - network - auth + network + auth api scheduler fakes diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst index 70f26038f..a0148e4b6 100644 --- a/doc/source/getting.started.rst +++ b/doc/source/getting.started.rst @@ -29,24 +29,37 @@ Related servers we rely on Optional servers -* OpenLDAP: By default, the auth server uses the RDBMS-backed datastore by setting FLAGS.auth_driver to 'nova.auth.dbdriver.DbDriver'. But OpenLDAP (or LDAP) could be configured. -* ReDIS: By default, this is not enabled as the auth driver. +* OpenLDAP: By default, the auth server uses the RDBMS-backed datastore by setting FLAGS.auth_driver to 'nova.auth.dbdriver.DbDriver'. But OpenLDAP (or LDAP) could be configured by specifying 'nova.auth.ldapdriver.LdapDriver'. There is a script in the sources(nova/auth/slap.sh) to install a very basic openldap server on ubuntu. +* ReDIS: There is a fake ldap driver that backends to redis. This was created for testing ldap implementation on systems that don't have an easy means to install ldap. -Python libraries we don't vendor +Python libraries that we use (from pip-requires): + +.. literalinclude:: ../../tools/pip-requires + +Other libraries: -* M2Crypto: python library interface for openssl -* curl * XenAPI: Needed only for Xen Cloud Platform or XenServer support. Available from http://wiki.xensource.com/xenwiki/XCP_SDK or http://community.citrix.com/cdn/xs/sdks. -Vendored python libaries (don't require any installation) +External unix tools that are required: + +* iptables +* ebtables +* gawk +* curl +* kvm +* libvirt +* dnsmasq +* vlan +* open-iscsi and iscsitarget (if you use iscsi volumes) +* aoetools and vblade-persist (if you use aoe-volumes) + +Nova uses cutting-edge versions of many packages. There are ubuntu packages in the nova-core ppa. You can use add this ppa to your sources list on an ubuntu machine with the following commands:: -* Twisted: just for the twisted.internet.defer package -* Tornado: scalable non blocking web server for api requests -* boto: python api for aws api -* IPy: library for managing ip addresses + sudo apt-get install -y python-software-properties + sudo add-apt-repository ppa:nova-core/ppa Recommended ------------------ +----------- * euca2ools: python implementation of aws ec2-tools and ami tools * build tornado to use C module for evented section @@ -55,7 +68,7 @@ Recommended Installation -------------- - Due to many changes it's best to rely on the `OpenStack wiki `_ for installation instructions. +Due to many changes it's best to rely on the `OpenStack wiki `_ for installation instructions. Configuration --------------- @@ -88,3 +101,4 @@ Launch nova components * nova-compute * nova-objectstore * nova-volume +* nova-scheduler -- cgit From 1119f7e0eeffc0bc2f918b0c5fb8eb87cd3e3784 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 2 Nov 2010 17:37:11 -0700 Subject: more descriptive title for cloudpipe --- doc/source/cloudpipe.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/cloudpipe.rst b/doc/source/cloudpipe.rst index 952a75351..d632a40e6 100644 --- a/doc/source/cloudpipe.rst +++ b/doc/source/cloudpipe.rst @@ -18,8 +18,8 @@ .. _cloudpipe: -Cloudpipe -========= +Cloudpipe -- Per Project Vpns +============================= Cloudpipe is a method for connecting end users to their project insnances in vlan mode. -- cgit From 2a16ae2f8479e469e413dfd036bac805163f3ac2 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 3 Nov 2010 15:05:10 +0100 Subject: Change socket type in nova.utils.get_my_ip() to SOCK_DGRAM. This way, we don't actually have to set up a connection. Also, change the destination host to an IP (chose one of Google's DNS's at random) rather than a hostname, so we avoid doing a DNS lookup. --- nova/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index bc495a691..e7892a212 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -154,8 +154,8 @@ def get_my_ip(): if getattr(FLAGS, 'fake_tests', None): return '127.0.0.1' try: - csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - csock.connect(('www.google.com', 80)) + csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + csock.connect(('8.8.8.8', 80)) (addr, port) = csock.getsockname() csock.close() return addr -- cgit From 2cbef8ffd80546f1dcd850322621b04395591d69 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 3 Nov 2010 14:30:13 -0400 Subject: Doc cleanups. --- doc/source/auth.rst | 21 ++++--- doc/source/compute.rst | 6 -- doc/source/development.guide.rst | 10 +++- doc/source/network.rst | 8 +-- doc/source/nova.rst | 16 ------ nova/network/manager.py | 116 +++++++++++++++++++++++---------------- 6 files changed, 93 insertions(+), 84 deletions(-) diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 61db39ec3..77d97f68b 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -20,12 +20,13 @@ Authentication and Authorization Programming Guide ================================================== -:: - - TOOD(todd): * Authorizztion - * RBAC - * Quotas +The :mod:`quota` Module +----------------------- +.. automodule:: nova.quota + :members: + :undoc-members: + :show-inheritance: Auth Manager ------------ @@ -68,7 +69,6 @@ The :mod:`signer` Module :undoc-members: :show-inheritance: - Related Tests ------------- @@ -81,13 +81,20 @@ The :mod:`auth_unittest` Module :show-inheritance: The :mod:`access_unittest` Module ---------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.tests.access_unittest :members: :undoc-members: :show-inheritance: +The :mod:`quota_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.quota_unittest + :members: + :undoc-members: + :show-inheritance: legacy docs ----------- diff --git a/doc/source/compute.rst b/doc/source/compute.rst index ff6aeab92..e4c6c6ae7 100644 --- a/doc/source/compute.rst +++ b/doc/source/compute.rst @@ -21,12 +21,6 @@ Virtualization Programming Guide This page contains the Compute Package documentation. - -:: - - TODO(todd): * document instance_types and power_states - - Manager ------- diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst index e4af2da56..704cbe399 100644 --- a/doc/source/development.guide.rst +++ b/doc/source/development.guide.rst @@ -20,9 +20,14 @@ Nova Development Guide Nova is written in python. -:: +Prerequisites +------------- +.. toctree:: + :maxdepth: 1 - TODO(todd): * Building packages + concepts.and.introduction + architecture + development.environment Contents @@ -31,7 +36,6 @@ Contents .. toctree:: :maxdepth: 1 - architecture services database volume diff --git a/doc/source/network.rst b/doc/source/network.rst index cbff22d3f..9426ece82 100644 --- a/doc/source/network.rst +++ b/doc/source/network.rst @@ -20,12 +20,8 @@ Networking Programming Guide :: - TODO(todd): - * document how managers work - * document backend (linux-net) - * document hardware specific commands - * pep 257 - * document flags + TODO(todd): * document hardware specific commands (maybe in admin guide?) + * document a map between flags and managers/backends The :mod:`Manager` module ------------------------- diff --git a/doc/source/nova.rst b/doc/source/nova.rst index 3963992ae..58125dc80 100644 --- a/doc/source/nova.rst +++ b/doc/source/nova.rst @@ -58,14 +58,6 @@ The :mod:`process` Module :undoc-members: :show-inheritance: -The :mod:`quota` Module ------------------------ - -.. automodule:: nova.quota - :members: - :undoc-members: - :show-inheritance: - The :mod:`rpc` Module --------------------- @@ -157,14 +149,6 @@ The :mod:`process_unittest` Module :undoc-members: :show-inheritance: -The :mod:`quota_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.quota_unittest - :members: - :undoc-members: - :show-inheritance: - The :mod:`real_flags` Module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/nova/network/manager.py b/nova/network/manager.py index 8a20cb491..b033bb0a4 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -17,7 +17,30 @@ # under the License. """ -Network Hosts are responsible for allocating ips and setting up network +Network Hosts are responsible for allocating ips and setting up network. + +There are multiple backend drivers that handle specific types of networking +topologies. All of the network commands are issued to a subclass of +:class:`NetworkManager`. + +**Related Flags** + +:network_driver: Driver to use for network creation +:flat_network_bridge: Bridge device for simple network instances +:flat_network_dns: Dns for simple network +:flat_network_dhcp_start: Dhcp start for FlatDhcp +:vlan_start: First VLAN for private networks +:vpn_ip: Public IP for the cloudpipe VPN servers +:vpn_start: First Vpn port for private networks +:cnt_vpn_clients: Number of addresses reserved for vpn clients +:network_size: Number of addresses in each private subnet +:floating_range: Floating IP address block +:fixed_range: Fixed IP address block +:date_dhcp_on_disassociate: Whether to update dhcp when fixed_ip + is disassociated +:fixed_ip_disassociate_timeout: Seconds after which a deallocated ip + is disassociated + """ import datetime @@ -63,15 +86,16 @@ flags.DEFINE_integer('fixed_ip_disassociate_timeout', 600, class AddressAlreadyAllocated(exception.Error): - """Address was already allocated""" + """Address was already allocated.""" pass class NetworkManager(manager.Manager): - """Implements common network manager functionality + """Implements common network manager functionality. - This class must be subclassed. + This class must be subclassed to support specific topologies. """ + def __init__(self, network_driver=None, *args, **kwargs): if not network_driver: network_driver = FLAGS.network_driver @@ -86,7 +110,7 @@ class NetworkManager(manager.Manager): self._on_set_network_host(ctxt, network['id']) def set_network_host(self, context, network_id): - """Safely sets the host of the network""" + """Safely sets the host of the network.""" logging.debug("setting network host") host = self.db.network_set_host(context, network_id, @@ -95,34 +119,34 @@ class NetworkManager(manager.Manager): return host def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool""" + """Gets a fixed ip from the pool.""" raise NotImplementedError() def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" raise NotImplementedError() def setup_fixed_ip(self, context, address): - """Sets up rules for fixed ip""" + """Sets up rules for fixed ip.""" raise NotImplementedError() def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a network""" + """Called when this host becomes the host for a network.""" raise NotImplementedError() def setup_compute_network(self, context, instance_id): - """Sets up matching network for compute hosts""" + """Sets up matching network for compute hosts.""" raise NotImplementedError() def allocate_floating_ip(self, context, project_id): - """Gets an floating ip from the pool""" + """Gets an floating ip from the pool.""" # TODO(vish): add floating ips through manage command return self.db.floating_ip_allocate_address(context, self.host, project_id) def associate_floating_ip(self, context, floating_address, fixed_address): - """Associates an floating ip to a fixed ip""" + """Associates an floating ip to a fixed ip.""" self.db.floating_ip_fixed_ip_associate(context, floating_address, fixed_address) @@ -130,18 +154,18 @@ class NetworkManager(manager.Manager): self.driver.ensure_floating_forward(floating_address, fixed_address) def disassociate_floating_ip(self, context, floating_address): - """Disassociates a floating ip""" + """Disassociates a floating ip.""" fixed_address = self.db.floating_ip_disassociate(context, floating_address) self.driver.unbind_floating_ip(floating_address) self.driver.remove_floating_forward(floating_address, fixed_address) def deallocate_floating_ip(self, context, floating_address): - """Returns an floating ip to the pool""" + """Returns an floating ip to the pool.""" self.db.floating_ip_deallocate(context, floating_address) def lease_fixed_ip(self, context, mac, address): - """Called by dhcp-bridge when ip is leased""" + """Called by dhcp-bridge when ip is leased.""" logging.debug("Leasing IP %s", address) fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) instance_ref = fixed_ip_ref['instance'] @@ -158,7 +182,7 @@ class NetworkManager(manager.Manager): logging.warn("IP %s leased that was already deallocated", address) def release_fixed_ip(self, context, mac, address): - """Called by dhcp-bridge when ip is released""" + """Called by dhcp-bridge when ip is released.""" logging.debug("Releasing IP %s", address) fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) instance_ref = fixed_ip_ref['instance'] @@ -183,26 +207,26 @@ class NetworkManager(manager.Manager): self.driver.update_dhcp(context, network_ref['id']) def get_network(self, context): - """Get the network for the current context""" + """Get the network for the current context.""" raise NotImplementedError() def create_networks(self, context, num_networks, network_size, *args, **kwargs): - """Create networks based on parameters""" + """Create networks based on parameters.""" raise NotImplementedError() @property def _bottom_reserved_ips(self): # pylint: disable-msg=R0201 - """Number of reserved ips at the bottom of the range""" + """Number of reserved ips at the bottom of the range.""" return 2 # network, gateway @property def _top_reserved_ips(self): # pylint: disable-msg=R0201 - """Number of reserved ips at the top of the range""" + """Number of reserved ips at the top of the range.""" return 1 # broadcast def _create_fixed_ips(self, context, network_id): - """Create all fixed ips for network""" + """Create all fixed ips for network.""" network_ref = self.db.network_get(context, network_id) # NOTE(vish): Should these be properties of the network as opposed # to properties of the manager class? @@ -222,10 +246,10 @@ class NetworkManager(manager.Manager): class FlatManager(NetworkManager): - """Basic network where no vlans are used""" + """Basic network where no vlans are used.""" def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool""" + """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute # with a network, or a cluster of computes with a network # and use that network here with a method like @@ -239,21 +263,21 @@ class FlatManager(NetworkManager): return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" self.db.fixed_ip_update(context, address, {'allocated': False}) self.db.fixed_ip_disassociate(context.elevated(), address) def setup_compute_network(self, context, instance_id): - """Network is created manually""" + """Network is created manually.""" pass def setup_fixed_ip(self, context, address): - """Currently no setup""" + """Currently no setup.""" pass def create_networks(self, context, cidr, num_networks, network_size, *args, **kwargs): - """Create networks based on parameters""" + """Create networks based on parameters.""" fixed_net = IPy.IP(cidr) for index in range(num_networks): start = index * network_size @@ -271,7 +295,7 @@ class FlatManager(NetworkManager): self._create_fixed_ips(context, network_ref['id']) def get_network(self, context): - """Get the network for the current context""" + """Get the network for the current context.""" # NOTE(vish): To support mutilple network hosts, This could randomly # select from multiple networks instead of just # returning the one. It could also potentially be done @@ -280,7 +304,7 @@ class FlatManager(NetworkManager): FLAGS.flat_network_bridge) def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a network""" + """Called when this host becomes the host for a network.""" net = {} net['injected'] = True net['bridge'] = FLAGS.flat_network_bridge @@ -289,19 +313,19 @@ class FlatManager(NetworkManager): class FlatDHCPManager(NetworkManager): - """Flat networking with dhcp""" + """Flat networking with dhcp.""" def setup_fixed_ip(self, context, address): - """Setup dhcp for this network""" + """Setup dhcp for this network.""" network_ref = db.fixed_ip_get_by_address(context, address) self.driver.update_dhcp(context, network_ref['id']) def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" self.db.fixed_ip_update(context, address, {'allocated': False}) def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a project""" + """Called when this host becomes the host for a project.""" super(FlatDHCPManager, self)._on_set_network_host(context, network_id) network_ref = self.db.network_get(context, network_id) self.db.network_update(context, @@ -313,11 +337,11 @@ class FlatDHCPManager(NetworkManager): class VlanManager(NetworkManager): - """Vlan network with dhcp""" + """Vlan network with dhcp.""" @defer.inlineCallbacks def periodic_tasks(self, context=None): - """Tasks to be run at a periodic interval""" + """Tasks to be run at a periodic interval.""" yield super(VlanManager, self).periodic_tasks(context) now = datetime.datetime.utcnow() timeout = FLAGS.fixed_ip_disassociate_timeout @@ -330,13 +354,13 @@ class VlanManager(NetworkManager): def init_host(self): """Do any initialization that needs to be run if this is a - standalone service. + standalone service. """ super(VlanManager, self).init_host() self.driver.init_host() def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool""" + """Gets a fixed ip from the pool.""" # TODO(vish): This should probably be getting project_id from # the instance, but it is another trip to the db. # Perhaps this method should take an instance_ref. @@ -356,11 +380,11 @@ class VlanManager(NetworkManager): return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" self.db.fixed_ip_update(context, address, {'allocated': False}) def setup_fixed_ip(self, context, address): - """Sets forwarding rules and dhcp for fixed ip""" + """Sets forwarding rules and dhcp for fixed ip.""" fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) network_ref = self.db.fixed_ip_get_network(context, address) if self.db.instance_is_vpn(context, fixed_ip_ref['instance_id']): @@ -370,19 +394,19 @@ class VlanManager(NetworkManager): self.driver.update_dhcp(context, network_ref['id']) def setup_compute_network(self, context, instance_id): - """Sets up matching network for compute hosts""" + """Sets up matching network for compute hosts.""" network_ref = db.network_get_by_instance(context, instance_id) self.driver.ensure_vlan_bridge(network_ref['vlan'], network_ref['bridge']) def restart_nets(self): - """Ensure the network for each user is enabled""" + """Ensure the network for each user is enabled.""" # TODO(vish): Implement this pass def create_networks(self, context, cidr, num_networks, network_size, vlan_start, vpn_start): - """Create networks based on parameters""" + """Create networks based on parameters.""" fixed_net = IPy.IP(cidr) for index in range(num_networks): vlan = vlan_start + index @@ -407,12 +431,12 @@ class VlanManager(NetworkManager): self._create_fixed_ips(context, network_ref['id']) def get_network(self, context): - """Get the network for the current context""" + """Get the network for the current context.""" return self.db.project_get_network(context.elevated(), context.project_id) def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a network""" + """Called when this host becomes the host for a network.""" network_ref = self.db.network_get(context, network_id) net = {} net['vpn_public_address'] = FLAGS.vpn_ip @@ -424,11 +448,11 @@ class VlanManager(NetworkManager): @property def _bottom_reserved_ips(self): - """Number of reserved ips at the bottom of the range""" + """Number of reserved ips at the bottom of the range.""" return super(VlanManager, self)._bottom_reserved_ips + 1 # vpn server @property def _top_reserved_ips(self): - """Number of reserved ips at the top of the range""" + """Number of reserved ips at the top of the range.""" parent_reserved = super(VlanManager, self)._top_reserved_ips return parent_reserved + FLAGS.cnt_vpn_clients -- cgit From 80a6dd72ef48ec4c7dc72b39073800bb7fe1e70a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 3 Nov 2010 14:04:36 -0700 Subject: prettier theme --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 27bc4ea6c..3f6ca2313 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -99,7 +99,7 @@ modindex_common_prefix = ['nova.'] # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'default' +html_theme = 'sphinxdoc' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the -- cgit From 179d980dffc03e1ee0449954eed40b1d5489f6e0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 3 Nov 2010 14:59:35 -0700 Subject: make sure context keys are not unicode so they can be passed as kwargs --- nova/rpc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/rpc.py b/nova/rpc.py index 895820cd0..05eaa0f99 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -262,6 +262,9 @@ def _unpack_context(msg): """Unpack context from msg.""" context_dict = {} for key in list(msg.keys()): + # NOTE(vish): Some versions of python don't like unicode keys + # in kwargs. + key = str(key) if key.startswith('_context_'): value = msg.pop(key) context_dict[key[9:]] = value -- cgit From 817690b03f2e498fb08eba3ca455719229f24640 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 3 Nov 2010 15:06:00 -0700 Subject: pep8 whitespace and line length fixes --- nova/adminclient.py | 4 ++-- nova/compute/manager.py | 3 ++- nova/db/api.py | 4 ++-- nova/db/sqlalchemy/models.py | 4 ++-- nova/volume/driver.py | 4 ++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index 0227cddd7..af55197fc 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -273,10 +273,10 @@ class NovaAdminClient(object): def get_user_roles(self, user, project=None): """Returns a list of roles for the given user. - + Omitting project will return any global roles that the user has. Specifying project will return only project specific roles. - + """ params = {'User': user} if project: diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3aed1e5a5..890d79fba 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -22,7 +22,8 @@ Handles all processes relating to instances (guest vms). The :py:class:`ComputeManager` class is a :py:class:`nova.manager.Manager` that handles RPC calls relating to creating instances. It is responsible for building a disk image, launching it via the underlying virtualization driver, -responding to calls to check it state, attaching persistent as well as termination. +responding to calls to check it state, attaching persistent as well as +termination. **Related Flags** diff --git a/nova/db/api.py b/nova/db/api.py index 6f1043d8f..8f9dc2443 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -395,7 +395,7 @@ def network_create_safe(context, values): The network is only returned if the create succeeds. If the create violates constraints because the network already exists, no exception is raised. - + """ return IMPL.network_create_safe(context, values) @@ -499,7 +499,7 @@ def export_device_create_safe(context, values): The device is not returned. If the create violates the unique constraints because the shelf_id and blade_id already exist, no exception is raised. - + """ return IMPL.export_device_create_safe(context, values) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index bc9a7480e..01b5cf350 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -411,10 +411,10 @@ class Network(BASE, NovaBase): class AuthToken(BASE, NovaBase): """Represents an authorization token for all API transactions. - + Fields are a string representing the actual token and a user id for mapping to the actual user - + """ __tablename__ = 'auth_tokens' token_hash = Column(String(255), primary_key=True) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 89e5c8d57..156aad2a0 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -114,7 +114,7 @@ class VolumeDriver(object): escaped_name)) def ensure_export(self, context, volume): - """Safely and synchronously recreates an export for a logical volume.""" + """Synchronously recreates an export for a logical volume.""" raise NotImplementedError() @defer.inlineCallbacks @@ -228,7 +228,7 @@ class ISCSIDriver(VolumeDriver): """Executes commands relating to ISCSI volumes.""" def ensure_export(self, context, volume): - """Safely and synchronously recreates an export for a logical volume.""" + """Synchronously recreates an export for a logical volume.""" iscsi_target = self.db.volume_get_iscsi_target_num(context, volume['id']) iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) -- cgit From 23463610cb180253697b500f11f01e686b20c1e8 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 3 Nov 2010 15:50:24 -0700 Subject: Fix for bug #640400, enables the exclusive flag on the temporary queues. --- nova/rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/rpc.py b/nova/rpc.py index 895820cd0..14fe010ac 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -206,6 +206,7 @@ class DirectConsumer(Consumer): self.routing_key = msg_id self.exchange = msg_id self.auto_delete = True + self.exclusive = True super(DirectConsumer, self).__init__(connection=connection) -- cgit From 6b8ee54df1a77c46b692cf43cc73009684cc9033 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 00:02:56 -0400 Subject: have "contents" look the same as other headings. --- doc/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 261919629..3c05d721d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -36,7 +36,7 @@ be found on the `OpenStack wiki`_. Contents --------- +======== .. toctree:: :maxdepth: 1 -- cgit From 8ff07424548ad4d25b1653351d0bffaac7bc0642 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 02:06:10 -0400 Subject: Cleanup nova-manage section. --- doc/source/concepts.and.introduction.rst | 117 +----------------- doc/source/nova.manage.rst | 200 +++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 112 deletions(-) create mode 100644 doc/source/nova.manage.rst diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index d73447ba9..205529812 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -35,11 +35,9 @@ focues on describing how Nova's implementation of those concepts is achieved. This page outlines concepts that you will need to understand as a user or administrator of an OpenStack installation. Each section links to more more -detailed information in the `Administration Guide`_, but you'll probably want -to read this section straight-through before tackling the specifics presented -in the administration guide. - -.. _`Administration Guide`: administration.guide.html +detailed information in the :doc:`administration.guide`, +but you'll probably want to read this section straight-through before tackling +the specifics presented in the administration guide. Concept: Users and Projects @@ -142,117 +140,12 @@ Nova is implemented by a number of related binaries. These binaries can run on Concept: nova-manage -------------------- -Introduction -~~~~~~~~~~~~ - The nova-manage command is used to perform many essential functions for administration and ongoing maintenance of nova, such as user creation, vpn management, and much more. -The standard pattern for executing a nova-manage command is: - -``nova-manage []`` - -For example, to obtain a list of all projects: - -``nova-manage project list`` - -User Maintenance -~~~~~~~~~~~~~~~~ - -* user admin: creates a new admin and prints exports - * arguments: name [access] [secret] -* user create: creates a new user and prints exports - * arguments: name [access] [secret] -* user delete: deletes an existing user - * arguments: name -* user exports: prints access and secrets for user in export format - * arguments: name -* user list: lists all users - * arguments: none -* user modify: update a users keys & admin flag - * arguments: accesskey secretkey admin - * leave any field blank to ignore it, admin should be 'T', 'F', or blank - -Project Maintenance -~~~~~~~~~~~~~~~~~~~ - -* project add: Adds user to project - * arguments: project user -* project create: Creates a new project - * arguments: name project_manager [description] -* project delete: Deletes an existing project - * arguments: project_id -* project environment: Exports environment variables to an sourcable file - * arguments: project_id user_id [filename='novarc] -* project list: lists all projects - * arguments: none -* project quota: Set or display quotas for project - * arguments: project_id [key] [value] -* project remove: Removes user from project - * arguments: project user -* project scrub: Deletes data associated with project - * arguments: project -* project zipfile: Exports credentials for project to a zip file - * arguments: project_id user_id [filename='nova.zip] - -User Role Management -~~~~~~~~~~~~~~~~~~~~ - -* role add: adds role to user - * if project is specified, adds project specific role - * arguments: user, role [project] -* role has: checks to see if user has role - * if project is specified, returns True if user has - the global role and the project role - * arguments: user, role [project] -* role remove: removes role from user - * if project is specified, removes project specific role - * arguments: user, role [project] - - -Nova Shell -~~~~~~~~~~ - -* shell bpython - * start a new bpython shell -* shell ipython - * start a new ipython shell -* shell python - * start a new python shell -* shell run - * ??? -* shell script: Runs the script from the specifed path with flags set properly. - * arguments: path - -VPN Management -~~~~~~~~~~~~~~ - -* vpn list: Print a listing of the VPNs for all projects. - * arguments: none -* vpn run: Start the VPN for a given project. - * arguments: project -* vpn spawn: Run all VPNs. - * arguments: none - - -Floating IP Management -~~~~~~~~~~~~~~~~~~~~~~ - -* floating create: Creates floating ips for host by range - * arguments: host ip_range -* floating delete: Deletes floating ips by range - * arguments: range -* floating list: Prints a listing of all floating ips - * arguments: none - -Network Management -~~~~~~~~~~~~~~~~~~ - -* network create: Creates fixed ips for host by range - * arguments: [fixed_range=FLAG], [num_networks=FLAG], - [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG] +See :doc:`nova.manage` in the Administration Guide for more +details. Concept: Flags diff --git a/doc/source/nova.manage.rst b/doc/source/nova.manage.rst new file mode 100644 index 000000000..eeb463f52 --- /dev/null +++ b/doc/source/nova.manage.rst @@ -0,0 +1,200 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +nova-manage +=========== + +Introduction +~~~~~~~~~~~~ + +The nova-manage command is used to perform many essential functions for +administration and ongoing maintenance of nova, such as user creation, +vpn management, and much more. + +The standard pattern for executing a nova-manage command is: + +``nova-manage []`` + +For example, to obtain a list of all projects: + +``nova-manage project list`` + +You can run without arguments to see a list of available command categories: + +``nova-manage`` + +You can run with a category argument to see a list of all commands in that +category: + +``nova-manage user`` + +User Maintenance +~~~~~~~~~~~~~~~~ + +Users, including admins, are created through the ``user`` commands. + +* user admin: creates a new admin and prints exports + * arguments: name [access] [secret] +* user create: creates a new user and prints exports + * arguments: name [access] [secret] +* user delete: deletes an existing user + * arguments: name +* user exports: prints access and secrets for user in export format + * arguments: name +* user list: lists all users + * arguments: none +* user modify: update a users keys & admin flag + * arguments: accesskey secretkey admin + * leave any field blank to ignore it, admin should be 'T', 'F', or blank + +Project Maintenance +~~~~~~~~~~~~~~~~~~~ + +* project add: Adds user to project + * arguments: project user +* project create: Creates a new project + * arguments: name project_manager [description] +* project delete: Deletes an existing project + * arguments: project_id +* project environment: Exports environment variables to an sourcable file + * arguments: project_id user_id [filename='novarc] +* project list: lists all projects + * arguments: none +* project quota: Set or display quotas for project + * arguments: project_id [key] [value] +* project remove: Removes user from project + * arguments: project user +* project scrub: Deletes data associated with project + * arguments: project +* project zipfile: Exports credentials for project to a zip file + * arguments: project_id user_id [filename='nova.zip] + +User Role Management +~~~~~~~~~~~~~~~~~~~~ + +* role add: adds role to user + * if project is specified, adds project specific role + * arguments: user, role [project] +* role has: checks to see if user has role + * if project is specified, returns True if user has + the global role and the project role + * arguments: user, role [project] +* role remove: removes role from user + * if project is specified, removes project specific role + * arguments: user, role [project] + + +Nova Shell +~~~~~~~~~~ + +* shell bpython + * start a new bpython shell +* shell ipython + * start a new ipython shell +* shell python + * start a new python shell +* shell run + * ??? +* shell script: Runs the script from the specifed path with flags set properly. + * arguments: path + +VPN Management +~~~~~~~~~~~~~~ + +* vpn list: Print a listing of the VPNs for all projects. + * arguments: none +* vpn run: Start the VPN for a given project. + * arguments: project +* vpn spawn: Run all VPNs. + * arguments: none + + +Floating IP Management +~~~~~~~~~~~~~~~~~~~~~~ + +* floating create: Creates floating ips for host by range + * arguments: host ip_range +* floating delete: Deletes floating ips by range + * arguments: range +* floating list: Prints a listing of all floating ips + * arguments: none + +Network Management +~~~~~~~~~~~~~~~~~~ + +* network create: Creates fixed ips for host by range + * arguments: [fixed_range=FLAG], [num_networks=FLAG], + [network_size=FLAG], [vlan_start=FLAG], + [vpn_start=FLAG] + + +Concept: Flags +-------------- + +python-gflags + + +Concept: Plugins +---------------- + +* Managers/Drivers: utils.import_object from string flag +* virt/connections: conditional loading from string flag +* db: LazyPluggable via string flag +* auth_manager: utils.import_class based on string flag +* Volumes: moving to pluggable driver instead of manager +* Network: pluggable managers +* Compute: same driver used, but pluggable at connection + + +Concept: IPC/RPC +---------------- + +Rabbit! + + +Concept: Fakes +-------------- + +* auth +* ldap + + +Concept: Scheduler +------------------ + +* simple +* random + + +Concept: Security Groups +------------------------ + +Security groups + + +Concept: Certificate Authority +------------------------------ + +Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns ` and decrypting bundled images. + + +Concept: Images +--------------- + +* launching +* bundling -- cgit From d14dafebf80cfc1776be5496b44a1970167114d8 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 02:06:28 -0400 Subject: add missing file. --- doc/source/development.environment.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 doc/source/development.environment.rst diff --git a/doc/source/development.environment.rst b/doc/source/development.environment.rst new file mode 100644 index 000000000..ee19ba2bf --- /dev/null +++ b/doc/source/development.environment.rst @@ -0,0 +1,23 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Setting up a development environment +==================================== + +:: + + TODO(anthony) -- cgit From bbd11f550765a91af17cb200a2b857f263765918 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 14:44:22 -0400 Subject: Nova quickstart: move vish's novascript into contrib, and convert reademe.md to a quickstart.rst --- contrib/nova.sh | 161 ++++++++++++++++++++++++++++++++++++++++++++++ doc/source/quickstart.rst | 79 ++++++++++++++++++++++- 2 files changed, 238 insertions(+), 2 deletions(-) create mode 100755 contrib/nova.sh diff --git a/contrib/nova.sh b/contrib/nova.sh new file mode 100755 index 000000000..9bc36d6fb --- /dev/null +++ b/contrib/nova.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +DIR=`pwd` +CMD=$1 +SOURCE_BRANCH=lp:nova +if [ -n "$2" ]; then + SOURCE_BRANCH=$2 +fi +DIRNAME=nova +NOVA_DIR=$DIR/$DIRNAME +if [ -n "$3" ]; then + NOVA_DIR=$DIR/$3 +fi + +if [ ! -n "$HOST_IP" ]; then + # NOTE(vish): This will just get the first ip in the list, so if you + # have more than one eth device set up, this will fail, and + # you should explicitly set HOST_IP in your environment + HOST_IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` +fi +TEST=0 +USE_MYSQL=0 +MYSQL_PASS=nova +USE_LDAP=0 +LIBVIRT_TYPE=qemu + +if [ "$USE_MYSQL" == 1 ]; then + SQL_CONN=mysql://root:$MYSQL_PASS@localhost/nova +else + SQL_CONN=sqlite:///$NOVA_DIR/nova.sqlite +fi + +if [ "$USE_LDAP" == 1 ]; then + AUTH=ldapdriver.LdapDriver +else + AUTH=dbdriver.DbDriver +fi + +mkdir -p /etc/nova +cat >/etc/nova/nova-manage.conf << NOVA_CONF_EOF +--verbose +--nodaemon +--dhcpbridge_flagfile=/etc/nova/nova-manage.conf +--FAKE_subdomain=ec2 +--cc_host=$HOST_IP +--routing_source_ip=$HOST_IP +--sql_connection=$SQL_CONN +--auth_driver=nova.auth.$AUTH +--libvirt_type=$LIBVIRT_TYPE +NOVA_CONF_EOF + +if [ "$CMD" == "branch" ]; then + sudo apt-get install -y bzr + rm -rf $NOVA_DIR + bzr branch $SOURCE_BRANCH $NOVA_DIR + cd $NOVA_DIR + mkdir -p $NOVA_DIR/instances + mkdir -p $NOVA_DIR/networks +fi + +# You should only have to run this once +if [ "$CMD" == "install" ]; then + sudo apt-get install -y python-software-properties + sudo add-apt-repository ppa:nova-core/ppa + sudo apt-get update + sudo apt-get install -y dnsmasq open-iscsi kpartx kvm gawk iptables ebtables + sudo apt-get install -y user-mode-linux kvm libvirt-bin + sudo apt-get install -y screen iscsitarget euca2ools vlan curl rabbitmq-server + sudo modprobe kvm + sudo /etc/init.d/libvirt-bin restart + sudo apt-get install -y python-twisted python-sqlalchemy python-mox python-greenlet python-carrot + sudo apt-get install -y python-daemon python-eventlet python-gflags python-tornado python-ipy + sudo apt-get install -y python-libvirt python-libxml2 python-routes + if [ "$USE_MYSQL" == 1 ]; then + cat < test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + euca-describe-instances + +To see output from the various workers, switch screen windows + +:: + + " + +will give you a list of running windows. + +When the instance is running, you should be able to ssh to it. + +:: + + chmod 600 test.pem + ssh -i test.pem root@10.0.0.3 + +When you exit screen + +:: + + + +nova will terminate. It may take a while for nova to finish cleaning up. If +you exit the process before it is done because there were some problems in your +build, you may have to clean up the nova processes manually. If you had any +instances running, you can attempt to kill them through the api: + +:: + + ./nova.sh terminate + +Then you can destroy the screen: + +:: + + ./nova.sh clean + +If things get particularly messed up, you might need to do some more intense +cleanup. Be careful, the following command will manually destroy all runnning +virsh instances and attempt to delete all vlans and bridges. -- cgit From 3a8dadd4d31dd564a08a9e285cc6f3318dc243b3 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 17:36:32 -0400 Subject: Getting Started Guide. --- doc/source/administration.guide.rst | 1 + doc/source/concepts.and.introduction.rst | 3 +- doc/source/getting.started.rst | 83 ++++++++++++++++++++++++++------ doc/source/index.rst | 4 +- doc/source/quickstart.rst | 8 +++ doc/source/reaching.out.rst | 28 +++++++++++ 6 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 doc/source/reaching.out.rst diff --git a/doc/source/administration.guide.rst b/doc/source/administration.guide.rst index 8c36d3d44..2d09bed71 100644 --- a/doc/source/administration.guide.rst +++ b/doc/source/administration.guide.rst @@ -30,5 +30,6 @@ Contents quickstart getting.started multi.node.install + nova.manage flags monitoring diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 205529812..5810280b4 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -144,8 +144,7 @@ The nova-manage command is used to perform many essential functions for administration and ongoing maintenance of nova, such as user creation, vpn management, and much more. -See :doc:`nova.manage` in the Administration Guide for more -details. +See doc:`nova.manage` in the Administration Guide for more details. Concept: Flags diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst index a0148e4b6..6873c49cd 100644 --- a/doc/source/getting.started.rst +++ b/doc/source/getting.started.rst @@ -18,19 +18,31 @@ Getting Started with Nova ========================= -This code base is continually changing so dependencies also change. +This code base is continually changing, so dependencies also change. If you +encounter any problems, see the :doc:`reaching.out` page. +The `contrib/nova.sh` script should be kept up to date, and may be a good +resource to review when debugging. Dependencies ------------ Related servers we rely on -* RabbitMQ: messaging queue, used for all communication between components +* **RabbitMQ**: messaging queue, used for all communication between components Optional servers -* OpenLDAP: By default, the auth server uses the RDBMS-backed datastore by setting FLAGS.auth_driver to 'nova.auth.dbdriver.DbDriver'. But OpenLDAP (or LDAP) could be configured by specifying 'nova.auth.ldapdriver.LdapDriver'. There is a script in the sources(nova/auth/slap.sh) to install a very basic openldap server on ubuntu. -* ReDIS: There is a fake ldap driver that backends to redis. This was created for testing ldap implementation on systems that don't have an easy means to install ldap. +* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by + setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP + (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. + There is a script in the sources (`nova/auth/slap.sh`) to install a very basic + openldap server on ubuntu. +* **ReDIS**: There is a fake ldap auth driver + `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was + created for testing ldap implementation on systems that don't have an easy + means to install ldap. +* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to + be avilable. Currently, only sqlite3 an mysql have been tested. Python libraries that we use (from pip-requires): @@ -38,7 +50,9 @@ Python libraries that we use (from pip-requires): Other libraries: -* XenAPI: Needed only for Xen Cloud Platform or XenServer support. Available from http://wiki.xensource.com/xenwiki/XCP_SDK or http://community.citrix.com/cdn/xs/sdks. +* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available + from http://wiki.xensource.com/xenwiki/XCP_SDK or + http://community.citrix.com/cdn/xs/sdks. External unix tools that are required: @@ -53,7 +67,9 @@ External unix tools that are required: * open-iscsi and iscsitarget (if you use iscsi volumes) * aoetools and vblade-persist (if you use aoe-volumes) -Nova uses cutting-edge versions of many packages. There are ubuntu packages in the nova-core ppa. You can use add this ppa to your sources list on an ubuntu machine with the following commands:: +Nova uses cutting-edge versions of many packages. There are ubuntu packages in +the nova-core ppa. You can use add this ppa to your sources list on an ubuntu +machine with the following commands:: sudo apt-get install -y python-software-properties sudo add-apt-repository ppa:nova-core/ppa @@ -68,34 +84,73 @@ Recommended Installation -------------- -Due to many changes it's best to rely on the `OpenStack wiki `_ for installation instructions. +You can install from packages for your particular Linux distribution if they are +available. Otherwise you can install from source by checking out the source +files from the `Nova Source Code Repository `_ +and running:: + + python setup.py install Configuration --------------- -These instructions are incomplete, but we are actively updating the `OpenStack wiki `_ with more configuration information. - -On the volume node +Configuring the host system +~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* Create a volume group (you can use an actual disk for the volume group as well) +As you read through the Administration Guide you will notice configuration hints +inline with documentation on the subsystem you are configuring. Presented in +this "Getting Started with Nova" document, we only provide what you need to +get started as quickly as possible. For a more detailed description of system +configuration, start reading through :doc:`multi.node.install`. -:: +* Create a volume group (you can use an actual disk for the volume group as + well):: # This creates a 1GB file to create volumes out of dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 losetup --show -f MY_FILE_PATH # replace /dev/loop0 below with whatever losetup returns + # nova-volumes is the default for the --volume_group flag vgcreate nova-volumes /dev/loop0 + +Configuring Nova +~~~~~~~~~~~~~~~~ + +Configuration of the entire system is performed through python-gflags. The +best way to track configuration is through the use of a flagfile. + +A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary +when you launch it. Flagfiles for nova are typically stored in +``/etc/nova/nova.conf``, and flags specific to a certain program are stored in +``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another +flagfile, so typically a file like ``nova-manage.conf`` would have as its first +line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before +specifying overrides or additional options. + +A sample configuration to test the system follows:: + + --verbose + --nodaemon + --FAKE_subdomain=ec2 + --auth_driver=nova.auth.dbdriver.DbDriver + Running --------- -Launch servers +There are many parts to the nova system, each with a specific function. They +are built to be highly-available, so there are may configurations they can be +run in (ie: on many machines, many listeners per machine, etc). This part +of the guide only gets you started quickly, to learn about HA options, see +:doc:`multi.node.install`. + +Launch supporting services * rabbitmq * redis (optional) +* mysql (optional) -Launch nova components +Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` * nova-api * nova-compute diff --git a/doc/source/index.rst b/doc/source/index.rst index 3c05d721d..9a62570ed 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -30,7 +30,8 @@ Nova is written with the following design guidelines in mind: This documentation is generated by the Sphinx toolkit and lives in the source tree. Additional documentation on Nova and other components of OpenStack can -be found on the `OpenStack wiki`_. +be found on the `OpenStack wiki`_. Also see the :doc:`reaching.out` page for +other ways to interact witht the community. .. _`OpenStack wiki`: http://wiki.openstack.org @@ -44,6 +45,7 @@ Contents concepts.and.introduction administration.guide development.guide + reaching.out Indices and tables diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 91e3c9ccb..511ed2483 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -18,6 +18,14 @@ Nova Quickstart =============== +:: + + TODO(todd): * Document the assumptions about pluggable interfaces + (sqlite3 instead of mysql, etc) + * Document env vars that can change things + (USE_MYSQL, HOST_IP) + + The `contrib/novascript.sh` file in the source distribution is a script that will quickly set up nova to run on a single machine. It is tested against Ubuntu only, but other distributions are forthcoming. diff --git a/doc/source/reaching.out.rst b/doc/source/reaching.out.rst new file mode 100644 index 000000000..0d8f675e0 --- /dev/null +++ b/doc/source/reaching.out.rst @@ -0,0 +1,28 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Reaching Out +============ + +The OpenStack community is a very friendly place. Feel free to ask questions. +This document points you to some of the places you may want to communicate +with people at. + +:: + + TODO(todd): * Write This + * Don't end a sentence with a preposition -- cgit From 74762113966fb873816afa7bc7c0f2e2e9eb2ec5 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 17:49:13 -0400 Subject: document purpose of documentation. --- doc/source/getting.started.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst index 6873c49cd..79d2cf204 100644 --- a/doc/source/getting.started.rst +++ b/doc/source/getting.started.rst @@ -23,6 +23,14 @@ encounter any problems, see the :doc:`reaching.out` page. The `contrib/nova.sh` script should be kept up to date, and may be a good resource to review when debugging. +The purpose of this document is to get a system installed that you can use to +test your setup assumptions. Working from this base installtion you can +tweak configurations and work with different flags to monitor interaction with +your hardware, network, and other factors that will allow you to determine +suitability for your deployment. After following this setup method, you should +be able to experiment with different managers, drivers, and flags to get the +best performance. + Dependencies ------------ -- cgit From 0de105e28ece8e742943abd3e6332d6e5694642c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 18:17:02 -0400 Subject: start adding info to multi-node admin guide. --- doc/source/multi.node.install.rst | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/doc/source/multi.node.install.rst b/doc/source/multi.node.install.rst index dad9dcd01..1d484e779 100644 --- a/doc/source/multi.node.install.rst +++ b/doc/source/multi.node.install.rst @@ -18,8 +18,40 @@ Multi-Node Nova =============== -http://etherpad.openstack.org/NovaMultinodeInstall +When you move beyond evaluating the technology and into building an actual +production environemnt, you will need to know how to configure your datacenter +and how to deploy components across your clusters. This guide should help you +through that process. + +Bare-metal Provisioning +----------------------- + +To install the base operating system you can use PXE booting. + +Deployment Technologies +----------------------- + +Once you have machines with a base operating system installation, you can deploy +code and configuration with your favorite tools: * Puppet * Chef -* PXE + +Types of Hosts +-------------- + +A single machine in your cluster can act as one or more of the following types +of host: + +Nova Services + +* Network +* Compute +* Volume +* API +* Objectstore + +Other supporting services + +* Datastore +* Message Queue -- cgit From 4ea4eac3f1a33c1f618c82c5c2312b4626aa7244 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 4 Nov 2010 19:00:46 -0400 Subject: Change directory structure for great justice! --- doc/source/adminguide/binaries.rst | 62 ++++++ doc/source/adminguide/flags.rst | 23 ++ doc/source/adminguide/getting.started.rst | 167 +++++++++++++++ doc/source/adminguide/index.rst | 36 ++++ doc/source/adminguide/monitoring.rst | 27 +++ doc/source/adminguide/multi.node.install.rst | 57 +++++ doc/source/adminguide/nova.manage.rst | 200 +++++++++++++++++ doc/source/adminguide/quickstart.rst | 104 +++++++++ doc/source/administration.guide.rst | 35 --- doc/source/api.rst | 271 ------------------------ doc/source/architecture.rst | 52 ----- doc/source/auth.rst | 258 ---------------------- doc/source/binaries.rst | 62 ------ doc/source/cloudpipe.rst | 77 ------- doc/source/compute.rst | 146 ------------- doc/source/concepts.and.introduction.rst | 2 +- doc/source/database.rst | 61 ------ doc/source/development.environment.rst | 23 -- doc/source/development.guide.rst | 60 ------ doc/source/devguide/api.rst | 271 ++++++++++++++++++++++++ doc/source/devguide/architecture.rst | 52 +++++ doc/source/devguide/auth.rst | 258 ++++++++++++++++++++++ doc/source/devguide/cloudpipe.rst | 77 +++++++ doc/source/devguide/compute.rst | 146 +++++++++++++ doc/source/devguide/database.rst | 61 ++++++ doc/source/devguide/development.environment.rst | 23 ++ doc/source/devguide/fakes.rst | 74 +++++++ doc/source/devguide/glance.rst | 27 +++ doc/source/devguide/index.rst | 60 ++++++ doc/source/devguide/network.rst | 124 +++++++++++ doc/source/devguide/nova.rst | 190 +++++++++++++++++ doc/source/devguide/objectstore.rst | 62 ++++++ doc/source/devguide/scheduler.rst | 62 ++++++ doc/source/devguide/services.rst | 72 +++++++ doc/source/devguide/volume.rst | 62 ++++++ doc/source/fakes.rst | 74 ------- doc/source/flags.rst | 23 -- doc/source/getting.started.rst | 167 --------------- doc/source/glance.rst | 27 --- doc/source/index.rst | 4 +- doc/source/modules.rst | 38 ---- doc/source/monitoring.rst | 27 --- doc/source/multi.node.install.rst | 57 ----- doc/source/network.rst | 124 ----------- doc/source/nova.manage.rst | 200 ----------------- doc/source/nova.rst | 190 ----------------- doc/source/objectstore.rst | 62 ------ doc/source/quickstart.rst | 104 --------- doc/source/scheduler.rst | 62 ------ doc/source/services.rst | 72 ------- doc/source/volume.rst | 62 ------ 51 files changed, 2300 insertions(+), 2337 deletions(-) create mode 100644 doc/source/adminguide/binaries.rst create mode 100644 doc/source/adminguide/flags.rst create mode 100644 doc/source/adminguide/getting.started.rst create mode 100644 doc/source/adminguide/index.rst create mode 100644 doc/source/adminguide/monitoring.rst create mode 100644 doc/source/adminguide/multi.node.install.rst create mode 100644 doc/source/adminguide/nova.manage.rst create mode 100644 doc/source/adminguide/quickstart.rst delete mode 100644 doc/source/administration.guide.rst delete mode 100644 doc/source/api.rst delete mode 100644 doc/source/architecture.rst delete mode 100644 doc/source/auth.rst delete mode 100644 doc/source/binaries.rst delete mode 100644 doc/source/cloudpipe.rst delete mode 100644 doc/source/compute.rst delete mode 100644 doc/source/database.rst delete mode 100644 doc/source/development.environment.rst delete mode 100644 doc/source/development.guide.rst create mode 100644 doc/source/devguide/api.rst create mode 100644 doc/source/devguide/architecture.rst create mode 100644 doc/source/devguide/auth.rst create mode 100644 doc/source/devguide/cloudpipe.rst create mode 100644 doc/source/devguide/compute.rst create mode 100644 doc/source/devguide/database.rst create mode 100644 doc/source/devguide/development.environment.rst create mode 100644 doc/source/devguide/fakes.rst create mode 100644 doc/source/devguide/glance.rst create mode 100644 doc/source/devguide/index.rst create mode 100644 doc/source/devguide/network.rst create mode 100644 doc/source/devguide/nova.rst create mode 100644 doc/source/devguide/objectstore.rst create mode 100644 doc/source/devguide/scheduler.rst create mode 100644 doc/source/devguide/services.rst create mode 100644 doc/source/devguide/volume.rst delete mode 100644 doc/source/fakes.rst delete mode 100644 doc/source/flags.rst delete mode 100644 doc/source/getting.started.rst delete mode 100644 doc/source/glance.rst delete mode 100644 doc/source/modules.rst delete mode 100644 doc/source/monitoring.rst delete mode 100644 doc/source/multi.node.install.rst delete mode 100644 doc/source/network.rst delete mode 100644 doc/source/nova.manage.rst delete mode 100644 doc/source/nova.rst delete mode 100644 doc/source/objectstore.rst delete mode 100644 doc/source/quickstart.rst delete mode 100644 doc/source/scheduler.rst delete mode 100644 doc/source/services.rst delete mode 100644 doc/source/volume.rst diff --git a/doc/source/adminguide/binaries.rst b/doc/source/adminguide/binaries.rst new file mode 100644 index 000000000..3ea07d9c2 --- /dev/null +++ b/doc/source/adminguide/binaries.rst @@ -0,0 +1,62 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _binaries: + +Nova Binaries +=============== + +The configuration of these binaries relies on "flagfiles" using the google +gflags package:: + + $ nova-xxxxx --flagfile flagfile + +The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. + +nova-manage +----------- + +Nova manage is a command line utility to administer the system. It will autmatically try to load a flagfile from /etc/nova/nova-manage.conf to save you having to type it. Info on the commands can be found :ref:`here `. + +nova-api +-------- + +Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. + +nova-objectstore +---------------- + +Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with glance and a simple image manager. + +nova-compute +------------ + +Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. + +nova-volume +----------- + +Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. + +nova-network +------------ + +Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: + + $ nova-network --network_manager=nova.network.manager.FlatManager + +IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/adminguide/flags.rst b/doc/source/adminguide/flags.rst new file mode 100644 index 000000000..4c950aa88 --- /dev/null +++ b/doc/source/adminguide/flags.rst @@ -0,0 +1,23 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Flags and Flagfiles +=================== + +* python-gflags +* flagfiles +* list of flags by component (see concepts list) diff --git a/doc/source/adminguide/getting.started.rst b/doc/source/adminguide/getting.started.rst new file mode 100644 index 000000000..65ba2d57e --- /dev/null +++ b/doc/source/adminguide/getting.started.rst @@ -0,0 +1,167 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Getting Started with Nova +========================= + +This code base is continually changing, so dependencies also change. If you +encounter any problems, see the :doc:`../reaching.out` page. +The `contrib/nova.sh` script should be kept up to date, and may be a good +resource to review when debugging. + +The purpose of this document is to get a system installed that you can use to +test your setup assumptions. Working from this base installtion you can +tweak configurations and work with different flags to monitor interaction with +your hardware, network, and other factors that will allow you to determine +suitability for your deployment. After following this setup method, you should +be able to experiment with different managers, drivers, and flags to get the +best performance. + +Dependencies +------------ + +Related servers we rely on + +* **RabbitMQ**: messaging queue, used for all communication between components + +Optional servers + +* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by + setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP + (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. + There is a script in the sources (`nova/auth/slap.sh`) to install a very basic + openldap server on ubuntu. +* **ReDIS**: There is a fake ldap auth driver + `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was + created for testing ldap implementation on systems that don't have an easy + means to install ldap. +* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to + be avilable. Currently, only sqlite3 an mysql have been tested. + +Python libraries that we use (from pip-requires): + +.. literalinclude:: ../../../tools/pip-requires + +Other libraries: + +* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available + from http://wiki.xensource.com/xenwiki/XCP_SDK or + http://community.citrix.com/cdn/xs/sdks. + +External unix tools that are required: + +* iptables +* ebtables +* gawk +* curl +* kvm +* libvirt +* dnsmasq +* vlan +* open-iscsi and iscsitarget (if you use iscsi volumes) +* aoetools and vblade-persist (if you use aoe-volumes) + +Nova uses cutting-edge versions of many packages. There are ubuntu packages in +the nova-core ppa. You can use add this ppa to your sources list on an ubuntu +machine with the following commands:: + + sudo apt-get install -y python-software-properties + sudo add-apt-repository ppa:nova-core/ppa + +Recommended +----------- + +* euca2ools: python implementation of aws ec2-tools and ami tools +* build tornado to use C module for evented section + + +Installation +-------------- + +You can install from packages for your particular Linux distribution if they are +available. Otherwise you can install from source by checking out the source +files from the `Nova Source Code Repository `_ +and running:: + + python setup.py install + +Configuration +--------------- + +Configuring the host system +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As you read through the Administration Guide you will notice configuration hints +inline with documentation on the subsystem you are configuring. Presented in +this "Getting Started with Nova" document, we only provide what you need to +get started as quickly as possible. For a more detailed description of system +configuration, start reading through :doc:`multi.node.install`. + +* Create a volume group (you can use an actual disk for the volume group as + well):: + + # This creates a 1GB file to create volumes out of + dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 + losetup --show -f MY_FILE_PATH + # replace /dev/loop0 below with whatever losetup returns + # nova-volumes is the default for the --volume_group flag + vgcreate nova-volumes /dev/loop0 + + +Configuring Nova +~~~~~~~~~~~~~~~~ + +Configuration of the entire system is performed through python-gflags. The +best way to track configuration is through the use of a flagfile. + +A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary +when you launch it. Flagfiles for nova are typically stored in +``/etc/nova/nova.conf``, and flags specific to a certain program are stored in +``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another +flagfile, so typically a file like ``nova-manage.conf`` would have as its first +line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before +specifying overrides or additional options. + +A sample configuration to test the system follows:: + + --verbose + --nodaemon + --FAKE_subdomain=ec2 + --auth_driver=nova.auth.dbdriver.DbDriver + +Running +--------- + +There are many parts to the nova system, each with a specific function. They +are built to be highly-available, so there are may configurations they can be +run in (ie: on many machines, many listeners per machine, etc). This part +of the guide only gets you started quickly, to learn about HA options, see +:doc:`multi.node.install`. + +Launch supporting services + +* rabbitmq +* redis (optional) +* mysql (optional) + +Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` + +* nova-api +* nova-compute +* nova-objectstore +* nova-volume +* nova-scheduler diff --git a/doc/source/adminguide/index.rst b/doc/source/adminguide/index.rst new file mode 100644 index 000000000..ad1788b8b --- /dev/null +++ b/doc/source/adminguide/index.rst @@ -0,0 +1,36 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Administration Guide +==================== + +How to deploy, monitor, and debug Nova. + + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + quickstart + getting.started + binaries + multi.node.install + nova.manage + flags + monitoring diff --git a/doc/source/adminguide/monitoring.rst b/doc/source/adminguide/monitoring.rst new file mode 100644 index 000000000..e7766a6e7 --- /dev/null +++ b/doc/source/adminguide/monitoring.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Monitoring +========== + +* components +* throughput +* exceptions +* hardware + +* ganglia +* syslog diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst new file mode 100644 index 000000000..1d484e779 --- /dev/null +++ b/doc/source/adminguide/multi.node.install.rst @@ -0,0 +1,57 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Multi-Node Nova +=============== + +When you move beyond evaluating the technology and into building an actual +production environemnt, you will need to know how to configure your datacenter +and how to deploy components across your clusters. This guide should help you +through that process. + +Bare-metal Provisioning +----------------------- + +To install the base operating system you can use PXE booting. + +Deployment Technologies +----------------------- + +Once you have machines with a base operating system installation, you can deploy +code and configuration with your favorite tools: + +* Puppet +* Chef + +Types of Hosts +-------------- + +A single machine in your cluster can act as one or more of the following types +of host: + +Nova Services + +* Network +* Compute +* Volume +* API +* Objectstore + +Other supporting services + +* Datastore +* Message Queue diff --git a/doc/source/adminguide/nova.manage.rst b/doc/source/adminguide/nova.manage.rst new file mode 100644 index 000000000..683cf5e14 --- /dev/null +++ b/doc/source/adminguide/nova.manage.rst @@ -0,0 +1,200 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +nova-manage +=========== + +Introduction +~~~~~~~~~~~~ + +The nova-manage command is used to perform many essential functions for +administration and ongoing maintenance of nova, such as user creation, +vpn management, and much more. + +The standard pattern for executing a nova-manage command is: + +``nova-manage []`` + +For example, to obtain a list of all projects: + +``nova-manage project list`` + +You can run without arguments to see a list of available command categories: + +``nova-manage`` + +You can run with a category argument to see a list of all commands in that +category: + +``nova-manage user`` + +User Maintenance +~~~~~~~~~~~~~~~~ + +Users, including admins, are created through the ``user`` commands. + +* user admin: creates a new admin and prints exports + * arguments: name [access] [secret] +* user create: creates a new user and prints exports + * arguments: name [access] [secret] +* user delete: deletes an existing user + * arguments: name +* user exports: prints access and secrets for user in export format + * arguments: name +* user list: lists all users + * arguments: none +* user modify: update a users keys & admin flag + * arguments: accesskey secretkey admin + * leave any field blank to ignore it, admin should be 'T', 'F', or blank + +Project Maintenance +~~~~~~~~~~~~~~~~~~~ + +* project add: Adds user to project + * arguments: project user +* project create: Creates a new project + * arguments: name project_manager [description] +* project delete: Deletes an existing project + * arguments: project_id +* project environment: Exports environment variables to an sourcable file + * arguments: project_id user_id [filename='novarc] +* project list: lists all projects + * arguments: none +* project quota: Set or display quotas for project + * arguments: project_id [key] [value] +* project remove: Removes user from project + * arguments: project user +* project scrub: Deletes data associated with project + * arguments: project +* project zipfile: Exports credentials for project to a zip file + * arguments: project_id user_id [filename='nova.zip] + +User Role Management +~~~~~~~~~~~~~~~~~~~~ + +* role add: adds role to user + * if project is specified, adds project specific role + * arguments: user, role [project] +* role has: checks to see if user has role + * if project is specified, returns True if user has + the global role and the project role + * arguments: user, role [project] +* role remove: removes role from user + * if project is specified, removes project specific role + * arguments: user, role [project] + + +Nova Shell +~~~~~~~~~~ + +* shell bpython + * start a new bpython shell +* shell ipython + * start a new ipython shell +* shell python + * start a new python shell +* shell run + * ??? +* shell script: Runs the script from the specifed path with flags set properly. + * arguments: path + +VPN Management +~~~~~~~~~~~~~~ + +* vpn list: Print a listing of the VPNs for all projects. + * arguments: none +* vpn run: Start the VPN for a given project. + * arguments: project +* vpn spawn: Run all VPNs. + * arguments: none + + +Floating IP Management +~~~~~~~~~~~~~~~~~~~~~~ + +* floating create: Creates floating ips for host by range + * arguments: host ip_range +* floating delete: Deletes floating ips by range + * arguments: range +* floating list: Prints a listing of all floating ips + * arguments: none + +Network Management +~~~~~~~~~~~~~~~~~~ + +* network create: Creates fixed ips for host by range + * arguments: [fixed_range=FLAG], [num_networks=FLAG], + [network_size=FLAG], [vlan_start=FLAG], + [vpn_start=FLAG] + + +Concept: Flags +-------------- + +python-gflags + + +Concept: Plugins +---------------- + +* Managers/Drivers: utils.import_object from string flag +* virt/connections: conditional loading from string flag +* db: LazyPluggable via string flag +* auth_manager: utils.import_class based on string flag +* Volumes: moving to pluggable driver instead of manager +* Network: pluggable managers +* Compute: same driver used, but pluggable at connection + + +Concept: IPC/RPC +---------------- + +Rabbit! + + +Concept: Fakes +-------------- + +* auth +* ldap + + +Concept: Scheduler +------------------ + +* simple +* random + + +Concept: Security Groups +------------------------ + +Security groups + + +Concept: Certificate Authority +------------------------------ + +Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns <../cloudpipe>` and decrypting bundled images. + + +Concept: Images +--------------- + +* launching +* bundling diff --git a/doc/source/adminguide/quickstart.rst b/doc/source/adminguide/quickstart.rst new file mode 100644 index 000000000..511ed2483 --- /dev/null +++ b/doc/source/adminguide/quickstart.rst @@ -0,0 +1,104 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Quickstart +=============== + +:: + + TODO(todd): * Document the assumptions about pluggable interfaces + (sqlite3 instead of mysql, etc) + * Document env vars that can change things + (USE_MYSQL, HOST_IP) + + +The `contrib/novascript.sh` file in the source distribution is a script that +will quickly set up nova to run on a single machine. It is tested against +Ubuntu only, but other distributions are forthcoming. + +Usage +----- + +Unless you want to spend a lot of time fiddling with permissions and sudoers, +you should probably run nova as root. + +:: + + sudo -i + +If you are concerned about security, nova runs just fine inside a virtual +machine. + +Use the script to install and run the current trunk. You can also specify a +specific branch by putting `lp:~someone/nova/some-branch` after the branch +command + +:: + + ./nova.sh branch + ./nova.sh install + ./nova.sh run + +The run command will drop you into a screen session with all of the workers +running in different windows You can use eucatools to run commands against the +cloud. + +:: + + euca-add-keypair test > test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + euca-describe-instances + +To see output from the various workers, switch screen windows + +:: + + " + +will give you a list of running windows. + +When the instance is running, you should be able to ssh to it. + +:: + + chmod 600 test.pem + ssh -i test.pem root@10.0.0.3 + +When you exit screen + +:: + + + +nova will terminate. It may take a while for nova to finish cleaning up. If +you exit the process before it is done because there were some problems in your +build, you may have to clean up the nova processes manually. If you had any +instances running, you can attempt to kill them through the api: + +:: + + ./nova.sh terminate + +Then you can destroy the screen: + +:: + + ./nova.sh clean + +If things get particularly messed up, you might need to do some more intense +cleanup. Be careful, the following command will manually destroy all runnning +virsh instances and attempt to delete all vlans and bridges. diff --git a/doc/source/administration.guide.rst b/doc/source/administration.guide.rst deleted file mode 100644 index 2d09bed71..000000000 --- a/doc/source/administration.guide.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Administration Guide -==================== - -How to deploy, monitor, and debug Nova. - - -Contents --------- - -.. toctree:: - :maxdepth: 1 - - quickstart - getting.started - multi.node.install - nova.manage - flags - monitoring diff --git a/doc/source/api.rst b/doc/source/api.rst deleted file mode 100644 index 0908d9e57..000000000 --- a/doc/source/api.rst +++ /dev/null @@ -1,271 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -API Endpoint Programming Guide -============================== - -:: - - TODO(todd): get actual docstrings from ec2/osapi_verions instead of @wsgify - -Nova has a system for managing multiple APIs on different subdomains. -Currently there is support for the OpenStack API, as well as the Amazon EC2 -API. - -Common Components ------------------ - -The :mod:`nova.api` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api.cloud - :members: - :undoc-members: - :show-inheritance: - -OpenStack API -------------- - -The :mod:`openstack` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack - :members: - :undoc-members: - :show-inheritance: - -The :mod:`auth` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.auth - :members: - :undoc-members: - :show-inheritance: - -The :mod:`backup_schedules` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.backup_schedules - :members: - :undoc-members: - :show-inheritance: - -The :mod:`faults` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.faults - :members: - :undoc-members: - :show-inheritance: - -The :mod:`flavors` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.flavors - :members: - :undoc-members: - :show-inheritance: - -The :mod:`images` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.images - :members: - :undoc-members: - :show-inheritance: - -The :mod:`ratelimiting` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.ratelimiting - :members: - :undoc-members: - :show-inheritance: - -The :mod:`servers` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.servers - :members: - :undoc-members: - :show-inheritance: - -The :mod:`sharedipgroups` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: nova.api.openstack.sharedipgroups - :members: - :undoc-members: - :show-inheritance: - -EC2 API -------- - -The :mod:`nova.api.ec2` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api.ec2 - :members: - :undoc-members: - :show-inheritance: - -The :mod:`admin` Module -~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api.ec2.admin - :members: - :undoc-members: - :show-inheritance: - -The :mod:`apirequest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api.ec2.apirequest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud` Module -~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api.ec2.cloud - :members: - :undoc-members: - :show-inheritance: - -The :mod:`images` Module -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api.ec2.images - :members: - :undoc-members: - :show-inheritance: - -The :mod:`metadatarequesthandler` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.api.ec2.metadatarequesthandler - :members: - :undoc-members: - :show-inheritance: - -Tests ------ - -The :mod:`api_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api_integration` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api_integration - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.cloud_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api.fakes` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.fakes - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api.test_wsgi` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.test_wsgi - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_api` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_api - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_auth` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_auth - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_faults` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_faults - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_flavors` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_flavors - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_images` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_images - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_ratelimiting` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_ratelimiting - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_servers` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_servers - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test_sharedipgroups` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.api.openstack.test_sharedipgroups - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst deleted file mode 100644 index 1e23e1361..000000000 --- a/doc/source/architecture.rst +++ /dev/null @@ -1,52 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova System Architecture -======================== - -Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received. - -Nova recently moved to using a sql-based central database that is shared by all components in the system. The amount and depth of the data fits into a sql database quite well. For small deployments this seems like an optimal solution. For larger deployments, and especially if security is a concern, nova will be moving towards multiple data stores with some kind of aggregation system. - -Components ----------- - -Below you will find a helpful explanation of the different components. - -:: - - /- ( LDAP ) - [ Auth Manager ] --- - | \- ( DB ) - | - | [ scheduler ] - [ volume ] - ( ATAoE/iSCSI ) - | / - [ Web Dashboard ] -> [ api ] -- < AMQP > ------ [ network ] - ( Flat/Vlan ) - | \ - < HTTP > [ scheduler ] - [ compute ] - ( libvirt/xen ) - | | - [ objectstore ] < - retrieves images - -* DB: sql database for data storage. Used by all components (LINKS NOT SHOWN) -* Web Dashboard: potential external component that talks to the api -* api: component that receives http requests, converts commands and communicates with other components via the queue or http (in the case of objectstore) -* Auth Manager: component responsible for users/projects/and roles. Can backend to DB or LDAP. This is not a separate binary, but rather a python class that is used by most components in the system. -* objectstore: twisted http server that replicates s3 api and allows storage and retrieval of images -* scheduler: decides which host gets each vm and volume -* volume: manages dynamically attachable block devices. -* network: manages ip forwarding, bridges, and vlans -* compute: manages communication with hypervisor and virtual machines. diff --git a/doc/source/auth.rst b/doc/source/auth.rst deleted file mode 100644 index 77d97f68b..000000000 --- a/doc/source/auth.rst +++ /dev/null @@ -1,258 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _auth: - -Authentication and Authorization Programming Guide -================================================== - -The :mod:`quota` Module ------------------------ - -.. automodule:: nova.quota - :members: - :undoc-members: - :show-inheritance: - -Auth Manager ------------- - -The :mod:`manager` Module -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.auth.manager - :members: - :undoc-members: - :show-inheritance: - -Drivers -------- - -The :mod:`ldapdriver` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.auth.ldapdriver - :members: - :undoc-members: - :show-inheritance: - -The :mod:`dbdriver` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.auth.dbdriver - :members: - :undoc-members: - :show-inheritance: - -API Integration ---------------- - -The :mod:`signer` Module -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.auth.signer - :members: - :undoc-members: - :show-inheritance: - -Related Tests -------------- - -The :mod:`auth_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.auth_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`access_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.access_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`quota_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.quota_unittest - :members: - :undoc-members: - :show-inheritance: - -legacy docs ------------ - -Nova provides RBAC (Role-based access control) of the AWS-type APIs. We define the following roles: - -Roles-Based Access Control of AWS-style APIs using SAML Assertions -“Achieving FIPS 199 Moderate certification of a hybrid cloud environment using CloudAudit and declarative C.I.A. classifications” - -Introduction ------------- - -We will investigate one method for integrating an AWS-style API with US eAuthentication-compatible federated authentication systems, to achieve access controls and limits based on traditional operational roles. -Additionally, we will look at how combining this approach, with an implementation of the CloudAudit APIs, will allow us to achieve a certification under FIPS 199 Moderate classification for a hybrid cloud environment. - -Relationship of US eAuth to RBAC --------------------------------- - -Typical implementations of US eAuth authentication systems are structured as follows:: - - [ MS Active Directory or other federated LDAP user store ] - --> backends to… - [ SUN Identity Manager or other SAML Policy Controller ] - --> maps URLs to groups… - [ Apache Policy Agent in front of eAuth-secured Web Application ] - -In more ideal implementations, the remainder of the application-specific account information is stored either in extended schema on the LDAP server itself, via the use of a translucent LDAP proxy, or in an independent datastore keyed off of the UID provided via SAML assertion. - -.. _auth_roles: - -Roles ------ - -AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: - -* Base User -* System Administrator/Developer (currently have the same permissions) -* Network Administrator -* Project Manager -* Cloud Administrator/IT-Security (currently have the same permissions) - -There is an additional, conceptual end-user that may or may not have API access: - -* (EXTERNAL) End-user / Third-party User - -Basic operations are available to any : - -* Describe Instances -* Describe Images -* Describe Volumes -* Describe Keypairs -* Create Keypair -* Delete Keypair -* Create, Upload, Delete: Buckets and Keys (Object Store) - -System Administrators/Developers/Project Manager: - -* Create, Attach, Delete Volume (Block Store) -* Launch, Reboot, Terminate Instance -* Register/Unregister Machine Image (project-wide) -* Request / Review CloudAudit Scans - -Project Manager: - -* Add and remove other users (currently no api) -* Set roles (currently no api) - -Network Administrator: - -* Change Machine Image properties (public / private) -* Change Firewall Rules, define Security Groups -* Allocate, Associate, Deassociate Public IP addresses - -Cloud Administrator/IT-Security: - -* All permissions - -Enhancements ------------- - -* SAML Token passing -* REST interfaces -* SOAP interfaces - -Wrapping the SAML token into the API calls. -Then store the UID (fetched via backchannel) into the instance metadata, providing end-to-end auditability of ownership and responsibility, without PII. - -CloudAudit APIs ---------------- - -* Request formats -* Response formats -* Stateless asynchronous queries - -CloudAudit queries may spawn long-running processes (similar to launching instances, etc.) They need to return a ReservationId in the same fashion, which can be returned in further queries for updates. -RBAC of CloudAudit API calls is critical, since detailed system information is a system vulnerability. - -Type declarations ------------------ -* Data declarations – Volumes and Objects -* System declarations – Instances - -Existing API calls to launch instances specific a single, combined “type” flag. We propose to extend this with three additional type declarations, mapping to the “Confidentiality, Integrity, Availability” classifications of FIPS 199. An example API call would look like:: - - RunInstances type=m1.large number=1 secgroup=default key=mykey confidentiality=low integrity=low availability=low - -These additional parameters would also apply to creation of block storage volumes (along with the existing parameter of ‘size’), and creation of object storage ‘buckets’. (C.I.A. classifications on a bucket would be inherited by the keys within this bucket.) - -Request Brokering ------------------ - - * Cloud Interop - * IMF Registration / PubSub - * Digital C&A - -Establishing declarative semantics for individual API calls will allow the cloud environment to seamlessly proxy these API calls to external, third-party vendors – when the requested CIA levels match. - -See related work within the Infrastructure 2.0 working group for more information on how the IMF Metadata specification could be utilized to manage registration of these vendors and their C&A credentials. - -Dirty Cloud – Hybrid Data Centers ---------------------------------- - -* CloudAudit bridge interfaces -* Anything in the ARP table - -A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. - -This interconnect is typically a bridged VPN connection. Any machines that can be bridged into a hybrid environment in this fashion (at Layer 2) must implement a minimum version of the CloudAudit spec, such that they can be queried to provide a complete picture of the IT-sec runtime environment. - -Network discovery protocols (ARP, CDP) can be applied in this case, and existing protocols (SNMP location data, DNS LOC records) overloaded to provide CloudAudit information. - -The Details ------------ - - * Preliminary Roles Definitions - * Categorization of available API calls - * SAML assertion vocabulary - -System limits -------------- - -The following limits need to be defined and enforced: - -* Total number of instances allowed (user / project) -* Total number of instances, per instance type (user / project) -* Total number of volumes (user / project) -* Maximum size of volume -* Cumulative size of all volumes -* Total use of object storage (GB) -* Total number of Public IPs - - -Further Challenges ------------------- - * Prioritization of users / jobs in shared computing environments - * Incident response planning - * Limit launch of instances to specific security groups based on AMI - * Store AMIs in LDAP for added property control - - - diff --git a/doc/source/binaries.rst b/doc/source/binaries.rst deleted file mode 100644 index 3ea07d9c2..000000000 --- a/doc/source/binaries.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _binaries: - -Nova Binaries -=============== - -The configuration of these binaries relies on "flagfiles" using the google -gflags package:: - - $ nova-xxxxx --flagfile flagfile - -The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. - -nova-manage ------------ - -Nova manage is a command line utility to administer the system. It will autmatically try to load a flagfile from /etc/nova/nova-manage.conf to save you having to type it. Info on the commands can be found :ref:`here `. - -nova-api --------- - -Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. - -nova-objectstore ----------------- - -Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with glance and a simple image manager. - -nova-compute ------------- - -Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. - -nova-volume ------------ - -Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. - -nova-network ------------- - -Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: - - $ nova-network --network_manager=nova.network.manager.FlatManager - -IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/cloudpipe.rst b/doc/source/cloudpipe.rst deleted file mode 100644 index d632a40e6..000000000 --- a/doc/source/cloudpipe.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -.. _cloudpipe: - -Cloudpipe -- Per Project Vpns -============================= - -Cloudpipe is a method for connecting end users to their project insnances in vlan mode. - -Overview --------- - -The support code for cloudpipe implements admin commands (via nova-manage) to automatically create a vm for a project that allows users to vpn into the private network of their project. Access to this vpn is provided through a public port on the network host for the project. This allows users to have free access to the virtual machines in their project without exposing those machines to the public internet. - -Cloudpipe Image ---------------- - -The cloudpipe image is basically just a linux instance with openvpn installed. It needs a simple script to grab user data from the metadata server, b64 decode it into a zip file, and run the autorun.sh script from inside the zip. The autorun script will configure and run openvpn to run using the data from nova. - -It is also useful to have a cron script that will periodically redownload the metadata and copy the new crl. This will keep revoked users from connecting and will disconnect any users that are connected with revoked certificates when their connection is renegotiated (every hour). - -Cloudpipe Launch ----------------- - -When you use nova-manage to launch a cloudpipe for a user, it goes through the following process: - -#. creates a keypair called -vpn and saves it in the keys directory -#. creates a security group -vpn and opens up 1194 and icmp -#. creates a cert and private key for the vpn instance and saves it in the CA/projects// directory -#. zips up the info and puts it b64 encoded as user data -#. launches an m1.tiny instance with the above settings using the flag-specified vpn image - -Vpn Access ----------- - -In vlan networking mode, the second ip in each private network is reserved for the cloudpipe instance. This gives a consistent ip to the instance so that nova-network can create forwarding rules for access from the outside world. The network for each project is given a specific high-numbered port on the public ip of the network host. This port is automatically forwarded to 1194 on the vpn instance. - -If specific high numbered ports do not work for your users, you can always allocate and associate a public ip to the instance, and then change the vpn_public_ip and vpn_public_port in the database. This will be turned into a nova-manage command or a flag soon. - - -Certificates and Revocation ---------------------------- - -If the use_project_ca flag is set (required to for cloudpipes to work securely), then each project has its own ca. This ca is used to sign the certificate for the vpn, and is also passed to the user for bundling images. When a certificate is revoked using nova-manage, a new Certificate Revocation List (crl) is generated. As long as cloudpipe has an updated crl, it will block revoked users from connecting to the vpn. - -The :mod:`cloudpipe` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.cloudpipe.pipelib - :members: - :undoc-members: - :show-inheritance: - -The :mod:`crypto` Module -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.crypto - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/source/compute.rst b/doc/source/compute.rst deleted file mode 100644 index e4c6c6ae7..000000000 --- a/doc/source/compute.rst +++ /dev/null @@ -1,146 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -Virtualization Programming Guide -================================ - -This page contains the Compute Package documentation. - -Manager -------- - -Documentation for the compute manager and related files. For reading about -a specific virtualization backend, read Drivers_. - - -The :mod:`manager` Module -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.compute.manager - :members: - :undoc-members: - :show-inheritance: - -The :mod:`connection` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.virt.connection - :members: - :undoc-members: - :show-inheritance: - -The :mod:`disk` Module -~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.compute.disk - :members: - :undoc-members: - :show-inheritance: - -The :mod:`images` Module -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.virt.images - :members: - :undoc-members: - :show-inheritance: - - -The :mod:`instance_types` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.compute.instance_types - :members: - :undoc-members: - :show-inheritance: - - -The :mod:`power_state` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.compute.power_state - :members: - :undoc-members: - :show-inheritance: - - -Drivers -------- - -Libvirt Implementations -~~~~~~~~~~~~~~~~~~~~~~~ - -The libvirt driver is capable of supporting KVM, QEMU, and UML. - -The :mod:`libvirt_conn` Module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: nova.virt.libvirt_conn - :members: - :undoc-members: - :show-inheritance: - -XEN -~~~ - -The :mod:`xenapi` Module -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: nova.virt.xenapi - :members: - :undoc-members: - :show-inheritance: - -FAKE -~~~~ - -.. automodule:: nova.virt.fake - :members: - :undoc-members: - :show-inheritance: - -Monitoring ----------- - -The :mod:`monitor` Module -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.compute.monitor - :members: - :undoc-members: - :show-inheritance: - - -Tests ------ - -The :mod:`compute_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.compute_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`virt_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.virt_unittest - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/concepts.and.introduction.rst b/doc/source/concepts.and.introduction.rst index 5810280b4..d409fbaec 100644 --- a/doc/source/concepts.and.introduction.rst +++ b/doc/source/concepts.and.introduction.rst @@ -35,7 +35,7 @@ focues on describing how Nova's implementation of those concepts is achieved. This page outlines concepts that you will need to understand as a user or administrator of an OpenStack installation. Each section links to more more -detailed information in the :doc:`administration.guide`, +detailed information in the :doc:`adminguide/index`, but you'll probably want to read this section straight-through before tackling the specifics presented in the administration guide. diff --git a/doc/source/database.rst b/doc/source/database.rst deleted file mode 100644 index b58ea147d..000000000 --- a/doc/source/database.rst +++ /dev/null @@ -1,61 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Database Programming Guide -========================== - -The :mod:`api` Module ---------------------- - -.. automodule:: nova.db.api - :members: - :undoc-members: - :show-inheritance: - - -Drivers -------- - -Driver: Sqlalchemy -~~~~~~~~~~~~~~~~~~ - -The :mod:`sqlalchemy.api` Module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: nova.db.sqlalchemy.api - -The :mod:`sqlalchemy.models` Module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: nova.db.sqlalchemy.models - :members: - :undoc-members: - :show-inheritance: - -The :mod:`sqlalchemy.session` Module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: nova.db.sqlalchemy.session - :members: - :undoc-members: - :show-inheritance: - -Tests ------ - -Tests are lacking for the db api layer and for the sqlalchemy driver. -Failures in the drivers would be dectected in other test cases, though. diff --git a/doc/source/development.environment.rst b/doc/source/development.environment.rst deleted file mode 100644 index ee19ba2bf..000000000 --- a/doc/source/development.environment.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Setting up a development environment -==================================== - -:: - - TODO(anthony) diff --git a/doc/source/development.guide.rst b/doc/source/development.guide.rst deleted file mode 100644 index 704cbe399..000000000 --- a/doc/source/development.guide.rst +++ /dev/null @@ -1,60 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova Development Guide -====================== - -Nova is written in python. - -Prerequisites -------------- -.. toctree:: - :maxdepth: 1 - - concepts.and.introduction - architecture - development.environment - - -Contents --------- - -.. toctree:: - :maxdepth: 1 - - services - database - volume - compute - network - auth - api - scheduler - fakes - nova - cloudpipe - objectstore - glance - - -Indices and tables ------------------- - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/doc/source/devguide/api.rst b/doc/source/devguide/api.rst new file mode 100644 index 000000000..0908d9e57 --- /dev/null +++ b/doc/source/devguide/api.rst @@ -0,0 +1,271 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +API Endpoint Programming Guide +============================== + +:: + + TODO(todd): get actual docstrings from ec2/osapi_verions instead of @wsgify + +Nova has a system for managing multiple APIs on different subdomains. +Currently there is support for the OpenStack API, as well as the Amazon EC2 +API. + +Common Components +----------------- + +The :mod:`nova.api` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.cloud + :members: + :undoc-members: + :show-inheritance: + +OpenStack API +------------- + +The :mod:`openstack` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack + :members: + :undoc-members: + :show-inheritance: + +The :mod:`auth` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.auth + :members: + :undoc-members: + :show-inheritance: + +The :mod:`backup_schedules` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.backup_schedules + :members: + :undoc-members: + :show-inheritance: + +The :mod:`faults` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.faults + :members: + :undoc-members: + :show-inheritance: + +The :mod:`flavors` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.flavors + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.images + :members: + :undoc-members: + :show-inheritance: + +The :mod:`ratelimiting` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.ratelimiting + :members: + :undoc-members: + :show-inheritance: + +The :mod:`servers` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.servers + :members: + :undoc-members: + :show-inheritance: + +The :mod:`sharedipgroups` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.sharedipgroups + :members: + :undoc-members: + :show-inheritance: + +EC2 API +------- + +The :mod:`nova.api.ec2` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2 + :members: + :undoc-members: + :show-inheritance: + +The :mod:`admin` Module +~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.admin + :members: + :undoc-members: + :show-inheritance: + +The :mod:`apirequest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.apirequest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud` Module +~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.cloud + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.images + :members: + :undoc-members: + :show-inheritance: + +The :mod:`metadatarequesthandler` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.metadatarequesthandler + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`api_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api_integration` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api_integration + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.cloud_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.fakes` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.fakes + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.test_wsgi` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.test_wsgi + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_api` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_api + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_auth` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_auth + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_faults` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_faults + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_flavors` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_flavors + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_images` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_images + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_ratelimiting` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_ratelimiting + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_servers` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_servers + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_sharedipgroups` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_sharedipgroups + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/devguide/architecture.rst b/doc/source/devguide/architecture.rst new file mode 100644 index 000000000..1e23e1361 --- /dev/null +++ b/doc/source/devguide/architecture.rst @@ -0,0 +1,52 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova System Architecture +======================== + +Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received. + +Nova recently moved to using a sql-based central database that is shared by all components in the system. The amount and depth of the data fits into a sql database quite well. For small deployments this seems like an optimal solution. For larger deployments, and especially if security is a concern, nova will be moving towards multiple data stores with some kind of aggregation system. + +Components +---------- + +Below you will find a helpful explanation of the different components. + +:: + + /- ( LDAP ) + [ Auth Manager ] --- + | \- ( DB ) + | + | [ scheduler ] - [ volume ] - ( ATAoE/iSCSI ) + | / + [ Web Dashboard ] -> [ api ] -- < AMQP > ------ [ network ] - ( Flat/Vlan ) + | \ + < HTTP > [ scheduler ] - [ compute ] - ( libvirt/xen ) + | | + [ objectstore ] < - retrieves images + +* DB: sql database for data storage. Used by all components (LINKS NOT SHOWN) +* Web Dashboard: potential external component that talks to the api +* api: component that receives http requests, converts commands and communicates with other components via the queue or http (in the case of objectstore) +* Auth Manager: component responsible for users/projects/and roles. Can backend to DB or LDAP. This is not a separate binary, but rather a python class that is used by most components in the system. +* objectstore: twisted http server that replicates s3 api and allows storage and retrieval of images +* scheduler: decides which host gets each vm and volume +* volume: manages dynamically attachable block devices. +* network: manages ip forwarding, bridges, and vlans +* compute: manages communication with hypervisor and virtual machines. diff --git a/doc/source/devguide/auth.rst b/doc/source/devguide/auth.rst new file mode 100644 index 000000000..77d97f68b --- /dev/null +++ b/doc/source/devguide/auth.rst @@ -0,0 +1,258 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _auth: + +Authentication and Authorization Programming Guide +================================================== + +The :mod:`quota` Module +----------------------- + +.. automodule:: nova.quota + :members: + :undoc-members: + :show-inheritance: + +Auth Manager +------------ + +The :mod:`manager` Module +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.manager + :members: + :undoc-members: + :show-inheritance: + +Drivers +------- + +The :mod:`ldapdriver` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.ldapdriver + :members: + :undoc-members: + :show-inheritance: + +The :mod:`dbdriver` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.dbdriver + :members: + :undoc-members: + :show-inheritance: + +API Integration +--------------- + +The :mod:`signer` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.signer + :members: + :undoc-members: + :show-inheritance: + +Related Tests +------------- + +The :mod:`auth_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.auth_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`access_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.access_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`quota_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.quota_unittest + :members: + :undoc-members: + :show-inheritance: + +legacy docs +----------- + +Nova provides RBAC (Role-based access control) of the AWS-type APIs. We define the following roles: + +Roles-Based Access Control of AWS-style APIs using SAML Assertions +“Achieving FIPS 199 Moderate certification of a hybrid cloud environment using CloudAudit and declarative C.I.A. classifications” + +Introduction +------------ + +We will investigate one method for integrating an AWS-style API with US eAuthentication-compatible federated authentication systems, to achieve access controls and limits based on traditional operational roles. +Additionally, we will look at how combining this approach, with an implementation of the CloudAudit APIs, will allow us to achieve a certification under FIPS 199 Moderate classification for a hybrid cloud environment. + +Relationship of US eAuth to RBAC +-------------------------------- + +Typical implementations of US eAuth authentication systems are structured as follows:: + + [ MS Active Directory or other federated LDAP user store ] + --> backends to… + [ SUN Identity Manager or other SAML Policy Controller ] + --> maps URLs to groups… + [ Apache Policy Agent in front of eAuth-secured Web Application ] + +In more ideal implementations, the remainder of the application-specific account information is stored either in extended schema on the LDAP server itself, via the use of a translucent LDAP proxy, or in an independent datastore keyed off of the UID provided via SAML assertion. + +.. _auth_roles: + +Roles +----- + +AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: + +* Base User +* System Administrator/Developer (currently have the same permissions) +* Network Administrator +* Project Manager +* Cloud Administrator/IT-Security (currently have the same permissions) + +There is an additional, conceptual end-user that may or may not have API access: + +* (EXTERNAL) End-user / Third-party User + +Basic operations are available to any : + +* Describe Instances +* Describe Images +* Describe Volumes +* Describe Keypairs +* Create Keypair +* Delete Keypair +* Create, Upload, Delete: Buckets and Keys (Object Store) + +System Administrators/Developers/Project Manager: + +* Create, Attach, Delete Volume (Block Store) +* Launch, Reboot, Terminate Instance +* Register/Unregister Machine Image (project-wide) +* Request / Review CloudAudit Scans + +Project Manager: + +* Add and remove other users (currently no api) +* Set roles (currently no api) + +Network Administrator: + +* Change Machine Image properties (public / private) +* Change Firewall Rules, define Security Groups +* Allocate, Associate, Deassociate Public IP addresses + +Cloud Administrator/IT-Security: + +* All permissions + +Enhancements +------------ + +* SAML Token passing +* REST interfaces +* SOAP interfaces + +Wrapping the SAML token into the API calls. +Then store the UID (fetched via backchannel) into the instance metadata, providing end-to-end auditability of ownership and responsibility, without PII. + +CloudAudit APIs +--------------- + +* Request formats +* Response formats +* Stateless asynchronous queries + +CloudAudit queries may spawn long-running processes (similar to launching instances, etc.) They need to return a ReservationId in the same fashion, which can be returned in further queries for updates. +RBAC of CloudAudit API calls is critical, since detailed system information is a system vulnerability. + +Type declarations +----------------- +* Data declarations – Volumes and Objects +* System declarations – Instances + +Existing API calls to launch instances specific a single, combined “type” flag. We propose to extend this with three additional type declarations, mapping to the “Confidentiality, Integrity, Availability” classifications of FIPS 199. An example API call would look like:: + + RunInstances type=m1.large number=1 secgroup=default key=mykey confidentiality=low integrity=low availability=low + +These additional parameters would also apply to creation of block storage volumes (along with the existing parameter of ‘size’), and creation of object storage ‘buckets’. (C.I.A. classifications on a bucket would be inherited by the keys within this bucket.) + +Request Brokering +----------------- + + * Cloud Interop + * IMF Registration / PubSub + * Digital C&A + +Establishing declarative semantics for individual API calls will allow the cloud environment to seamlessly proxy these API calls to external, third-party vendors – when the requested CIA levels match. + +See related work within the Infrastructure 2.0 working group for more information on how the IMF Metadata specification could be utilized to manage registration of these vendors and their C&A credentials. + +Dirty Cloud – Hybrid Data Centers +--------------------------------- + +* CloudAudit bridge interfaces +* Anything in the ARP table + +A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. + +This interconnect is typically a bridged VPN connection. Any machines that can be bridged into a hybrid environment in this fashion (at Layer 2) must implement a minimum version of the CloudAudit spec, such that they can be queried to provide a complete picture of the IT-sec runtime environment. + +Network discovery protocols (ARP, CDP) can be applied in this case, and existing protocols (SNMP location data, DNS LOC records) overloaded to provide CloudAudit information. + +The Details +----------- + + * Preliminary Roles Definitions + * Categorization of available API calls + * SAML assertion vocabulary + +System limits +------------- + +The following limits need to be defined and enforced: + +* Total number of instances allowed (user / project) +* Total number of instances, per instance type (user / project) +* Total number of volumes (user / project) +* Maximum size of volume +* Cumulative size of all volumes +* Total use of object storage (GB) +* Total number of Public IPs + + +Further Challenges +------------------ + * Prioritization of users / jobs in shared computing environments + * Incident response planning + * Limit launch of instances to specific security groups based on AMI + * Store AMIs in LDAP for added property control + + + diff --git a/doc/source/devguide/cloudpipe.rst b/doc/source/devguide/cloudpipe.rst new file mode 100644 index 000000000..d632a40e6 --- /dev/null +++ b/doc/source/devguide/cloudpipe.rst @@ -0,0 +1,77 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +.. _cloudpipe: + +Cloudpipe -- Per Project Vpns +============================= + +Cloudpipe is a method for connecting end users to their project insnances in vlan mode. + +Overview +-------- + +The support code for cloudpipe implements admin commands (via nova-manage) to automatically create a vm for a project that allows users to vpn into the private network of their project. Access to this vpn is provided through a public port on the network host for the project. This allows users to have free access to the virtual machines in their project without exposing those machines to the public internet. + +Cloudpipe Image +--------------- + +The cloudpipe image is basically just a linux instance with openvpn installed. It needs a simple script to grab user data from the metadata server, b64 decode it into a zip file, and run the autorun.sh script from inside the zip. The autorun script will configure and run openvpn to run using the data from nova. + +It is also useful to have a cron script that will periodically redownload the metadata and copy the new crl. This will keep revoked users from connecting and will disconnect any users that are connected with revoked certificates when their connection is renegotiated (every hour). + +Cloudpipe Launch +---------------- + +When you use nova-manage to launch a cloudpipe for a user, it goes through the following process: + +#. creates a keypair called -vpn and saves it in the keys directory +#. creates a security group -vpn and opens up 1194 and icmp +#. creates a cert and private key for the vpn instance and saves it in the CA/projects// directory +#. zips up the info and puts it b64 encoded as user data +#. launches an m1.tiny instance with the above settings using the flag-specified vpn image + +Vpn Access +---------- + +In vlan networking mode, the second ip in each private network is reserved for the cloudpipe instance. This gives a consistent ip to the instance so that nova-network can create forwarding rules for access from the outside world. The network for each project is given a specific high-numbered port on the public ip of the network host. This port is automatically forwarded to 1194 on the vpn instance. + +If specific high numbered ports do not work for your users, you can always allocate and associate a public ip to the instance, and then change the vpn_public_ip and vpn_public_port in the database. This will be turned into a nova-manage command or a flag soon. + + +Certificates and Revocation +--------------------------- + +If the use_project_ca flag is set (required to for cloudpipes to work securely), then each project has its own ca. This ca is used to sign the certificate for the vpn, and is also passed to the user for bundling images. When a certificate is revoked using nova-manage, a new Certificate Revocation List (crl) is generated. As long as cloudpipe has an updated crl, it will block revoked users from connecting to the vpn. + +The :mod:`cloudpipe` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.cloudpipe.pipelib + :members: + :undoc-members: + :show-inheritance: + +The :mod:`crypto` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.crypto + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/devguide/compute.rst b/doc/source/devguide/compute.rst new file mode 100644 index 000000000..e4c6c6ae7 --- /dev/null +++ b/doc/source/devguide/compute.rst @@ -0,0 +1,146 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +Virtualization Programming Guide +================================ + +This page contains the Compute Package documentation. + +Manager +------- + +Documentation for the compute manager and related files. For reading about +a specific virtualization backend, read Drivers_. + + +The :mod:`manager` Module +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.manager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`connection` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.connection + :members: + :undoc-members: + :show-inheritance: + +The :mod:`disk` Module +~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.disk + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.images + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`instance_types` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.instance_types + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`power_state` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.power_state + :members: + :undoc-members: + :show-inheritance: + + +Drivers +------- + +Libvirt Implementations +~~~~~~~~~~~~~~~~~~~~~~~ + +The libvirt driver is capable of supporting KVM, QEMU, and UML. + +The :mod:`libvirt_conn` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: nova.virt.libvirt_conn + :members: + :undoc-members: + :show-inheritance: + +XEN +~~~ + +The :mod:`xenapi` Module +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: nova.virt.xenapi + :members: + :undoc-members: + :show-inheritance: + +FAKE +~~~~ + +.. automodule:: nova.virt.fake + :members: + :undoc-members: + :show-inheritance: + +Monitoring +---------- + +The :mod:`monitor` Module +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.monitor + :members: + :undoc-members: + :show-inheritance: + + +Tests +----- + +The :mod:`compute_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.compute_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`virt_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.virt_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/devguide/database.rst b/doc/source/devguide/database.rst new file mode 100644 index 000000000..b58ea147d --- /dev/null +++ b/doc/source/devguide/database.rst @@ -0,0 +1,61 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Database Programming Guide +========================== + +The :mod:`api` Module +--------------------- + +.. automodule:: nova.db.api + :members: + :undoc-members: + :show-inheritance: + + +Drivers +------- + +Driver: Sqlalchemy +~~~~~~~~~~~~~~~~~~ + +The :mod:`sqlalchemy.api` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: nova.db.sqlalchemy.api + +The :mod:`sqlalchemy.models` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: nova.db.sqlalchemy.models + :members: + :undoc-members: + :show-inheritance: + +The :mod:`sqlalchemy.session` Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: nova.db.sqlalchemy.session + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +Tests are lacking for the db api layer and for the sqlalchemy driver. +Failures in the drivers would be dectected in other test cases, though. diff --git a/doc/source/devguide/development.environment.rst b/doc/source/devguide/development.environment.rst new file mode 100644 index 000000000..ee19ba2bf --- /dev/null +++ b/doc/source/devguide/development.environment.rst @@ -0,0 +1,23 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Setting up a development environment +==================================== + +:: + + TODO(anthony) diff --git a/doc/source/devguide/fakes.rst b/doc/source/devguide/fakes.rst new file mode 100644 index 000000000..5988e2ab8 --- /dev/null +++ b/doc/source/devguide/fakes.rst @@ -0,0 +1,74 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Fakes Programming Guide +======================= + +:: + + TODO(todd): * document general info about fakes + + +The :mod:`virt.fake` Module +--------------------------- + +.. automodule:: nova.virt.fake + :members: + :undoc-members: + :show-inheritance: + :noindex: + +The :mod:`fakeldap` Module +-------------------------- + +.. automodule:: nova.auth.fakeldap + :members: + :undoc-members: + :show-inheritance: + +The :mod:`fakerabbit` Module +---------------------------- + +.. automodule:: nova.fakerabbit + :members: + :undoc-members: + :show-inheritance: + +The :class:`FakeAOEDriver` Class +-------------------------------- + +.. autoclass:: nova.volume.driver.FakeAOEDriver + :members: + :undoc-members: + :show-inheritance: + +The :class:`service_unittest.FakeManager` Class +----------------------------------------------- + +.. autoclass:: nova.tests.service_unittest.FakeManager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.openstack.fakes` Module +------------------------------------------------ + +.. automodule:: nova.tests.api.openstack.fakes + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/devguide/glance.rst b/doc/source/devguide/glance.rst new file mode 100644 index 000000000..3d0f4ebec --- /dev/null +++ b/doc/source/devguide/glance.rst @@ -0,0 +1,27 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Glance Integration Programming Guide +==================================== + +The :mod:`image.service` Module +------------------------------- + +.. automodule:: nova.image.service + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/devguide/index.rst b/doc/source/devguide/index.rst new file mode 100644 index 000000000..6232a3181 --- /dev/null +++ b/doc/source/devguide/index.rst @@ -0,0 +1,60 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Development Guide +====================== + +Nova is written in python. + +Prerequisites +------------- +.. toctree:: + :maxdepth: 1 + + ../concepts.and.introduction + architecture + development.environment + + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + services + database + volume + compute + network + auth + api + scheduler + fakes + nova + cloudpipe + objectstore + glance + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/devguide/network.rst b/doc/source/devguide/network.rst new file mode 100644 index 000000000..9426ece82 --- /dev/null +++ b/doc/source/devguide/network.rst @@ -0,0 +1,124 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Networking Programming Guide +============================ + +:: + + TODO(todd): * document hardware specific commands (maybe in admin guide?) + * document a map between flags and managers/backends + +The :mod:`Manager` module +------------------------- + +.. automodule:: nova.network.manager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`LinuxNet` driver +-------------------------- + +.. automodule:: nova.network.linux_net + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`network_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.network_unittest + :members: + :undoc-members: + :show-inheritance: + + +legacy docs +----------- + +The nova networking components manage private networks, public IP addressing, VPN connectivity, and firewall rules. + +Components +---------- +There are several key components: + +* NetworkController (Manages address and vlan allocation) +* RoutingNode (NATs public IPs to private IPs, and enforces firewall rules) +* AddressingNode (runs DHCP services for private networks) +* BridgingNode (a subclass of the basic nova ComputeNode) +* TunnelingNode (provides VPN connectivity) + +Component Diagram +----------------- + +Overview:: + + (PUBLIC INTERNET) + | \ + / \ / \ + [RoutingNode] ... [RN] [TunnelingNode] ... [TN] + | \ / | | + | < AMQP > | | + [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] + \ | \ / + / \ / \ / \ / \ + [BridgingNode] ... [BridgingNode] + + + [NetworkController] ... [NetworkController] + \ / + < AMQP > + | + / \ + [CloudController]...[CloudController] + +While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). + +State Model +----------- +Network State consists of the following facts: + +* VLAN assignment (to a project) +* Private Subnet assignment (to a security group) in a VLAN +* Private IP assignments (to running instances) +* Public IP allocations (to a project) +* Public IP associations (to a private IP / running instance) + +While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. + +The Public Traffic Path +----------------------- + +Public Traffic:: + + (PUBLIC INTERNET) + | + <-- [RoutingNode] + | + [AddressingNode] --> | + ( VLAN ) + | <-- [BridgingNode] + | + + +The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. + +Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). diff --git a/doc/source/devguide/nova.rst b/doc/source/devguide/nova.rst new file mode 100644 index 000000000..58125dc80 --- /dev/null +++ b/doc/source/devguide/nova.rst @@ -0,0 +1,190 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Libraries Programming Guide +================================ + +The :mod:`adminclient` Module +----------------------------- + +.. automodule:: nova.adminclient + :members: + :undoc-members: + :show-inheritance: + +The :mod:`context` Module +------------------------- + +.. automodule:: nova.context + :members: + :undoc-members: + :show-inheritance: + +The :mod:`exception` Module +--------------------------- + +.. automodule:: nova.exception + :members: + :undoc-members: + :show-inheritance: + +The :mod:`flags` Module +----------------------- + +.. automodule:: nova.flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`process` Module +------------------------- + +.. automodule:: nova.process + :members: + :undoc-members: + :show-inheritance: + +The :mod:`rpc` Module +--------------------- + +.. automodule:: nova.rpc + :members: + :undoc-members: + :show-inheritance: + +The :mod:`server` Module +------------------------ + +.. automodule:: nova.server + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test` Module +---------------------- + +.. automodule:: nova.test + :members: + :undoc-members: + :show-inheritance: + +The :mod:`twistd` Module +------------------------ + +.. automodule:: nova.twistd + :members: + :undoc-members: + :show-inheritance: + +The :mod:`utils` Module +----------------------- + +.. automodule:: nova.utils + :members: + :undoc-members: + :show-inheritance: + +The :mod:`validate` Module +-------------------------- + +.. automodule:: nova.validate + :members: + :undoc-members: + :show-inheritance: + +The :mod:`wsgi` Module +---------------------- + +.. automodule:: nova.wsgi + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`declare_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.declare_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`fake_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.fake_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`flags_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.flags_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`process_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.process_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`real_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.real_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`rpc_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.rpc_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`runtime_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.runtime_flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`twistd_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.twistd_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`validator_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.validator_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/devguide/objectstore.rst b/doc/source/devguide/objectstore.rst new file mode 100644 index 000000000..4087b5dd0 --- /dev/null +++ b/doc/source/devguide/objectstore.rst @@ -0,0 +1,62 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Objectstore Programming Guide +============================= + +The :mod:`handler` Module +------------------------- + +.. automodule:: nova.objectstore.handler + :members: + :undoc-members: + :show-inheritance: + +The :mod:`bucket` Module +------------------------ + +.. automodule:: nova.objectstore.bucket + :members: + :undoc-members: + :show-inheritance: + +The :mod:`stored` Module +------------------------ + +.. automodule:: nova.objectstore.stored + :members: + :undoc-members: + :show-inheritance: + +The :mod:`image` Module +----------------------- + +.. automodule:: nova.objectstore.image + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`objectstore_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.objectstore_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/devguide/scheduler.rst b/doc/source/devguide/scheduler.rst new file mode 100644 index 000000000..df820c2f2 --- /dev/null +++ b/doc/source/devguide/scheduler.rst @@ -0,0 +1,62 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Scheduler Programming Guide +=========================== + +The :mod:`manager` Module +------------------------- + +.. automodule:: nova.scheduler.manager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`driver` Module +------------------------ + +.. automodule:: nova.scheduler.driver + :members: + :undoc-members: + :show-inheritance: + +The :mod:`chance` Module +------------------------ + +.. automodule:: nova.scheduler.chance + :members: + :undoc-members: + :show-inheritance: + +The :mod:`simple` Module +------------------------ + +.. automodule:: nova.scheduler.simple + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`scheduler_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.scheduler_unittest + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/devguide/services.rst b/doc/source/devguide/services.rst new file mode 100644 index 000000000..517a51901 --- /dev/null +++ b/doc/source/devguide/services.rst @@ -0,0 +1,72 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _service_manager_driver: + +Services Managers and Drivers +============================= + +The responsibilities of Services, Managers, and Drivers, can be a bit confusing to people that are new to nova. This document attempts to outline the division of responsibilities to make understanding the system a little bit easier. + +Currently, Managers and Drivers are specified by flags and loaded using utils.load_object(). This method allows for them to be implemented as singletons, classes, modules or objects. As long as the path specified by the flag leads to an object (or a callable that returns an object) that responds to getattr, it should work as a manager or driver. + +Service +------- + +A service is a very thin wrapper around a Manager object. It exposes the manager's public methods to other components of the system via rpc. It will report state periodically to the database and is responsible for initiating any periodic tasts that need to be executed on a given host. + +The :mod:`service` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.service + :members: + :undoc-members: + :show-inheritance: + +Manager +------- + +Managers are responsible for a certain aspect of the sytem. It is a logical grouping of code relating to a portion of the system. In general other components should be using the manager to make changes to the components that it is responsible for. + +For example, other components that need to deal with volumes in some way, should do so by calling methods on the VolumeManager instead of directly changing fields in the database. This allows us to keep all of the code relating to volumes in the same place. + +We have adopted a basic strategy of Smart managers and dumb data, which means rather than attaching methods to data objects, components should call manager methods that act on the data. + +Methods on managers that can be executed locally should be called directly. If a particular method must execute on a remote host, this should be done via rpc to the service that wraps the manager + +Managers should be responsible for most of the db access, and non-implementation specific data. Anything implementation specific that can't be generalized should be done by the Driver. + +In general, we prefer to have one manager with multiple drivers for different implementations, but sometimes it makes sense to have multiple managers. You can think of it this way: Abstract different overall strategies at the manager level(FlatNetwork vs VlanNetwork), and different implementations at the driver level(LinuxNetDriver vs CiscoNetDriver). + +Managers will often provide methods for initial setup of a host or periodic tasksto a wrapping service. + +The :mod:`manager` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.manager + :members: + :undoc-members: + :show-inheritance: + +Driver +------ + +A manager will generally load a driver for some of its tasks. The driver is responsible for specific implementation details. Anything running shell commands on a host, or dealing with other non-python code should probably be happening in a driver. + +Drivers should minimize touching the database, although it is currently acceptable for implementation specific data. This may be reconsidered at some point. + +It usually makes sense to define an Abstract Base Class for the specific driver (i.e. VolumeDriver), to define the methods that a different driver would need to implement. diff --git a/doc/source/devguide/volume.rst b/doc/source/devguide/volume.rst new file mode 100644 index 000000000..39b33a500 --- /dev/null +++ b/doc/source/devguide/volume.rst @@ -0,0 +1,62 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Volume Programming Guide +========================= + +:: + + TODO(todd): rework after iSCSI merge (see 'Old Docs') + + +The :mod:`nova.volume.manager` Module +------------------------------------- + +.. automodule:: nova.volume.manager + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.volume.driver` Module +------------------------------------- + +.. automodule:: nova.volume.driver + :members: + :undoc-members: + :show-inheritance: + :exclude-members: FakeAOEDriver + +Tests +----- + +.. automodule:: nova.tests.volume_unittest + :members: + :undoc-members: + :show-inheritance: + +Old Docs +-------- + +Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. + +Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. + +AoE exports are numbered according to a "shelf and blade" syntax. In order to avoid collisions, we currently perform an AoE-discover of existing exports, and then grab the next unused number. (This obviously has race condition problems, and should be replaced by allocating a shelf-id to each storage node.) + +The underlying volumes are LVM logical volumes, created on demand within a single large volume group. + + diff --git a/doc/source/fakes.rst b/doc/source/fakes.rst deleted file mode 100644 index 5988e2ab8..000000000 --- a/doc/source/fakes.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Fakes Programming Guide -======================= - -:: - - TODO(todd): * document general info about fakes - - -The :mod:`virt.fake` Module ---------------------------- - -.. automodule:: nova.virt.fake - :members: - :undoc-members: - :show-inheritance: - :noindex: - -The :mod:`fakeldap` Module --------------------------- - -.. automodule:: nova.auth.fakeldap - :members: - :undoc-members: - :show-inheritance: - -The :mod:`fakerabbit` Module ----------------------------- - -.. automodule:: nova.fakerabbit - :members: - :undoc-members: - :show-inheritance: - -The :class:`FakeAOEDriver` Class --------------------------------- - -.. autoclass:: nova.volume.driver.FakeAOEDriver - :members: - :undoc-members: - :show-inheritance: - -The :class:`service_unittest.FakeManager` Class ------------------------------------------------ - -.. autoclass:: nova.tests.service_unittest.FakeManager - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api.openstack.fakes` Module ------------------------------------------------- - -.. automodule:: nova.tests.api.openstack.fakes - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/source/flags.rst b/doc/source/flags.rst deleted file mode 100644 index 4c950aa88..000000000 --- a/doc/source/flags.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Flags and Flagfiles -=================== - -* python-gflags -* flagfiles -* list of flags by component (see concepts list) diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst deleted file mode 100644 index 79d2cf204..000000000 --- a/doc/source/getting.started.rst +++ /dev/null @@ -1,167 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Getting Started with Nova -========================= - -This code base is continually changing, so dependencies also change. If you -encounter any problems, see the :doc:`reaching.out` page. -The `contrib/nova.sh` script should be kept up to date, and may be a good -resource to review when debugging. - -The purpose of this document is to get a system installed that you can use to -test your setup assumptions. Working from this base installtion you can -tweak configurations and work with different flags to monitor interaction with -your hardware, network, and other factors that will allow you to determine -suitability for your deployment. After following this setup method, you should -be able to experiment with different managers, drivers, and flags to get the -best performance. - -Dependencies ------------- - -Related servers we rely on - -* **RabbitMQ**: messaging queue, used for all communication between components - -Optional servers - -* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by - setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP - (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. - There is a script in the sources (`nova/auth/slap.sh`) to install a very basic - openldap server on ubuntu. -* **ReDIS**: There is a fake ldap auth driver - `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was - created for testing ldap implementation on systems that don't have an easy - means to install ldap. -* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to - be avilable. Currently, only sqlite3 an mysql have been tested. - -Python libraries that we use (from pip-requires): - -.. literalinclude:: ../../tools/pip-requires - -Other libraries: - -* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available - from http://wiki.xensource.com/xenwiki/XCP_SDK or - http://community.citrix.com/cdn/xs/sdks. - -External unix tools that are required: - -* iptables -* ebtables -* gawk -* curl -* kvm -* libvirt -* dnsmasq -* vlan -* open-iscsi and iscsitarget (if you use iscsi volumes) -* aoetools and vblade-persist (if you use aoe-volumes) - -Nova uses cutting-edge versions of many packages. There are ubuntu packages in -the nova-core ppa. You can use add this ppa to your sources list on an ubuntu -machine with the following commands:: - - sudo apt-get install -y python-software-properties - sudo add-apt-repository ppa:nova-core/ppa - -Recommended ------------ - -* euca2ools: python implementation of aws ec2-tools and ami tools -* build tornado to use C module for evented section - - -Installation --------------- - -You can install from packages for your particular Linux distribution if they are -available. Otherwise you can install from source by checking out the source -files from the `Nova Source Code Repository `_ -and running:: - - python setup.py install - -Configuration ---------------- - -Configuring the host system -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As you read through the Administration Guide you will notice configuration hints -inline with documentation on the subsystem you are configuring. Presented in -this "Getting Started with Nova" document, we only provide what you need to -get started as quickly as possible. For a more detailed description of system -configuration, start reading through :doc:`multi.node.install`. - -* Create a volume group (you can use an actual disk for the volume group as - well):: - - # This creates a 1GB file to create volumes out of - dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 - losetup --show -f MY_FILE_PATH - # replace /dev/loop0 below with whatever losetup returns - # nova-volumes is the default for the --volume_group flag - vgcreate nova-volumes /dev/loop0 - - -Configuring Nova -~~~~~~~~~~~~~~~~ - -Configuration of the entire system is performed through python-gflags. The -best way to track configuration is through the use of a flagfile. - -A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary -when you launch it. Flagfiles for nova are typically stored in -``/etc/nova/nova.conf``, and flags specific to a certain program are stored in -``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another -flagfile, so typically a file like ``nova-manage.conf`` would have as its first -line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before -specifying overrides or additional options. - -A sample configuration to test the system follows:: - - --verbose - --nodaemon - --FAKE_subdomain=ec2 - --auth_driver=nova.auth.dbdriver.DbDriver - -Running ---------- - -There are many parts to the nova system, each with a specific function. They -are built to be highly-available, so there are may configurations they can be -run in (ie: on many machines, many listeners per machine, etc). This part -of the guide only gets you started quickly, to learn about HA options, see -:doc:`multi.node.install`. - -Launch supporting services - -* rabbitmq -* redis (optional) -* mysql (optional) - -Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` - -* nova-api -* nova-compute -* nova-objectstore -* nova-volume -* nova-scheduler diff --git a/doc/source/glance.rst b/doc/source/glance.rst deleted file mode 100644 index 3d0f4ebec..000000000 --- a/doc/source/glance.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Glance Integration Programming Guide -==================================== - -The :mod:`image.service` Module -------------------------------- - -.. automodule:: nova.image.service - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/index.rst b/doc/source/index.rst index 9a62570ed..0dcc1945e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -43,8 +43,8 @@ Contents :maxdepth: 1 concepts.and.introduction - administration.guide - development.guide + adminguide/index + devguide/index reaching.out diff --git a/doc/source/modules.rst b/doc/source/modules.rst deleted file mode 100644 index 9b31664b6..000000000 --- a/doc/source/modules.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova Documentation -================== - -This page contains the Nova Modules documentation. - -Modules: --------- - -.. toctree:: - :maxdepth: 4 - - auth - compute - fakes - nova - volume - -Removed: --------- - -* endpoint diff --git a/doc/source/monitoring.rst b/doc/source/monitoring.rst deleted file mode 100644 index e7766a6e7..000000000 --- a/doc/source/monitoring.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Monitoring -========== - -* components -* throughput -* exceptions -* hardware - -* ganglia -* syslog diff --git a/doc/source/multi.node.install.rst b/doc/source/multi.node.install.rst deleted file mode 100644 index 1d484e779..000000000 --- a/doc/source/multi.node.install.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Multi-Node Nova -=============== - -When you move beyond evaluating the technology and into building an actual -production environemnt, you will need to know how to configure your datacenter -and how to deploy components across your clusters. This guide should help you -through that process. - -Bare-metal Provisioning ------------------------ - -To install the base operating system you can use PXE booting. - -Deployment Technologies ------------------------ - -Once you have machines with a base operating system installation, you can deploy -code and configuration with your favorite tools: - -* Puppet -* Chef - -Types of Hosts --------------- - -A single machine in your cluster can act as one or more of the following types -of host: - -Nova Services - -* Network -* Compute -* Volume -* API -* Objectstore - -Other supporting services - -* Datastore -* Message Queue diff --git a/doc/source/network.rst b/doc/source/network.rst deleted file mode 100644 index 9426ece82..000000000 --- a/doc/source/network.rst +++ /dev/null @@ -1,124 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Networking Programming Guide -============================ - -:: - - TODO(todd): * document hardware specific commands (maybe in admin guide?) - * document a map between flags and managers/backends - -The :mod:`Manager` module -------------------------- - -.. automodule:: nova.network.manager - :members: - :undoc-members: - :show-inheritance: - -The :mod:`LinuxNet` driver --------------------------- - -.. automodule:: nova.network.linux_net - :members: - :undoc-members: - :show-inheritance: - -Tests ------ - -The :mod:`network_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.network_unittest - :members: - :undoc-members: - :show-inheritance: - - -legacy docs ------------ - -The nova networking components manage private networks, public IP addressing, VPN connectivity, and firewall rules. - -Components ----------- -There are several key components: - -* NetworkController (Manages address and vlan allocation) -* RoutingNode (NATs public IPs to private IPs, and enforces firewall rules) -* AddressingNode (runs DHCP services for private networks) -* BridgingNode (a subclass of the basic nova ComputeNode) -* TunnelingNode (provides VPN connectivity) - -Component Diagram ------------------ - -Overview:: - - (PUBLIC INTERNET) - | \ - / \ / \ - [RoutingNode] ... [RN] [TunnelingNode] ... [TN] - | \ / | | - | < AMQP > | | - [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] - \ | \ / - / \ / \ / \ / \ - [BridgingNode] ... [BridgingNode] - - - [NetworkController] ... [NetworkController] - \ / - < AMQP > - | - / \ - [CloudController]...[CloudController] - -While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). - -State Model ------------ -Network State consists of the following facts: - -* VLAN assignment (to a project) -* Private Subnet assignment (to a security group) in a VLAN -* Private IP assignments (to running instances) -* Public IP allocations (to a project) -* Public IP associations (to a private IP / running instance) - -While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. - -The Public Traffic Path ------------------------ - -Public Traffic:: - - (PUBLIC INTERNET) - | - <-- [RoutingNode] - | - [AddressingNode] --> | - ( VLAN ) - | <-- [BridgingNode] - | - - -The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. - -Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). diff --git a/doc/source/nova.manage.rst b/doc/source/nova.manage.rst deleted file mode 100644 index eeb463f52..000000000 --- a/doc/source/nova.manage.rst +++ /dev/null @@ -1,200 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - - -nova-manage -=========== - -Introduction -~~~~~~~~~~~~ - -The nova-manage command is used to perform many essential functions for -administration and ongoing maintenance of nova, such as user creation, -vpn management, and much more. - -The standard pattern for executing a nova-manage command is: - -``nova-manage []`` - -For example, to obtain a list of all projects: - -``nova-manage project list`` - -You can run without arguments to see a list of available command categories: - -``nova-manage`` - -You can run with a category argument to see a list of all commands in that -category: - -``nova-manage user`` - -User Maintenance -~~~~~~~~~~~~~~~~ - -Users, including admins, are created through the ``user`` commands. - -* user admin: creates a new admin and prints exports - * arguments: name [access] [secret] -* user create: creates a new user and prints exports - * arguments: name [access] [secret] -* user delete: deletes an existing user - * arguments: name -* user exports: prints access and secrets for user in export format - * arguments: name -* user list: lists all users - * arguments: none -* user modify: update a users keys & admin flag - * arguments: accesskey secretkey admin - * leave any field blank to ignore it, admin should be 'T', 'F', or blank - -Project Maintenance -~~~~~~~~~~~~~~~~~~~ - -* project add: Adds user to project - * arguments: project user -* project create: Creates a new project - * arguments: name project_manager [description] -* project delete: Deletes an existing project - * arguments: project_id -* project environment: Exports environment variables to an sourcable file - * arguments: project_id user_id [filename='novarc] -* project list: lists all projects - * arguments: none -* project quota: Set or display quotas for project - * arguments: project_id [key] [value] -* project remove: Removes user from project - * arguments: project user -* project scrub: Deletes data associated with project - * arguments: project -* project zipfile: Exports credentials for project to a zip file - * arguments: project_id user_id [filename='nova.zip] - -User Role Management -~~~~~~~~~~~~~~~~~~~~ - -* role add: adds role to user - * if project is specified, adds project specific role - * arguments: user, role [project] -* role has: checks to see if user has role - * if project is specified, returns True if user has - the global role and the project role - * arguments: user, role [project] -* role remove: removes role from user - * if project is specified, removes project specific role - * arguments: user, role [project] - - -Nova Shell -~~~~~~~~~~ - -* shell bpython - * start a new bpython shell -* shell ipython - * start a new ipython shell -* shell python - * start a new python shell -* shell run - * ??? -* shell script: Runs the script from the specifed path with flags set properly. - * arguments: path - -VPN Management -~~~~~~~~~~~~~~ - -* vpn list: Print a listing of the VPNs for all projects. - * arguments: none -* vpn run: Start the VPN for a given project. - * arguments: project -* vpn spawn: Run all VPNs. - * arguments: none - - -Floating IP Management -~~~~~~~~~~~~~~~~~~~~~~ - -* floating create: Creates floating ips for host by range - * arguments: host ip_range -* floating delete: Deletes floating ips by range - * arguments: range -* floating list: Prints a listing of all floating ips - * arguments: none - -Network Management -~~~~~~~~~~~~~~~~~~ - -* network create: Creates fixed ips for host by range - * arguments: [fixed_range=FLAG], [num_networks=FLAG], - [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG] - - -Concept: Flags --------------- - -python-gflags - - -Concept: Plugins ----------------- - -* Managers/Drivers: utils.import_object from string flag -* virt/connections: conditional loading from string flag -* db: LazyPluggable via string flag -* auth_manager: utils.import_class based on string flag -* Volumes: moving to pluggable driver instead of manager -* Network: pluggable managers -* Compute: same driver used, but pluggable at connection - - -Concept: IPC/RPC ----------------- - -Rabbit! - - -Concept: Fakes --------------- - -* auth -* ldap - - -Concept: Scheduler ------------------- - -* simple -* random - - -Concept: Security Groups ------------------------- - -Security groups - - -Concept: Certificate Authority ------------------------------- - -Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns ` and decrypting bundled images. - - -Concept: Images ---------------- - -* launching -* bundling diff --git a/doc/source/nova.rst b/doc/source/nova.rst deleted file mode 100644 index 58125dc80..000000000 --- a/doc/source/nova.rst +++ /dev/null @@ -1,190 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova Libraries Programming Guide -================================ - -The :mod:`adminclient` Module ------------------------------ - -.. automodule:: nova.adminclient - :members: - :undoc-members: - :show-inheritance: - -The :mod:`context` Module -------------------------- - -.. automodule:: nova.context - :members: - :undoc-members: - :show-inheritance: - -The :mod:`exception` Module ---------------------------- - -.. automodule:: nova.exception - :members: - :undoc-members: - :show-inheritance: - -The :mod:`flags` Module ------------------------ - -.. automodule:: nova.flags - :members: - :undoc-members: - :show-inheritance: - -The :mod:`process` Module -------------------------- - -.. automodule:: nova.process - :members: - :undoc-members: - :show-inheritance: - -The :mod:`rpc` Module ---------------------- - -.. automodule:: nova.rpc - :members: - :undoc-members: - :show-inheritance: - -The :mod:`server` Module ------------------------- - -.. automodule:: nova.server - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test` Module ----------------------- - -.. automodule:: nova.test - :members: - :undoc-members: - :show-inheritance: - -The :mod:`twistd` Module ------------------------- - -.. automodule:: nova.twistd - :members: - :undoc-members: - :show-inheritance: - -The :mod:`utils` Module ------------------------ - -.. automodule:: nova.utils - :members: - :undoc-members: - :show-inheritance: - -The :mod:`validate` Module --------------------------- - -.. automodule:: nova.validate - :members: - :undoc-members: - :show-inheritance: - -The :mod:`wsgi` Module ----------------------- - -.. automodule:: nova.wsgi - :members: - :undoc-members: - :show-inheritance: - -Tests ------ - -The :mod:`declare_flags` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.declare_flags - :members: - :undoc-members: - :show-inheritance: - -The :mod:`fake_flags` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.fake_flags - :members: - :undoc-members: - :show-inheritance: - -The :mod:`flags_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.flags_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`process_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.process_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`real_flags` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.real_flags - :members: - :undoc-members: - :show-inheritance: - -The :mod:`rpc_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.rpc_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`runtime_flags` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.runtime_flags - :members: - :undoc-members: - :show-inheritance: - -The :mod:`twistd_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.twistd_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`validator_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.validator_unittest - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/objectstore.rst b/doc/source/objectstore.rst deleted file mode 100644 index 4087b5dd0..000000000 --- a/doc/source/objectstore.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Objectstore Programming Guide -============================= - -The :mod:`handler` Module -------------------------- - -.. automodule:: nova.objectstore.handler - :members: - :undoc-members: - :show-inheritance: - -The :mod:`bucket` Module ------------------------- - -.. automodule:: nova.objectstore.bucket - :members: - :undoc-members: - :show-inheritance: - -The :mod:`stored` Module ------------------------- - -.. automodule:: nova.objectstore.stored - :members: - :undoc-members: - :show-inheritance: - -The :mod:`image` Module ------------------------ - -.. automodule:: nova.objectstore.image - :members: - :undoc-members: - :show-inheritance: - -Tests ------ - -The :mod:`objectstore_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.objectstore_unittest - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst deleted file mode 100644 index 511ed2483..000000000 --- a/doc/source/quickstart.rst +++ /dev/null @@ -1,104 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova Quickstart -=============== - -:: - - TODO(todd): * Document the assumptions about pluggable interfaces - (sqlite3 instead of mysql, etc) - * Document env vars that can change things - (USE_MYSQL, HOST_IP) - - -The `contrib/novascript.sh` file in the source distribution is a script that -will quickly set up nova to run on a single machine. It is tested against -Ubuntu only, but other distributions are forthcoming. - -Usage ------ - -Unless you want to spend a lot of time fiddling with permissions and sudoers, -you should probably run nova as root. - -:: - - sudo -i - -If you are concerned about security, nova runs just fine inside a virtual -machine. - -Use the script to install and run the current trunk. You can also specify a -specific branch by putting `lp:~someone/nova/some-branch` after the branch -command - -:: - - ./nova.sh branch - ./nova.sh install - ./nova.sh run - -The run command will drop you into a screen session with all of the workers -running in different windows You can use eucatools to run commands against the -cloud. - -:: - - euca-add-keypair test > test.pem - euca-run-instances -k test -t m1.tiny ami-tiny - euca-describe-instances - -To see output from the various workers, switch screen windows - -:: - - " - -will give you a list of running windows. - -When the instance is running, you should be able to ssh to it. - -:: - - chmod 600 test.pem - ssh -i test.pem root@10.0.0.3 - -When you exit screen - -:: - - - -nova will terminate. It may take a while for nova to finish cleaning up. If -you exit the process before it is done because there were some problems in your -build, you may have to clean up the nova processes manually. If you had any -instances running, you can attempt to kill them through the api: - -:: - - ./nova.sh terminate - -Then you can destroy the screen: - -:: - - ./nova.sh clean - -If things get particularly messed up, you might need to do some more intense -cleanup. Be careful, the following command will manually destroy all runnning -virsh instances and attempt to delete all vlans and bridges. diff --git a/doc/source/scheduler.rst b/doc/source/scheduler.rst deleted file mode 100644 index df820c2f2..000000000 --- a/doc/source/scheduler.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Scheduler Programming Guide -=========================== - -The :mod:`manager` Module -------------------------- - -.. automodule:: nova.scheduler.manager - :members: - :undoc-members: - :show-inheritance: - -The :mod:`driver` Module ------------------------- - -.. automodule:: nova.scheduler.driver - :members: - :undoc-members: - :show-inheritance: - -The :mod:`chance` Module ------------------------- - -.. automodule:: nova.scheduler.chance - :members: - :undoc-members: - :show-inheritance: - -The :mod:`simple` Module ------------------------- - -.. automodule:: nova.scheduler.simple - :members: - :undoc-members: - :show-inheritance: - -Tests ------ - -The :mod:`scheduler_unittest` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.tests.scheduler_unittest - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/services.rst b/doc/source/services.rst deleted file mode 100644 index 517a51901..000000000 --- a/doc/source/services.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _service_manager_driver: - -Services Managers and Drivers -============================= - -The responsibilities of Services, Managers, and Drivers, can be a bit confusing to people that are new to nova. This document attempts to outline the division of responsibilities to make understanding the system a little bit easier. - -Currently, Managers and Drivers are specified by flags and loaded using utils.load_object(). This method allows for them to be implemented as singletons, classes, modules or objects. As long as the path specified by the flag leads to an object (or a callable that returns an object) that responds to getattr, it should work as a manager or driver. - -Service -------- - -A service is a very thin wrapper around a Manager object. It exposes the manager's public methods to other components of the system via rpc. It will report state periodically to the database and is responsible for initiating any periodic tasts that need to be executed on a given host. - -The :mod:`service` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.service - :members: - :undoc-members: - :show-inheritance: - -Manager -------- - -Managers are responsible for a certain aspect of the sytem. It is a logical grouping of code relating to a portion of the system. In general other components should be using the manager to make changes to the components that it is responsible for. - -For example, other components that need to deal with volumes in some way, should do so by calling methods on the VolumeManager instead of directly changing fields in the database. This allows us to keep all of the code relating to volumes in the same place. - -We have adopted a basic strategy of Smart managers and dumb data, which means rather than attaching methods to data objects, components should call manager methods that act on the data. - -Methods on managers that can be executed locally should be called directly. If a particular method must execute on a remote host, this should be done via rpc to the service that wraps the manager - -Managers should be responsible for most of the db access, and non-implementation specific data. Anything implementation specific that can't be generalized should be done by the Driver. - -In general, we prefer to have one manager with multiple drivers for different implementations, but sometimes it makes sense to have multiple managers. You can think of it this way: Abstract different overall strategies at the manager level(FlatNetwork vs VlanNetwork), and different implementations at the driver level(LinuxNetDriver vs CiscoNetDriver). - -Managers will often provide methods for initial setup of a host or periodic tasksto a wrapping service. - -The :mod:`manager` Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: nova.manager - :members: - :undoc-members: - :show-inheritance: - -Driver ------- - -A manager will generally load a driver for some of its tasks. The driver is responsible for specific implementation details. Anything running shell commands on a host, or dealing with other non-python code should probably be happening in a driver. - -Drivers should minimize touching the database, although it is currently acceptable for implementation specific data. This may be reconsidered at some point. - -It usually makes sense to define an Abstract Base Class for the specific driver (i.e. VolumeDriver), to define the methods that a different driver would need to implement. diff --git a/doc/source/volume.rst b/doc/source/volume.rst deleted file mode 100644 index 39b33a500..000000000 --- a/doc/source/volume.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Volume Programming Guide -========================= - -:: - - TODO(todd): rework after iSCSI merge (see 'Old Docs') - - -The :mod:`nova.volume.manager` Module -------------------------------------- - -.. automodule:: nova.volume.manager - :members: - :undoc-members: - :show-inheritance: - -The :mod:`nova.volume.driver` Module -------------------------------------- - -.. automodule:: nova.volume.driver - :members: - :undoc-members: - :show-inheritance: - :exclude-members: FakeAOEDriver - -Tests ------ - -.. automodule:: nova.tests.volume_unittest - :members: - :undoc-members: - :show-inheritance: - -Old Docs --------- - -Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. - -Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. - -AoE exports are numbered according to a "shelf and blade" syntax. In order to avoid collisions, we currently perform an AoE-discover of existing exports, and then grab the next unused number. (This obviously has race condition problems, and should be replaced by allocating a shelf-id to each storage node.) - -The underlying volumes are LVM logical volumes, created on demand within a single large volume group. - - -- cgit From 990f4451e7779654188afa514baef270182f8697 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 5 Nov 2010 14:09:36 -0400 Subject: update types of services that may run on machines. --- doc/source/adminguide/getting.started.rst | 1 + doc/source/adminguide/multi.node.install.rst | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/adminguide/getting.started.rst b/doc/source/adminguide/getting.started.rst index 65ba2d57e..1a5e913ce 100644 --- a/doc/source/adminguide/getting.started.rst +++ b/doc/source/adminguide/getting.started.rst @@ -157,6 +157,7 @@ Launch supporting services * rabbitmq * redis (optional) * mysql (optional) +* openldap (optional) Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst index 1d484e779..5ea0b6038 100644 --- a/doc/source/adminguide/multi.node.install.rst +++ b/doc/source/adminguide/multi.node.install.rst @@ -53,5 +53,6 @@ Nova Services Other supporting services -* Datastore * Message Queue +* Database (optional) +* Authentication database (optional) -- cgit From f9bd8f41807a27d62b538707b9aaff44746d8933 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 5 Nov 2010 14:30:46 -0400 Subject: Change order of secions so puppeting is last, add more initial setup tasks. --- doc/source/adminguide/multi.node.install.rst | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst index 5ea0b6038..eaf8dd966 100644 --- a/doc/source/adminguide/multi.node.install.rst +++ b/doc/source/adminguide/multi.node.install.rst @@ -28,15 +28,6 @@ Bare-metal Provisioning To install the base operating system you can use PXE booting. -Deployment Technologies ------------------------ - -Once you have machines with a base operating system installation, you can deploy -code and configuration with your favorite tools: - -* Puppet -* Chef - Types of Hosts -------------- @@ -56,3 +47,19 @@ Other supporting services * Message Queue * Database (optional) * Authentication database (optional) + +Initial Setup +------------- + +* Networking +* Cloudadmin User Creation + +Deployment Technologies +----------------------- + +Once you have machines with a base operating system installation, you can deploy +code and configuration with your favorite tools to specify which machines in +your cluster have which roles: + +* Puppet +* Chef -- cgit From 0a4785be35bf1fc66e41b7565edf5e3936b70673 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 5 Nov 2010 11:56:12 -0700 Subject: additions to home page --- doc/source/index.rst | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 0dcc1945e..5dec404ef 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -27,6 +27,7 @@ Nova is written with the following design guidelines in mind: * **Fault-Tollerant**: Isloated processes avoid cascading failures * **Recoverable**: Failures should be easy to diagnose, debug, and rectify * **Open Standards**: Be a reference implementation for a community-driven api +* **API Compatibility**: Nova strives to provide API-compatible with popular systems like Amazon EC2 This documentation is generated by the Sphinx toolkit and lives in the source tree. Additional documentation on Nova and other components of OpenStack can @@ -47,6 +48,115 @@ Contents devguide/index reaching.out +Recommended System Configuration +================================ + +Although Nova can be run on a variety of system architectures, for most users the following will be simplest: + +* Ubuntu Lucid +* 10GB Hard Disk Space +* 512MB RAM + +For development, Nova can run from within a VM. + +Quickstart +========== + +To make getting started with Nova easier, we provide a setup script that makes it easy to download and run the most recent version of Nova on your local machine. + +* sudo -i # become root +* cd # go to home directory +* git clone git://github.com/vishvananda/novascript.git +* cd novascript + +Inside this directory, you will find nova.sh, which is a utility designed to assist with getting your development environment up and running. + +Use nova.sh to install and run the current trunk. You can also specify a specific branch by putting lp:~someone/nova/some-branch after the branch command + +* ./nova.sh branch +* ./nova.sh install +* ./nova.sh run + +The run command will drop you into a screen session with all of the workers running in different windows You can use eucatools to run commands against the cloud. + +* euca-add-keypair test > test.pem +* euca-run-instances -k test -t m1.tiny ami-tiny +* euca-describe-instances + +To see output from the various workers, switch screen windows + +* " + +will give you a list of running windows. + +When the instance is running, you should be able to ssh to it. + +* chmod 600 test.pem +* ssh -i test.pem root@10.0.0.3 + +When you exit screen + +* + +nova will terminate. It may take a while for nova to finish cleaning up. If you exit the process before it is done because there were some problems in your build, you may have to clean up the nova processes manually. If you had any instances running, you can attempt to kill them through the api: + +* ./nova.sh terminate + +Then you can destroy the screen: + +* ./nova.sh clean + +If things get particularly messed up, you might need to do some more intense cleanup. Be careful, the following command will manually destroy all runnning virsh instances and attempt to delete all vlans and bridges. + +* ./nova.sh scrub + +You can edit files in the install directory or do a bzr pull to pick up new versions. You only need to do + +* ./nova.sh run + +to run nova after the first install. The database should be cleaned up on each run. + +Notes +----- + +The script starts nova-volume in fake mode, so it will not create any actual volumes. + +if you want to USE_VENV because you have different versions of python packages on your system that you want to keep, you should run install before branch: + +* ./nova.sh install +* ./nova.sh branch +* ./nova.sh run + +A sample image should be downloaded by the script, but if necessary you can download it by hand: + +* wget http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz + +untar the file to create a usable images directory + +* tar -zxf /path/to/images.tgz + +If you want to be able to contact the metadata server and route to the outside world from instances, you will need to make sure $HOST_IP is set properly. The script attemps to grab it from ifconfig, but if you have multiple adapters set up, it may fail. Fix it with export HOST_IP="": + +Customization +------------- + +You can make nova use mysql instead of sqlite with USE_MYSQL, it will attempt to install mysql with the specified root password and create a database called nova. + +If you are running nova on bare metal that supports hardware virtualization, you should probably edit the libvirt line near the top + +* LIBVIRT_TYPE=kvm + +If you are running in a virtual machine and software emulation is too slow for you, you can use user mode linux. + +* LIBVIRT_TYPE=uml + +You will need a few bleeding edge packages to make it work, so you should make sure to use the PPA. + +* USE_PPA=1 + +If you have any issues, there is usually someone in #openstack on irc.freenode.net that can help you out. + + Indices and tables ================== -- cgit From c5e616f5908c0c5966c0a3612b9cc565387f50a9 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 5 Nov 2010 15:11:04 -0400 Subject: Format TODO items for sphinx todo extension. --- doc/source/adminguide/quickstart.rst | 9 ++++----- doc/source/devguide/api.rst | 4 +--- doc/source/devguide/development.environment.rst | 4 +--- doc/source/devguide/fakes.rst | 4 +--- doc/source/devguide/network.rst | 6 +++--- doc/source/devguide/volume.rst | 4 +--- doc/source/index.rst | 5 +++++ doc/source/reaching.out.rst | 5 +---- 8 files changed, 17 insertions(+), 24 deletions(-) diff --git a/doc/source/adminguide/quickstart.rst b/doc/source/adminguide/quickstart.rst index 511ed2483..718ea7d14 100644 --- a/doc/source/adminguide/quickstart.rst +++ b/doc/source/adminguide/quickstart.rst @@ -18,12 +18,11 @@ Nova Quickstart =============== -:: +.. todo:: - TODO(todd): * Document the assumptions about pluggable interfaces - (sqlite3 instead of mysql, etc) - * Document env vars that can change things - (USE_MYSQL, HOST_IP) + * Document the assumptions about pluggable interfaces (sqlite3 instead of + mysql, etc) (todd) + * Document env vars that can change things (USE_MYSQL, HOST_IP) (todd) The `contrib/novascript.sh` file in the source distribution is a script that diff --git a/doc/source/devguide/api.rst b/doc/source/devguide/api.rst index 0908d9e57..d1d50d745 100644 --- a/doc/source/devguide/api.rst +++ b/doc/source/devguide/api.rst @@ -18,9 +18,7 @@ API Endpoint Programming Guide ============================== -:: - - TODO(todd): get actual docstrings from ec2/osapi_verions instead of @wsgify +.. todo:: get actual docstrings from ec2/osapi_verions instead of @wsgify Nova has a system for managing multiple APIs on different subdomains. Currently there is support for the OpenStack API, as well as the Amazon EC2 diff --git a/doc/source/devguide/development.environment.rst b/doc/source/devguide/development.environment.rst index ee19ba2bf..34104c964 100644 --- a/doc/source/devguide/development.environment.rst +++ b/doc/source/devguide/development.environment.rst @@ -18,6 +18,4 @@ Setting up a development environment ==================================== -:: - - TODO(anthony) +.. todo:: write this diff --git a/doc/source/devguide/fakes.rst b/doc/source/devguide/fakes.rst index 5988e2ab8..61622754c 100644 --- a/doc/source/devguide/fakes.rst +++ b/doc/source/devguide/fakes.rst @@ -18,9 +18,7 @@ Fakes Programming Guide ======================= -:: - - TODO(todd): * document general info about fakes +.. todo:: document general info about fakes The :mod:`virt.fake` Module diff --git a/doc/source/devguide/network.rst b/doc/source/devguide/network.rst index 9426ece82..318286383 100644 --- a/doc/source/devguide/network.rst +++ b/doc/source/devguide/network.rst @@ -18,10 +18,10 @@ Networking Programming Guide ============================ -:: +.. todo:: - TODO(todd): * document hardware specific commands (maybe in admin guide?) - * document a map between flags and managers/backends + * document hardware specific commands (maybe in admin guide?) (todd) + * document a map between flags and managers/backends (todd) The :mod:`Manager` module ------------------------- diff --git a/doc/source/devguide/volume.rst b/doc/source/devguide/volume.rst index 39b33a500..19b750870 100644 --- a/doc/source/devguide/volume.rst +++ b/doc/source/devguide/volume.rst @@ -18,9 +18,7 @@ Volume Programming Guide ========================= -:: - - TODO(todd): rework after iSCSI merge (see 'Old Docs') +.. todo:: rework after iSCSI merge (see 'Old Docs') (todd or vish) The :mod:`nova.volume.manager` Module diff --git a/doc/source/index.rst b/doc/source/index.rst index 0dcc1945e..58c3cad58 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -48,6 +48,11 @@ Contents reaching.out +Outstanding Documentation Tasks +=============================== + +.. todolist:: + Indices and tables ================== diff --git a/doc/source/reaching.out.rst b/doc/source/reaching.out.rst index 0d8f675e0..064d2d87a 100644 --- a/doc/source/reaching.out.rst +++ b/doc/source/reaching.out.rst @@ -22,7 +22,4 @@ The OpenStack community is a very friendly place. Feel free to ask questions. This document points you to some of the places you may want to communicate with people at. -:: - - TODO(todd): * Write This - * Don't end a sentence with a preposition +.. todo:: Write reaching.out.rst (todd) -- cgit From dd505245c27d2abd2d710a44a120b21cacb9089c Mon Sep 17 00:00:00 2001 From: root Date: Fri, 5 Nov 2010 12:49:42 -0700 Subject: add in custom todo, and custom css --- doc/ext/__init__.py | 0 doc/ext/nova_todo.py | 86 +++++++++++++++++++++++++++++++++++++++++++ doc/source/_static/tweaks.css | 44 ++++++++++++++++++++++ doc/source/_theme/layout.html | 2 + doc/source/_theme/theme.conf | 5 +++ doc/source/conf.py | 8 ++-- 6 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 doc/ext/__init__.py create mode 100644 doc/ext/nova_todo.py create mode 100644 doc/source/_static/tweaks.css create mode 100644 doc/source/_theme/layout.html create mode 100644 doc/source/_theme/theme.conf diff --git a/doc/ext/__init__.py b/doc/ext/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/doc/ext/nova_todo.py b/doc/ext/nova_todo.py new file mode 100644 index 000000000..7a06b1bf9 --- /dev/null +++ b/doc/ext/nova_todo.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# This is a hack of the builtin todo extension, to make the todo_list more user friendly + +from sphinx.ext.todo import * +from docutils.parsers.rst import directives + +def _(s): + return s + +def process_todo_nodes(app, doctree, fromdocname): + if not app.config['todo_include_todos']: + for node in doctree.traverse(todo_node): + node.parent.remove(node) + + # Replace all todolist nodes with a list of the collected todos. + # Augment each todo with a backlink to the original location. + env = app.builder.env + + if not hasattr(env, 'todo_all_todos'): + env.todo_all_todos = [] + + my_todo_list = nodes.bullet_list("", nodes.Text('','')); + + # remove the item that was added in the constructor, since I'm tired of + # reading through docutils for the proper way to construct an empty list + my_todo_list.remove(my_todo_list[0]) + + my_todo_list.set_class('todo_list') + for node in doctree.traverse(todolist): + if not app.config['todo_include_todos']: + node.replace_self([]) + continue + + content = [] + + for todo_info in env.todo_all_todos: + para = nodes.paragraph() + filename = env.doc2path(todo_info['docname'], base=None) + + # Create a reference + newnode = nodes.reference('', '') + + link = _('%s, line %d') % (filename, todo_info['lineno']); + innernode = nodes.emphasis(link, link) + newnode['refdocname'] = todo_info['docname'] + + try: + newnode['refuri'] = app.builder.get_relative_uri( + fromdocname, todo_info['docname']) + newnode['refuri'] += '#' + todo_info['target']['refid'] + except NoUri: + # ignore if no URI can be determined, e.g. for LaTeX output + pass + + newnode.append(innernode) + para += newnode + para.set_class("link") + + todo_entry = todo_info['todo'] + + env.resolve_references(todo_entry, todo_info['docname'], app.builder) + + item = nodes.list_item("", para) + todo_entry[1].set_class("details") + item.append(todo_entry[1]) + + my_todo_list.insert(0, item) + + + node.replace_self(my_todo_list) + +def setup(app): + app.add_config_value('todo_include_todos', False, False) + + app.add_node(todolist) + app.add_node(todo_node, + html=(visit_todo_node, depart_todo_node), + latex=(visit_todo_node, depart_todo_node), + text=(visit_todo_node, depart_todo_node)) + + app.add_directive('todo', Todo) + app.add_directive('todolist', TodoList) + app.connect('doctree-read', process_todos) + app.connect('doctree-resolved', process_todo_nodes) + app.connect('env-purge-doc', purge_todos) + diff --git a/doc/source/_static/tweaks.css b/doc/source/_static/tweaks.css new file mode 100644 index 000000000..d1cbed96d --- /dev/null +++ b/doc/source/_static/tweaks.css @@ -0,0 +1,44 @@ +ul.todo_list { + list-style-type: none; + margin: 0; + padding: 0; +} + +ul.todo_list li { + display: block; + margin: 0; + padding: 7px 0; + border-top: 1px solid #eee; +} + +ul.todo_list li p { + display: inline; +} + +ul.todo_list li p.link { + font-weight: bold; +} + +ul.todo_list li p.details { + font-style: italic; +} + +ul.todo_list li { +} + +div.admonition { + border: 1px solid #FF6666; +} + +div.admonition p.admonition-title { + background-color: #FF6666; + border-bottom: 1px solid #FF6666; +} + +em { + font-style: normal; +} + +table.docutils { + font-size: 11px; +} \ No newline at end of file diff --git a/doc/source/_theme/layout.html b/doc/source/_theme/layout.html new file mode 100644 index 000000000..ed1cab0a6 --- /dev/null +++ b/doc/source/_theme/layout.html @@ -0,0 +1,2 @@ +{% extends "sphinxdoc/layout.html" %} +{% set css_files = css_files + ['_static/tweaks.css'] %} diff --git a/doc/source/_theme/theme.conf b/doc/source/_theme/theme.conf new file mode 100644 index 000000000..e039fe01f --- /dev/null +++ b/doc/source/_theme/theme.conf @@ -0,0 +1,5 @@ +[theme] +inherit = sphinxdoc +stylesheet = sphinxdoc.css +pygments_style = friendly + diff --git a/doc/source/conf.py b/doc/source/conf.py index 3f6ca2313..177c6568e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -17,13 +17,14 @@ import sys, os # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../../')) - +sys.path.insert(0, '../') +sys.path.insert(0, './') # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'ext.nova_todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig'] todo_include_todos = True # Add any paths that contain templates here, relative to this directory. @@ -99,7 +100,8 @@ modindex_common_prefix = ['nova.'] # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'sphinxdoc' +html_theme_path = ["."] +html_theme = '_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the -- cgit From 3ef09f689b7f151aa9225b69d85c96fd39fd2c9c Mon Sep 17 00:00:00 2001 From: root Date: Fri, 5 Nov 2010 13:25:40 -0700 Subject: use include to grab todd's quickstart --- doc/source/index.rst | 98 +--------------------------------------------------- 1 file changed, 1 insertion(+), 97 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 5dec404ef..bfc2984fe 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -59,103 +59,7 @@ Although Nova can be run on a variety of system architectures, for most users th For development, Nova can run from within a VM. -Quickstart -========== - -To make getting started with Nova easier, we provide a setup script that makes it easy to download and run the most recent version of Nova on your local machine. - -* sudo -i # become root -* cd # go to home directory -* git clone git://github.com/vishvananda/novascript.git -* cd novascript - -Inside this directory, you will find nova.sh, which is a utility designed to assist with getting your development environment up and running. - -Use nova.sh to install and run the current trunk. You can also specify a specific branch by putting lp:~someone/nova/some-branch after the branch command - -* ./nova.sh branch -* ./nova.sh install -* ./nova.sh run - -The run command will drop you into a screen session with all of the workers running in different windows You can use eucatools to run commands against the cloud. - -* euca-add-keypair test > test.pem -* euca-run-instances -k test -t m1.tiny ami-tiny -* euca-describe-instances - -To see output from the various workers, switch screen windows - -* " - -will give you a list of running windows. - -When the instance is running, you should be able to ssh to it. - -* chmod 600 test.pem -* ssh -i test.pem root@10.0.0.3 - -When you exit screen - -* - -nova will terminate. It may take a while for nova to finish cleaning up. If you exit the process before it is done because there were some problems in your build, you may have to clean up the nova processes manually. If you had any instances running, you can attempt to kill them through the api: - -* ./nova.sh terminate - -Then you can destroy the screen: - -* ./nova.sh clean - -If things get particularly messed up, you might need to do some more intense cleanup. Be careful, the following command will manually destroy all runnning virsh instances and attempt to delete all vlans and bridges. - -* ./nova.sh scrub - -You can edit files in the install directory or do a bzr pull to pick up new versions. You only need to do - -* ./nova.sh run - -to run nova after the first install. The database should be cleaned up on each run. - -Notes ------ - -The script starts nova-volume in fake mode, so it will not create any actual volumes. - -if you want to USE_VENV because you have different versions of python packages on your system that you want to keep, you should run install before branch: - -* ./nova.sh install -* ./nova.sh branch -* ./nova.sh run - -A sample image should be downloaded by the script, but if necessary you can download it by hand: - -* wget http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz - -untar the file to create a usable images directory - -* tar -zxf /path/to/images.tgz - -If you want to be able to contact the metadata server and route to the outside world from instances, you will need to make sure $HOST_IP is set properly. The script attemps to grab it from ifconfig, but if you have multiple adapters set up, it may fail. Fix it with export HOST_IP="": - -Customization -------------- - -You can make nova use mysql instead of sqlite with USE_MYSQL, it will attempt to install mysql with the specified root password and create a database called nova. - -If you are running nova on bare metal that supports hardware virtualization, you should probably edit the libvirt line near the top - -* LIBVIRT_TYPE=kvm - -If you are running in a virtual machine and software emulation is too slow for you, you can use user mode linux. - -* LIBVIRT_TYPE=uml - -You will need a few bleeding edge packages to make it work, so you should make sure to use the PPA. - -* USE_PPA=1 - -If you have any issues, there is usually someone in #openstack on irc.freenode.net that can help you out. - +.. include:: adminguide/quickstart.rst Indices and tables -- cgit From 64ce9316b694ae286ce8caaeb9c486dc9af12c50 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 5 Nov 2010 13:44:22 -0700 Subject: small tweaks before context switch --- doc/source/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index bfc2984fe..4da06740c 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -18,7 +18,7 @@ Welcome to Nova's documentation! ================================ -Nova is a cloud computing fabric controller, the main part of an IaaS system. +Nova is a cloud computing fabric controller, the main part of an IaaS system. Individuals and organizations can use it to host and manage their own cloud computing systems. Nova originated as a project out of NASA Ames Research Laboratory. Nova is written with the following design guidelines in mind: @@ -32,7 +32,7 @@ Nova is written with the following design guidelines in mind: This documentation is generated by the Sphinx toolkit and lives in the source tree. Additional documentation on Nova and other components of OpenStack can be found on the `OpenStack wiki`_. Also see the :doc:`reaching.out` page for -other ways to interact witht the community. +other ways to interact with the community. .. _`OpenStack wiki`: http://wiki.openstack.org -- cgit From 07f5d3abe26b57ef8d82ddaab80141e0130d95ac Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 5 Nov 2010 17:33:08 -0400 Subject: Community contact info. --- doc/source/reaching.out.rst | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/doc/source/reaching.out.rst b/doc/source/reaching.out.rst index 064d2d87a..ee7035aa3 100644 --- a/doc/source/reaching.out.rst +++ b/doc/source/reaching.out.rst @@ -22,4 +22,43 @@ The OpenStack community is a very friendly place. Feel free to ask questions. This document points you to some of the places you may want to communicate with people at. -.. todo:: Write reaching.out.rst (todd) + +#openstack on Freenode IRC Network +---------------------------------- + +There is a very active chat channel at ``_. This +is usually the best place to ask questions and find your way around. + +OpenStack Wiki +-------------- + +The wiki is a living source of knowledge. It is edited by the community, and +has collections of links and other sources of information. + +`OpenStack Wiki `_ + +Nova on Launchpad +----------------- + +Launchapd is a code hosting service that hosts the Nova source code. From +launchpad you can report bugs, ask questions, and register blueprints (feature +requests). + +`Launchpad Nova Page `_ + +OpenStack Blog +-------------- + +The OpenStack blog includes a weekly newsletter that aggregates OpenStack news +from around the internet, as well as providing inside information on upcoming +events and posts from OpenStack contributors. + +`OpenStack Blog `_ + +See also: `Planet OpenStack `_, aggregating blogs +about OpenStack from around the internet into a single feed. + +Twitter +------- + +Because all the cool kids do it: `@openstack `_. -- cgit From 9675be4ffd54db83b7417c968943238958ace9db Mon Sep 17 00:00:00 2001 From: root Date: Fri, 5 Nov 2010 15:29:54 -0700 Subject: add twitter feed to the home page --- doc/source/_static/jquery.tweet.js | 154 +++++++++++++++++++++++++++++++++++++ doc/source/_static/tweaks.css | 10 ++- doc/source/_theme/layout.html | 83 ++++++++++++++++++++ 3 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 doc/source/_static/jquery.tweet.js diff --git a/doc/source/_static/jquery.tweet.js b/doc/source/_static/jquery.tweet.js new file mode 100644 index 000000000..c93fea876 --- /dev/null +++ b/doc/source/_static/jquery.tweet.js @@ -0,0 +1,154 @@ +(function($) { + + $.fn.tweet = function(o){ + var s = { + username: ["seaofclouds"], // [string] required, unless you want to display our tweets. :) it can be an array, just do ["username1","username2","etc"] + list: null, //[string] optional name of list belonging to username + avatar_size: null, // [integer] height and width of avatar if displayed (48px max) + count: 3, // [integer] how many tweets to display? + intro_text: null, // [string] do you want text BEFORE your your tweets? + outro_text: null, // [string] do you want text AFTER your tweets? + join_text: null, // [string] optional text in between date and tweet, try setting to "auto" + auto_join_text_default: "i said,", // [string] auto text for non verb: "i said" bullocks + auto_join_text_ed: "i", // [string] auto text for past tense: "i" surfed + auto_join_text_ing: "i am", // [string] auto tense for present tense: "i was" surfing + auto_join_text_reply: "i replied to", // [string] auto tense for replies: "i replied to" @someone "with" + auto_join_text_url: "i was looking at", // [string] auto tense for urls: "i was looking at" http:... + loading_text: null, // [string] optional loading text, displayed while tweets load + query: null // [string] optional search query + }; + + if(o) $.extend(s, o); + + $.fn.extend({ + linkUrl: function() { + var returning = []; + var regexp = /((ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?)/gi; + this.each(function() { + returning.push(this.replace(regexp,"$1")); + }); + return $(returning); + }, + linkUser: function() { + var returning = []; + var regexp = /[\@]+([A-Za-z0-9-_]+)/gi; + this.each(function() { + returning.push(this.replace(regexp,"@$1")); + }); + return $(returning); + }, + linkHash: function() { + var returning = []; + var regexp = / [\#]+([A-Za-z0-9-_]+)/gi; + this.each(function() { + returning.push(this.replace(regexp, ' #$1')); + }); + return $(returning); + }, + capAwesome: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/\b(awesome)\b/gi, '$1')); + }); + return $(returning); + }, + capEpic: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/\b(epic)\b/gi, '$1')); + }); + return $(returning); + }, + makeHeart: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/(<)+[3]/gi, "♥")); + }); + return $(returning); + } + }); + + function relative_time(time_value) { + var parsed_date = Date.parse(time_value); + var relative_to = (arguments.length > 1) ? arguments[1] : new Date(); + var delta = parseInt((relative_to.getTime() - parsed_date) / 1000); + var pluralize = function (singular, n) { + return '' + n + ' ' + singular + (n == 1 ? '' : 's'); + }; + if(delta < 60) { + return 'less than a minute ago'; + } else if(delta < (45*60)) { + return 'about ' + pluralize("minute", parseInt(delta / 60)) + ' ago'; + } else if(delta < (24*60*60)) { + return 'about ' + pluralize("hour", parseInt(delta / 3600)) + ' ago'; + } else { + return 'about ' + pluralize("day", parseInt(delta / 86400)) + ' ago'; + } + } + + function build_url() { + var proto = ('https:' == document.location.protocol ? 'https:' : 'http:'); + if (s.list) { + return proto+"//api.twitter.com/1/"+s.username[0]+"/lists/"+s.list+"/statuses.json?per_page="+s.count+"&callback=?"; + } else if (s.query == null && s.username.length == 1) { + return proto+'//twitter.com/status/user_timeline/'+s.username[0]+'.json?count='+s.count+'&callback=?'; + } else { + var query = (s.query || 'from:'+s.username.join('%20OR%20from:')); + return proto+'//search.twitter.com/search.json?&q='+query+'&rpp='+s.count+'&callback=?'; + } + } + + return this.each(function(){ + var list = $('
    ').appendTo(this); + var intro = '

    '+s.intro_text+'

    '; + var outro = '

    '+s.outro_text+'

    '; + var loading = $('

    '+s.loading_text+'

    '); + + if(typeof(s.username) == "string"){ + s.username = [s.username]; + } + + if (s.loading_text) $(this).append(loading); + $.getJSON(build_url(), function(data){ + if (s.loading_text) loading.remove(); + if (s.intro_text) list.before(intro); + $.each((data.results || data), function(i,item){ + // auto join text based on verb tense and content + if (s.join_text == "auto") { + if (item.text.match(/^(@([A-Za-z0-9-_]+)) .*/i)) { + var join_text = s.auto_join_text_reply; + } else if (item.text.match(/(^\w+:\/\/[A-Za-z0-9-_]+\.[A-Za-z0-9-_:%&\?\/.=]+) .*/i)) { + var join_text = s.auto_join_text_url; + } else if (item.text.match(/^((\w+ed)|just) .*/im)) { + var join_text = s.auto_join_text_ed; + } else if (item.text.match(/^(\w*ing) .*/i)) { + var join_text = s.auto_join_text_ing; + } else { + var join_text = s.auto_join_text_default; + } + } else { + var join_text = s.join_text; + }; + + var from_user = item.from_user || item.user.screen_name; + var profile_image_url = item.profile_image_url || item.user.profile_image_url; + var join_template = ' '+join_text+' '; + var join = ((s.join_text) ? join_template : ' '); + var avatar_template = ''+from_user+'\'s avatar'; + var avatar = (s.avatar_size ? avatar_template : ''); + var date = ''+relative_time(item.created_at)+''; + var text = '' +$([item.text]).linkUrl().linkUser().linkHash().makeHeart().capAwesome().capEpic()[0]+ ''; + + // until we create a template option, arrange the items below to alter a tweet's display. + list.append('
  • ' + avatar + date + join + text + '
    • '); + + list.children('li:first').addClass('tweet_first'); + list.children('li:odd').addClass('tweet_even'); + list.children('li:even').addClass('tweet_odd'); + }); + if (s.outro_text) list.after(outro); + }); + + }); + }; +})(jQuery); \ No newline at end of file diff --git a/doc/source/_static/tweaks.css b/doc/source/_static/tweaks.css index d1cbed96d..630344555 100644 --- a/doc/source/_static/tweaks.css +++ b/doc/source/_static/tweaks.css @@ -41,4 +41,12 @@ em { table.docutils { font-size: 11px; -} \ No newline at end of file +} + +.tweet_list li { + margin-bottom: 10px; +} + +.tweet_list li .tweet_avatar { + float: left; +} diff --git a/doc/source/_theme/layout.html b/doc/source/_theme/layout.html index ed1cab0a6..d30607ed3 100644 --- a/doc/source/_theme/layout.html +++ b/doc/source/_theme/layout.html @@ -1,2 +1,85 @@ {% extends "sphinxdoc/layout.html" %} {% set css_files = css_files + ['_static/tweaks.css'] %} +{% set script_files = script_files + ['_static/jquery.tweet.js'] %} +{% block extrahead %} + +{% endblock %} + +{%- macro sidebar() %} + {%- if not embedded %}{% if not theme_nosidebar|tobool %} +
    +
    + {%- block sidebarlogo %} + {%- if logo %} + + {%- endif %} + {%- endblock %} + {%- block sidebartoc %} + {%- if display_toc %} +

    {{ _('Table Of Contents') }}

    + {{ toc }} + {%- endif %} + {%- endblock %} + {%- block sidebarrel %} + {%- if prev %} +

    {{ _('Previous topic') }}

    +

    {{ prev.title }}

    + {%- endif %} + {%- if next %} +

    {{ _('Next topic') }}

    +

    {{ next.title }}

    + {%- endif %} + {%- endblock %} + {%- block sidebarsourcelink %} + {%- if show_source and has_source and sourcename %} +

    {{ _('This Page') }}

    + + {%- endif %} + {%- endblock %} + {%- if customsidebar %} + {% include customsidebar %} + {%- endif %} + {%- block sidebarsearch %} + {%- if pagename != "search" %} + + + {%- endif %} + + {%- if pagename == "index" %} +

    {{ _('Twitter Feed') }}

    + + {%- endif %} + + + {%- endblock %} +
    +
    + {%- endif %}{% endif %} +{%- endmacro %} -- cgit From c3054a9a55371fe3acf37589df7d81c6381d7220 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 5 Nov 2010 18:31:26 -0400 Subject: Document nova.sh environment. --- doc/source/adminguide/quickstart.rst | 52 +++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/doc/source/adminguide/quickstart.rst b/doc/source/adminguide/quickstart.rst index 718ea7d14..7e746e3f2 100644 --- a/doc/source/adminguide/quickstart.rst +++ b/doc/source/adminguide/quickstart.rst @@ -18,16 +18,54 @@ Nova Quickstart =============== -.. todo:: +The `contrib/nova.sh` file in the source distribution is a script that +will quickly set up nova to run on a single machine. It is tested against +Ubuntu only, but other distributions are forthcoming. - * Document the assumptions about pluggable interfaces (sqlite3 instead of - mysql, etc) (todd) - * Document env vars that can change things (USE_MYSQL, HOST_IP) (todd) +Environment Variables +--------------------- +By tweaking the environment that nova.sh run in, you can build slightly +different configurations (though for more complex setups you should see +:doc:`/adminguide/getting.started` and :doc:`/adminguide/multi.node.install`). -The `contrib/novascript.sh` file in the source distribution is a script that -will quickly set up nova to run on a single machine. It is tested against -Ubuntu only, but other distributions are forthcoming. +HOST_IP +~~~~~~~ + +**Default**: address of first interface from the ifconfig command +**Values**: 127.0.0.1, or any other valid address + +TEST +~~~~ + +**Default**: 0 +**Values**: 1, run tests after checkout and initial setup + +USE_MYSQL +~~~~~~~~~ + +**Default**: 0, use sqlite3 +**Values**: 1, use mysql instead of sqlite3 + +MYSQL_PASS +~~~~~~~~~~ + +Only useful if $USE_MYSQL=1. + +**Default**: nova +**Values**: value of root password for mysql + +USE_LDAP +~~~~~~~~ + +**Default**: 0, use :mod:`nova.auth.dbdriver` +**Values**: 1, use :mod:`nova.auth.ldapdriver` + +LIBVIRT_TYPE +~~~~~~~~~~~~ + +**Default**: qemu +**Values**: uml, kvm Usage ----- -- cgit From 9c2af052e4ca34a14b84a9dafcb72830e3351a60 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 5 Nov 2010 15:36:19 -0700 Subject: shrink tweet text a bit --- doc/source/_static/tweaks.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/_static/tweaks.css b/doc/source/_static/tweaks.css index 630344555..ea2ed3cf5 100644 --- a/doc/source/_static/tweaks.css +++ b/doc/source/_static/tweaks.css @@ -44,7 +44,9 @@ table.docutils { } .tweet_list li { - margin-bottom: 10px; + font-size: 0.9em; + border-bottom: 1px solid #eee; + padding: 5px 0; } .tweet_list li .tweet_avatar { -- cgit From d24c7090448ebdb9774b69490df8a983e3d27660 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 5 Nov 2010 15:52:59 -0700 Subject: fix default twitter username --- doc/source/_theme/layout.html | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/_theme/layout.html b/doc/source/_theme/layout.html index d30607ed3..e3eb54b71 100644 --- a/doc/source/_theme/layout.html +++ b/doc/source/_theme/layout.html @@ -5,6 +5,7 @@