diff options
| author | Sleepsonthefloor <sleepsonthefloor@gmail.com> | 2010-08-14 06:44:45 -0700 |
|---|---|---|
| committer | Sleepsonthefloor <sleepsonthefloor@gmail.com> | 2010-08-14 06:44:45 -0700 |
| commit | 08a7da895690757a17fdab5e4aadeaf6ba9133d3 (patch) | |
| tree | c564ddfc4a7ac80170a3626ffd120f764e931f07 | |
| parent | a860a07068d4d643c42973625c454c6b09e883cb (diff) | |
| parent | 3ee748bb6f55ad341606919901c4c17a82d069fd (diff) | |
refactor to have base helper class with shared session and engine
| -rw-r--r-- | nova/auth/manager.py | 14 | ||||
| -rw-r--r-- | nova/compute/model.py | 12 | ||||
| -rw-r--r-- | nova/compute/service.py | 203 | ||||
| -rw-r--r-- | nova/datastore.old.py | 261 | ||||
| -rw-r--r-- | nova/datastore.py | 209 | ||||
| -rw-r--r-- | nova/models.py | 191 | ||||
| -rw-r--r-- | nova/network/model.py | 12 | ||||
| -rw-r--r-- | nova/network/service.py | 28 | ||||
| -rw-r--r-- | nova/network/vpn.py | 2 | ||||
| -rw-r--r-- | nova/test.py | 6 | ||||
| -rw-r--r-- | nova/tests/compute_unittest.py | 71 | ||||
| -rw-r--r-- | nova/virt/fake.py | 4 | ||||
| -rw-r--r-- | nova/volume/service.py | 2 |
13 files changed, 481 insertions, 534 deletions
diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 064fd78bc..4a813c861 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -31,6 +31,7 @@ import zipfile from nova import crypto from nova import exception from nova import flags +from nova import models from nova import utils from nova.auth import signer from nova.network import vpn @@ -201,6 +202,11 @@ class Project(AuthBase): ip, port = AuthManager().get_project_vpn_data(self) return port + @property + def network(self): + session = models.create_session() + return session.query(models.Network).filter_by(project_id=self.id).first() + def has_manager(self, user): return AuthManager().is_project_manager(user, self) @@ -521,7 +527,13 @@ class AuthManager(object): description, member_users) if project_dict: - return Project(**project_dict) + project = Project(**project_dict) + # FIXME(ja): EVIL HACK - this should poll from a pool + session = models.create_session() + net = models.Network(project_id=project.id, kind='vlan') + session.add(net) + session.commit() + return project def add_to_project(self, user, project): """Add user to project""" diff --git a/nova/compute/model.py b/nova/compute/model.py index 266a93b9a..54d816a9c 100644 --- a/nova/compute/model.py +++ b/nova/compute/model.py @@ -63,13 +63,11 @@ class InstanceDirectory(object): def __getitem__(self, item): return self.get(item) - @datastore.absorb_connection_error def by_project(self, project): """returns a list of instance objects for a project""" for instance_id in datastore.Redis.instance().smembers('project:%s:instances' % project): yield Instance(instance_id) - @datastore.absorb_connection_error def by_node(self, node): """returns a list of instances for a node""" for instance_id in datastore.Redis.instance().smembers('node:%s:instances' % node): @@ -90,12 +88,10 @@ class InstanceDirectory(object): """returns the instance a volume is attached to""" pass - @datastore.absorb_connection_error def exists(self, instance_id): return datastore.Redis.instance().sismember('instances', instance_id) @property - @datastore.absorb_connection_error def all(self): """returns a list of all instances""" for instance_id in datastore.Redis.instance().smembers('instances'): @@ -107,7 +103,7 @@ class InstanceDirectory(object): return self.get(instance_id) -class Instance(datastore.BasicModel): +class Instance(): """Wrapper around stored properties of an instance""" def __init__(self, instance_id): @@ -168,7 +164,7 @@ class Instance(datastore.BasicModel): self.unassociate_with("ip", self.state['private_dns_name']) return super(Instance, self).destroy() -class Host(datastore.BasicModel): +class Host(): """A Host is the machine where a Daemon is running.""" def __init__(self, hostname): @@ -185,7 +181,7 @@ class Host(datastore.BasicModel): return self.hostname -class Daemon(datastore.BasicModel): +class Daemon(): """A Daemon is a job (compute, api, network, ...) that runs on a host.""" def __init__(self, host_or_combined, binpath=None): @@ -235,7 +231,7 @@ class Daemon(datastore.BasicModel): for x in cls.associated_to("host", hostname): yield x -class SessionToken(datastore.BasicModel): +class SessionToken(): """This is a short-lived auth token that is passed through web requests""" def __init__(self, session_token): diff --git a/nova/compute/service.py b/nova/compute/service.py index 820116453..dc6a93bdb 100644 --- a/nova/compute/service.py +++ b/nova/compute/service.py @@ -38,7 +38,7 @@ from nova import process from nova import service from nova import utils from nova.compute import disk -from nova.compute import model +from nova import models from nova.compute import power_state from nova.compute.instance_types import INSTANCE_TYPES from nova.network import service as network_service @@ -61,7 +61,6 @@ class ComputeService(service.Service): super(ComputeService, self).__init__() self._instances = {} self._conn = virt_connection.get_connection() - self.instdir = model.InstanceDirectory() # TODO(joshua): This needs to ensure system state, specifically: modprobe aoe def noop(self): @@ -69,11 +68,15 @@ class ComputeService(service.Service): return defer.succeed('PONG') def get_instance(self, instance_id): - # inst = self.instdir.get(instance_id) - # return inst - if self.instdir.exists(instance_id): - return Instance.fromName(self._conn, instance_id) - return None + session = models.create_session() + return session.query(models.Instance).filter_by(id=instance_id).one() + + def update_state(self, instance_id): + session = models.create_session() + inst = session.query(models.Instance).filter_by(id=instance_id).one() + # FIXME(ja): include other fields from state? + inst.state = self._conn.get_info(instance_id)['state'] + session.flush() @exception.wrap_exception def adopt_instances(self): @@ -88,14 +91,6 @@ class ComputeService(service.Service): pass return defer.succeed(len(self._instances)) - @exception.wrap_exception - def describe_instances(self): - retval = {} - for inst in self.instdir.by_node(FLAGS.node_name): - retval[inst['instance_id']] = ( - Instance.fromName(self._conn, inst['instance_id'])) - return retval - @defer.inlineCallbacks def report_state(self, nodename, daemon): # TODO(termie): make this pattern be more elegant. -todd @@ -112,71 +107,93 @@ class ComputeService(service.Service): logging.exception("model server went away") yield + @defer.inlineCallbacks @exception.wrap_exception def run_instance(self, instance_id, **_kwargs): """ launch a new instance with specified options """ logging.debug("Starting instance %s..." % (instance_id)) - inst = self.instdir.get(instance_id) - # TODO: Get the real security group of launch in here - security_group = "default" + session = models.create_session() + inst = session.query(models.Instance).filter_by(id=instance_id).first() # NOTE(vish): passing network type allows us to express the # network without making a call to network to find # out which type of network to setup - network_service.setup_compute_network( - inst.get('network_type', 'vlan'), - inst['user_id'], - inst['project_id'], - security_group) - - inst['node_name'] = FLAGS.node_name - inst.save() + network_service.setup_compute_network(inst) + inst.node_name = FLAGS.node_name + session.commit() + # TODO(vish) check to make sure the availability zone matches - new_inst = Instance(self._conn, name=instance_id, data=inst) - logging.info("Instances current state is %s", new_inst.state) - if new_inst.is_running(): - raise exception.Error("Instance is already running") - new_inst.spawn() + inst.set_state(power_state.NOSTATE, 'spawning') + session.commit() + + try: + yield self._conn.spawn(inst) + except Exception, ex: + logging.debug(ex) + inst.set_state(power_state.SHUTDOWN) + + self.update_state(instance_id) + @defer.inlineCallbacks @exception.wrap_exception def terminate_instance(self, instance_id): """ terminate an instance on this machine """ logging.debug("Got told to terminate instance %s" % instance_id) - instance = self.get_instance(instance_id) - # inst = self.instdir.get(instance_id) - if not instance: - raise exception.Error( - 'trying to terminate unknown instance: %s' % instance_id) - d = instance.destroy() - # d.addCallback(lambda x: inst.destroy()) - return d + session = models.create_session() + instance = session.query(models.Instance).filter_by(id=instance_id).one() + + if instance.state == power_state.SHUTOFF: + # self.datamodel.destroy() FIXME: RE-ADD ????? + raise exception.Error('trying to destroy already destroyed' + ' instance: %s' % instance_id) + + instance.set_state(power_state.NOSTATE, 'shutting_down') + yield self._conn.destroy(instance) + # FIXME(ja): should we keep it in a terminated state for a bit? + session.delete(instance) + session.flush() + @defer.inlineCallbacks @exception.wrap_exception def reboot_instance(self, instance_id): """ reboot an instance on this server KVM doesn't support reboot, so we terminate and restart """ + self.update_state(instance_id) instance = self.get_instance(instance_id) - if not instance: + + # FIXME(ja): this is only checking the model state - not state on disk? + if instance.state != power_state.RUNNING: raise exception.Error( - 'trying to reboot unknown instance: %s' % instance_id) - return instance.reboot() + 'trying to reboot a non-running' + 'instance: %s (state: %s excepted: %s)' % (instance.id, instance.state, power_state.RUNNING)) + + logging.debug('rebooting instance %s' % instance.id) + instance.set_state(power_state.NOSTATE, 'rebooting') + yield self._conn.reboot(instance) + self.update_state(instance_id) - @defer.inlineCallbacks @exception.wrap_exception def get_console_output(self, instance_id): """ send the console output for an instance """ + # FIXME: Abstract this for Xen + logging.debug("Getting console output for %s" % (instance_id)) - inst = self.instdir.get(instance_id) - instance = self.get_instance(instance_id) - if not instance: - raise exception.Error( - 'trying to get console log for unknown: %s' % instance_id) - rv = yield instance.console_output() + session = models.create_session() + inst = self.get_instance(instance_id) + + if FLAGS.connection_type == 'libvirt': + fname = os.path.abspath( + os.path.join(FLAGS.instances_path, inst.id, 'console.log')) + with open(fname, 'r') as f: + output = f.read() + else: + output = 'FAKE CONSOLE OUTPUT' + # TODO(termie): this stuff belongs in the API layer, no need to # munge the data we send to ourselves output = {"InstanceId" : instance_id, "Timestamp" : "2", - "output" : base64.b64encode(rv)} - defer.returnValue(output) + "output" : base64.b64encode(output)} + return output @defer.inlineCallbacks @exception.wrap_exception @@ -270,29 +287,6 @@ class Instance(object): self.datamodel.save() logging.debug("Finished init of Instance with id of %s" % name) - @classmethod - def fromName(cls, conn, name): - """ use the saved data for reloading the instance """ - instdir = model.InstanceDirectory() - instance = instdir.get(name) - return cls(conn=conn, name=name, data=instance) - - def set_state(self, state_code, state_description=None): - self.datamodel['state'] = state_code - if not state_description: - state_description = power_state.name(state_code) - self.datamodel['state_description'] = state_description - self.datamodel.save() - - @property - def state(self): - # it is a string in datamodel - return int(self.datamodel['state']) - - @property - def name(self): - return self.datamodel['name'] - def is_pending(self): return (self.state == power_state.NOSTATE or self.state == 'pending') @@ -303,64 +297,3 @@ class Instance(object): logging.debug("Instance state is: %s" % self.state) return (self.state == power_state.RUNNING or self.state == 'running') - def describe(self): - return self.datamodel - - def info(self): - result = self._conn.get_info(self.name) - result['node_name'] = FLAGS.node_name - return result - - def update_state(self): - self.datamodel.update(self.info()) - self.set_state(self.state) - self.datamodel.save() # Extra, but harmless - - @defer.inlineCallbacks - @exception.wrap_exception - def destroy(self): - if self.is_destroyed(): - self.datamodel.destroy() - raise exception.Error('trying to destroy already destroyed' - ' instance: %s' % self.name) - - self.set_state(power_state.NOSTATE, 'shutting_down') - yield self._conn.destroy(self) - self.datamodel.destroy() - - @defer.inlineCallbacks - @exception.wrap_exception - def reboot(self): - if not self.is_running(): - raise exception.Error( - 'trying to reboot a non-running' - 'instance: %s (state: %s)' % (self.name, self.state)) - - logging.debug('rebooting instance %s' % self.name) - self.set_state(power_state.NOSTATE, 'rebooting') - yield self._conn.reboot(self) - self.update_state() - - @defer.inlineCallbacks - @exception.wrap_exception - def spawn(self): - self.set_state(power_state.NOSTATE, 'spawning') - logging.debug("Starting spawn in Instance") - try: - yield self._conn.spawn(self) - except Exception, ex: - logging.debug(ex) - self.set_state(power_state.SHUTDOWN) - self.update_state() - - @exception.wrap_exception - def console_output(self): - # FIXME: Abstract this for Xen - if FLAGS.connection_type == 'libvirt': - fname = os.path.abspath( - os.path.join(self.datamodel['basepath'], 'console.log')) - with open(fname, 'r') as f: - console = f.read() - else: - console = 'FAKE CONSOLE OUTPUT' - return defer.succeed(console) diff --git a/nova/datastore.old.py b/nova/datastore.old.py new file mode 100644 index 000000000..751c5eeeb --- /dev/null +++ b/nova/datastore.old.py @@ -0,0 +1,261 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Datastore: + +MAKE Sure that ReDIS is running, and your flags are set properly, +before trying to run this. +""" + +import logging + +from nova import exception +from nova import flags +from nova import utils + + +FLAGS = flags.FLAGS +flags.DEFINE_string('redis_host', '127.0.0.1', + 'Host that redis is running on.') +flags.DEFINE_integer('redis_port', 6379, + 'Port that redis is running on.') +flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') + + +class Redis(object): + def __init__(self): + if hasattr(self.__class__, '_instance'): + raise Exception('Attempted to instantiate singleton') + + @classmethod + def instance(cls): + if not hasattr(cls, '_instance'): + inst = redis.Redis(host=FLAGS.redis_host, + port=FLAGS.redis_port, + db=FLAGS.redis_db) + cls._instance = inst + return cls._instance + + +class ConnectionError(exception.Error): + pass + + +def absorb_connection_error(fn): + def _wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except redis.exceptions.ConnectionError, ce: + raise ConnectionError(str(ce)) + return _wrapper + + +class BasicModel(object): + """ + All Redis-backed data derives from this class. + + You MUST specify an identifier() property that returns a unique string + per instance. + + You MUST have an initializer that takes a single argument that is a value + returned by identifier() to load a new class with. + + You may want to specify a dictionary for default_state(). + + You may also specify override_type at the class left to use a key other + than __class__.__name__. + + You override save and destroy calls to automatically build and destroy + associations. + """ + + override_type = None + + @absorb_connection_error + def __init__(self): + state = Redis.instance().hgetall(self.__redis_key) + if state: + self.initial_state = state + self.state = dict(self.initial_state) + else: + self.initial_state = {} + self.state = self.default_state() + + + def default_state(self): + """You probably want to define this in your subclass""" + return {} + + @classmethod + def _redis_name(cls): + return cls.override_type or cls.__name__.lower() + + @classmethod + def lookup(cls, identifier): + rv = cls(identifier) + if rv.is_new_record(): + return None + else: + return rv + + @classmethod + @absorb_connection_error + def all(cls): + """yields all objects in the store""" + redis_set = cls._redis_set_name(cls.__name__) + for identifier in Redis.instance().smembers(redis_set): + yield cls(identifier) + + @classmethod + def associated_to(cls, foreign_type, foreign_id): + for identifier in cls.associated_keys(foreign_type, foreign_id): + yield cls(identifier) + + @classmethod + @absorb_connection_error + def associated_keys(cls, foreign_type, foreign_id): + redis_set = cls._redis_association_name(foreign_type, foreign_id) + return Redis.instance().smembers(redis_set) or [] + + @classmethod + def _redis_set_name(cls, kls_name): + # stupidly pluralize (for compatiblity with previous codebase) + return kls_name.lower() + "s" + + @classmethod + def _redis_association_name(cls, foreign_type, foreign_id): + return cls._redis_set_name("%s:%s:%s" % + (foreign_type, foreign_id, cls._redis_name())) + + @property + def identifier(self): + """You DEFINITELY want to define this in your subclass""" + raise NotImplementedError("Your subclass should define identifier") + + @property + def __redis_key(self): + return '%s:%s' % (self._redis_name(), self.identifier) + + def __repr__(self): + return "<%s:%s>" % (self.__class__.__name__, self.identifier) + + def keys(self): + return self.state.keys() + + def copy(self): + copyDict = {} + for item in self.keys(): + copyDict[item] = self[item] + return copyDict + + def get(self, item, default): + return self.state.get(item, default) + + def update(self, update_dict): + return self.state.update(update_dict) + + def setdefault(self, item, default): + return self.state.setdefault(item, default) + + def __contains__(self, item): + return item in self.state + + def __getitem__(self, item): + return self.state[item] + + def __setitem__(self, item, val): + self.state[item] = val + return self.state[item] + + def __delitem__(self, item): + """We don't support this""" + raise Exception("Silly monkey, models NEED all their properties.") + + def is_new_record(self): + return self.initial_state == {} + + @absorb_connection_error + def add_to_index(self): + """Each insance of Foo has its id tracked int the set named Foos""" + set_name = self.__class__._redis_set_name(self.__class__.__name__) + Redis.instance().sadd(set_name, self.identifier) + + @absorb_connection_error + def remove_from_index(self): + """Remove id of this instance from the set tracking ids of this type""" + set_name = self.__class__._redis_set_name(self.__class__.__name__) + Redis.instance().srem(set_name, self.identifier) + + @absorb_connection_error + def associate_with(self, foreign_type, foreign_id): + """Add this class id into the set foreign_type:foreign_id:this_types""" + # note the extra 's' on the end is for plurality + # to match the old data without requiring a migration of any sort + self.add_associated_model_to_its_set(foreign_type, foreign_id) + redis_set = self.__class__._redis_association_name(foreign_type, + foreign_id) + Redis.instance().sadd(redis_set, self.identifier) + + @absorb_connection_error + def unassociate_with(self, foreign_type, foreign_id): + """Delete from foreign_type:foreign_id:this_types set""" + redis_set = self.__class__._redis_association_name(foreign_type, + foreign_id) + Redis.instance().srem(redis_set, self.identifier) + + def add_associated_model_to_its_set(self, model_type, model_id): + """ + When associating an X to a Y, save Y for newer timestamp, etc, and to + make sure to save it if Y is a new record. + If the model_type isn't found as a usable class, ignore it, this can + happen when associating to things stored in LDAP (user, project, ...). + """ + table = globals() + klsname = model_type.capitalize() + if table.has_key(klsname): + model_class = table[klsname] + model_inst = model_class(model_id) + model_inst.save() + + @absorb_connection_error + def save(self): + """ + update the directory with the state from this model + also add it to the index of items of the same type + then set the initial_state = state so new changes are tracked + """ + # TODO(ja): implement hmset in redis-py and use it + # instead of multiple calls to hset + if self.is_new_record(): + self["create_time"] = utils.isotime() + for key, val in self.state.iteritems(): + Redis.instance().hset(self.__redis_key, key, val) + self.add_to_index() + self.initial_state = dict(self.state) + return True + + @absorb_connection_error + def destroy(self): + """deletes all related records from datastore.""" + logging.info("Destroying datamodel for %s %s", + self.__class__.__name__, self.identifier) + Redis.instance().delete(self.__redis_key) + self.remove_from_index() + return True + diff --git a/nova/datastore.py b/nova/datastore.py index 5dc6ed107..8e2519429 100644 --- a/nova/datastore.py +++ b/nova/datastore.py @@ -26,10 +26,7 @@ before trying to run this. import logging import redis -from nova import exception from nova import flags -from nova import utils - FLAGS = flags.FLAGS flags.DEFINE_string('redis_host', '127.0.0.1', @@ -54,209 +51,3 @@ class Redis(object): return cls._instance -class ConnectionError(exception.Error): - pass - - -def absorb_connection_error(fn): - def _wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except redis.exceptions.ConnectionError, ce: - raise ConnectionError(str(ce)) - return _wrapper - - -class BasicModel(object): - """ - All Redis-backed data derives from this class. - - You MUST specify an identifier() property that returns a unique string - per instance. - - You MUST have an initializer that takes a single argument that is a value - returned by identifier() to load a new class with. - - You may want to specify a dictionary for default_state(). - - You may also specify override_type at the class left to use a key other - than __class__.__name__. - - You override save and destroy calls to automatically build and destroy - associations. - """ - - override_type = None - - @absorb_connection_error - def __init__(self): - state = Redis.instance().hgetall(self.__redis_key) - if state: - self.initial_state = state - self.state = dict(self.initial_state) - else: - self.initial_state = {} - self.state = self.default_state() - - - def default_state(self): - """You probably want to define this in your subclass""" - return {} - - @classmethod - def _redis_name(cls): - return cls.override_type or cls.__name__.lower() - - @classmethod - def lookup(cls, identifier): - rv = cls(identifier) - if rv.is_new_record(): - return None - else: - return rv - - @classmethod - @absorb_connection_error - def all(cls): - """yields all objects in the store""" - redis_set = cls._redis_set_name(cls.__name__) - for identifier in Redis.instance().smembers(redis_set): - yield cls(identifier) - - @classmethod - def associated_to(cls, foreign_type, foreign_id): - for identifier in cls.associated_keys(foreign_type, foreign_id): - yield cls(identifier) - - @classmethod - @absorb_connection_error - def associated_keys(cls, foreign_type, foreign_id): - redis_set = cls._redis_association_name(foreign_type, foreign_id) - return Redis.instance().smembers(redis_set) or [] - - @classmethod - def _redis_set_name(cls, kls_name): - # stupidly pluralize (for compatiblity with previous codebase) - return kls_name.lower() + "s" - - @classmethod - def _redis_association_name(cls, foreign_type, foreign_id): - return cls._redis_set_name("%s:%s:%s" % - (foreign_type, foreign_id, cls._redis_name())) - - @property - def identifier(self): - """You DEFINITELY want to define this in your subclass""" - raise NotImplementedError("Your subclass should define identifier") - - @property - def __redis_key(self): - return '%s:%s' % (self._redis_name(), self.identifier) - - def __repr__(self): - return "<%s:%s>" % (self.__class__.__name__, self.identifier) - - def keys(self): - return self.state.keys() - - def copy(self): - copyDict = {} - for item in self.keys(): - copyDict[item] = self[item] - return copyDict - - def get(self, item, default): - return self.state.get(item, default) - - def update(self, update_dict): - return self.state.update(update_dict) - - def setdefault(self, item, default): - return self.state.setdefault(item, default) - - def __contains__(self, item): - return item in self.state - - def __getitem__(self, item): - return self.state[item] - - def __setitem__(self, item, val): - self.state[item] = val - return self.state[item] - - def __delitem__(self, item): - """We don't support this""" - raise Exception("Silly monkey, models NEED all their properties.") - - def is_new_record(self): - return self.initial_state == {} - - @absorb_connection_error - def add_to_index(self): - """Each insance of Foo has its id tracked int the set named Foos""" - set_name = self.__class__._redis_set_name(self.__class__.__name__) - Redis.instance().sadd(set_name, self.identifier) - - @absorb_connection_error - def remove_from_index(self): - """Remove id of this instance from the set tracking ids of this type""" - set_name = self.__class__._redis_set_name(self.__class__.__name__) - Redis.instance().srem(set_name, self.identifier) - - @absorb_connection_error - def associate_with(self, foreign_type, foreign_id): - """Add this class id into the set foreign_type:foreign_id:this_types""" - # note the extra 's' on the end is for plurality - # to match the old data without requiring a migration of any sort - self.add_associated_model_to_its_set(foreign_type, foreign_id) - redis_set = self.__class__._redis_association_name(foreign_type, - foreign_id) - Redis.instance().sadd(redis_set, self.identifier) - - @absorb_connection_error - def unassociate_with(self, foreign_type, foreign_id): - """Delete from foreign_type:foreign_id:this_types set""" - redis_set = self.__class__._redis_association_name(foreign_type, - foreign_id) - Redis.instance().srem(redis_set, self.identifier) - - def add_associated_model_to_its_set(self, model_type, model_id): - """ - When associating an X to a Y, save Y for newer timestamp, etc, and to - make sure to save it if Y is a new record. - If the model_type isn't found as a usable class, ignore it, this can - happen when associating to things stored in LDAP (user, project, ...). - """ - table = globals() - klsname = model_type.capitalize() - if table.has_key(klsname): - model_class = table[klsname] - model_inst = model_class(model_id) - model_inst.save() - - @absorb_connection_error - def save(self): - """ - update the directory with the state from this model - also add it to the index of items of the same type - then set the initial_state = state so new changes are tracked - """ - # TODO(ja): implement hmset in redis-py and use it - # instead of multiple calls to hset - if self.is_new_record(): - self["create_time"] = utils.isotime() - for key, val in self.state.iteritems(): - Redis.instance().hset(self.__redis_key, key, val) - self.add_to_index() - self.initial_state = dict(self.state) - return True - - @absorb_connection_error - def destroy(self): - """deletes all related records from datastore.""" - logging.info("Destroying datamodel for %s %s", - self.__class__.__name__, self.identifier) - Redis.instance().delete(self.__redis_key) - self.remove_from_index() - return True - diff --git a/nova/models.py b/nova/models.py index 4c739488a..79273965b 100644 --- a/nova/models.py +++ b/nova/models.py @@ -1,111 +1,56 @@ from sqlalchemy.orm import relationship, backref, validates from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime, Boolean, Text from sqlalchemy.ext.declarative import declarative_base -from auth import * +from nova import auth Base = declarative_base() -class User(Base): - # sqlalchemy - __tablename__ = 'users' - sid = Column(String, primary_key=True) +class NovaBase(object): + created_at = Column(DateTime) + updated_at = Column(DateTime) - # backwards compatibility + _session = None + _engine = None @classmethod - def safe_id(cls, obj): - """Safe get object id - - This method will return the id of the object if the object - is of this class, otherwise it will return the original object. - This allows methods to accept objects or ids as paramaters. - - """ - if isinstance(obj, cls): - return obj.id - else: - return obj - -# def __init__(self, id, name, access, secret, admin): -# self.id = id -# self.name = name -# self.access = access -# self.secret = secret -# self.admin = admin - - def __getattr__(self, name): - if name == 'id': - return self.uid - else: raise AttributeError, name - - def is_superuser(self): - return AuthManager().is_superuser(self) - - def is_admin(self): - return AuthManager().is_admin(self) - - def has_role(self, role): - return AuthManager().has_role(self, role) - - def add_role(self, role): - return AuthManager().add_role(self, role) - - def remove_role(self, role): - return AuthManager().remove_role(self, role) - - def is_project_member(self, project): - return AuthManager().is_project_member(self, project) - - def is_project_manager(self, project): - return AuthManager().is_project_manager(self, project) - - def generate_key_pair(self, name): - return AuthManager().generate_key_pair(self.id, name) - - def create_key_pair(self, name, public_key, fingerprint): - return AuthManager().create_key_pair(self.id, - name, - public_key, - fingerprint) - - def get_key_pair(self, name): - return AuthManager().get_key_pair(self.id, name) - - def delete_key_pair(self, name): - return AuthManager().delete_key_pair(self.id, name) - - def get_key_pairs(self): - return AuthManager().get_key_pairs(self.id) - - def __repr__(self): - return "User('%s', '%s', '%s', '%s', %s)" % (self.id, - self.name, - self.access, - self.secret, - self.admin) + def create_engine(cls): + if NovaBase._engine is not None: + return _engine + from sqlalchemy import create_engine + NovaBase._engine = create_engine('sqlite:///:memory:', echo=True) + Base.metadata.create_all(NovaBase._engine) + return NovaBase._engine + @classmethod + def get_session(cls): + from sqlalchemy.orm import sessionmaker + if NovaBase._session == None: + NovaBase.create_engine(); + NovaBase._session = sessionmaker(bind=NovaBase._engine)() + return NovaBase._session + @classmethod + def all(cls): + session = NovaBase.get_session() + return session.query(cls).all() -class Project(Base): - __tablename__ = 'projects' - sid = Column(String, primary_key=True) + def save(self): + session = NovaBase.get_session() + session.add(self) + session.commit() -class Image(Base): +class Image(Base, NovaBase): __tablename__ = 'images' - user_sid = Column(String, ForeignKey('users.sid'), nullable=False) - project_sid = Column(String, ForeignKey('projects.sid'), nullable=False) + user_id = Column(String)#, ForeignKey('users.id'), nullable=False) + project_id = Column(String)#, ForeignKey('projects.id'), nullable=False) - sid = Column(String, primary_key=True) + id = Column(String, primary_key=True) image_type = Column(String) public = Column(Boolean, default=False) state = Column(String) location = Column(String) arch = Column(String) - default_kernel_sid = Column(String) - default_ramdisk_sid = Column(String) - - created_at = Column(DateTime) - updated_at = Column(DateTime) # auto update on change FIXME - + default_kernel_id = Column(String) + default_ramdisk_id = Column(String) @validates('image_type') def validate_image_type(self, key, image_type): @@ -115,13 +60,13 @@ class Image(Base): def validate_state(self, key, state): assert(state in ['available', 'pending', 'disabled']) - @validates('default_kernel_sid') - def validate_kernel_sid(self, key, val): + @validates('default_kernel_id') + def validate_kernel_id(self, key, val): if val != 'machine': assert(val is None) - @validates('default_ramdisk_sid') - def validate_ramdisk_sid(self, key, val): + @validates('default_ramdisk_id') + def validate_ramdisk_id(self, key, val): if val != 'machine': assert(val is None) @@ -130,29 +75,40 @@ class Network(Base): id = Column(Integer, primary_key=True) bridge = Column(String) vlan = Column(String) + kind = Column(String) #vpn_port = Column(Integer) - project_sid = Column(String, ForeignKey('projects.sid'), nullable=False) + project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) class PhysicalNode(Base): __tablename__ = 'physical_nodes' id = Column(Integer, primary_key=True) -class Instance(Base): +class Instance(Base, NovaBase): __tablename__ = 'instances' id = Column(Integer, primary_key=True) - user_sid = Column(String, ForeignKey('users.sid'), nullable=False) - project_sid = Column(String, ForeignKey('projects.sid')) + user_id = Column(String) #, ForeignKey('users.id'), nullable=False) + project_id = Column(String) #, ForeignKey('projects.id')) + + @property + def user(self): + return auth.manager.AuthManager().get_user(self.user_id) + + @property + def project(self): + return auth.manager.AuthManager().get_project(self.project_id) - image_sid = Column(Integer, ForeignKey('images.sid'), nullable=False) - kernel_sid = Column(String, ForeignKey('images.sid'), nullable=True) - ramdisk_sid = Column(String, ForeignKey('images.sid'), nullable=True) + image_id = Column(Integer, ForeignKey('images.id'), nullable=False) + kernel_id = Column(String, ForeignKey('images.id'), nullable=True) + ramdisk_id = Column(String, ForeignKey('images.id'), nullable=True) launch_index = Column(Integer) key_name = Column(String) key_data = Column(Text) + security_group = Column(String) - state = Column(String) + state = Column(Integer) + state_description = Column(String) hostname = Column(String) physical_node_id = Column(Integer) @@ -161,7 +117,13 @@ class Instance(Base): user_data = Column(Text) -# user = relationship(User, backref=backref('instances', order_by=id)) + def set_state(self, state_code, state_description=None): + from nova.compute import power_state + self.state = state_code + if not state_description: + state_description = power_state.name(state_code) + self.state_description = state_description + # ramdisk = relationship(Ramdisk, backref=backref('instances', order_by=id)) # kernel = relationship(Kernel, backref=backref('instances', order_by=id)) # project = relationship(Project, backref=backref('instances', order_by=id)) @@ -171,9 +133,9 @@ class Instance(Base): # power_state = what we have # task_state = transitory and may trigger power state transition - @validates('state') - def validate_state(self, key, state): - assert(state in ['nostate', 'running', 'blocked', 'paused', 'shutdown', 'shutoff', 'crashed']) + #@validates('state') + #def validate_state(self, key, state): + # assert(state in ['nostate', 'running', 'blocked', 'paused', 'shutdown', 'shutoff', 'crashed']) class Volume(Base): __tablename__ = 'volumes' @@ -182,17 +144,18 @@ class Volume(Base): blade_id = Column(Integer) -if __name__ == '__main__': - from sqlalchemy import create_engine - engine = create_engine('sqlite:///:memory:', echo=True) - Base.metadata.create_all(engine) +def create_engine(): + return NovaBase.get_engine(); + +def create_session(engine=None): + return NovaBase.get_session() - from sqlalchemy.orm import sessionmaker - Session = sessionmaker(bind=engine) - session = Session() +if __name__ == '__main__': + engine = create_engine() + session = create_session(engine) - instance = Instance(image_sid='as', ramdisk_sid='AS', user_sid='anthony') - user = User(sid='anthony') + instance = Instance(image_id='as', ramdisk_id='AS', user_id='anthony') + user = User(id='anthony') session.add(instance) session.commit() diff --git a/nova/network/model.py b/nova/network/model.py index ce9345067..c5c8ce443 100644 --- a/nova/network/model.py +++ b/nova/network/model.py @@ -56,7 +56,7 @@ flags.DEFINE_integer('cloudpipe_start_port', 12000, logging.getLogger().setLevel(logging.DEBUG) -class Vlan(datastore.BasicModel): +class Vlan(): """Tracks vlans assigned to project it the datastore""" def __init__(self, project, vlan): # pylint: disable=W0231 """ @@ -79,7 +79,6 @@ class Vlan(datastore.BasicModel): return instance @classmethod - @datastore.absorb_connection_error def lookup(cls, project): """Returns object by project if it exists in datastore or None""" set_name = cls._redis_set_name(cls.__name__) @@ -90,14 +89,12 @@ class Vlan(datastore.BasicModel): return None @classmethod - @datastore.absorb_connection_error def dict_by_project(cls): """A hash of project:vlan""" set_name = cls._redis_set_name(cls.__name__) return datastore.Redis.instance().hgetall(set_name) or {} @classmethod - @datastore.absorb_connection_error def dict_by_vlan(cls): """A hash of vlan:project""" set_name = cls._redis_set_name(cls.__name__) @@ -108,14 +105,12 @@ class Vlan(datastore.BasicModel): return retvals @classmethod - @datastore.absorb_connection_error def all(cls): set_name = cls._redis_set_name(cls.__name__) elements = datastore.Redis.instance().hgetall(set_name) for project in elements: yield cls(project, elements[project]) - @datastore.absorb_connection_error def save(self): """ Vlan saves state into a giant hash named "vlans", with keys of @@ -127,7 +122,6 @@ class Vlan(datastore.BasicModel): self.project_id, self.vlan_id) - @datastore.absorb_connection_error def destroy(self): """Removes the object from the datastore""" set_name = self._redis_set_name(self.__class__.__name__) @@ -143,7 +137,7 @@ class Vlan(datastore.BasicModel): network[start + FLAGS.network_size - 1]) -class Address(datastore.BasicModel): +class Address(): """Represents a fixed ip in the datastore""" override_type = "address" @@ -197,7 +191,7 @@ class PublicAddress(Address): # CLEANUP: # TODO(ja): does vlanpool "keeper" need to know the min/max - # shouldn't FLAGS always win? -class BaseNetwork(datastore.BasicModel): +class BaseNetwork(): """Implements basic logic for allocating ips in a network""" override_type = 'network' address_class = Address diff --git a/nova/network/service.py b/nova/network/service.py index 9c0f5520b..b6777efc7 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -29,6 +29,7 @@ from nova.exception import NotFound from nova.network import exception from nova.network import model from nova.network import vpn +from nova.network import linux_net FLAGS = flags.FLAGS @@ -61,13 +62,10 @@ def type_to_class(network_type): raise NotFound("Couldn't find %s network type" % network_type) -def setup_compute_network(network_type, user_id, project_id, security_group): +def setup_compute_network(instance): """Sets up the network on a compute host""" - srv = type_to_class(network_type) - srv.setup_compute_network(network_type, - user_id, - project_id, - security_group) + srv = type_to_class(instance.project.network.kind) + srv.setup_compute_network(instance) def get_host_for_project(project_id): @@ -118,8 +116,7 @@ class BaseNetworkService(service.Service): pass @classmethod - def setup_compute_network(cls, user_id, project_id, security_group, - *args, **kwargs): + def setup_compute_network(cls, instance, *args, **kwargs): """Sets up matching network for compute hosts""" raise NotImplementedError() @@ -147,8 +144,7 @@ class FlatNetworkService(BaseNetworkService): """Basic network where no vlans are used""" @classmethod - def setup_compute_network(cls, user_id, project_id, security_group, - *args, **kwargs): + def setup_compute_network(cls, instance, *args, **kwargs): """Network is created manually""" pass @@ -245,13 +241,11 @@ class VlanNetworkService(BaseNetworkService): vpn.NetworkData.create(project_id) @classmethod - def setup_compute_network(cls, user_id, project_id, security_group, - *args, **kwargs): + def setup_compute_network(cls, instance, *args, **kwargs): """Sets up matching network for compute hosts""" # NOTE(vish): Use BridgedNetwork instead of DHCPNetwork because # we don't want to run dnsmasq on the client machines - net = model.BridgedNetwork.get_network_for_project( - user_id, - project_id, - security_group) - net.express() + net = instance.project.network + # FIXME(ja): hack - uncomment this: + #linux_net.vlan_create(net) + #linux_net.bridge_create(net) diff --git a/nova/network/vpn.py b/nova/network/vpn.py index a0e2a7fa1..5eb1c2b20 100644 --- a/nova/network/vpn.py +++ b/nova/network/vpn.py @@ -39,7 +39,7 @@ class NoMorePorts(exception.Error): pass -class NetworkData(datastore.BasicModel): +class NetworkData(): """Manages network host, and vpn ip and port for projects""" def __init__(self, project_id): self.project_id = project_id diff --git a/nova/test.py b/nova/test.py index c7e08734f..9cb826253 100644 --- a/nova/test.py +++ b/nova/test.py @@ -39,6 +39,12 @@ FLAGS = flags.FLAGS flags.DEFINE_bool('fake_tests', True, 'should we use everything for testing') +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base + +engine = create_engine('sqlite:///:memory:', echo=True) +Base = declarative_base() +Base.metadata.create_all(engine) def skip_if_fake(func): """Decorator that skips a test if running in fake mode""" diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index da0f82e3a..44cc6ac15 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -25,7 +25,8 @@ from nova import exception from nova import flags from nova import test from nova import utils -from nova.compute import model +from nova import models +from nova.auth import manager from nova.compute import service @@ -60,51 +61,51 @@ class ComputeConnectionTestCase(test.TrialTestCase): self.flags(connection_type='fake', fake_storage=True) self.compute = service.ComputeService() + self.manager = manager.AuthManager() + user = self.manager.create_user('fake', 'fake', 'fake') + project = self.manager.create_project('fake', 'fake', 'fake') + + def tearDown(self): + self.manager.delete_user('fake') + self.manager.delete_project('fake') def create_instance(self): - instdir = model.InstanceDirectory() - inst = instdir.new() + inst = models.Instance(user_id='fake', project_id='fake', image_id='ami-test') + inst.save(); # TODO(ja): add ami, ari, aki, user_data - inst['reservation_id'] = 'r-fakeres' - inst['launch_time'] = '10' - inst['user_id'] = 'fake' - inst['project_id'] = 'fake' - inst['instance_type'] = 'm1.tiny' - inst['node_name'] = FLAGS.node_name - inst['mac_address'] = utils.generate_mac() - inst['ami_launch_index'] = 0 - inst.save() - return inst['instance_id'] + # inst['reservation_id'] = 'r-fakeres' + # inst['launch_time'] = '10' + #inst['user_id'] = 'fake' + #inst['project_id'] = 'fake' + #inst['instance_type'] = 'm1.tiny' + #inst['node_name'] = FLAGS.node_name + #inst['mac_address'] = utils.generate_mac() + #inst['ami_launch_index'] = 0 + #inst.save() + return inst.id @defer.inlineCallbacks def test_run_describe_terminate(self): instance_id = self.create_instance() - rv = yield self.compute.run_instance(instance_id) + yield self.compute.run_instance(instance_id) - rv = yield self.compute.describe_instances() - logging.info("Running instances: %s", rv) - self.assertEqual(rv[instance_id].name, instance_id) + instances = models.Instance.all() + logging.info("Running instances: %s", instances) + self.assertEqual(len(instances), 1) - rv = yield self.compute.terminate_instance(instance_id) + yield self.compute.terminate_instance(instance_id) - rv = yield self.compute.describe_instances() - logging.info("After terminating instances: %s", rv) - self.assertEqual(rv, {}) + instances = models.Instance.all() + logging.info("After terminating instances: %s", instances) + self.assertEqual(len(instances), 0) @defer.inlineCallbacks def test_reboot(self): instance_id = self.create_instance() - rv = yield self.compute.run_instance(instance_id) - - rv = yield self.compute.describe_instances() - self.assertEqual(rv[instance_id].name, instance_id) - + yield self.compute.run_instance(instance_id) yield self.compute.reboot_instance(instance_id) - - rv = yield self.compute.describe_instances() - self.assertEqual(rv[instance_id].name, instance_id) - rv = yield self.compute.terminate_instance(instance_id) + yield self.compute.terminate_instance(instance_id) @defer.inlineCallbacks def test_console_output(self): @@ -118,10 +119,6 @@ class ComputeConnectionTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_instance_existing(self): instance_id = self.create_instance() - rv = yield self.compute.run_instance(instance_id) - - rv = yield self.compute.describe_instances() - self.assertEqual(rv[instance_id].name, instance_id) - - self.assertRaises(exception.Error, self.compute.run_instance, instance_id) - rv = yield self.compute.terminate_instance(instance_id) + yield self.compute.run_instance(instance_id) + self.assertFailure(self.compute.run_instance(instance_id), exception.Error) + yield self.compute.terminate_instance(instance_id) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index d9ae5ac96..90ea9d053 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -46,14 +46,14 @@ class FakeConnection(object): def spawn(self, instance): fake_instance = FakeInstance() - self.instances[instance.name] = fake_instance + self.instances[instance.id] = fake_instance fake_instance._state = power_state.RUNNING def reboot(self, instance): pass def destroy(self, instance): - del self.instances[instance.name] + del self.instances[instance.id] def get_info(self, instance_id): i = self.instances[instance_id] diff --git a/nova/volume/service.py b/nova/volume/service.py index 66163a812..1086b4cd0 100644 --- a/nova/volume/service.py +++ b/nova/volume/service.py @@ -142,7 +142,7 @@ class VolumeService(service.Service): "sudo vgcreate %s %s" % (FLAGS.volume_group, FLAGS.storage_dev)) -class Volume(datastore.BasicModel): +class Volume(): def __init__(self, volume_id=None): self.volume_id = volume_id |
