diff options
96 files changed, 1387 insertions, 2966 deletions
diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 2bc407658..392b328b1 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -25,7 +25,6 @@ from eventlet.green import urllib2 import exceptions import gettext -import logging import os import sys import time @@ -130,6 +129,7 @@ class AjaxConsoleProxy(object): if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() server = wsgi.Server() acp = AjaxConsoleProxy() acp.register_listeners() diff --git a/bin/nova-api b/bin/nova-api index 11176a021..d5efb4687 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -39,11 +39,15 @@ from nova import log as logging from nova import version from nova import wsgi -logging.basicConfig() LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS +flags.DEFINE_string('ec2_listen', "0.0.0.0", + 'IP address for EC2 API to listen') +flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') +flags.DEFINE_string('osapi_listen', "0.0.0.0", + 'IP address for OpenStack API to listen') +flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') API_ENDPOINTS = ['ec2', 'osapi'] @@ -57,21 +61,15 @@ def run_app(paste_config_file): LOG.debug(_("No paste configuration for app: %s"), api) continue LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) return - # NOTE(todd): redo logging config, verbose could be set in paste config - logging.basicConfig() server = wsgi.Server() for app in apps: server.start(*app) @@ -80,8 +78,13 @@ def run_app(paste_config_file): if __name__ == '__main__': FLAGS(sys.argv) + logging.setup() LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) conf = wsgi.paste_config_file('nova-api.conf') if conf: run_app(conf) diff --git a/bin/nova-combined b/bin/nova-combined deleted file mode 100755 index 913c866bf..000000000 --- a/bin/nova-combined +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Combined starter script for Nova services.""" - -import eventlet -eventlet.monkey_patch() - -import gettext -import os -import sys - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import wsgi - - -FLAGS = flags.FLAGS - - -if __name__ == '__main__': - utils.default_flagfile() - FLAGS(sys.argv) - logging.basicConfig() - - compute = service.Service.create(binary='nova-compute') - network = service.Service.create(binary='nova-network') - volume = service.Service.create(binary='nova-volume') - scheduler = service.Service.create(binary='nova-scheduler') - #objectstore = service.Service.create(binary='nova-objectstore') - - service.serve(compute, network, volume, scheduler) - - apps = [] - paste_config_file = wsgi.paste_config_file('nova-api.conf') - for api in ['osapi', 'ec2']: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - continue - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) - if len(apps) > 0: - logging.basicConfig() - server = wsgi.Server() - for app in apps: - server.start(*app) - server.wait() diff --git a/bin/nova-compute b/bin/nova-compute index d2d352da2..95fa393b1 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-console b/bin/nova-console index 802cc80b6..40608b995 100755 --- a/bin/nova-console +++ b/bin/nova-console @@ -35,10 +35,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index d38ba2543..04a1771f0 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -102,7 +102,7 @@ def main(): flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) - logging.basicConfig() + logging.setup() interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') if int(os.environ.get('TESTING', '0')): FLAGS.fake_rabbit = True @@ -113,7 +113,9 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', - 'nova.sqlite')) + 'nova', + 'tests', + 'tests.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] if action in ['add', 'del', 'old']: diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 173b39bdb..6c63bd26b 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -35,6 +35,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova import wsgi from nova.api import direct @@ -48,6 +49,7 @@ flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() direct.register_service('compute', compute_api.API()) direct.register_service('reflect', direct.Reflection()) diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 036b41e48..404ae37f4 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -41,6 +41,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova.objectstore import image @@ -92,6 +93,7 @@ def main(): """Main entry point.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() images = get_images() if len(argv) == 2: diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 7dca02014..24cc9fd23 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -41,9 +41,6 @@ from nova import utils from nova import twistd from nova.compute import monitor -# TODO(todd): shouldn't this be done with flags? And what about verbose? -logging.getLogger('boto').setLevel(logging.WARN) - LOG = logging.getLogger('nova.instancemonitor') diff --git a/bin/nova-manage b/bin/nova-manage index 6d67252b8..5189de0e1 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -86,8 +86,6 @@ from nova.auth import manager from nova.cloudpipe import pipelib from nova.db import migration - -logging.basicConfig() FLAGS = flags.FLAGS flags.DECLARE('fixed_range', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') @@ -710,6 +708,7 @@ def main(): """Parse options and call the appropriate class/method.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() script_name = argv.pop(0) if len(argv) < 1: diff --git a/bin/nova-network b/bin/nova-network index 0143846a7..101761ef7 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index f4c0eaed6..0c205a80f 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index ad3ddc405..8dcdbc500 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/contrib/nova.sh b/contrib/nova.sh index 9259035ca..1187f2728 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -66,7 +66,7 @@ if [ "$CMD" == "install" ]; then sudo apt-get install -y user-mode-linux kvm libvirt-bin sudo apt-get install -y screen euca2ools vlan curl rabbitmq-server sudo apt-get install -y lvm2 iscsitarget open-iscsi - sudo apt-get install -y socat + sudo apt-get install -y socat unzip echo "ISCSITARGET_ENABLE=true" | sudo tee /etc/default/iscsitarget sudo /etc/init.d/iscsitarget restart sudo modprobe kvm @@ -111,8 +111,7 @@ if [ "$CMD" == "run" ]; then --nodaemon --dhcpbridge_flagfile=$NOVA_DIR/bin/nova.conf --network_manager=nova.network.manager.$NET_MAN ---cc_host=$HOST_IP ---routing_source_ip=$HOST_IP +--my_ip=$HOST_IP --sql_connection=$SQL_CONN --auth_driver=nova.auth.$AUTH --libvirt_type=$LIBVIRT_TYPE @@ -151,7 +150,6 @@ NOVA_CONF_EOF mkdir -p $NOVA_DIR/instances rm -rf $NOVA_DIR/networks mkdir -p $NOVA_DIR/networks - $NOVA_DIR/tools/clean-vlans if [ ! -d "$NOVA_DIR/images" ]; then ln -s $DIR/images $NOVA_DIR/images fi @@ -169,10 +167,14 @@ NOVA_CONF_EOF # create a project called 'admin' with project manager of 'admin' $NOVA_DIR/bin/nova-manage project create admin admin # export environment variables for project 'admin' and user 'admin' - $NOVA_DIR/bin/nova-manage project environment admin admin $NOVA_DIR/novarc + $NOVA_DIR/bin/nova-manage project zipfile admin admin $NOVA_DIR/nova.zip + unzip -o $NOVA_DIR/nova.zip -d $NOVA_DIR/ # create a small network $NOVA_DIR/bin/nova-manage network create 10.0.0.0/8 1 32 + # create some floating ips + $NOVA_DIR/bin/nova-manage floating create `hostname` 10.6.0.0/27 + # nova api crashes if we start it with a regular screen command, # so send the start command by forcing text into the window. screen_it api "$NOVA_DIR/bin/nova-api" diff --git a/contrib/puppet/files/etc/default/nova-compute b/contrib/puppet/files/etc/default/nova-compute deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-compute +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/default/nova-volume b/contrib/puppet/files/etc/default/nova-volume deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-volume +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/issue b/contrib/puppet/files/etc/issue deleted file mode 100644 index 8c567221b..000000000 --- a/contrib/puppet/files/etc/issue +++ /dev/null @@ -1,5 +0,0 @@ ------------------------------------------------ - - Welcome to your OpenStack installation! - ------------------------------------------------ diff --git a/contrib/puppet/files/etc/libvirt/qemu.conf b/contrib/puppet/files/etc/libvirt/qemu.conf deleted file mode 100644 index 7839f12e5..000000000 --- a/contrib/puppet/files/etc/libvirt/qemu.conf +++ /dev/null @@ -1,170 +0,0 @@ -# Master configuration file for the QEMU driver. -# All settings described here are optional - if omitted, sensible -# defaults are used. - -# VNC is configured to listen on 127.0.0.1 by default. -# To make it listen on all public interfaces, uncomment -# this next option. -# -# NB, strong recommendation to enable TLS + x509 certificate -# verification when allowing public access -# -# vnc_listen = "0.0.0.0" - - -# Enable use of TLS encryption on the VNC server. This requires -# a VNC client which supports the VeNCrypt protocol extension. -# Examples include vinagre, virt-viewer, virt-manager and vencrypt -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to setup CA and issue a server certificate -# before enabling this. -# -# vnc_tls = 1 - - -# Use of TLS requires that x509 certificates be issued. The -# default it to keep them in /etc/pki/libvirt-vnc. This directory -# must contain -# -# ca-cert.pem - the CA master certificate -# server-cert.pem - the server certificate signed with ca-cert.pem -# server-key.pem - the server private key -# -# This option allows the certificate directory to be changed -# -# vnc_tls_x509_cert_dir = "/etc/pki/libvirt-vnc" - - -# The default TLS configuration only uses certificates for the server -# allowing the client to verify the server's identity and establish -# and encrypted channel. -# -# It is possible to use x509 certificates for authentication too, by -# issuing a x509 certificate to every client who needs to connect. -# -# Enabling this option will reject any client who does not have a -# certificate signed by the CA in /etc/pki/libvirt-vnc/ca-cert.pem -# -# vnc_tls_x509_verify = 1 - - -# The default VNC password. Only 8 letters are significant for -# VNC passwords. This parameter is only used if the per-domain -# XML config does not already provide a password. To allow -# access without passwords, leave this commented out. An empty -# string will still enable passwords, but be rejected by QEMU -# effectively preventing any use of VNC. Obviously change this -# example here before you set this -# -# vnc_password = "XYZ12345" - - -# Enable use of SASL encryption on the VNC server. This requires -# a VNC client which supports the SASL protocol extension. -# Examples include vinagre, virt-viewer and virt-manager -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to configure /etc/sasl2/qemu.conf to choose -# the desired SASL plugin (eg, GSSPI for Kerberos) -# -# vnc_sasl = 1 - - -# The default SASL configuration file is located in /etc/sasl2/ -# When running libvirtd unprivileged, it may be desirable to -# override the configs in this location. Set this parameter to -# point to the directory, and create a qemu.conf in that location -# -# vnc_sasl_dir = "/some/directory/sasl2" - - - - -# The default security driver is SELinux. If SELinux is disabled -# on the host, then the security driver will automatically disable -# itself. If you wish to disable QEMU SELinux security driver while -# leaving SELinux enabled for the host in general, then set this -# to 'none' instead -# -# security_driver = "selinux" - - -# The user ID for QEMU processes run by the system instance -user = "root" - -# The group ID for QEMU processes run by the system instance -group = "root" - -# Whether libvirt should dynamically change file ownership -# to match the configured user/group above. Defaults to 1. -# Set to 0 to disable file ownership changes. -#dynamic_ownership = 1 - - -# What cgroup controllers to make use of with QEMU guests -# -# - 'cpu' - use for schedular tunables -# - 'devices' - use for device whitelisting -# -# NB, even if configured here, they won't be used unless -# the adminsitrator has mounted cgroups. eg -# -# mkdir /dev/cgroup -# mount -t cgroup -o devices,cpu none /dev/cgroup -# -# They can be mounted anywhere, and different controlers -# can be mounted in different locations. libvirt will detect -# where they are located. -# -# cgroup_controllers = [ "cpu", "devices" ] - -# This is the basic set of devices allowed / required by -# all virtual machines. -# -# As well as this, any configured block backed disks, -# all sound device, and all PTY devices are allowed. -# -# This will only need setting if newer QEMU suddenly -# wants some device we don't already know a bout. -# -#cgroup_device_acl = [ -# "/dev/null", "/dev/full", "/dev/zero", -# "/dev/random", "/dev/urandom", -# "/dev/ptmx", "/dev/kvm", "/dev/kqemu", -# "/dev/rtc", "/dev/hpet", "/dev/net/tun", -#] - -# The default format for Qemu/KVM guest save images is raw; that is, the -# memory from the domain is dumped out directly to a file. If you have -# guests with a large amount of memory, however, this can take up quite -# a bit of space. If you would like to compress the images while they -# are being saved to disk, you can also set "lzop", "gzip", "bzip2", or "xz" -# for save_image_format. Note that this means you slow down the process of -# saving a domain in order to save disk space; the list above is in descending -# order by performance and ascending order by compression ratio. -# -# save_image_format = "raw" - -# If provided by the host and a hugetlbfs mount point is configured, -# a guest may request huge page backing. When this mount point is -# unspecified here, determination of a host mount point in /proc/mounts -# will be attempted. Specifying an explicit mount overrides detection -# of the same in /proc/mounts. Setting the mount point to "" will -# disable guest hugepage backing. -# -# NB, within this mount point, guests will create memory backing files -# in a location of $MOUNTPOINT/libvirt/qemu - -# hugetlbfs_mount = "/dev/hugepages" - -# mac_filter enables MAC addressed based filtering on bridge ports. -# This currently requires ebtables to be installed. -# -# mac_filter = 1 - -# By default, PCI devices below non-ACS switch are not allowed to be assigned -# to guests. By setting relaxed_acs_check to 1 such devices will be allowed to -# be assigned to guests. -# -# relaxed_acs_check = 1 diff --git a/contrib/puppet/files/etc/lvm/lvm.conf b/contrib/puppet/files/etc/lvm/lvm.conf deleted file mode 100644 index 4e814ad49..000000000 --- a/contrib/puppet/files/etc/lvm/lvm.conf +++ /dev/null @@ -1,463 +0,0 @@ -# This is an example configuration file for the LVM2 system. -# It contains the default settings that would be used if there was no -# /etc/lvm/lvm.conf file. -# -# Refer to 'man lvm.conf' for further information including the file layout. -# -# To put this file in a different directory and override /etc/lvm set -# the environment variable LVM_SYSTEM_DIR before running the tools. - - -# This section allows you to configure which block devices should -# be used by the LVM system. -devices { - - # Where do you want your volume groups to appear ? - dir = "/dev" - - # An array of directories that contain the device nodes you wish - # to use with LVM2. - scan = [ "/dev" ] - - # If several entries in the scanned directories correspond to the - # same block device and the tools need to display a name for device, - # all the pathnames are matched against each item in the following - # list of regular expressions in turn and the first match is used. - preferred_names = [ ] - - # Try to avoid using undescriptive /dev/dm-N names, if present. - # preferred_names = [ "^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d" ] - - # A filter that tells LVM2 to only use a restricted set of devices. - # The filter consists of an array of regular expressions. These - # expressions can be delimited by a character of your choice, and - # prefixed with either an 'a' (for accept) or 'r' (for reject). - # The first expression found to match a device name determines if - # the device will be accepted or rejected (ignored). Devices that - # don't match any patterns are accepted. - - # Be careful if there there are symbolic links or multiple filesystem - # entries for the same device as each name is checked separately against - # the list of patterns. The effect is that if any name matches any 'a' - # pattern, the device is accepted; otherwise if any name matches any 'r' - # pattern it is rejected; otherwise it is accepted. - - # Don't have more than one filter line active at once: only one gets used. - - # Run vgscan after you change this parameter to ensure that - # the cache file gets regenerated (see below). - # If it doesn't do what you expect, check the output of 'vgscan -vvvv'. - - - # By default we accept every block device: - filter = [ "r|/dev/etherd/.*|", "r|/dev/block/.*|", "a/.*/" ] - - # Exclude the cdrom drive - # filter = [ "r|/dev/cdrom|" ] - - # When testing I like to work with just loopback devices: - # filter = [ "a/loop/", "r/.*/" ] - - # Or maybe all loops and ide drives except hdc: - # filter =[ "a|loop|", "r|/dev/hdc|", "a|/dev/ide|", "r|.*|" ] - - # Use anchors if you want to be really specific - # filter = [ "a|^/dev/hda8$|", "r/.*/" ] - - # The results of the filtering are cached on disk to avoid - # rescanning dud devices (which can take a very long time). - # By default this cache is stored in the /etc/lvm/cache directory - # in a file called '.cache'. - # It is safe to delete the contents: the tools regenerate it. - # (The old setting 'cache' is still respected if neither of - # these new ones is present.) - cache_dir = "/etc/lvm/cache" - cache_file_prefix = "" - - # You can turn off writing this cache file by setting this to 0. - write_cache_state = 1 - - # Advanced settings. - - # List of pairs of additional acceptable block device types found - # in /proc/devices with maximum (non-zero) number of partitions. - # types = [ "fd", 16 ] - - # If sysfs is mounted (2.6 kernels) restrict device scanning to - # the block devices it believes are valid. - # 1 enables; 0 disables. - sysfs_scan = 1 - - # By default, LVM2 will ignore devices used as components of - # software RAID (md) devices by looking for md superblocks. - # 1 enables; 0 disables. - md_component_detection = 1 - - # By default, if a PV is placed directly upon an md device, LVM2 - # will align its data blocks with the md device's stripe-width. - # 1 enables; 0 disables. - md_chunk_alignment = 1 - - # By default, the start of a PV's data area will be a multiple of - # the 'minimum_io_size' or 'optimal_io_size' exposed in sysfs. - # - minimum_io_size - the smallest request the device can perform - # w/o incurring a read-modify-write penalty (e.g. MD's chunk size) - # - optimal_io_size - the device's preferred unit of receiving I/O - # (e.g. MD's stripe width) - # minimum_io_size is used if optimal_io_size is undefined (0). - # If md_chunk_alignment is enabled, that detects the optimal_io_size. - # This setting takes precedence over md_chunk_alignment. - # 1 enables; 0 disables. - data_alignment_detection = 1 - - # Alignment (in KB) of start of data area when creating a new PV. - # If a PV is placed directly upon an md device and md_chunk_alignment or - # data_alignment_detection is enabled this parameter is ignored. - # Set to 0 for the default alignment of 64KB or page size, if larger. - data_alignment = 0 - - # By default, the start of the PV's aligned data area will be shifted by - # the 'alignment_offset' exposed in sysfs. This offset is often 0 but - # may be non-zero; e.g.: certain 4KB sector drives that compensate for - # windows partitioning will have an alignment_offset of 3584 bytes - # (sector 7 is the lowest aligned logical block, the 4KB sectors start - # at LBA -1, and consequently sector 63 is aligned on a 4KB boundary). - # 1 enables; 0 disables. - data_alignment_offset_detection = 1 - - # If, while scanning the system for PVs, LVM2 encounters a device-mapper - # device that has its I/O suspended, it waits for it to become accessible. - # Set this to 1 to skip such devices. This should only be needed - # in recovery situations. - ignore_suspended_devices = 0 -} - -# This section that allows you to configure the nature of the -# information that LVM2 reports. -log { - - # Controls the messages sent to stdout or stderr. - # There are three levels of verbosity, 3 being the most verbose. - verbose = 0 - - # Should we send log messages through syslog? - # 1 is yes; 0 is no. - syslog = 1 - - # Should we log error and debug messages to a file? - # By default there is no log file. - #file = "/var/log/lvm2.log" - - # Should we overwrite the log file each time the program is run? - # By default we append. - overwrite = 0 - - # What level of log messages should we send to the log file and/or syslog? - # There are 6 syslog-like log levels currently in use - 2 to 7 inclusive. - # 7 is the most verbose (LOG_DEBUG). - level = 0 - - # Format of output messages - # Whether or not (1 or 0) to indent messages according to their severity - indent = 1 - - # Whether or not (1 or 0) to display the command name on each line output - command_names = 0 - - # A prefix to use before the message text (but after the command name, - # if selected). Default is two spaces, so you can see/grep the severity - # of each message. - prefix = " " - - # To make the messages look similar to the original LVM tools use: - # indent = 0 - # command_names = 1 - # prefix = " -- " - - # Set this if you want log messages during activation. - # Don't use this in low memory situations (can deadlock). - # activation = 0 -} - -# Configuration of metadata backups and archiving. In LVM2 when we -# talk about a 'backup' we mean making a copy of the metadata for the -# *current* system. The 'archive' contains old metadata configurations. -# Backups are stored in a human readeable text format. -backup { - - # Should we maintain a backup of the current metadata configuration ? - # Use 1 for Yes; 0 for No. - # Think very hard before turning this off! - backup = 1 - - # Where shall we keep it ? - # Remember to back up this directory regularly! - backup_dir = "/etc/lvm/backup" - - # Should we maintain an archive of old metadata configurations. - # Use 1 for Yes; 0 for No. - # On by default. Think very hard before turning this off. - archive = 1 - - # Where should archived files go ? - # Remember to back up this directory regularly! - archive_dir = "/etc/lvm/archive" - - # What is the minimum number of archive files you wish to keep ? - retain_min = 10 - - # What is the minimum time you wish to keep an archive file for ? - retain_days = 30 -} - -# Settings for the running LVM2 in shell (readline) mode. -shell { - - # Number of lines of history to store in ~/.lvm_history - history_size = 100 -} - - -# Miscellaneous global LVM2 settings -global { - - # The file creation mask for any files and directories created. - # Interpreted as octal if the first digit is zero. - umask = 077 - - # Allow other users to read the files - #umask = 022 - - # Enabling test mode means that no changes to the on disk metadata - # will be made. Equivalent to having the -t option on every - # command. Defaults to off. - test = 0 - - # Default value for --units argument - units = "h" - - # Since version 2.02.54, the tools distinguish between powers of - # 1024 bytes (e.g. KiB, MiB, GiB) and powers of 1000 bytes (e.g. - # KB, MB, GB). - # If you have scripts that depend on the old behaviour, set this to 0 - # temporarily until you update them. - si_unit_consistency = 1 - - # Whether or not to communicate with the kernel device-mapper. - # Set to 0 if you want to use the tools to manipulate LVM metadata - # without activating any logical volumes. - # If the device-mapper kernel driver is not present in your kernel - # setting this to 0 should suppress the error messages. - activation = 1 - - # If we can't communicate with device-mapper, should we try running - # the LVM1 tools? - # This option only applies to 2.4 kernels and is provided to help you - # switch between device-mapper kernels and LVM1 kernels. - # The LVM1 tools need to be installed with .lvm1 suffices - # e.g. vgscan.lvm1 and they will stop working after you start using - # the new lvm2 on-disk metadata format. - # The default value is set when the tools are built. - # fallback_to_lvm1 = 0 - - # The default metadata format that commands should use - "lvm1" or "lvm2". - # The command line override is -M1 or -M2. - # Defaults to "lvm2". - # format = "lvm2" - - # Location of proc filesystem - proc = "/proc" - - # Type of locking to use. Defaults to local file-based locking (1). - # Turn locking off by setting to 0 (dangerous: risks metadata corruption - # if LVM2 commands get run concurrently). - # Type 2 uses the external shared library locking_library. - # Type 3 uses built-in clustered locking. - # Type 4 uses read-only locking which forbids any operations that might - # change metadata. - locking_type = 1 - - # Set to 0 to fail when a lock request cannot be satisfied immediately. - wait_for_locks = 1 - - # If using external locking (type 2) and initialisation fails, - # with this set to 1 an attempt will be made to use the built-in - # clustered locking. - # If you are using a customised locking_library you should set this to 0. - fallback_to_clustered_locking = 1 - - # If an attempt to initialise type 2 or type 3 locking failed, perhaps - # because cluster components such as clvmd are not running, with this set - # to 1 an attempt will be made to use local file-based locking (type 1). - # If this succeeds, only commands against local volume groups will proceed. - # Volume Groups marked as clustered will be ignored. - fallback_to_local_locking = 1 - - # Local non-LV directory that holds file-based locks while commands are - # in progress. A directory like /tmp that may get wiped on reboot is OK. - locking_dir = "/var/lock/lvm" - - # Whenever there are competing read-only and read-write access requests for - # a volume group's metadata, instead of always granting the read-only - # requests immediately, delay them to allow the read-write requests to be - # serviced. Without this setting, write access may be stalled by a high - # volume of read-only requests. - # NB. This option only affects locking_type = 1 viz. local file-based - # locking. - prioritise_write_locks = 1 - - # Other entries can go here to allow you to load shared libraries - # e.g. if support for LVM1 metadata was compiled as a shared library use - # format_libraries = "liblvm2format1.so" - # Full pathnames can be given. - - # Search this directory first for shared libraries. - # library_dir = "/lib/lvm2" - - # The external locking library to load if locking_type is set to 2. - # locking_library = "liblvm2clusterlock.so" -} - -activation { - # Set to 0 to disable udev syncronisation (if compiled into the binaries). - # Processes will not wait for notification from udev. - # They will continue irrespective of any possible udev processing - # in the background. You should only use this if udev is not running - # or has rules that ignore the devices LVM2 creates. - # The command line argument --nodevsync takes precedence over this setting. - # If set to 1 when udev is not running, and there are LVM2 processes - # waiting for udev, run 'dmsetup udevcomplete_all' manually to wake them up. - udev_sync = 1 - - # How to fill in missing stripes if activating an incomplete volume. - # Using "error" will make inaccessible parts of the device return - # I/O errors on access. You can instead use a device path, in which - # case, that device will be used to in place of missing stripes. - # But note that using anything other than "error" with mirrored - # or snapshotted volumes is likely to result in data corruption. - missing_stripe_filler = "error" - - # How much stack (in KB) to reserve for use while devices suspended - reserved_stack = 256 - - # How much memory (in KB) to reserve for use while devices suspended - reserved_memory = 8192 - - # Nice value used while devices suspended - process_priority = -18 - - # If volume_list is defined, each LV is only activated if there is a - # match against the list. - # "vgname" and "vgname/lvname" are matched exactly. - # "@tag" matches any tag set in the LV or VG. - # "@*" matches if any tag defined on the host is also set in the LV or VG - # - # volume_list = [ "vg1", "vg2/lvol1", "@tag1", "@*" ] - - # Size (in KB) of each copy operation when mirroring - mirror_region_size = 512 - - # Setting to use when there is no readahead value stored in the metadata. - # - # "none" - Disable readahead. - # "auto" - Use default value chosen by kernel. - readahead = "auto" - - # 'mirror_image_fault_policy' and 'mirror_log_fault_policy' define - # how a device failure affecting a mirror is handled. - # A mirror is composed of mirror images (copies) and a log. - # A disk log ensures that a mirror does not need to be re-synced - # (all copies made the same) every time a machine reboots or crashes. - # - # In the event of a failure, the specified policy will be used to determine - # what happens. This applies to automatic repairs (when the mirror is being - # monitored by dmeventd) and to manual lvconvert --repair when - # --use-policies is given. - # - # "remove" - Simply remove the faulty device and run without it. If - # the log device fails, the mirror would convert to using - # an in-memory log. This means the mirror will not - # remember its sync status across crashes/reboots and - # the entire mirror will be re-synced. If a - # mirror image fails, the mirror will convert to a - # non-mirrored device if there is only one remaining good - # copy. - # - # "allocate" - Remove the faulty device and try to allocate space on - # a new device to be a replacement for the failed device. - # Using this policy for the log is fast and maintains the - # ability to remember sync state through crashes/reboots. - # Using this policy for a mirror device is slow, as it - # requires the mirror to resynchronize the devices, but it - # will preserve the mirror characteristic of the device. - # This policy acts like "remove" if no suitable device and - # space can be allocated for the replacement. - # - # "allocate_anywhere" - Not yet implemented. Useful to place the log device - # temporarily on same physical volume as one of the mirror - # images. This policy is not recommended for mirror devices - # since it would break the redundant nature of the mirror. This - # policy acts like "remove" if no suitable device and space can - # be allocated for the replacement. - - mirror_log_fault_policy = "allocate" - mirror_device_fault_policy = "remove" -} - - -#################### -# Advanced section # -#################### - -# Metadata settings -# -# metadata { - # Default number of copies of metadata to hold on each PV. 0, 1 or 2. - # You might want to override it from the command line with 0 - # when running pvcreate on new PVs which are to be added to large VGs. - - # pvmetadatacopies = 1 - - # Approximate default size of on-disk metadata areas in sectors. - # You should increase this if you have large volume groups or - # you want to retain a large on-disk history of your metadata changes. - - # pvmetadatasize = 255 - - # List of directories holding live copies of text format metadata. - # These directories must not be on logical volumes! - # It's possible to use LVM2 with a couple of directories here, - # preferably on different (non-LV) filesystems, and with no other - # on-disk metadata (pvmetadatacopies = 0). Or this can be in - # addition to on-disk metadata areas. - # The feature was originally added to simplify testing and is not - # supported under low memory situations - the machine could lock up. - # - # Never edit any files in these directories by hand unless you - # you are absolutely sure you know what you are doing! Use - # the supplied toolset to make changes (e.g. vgcfgrestore). - - # dirs = [ "/etc/lvm/metadata", "/mnt/disk2/lvm/metadata2" ] -#} - -# Event daemon -# -dmeventd { - # mirror_library is the library used when monitoring a mirror device. - # - # "libdevmapper-event-lvm2mirror.so" attempts to recover from - # failures. It removes failed devices from a volume group and - # reconfigures a mirror as necessary. If no mirror library is - # provided, mirrors are not monitored through dmeventd. - - mirror_library = "libdevmapper-event-lvm2mirror.so" - - # snapshot_library is the library used when monitoring a snapshot device. - # - # "libdevmapper-event-lvm2snapshot.so" monitors the filling of - # snapshots and emits a warning through syslog, when the use of - # snapshot exceedes 80%. The warning is repeated when 85%, 90% and - # 95% of the snapshot are filled. - - snapshot_library = "libdevmapper-event-lvm2snapshot.so" -} diff --git a/contrib/puppet/files/etc/nova.conf b/contrib/puppet/files/etc/nova.conf deleted file mode 100644 index a0d64078c..000000000 --- a/contrib/puppet/files/etc/nova.conf +++ /dev/null @@ -1,28 +0,0 @@ ---ec2_url=http://192.168.255.1:8773/services/Cloud ---rabbit_host=192.168.255.1 ---redis_host=192.168.255.1 ---s3_host=192.168.255.1 ---vpn_ip=192.168.255.1 ---datastore_path=/var/lib/nova/keeper ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vlan_start=2000 ---vlan_end=3000 ---private_range=192.168.0.0/16 ---public_range=10.0.0.0/24 ---volume_group=vgdata ---storage_dev=/dev/sdc ---bridge_dev=eth2 ---aoe_eth_dev=eth2 ---public_interface=vlan0 ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---vpn_image_id=ami-cloudpipe ---daemonize ---verbose ---syslog ---prefix=nova diff --git a/contrib/puppet/files/production/boto.cfg b/contrib/puppet/files/production/boto.cfg deleted file mode 100644 index f4a2de2b6..000000000 --- a/contrib/puppet/files/production/boto.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[Boto] -debug = 0 -num_retries = 1 diff --git a/contrib/puppet/files/production/libvirt.qemu.xml.template b/contrib/puppet/files/production/libvirt.qemu.xml.template deleted file mode 100644 index 114dfdc01..000000000 --- a/contrib/puppet/files/production/libvirt.qemu.xml.template +++ /dev/null @@ -1,35 +0,0 @@ -<domain type='%(type)s'> - <name>%(name)s</name> - <os> - <type>hvm</type> - <kernel>%(basepath)s/kernel</kernel> - <initrd>%(basepath)s/ramdisk</initrd> - <cmdline>root=/dev/vda1 console=ttyS0</cmdline> - </os> - <features> - <acpi/> - </features> - <memory>%(memory_kb)s</memory> - <vcpu>%(vcpus)s</vcpu> - <devices> - <disk type='file'> - <source file='%(basepath)s/disk'/> - <target dev='vda' bus='virtio'/> - </disk> - <interface type='bridge'> - <source bridge='%(bridge_name)s'/> - <mac address='%(mac_address)s'/> - <!-- <model type='virtio'/> CANT RUN virtio network right now --> - <!-- - <filterref filter="nova-instance-%(name)s"> - <parameter name="IP" value="%(ip_address)s" /> - <parameter name="DHCPSERVER" value="%(dhcp_server)s" /> - </filterref> - --> - </interface> - <serial type="file"> - <source path='%(basepath)s/console.log'/> - <target port='1'/> - </serial> - </devices> -</domain> diff --git a/contrib/puppet/files/production/my.cnf b/contrib/puppet/files/production/my.cnf deleted file mode 100644 index 8777bc480..000000000 --- a/contrib/puppet/files/production/my.cnf +++ /dev/null @@ -1,137 +0,0 @@ -# -# The MySQL database server configuration file. -# -# You can copy this to one of: -# - "/etc/mysql/my.cnf" to set global options, -# - "~/.my.cnf" to set user-specific options. -# -# One can use all long options that the program supports. -# Run program with --help to get a list of available options and with -# --print-defaults to see which it would actually understand and use. -# -# For explanations see -# http://dev.mysql.com/doc/mysql/en/server-system-variables.html - -# This will be passed to all mysql clients -# It has been reported that passwords should be enclosed with ticks/quotes -# escpecially if they contain "#" chars... -# Remember to edit /etc/mysql/debian.cnf when changing the socket location. -[client] -port = 3306 -socket = /var/run/mysqld/mysqld.sock - -# Here is entries for some specific programs -# The following values assume you have at least 32M ram - -# This was formally known as [safe_mysqld]. Both versions are currently parsed. -[mysqld_safe] -socket = /var/run/mysqld/mysqld.sock -nice = 0 - -[mysqld] -# -# * Basic Settings -# - -# -# * IMPORTANT -# If you make changes to these settings and your system uses apparmor, you may -# also need to also adjust /etc/apparmor.d/usr.sbin.mysqld. -# - -user = mysql -socket = /var/run/mysqld/mysqld.sock -port = 3306 -basedir = /usr -datadir = /var/lib/mysql -tmpdir = /tmp -skip-external-locking -# -# Instead of skip-networking the default is now to listen only on -# localhost which is more compatible and is not less secure. -# bind-address = 127.0.0.1 -# -# * Fine Tuning -# -innodb_buffer_pool_size = 12G -#innodb_log_file_size = 256M -innodb_log_buffer_size=4M -innodb_flush_log_at_trx_commit=2 -innodb_thread_concurrency=8 -innodb_flush_method=O_DIRECT -key_buffer = 128M -max_allowed_packet = 256M -thread_stack = 8196K -thread_cache_size = 32 -# This replaces the startup script and checks MyISAM tables if needed -# the first time they are touched -myisam-recover = BACKUP -max_connections = 1000 -table_cache = 1024 -#thread_concurrency = 10 -# -# * Query Cache Configuration -# -query_cache_limit = 32M -query_cache_size = 256M -# -# * Logging and Replication -# -# Both location gets rotated by the cronjob. -# Be aware that this log type is a performance killer. -# As of 5.1 you can enable the log at runtime! -#general_log_file = /var/log/mysql/mysql.log -#general_log = 1 - -log_error = /var/log/mysql/error.log - -# Here you can see queries with especially long duration -log_slow_queries = /var/log/mysql/mysql-slow.log -long_query_time = 2 -#log-queries-not-using-indexes -# -# The following can be used as easy to replay backup logs or for replication. -# note: if you are setting up a replication slave, see README.Debian about -# other settings you may need to change. -server-id = 1 -log_bin = /var/log/mysql/mysql-bin.log -expire_logs_days = 10 -max_binlog_size = 50M -#binlog_do_db = include_database_name -#binlog_ignore_db = include_database_name -# -# * InnoDB -# -sync_binlog=1 -# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. -# Read the manual for more InnoDB related options. There are many! -# -# * Security Features -# -# Read the manual, too, if you want chroot! -# chroot = /var/lib/mysql/ -# -# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". -# -# ssl-ca=/etc/mysql/cacert.pem -# ssl-cert=/etc/mysql/server-cert.pem -# ssl-key=/etc/mysql/server-key.pem - - - -[mysqldump] -quick -quote-names -max_allowed_packet = 256M - -[mysql] -#no-auto-rehash # faster start of mysql but no tab completition - -[isamchk] -key_buffer = 128M - -# -# * IMPORTANT: Additional settings that can override those from this file! -# The files must end with '.cnf', otherwise they'll be ignored. -# -!includedir /etc/mysql/conf.d/ diff --git a/contrib/puppet/files/production/nova-iptables b/contrib/puppet/files/production/nova-iptables deleted file mode 100755 index 61e2ca2b9..000000000 --- a/contrib/puppet/files/production/nova-iptables +++ /dev/null @@ -1,187 +0,0 @@ -#! /bin/sh - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# NOTE(vish): This script sets up some reasonable defaults for iptables and -# creates nova-specific chains. If you use this script you should -# run nova-network and nova-compute with --use_nova_chains=True - - -# NOTE(vish): If you run public nova-api on a different port, make sure to -# change the port here - -if [ -f /etc/default/nova-iptables ] ; then - . /etc/default/nova-iptables -fi - -export LC_ALL=C - -API_PORT=${API_PORT:-"8773"} - -if [ ! -n "$IP" ]; then - # NOTE(vish): IP address is what address the services ALLOW on. - # This will just get the first ip in the list, so if you - # have more than one eth device set up, this will fail, and - # you should explicitly pass in the ip of the instance - IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` -fi - -if [ ! -n "$PRIVATE_RANGE" ]; then - #NOTE(vish): PRIVATE_RANGE: range is ALLOW to access DHCP - PRIVATE_RANGE="192.168.0.0/12" -fi - -if [ ! -n "$MGMT_IP" ]; then - # NOTE(vish): Management IP is the ip over which to allow ssh traffic. It - # will also allow traffic to nova-api - MGMT_IP="$IP" -fi - -if [ ! -n "$DMZ_IP" ]; then - # NOTE(vish): DMZ IP is the ip over which to allow api & objectstore access - DMZ_IP="$IP" -fi - -clear_nova_iptables() { - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - iptables -F - iptables -t nat -F - iptables -F services - iptables -X services - # HACK: re-adding fail2ban rules :( - iptables -N fail2ban-ssh - iptables -A INPUT -p tcp -m multiport --dports 22 -j fail2ban-ssh - iptables -A fail2ban-ssh -j RETURN -} - -load_nova_iptables() { - - iptables -P INPUT DROP - iptables -A INPUT -m state --state INVALID -j DROP - iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - # NOTE(ja): allow localhost for everything - iptables -A INPUT -d 127.0.0.1/32 -j ACCEPT - # NOTE(ja): 22 only allowed MGMT_IP before, but we widened it to any - # address, since ssh should be listening only on internal - # before we re-add this rule we will need to add - # flexibility for RSYNC between omega/stingray - iptables -A INPUT -m tcp -p tcp --dport 22 -j ACCEPT - iptables -A INPUT -m udp -p udp --dport 123 -j ACCEPT - iptables -A INPUT -p icmp -j ACCEPT - iptables -N services - iptables -A INPUT -j services - iptables -A INPUT -p tcp -j REJECT --reject-with tcp-reset - iptables -A INPUT -j REJECT --reject-with icmp-port-unreachable - - iptables -P FORWARD DROP - iptables -A FORWARD -m state --state INVALID -j DROP - iptables -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT - iptables -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu - - # NOTE(vish): DROP on output is too restrictive for now. We need to add - # in a bunch of more specific output rules to use it. - # iptables -P OUTPUT DROP - iptables -A OUTPUT -m state --state INVALID -j DROP - iptables -A OUTPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - - if [ -n "$GANGLIA" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 8649 -j ACCEPT - iptables -A services -m udp -p udp -d $IP --dport 8649 -j ACCEPT - fi - - # if [ -n "$WEB" ] || [ -n "$ALL" ]; then - # # NOTE(vish): This opens up ports for web access, allowing web-based - # # dashboards to work. - # iptables -A services -m tcp -p tcp -d $IP --dport 80 -j ACCEPT - # iptables -A services -m tcp -p tcp -d $IP --dport 443 -j ACCEPT - # fi - - if [ -n "$OBJECTSTORE" ] || [ -n "$ALL" ]; then - # infrastructure - iptables -A services -m tcp -p tcp -d $IP --dport 3333 -j ACCEPT - # clients - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport 3333 -j ACCEPT - fi - - if [ -n "$API" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport $API_PORT -j ACCEPT - if [ "$IP" != "$DMZ_IP" ]; then - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport $API_PORT -j ACCEPT - fi - if [ "$IP" != "$MGMT_IP" ] && [ "$DMZ_IP" != "$MGMT_IP" ]; then - iptables -A services -m tcp -p tcp -d $MGMT_IP --dport $API_PORT -j ACCEPT - fi - fi - - if [ -n "$REDIS" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 6379 -j ACCEPT - fi - - if [ -n "$MYSQL" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3306 -j ACCEPT - fi - - if [ -n "$RABBITMQ" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 4369 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 5672 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 53284 -j ACCEPT - fi - - if [ -n "$DNSMASQ" ] || [ -n "$ALL" ]; then - # NOTE(vish): this could theoretically be setup per network - # for each host, but it seems like overkill - iptables -A services -m tcp -p tcp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp --dport 67 -j ACCEPT - fi - - if [ -n "$LDAP" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 389 -j ACCEPT - fi - - if [ -n "$ISCSI" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3260 -j ACCEPT - iptables -A services -m tcp -p tcp -d 127.0.0.0/16 --dport 3260 -j ACCEPT - fi -} - - -case "$1" in - start) - echo "Starting nova-iptables: " - load_nova_iptables - ;; - stop) - echo "Clearing nova-iptables: " - clear_nova_iptables - ;; - restart) - echo "Restarting nova-iptables: " - clear_nova_iptables - load_nova_iptables - ;; - *) - echo "Usage: $NAME {start|stop|restart}" >&2 - exit 1 - ;; -esac - -exit 0 diff --git a/contrib/puppet/files/production/nova-iscsi-dev.sh b/contrib/puppet/files/production/nova-iscsi-dev.sh deleted file mode 100644 index 8eda10d2e..000000000 --- a/contrib/puppet/files/production/nova-iscsi-dev.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# FILE: /etc/udev/scripts/iscsidev.sh - -BUS=${1} -HOST=${BUS%%:*} - -[ -e /sys/class/iscsi_host ] || exit 1 - -file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" - -target_name=$(cat ${file}) - -# This is not an open-scsi drive -if [ -z "${target_name}" ]; then - exit 1 -fi - -echo "${target_name##*:}" diff --git a/contrib/puppet/files/production/setup_data.sh b/contrib/puppet/files/production/setup_data.sh deleted file mode 100755 index 1fbbac41c..000000000 --- a/contrib/puppet/files/production/setup_data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -/root/slap.sh -mysql -e "DROP DATABASE nova" -mysql -e "CREATE DATABASE nova" -mysql -e "GRANT ALL on nova.* to nova@'%' identified by 'TODO:CHANGEME:CMON'" -touch /root/installed diff --git a/contrib/puppet/files/production/slap.sh b/contrib/puppet/files/production/slap.sh deleted file mode 100755 index f8ea16949..000000000 --- a/contrib/puppet/files/production/slap.sh +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# LDAP INSTALL SCRIPT - SHOULD BE IDEMPOTENT, but it SCRUBS all USERS - -apt-get install -y slapd ldap-utils python-ldap - -cat >/etc/ldap/schema/openssh-lpk_openldap.schema <<LPK_SCHEMA_EOF -# -# LDAP Public Key Patch schema for use with openssh-ldappubkey -# Author: Eric AUGE <eau@phear.org> -# -# Based on the proposal of : Mark Ruijter -# - - -# octetString SYNTAX -attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' - DESC 'MANDATORY: OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -# printableString SYNTAX yes|no -objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY - DESC 'MANDATORY: OpenSSH LPK objectclass' - MAY ( sshPublicKey $ uid ) - ) -LPK_SCHEMA_EOF - -cat >/etc/ldap/schema/nova.schema <<NOVA_SCHEMA_EOF -# -# Person object for Nova -# inetorgperson with extra attributes -# Author: Vishvananda Ishaya <vishvananda@yahoo.com> -# -# - -# using internet experimental oid arc as per BP64 3.1 -objectidentifier novaSchema 1.3.6.1.3.1.666.666 -objectidentifier novaAttrs novaSchema:3 -objectidentifier novaOCs novaSchema:4 - -attributetype ( - novaAttrs:1 - NAME 'accessKey' - DESC 'Key for accessing data' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:2 - NAME 'secretKey' - DESC 'Secret key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - -objectClass ( - novaOCs:1 - NAME 'novaUser' - DESC 'access and secret keys' - AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) - ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) - ) - -NOVA_SCHEMA_EOF - -mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig -cat >/etc/ldap/slapd.conf <<SLAPD_CONF_EOF -# slapd.conf - Configuration file for LDAP SLAPD -########## -# Basics # -########## -include /etc/ldap/schema/core.schema -include /etc/ldap/schema/cosine.schema -include /etc/ldap/schema/inetorgperson.schema -include /etc/ldap/schema/openssh-lpk_openldap.schema -include /etc/ldap/schema/nova.schema -pidfile /var/run/slapd/slapd.pid -argsfile /var/run/slapd/slapd.args -loglevel none -modulepath /usr/lib/ldap -# modulepath /usr/local/libexec/openldap -moduleload back_hdb -########################## -# Database Configuration # -########################## -database hdb -suffix "dc=example,dc=com" -rootdn "cn=Manager,dc=example,dc=com" -rootpw changeme -directory /var/lib/ldap -# directory /usr/local/var/openldap-data -index objectClass,cn eq -######## -# ACLs # -######## -access to attrs=userPassword - by anonymous auth - by self write - by * none -access to * - by self write - by * none -SLAPD_CONF_EOF - -mv /etc/ldap/ldap.conf /etc/ldap/ldap.conf.orig - -cat >/etc/ldap/ldap.conf <<LDAP_CONF_EOF -# LDAP Client Settings -URI ldap://localhost -BASE dc=example,dc=com -BINDDN cn=Manager,dc=example,dc=com -SIZELIMIT 0 -TIMELIMIT 0 -LDAP_CONF_EOF - -cat >/etc/ldap/base.ldif <<BASE_LDIF_EOF -# This is the root of the directory tree -dn: dc=example,dc=com -description: Example.Com, your trusted non-existent corporation. -dc: example -o: Example.Com -objectClass: top -objectClass: dcObject -objectClass: organization - -# Subtree for users -dn: ou=Users,dc=example,dc=com -ou: Users -description: Users -objectClass: organizationalUnit - -# Subtree for groups -dn: ou=Groups,dc=example,dc=com -ou: Groups -description: Groups -objectClass: organizationalUnit - -# Subtree for system accounts -dn: ou=System,dc=example,dc=com -ou: System -description: Special accounts used by software applications. -objectClass: organizationalUnit - -# Special Account for Authentication: -dn: uid=authenticate,ou=System,dc=example,dc=com -uid: authenticate -ou: System -description: Special account for authenticating users -userPassword: {MD5}TODO-000000000000000000000000000== -objectClass: account -objectClass: simpleSecurityObject - -# create the sysadmin entry - -dn: cn=developers,ou=Groups,dc=example,dc=com -objectclass: groupOfNames -cn: developers -description: IT admin group -member: uid=admin,ou=Users,dc=example,dc=com - -dn: cn=sysadmins,ou=Groups,dc=example,dc=com -objectclass: groupOfNames -cn: sysadmins -description: IT admin group -member: uid=admin,ou=Users,dc=example,dc=com - -dn: cn=netadmins,ou=Groups,dc=example,dc=com -objectclass: groupOfNames -cn: netadmins -description: Network admin group -member: uid=admin,ou=Users,dc=example,dc=com - -dn: cn=cloudadmins,ou=Groups,dc=example,dc=com -objectclass: groupOfNames -cn: cloudadmins -description: Cloud admin group -member: uid=admin,ou=Users,dc=example,dc=com - -dn: cn=itsec,ou=Groups,dc=example,dc=com -objectclass: groupOfNames -cn: itsec -description: IT security users group -member: uid=admin,ou=Users,dc=example,dc=com -BASE_LDIF_EOF - -/etc/init.d/slapd stop -rm -rf /var/lib/ldap/* -rm -rf /etc/ldap/slapd.d/* -slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d -cp /usr/share/slapd/DB_CONFIG /var/lib/ldap/DB_CONFIG -slapadd -v -l /etc/ldap/base.ldif -chown -R openldap:openldap /etc/ldap/slapd.d -chown -R openldap:openldap /var/lib/ldap -/etc/init.d/slapd start diff --git a/contrib/puppet/fileserver.conf b/contrib/puppet/fileserver.conf deleted file mode 100644 index 6e2984b8c..000000000 --- a/contrib/puppet/fileserver.conf +++ /dev/null @@ -1,8 +0,0 @@ -# fileserver.conf - -[files] -path /srv/cloud/puppet/files -allow 10.0.0.0/24 - -[plugins] - diff --git a/contrib/puppet/manifests/classes/apt.pp b/contrib/puppet/manifests/classes/apt.pp deleted file mode 100644 index 03022aeef..000000000 --- a/contrib/puppet/manifests/classes/apt.pp +++ /dev/null @@ -1 +0,0 @@ -exec { "update-apt": command => "/usr/bin/apt-get update" } diff --git a/contrib/puppet/manifests/classes/issue.pp b/contrib/puppet/manifests/classes/issue.pp deleted file mode 100644 index 8bb37ee3f..000000000 --- a/contrib/puppet/manifests/classes/issue.pp +++ /dev/null @@ -1,14 +0,0 @@ -class issue { - file { "/etc/issue": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } - file { "/etc/issue.net": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } -} diff --git a/contrib/puppet/manifests/classes/kern_module.pp b/contrib/puppet/manifests/classes/kern_module.pp deleted file mode 100644 index 00ec0636c..000000000 --- a/contrib/puppet/manifests/classes/kern_module.pp +++ /dev/null @@ -1,34 +0,0 @@ -# via http://projects.puppetlabs.com/projects/puppet/wiki/Kernel_Modules_Patterns - -define kern_module ($ensure) { - $modulesfile = $operatingsystem ? { ubuntu => "/etc/modules", redhat => "/etc/rc.modules" } - case $operatingsystem { - redhat: { file { "/etc/rc.modules": ensure => file, mode => 755 } } - } - case $ensure { - present: { - exec { "insert_module_${name}": - command => $operatingsystem ? { - ubuntu => "/bin/echo '${name}' >> '${modulesfile}'", - redhat => "/bin/echo '/sbin/modprobe ${name}' >> '${modulesfile}' " - }, - unless => "/bin/grep -qFx '${name}' '${modulesfile}'" - } - exec { "/sbin/modprobe ${name}": unless => "/bin/grep -q '^${name} ' '/proc/modules'" } - } - absent: { - exec { "/sbin/modprobe -r ${name}": onlyif => "/bin/grep -q '^${name} ' '/proc/modules'" } - exec { "remove_module_${name}": - command => $operatingsystem ? { - ubuntu => "/usr/bin/perl -ni -e 'print unless /^\\Q${name}\\E\$/' '${modulesfile}'", - redhat => "/usr/bin/perl -ni -e 'print unless /^\\Q/sbin/modprobe ${name}\\E\$/' '${modulesfile}'" - }, - onlyif => $operatingsystem ? { - ubuntu => "/bin/grep -qFx '${name}' '${modulesfile}'", - redhat => "/bin/grep -q '^/sbin/modprobe ${name}' '${modulesfile}'" - } - } - } - default: { err ( "unknown ensure value ${ensure}" ) } - } -} diff --git a/contrib/puppet/manifests/classes/loopback.pp b/contrib/puppet/manifests/classes/loopback.pp deleted file mode 100644 index e0fa9d541..000000000 --- a/contrib/puppet/manifests/classes/loopback.pp +++ /dev/null @@ -1,6 +0,0 @@ -define loopback($num) { - exec { "mknod -m 0660 /dev/loop${num} b 7 ${num}; chown root:disk /dev/loop${num}": - creates => "/dev/loop${num}", - path => ["/usr/bin", "/usr/sbin", "/bin"] - } -} diff --git a/contrib/puppet/manifests/classes/lvm.pp b/contrib/puppet/manifests/classes/lvm.pp deleted file mode 100644 index 5a407abcb..000000000 --- a/contrib/puppet/manifests/classes/lvm.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvm { - file { "/etc/lvm/lvm.conf": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/lvm.conf", - } -} diff --git a/contrib/puppet/manifests/classes/lvmconf.pp b/contrib/puppet/manifests/classes/lvmconf.pp deleted file mode 100644 index 4aa7ddfdc..000000000 --- a/contrib/puppet/manifests/classes/lvmconf.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvmconf { - file { "/etc/lvm/lvm.conf": - owner => "root", group => "root", mode => 644, - source => "puppet://${puppet_server}/files/etc/lvm/lvm.conf", - ensure => present - } -} - diff --git a/contrib/puppet/manifests/classes/nova.pp b/contrib/puppet/manifests/classes/nova.pp deleted file mode 100644 index e942860f4..000000000 --- a/contrib/puppet/manifests/classes/nova.pp +++ /dev/null @@ -1,464 +0,0 @@ -import "kern_module" -import "apt" -import "loopback" - -#$head_node_ip = "undef" -#$rabbit_ip = "undef" -#$vpn_ip = "undef" -#$public_interface = "undef" -#$vlan_start = "5000" -#$vlan_end = "6000" -#$private_range = "10.0.0.0/16" -#$public_range = "192.168.177.0/24" - -define nova_iptables($services, $ip="", $private_range="", $mgmt_ip="", $dmz_ip="") { - file { "/etc/init.d/nova-iptables": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iptables", - } - - file { "/etc/default/nova-iptables": - owner => "root", mode => 644, - content => template("nova-iptables.erb") - } -} - -define nova_conf_pointer($name) { - file { "/etc/nova/nova-${name}.conf": - owner => "nova", mode => 400, - content => "--flagfile=/etc/nova/nova.conf" - } -} - -class novaconf { - file { "/etc/nova/nova.conf": - owner => "nova", mode => 400, - content => template("production/nova-common.conf.erb", "production/nova-${cluster_name}.conf.erb") - } - nova_conf_pointer{'manage': name => 'manage'} -} - -class novadata { - package { "rabbitmq-server": ensure => present } - - file { "/etc/rabbitmq/rabbitmq.conf": - owner => "root", mode => 644, - content => "NODENAME=rabbit@localhost", - } - - service { "rabbitmq-server": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/rabbitmq/rabbitmq.conf"], - Package["rabbitmq-server"] - ] - } - - package { "mysql-server": ensure => present } - - file { "/etc/mysql/my.cnf": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/my.cnf", - } - - service { "mysql": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/mysql/my.cnf"], - Package["mysql-server"] - ] - } - - file { "/root/slap.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/slap.sh", - } - - file { "/root/setup_data.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/setup_data.sh", - } - - # setup compute data - exec { "setup_data": - command => "/root/setup_data.sh", - path => "/usr/bin:/bin", - unless => "test -f /root/installed", - require => [ - Service["mysql"], - File["/root/slap.sh"], - File["/root/setup_data.sh"] - ] - } -} - -define nscheduler($version) { - package { "nova-scheduler": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'scheduler': name => 'scheduler'} - exec { "update-rc.d -f nova-scheduler remove; update-rc.d nova-scheduler defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-scheduler", - unless => "test -f /etc/rc2.d/S50nova-scheduler" - } - service { "nova-scheduler": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-scheduler"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-scheduler.conf"] - ] - } - -} - -define napi($version, $api_servers, $api_base_port) { - file { "/etc/boto.cfg": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/boto.cfg", - } - - file { "/var/lib/nova/CA/genvpn.sh": - owner => "nova", mode => 755, - source => "puppet://${puppet_server}/files/production/genvpn.sh", - } - - package { "python-greenlet": ensure => present } - package { "nova-api": ensure => $version, require => [Exec["update-apt"], Package["python-greenlet"]] } - nova_conf_pointer{'api': name => 'api'} - - exec { "update-rc.d -f nova-api remove; update-rc.d nova-api defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-api", - unless => "test -f /etc/rc2.d/S50nova-api" - } - - service { "nova-netsync": - start => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock start", - stop => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock stop", - ensure => running, - hasstatus => false, - pattern => "nova-netsync", - require => Service["nova-api"], - subscribe => File["/etc/nova/nova.conf"] - } - service { "nova-api": - start => "monit start all -g nova_api", - stop => "monit stop all -g nova_api", - restart => "monit restart all -g nova_api", - # ensure => running, - # hasstatus => true, - require => Service["monit"], - subscribe => [ - Package["nova-objectstore"], - File["/etc/boto.cfg"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } - - # the haproxy & monit's template use $api_servers and $api_base_port - - package { "haproxy": ensure => present } - file { "/etc/default/haproxy": - owner => "root", mode => 644, - content => "ENABLED=1", - require => Package['haproxy'] - } - file { "/etc/haproxy/haproxy.cfg": - owner => "root", mode => 644, - content => template("/srv/cloud/puppet/templates/haproxy.cfg.erb"), - require => Package['haproxy'] - } - service { "haproxy": - ensure => true, - enable => true, - hasstatus => true, - subscribe => [ - Package["haproxy"], - File["/etc/default/haproxy"], - File["/etc/haproxy/haproxy.cfg"], - ] - } - - package { "socat": ensure => present } - - file { "/usr/local/bin/gmetric_haproxy.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/ganglia/gmetric_scripts/gmetric_haproxy.sh", - } - - cron { "gmetric_haproxy": - command => "/usr/local/bin/gmetric_haproxy.sh", - user => root, - minute => "*/3", - } - - package { "monit": ensure => present } - - file { "/etc/default/monit": - owner => "root", mode => 644, - content => "startup=1", - require => Package['monit'] - } - file { "/etc/monit/monitrc": - owner => "root", mode => 600, - content => template("/srv/cloud/puppet/templates/monitrc-nova-api.erb"), - require => Package['monit'] - } - service { "monit": - ensure => true, - pattern => "sbin/monit", - subscribe => [ - Package["monit"], - File["/etc/default/monit"], - File["/etc/monit/monitrc"], - ] - } - -} - - -define nnetwork($version) { - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - # EVIL HACK: custom binary because dnsmasq 2.52 segfaulted accessing dereferenced object - file { "/usr/sbin/dnsmasq": - owner => "root", group => "root", - source => "puppet://${puppet_server}/files/production/dnsmasq", - } - - package { "nova-network": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'dhcpbridge': name => 'dhcpbridge'} - nova_conf_pointer{'network': name => "network" } - - exec { "update-rc.d -f nova-network remove; update-rc.d nova-network defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-network", - unless => "test -f /etc/rc2.d/S50nova-network" - } - service { "nova-network": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-network"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-network.conf"] - ] - } -} - -define nobjectstore($version) { - package { "nova-objectstore": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'objectstore': name => 'objectstore'} - exec { "update-rc.d -f nova-objectstore remove; update-rc.d nova-objectstore defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-objectstore", - unless => "test -f /etc/rc2.d/S50nova-objectstore" - } - service { "nova-objectstore": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-objectstore"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } -} - -define ncompute($version) { - include ganglia-python - include ganglia-compute - - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - - # LIBVIRT has to be restarted when ebtables / gawk is installed - service { "libvirt-bin": - ensure => running, - pattern => "sbin/libvirtd", - subscribe => [ - Package["ebtables"], - Kern_module["kvm_intel"] - ], - require => [ - Package["libvirt-bin"], - Package["ebtables"], - Package["gawk"], - Kern_module["kvm_intel"], - File["/dev/kvm"] - ] - } - - package { "libvirt-bin": ensure => "0.8.3-1ubuntu14~ppalucid2" } - package { "ebtables": ensure => present } - package { "gawk": ensure => present } - - # ensure proper permissions on /dev/kvm - file { "/dev/kvm": - owner => "root", - group => "kvm", - mode => 660 - } - - # require hardware virt - kern_module { "kvm_intel": - ensure => present, - } - - # increase loopback devices - file { "/etc/modprobe.d/loop.conf": - owner => "root", mode => 644, - content => "options loop max_loop=40" - } - - nova_conf_pointer{'compute': name => 'compute'} - - loopback{loop0: num => 0} - loopback{loop1: num => 1} - loopback{loop2: num => 2} - loopback{loop3: num => 3} - loopback{loop4: num => 4} - loopback{loop5: num => 5} - loopback{loop6: num => 6} - loopback{loop7: num => 7} - loopback{loop8: num => 8} - loopback{loop9: num => 9} - loopback{loop10: num => 10} - loopback{loop11: num => 11} - loopback{loop12: num => 12} - loopback{loop13: num => 13} - loopback{loop14: num => 14} - loopback{loop15: num => 15} - loopback{loop16: num => 16} - loopback{loop17: num => 17} - loopback{loop18: num => 18} - loopback{loop19: num => 19} - loopback{loop20: num => 20} - loopback{loop21: num => 21} - loopback{loop22: num => 22} - loopback{loop23: num => 23} - loopback{loop24: num => 24} - loopback{loop25: num => 25} - loopback{loop26: num => 26} - loopback{loop27: num => 27} - loopback{loop28: num => 28} - loopback{loop29: num => 29} - loopback{loop30: num => 30} - loopback{loop31: num => 31} - loopback{loop32: num => 32} - loopback{loop33: num => 33} - loopback{loop34: num => 34} - loopback{loop35: num => 35} - loopback{loop36: num => 36} - loopback{loop37: num => 37} - loopback{loop38: num => 38} - loopback{loop39: num => 39} - - package { "python-libvirt": ensure => "0.8.3-1ubuntu14~ppalucid2" } - - package { "nova-compute": - ensure => "$version", - require => Package["python-libvirt"] - } - - #file { "/usr/share/nova/libvirt.qemu.xml.template": - # owner => "nova", mode => 400, - # source => "puppet://${puppet_server}/files/production/libvirt.qemu.xml.template", - #} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-compute remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-compute" - } - service { "nova-compute": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-compute"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-compute.conf"], - #File["/usr/share/nova/libvirt.qemu.xml.template"], - Service["libvirt-bin"], - Kern_module["kvm_intel"] - ] - } -} - -define nvolume($version) { - - package { "nova-volume": ensure => $version, require => Exec["update-apt"] } - - nova_conf_pointer{'volume': name => 'volume'} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-volume remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-volume" - } - - file { "/etc/default/iscsitarget": - owner => "root", mode => 644, - content => "ISCSITARGET_ENABLE=true" - } - - package { "iscsitarget": ensure => present } - - file { "/dev/iscsi": ensure => directory } # FIXME(vish): owner / mode? - file { "/usr/sbin/nova-iscsi-dev.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iscsi-dev.sh" - } - file { "/etc/udev/rules.d/55-openiscsi.rules": - owner => "root", mode => 644, - content => 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/usr/sbin/nova-iscsi-dev.sh %b",SYMLINK+="iscsi/%c%n"' - } - - service { "iscsitarget": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/default/iscsitarget"], - Package["iscsitarget"] - ] - } - - service { "nova-volume": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-volume"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-volume.conf"] - ] - } -} - -class novaspool { - # This isn't in release yet - #cron { logspool: - # command => "/usr/bin/nova-logspool /var/log/nova.log /var/lib/nova/spool", - # user => "nova" - #} - #cron { spoolsentry: - # command => "/usr/bin/nova-spoolsentry ${sentry_url} ${sentry_key} /var/lib/nova/spool", - # user => "nova" - #} -} diff --git a/contrib/puppet/manifests/classes/swift.pp b/contrib/puppet/manifests/classes/swift.pp deleted file mode 100644 index 64ffb6fa3..000000000 --- a/contrib/puppet/manifests/classes/swift.pp +++ /dev/null @@ -1,7 +0,0 @@ -class swift { - package { "memcached": ensure => present } - service { "memcached": require => Package['memcached'] } - - package { "swift-proxy": ensure => present } -} - diff --git a/contrib/puppet/manifests/site.pp b/contrib/puppet/manifests/site.pp deleted file mode 100644 index ca07a34ad..000000000 --- a/contrib/puppet/manifests/site.pp +++ /dev/null @@ -1,120 +0,0 @@ -# site.pp - -import "templates" -import "classes/*" - -node novabase inherits default { -# $puppet_server = "192.168.0.10" - $cluster_name = "openstack001" - $ganglia_udp_send_channel = "openstack001.example.com" - $syslog = "192.168.0.10" - - # THIS STUFF ISN'T IN RELEASE YET - #$sentry_url = "http://192.168.0.19/sentry/store/" - #$sentry_key = "TODO:SENTRYPASS" - - $local_network = "192.168.0.0/16" - $vpn_ip = "192.168.0.2" - $public_interface = "eth0" - include novanode -# include nova-common - include opsmetrics - -# non-nova stuff such as nova-dash inherit from novanode -# novaspool needs a better home -# include novaspool -} - -# Builder -node "nova000.example.com" inherits novabase { - $syslog = "server" - include ntp - include syslog-server -} - -# Non-Nova nodes - -node - "blog.example.com", - "wiki.example.com" -inherits novabase { - include ganglia-python - include ganglia-apache - include ganglia-mysql -} - - -node "nova001.example.com" -inherits novabase { - include novabase - - nova_iptables { nova: - services => [ - "ganglia", - "mysql", - "rabbitmq", - "ldap", - "api", - "objectstore", - "nrpe", - ], - ip => "192.168.0.10", - } - - nobjectstore { nova: version => "0.9.0" } - nscheduler { nova: version => "0.9.0" } - napi { nova: - version => "0.9.0", - api_servers => 10, - api_base_port => 8000 - } -} - -node "nova002.example.com" -inherits novabase { - include novaconf - - nova_iptables { nova: - services => [ - "ganglia", - "dnsmasq", - "nrpe" - ], - ip => "192.168.4.2", - private_range => "192.168.0.0/16", - } - - nnetwork { nova: version => "0.9.0" } -} - -node - "nova003.example.com", - "nova004.example.com", - "nova005.example.com", - "nova006.example.com", - "nova007.example.com", - "nova008.example.com", - "nova009.example.com", - "nova010.example.com", - "nova011.example.com", - "nova012.example.com", - "nova013.example.com", - "nova014.example.com", - "nova015.example.com", - "nova016.example.com", - "nova017.example.com", - "nova018.example.com", - "nova019.example.com", -inherits novabase { - include novaconf - ncompute { nova: version => "0.9.0" } - nvolume { nova: version => "0.9.0" } -} - -#node -# "nova020.example.com" -# "nova021.example.com" -#inherits novanode { -# include novaconf - #ncompute { nova: version => "0.9.0" } -#} diff --git a/contrib/puppet/manifests/templates.pp b/contrib/puppet/manifests/templates.pp deleted file mode 100644 index 90e433013..000000000 --- a/contrib/puppet/manifests/templates.pp +++ /dev/null @@ -1,21 +0,0 @@ -# templates.pp - -import "classes/*" - -class baseclass { -# include dns-client # FIXME: missing resolv.conf.erb?? - include issue -} - -node default { - $nova_site = "undef" - $nova_ns1 = "undef" - $nova_ns2 = "undef" -# include baseclass -} - -# novanode handles the system-level requirements for Nova/Swift nodes -class novanode { - include baseclass - include lvmconf -} diff --git a/contrib/puppet/puppet.conf b/contrib/puppet/puppet.conf deleted file mode 100644 index 92af920e3..000000000 --- a/contrib/puppet/puppet.conf +++ /dev/null @@ -1,11 +0,0 @@ -[main] -logdir=/var/log/puppet -vardir=/var/lib/puppet -ssldir=/var/lib/puppet/ssl -rundir=/var/run/puppet -factpath=$vardir/lib/facter -pluginsync=false - -[puppetmasterd] -templatedir=/var/lib/nova/contrib/puppet/templates -autosign=true diff --git a/contrib/puppet/templates/haproxy.cfg.erb b/contrib/puppet/templates/haproxy.cfg.erb deleted file mode 100644 index bd9991de7..000000000 --- a/contrib/puppet/templates/haproxy.cfg.erb +++ /dev/null @@ -1,39 +0,0 @@ -# this config needs haproxy-1.1.28 or haproxy-1.2.1 - -global - log 127.0.0.1 local0 - log 127.0.0.1 local1 notice - #log loghost local0 info - maxconn 4096 - #chroot /usr/share/haproxy - stats socket /var/run/haproxy.sock - user haproxy - group haproxy - daemon - #debug - #quiet - -defaults - log global - mode http - option httplog - option dontlognull - retries 3 - option redispatch - stats enable - stats uri /haproxy - maxconn 2000 - contimeout 5000 - clitimeout 50000 - srvtimeout 50000 - - -listen nova-api 0.0.0.0:8773 - option httpchk GET / HTTP/1.0\r\nHost:\ example.com - option forwardfor - reqidel ^X-Forwarded-For:.* - balance roundrobin -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset -%> - server api_<%= port %> 127.0.0.1:<%= port %> maxconn 1 check -<% end -%> - option httpclose # disable keep-alive diff --git a/contrib/puppet/templates/monitrc-nova-api.erb b/contrib/puppet/templates/monitrc-nova-api.erb deleted file mode 100644 index fe2626327..000000000 --- a/contrib/puppet/templates/monitrc-nova-api.erb +++ /dev/null @@ -1,138 +0,0 @@ -############################################################################### -## Monit control file -############################################################################### -## -## Comments begin with a '#' and extend through the end of the line. Keywords -## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. -## -## Below you will find examples of some frequently used statements. For -## information about the control file, a complete list of statements and -## options please have a look in the monit manual. -## -## -############################################################################### -## Global section -############################################################################### -## -## Start monit in the background (run as a daemon): -# -set daemon 60 # check services at 1-minute intervals - with start delay 30 # optional: delay the first check by half a minute - # (by default check immediately after monit start) - - -## Set syslog logging with the 'daemon' facility. If the FACILITY option is -## omitted, monit will use 'user' facility by default. If you want to log to -## a stand alone log file instead, specify the path to a log file -# -set logfile syslog facility log_daemon -# -# -### Set the location of monit id file which saves the unique id specific for -### given monit. The id is generated and stored on first monit start. -### By default the file is placed in $HOME/.monit.id. -# -# set idfile /var/.monit.id -# -### Set the location of monit state file which saves the monitoring state -### on each cycle. By default the file is placed in $HOME/.monit.state. If -### state file is stored on persistent filesystem, monit will recover the -### monitoring state across reboots. If it is on temporary filesystem, the -### state will be lost on reboot. -# -# set statefile /var/.monit.state -# -## Set the list of mail servers for alert delivery. Multiple servers may be -## specified using comma separator. By default monit uses port 25 - this -## is possible to override with the PORT option. -# -# set mailserver mail.bar.baz, # primary mailserver -# backup.bar.baz port 10025, # backup mailserver on port 10025 -# localhost # fallback relay -# -# -## By default monit will drop alert events if no mail servers are available. -## If you want to keep the alerts for a later delivery retry, you can use the -## EVENTQUEUE statement. The base directory where undelivered alerts will be -## stored is specified by the BASEDIR option. You can limit the maximal queue -## size using the SLOTS option (if omitted, the queue is limited by space -## available in the back end filesystem). -# -# set eventqueue -# basedir /var/monit # set the base directory where events will be stored -# slots 100 # optionaly limit the queue size -# -# -## Send status and events to M/Monit (Monit central management: for more -## informations about M/Monit see http://www.tildeslash.com/mmonit). -# -# set mmonit http://monit:monit@192.168.1.10:8080/collector -# -# -## Monit by default uses the following alert mail format: -## -## --8<-- -## From: monit@$HOST # sender -## Subject: monit alert -- $EVENT $SERVICE # subject -## -## $EVENT Service $SERVICE # -## # -## Date: $DATE # -## Action: $ACTION # -## Host: $HOST # body -## Description: $DESCRIPTION # -## # -## Your faithful employee, # -## monit # -## --8<-- -## -## You can override this message format or parts of it, such as subject -## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. -## are expanded at runtime. For example, to override the sender: -# -# set mail-format { from: monit@foo.bar } -# -# -## You can set alert recipients here whom will receive alerts if/when a -## service defined in this file has errors. Alerts may be restricted on -## events by using a filter as in the second example below. -# -# set alert sysadm@foo.bar # receive all alerts -# set alert manager@foo.bar only on { timeout } # receive just service- -# # timeout alert -# -# -## Monit has an embedded web server which can be used to view status of -## services monitored, the current configuration, actual services parameters -## and manage services from a web interface. -# - set httpd port 2812 and - use address localhost # only accept connection from localhost - allow localhost # allow localhost to connect to the server and -# allow admin:monit # require user 'admin' with password 'monit' -# allow @monit # allow users of group 'monit' to connect (rw) -# allow @users readonly # allow users of group 'users' to connect readonly -# -# -############################################################################### -## Services -############################################################################### - -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset %> - -check process nova_api_<%= port %> with pidfile /var/run/nova/nova-api-<%= port %>.pid - group nova_api - start program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock start" - as uid nova - stop program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock stop" - as uid nova - if failed port <%= port %> protocol http - with timeout 15 seconds - for 4 cycles - then restart - if totalmem > 300 Mb then restart - if cpu is greater than 60% for 2 cycles then alert - if cpu > 80% for 3 cycles then restart - if 3 restarts within 5 cycles then timeout - -<% end %> diff --git a/contrib/puppet/templates/nova-iptables.erb b/contrib/puppet/templates/nova-iptables.erb deleted file mode 100644 index 2fc066305..000000000 --- a/contrib/puppet/templates/nova-iptables.erb +++ /dev/null @@ -1,10 +0,0 @@ -<% services.each do |service| -%> -<%= service.upcase %>=1 -<% end -%> -<% if ip && ip != "" %>IP="<%=ip%>"<% end %> -<% if private_range && private_range != "" %>PRIVATE_RANGE="<%=private_range%>"<% end %> -<% if mgmt_ip && mgmt_ip != "" %>MGMT_IP="<%=mgmt_ip%>"<% end %> -<% if dmz_ip && dmz_ip != "" %>DMZ_IP="<%=dmz_ip%>"<% end %> - -# warning: this file is auto-generated by puppet - diff --git a/contrib/puppet/templates/production/nova-common.conf.erb b/contrib/puppet/templates/production/nova-common.conf.erb deleted file mode 100644 index 23ee0c5e8..000000000 --- a/contrib/puppet/templates/production/nova-common.conf.erb +++ /dev/null @@ -1,55 +0,0 @@ -# global ---dmz_net=192.168.0.0 ---dmz_mask=255.255.0.0 ---dmz_cidr=192.168.0.0/16 ---ldap_user_dn=cn=Administrators,dc=example,dc=com ---ldap_user_unit=Users ---ldap_user_subtree=ou=Users,dc=example,dc=com ---ldap_project_subtree=ou=Groups,dc=example,dc=com ---role_project_subtree=ou=Groups,dc=example,dc=com ---ldap_cloudadmin=cn=NovaAdmins,ou=Groups,dc=example,dc=com ---ldap_itsec=cn=NovaSecurity,ou=Groups,dc=example,dc=com ---ldap_sysadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_netadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_developer=cn=developers,ou=Groups,dc=example,dc=com ---verbose ---daemonize ---syslog ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---scheduler_driver=nova.scheduler.simple.SimpleScheduler ---libvirt_xml_template=/usr/share/nova/libvirt.qemu.xml.template ---credentials_template=/usr/share/nova/novarc.template ---boot_script_template=/usr/share/nova/bootscript.template ---vpn_client_template=/usr/share/nova/client.ovpn.template ---max_cores=40 ---max_gigabytes=2000 ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vpn_start=11000 ---volume_group=vgdata ---volume_manager=nova.volume.manager.ISCSIManager ---volume_driver=nova.volume.driver.ISCSIDriver ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---dhcpbridge=/usr/bin/nova-dhcpbridge ---vpn_image_id=ami-cloudpipe ---dhcpbridge_flagfile=/etc/nova/nova.conf ---credential_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=NOVA/CN=%s-%s ---auth_driver=nova.auth.ldapdriver.LdapDriver ---quota_cores=17 ---quota_floating_ips=5 ---quota_instances=6 ---quota_volumes=10 ---quota_gigabytes=100 ---use_nova_chains=True ---input_chain=services ---use_project_ca=True ---fixed_ip_disassociate_timeout=300 ---api_max_requests=1 ---api_listen_ip=127.0.0.1 ---user_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=%s-%s-%s ---project_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-ca-%s-%s ---vpn_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-vpn-%s-%s diff --git a/contrib/puppet/templates/production/nova-nova.conf.erb b/contrib/puppet/templates/production/nova-nova.conf.erb deleted file mode 100644 index 8683fefde..000000000 --- a/contrib/puppet/templates/production/nova-nova.conf.erb +++ /dev/null @@ -1,21 +0,0 @@ ---fixed_range=192.168.0.0/16 ---iscsi_ip_prefix=192.168.4 ---floating_range=10.0.0.0/24 ---rabbit_host=192.168.0.10 ---s3_host=192.168.0.10 ---cc_host=192.168.0.10 ---cc_dmz=192.168.24.10 ---s3_dmz=192.168.24.10 ---ec2_url=http://192.168.0.1:8773/services/Cloud ---vpn_ip=192.168.0.2 ---ldap_url=ldap://192.168.0.10 ---sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---other_sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---routing_source_ip=192.168.0.2 ---bridge_dev=eth1 ---public_interface=eth0 ---vlan_start=3100 ---num_networks=700 ---rabbit_userid=TODO:RABBIT ---rabbit_password=TODO:CHANGEME ---ldap_password=TODO:CHANGEME diff --git a/etc/nova-api.conf b/etc/nova-api.conf index f0e749805..9f7e93d4c 100644 --- a/etc/nova-api.conf +++ b/etc/nova-api.conf @@ -1,6 +1,3 @@ -[DEFAULT] -verbose = 1 - ####### # EC2 # ####### diff --git a/nova/__init__.py b/nova/__init__.py index 8745617bc..256db55a9 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -30,5 +30,3 @@ .. moduleauthor:: Manish Singh <yosh@gimp.org> .. moduleauthor:: Andy Smith <andy@anarkystic.com> """ - -from exception import * diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 00b527d62..2b1acba5a 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -46,6 +46,11 @@ def _underscore_to_xmlcase(str): return res[:1].lower() + res[1:] +def _database_to_isoformat(datetimeobj): + """Return a xs:dateTime parsable string from datatime""" + return datetimeobj.strftime("%Y-%m-%dT%H:%M:%SZ") + + def _try_convert(value): """Return a non-string if possible""" if value == 'None': @@ -173,7 +178,8 @@ class APIRequest(object): elif isinstance(data, bool): data_el.appendChild(xml.createTextNode(str(data).lower())) elif isinstance(data, datetime.datetime): - data_el.appendChild(xml.createTextNode(data.isoformat())) + data_el.appendChild( + xml.createTextNode(_database_to_isoformat(data))) elif data != None: data_el.appendChild(xml.createTextNode(str(data))) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 882cdcfc9..7458d307a 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -198,8 +198,9 @@ class CloudController(object): return self._describe_availability_zones(context, **kwargs) def _describe_availability_zones(self, context, **kwargs): - enabled_services = db.service_get_all(context) - disabled_services = db.service_get_all(context, True) + ctxt = context.elevated() + enabled_services = db.service_get_all(ctxt) + disabled_services = db.service_get_all(ctxt, True) available_zones = [] for zone in [service.availability_zone for service in enabled_services]: @@ -318,14 +319,19 @@ class CloudController(object): def describe_security_groups(self, context, group_name=None, **kwargs): self.compute_api.ensure_default_security_group(context) - if context.is_admin: + if group_name: + groups = [] + for name in group_name: + group = db.security_group_get_by_name(context, + context.project_id, + name) + groups.append(group) + elif context.is_admin: groups = db.security_group_get_all(context) else: groups = db.security_group_get_by_project(context, context.project_id) groups = [self._format_security_group(context, g) for g in groups] - if not group_name is None: - groups = [g for g in groups if g.name in group_name] return {'securityGroupInfo': list(sorted(groups, @@ -529,8 +535,9 @@ class CloudController(object): def get_ajax_console(self, context, instance_id, **kwargs): ec2_id = instance_id[0] - internal_id = ec2_id_to_id(ec2_id) - return self.compute_api.get_ajax_console(context, internal_id) + instance_id = ec2_id_to_id(ec2_id) + return self.compute_api.get_ajax_console(context, + instance_id=instance_id) def describe_volumes(self, context, volume_id=None, **kwargs): if volume_id: @@ -669,7 +676,8 @@ class CloudController(object): instances = [] for ec2_id in instance_id: internal_id = ec2_id_to_id(ec2_id) - instance = self.compute_api.get(context, internal_id) + instance = self.compute_api.get(context, + instance_id=internal_id) instances.append(instance) else: instances = self.compute_api.get_all(context, **kwargs) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 0b6ef8fc5..1dfdd5318 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -19,7 +19,6 @@ import datetime import hashlib import json import time -import logging import webob.exc import webob.dec @@ -121,8 +120,8 @@ class AuthMiddleware(wsgi.Middleware): req - webob.Request object """ ctxt = context.get_admin_context() - user = self.auth.get_user_from_access_key(username) - if user and user.secret == key: + user = self.auth.get_user_from_access_key(key) + if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, time.time())).hexdigest() token_dict = {} diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index 197125d86..7abb5f884 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -15,7 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import time from webob import exc diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 9d56bc508..cf85a496f 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -15,8 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from webob import exc from nova import compute diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ce9601ecb..63e047b39 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -33,7 +33,6 @@ import nova.api.openstack LOG = logging.getLogger('server') -LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS @@ -64,23 +63,21 @@ def _translate_detail_keys(inst): inst_dict['addresses'] = dict(public=[], private=[]) # grab single private fixed ip - try: - private_ip = inst['fixed_ip']['address'] - if private_ip: - inst_dict['addresses']['private'].append(private_ip) - except KeyError: - LOG.debug(_("Failed to read private ip")) + private_ips = utils.get_from_path(inst, 'fixed_ip/address') + inst_dict['addresses']['private'] = private_ips # grab all public floating ips - try: - for floating in inst['fixed_ip']['floating_ips']: - inst_dict['addresses']['public'].append(floating['address']) - except KeyError: - LOG.debug(_("Failed to read public ip(s)")) + public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') + inst_dict['addresses']['public'] = public_ips - inst_dict['metadata'] = {} inst_dict['hostId'] = '' + # Return the metadata as a dictionary + metadata = {} + for item in inst['metadata']: + metadata[item['key']] = item['value'] + inst_dict['metadata'] = metadata + return dict(server=inst_dict) @@ -148,9 +145,10 @@ class Controller(wsgi.Controller): try: return image['properties'][param] except KeyError: - raise exception.NotFound( + LOG.debug( _("%(param)s property not found for image %(_image_id)s") % locals()) + return None image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) @@ -162,18 +160,29 @@ class Controller(wsgi.Controller): if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) - key_pairs = auth_manager.AuthManager.get_key_pairs( - req.environ['nova.context']) + context = req.environ['nova.context'] + key_pairs = auth_manager.AuthManager.get_key_pairs(context) if not key_pairs: raise exception.NotFound(_("No keypairs defined")) key_pair = key_pairs[0] image_id = common.get_image_id_from_image_hash(self._image_service, - req.environ['nova.context'], env['server']['imageId']) + context, env['server']['imageId']) kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( req, image_id) + + # Metadata is a list, not a Dictionary, because we allow duplicate keys + # (even though JSON can't encode this) + # In future, we may not allow duplicate keys. + # However, the CloudServers API is not definitive on this front, + # and we want to be compatible. + metadata = [] + if env['server']['metadata']: + for k, v in env['server']['metadata'].items(): + metadata.append({'key': k, 'value': v}) + instances = self.compute_api.create( - req.environ['nova.context'], + context, instance_types.get_by_flavor_id(env['server']['flavorId']), image_id, kernel_id=kernel_id, @@ -182,6 +191,7 @@ class Controller(wsgi.Controller): display_description=env['server']['name'], key_name=key_pair['name'], key_data=key_pair['public_key'], + metadata=metadata, onset_files=env.get('onset_files', [])) return _translate_keys(instances[0]) diff --git a/nova/api/openstack/shared_ip_groups.py b/nova/api/openstack/shared_ip_groups.py index bd3cc23a8..5d78f9377 100644 --- a/nova/api/openstack/shared_ip_groups.py +++ b/nova/api/openstack/shared_ip_groups.py @@ -15,8 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from webob import exc from nova import wsgi diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 830464ffd..d5206da20 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -14,7 +14,6 @@ # under the License. import common -import logging from nova import flags from nova import wsgi diff --git a/nova/compute/api.py b/nova/compute/api.py index 81ea6dc53..d9431c679 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -85,7 +85,7 @@ class API(base.Base): min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', - availability_zone=None, user_data=None, + availability_zone=None, user_data=None, metadata=[], onset_files=None): """Create the number of instances requested if quota and other arguments check out ok. @@ -100,6 +100,30 @@ class API(base.Base): "run %s more instances of this type.") % num_instances, "InstanceLimitExceeded") + num_metadata = len(metadata) + quota_metadata = quota.allowed_metadata_items(context, num_metadata) + if quota_metadata < num_metadata: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " tried to set %(num_metadata)s metadata properties") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + + # Because metadata is stored in the DB, we hard-code the size limits + # In future, we may support more variable length strings, so we act + # as if this is quota-controlled for forwards compatibility + for metadata_item in metadata: + k = metadata_item['key'] + v = metadata_item['value'] + if len(k) > 255 or len(v) > 255: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " metadata property key or value too long") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + image = self.image_service.show(context, image_id) if kernel_id is None: kernel_id = image.get('kernel_id', None) @@ -154,6 +178,7 @@ class API(base.Base): 'key_name': key_name, 'key_data': key_data, 'locked': False, + 'metadata': metadata, 'availability_zone': availability_zone} elevated = context.elevated() instances = [] @@ -447,7 +472,7 @@ class API(base.Base): {'method': 'authorize_ajax_console', 'args': {'token': output['token'], 'host': output['host'], 'port': output['port']}}) - return {'url': '%s?token=%s' % (FLAGS.ajax_console_proxy_url, + return {'url': '%s/?token=%s' % (FLAGS.ajax_console_proxy_url, output['token'])} def get_console_output(self, context, instance_id): diff --git a/nova/console/manager.py b/nova/console/manager.py index 5697e7cb1..57c75cf4f 100644 --- a/nova/console/manager.py +++ b/nova/console/manager.py @@ -20,11 +20,11 @@ Console Proxy Service """ import functools -import logging import socket from nova import exception from nova import flags +from nova import log as logging from nova import manager from nova import rpc from nova import utils diff --git a/nova/console/xvp.py b/nova/console/xvp.py index ee66dac46..cd257e0a6 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -20,7 +20,6 @@ XVP (Xenserver VNC Proxy) driver. """ import fcntl -import logging import os import signal import subprocess @@ -31,6 +30,7 @@ from nova import context from nova import db from nova import exception from nova import flags +from nova import log as logging from nova import utils flags.DEFINE_string('console_xvp_conf_template', diff --git a/nova/db/api.py b/nova/db/api.py index d7f3746d2..0a010e727 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -85,8 +85,8 @@ def service_get(context, service_id): def service_get_all(context, disabled=False): - """Get all service.""" - return IMPL.service_get_all(context, None, disabled) + """Get all services.""" + return IMPL.service_get_all(context, disabled) def service_get_all_by_topic(context, topic): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..d8751bef4 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -136,15 +136,12 @@ def service_get(context, service_id, session=None): @require_admin_context -def service_get_all(context, session=None, disabled=False): - if not session: - session = get_session() - - result = session.query(models.Service).\ +def service_get_all(context, disabled=False): + session = get_session() + return session.query(models.Service).\ filter_by(deleted=can_read_deleted(context)).\ filter_by(disabled=disabled).\ all() - return result @require_admin_context @@ -715,6 +712,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ filter_by(id=instance_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -723,6 +721,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ + options(joinedload('metadata')).\ filter_by(project_id=context.project_id).\ filter_by(id=instance_id).\ filter_by(deleted=False).\ @@ -1046,7 +1045,8 @@ def network_create_safe(context, values): @require_admin_context def network_disassociate(context, network_id): - network_update(context, network_id, {'project_id': None}) + network_update(context, network_id, {'project_id': None, + 'host': None}) @require_admin_context diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py new file mode 100644 index 000000000..4cb07e0d8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +quotas = Table('quotas', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# + +instance_metadata_table = Table('instance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('value', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +# +# New columns +# +quota_metadata_items = Column('metadata_items', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (instance_metadata_table, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + quotas.create_column(quota_metadata_items) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 4485ee9e4..1882efeba 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -259,6 +259,7 @@ class Quota(BASE, NovaBase): volumes = Column(Integer) gigabytes = Column(Integer) floating_ips = Column(Integer) + metadata_items = Column(Integer) class ExportDevice(BASE, NovaBase): @@ -539,6 +540,20 @@ class Console(BASE, NovaBase): pool = relationship(ConsolePool, backref=backref('consoles')) +class InstanceMetadata(BASE, NovaBase): + """Represents a metadata key/value pair for an instance""" + __tablename__ = 'instance_metadata' + id = Column(Integer, primary_key=True) + key = Column(String(255)) + value = Column(String(255)) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=False) + instance = relationship(Instance, backref="metadata", + foreign_keys=instance_id, + primaryjoin='and_(' + 'InstanceMetadata.instance_id == Instance.id,' + 'InstanceMetadata.deleted == False)') + + class Zone(BASE, NovaBase): """Represents a child zone of this zone.""" __tablename__ = 'zones' @@ -560,7 +575,8 @@ def register_models(): Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, - Project, Certificate, ConsolePool, Console, Zone) + Project, Certificate, ConsolePool, Console, Zone, + InstanceMetadata) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/log.py b/nova/log.py index 6b201ffcc..591d26c63 100644 --- a/nova/log.py +++ b/nova/log.py @@ -65,6 +65,7 @@ flags.DEFINE_string('logging_exception_prefix', flags.DEFINE_list('default_log_levels', ['amqplib=WARN', 'sqlalchemy=WARN', + 'boto=WARN', 'eventlet.wsgi.server=WARN'], 'list of logger=LEVEL pairs') @@ -117,7 +118,7 @@ def _get_binary_name(): return os.path.basename(inspect.stack()[-1][1]) -def get_log_file_path(binary=None): +def _get_log_file_path(binary=None): if FLAGS.logfile: return FLAGS.logfile if FLAGS.logdir: @@ -125,25 +126,6 @@ def get_log_file_path(binary=None): return '%s.log' % (os.path.join(FLAGS.logdir, binary),) -def basicConfig(): - logging.basicConfig() - for handler in logging.root.handlers: - handler.setFormatter(_formatter) - if FLAGS.verbose: - logging.root.setLevel(logging.DEBUG) - else: - logging.root.setLevel(logging.INFO) - if FLAGS.use_syslog: - syslog = SysLogHandler(address='/dev/log') - syslog.setFormatter(_formatter) - logging.root.addHandler(syslog) - logpath = get_log_file_path() - if logpath: - logfile = WatchedFileHandler(logpath) - logfile.setFormatter(_formatter) - logging.root.addHandler(logfile) - - class NovaLogger(logging.Logger): """ NovaLogger manages request context and formatting. @@ -151,23 +133,19 @@ class NovaLogger(logging.Logger): This becomes the class that is instanciated by logging.getLogger. """ def __init__(self, name, level=NOTSET): - level_name = self._get_level_from_flags(name, FLAGS) - level = globals()[level_name] logging.Logger.__init__(self, name, level) + self.setup_from_flags() - def _get_level_from_flags(self, name, FLAGS): - # if exactly "nova", or a child logger, honor the verbose flag - if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: - return 'DEBUG' + def setup_from_flags(self): + """Setup logger from flags""" + level = NOTSET for pair in FLAGS.default_log_levels: - logger, _sep, level = pair.partition('=') + logger, _sep, level_name = pair.partition('=') # NOTE(todd): if we set a.b, we want a.b.c to have the same level # (but not a.bc, so we check the dot) - if name == logger: - return level - if name.startswith(logger) and name[len(logger)] == '.': - return level - return 'INFO' + if self.name == logger or self.name.startswith("%s." % logger): + level = globals()[level_name] + self.setLevel(level) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): """Extract context from any log call""" @@ -176,12 +154,12 @@ class NovaLogger(logging.Logger): if context: extra.update(_dictify_context(context)) extra.update({"nova_version": version.version_string_with_vcs()}) - logging.Logger._log(self, level, msg, args, exc_info, extra) + return logging.Logger._log(self, level, msg, args, exc_info, extra) def addHandler(self, handler): """Each handler gets our custom formatter""" handler.setFormatter(_formatter) - logging.Logger.addHandler(self, handler) + return logging.Logger.addHandler(self, handler) def audit(self, msg, *args, **kwargs): """Shortcut for our AUDIT level""" @@ -208,23 +186,6 @@ class NovaLogger(logging.Logger): self.error(message, **kwargs) -def handle_exception(type, value, tb): - logging.root.critical(str(value), exc_info=(type, value, tb)) - - -sys.excepthook = handle_exception -logging.setLoggerClass(NovaLogger) - - -class NovaRootLogger(NovaLogger): - pass - -if not isinstance(logging.root, NovaRootLogger): - logging.root = NovaRootLogger("nova.root", WARNING) - NovaLogger.root = logging.root - NovaLogger.manager.root = logging.root - - class NovaFormatter(logging.Formatter): """ A nova.context.RequestContext aware formatter configured through flags. @@ -271,8 +232,73 @@ class NovaFormatter(logging.Formatter): _formatter = NovaFormatter() +class NovaRootLogger(NovaLogger): + def __init__(self, name, level=NOTSET): + self.logpath = None + self.filelog = None + self.streamlog = StreamHandler() + self.syslog = None + NovaLogger.__init__(self, name, level) + + def setup_from_flags(self): + """Setup logger from flags""" + global _filelog + if FLAGS.use_syslog: + self.syslog = SysLogHandler(address='/dev/log') + self.addHandler(self.syslog) + elif self.syslog: + self.removeHandler(self.syslog) + logpath = _get_log_file_path() + if logpath: + self.removeHandler(self.streamlog) + if logpath != self.logpath: + self.removeHandler(self.filelog) + self.filelog = WatchedFileHandler(logpath) + self.addHandler(self.filelog) + self.logpath = logpath + else: + self.removeHandler(self.filelog) + self.addHandler(self.streamlog) + if FLAGS.verbose: + self.setLevel(DEBUG) + else: + self.setLevel(INFO) + + +def handle_exception(type, value, tb): + logging.root.critical(str(value), exc_info=(type, value, tb)) + + +def reset(): + """Resets logging handlers. Should be called if FLAGS changes.""" + for logger in NovaLogger.manager.loggerDict.itervalues(): + if isinstance(logger, NovaLogger): + logger.setup_from_flags() + + +def setup(): + """Setup nova logging.""" + if not isinstance(logging.root, NovaRootLogger): + logging._acquireLock() + for handler in logging.root.handlers: + logging.root.removeHandler(handler) + logging.root = NovaRootLogger("nova") + NovaLogger.root = logging.root + NovaLogger.manager.root = logging.root + for logger in NovaLogger.manager.loggerDict.itervalues(): + logger.root = logging.root + if isinstance(logger, logging.Logger): + NovaLogger.manager._fixupParents(logger) + NovaLogger.manager.loggerDict["nova"] = logging.root + logging._releaseLock() + sys.excepthook = handle_exception + reset() + + +root = logging.root +logging.setLoggerClass(NovaLogger) + + def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" - if len(logging.root.handlers) == 0: - basicConfig() logging.root.log(AUDIT, msg, *args, **kwargs) diff --git a/nova/network/manager.py b/nova/network/manager.py index c6eba225e..1df193be0 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -322,6 +322,16 @@ class FlatManager(NetworkManager): """ timeout_fixed_ips = False + def init_host(self): + """Do any initialization that needs to be run if this is a + standalone service. + """ + #Fix for bug 723298 - do not call init_host on superclass + #Following code has been copied for NetworkManager.init_host + ctxt = context.get_admin_context() + for network in self.db.host_get_networks(ctxt, self.host): + self._on_set_network_host(ctxt, network['id']) + def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute @@ -406,6 +416,22 @@ class FlatManager(NetworkManager): net['dns'] = FLAGS.flat_network_dns self.db.network_update(context, network_id, net) + def allocate_floating_ip(self, context, project_id): + #Fix for bug 723298 + raise NotImplementedError() + + def associate_floating_ip(self, context, floating_address, fixed_address): + #Fix for bug 723298 + raise NotImplementedError() + + def disassociate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + + def deallocate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + class FlatDHCPManager(FlatManager): """Flat networking with dhcp. diff --git a/nova/quota.py b/nova/quota.py index 3884eb308..6b52a97fa 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -35,6 +35,8 @@ flags.DEFINE_integer('quota_gigabytes', 1000, 'number of volume gigabytes allowed per project') flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') +flags.DEFINE_integer('quota_metadata_items', 128, + 'number of metadata items allowed per instance') def get_quota(context, project_id): @@ -42,7 +44,8 @@ def get_quota(context, project_id): 'cores': FLAGS.quota_cores, 'volumes': FLAGS.quota_volumes, 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips} + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items} try: quota = db.quota_get(context, project_id) for key in rval.keys(): @@ -94,6 +97,15 @@ def allowed_floating_ips(context, num_floating_ips): return min(num_floating_ips, allowed_floating_ips) +def allowed_metadata_items(context, num_metadata_items): + """Check quota; return min(num_metadata_items,allowed_metadata_items)""" + project_id = context.project_id + context = context.elevated() + quota = get_quota(context, project_id) + num_allowed_metadata_items = quota['metadata_items'] + return min(num_metadata_items, num_allowed_metadata_items) + + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass diff --git a/nova/service.py b/nova/service.py index 59648adf2..cc88ac233 100644 --- a/nova/service.py +++ b/nova/service.py @@ -50,10 +50,6 @@ flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) -flags.DEFINE_string('pidfile', None, - 'pidfile to use for this service') - - flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) @@ -181,6 +177,13 @@ class Service(object): pass self.timers = [] + def wait(self): + for x in self.timers: + try: + x.wait() + except Exception: + pass + def periodic_tasks(self): """Tasks to be run at a periodic interval""" self.manager.periodic_tasks(context.get_admin_context()) @@ -214,9 +217,6 @@ class Service(object): def serve(*services): - FLAGS(sys.argv) - logging.basicConfig() - if not services: services = [Service.create()] diff --git a/nova/test.py b/nova/test.py index a12cf9d32..e0e203647 100644 --- a/nova/test.py +++ b/nova/test.py @@ -23,6 +23,7 @@ and some black magic for inline callbacks. """ import datetime +import uuid import unittest import mox @@ -32,9 +33,10 @@ from nova import context from nova import db from nova import fakerabbit from nova import flags +from nova import log as logging from nova import rpc +from nova import service from nova.network import manager as network_manager -from nova.tests import fake_flags FLAGS = flags.FLAGS @@ -80,6 +82,7 @@ class TestCase(unittest.TestCase): self.stubs = stubout.StubOutForTesting() self.flag_overrides = {} self.injected = [] + self._services = [] self._monkey_patch_attach() self._original_flags = FLAGS.FlagValuesDict() @@ -91,25 +94,42 @@ class TestCase(unittest.TestCase): self.stubs.UnsetAll() self.stubs.SmartUnsetAll() self.mox.VerifyAll() - # NOTE(vish): Clean up any ips associated during the test. - ctxt = context.get_admin_context() - db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, - self.start) - db.network_disassociate_all(ctxt) + super(TestCase, self).tearDown() + finally: + try: + # Clean up any ips associated during the test. + ctxt = context.get_admin_context() + db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, + self.start) + db.network_disassociate_all(ctxt) + + db.security_group_destroy_all(ctxt) + except Exception: + pass + + # Clean out fake_rabbit's queue if we used it + if FLAGS.fake_rabbit: + fakerabbit.reset_all() + + # Reset any overriden flags + self.reset_flags() + + # Reset our monkey-patches rpc.Consumer.attach_to_eventlet = self.originalAttach + + # Stop any timers for x in self.injected: try: x.stop() except AssertionError: pass - if FLAGS.fake_rabbit: - fakerabbit.reset_all() - - db.security_group_destroy_all(ctxt) - super(TestCase, self).tearDown() - finally: - self.reset_flags() + # Kill any services + for x in self._services: + try: + x.kill() + except Exception: + pass def flags(self, **kw): """Override flag variables for a test""" @@ -127,6 +147,15 @@ class TestCase(unittest.TestCase): for k, v in self._original_flags.iteritems(): setattr(FLAGS, k, v) + def start_service(self, name, host=None, **kwargs): + host = host and host or uuid.uuid4().hex + kwargs.setdefault('host', host) + kwargs.setdefault('binary', 'nova-%s' % name) + svc = service.Service.create(**kwargs) + svc.start() + self._services.append(svc) + return svc + def _monkey_patch_attach(self): self.originalAttach = rpc.Consumer.attach_to_eventlet diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e0b7b8029..fb282f1c9 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -221,8 +221,7 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): auth_data = {} - def add_user(self, user): - key = user.id + def add_user(self, key, user): FakeAuthManager.auth_data[key] = user def get_user(self, uid): @@ -235,10 +234,7 @@ class FakeAuthManager(object): return None def get_user_from_access_key(self, key): - for k, v in FakeAuthManager.auth_data.iteritems(): - if v.access == key: - return v - return None + return FakeAuthManager.auth_data.get(key, None) class FakeRateLimiter(object): diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index eab78b50c..0dd65d321 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -48,7 +48,7 @@ class Test(unittest.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -62,7 +62,7 @@ class Test(unittest.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -144,7 +144,7 @@ class TestLimiter(unittest.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..054996658 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -28,6 +28,7 @@ import nova.api.openstack from nova.api.openstack import servers import nova.db.api from nova.db.sqlalchemy.models import Instance +from nova.db.sqlalchemy.models import InstanceMetadata import nova.rpc from nova.tests.api.openstack import fakes @@ -64,6 +65,9 @@ def instance_address(context, instance_id): def stub_instance(id, user_id=1, private_address=None, public_addresses=None): + metadata = [] + metadata.append(InstanceMetadata(key='seq', value=id)) + if public_addresses == None: public_addresses = list() @@ -84,7 +88,7 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "vcpus": 0, "local_gb": 0, "hostname": "", - "host": "", + "host": None, "instance_type": "", "user_data": "", "reservation_id": "", @@ -95,7 +99,8 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "availability_zone": "", "display_name": "server%s" % id, "display_description": "", - "locked": False} + "locked": False, + "metadata": metadata} instance["fixed_ip"] = { "address": private_address, @@ -214,7 +219,8 @@ class ServersTest(unittest.TestCase): "get_image_id_from_image_hash", image_id_from_hash) body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, metadata={}, + name='server_test', imageId=2, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, personality={})) req = webob.Request.blank('/v1.0/servers') req.method = 'POST' @@ -291,6 +297,7 @@ class ServersTest(unittest.TestCase): self.assertEqual(s['id'], i) self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s['imageId'], 10) + self.assertEqual(s['metadata']['seq'], i) i += 1 def test_server_pause(self): diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 1097488ec..2b1919407 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -39,5 +39,5 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///nova.sqlite' +FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index fa27825cd..d5c54a1c3 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -20,6 +20,7 @@ import boto from boto.ec2 import regioninfo +import datetime import httplib import random import StringIO @@ -127,6 +128,28 @@ class ApiEc2TestCase(test.TestCase): self.ec2.new_http_connection(host, is_secure).AndReturn(self.http) return self.http + def test_return_valid_isoformat(self): + """ + Ensure that the ec2 api returns datetime in xs:dateTime + (which apparently isn't datetime.isoformat()) + NOTE(ken-pepple): https://bugs.launchpad.net/nova/+bug/721297 + """ + conv = apirequest._database_to_isoformat + # sqlite database representation with microseconds + time_to_convert = datetime.datetime.strptime( + "2011-02-21 20:14:10.634276", + "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T20:14:10Z') + # mysqlite database representation + time_to_convert = datetime.datetime.strptime( + "2011-02-21 19:56:18", + "%Y-%m-%d %H:%M:%S") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T19:56:18Z') + def test_xmlns_version_matches_request_version(self): self.expect_http(api_version='2010-10-30') self.mox.ReplayAll() diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 35ffffb67..2a7817032 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -327,15 +327,6 @@ class AuthManagerTestCase(object): class AuthManagerLdapTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' - def __init__(self, *args, **kwargs): - AuthManagerTestCase.__init__(self) - test.TestCase.__init__(self, *args, **kwargs) - import nova.auth.fakeldap as fakeldap - if FLAGS.flush_db: - LOG.info("Flushing datastore") - r = fakeldap.Store.instance() - r.flushdb() - class AuthManagerDbTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 445cc6e8b..061910013 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -65,18 +65,21 @@ class CloudTestCase(test.TestCase): self.cloud = cloud.CloudController() # set up services - self.compute = service.Service.create(binary='nova-compute') - self.compute.start() - self.network = service.Service.create(binary='nova-network') - self.network.start() + self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') + self.network = self.start_service('network') self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('proj', 'admin', 'proj') self.context = context.RequestContext(user=self.user, project=self.project) + host = self.network.get_network_host(self.context.elevated()) def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.compute.kill() @@ -102,7 +105,7 @@ class CloudTestCase(test.TestCase): address = "10.10.10.10" db.floating_ip_create(self.context, {'address': address, - 'host': FLAGS.host}) + 'host': self.network.host}) self.cloud.allocate_address(self.context) self.cloud.describe_addresses(self.context) self.cloud.release_address(self.context, @@ -115,9 +118,9 @@ class CloudTestCase(test.TestCase): address = "10.10.10.10" db.floating_ip_create(self.context, {'address': address, - 'host': FLAGS.host}) + 'host': self.network.host}) self.cloud.allocate_address(self.context) - inst = db.instance_create(self.context, {'host': FLAGS.host}) + inst = db.instance_create(self.context, {'host': self.compute.host}) fixed = self.network.allocate_fixed_ip(self.context, inst['id']) ec2_id = cloud.id_to_ec2_id(inst['id']) self.cloud.associate_address(self.context, @@ -133,6 +136,22 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_security_groups(self): + """Makes sure describe_security_groups works and filters results.""" + sec = db.security_group_create(self.context, + {'project_id': self.context.project_id, + 'name': 'test'}) + result = self.cloud.describe_security_groups(self.context) + # NOTE(vish): should have the default group as well + self.assertEqual(len(result['securityGroupInfo']), 2) + result = self.cloud.describe_security_groups(self.context, + group_name=[sec['name']]) + self.assertEqual(len(result['securityGroupInfo']), 1) + self.assertEqual( + result['securityGroupInfo'][0]['groupName'], + sec['name']) + db.security_group_destroy(self.context, sec['id']) + def test_describe_volumes(self): """Makes sure describe_volumes works and filters results.""" vol1 = db.volume_create(self.context, {}) @@ -203,27 +222,32 @@ class CloudTestCase(test.TestCase): 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) + greenthread.sleep(0.3) instance_id = rv['instancesSet'][0]['instanceId'] output = self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) + instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_ajax_console(self): + image_id = FLAGS.default_image kwargs = {'image_id': image_id} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) - self.assertEquals(b64decode(output['output']), - 'http://fakeajaxconsole.com/?token=FAKETOKEN') + greenthread.sleep(0.3) + output = self.cloud.get_ajax_console(context=self.context, + instance_id=[instance_id]) + self.assertEquals(output['url'], + '%s/?token=FAKETOKEN' % FLAGS.ajax_console_proxy_url) # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) - rv = yield self.cloud.terminate_instances(self.context, [instance_id]) + rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_key_generation(self): result = self._create_key('test') @@ -286,70 +310,6 @@ class CloudTestCase(test.TestCase): LOG.debug(_("Terminating instance %s"), instance_id) rv = self.compute.terminate_instance(instance_id) - def test_describe_instances(self): - """Makes sure describe_instances works.""" - instance1 = db.instance_create(self.context, {'host': 'host2'}) - comp1 = db.service_create(self.context, {'host': 'host2', - 'availability_zone': 'zone1', - 'topic': "compute"}) - result = self.cloud.describe_instances(self.context) - self.assertEqual(result['reservationSet'][0] - ['instancesSet'][0] - ['placement']['availabilityZone'], 'zone1') - db.instance_destroy(self.context, instance1['id']) - db.service_destroy(self.context, comp1['id']) - - def test_instance_update_state(self): - # TODO(termie): what is this code even testing? - def instance(num): - return { - 'reservation_id': 'r-1', - 'instance_id': 'i-%s' % num, - 'image_id': 'ami-%s' % num, - 'private_dns_name': '10.0.0.%s' % num, - 'dns_name': '10.0.0%s' % num, - 'ami_launch_index': str(num), - 'instance_type': 'fake', - 'availability_zone': 'fake', - 'key_name': None, - 'kernel_id': 'fake', - 'ramdisk_id': 'fake', - 'groups': ['default'], - 'product_codes': None, - 'state': 0x01, - 'user_data': ''} - rv = self.cloud._format_describe_instances(self.context) - logging.error(str(rv)) - self.assertEqual(len(rv['reservationSet']), 0) - - # simulate launch of 5 instances - # self.cloud.instances['pending'] = {} - #for i in xrange(5): - # inst = instance(i) - # self.cloud.instances['pending'][inst['instance_id']] = inst - - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - # report 4 nodes each having 1 of the instances - #for i in xrange(4): - # self.cloud.update_state('instances', - # {('node-%s' % i): {('i-%s' % i): - # instance(i)}}) - - # one instance should be pending still - #self.assert_(len(self.cloud.instances['pending'].keys()) == 1) - - # check that the reservations collapse - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - - # check that we can get metadata for each instance - #for i in xrange(4): - # data = self.cloud.get_metadata(instance(i)['private_dns_name']) - # self.assert_(data['meta-data']['ami-id'] == 'ami-%s' % i) - @staticmethod def _fake_set_image_description(ctxt, image_id, description): from nova.objectstore import handler diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 85bf94458..49ff24413 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -21,7 +21,6 @@ Tests For Console proxy. """ import datetime -import logging from nova import context from nova import db @@ -38,7 +37,6 @@ FLAGS = flags.FLAGS class ConsoleTestCase(test.TestCase): """Test case for console proxy""" def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) super(ConsoleTestCase, self).setUp() self.flags(console_driver='nova.console.fake.FakeConsoleProxy', stub_compute=True) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 8a74b2296..7656f5396 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -19,7 +19,6 @@ """Tests for Direct API.""" import json -import logging import webob diff --git a/nova/tests/test_localization.py b/nova/tests/test_localization.py index 6992773f5..393d71038 100644 --- a/nova/tests/test_localization.py +++ b/nova/tests/test_localization.py @@ -15,7 +15,6 @@ # under the License. import glob -import logging import os import re import sys diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index c2c9d7772..122351ff6 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -1,9 +1,12 @@ import cStringIO from nova import context +from nova import flags from nova import log from nova import test +FLAGS = flags.FLAGS + def _fake_context(): return context.RequestContext(1, 1) @@ -14,15 +17,11 @@ class RootLoggerTestCase(test.TestCase): super(RootLoggerTestCase, self).setUp() self.log = log.logging.root - def tearDown(self): - super(RootLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_is_nova_instance(self): self.assert_(isinstance(self.log, log.NovaLogger)) - def test_name_is_nova_root(self): - self.assertEqual("nova.root", self.log.name) + def test_name_is_nova(self): + self.assertEqual("nova", self.log.name) def test_handlers_have_nova_formatter(self): formatters = [] @@ -45,25 +44,36 @@ class RootLoggerTestCase(test.TestCase): log.audit("foo", context=_fake_context()) self.assert_(True) # didn't raise exception + def test_will_be_verbose_if_verbose_flag_set(self): + self.flags(verbose=True) + log.reset() + self.assertEqual(log.DEBUG, self.log.level) + + def test_will_not_be_verbose_if_verbose_flag_not_set(self): + self.flags(verbose=False) + log.reset() + self.assertEqual(log.INFO, self.log.level) + class LogHandlerTestCase(test.TestCase): def test_log_path_logdir(self): - self.flags(logdir='/some/path') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.flags(logdir='/some/path', logfile=None) + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_logfile(self): self.flags(logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_none(self): - self.assertTrue(log.get_log_file_path(binary='foo-bar') is None) + self.flags(logdir=None, logfile=None) + self.assertTrue(log._get_log_file_path(binary='foo-bar') is None) def test_log_path_logfile_overrides_logdir(self): self.flags(logdir='/some/other/path', logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') @@ -76,13 +86,15 @@ class NovaFormatterTestCase(test.TestCase): logging_debug_format_suffix="--DBG") self.log = log.logging.root self.stream = cStringIO.StringIO() - handler = log.StreamHandler(self.stream) - self.log.addHandler(handler) + self.handler = log.StreamHandler(self.stream) + self.log.addHandler(self.handler) + self.level = self.log.level self.log.setLevel(log.DEBUG) def tearDown(self): + self.log.setLevel(self.level) + self.log.removeHandler(self.handler) super(NovaFormatterTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} def test_uncontextualized_log(self): self.log.info("foo") @@ -102,30 +114,15 @@ class NovaFormatterTestCase(test.TestCase): class NovaLoggerTestCase(test.TestCase): def setUp(self): super(NovaLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova-test=AUDIT"], verbose=False) + levels = FLAGS.default_log_levels + levels.append("nova-test=AUDIT") + self.flags(default_log_levels=levels, + verbose=True) self.log = log.getLogger('nova-test') - def tearDown(self): - super(NovaLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_has_level_from_flags(self): self.assertEqual(log.AUDIT, self.log.level) def test_child_log_has_level_of_parent_flag(self): l = log.getLogger('nova-test.foo') self.assertEqual(log.AUDIT, l.level) - - -class VerboseLoggerTestCase(test.TestCase): - def setUp(self): - super(VerboseLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova.test=AUDIT"], verbose=True) - self.log = log.getLogger('nova.test') - - def tearDown(self): - super(VerboseLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - - def test_will_be_verbose_if_named_nova_and_verbose_flag_set(self): - self.assertEqual(log.DEBUG, self.log.level) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f3..53cfea276 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -117,6 +117,9 @@ class NetworkTestCase(test.TestCase): utils.to_global_ipv6( network_ref['cidr_v6'], instance_ref['mac_address'])) + self._deallocate_address(0, address) + db.instance_destroy(context.get_admin_context(), + instance_ref['id']) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 9548a8c13..1e42fddf3 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -16,6 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nova import compute from nova import context from nova import db from nova import flags @@ -87,6 +88,18 @@ class QuotaTestCase(test.TestCase): num_instances = quota.allowed_instances(self.context, 100, instance_types.INSTANCE_TYPES['m1.small']) self.assertEqual(num_instances, 10) + + # metadata_items + too_many_items = FLAGS.quota_metadata_items + 1000 + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, FLAGS.quota_metadata_items) + db.quota_update(self.context, self.project.id, {'metadata_items': 5}) + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, 5) + + # Cleanup db.quota_destroy(self.context, self.project.id) def test_too_many_instances(self): @@ -151,3 +164,15 @@ class QuotaTestCase(test.TestCase): self.assertRaises(quota.QuotaError, self.cloud.allocate_address, self.context) db.floating_ip_destroy(context.get_admin_context(), address) + + def test_too_many_metadata_items(self): + metadata = {} + for i in range(FLAGS.quota_metadata_items + 1): + metadata['key%s' % i] = 'value%s' % i + self.assertRaises(quota.QuotaError, compute.API().create, + self.context, + min_count=1, + max_count=1, + instance_type='m1.small', + image_id='fake', + metadata=metadata) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 9d458244b..b6888c4d2 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -150,6 +150,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" @@ -176,18 +177,8 @@ class SimpleDriverTestCase(test.TestCase): def test_doesnt_report_disabled_hosts_as_up(self): """Ensures driver doesn't find hosts before they are enabled""" - # NOTE(vish): constructing service without create method - # because we are going to use it without queue - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') db.service_update(self.context, s1['id'], {'disabled': True}) @@ -199,18 +190,8 @@ class SimpleDriverTestCase(test.TestCase): def test_reports_enabled_hosts_as_up(self): """Ensures driver can find the hosts that are up""" - # NOTE(vish): constructing service without create method - # because we are going to use it without queue - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(2, len(hosts)) compute1.kill() @@ -218,16 +199,8 @@ class SimpleDriverTestCase(test.TestCase): def test_least_busy_host_gets_instance(self): """Ensures the host with less cores gets the next one""" - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance() @@ -241,16 +214,8 @@ class SimpleDriverTestCase(test.TestCase): def test_specific_host_gets_instance(self): """Ensures if you set availability_zone it launches on that zone""" - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance(availability_zone='nova:host1') @@ -263,11 +228,7 @@ class SimpleDriverTestCase(test.TestCase): compute2.kill() def test_wont_sechedule_if_specified_host_is_down(self): - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() + compute1 = self.start_service('compute', host='host1') s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') now = datetime.datetime.utcnow() delta = datetime.timedelta(seconds=FLAGS.service_down_time * 2) @@ -282,11 +243,7 @@ class SimpleDriverTestCase(test.TestCase): compute1.kill() def test_will_schedule_on_disabled_host_if_specified(self): - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() + compute1 = self.start_service('compute', host='host1') s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') db.service_update(self.context, s1['id'], {'disabled': True}) instance_id2 = self._create_instance(availability_zone='nova:host1') @@ -298,16 +255,8 @@ class SimpleDriverTestCase(test.TestCase): def test_too_many_cores(self): """Ensures we don't go over max cores""" - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') instance_ids1 = [] instance_ids2 = [] for index in xrange(FLAGS.max_cores): @@ -322,6 +271,7 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.driver.schedule_run_instance, self.context, instance_id) + db.instance_destroy(self.context, instance_id) for instance_id in instance_ids1: compute1.terminate_instance(self.context, instance_id) for instance_id in instance_ids2: @@ -331,16 +281,8 @@ class SimpleDriverTestCase(test.TestCase): def test_least_busy_host_gets_volume(self): """Ensures the host with less gigabytes gets the next one""" - volume1 = service.Service('host1', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume1.start() - volume2 = service.Service('host2', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume2.start() + volume1 = self.start_service('volume', host='host1') + volume2 = self.start_service('volume', host='host2') volume_id1 = self._create_volume() volume1.create_volume(self.context, volume_id1) volume_id2 = self._create_volume() @@ -354,16 +296,8 @@ class SimpleDriverTestCase(test.TestCase): def test_too_many_gigabytes(self): """Ensures we don't go over max gigabytes""" - volume1 = service.Service('host1', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume1.start() - volume2 = service.Service('host2', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume2.start() + volume1 = self.start_service('volume', host='host1') + volume2 = self.start_service('volume', host='host2') volume_ids1 = [] volume_ids2 = [] for index in xrange(FLAGS.max_gigabytes): diff --git a/contrib/puppet/files/production/genvpn.sh b/nova/tests/test_test.py index 538c3cd33..e237674e6 100644 --- a/contrib/puppet/files/production/genvpn.sh +++ b/nova/tests/test_test.py @@ -1,4 +1,3 @@ -#!/bin/bash # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the @@ -17,19 +16,25 @@ # License for the specific language governing permissions and limitations # under the License. -# This gets zipped and run on the cloudpipe-managed OpenVPN server -NAME=$1 -SUBJ=$2 +"""Tests for the testing base code.""" -mkdir -p projects/$NAME -cd projects/$NAME +from nova import rpc +from nova import test -# generate a server priv key -openssl genrsa -out server.key 2048 -# generate a server CSR -openssl req -new -key server.key -out server.csr -batch -subj "$SUBJ" +class IsolationTestCase(test.TestCase): + """Ensure that things are cleaned up after failed tests. -if [ "`id -u`" != "`grep nova /etc/passwd | cut -d':' -f3`" ]; then - sudo chown -R nova:nogroup . -fi + These tests don't really do much here, but if isolation fails a bunch + of other tests should fail. + + """ + def test_service_isolation(self): + self.start_service('compute') + + def test_rpc_consumer_isolation(self): + connection = rpc.Connection.instance(new=True) + consumer = rpc.TopicConsumer(connection, topic='compute') + consumer.register_callback( + lambda x, y: self.fail('I should never be called')) + consumer.attach_to_eventlet() diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py new file mode 100644 index 000000000..34a407f1a --- /dev/null +++ b/nova/tests/test_utils.py @@ -0,0 +1,174 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class GetFromPathTestCase(test.TestCase): + def test_tolerates_nones(self): + f = utils.get_from_path + + input = [] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [None] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + def test_does_select(self): + f = utils.get_from_path + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + f(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], f(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], f(input, "a/b/c")) + + self.assertEquals([], f(input, "a/b/c/d")) + self.assertEquals([], f(input, "c/a/b/d")) + self.assertEquals([], f(input, "i/r/t")) + + def test_flattens_lists(self): + f = utils.get_from_path + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + + def test_bad_xpath(self): + f = utils.get_from_path + + self.assertRaises(exception.Error, f, [], None) + self.assertRaises(exception.Error, f, [], "") + self.assertRaises(exception.Error, f, [], "/") + self.assertRaises(exception.Error, f, [], "/a") + self.assertRaises(exception.Error, f, [], "/a/") + self.assertRaises(exception.Error, f, [], "//") + self.assertRaises(exception.Error, f, [], "//a") + self.assertRaises(exception.Error, f, [], "a//a") + self.assertRaises(exception.Error, f, [], "a//a/") + self.assertRaises(exception.Error, f, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + f = utils.get_from_path + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = f(inst, 'fixed_ip/address') + public_ips = f(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + f = utils.get_from_path + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..5b3247df9 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -204,6 +204,7 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, testuri) + db.instance_destroy(user_context, instance_ref['id']) def tearDown(self): super(LibvirtConnTestCase, self).tearDown() @@ -365,6 +366,7 @@ class IptablesFirewallTestCase(test.TestCase): '--dports 80:81 -j ACCEPT' % security_group_chain \ in self.out_rules, "TCP port 80/81 acceptance rule wasn't added") + db.instance_destroy(admin_ctxt, instance_ref['id']) class NWFilterTestCase(test.TestCase): @@ -514,3 +516,4 @@ class NWFilterTestCase(test.TestCase): self.fw.apply_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/twistd.py b/nova/twistd.py index 60ff7879a..c07ed991f 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -148,6 +148,7 @@ def WrapTwistedOptions(wrapped): options.insert(0, '') args = FLAGS(options) + logging.setup() argv = args[1:] # ignore subcommands @@ -258,7 +259,6 @@ def serve(filename): print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) - logging.basicConfig() logging.debug(_("Full set of FLAGS:")) for flag in FLAGS: logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) diff --git a/nova/utils.py b/nova/utils.py index 42efa0008..0cf91e0cc 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -2,6 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -31,6 +32,7 @@ import string import struct import sys import time +import types from xml.sax import saxutils import re import netaddr @@ -55,7 +57,7 @@ def import_class(import_str): __import__(mod_str) return getattr(sys.modules[mod_str], class_str) except (ImportError, ValueError, AttributeError), exc: - logging.debug(_('Inner Exception: %s'), exc) + LOG.debug(_('Inner Exception: %s'), exc) raise exception.NotFound(_('Class %s cannot be found') % class_str) @@ -499,3 +501,52 @@ def ensure_b64_encoding(val): return val except TypeError: return base64.b64encode(val) + + +def get_from_path(items, path): + """ Returns a list of items matching the specified path. Takes an + XPath-like expression e.g. prop1/prop2/prop3, and for each item in items, + looks up items[prop1][prop2][prop3]. Like XPath, if any of the + intermediate results are lists it will treat each list item individually. + A 'None' in items or any child expressions will be ignored, this function + will not throw because of None (anywhere) in items. The returned list + will contain no None values.""" + + if path is None: + raise exception.Error("Invalid mini_xpath") + + (first_token, sep, remainder) = path.partition("/") + + if first_token == "": + raise exception.Error("Invalid mini_xpath") + + results = [] + + if items is None: + return results + + if not isinstance(items, types.ListType): + # Wrap single objects in a list + items = [items] + + for item in items: + if item is None: + continue + get_method = getattr(item, "get", None) + if get_method is None: + continue + child = get_method(first_token) + if child is None: + continue + if isinstance(child, types.ListType): + # Flatten intermediate lists + for x in child: + results.append(x) + else: + results.append(child) + + if not sep: + # No more tokens + return results + else: + return get_from_path(results, remainder) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 92749f38a..4346dffc1 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -319,7 +319,9 @@ class FakeConnection(object): return 'FAKE CONSOLE OUTPUT' def get_ajax_console(self, instance): - return 'http://fakeajaxconsole.com/?token=FAKETOKEN' + return {'token': 'FAKETOKEN', + 'host': 'fakeajaxconsole.com', + 'port': 6969} def get_console_pool_info(self, console_type): return {'address': '127.0.0.1', diff --git a/nova/wsgi.py b/nova/wsgi.py index e01cc1e1e..1eb66d067 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -59,7 +59,6 @@ class Server(object): """Server class to manage multiple WSGI sockets and applications.""" def __init__(self, threads=1000): - logging.basicConfig() self.pool = eventlet.GreenPool(threads) def start(self, application, port, host='0.0.0.0', backlog=128): @@ -515,10 +514,3 @@ def load_paste_app(filename, appname): except LookupError: pass return app - - -def paste_config_to_flags(config, mixins): - for k, v in mixins.iteritems(): - value = config.get(k, v) - converted_value = FLAGS[k].parser.Parse(value) - setattr(FLAGS, k, converted_value) diff --git a/run_tests.py b/run_tests.py index 24786e8ad..8025548e5 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,6 +17,44 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + import gettext import os import unittest @@ -27,16 +65,197 @@ from nose import result from nose import core from nova import log as logging +from nova.tests import fake_flags + + +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_OUT_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout def getDescription(self, test): return str(test) + # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + if self.showAll: + self.colorizer.write("OK", 'green') + self.stream.writeln() + elif self.dots: + self.stream.write('.') + self.stream.flush() + + # NOTE(vish): copied from unittest with edit to add color + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + if self.showAll: + self.colorizer.write("FAIL", 'red') + self.stream.writeln() + elif self.dots: + self.stream.write('F') + self.stream.flush() + + # NOTE(vish): copied from nose with edit to add color + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + if self.showAll: + self.colorizer.write("ERROR", 'red') + self.stream.writeln() + elif self.dots: + stream.write('E') + def startTest(self, test): unittest.TestResult.startTest(self, test) current_case = test.test.__class__.__name__ @@ -60,13 +279,27 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - logging.basicConfig() + logging.setup() + testdir = os.path.abspath(os.path.join("nova", "tests")) + testdb = os.path.join(testdir, "tests.sqlite") + if os.path.exists(testdb): + os.unlink(testdb) + # If any argument looks like a test name but doesn't have "nova.tests" in + # front of it, automatically add that so we don't have to type as much + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('nova.tests.%s' % x) + else: + argv.append(x) + c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, + workingDir=testdir, plugins=core.DefaultPluginManager()) runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, config=c) - sys.exit(not core.run(config=c, testRunner=runner)) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) diff --git a/run_tests.sh b/run_tests.sh index 4e21fe945..d4586a57e 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -39,8 +39,18 @@ done function run_tests { # Just run the test suites in current environment - ${wrapper} rm -f nova.sqlite - ${wrapper} $NOSETESTS 2> run_tests.err.log + ${wrapper} $NOSETESTS 2> run_tests.log + # If we get some short import error right away, print the error log directly + RESULT=$? + if [ "$RESULT" -ne "0" ]; + then + ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` + if [ "$ERRSIZE" -lt "40" ]; + then + cat run_tests.err.log + fi + fi + return $RESULT } NOSETESTS="python run_tests.py $noseargs" @@ -73,7 +83,9 @@ fi if [ -z "$noseargs" ]; then - run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py bin/* nova setup.py || exit 1 + srcfiles=`find bin -type f ! -name "nova.conf*"` + srcfiles+=" nova setup.py" + run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py ${srcfiles} || exit 1 else run_tests fi @@ -98,7 +98,6 @@ DistUtilsExtra.auto.setup(name='nova', test_suite='nose.collector', scripts=['bin/nova-ajax-console-proxy', 'bin/nova-api', - 'bin/nova-combined', 'bin/nova-compute', 'bin/nova-console', 'bin/nova-dhcpbridge', diff --git a/smoketests/base.py b/smoketests/base.py index 610270c5c..204b4a1eb 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -17,19 +17,21 @@ # under the License. import boto -import boto_v6 import commands import httplib import os import paramiko -import random import sys +import time import unittest from boto.ec2.regioninfo import RegionInfo from smoketests import flags +SUITE_NAMES = '[image, instance, volume]' FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) +boto_v6 = None class SmokeTestCase(unittest.TestCase): @@ -39,12 +41,10 @@ class SmokeTestCase(unittest.TestCase): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.WarningPolicy()) client.connect(ip, username='root', pkey=key) - stdin, stdout, stderr = client.exec_command('uptime') - print 'uptime: ', stdout.read() return client - def can_ping(self, ip): - """ Attempt to ping the specified IP, and give up after 1 second. """ + def can_ping(self, ip, command="ping"): + """Attempt to ping the specified IP, and give up after 1 second.""" # NOTE(devcamcar): ping timeout flag is different in OSX. if sys.platform == 'darwin': @@ -52,10 +52,41 @@ class SmokeTestCase(unittest.TestCase): else: timeout_flag = 'w' - status, output = commands.getstatusoutput('ping -c1 -%s1 %s' % - (timeout_flag, ip)) + status, output = commands.getstatusoutput('%s -c1 -%s1 %s' % + (command, timeout_flag, ip)) return status == 0 + def wait_for_running(self, instance, tries=60, wait=1): + """Wait for instance to be running""" + for x in xrange(tries): + instance.update() + if instance.state.startswith('running'): + return True + time.sleep(wait) + else: + return False + + def wait_for_ping(self, ip, command="ping", tries=120): + """Wait for ip to be pingable""" + for x in xrange(tries): + if self.can_ping(ip, command): + return True + else: + return False + + def wait_for_ssh(self, ip, key_name, tries=30, wait=5): + """Wait for ip to be sshable""" + for x in xrange(tries): + try: + conn = self.connect_ssh(ip, key_name) + conn.close() + except Exception, e: + time.sleep(wait) + else: + return True + else: + return False + def connection_for_env(self, **kwargs): """ Returns a boto ec2 connection for the current environment. @@ -144,8 +175,21 @@ class SmokeTestCase(unittest.TestCase): return True +TEST_DATA = {} + + +class UserSmokeTestCase(SmokeTestCase): + def setUp(self): + global TEST_DATA + self.conn = self.connection_for_env() + self.data = TEST_DATA + + def run_tests(suites): argv = FLAGS(sys.argv) + if FLAGS.use_ipv6: + global boto_v6 + boto_v6 = __import__('boto_v6') if not os.getenv('EC2_ACCESS_KEY'): print >> sys.stderr, 'Missing EC2 environment variables. Please ' \ diff --git a/smoketests/flags.py b/smoketests/flags.py index 35f432a77..5f3c8505e 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -35,5 +35,5 @@ DEFINE_bool = DEFINE_bool # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 DEFINE_string('region', 'nova', 'Region to use') -DEFINE_string('test_image', 'ami-tiny', 'Image to use for launch tests') -DEFINE_string('use_ipv6', True, 'use the ipv6 or not') +DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') +DEFINE_bool('use_ipv6', False, 'use the ipv6 or not') diff --git a/smoketests/netadmin_smoketests.py b/smoketests/netadmin_smoketests.py new file mode 100644 index 000000000..38beb8fdc --- /dev/null +++ b/smoketests/netadmin_smoketests.py @@ -0,0 +1,194 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import sys +import time +import unittest + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from smoketests import flags +from smoketests import base + + +FLAGS = flags.FLAGS + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_GROUP = '%s_group' % TEST_PREFIX + + +class AddressTests(base.UserSmokeTestCase): + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_001_can_allocate_floating_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + def test_002_can_associate_ip_with_instance(self): + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + self.assertTrue(result) + + def test_003_can_ssh_with_public_ip(self): + ssh_authorized = False + groups = self.conn.get_all_security_groups(['default']) + for rule in groups[0].rules: + if (rule.ip_protocol == 'tcp' and + rule.from_port <= 22 and rule.to_port >= 22): + ssh_authorized = True + if not ssh_authorized: + self.conn.authorize_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + try: + if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): + self.fail('could not ssh to public ip') + finally: + if not ssh_authorized: + self.conn.revoke_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + + def test_004_can_disassociate_ip_from_instance(self): + result = self.conn.disassociate_address(self.data['public_ip']) + self.assertTrue(result) + + def test_005_can_deallocate_floating_ip(self): + result = self.conn.release_address(self.data['public_ip']) + self.assertTrue(result) + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + self.conn.terminate_instances([self.data['instance'].id]) + + +class SecurityGroupTests(base.UserSmokeTestCase): + + def __public_instance_is_accessible(self): + id_url = "latest/meta-data/instance-id" + options = "-s --max-time 1" + command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) + instance_id = commands.getoutput(command).strip() + if not instance_id: + return False + if instance_id != self.data['instance'].id: + raise Exception("Wrong instance id") + return True + + def test_001_can_create_security_group(self): + self.conn.create_security_group(TEST_GROUP, description='test') + + groups = self.conn.get_all_security_groups() + self.assertTrue(TEST_GROUP in [group.name for group in groups]) + + def test_002_can_launch_instance_in_security_group(self): + with open("proxy.sh") as f: + user_data = f.read() + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + security_groups=[TEST_GROUP], + user_data=user_data, + instance_type='m1.tiny') + + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_003_can_authorize_security_group_ingress(self): + self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + + def test_004_can_access_metadata_over_public_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + start_time = time.time() + try: + while not self.__public_instance_is_accessible(): + # 1 minute to launch + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + finally: + result = self.conn.disassociate_address(self.data['public_ip']) + + def test_005_can_revoke_security_group_ingress(self): + self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + start_time = time.time() + while self.__public_instance_is_accessible(): + # 1 minute to teardown + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + + def test_999_tearDown(self): + self.conn.delete_key_pair(TEST_KEY) + self.conn.delete_security_group(TEST_GROUP) + groups = self.conn.get_all_security_groups() + self.assertFalse(TEST_GROUP in [group.name for group in groups]) + self.conn.terminate_instances([self.data['instance'].id]) + self.assertTrue(self.conn.release_address(self.data['public_ip'])) + + +if __name__ == "__main__": + suites = {'address': unittest.makeSuite(AddressTests), + 'security_group': unittest.makeSuite(SecurityGroupTests) + } + sys.exit(base.run_tests(suites)) diff --git a/smoketests/proxy.sh b/smoketests/proxy.sh new file mode 100755 index 000000000..9b3f3108a --- /dev/null +++ b/smoketests/proxy.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This is a simple shell script that uses netcat to set up a proxy to the +# metadata server on port 80 and to a google ip on port 8080. This is meant +# to be passed in by a script to an instance via user data, so that +# automatic testing of network connectivity can be performed. + +# Example usage: +# euca-run-instances -t m1.tiny -f proxy.sh ami-tty + +mkfifo backpipe1 +mkfifo backpipe2 + +# NOTE(vish): proxy metadata on port 80 +while true; do + nc -l -p 80 0<backpipe1 | nc 169.254.169.254 80 1>backpipe1 +done & + +# NOTE(vish): proxy google on port 8080 +while true; do + nc -l -p 8080 0<backpipe2 | nc 74.125.19.99 80 1>backpipe2 +done & diff --git a/smoketests/public_network_smoketests.py b/smoketests/public_network_smoketests.py index bfc2b20ba..5a4c67642 100644 --- a/smoketests/public_network_smoketests.py +++ b/smoketests/public_network_smoketests.py @@ -24,9 +24,16 @@ import sys import time import unittest +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from smoketests import flags from smoketests import base -from smoketests import user_smoketests #Note that this test should run from #public network (outside of private network segments) @@ -42,7 +49,7 @@ TEST_KEY2 = '%s_key2' % TEST_PREFIX TEST_DATA = {} -class InstanceTestsFromPublic(user_smoketests.UserSmokeTestCase): +class InstanceTestsFromPublic(base.UserSmokeTestCase): def test_001_can_create_keypair(self): key = self.create_key_pair(self.conn, TEST_KEY) self.assertEqual(key.name, TEST_KEY) diff --git a/smoketests/user_smoketests.py b/smoketests/sysadmin_smoketests.py index d5a3a7556..e3b84d3d3 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/sysadmin_smoketests.py @@ -19,7 +19,6 @@ import commands import os import random -import socket import sys import time import unittest @@ -36,10 +35,8 @@ from smoketests import flags from smoketests import base -SUITE_NAMES = '[image, instance, volume]' FLAGS = flags.FLAGS -flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', 'Local kernel file to use for bundling tests') flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', @@ -49,17 +46,7 @@ TEST_PREFIX = 'test%s' % int(random.random() * 1000000) TEST_BUCKET = '%s_bucket' % TEST_PREFIX TEST_KEY = '%s_key' % TEST_PREFIX TEST_GROUP = '%s_group' % TEST_PREFIX -TEST_DATA = {} - - -class UserSmokeTestCase(base.SmokeTestCase): - def setUp(self): - global TEST_DATA - self.conn = self.connection_for_env() - self.data = TEST_DATA - - -class ImageTests(UserSmokeTestCase): +class ImageTests(base.UserSmokeTestCase): def test_001_can_bundle_image(self): self.assertTrue(self.bundle_image(FLAGS.bundle_image)) @@ -91,7 +78,6 @@ class ImageTests(UserSmokeTestCase): break time.sleep(1) else: - print image.state self.assert_(False) # wasn't available within 10 seconds self.assert_(image.type == 'machine') @@ -133,7 +119,7 @@ class ImageTests(UserSmokeTestCase): self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) -class InstanceTests(UserSmokeTestCase): +class InstanceTests(base.UserSmokeTestCase): def test_001_can_create_keypair(self): key = self.create_key_pair(self.conn, TEST_KEY) self.assertEqual(key.name, TEST_KEY) @@ -143,109 +129,44 @@ class InstanceTests(UserSmokeTestCase): key_name=TEST_KEY, instance_type='m1.tiny') self.assertEqual(len(reservation.instances), 1) - self.data['instance_id'] = reservation.instances[0].id + self.data['instance'] = reservation.instances[0] def test_003_instance_runs_within_60_seconds(self): - reservations = self.conn.get_all_instances([self.data['instance_id']]) - instance = reservations[0].instances[0] + instance = self.data['instance'] # allow 60 seconds to exit pending with IP - for x in xrange(60): - instance.update() - if instance.state == u'running': - break - time.sleep(1) - else: + if not self.wait_for_running(self.data['instance']): self.fail('instance failed to start') - ip = reservations[0].instances[0].private_dns_name + self.data['instance'].update() + ip = self.data['instance'].private_dns_name self.failIf(ip == '0.0.0.0') - self.data['private_ip'] = ip if FLAGS.use_ipv6: - ipv6 = reservations[0].instances[0].dns_name_v6 + ipv6 = self.data['instance'].dns_name_v6 self.failIf(ipv6 is None) - self.data['ip_v6'] = ipv6 def test_004_can_ping_private_ip(self): - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 %s' % self.data['private_ip']) - if status == 0: - break - else: + if not self.wait_for_ping(self.data['instance'].private_dns_name): self.fail('could not ping instance') if FLAGS.use_ipv6: - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping6 -c1 %s' % self.data['ip_v6']) - if status == 0: - break - else: - self.fail('could not ping instance') + if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + self.fail('could not ping instance v6') def test_005_can_ssh_to_private_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): self.fail('could not ssh to instance') if FLAGS.use_ipv6: - for x in xrange(30): - try: - conn = self.connect_ssh( - self.data['ip_v6'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].ip_v6, + TEST_KEY): self.fail('could not ssh to instance v6') - def test_006_can_allocate_elastic_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - def test_007_can_associate_ip_with_instance(self): - result = self.conn.associate_address(self.data['instance_id'], - self.data['public_ip']) - self.assertTrue(result) - - def test_008_can_ssh_with_public_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) - conn.close() - except socket.error: - time.sleep(1) - else: - break - else: - self.fail('could not ssh to instance') - - def test_009_can_disassociate_ip_from_instance(self): - result = self.conn.disassociate_address(self.data['public_ip']) - self.assertTrue(result) - - def test_010_can_deallocate_elastic_ip(self): - result = self.conn.release_address(self.data['public_ip']) - self.assertTrue(result) - def test_999_tearDown(self): self.delete_key_pair(self.conn, TEST_KEY) - if self.data.has_key('instance_id'): - self.conn.terminate_instances([self.data['instance_id']]) + self.conn.terminate_instances([self.data['instance'].id]) -class VolumeTests(UserSmokeTestCase): +class VolumeTests(base.UserSmokeTestCase): def setUp(self): super(VolumeTests, self).setUp() self.device = '/dev/vdb' @@ -255,55 +176,65 @@ class VolumeTests(UserSmokeTestCase): reservation = self.conn.run_instances(FLAGS.test_image, instance_type='m1.tiny', key_name=TEST_KEY) - instance = reservation.instances[0] - self.data['instance'] = instance - for x in xrange(120): - if self.can_ping(instance.private_dns_name): - break - else: - self.fail('unable to start instance') + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') def test_001_can_create_volume(self): volume = self.conn.create_volume(1, 'nova') self.assertEqual(volume.size, 1) self.data['volume'] = volume # Give network time to find volume. - time.sleep(5) + time.sleep(10) def test_002_can_attach_volume(self): volume = self.data['volume'] for x in xrange(10): - if volume.status == u'available': - break - time.sleep(5) volume.update() + if volume.status.startswith('available'): + break + time.sleep(1) else: self.fail('cannot attach volume with state %s' % volume.status) volume.attach(self.data['instance'].id, self.device) - # Volumes seems to report "available" too soon. + # wait for x in xrange(10): - if volume.status == u'in-use': - break - time.sleep(5) volume.update() + if volume.status.startswith('in-use'): + break + time.sleep(1) + else: + self.fail('volume never got to in use') - self.assertEqual(volume.status, u'in-use') + self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. - time.sleep(5) + time.sleep(10) def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name conn = self.connect_ssh(ip, TEST_KEY) - commands = [] - commands.append('mkdir -p /mnt/vol') - commands.append('mkfs.ext2 %s' % self.device) - commands.append('mount %s /mnt/vol' % self.device) - commands.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) + # NOTE(vish): this will create an dev for images that don't have + # udev rules + stdin, stdout, stderr = conn.exec_command( + 'grep %s /proc/partitions | ' + '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' + % self.device.rpartition('/')[2]) + exec_list = [] + exec_list.append('mkdir -p /mnt/vol') + exec_list.append('/sbin/mke2fs %s' % self.device) + exec_list.append('mount %s /mnt/vol' % self.device) + exec_list.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) out = stdout.read() conn.close() if not out.strip().endswith('success'): @@ -327,7 +258,7 @@ class VolumeTests(UserSmokeTestCase): "df -h | grep %s | awk {'print $2'}" % self.device) out = stdout.read() conn.close() - if not out.strip() == '1008M': + if not out.strip() == '1007.9M': self.fail('Volume is not the right size: %s %s' % (out, stderr.read())) @@ -354,79 +285,9 @@ class VolumeTests(UserSmokeTestCase): self.conn.delete_key_pair(TEST_KEY) -class SecurityGroupTests(UserSmokeTestCase): - - def __public_instance_is_accessible(self): - id_url = "latest/meta-data/instance-id" - options = "-s --max-time 1" - command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) - instance_id = commands.getoutput(command).strip() - if not instance_id: - return False - if instance_id != self.data['instance_id']: - raise Exception("Wrong instance id") - return True - - def test_001_can_create_security_group(self): - self.conn.create_security_group(TEST_GROUP, description='test') - - groups = self.conn.get_all_security_groups() - self.assertTrue(TEST_GROUP in [group.name for group in groups]) - - def test_002_can_launch_instance_in_security_group(self): - self.create_key_pair(self.conn, TEST_KEY) - reservation = self.conn.run_instances(FLAGS.test_image, - key_name=TEST_KEY, - security_groups=[TEST_GROUP], - instance_type='m1.tiny') - - self.data['instance_id'] = reservation.instances[0].id - - def test_003_can_authorize_security_group_ingress(self): - self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - - def test_004_can_access_instance_over_public_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - result = self.conn.associate_address(self.data['instance_id'], - self.data['public_ip']) - start_time = time.time() - while not self.__public_instance_is_accessible(): - # 1 minute to launch - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_005_can_revoke_security_group_ingress(self): - self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - start_time = time.time() - while self.__public_instance_is_accessible(): - # 1 minute to teardown - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_999_tearDown(self): - self.conn.delete_key_pair(TEST_KEY) - self.conn.delete_security_group(TEST_GROUP) - groups = self.conn.get_all_security_groups() - self.assertFalse(TEST_GROUP in [group.name for group in groups]) - self.conn.terminate_instances([self.data['instance_id']]) - self.assertTrue(self.conn.release_address(self.data['public_ip'])) - - if __name__ == "__main__": suites = {'image': unittest.makeSuite(ImageTests), 'instance': unittest.makeSuite(InstanceTests), - 'security_group': unittest.makeSuite(SecurityGroupTests), 'volume': unittest.makeSuite(VolumeTests) } sys.exit(base.run_tests(suites)) |