Merge "Add common filter/filter handler for filter scheduler"

7 files changed, 423 insertions, 0 deletions

diff --git a/openstack/common/scheduler/__init__.py b/openstack/common/scheduler/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/openstack/common/scheduler/__init__.py
diff --git a/openstack/common/scheduler/filter.py b/openstack/common/scheduler/filter.py
new file mode 100644
index 0000000..0bdb10d
--- /dev/null
+++ b/openstack/common/scheduler/filter.py
@@ -0,0 +1,71 @@
+# Copyright (c) 2011-2012 OpenStack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Filter support
+"""
+
+import inspect
+
+from stevedore import extension
+
+
+class BaseFilter(object):
+    """Base class for all filter classes."""
+    def _filter_one(self, obj, filter_properties):
+        """Return True if it passes the filter, False otherwise.
+        Override this in a subclass.
+        """
+        return True
+
+    def filter_all(self, filter_obj_list, filter_properties):
+        """Yield objects that pass the filter.
+
+        Can be overriden in a subclass, if you need to base filtering
+        decisions on all objects.  Otherwise, one can just override
+        _filter_one() to filter a single object.
+        """
+        for obj in filter_obj_list:
+            if self._filter_one(obj, filter_properties):
+                yield obj
+
+
+class BaseFilterHandler(object):
+    """ Base class to handle loading filter classes.
+
+    This class should be subclassed where one needs to use filters.
+    """
+    def __init__(self, filter_class_type, filter_namespace):
+        self.namespace = filter_namespace
+        self.filter_class_type = filter_class_type
+        self.filter_manager = extension.ExtensionManager(filter_namespace)
+
+    def _is_correct_class(self, obj):
+        """Return whether an object is a class of the correct type and
+        is not prefixed with an underscore.
+        """
+        return (inspect.isclass(obj) and
+                not obj.__name__.startswith('_') and
+                issubclass(obj, self.filter_class_type))
+
+    def get_all_classes(self):
+        return [x.plugin for x in self.filter_manager
+                if self._is_correct_class(x.plugin)]
+
+    def get_filtered_objects(self, filter_classes, objs,
+                             filter_properties):
+        for filter_cls in filter_classes:
+            objs = filter_cls().filter_all(objs, filter_properties)
+        return list(objs)
diff --git a/openstack/common/scheduler/filters/__init__.py b/openstack/common/scheduler/filters/__init__.py
new file mode 100644
index 0000000..7476069
--- /dev/null
+++ b/openstack/common/scheduler/filters/__init__.py
@@ -0,0 +1,41 @@
+# Copyright (c) 2011 OpenStack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Scheduler host filters
+"""
+
+from openstack.common import log as logging
+from openstack.common.scheduler import filter
+
+LOG = logging.getLogger(__name__)
+
+
+class BaseHostFilter(filter.BaseFilter):
+    """Base class for host filters."""
+    def _filter_one(self, obj, filter_properties):
+        """Return True if the object passes the filter, otherwise False."""
+        return self.host_passes(obj, filter_properties)
+
+    def host_passes(self, host_state, filter_properties):
+        """Return True if the HostState passes the filter, otherwise False.
+        Override this in a subclass.
+        """
+        raise NotImplementedError()
+
+
+class HostFilterHandler(filter.BaseFilterHandler):
+    def __init__(self, namespace):
+        super(HostFilterHandler, self).__init__(BaseHostFilter, namespace)
diff --git a/openstack/common/scheduler/filters/availability_zone_filter.py b/openstack/common/scheduler/filters/availability_zone_filter.py
new file mode 100644
index 0000000..5659a6c
--- /dev/null
+++ b/openstack/common/scheduler/filters/availability_zone_filter.py
@@ -0,0 +1,30 @@
+# Copyright (c) 2011-2012 OpenStack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from openstack.common.scheduler import filters
+
+
+class AvailabilityZoneFilter(filters.BaseHostFilter):
+    """Filters Hosts by availability zone."""
+
+    def host_passes(self, host_state, filter_properties):
+        spec = filter_properties.get('request_spec', {})
+        props = spec.get('resource_properties', [])
+        availability_zone = props.get('availability_zone')
+
+        if availability_zone:
+            return availability_zone == host_state.service['availability_zone']
+        return True
diff --git a/openstack/common/scheduler/filters/capabilities_filter.py b/openstack/common/scheduler/filters/capabilities_filter.py
new file mode 100644
index 0000000..8a658fa
--- /dev/null
+++ b/openstack/common/scheduler/filters/capabilities_filter.py
@@ -0,0 +1,63 @@
+# Copyright (c) 2011 OpenStack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from openstack.common import log as logging
+from openstack.common.scheduler import filters
+from openstack.common.scheduler.filters import extra_specs_ops
+
+
+LOG = logging.getLogger(__name__)
+
+
+class CapabilitiesFilter(filters.BaseHostFilter):
+    """HostFilter to work with resource (instance & volume) type records."""
+
+    def _satisfies_extra_specs(self, capabilities, resource_type):
+        """Check that the capabilities provided by the services
+        satisfy the extra specs associated with the instance type"""
+        extra_specs = resource_type.get('extra_specs', [])
+        if not extra_specs:
+            return True
+
+        for key, req in extra_specs.iteritems():
+            # Either not scope format, or in capabilities scope
+            scope = key.split(':')
+            if len(scope) > 1 and scope[0] != "capabilities":
+                continue
+            elif scope[0] == "capabilities":
+                del scope[0]
+
+            cap = capabilities
+            for index in range(0, len(scope)):
+                try:
+                    cap = cap.get(scope[index], None)
+                except AttributeError:
+                    return False
+                if cap is None:
+                    return False
+            if not extra_specs_ops.match(cap, req):
+                return False
+        return True
+
+    def host_passes(self, host_state, filter_properties):
+        """Return a list of hosts that can create instance_type."""
+        # Note(zhiteng) Currently only Cinder and Nova are using
+        # this filter, so the resource type is either instance or
+        # volume.
+        resource_type = filter_properties.get('resource_type')
+        if not self._satisfies_extra_specs(host_state.capabilities,
+                                           resource_type):
+            return False
+        return True
diff --git a/openstack/common/scheduler/filters/extra_specs_ops.py b/openstack/common/scheduler/filters/extra_specs_ops.py
new file mode 100644
index 0000000..f4d4ff4
--- /dev/null
+++ b/openstack/common/scheduler/filters/extra_specs_ops.py
@@ -0,0 +1,68 @@
+# Copyright (c) 2011 OpenStack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import operator
+
+# 1. The following operations are supported:
+#   =, s==, s!=, s>=, s>, s<=, s<, <in>, <or>, ==, !=, >=, <=
+# 2. Note that <or> is handled in a different way below.
+# 3. If the first word in the extra_specs is not one of the operators,
+#   it is ignored.
+_op_methods = {'=': lambda x, y: float(x) >= float(y),
+               '<in>': lambda x, y: y in x,
+               '==': lambda x, y: float(x) == float(y),
+               '!=': lambda x, y: float(x) != float(y),
+               '>=': lambda x, y: float(x) >= float(y),
+               '<=': lambda x, y: float(x) <= float(y),
+               's==': operator.eq,
+               's!=': operator.ne,
+               's<': operator.lt,
+               's<=': operator.le,
+               's>': operator.gt,
+               's>=': operator.ge}
+
+
+def match(value, req):
+    words = req.split()
+
+    op = method = None
+    if words:
+        op = words.pop(0)
+        method = _op_methods.get(op)
+
+    if op != '<or>' and not method:
+        return value == req
+
+    if value is None:
+        return False
+
+    if op == '<or>':  # Ex: <or> v1 <or> v2 <or> v3
+        while True:
+            if words.pop(0) == value:
+                return True
+            if not words:
+                break
+            op = words.pop(0)  # remove a keyword <or>
+            if not words:
+                break
+        return False
+
+    try:
+        if words and method(value, words[0]):
+            return True
+    except ValueError:
+        pass
+
+    return False
diff --git a/openstack/common/scheduler/filters/json_filter.py b/openstack/common/scheduler/filters/json_filter.py
new file mode 100644
index 0000000..273d55e
--- /dev/null
+++ b/openstack/common/scheduler/filters/json_filter.py
@@ -0,0 +1,150 @@
+# Copyright (c) 2011 OpenStack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import operator
+
+from openstack.common import jsonutils
+from openstack.common.scheduler import filters
+
+
+class JsonFilter(filters.BaseHostFilter):
+    """Host Filter to allow simple JSON-based grammar for
+    selecting hosts.
+    """
+    def _op_compare(self, args, op):
+        """Returns True if the specified operator can successfully
+        compare the first item in the args with all the rest. Will
+        return False if only one item is in the list.
+        """
+        if len(args) < 2:
+            return False
+        if op is operator.contains:
+            bad = not args[0] in args[1:]
+        else:
+            bad = [arg for arg in args[1:]
+                   if not op(args[0], arg)]
+        return not bool(bad)
+
+    def _equals(self, args):
+        """First term is == all the other terms."""
+        return self._op_compare(args, operator.eq)
+
+    def _less_than(self, args):
+        """First term is < all the other terms."""
+        return self._op_compare(args, operator.lt)
+
+    def _greater_than(self, args):
+        """First term is > all the other terms."""
+        return self._op_compare(args, operator.gt)
+
+    def _in(self, args):
+        """First term is in set of remaining terms"""
+        return self._op_compare(args, operator.contains)
+
+    def _less_than_equal(self, args):
+        """First term is <= all the other terms."""
+        return self._op_compare(args, operator.le)
+
+    def _greater_than_equal(self, args):
+        """First term is >= all the other terms."""
+        return self._op_compare(args, operator.ge)
+
+    def _not(self, args):
+        """Flip each of the arguments."""
+        return [not arg for arg in args]
+
+    def _or(self, args):
+        """True if any arg is True."""
+        return any(args)
+
+    def _and(self, args):
+        """True if all args are True."""
+        return all(args)
+
+    commands = {
+        '=': _equals,
+        '<': _less_than,
+        '>': _greater_than,
+        'in': _in,
+        '<=': _less_than_equal,
+        '>=': _greater_than_equal,
+        'not': _not,
+        'or': _or,
+        'and': _and,
+    }
+
+    def _parse_string(self, string, host_state):
+        """Strings prefixed with $ are capability lookups in the
+        form '$variable' where 'variable' is an attribute in the
+        HostState class.  If $variable is a dictionary, you may
+        use: $variable.dictkey
+        """
+        if not string:
+            return None
+        if not string.startswith("$"):
+            return string
+
+        path = string[1:].split(".")
+        obj = getattr(host_state, path[0], None)
+        if obj is None:
+            return None
+        for item in path[1:]:
+            obj = obj.get(item, None)
+            if obj is None:
+                return None
+        return obj
+
+    def _process_filter(self, query, host_state):
+        """Recursively parse the query structure."""
+        if not query:
+            return True
+        cmd = query[0]
+        method = self.commands[cmd]
+        cooked_args = []
+        for arg in query[1:]:
+            if isinstance(arg, list):
+                arg = self._process_filter(arg, host_state)
+            elif isinstance(arg, basestring):
+                arg = self._parse_string(arg, host_state)
+            if arg is not None:
+                cooked_args.append(arg)
+        result = method(self, cooked_args)
+        return result
+
+    def host_passes(self, host_state, filter_properties):
+        """Return a list of hosts that can fulfill the requirements
+        specified in the query.
+        """
+        # TODO(zhiteng) Add description for filter_properties structure
+        # and scheduler_hints.
+        try:
+            query = filter_properties['scheduler_hints']['query']
+        except KeyError:
+            query = None
+        if not query:
+            return True
+
+        # NOTE(comstud): Not checking capabilities or service for
+        # enabled/disabled so that a provided json filter can decide
+
+        result = self._process_filter(jsonutils.loads(query), host_state)
+        if isinstance(result, list):
+            # If any succeeded, include the host
+            result = any(result)
+        if result:
+            # Filter it out.
+            return True
+        return False