schema: support plugin versioning

Update API schema server and client code to support plugin versioning.

https://fedorahosted.org/freeipa/ticket/4427

Reviewed-By: David Kupka <dkupka@redhat.com>

4 files changed, 127 insertions, 77 deletions

diff --git a/API.txt b/API.txt
index 9c979959a..76e58aeec 100644
--- a/API.txt
+++ b/API.txt
@@ -881,7 +881,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: Output('truncated', type=[<type 'bool'>])
 command: class_show/1
 args: 1,3,3
-arg: Str('name')
+arg: Str('full_name')
 option: Flag('all', autofill=True, cli_name='all', default=False)
 option: Flag('raw', autofill=True, cli_name='raw', default=False)
 option: Str('version?')
@@ -890,7 +890,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: PrimaryKey('value')
 command: command_defaults/1
 args: 1,3,1
-arg: Str('name')
+arg: Str('full_name')
 option: Dict('kw?')
 option: Str('params*')
 option: Str('version?')
@@ -908,7 +908,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: Output('truncated', type=[<type 'bool'>])
 command: command_show/1
 args: 1,3,3
-arg: Str('name')
+arg: Str('full_name')
 option: Flag('all', autofill=True, cli_name='all', default=False)
 option: Flag('raw', autofill=True, cli_name='raw', default=False)
 option: Str('version?')
@@ -3352,7 +3352,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: PrimaryKey('value')
 command: output_find/1
 args: 2,4,4
-arg: Str('commandname', cli_name='command')
+arg: Str('commandfull_name', cli_name='command')
 arg: Str('criteria?')
 option: Flag('all', autofill=True, cli_name='all', default=False)
 option: Flag('pkey_only?', autofill=True, default=False)
@@ -3364,7 +3364,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: Output('truncated', type=[<type 'bool'>])
 command: output_show/1
 args: 2,3,3
-arg: Str('commandname', cli_name='command')
+arg: Str('commandfull_name', cli_name='command')
 arg: Str('name')
 option: Flag('all', autofill=True, cli_name='all', default=False)
 option: Flag('raw', autofill=True, cli_name='raw', default=False)
@@ -3374,7 +3374,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: PrimaryKey('value')
 command: param_find/1
 args: 2,4,4
-arg: Str('metaobjectname', cli_name='metaobject')
+arg: Str('metaobjectfull_name', cli_name='metaobject')
 arg: Str('criteria?')
 option: Flag('all', autofill=True, cli_name='all', default=False)
 option: Flag('pkey_only?', autofill=True, default=False)
@@ -3386,7 +3386,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: Output('truncated', type=[<type 'bool'>])
 command: param_show/1
 args: 2,3,3
-arg: Str('metaobjectname', cli_name='metaobject')
+arg: Str('metaobjectfull_name', cli_name='metaobject')
 arg: Str('name')
 option: Flag('all', autofill=True, cli_name='all', default=False)
 option: Flag('raw', autofill=True, cli_name='raw', default=False)
@@ -5253,7 +5253,7 @@ output: Output('summary', type=[<type 'unicode'>, <type 'NoneType'>])
 output: Output('truncated', type=[<type 'bool'>])
 command: topic_show/1
 args: 1,3,3
-arg: Str('name')
+arg: Str('full_name')
 option: Flag('all', autofill=True, cli_name='all', default=False)
 option: Flag('raw', autofill=True, cli_name='raw', default=False)
 option: Str('version?')
diff --git a/VERSION b/VERSION
index 967bc5c82..d4d7228ed 100644
--- a/VERSION
+++ b/VERSION
@@ -90,5 +90,5 @@ IPA_DATA_VERSION=20100614120000
 #                                                      #
 ########################################################
 IPA_API_VERSION_MAJOR=2
-IPA_API_VERSION_MINOR=201
-# Last change: plugable: support plugin versioning
+IPA_API_VERSION_MINOR=202
+# Last change: schema: support plugin versioning
diff --git a/ipaclient/remote_plugins/schema.py b/ipaclient/remote_plugins/schema.py
index 135c16705..bf44f9f8e 100644
--- a/ipaclient/remote_plugins/schema.py
+++ b/ipaclient/remote_plugins/schema.py
@@ -133,6 +133,14 @@ class _SchemaMethod(Method, _SchemaCommand):
         ),
     )
 
+    @property
+    def obj_name(self):
+        return self.api.Object[self.obj_full_name].name
+
+    @property
+    def obj_version(self):
+        return self.api.Object[self.obj_full_name].version
+
     def get_output_params(self):
         seen = set()
         for output_param in super(_SchemaMethod, self).get_output_params():
@@ -151,14 +159,13 @@ class _SchemaPlugin(object):
     bases = None
     schema_key = None
 
-    def __init__(self, name):
-        self.name = name
-        self.version = '1'
-        self.full_name = '{}/{}'.format(self.name, self.version)
+    def __init__(self, full_name):
+        self.name, _slash, self.version = full_name.partition('/')
+        self.full_name = full_name
         self.__class = None
 
     def _create_default_from(self, api, name, keys):
-        cmd_name = self.name
+        cmd_name = self.full_name
 
         def get_default(*args):
             kw = dict(zip(keys, args))
@@ -176,7 +183,7 @@ class _SchemaPlugin(object):
             def callback():
                 return get_default()
 
-        callback.__name__ = '{0}_{1}_default'.format(cmd_name, name)
+        callback.__name__ = '{0}_{1}_default'.format(self.name, name)
 
         return DefaultFrom(callback, *keys)
 
@@ -247,11 +254,13 @@ class _SchemaPlugin(object):
     def _create_class(self, api, schema):
         class_dict = {}
 
-        class_dict['name'] = self.name
+        class_dict['name'] = str(schema['name'])
+        class_dict['version'] = str(schema['version'])
+        class_dict['full_name'] = str(schema['full_name'])
         if 'doc' in schema:
             class_dict['doc'] = schema['doc']
         if 'topic_topic' in schema:
-            class_dict['topic'] = str(schema['topic_topic'])
+            class_dict['topic'] = str(schema['topic_topic']).partition('/')[0]
         else:
             class_dict['topic'] = None
 
@@ -262,7 +271,7 @@ class _SchemaPlugin(object):
 
     def __call__(self, api):
         if self.__class is None:
-            schema = api._schema[self.schema_key][self.name]
+            schema = api._schema[self.schema_key][self.full_name]
             name, bases, class_dict = self._create_class(api, schema)
             self.__class = type(name, bases, class_dict)
 
@@ -306,7 +315,7 @@ class _SchemaCommandPlugin(_SchemaPlugin):
             bases = (_SchemaMethod,)
 
         if 'obj_class' in schema:
-            class_dict['obj_name'] = str(schema['obj_class'])
+            class_dict['obj_full_name'] = str(schema['obj_class'])
         if 'attr_name' in schema:
             class_dict['attr_name'] = str(schema['attr_name'])
         if 'exclude' in schema and u'cli' in schema['exclude']:
@@ -345,7 +354,7 @@ def get_package(api):
             client.disconnect()
 
         for key in ('commands', 'classes', 'topics'):
-            schema[key] = {str(s.pop('name')): s for s in schema[key]}
+            schema[key] = {str(s['full_name']): s for s in schema[key]}
 
         object.__setattr__(api, '_schema', schema)
 
@@ -369,13 +378,13 @@ def get_package(api):
     module.register = plugable.Registry()
     for key, plugin_cls in (('commands', _SchemaCommandPlugin),
                             ('classes', _SchemaObjectPlugin)):
-        for name in schema[key]:
-            plugin = plugin_cls(name)
+        for full_name in schema[key]:
+            plugin = plugin_cls(full_name)
             plugin = module.register()(plugin)
-            setattr(module, name, plugin)
     sys.modules[module_name] = module
 
-    for name, topic in six.iteritems(schema['topics']):
+    for full_name, topic in six.iteritems(schema['topics']):
+        name = str(topic['name'])
         module_name = '.'.join((package_name, name))
         try:
             module = sys.modules[module_name]
@@ -384,7 +393,7 @@ def get_package(api):
             module.__file__ = os.path.join(package_dir, '{}.py'.format(name))
         module.__doc__ = topic.get('doc')
         if 'topic_topic' in topic:
-            module.topic = str(topic['topic_topic'])
+            module.topic = str(topic['topic_topic']).partition('/')[0]
         else:
             module.topic = None
 
diff --git a/ipaserver/plugins/schema.py b/ipaserver/plugins/schema.py
index 96a822491..c7aa5f36c 100644
--- a/ipaserver/plugins/schema.py
+++ b/ipaserver/plugins/schema.py
@@ -48,7 +48,6 @@ class BaseMetaObject(Object):
         Str(
             'name',
             label=_("Name"),
-            primary_key=True,
             normalizer=lambda name: name.replace(u'-', u'_'),
             flags={'no_search'},
         ),
@@ -99,7 +98,8 @@ class BaseMetaObject(Object):
                           criteria in r.get('doc', u'').lower()))
 
         if not kwargs.get('all', False) and kwargs.get('pkey_only', False):
-            result = ({'name': r['name']} for r in result)
+            key = self.primary_key.name
+            result = ({key: r[key]} for r in result)
 
         return result
 
@@ -136,6 +136,24 @@ class MetaObject(BaseMetaObject):
         ),
     )
 
+    def get_params(self):
+        for param in super(MetaObject, self).get_params():
+            yield param
+
+            if param.name == 'name':
+                yield Str(
+                    'version',
+                    label=_("Version"),
+                    flags={'no_search'},
+                )
+                yield Str(
+                    'full_name',
+                    label=_("Full name"),
+                    primary_key=True,
+                    normalizer=lambda name: name.replace(u'-', u'_'),
+                    flags={'no_search'},
+                )
+
 
 class MetaRetrieve(BaseMetaRetrieve):
     pass
@@ -161,6 +179,8 @@ class metaobject(MetaObject):
     def _get_obj(self, metaobj, all=False, **kwargs):
         obj = dict()
         obj['name'] = unicode(metaobj.name)
+        obj['version'] = unicode(metaobj.version)
+        obj['full_name'] = unicode(metaobj.full_name)
 
         if all:
             params = [unicode(p.name) for p in self._iter_params(metaobj)]
@@ -213,10 +233,10 @@ class command(metaobject):
             except errors.NotFound:
                 pass
             else:
-                obj['topic_topic'] = topic['name']
+                obj['topic_topic'] = topic['full_name']
 
         if isinstance(cmd, Method):
-            obj['obj_class'] = unicode(cmd.obj_name)
+            obj['obj_class'] = unicode(cmd.obj_full_name)
             obj['attr_name'] = unicode(cmd.attr_name)
 
         if cmd.NO_CLI:
@@ -327,61 +347,76 @@ class topic_(MetaObject):
     def __init__(self, api):
         super(topic_, self).__init__(api)
         self.__topics = None
+        self.__topics_by_key = None
 
-    def __get_topics(self):
-        if self.__topics is None:
-            topics = {}
-            object.__setattr__(self, '_topic___topics', topics)
-
-            for command in self.api.Command():
-                topic_value = command.topic
-                if topic_value is None:
-                    continue
-                topic_name = unicode(topic_value)
+    def __make_topics(self):
+        if self.__topics is not None and self.__topics_by_key is not None:
+            return
 
-                while topic_name not in topics:
-                    topic = topics[topic_name] = {'name': topic_name}
-
-                    for package in self.api.packages:
-                        module_name = '.'.join((package.__name__, topic_name))
-                        try:
-                            module = sys.modules[module_name]
-                        except KeyError:
-                            try:
-                                module = importlib.import_module(module_name)
-                            except ImportError:
-                                continue
-
-                        if module.__doc__ is not None:
-                            topic['doc'] = unicode(module.__doc__).strip()
+        object.__setattr__(self, '_topic___topics', [])
+        topics = self.__topics
+        object.__setattr__(self, '_topic___topics_by_key', {})
+        topics_by_key = self.__topics_by_key
 
+        for command in self.api.Command():
+            topic_value = command.topic
+            if topic_value is None:
+                continue
+            topic_name = unicode(topic_value)
+
+            while topic_name not in topics_by_key:
+                topic_version = u'1'
+                topic_full_name = u'{}/{}'.format(topic_name,
+                                                  topic_version)
+                topic = {
+                    'name': topic_name,
+                    'version': topic_version,
+                    'full_name': topic_full_name,
+                }
+                topics.append(topic)
+                topics_by_key[topic_name] = topic
+                topics_by_key[topic_full_name] = topic
+
+                for package in self.api.packages:
+                    module_name = '.'.join((package.__name__, topic_name))
+                    try:
+                        module = sys.modules[module_name]
+                    except KeyError:
                         try:
-                            topic_value = module.topic
-                        except AttributeError:
+                            module = importlib.import_module(module_name)
+                        except ImportError:
                             continue
-                        if topic_value is not None:
-                            topic_name = unicode(topic_value)
-                            topic['topic_topic'] = topic_name
-                        else:
-                            topic.pop('topic_topic', None)
 
-        return self.__topics
+                    if module.__doc__ is not None:
+                        topic['doc'] = unicode(module.__doc__).strip()
+
+                    try:
+                        topic_value = module.topic
+                    except AttributeError:
+                        continue
+                    if topic_value is not None:
+                        topic_name = unicode(topic_value)
+                        topic['topic_topic'] = topic_full_name
+                    else:
+                        topic.pop('topic_topic', None)
 
     def _get_obj(self, topic, **kwargs):
         return topic
 
-    def _retrieve(self, name, **kwargs):
+    def _retrieve(self, full_name, **kwargs):
+        self.__make_topics()
         try:
-            return self.__get_topics()[name]
+            return self.__topics_by_key[full_name]
         except KeyError:
             raise errors.NotFound(
                 reason=_("%(pkey)s: %(oname)s not found") % {
-                    'pkey': name, 'oname': self.name,
+                    'pkey': full_name, 'oname': self.name,
                 }
             )
 
     def _search(self, **kwargs):
-        return self.__get_topics().values()
+        self.__make_topics()
+        return iter(self.__topics)
 
 
 @register()
@@ -413,6 +448,12 @@ class BaseParam(BaseMetaObject):
         ),
     )
 
+    def get_params(self):
+        for param in super(BaseParam, self).get_params():
+            if param.name == 'name':
+                param = param.clone(primary_key=True)
+            yield param
+
     @property
     def parent(self):
         raise AttributeError('parent')
@@ -578,12 +619,12 @@ class param(BaseParam):
 
         return obj
 
-    def _retrieve(self, metaobjectname, name, **kwargs):
+    def _retrieve(self, metaobjectfull_name, name, **kwargs):
         try:
-            metaobj = self.api.Command[metaobjectname]
+            metaobj = self.api.Command[metaobjectfull_name]
             plugin = self.api.Object['command']
         except KeyError:
-            metaobj = self.api.Object[metaobjectname]
+            metaobj = self.api.Object[metaobjectfull_name]
             plugin = self.api.Object['class']
 
         for param in plugin._iter_params(metaobj):
@@ -596,12 +637,12 @@ class param(BaseParam):
             }
         )
 
-    def _search(self, metaobjectname, **kwargs):
+    def _search(self, metaobjectfull_name, **kwargs):
         try:
-            metaobj = self.api.Command[metaobjectname]
+            metaobj = self.api.Command[metaobjectfull_name]
             plugin = self.api.Object['command']
         except KeyError:
-            metaobj = self.api.Object[metaobjectname]
+            metaobj = self.api.Object[metaobjectfull_name]
             plugin = self.api.Object['class']
 
         return ((metaobj, param) for param in plugin._iter_params(metaobj))
@@ -668,8 +709,8 @@ class output(BaseParam):
 
         return obj
 
-    def _retrieve(self, commandname, name, **kwargs):
-        cmd = self.api.Command[commandname]
+    def _retrieve(self, commandfull_name, name, **kwargs):
+        cmd = self.api.Command[commandfull_name]
         try:
             return (cmd, cmd.output[name])
         except KeyError:
@@ -679,8 +720,8 @@ class output(BaseParam):
                 }
             )
 
-    def _search(self, commandname, **kwargs):
-        cmd = self.api.Command[commandname]
+    def _search(self, commandfull_name, **kwargs):
+        cmd = self.api.Command[commandfull_name]
         return ((cmd, output) for output in cmd.output())