Parse comma-separated lists of values in all parameter types. This can be enabled for a specific parameter by setting the "csv" option to True.

Remove "List" parameter type and replace all occurences of it with appropriate
multi-valued parameter ("Str" in most cases) with csv enabled.

Add new parameter type "Any", capable of holding values of any type. This is
needed by the "batch" command, as "Str" is not suitable type for the "methods"
parameter.

ticket 2007

13 files changed, 112 insertions, 99 deletions

diff --git a/ipalib/__init__.py b/ipalib/__init__.py
index c54717bf8..29ba0bb90 100644
--- a/ipalib/__init__.py
+++ b/ipalib/__init__.py
@@ -878,7 +878,7 @@ from backend import Backend
 from frontend import Command, LocalOrRemote, Updater
 from frontend import Object, Method, Property
 from crud import Create, Retrieve, Update, Delete, Search
-from parameters import DefaultFrom, Bool, Flag, Int, Float, Bytes, Str, IA5Str, Password,List
+from parameters import DefaultFrom, Bool, Flag, Int, Float, Bytes, Str, IA5Str, Password
 from parameters import BytesEnum, StrEnum, AccessTime, File
 from errors import SkipPluginModule
 from text import _, ngettext, GettextFactory, NGettextFactory
diff --git a/ipalib/parameters.py b/ipalib/parameters.py
index 96dde7fc2..287304d3b 100644
--- a/ipalib/parameters.py
+++ b/ipalib/parameters.py
@@ -381,6 +381,9 @@ class Param(ReadOnly):
         ('hint', (str, Gettext), None),
         ('alwaysask', bool, False),
         ('sortorder', int, 2), # see finalize()
+        ('csv', bool, False),
+        ('csv_separator', str, ','),
+        ('csv_skipspace', bool, True),
 
         # The 'default' kwarg gets appended in Param.__init__():
         # ('default', self.type, None),
@@ -493,6 +496,10 @@ class Param(ReadOnly):
                 )
             )
 
+        # Check that if csv is set, multivalue is set too
+        if self.csv and not self.multivalue:
+            raise ValueError('%s: cannot have csv without multivalue' % self.nice)
+
         # Check that all the rules are callable
         self.class_rules = tuple(class_rules)
         self.rules = rules
@@ -663,6 +670,23 @@ class Param(ReadOnly):
         kw.update(overrides)
         return klass(name, *self.rules, **kw)
 
+    # The following 2 functions were taken from the Python
+    # documentation at http://docs.python.org/library/csv.html
+    def __utf_8_encoder(self, unicode_csv_data):
+        for line in unicode_csv_data:
+            yield line.encode('utf-8')
+
+    def __unicode_csv_reader(self, unicode_csv_data, dialect=csv.excel, **kwargs):
+        # csv.py doesn't do Unicode; encode temporarily as UTF-8:
+        csv_reader = csv.reader(self.__utf_8_encoder(unicode_csv_data),
+                                dialect=dialect,
+                                delimiter=self.csv_separator, escapechar='\\',
+                                skipinitialspace=self.csv_skipspace,
+                                **kwargs)
+        for row in csv_reader:
+            # decode UTF-8 back to Unicode, cell by cell:
+            yield [unicode(cell, 'utf-8') for cell in row]
+
     def normalize(self, value):
         """
         Normalize ``value`` using normalizer callback.
@@ -686,15 +710,20 @@ class Param(ReadOnly):
 
         :param value: A proposed value for this parameter.
         """
+        if self.multivalue:
+            if self.csv and isinstance(value, basestring):
+                csvreader = self.__unicode_csv_reader([unicode(value)])
+                value = tuple(csvreader.next()) #pylint: disable=E1101
+            elif type(value) not in (tuple, list):
+                value = (value,)
         if self.normalizer is None:
             return value
         if self.multivalue:
-            if type(value) in (tuple, list):
-                return tuple(
-                    self._normalize_scalar(v) for v in value
-                )
-            return (self._normalize_scalar(value),)  # Return a tuple
-        return self._normalize_scalar(value)
+            return tuple(
+                self._normalize_scalar(v) for v in value
+            )
+        else:
+            return self._normalize_scalar(value)
 
     def _normalize_scalar(self, value):
         """
@@ -1525,47 +1554,12 @@ class StrEnum(Enum):
     type = unicode
 
 
-class List(Param):
+class Any(Param):
     """
-    Base class for parameters as a list of values. The input is a delimited
-    string.
+    A parameter capable of holding values of any type. For internal use only.
     """
-    type = tuple
 
-    kwargs = Param.kwargs + (
-        ('separator', str, ','),
-        ('skipspace', bool, True),
-    )
-
-    # The following 2 functions were taken from the Python
-    # documentation at http://docs.python.org/library/csv.html
-    def __utf_8_encoder(self, unicode_csv_data):
-        for line in unicode_csv_data:
-            yield line.encode('utf-8')
-
-    def __unicode_csv_reader(self, unicode_csv_data, dialect=csv.excel, **kwargs):
-        # csv.py doesn't do Unicode; encode temporarily as UTF-8:
-        csv_reader = csv.reader(self.__utf_8_encoder(unicode_csv_data),
-                                dialect=dialect,
-                                delimiter=self.separator, escapechar='\\',
-                                skipinitialspace=self.skipspace,
-                                **kwargs)
-        for row in csv_reader:
-            # decode UTF-8 back to Unicode, cell by cell:
-            yield [unicode(cell, 'utf-8') for cell in row]
-
-    def __init__(self, name, *rules, **kw):
-        kw['multivalue'] = True
-        super(List, self).__init__(name, *rules, **kw)
-
-    def normalize(self, value):
-        if value and not type(value) in (list, tuple):
-            reader = self.__unicode_csv_reader([value])
-            value = []
-            for row in reader:
-                value = value + row
-            value = tuple(value)
-        return super(List, self).normalize(value)
+    type = object
 
     def _convert_scalar(self, value, index=None):
         return value
diff --git a/ipalib/plugins/aci.py b/ipalib/plugins/aci.py
index 04f25f289..7ace05eb4 100644
--- a/ipalib/plugins/aci.py
+++ b/ipalib/plugins/aci.py
@@ -120,7 +120,7 @@ targetattr REPLACES the current attributes, it does not add to them.
 
 from ipalib import api, crud, errors
 from ipalib import Object, Command
-from ipalib import Flag, Int, List, Str, StrEnum
+from ipalib import Flag, Int, Str, StrEnum
 from ipalib.aci import ACI
 from ipalib import output
 from ipalib import _, ngettext
@@ -430,18 +430,20 @@ class aci(Object):
             doc=_('User group ACI grants access to'),
             flags=('virtual_attribute',),
         ),
-        List('permissions', validate_permissions,
+        Str('permissions+', validate_permissions,
             cli_name='permissions',
             label=_('Permissions'),
             doc=_('comma-separated list of permissions to grant' \
                 '(read, write, add, delete, all)'),
+            csv=True,
             normalizer=_normalize_permissions,
             flags=('virtual_attribute',),
         ),
-        List('attrs?',
+        Str('attrs*',
             cli_name='attrs',
             label=_('Attributes'),
             doc=_('Comma-separated list of attributes'),
+            csv=True,
             flags=('virtual_attribute',),
         ),
         StrEnum('type?',
diff --git a/ipalib/plugins/automember.py b/ipalib/plugins/automember.py
index db58a7aa3..ff5a2bafb 100644
--- a/ipalib/plugins/automember.py
+++ b/ipalib/plugins/automember.py
@@ -109,18 +109,18 @@ INCLUDE_RE = 'automemberinclusiveregex'
 EXCLUDE_RE = 'automemberexclusiveregex'
 
 regex_attrs = (
-    List('automemberinclusiveregex?',
+    Str('automemberinclusiveregex*',
         cli_name='inclusive_regex',
         label=_('Inclusive Regex'),
         doc=_('Inclusive Regex'),
-        multivalue=True,
+        csv=True,
         alwaysask=True,
     ),
-    List('automemberexclusiveregex?',
+    Str('automemberexclusiveregex*',
         cli_name='exclusive_regex',
         label=_('Exclusive Regex'),
         doc=_('Exclusive Regex'),
-        multivalue=True,
+        csv=True,
         alwaysask=True,
     ),
     Str('key',
diff --git a/ipalib/plugins/baseldap.py b/ipalib/plugins/baseldap.py
index 1766077be..cda0c497f 100644
--- a/ipalib/plugins/baseldap.py
+++ b/ipalib/plugins/baseldap.py
@@ -27,7 +27,7 @@ from copy import deepcopy
 
 from ipalib import api, crud, errors
 from ipalib import Method, Object, Command
-from ipalib import Flag, Int, List, Str
+from ipalib import Flag, Int, Str
 from ipalib.base import NameSpace
 from ipalib.cli import to_cli, from_cli
 from ipalib import output
@@ -1298,8 +1298,9 @@ class LDAPModMember(LDAPQuery):
                 ldap_obj = self.api.Object[ldap_obj_name]
                 name = to_cli(ldap_obj_name)
                 doc = self.member_param_doc % ldap_obj.object_name_plural
-                yield List('%s?' % name, cli_name='%ss' % name, doc=doc,
-                           label=_('member %s') % ldap_obj.object_name, alwaysask=True)
+                yield Str('%s*' % name, cli_name='%ss' % name, doc=doc,
+                          label=_('member %s') % ldap_obj.object_name,
+                          csv=True, alwaysask=True)
 
     def get_member_dns(self, **options):
         dns = {}
@@ -1593,18 +1594,18 @@ class LDAPSearch(BaseLDAPCommand, crud.Search):
                     ldap_obj.object_name_plural
                 )
                 name = '%s%s' % (relationship[1], to_cli(ldap_obj_name))
-                yield List(
-                    '%s?' % name, cli_name='%ss' % name, doc=doc,
-                    label=ldap_obj.object_name
+                yield Str(
+                    '%s*' % name, cli_name='%ss' % name, doc=doc,
+                    label=ldap_obj.object_name, csv=True
                 )
                 doc = self.member_param_excl_doc % (
                     self.obj.object_name_plural, relationship[0].lower(),
                     ldap_obj.object_name_plural
                 )
                 name = '%s%s' % (relationship[2], to_cli(ldap_obj_name))
-                yield List(
-                    '%s?' % name, cli_name='%ss' % name, doc=doc,
-                    label=ldap_obj.object_name
+                yield Str(
+                    '%s*' % name, cli_name='%ss' % name, doc=doc,
+                    label=ldap_obj.object_name, csv=True
                 )
 
     def get_member_filter(self, ldap, **options):
@@ -1784,8 +1785,9 @@ class LDAPModReverseMember(LDAPQuery):
                 ldap_obj = self.api.Object[ldap_obj_name]
                 name = to_cli(ldap_obj_name)
                 doc = self.reverse_param_doc % ldap_obj.object_name_plural
-                yield List('%s?' % name, cli_name='%ss' % name, doc=doc,
-                           label=ldap_obj.object_name, alwaysask=True)
+                yield Str('%s*' % name, cli_name='%ss' % name, doc=doc,
+                          label=ldap_obj.object_name, csv=True,
+                          alwaysask=True)
 
 
 class LDAPAddReverseMember(LDAPModReverseMember):
diff --git a/ipalib/plugins/batch.py b/ipalib/plugins/batch.py
index 72d7fe39e..4c5a6bd1e 100644
--- a/ipalib/plugins/batch.py
+++ b/ipalib/plugins/batch.py
@@ -47,7 +47,7 @@ And then a nested response for each IPA command method sent in the request
 
 from ipalib import api, errors
 from ipalib import Command
-from ipalib import Str, List
+from ipalib.parameters import Str, Any
 from ipalib.output import Output
 from ipalib import output
 from ipalib.text import _
@@ -58,10 +58,10 @@ class batch(Command):
     NO_CLI = True
 
     takes_args = (
-        List('methods?',
-             doc=_('Nested Methods to execute'),
-             ),
-        )
+        Any('methods*',
+            doc=_('Nested Methods to execute'),
+        ),
+    )
 
     take_options = (
         Str('version',
@@ -71,7 +71,7 @@ class batch(Command):
             flags=['no_option', 'no_output'],
             default=API_VERSION,
             autofill=True,
-            )
+        ),
     )
 
     has_output = (
diff --git a/ipalib/plugins/config.py b/ipalib/plugins/config.py
index 2b7dd6a2f..ee4505f49 100644
--- a/ipalib/plugins/config.py
+++ b/ipalib/plugins/config.py
@@ -148,15 +148,17 @@ class config(LDAPObject):
             doc=_('Base for certificate subjects (OU=Test,O=Example)'),
             flags=['no_update'],
         ),
-        List('ipagroupobjectclasses?',
+        Str('ipagroupobjectclasses*',
             cli_name='groupobjectclasses',
             label=_('Default group objectclasses'),
             doc=_('Default group objectclasses (comma-separated list)'),
+            csv=True,
         ),
-        List('ipauserobjectclasses?',
+        Str('ipauserobjectclasses*',
             cli_name='userobjectclasses',
             label=_('Default user objectclasses'),
             doc=_('Default user objectclasses (comma-separated list)'),
+            csv=True,
         ),
         Int('ipapwdexpadvnotify?',
             cli_name='pwdexpnotify',
diff --git a/ipalib/plugins/delegation.py b/ipalib/plugins/delegation.py
index fad0be362..5fe1511f2 100644
--- a/ipalib/plugins/delegation.py
+++ b/ipalib/plugins/delegation.py
@@ -19,7 +19,7 @@
 
 import copy
 from ipalib import api, _, ngettext
-from ipalib import Flag, Str, List
+from ipalib import Flag, Str
 from ipalib.request import context
 from ipalib import api, crud, errors
 from ipalib import output
@@ -110,16 +110,18 @@ class delegation(Object):
             doc=_('Delegation name'),
             primary_key=True,
         ),
-        List('permissions?',
+        Str('permissions*',
             cli_name='permissions',
             label=_('Permissions'),
             doc=_('Comma-separated list of permissions to grant ' \
                 '(read, write). Default is write.'),
+            csv=True,
         ),
-        List('attrs',
+        Str('attrs+',
             cli_name='attrs',
             label=_('Attributes'),
             doc=_('Comma-separated list of attributes'),
+            csv=True,
             normalizer=lambda value: value.lower(),
         ),
         Str('memberof',
diff --git a/ipalib/plugins/dns.py b/ipalib/plugins/dns.py
index b3a6abc67..7acb3df8a 100644
--- a/ipalib/plugins/dns.py
+++ b/ipalib/plugins/dns.py
@@ -24,7 +24,7 @@ import re
 
 from ipalib import api, errors, output
 from ipalib import Command
-from ipalib import Flag, Bool, Int, List, Str, StrEnum
+from ipalib import Flag, Bool, Int, Str, StrEnum
 from ipalib.plugins.baseldap import *
 from ipalib import _, ngettext
 from ipalib.util import validate_zonemgr, normalize_zonemgr, validate_hostname
@@ -1040,16 +1040,16 @@ class dnsrecord_cmd_w_record_options(Command):
         validator = _record_validators.get(rec_type)
         normalizer = _record_normalizers.get(rec_type)
         if validator:
-            return List(
-                '%srecord?' % rec_type.lower(), validator, normalizer=normalizer,
+            return Str(
+                '%srecord*' % rec_type.lower(), validator, normalizer=normalizer,
                 cli_name='%s_rec' % rec_type.lower(), doc=doc,
-                label='%s record' % rec_type, attribute=True
+                label='%s record' % rec_type, csv=True, attribute=True
             )
         else:
-            return List(
-                '%srecord?' % rec_type.lower(), cli_name='%s_rec' % rec_type.lower(),
+            return Str(
+                '%srecord*' % rec_type.lower(), cli_name='%s_rec' % rec_type.lower(),
                 normalizer=normalizer, doc=doc, label='%s record' % rec_type,
-                attribute=True
+                csv=True, attribute=True
             )
 
     def prompt_record_options(self, rec_type_list):
diff --git a/ipalib/plugins/hbactest.py b/ipalib/plugins/hbactest.py
index 6bbdada4e..fbc3dbb2e 100644
--- a/ipalib/plugins/hbactest.py
+++ b/ipalib/plugins/hbactest.py
@@ -18,7 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from ipalib import api, errors, output
-from ipalib import Command, List, Str, Flag
+from ipalib import Command, Str, Flag
 from types import NoneType
 from ipalib.cli import to_cli
 from ipalib import _, ngettext
@@ -186,9 +186,10 @@ class hbactest(Command):
             cli_name='service',
             label=_('Service'),
         ),
-        List('rules?',
+        Str('rules*',
              cli_name='rules',
              label=_('Rules to test. If not specified, --enabled is assumed'),
+             csv=True,
         ),
         Flag('nodetail?',
              cli_name='nodetail',
diff --git a/ipalib/plugins/migration.py b/ipalib/plugins/migration.py
index 5d6631f58..9fe72d587 100644
--- a/ipalib/plugins/migration.py
+++ b/ipalib/plugins/migration.py
@@ -21,7 +21,7 @@ import re
 import ldap as _ldap
 
 from ipalib import api, errors, output
-from ipalib import Command, List, Password, Str, Flag, StrEnum
+from ipalib import Command, Password, Str, Flag, StrEnum
 from ipalib.cli import to_cli
 from ipalib.dn import *
 if api.env.in_server and api.env.context in ['lite', 'server']:
@@ -351,45 +351,51 @@ class migrate_ds(Command):
             default=u'ou=groups',
             autofill=True,
         ),
-        List('userobjectclass?',
+        Str('userobjectclass*',
             cli_name='user_objectclass',
             label=_('User object class'),
             doc=_('Comma-separated list of objectclasses used to search for user entries in DS'),
+            csv=True,
             default=(u'person',),
             autofill=True,
         ),
-        List('groupobjectclass?',
+        Str('groupobjectclass*',
             cli_name='group_objectclass',
             label=_('Group object class'),
             doc=_('Comma-separated list of objectclasses used to search for group entries in DS'),
+            csv=True,
             default=(u'groupOfUniqueNames', u'groupOfNames'),
             autofill=True,
         ),
-        List('userignoreobjectclass?',
+        Str('userignoreobjectclass*',
             cli_name='user_ignore_objectclass',
             label=_('Ignore user object class'),
             doc=_('Comma-separated list of objectclasses to be ignored for user entries in DS'),
+            csv=True,
             default=tuple(),
             autofill=True,
         ),
-        List('userignoreattribute?',
+        Str('userignoreattribute*',
             cli_name='user_ignore_attribute',
             label=_('Ignore user attribute'),
             doc=_('Comma-separated list of attributes to be ignored for user entries in DS'),
+            csv=True,
             default=tuple(),
             autofill=True,
         ),
-        List('groupignoreobjectclass?',
+        Str('groupignoreobjectclass*',
             cli_name='group_ignore_objectclass',
             label=_('Ignore group object class'),
             doc=_('Comma-separated list of objectclasses to be ignored for group entries in DS'),
+            csv=True,
             default=tuple(),
             autofill=True,
         ),
-        List('groupignoreattribute?',
+        Str('groupignoreattribute*',
             cli_name='group_ignore_attribute',
             label=_('Ignore group attribute'),
             doc=_('Comma-separated list of attributes to be ignored for group entries in DS'),
+            csv=True,
             default=tuple(),
             autofill=True,
         ),
@@ -457,9 +463,9 @@ can use their Kerberos accounts.''')
             ldap_obj = self.api.Object[ldap_obj_name]
             name = 'exclude_%ss' % to_cli(ldap_obj_name)
             doc = self.exclude_doc % ldap_obj.object_name_plural
-            yield List(
-                '%s?' % name, cli_name=name, doc=doc, default=tuple(),
-                autofill=True
+            yield Str(
+                '%s*' % name, cli_name=name, doc=doc, csv=True,
+                default=tuple(), autofill=True
             )
 
     def normalize_options(self, options):
@@ -470,7 +476,7 @@ can use their Kerberos accounts.''')
         plugin doesn't like that - convert back to empty lists.
         """
         for p in self.params():
-            if isinstance(p, List):
+            if p.csv:
                 if options[p.name]:
                     options[p.name] = tuple(
                         v.lower() for v in options[p.name]
diff --git a/ipalib/plugins/permission.py b/ipalib/plugins/permission.py
index e51e3b859..c48979f9d 100644
--- a/ipalib/plugins/permission.py
+++ b/ipalib/plugins/permission.py
@@ -114,16 +114,18 @@ class permission(LDAPObject):
             label=_('Permission name'),
             primary_key=True,
         ),
-        List('permissions',
+        Str('permissions+',
             cli_name='permissions',
             label=_('Permissions'),
             doc=_('Comma-separated list of permissions to grant ' \
                 '(read, write, add, delete, all)'),
+            csv=True,
         ),
-        List('attrs?',
+        Str('attrs*',
             cli_name='attrs',
             label=_('Attributes'),
             doc=_('Comma-separated list of attributes'),
+            csv=True,
             normalizer=lambda value: value.lower(),
             flags=('ask_create', 'ask_update'),
         ),
diff --git a/ipalib/plugins/selfservice.py b/ipalib/plugins/selfservice.py
index f68212539..902e16baf 100644
--- a/ipalib/plugins/selfservice.py
+++ b/ipalib/plugins/selfservice.py
@@ -19,7 +19,7 @@
 
 import copy
 from ipalib import api, _, ngettext
-from ipalib import Flag, Str, List
+from ipalib import Flag, Str
 from ipalib.request import context
 from ipalib import api, crud, errors
 from ipalib import output
@@ -83,16 +83,18 @@ class selfservice(Object):
             doc=_('Self-service name'),
             primary_key=True,
         ),
-        List('permissions?',
+        Str('permissions*',
             cli_name='permissions',
             label=_('Permissions'),
             doc=_('Comma-separated list of permissions to grant ' \
                 '(read, write). Default is write.'),
+            csv=True,
         ),
-        List('attrs',
+        Str('attrs+',
             cli_name='attrs',
             label=_('Attributes'),
             doc=_('Comma-separated list of attributes'),
+            csv=True,
             normalizer=lambda value: value.lower(),
         ),
     )