Fix LDAP object parameter encoding

Parameters in LDAP objects missed an information if they are real
LDAP attributes or not. Real LDAP attributes are written to
entry_attrs dictionary in plugin callbacks and are being encoded.
This causes issues when plugin callbacks does not expect that
the parameters values are already encoded for submission to LDAP.

This patch introduces a new flag "noattribute" used to mark that
a parameter is not an LDAP attribute and thus should not be encoded
or added to entry_attrs. Param documentation is improved to describe
the meaning of this and other Param flags or attributes.

https://fedorahosted.org/freeipa/ticket/2097

6 files changed, 95 insertions, 32 deletions

diff --git a/ipalib/crud.py b/ipalib/crud.py
index 97d6430d7..833914cfa 100644
--- a/ipalib/crud.py
+++ b/ipalib/crud.py
@@ -139,15 +139,16 @@ class Create(Method):
             for option in super(Create, self).get_options():
                 yield option
         for option in self.obj.params_minus(self.args):
+            attribute = 'virtual_attribute' not in option.flags
             if 'no_create' in option.flags:
                 continue
             if 'ask_create' in option.flags:
                 yield option.clone(
-                    attribute=True, query=True, required=False,
+                    attribute=attribute, query=True, required=False,
                     autofill=False, alwaysask=True
                 )
             else:
-                yield option.clone(attribute=True)
+                yield option.clone(attribute=attribute)
         if not self.extra_options_first:
             for option in super(Create, self).get_options():
                 yield option
@@ -183,19 +184,20 @@ class Update(PKQuery):
             for option in super(Update, self).get_options():
                 yield option
         for option in self.obj.params_minus_pk():
+            attribute = 'virtual_attribute' not in option.flags
             if 'no_update' in option.flags:
                 continue
             if 'ask_update' in option.flags:
                 yield option.clone(
-                    attribute=True, query=True, required=False,
+                    attribute=attribute, query=True, required=False,
                     autofill=False, alwaysask=True
                 )
             elif 'req_update' in option.flags:
                 yield option.clone(
-                    attribute=True, required=True, alwaysask=False,
+                    attribute=attribute, required=True, alwaysask=False,
                 )
             else:
-                yield option.clone(attribute=True, required=False, autofill=False)
+                yield option.clone(attribute=attribute, required=False, autofill=False)
         if not self.extra_options_first:
             for option in super(Update, self).get_options():
                 yield option
@@ -224,21 +226,22 @@ class Search(Method):
             for option in super(Search, self).get_options():
                 yield option
         for option in self.obj.params_minus(self.args):
+            attribute = 'virtual_attribute' not in option.flags
             if 'no_search' in option.flags:
                 continue
             if 'ask_search' in option.flags:
                 yield option.clone(
-                    attribute=True, query=True, required=False,
+                    attribute=attribute, query=True, required=False,
                     autofill=False, alwaysask=True
                 )
             elif isinstance(option, parameters.Flag):
                 yield option.clone_retype(
                     option.name, parameters.Bool,
-                    attribute=True, query=True, required=False, autofill=False
+                    attribute=attribute, query=True, required=False, autofill=False
                 )
             else:
                 yield option.clone(
-                    attribute=True, query=True, required=False, autofill=False
+                    attribute=attribute, query=True, required=False, autofill=False
                 )
         if not self.extra_options_first:
             for option in super(Search, self).get_options():
diff --git a/ipalib/parameters.py b/ipalib/parameters.py
index 1f3fdfde7..96dde7fc2 100644
--- a/ipalib/parameters.py
+++ b/ipalib/parameters.py
@@ -285,6 +285,68 @@ def _(message):
 class Param(ReadOnly):
     """
     Base class for all parameters.
+
+    Param attributes:
+    =================
+    The behavior of Param class and subclasses can be controlled using the
+    following set of attributes:
+
+      - cli_name: option name in CLI
+      - cli_short_name: one character version of cli_name
+      - label: very short description of the parameter. This value is used in
+        when the Command output is printed to CLI or in a Command help
+      - doc: parameter long description used in help
+      - required: the parameter is marked as required for given Command
+      - multivalue: indicates if the attribute is multivalued
+      - primary_key: Command's parameter primary key is used for unique
+        identification of an LDAP object and for sorting
+      - normalizer: a custom function for Param value normalization
+      - encoder: a custom function used to override Param subclass default
+        encoder
+      - default_from: a custom function for generating default values of
+        parameter instance
+      - create_default: a custom function for generating default values of
+        parameter instance. Unlike default_from attribute, this function
+        is not wrapped. `Param.get_default()` documentation provides further
+        details
+      - autofill: by default, only `required` parameters get a default value
+        from default_from or create_default functions. When autofill is
+        enabled, optional attributes get the default value filled too
+      - query: this attribute is controlled by framework. When the `query`
+        is enabled, framework assumes that the value is only queried and not
+        inserted in the LDAP. Validation is then relaxed - custom
+        parameter validators are skipped and only basic class validators are
+        executed to check the parameter value
+      - attribute: this attribute is controlled by framework and enabled for
+        all LDAP objects parameters (unless parameter has "virtual_attribute"
+        flag). All parameters with enabled `attribute` are being encoded and
+        placed to an entry passed to LDAP Create/Update calls
+      - include: a list of contexts where this parameter should be included.
+        `Param.use_in_context()` provides further information.
+      - exclude: a list of contexts where this parameter should be excluded.
+        `Param.use_in_context()` provides further information.
+      - flags: there are several flags that can be used to further tune the
+        parameter behavior:
+            * no_display (Output parameters only): do not display the parameter
+            * no_create: do not include the parameter for crud.Create based
+              commands
+            * no_update: do not include the parameter for crud.update based
+              commands
+            * virtual_attribute: the parameter is not stored physically in the
+              LDAP and thus attribute `attribute` is not enabled
+            * suppress_empty (Output parameters only): do not display parameter
+              value when empty
+            * ask_create: CLI asks for parameter value even when the parameter
+              is not `required`. Applied for all crud.Create based commands
+            * ask_update: CLI asks for parameter value even when the parameter
+              is not `required`. Applied for all crud.Update based commands
+            * req_update: The parameter is `required` in all crud.Update based
+              commands
+      - hint: This attribute is currently not used
+      - alwaysask: when enabled, CLI asks for parameter value even when the
+        parameter is not `required`
+      - sortorder: used to sort a list of parameters for Command. See
+        `Command.finalize()` for further information
     """
 
     # This is a dummy type so that most of the functionality of Param can be
diff --git a/ipalib/plugins/aci.py b/ipalib/plugins/aci.py
index 429ae6eb5..585dab837 100644
--- a/ipalib/plugins/aci.py
+++ b/ipalib/plugins/aci.py
@@ -416,16 +416,19 @@ class aci(Object):
             cli_name='name',
             label=_('ACI name'),
             primary_key=True,
+            flags=('virtual_attribute',),
         ),
         Str('permission?',
             cli_name='permission',
             label=_('Permission'),
             doc=_('Permission ACI grants access to'),
+            flags=('virtual_attribute',),
         ),
         Str('group?',
             cli_name='group',
             label=_('User group'),
             doc=_('User group ACI grants access to'),
+            flags=('virtual_attribute',),
         ),
         List('permissions', validate_permissions,
             cli_name='permissions',
@@ -433,42 +436,50 @@ class aci(Object):
             doc=_('comma-separated list of permissions to grant' \
                 '(read, write, add, delete, all)'),
             normalizer=_normalize_permissions,
+            flags=('virtual_attribute',),
         ),
         List('attrs?',
             cli_name='attrs',
             label=_('Attributes'),
             doc=_('Comma-separated list of attributes'),
+            flags=('virtual_attribute',),
         ),
         StrEnum('type?',
             cli_name='type',
             label=_('Type'),
             doc=_('type of IPA object (user, group, host, hostgroup, service, netgroup)'),
             values=(u'user', u'group', u'host', u'service', u'hostgroup', u'netgroup', u'dnsrecord'),
+            flags=('virtual_attribute',),
         ),
         Str('memberof?',
             cli_name='memberof',
             label=_('Member of'),  # FIXME: Does this label make sense?
             doc=_('Member of a group'),
+            flags=('virtual_attribute',),
         ),
         Str('filter?',
             cli_name='filter',
             label=_('Filter'),
             doc=_('Legal LDAP filter (e.g. ou=Engineering)'),
+            flags=('virtual_attribute',),
         ),
         Str('subtree?',
             cli_name='subtree',
             label=_('Subtree'),
             doc=_('Subtree to apply ACI to'),
+            flags=('virtual_attribute',),
         ),
         Str('targetgroup?',
             cli_name='targetgroup',
             label=_('Target group'),
             doc=_('Group to apply ACI to'),
+            flags=('virtual_attribute',),
         ),
         Flag('selfaci?',
              cli_name='self',
              label=_('Target your own entry (self)'),
              doc=_('Apply ACI to your own entry (self)'),
+             flags=('virtual_attribute',),
         ),
     )
 
diff --git a/ipalib/plugins/dns.py b/ipalib/plugins/dns.py
index 0a0bcb79c..bfc8090ca 100644
--- a/ipalib/plugins/dns.py
+++ b/ipalib/plugins/dns.py
@@ -665,6 +665,7 @@ class dnszone(LDAPObject):
         Str('name_from_ip?', _validate_ipnet,
             label=_('Reverse zone IP network'),
             doc=_('IP network to create reverse zone name from'),
+            flags=('virtual_attribute',),
         ),
         Str('idnssoamname',
             cli_name='name_server',
@@ -780,9 +781,6 @@ class dnszone_add(LDAPCreate):
         if not dns_container_exists(self.api.Backend.ldap2):
             raise errors.NotFound(reason=_('DNS is not configured'))
 
-        if 'name_from_ip' in entry_attrs:
-            del entry_attrs['name_from_ip']
-
         entry_attrs['idnszoneactive'] = 'TRUE'
 
         # Check nameserver has a forward record
@@ -832,11 +830,6 @@ class dnszone_mod(LDAPUpdate):
             self.obj.params['name_from_ip'](unicode(options['name_from_ip']))
         return super(dnszone_mod, self).args_options_2_params(*args, **options)
 
-    def pre_callback(self, ldap, dn, entry_attrs, *keys, **options):
-        if 'name_from_ip' in entry_attrs:
-            del entry_attrs['name_from_ip']
-        return dn
-
 api.register(dnszone_mod)
 
 
diff --git a/ipalib/plugins/host.py b/ipalib/plugins/host.py
index 6557880aa..33d60ad1c 100644
--- a/ipalib/plugins/host.py
+++ b/ipalib/plugins/host.py
@@ -290,12 +290,12 @@ class host(LDAPObject):
         ),
         Flag('random?',
             doc=_('Generate a random password to be used in bulk enrollment'),
-            flags=['no_search'],
+            flags=('no_search', 'virtual_attribute'),
             default=False,
         ),
         Str('randompassword?',
             label=_('Random password'),
-            flags=['no_create', 'no_update', 'no_search'],
+            flags=('no_create', 'no_update', 'no_search', 'virtual_attribute'),
         ),
         Bytes('usercertificate?', validate_certificate,
             cli_name='certificate',
@@ -432,12 +432,10 @@ class host_add(LDAPCreate):
                 entry_attrs['objectclass'].remove('krbprincipalaux')
             if 'krbprincipal' in entry_attrs['objectclass']:
                 entry_attrs['objectclass'].remove('krbprincipal')
-        if 'random' in options:
-            if options.get('random'):
-                entry_attrs['userpassword'] = ipa_generate_password()
-                # save the password so it can be displayed in post_callback
-                setattr(context, 'randompassword', entry_attrs['userpassword'])
-            del entry_attrs['random']
+        if options.get('random'):
+            entry_attrs['userpassword'] = ipa_generate_password()
+            # save the password so it can be displayed in post_callback
+            setattr(context, 'randompassword', entry_attrs['userpassword'])
         cert = options.get('usercertificate')
         if cert:
             cert = x509.normalize_certificate(cert)
@@ -680,11 +678,9 @@ class host_mod(LDAPUpdate):
                         raise nsprerr
 
             entry_attrs['usercertificate'] = cert
-        if 'random' in options:
-            if options.get('random'):
-                entry_attrs['userpassword'] = ipa_generate_password()
-                setattr(context, 'randompassword', entry_attrs['userpassword'])
-            del entry_attrs['random']
+        if options.get('random'):
+            entry_attrs['userpassword'] = ipa_generate_password()
+            setattr(context, 'randompassword', entry_attrs['userpassword'])
 
         return dn
 
diff --git a/ipalib/plugins/pwpolicy.py b/ipalib/plugins/pwpolicy.py
index 6c5e6c56b..db42bca04 100644
--- a/ipalib/plugins/pwpolicy.py
+++ b/ipalib/plugins/pwpolicy.py
@@ -263,6 +263,7 @@ class pwpolicy(LDAPObject):
             label=_('Priority'),
             doc=_('Priority of the policy (higher number means lower priority'),
             minvalue=0,
+            flags=('virtual_attribute',),
         ),
     ) + lockout_params
 
@@ -344,8 +345,6 @@ class pwpolicy_add(LDAPCreate):
             keys[-1], krbpwdpolicyreference=dn,
             cospriority=options.get('cospriority')
         )
-        if 'cospriority' in entry_attrs:
-            del entry_attrs['cospriority']
         return dn
 
     def post_callback(self, ldap, dn, entry_attrs, *keys, **options):
@@ -406,7 +405,6 @@ class pwpolicy_mod(LDAPUpdate):
                     raise e
             else:
                 setattr(context, 'cosupdate', True)
-            del entry_attrs['cospriority']
         return dn
 
     def post_callback(self, ldap, dn, entry_attrs, *keys, **options):