summaryrefslogtreecommitdiffstats
path: root/ipalib/plugins
diff options
context:
space:
mode:
authorRob Crittenden <rcritten@redhat.com>2013-01-29 11:19:30 -0500
committerMartin Kosek <mkosek@redhat.com>2013-02-05 17:18:13 +0100
commit53c94361d6d119a68a645ac3e5f702c31afab831 (patch)
tree412a1c4b2589ce37b0f8dca5e7ea92875b68d275 /ipalib/plugins
parent9b7930b93d044378812e577405aae29bb445d2be (diff)
downloadfreeipa-53c94361d6d119a68a645ac3e5f702c31afab831.tar.gz
freeipa-53c94361d6d119a68a645ac3e5f702c31afab831.tar.xz
freeipa-53c94361d6d119a68a645ac3e5f702c31afab831.zip
Improve migration performance
Add new users to the default users group in batches of 100. The biggest overhead of migration is in calculating the modlist when managing the default user's group and applying the changes. A significant amount of time can be saved by not doing this on every add operation. Some other minor improvements include: Add a negative cache for groups not found in the remote LDAP server. Replace call to user_mod with a direct LDAP update. Catch some occurances of LimitError and handle more gracefully. I also added some debug logging to report on migration status and performance. https://fedorahosted.org/freeipa/ticket/3386
Diffstat (limited to 'ipalib/plugins')
-rw-r--r--ipalib/plugins/migration.py96
1 files changed, 88 insertions, 8 deletions
diff --git a/ipalib/plugins/migration.py b/ipalib/plugins/migration.py
index 81df59a23..ccb484e4a 100644
--- a/ipalib/plugins/migration.py
+++ b/ipalib/plugins/migration.py
@@ -31,6 +31,7 @@ if api.env.in_server and api.env.context in ['lite', 'server']:
raise e
from ipalib import _
from ipapython.dn import DN
+import datetime
__doc__ = _("""
Migration to IPA
@@ -72,6 +73,12 @@ If a base DN is not provided with --basedn then IPA will use either
the value of defaultNamingContext if it is set or the first value
in namingContexts set in the root of the remote LDAP server.
+Users are added as members to the default user group. This can be a
+time-intensive task so during migration this is done in a batch
+mode for every 100 users. As a result there will be a window in which
+users will be added to IPA but will not be members of the default
+user group.
+
EXAMPLES:
The simplest migration, accepting all defaults:
@@ -102,11 +109,27 @@ EXAMPLES:
--user-ignore-objectclass=radiusprofile \\
--user-ignore-attribute=radiusgroupname \\
ldap://ds.example.com:389
+
+LOGGING
+
+Migration will log warnings and errors to the Apache error log. This
+file should be evaluated post-migration to correct or investigate any
+issues that were discovered.
+
+For every 100 users migrated an info-level message will be displayed to
+give the current progress and duration to make it possible to track
+the progress of migration.
+
+If the log level is debug, either by setting debug = True in
+/etc/ipa/default.conf or /etc/ipa/server.conf, then an entry will be printed
+for each user added plus a summary when the default user group is
+updated.
""")
# USER MIGRATION CALLBACKS AND VARS
_krb_err_msg = _('Kerberos principal %s already exists. Use \'ipa user-mod\' to set it manually.')
+_krb_failed_msg = _('Unable to determine if Kerberos principal %s already exists. Use \'ipa user-mod\' to set it manually.')
_grp_err_msg = _('Failed to add user to the default group. Use \'ipa group-add-member\' to add manually.')
_ref_err_msg = _('Migration of LDAP search reference is not supported.')
_dn_err_msg = _('Malformed DN')
@@ -123,13 +146,17 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
has_upg = ctx['has_upg']
search_bases = kwargs.get('search_bases', None)
valid_gids = kwargs['valid_gids']
+ invalid_gids = kwargs['invalid_gids']
if 'gidnumber' not in entry_attrs:
raise errors.NotFound(reason=_('%(user)s is not a POSIX user') % dict(user=pkey))
else:
# See if the gidNumber at least points to a valid group on the remote
# server.
- if entry_attrs['gidnumber'][0] not in valid_gids:
+ if entry_attrs['gidnumber'][0] in invalid_gids:
+ api.log.warn('GID number %s of migrated user %s does not point to a known group.' \
+ % (entry_attrs['gidnumber'][0], pkey))
+ elif entry_attrs['gidnumber'][0] not in valid_gids:
try:
(remote_dn, remote_entry) = ds_ldap.find_entry_by_attr(
'gidnumber', entry_attrs['gidnumber'][0], 'posixgroup',
@@ -139,10 +166,13 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
except errors.NotFound:
api.log.warn('GID number %s of migrated user %s does not point to a known group.' \
% (entry_attrs['gidnumber'][0], pkey))
+ invalid_gids.append(entry_attrs['gidnumber'][0])
except errors.SingleMatchExpected, e:
# GID number matched more groups, this should not happen
api.log.warn('GID number %s of migrated user %s should match 1 group, but it matched %d groups' \
% (entry_attrs['gidnumber'][0], pkey, e.found))
+ except errors.LimitsExceeded, e:
+ api.log.warn('Search limit exceeded searching for GID %s' % entry_attrs['gidnumber'][0])
# We don't want to create a UPG so set the magic value in description
# to let the DS plugin know.
@@ -182,6 +212,8 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
)
except errors.NotFound:
entry_attrs['krbprincipalname'] = principal
+ except errors.LimitsExceeded:
+ failed[pkey] = unicode(_krb_failed_msg % principal)
else:
failed[pkey] = unicode(_krb_err_msg % principal)
@@ -232,19 +264,52 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
def _post_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx):
assert isinstance(dn, DN)
- # add user to the default group
- try:
- ldap.add_entry_to_group(dn, ctx['def_group_dn'])
- except errors.ExecutionError, e:
- failed[pkey] = unicode(_grp_err_msg)
+
+ _update_default_group(ldap, pkey, config, ctx, False)
+
if 'description' in entry_attrs and NO_UPG_MAGIC in entry_attrs['description']:
entry_attrs['description'].remove(NO_UPG_MAGIC)
- kw = {'setattr': unicode('description=%s' % ','.join(entry_attrs['description']))}
+ update_attrs = dict(description = entry_attrs['description'])
try:
- api.Command['user_mod'](pkey, **kw)
+ ldap.update_entry(dn, update_attrs)
except (errors.EmptyModlist, errors.NotFound):
pass
+def _update_default_group(ldap, pkey, config, ctx, force):
+ migrate_cnt = ctx['migrate_cnt']
+ group_dn = ctx['def_group_dn']
+
+ # Purposely let this fire when migrate_cnt == 0 so on re-running migration
+ # it can catch any users migrated but not added to the default group.
+ if force or migrate_cnt % 100 == 0:
+ s = datetime.datetime.now()
+ searchfilter = "(&(objectclass=posixAccount)(!(memberof=%s)))" % group_dn
+ try:
+ (result, truncated) = ldap.find_entries(searchfilter,
+ [''], api.env.container_user, scope=_ldap.SCOPE_SUBTREE,
+ time_limit = -1)
+ except errors.NotFound:
+ return
+ new_members = []
+ (group_dn, group_entry_attrs) = ldap.get_entry(group_dn, ['member'])
+ for m in result:
+ if m[0] not in group_entry_attrs.get('member', []):
+ new_members.append(m[0])
+ if len(new_members) > 0:
+ members = group_entry_attrs.get('member', [])
+ members.extend(new_members)
+ group_entry_attrs['member'] = members
+
+ try:
+ ldap.update_entry(group_dn, group_entry_attrs)
+ except errors.EmptyModlist:
+ pass
+
+ e = datetime.datetime.now()
+ d = e - s
+ mode = " (forced)" if force else ""
+ api.log.debug('Adding %d users to group%s duration %s' % (len(new_members), mode, d))
+
# GROUP MIGRATION CALLBACKS AND VARS
def _pre_migrate_group(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs):
@@ -619,6 +684,7 @@ can use their Kerberos accounts.''')
migrated = {} # {'OBJ': ['PKEY1', 'PKEY2', ...], ...}
failed = {} # {'OBJ': {'PKEY1': 'Failed 'cos blabla', ...}, ...}
search_bases = self._get_search_bases(options, ds_base_dn, self.migrate_order)
+ migration_start = datetime.datetime.now()
for ldap_obj_name in self.migrate_order:
ldap_obj = self.api.Object[ldap_obj_name]
@@ -682,7 +748,11 @@ can use their Kerberos accounts.''')
context['has_upg'] = ldap.has_upg()
valid_gids = []
+ invalid_gids = []
+ migrate_cnt = 0
for (dn, entry_attrs) in entries:
+ context['migrate_cnt'] = migrate_cnt
+ s = datetime.datetime.now()
if dn is None: # LDAP search reference
failed[ldap_obj_name][entry_attrs[0]] = unicode(_ref_err_msg)
continue
@@ -724,6 +794,7 @@ can use their Kerberos accounts.''')
config, context, schema = options['schema'],
search_bases = search_bases,
valid_gids = valid_gids,
+ invalid_gids = invalid_gids,
**blacklists
)
assert isinstance(dn, DN)
@@ -755,6 +826,15 @@ can use their Kerberos accounts.''')
ldap, pkey, dn, entry_attrs, failed[ldap_obj_name],
config, context,
)
+ e = datetime.datetime.now()
+ d = e - s
+ total_dur = e - migration_start
+ migrate_cnt += 1
+ if migrate_cnt > 0 and migrate_cnt % 100 == 0:
+ api.log.info("%d %ss migrated. %s elapsed." % (migrate_cnt, ldap_obj_name, total_dur))
+ api.log.debug("%d %ss migrated, duration: %s (total %s)" % (migrate_cnt, ldap_obj_name, d, total_dur))
+
+ _update_default_group(ldap, pkey, config, context, True)
return (migrated, failed)