- move some documentation files around

6 files changed, 909 insertions, 0 deletions

diff --git a/doc/devel/index-memberof.ldif b/doc/devel/index-memberof.ldif
new file mode 100644
index 0000000..b82f7fd
--- /dev/null
+++ b/doc/devel/index-memberof.ldif
@@ -0,0 +1,21 @@
+#
+#  Modify the default for a 389-based directory so that newly-created
+#  databases will index the 'memberOf' attribute by default.  Older
+#  versions did not do so automatically, which causes a dramatic speed
+#  hit if you try to use these plugins, which frequently perform internal
+#  searches using memberOf in the filter.
+#
+dn: cn=memberOf,cn=default indexes, cn=config,cn=ldbm database,cn=plugins,cn=config
+objectClass: top
+objectClass: nsIndex
+cn: memberOf
+nsSystemIndex: false
+nsIndexType: eq
+
+dn: cn=memberOf, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config
+objectClass: top
+objectClass: nsIndex
+cn: memberOf
+nsSystemIndex: false
+nsIndexType: eq
+
diff --git a/doc/devel/migrate-nis.sh b/doc/devel/migrate-nis.sh
new file mode 100755
index 0000000..d8dcb63
--- /dev/null
+++ b/doc/devel/migrate-nis.sh
@@ -0,0 +1,387 @@
+#!/bin/sh
+
+domain=`domainname`
+server=`ypwhich -d $domain`
+suffix=dc=example,dc=com
+people=cn=Users
+groups=cn=Group
+ipa=false
+realm=`echo "$domain" | tr '[a-z]' '[A-Z]'`
+rfc2307bis=false
+mergegroups=true
+maps=
+automap=false
+help=false
+email=false
+containers=false
+entries=true
+
+object_from_attr()
+{
+	case "$1" in
+	cn)
+		containerobject=nsContainer
+		;;
+	dc)
+		containerobject=domain
+		;;
+	ou)
+		containerobject=organizationalUnit
+		;;
+	*)
+		containerobject=extensibleObject
+		;;
+	esac
+	echo $containerobject
+}
+
+migrate_passwd() {
+	if $containers ; then
+		nameattr=`echo "$people" | cut -f1 -d=`
+		nameval=`echo "$people" | cut -f2- -d=`
+		containerclass=`object_from_attr "$nameattr"`
+		grep -v '^$' <<- EOF
+		dn: $people,$suffix
+		${nameattr}: ${nameval}
+		objectClass: $containerclass
+		EOF
+		echo
+	fi
+	while read key value ; do
+		if ! $entries ; then
+			continue
+		fi
+		uid=`echo "$value" | cut -d: -f1`
+		userpassword=`echo "$value" | cut -d: -f2`
+		uidnumber=`echo "$value" | cut -d: -f3`
+		gidnumber=`echo "$value" | cut -d: -f4`
+		gecos=`echo "$value" | cut -d: -f5`
+		homedirectory=`echo "$value" | cut -d: -f6`
+		loginshell=`echo "$value" | cut -d: -f7`
+		cn=`echo "$gecos" | cut -d, -f1`
+		givenname=`echo "$gecos" | awk '{print $1}'`
+		sn=`echo "$gecos" | awk '{print $NF}'`
+		grep -v '^$' <<- EOF
+			dn: uid=$uid,$people,$suffix
+			objectClass: posixAccount
+			uid: $uid
+			uidNumber: $uidnumber
+			gidNumber: $gidnumber
+			homeDirectory: $homedirectory
+			${userpassword:+userPassword: "{CRYPT}"$userpassword}
+			${loginshell:+loginShell: $loginshell}
+		EOF
+		if $rfc2307bis || $ipa || $email ; then
+			grep -v '^$' <<- EOF
+			objectClass: inetOrgPerson
+			objectClass: inetUser
+			objectClass: organizationalPerson
+			objectClass: person
+			cn: ${cn:-$uid}
+			sn: ${sn:-$uid}
+			givenName: ${givenname:-$uid}
+			mail: ${uid}@${domain}
+			EOF
+		fi
+		if $ipa ; then
+			grep -v '^$' <<- EOF
+				objectClass: krbprincipalaux
+				krbPrincipalName: $uid@$realm
+			EOF
+		fi
+		echo
+	done
+}
+
+migrate_group() {
+	if $containers ; then
+		nameattr=`echo "$groups" | cut -f1 -d=`
+		nameval=`echo "$groups" | cut -f2- -d=`
+		containerclass=`object_from_attr "$nameattr"`
+		grep -v '^$' <<- EOF
+		dn: $groups,$suffix
+		${nameattr}: ${nameval}
+		objectClass: $containerclass
+		EOF
+		echo
+	fi
+	while read key value ; do
+		if ! $entries ; then
+			continue
+		fi
+		gid=`echo "$value" | cut -d: -f1`
+		userpassword=`echo "$value" | cut -d: -f2`
+		gidnumber=`echo "$value" | cut -d: -f3`
+		members=`echo "$value" | cut -d: -f4`
+		grep -v '^$' <<- EOF
+			dn: cn=$gid,$groups,$suffix
+			objectClass: posixGroup
+			cn: $gid
+			gidNumber: $gidnumber
+			${userpassword:+userPassword: "{CRYPT}"$userpassword}
+		EOF
+		if $rfc2307bis || $ipa ; then
+			grep -v '^$' <<- EOF
+				objectClass: groupOfNames
+			EOF
+			for member in `echo "$members" | sed 's:,: :g'` ; do
+				echo member: uid=$member,$people,$suffix
+			done
+		else
+			for member in `echo "$members" | sed 's:,: :g'` ; do
+				echo memberUid: $member
+			done
+		fi
+		echo
+	done
+}
+
+migrate_automount() {
+	if $containers ; then
+		grep -v '^$' <<- EOF
+			dn: automountMapName=$1,$suffix
+			objectClass: automountMap
+			automountMapName: $1
+		EOF
+		echo
+	fi
+	while read key value ; do
+		if ! $entries ; then
+			continue
+		fi
+		grep -v '^$' <<- EOF
+			dn: automountKey=$key,automountMap=$1,$suffix
+			objectClass: automount
+			automountKey: $key
+			automountInformation: $value
+		EOF
+		echo
+	done
+}
+
+migrate_nis() {
+	if $containers ; then
+		grep -v '^$' <<- EOF
+			dn: nisMapName=$1,$suffix
+			objectClass: nisMap
+			automountMapName: $1
+		EOF
+		echo
+	fi
+	while read key value ; do
+		if ! $entries ; then
+			continue
+		fi
+		grep -v '^$' <<- EOF
+			dn: cn=$key,automountMap=$1,$suffix
+			objectClass: nisObject
+			nisMapName: $1
+			cn: $key
+			nisEntry: $value
+		EOF
+		echo
+	done
+}
+
+mergegroups() {
+	if $mergegroups ; then
+		awk -F: '
+		BEGIN { OFS=":" }
+		{ 
+			if ((length(NAMES[$3]) == 0) ||
+			    (length(NAMES[$3]) > length($1))) {
+				NAMES[$3] = $1
+			}
+			GIDS[$3] = $3
+			PASS[$3] = $2
+			if (length(MEMBERS[$3]) > 0) {
+				MEMBERS[$3] = MEMBERS[$3] "," $4
+			} else {
+				MEMBERS[$3] = $4
+			}
+		}
+		END {
+			for (GID in GIDS) {
+				print NAMES[GID],PASS[GID],GID,MEMBERS[GID]
+			}
+		}'
+	else
+		cat
+	fi
+}
+
+get_map() {
+	case "$1" in
+	passwd*)
+		ypcat -k ${server:+-h $server} ${domain:+-d $domain} passwd.byname | sort
+		;;
+	group*)
+		ypcat -k ${server:+-h $server} ${domain:+-d $domain} group.byname | mergegroups | sort
+		;;
+	*)
+		ypcat -k ${server:+-h $server} ${domain:+-d $domain} "$1" | sort
+		;;
+	esac
+}
+
+migrate_map() {
+	case "$1" in
+	passwd*)
+		(get_map "$1" || echo) | migrate_passwd
+		;;
+	group*)
+		(get_map "$1" || echo) | migrate_group
+		;;
+	auto.*|auto_*)
+		(get_map "$1" || echo) | migrate_automount "$1"
+		;;
+	*)
+		(get_map "$1" || echo) | migrate_nis "$1"
+		;;
+	esac
+}
+
+while test $# -gt 0 ; do
+	case "$1" in
+	--domain=*)
+		domain=`echo "$1" | cut -f2- -d=`
+		automap=false
+		;;
+	--domain)
+		shift
+		domain="$1"
+		automap=false
+		;;
+	--server=*)
+		server=`echo "$1" | cut -f2- -d=`
+		automap=false
+		;;
+	--server)
+		shift
+		server="$1"
+		automap=false
+		;;
+	--suffix=*)
+		suffix=`echo "$1" | cut -f2- -d=`
+		;;
+	--suffix)
+		shift
+		suffix="$1"
+		;;
+	--people=*)
+		people=`echo "$1" | cut -f2- -d=`
+		;;
+	--people)
+		shift
+		people="$1"
+		;;
+	--groups=*)
+		groups=`echo "$1" | cut -f2- -d=`
+		;;
+	--groups)
+		shift
+		groups="$1"
+		;;
+	--nomergegroups)
+		mergegroups=false
+		;;
+	--rfc2307bis)
+		rfc2307bis=true
+		;;
+	--ipa)
+		ipa=true
+		;;
+	--email)
+		email=true
+		;;
+	--realm=*)
+		realm=`echo "$1" | cut -f2- -d= | tr '[a-z]' '[A-Z]'`
+		automap=false
+		;;
+	--realm)
+		shift
+		realm=`echo "$1"                | tr '[a-z]' '[A-Z]'`
+		automap=false
+		;;
+	-a|--all)
+		automap=true
+		;;
+	--containers)
+		containers=true
+		;;
+	--just-containers)
+		containers=true
+		entries=false
+		;;
+	-*|-h|--help)
+		help=true
+		;;
+	*)
+		maps="${maps:+$maps }$1"
+		;;
+	esac
+	shift
+done
+
+if $automap && test -z "$maps" ; then
+	maps=`./ypmaplist.py`
+fi
+if $help || test -z "$maps" ; then
+	echo `basename $0`: create LDIF from NIS maps
+	echo Usage: `basename $0` "[options] [mapname [...]]"
+	cat <<- EOF
+	Options:
+	-h --help		Print this text.
+	--domain		Query maps for a non-default domain (default is
+	 			"$domain").
+	--server		Query a non-default server (default is
+	 			"$server").
+	--suffix		Store entries under a non-default suffix (default is
+	 			"$suffix").
+	--people		Store account entries under a non-default container
+	 			under the suffix (default is "$people").
+	--groups		Store group entries under a non-default container
+	 			under the suffix (default is "$groups").
+	--nomergegroups		Don't merge group entries which have the same GID.
+	--rfc2307bis		Use groupOfNames groups, create user account
+	 			entries which are also inetOrgPerson entries.
+	--ipa			Use groupOfNames groups, create user account
+	 			entries which are also inetOrgPerson and Kerberos
+	 			user entries.
+	--realm			Use a non-default Kerberos realm name (default is
+	 			"$realm").
+	--email			Add email addresses by default (default domain for
+	 			mail addresses is "$domain").
+	-a --all		Attempt to migrate all maps in the local domain.
+	 			(Can not be used with either the --server or
+	 			the --domain options.)
+	--containers		Create containers for maps in addition to entries.
+	--just-containers	Create containers for maps, but not for entries.
+	EOF
+else
+	seen_passwd=false
+	seen_group=false
+	for map in $maps ; do
+		seen_before=false
+		case "$map" in
+		*.by*)
+			base=`echo "$map" | sed 's,\.by.*,,g'`
+			case $base in
+			passwd)
+				if $seen_passwd ; then
+					seen_before=true
+				fi
+				seen_passwd=true
+				;;
+			group)
+				if $seen_group ; then
+					seen_before=true
+				fi
+				seen_group=true
+				;;
+			esac
+			;;
+		esac
+		$seen_before || migrate_map "$map"
+	done
+fi
diff --git a/doc/devel/nis-compatibility.txt b/doc/devel/nis-compatibility.txt
new file mode 100644
index 0000000..c600450
--- /dev/null
+++ b/doc/devel/nis-compatibility.txt
@@ -0,0 +1,77 @@
+NIS itself consists of approximately a dozen RPC functions.  In RPC
+parlance, the yppasswd protocol, which provides a means of remotely
+changing passwords and GECOS information, is a wholly different
+protocol, and no attempt has been made to implement any part of it in
+this plugin.  The rationale for this is that browser-based self-service
+solutions for this problem are assumed to be plentiful, and as the
+yppasswd protocol doesn't encrypt passwords when they're sent over the
+network, it's best avoided if possible.
+
+Getting back to NIS, the protocol consists of several remote procedure
+call interfaces which a server is expected to provide, and compatibility
+can most simply be described by noting differences between this
+implementation and typical NIS server implementations.
+
+YPPROC_NULL
+  This function doesn't actually do anything.  Fully implemented.
+
+YPPROC_DOMAIN
+YPPROC_DOMAIN_NONACK
+  These functions are used by a client's ypbind daemon to verify that a
+  particular server contains maps for a given domain.  Fully implemented.
+
+YPPROC_MATCH
+  This function searches for one entry in a map, and is typically what
+  the "ypmatch" command-line utility uses.  Fully implemented.
+
+YPPROC_FIRST
+  This function retrieves the first entry in a map.  Fully implemented.
+
+YPPROC_NEXT
+  This function retrieves the entry in a map which immediately follows
+  another, specified, entry in the map.  The combination of YPPROC_FIRST
+  and YPPROC_NEXT provides one mechanism for clients to walk the entire
+  set of entries in a map.  If a single key occurs more than once in a
+  map, however, a client will either "skip over" entries which are
+  between instances of that key or may become stuck endlessly traversing
+  that same set of entries.
+ 
+  While typical NIS servers have a defined order for entries in maps
+  (that order usually being the same as the ordering used in the files
+  which were used to generate the maps), a directory server does not
+  define such an order, so while this function can still be used to
+  retrieve all of the entries in a map, no guarantee can be made with
+  respect to the order in which those entries will be found.
+
+YPPROC_ALL
+  This function retrieves all of the entries in a map, and is typically
+  what the "ypcat" command-line utility uses.  Fully implemented.
+
+YPPROC_MASTER
+  This function retrieves the name of the NIS server which is the master
+  (in replication scenarios) for a specified map.  Because replication
+  of data is handled at the directory server level, this function is a
+  no-op.
+
+YPPROC_CLEAR
+  This function typically instructs a NIS server to close and reopen the
+  file from which it is reading the contents of a map.  Because the
+  plugin updates its maps at the same time the data on which those maps
+  depend is modified, this function is a no-op.
+
+YPPROC_XFR
+  This function typically instructs a NIS server to re-fetch a map from
+  the server which is the master for the map.  Because replication of
+  data is handled at the directory server level, this function is a
+  no-op.
+
+YPPROC_ORDER
+  This function retrieves the time that a specified map was last
+  modified, typically used by a replica NIS server to determine when it
+  needs to fetch an updated set of map entries.  Fully implemented.
+
+YPPROC_MAPLIST
+  This function retrieves the list of maps defined for the specified
+  domain.  While there is no typical command-line utility which fetches
+  this information, the source tree includes a python script which calls
+  this function and prints its results.  Fully implemented.
diff --git a/doc/devel/nis-design.txt b/doc/devel/nis-design.txt
new file mode 100644
index 0000000..f98bf57
--- /dev/null
+++ b/doc/devel/nis-design.txt
@@ -0,0 +1,289 @@
+= Design Overview =
+
+The NIS Server plugin's aim is to serve up data from the directory
+server using the NIS protocols.  It does this by doing what any gateway
+would do: it queries the directory server for entries which would
+correspond to the contents of maps, reads the contents of various
+attributes from those entries, and uses that data to synthesize entries
+for maps which it serves to clients.
+
+In broad strokes, one design might look like this:
+
+   ┌──────────┐   NIS   ┌───────────┐   LDAP   ┌────────────────────┐
+   │  Client  │─────────│  Gateway  │──────────│  Directory Server  │
+   └──────────┘         └───────────┘          └────────────────────┘
+
+The links in this diagram represent network traffic.  The client uses
+the NIS protocol to communicate with the gateway, and the gateway uses
+the LDAP protocol to communicate with the directory server.
+
+This implementation requires that the gateway be robust against
+variations in directory server availability, be flexible enough to use
+any of a number of methods of authenticating to the directory server,
+and may additionally require the presence of specific extensions on the
+server in order to be able to be even reasonably certain of consistency
+with the directory's contents.
+
+In order to sidestep these requirements, and the complexity they add to
+an implementation, we decided to implement the gateway as a plugin.  As
+a plugin, the gateway starts and stops with the directory server, it
+does not need to authenticate as a normal client would, and it can be
+expected to work with a server which can use it.
+
+Taking just the gateway and directory server portions of the above
+diagram, and breaking them down further, we can come to this:
+
+   ┌──────────────┐   ┌─────────┐   ┌────────────────────────────┐
+   │ NIS Protocol │───│ Mapping │───│ Directory Server Back Ends │
+   └──────────────┘   └─────────┘   └────────────────────────────┘
+
+The links in this diagram are now API calls.  We've relegated the work
+of reading a query (parsed from the NIS client by the NIS Protocol
+handler), converting that query to a directory server search operation,
+and marshalling the results of that search into a format suitable for
+transmission as a NIS response, all to the Mapping module.  The
+directory server back ends are exposed by SLAPI, of course.
+
+This approach does have its problems, though.
+
+NIS, as a protocol, requires that the server be able to supply a few
+bits of information which can't readily (or shouldn't) be retrieved this
+way, information which is not normally kept in a directory server.
+
+NIS requires that a server be able to report a revision number for a
+map, which is used as an indicator of the time when the map was last
+modified.  A slave server can use this information to poll for changes
+in map contents on the master, possibly beginning a full map enumeration
+to read those new contents in order to serve its clients.
+
+A directory server, if it stores revision information at all, stores
+it on a per-entry basis.  So when a gateway designed as we diagrammed
+above is asked for this information, it has at least these options:
+  a) always use the current time
+     - This causes frequent map updates on clients when they don't need
+       them, and completely unnecessary network traffic.
+  b) always use the same value
+     - This keeps clients from ever noticing that a map has changed.
+  c) return the latest revision of any of the results which formed the
+     contents of the map
+     - This could severely load a directory server if the information
+       needs to be generated by reading the last-modified timestamps
+       from many directory server entries.
+
+NIS also requires that a server be able to answer whether or not it
+services a specified domain, and which maps it serves for a domain that
+it serves.  While the mapping module could search the directory's
+configuration space whenever it is asked these questions, the first
+question is asked regularly by each running copy of ypbind, which could
+also bog servers down (though admittedly, less than the previous case).
+
+If we break the mapping portion up further, we can introduce a map
+cache.  In this module we can maintain a cache of the NIS server's data
+set, taking care to construct it at startup-time, updating it as the
+contents of the directory server change, and always serving clients
+using data from the cache.
+
+   ┌──────────────┐  ┌───────────┐  ┌──────────────┐  ┌──────┐
+   │ NIS Protocol │──│ Map Cache │──│ Map Back End │──│ Data │
+   └──────────────┘  └───────────┘  └──────────────┘  └──────┘
+
+Which takes us to the current design.  The NIS protocol handler reads
+data from the map cache, and the map back end uses SLAPI to obtain data
+which is used to populate the map cache at startup-time, as well as to
+watch for changes in the directory's contents which would need to be
+reflected in the map cache.
+
+= Components =
+
+== Protocol Handler ==
+
+This NIS protocol handler module takes the opportunity to set up
+listening sockets (listening on the port specified in the plugin
+configuration entry's "nsslapd-pluginarg0" attribute, or an unused port
+if none is specified) and register with the local portmapper at module
+initialization time.  The plugin then starts a listening thread to
+service its clients.
+
+The plugin listens for datagram queries from clients, processing them as
+they come in, as well as accepting connections from clients.  Because
+connected clients may not always transmit an entire request at once, and
+because the server may find itself unable to transmit an entire response
+at once, it buffers traffic for connected clients, multiplexing the work
+it does for all of its clients from inside of its thread.  The actual
+protocol datagram parsing is performed by libnsl, which is provided as a
+part of the C library.
+
+Datagram responses which exceed the "nis-max-dgram-size" threshold (by
+default, 1024 bytes, or 1 kilobyte) are simply dropped.  Response
+records larger than "nis-max-value-size" (by default, 256 kilobytes) are
+also ignored, even for connected clients.
+
+Client access is limited by the local tcp_wrappers configuration on the
+directory server, with a tcp_wrappers service name as dictated by the
+"nis-tcp-wrappers-name" attribute (by default, "nis-plugin") in the
+plugin's configuration.  If the tcp_wrappers configuration denies access
+for the client, a connected client's connection will be closed, and a
+datagram client's request will be discarded.
+
+Client requests are also limited based on a client's address using
+"securenet"-style settings in the module's configuration entry's
+"nis-securenet" attribute.  If no values are specified, access is
+allowed to all clients.  If the securenet configuration denies access
+for the client, a connected client's connection will be closed, and a
+datagram client's request will be discarded.
+
+Client requests are further classed as "secure" or not, based on the
+query's originating port.  This information is used elsewhere for
+additional access control.
+
+== NIS Layer ==
+
+The NIS layer processes complete requests, whether they come in from
+connected or datagram clients, fetches the requested information from
+the map cache, and uses callbacks provided by the protocol handler to
+respond.  Before doing so, if a value is being retrieved from a map, it
+checks if the map's contents are restricted to "secure" clients.  If
+they are, but the client is not a "secure" client, the NIS layer will
+respond as if no data were present in the map.
+
+== Map Cache ==
+
+The map cache keeps a dynamically-constructed set of maps in memory,
+grouped by domain, and for each map maintains information regarding the
+last time its contents were modified (to answer client requests for a
+map's order) and whether or not the map's contents should be restricted
+to "secure" clients.  The map cache can quickly answer whether or not a
+domain is being served by checking whether or not any maps are defined
+for it.  The definitions of which maps are served for which domains is
+configurable via internal APIs -- the map cache itself has no forehand
+knowledge of domain names, map names, or formats, as it merely models
+data in the way that a conventional NIS server might.
+
+Forcing queries to use the cache provides a couple of benefits over an
+alternate approach of performing an LDAP query for each NIS query:
+* While the directory server is generally only case-preserving, the NIS
+  server can be case-sensitive, which is preferred by NIS clients and
+  a requirement for some customers.
+* Because the query used is never used to construct an LDAP filter or
+  query, we don't have to worry about escaping text to avoid string
+  injection attacks.
+
+=== Internal Representation ===
+
+At the topmost level, the map cache is a table.  Each entry in the table
+is the name of a domain and a table of maps.
+
+Each entry in a domain's table of maps contains the map's name, the time
+the map was last modified, a note indicating whether or not the map is a
+"secure" map, a linked list of map entries, and a set of indexes into
+the list.  Each map can also hold a data pointer on behalf of the
+backend.
+
+Each item in the map's list of entries contains an array of NIS keys, an
+array of corresponding values, a unique identifier (which, currently,
+stores the NDN of the directory server entry which was used to create
+this list item) and a data pointer which is kept on behalf of the
+backend.
+
+The map indexes its entry list using an entry's unique identifier, and
+each of its keys.
+
+== Back End ==
+
+The backend interface module sets up, populates, and maintains the map
+cache.  At startup time, it configures the map cache with the list of
+domains and maps, and populates the maps with initial data.  Using
+postoperation plugin hooks, the backend interface also notes when
+entries are added, modified, renamed (modrdn'd), or deleted from the
+directory server.  It uses this information to create or destroy maps in
+the map cache, and to add, remove, or update entries in the map cache's
+maps, thereby ensuring that the map cache always reflects the current
+contents of the directory server.
+
+The backend interface reads the configuration it should use for the map
+cache from its configuration area in the directory server.  Beneath the
+plugin's entry, the backend checks for entries with these attributes:
+* nis-domain
+* nis-map
+* nis-secure
+* nis-base
+* nis-filter
+* nis-key-format
+* nis-keys-format
+* nis-value-format
+* nis-values-format
+The backend then instructs the map cache to prepare to hold a map in the
+given domain (or domains) with the given map name (or names), and then
+performs a subtree search under the specified base (or bases, if there's
+more than one ''nis-base'' value) for entries which match the provided
+filter.  Each entry found is then "added" to the map, using the format
+specifiers stored in the ''nis-key-format'' and ''nis-keys-format''
+attributes to construct the keys for the entry in the map, with the
+corresponding value in the map being constructed using the format
+specifiers stored in the ''nis-value-format'' and ''nis-values-format''
+attributes.  The map is also marked as a "secure" map according to the
+''nis-secure'' attribute, if so set.
+
+For each ''nis-key-format'' value, exactly one entry will be created in a
+NIS map.  (If a ''nis-key-format'' does not yield a single value, the
+directory server entry will not appear in the NIS map.)  For each
+''nis-keys-format'' value, any number of entries will be created in a NIS
+map.  The method by which these attributes (and the ''nis-value-format''
+and ''nis-value-formats'') are interpreted is described below.
+
+Should one of the directory server entries which was used to construct
+one or more NIS map entries be modified or removed, the corresponding
+entries in every applicable NIS map are updated or removed.  Likewise,
+if an entry is added to the directory server which would correspond to
+an entry in a NIS map, entries are created in the corresponding NIS
+maps.
+
+== Formatting Data for NIS ==
+
+The ''nis-key-format'' and ''nis-value-format'' specifiers resemble an RPM
+format specifier, and can include the values of multiple attributes in
+any part of the specifier.  The backend composes the string using the
+attribute values stored in the directory server entry, using the format
+specifier as a guide.  In this way, the NIS map's contents can be
+constructed to almost any specification, and can make use of data stored
+using any schema.
+
+An example specification for the ''nis-value-format'' for a user's entry
+could look something like this:
+  %{uid}:%{userPassword:-*}:%{uidNumber}:%{gidNumber}:%{gecos:-%{cn:-}}:%{homeDirectory}:%{loginShell:-/bin/sh}
+The syntax borrows from RPM's syntax, which in turn borrows from shell
+syntax, to allow the specification of alternate values to be used when
+the directory server entry doesn't include a ''userPassword'' or ''gecos''
+attribute.  Additional operators include "#", "##", "%", "%%", "/",
+"//", which operate in ways similar to their shell counterparts (with
+one notable exception being that patterns for the "/" operator can not
+currently be anchored to the beginning or end of the string).
+
+A format specifier can actually be interpreted in two ways: it can be
+interpreted as a single value (when given as ''nis-key-format'' or
+''nis-value-format''), or it can be interpreted as providing a list of
+values (''nis-keys-format'' or ''nis-values-format'').  When the format
+specifier is being interpreted as a single value, any reference to an
+attribute value which does not also specify an alternate value will
+cause the directory server entry to be ignored if the referenced
+attribute has no value defined for that entry, or contains multiple
+values.  In the above example, the entry would be ignored if the ''uid'',
+''uidNumber'', ''gidNumber'', or ''homeDirectory'' attributes of the entry did
+not each contain exactly one value.
+
+The ''nis-filter'', ''nis-key-format'', and ''nis-value-format'' settings have
+sensible defaults for the maps which we expect to be commonly used --
+this is important because it's easy to subtly construct malformed result
+specifiers which could trigger undefined behavior on clients -- for
+example by leaving the user's numeric UID empty in a passwd entry, which
+may be treated as "0" by inattentive clients.
+
+The format specifier syntax further defines "functions" which can be
+used to concatenate lists of multiple values into a single result, for
+example for groups:
+  %{cn}:%{userPassword:-*}:%{gidNumber}:%merge(",","%{memberUid}")
+This filter takes advantage of a built-in ''merge'' function, which
+processes zero or more single or list values and concatenates them
+together with a "," separator, to generate the list of the group's
+members.  The available functions are described more fully in
+"format-specifiers.txt".
diff --git a/doc/devel/sch-design.txt b/doc/devel/sch-design.txt
new file mode 100644
index 0000000..ecaf14c
--- /dev/null
+++ b/doc/devel/sch-design.txt
@@ -0,0 +1,127 @@
+= Design Overview =
+
+The Schema Compatibility plugin's aim is to synthesize a modified view
+of the contents of one part of the directory and to make that view
+visible in another part of the directory.  It does this by precomputing
+the contents of its entries at startup-time and refreshing its entries
+as needed.
+
+== Map Cache ==
+
+The map cache keeps a dynamically-constructed group of sets of entries
+in memory, grouped by the name of their top-level container, and for
+each set of entries maintains a copy of the set's releative
+distinguished name.  The map cache can be used to quickly answer whether
+or not a particular search request crosses into the portion of the DIT
+which is "served" by the data in the cache.
+
+=== Internal Representation ===
+
+At the topmost level, the map cache is a table.  Each entry in the table
+contains the name of a top-level container entry and a table of sets.
+
+Each set in a container's table of sets contains the relative
+distinguished name (RDN) of the set, a linked list of map entries, and a
+set of indexes into the list.  Each set can also hold a data pointer on
+behalf of the backend.
+
+Each item in a set's list of entries contains the entry's relative
+distinguished name (relative to the set, which is itself relative to the
+group-of-sets container), the entry's whole distinguished name, a unique
+identifier (which, currently, stores the NDN of the directory server
+entry which was used to create this entry) and a data pointer which is
+kept on behalf of the backend (which, currently, stores a Slapi_DN for
+the directory server entry which was used to create this entry and the
+full Slapi_Entry for the generated entry).
+
+The set indexes its entry list using an entry's unique identifier and
+its relative distinguished name.
+
+== Back End ==
+
+The backend interface module sets up, populates, and maintains the map
+cache.  At startup time, it configures the map cache with the list of
+groups of sets of entries, and populates the sets with initial data.
+Using postoperation plugin hooks, the backend interface also notes when
+entries are added, modified, renamed (modrdn'd), or deleted from the
+directory server.  It uses this information to create or destroy sets in
+the map cache, and to add, remove, or update entries in the map cache's
+sets, thereby ensuring that the synthetic data in the map cache is
+always up to date with respect to the current contents of the directory
+server.
+
+The backend interface reads the configuration it should use for the map
+cache from its configuration area in the directory server.  Beneath the
+plugin's entry, the backend checks for entries with these attributes:
+* schema-compat-search-base
+* schema-compat-search-filter
+* schema-compat-container-group
+* schema-compat-container-rdn
+* schema-compat-entry-rdn
+* schema-compat-entry-attribute
+* schema-compat-check-access
+
+The backend then instructs the map cache to prepare to hold a set of
+entries in the given container group (or container groups) with the
+given subcontainer RDN name (or names), and then performs a subtree
+search under the specified base (or bases, if there's more than one
+''schema-compat-search-base'' value) for entries which match the provided
+filter (''schema-compat-search-filter'').
+
+For each entry found, a new entry is generated and "added" to the
+subcontainer, using the format specifier stored in the
+''schema-compat-entry-rdn'' and ''schema-compat-entry-attribute''
+attributes to construct the RDN and attribute values for the entry in
+the set.
+
+Should one of the directory server entries which was used to construct
+one or more entries be modified or removed, the corresponding entries in
+every applicable container are updated or removed.  Likewise, if an
+entry is added to the directory server which would correspond to an
+entry in a container, entries are created in the corresponding
+container.
+
+== Specifying Entry Contents ==
+
+The ''schema-compat-entry-rdn'' specifier resembles an RPM format
+specifier, and can include the values of multiple attributes in any part
+of the specifier.  The backend composes the string using the attribute
+values stored in the directory server entry, using the format specifier
+as a guide, and names the resulting entry using the subcontainer's name
+and the generated RDN.  Attributes specified using values of the
+''schema-compat-entry-attribute'' attribute are then added.  If the
+resulting entry fails schema checks, it is automatically given the
+''extensibleObject'' object class.
+
+An example specification for the ''schema-compat-entry-rdn'' for a user's
+entry could look something like this:
+  uid=%{uid}
+The syntax borrows from RPM's syntax, which in turn borrows from shell
+syntax, to allow the specification of alternate values to be used when
+the directory server entry doesn't include a "uid" attribute.
+Additional operators include "#", "##", "%", "%%", "/", "//", which
+operate in ways similar to their shell counterparts (with one notable
+exception being that patterns for the "/" operator can not currently be
+anchored to the beginning or end of the string).
+
+After the RDN is determined, attributes can be added.  Most often, this
+will be done either by specifying a specific value (typically for the
+entry's object classes) or an existing attribute, like so:
+  objectclass=posixAccount
+  cn=%{cn}
+
+A format specifier can actually be interpreted in two ways: it can be
+interpreted as a single value (as it is for ''schema-compat-entry-rdn''
+values), or it can be interpreted as providing a list of values (as it
+is for ''schema-compat-entry-attribute'' values).  When the format
+specifier is being interpreted as a single value, any reference to an
+attribute value which does not also specify an alternate value will
+cause the directory server entry to be ignored if the referenced
+attribute has no value defined for that entry, or contains multiple
+values.
+
+The format specifier syntax further defines "functions" which can be
+used to manipulate values for attributes in the entry being examined,
+follow references to other entries and retrieve values from them, and to
+manipulate those values into whatever format is required.  The available
+functions are described more fully in "format-specifiers.txt".
diff --git a/doc/devel/todo.txt b/doc/devel/todo.txt
new file mode 100644
index 0000000..c8cdb8a
--- /dev/null
+++ b/doc/devel/todo.txt
@@ -0,0 +1,8 @@
+* Finish figuring out sane default key and value formats for the set of maps
+  for which we're trying to provide default settings. (NIS)
+* More tests.
+* Stop using an internal function to send the LDAP result for failed searches.
+* Reintroduce / and // operator support in attribute references.
+* Look at moving the schema compatibility plugin to using a private database,
+  so that it doesn't have to store everything it has in memory, we get working
+  indexing, and so on and so forth.