Merge 2610c05b5b95cc7036b3d6dfb894c6cfbdb68483 as Samba-4.0alpha16samba-4.0.0alpha16

1484 files changed, 78250 insertions, 25241 deletions

diff --git a/.gitignore b/.gitignore
index a8eacd562f4..383bb1e098f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,8 +71,8 @@ source3/exports/libtalloc.syms
 source3/exports/libtdb.syms
 source3/exports/libwbclient.syms
 source3/include/build_env.h
-source3/include/config.h
-source3/include/config.h.in
+source3/include/autoconf/config.h
+source3/include/autoconf/config.h.in
 source3/include/includes.h.gch
 source3/include/stamp-h
 source3/include/version.h
diff --git a/Makefile b/Makefile
index cbbf045062e..3b40c73d957 100644
--- a/Makefile
+++ b/Makefile
@@ -68,7 +68,7 @@ ctags:
 
 # this allows for things like "make bin/smbtorture"
 bin/%:: FORCE
-	$(WAF) --targets=`basename $@`
+	$(WAF) --targets=$@
 FORCE:
 
 pydoctor:
diff --git a/VERSION b/VERSION
index 1c5ad2f146e..96059d6c1b0 100644
--- a/VERSION
+++ b/VERSION
@@ -57,7 +57,7 @@ SAMBA_VERSION_TP_RELEASE=
 # e.g. SAMBA_VERSION_ALPHA_RELEASE=1                   #
 #  ->  "4.0.0alpha1"                                   #
 ########################################################
-SAMBA_VERSION_ALPHA_RELEASE=15
+SAMBA_VERSION_ALPHA_RELEASE=16
 
 ########################################################
 # For 'pre' releases the version will be               #
diff --git a/WHATSNEW.txt b/WHATSNEW.txt
index 9c728ba9c3d..0251ffdd72f 100644
--- a/WHATSNEW.txt
+++ b/WHATSNEW.txt
@@ -1,212 +1,129 @@
-                   =================================
-                   Release Notes for Samba 3.6.0pre1
-                             July 28, 2010
-                   =================================
+What's new in Samba 4 alpha16
+=============================
 
+Samba 4.0 will be the next version of the Samba suite and incorporates
+all the technology found in both the Samba4 alpha series and the
+stable 3.x series. The primary additional features over Samba 3.6 are
+support for the Active Directory logon protocols used by Windows 2000
+and above.
 
-This is the first preview release of Samba 3.6.  This is *not*
-intended for production environments and is designed for testing
-purposes only.  Please report any defects via the Samba bug reporting
-system at https://bugzilla.samba.org/.
+WARNINGS
+========
 
+Samba4 alpha16 is not a final Samba release, however we are now making
+good progress towards a Samba 4.0 release, of which this is a preview.
+Be aware the this release contains both the technology of Samba 3.6
+(that you can reasonably expect to upgrade existing Samba 3.x releases
+to) and the AD domain controller work previously known as 'samba4'.
+No migration path is currently provided between these two sets of
+technology.  These missing migration paths will be the focus of
+development between now and a Samba 4.0 release.
 
-Major enhancements in Samba 3.6.0 include:
+While binaries for the stable file server are provided in this
+release, for a stable, supported file server, Samba3 domain or AD
+domain member installation, please run a Samba 3.x release, as we are
+still bedding down the new single build system.
 
+Samba4 is subjected to an awesome battery of tests on an automated
+basis, we have found Samba 4.0 to be very stable in it's behavior.
+However, we still recommend against upgrading production servers from
+Samba 3.x release to Samba 4.0 alpha at this stage.
 
-Changed security defaults
--------------------------
+If you are upgrading, or looking to develop, test or deploy Samba 4.0
+alpha releases, you should backup all configuration and data.
 
-Samba 3.6 has adopted a number of improved security defaults that will
-impact on existing users of Samba.
+NEW FEATURES
+============
 
- client ntlmv2 auth = yes
- client use spnego principal = no
- send spnego principal = no
+Samba 4.0 alpha supports the server-side of the Active Directory logon
+environment used by Windows 2000 and later, so we can do full domain
+join and domain logon operations with these clients.
 
-The impact of 'client ntlmv2 auth = yes' is that by default we will not
-use NTLM authentication as a client.  This applies to the Samba client
-tools such as smbclient and winbind, but does not change the separately
-released in-kernel CIFS client.  To re-enable the poorer NTLM encryption
-set '--option=clientusentlmv2auth=no' on your smbclient command line, or
-set 'client ntlmv2 auth = no' in your smb.conf
+Our Domain Controller (DC) implementation includes our own built-in
+LDAP server and Kerberos Key Distribution Center (KDC) as well as the
+Samba3-like logon services provided over CIFS.  We correctly generate
+the infamous Kerberos PAC, and include it with the Kerberos tickets we
+issue.
 
-The impact of 'client use spnego principal = no' is that we may be able
-to use Kerberos to communicate with a server less often in smbclient,
-winbind and other Samba client tools.  We may fall back to NTLMSSP in
-more situations where we would previously rely on the insecure
-indication from the 'NegProt' CIFS packet.  This mostly occursed when
-connecting to a name alias not recorded as a servicePrincipalName for
-the server.  This indication is not available from Windows 2008 or later
-in any case, and is not used by modern Windows clients, so this makes
-Samba's behaviour consistent with other clients and against all servers.
+Samba 4.0 alpha ships with two distinct file servers.  The file server
+from the Samba 3.x series is 'smbd', and works with the binaries users
+would expect from that series (nmbd, winbindd, smbpasswd).
 
-The impact of 'send spnego principal = no' is to match Windows 2008 and
-not to send this principal, making existing clients give more consistent
-behaviour (more likely to fall back to NTLMSSP) between Samba and
-Windows 2008, and between Windows versions that did and no longer use
-this insecure hint.
+Samba 4.0 also ships with a new file server, which is tuned to match
+the requirements of an AD domain controller.  Users should not use the
+file server in the 'samba' binary for non-DC related tasks.
 
+A new scripting interface has been added to Samba 4, allowing Python
+programs to interface to Samba's internals, and many tools and
+internal workings of the DC code is now implemented in python.
 
-SMB2 support
-------------
 
-SMB2 support in 3.6.0 is fully functional (with one omission),
-and can be enabled by setting:
+CHANGES SINCE alpha15
+=====================
 
-max protocol = SMB2
+For a list of changes since alpha 15, please see the git log.
 
-in the [global] section of your smb.conf and re-starting
-Samba. All features should work over SMB2 except the modification
-of user quotas using the Windows quota management tools.
+$ git clone git://git.samba.org/samba.git
+$ cd samba.git
+$ git log release-4-0-0alpha15..release-4-0-0alpha16
 
-As this is the first release containing what we consider
-to be a fully featured SMB2 protocol, we are not enabling
-this by default, but encourage users to enable SMB2 and
-test it. Once we have enough confirmation from Samba
-users and OEMs that SMB2 support is stable in wide user
-testing we will enable SMB2 by default in a future Samba
-release.
+The biggest user-visible change is that binaries from the Samba 3.x
+series of development are now built and installed.  These binaries
+(smbd, nmbd, winbindd, net, testparm, etc) operate much as they do in
+the Samba 3.6 release candidates. 
 
+CHANGES
+=======
 
-Internal Winbind passdb changes
--------------------------------
+smb.conf changes
+----------------
 
-Winbind has been changed to use the internal samr and lsa rpc pipe to get
-local user and group information instead of calling passdb functions. The
-reason is to use more of our infrastructure and test this infrastructure by
-using it. With this approach more code in Winbind is shared.
+   Parameter Name                      Description     Default
+   --------------                      -----------     -------
+   announce version 		       Removed
+   announce as	 		       Removed
 
+Those familiar with Samba 3 can find a list of user-visible changes
+between the two technology sets provided in the NEWS file.
 
-New Spoolss code
-----------------
+KNOWN ISSUES
+============
 
-The spoolss and the old RAP printing code have been completely
-overhauled and refactored.
-
-All calls from lanman/printing code has been changed to go through the
-spoolss RPC interfaces, this allows us to keep all checks in one place
-and avoid special cases in the main printing code.
-Printing code has been therefore confined within the spoolss code.
-
-All the printing code, including the spoolss RPC interfaces has been
-changed to use the winreg RPC interfaces to store all data.
-All data has been migrated from custom, arbitrary TDB files to the
-registry interface. This transition allow us to present correct data to
-windows client accessing the server registry through the winreg RPC
-interfaces to query for printer data. Data is served out from a real
-registry implementation and therefore arguably 100% forward compatible.
-
-Migration code from the previous TDB files formats is provided. This
-code is automatically invoked the first time the new code is run on the
-server. Although manual migration is also available using the 'net
-printer migrate' command.
-
-These changes not only make all the spoolss code much more closer to
-"the spec", it also greatly improves our internal testing of both
-spoolss and winreg interfaces, and reduces overall code duplication.
-
-As part of this work, new tests have been also added to increase
-coverage.
-
-This code will also allow, in future, an easy transition to split out
-the spooling functions into a separate daemon for those OEMs that do not
-need printing functionality in their appliances, reducing the code
-footprint.
-
-
-ID Mapping Changes
-------------------
-
-The id mapping configuration has been a source of much grief in the past.
-For this release, id mapping has ben rewritten yet again with the goal
-of making the configuration more simple and more coherent while keeping
-the needed flexibility and even adding to the flexibility in some respects.
-
-The major change that implies the configuration simplifications is at
-the heart of the id mapping system: The separation of the "idmap alloc
-system" that is responsible for the unix id counters in the tdb, tdb2
-and ldap idmap backends from the id mapping code itself has been removed.
-The sids_to_unixids operation is now atomic and encapsulates (if needed)
-the action of allocating a unix id for a mapping that is to be created.
-Consequently all idmap alloc configuration parameters have vanished and
-it is hence now also not possible any more to specify an idmap alloc
-backend different from the idmap backend. Each idmap backend uses its
-own idmap unixid creation mechanism transparently.
-
-As a consequence of the id mapping changes, the methods that are used
-for storing and deleting id mappings have been removed from the winbindd
-API. The "net idmap dump/restore" commands have been rewritten to
-not speak through winbindd any more but directly act on the databases.
-This is currently available for the tdb and tdb2 backends, the implementation
-for ldap still missing.
-
-The allocate_id functionality is preserved for the unix id creator of the
-default idmap configuration is also used as the source of unix ids
-for the group mapping database and for the posix attributes in a
-ldapsam:editposix setup.
-
-As part of the changes, the default idmap configuration has been
-changed to be more coherent with the per-domain configuration.
-The parameters "idmap uid", "idmap gid" and "idmap range" are now
-deprecated in favour of the systematic "idmap config * : range"
-and "idmap config * : backend" parameters. The reason for this change
-is that the old options only provided an incomplete and hence deceiving
-backwards compatibility, which was a source of many problems with
-updgrades. By introducing this change in configuration, it should be
-brought to the conciousness of the users that even the simple
-id mapping is not working exactly as in Samba 3.0 versions any more.
-
-
-SMB Traffic Analyzer
---------------------
-
-Added the new SMB Traffic Analyzer (SMBTA) VFS module protocol 2
-featuring encryption, multiple arguments, and easier parseability. A new
-tool 'smbta-util' has been created to control the encryption behaviour
-of SMBTA. For compatibility, SMBTA by default operates on version 1.
-There are programs consuming the data that the module sends.
-
-More information can be found on
-http://holger123.wordpress.com/smb-traffic-analyzer/
-
-
-NFS quota backend on Linux
---------------------------
-
-A new nfs quota backend for Linux has been added that is based
-on the existing Solaris/FreeBSD implementation. This allows samba
-to communicate correct diskfree information for nfs imports that
-are re-exported as samba shares.
-
-
-######################################################################
-Changes
-#######
+- Domain member support in the 'samba' binary is in it's infancy, and
+  is not comparable to the support found in winbindd.  As such, do not
+  use the 'samba' binary (provided for the AD server) on a member
+  server.
 
-smb.conf changes
-----------------
+- There is no printing support in the 'samba' binary (use smbd instead)
 
-   Parameter Name                      Description     Default
-   --------------                      -----------     -------
+- There is no NetBIOS browsing support in the 'samba' binary (use nmbd
+  and smbd instead)
+
+- Clock Synchronisation is critical.  Many 'wrong password' errors are
+  actually due to Kerberos objecting to a clock skew between client
+  and server.  (The NTP work in the previous alphas are partly to assist
+  with this problem).
+
+- The DRS replication code may fail.  Please contact the team if you
+  experience issues with DRS replication, as we have fixed many issues
+  here in response to feedback from our production users. 
+
+- Users upgrading existing databases to Samba4 should carefully
+  consult upgrading-samba4.txt.  We have made a number of changes in
+  this release that should make it easier to upgrade in future.
+  Btw: there exists also a script under the "setup" directory of the
+  source distribution called "upgrade_from_s3" which should allow a step-up
+  from Samba3 to Samba4. It's not included yet in the binary distributions
+  since it's completely experimental!
 
-   async smb echo handler	       New	       No
-   client ntlmv2 auth		       Changed Default Yes
-   client use spnego principal	       New	       No
-   ctdb locktime warn threshold	       New	       0
-   idmap alloc backend		       Removed
-   log writeable files on exit	       New	       No
-   multicast dns register	       New	       Yes
-   ncalrpc dir			       New
-   send spnego principal	       New	       No
-   smb2 max credits		       New	       128
-   smb2 max read		       New	       1048576
-   smb2 max trans		       New	       1048576
-   smb2 max write		       New	       1048576
-   username map cache time	       New	       0
-   winbind max clients		       New	       200
-
-
-######################################################################
+RUNNING Samba 4.0 as an AD DC
+=============================
+
+A short guide to setting up Samba 4 as an AD DC can be found on the wiki:
+
+  http://wiki.samba.org/index.php/Samba4/HOWTO
+
+#######################################
 Reporting bugs & Development Discussion
 #######################################
 
@@ -216,7 +133,7 @@ joining the #samba-technical IRC channel on irc.freenode.net.
 If you do report problems then please try to send high quality
 feedback. If you don't provide vital information to help us track down
 the problem then you will probably be ignored.  All bug reports should
-be filed under the Samba 3.6 product in the project's Bugzilla
+be filed under the Samba 4.0 product in the project's Bugzilla
 database (https://bugzilla.samba.org/).
 
 
diff --git a/WHATSNEW4.txt b/WHATSNEW4.txt
deleted file mode 100644
index 78638331f5e..00000000000
--- a/WHATSNEW4.txt
+++ /dev/null
@@ -1,132 +0,0 @@
-What's new in Samba 4 alpha15
-=============================
-
-Samba 4 is the ambitious next version of the Samba suite that is being
-developed in parallel to the stable 3.x series. The main emphasis in
-this branch is support for the Active Directory logon protocols used
-by Windows 2000 and above.
-
-Samba4 alpha15 follows on from the alpha release series we have been
-publishing since September 2007.
-
-WARNINGS
-========
-
-Samba4 alpha15 is not a final Samba release.  That is more a reference
-to Samba4's lack of the features we expect you will need than a
-statement of code quality, but clearly it hasn't seen a broad
-deployment yet.  If you were to upgrade Samba3 (or indeed Windows) to
-Samba4, you would find many things work, but that other key features
-you may have relied on simply are not there yet.
-
-For example, while Samba 3 is an excellent member of a Active
-Directory domain, Samba4 is happier as a domain controller, and it is
-in this role where it has seen deployment into production.
-
-Samba4 is subjected to an awesome battery of tests on an
-automated basis, we have found Samba4 to be very stable in it's
-behaviour.  We have to recommend against upgrading production servers
-from Samba 3 to Samba 4 at this stage, because there may be the features on
-which you may rely that are not present, or the mapping of
-your configuration and user database may not be complete.
-
-If you are upgrading, or looking to develop, test or deploy Samba4, you should
-backup all configuration and data.
-
-NEW FEATURES
-============
-
-Samba4 supports the server-side of the Active Directory logon environment
-used by Windows 2000 and later, so we can do full domain join
-and domain logon operations with these clients.
-
-Our Domain Controller (DC) implementation includes our own built-in
-LDAP server and Kerberos Key Distribution Center (KDC) as well as the
-Samba3-like logon services provided over CIFS.  We correctly generate
-the infamous Kerberos PAC, and include it with the Kerberos tickets we
-issue.
-
-The new VFS features in Samba 4 adapts the filesystem on the server to
-match the Windows client semantics, allowing Samba 4 to better match
-windows behaviour and application expectations.  This includes file
-annotation information (in streams) and NT ACLs in particular.  The
-VFS is backed with an extensive automated test suite.
-
-A new scripting interface has been added to Samba 4, allowing
-Python programs to interface to Samba's internals.
-
-The Samba 4 architecture is based around an LDAP-like database that
-can use a range of modular backends.  One of the backends supports
-standards compliant LDAP servers (including OpenLDAP), and we are
-working on modules to map between AD-like behaviours and this backend.
-We are aiming for Samba 4 to be powerful frontend to large
-directories.
-
-CHANGES SINCE alpha14
-=====================
-
-For a list of changes since alpha 14, please see the git log.
-
-$ git clone git://git.samba.org/samba.git
-$ cd samba.git
-$ git log release-4-0-0alpha14..release-4-0-0alpha15
-
-CHANGES
-=======
-
-Those familiar with Samba 3 can find a list of user-visible changes
-since that release series in the NEWS file.
-
-KNOWN ISSUES
-============
-
-- Domain member support is in it's infancy, and is not comparable to
-  the support found in Samba3.
-
-- There is no printing support in the current release.
-
-- There is no NetBIOS browsing support in the current release
-
-- The Samba4 port of the CTDB clustering support is not yet complete
-
-- Clock Synchronisation is critical.  Many 'wrong password' errors are
-  actually due to Kerberos objecting to a clock skew between client
-  and server.  (The NTP work in the previous alphas are partly to assist
-  with this problem).
-
-- The DRS replication code fails, and is very new
-
-- Users upgrading existing databases to Samba4 should carefully
-  consult upgrading-samba4.txt.  We have made a number of changes in
-  this release that should make it easier to upgrade in future.
-  Btw: there exists also a script under the "setup" directory of the
-  source distribution called "upgrade_from_s3" which should allow a step-up
-  from Samba3 to Samba4. It's not included yet in the binary distributions
-  since it's completely experimental!
-
-RUNNING Samba4
-==============
-
-A short guide to setting up Samba 4 can be found on the wiki:
-
-  http://wiki.samba.org/index.php/Samba4/HOWTO
-
-DEVELOPMENT and FEEDBACK
-========================
-
-We need your help! Projects as Samba 4 live from the community feedback. If you
-provide expressive bug reports, some documentation snippets on the wiki or some
-real code patches - all is appreciated if it meets our quality criterias. Here
-you can find further references:
-
-Bugs can be filed at https://bugzilla.samba.org/ but please be aware
-that many features are simply not expected to work at this stage.
-
-The Samba Wiki at http://wiki.samba.org should detail some of these
-development plans.
-
-Development and general discussion about Samba 4 happens mainly on
-the #samba-technical IRC channel (on irc.freenode.net) and
-the samba-technical mailing list (see http://lists.samba.org/ for
-details).
-
diff --git a/auth/auth_sam_reply.h b/auth/auth_sam_reply.h
index dea6501257b..bd92872009d 100644
--- a/auth/auth_sam_reply.h
+++ b/auth/auth_sam_reply.h
@@ -1,3 +1,25 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   Convert a server info struct into the form for PAC and NETLOGON replies
+
+   Copyright (C) Andrew Bartlett <abartlet@samba.org> 2004
+   Copyright (C) Stefan Metzmacher <metze@samba.org>  2005
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef __AUTH_AUTH_SAM_REPLY_H__
 #define __AUTH_AUTH_SAM_REPLY_H__
 
diff --git a/auth/common_auth.h b/auth/common_auth.h
index c64b46c7cf0..b2db23c7200 100644
--- a/auth/common_auth.h
+++ b/auth/common_auth.h
@@ -17,6 +17,9 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#ifndef AUTH_COMMON_AUTH_H
+#define AUTH_COMMON_AUTH_H
+
 #define USER_INFO_CASE_INSENSITIVE_USERNAME 0x01 /* username may be in any case */
 #define USER_INFO_CASE_INSENSITIVE_PASSWORD 0x02 /* password may be in any case */
 #define USER_INFO_DONT_CHECK_UNIX_ACCOUNT   0x04 /* don't check unix account status */
@@ -59,3 +62,5 @@ struct auth_usersupplied_info
 	} password;
 	uint32_t flags;
 };
+
+#endif
diff --git a/auth/kerberos/gssapi_pac.c b/auth/kerberos/gssapi_pac.c
new file mode 100644
index 00000000000..70bc9e576a0
--- /dev/null
+++ b/auth/kerberos/gssapi_pac.c
@@ -0,0 +1,152 @@
+/*
+   Unix SMB/CIFS implementation.
+   kerberos authorization data (PAC) utility library
+   Copyright (C) Andrew Bartlett <abartlet@samba.org> 2011
+   Copyright (C) Simo Sorce 2010.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#ifdef HAVE_KRB5
+
+#include "libcli/auth/krb5_wrap.h"
+
+#if 0
+/* FIXME - need proper configure/waf test
+ * to determine if gss_mech_krb5 and friends
+ * exist. JRA.
+ */
+/*
+ * These are not exported by Solaris -lkrb5
+ * Maybe move to libreplace somewhere?
+ */
+static const gss_OID_desc krb5_gss_oid_array[] = {
+	/* this is the official, rfc-specified OID */
+	{ 9, "\052\206\110\206\367\022\001\002\002" },
+	/* this is the pre-RFC mech OID */
+	{ 5, "\053\005\001\005\002" },
+	/* this is the unofficial, incorrect mech OID emitted by MS */
+	{ 9, "\052\206\110\202\367\022\001\002\002" },
+	{ 0, 0 }
+};
+
+const gss_OID_desc * const gss_mech_krb5              = krb5_gss_oid_array+0;
+const gss_OID_desc * const gss_mech_krb5_old          = krb5_gss_oid_array+1;
+const gss_OID_desc * const gss_mech_krb5_wrong        = krb5_gss_oid_array+2;
+#endif
+
+/* The Heimdal OID for getting the PAC */
+#define EXTRACT_PAC_AUTHZ_DATA_FROM_SEC_CONTEXT_OID_LENGTH 8
+/*					                EXTRACTION OID		   AUTHZ ID */
+#define EXTRACT_PAC_AUTHZ_DATA_FROM_SEC_CONTEXT_OID "\x2a\x85\x70\x2b\x0d\x03" "\x81\x00"
+
+NTSTATUS gssapi_obtain_pac_blob(TALLOC_CTX *mem_ctx,
+				gss_ctx_id_t gssapi_context,
+				gss_name_t gss_client_name,
+				DATA_BLOB *pac_blob)
+{
+	NTSTATUS status;
+	OM_uint32 gss_maj, gss_min;
+#ifdef HAVE_GSS_GET_NAME_ATTRIBUTE
+	gss_buffer_desc pac_buffer;
+	gss_buffer_desc pac_display_buffer;
+	gss_buffer_desc pac_name = {
+		.value = discard_const("urn:mspac:"),
+		.length = sizeof("urn:mspac:")-1
+	};
+	int more = -1;
+	int authenticated = false;
+	int complete = false;
+
+	gss_maj = gss_get_name_attribute(
+		&gss_min, gss_client_name, &pac_name,
+		&authenticated, &complete,
+		&pac_buffer, &pac_display_buffer, &more);
+
+	if (gss_maj != 0) {
+		DEBUG(0, ("obtaining PAC via GSSAPI gss_get_name_attribute failed: %s\n",
+			  gssapi_error_string(mem_ctx, gss_maj, gss_min, gss_mech_krb5)));
+		return NT_STATUS_ACCESS_DENIED;
+	} else if (authenticated && complete) {
+		/* The PAC blob is returned directly */
+		*pac_blob = data_blob_talloc(mem_ctx, pac_buffer.value,
+					    pac_buffer.length);
+
+		if (!pac_blob->data) {
+			status = NT_STATUS_NO_MEMORY;
+		} else {
+			status = NT_STATUS_OK;
+		}
+
+		gss_maj = gss_release_buffer(&gss_min, &pac_buffer);
+		gss_maj = gss_release_buffer(&gss_min, &pac_display_buffer);
+		return status;
+	} else {
+		DEBUG(0, ("obtaining PAC via GSSAPI failed: authenticated: %s, complete: %s, more: %s\n",
+			  authenticated ? "true" : "false",
+			  complete ? "true" : "false",
+			  more ? "true" : "false"));
+		return NT_STATUS_ACCESS_DENIED;
+	}
+
+#elif defined(HAVE_GSS_INQUIRE_SEC_CONTEXT_BY_OID)
+	gss_OID_desc pac_data_oid = {
+		.elements = discard_const(EXTRACT_PAC_AUTHZ_DATA_FROM_SEC_CONTEXT_OID),
+		.length = EXTRACT_PAC_AUTHZ_DATA_FROM_SEC_CONTEXT_OID_LENGTH
+	};
+
+	gss_buffer_set_t set = GSS_C_NO_BUFFER_SET;
+
+	/* If we didn't have the routine to get a verified, validated
+	 * PAC (supplied only by MIT at the time of writing), then try
+	 * with the Heimdal OID (fetches the PAC directly and always
+	 * validates) */
+	gss_maj = gss_inquire_sec_context_by_oid(
+				&gss_min, gssapi_context,
+				&pac_data_oid, &set);
+
+	/* First check for the error MIT gives for an unknown OID */
+	if (gss_maj == GSS_S_UNAVAILABLE) {
+		DEBUG(1, ("unable to obtain a PAC against this GSSAPI library.  "
+			  "GSSAPI secured connections are available only with Heimdal or MIT Kerberos >= 1.8\n"));
+	} else if (gss_maj != 0) {
+		DEBUG(2, ("obtaining PAC via GSSAPI gss_inqiure_sec_context_by_oid (Heimdal OID) failed: %s\n",
+			  gssapi_error_string(mem_ctx, gss_maj, gss_min, gss_mech_krb5)));
+	} else {
+		if (set == GSS_C_NO_BUFFER_SET) {
+			DEBUG(0, ("gss_inquire_sec_context_by_oid returned unknown "
+				  "data in results.\n"));
+			return NT_STATUS_INTERNAL_ERROR;
+		}
+
+		/* The PAC blob is returned directly */
+		*pac_blob = data_blob_talloc(mem_ctx, set->elements[0].value,
+					    set->elements[0].length);
+		if (!pac_blob->data) {
+			status = NT_STATUS_NO_MEMORY;
+		} else {
+			status = NT_STATUS_OK;
+		}
+
+		gss_maj = gss_release_buffer_set(&gss_min, &set);
+		return status;
+	}
+#else
+	DEBUG(1, ("unable to obtain a PAC against this GSSAPI library.  "
+		  "GSSAPI secured connections are available only with Heimdal or MIT Kerberos >= 1.8\n"));
+#endif
+	return NT_STATUS_ACCESS_DENIED;
+}
+#endif
diff --git a/libcli/auth/kerberos_pac.c b/auth/kerberos/kerberos_pac.c
index 6a41eb1aceb..79d51b26453 100644
--- a/libcli/auth/kerberos_pac.c
+++ b/auth/kerberos/kerberos_pac.c
@@ -279,61 +279,67 @@ NTSTATUS kerberos_decode_pac(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	/* verify by service_key */
-	ret = check_pac_checksum(mem_ctx,
-				 modified_pac_blob, srv_sig_ptr,
-				 context,
-				 service_keyblock);
-	if (ret) {
-		DEBUG(1, ("PAC Decode: Failed to verify the service "
-			  "signature: %s\n", error_message(ret)));
-		return NT_STATUS_ACCESS_DENIED;
-	}
-
-	if (krbtgt_keyblock) {
-		/* verify the service key checksum by krbtgt_key */
+	if (service_keyblock) {
+		/* verify by service_key */
 		ret = check_pac_checksum(mem_ctx,
-					 srv_sig_ptr->signature, kdc_sig_ptr,
-					 context, krbtgt_keyblock);
+					 modified_pac_blob, srv_sig_ptr,
+					 context,
+					 service_keyblock);
 		if (ret) {
-			DEBUG(1, ("PAC Decode: Failed to verify the KDC signature: %s\n",
-				  smb_get_krb5_error_message(context, ret, mem_ctx)));
+			DEBUG(1, ("PAC Decode: Failed to verify the service "
+				  "signature: %s\n", error_message(ret)));
 			return NT_STATUS_ACCESS_DENIED;
 		}
-	}
 
-	/* Convert to NT time, so as not to loose accuracy in comparison */
-	unix_to_nt_time(&tgs_authtime_nttime, tgs_authtime);
-
-	if (tgs_authtime_nttime != logon_name->logon_time) {
-		DEBUG(2, ("PAC Decode: "
-			  "Logon time mismatch between ticket and PAC!\n"));
-		DEBUG(2, ("PAC Decode: PAC: %s\n",
-			  nt_time_string(mem_ctx, logon_name->logon_time)));
-		DEBUG(2, ("PAC Decode: Ticket: %s\n",
-			  nt_time_string(mem_ctx, tgs_authtime_nttime)));
-		return NT_STATUS_ACCESS_DENIED;
+		if (krbtgt_keyblock) {
+			/* verify the service key checksum by krbtgt_key */
+			ret = check_pac_checksum(mem_ctx,
+						 srv_sig_ptr->signature, kdc_sig_ptr,
+						 context, krbtgt_keyblock);
+			if (ret) {
+				DEBUG(1, ("PAC Decode: Failed to verify the KDC signature: %s\n",
+					  smb_get_krb5_error_message(context, ret, mem_ctx)));
+				return NT_STATUS_ACCESS_DENIED;
+			}
+		}
 	}
 
-	ret = smb_krb5_parse_name_norealm(context,
-					  logon_name->account_name,
-					  &client_principal_pac);
-	if (ret) {
-		DEBUG(2, ("Could not parse name from PAC: [%s]:%s\n",
-			  logon_name->account_name, error_message(ret)));
-		return NT_STATUS_INVALID_PARAMETER;
+	if (tgs_authtime) {
+		/* Convert to NT time, so as not to loose accuracy in comparison */
+		unix_to_nt_time(&tgs_authtime_nttime, tgs_authtime);
+
+		if (tgs_authtime_nttime != logon_name->logon_time) {
+			DEBUG(2, ("PAC Decode: "
+				  "Logon time mismatch between ticket and PAC!\n"));
+			DEBUG(2, ("PAC Decode: PAC: %s\n",
+				  nt_time_string(mem_ctx, logon_name->logon_time)));
+			DEBUG(2, ("PAC Decode: Ticket: %s\n",
+				  nt_time_string(mem_ctx, tgs_authtime_nttime)));
+			return NT_STATUS_ACCESS_DENIED;
+		}
 	}
 
-	bool_ret = smb_krb5_principal_compare_any_realm(context,
-							client_principal,
-							client_principal_pac);
+	if (client_principal) {
+		ret = smb_krb5_parse_name_norealm(context,
+						  logon_name->account_name,
+						  &client_principal_pac);
+		if (ret) {
+			DEBUG(2, ("Could not parse name from PAC: [%s]:%s\n",
+				  logon_name->account_name, error_message(ret)));
+			return NT_STATUS_INVALID_PARAMETER;
+		}
+
+		bool_ret = smb_krb5_principal_compare_any_realm(context,
+								client_principal,
+								client_principal_pac);
 
-	krb5_free_principal(context, client_principal_pac);
+		krb5_free_principal(context, client_principal_pac);
 
-	if (!bool_ret) {
-		DEBUG(2, ("Name in PAC [%s] does not match principal name "
-			  "in ticket\n", logon_name->account_name));
-		return NT_STATUS_ACCESS_DENIED;
+		if (!bool_ret) {
+			DEBUG(2, ("Name in PAC [%s] does not match principal name "
+				  "in ticket\n", logon_name->account_name));
+			return NT_STATUS_ACCESS_DENIED;
+		}
 	}
 
 	DEBUG(3,("Found account name from PAC: %s [%s]\n",
diff --git a/auth/kerberos/wscript_build b/auth/kerberos/wscript_build
new file mode 100644
index 00000000000..10707708f09
--- /dev/null
+++ b/auth/kerberos/wscript_build
@@ -0,0 +1,3 @@
+bld.SAMBA_SUBSYSTEM('KRB5_PAC',
+                    source='gssapi_pac.c kerberos_pac.c',
+                    deps='gssapi_krb5 krb5 ndr-krb5pac')
diff --git a/auth/wscript_build b/auth/wscript_build
index 75580fd2991..0472a20a012 100644
--- a/auth/wscript_build
+++ b/auth/wscript_build
@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 
-bld.SAMBA_SUBSYSTEM('auth_sam_reply',
-	source='auth_sam_reply.c',
-        deps='talloc',
-	autoproto='auth_sam_reply.h'
-	)
+bld.SAMBA_LIBRARY('auth_sam_reply',
+                  source='auth_sam_reply.c',
+                  deps='talloc security samba-util',
+                  autoproto='auth_sam_reply.h',
+                  private_library=True
+                  )
diff --git a/buildtools/scripts/Makefile.waf b/buildtools/scripts/Makefile.waf
index 716ab932706..5fc939c99e0 100644
--- a/buildtools/scripts/Makefile.waf
+++ b/buildtools/scripts/Makefile.waf
@@ -62,7 +62,7 @@ ctags:
 	$(WAF) ctags
 
 bin/%:: FORCE
-	$(WAF) --targets=`basename $@`
+	$(WAF) --targets=$@
 FORCE:
 
 configure: autogen-waf.sh BUILDTOOLS/scripts/configure.waf
diff --git a/buildtools/wafsamba/pkgconfig.py b/buildtools/wafsamba/pkgconfig.py
index 09bfcb9c6be..8a3f807dc55 100644
--- a/buildtools/wafsamba/pkgconfig.py
+++ b/buildtools/wafsamba/pkgconfig.py
@@ -53,6 +53,7 @@ def PKG_CONFIG_FILES(bld, pc_files, vnum=None):
                                 rule=subst_at_vars,
                                 source=f+'.in',
                                 target=f)
+        bld.add_manual_dependency(bld.path.find_or_declare(f), bld.env['PREFIX'])
         t.vars = []
         if t.env.RPATH_ON_INSTALL:
             t.env.LIB_RPATH = t.env.RPATH_ST % t.env.LIBDIR
diff --git a/buildtools/wafsamba/samba3.py b/buildtools/wafsamba/samba3.py
index cb459ad115a..d77cd4d41e9 100644
--- a/buildtools/wafsamba/samba3.py
+++ b/buildtools/wafsamba/samba3.py
@@ -51,14 +51,18 @@ def s3_fix_kwargs(bld, kwargs):
     s3reldir = os_path_relpath(s3dir, bld.curdir)
 
     # the extra_includes list is relative to the source3 directory
-    extra_includes = [ '.', 'include', 'lib' ]
+    extra_includes = [ '.', 'include', 'lib', '../lib/tdb_compat' ]
     if bld.env.use_intree_heimdal:
         extra_includes += [ '../source4/heimdal/lib/com_err',
                             '../source4/heimdal/lib/gssapi',
                             '../source4/heimdal_build' ]
 
-    if not bld.CONFIG_SET('USING_SYSTEM_TDB'):
-        extra_includes += [ '../lib/tdb/include' ]
+    if bld.CONFIG_SET('BUILD_TDB2'):
+        if not bld.CONFIG_SET('USING_SYSTEM_TDB2'):
+            extra_includes += [ '../lib/tdb2' ]
+    else:
+        if not bld.CONFIG_SET('USING_SYSTEM_TDB'):
+            extra_includes += [ '../lib/tdb/include' ]
 
     if not bld.CONFIG_SET('USING_SYSTEM_TEVENT'):
         extra_includes += [ '../lib/tevent' ]
@@ -89,13 +93,13 @@ def s3_fix_kwargs(bld, kwargs):
 
 def SAMBA3_LIBRARY(bld, name, *args, **kwargs):
 	s3_fix_kwargs(bld, kwargs)
-	kwargs['allow_undefined_symbols'] = True
 	return bld.SAMBA_LIBRARY(name, *args, **kwargs)
 Build.BuildContext.SAMBA3_LIBRARY = SAMBA3_LIBRARY
 
 def SAMBA3_MODULE(bld, name, *args, **kwargs):
 	s3_fix_kwargs(bld, kwargs)
-	kwargs['allow_undefined_symbols'] = True
+        if not 'allow_undefined_symbols' in kwargs:
+            kwargs['allow_undefined_symbols'] = True
 	return bld.SAMBA_MODULE(name, *args, **kwargs)
 Build.BuildContext.SAMBA3_MODULE = SAMBA3_MODULE
 
diff --git a/buildtools/wafsamba/samba_autoconf.py b/buildtools/wafsamba/samba_autoconf.py
index 174ca142102..207d7320b95 100644
--- a/buildtools/wafsamba/samba_autoconf.py
+++ b/buildtools/wafsamba/samba_autoconf.py
@@ -597,7 +597,7 @@ def SAMBA_CONFIG_H(conf, path=None):
 
     if Options.options.developer:
         # we add these here to ensure that -Wstrict-prototypes is not set during configure
-        conf.ADD_CFLAGS('-Wall -g -Wshadow -Wstrict-prototypes -Wpointer-arith -Wcast-align -Wwrite-strings -Werror-implicit-function-declaration -Wformat=2 -Wno-format-y2k -Wmissing-prototypes',
+        conf.ADD_CFLAGS('-Wall -g -Wshadow -Wstrict-prototypes -Wpointer-arith -Wcast-align -Wwrite-strings -Werror-implicit-function-declaration -Wformat=2 -Wno-format-y2k -Wmissing-prototypes -fno-common',
                         testflags=True)
         if os.getenv('TOPLEVEL_BUILD'):
             conf.ADD_CFLAGS('-Wcast-qual', testflags=True)
diff --git a/buildtools/wafsamba/samba_deps.py b/buildtools/wafsamba/samba_deps.py
index adeb3645ce1..0bd2c6cf67b 100644
--- a/buildtools/wafsamba/samba_deps.py
+++ b/buildtools/wafsamba/samba_deps.py
@@ -89,6 +89,10 @@ def build_dependencies(self):
             for f in self.env.undefined_ldflags:
                 self.ldflags.remove(f)
 
+        if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ignore_ldflags:
+            for f in self.env.undefined_ignore_ldflags:
+                self.ldflags.append(f)
+
         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
               self.sname, self.uselib, self.uselib_local, self.add_objects)
 
diff --git a/buildtools/wafsamba/samba_install.py b/buildtools/wafsamba/samba_install.py
index d755d010784..26d0a371e6d 100644
--- a/buildtools/wafsamba/samba_install.py
+++ b/buildtools/wafsamba/samba_install.py
@@ -137,9 +137,9 @@ def install_library(self):
                    os.path.join(self.path.abspath(bld.env), inst_name))
     if install_link and install_link != install_name:
         # and the symlink if needed
-        bld.symlink_as(os.path.join(install_path, install_link), install_name)
+        bld.symlink_as(os.path.join(install_path, install_link), os.path.basename(install_name))
     if dev_link:
-        bld.symlink_as(os.path.join(install_path, dev_link), install_name)
+        bld.symlink_as(os.path.join(install_path, dev_link), os.path.basename(install_name))
 
 
 @feature('cshlib')
diff --git a/buildtools/wafsamba/samba_patterns.py b/buildtools/wafsamba/samba_patterns.py
index 37ef4198a68..f064608f3a3 100644
--- a/buildtools/wafsamba/samba_patterns.py
+++ b/buildtools/wafsamba/samba_patterns.py
@@ -10,7 +10,7 @@ def write_version_header(task):
     src = task.inputs[0].srcpath(task.env)
     tgt = task.outputs[0].bldpath(task.env)
 
-    version = samba_version_file(src, task.env.srcdir, env=task.env)
+    version = samba_version_file(src, task.env.srcdir, env=task.env, is_install=task.env.is_install)
     string = str(version)
 
     f = open(tgt, 'w')
@@ -26,4 +26,5 @@ def SAMBA_MKVERSION(bld, target):
                             source= 'VERSION',
                             target=target,
                             always=True)
+    t.env.is_install = bld.is_install
 Build.BuildContext.SAMBA_MKVERSION = SAMBA_MKVERSION
diff --git a/buildtools/wafsamba/samba_version.py b/buildtools/wafsamba/samba_version.py
index 0b0c159f55f..0c39ed44ecd 100644
--- a/buildtools/wafsamba/samba_version.py
+++ b/buildtools/wafsamba/samba_version.py
@@ -90,7 +90,7 @@ def git_version_summary(path, env=None):
 
 class SambaVersion(object):
 
-    def __init__(self, version_dict, path, env=None):
+    def __init__(self, version_dict, path, env=None, is_install=True):
         '''Determine the version number of samba
 
 See VERSION for the format.  Entries on that file are 
@@ -152,7 +152,10 @@ also accepted as dictionary entries here
             SAMBA_VERSION_STRING += ("rc%u" % self.RC_RELEASE)
 
         if self.IS_SNAPSHOT:
-            if os.path.exists(os.path.join(path, ".git")):
+            if not is_install:
+                suffix = "DEVELOPERBUILD"
+                self.vcs_fields = {}
+            elif os.path.exists(os.path.join(path, ".git")):
                 suffix, self.vcs_fields = git_version_summary(path, env=env)
             elif os.path.exists(os.path.join(path, ".bzr")):
                 suffix, self.vcs_fields = bzr_version_summary(path)
@@ -234,7 +237,7 @@ also accepted as dictionary entries here
         return string
 
 
-def samba_version_file(version_file, path, env=None):
+def samba_version_file(version_file, path, env=None, is_install=True):
     '''Parse the version information from a VERSION file'''
 
     f = open(version_file, 'r')
@@ -254,16 +257,16 @@ def samba_version_file(version_file, path, env=None):
             print("Failed to parse line %s from %s" % (line, version_file))
             raise
 
-    return SambaVersion(version_dict, path, env=env)
+    return SambaVersion(version_dict, path, env=env, is_install=is_install)
 
 
 
-def load_version(env=None):
+def load_version(env=None, is_install=True):
     '''load samba versions either from ./VERSION or git
     return a version object for detailed breakdown'''
     if not env:
         env = samba_utils.LOAD_ENVIRONMENT()
 
-    version = samba_version_file("./VERSION", ".", env)
+    version = samba_version_file("./VERSION", ".", env, is_install=is_install)
     Utils.g_module.VERSION = version.STRING
     return version
diff --git a/buildtools/wafsamba/samba_wildcard.py b/buildtools/wafsamba/samba_wildcard.py
index 5bf12672a95..75ab5defcdc 100644
--- a/buildtools/wafsamba/samba_wildcard.py
+++ b/buildtools/wafsamba/samba_wildcard.py
@@ -17,7 +17,7 @@ def run_task(t, k):
 def run_named_build_task(cmd):
 	'''run a named build task, matching the cmd name using fnmatch
 	wildcards against inputs and outputs of all build tasks'''
-	bld = fake_build_environment()
+	bld = fake_build_environment(info=False)
 	found = False
 	cwd_node = bld.root.find_dir(os.getcwd())
 	top_node = bld.root.find_dir(bld.srcnode.abspath())
@@ -50,6 +50,28 @@ def run_named_build_task(cmd):
 		raise Utils.WafError("Unable to find build target matching %s" % cmd)
 
 
+def rewrite_compile_targets():
+	'''cope with the bin/ form of compile target'''
+	if not Options.options.compile_targets:
+		return
+
+	bld = fake_build_environment(info=False)
+	targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
+	tlist = []
+
+	for t in Options.options.compile_targets.split(','):
+		if not os.path.islink(t):
+			tlist.append(t)
+			continue
+		link = os.readlink(t)
+		list = link.split('/')
+		for name in [list[-1], '/'.join(list[-2:])]:
+			if name in targets:
+				tlist.append(name)
+				continue
+	Options.options.compile_targets = ",".join(tlist)
+
+
 
 def wildcard_main(missing_cmd_fn):
 	'''this replaces main from Scripting, allowing us to override the
@@ -60,6 +82,9 @@ def wildcard_main(missing_cmd_fn):
 	   '''
 	Scripting.commands = Options.arg_line[:]
 
+	# rewrite the compile targets to cope with the bin/xx form
+	rewrite_compile_targets()
+
 	while Scripting.commands:
 		x = Scripting.commands.pop(0)
 
@@ -99,7 +124,7 @@ def wildcard_main(missing_cmd_fn):
 
 
 
-def fake_build_environment():
+def fake_build_environment(info=True, flush=False):
 	"""create all the tasks for the project, but do not run the build
 	return the build context in use"""
 	bld = getattr(Utils.g_module, 'build_context', Utils.Context)()
@@ -119,10 +144,12 @@ def fake_build_environment():
 	bld.load_dirs(proj[SRCDIR], proj[BLDDIR])
 	bld.load_envs()
 
-	Logs.info("Waf: Entering directory `%s'" % bld.bldnode.abspath())
+	if info:
+		Logs.info("Waf: Entering directory `%s'" % bld.bldnode.abspath())
 	bld.add_subdirs([os.path.split(Utils.g_module.root_path)[0]])
 
 	bld.pre_build()
-	bld.flush()
+	if flush:
+		bld.flush()
 	return bld
 
diff --git a/buildtools/wafsamba/symbols.py b/buildtools/wafsamba/symbols.py
index ed195176e65..dfab71c2c43 100644
--- a/buildtools/wafsamba/symbols.py
+++ b/buildtools/wafsamba/symbols.py
@@ -447,7 +447,7 @@ def check_dependencies(bld, t):
 
     deps = set(t.samba_deps)
     for d in t.samba_deps:
-        if targets[d] in [ 'EMPTY', 'DISABLED', 'SYSLIB' ]:
+        if targets[d] in [ 'EMPTY', 'DISABLED', 'SYSLIB', 'GENERATOR' ]:
             continue
         bld.ASSERT(d in bld.env.public_symbols, "Failed to find symbol list for dependency '%s'" % d)
         diff = remaining.intersection(bld.env.public_symbols[d])
diff --git a/buildtools/wafsamba/wafsamba.py b/buildtools/wafsamba/wafsamba.py
index 2a1c82a3077..d76afc49847 100644
--- a/buildtools/wafsamba/wafsamba.py
+++ b/buildtools/wafsamba/wafsamba.py
@@ -628,17 +628,6 @@ def ENABLE_TIMESTAMP_DEPENDENCIES(conf):
     Utils.h_file = h_file
 
 
-
-t = Task.simple_task_type('copy_script', 'rm -f "${LINK_TARGET}" && ln -s "${SRC[0].abspath(env)}" ${LINK_TARGET}',
-                          shell=True, color='PINK', ext_in='.bin')
-t.quiet = True
-
-@feature('copy_script')
-@before('apply_link')
-def copy_script(self):
-    tsk = self.create_task('copy_script', self.allnodes[0])
-    tsk.env.TARGET = self.target
-
 def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
     '''used to copy scripts from the source tree into the build directory
        for use by selftest'''
@@ -653,15 +642,17 @@ def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
         target = os.path.join(installdir, iname)
         tgtdir = os.path.dirname(os.path.join(bld.srcnode.abspath(bld.env), '..', target))
         mkdir_p(tgtdir)
-        t = bld(features='copy_script',
-                source       = s,
-                target       = target,
-                always       = True,
-                install_path = None)
-        t.env.LINK_TARGET = target
-
+        link_src = os.path.normpath(os.path.join(bld.curdir, s))
+        link_dst = os.path.join(tgtdir, os.path.basename(iname))
+        if os.path.islink(link_dst) and os.readlink(link_dst) == link_src:
+            continue
+        if os.path.exists(link_dst):
+            os.unlink(link_dst)
+        Logs.info("symlink: %s -> %s/%s" % (s, installdir, iname))
+        os.symlink(link_src, link_dst)
 Build.BuildContext.SAMBA_SCRIPT = SAMBA_SCRIPT
 
+
 def copy_and_fix_python_path(task):
     pattern='sys.path.insert(0, "bin/python")'
     if task.env["PYTHONARCHDIR"] in sys.path and task.env["PYTHONDIR"] in sys.path:
@@ -701,6 +692,8 @@ def install_file(bld, destdir, file, chmod=MODE_644, flat=False,
                             rule=copy_and_fix_python_path,
                             source=file,
                             target=inst_file)
+        bld.add_manual_dependency(bld.path.find_or_declare(inst_file), bld.env["PYTHONARCHDIR"])
+        bld.add_manual_dependency(bld.path.find_or_declare(inst_file), bld.env["PYTHONDIR"])
         file = inst_file
     if base_name:
         file = os.path.join(base_name, file)
diff --git a/docs-xml/manpages-3/idmap_ad.8.xml b/docs-xml/manpages-3/idmap_ad.8.xml
index 3ecb07e590c..fbadaf2d55c 100644
--- a/docs-xml/manpages-3/idmap_ad.8.xml
+++ b/docs-xml/manpages-3/idmap_ad.8.xml
@@ -34,7 +34,7 @@
 	to configure it separately for each domain for which one wants
 	to use it, using disjoint ranges. One usually needs to configure
 	a writeable default idmap range, using for example the
-	<parameter>tdb</parameter> or <parameter>ldap</parameter>)
+	<parameter>tdb</parameter> or <parameter>ldap</parameter>
 	backend, in order to be able to map the BUILTIN sids and
 	possibly other trusted domains. The writeable default config
 	is also needed in order to be able to create group mappings.
@@ -85,9 +85,8 @@
 
 	<programlisting>
 	[global]
-	idmap backend = tdb
-	idmap uid = 1000000-1999999
-	idmap gid = 1000000-1999999
+	idmap config * : backend = tdb
+	idmap config * : range = 1000000-1999999
 
 	idmap config CORP : backend  = ad
 	idmap config CORP : range = 1000-999999
diff --git a/docs-xml/manpages-3/idmap_adex.8.xml b/docs-xml/manpages-3/idmap_adex.8.xml
index 7349caa61e3..16d12cdbbc9 100644
--- a/docs-xml/manpages-3/idmap_adex.8.xml
+++ b/docs-xml/manpages-3/idmap_adex.8.xml
@@ -66,9 +66,8 @@
 
 	<programlisting>
 	[global]
-	idmap backend = adex
-	idmap uid = 1000-4000000000
-	idmap gid = 1000-4000000000
+	idmap config * : backend = adex
+	idmap config * : range = 1000-4000000000
 
 	winbind nss info = adex
 	winbind normalize names = yes
diff --git a/docs-xml/manpages-3/idmap_autorid.8.xml b/docs-xml/manpages-3/idmap_autorid.8.xml
index 38790ea2b04..b5a9bde2e66 100644
--- a/docs-xml/manpages-3/idmap_autorid.8.xml
+++ b/docs-xml/manpages-3/idmap_autorid.8.xml
@@ -88,9 +88,8 @@
 	workgroup = CUSTOMER
 	realm = CUSTOMER.COM
 
-	idmap backend = autorid
-	idmap uid = 1000000-1999999
-	idmap gid = 1000000-1999999
+	idmap config * : backend = autorid
+	idmap config * : range = 1000000-1999999
 
 	</programlisting>
 
@@ -98,7 +97,7 @@
 		This example shows how to configure idmap_autorid as default
 		for all domains with a potentially large amount of users
 		plus a specific configuration for a trusted domain
-		that uses the SFU mapping scheme. Please note that idmap uid/gid
+		that uses the SFU mapping scheme. Please note that idmap 
 		ranges and sfu ranges are not allowed to overlap.
 	</para>
 
@@ -108,10 +107,9 @@
 	workgroup = CUSTOMER
 	realm = CUSTOMER.COM
 
-	idmap backend = autorid
-	autorid:rangesize = 1000000
-	idmap uid = 1000000-19999999
-	idmap gid = 1000000-19999999
+	idmap config * : backend = autorid
+	idmap config * : range = 1000000-19999999
+	idmap config * : rangesize = 1000000
 
 	idmap config TRUSTED : backend  = ad
 	idmap config TRUSTED : range    = 50000 - 99999
diff --git a/docs-xml/manpages-3/idmap_hash.8.xml b/docs-xml/manpages-3/idmap_hash.8.xml
index 2bbae713544..f3ec6a7bc25 100644
--- a/docs-xml/manpages-3/idmap_hash.8.xml
+++ b/docs-xml/manpages-3/idmap_hash.8.xml
@@ -52,9 +52,8 @@
 
 	<programlisting>
 	[global]
-	idmap backend = hash
-	idmap uid = 1000-4000000000
-	idmap gid = 1000-4000000000
+	idmap config * : backend = hash
+	idmap config * : range = 1000-4000000000
 
 	winbind nss info = hash
 	winbind normalize names = yes
diff --git a/docs-xml/manpages-3/idmap_ldap.8.xml b/docs-xml/manpages-3/idmap_ldap.8.xml
index e3588b95bcf..2c0fcfd17c6 100644
--- a/docs-xml/manpages-3/idmap_ldap.8.xml
+++ b/docs-xml/manpages-3/idmap_ldap.8.xml
@@ -27,26 +27,9 @@
 	<para>
 	In contrast to read only backends like idmap_rid, it is an allocating
 	backend: This means that it needs to allocate new user and group IDs in
-	order to create new mappings. The allocator can be provided by the
-	idmap_ldap backend itself or by any other allocating backend like
-	idmap_tdb or idmap_tdb2. This is configured with the
-	parameter <parameter>idmap alloc backend</parameter>.
+	order to create new mappings.
 	</para>
 
-	<para>
-	Note that in order for this (or any other allocating) backend to
-	function at all, the default backend needs to be writeable.
-	The ranges used for uid and gid allocation are the default ranges
-	configured by &quot;idmap uid&quot; and &quot;idmap gid&quot;.
-	</para>
-
-	<para>
-	Furthermore, since there is only one global allocating backend
-	responsible for all domains using writeable idmap backends,
-	any explicitly configured domain with idmap backend ldap
-	should have the same range as the default range, since it needs
-	to use the global uid / gid allocator. See the example below.
-	</para>
 </refsynopsisdiv>
 
 <refsect1>
@@ -56,7 +39,7 @@
 		<varlistentry>
 		<term>ldap_base_dn = DN</term>
 		<listitem><para>
-			Defines the directory base suffix to use when searching for
+			Defines the directory base suffix to use for
 			SID/uid/gid mapping entries.  If not defined, idmap_ldap will default
 			to using the &quot;ldap idmap suffix&quot; option from smb.conf.
 		</para></listitem>
@@ -65,15 +48,21 @@
 		<varlistentry>
 		<term>ldap_user_dn = DN</term>
 		<listitem><para>
-			Defines the user DN to be used for authentication. If absent an
-			anonymous bind will be performed.
+			Defines the user DN to be used for authentication.
+			The secret for authenticating this user should be
+			stored with net idmap secret
+			(see <citerefentry><refentrytitle>net</refentrytitle>
+			<manvolnum>8</manvolnum></citerefentry>).
+			If absent, the ldap credentials from the ldap passdb configuration
+			are used, and if these are also absent, an anonymous
+			bind will be performed as last fallback.
 		</para></listitem>
 		</varlistentry>
 
 		<varlistentry>
 		<term>ldap_url = ldap://server/</term>
 		<listitem><para>
-			Specifies the LDAP server to use when searching for existing
+			Specifies the LDAP server to use for
 			SID/uid/gid map entries. If not defined, idmap_ldap will
 			assume that ldap://localhost/ should be used.
 		</para></listitem>
@@ -84,64 +73,50 @@
                 <listitem><para>
 			Defines the available matching uid and gid range for which the
 			backend is authoritative.
-			If the parameter is absent, Winbind fails over to use the
-			&quot;idmap uid&quot; and &quot;idmap gid&quot; options
-			from smb.conf.
                 </para></listitem>
                 </varlistentry>
 	</variablelist>
 </refsect1>
 
 <refsect1>
-	<title>IDMAP ALLOC OPTIONS</title>
-
-	<variablelist>
-		<varlistentry>
-		<term>ldap_base_dn = DN</term>
-		<listitem><para>
-			Defines the directory base suffix under which new SID/uid/gid mapping
-			entries should be stored.  If not defined, idmap_ldap will default
-			to using the &quot;ldap idmap suffix&quot; option from smb.conf.
-		</para></listitem>
-		</varlistentry>
-
-		<varlistentry>
-		<term>ldap_user_dn = DN</term>
-		<listitem><para>
-			Defines the user DN to be used for authentication. If absent an
-			anonymous bind will be performed.
-		</para></listitem>
-		</varlistentry>
-
-		<varlistentry>
-		<term>ldap_url = ldap://server/</term>
-		<listitem><para>
-			Specifies the LDAP server to which modify/add/delete requests should
-			be sent.  If not defined, idmap_ldap will assume that ldap://localhost/
-			should be used.
-		</para></listitem>
-		</varlistentry>
-	</variablelist>
-</refsect1>
-
-<refsect1>
 	<title>EXAMPLES</title>
 
 	<para>
-	The follow sets of a LDAP configuration which uses two LDAP
-	directories, one for storing the ID mappings and one for retrieving
-	new IDs.
+	The following example shows how an ldap directory is used as the 
+	default idmap backend. It also configures the idmap range and base 
+	directory suffix. The secret for the ldap_user_dn has to be set with
+	&quot;net idmap secret '*' password&quot;.
 	</para>
 
 	<programlisting>
 	[global]
-	idmap backend = ldap:ldap://localhost/
-	idmap uid = 1000000-1999999
-	idmap gid = 1000000-1999999
+	idmap config * : backend      = ldap
+	idmap config * : range        = 1000000-1999999
+	idmap config * : ldap_url     = ldap://localhost/
+	idmap config * : ldap_base_dn = ou=idmap,dc=example,dc=com
+	idmap config * : ldap_user_dn = cn=idmap_admin,dc=example,dc=com
+	</programlisting>
+
+	<para>
+	This example shows how ldap can be used as a readonly backend while
+	tdb is the default backend used to store the mappings.
+	It adds an explicit configuration for some domain DOM1, that
+	uses the ldap idmap backend. Note that a range disjoint from the
+	default range is used.
+	</para>
 
-	idmap alloc backend = ldap
-	idmap alloc config : ldap_url	= ldap://id-master/
-	idmap alloc config : ldap_base_dn = ou=idmap,dc=example,dc=com
+	<programlisting>
+	[global]
+	# "backend = tdb" is redundant here since it is the default
+	idmap config * : backend = tdb
+	idmap config * : range = 1000000-1999999
+
+	idmap config DOM1 : backend = ldap
+	idmap config DOM1 : range = 2000000-2999999
+	idmap config DOM1 : read only = yes
+	idmap config DOM1 : ldap_url = ldap://server/
+	idmap config DOM1 : ldap_base_dn = ou=idmap,dc=dom1,dc=example,dc=com
+	idmap config DOM1 : ldap_user_dn = cn=idmap_admin,dc=dom1,dc=example,dc=com
 	</programlisting>
 </refsect1>
 
diff --git a/docs-xml/manpages-3/idmap_nss.8.xml b/docs-xml/manpages-3/idmap_nss.8.xml
index a7fdca033b1..576eef666a9 100644
--- a/docs-xml/manpages-3/idmap_nss.8.xml
+++ b/docs-xml/manpages-3/idmap_nss.8.xml
@@ -38,9 +38,8 @@
 
 	<programlisting>
 	[global]
-	idmap backend = tdb
-	idmap uid = 1000000-1999999
-	idmap gid = 1000000-1999999
+	idmap config * : backend = tdb
+	idmap config * : range = 1000000-1999999
 
 	idmap config SAMBA : backend  = nss
 	idmap config SAMBA : range = 1000-999999
diff --git a/docs-xml/manpages-3/idmap_rid.8.xml b/docs-xml/manpages-3/idmap_rid.8.xml
index a2a1c58a6f4..3f8735288ca 100644
--- a/docs-xml/manpages-3/idmap_rid.8.xml
+++ b/docs-xml/manpages-3/idmap_rid.8.xml
@@ -28,7 +28,7 @@
 	Currently, there should to be an explicit idmap configuration for each
 	domain that should use the idmap_rid backend, using disjoint ranges.
 	One usually needs to define a writeable default idmap range, using
-	a backent like <parameter>tdb</parameter> or <parameter>ldap</parameter>
+	a backend like <parameter>tdb</parameter> or <parameter>ldap</parameter>
 	that can create unix ids, in order to be able to map the BUILTIN sids
 	and other domains, and also in order to be able to create group mappings.
 	See the example below.
@@ -106,9 +106,8 @@
 	security = domain
 	workgroup = MAIN
 
-	idmap backend = tdb
-	idmap uid = 1000000-1999999
-	idmap gid = 1000000-1999999
+	idmap config * : backend        = tdb
+	idmap config * : range          = 1000000-1999999
 
 	idmap config MAIN : backend     = rid
 	idmap config MAIN : range       = 10000 - 49999
diff --git a/docs-xml/manpages-3/idmap_tdb.8.xml b/docs-xml/manpages-3/idmap_tdb.8.xml
index 06a29679c85..c67d6cb9bc2 100644
--- a/docs-xml/manpages-3/idmap_tdb.8.xml
+++ b/docs-xml/manpages-3/idmap_tdb.8.xml
@@ -27,25 +27,7 @@
 	<para>
 	In contrast to read only backends like idmap_rid, it is an allocating
 	backend: This means that it needs to allocate new user and group IDs in
-	order to create new mappings. The allocator can be provided by the
-	idmap_tdb backend itself or by any other allocating backend like
-	idmap_ldap or idmap_tdb2. This is configured with the
-	parameter <parameter>idmap alloc backend</parameter>.
-	</para>
-
-	<para>
-	Note that in order for this (or any other allocating) backend to
-	function at all, the default backend needs to be writeable.
-	The ranges used for uid and gid allocation are the default ranges
-	configured by &quot;idmap uid&quot; and &quot;idmap gid&quot;.
-	</para>
-
-	<para>
-	Furthermore, since there is only one global allocating backend
-	responsible for all domains using writeable idmap backends,
-	any explicitly configured domain with idmap backend tdb
-	should have the same range as the default range, since it needs
-	to use the global uid / gid allocator. See the example below.
+	order to create new mappings.
 	</para>
 </refsynopsisdiv>
 
@@ -58,9 +40,6 @@
                 <listitem><para>
 			Defines the available matching uid and gid range for which the
 			backend is authoritative.
-			If the parameter is absent, Winbind fails over to use
-			the &quot;idmap uid&quot; and &quot;idmap gid&quot; options
-			from smb.conf.
                 </para></listitem>
                 </varlistentry>
 	</variablelist>
@@ -71,38 +50,14 @@
 
 	<para>
 	This example shows how tdb is used as a the default idmap backend.
-	It configures the idmap range through the global options for all
-	domains encountered. This same range is used for uid/gid allocation.
-	</para>
-
-	<programlisting>
-	[global]
-	# "idmap backend = tdb" is redundant here since it is the default
-	idmap backend = tdb
-	idmap uid = 1000000-2000000
-	idmap gid = 1000000-2000000
-	</programlisting>
-
-	<para>
-	This (rather theoretical) example shows how tdb can be used as the
-	allocating backend while ldap is the default backend used to store
-	the mappings.
-	It adds an explicit configuration for some domain DOM1, that
-	uses the tdb idmap backend. Note that the same range as the
-	default uid/gid range is used, since the allocator has to serve
-	both the default backend and the explicitly configured domain DOM1.
+	This configured range is used for uid and gid allocation.
 	</para>
 
 	<programlisting>
 	[global]
-	idmap backend = ldap
-	idmap uid = 1000000-2000000
-	idmap gid = 1000000-2000000
-	# use a different uid/gid allocator:
-	idmap alloc backend = tdb
-
-	idmap config DOM1 : backend = tdb
-	idmap config DOM1 : range = 1000000-2000000
+	# "backend = tdb" is redundant here since it is the default
+	idmap config * : backend = tdb
+	idmap config * : range = 1000000-2000000
 	</programlisting>
 </refsect1>
 
diff --git a/docs-xml/manpages-3/idmap_tdb2.8.xml b/docs-xml/manpages-3/idmap_tdb2.8.xml
index a5d1080474e..980ffe6eff8 100644
--- a/docs-xml/manpages-3/idmap_tdb2.8.xml
+++ b/docs-xml/manpages-3/idmap_tdb2.8.xml
@@ -28,25 +28,7 @@
 	<para>
 	In contrast to read only backends like idmap_rid, it is an allocating
 	backend: This means that it needs to allocate new user and group IDs in
-	order to create new mappings. The allocator can be provided by the
-	idmap_tdb2 backend itself or by any other allocating backend like
-	idmap_tdb or idmap_ldap. This is configured with the
-	parameter <parameter>idmap alloc backend</parameter>.
-	</para>
-
-	<para>
-	Note that in order for this (or any other allocating) backend to
-	function at all, the default backend needs to be writeable.
-	The ranges used for uid and gid allocation are the default ranges
-	configured by &quot;idmap uid&quot; and &quot;idmap gid&quot;.
-	</para>
-
-	<para>
-	Furthermore, since there is only one global allocating backend
-	responsible for all domains using writeable idmap backends,
-	any explicitly configured domain with idmap backend tdb2
-	should have the same range as the default range, since it needs
-	to use the global uid / gid allocator. See the example below.
+	order to create new mappings.
 	</para>
 </refsynopsisdiv>
 
@@ -59,9 +41,6 @@
 		<listitem><para>
 			Defines the available matching uid and gid range for which the
 			backend is authoritative.
-			If the parameter is absent, Winbind fails over to use
-			the &quot;idmap uid&quot; and &quot;idmap gid&quot; options
-			from smb.conf.
 		</para></listitem>
 		</varlistentry>
 	</variablelist>
@@ -108,14 +87,13 @@
 	<para>
 	This example shows how tdb2 is used as a the default idmap backend.
 	It configures the idmap range through the global options for all
-	domains encountered. This same range is used for uid/gid allocation.
+	domains encountered.
 	</para>
 
 	<programlisting>
 	[global]
-	idmap backend = tdb2
-	idmap uid = 1000000-2000000
-	idmap gid = 1000000-2000000
+	idmap config * : backend = tdb2
+	idmap config * : range = 1000000-2000000
 	</programlisting>
 </refsect1>
 
diff --git a/docs-xml/manpages-3/net.8.xml b/docs-xml/manpages-3/net.8.xml
index d69a3b72205..2c6c11ecf84 100644
--- a/docs-xml/manpages-3/net.8.xml
+++ b/docs-xml/manpages-3/net.8.xml
@@ -795,8 +795,122 @@ appear in your system.
 <para>List all interdomain trust relationships.</para>
 
 </refsect3>
+</refsect2>
+
+<refsect2>
+<title>RPC TRUST</title>
+
+<refsect3>
+<title>RPC TRUST CREATE</title>
+
+<para>Create a trust object by calling lsaCreateTrustedDomainEx2.
+The can be done on a single server or on two servers at once with the
+possibility to use a random trust password.</para>
+
+<variablelist><title>Options:</title>
+<varlistentry>
+<term>otherserver</term>
+<listitem><para>Domain controller of the second domain</para></listitem>
+</varlistentry>
+
+<varlistentry>
+<term>otheruser</term>
+<listitem><para>Admin user in the second domain</para></listitem>
+</varlistentry>
+
+<varlistentry>
+<term>otherdomainsid</term>
+<listitem><para>SID of the second domain</para></listitem>
+</varlistentry>
+
+<varlistentry>
+<term>other_netbios_domain</term>
+<listitem><para>NetBIOS (short) name of the second domain</para></listitem>
+</varlistentry>
+
+<varlistentry>
+<term>otherdomain</term>
+<listitem><para>DNS (full) name of the second domain</para></listitem>
+</varlistentry>
+
+<varlistentry>
+<term>trustpw</term>
+<listitem><para>Trust password</para></listitem>
+</varlistentry>
+</variablelist>
+
+<variablelist><title>Examples:</title>
+<varlistentry>
+<term>Create a trust object on srv1.dom1.dom for the domain dom2</term>
+<listitem><literallayout>
+net rpc trust create \
+    otherdomainsid=S-x-x-xx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxx \
+    other_netbios_domain=dom2 \
+    otherdomain=dom2.dom \
+    trustpw=12345678 \
+    -S srv1.dom1.dom
+</literallayout></listitem>
+</varlistentry>
+<varlistentry>
+<term>Create a trust relationship between dom1 and dom2</term>
+<listitem><literallayout>
+net rpc trust create \
+    otherserver=srv2.dom2.test \
+    otheruser=dom2adm \
+    -S srv1.dom1.dom
+</literallayout></listitem>
+</varlistentry>
+</variablelist>
+</refsect3>
 
 <refsect3>
+<title>RPC TRUST DELETE</title>
+
+<para>Delete a trust trust object by calling lsaDeleteTrustedDomain.
+The can be done on a single server or on two servers at once.</para>
+
+<variablelist><title>Options:</title>
+<varlistentry>
+<term>otherserver</term>
+<listitem><para>Domain controller of the second domain</para></listitem>
+</varlistentry>
+
+<varlistentry>
+<term>otheruser</term>
+<listitem><para>Admin user in the second domain</para></listitem>
+</varlistentry>
+
+<varlistentry>
+<term>otherdomainsid</term>
+<listitem><para>SID of the second domain</para></listitem>
+</varlistentry>
+</variablelist>
+
+<variablelist><title>Examples:</title>
+<varlistentry>
+<term>Delete a trust object on srv1.dom1.dom for the domain dom2</term>
+<listitem><literallayout>
+net rpc trust delete \
+    otherdomainsid=S-x-x-xx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxx \
+    -S srv1.dom1.dom
+</literallayout></listitem>
+</varlistentry>
+<varlistentry>
+<term>Delete a trust relationship between dom1 and dom2</term>
+<listitem><literallayout>
+net rpc trust delete \
+    otherserver=srv2.dom2.test \
+    otheruser=dom2adm \
+    -S srv1.dom1.dom
+</literallayout></listitem>
+</varlistentry>
+</variablelist>
+</refsect3>
+
+</refsect2>
+
+<refsect2>
+<refsect3>
 <title>RPC RIGHTS</title>
 
 <para>This subcommand is used to view and manage Samba's rights assignments (also 
@@ -1734,8 +1848,10 @@ Manipulate Samba's registry.
 <para>The registry commands are:
 <simplelist>
 <member>net registry enumerate   - Enumerate registry keys and values.</member>
+<member>net registry enumerate_recursive - Enumerate registry key and its subkeys.</member>
 <member>net registry createkey   - Create a new registry key.</member>
 <member>net registry deletekey   - Delete a registry key.</member>
+<member>net registry deletekey_recursive - Delete a registry key with subkeys.</member>
 <member>net registry getvalue    - Print a registry value.</member>
 <member>net registry getvalueraw - Print a registry value (raw format).</member>
 <member>net registry setvalue    - Set a new registry value.</member>
@@ -1758,7 +1874,13 @@ string.</member>
 
 <refsect3>
   <title>REGISTRY ENUMERATE <replaceable>key</replaceable> </title>
-  <para>Enumerate subkeys and values of <emphasis>key</emphasis>
+  <para>Enumerate subkeys and values of <emphasis>key</emphasis>.
+  </para>
+</refsect3>
+
+<refsect3>
+  <title>REGISTRY ENUMERATE_RECURSIVE <replaceable>key</replaceable> </title>
+  <para>Enumerate values of <emphasis>key</emphasis> and its subkeys.
   </para>
 </refsect3>
 
@@ -1770,6 +1892,13 @@ string.</member>
 
 <refsect3>
   <title>REGISTRY DELETEKEY <replaceable>key</replaceable> </title>
+  <para>Delete the given <emphasis>key</emphasis> and its
+  values from the registry, if it has no subkeys.
+  </para>
+</refsect3>
+
+<refsect3>
+  <title>REGISTRY DELETEKEY_RECURSIVE <replaceable>key</replaceable> </title>
   <para>Delete the given <emphasis>key</emphasis> and all of its
   subkeys and values from the registry.
   </para>
diff --git a/docs-xml/manpages-3/smb.conf.5.xml b/docs-xml/manpages-3/smb.conf.5.xml
index 2b930654436..f5f252ba46d 100644
--- a/docs-xml/manpages-3/smb.conf.5.xml
+++ b/docs-xml/manpages-3/smb.conf.5.xml
@@ -503,12 +503,16 @@ chmod 1770 /usr/local/samba/lib/usershares
 		<varlistentry>
 		<term>%I</term>
 		<listitem><para>the IP address of the client machine.</para>
+		<para>Before 3.6.0 it could contain IPv4 mapped IPv6 addresses,
+			now it only contains IPv4 or IPv6 addresses.</para>
 		</listitem>
 		</varlistentry>
 
 		<varlistentry>
 		<term>%i</term>
 		<listitem><para>the local IP address to which a client connected.</para>
+		<para>Before 3.6.0 it could contain IPv4 mapped IPv6 addresses,
+			now it only contains IPv4 or IPv6 addresses.</para>
 		</listitem>
 		</varlistentry>
 
diff --git a/docs-xml/manpages-3/vfs_gpfs.8.xml b/docs-xml/manpages-3/vfs_gpfs.8.xml
index 5f89ffac779..a6b31240eea 100644
--- a/docs-xml/manpages-3/vfs_gpfs.8.xml
+++ b/docs-xml/manpages-3/vfs_gpfs.8.xml
@@ -276,6 +276,26 @@
 		</listitem>
 		</varlistentry>
 
+		<varlistentry>
+		<term>gpfs:syncio = [yes|no]</term>
+		<listitem>
+		<para>This parameter makes Samba open all files with O_SYNC.
+		  This triggers optimizations in GPFS for workloads that
+		  heavily share files.</para>
+
+		<para>Following is the behaviour of Samba for different
+		  values:
+		</para>
+		<itemizedlist>
+		<listitem><para><command>yes</command>Open files with O_SYNC
+		</para></listitem>
+		<listitem><para><command>no (default)</command>Open files as
+		    normal Samba would do
+		</para></listitem>
+		</itemizedlist>
+		</listitem>
+		</varlistentry>
+
 	</variablelist>
 </refsect1>
 
diff --git a/docs-xml/manpages-3/vfs_smb_traffic_analyzer.8.xml b/docs-xml/manpages-3/vfs_smb_traffic_analyzer.8.xml
index 882ee6af3f8..8df9751aa4d 100644
--- a/docs-xml/manpages-3/vfs_smb_traffic_analyzer.8.xml
+++ b/docs-xml/manpages-3/vfs_smb_traffic_analyzer.8.xml
@@ -45,6 +45,8 @@
 	protocol, supporting only a small list of VFS operations, and had
 	several drawbacks. The protocol version 2 is a try to solve the
 	problems version 1 had while at the same time adding new features.
+	With the release of Samba 3.6.0, the module will run protocol version 2
+	by default.
 	</para>
 </refsect1>
 
@@ -77,6 +79,7 @@
 	<listitem><para><command>SHARE</command> - the name of the share on which the VFS operation occured</para></listitem>
 	<listitem><para><command>FILENAME</command> - the name of the file that was used by the VFS operation</para></listitem>
 	<listitem><para><command>TIMESTAMP</command> - a timestamp, formatted as "yyyy-mm-dd hh-mm-ss.ms" indicating when the VFS operation occured</para></listitem>
+	<listitem><para><command>IP</command> - The IP Address (v4 or v6) of the client machine that initiated the VFS operation.</para></listitem>
 	</itemizedlist>
 
 	</para>
@@ -136,7 +139,7 @@
 		The module now can identify itself against the receiver with a sub-release number, where
 		the receiver may run with a different sub-release number than the module. However, as
 		long as both run on the V2.x protocol, the receiver will not crash, even if the module
-		uses features only implemented in the newer subrelease. Ultimativly, if the module uses
+		uses features only implemented in the newer subrelease. Ultimatively, if the module uses
 		a new feature from a newer subrelease, and the receiver runs an older protocol, it is just
 		ignoring the functionality. Of course it is best to have both the receiver and the module
 		running the same subrelease of the protocol.
@@ -150,7 +153,7 @@
 		</listitem>
 		<listitem>
 		<para>
-		The module now potientially has the ability to create data on every VFS function. As of
+		The module now potentially has the ability to create data on every VFS function. As of
 		protocol V2.0, there is support for 8 VFS functions, namely write,read,pread,pwrite,
 		rename,chdir,mkdir and rmdir. Supporting more VFS functions is one of the targets for the
 		upcoming sub-releases.
@@ -224,9 +227,9 @@
 		<varlistentry>
 		<term>smb_traffic_analyzer:protocol_version = STRING</term>
 		<listitem>
-		<para>If STRING matches to V1 or is not given at all, the module
-		will use version 1 of the protocol. If STRING matches to "V2"
-		the module will use version 2 of the protocol.
+		<para>If STRING matches to V1, the module will use version 1 of the
+		protocol. If STRING is not given, the module will use version 2 of the
+		protocol, which is the default.
 		</para>
 		</listitem>
 		</varlistentry>
@@ -241,7 +244,6 @@
 	<smbconfsection name="[example_share]"/>
 	<smbconfoption name="path">/data/example</smbconfoption>
 	<smbconfoption name="vfs_objects">smb_traffic_analyzer</smbconfoption>
-	<smbconfoption name="smb_traffic_analyzer:protocol_version">V2</smbconfoption>
 	<smbconfoption name="smb_traffic_analyzer:host">examplehost</smbconfoption>
 	<smbconfoption name="smb_traffic_analyzer:port">3491</smbconfoption>
 	</programlisting>
diff --git a/docs-xml/manpages-3/wbinfo.1.xml b/docs-xml/manpages-3/wbinfo.1.xml
index c1b2c1f6278..0701d08a56d 100644
--- a/docs-xml/manpages-3/wbinfo.1.xml
+++ b/docs-xml/manpages-3/wbinfo.1.xml
@@ -423,7 +423,7 @@
 		<term>-U|--uid-to-sid <replaceable>uid</replaceable></term>
 		<listitem><para>Try to convert a UNIX user id to a Windows NT
 		SID.  If the uid specified does not refer to one within
-		the idmap uid range then the operation will fail. </para></listitem>
+		the idmap range then the operation will fail. </para></listitem>
 		</varlistentry>
 
 		<varlistentry>
diff --git a/docs-xml/manpages-3/winbindd.8.xml b/docs-xml/manpages-3/winbindd.8.xml
index c9fd4d8e656..df44e4437d9 100644
--- a/docs-xml/manpages-3/winbindd.8.xml
+++ b/docs-xml/manpages-3/winbindd.8.xml
@@ -45,10 +45,9 @@
 	<para>Even if winbind is not used for nsswitch, it still provides a
 	service to <command>smbd</command>, <command>ntlm_auth</command>
 	and the <command>pam_winbind.so</command> PAM module, by managing connections to
-	domain controllers.  In this configuraiton the
-	<smbconfoption name="idmap uid"/> and
-	<smbconfoption name="idmap gid"/>
-	parameters are not required. (This is known as `netlogon proxy only mode'.)</para>
+	domain controllers.  In this configuration the
+	<smbconfoption name="idmap config * : range"/>
+	parameter is not required. (This is known as `netlogon proxy only mode'.)</para>
 
         <para> The Name Service Switch allows user
 	and system information to be obtained from different databases
@@ -246,11 +245,9 @@ hosts:		files wins
 		<listitem><para>
 		<smbconfoption name="winbind separator"/></para></listitem>
 		<listitem><para>
-		<smbconfoption name="idmap uid"/></para></listitem>
+		<smbconfoption name="idmap config * : range"/></para></listitem>
 		<listitem><para>
-		<smbconfoption name="idmap gid"/></para></listitem>
-		<listitem><para>
-		<smbconfoption name="idmap backend"/></para></listitem>
+		<smbconfoption name="idmap config * : backend"/></para></listitem>
 		<listitem><para>
 		<smbconfoption name="winbind cache time"/></para></listitem>
 		<listitem><para>
@@ -340,8 +337,7 @@ auth  required    /lib/security/pam_unix.so \
         winbind cache time = 10
         template shell = /bin/bash
         template homedir = /home/%D/%U
-        idmap uid = 10000-20000
-        idmap gid = 10000-20000
+        idmap config * : range = 10000-20000
         workgroup = DOMAIN
         security = domain
         password server = *
@@ -374,7 +370,7 @@ auth  required    /lib/security/pam_unix.so \
 	<para>If more than one UNIX machine is running <command>winbindd</command>,
 	then in general the user and groups ids allocated by winbindd will not
 	be the same.  The user and group ids will only be valid for the local
-	machine, unless a shared <smbconfoption name="idmap backend"/> is configured.</para>
+	machine, unless a shared <smbconfoption name="idmap config * : backend"/> is configured.</para>
 
 	<para>If the the Windows NT SID to UNIX user and group id mapping
 	file is damaged or destroyed then the mappings will be lost. </para>
diff --git a/docs-xml/smbdotconf/logon/enableprivileges.xml b/docs-xml/smbdotconf/logon/enableprivileges.xml
index 3e958e0ce9a..0fbc504c526 100644
--- a/docs-xml/smbdotconf/logon/enableprivileges.xml
+++ b/docs-xml/smbdotconf/logon/enableprivileges.xml
@@ -5,7 +5,7 @@
                  xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
 <description>
 	<para>
-	This parameter controls whether or not smbd will honor privileges assigned to specific SIDs via either
+	This deprecated parameter controls whether or not smbd will honor privileges assigned to specific SIDs via either
 	 <command>net rpc rights</command> or one of the Windows user and group manager tools.  This parameter is
 	enabled by default. It can be disabled to prevent members of the Domain Admins group from being able to
 	assign privileges to users or groups which can then result in certain smbd operations running as root that
diff --git a/docs-xml/smbdotconf/misc/asyncsmbechohandler.xml b/docs-xml/smbdotconf/misc/asyncsmbechohandler.xml
new file mode 100644
index 00000000000..d10dac90b3e
--- /dev/null
+++ b/docs-xml/smbdotconf/misc/asyncsmbechohandler.xml
@@ -0,0 +1,15 @@
+<samba:parameter name="async smb echo handler"
+                 context="G"
+				 type="boolean"
+                 advanced="1"
+                 xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
+<description>
+	<para>This parameter specifies whether Samba should fork the
+	  async smb echo handler. It can be beneficial if your file
+	  system can block syscalls for a very long time. In some
+	  circumstances, it prolongs the timeout that Windows uses to
+	  determine whether a connection is dead.
+	</para>
+</description>
+<value type="default">no</value>
+</samba:parameter>
diff --git a/docs-xml/smbdotconf/misc/ncalrpcdir.xml b/docs-xml/smbdotconf/misc/ncalrpcdir.xml
new file mode 100644
index 00000000000..b53acef9e26
--- /dev/null
+++ b/docs-xml/smbdotconf/misc/ncalrpcdir.xml
@@ -0,0 +1,13 @@
+<samba:parameter name="ncalrpc dir"
+                 context="G"
+                 advanced="1" developer="1"
+		 type="string"
+                 xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
+<description>
+	<para>This directory will hold a series of named pipes to allow RPC over inter-process communication.  </para>.  
+	<para>This will allow Samba and other unix processes to interact over DCE/RPC without using TCP/IP.  Additionally a sub-directory 'np' has restricted permissions, and allows a trusted communication channel between Samba processes</para>
+</description>
+
+<value type="default">${prefix}/var/ncalrpc</value>
+<value type="example">/var/run/samba/ncalrpc</value>
+</samba:parameter>
diff --git a/docs-xml/smbdotconf/misc/timeoffset.xml b/docs-xml/smbdotconf/misc/timeoffset.xml
deleted file mode 100644
index 1afc514e60d..00000000000
--- a/docs-xml/smbdotconf/misc/timeoffset.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<samba:parameter name="time offset"
-	         context="G"
-			 type="integer"
-		 advanced="1" developer="1"
-		 xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
-<description>
-	<para>This parameter is a setting in minutes to add 
-	to the normal GMT to local time conversion. This is useful if 
-	you are serving a lot of PCs that have incorrect daylight 
-	saving time handling.</para>
-</description>
-
-<value type="default">0</value>
-<value type="example">60</value>
-</samba:parameter>
diff --git a/docs-xml/smbdotconf/protocol/announceas.xml b/docs-xml/smbdotconf/protocol/announceas.xml
deleted file mode 100644
index 88914961941..00000000000
--- a/docs-xml/smbdotconf/protocol/announceas.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<samba:parameter name="announce as"
-                 context="G"
-				 type="string"
-                 developer="1"
-                 xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
-<description>
-    <para>This specifies what type of server <citerefentry><refentrytitle>nmbd</refentrytitle>
-    <manvolnum>8</manvolnum></citerefentry> will announce itself as, to a network neighborhood browse 
-    list. By default this is set to Windows NT. The valid options 
-    are : &quot;NT Server&quot; (which can also be written as &quot;NT&quot;), 
-    &quot;NT Workstation&quot;, &quot;Win95&quot; or &quot;WfW&quot; meaning Windows NT Server, 
-    Windows NT Workstation, Windows 95 and Windows for Workgroups 
-    respectively. Do not change this parameter unless you have a 
-    specific need to stop Samba appearing as an NT server as this 
-    may prevent Samba servers from participating as browser servers 
-	correctly.</para>
-</description>
-
-<value type="default">NT Server</value>
-<value type="example">Win95</value>
-</samba:parameter>
diff --git a/docs-xml/smbdotconf/protocol/announceversion.xml b/docs-xml/smbdotconf/protocol/announceversion.xml
deleted file mode 100644
index ecdcd4c7349..00000000000
--- a/docs-xml/smbdotconf/protocol/announceversion.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<samba:parameter name="announce version"
-                 context="G"
-                 developer="1"
-				 type="string"
-                 xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
-<description>
-    <para>This specifies the major and minor version numbers 
-    that nmbd will use when announcing itself as a server. The default 
-    is 4.9.  Do not change this parameter unless you have a specific 
-	need to set a Samba server to be a downlevel server.</para>
-</description>
-<value type="default">4.9</value>
-<value type="example">2.0</value>
-</samba:parameter>
diff --git a/docs-xml/smbdotconf/protocol/eventloglist.xml b/docs-xml/smbdotconf/protocol/eventloglist.xml
index e98559bc179..101821ad12a 100644
--- a/docs-xml/smbdotconf/protocol/eventloglist.xml
+++ b/docs-xml/smbdotconf/protocol/eventloglist.xml
@@ -6,7 +6,7 @@
     <para>This option defines a list of log names that Samba will 
     report to the Microsoft EventViewer utility.  The listed 
     eventlogs will be associated with tdb file on disk in the 
-    <filename>$(lockdir)/eventlog</filename>.
+    <filename>$(statedir)/eventlog</filename>.
     </para>
 
     <para>
diff --git a/docs-xml/smbdotconf/protocol/usespnego.xml b/docs-xml/smbdotconf/protocol/usespnego.xml
index 8fb559c1775..e16c7ce2be1 100644
--- a/docs-xml/smbdotconf/protocol/usespnego.xml
+++ b/docs-xml/smbdotconf/protocol/usespnego.xml
@@ -4,7 +4,7 @@
                  developer="1"
                  xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
 <description>
-    <para>This variable controls controls whether samba will try 
+    <para>This deprecated variable controls controls whether samba will try 
     to use Simple and Protected NEGOciation (as specified by rfc2478) with 
     WindowsXP and Windows2000 clients to agree upon an authentication mechanism. 
 </para>
diff --git a/docs-xml/smbdotconf/security/passwordlevel.xml b/docs-xml/smbdotconf/security/passwordlevel.xml
index 1da11e406bd..eee838f65c2 100644
--- a/docs-xml/smbdotconf/security/passwordlevel.xml
+++ b/docs-xml/smbdotconf/security/passwordlevel.xml
@@ -13,7 +13,7 @@
     text passwords even when NT LM 0.12 selected by the protocol
     negotiation request/response.</para>
 
-    <para>This parameter defines the maximum number of characters 
+    <para>This deprecated parameter defines the maximum number of characters 
     that may be upper case in passwords.</para>
 
     <para>For example, say the password given was &quot;FRED&quot;. If <parameter moreinfo="none">
diff --git a/docs-xml/smbdotconf/security/passwordserver.xml b/docs-xml/smbdotconf/security/passwordserver.xml
index 0e92af9ebae..09d335c3ac0 100644
--- a/docs-xml/smbdotconf/security/passwordserver.xml
+++ b/docs-xml/smbdotconf/security/passwordserver.xml
@@ -10,54 +10,24 @@
     it is possible to get Samba
     to do all its username/password validation using a specific remote server.</para>
 
-    <para>This option sets the name or IP address of the password server to use. 
-    New syntax has been added to support defining the port to use when connecting 
-    to the server the case of an ADS realm.  To define a port other than the
-    default LDAP port of 389, add the port number using a colon after the 
-    name or IP address (e.g. 192.168.1.100:389).  If you do not specify a port,
-    Samba will use the standard LDAP port of tcp/389.  Note that port numbers
-    have no effect on password servers for Windows NT 4.0 domains or netbios 
-    connections.</para>
-
-    <para>If parameter is a name, it is looked up using the 
-    parameter <smbconfoption name="name resolve order"/> and so may resolved
-    by any method and order described in that parameter.</para>
-
-    <para>The password server must be a machine capable of using 
-    the &quot;LM1.2X002&quot; or the &quot;NT LM 0.12&quot; protocol, and it must be in 
-    user level security mode.</para>
-
-    <note><para>Using a password server  means your UNIX box (running
-    Samba) is only as secure as your  password server. <emphasis>DO NOT
-    CHOOSE A PASSWORD SERVER THAT  YOU DON'T COMPLETELY TRUST</emphasis>.
-    </para></note>
-		
-    <para>Never point a Samba server at itself for password serving.
-    This will cause a loop and could lock up your Samba  server!</para>
-
-    <para>The name of the password server takes the standard 
-    substitutions, but probably the only useful one is <parameter moreinfo="none">%m
-    </parameter>, which means the Samba server will use the incoming 
-    client as the password server. If you use this then you better 
-    trust your clients, and you had better restrict them with hosts allow!</para>
-
     <para>If the <parameter moreinfo="none">security</parameter> parameter is set to
-    <constant>domain</constant> or <constant>ads</constant>, then the list of machines in this 
-    option must be a list of Primary or Backup Domain controllers for the
-    Domain or the character '*', as the Samba server is effectively
-    in that domain, and will use cryptographically authenticated RPC calls
-    to authenticate the user logging on. The advantage of using <command moreinfo="none">
-    security = domain</command> is that if you list several hosts in the 
-    <parameter moreinfo="none">password server</parameter> option then <command moreinfo="none">smbd
-    </command> will try each in turn till it finds one that responds.  This
-    is useful in case your primary server goes down.</para>
+    <constant>domain</constant> or <constant>ads</constant>, then this option 
+    <emphasis>should not</emphasis> be used, as the default '*' indicates to Samba 
+    to determine the best DC to contact dynamically, just as all other hosts in an 
+    AD domain do.  This allows the domain to be maintained without modification to 
+    the smb.conf file.  The cryptograpic protection on the authenticated RPC calls
+    used to verify passwords ensures that this default is safe.</para>
 
-    <para>If the <parameter moreinfo="none">password server</parameter> option is set 
-    to the character '*', then Samba will attempt to auto-locate the 
-    Primary or Backup Domain controllers to authenticate against by 
-    doing a query for the name <constant>WORKGROUP&lt;1C&gt;</constant> 
-    and then contacting each server returned in the list of IP 
-    addresses from the name resolution source. </para>
+    <para><emphasis>It is strongly recommended that you use the
+    default of '*'</emphasis>, however if in your particular
+    environment you have reason to specify a particular DC list, then
+    the list of machines in this option must be a list of names or IP
+    addresses of Domain controllers for the Domain. If you use the
+    default of '*', or list several hosts in the <parameter
+    moreinfo="none">password server</parameter> option then <command
+    moreinfo="none">smbd </command> will try each in turn till it
+    finds one that responds.  This is useful in case your primary
+    server goes down.</para>
 
     <para>If the list of servers contains both names/IP's and the '*'
     character, the list is treated as a list of preferred 
@@ -65,10 +35,12 @@
     will be added to the list as well.  Samba will not attempt to optimize 
     this list by locating the closest DC.</para>
 		
+    <para>If parameter is a name, it is looked up using the 
+    parameter <smbconfoption name="name resolve order"/> and so may resolved
+    by any method and order described in that parameter.</para>
+
     <para>If the <parameter moreinfo="none">security</parameter> parameter is 
-    set to <constant>server</constant>, then there are different
-    restrictions that <command moreinfo="none">security = domain</command> doesn't 
-    suffer from:</para>
+    set to <constant>server</constant>, these additional restrictions apply:</para>
 
     <itemizedlist>
 	<listitem>
@@ -82,12 +54,34 @@
 	</listitem>
 	    
 	<listitem>
-	    <para>If you are using a Windows NT server as your 
-	    password server then you will have to ensure that your users 
+	    <para>You will have to ensure that your users 
 	    are able to login from the Samba server, as when in <command moreinfo="none">
 	    security = server</command>  mode the network logon will appear to 
-	    come from there rather than from the users workstation.</para>
+	    come from the Samba server rather than from the users workstation.</para>
 	</listitem>
+
+	<listitem>
+	    <para>The client must not select NTLMv2 authentication.</para>
+	</listitem>
+
+	<listitem>
+	  <para>The password server must be a machine capable of using 
+	  the &quot;LM1.2X002&quot; or the &quot;NT LM 0.12&quot; protocol, and it must be in 
+	  user level security mode.</para>
+	</listitem>
+
+	<listitem>
+	  <para>Using a password server  means your UNIX box (running
+	  Samba) is only as secure as (a host masqurading as) your password server. <emphasis>DO NOT
+	  CHOOSE A PASSWORD SERVER THAT  YOU DON'T COMPLETELY TRUST</emphasis>.
+	  </para>
+	</listitem>
+		
+	<listitem>
+	  <para>Never point a Samba server at itself for password serving.
+	  This will cause a loop and could lock up your Samba  server!</para>
+	</listitem>
+
     </itemizedlist>
 </description>
 
diff --git a/docs-xml/smbdotconf/security/security.xml b/docs-xml/smbdotconf/security/security.xml
index 514ea54e0ff..55e147e8dc4 100644
--- a/docs-xml/smbdotconf/security/security.xml
+++ b/docs-xml/smbdotconf/security/security.xml
@@ -22,32 +22,18 @@
     the most common setting needed when talking to Windows 98 and 
     Windows NT.</para>
 
-    <para>The alternatives are <command moreinfo="none">security = share</command>,
-    <command moreinfo="none">security = server</command> or <command moreinfo="none">security = domain
-    </command>.</para>
+    <para>The alternatives are
+    <command moreinfo="none">security = ads</command> or <command moreinfo="none">security = domain
+    </command>, which support joining Samba to a Windows domain, along with <command moreinfo="none">security = share</command> and <command moreinfo="none">security = server</command>, both of which are deprecated.</para>
 
     <para>In versions of Samba prior to 2.0.0, the default was 
     <command moreinfo="none">security = share</command> mainly because that was
     the only option at one stage.</para>
 
-    <para>There is a bug in WfWg that has relevance to this 
-    setting. When in user or server level security a WfWg client 
-    will totally ignore the username and password you type in the &quot;connect 
-    drive&quot; dialog box. This makes it very difficult (if not impossible) 
-    to connect to a Samba service as anyone except the user that 
-    you are logged into WfWg as.</para>
-
-    <para>If your PCs use usernames that are the same as their 
-    usernames on the UNIX machine then you will want to use 
-    <command moreinfo="none">security = user</command>. If you mostly use usernames 
-    that don't exist on the UNIX box then use <command moreinfo="none">security = 
-    share</command>.</para>
-
-    <para>You should also use <command moreinfo="none">security = share</command> if you 
+    <para>You should use <command moreinfo="none">security = user</command> and 
+    <smbconfoption name="map to guest"/> if you 
     want to mainly setup shares without a password (guest shares). This 
-    is commonly used for a shared printer server. It is more difficult 
-    to setup guest shares with <command moreinfo="none">security = user</command>, see 
-    the <smbconfoption name="map to guest"/> parameter for details.</para>
+    is commonly used for a shared printer server. </para>
 		
     <para>It is possible to use <command moreinfo="none">smbd</command> in a <emphasis>
     hybrid mode</emphasis> where it is offers both user and share 
@@ -56,7 +42,62 @@
     <para>The different settings will now be explained.</para>
 
 
+    <para><anchor id="SECURITYEQUALSUSER"/><emphasis>SECURITY = USER</emphasis></para>
+
+    <para>This is the default security setting in Samba. 
+    With user-level security a client must first &quot;log-on&quot; with a 
+    valid username and password (which can be mapped using the <smbconfoption name="username map"/> 
+    parameter). Encrypted passwords (see the <smbconfoption name="encrypted passwords"/> parameter) can also
+    be used in this security mode. Parameters such as <smbconfoption name="user"/> and <smbconfoption
+	name="guest only"/> if set	are then applied and 
+    may change the UNIX user to use on this connection, but only after 
+    the user has been successfully authenticated.</para>
+
+    <para><emphasis>Note</emphasis> that the name of the resource being 
+    requested is <emphasis>not</emphasis> sent to the server until after 
+    the server has successfully authenticated the client. This is why 
+    guest shares don't work in user level security without allowing 
+    the server to automatically map unknown users into the <smbconfoption name="guest account"/>. 
+    See the <smbconfoption name="map to guest"/> parameter for details on doing this.</para>
+
+    <para>See also the section <link linkend="VALIDATIONSECT">NOTE ABOUT USERNAME/PASSWORD VALIDATION</link>.</para>
+
+    <para><anchor id="SECURITYEQUALSDOMAIN"/><emphasis>SECURITY = DOMAIN</emphasis></para>
+
+    <para>This mode will only work correctly if <citerefentry><refentrytitle>net</refentrytitle>
+    <manvolnum>8</manvolnum></citerefentry> has been used to add this
+    machine into a Windows NT Domain. It expects the <smbconfoption name="encrypted passwords"/>
+	parameter to be set to <constant>yes</constant>. In this 
+    mode Samba will try to validate the username/password by passing
+    it to a Windows NT Primary or Backup Domain Controller, in exactly 
+    the same way that a Windows NT Server would do.</para>
+
+    <para><emphasis>Note</emphasis> that a valid UNIX user must still 
+    exist as well as the account on the Domain Controller to allow 
+    Samba to have a valid UNIX account to map file access to.</para>
+
+    <para><emphasis>Note</emphasis> that from the client's point 
+    of view <command moreinfo="none">security = domain</command> is the same 
+    as <command moreinfo="none">security = user</command>. It only 
+    affects how the server deals with the authentication, 
+    it does not in any way affect what the client sees.</para>
+
+    <para><emphasis>Note</emphasis> that the name of the resource being 
+    requested is <emphasis>not</emphasis> sent to the server until after 
+    the server has successfully authenticated the client. This is why 
+    guest shares don't work in user level security without allowing 
+    the server to automatically map unknown users into the <smbconfoption name="guest account"/>. 
+    See the <smbconfoption name="map to guest"/> parameter for details on doing this.</para>
+
+    <para>See also the section <link linkend="VALIDATIONSECT">
+    NOTE ABOUT USERNAME/PASSWORD VALIDATION</link>.</para>
+
+    <para>See also the <smbconfoption name="password server"/> parameter and
+	 the <smbconfoption name="encrypted passwords"/> parameter.</para>
+
     <para><anchor id="SECURITYEQUALSSHARE"/><emphasis>SECURITY = SHARE</emphasis></para> 
+
+    <note><para>This option is deprecated as it is incompatible with SMB2</para></note>
 		
     <para>When clients connect to a share level security server, they 
     need not log onto the server with a valid username and password before 
@@ -135,63 +176,10 @@
     <para>See also the section <link linkend="VALIDATIONSECT">
     NOTE ABOUT USERNAME/PASSWORD VALIDATION</link>.</para>
 
-    <para><anchor id="SECURITYEQUALSUSER"/><emphasis>SECURITY = USER</emphasis></para>
-
-    <para>This is the default security setting in Samba 3.0. 
-    With user-level security a client must first &quot;log-on&quot; with a 
-    valid username and password (which can be mapped using the <smbconfoption name="username map"/> 
-    parameter). Encrypted passwords (see the <smbconfoption name="encrypted passwords"/> parameter) can also
-    be used in this security mode. Parameters such as <smbconfoption name="user"/> and <smbconfoption
-	name="guest only"/> if set	are then applied and 
-    may change the UNIX user to use on this connection, but only after 
-    the user has been successfully authenticated.</para>
-
-    <para><emphasis>Note</emphasis> that the name of the resource being 
-    requested is <emphasis>not</emphasis> sent to the server until after 
-    the server has successfully authenticated the client. This is why 
-    guest shares don't work in user level security without allowing 
-    the server to automatically map unknown users into the <smbconfoption name="guest account"/>. 
-    See the <smbconfoption name="map to guest"/> parameter for details on doing this.</para>
-
-    <para>See also the section <link linkend="VALIDATIONSECT">NOTE ABOUT USERNAME/PASSWORD VALIDATION</link>.</para>
-
-    <para><anchor id="SECURITYEQUALSDOMAIN"/><emphasis>SECURITY = DOMAIN</emphasis></para>
-
-    <para>This mode will only work correctly if <citerefentry><refentrytitle>net</refentrytitle>
-    <manvolnum>8</manvolnum></citerefentry> has been used to add this
-    machine into a Windows NT Domain. It expects the <smbconfoption name="encrypted passwords"/>
-	parameter to be set to <constant>yes</constant>. In this 
-    mode Samba will try to validate the username/password by passing
-    it to a Windows NT Primary or Backup Domain Controller, in exactly 
-    the same way that a Windows NT Server would do.</para>
-
-    <para><emphasis>Note</emphasis> that a valid UNIX user must still 
-    exist as well as the account on the Domain Controller to allow 
-    Samba to have a valid UNIX account to map file access to.</para>
-
-    <para><emphasis>Note</emphasis> that from the client's point 
-    of view <command moreinfo="none">security = domain</command> is the same 
-    as <command moreinfo="none">security = user</command>. It only 
-    affects how the server deals with the authentication, 
-    it does not in any way affect what the client sees.</para>
-
-    <para><emphasis>Note</emphasis> that the name of the resource being 
-    requested is <emphasis>not</emphasis> sent to the server until after 
-    the server has successfully authenticated the client. This is why 
-    guest shares don't work in user level security without allowing 
-    the server to automatically map unknown users into the <smbconfoption name="guest account"/>. 
-    See the <smbconfoption name="map to guest"/> parameter for details on doing this.</para>
-
-    <para>See also the section <link linkend="VALIDATIONSECT">
-    NOTE ABOUT USERNAME/PASSWORD VALIDATION</link>.</para>
-
-    <para>See also the <smbconfoption name="password server"/> parameter and
-	 the <smbconfoption name="encrypted passwords"/> parameter.</para>
-
     <para><anchor id="SECURITYEQUALSSERVER"/><emphasis>SECURITY = SERVER</emphasis></para>
 
     <para>
-	In this mode Samba will try to validate the username/password by passing it to another SMB server, such as an
+	In this depicted mode Samba will try to validate the username/password by passing it to another SMB server, such as an
 	NT box. If this fails it will revert to <command moreinfo="none">security = user</command>. It expects the
 	<smbconfoption name="encrypted passwords"/> parameter to be set to <constant>yes</constant>, unless the remote
 	server does not support them.  However note that if encrypted passwords have been negotiated then Samba cannot
@@ -203,19 +191,24 @@
 	<note><para>This mode of operation has
     significant pitfalls since it is more vulnerable to
     man-in-the-middle attacks and server impersonation.  In particular,
-    this mode of operation can cause significant resource consuption on
+    this mode of operation can cause significant resource consumption on
     the PDC, as it must maintain an active connection for the duration
     of the user's session.  Furthermore, if this connection is lost,
-    there is no way to reestablish it, and futher authentications to the
+    there is no way to reestablish it, and further authentications to the
     Samba server may fail (from a single client, till it disconnects).
 	</para></note>
 
+	<note><para>If the client selects NTLMv2 authentication, then this mode of operation <emphasis>will fail</emphasis>
+	</para></note>
+
 	<note><para>From the client's point of 
     view, <command moreinfo="none">security = server</command> is the
     same as <command moreinfo="none">security = user</command>.  It
     only affects how the server deals  with the authentication, it does
 	not in any way affect what the  client sees.</para></note>
 
+    <note><para>This option is deprecated, and may be removed in future</para></note>
+
     <para><emphasis>Note</emphasis> that the name of the resource being 
     requested is <emphasis>not</emphasis> sent to the server until after 
     the server has successfully authenticated the client. This is why 
diff --git a/docs-xml/smbdotconf/security/username.xml b/docs-xml/smbdotconf/security/username.xml
index 3a45d4d72fc..19d8a2ecfd5 100644
--- a/docs-xml/smbdotconf/security/username.xml
+++ b/docs-xml/smbdotconf/security/username.xml
@@ -9,7 +9,7 @@
     list, in which case the supplied password will be tested against 
     each username in turn (left to right).</para>
 
-    <para>The <parameter moreinfo="none">username</parameter> line is needed only when 
+    <para>The deprecated <parameter moreinfo="none">username</parameter> line is needed only when 
     the PC is unable to supply its own username. This is the case 
     for the COREPLUS protocol or where your users have different WfWg 
     usernames to UNIX usernames. In both these cases you may also be 
diff --git a/docs-xml/smbdotconf/winbind/idmapallocconfig.xml b/docs-xml/smbdotconf/winbind/idmapallocconfig.xml
deleted file mode 100644
index 013904122c0..00000000000
--- a/docs-xml/smbdotconf/winbind/idmapallocconfig.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<samba:parameter name="idmap alloc config"
-                 context="G"
-		 type="string"
-                 advanced="1" developer="1" hide="1"
-                 xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
-<description>
-	<para>
-	The idmap alloc config prefix provides a means of managing settings
-	for the backend defined by the <smbconfoption name="idmap alloc backend"/> 
-	parameter.  Refer to the man page for each idmap plugin regarding
-	specific configuration details.
-	</para>
-</description>
-</samba:parameter>
diff --git a/docs-xml/smbdotconf/winbind/idmapbackend.xml b/docs-xml/smbdotconf/winbind/idmapbackend.xml
index 824476f4548..bd96dfedd8a 100644
--- a/docs-xml/smbdotconf/winbind/idmapbackend.xml
+++ b/docs-xml/smbdotconf/winbind/idmapbackend.xml
@@ -11,39 +11,8 @@
 
 	<para>
 	This option specifies the default backend that is used when no special
-	configuration set by <smbconfoption name="idmap config"/> matches the
-	specific request.
-	</para>
-
-	<para>
-	This default backend also specifies the place where winbind-generated
-	idmap entries will be stored. So it is highly recommended that you
-	specify a writable backend like <citerefentry>
-	<refentrytitle>idmap_tdb</refentrytitle> <manvolnum>8</manvolnum>
-	</citerefentry> or <citerefentry>
-	<refentrytitle>idmap_ldap</refentrytitle> <manvolnum>8</manvolnum>
-	</citerefentry> as the idmap backend. The <citerefentry>
-	<refentrytitle>idmap_rid</refentrytitle> <manvolnum>8</manvolnum>
-	</citerefentry> and <citerefentry>
-	<refentrytitle>idmap_ad</refentrytitle> <manvolnum>8</manvolnum>
-	</citerefentry> backends are not writable and thus will generate
-	unexpected results if set as idmap backend.
-	</para>
-
-	<para>
-	To use the rid and ad backends, please specify them via the
-	<smbconfoption name="idmap config"/> parameter, possibly also for the
-	domain your machine is member of, specified by <smbconfoption
-	name="workgroup"/>.
-	</para>
-
-	<para>Examples of SID/uid/gid backends include tdb (<citerefentry>
-	<refentrytitle>idmap_tdb</refentrytitle><manvolnum>8</manvolnum></citerefentry>),
-	ldap (<citerefentry><refentrytitle>idmap_ldap</refentrytitle>
-	<manvolnum>8</manvolnum></citerefentry>), rid (<citerefentry>
-	<refentrytitle>idmap_rid</refentrytitle><manvolnum>8</manvolnum></citerefentry>),
-	and ad (<citerefentry><refentrytitle>idmap_ad</refentrytitle>
-	<manvolnum>8</manvolnum></citerefentry>).
+	configuration set, but it is now deprecated in favour of the new
+	spelling <smbconfoption name="idmap config * :  backend"/>.
 	</para>
 </description>
 
diff --git a/docs-xml/smbdotconf/winbind/idmapconfig.xml b/docs-xml/smbdotconf/winbind/idmapconfig.xml
index f6e97b9d97c..69bddf0ebf7 100644
--- a/docs-xml/smbdotconf/winbind/idmapconfig.xml
+++ b/docs-xml/smbdotconf/winbind/idmapconfig.xml
@@ -6,44 +6,108 @@
 <description>
 
 	<para>
-	The idmap config prefix provides a means of managing each trusted
-        domain separately. The idmap config prefix should be followed by the
-        name of the domain, a colon, and a setting specific to the chosen
-        backend. There are three options available for all domains:
+	ID mapping in Samba is the mapping between Windows SIDs and Unix user
+	and group IDs. This is performed by Winbindd with a configurable plugin
+	interface. Samba's ID mapping is configured by options starting with the
+	<smbconfoption name="idmap config"/> prefix.
+	An idmap option consists of the <smbconfoption name="idmap config"/>
+	prefix, followed by a domain name or the asterisk character (*),
+	a colon, and the name of an idmap setting for the chosen domain.
 	</para>
 
-	<variablelist>  
+	<para>
+	The idmap configuration is hence divided into groups, one group
+	for each domain to be configured, and one group with the the
+	asterisk instead of a proper domain name, which speifies the
+	default configuration that is used to catch all domains that do
+	not have an explicit idmap configuration of their own.
+	</para>
+
+	<para>
+	There are three general options available:
+	</para>
+
+	<variablelist>
 		<varlistentry>
 		<term>backend = backend_name</term>
 		<listitem><para>
-			Specifies the name of the idmap plugin to use as the 
-			SID/uid/gid backend for this domain.
+		This specifies the name of the idmap plugin to use as the
+		SID/uid/gid backend for this domain. The standard backends are
+		tdb
+		(<citerefentry><refentrytitle>idmap_tdb</refentrytitle> <manvolnum>8</manvolnum> </citerefentry>),
+		tdb2
+		(<citerefentry><refentrytitle>idmap_tdb2</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		ldap
+		(<citerefentry><refentrytitle>idmap_ldap</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		,
+		rid
+		(<citerefentry><refentrytitle>idmap_rid</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		,
+		hash
+		(<citerefentry><refentrytitle>idmap_hash</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		,
+		autorid
+		(<citerefentry><refentrytitle>idmap_autorid</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		,
+		ad
+		(<citerefentry><refentrytitle>idmap_ad</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		,
+		adex
+		(<citerefentry><refentrytitle>idmap_adex</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		,
+		and nss.
+		(<citerefentry><refentrytitle>idmap_nss</refentrytitle> <manvolnum>8</manvolnum></citerefentry>),
+		The corresponding manual pages contain the details, but
+		here is a summary.
+		</para>
+		<para>
+		The first three of these create mappings of their own using
+		internal unixid counters and store the mappings in a database.
+		These are suitable for use in the default idmap configuration.
+		The rid and hash backends use a pure algorithmic calculation
+		to determine the unixid for a SID. The autorid module is a
+		mixture of the tdb and rid backend. It creates ranges for
+		each domain encountered and then uses the rid algorithm for each
+		of these automatically configured domains individually.
+		The ad and adex
+		backends both use unix IDs stored in Active Directory via
+		the standard schema extensions. The nss backend reverses
+		the standard winbindd setup and gets the unixids via names
+		from nsswitch which can be useful in an ldap setup.
 		</para></listitem>
 		</varlistentry>
 
 		<varlistentry>
 		<term>range = low - high</term>
-                <listitem><para>
+		<listitem><para>
 		Defines the available matching uid and gid range for which the
-		backend is authoritative.  Note that the range commonly
-		matches the allocation range due to the fact that the same
-		backend will store and retrieve SID/uid/gid mapping entries.
-                </para>
+		backend is authoritative. For allocating backends, this also
+		defines the start and the end of the range for allocating
+		new unid IDs.
+		</para>
 		<para>
 		winbind uses this parameter to find the backend that is
-                authoritative for a unix ID to SID mapping, so it must be set
-                for each individually configured domain, and it must be
-                disjoint from the ranges set via <smbconfoption name="idmap
-                uid"/> and <smbconfoption name="idmap gid"/>.
+		authoritative for a unix ID to SID mapping, so it must be set
+		for each individually configured domain and for the default
+		configuration. The configured ranges must be mutually disjoint.
 		</para></listitem>
+		</varlistentry>
 
+		<varlistentry>
+		<term>read only = yes|no</term>
+		<listitem><para>
+		This option can be used to turn the writing backends
+		tdb, tdb2, and ldap into read only mode. This can be useful
+		e.g. in cases where a pre-filled database exists that should
+		not be extended automatically.
+		</para></listitem>
 		</varlistentry>
 	</variablelist>
 
 	<para>
 	The following example illustrates how to configure the <citerefentry>
 	<refentrytitle>idmap_ad</refentrytitle> <manvolnum>8</manvolnum>
-	</citerefentry> for the CORP domain and the
+	</citerefentry> backend for the CORP domain and the
 	<citerefentry><refentrytitle>idmap_tdb</refentrytitle>
 	<manvolnum>8</manvolnum></citerefentry> backend for all other
 	domains. This configuration assumes that the admin of CORP assigns
@@ -53,9 +117,8 @@
 	</para>
 
 	<programlisting>
-	idmap backend = tdb
-	idmap uid = 1000000-1999999
-	idmap gid = 1000000-1999999
+	idmap config * : backend = tdb
+	idmap config * : range = 1000000-1999999
 
 	idmap config CORP : backend  = ad
 	idmap config CORP : range = 1000-999999
diff --git a/docs-xml/smbdotconf/winbind/idmapgid.xml b/docs-xml/smbdotconf/winbind/idmapgid.xml
index ef3ae4fde11..27648a253d9 100644
--- a/docs-xml/smbdotconf/winbind/idmapgid.xml
+++ b/docs-xml/smbdotconf/winbind/idmapgid.xml
@@ -5,16 +5,13 @@
                  xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
 		<synonym>winbind gid</synonym>
 <description>
-	<para>The idmap gid parameter specifies the range of group ids 
-	that are allocated for the purpose of mapping UNX groups to NT group 
-	SIDs. This range of group ids should have no 
-	existing local or NIS groups within it as strange conflicts can 
-	occur otherwise.</para>
-
-	<para>See also the <smbconfoption name="idmap backend"/>, and
-	<smbconfoption name="idmap config"/> options.
+	<para>
+	The idmap gid parameter specifies the range of group ids
+	for the default idmap configuration. It is now deprecated
+	in favour of <smbconfoption name="idmap config * : range"/>.
 	</para>
 
+	<para>See the <smbconfoption name="idmap config"/> option.</para>
 </description>
 
 <value type="default"></value>
diff --git a/docs-xml/smbdotconf/winbind/idmapuid.xml b/docs-xml/smbdotconf/winbind/idmapuid.xml
index 2c53817375e..ce5a4dea059 100644
--- a/docs-xml/smbdotconf/winbind/idmapuid.xml
+++ b/docs-xml/smbdotconf/winbind/idmapuid.xml
@@ -6,14 +6,12 @@
 <synonym>winbind uid</synonym>
 <description>
 	<para>
-	The idmap uid parameter specifies the range of user ids that are 
-	allocated for use in mapping UNIX users to NT user SIDs. This 
-	range of ids should have no existing local
-	or NIS users within it as strange conflicts can occur otherwise.</para>
-
-	<para>See also the <smbconfoption name="idmap backend"/> and
-	<smbconfoption name="idmap config"/> options.
+	The idmap uid parameter specifies the range of user ids for
+	the default idmap configuration. It is now deprecated in favour
+	of <smbconfoption name="idmap config * : range"/>.
 	</para>
+
+	<para>See the <smbconfoption name="idmap config"/> option.</para>
 </description>
 
 <value type="default"></value>
diff --git a/source4/dynconfig/dynconfig.c b/dynconfig/dynconfig.c
index 65e57e0fc17..4bcdab35961 100644
--- a/source4/dynconfig/dynconfig.c
+++ b/dynconfig/dynconfig.c
@@ -1,24 +1,25 @@
-/* 
+/*
    Unix SMB/CIFS implementation.
    Copyright (C) 2001 by Martin Pool <mbp@samba.org>
    Copyright (C) Jim McDonough (jmcd@us.ibm.com)  2003.
    Copyright (C) Stefan Metzmacher	2003
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
+#include "nsswitch/winbind_struct_protocol.h"
 
 /**
  * @file dynconfig.c
@@ -41,6 +42,9 @@
  **/
 
 #include "dynconfig.h"
+#ifdef strdup
+#undef strdup
+#endif
 
 #define DEFINE_DYN_CONFIG_PARAM(name) \
 const char *dyn_##name = name; \
@@ -99,13 +103,15 @@ DEFINE_DYN_CONFIG_PARAM(SMB_PASSWD_FILE)
 DEFINE_DYN_CONFIG_PARAM(PRIVATE_DIR)
 DEFINE_DYN_CONFIG_PARAM(LOCALEDIR)
 DEFINE_DYN_CONFIG_PARAM(NMBDSOCKETDIR)
-
-/* these are not in s3 */
 DEFINE_DYN_CONFIG_PARAM(DATADIR)
 DEFINE_DYN_CONFIG_PARAM(SETUPDIR)
-DEFINE_DYN_CONFIG_PARAM(WINBINDD_SOCKET_DIR)
+DEFINE_DYN_CONFIG_PARAM(WINBINDD_SOCKET_DIR) /* from winbind_struct_protocol.h in s3 autoconf */
+
+/* these are not in s3 */
+#if (_SAMBA_BUILD_ >= 4)
 DEFINE_DYN_CONFIG_PARAM(WINBINDD_PRIVILEGED_SOCKET_DIR)
 DEFINE_DYN_CONFIG_PARAM(NTP_SIGND_SOCKET_DIR)
 DEFINE_DYN_CONFIG_PARAM(PYTHONDIR)
 DEFINE_DYN_CONFIG_PARAM(PYTHONARCHDIR)
 DEFINE_DYN_CONFIG_PARAM(SCRIPTSBINDIR)
+#endif
diff --git a/source4/dynconfig/dynconfig.h b/dynconfig/dynconfig.h
index ee710f3de92..e53cd8ff5fa 100644
--- a/source4/dynconfig/dynconfig.h
+++ b/dynconfig/dynconfig.h
@@ -1,19 +1,19 @@
-/* 
+/*
    Unix SMB/CIFS implementation.
    Copyright (C) 2001 by Martin Pool <mbp@samba.org>
    Copyright (C) Jim McDonough (jmcd@us.ibm.com)  2003.
-   
-   
+
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
diff --git a/source4/dynconfig/wscript b/dynconfig/wscript
index f815e81a2d3..35af13db7ec 100755
--- a/source4/dynconfig/wscript
+++ b/dynconfig/wscript
@@ -5,12 +5,15 @@ from samba_utils import EXPAND_VARIABLES, os_path_relpath
 
 # list of directory options to offer in configure
 dir_options = {
-    'with-piddir'                         : [ '${PREFIX}/var/run', 'where to put pid files' ],
+    'with-piddir'                         : [ '${LOCALSTATEDIR}/run', 'where to put pid files' ],
     'with-privatedir'                     : [ '${PREFIX}/private', 'Where to put sam.ldb and other private files' ],
-    'with-winbindd-socket-dir'            : [ '${PREFIX}/var/lib/winbindd', 'winbind socket directory' ],
-    'with-winbindd-privileged-socket-dir' : [ '${PREFIX}/var/lib/winbindd_privileged', 'winbind privileged socket directory'],
-    'with-ntp-signd-socket-dir'           : [ '${PREFIX}/var/run/ntp_signd', 'NTP signed directory'],
-    'with-lockdir'                        : [ '${PREFIX}/var/locks', 'where to put lock files' ]
+    'with-sockets-dir'                    : [ '${LOCALSTATEDIR}/run', 'sockets directory' ],
+    'with-winbindd-privileged-socket-dir' : [ '${LOCALSTATEDIR}/lib/winbindd_privileged', 'winbind privileged socket directory'],
+    'with-lockdir'                        : [ '${LOCALSTATEDIR}/lock', 'where to put short term disposable state files' ],
+    'with-cachedir'                       : [ '${LOCALSTATEDIR}/cache', 'where to put cache files' ],
+    'with-logfilebase'                    : [ '${LOCALSTATEDIR}', 'Where to put log files' ],
+    'with-pammodulesdir'                  : [ '${LIBDIR}', 'Which directory to use for PAM modules' ],
+    'with-statedir'                       : [ '${LOCALSTATEDIR}/locks', 'where to put persistent state files' ],
     }
 
 # list of cflags to use for dynconfig.c
@@ -22,56 +25,30 @@ dyn_cflags = {
     'CONFIGFILE'                     : '${SYSCONFDIR}/smb.conf',
     'LMHOSTSFILE'                    : '${SYSCONFDIR}/lmhosts',
     'PRIVATE_DIR'                    : '${PRIVATEDIR}',
-    'LOGFILEBASE'                    : '${LOCALSTATEDIR}',
-    'LOCKDIR'                        : '${LOCALSTATEDIR}/locks',
-    'PIDDIR'                         : '${LOCALSTATEDIR}/run',
+    'LOGFILEBASE'                    : '${LOGFILEBASE}',
+    'LOCKDIR'                        : '${LOCKDIR}',
+    'PIDDIR'                         : '${PIDDIR}',
     'DATADIR'                        : '${DATADIR}',
     'LOCALEDIR'                      : '${LOCALEDIR}',
     'SETUPDIR'                       : '${DATADIR}/setup',
-    'WINBINDD_SOCKET_DIR'            : '${WINBINDD_SOCKET_DIR}',
+    'WINBINDD_SOCKET_DIR'            : '${SOCKETS_DIR}/winbindd',
     'WINBINDD_PRIVILEGED_SOCKET_DIR' : '${WINBINDD_PRIVILEGED_SOCKET_DIR}',
-    'NTP_SIGND_SOCKET_DIR'           : '${NTP_SIGND_SOCKET_DIR}',
-    'NCALRPCDIR'                     : '${LOCALSTATEDIR}/ncalrpc',
+    'NTP_SIGND_SOCKET_DIR'           : '${SOCKETS_DIR}/ntp_signd',
+    'NCALRPCDIR'                     : '${SOCKETS_DIR}/ncalrpc',
     'PYTHONDIR'                      : '${PYTHONDIR}',
     'PYTHONARCHDIR'                  : '${PYTHONARCHDIR}',
     'MODULESDIR'                     : '${PREFIX}/modules',
     'INCLUDEDIR'                     : '${PREFIX}/include',
     'PKGCONFIGDIR'                   : '${LIBDIR}/pkgconfig',
     'SWATDIR'                        : '${DATADIR}/swat',
-    'CODEPAGEDIR'                    : '${LIBDIR}/samba',
+    'CODEPAGEDIR'                    : '${DATADIR}/codepages',
     'LIBDIR'                         : '${LIBDIR}',
-    'LIBEXECDIR'                     : '${MODULESDIR}',
-    'STATEDIR'                       : '${LOCALSTATEDIR}',
-    'CACHEDIR'                       : '${LOCKDIR}',
+    'LIBEXECDIR'                     : '${LIBEXECDIR}',
+    'STATEDIR'                       : '${STATEDIR}',
+    'CACHEDIR'                       : '${CACHEDIR}',
     'SMB_PASSWD_FILE'                : '${PRIVATEDIR}/smbpasswd',
-    'NMBDSOCKETDIR'                  : '${LOCALSTATEDIR}/nmbd',
-    }
-
-# changes for when FHS is enabled
-dyn_cflags_fhs = {
-    'BINDIR'                         : '${BINDIR}',
-    'SBINDIR'                        : '${SBINDIR}',
-    'SCRIPTSBINDIR'                  : '${SBINDIR}',
-    'CONFIGDIR'                      : '${SYSCONFDIR}/samba',
-    'CONFIGFILE'                     : '${SYSCONFDIR}/samba/smb.conf',
-    'LMHOSTSFILE'                    : '${SYSCONFDIR}/samba/lmhosts',
-    'PRIVATE_DIR'                    : '${LOCALSTATEDIR}/lib/samba/private',
-    'LOGFILEBASE'                    : '${LOCALSTATEDIR}/log/samba',
-    'LOCKDIR'                        : '${LOCALSTATEDIR}/lib/samba',
-    'PIDDIR'                         : '${LOCALSTATEDIR}/run/samba',
-    'SETUPDIR'                       : '${DATADIR}/samba/setup',
-    'WINBINDD_SOCKET_DIR'            : '${LOCALSTATEDIR}/run/samba/winbindd',
-    'WINBINDD_PRIVILEGED_SOCKET_DIR' : '${LOCALSTATEDIR}/run/samba/winbindd_privileged',
-    'NTP_SIGND_SOCKET_DIR'           : '${LOCALSTATEDIR}/run/samba/ntp_signd',
-    'NCALRPCDIR'                     : '${LOCALSTATEDIR}/run/samba/ncalrpc',
-    'PYTHONARCHDIR'                  : '${PYTHONARCHDIR}',
-    'MODULESDIR'                     : '${LIBDIR}/samba',
-    'LIBEXECDIR'                     : '${MODULESDIR}',
-    'INCLUDEDIR'                     : '${INCLUDEDIR}/samba-4.0',
-    'PKGCONFIGDIR'                   : '${LIBDIR}/pkgconfig',
-    'SWATDIR'                        : '${DATADIR}/swat',
-    'CODEPAGEDIR'                    : '${DATADIR}/samba',
-    'NMBDSOCKETDIR'                  : '${LOCALSTATEDIR}/run/samba/nmbd',
+    'NMBDSOCKETDIR'                  : '${SOCKETS_DIR}/nmbd',
+    'PAMMODULESDIR'                  : '${PAMMODULESDIR}',
     }
 
 def get_varname(v):
@@ -85,9 +62,6 @@ def get_varname(v):
 
 def set_options(opt):
     # get all the basic GNU options from the gnu_dirs tool
-    opt.add_option('--enable-fhs',
-                   help=("Use FHS-compliant paths (default no)"),
-                   action="store_true", dest='ENABLE_FHS', default=False)
     for option in dir_options.keys():
         default = dir_options[option][0]
         help    = dir_options[option][1]
@@ -110,18 +84,6 @@ def configure(conf):
         conf.ASSERT(v != '', "Empty dynconfig value for %s" % f)
         conf.env[f] = v
 
-    if Options.options.ENABLE_FHS:
-        for f in dyn_cflags_fhs.keys():
-            v = EXPAND_VARIABLES(conf, dyn_cflags_fhs[f])
-            conf.ASSERT(v != '', "Empty dynconfig value for %s" % f)
-            conf.env[f] = v
-
-    if (not Options.options.ENABLE_FHS and
-        (conf.env.PREFIX == '/usr' or conf.env.PREFIX == '/usr/local')):
-        print("ERROR: Don't install directly under /usr or /usr/local without using the FHS option (--enable-fhs)")
-        sys.exit(1)
-
-
 def dynconfig_cflags(bld, list=None):
     '''work out the extra CFLAGS for dynconfig.c'''
     cflags = []
diff --git a/examples/LDAP/samba-nds.schema b/examples/LDAP/samba-nds.schema
index 7ba8665b8c9..369670bca2c 100644
--- a/examples/LDAP/samba-nds.schema
+++ b/examples/LDAP/samba-nds.schema
@@ -35,7 +35,7 @@ attributeTypes: ( 1.3.6.1.4.1.7165.2.1.25 NAME 'sambaNTPassword' DESC 'MD4 hash
 dn: cn=schema
 changetype: modify
 add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.26 NAME 'sambaAcctFlags' DESC 'Account Flags' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{16} SINGLE-VALUE )
+attributeTypes: ( 1.3.6.1.4.1.7165.2.1.26 NAME 'sambaAcctFlags' DESC 'Account Flags' EQUALITY caseIgnoreIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{16} SINGLE-VALUE )
 
 ##
 ## Password timestamps & policies
@@ -128,7 +128,7 @@ attributeTypes: ( 1.3.6.1.4.1.7165.2.1.47 NAME 'sambaMungedDial' DESC 'Base64 en
 dn: cn=schema
 changetype: modify
 add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.54 NAME 'sambaPasswordHistory' DESC 'Concatenated MD4 hashes of the unicode passwords used on this account' EQUALITY caseIgnoreIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{1024} )
+attributeTypes: ( 1.3.6.1.4.1.7165.2.1.54 NAME 'sambaPasswordHistory' DESC 'Concatenated MD5 hashes of the salted NT passwords used on this account' EQUALITY caseIgnoreIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{1024} )
 
 ##
 ## SID, of any type
@@ -137,7 +137,7 @@ attributeTypes: ( 1.3.6.1.4.1.7165.2.1.54 NAME 'sambaPasswordHistory' DESC 'Conc
 dn: cn=schema
 changetype: modify
 add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.20 NAME 'sambaSID' DESC 'Security ID' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64} SINGLE-VALUE )
+attributeTypes: ( 1.3.6.1.4.1.7165.2.1.20 NAME 'sambaSID' DESC 'Security ID' EQUALITY caseIgnoreIA5Match SUBSTR caseExactIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64} SINGLE-VALUE )
 
 ##
 ## Primary group SID, compatible with ntSid
@@ -287,47 +287,13 @@ attributeTypes: ( 1.3.6.1.4.1.7165.2.1.67 NAME 'sambaRefuseMachinePwdChange' DES
 dn: cn=schema
 changetype: modify
 add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.70 NAME 'sambaTrustType' DESC 'Type of trust' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )
+attributeTypes: ( 1.3.6.1.4.1.7165.2.1.68 NAME 'sambaClearTextPassword' DESC 'Clear text password (used for trusted domain passwords)' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 )
 
 dn: cn=schema
 changetype: modify
 add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.71 NAME 'sambaTrustAttributes' DESC 'Trust attributes for a trusted domain' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )
-
-dn: cn=schema
-changetype: modify
-add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.72 NAME 'sambaTrustDirection' DESC 'Direction of a trust' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )
-
-dn: cn=schema
-changetype: modify
-add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.73 NAME 'sambaTrustPartner' DESC 'Fully qualified name of the domain with which a trust exists' EQUALITY caseIgnoreMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} )
-
-dn: cn=schema
-changetype: modify
-add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.74 NAME 'sambaFlatName' DESC 'NetBIOS name of a domain' EQUALITY caseIgnoreMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} )
-
-dn: cn=schema
-changetype: modify
-add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.75 NAME 'sambaTrustAuthOutgoing' DESC 'Authentication information for the outgoing portion of a trust' EQUALITY caseExactMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{1050} )
-
-dn: cn=schema
-changetype: modify
-add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.76 NAME 'sambaTrustAuthIncoming' DESC 'Authentication information for the incoming portion of a trust' EQUALITY caseExactMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{1050} )
-
-dn: cn=schema
-changetype: modify
-add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.77 NAME 'sambaSecurityIdentifier' DESC 'SID of a trusted domain' EQUALITY caseIgnoreIA5Match SUBSTR caseExactIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64} SINGLE-VALUE )
-
-dn: cn=schema
-changetype: modify
-add: attributetypes
-attributeTypes: ( 1.3.6.1.4.1.7165.2.1.78 NAME 'sambaTrustForestTrustInfo' DESC 'Forest trust information for a trusted domain object' EQUALITY caseExactMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{1050} )
+attributeTypes:  ( 1.3.6.1.4.1.7165.2.1.69 NAME 'sambaPreviousClearTextPassword' DESC 'Previous clear text password (used for trusted domain passwords)' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 )
+ 
 
 #######################################################################
 ##              objectClasses used by Samba 3.0 schema               ##
@@ -357,6 +323,16 @@ add: objectClasses
 objectClasses: ( 1.3.6.1.4.1.7165.2.2.14 NAME 'sambaTrustPassword' DESC 'Samba Trust Password' SUP top STRUCTURAL MUST ( sambaDomainName $ sambaNTPassword $ sambaTrustFlags ) MAY ( sambaSID $ sambaPwdLastSet ))
 
 ##
+## Trust password for trusted domains
+## (to be stored beneath the trusting sambaDomain object in the DIT)
+##
+
+dn: cn=schema
+changetype: modify
+add: objectClasses
+objectClasses: ( 1.3.6.1.4.1.7165.2.2.15 NAME 'sambaTrustedDomainPassword' SUP top STRUCTURAL DESC 'Samba Trusted Domain Password' MUST ( sambaDomainName $ sambaSID $ sambaClearTextPassword $ sambaPwdLastSet ) MAY  ( sambaPreviousClearTextPassword ))
+
+##
 ## Whole-of-domain info
 ##
 
@@ -372,22 +348,22 @@ objectClasses: ( 1.3.6.1.4.1.7165.2.2.5 NAME 'sambaDomain' DESC 'Samba Domain In
 dn: cn=schema
 changetype: modify
 add: objectClasses
-objectClasses: ( 1.3.6.1.4.1.7165.2.2.7 NAME 'sambaUnixIdPool' DESC 'Pool for allocating UNIX uids/gids' SUP top AUXILIARY MUST ( uidNumber $ gidNumber ))
+objectClasses: ( 1.3.6.1.4.1.7165.1.2.2.7 NAME 'sambaUnixIdPool' DESC 'Pool for allocating UNIX uids/gids' SUP top AUXILIARY MUST ( uidNumber $ gidNumber ))
 
 dn: cn=schema
 changetype: modify
 add: objectClasses
-objectClasses: ( 1.3.6.1.4.1.7165.2.2.8 NAME 'sambaIdmapEntry' DESC 'Mapping from a SID to an ID' SUP top AUXILIARY MUST ( sambaSID ) MAY ( uidNumber $ gidNumber ))
+objectClasses: ( 1.3.6.1.4.1.7165.1.2.2.8 NAME 'sambaIdmapEntry' DESC 'Mapping from a SID to an ID' SUP top AUXILIARY MUST ( sambaSID ) MAY ( uidNumber $ gidNumber ))
 
 dn: cn=schema
 changetype: modify
 add: objectClasses
-objectClasses: ( 1.3.6.1.4.1.7165.2.2.9 NAME 'sambaSidEntry' DESC 'Structural Class for a SID' SUP top STRUCTURAL MUST ( sambaSID ))
+objectClasses: ( 1.3.6.1.4.1.7165.1.2.2.9 NAME 'sambaSidEntry' DESC 'Structural Class for a SID' SUP top STRUCTURAL MUST ( sambaSID ))
 
 dn: cn=schema
 changetype: modify
 add: objectClasses
-objectClasses: ( 1.3.6.1.4.1.7165.2.2.10 NAME 'sambaConfig' DESC 'Samba Configuration Section' SUP top AUXILIARY MAY ( description ))
+objectClasses: ( 1.3.6.1.4.1.7165.1.2.2.10 NAME 'sambaConfig' DESC 'Samba Configuration Section' SUP top AUXILIARY MAY ( description ))
 
 dn: cn=schema
 changetype: modify
@@ -403,8 +379,3 @@ dn: cn=schema
 changetype: modify
 add: objectClasses
 objectClasses: ( 1.3.6.1.4.1.7165.2.2.13 NAME 'sambaPrivilege' DESC 'Samba Privilege' SUP top AUXILIARY MUST ( sambaSID ) MAY ( sambaPrivilegeList ))
-
-dn: cn=schema
-changetype: modify
-add: objectClasses
-objectClasses: ( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL DESC 'Samba Trusted Domain Object' MUST ( cn ) MAY ( sambaTrustTyp e $ sambaTrustAttributes $ sambaTrustDirection $ sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $ sambaTrustAuthIncoming $ sambaSecurityIdentifier $ sambaTrustForestTrustInfo ) )
diff --git a/examples/LDAP/samba-schema-FDS.ldif b/examples/LDAP/samba-schema-FDS.ldif
index 981dadaeec0..7513a49fe73 100644
--- a/examples/LDAP/samba-schema-FDS.ldif
+++ b/examples/LDAP/samba-schema-FDS.ldif
@@ -183,4 +183,4 @@ objectClasses: ( 1.3.6.1.4.1.7165.2.2.15 NAME 'sambaTrustedDomainPassword' SUP t
 ##
 ## used for IPA_ldapsam
 ##
-objectClasses: ( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL DESC 'Samba Trusted Domain Object' MUST ( cn ) MAY ( sambaTrustTyp e $ sambaTrustAttributes $ sambaTrustDirection $ sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $ sambaTrustAuthIncoming $ sambaSecurityIdentifier $ sambaTrustForestTrustInfo ) )
+objectClasses: ( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL DESC 'Samba Trusted Domain Object' MUST ( cn ) MAY ( sambaTrustType $ sambaTrustAttributes $ sambaTrustDirection $ sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $ sambaTrustAuthIncoming $ sambaSecurityIdentifier $ sambaTrustForestTrustInfo ) )
diff --git a/examples/LDAP/samba-schema-netscapeds5.x b/examples/LDAP/samba-schema-netscapeds5.x
index 8bf0e93269d..55c2aff601f 100644
--- a/examples/LDAP/samba-schema-netscapeds5.x
+++ b/examples/LDAP/samba-schema-netscapeds5.x
@@ -36,7 +36,7 @@ objectClasses: ( 1.3.6.1.4.1.7165.2.2.7 NAME 'sambaUnixIdPool' SUP top AUXILIARY
 objectClasses: ( 1.3.6.1.4.1.7165.2.2.8 NAME 'sambaIdmapEntry' SUP top AUXILIARY DESC 'Mapping from a SID to an ID' MUST ( sambaSID ) MAY ( uidNumber $ gidNumber )  X-ORIGIN 'user defined' )
 objectClasses: ( 1.3.6.1.4.1.7165.2.2.9 NAME 'sambaSidEntry' SUP top STRUCTURAL DESC 'Structural Class for a SID' MUST ( sambaSID )  X-ORIGIN 'user defined' )
 objectClasses: ( 1.3.6.1.4.1.7165.2.2.15 NAME 'sambaTrustedDomainPassword' SUP top STRUCTURAL DESC 'Samba Trusted Domain Password' MUST ( sambaDomainName $ sambaSID $ sambaClearTextPassword $ sambaPwdLastSet ) MAY  ( sambaPreviousClearTextPassword ) X-ORIGIN 'user defined')
-objectClasses: ( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL DESC 'Samba Trusted Domain Object' MUST ( cn ) MAY ( sambaTrustTyp e $ sambaTrustAttributes $ sambaTrustDirection $ sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $ sambaTrustAuthIncoming $ sambaSecurityIdentifier $ sambaTrustForestTrustInfo ) X-ORIGIN 'user defined' )
+objectClasses: ( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL DESC 'Samba Trusted Domain Object' MUST ( cn ) MAY ( sambaTrustType $ sambaTrustAttributes $ sambaTrustDirection $ sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $ sambaTrustAuthIncoming $ sambaSecurityIdentifier $ sambaTrustForestTrustInfo ) X-ORIGIN 'user defined' )
 attributeTypes: ( 1.3.6.1.4.1.7165.2.1.24 NAME 'sambaLMPassword' DESC 'LanManager Password' EQUALITY caseIgnoreIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{32} SINGLE-VALUE X-ORIGIN 'user defined' )
 attributeTypes: ( 1.3.6.1.4.1.7165.2.1.25 NAME 'sambaNTPassword' DESC 'MD4 hash of the unicode password' EQUALITY caseIgnoreIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{32} SINGLE-VALUE X-ORIGIN 'user defined' )
 attributeTypes: ( 1.3.6.1.4.1.7165.2.1.26 NAME 'sambaAcctFlags'	DESC 'Account Flags' EQUALITY caseIgnoreIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{16} SINGLE-VALUE X-ORIGIN 'user defined' )
diff --git a/examples/LDAP/samba.schema b/examples/LDAP/samba.schema
index df5267a9c60..716c191289b 100644
--- a/examples/LDAP/samba.schema
+++ b/examples/LDAP/samba.schema
@@ -621,10 +621,10 @@ objectclass ( 1.3.6.1.4.1.7165.2.2.12 NAME 'sambaConfigOption' SUP top STRUCTURA
 ##
 ## used for IPA_ldapsam
 ##
-objectclasses ( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL
+objectclass ( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL
 	DESC 'Samba Trusted Domain Object'
 	MUST ( cn )
-	MAY ( sambaTrustTyp e $ sambaTrustAttributes $ sambaTrustDirection $
+	MAY ( sambaTrustType $ sambaTrustAttributes $ sambaTrustDirection $
 	      sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $
 	      sambaTrustAuthIncoming $ sambaSecurityIdentifier $
 	      sambaTrustForestTrustInfo) )
diff --git a/examples/LDAP/samba.schema.oc.IBM-DS b/examples/LDAP/samba.schema.oc.IBM-DS
index ba31a31aee9..1301ad4cc74 100644
--- a/examples/LDAP/samba.schema.oc.IBM-DS
+++ b/examples/LDAP/samba.schema.oc.IBM-DS
@@ -20,4 +20,4 @@ objectclasses=( 1.3.6.1.4.1.7165.2.2.12 NAME 'sambaConfigOption' SUP top STRUCTU
 
 objectclasses=( 1.3.6.1.4.1.7165.2.2.14 NAME 'sambaTrustPassword' SUP top STRUCTURAL DESC 'Samba Trust Password' MUST ( sambaDomainName $ sambaNTPassword $ sambaTrustFlags ) MAY ( sambaSID $ sambaPwdLastSet ))
 
-objectclasses=( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL DESC 'Samba Trusted Domain Object' MUST ( cn ) MAY ( sambaTrustTyp e $ sambaTrustAttributes $ sambaTrustDirection $ sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $ sambaTrustAuthIncoming $ sambaSecurityIdentifier $ sambaTrustForestTrustInfo ) )
+objectclasses=( 1.3.6.1.4.1.7165.2.2.16 NAME 'sambaTrustedDomain' SUP top STRUCTURAL DESC 'Samba Trusted Domain Object' MUST ( cn ) MAY ( sambaTrustType $ sambaTrustAttributes $ sambaTrustDirection $ sambaTrustPartner $ sambaFlatName $ sambaTrustAuthOutgoing $ sambaTrustAuthIncoming $ sambaSecurityIdentifier $ sambaTrustForestTrustInfo ) )
diff --git a/examples/VFS/shadow_copy_test.c b/examples/VFS/shadow_copy_test.c
index fab7007046f..d4da23846fc 100644
--- a/examples/VFS/shadow_copy_test.c
+++ b/examples/VFS/shadow_copy_test.c
@@ -59,7 +59,7 @@ static int test_get_shadow_copy_data(vfs_handle_struct *handle, files_struct *fs
 	
 	if (labels) {	
 		if (num) {
-			shadow_copy_data->labels = TALLOC_ZERO_ARRAY(shadow_copy_data->mem_ctx,SHADOW_COPY_LABEL,num);
+			shadow_copy_data->labels = talloc_zero_array(shadow_copy_data->mem_ctx,SHADOW_COPY_LABEL,num);
 		} else {
 			shadow_copy_data->labels = NULL;
 		}
diff --git a/examples/VFS/skel_opaque.c b/examples/VFS/skel_opaque.c
index f60f0fe1628..f8399935d38 100644
--- a/examples/VFS/skel_opaque.c
+++ b/examples/VFS/skel_opaque.c
@@ -293,7 +293,7 @@ static int skel_chdir(vfs_handle_struct *handle,  const char *path)
 	return -1;
 }
 
-static char *skel_getwd(vfs_handle_struct *handle,  char *buf)
+static char *skel_getwd(vfs_handle_struct *handle)
 {
 	errno = ENOSYS;
 	return NULL;
diff --git a/examples/VFS/skel_transparent.c b/examples/VFS/skel_transparent.c
index f06e02afe5f..7c91a615b1f 100644
--- a/examples/VFS/skel_transparent.c
+++ b/examples/VFS/skel_transparent.c
@@ -280,9 +280,9 @@ static int skel_chdir(vfs_handle_struct *handle,  const char *path)
 	return SMB_VFS_NEXT_CHDIR(handle, path);
 }
 
-static char *skel_getwd(vfs_handle_struct *handle,  char *buf)
+static char *skel_getwd(vfs_handle_struct *handle)
 {
-	return SMB_VFS_NEXT_GETWD(handle, buf);
+	return SMB_VFS_NEXT_GETWD(handle);
 }
 
 static int skel_ntimes(vfs_handle_struct *handle,
diff --git a/howto-ol-backend-s4.txt b/howto-ol-backend-s4.txt
deleted file mode 100644
index 04b8ab68212..00000000000
--- a/howto-ol-backend-s4.txt
+++ /dev/null
@@ -1,131 +0,0 @@
-Samba4  OpenLDAP-Backend Quick-Howto
-====================================
-
-oliver@itc.li  -  August 2009
-
-
-This Mini-Howto describes in a very simplified way 
-how to setup Samba 4 (S4) (pre)Alpha 13 with the
-OpenLDAP (OL) -Backend.
-Use of OpenLDAP from CVS after 2010-04-22 is required
-
-The current instructions are at:
-
-http://wiki.samba.org/index.php/Samba4/LDAP_Backend/OpenLDAP
-
-1.) Download and compile OpenLDAP. 
-
-The use of (older) Versions shipped with Distributions often
-causes trouble, so dont use them. Configure-Example:
-
-#> ./configure --enable-overlays=yes --with-tls=yes --with-cyrus-sasl=yes
-#> make depend && make && make install
-
-Note: openssl and cyrus-sasl libs should be installed
-before compilation.
-
-
-
-
-2.) Final provision:
-
-(you can add --adminpass=<yourpass> to the parameters,
-otherwise a random password will be generated for 
-cn=Administrator,cn=users,<Your Base-DN>):
-
-#> setup/provision \
-   --ldap-backend-type=openldap \
-   --slapd-path="/usr/local/libexec/slapd"
-   --username=samba-admin --realm=ldap.local.site \
-   --domain=LDAP --server-role='domain controller'\
-   --adminpass=linux
-
-At the End of the final provision you should get
-the following output (only partial here). Read it carefully:
-
---------
-...
-A Kerberos configuration suitable for Samba 4 has been generated at /usr/local/samba/private/krb5.conf
-
-Use later the following commandline to start slapd, then Samba:
-/usr/local/libexec/slapd -f /usr/local/samba/private/ldap/slapd.conf -h ldapi://%2Fusr%2Flocal%2Fsamba%2Fprivate%2Fldap%2Fldapi
-
-This slapd-Commandline is also stored under: /usr/local/samba/private/ldap/slapd_command_file.sh
-Please install the phpLDAPadmin configuration located at /usr/local/samba/private/phpldapadmin-config.php into /etc/phpldapadmin/config.php
-Once the above files are installed, your Samba4 server will be ready to use
-Server Role:    domain controller
-Hostname:       ldapmaster
-NetBIOS Domain: LDAP
-DNS Domain:     ldap.local.site
-DOMAIN SID:     S-1-5-21-429312062-2328781357-2130201529
-Admin password: linux
-
---------
-
-Our slapd in "provision-mode" wiil be shut down automatically 
-after final provision ends.
-
-
-3.) Run OL and S4:
-
-After you completed the other necessary steps (krb and named-specific),
-start first OL with the commandline displayed in the output under (3),
-(remember: the slapd-Commandline is also stored in the file ../slapd_command_file.sh)
-then S4.
-
-
-
-4.) Special Setup-Types:
-
-OpenLDAP-Online Configuration is now in use by default (olc):
-
-The olc will be setup automatically
-under ../private/slapd.d/.
-olc is accessible via "cn=samba-admin,cn=samba" and Base-DN "cn=config"
-olc is intended primarily for use in conjunction with MMR
-
-Attention: You have to start OL with the commandline
-displayed in the output under (3), but you have to set a 
-listening port of slapd manually:
-
-(e.g. -h ldap://ldapmaster.ldap.local.site:9000)
-
-Attention: You _should_not_ edit the olc-Sections
-"config" and "ldif", as these are vital to the olc itself.
-
-
-b) MultiMaster-Configuration (MMR):
-Use the provision Parameter:
-
- --ol-mmr-urls=<list of whitespace separated ldap-urls (and Ports <> 389!).
-
-e.g.:
---ol-mmr-urls="ldap://ldapmaster1.ldap.local.site:9000 \ 
-   ldap://ldapmaster2.ldap.local.site:9000"
-
-Attention: You have to start OL with the commandline
-displayed in the output under (3), but you have to set a 
-listening port of slapd manually
-(e.g. -h ldap://ldapmaster1.ldap.local.site:9000)
-
-The Ports must be different from 389, as these are occupied by S4.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/install_with_python.sh b/install_with_python.sh
new file mode 100644
index 00000000000..d7ae55da123
--- /dev/null
+++ b/install_with_python.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+# this script installs a private copy of python in the same prefix as Samba
+
+if [ $# -lt 1 ]; then
+cat <<EOF
+Usage: install_with_python.sh PREFIX [CONFIGURE OPTIONS]
+EOF
+exit 1;
+fi
+
+PREFIX="$1"
+shift
+
+PATH=$PREFIX/python/bin:$PATH
+export PATH
+
+VERSION="Python-2.6.5"
+
+do_install_python() {
+       mkdir -p python_install || exit 1
+       rsync -avz samba.org::ftp/tridge/python/$VERSION.tar python_install || exit 1
+       cd python_install || exit 1;
+       rm -rf $VERSION || exit 1
+       tar -xf $VERSION.tar || exit 1
+       cd $VERSION || exit 1
+       ./configure --prefix=$PREFIX/python --enable-shared --disable-ipv6 || exit 1
+       make || exit 1
+       make install || exit 1
+       cd ../.. || exit 1
+       rm -rf python_install || exit 1
+}
+
+if ! test -d $PREFIX/python; then
+   # needs to be installed
+   do_install_python
+fi
+
+`dirname $0`/configure --prefix=$PREFIX $@ || exit 1
+make -j || exit 1
+make install || exit 1
diff --git a/lib/addns/dnsgss.c b/lib/addns/dnsgss.c
index c9037417da9..19b734a6a31 100644
--- a/lib/addns/dnsgss.c
+++ b/lib/addns/dnsgss.c
@@ -92,7 +92,7 @@ static DNS_ERROR dns_negotiate_gss_ctx_int( TALLOC_CTX *mem_ctx,
 	DNS_ERROR err;
 
 	gss_OID_desc krb5_oid_desc =
-		{ 9, (char *)"\x2a\x86\x48\x86\xf7\x12\x01\x02\x02" };
+		{ 9, (const char *)"\x2a\x86\x48\x86\xf7\x12\x01\x02\x02" };
 
 	*ctx = GSS_C_NO_CONTEXT;
 	input_ptr = NULL;
@@ -222,7 +222,7 @@ DNS_ERROR dns_negotiate_sec_ctx( const char *target_realm,
 	gss_name_t targ_name;
 
 	gss_OID_desc nt_host_oid_desc =
-		{10, (char *)"\x2a\x86\x48\x86\xf7\x12\x01\x02\x02\x01"};
+		{10, (const char *)"\x2a\x86\x48\x86\xf7\x12\x01\x02\x02\x01"};
 
 	TALLOC_CTX *mem_ctx;
 
diff --git a/lib/addns/dnsmarshall.c b/lib/addns/dnsmarshall.c
index 5530290c57a..59d6470f345 100644
--- a/lib/addns/dnsmarshall.c
+++ b/lib/addns/dnsmarshall.c
@@ -39,7 +39,7 @@ struct dns_buffer *dns_create_buffer(TALLOC_CTX *mem_ctx)
 	 */
 	result->size = 2;
 
-	if (!(result->data = TALLOC_ARRAY(result, uint8, result->size))) {
+	if (!(result->data = talloc_array(result, uint8, result->size))) {
 		TALLOC_FREE(result);
 		return NULL;
 	}
@@ -78,7 +78,7 @@ void dns_marshall_buffer(struct dns_buffer *buf, const uint8 *data,
 
 		new_size += (64 - (new_size % 64));
 
-		if (!(new_data = TALLOC_REALLOC_ARRAY(buf, buf->data, uint8,
+		if (!(new_data = talloc_realloc(buf, buf->data, uint8,
 						      new_size))) {
 			buf->error = ERROR_DNS_NO_MEMORY;
 			return;
@@ -223,7 +223,7 @@ static void dns_unmarshall_label(TALLOC_CTX *mem_ctx,
 
 	label->len = len;
 
-	if (!(label->label = TALLOC_ARRAY(label, char, len+1))) {
+	if (!(label->label = talloc_array(label, char, len+1))) {
 		buf->error = ERROR_DNS_NO_MEMORY;
 		goto error;
 	}
@@ -329,7 +329,7 @@ static void dns_unmarshall_rr(TALLOC_CTX *mem_ctx,
 	if (!(ERR_DNS_IS_OK(buf->error))) return;
 
 	if (r->data_length != 0) {
-		if (!(r->data = TALLOC_ARRAY(r, uint8, r->data_length))) {
+		if (!(r->data = talloc_array(r, uint8, r->data_length))) {
 			buf->error = ERROR_DNS_NO_MEMORY;
 			return;
 		}
@@ -390,7 +390,7 @@ DNS_ERROR dns_unmarshall_request(TALLOC_CTX *mem_ctx,
 	uint16 i;
 	DNS_ERROR err;
 
-	if (!(req = TALLOC_ZERO_P(mem_ctx, struct dns_request))) {
+	if (!(req = talloc_zero(mem_ctx, struct dns_request))) {
 		return ERROR_DNS_NO_MEMORY;
 	}
 
@@ -406,22 +406,22 @@ DNS_ERROR dns_unmarshall_request(TALLOC_CTX *mem_ctx,
 	err = ERROR_DNS_NO_MEMORY;
 
 	if ((req->num_questions != 0) &&
-	    !(req->questions = TALLOC_ARRAY(req, struct dns_question *,
+	    !(req->questions = talloc_array(req, struct dns_question *,
 					    req->num_questions))) {
 		goto error;
 	}
 	if ((req->num_answers != 0) &&
-	    !(req->answers = TALLOC_ARRAY(req, struct dns_rrec *,
+	    !(req->answers = talloc_array(req, struct dns_rrec *,
 					  req->num_answers))) {
 		goto error;
 	}
 	if ((req->num_auths != 0) &&
-	    !(req->auths = TALLOC_ARRAY(req, struct dns_rrec *,
+	    !(req->auths = talloc_array(req, struct dns_rrec *,
 					req->num_auths))) {
 		goto error;
 	}
 	if ((req->num_additionals != 0) &&
-	    !(req->additionals = TALLOC_ARRAY(req, struct dns_rrec *,
+	    !(req->additionals = talloc_array(req, struct dns_rrec *,
 					      req->num_additionals))) {
 		goto error;
 	}
diff --git a/lib/addns/dnsrecord.c b/lib/addns/dnsrecord.c
index 559c2644d44..2240d08fb91 100644
--- a/lib/addns/dnsrecord.c
+++ b/lib/addns/dnsrecord.c
@@ -31,8 +31,8 @@ DNS_ERROR dns_create_query( TALLOC_CTX *mem_ctx, const char *name,
 	struct dns_question *q;
 	DNS_ERROR err;
 
-	if (!(req = TALLOC_ZERO_P(mem_ctx, struct dns_request)) ||
-	    !(req->questions = TALLOC_ARRAY(req, struct dns_question *, 1)) ||
+	if (!(req = talloc_zero(mem_ctx, struct dns_request)) ||
+	    !(req->questions = talloc_array(req, struct dns_question *, 1)) ||
 	    !(req->questions[0] = talloc(req->questions,
 					 struct dns_question))) {
 		TALLOC_FREE(req);
@@ -64,8 +64,8 @@ DNS_ERROR dns_create_update( TALLOC_CTX *mem_ctx, const char *name,
 	struct dns_zone *z;
 	DNS_ERROR err;
 
-	if (!(req = TALLOC_ZERO_P(mem_ctx, struct dns_update_request)) ||
-	    !(req->zones = TALLOC_ARRAY(req, struct dns_zone *, 1)) ||
+	if (!(req = talloc_zero(mem_ctx, struct dns_update_request)) ||
+	    !(req->zones = talloc_array(req, struct dns_zone *, 1)) ||
 	    !(req->zones[0] = talloc(req->zones, struct dns_zone))) {
 		TALLOC_FREE(req);
 		return ERROR_DNS_NO_MEMORY;
@@ -131,8 +131,8 @@ DNS_ERROR dns_create_a_record(TALLOC_CTX *mem_ctx, const char *host,
 		return ERROR_DNS_SUCCESS;
 	}
 
-	ip = ((struct sockaddr_in *)pss)->sin_addr;
-	if (!(data = (uint8 *)TALLOC_MEMDUP(mem_ctx, (const void *)&ip.s_addr,
+	ip = ((const struct sockaddr_in *)pss)->sin_addr;
+	if (!(data = (uint8 *)talloc_memdup(mem_ctx, (const void *)&ip.s_addr,
 					    sizeof(ip.s_addr)))) {
 		return ERROR_DNS_NO_MEMORY;
 	}
@@ -240,7 +240,7 @@ DNS_ERROR dns_unmarshall_tkey_record(TALLOC_CTX *mem_ctx, struct dns_rrec *rec,
 	if (!ERR_DNS_IS_OK(buf.error)) goto error;
 
 	if (tkey->key_length) {
-		if (!(tkey->key = TALLOC_ARRAY(tkey, uint8, tkey->key_length))) {
+		if (!(tkey->key = talloc_array(tkey, uint8, tkey->key_length))) {
 			buf.error = ERROR_DNS_NO_MEMORY;
 			goto error;
 		}
@@ -308,7 +308,7 @@ DNS_ERROR dns_add_rrec(TALLOC_CTX *mem_ctx, struct dns_rrec *rec,
 {
 	struct dns_rrec **new_records;
 
-	if (!(new_records = TALLOC_REALLOC_ARRAY(mem_ctx, *records,
+	if (!(new_records = talloc_realloc(mem_ctx, *records,
 						 struct dns_rrec *,
 						 (*num_records)+1))) {
 		return ERROR_DNS_NO_MEMORY;
diff --git a/lib/addns/dnssock.c b/lib/addns/dnssock.c
index 42b4e2d40f0..aaeb3f03fa6 100644
--- a/lib/addns/dnssock.c
+++ b/lib/addns/dnssock.c
@@ -250,7 +250,7 @@ static DNS_ERROR dns_receive_tcp(TALLOC_CTX *mem_ctx,
 	DNS_ERROR err;
 	uint16 len;
 
-	if (!(buf = TALLOC_ZERO_P(mem_ctx, struct dns_buffer))) {
+	if (!(buf = talloc_zero(mem_ctx, struct dns_buffer))) {
 		return ERROR_DNS_NO_MEMORY;
 	}
 
@@ -262,7 +262,7 @@ static DNS_ERROR dns_receive_tcp(TALLOC_CTX *mem_ctx,
 	buf->size = ntohs(len);
 
 	if (buf->size) {
-		if (!(buf->data = TALLOC_ARRAY(buf, uint8, buf->size))) {
+		if (!(buf->data = talloc_array(buf, uint8, buf->size))) {
 			TALLOC_FREE(buf);
 			return ERROR_DNS_NO_MEMORY;
 		}
@@ -287,7 +287,7 @@ static DNS_ERROR dns_receive_udp(TALLOC_CTX *mem_ctx,
 	struct dns_buffer *buf;
 	ssize_t received;
 
-	if (!(buf = TALLOC_ZERO_P(mem_ctx, struct dns_buffer))) {
+	if (!(buf = talloc_zero(mem_ctx, struct dns_buffer))) {
 		return ERROR_DNS_NO_MEMORY;
 	}
 
@@ -295,7 +295,7 @@ static DNS_ERROR dns_receive_udp(TALLOC_CTX *mem_ctx,
 	 * UDP based DNS can only be 512 bytes
 	 */
 
-	if (!(buf->data = TALLOC_ARRAY(buf, uint8, 512))) {
+	if (!(buf->data = talloc_array(buf, uint8, 512))) {
 		TALLOC_FREE(buf);
 		return ERROR_DNS_NO_MEMORY;
 	}
diff --git a/lib/addns/dnsutils.c b/lib/addns/dnsutils.c
index 37b862c7f07..43305a98730 100644
--- a/lib/addns/dnsutils.c
+++ b/lib/addns/dnsutils.c
@@ -53,7 +53,7 @@ static DNS_ERROR LabelList( TALLOC_CTX *mem_ctx,
 		return ERROR_DNS_INVALID_NAME;
 	}
 
-	if (!(result = TALLOC_ZERO_P(mem_ctx, struct dns_domain_label))) {
+	if (!(result = talloc_zero(mem_ctx, struct dns_domain_label))) {
 		return ERROR_DNS_NO_MEMORY;
 	}
 
@@ -138,7 +138,7 @@ char *dns_generate_keyname( TALLOC_CTX *mem_ctx )
 	/*
 	 * uuid_unparse gives 36 bytes plus '\0'
 	 */
-	if (!(result = TALLOC_ARRAY(mem_ctx, char, 37))) {
+	if (!(result = talloc_array(mem_ctx, char, 37))) {
 		return NULL;
 	}
 
diff --git a/lib/async_req/async_sock.c b/lib/async_req/async_sock.c
index 86053d94e87..dfb1a1cdbd7 100644
--- a/lib/async_req/async_sock.c
+++ b/lib/async_req/async_sock.c
@@ -386,6 +386,7 @@ struct writev_state {
 	int count;
 	size_t total_size;
 	uint16_t flags;
+	bool err_on_readability;
 };
 
 static void writev_trigger(struct tevent_req *req, void *private_data);
@@ -413,10 +414,8 @@ struct tevent_req *writev_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
 	if (state->iov == NULL) {
 		goto fail;
 	}
-	state->flags = TEVENT_FD_WRITE;
-	if (err_on_readability) {
-		state->flags |= TEVENT_FD_READ;
-	}
+	state->flags = TEVENT_FD_WRITE|TEVENT_FD_READ;
+	state->err_on_readability = err_on_readability;
 
 	if (queue == NULL) {
 		struct tevent_fd *fde;
@@ -462,8 +461,35 @@ static void writev_handler(struct tevent_context *ev, struct tevent_fd *fde,
 	to_write = 0;
 
 	if ((state->flags & TEVENT_FD_READ) && (flags & TEVENT_FD_READ)) {
-		tevent_req_error(req, EPIPE);
-		return;
+		int ret, value;
+
+		if (state->err_on_readability) {
+			/* Readable and the caller wants an error on read. */
+			tevent_req_error(req, EPIPE);
+			return;
+		}
+
+		/* Might be an error. Check if there are bytes to read */
+		ret = ioctl(state->fd, FIONREAD, &value);
+		/* FIXME - should we also check
+		   for ret == 0 and value == 0 here ? */
+		if (ret == -1) {
+			/* There's an error. */
+			tevent_req_error(req, EPIPE);
+			return;
+		}
+		/* A request for TEVENT_FD_READ will succeed from now and
+		   forevermore until the bytes are read so if there was
+		   an error we'll wait until we do read, then get it in
+		   the read callback function. Until then, remove TEVENT_FD_READ
+		   from the flags we're waiting for. */
+		state->flags &= ~TEVENT_FD_READ;
+		TEVENT_FD_NOT_READABLE(fde);
+
+		/* If not writable, we're done. */
+		if (!(flags & TEVENT_FD_WRITE)) {
+			return;
+		}
 	}
 
 	for (i=0; i<state->count; i++) {
diff --git a/lib/ccan/array_size/LICENSE b/lib/ccan/array_size/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/array_size/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/array_size/_info b/lib/ccan/array_size/_info
new file mode 100644
index 00000000000..af7ef1cfd29
--- /dev/null
+++ b/lib/ccan/array_size/_info
@@ -0,0 +1,46 @@
+#include <stdio.h>
+#include <string.h>
+#include "config.h"
+
+/**
+ * array_size - routine for safely deriving the size of a visible array.
+ *
+ * This provides a simple ARRAY_SIZE() macro, which (given a good compiler)
+ * will also break compile if you try to use it on a pointer.
+ *
+ * This can ensure your code is robust to changes, without needing a gratuitous
+ * macro or constant.
+ *
+ * Example:
+ *	// Outputs "Initialized 32 values"
+ *	#include <ccan/array_size/array_size.h>
+ *	#include <stdlib.h>
+ *	#include <stdio.h>
+ *
+ *	// We currently use 32 random values.
+ *	static unsigned int vals[32];
+ *
+ *	int main(void)
+ *	{
+ *		unsigned int i;
+ *		for (i = 0; i < ARRAY_SIZE(vals); i++)
+ *			vals[i] = random();
+ *		printf("Initialized %u values\n", i);
+ *		return 0;
+ *	}
+ *
+ * License: LGPL (2 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/array_size/array_size.h b/lib/ccan/array_size/array_size.h
new file mode 100644
index 00000000000..0876945c5e5
--- /dev/null
+++ b/lib/ccan/array_size/array_size.h
@@ -0,0 +1,25 @@
+#ifndef CCAN_ARRAY_SIZE_H
+#define CCAN_ARRAY_SIZE_H
+#include "config.h"
+#include <ccan/build_assert/build_assert.h>
+
+/**
+ * ARRAY_SIZE - get the number of elements in a visible array
+ * @arr: the array whose size you want.
+ *
+ * This does not work on pointers, or arrays declared as [], or
+ * function parameters.  With correct compiler support, such usage
+ * will cause a build error (see build_assert).
+ */
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + _array_size_chk(arr))
+
+#if HAVE_BUILTIN_TYPES_COMPATIBLE_P && HAVE_TYPEOF
+/* Two gcc extensions.
+ * &a[0] degrades to a pointer: a different type from an array */
+#define _array_size_chk(arr)						\
+	BUILD_ASSERT_OR_ZERO(!__builtin_types_compatible_p(typeof(arr),	\
+							typeof(&(arr)[0])))
+#else
+#define _array_size_chk(arr) 0
+#endif
+#endif /* CCAN_ALIGNOF_H */
diff --git a/lib/ccan/array_size/test/compile_fail-function-param.c b/lib/ccan/array_size/test/compile_fail-function-param.c
new file mode 100644
index 00000000000..cb64d984247
--- /dev/null
+++ b/lib/ccan/array_size/test/compile_fail-function-param.c
@@ -0,0 +1,24 @@
+#include <ccan/array_size/array_size.h>
+#include <stdlib.h>
+
+struct foo {
+	unsigned int a, b;
+};
+
+int check_parameter(const struct foo array[4]);
+int check_parameter(const struct foo array[4])
+{
+#ifdef FAIL
+	return (ARRAY_SIZE(array) == 4);
+#if !HAVE_TYPEOF || !HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if _array_size_chk is a noop."
+#endif
+#else
+	return sizeof(array) == 4 * sizeof(struct foo);
+#endif
+}
+
+int main(int argc, char *argv[])
+{
+	return check_parameter(NULL);
+}
diff --git a/lib/ccan/array_size/test/compile_fail.c b/lib/ccan/array_size/test/compile_fail.c
new file mode 100644
index 00000000000..37d315f2198
--- /dev/null
+++ b/lib/ccan/array_size/test/compile_fail.c
@@ -0,0 +1,14 @@
+#include <ccan/array_size/array_size.h>
+
+int main(int argc, char *argv[8])
+{
+	char array[100];
+#ifdef FAIL
+	return ARRAY_SIZE(argv) + ARRAY_SIZE(array);
+#if !HAVE_TYPEOF || !HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if _array_size_chk is a noop."
+#endif
+#else
+	return ARRAY_SIZE(array);
+#endif
+}
diff --git a/lib/ccan/array_size/test/run.c b/lib/ccan/array_size/test/run.c
new file mode 100644
index 00000000000..37b4200b44f
--- /dev/null
+++ b/lib/ccan/array_size/test/run.c
@@ -0,0 +1,33 @@
+#include <ccan/array_size/array_size.h>
+#include <ccan/tap/tap.h>
+
+static char array1[1];
+static int array2[2];
+static unsigned long array3[3][5];
+struct foo {
+	unsigned int a, b;
+	char string[100];
+};
+static struct foo array4[4];
+
+/* Make sure they can be used in initializers. */
+static int array1_size = ARRAY_SIZE(array1);
+static int array2_size = ARRAY_SIZE(array2);
+static int array3_size = ARRAY_SIZE(array3);
+static int array4_size = ARRAY_SIZE(array4);
+
+int main(int argc, char *argv[])
+{
+	plan_tests(8);
+	ok1(array1_size == 1);
+	ok1(array2_size == 2);
+	ok1(array3_size == 3);
+	ok1(array4_size == 4);
+
+	ok1(ARRAY_SIZE(array1) == 1);
+	ok1(ARRAY_SIZE(array2) == 2);
+	ok1(ARRAY_SIZE(array3) == 3);
+	ok1(ARRAY_SIZE(array4) == 4);
+
+	return exit_status();
+}
diff --git a/lib/ccan/asearch/LICENSE b/lib/ccan/asearch/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/asearch/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/asearch/_info b/lib/ccan/asearch/_info
new file mode 100644
index 00000000000..857475016fb
--- /dev/null
+++ b/lib/ccan/asearch/_info
@@ -0,0 +1,58 @@
+#include <stdio.h>
+#include <string.h>
+#include "config.h"
+
+/**
+ * asearch - typesafe binary search (bsearch)
+ *
+ * An ordered array of objects can be efficiently searched using a binary
+ * search algorithm; the time taken is around log(number of elements).
+ *
+ * This version uses macros to be typesafe on platforms which support it.
+ *
+ * License: LGPL
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Example:
+ *	#include <ccan/asearch/asearch.h>
+ *	#include <stdio.h>
+ *	#include <string.h>
+ *
+ *	static int cmp(const char *key, char *const *elem)
+ *	{
+ *		return strcmp(key, *elem);
+ *	}
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		char **p;
+ *
+ *		if (argc < 2) {
+ *			fprintf(stderr, "Usage: %s <key> <list>...\n"
+ *				"Print position of key in (sorted) list\n",
+ *				argv[0]);
+ *			exit(1);
+ *		}
+ *
+ *		p = asearch(argv[1], &argv[2], argc-2, cmp);
+ *		if (!p) {
+ *			printf("Not found!\n");
+ *			return 1;
+ *		}
+ *		printf("%u\n", p - &argv[2]);
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/typesafe_cb\n");
+		printf("ccan/array_size\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/asearch/asearch.h b/lib/ccan/asearch/asearch.h
new file mode 100644
index 00000000000..d252284e7d8
--- /dev/null
+++ b/lib/ccan/asearch/asearch.h
@@ -0,0 +1,37 @@
+#ifndef CCAN_ASEARCH_H
+#define CCAN_ASEARCH_H
+#include <stdlib.h>
+#include <ccan/typesafe_cb/typesafe_cb.h>
+
+/**
+ * asearch - search an array of elements
+ * @key: pointer to item being searched for
+ * @base: pointer to data to sort
+ * @num: number of elements
+ * @cmp: pointer to comparison function
+ *
+ * This function does a binary search on the given array.  The
+ * contents of the array should already be in ascending sorted order
+ * under the provided comparison function.
+ *
+ * Note that the key need not have the same type as the elements in
+ * the array, e.g. key could be a string and the comparison function
+ * could compare the string with the struct's name field.  However, if
+ * the key and elements in the array are of the same type, you can use
+ * the same comparison function for both sort() and asearch().
+ */
+#if HAVE_TYPEOF
+#define asearch(key, base, num, cmp)					\
+	((__typeof__(*(base))*)(bsearch((key), (base), (num), sizeof(*(base)), \
+		typesafe_cb_cast(int (*)(const void *, const void *),	\
+				 int (*)(const __typeof__(*(key)) *,	\
+					 const __typeof__(*(base)) *),	\
+				 (cmp)))))
+
+#else
+#define asearch(key, base, num, cmp)				\
+	(bsearch((key), (base), (num), sizeof(*(base)),		\
+		 (int (*)(const void *, const void *))(cmp)))
+#endif
+
+#endif /* CCAN_ASEARCH_H */
diff --git a/lib/ccan/asearch/test/compile_fail-return-value-const.c b/lib/ccan/asearch/test/compile_fail-return-value-const.c
new file mode 100644
index 00000000000..2edee935012
--- /dev/null
+++ b/lib/ccan/asearch/test/compile_fail-return-value-const.c
@@ -0,0 +1,25 @@
+#include <ccan/asearch/asearch.h>
+#include <ccan/array_size/array_size.h>
+#include <string.h>
+
+static int cmp(const char *key, const char *const *elem)
+{
+	return strcmp(key, *elem);
+}
+
+int main(void)
+{
+	const char key[] = "key";
+	const char *elems[] = { "a", "big", "list", "of", "things" };
+
+#ifdef FAIL
+	char **p;
+#if !HAVE_TYPEOF
+#error "Unfortunately we don't fail if no typeof."
+#endif
+#else
+	const char **p;
+#endif
+	p = asearch(key, elems, ARRAY_SIZE(elems), cmp);
+	return p ? 0 : 1;
+}
diff --git a/lib/ccan/asearch/test/compile_fail-return-value.c b/lib/ccan/asearch/test/compile_fail-return-value.c
new file mode 100644
index 00000000000..4aef5327a83
--- /dev/null
+++ b/lib/ccan/asearch/test/compile_fail-return-value.c
@@ -0,0 +1,22 @@
+#include <ccan/asearch/asearch.h>
+
+static int cmp(const char *key, char *const *elem)
+{
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	const char key[] = "key";
+
+#ifdef FAIL
+	int **p;
+#if !HAVE_TYPEOF
+#error "Unfortunately we don't fail if no typeof."
+#endif
+#else
+	char **p;
+#endif
+	p = asearch(key, argv+1, argc-1, cmp);
+	return p ? 0 : 1;
+}
diff --git a/lib/ccan/asearch/test/run-strings.c b/lib/ccan/asearch/test/run-strings.c
new file mode 100644
index 00000000000..3ec453842f8
--- /dev/null
+++ b/lib/ccan/asearch/test/run-strings.c
@@ -0,0 +1,22 @@
+#include <ccan/asearch/asearch.h>
+#include <ccan/array_size/array_size.h>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+
+static int cmp(const int *key, const char *const *elem)
+{
+	return *key - atoi(*elem);
+}
+
+int main(void)
+{
+	const char *args[] = { "1", "4", "7", "9" };
+	int key = 7;
+	const char **p;
+
+	plan_tests(1);
+	p = asearch(&key, args, ARRAY_SIZE(args), cmp);
+	ok1(p == &args[2]);
+
+	return exit_status();
+}
diff --git a/lib/ccan/asearch/test/run.c b/lib/ccan/asearch/test/run.c
new file mode 100644
index 00000000000..2a896fccfe8
--- /dev/null
+++ b/lib/ccan/asearch/test/run.c
@@ -0,0 +1,40 @@
+#include <ccan/asearch/asearch.h>
+#include <ccan/array_size/array_size.h>
+#include <ccan/tap/tap.h>
+#include <limits.h>
+
+static int test_cmp(const int *key, const int *elt)
+{
+	if (*key < *elt)
+		return -1;
+	else if (*key > *elt)
+		return 1;
+	return 0;
+}
+
+int main(void)
+{
+	const int arr[] = { INT_MIN, 0, 1, 2, 3, 4, 5, 6, INT_MAX };
+	unsigned int start, num, i, total = 0;
+	int key;
+
+	plan_tests(285);
+
+	for (start = 0; start < ARRAY_SIZE(arr); start++) {
+		for (num = 0; num < ARRAY_SIZE(arr) - start; num++) {
+			key = 7;
+			ok1(asearch(&key, &arr[start], num, test_cmp) == NULL);
+			total++;
+			for (i = start; i < start+num; i++) {
+				const int *ret;
+				key = arr[i];
+				ret = asearch(&key, &arr[start], num, test_cmp);
+				ok1(ret);
+				ok1(ret && *ret == key);
+				total++;
+			}
+		}
+	}
+	diag("Tested %u searches\n", total);
+	return exit_status();
+}
diff --git a/lib/ccan/build_assert/LICENSE b/lib/ccan/build_assert/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/build_assert/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/build_assert/_info b/lib/ccan/build_assert/_info
new file mode 100644
index 00000000000..0906af07e60
--- /dev/null
+++ b/lib/ccan/build_assert/_info
@@ -0,0 +1,49 @@
+#include <stdio.h>
+#include <string.h>
+#include "config.h"
+
+/**
+ * build_assert - routines for build-time assertions
+ *
+ * This code provides routines which will cause compilation to fail should some
+ * assertion be untrue: such failures are preferable to run-time assertions,
+ * but much more limited since they can only depends on compile-time constants.
+ *
+ * These assertions are most useful when two parts of the code must be kept in
+ * sync: it is better to avoid such cases if possible, but seconds best is to
+ * detect invalid changes at build time.
+ *
+ * For example, a tricky piece of code might rely on a certain element being at
+ * the start of the structure.  To ensure that future changes don't break it,
+ * you would catch such changes in your code like so:
+ *
+ * Example:
+ *	#include <stddef.h>
+ *	#include <ccan/build_assert/build_assert.h>
+ *
+ *	struct foo {
+ *		char string[5];
+ *		int x;
+ *	};
+ *
+ *	static char *foo_string(struct foo *foo)
+ *	{
+ *		// This trick requires that the string be first in the structure
+ *		BUILD_ASSERT(offsetof(struct foo, string) == 0);
+ *		return (char *)foo;
+ *	}
+ *
+ * License: LGPL (2 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0)
+		/* Nothing. */
+		return 0;
+
+	return 1;
+}
diff --git a/lib/ccan/build_assert/build_assert.h b/lib/ccan/build_assert/build_assert.h
new file mode 100644
index 00000000000..24e59c44cd9
--- /dev/null
+++ b/lib/ccan/build_assert/build_assert.h
@@ -0,0 +1,39 @@
+#ifndef CCAN_BUILD_ASSERT_H
+#define CCAN_BUILD_ASSERT_H
+
+/**
+ * BUILD_ASSERT - assert a build-time dependency.
+ * @cond: the compile-time condition which must be true.
+ *
+ * Your compile will fail if the condition isn't true, or can't be evaluated
+ * by the compiler.  This can only be used within a function.
+ *
+ * Example:
+ *	#include <stddef.h>
+ *	...
+ *	static char *foo_to_char(struct foo *foo)
+ *	{
+ *		// This code needs string to be at start of foo.
+ *		BUILD_ASSERT(offsetof(struct foo, string) == 0);
+ *		return (char *)foo;
+ *	}
+ */
+#define BUILD_ASSERT(cond) \
+	do { (void) sizeof(char [1 - 2*!(cond)]); } while(0)
+
+/**
+ * BUILD_ASSERT_OR_ZERO - assert a build-time dependency, as an expression.
+ * @cond: the compile-time condition which must be true.
+ *
+ * Your compile will fail if the condition isn't true, or can't be evaluated
+ * by the compiler.  This can be used in an expression: its value is "0".
+ *
+ * Example:
+ *	#define foo_to_char(foo)					\
+ *		 ((char *)(foo)						\
+ *		  + BUILD_ASSERT_OR_ZERO(offsetof(struct foo, string) == 0))
+ */
+#define BUILD_ASSERT_OR_ZERO(cond) \
+	(sizeof(char [1 - 2*!(cond)]) - 1)
+
+#endif /* CCAN_BUILD_ASSERT_H */
diff --git a/lib/ccan/build_assert/test/compile_fail-expr.c b/lib/ccan/build_assert/test/compile_fail-expr.c
new file mode 100644
index 00000000000..109215b8aa7
--- /dev/null
+++ b/lib/ccan/build_assert/test/compile_fail-expr.c
@@ -0,0 +1,10 @@
+#include <ccan/build_assert/build_assert.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	return BUILD_ASSERT_OR_ZERO(1 == 0);
+#else
+	return 0;
+#endif
+}
diff --git a/lib/ccan/build_assert/test/compile_fail.c b/lib/ccan/build_assert/test/compile_fail.c
new file mode 100644
index 00000000000..37d95eddc9b
--- /dev/null
+++ b/lib/ccan/build_assert/test/compile_fail.c
@@ -0,0 +1,9 @@
+#include <ccan/build_assert/build_assert.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	BUILD_ASSERT(1 == 0);
+#endif
+	return 0;
+}
diff --git a/lib/ccan/build_assert/test/compile_ok.c b/lib/ccan/build_assert/test/compile_ok.c
new file mode 100644
index 00000000000..4105484d1fe
--- /dev/null
+++ b/lib/ccan/build_assert/test/compile_ok.c
@@ -0,0 +1,7 @@
+#include <ccan/build_assert/build_assert.h>
+
+int main(int argc, char *argv[])
+{
+	BUILD_ASSERT(1 == 1);
+	return 0;
+}
diff --git a/lib/ccan/build_assert/test/run-BUILD_ASSERT_OR_ZERO.c b/lib/ccan/build_assert/test/run-BUILD_ASSERT_OR_ZERO.c
new file mode 100644
index 00000000000..41858213317
--- /dev/null
+++ b/lib/ccan/build_assert/test/run-BUILD_ASSERT_OR_ZERO.c
@@ -0,0 +1,9 @@
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+int main(int argc, char *argv[])
+{
+	plan_tests(1);
+	ok1(BUILD_ASSERT_OR_ZERO(1 == 1) == 0);
+	return exit_status();
+}
diff --git a/lib/ccan/build_assert/test/run-EXPR_BUILD_ASSERT.c b/lib/ccan/build_assert/test/run-EXPR_BUILD_ASSERT.c
new file mode 100644
index 00000000000..91bbbbbf75f
--- /dev/null
+++ b/lib/ccan/build_assert/test/run-EXPR_BUILD_ASSERT.c
@@ -0,0 +1,9 @@
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+int main(int argc, char *argv[])
+{
+	plan_tests(1);
+	ok1(EXPR_BUILD_ASSERT(1 == 1) == 0);
+	return exit_status();
+}
diff --git a/lib/ccan/cast/LICENSE b/lib/ccan/cast/LICENSE
new file mode 100644
index 00000000000..cca7fc278f5
--- /dev/null
+++ b/lib/ccan/cast/LICENSE
@@ -0,0 +1,165 @@
+		   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/lib/ccan/cast/_info b/lib/ccan/cast/_info
new file mode 100644
index 00000000000..5f82a05b8fa
--- /dev/null
+++ b/lib/ccan/cast/_info
@@ -0,0 +1,84 @@
+#include <string.h>
+#include "config.h"
+
+/**
+ * cast - routines for safer casting.
+ *
+ * Often you want to cast in a limited way, such as removing a const or
+ * switching between integer types.  However, normal casts will work on
+ * almost any type, making them dangerous when the code changes.
+ *
+ * These C++-inspired macros serve two purposes: they make it clear the
+ * exact reason for the cast, and they also (with some compilers) cause
+ * errors when misused.
+ *
+ * Based on Jan Engelhardt's libHX macros: http://libhx.sourceforge.net/
+ *
+ * Author: Jan Engelhardt
+ * Maintainer: Rusty Russell <rusty@rustcorp.com.au>
+ * License: LGPL
+ *
+ * Example:
+ *	// Given "test" contains "3 t's in 'test string'
+ *	#include <ccan/cast/cast.h>
+ *	#include <stdint.h>
+ *	#include <stdio.h>
+ *
+ *	// Find char @orig in @str, if @repl, replace them.  Return number.
+ *	static size_t find_chars(char *str, char orig, char repl)
+ *	{
+ *		size_t i, count = 0;
+ *		for (i = 0; str[i]; i++) {
+ *			if (str[i] == orig) {
+ *				count++;
+ *				if (repl)
+ *					str[i] = repl;
+ *			}
+ *		}
+ *		return count;
+ *	}
+ *
+ *	// Terrible hash function.
+ *	static uint64_t hash_string(const unsigned char *str)
+ *	{
+ *		size_t i;
+ *		uint64_t hash = 0;
+ *		for (i = 0; str[i]; i++)
+ *			hash += str[i];
+ *		return hash;
+ *	}
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		uint64_t hash;
+ *
+ *		// find_chars wants a non-const string, but doesn't
+ *		// need it if repl == 0.
+ *		printf("%zu %c's in 'test string'\n",
+ *		       find_chars(cast_const(char *, "test string"),
+ *				  argv[1][0], 0),
+ *		       argv[1][0]);
+ *
+ *		// hash_string wants an unsigned char.
+ *		hash = hash_string(cast_signed(unsigned char *, argv[1]));
+ *
+ *		// Need a long long to hand to printf.
+ *		printf("Hash of '%s' = %llu\n", argv[1],
+ *		       cast_static(unsigned long long, hash));
+ *		return 0;
+ *	}
+ *
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/cast/cast.h b/lib/ccan/cast/cast.h
new file mode 100644
index 00000000000..daebd857239
--- /dev/null
+++ b/lib/ccan/cast/cast.h
@@ -0,0 +1,129 @@
+#ifndef CCAN_CAST_H
+#define CCAN_CAST_H
+#include "config.h"
+#include <stdint.h>
+#include <ccan/build_assert/build_assert.h>
+
+/**
+ * cast_signed - cast a (const) char * to/from (const) signed/unsigned char *.
+ * @type: some char * variant.
+ * @expr: expression (of some char * variant) to cast.
+ *
+ * Some libraries insist on an unsigned char in various places; cast_signed
+ * makes sure (with suitable compiler) that the expression you are casting
+ * only differs in signed/unsigned, not in type or const-ness.
+ */
+#define cast_signed(type, expr)						\
+	((type)(expr)							\
+	 + BUILD_ASSERT_OR_ZERO(cast_sign_compatible(type, (expr))))
+
+/**
+ * cast_const - remove a const qualifier from a pointer.
+ * @type: some pointer type.
+ * @expr: expression to cast.
+ *
+ * This ensures that you are only removing the const qualifier from an
+ * expression.  The expression must otherwise match @type.
+ *
+ * If @type is a pointer to a pointer, you must use cast_const2 (etc).
+ *
+ * Example:
+ *	// Dumb open-coded strstr variant.
+ *	static char *find_needle(const char *haystack)
+ *	{
+ *		size_t i;
+ *		for (i = 0; i < strlen(haystack); i++)
+ *		if (memcmp("needle", haystack+i, strlen("needle")) == 0)
+ *			return cast_const(char *, haystack+i);
+ *		return NULL;
+ *	}
+ */
+#define cast_const(type, expr)						\
+	((type)((intptr_t)(expr)					\
+		+ BUILD_ASSERT_OR_ZERO(cast_const_compat1((expr), type))))
+
+/**
+ * cast_const2 - remove a const qualifier from a pointer to a pointer.
+ * @type: some pointer to pointer type.
+ * @expr: expression to cast.
+ *
+ * This ensures that you are only removing the const qualifier from an
+ * expression.  The expression must otherwise match @type.
+ */
+#define cast_const2(type, expr)						\
+	((type)((intptr_t)(expr)					\
+		+ BUILD_ASSERT_OR_ZERO(cast_const_compat2((expr), type))))
+
+/**
+ * cast_const3 - remove a const from a pointer to a pointer to a pointer..
+ * @type: some pointer to pointer to pointer type.
+ * @expr: expression to cast.
+ *
+ * This ensures that you are only removing the const qualifier from an
+ * expression.  The expression must otherwise match @type.
+ */
+#define cast_const3(type, expr)						\
+	((type)((intptr_t)(expr)					\
+		+ BUILD_ASSERT_OR_ZERO(cast_const_compat3((expr), type))))
+
+
+/**
+ * cast_static - explicit mimic of implicit cast.
+ * @type: some type.
+ * @expr: expression to cast.
+ *
+ * This ensures that the cast is not to or from a pointer: it can only be
+ * an implicit cast, such as a pointer to a similar const pointer, or between
+ * integral types.
+ */
+#if HAVE_COMPOUND_LITERALS
+#define cast_static(type, expr)			\
+	((struct { type x; }){(expr)}.x)
+#else
+#define cast_static(type, expr)			\
+	((type)(expr))
+#endif
+
+/* Herein lies the gcc magic to evoke compile errors. */
+#if HAVE_BUILTIN_CHOOSE_EXPR && HAVE_BUILTIN_TYPES_COMPATIBLE_P && HAVE_TYPEOF
+#define cast_sign_compatible(t, e) \
+  __builtin_choose_expr(						\
+	  __builtin_types_compatible_p(__typeof__(t), char *) ||	\
+	  __builtin_types_compatible_p(__typeof__(t), signed char *) || \
+	  __builtin_types_compatible_p(__typeof__(t), unsigned char *), \
+	  /* if type is not const qualified */				\
+	  __builtin_types_compatible_p(__typeof__(e), char *) ||	\
+	  __builtin_types_compatible_p(__typeof__(e), signed char *) || \
+	  __builtin_types_compatible_p(__typeof__(e), unsigned char *), \
+	  /* and if it is... */						\
+	  __builtin_types_compatible_p(__typeof__(e), const char *) ||	\
+	  __builtin_types_compatible_p(__typeof__(e), const signed char *) || \
+	  __builtin_types_compatible_p(__typeof__(e), const unsigned char *) ||\
+	  __builtin_types_compatible_p(__typeof__(e), char *) ||	\
+	  __builtin_types_compatible_p(__typeof__(e), signed char *) ||	\
+	  __builtin_types_compatible_p(__typeof__(e), unsigned char *)	\
+	  )
+
+#define cast_const_strip1(expr)			\
+	__typeof__(*(struct { int z; __typeof__(expr) x; }){0}.x)
+#define cast_const_strip2(expr) \
+	__typeof__(**(struct { int z; __typeof__(expr) x; }){0}.x)
+#define cast_const_strip3(expr) \
+	__typeof__(***(struct { int z; __typeof__(expr) x; }){0}.x)
+#define cast_const_compat1(expr, type)					\
+	__builtin_types_compatible_p(cast_const_strip1(expr),		\
+				     cast_const_strip1(type))
+#define cast_const_compat2(expr, type)					\
+	__builtin_types_compatible_p(cast_const_strip2(expr),		\
+				     cast_const_strip2(type))
+#define cast_const_compat3(expr, type)					\
+	__builtin_types_compatible_p(cast_const_strip3(expr),		\
+				     cast_const_strip3(type))
+#else
+#define cast_sign_compatible(type, expr)		\
+	(sizeof(*(type)0) == 1 && sizeof(*(expr)) == 1)
+#define cast_const_compat1(expr, type)		(1)
+#define cast_const_compat2(expr, type)		(1)
+#define cast_const_compat3(expr, type)		(1)
+#endif
+#endif /* CCAN_CAST_H */
diff --git a/lib/ccan/cast/test/compile_fail-cast_const.c b/lib/ccan/cast/test/compile_fail-cast_const.c
new file mode 100644
index 00000000000..277f3de1c44
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_const.c
@@ -0,0 +1,29 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+/* Note: this *isn't* sizeof(char) on all platforms. */
+struct char_struct {
+	char c;
+};
+
+int main(int argc, char *argv[])
+{
+	char *uc;
+	const
+#ifdef FAIL
+		struct char_struct
+#else
+		char
+#endif
+		*p = NULL;
+
+	uc = cast_const(char *, p);
+	(void) uc; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
+
+#ifdef FAIL
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_const can only use size"
+#endif
+#endif
diff --git a/lib/ccan/cast/test/compile_fail-cast_const2.c b/lib/ccan/cast/test/compile_fail-cast_const2.c
new file mode 100644
index 00000000000..e671e88eda5
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_const2.c
@@ -0,0 +1,29 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+/* Note: this *isn't* sizeof(char) on all platforms. */
+struct char_struct {
+	char c;
+};
+
+int main(int argc, char *argv[])
+{
+	char **uc;
+	const
+#ifdef FAIL
+		struct char_struct
+#else
+		char
+#endif
+		**p = NULL;
+
+	uc = cast_const2(char **, p);
+	(void) uc; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
+
+#ifdef FAIL
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_const can only use size"
+#endif
+#endif
diff --git a/lib/ccan/cast/test/compile_fail-cast_const3.c b/lib/ccan/cast/test/compile_fail-cast_const3.c
new file mode 100644
index 00000000000..e958e2dde5b
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_const3.c
@@ -0,0 +1,29 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+/* Note: this *isn't* sizeof(char) on all platforms. */
+struct char_struct {
+	char c;
+};
+
+int main(int argc, char *argv[])
+{
+	char ***uc;
+	const
+#ifdef FAIL
+		struct char_struct
+#else
+		char
+#endif
+		***p = NULL;
+
+	uc = cast_const3(char ***, p);
+	(void) uc; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
+
+#ifdef FAIL
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_const can only use size"
+#endif
+#endif
diff --git a/lib/ccan/cast/test/compile_fail-cast_signed-const.c b/lib/ccan/cast/test/compile_fail-cast_signed-const.c
new file mode 100644
index 00000000000..9971dc8eb32
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_signed-const.c
@@ -0,0 +1,22 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	unsigned char *uc;
+#ifdef FAIL
+	const
+#endif
+	char
+		*p = NULL;
+
+	uc = cast_signed(unsigned char *, p);
+	(void) uc; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
+
+#ifdef FAIL
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_const can only use size"
+#endif
+#endif
diff --git a/lib/ccan/cast/test/compile_fail-cast_signed-sizesame.c b/lib/ccan/cast/test/compile_fail-cast_signed-sizesame.c
new file mode 100644
index 00000000000..2bc40b2f462
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_signed-sizesame.c
@@ -0,0 +1,29 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+/* Note: this *isn't* sizeof(char) on all platforms. */
+struct char_struct {
+	char c;
+};
+
+int main(int argc, char *argv[])
+{
+	unsigned char *uc;
+#ifdef FAIL
+	struct char_struct
+#else
+	char
+#endif
+		*p = NULL;
+
+	uc = cast_signed(unsigned char *, p);
+
+	(void) uc; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
+
+#ifdef FAIL
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_signed can only use size"
+#endif
+#endif
diff --git a/lib/ccan/cast/test/compile_fail-cast_signed.c b/lib/ccan/cast/test/compile_fail-cast_signed.c
new file mode 100644
index 00000000000..66bcc0a1b5e
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_signed.c
@@ -0,0 +1,17 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	unsigned char *uc;
+#ifdef FAIL
+	int
+#else
+	char
+#endif
+		*p = NULL;
+
+	uc = cast_signed(unsigned char *, p);
+	(void) uc; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
diff --git a/lib/ccan/cast/test/compile_fail-cast_static-2.c b/lib/ccan/cast/test/compile_fail-cast_static-2.c
new file mode 100644
index 00000000000..8a120253845
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_static-2.c
@@ -0,0 +1,23 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	char *c;
+#ifdef FAIL
+	long
+#else
+	char
+#endif
+		*p = 0;
+
+	c = cast_static(char *, p);
+	(void) c; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
+
+#ifdef FAIL
+#if !HAVE_COMPOUND_LITERALS
+#error "Unfortunately we don't fail if cast_static is a noop"
+#endif
+#endif
diff --git a/lib/ccan/cast/test/compile_fail-cast_static-3.c b/lib/ccan/cast/test/compile_fail-cast_static-3.c
new file mode 100644
index 00000000000..6296b752769
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_static-3.c
@@ -0,0 +1,21 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	char *c;
+#ifdef FAIL
+	const
+#endif
+		char *p = 0;
+
+	c = cast_static(char *, p);
+	(void) c; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
+
+#ifdef FAIL
+#if !HAVE_COMPOUND_LITERALS
+#error "Unfortunately we don't fail if cast_static is a noop"
+#endif
+#endif
diff --git a/lib/ccan/cast/test/compile_fail-cast_static.c b/lib/ccan/cast/test/compile_fail-cast_static.c
new file mode 100644
index 00000000000..0f9e4780470
--- /dev/null
+++ b/lib/ccan/cast/test/compile_fail-cast_static.c
@@ -0,0 +1,17 @@
+#include <ccan/cast/cast.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	char c;
+#ifdef FAIL
+	char *
+#else
+	long
+#endif
+		x = 0;
+
+	c = cast_static(char, x);
+	(void) c; /* Suppress unused-but-set-variable warning. */
+	return 0;
+}
diff --git a/lib/ccan/cast/test/compile_ok-cast_void.c b/lib/ccan/cast/test/compile_ok-cast_void.c
new file mode 100644
index 00000000000..c649d283b3b
--- /dev/null
+++ b/lib/ccan/cast/test/compile_ok-cast_void.c
@@ -0,0 +1,12 @@
+#include <ccan/cast/cast.h>
+
+static void *remove_void(const void *p)
+{
+	return cast_const(void *, p);
+}
+
+int main(void)
+{
+	void *p = remove_void("foo");
+	return !p;
+}
diff --git a/lib/ccan/compiler/LICENSE b/lib/ccan/compiler/LICENSE
new file mode 100644
index 00000000000..cca7fc278f5
--- /dev/null
+++ b/lib/ccan/compiler/LICENSE
@@ -0,0 +1,165 @@
+		   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/lib/ccan/compiler/_info b/lib/ccan/compiler/_info
new file mode 100644
index 00000000000..c55ba22f086
--- /dev/null
+++ b/lib/ccan/compiler/_info
@@ -0,0 +1,64 @@
+#include <string.h>
+#include <stdio.h>
+#include "config.h"
+
+/**
+ * compiler - macros for common compiler extensions
+ *
+ * Abstracts away some compiler hints.  Currently these include:
+ * - COLD
+ *	For functions not called in fast paths (aka. cold functions)
+ * - PRINTF_FMT
+ *	For functions which take printf-style parameters.
+ * - IDEMPOTENT
+ *	For functions which return the same value for same parameters.
+ * - NEEDED
+ *	For functions and variables which must be emitted even if unused.
+ * - UNNEEDED
+ *	For functions and variables which need not be emitted if unused.
+ * - UNUSED
+ *	For parameters which are not used.
+ * - IS_COMPILE_CONSTANT
+ *	For using different tradeoffs for compiletime vs runtime evaluation.
+ *
+ * License: LGPL (3 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Example:
+ *	#include <ccan/compiler/compiler.h>
+ *	#include <stdio.h>
+ *	#include <stdarg.h>
+ *
+ *	// Example of a (slow-path) logging function.
+ *	static int log_threshold = 2;
+ *	static void COLD PRINTF_FMT(2,3)
+ *		logger(int level, const char *fmt, ...)
+ *	{
+ *		va_list ap;
+ *		va_start(ap, fmt);
+ *		if (level >= log_threshold)
+ *			vfprintf(stderr, fmt, ap);
+ *		va_end(ap);
+ *	}
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		if (argc != 1) {
+ *			logger(3, "Don't want %i arguments!\n", argc-1);
+ *			return 1;
+ *		}
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/compiler/compiler.h b/lib/ccan/compiler/compiler.h
new file mode 100644
index 00000000000..74e0f1835c1
--- /dev/null
+++ b/lib/ccan/compiler/compiler.h
@@ -0,0 +1,216 @@
+#ifndef CCAN_COMPILER_H
+#define CCAN_COMPILER_H
+#include "config.h"
+
+#ifndef COLD
+#if HAVE_ATTRIBUTE_COLD
+/**
+ * COLD - a function is unlikely to be called.
+ *
+ * Used to mark an unlikely code path and optimize appropriately.
+ * It is usually used on logging or error routines.
+ *
+ * Example:
+ * static void COLD moan(const char *reason)
+ * {
+ *	fprintf(stderr, "Error: %s (%s)\n", reason, strerror(errno));
+ * }
+ */
+#define COLD __attribute__((cold))
+#else
+#define COLD
+#endif
+#endif
+
+#ifndef NORETURN
+#if HAVE_ATTRIBUTE_NORETURN
+/**
+ * NORETURN - a function does not return
+ *
+ * Used to mark a function which exits; useful for suppressing warnings.
+ *
+ * Example:
+ * static void NORETURN fail(const char *reason)
+ * {
+ *	fprintf(stderr, "Error: %s (%s)\n", reason, strerror(errno));
+ *	exit(1);
+ * }
+ */
+#define NORETURN __attribute__((noreturn))
+#else
+#define NORETURN
+#endif
+#endif
+
+#ifndef PRINTF_FMT
+#if HAVE_ATTRIBUTE_PRINTF
+/**
+ * PRINTF_FMT - a function takes printf-style arguments
+ * @nfmt: the 1-based number of the function's format argument.
+ * @narg: the 1-based number of the function's first variable argument.
+ *
+ * This allows the compiler to check your parameters as it does for printf().
+ *
+ * Example:
+ * void PRINTF_FMT(2,3) my_printf(const char *prefix, const char *fmt, ...);
+ */
+#define PRINTF_FMT(nfmt, narg) \
+	__attribute__((format(__printf__, nfmt, narg)))
+#else
+#define PRINTF_FMT(nfmt, narg)
+#endif
+#endif
+
+#ifndef IDEMPOTENT
+#if HAVE_ATTRIBUTE_CONST
+/**
+ * IDEMPOTENT - a function's return depends only on its argument
+ *
+ * This allows the compiler to assume that the function will return the exact
+ * same value for the exact same arguments.  This implies that the function
+ * must not use global variables, or dereference pointer arguments.
+ */
+#define IDEMPOTENT __attribute__((const))
+#else
+#define IDEMPOTENT
+#endif
+#endif
+
+#if HAVE_ATTRIBUTE_UNUSED
+#ifndef UNNEEDED
+/**
+ * UNNEEDED - a variable/function may not be needed
+ *
+ * This suppresses warnings about unused variables or functions, but tells
+ * the compiler that if it is unused it need not emit it into the source code.
+ *
+ * Example:
+ * // With some preprocessor options, this is unnecessary.
+ * static UNNEEDED int counter;
+ *
+ * // With some preprocessor options, this is unnecessary.
+ * static UNNEEDED void add_to_counter(int add)
+ * {
+ *	counter += add;
+ * }
+ */
+#define UNNEEDED __attribute__((unused))
+#endif
+
+#ifndef NEEDED
+#if HAVE_ATTRIBUTE_USED
+/**
+ * NEEDED - a variable/function is needed
+ *
+ * This suppresses warnings about unused variables or functions, but tells
+ * the compiler that it must exist even if it (seems) unused.
+ *
+ * Example:
+ *	// Even if this is unused, these are vital for debugging.
+ *	static NEEDED int counter;
+ *	static NEEDED void dump_counter(void)
+ *	{
+ *		printf("Counter is %i\n", counter);
+ *	}
+ */
+#define NEEDED __attribute__((used))
+#else
+/* Before used, unused functions and vars were always emitted. */
+#define NEEDED __attribute__((unused))
+#endif
+#endif
+
+#ifndef UNUSED
+/**
+ * UNUSED - a parameter is unused
+ *
+ * Some compilers (eg. gcc with -W or -Wunused) warn about unused
+ * function parameters.  This suppresses such warnings and indicates
+ * to the reader that it's deliberate.
+ *
+ * Example:
+ *	// This is used as a callback, so needs to have this prototype.
+ *	static int some_callback(void *unused UNUSED)
+ *	{
+ *		return 0;
+ *	}
+ */
+#define UNUSED __attribute__((unused))
+#endif
+#else
+#ifndef UNNEEDED
+#define UNNEEDED
+#endif
+#ifndef NEEDED
+#define NEEDED
+#endif
+#ifndef UNUSED
+#define UNUSED
+#endif
+#endif
+
+#ifndef IS_COMPILE_CONSTANT
+#if HAVE_BUILTIN_CONSTANT_P
+/**
+ * IS_COMPILE_CONSTANT - does the compiler know the value of this expression?
+ * @expr: the expression to evaluate
+ *
+ * When an expression manipulation is complicated, it is usually better to
+ * implement it in a function.  However, if the expression being manipulated is
+ * known at compile time, it is better to have the compiler see the entire
+ * expression so it can simply substitute the result.
+ *
+ * This can be done using the IS_COMPILE_CONSTANT() macro.
+ *
+ * Example:
+ *	enum greek { ALPHA, BETA, GAMMA, DELTA, EPSILON };
+ *
+ *	// Out-of-line version.
+ *	const char *greek_name(enum greek greek);
+ *
+ *	// Inline version.
+ *	static inline const char *_greek_name(enum greek greek)
+ *	{
+ *		switch (greek) {
+ *		case ALPHA: return "alpha";
+ *		case BETA: return "beta";
+ *		case GAMMA: return "gamma";
+ *		case DELTA: return "delta";
+ *		case EPSILON: return "epsilon";
+ *		default: return "**INVALID**";
+ *		}
+ *	}
+ *
+ *	// Use inline if compiler knows answer.  Otherwise call function
+ *	// to avoid copies of the same code everywhere.
+ *	#define greek_name(g)						\
+ *		 (IS_COMPILE_CONSTANT(greek) ? _greek_name(g) : greek_name(g))
+ */
+#define IS_COMPILE_CONSTANT(expr) __builtin_constant_p(expr)
+#else
+/* If we don't know, assume it's not. */
+#define IS_COMPILE_CONSTANT(expr) 0
+#endif
+#endif
+
+#ifndef WARN_UNUSED_RESULT
+#if HAVE_WARN_UNUSED_RESULT
+/**
+ * WARN_UNUSED_RESULT - warn if a function return value is unused.
+ *
+ * Used to mark a function where it is extremely unlikely that the caller
+ * can ignore the result, eg realloc().
+ *
+ * Example:
+ * // buf param may be freed by this; need return value!
+ * static char *WARN_UNUSED_RESULT enlarge(char *buf, unsigned *size)
+ * {
+ *	return realloc(buf, (*size) *= 2);
+ * }
+ */
+#define WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#else
+#define WARN_UNUSED_RESULT
+#endif
+#endif
+#endif /* CCAN_COMPILER_H */
diff --git a/lib/ccan/compiler/test/compile_fail-printf.c b/lib/ccan/compiler/test/compile_fail-printf.c
new file mode 100644
index 00000000000..8f34ae5a125
--- /dev/null
+++ b/lib/ccan/compiler/test/compile_fail-printf.c
@@ -0,0 +1,22 @@
+#include <ccan/compiler/compiler.h>
+
+static void PRINTF_FMT(2,3) my_printf(int x, const char *fmt, ...)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i = 0;
+
+	my_printf(1, "Not a pointer "
+#ifdef FAIL
+		  "%p",
+#if !HAVE_ATTRIBUTE_PRINTF
+#error "Unfortunately we don't fail if !HAVE_ATTRIBUTE_PRINTF."
+#endif
+#else
+		  "%i",
+#endif
+		  i);
+	return 0;
+}
diff --git a/lib/ccan/compiler/test/run-is_compile_constant.c b/lib/ccan/compiler/test/run-is_compile_constant.c
new file mode 100644
index 00000000000..a66f2e13e6c
--- /dev/null
+++ b/lib/ccan/compiler/test/run-is_compile_constant.c
@@ -0,0 +1,15 @@
+#include <ccan/compiler/compiler.h>
+#include <ccan/tap/tap.h>
+
+int main(int argc, char *argv[])
+{
+	plan_tests(2);
+
+	ok1(!IS_COMPILE_CONSTANT(argc));
+#if HAVE_BUILTIN_CONSTANT_P
+	ok1(IS_COMPILE_CONSTANT(7));
+#else
+	pass("If !HAVE_BUILTIN_CONSTANT_P, IS_COMPILE_CONSTANT always false");
+#endif
+	return exit_status();
+}
diff --git a/lib/ccan/endian/LICENSE b/lib/ccan/endian/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/endian/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/endian/_info b/lib/ccan/endian/_info
new file mode 100644
index 00000000000..5d4d65ff741
--- /dev/null
+++ b/lib/ccan/endian/_info
@@ -0,0 +1,53 @@
+#include <stdio.h>
+#include <string.h>
+#include "config.h"
+
+/**
+ * endian - endian conversion macros for simple types
+ *
+ * Portable protocols (such as on-disk formats, or network protocols)
+ * are often defined to be a particular endian: little-endian (least
+ * significant bytes first) or big-endian (most significant bytes
+ * first).
+ *
+ * Similarly, some CPUs lay out values in memory in little-endian
+ * order (most commonly, Intel's 8086 and derivatives), or big-endian
+ * order (almost everyone else).
+ *
+ * This module provides conversion routines, inspired by the linux kernel.
+ *
+ * Example:
+ *	#include <stdio.h>
+ *	#include <err.h>
+ *	#include <ccan/endian/endian.h>
+ *
+ *	//
+ *	int main(int argc, char *argv[])
+ *	{
+ *		uint32_t value;
+ *
+ *		if (argc != 2)
+ *			errx(1, "Usage: %s <value>", argv[0]);
+ *
+ *		value = atoi(argv[1]);
+ *		printf("native:        %08x\n", value);
+ *		printf("little-endian: %08x\n", cpu_to_le32(value));
+ *		printf("big-endian:    %08x\n", cpu_to_be32(value));
+ *		printf("byte-reversed: %08x\n", bswap_32(value));
+ *		exit(0);
+ *	}
+ *
+ * License: LGPL (2 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0)
+		/* Nothing */
+		return 0;
+
+	return 1;
+}
diff --git a/lib/ccan/endian/endian.h b/lib/ccan/endian/endian.h
new file mode 100644
index 00000000000..baee60be144
--- /dev/null
+++ b/lib/ccan/endian/endian.h
@@ -0,0 +1,226 @@
+#ifndef CCAN_ENDIAN_H
+#define CCAN_ENDIAN_H
+#include <stdint.h>
+#include "config.h"
+
+#if HAVE_BYTESWAP_H
+#include <byteswap.h>
+#else
+/**
+ * bswap_16 - reverse bytes in a uint16_t value.
+ * @val: value whose bytes to swap.
+ *
+ * Example:
+ *	// Output contains "1024 is 4 as two bytes reversed"
+ *	printf("1024 is %u as two bytes reversed\n", bswap_16(1024));
+ */
+static inline uint16_t bswap_16(uint16_t val)
+{
+	return ((val & (uint16_t)0x00ffU) << 8)
+		| ((val & (uint16_t)0xff00U) >> 8);
+}
+
+/**
+ * bswap_32 - reverse bytes in a uint32_t value.
+ * @val: value whose bytes to swap.
+ *
+ * Example:
+ *	// Output contains "1024 is 262144 as four bytes reversed"
+ *	printf("1024 is %u as four bytes reversed\n", bswap_32(1024));
+ */
+static inline uint32_t bswap_32(uint32_t val)
+{
+	return ((val & (uint32_t)0x000000ffUL) << 24)
+		| ((val & (uint32_t)0x0000ff00UL) <<  8)
+		| ((val & (uint32_t)0x00ff0000UL) >>  8)
+		| ((val & (uint32_t)0xff000000UL) >> 24);
+}
+#endif /* !HAVE_BYTESWAP_H */
+
+#if !HAVE_BSWAP_64
+/**
+ * bswap_64 - reverse bytes in a uint64_t value.
+ * @val: value whose bytes to swap.
+ *
+ * Example:
+ *	// Output contains "1024 is 1125899906842624 as eight bytes reversed"
+ *	printf("1024 is %llu as eight bytes reversed\n",
+ *		(unsigned long long)bswap_64(1024));
+ */
+static inline uint64_t bswap_64(uint64_t val)
+{
+	return ((val & (uint64_t)0x00000000000000ffULL) << 56)
+		| ((val & (uint64_t)0x000000000000ff00ULL) << 40)
+		| ((val & (uint64_t)0x0000000000ff0000ULL) << 24)
+		| ((val & (uint64_t)0x00000000ff000000ULL) <<  8)
+		| ((val & (uint64_t)0x000000ff00000000ULL) >>  8)
+		| ((val & (uint64_t)0x0000ff0000000000ULL) >> 24)
+		| ((val & (uint64_t)0x00ff000000000000ULL) >> 40)
+		| ((val & (uint64_t)0xff00000000000000ULL) >> 56);
+}
+#endif
+
+/* Sanity check the defines.  We don't handle weird endianness. */
+#if !HAVE_LITTLE_ENDIAN && !HAVE_BIG_ENDIAN
+#error "Unknown endian"
+#elif HAVE_LITTLE_ENDIAN && HAVE_BIG_ENDIAN
+#error "Can't compile for both big and little endian."
+#endif
+
+/**
+ * cpu_to_le64 - convert a uint64_t value to little-endian
+ * @native: value to convert
+ */
+static inline uint64_t cpu_to_le64(uint64_t native)
+{
+#if HAVE_LITTLE_ENDIAN
+	return native;
+#else
+	return bswap_64(native);
+#endif
+}
+
+/**
+ * cpu_to_le32 - convert a uint32_t value to little-endian
+ * @native: value to convert
+ */
+static inline uint32_t cpu_to_le32(uint32_t native)
+{
+#if HAVE_LITTLE_ENDIAN
+	return native;
+#else
+	return bswap_32(native);
+#endif
+}
+
+/**
+ * cpu_to_le16 - convert a uint16_t value to little-endian
+ * @native: value to convert
+ */
+static inline uint16_t cpu_to_le16(uint16_t native)
+{
+#if HAVE_LITTLE_ENDIAN
+	return native;
+#else
+	return bswap_16(native);
+#endif
+}
+
+/**
+ * le64_to_cpu - convert a little-endian uint64_t value
+ * @le_val: little-endian value to convert
+ */
+static inline uint64_t le64_to_cpu(uint64_t le_val)
+{
+#if HAVE_LITTLE_ENDIAN
+	return le_val;
+#else
+	return bswap_64(le_val);
+#endif
+}
+
+/**
+ * le32_to_cpu - convert a little-endian uint32_t value
+ * @le_val: little-endian value to convert
+ */
+static inline uint32_t le32_to_cpu(uint32_t le_val)
+{
+#if HAVE_LITTLE_ENDIAN
+	return le_val;
+#else
+	return bswap_32(le_val);
+#endif
+}
+
+/**
+ * le16_to_cpu - convert a little-endian uint16_t value
+ * @le_val: little-endian value to convert
+ */
+static inline uint16_t le16_to_cpu(uint16_t le_val)
+{
+#if HAVE_LITTLE_ENDIAN
+	return le_val;
+#else
+	return bswap_16(le_val);
+#endif
+}
+
+/**
+ * cpu_to_be64 - convert a uint64_t value to big endian.
+ * @native: value to convert
+ */
+static inline uint64_t cpu_to_be64(uint64_t native)
+{
+#if HAVE_LITTLE_ENDIAN
+	return bswap_64(native);
+#else
+	return native;
+#endif
+}
+
+/**
+ * cpu_to_be32 - convert a uint32_t value to big endian.
+ * @native: value to convert
+ */
+static inline uint32_t cpu_to_be32(uint32_t native)
+{
+#if HAVE_LITTLE_ENDIAN
+	return bswap_32(native);
+#else
+	return native;
+#endif
+}
+
+/**
+ * cpu_to_be16 - convert a uint16_t value to big endian.
+ * @native: value to convert
+ */
+static inline uint16_t cpu_to_be16(uint16_t native)
+{
+#if HAVE_LITTLE_ENDIAN
+	return bswap_16(native);
+#else
+	return native;
+#endif
+}
+
+/**
+ * be64_to_cpu - convert a big-endian uint64_t value
+ * @be_val: big-endian value to convert
+ */
+static inline uint64_t be64_to_cpu(uint64_t be_val)
+{
+#if HAVE_LITTLE_ENDIAN
+	return bswap_64(be_val);
+#else
+	return be_val;
+#endif
+}
+
+/**
+ * be32_to_cpu - convert a big-endian uint32_t value
+ * @be_val: big-endian value to convert
+ */
+static inline uint32_t be32_to_cpu(uint32_t be_val)
+{
+#if HAVE_LITTLE_ENDIAN
+	return bswap_32(be_val);
+#else
+	return be_val;
+#endif
+}
+
+/**
+ * be16_to_cpu - convert a big-endian uint16_t value
+ * @be_val: big-endian value to convert
+ */
+static inline uint16_t be16_to_cpu(uint16_t be_val)
+{
+#if HAVE_LITTLE_ENDIAN
+	return bswap_16(be_val);
+#else
+	return be_val;
+#endif
+}
+
+#endif /* CCAN_ENDIAN_H */
diff --git a/lib/ccan/endian/test/run.c b/lib/ccan/endian/test/run.c
new file mode 100644
index 00000000000..a00fce74e45
--- /dev/null
+++ b/lib/ccan/endian/test/run.c
@@ -0,0 +1,106 @@
+#include <ccan/endian/endian.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <ccan/tap/tap.h>
+
+int main(int argc, char *argv[])
+{
+	union {
+		uint64_t u64;
+		unsigned char u64_bytes[8];
+	} u64;
+	union {
+		uint32_t u32;
+		unsigned char u32_bytes[4];
+	} u32;
+	union {
+		uint16_t u16;
+		unsigned char u16_bytes[2];
+	} u16;
+
+	plan_tests(48);
+
+	/* Straight swap tests. */
+	u64.u64_bytes[0] = 0x00;
+	u64.u64_bytes[1] = 0x11;
+	u64.u64_bytes[2] = 0x22;
+	u64.u64_bytes[3] = 0x33;
+	u64.u64_bytes[4] = 0x44;
+	u64.u64_bytes[5] = 0x55;
+	u64.u64_bytes[6] = 0x66;
+	u64.u64_bytes[7] = 0x77;
+	u64.u64 = bswap_64(u64.u64);
+	ok1(u64.u64_bytes[7] == 0x00);
+	ok1(u64.u64_bytes[6] == 0x11);
+	ok1(u64.u64_bytes[5] == 0x22);
+	ok1(u64.u64_bytes[4] == 0x33);
+	ok1(u64.u64_bytes[3] == 0x44);
+	ok1(u64.u64_bytes[2] == 0x55);
+	ok1(u64.u64_bytes[1] == 0x66);
+	ok1(u64.u64_bytes[0] == 0x77);
+
+	u32.u32_bytes[0] = 0x00;
+	u32.u32_bytes[1] = 0x11;
+	u32.u32_bytes[2] = 0x22;
+	u32.u32_bytes[3] = 0x33;
+	u32.u32 = bswap_32(u32.u32);
+	ok1(u32.u32_bytes[3] == 0x00);
+	ok1(u32.u32_bytes[2] == 0x11);
+	ok1(u32.u32_bytes[1] == 0x22);
+	ok1(u32.u32_bytes[0] == 0x33);
+
+	u16.u16_bytes[0] = 0x00;
+	u16.u16_bytes[1] = 0x11;
+	u16.u16 = bswap_16(u16.u16);
+	ok1(u16.u16_bytes[1] == 0x00);
+	ok1(u16.u16_bytes[0] == 0x11);
+
+	/* Endian tests. */
+	u64.u64 = cpu_to_le64(0x0011223344556677ULL);
+	ok1(u64.u64_bytes[0] == 0x77);
+	ok1(u64.u64_bytes[1] == 0x66);
+	ok1(u64.u64_bytes[2] == 0x55);
+	ok1(u64.u64_bytes[3] == 0x44);
+	ok1(u64.u64_bytes[4] == 0x33);
+	ok1(u64.u64_bytes[5] == 0x22);
+	ok1(u64.u64_bytes[6] == 0x11);
+	ok1(u64.u64_bytes[7] == 0x00);
+	ok1(le64_to_cpu(u64.u64) == 0x0011223344556677ULL);
+
+	u64.u64 = cpu_to_be64(0x0011223344556677ULL);
+	ok1(u64.u64_bytes[7] == 0x77);
+	ok1(u64.u64_bytes[6] == 0x66);
+	ok1(u64.u64_bytes[5] == 0x55);
+	ok1(u64.u64_bytes[4] == 0x44);
+	ok1(u64.u64_bytes[3] == 0x33);
+	ok1(u64.u64_bytes[2] == 0x22);
+	ok1(u64.u64_bytes[1] == 0x11);
+	ok1(u64.u64_bytes[0] == 0x00);
+	ok1(be64_to_cpu(u64.u64) == 0x0011223344556677ULL);
+
+	u32.u32 = cpu_to_le32(0x00112233);
+	ok1(u32.u32_bytes[0] == 0x33);
+	ok1(u32.u32_bytes[1] == 0x22);
+	ok1(u32.u32_bytes[2] == 0x11);
+	ok1(u32.u32_bytes[3] == 0x00);
+	ok1(le32_to_cpu(u32.u32) == 0x00112233);
+
+	u32.u32 = cpu_to_be32(0x00112233);
+	ok1(u32.u32_bytes[3] == 0x33);
+	ok1(u32.u32_bytes[2] == 0x22);
+	ok1(u32.u32_bytes[1] == 0x11);
+	ok1(u32.u32_bytes[0] == 0x00);
+	ok1(be32_to_cpu(u32.u32) == 0x00112233);
+
+	u16.u16 = cpu_to_le16(0x0011);
+	ok1(u16.u16_bytes[0] == 0x11);
+	ok1(u16.u16_bytes[1] == 0x00);
+	ok1(le16_to_cpu(u16.u16) == 0x0011);
+
+	u16.u16 = cpu_to_be16(0x0011);
+	ok1(u16.u16_bytes[1] == 0x11);
+	ok1(u16.u16_bytes[0] == 0x00);
+	ok1(be16_to_cpu(u16.u16) == 0x0011);
+
+	exit(exit_status());
+}
diff --git a/lib/ccan/hash/_info b/lib/ccan/hash/_info
new file mode 100644
index 00000000000..5aeb9121369
--- /dev/null
+++ b/lib/ccan/hash/_info
@@ -0,0 +1,31 @@
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * hash - routines for hashing bytes
+ *
+ * When creating a hash table it's important to have a hash function
+ * which mixes well and is fast.  This package supplies such functions.
+ *
+ * The hash functions come in two flavors: the normal ones and the
+ * stable ones.  The normal ones can vary from machine-to-machine and
+ * may change if we find better or faster hash algorithms in future.
+ * The stable ones will always give the same results on any computer,
+ * and on any version of this package.
+ *
+ * License: Public Domain
+ * Maintainer: Rusty Russell <rusty@rustcorp.com.au>
+ * Author: Bob Jenkins <bob_jenkins@burtleburtle.net>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/hash/hash.c b/lib/ccan/hash/hash.c
new file mode 100644
index 00000000000..0fd6109513e
--- /dev/null
+++ b/lib/ccan/hash/hash.c
@@ -0,0 +1,925 @@
+/*
+-------------------------------------------------------------------------------
+lookup3.c, by Bob Jenkins, May 2006, Public Domain.
+
+These are functions for producing 32-bit hashes for hash table lookup.
+hash_word(), hashlittle(), hashlittle2(), hashbig(), mix(), and final()
+are externally useful functions.  Routines to test the hash are included
+if SELF_TEST is defined.  You can use this free for any purpose.  It's in
+the public domain.  It has no warranty.
+
+You probably want to use hashlittle().  hashlittle() and hashbig()
+hash byte arrays.  hashlittle() is is faster than hashbig() on
+little-endian machines.  Intel and AMD are little-endian machines.
+On second thought, you probably want hashlittle2(), which is identical to
+hashlittle() except it returns two 32-bit hashes for the price of one.
+You could implement hashbig2() if you wanted but I haven't bothered here.
+
+If you want to find a hash of, say, exactly 7 integers, do
+  a = i1;  b = i2;  c = i3;
+  mix(a,b,c);
+  a += i4; b += i5; c += i6;
+  mix(a,b,c);
+  a += i7;
+  final(a,b,c);
+then use c as the hash value.  If you have a variable length array of
+4-byte integers to hash, use hash_word().  If you have a byte array (like
+a character string), use hashlittle().  If you have several byte arrays, or
+a mix of things, see the comments above hashlittle().
+
+Why is this so big?  I read 12 bytes at a time into 3 4-byte integers,
+then mix those integers.  This is fast (you can do a lot more thorough
+mixing with 12*3 instructions on 3 integers than you can with 3 instructions
+on 1 byte), but shoehorning those bytes into integers efficiently is messy.
+-------------------------------------------------------------------------------
+*/
+//#define SELF_TEST 1
+
+#if 0
+#include <stdio.h>      /* defines printf for tests */
+#include <time.h>       /* defines time_t for timings in the test */
+#include <stdint.h>     /* defines uint32_t etc */
+#include <sys/param.h>  /* attempt to define endianness */
+
+#ifdef linux
+# include <endian.h>    /* attempt to define endianness */
+#endif
+
+/*
+ * My best guess at if you are big-endian or little-endian.  This may
+ * need adjustment.
+ */
+#if (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
+     __BYTE_ORDER == __LITTLE_ENDIAN) || \
+    (defined(i386) || defined(__i386__) || defined(__i486__) || \
+     defined(__i586__) || defined(__i686__) || defined(__x86_64) || \
+     defined(vax) || defined(MIPSEL))
+# define HASH_LITTLE_ENDIAN 1
+# define HASH_BIG_ENDIAN 0
+#elif (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && \
+       __BYTE_ORDER == __BIG_ENDIAN) || \
+      (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel))
+# define HASH_LITTLE_ENDIAN 0
+# define HASH_BIG_ENDIAN 1
+#else
+# error Unknown endian
+#endif
+#endif /* old hash.c headers. */
+
+#include "hash.h"
+
+#if HAVE_LITTLE_ENDIAN
+#define HASH_LITTLE_ENDIAN 1
+#define HASH_BIG_ENDIAN 0
+#elif HAVE_BIG_ENDIAN
+#define HASH_LITTLE_ENDIAN 0
+#define HASH_BIG_ENDIAN 1
+#else
+#error Unknown endian
+#endif
+
+#define hashsize(n) ((uint32_t)1<<(n))
+#define hashmask(n) (hashsize(n)-1)
+#define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k))))
+
+/*
+-------------------------------------------------------------------------------
+mix -- mix 3 32-bit values reversibly.
+
+This is reversible, so any information in (a,b,c) before mix() is
+still in (a,b,c) after mix().
+
+If four pairs of (a,b,c) inputs are run through mix(), or through
+mix() in reverse, there are at least 32 bits of the output that
+are sometimes the same for one pair and different for another pair.
+This was tested for:
+* pairs that differed by one bit, by two bits, in any combination
+  of top bits of (a,b,c), or in any combination of bottom bits of
+  (a,b,c).
+* "differ" is defined as +, -, ^, or ~^.  For + and -, I transformed
+  the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
+  is commonly produced by subtraction) look like a single 1-bit
+  difference.
+* the base values were pseudorandom, all zero but one bit set, or
+  all zero plus a counter that starts at zero.
+
+Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that
+satisfy this are
+    4  6  8 16 19  4
+    9 15  3 18 27 15
+   14  9  3  7 17  3
+Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing
+for "differ" defined as + with a one-bit base and a two-bit delta.  I
+used http://burtleburtle.net/bob/hash/avalanche.html to choose
+the operations, constants, and arrangements of the variables.
+
+This does not achieve avalanche.  There are input bits of (a,b,c)
+that fail to affect some output bits of (a,b,c), especially of a.  The
+most thoroughly mixed value is c, but it doesn't really even achieve
+avalanche in c.
+
+This allows some parallelism.  Read-after-writes are good at doubling
+the number of bits affected, so the goal of mixing pulls in the opposite
+direction as the goal of parallelism.  I did what I could.  Rotates
+seem to cost as much as shifts on every machine I could lay my hands
+on, and rotates are much kinder to the top and bottom bits, so I used
+rotates.
+-------------------------------------------------------------------------------
+*/
+#define mix(a,b,c) \
+{ \
+  a -= c;  a ^= rot(c, 4);  c += b; \
+  b -= a;  b ^= rot(a, 6);  a += c; \
+  c -= b;  c ^= rot(b, 8);  b += a; \
+  a -= c;  a ^= rot(c,16);  c += b; \
+  b -= a;  b ^= rot(a,19);  a += c; \
+  c -= b;  c ^= rot(b, 4);  b += a; \
+}
+
+/*
+-------------------------------------------------------------------------------
+final -- final mixing of 3 32-bit values (a,b,c) into c
+
+Pairs of (a,b,c) values differing in only a few bits will usually
+produce values of c that look totally different.  This was tested for
+* pairs that differed by one bit, by two bits, in any combination
+  of top bits of (a,b,c), or in any combination of bottom bits of
+  (a,b,c).
+* "differ" is defined as +, -, ^, or ~^.  For + and -, I transformed
+  the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
+  is commonly produced by subtraction) look like a single 1-bit
+  difference.
+* the base values were pseudorandom, all zero but one bit set, or
+  all zero plus a counter that starts at zero.
+
+These constants passed:
+ 14 11 25 16 4 14 24
+ 12 14 25 16 4 14 24
+and these came close:
+  4  8 15 26 3 22 24
+ 10  8 15 26 3 22 24
+ 11  8 15 26 3 22 24
+-------------------------------------------------------------------------------
+*/
+#define final(a,b,c) \
+{ \
+  c ^= b; c -= rot(b,14); \
+  a ^= c; a -= rot(c,11); \
+  b ^= a; b -= rot(a,25); \
+  c ^= b; c -= rot(b,16); \
+  a ^= c; a -= rot(c,4);  \
+  b ^= a; b -= rot(a,14); \
+  c ^= b; c -= rot(b,24); \
+}
+
+/*
+--------------------------------------------------------------------
+ This works on all machines.  To be useful, it requires
+ -- that the key be an array of uint32_t's, and
+ -- that the length be the number of uint32_t's in the key
+
+ The function hash_word() is identical to hashlittle() on little-endian
+ machines, and identical to hashbig() on big-endian machines,
+ except that the length has to be measured in uint32_ts rather than in
+ bytes.  hashlittle() is more complicated than hash_word() only because
+ hashlittle() has to dance around fitting the key bytes into registers.
+--------------------------------------------------------------------
+*/
+uint32_t hash_u32(
+const uint32_t *k,                   /* the key, an array of uint32_t values */
+size_t          length,               /* the length of the key, in uint32_ts */
+uint32_t        initval)         /* the previous hash, or an arbitrary value */
+{
+  uint32_t a,b,c;
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + (((uint32_t)length)<<2) + initval;
+
+  /*------------------------------------------------- handle most of the key */
+  while (length > 3)
+  {
+    a += k[0];
+    b += k[1];
+    c += k[2];
+    mix(a,b,c);
+    length -= 3;
+    k += 3;
+  }
+
+  /*------------------------------------------- handle the last 3 uint32_t's */
+  switch(length)                     /* all the case statements fall through */
+  {
+  case 3 : c+=k[2];
+  case 2 : b+=k[1];
+  case 1 : a+=k[0];
+    final(a,b,c);
+  case 0:     /* case 0: nothing left to add */
+    break;
+  }
+  /*------------------------------------------------------ report the result */
+  return c;
+}
+
+/*
+-------------------------------------------------------------------------------
+hashlittle() -- hash a variable-length key into a 32-bit value
+  k       : the key (the unaligned variable-length array of bytes)
+  length  : the length of the key, counting by bytes
+  val2    : IN: can be any 4-byte value OUT: second 32 bit hash.
+Returns a 32-bit value.  Every bit of the key affects every bit of
+the return value.  Two keys differing by one or two bits will have
+totally different hash values.  Note that the return value is better
+mixed than val2, so use that first.
+
+The best hash table sizes are powers of 2.  There is no need to do
+mod a prime (mod is sooo slow!).  If you need less than 32 bits,
+use a bitmask.  For example, if you need only 10 bits, do
+  h = (h & hashmask(10));
+In which case, the hash table should have hashsize(10) elements.
+
+If you are hashing n strings (uint8_t **)k, do it like this:
+  for (i=0, h=0; i<n; ++i) h = hashlittle( k[i], len[i], h);
+
+By Bob Jenkins, 2006.  bob_jenkins@burtleburtle.net.  You may use this
+code any way you wish, private, educational, or commercial.  It's free.
+
+Use for hash table lookup, or anything where one collision in 2^^32 is
+acceptable.  Do NOT use for cryptographic purposes.
+-------------------------------------------------------------------------------
+*/
+
+static uint32_t hashlittle( const void *key, size_t length, uint32_t *val2 )
+{
+  uint32_t a,b,c;                                          /* internal state */
+  union { const void *ptr; size_t i; } u;     /* needed for Mac Powerbook G4 */
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + ((uint32_t)length) + *val2;
+
+  u.ptr = key;
+  if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) {
+    const uint32_t *k = (const uint32_t *)key;         /* read 32-bit chunks */
+    const uint8_t  *k8;
+
+    /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      b += k[1];
+      c += k[2];
+      mix(a,b,c);
+      length -= 12;
+      k += 3;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    /*
+     * "k[2]&0xffffff" actually reads beyond the end of the string, but
+     * then masks off the part it's not allowed to read.  Because the
+     * string is aligned, the masked-off tail is in the same word as the
+     * rest of the string.  Every machine with memory protection I've seen
+     * does it on word boundaries, so is OK with this.  But VALGRIND will
+     * still catch it and complain.  The masking trick does make the hash
+     * noticably faster for short strings (like English words).
+     *
+     * Not on my testing with gcc 4.5 on an intel i5 CPU, at least --RR.
+     */
+#if 0
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
+    case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break;
+    case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break;
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=k[1]&0xffffff; a+=k[0]; break;
+    case 6 : b+=k[1]&0xffff; a+=k[0]; break;
+    case 5 : b+=k[1]&0xff; a+=k[0]; break;
+    case 4 : a+=k[0]; break;
+    case 3 : a+=k[0]&0xffffff; break;
+    case 2 : a+=k[0]&0xffff; break;
+    case 1 : a+=k[0]&0xff; break;
+    case 0 : return c;              /* zero length strings require no mixing */
+    }
+
+#else /* make valgrind happy */
+
+    k8 = (const uint8_t *)k;
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=((uint32_t)k8[10])<<16;  /* fall through */
+    case 10: c+=((uint32_t)k8[9])<<8;    /* fall through */
+    case 9 : c+=k8[8];                   /* fall through */
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=((uint32_t)k8[6])<<16;   /* fall through */
+    case 6 : b+=((uint32_t)k8[5])<<8;    /* fall through */
+    case 5 : b+=k8[4];                   /* fall through */
+    case 4 : a+=k[0]; break;
+    case 3 : a+=((uint32_t)k8[2])<<16;   /* fall through */
+    case 2 : a+=((uint32_t)k8[1])<<8;    /* fall through */
+    case 1 : a+=k8[0]; break;
+    case 0 : return c;
+    }
+
+#endif /* !valgrind */
+
+  } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) {
+    const uint16_t *k = (const uint16_t *)key;         /* read 16-bit chunks */
+    const uint8_t  *k8;
+
+    /*--------------- all but last block: aligned reads and different mixing */
+    while (length > 12)
+    {
+      a += k[0] + (((uint32_t)k[1])<<16);
+      b += k[2] + (((uint32_t)k[3])<<16);
+      c += k[4] + (((uint32_t)k[5])<<16);
+      mix(a,b,c);
+      length -= 12;
+      k += 6;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    k8 = (const uint8_t *)k;
+    switch(length)
+    {
+    case 12: c+=k[4]+(((uint32_t)k[5])<<16);
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 11: c+=((uint32_t)k8[10])<<16;     /* fall through */
+    case 10: c+=k[4];
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 9 : c+=k8[8];                      /* fall through */
+    case 8 : b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 7 : b+=((uint32_t)k8[6])<<16;      /* fall through */
+    case 6 : b+=k[2];
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 5 : b+=k8[4];                      /* fall through */
+    case 4 : a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 3 : a+=((uint32_t)k8[2])<<16;      /* fall through */
+    case 2 : a+=k[0];
+             break;
+    case 1 : a+=k8[0];
+             break;
+    case 0 : return c;                     /* zero length requires no mixing */
+    }
+
+  } else {                        /* need to read the key one byte at a time */
+    const uint8_t *k = (const uint8_t *)key;
+
+    /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      a += ((uint32_t)k[1])<<8;
+      a += ((uint32_t)k[2])<<16;
+      a += ((uint32_t)k[3])<<24;
+      b += k[4];
+      b += ((uint32_t)k[5])<<8;
+      b += ((uint32_t)k[6])<<16;
+      b += ((uint32_t)k[7])<<24;
+      c += k[8];
+      c += ((uint32_t)k[9])<<8;
+      c += ((uint32_t)k[10])<<16;
+      c += ((uint32_t)k[11])<<24;
+      mix(a,b,c);
+      length -= 12;
+      k += 12;
+    }
+
+    /*-------------------------------- last block: affect all 32 bits of (c) */
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=((uint32_t)k[11])<<24;
+    case 11: c+=((uint32_t)k[10])<<16;
+    case 10: c+=((uint32_t)k[9])<<8;
+    case 9 : c+=k[8];
+    case 8 : b+=((uint32_t)k[7])<<24;
+    case 7 : b+=((uint32_t)k[6])<<16;
+    case 6 : b+=((uint32_t)k[5])<<8;
+    case 5 : b+=k[4];
+    case 4 : a+=((uint32_t)k[3])<<24;
+    case 3 : a+=((uint32_t)k[2])<<16;
+    case 2 : a+=((uint32_t)k[1])<<8;
+    case 1 : a+=k[0];
+             break;
+    case 0 : return c;
+    }
+  }
+
+  final(a,b,c);
+  *val2 = b;
+  return c;
+}
+
+/*
+ * hashbig():
+ * This is the same as hash_word() on big-endian machines.  It is different
+ * from hashlittle() on all machines.  hashbig() takes advantage of
+ * big-endian byte ordering.
+ */
+static uint32_t hashbig( const void *key, size_t length, uint32_t *val2)
+{
+  uint32_t a,b,c;
+  union { const void *ptr; size_t i; } u; /* to cast key to (size_t) happily */
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + ((uint32_t)length) + *val2;
+
+  u.ptr = key;
+  if (HASH_BIG_ENDIAN && ((u.i & 0x3) == 0)) {
+    const uint32_t *k = (const uint32_t *)key;         /* read 32-bit chunks */
+    const uint8_t  *k8;
+
+    /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      b += k[1];
+      c += k[2];
+      mix(a,b,c);
+      length -= 12;
+      k += 3;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    /*
+     * "k[2]<<8" actually reads beyond the end of the string, but
+     * then shifts out the part it's not allowed to read.  Because the
+     * string is aligned, the illegal read is in the same word as the
+     * rest of the string.  Every machine with memory protection I've seen
+     * does it on word boundaries, so is OK with this.  But VALGRIND will
+     * still catch it and complain.  The masking trick does make the hash
+     * noticably faster for short strings (like English words).
+     *
+     * Not on my testing with gcc 4.5 on an intel i5 CPU, at least --RR.
+     */
+#if 0
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break;
+    case 10: c+=k[2]&0xffff0000; b+=k[1]; a+=k[0]; break;
+    case 9 : c+=k[2]&0xff000000; b+=k[1]; a+=k[0]; break;
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=k[1]&0xffffff00; a+=k[0]; break;
+    case 6 : b+=k[1]&0xffff0000; a+=k[0]; break;
+    case 5 : b+=k[1]&0xff000000; a+=k[0]; break;
+    case 4 : a+=k[0]; break;
+    case 3 : a+=k[0]&0xffffff00; break;
+    case 2 : a+=k[0]&0xffff0000; break;
+    case 1 : a+=k[0]&0xff000000; break;
+    case 0 : return c;              /* zero length strings require no mixing */
+    }
+
+#else  /* make valgrind happy */
+
+    k8 = (const uint8_t *)k;
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=((uint32_t)k8[10])<<8;  /* fall through */
+    case 10: c+=((uint32_t)k8[9])<<16;  /* fall through */
+    case 9 : c+=((uint32_t)k8[8])<<24;  /* fall through */
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=((uint32_t)k8[6])<<8;   /* fall through */
+    case 6 : b+=((uint32_t)k8[5])<<16;  /* fall through */
+    case 5 : b+=((uint32_t)k8[4])<<24;  /* fall through */
+    case 4 : a+=k[0]; break;
+    case 3 : a+=((uint32_t)k8[2])<<8;   /* fall through */
+    case 2 : a+=((uint32_t)k8[1])<<16;  /* fall through */
+    case 1 : a+=((uint32_t)k8[0])<<24; break;
+    case 0 : return c;
+    }
+
+#endif /* !VALGRIND */
+
+  } else {                        /* need to read the key one byte at a time */
+    const uint8_t *k = (const uint8_t *)key;
+
+    /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += ((uint32_t)k[0])<<24;
+      a += ((uint32_t)k[1])<<16;
+      a += ((uint32_t)k[2])<<8;
+      a += ((uint32_t)k[3]);
+      b += ((uint32_t)k[4])<<24;
+      b += ((uint32_t)k[5])<<16;
+      b += ((uint32_t)k[6])<<8;
+      b += ((uint32_t)k[7]);
+      c += ((uint32_t)k[8])<<24;
+      c += ((uint32_t)k[9])<<16;
+      c += ((uint32_t)k[10])<<8;
+      c += ((uint32_t)k[11]);
+      mix(a,b,c);
+      length -= 12;
+      k += 12;
+    }
+
+    /*-------------------------------- last block: affect all 32 bits of (c) */
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=k[11];
+    case 11: c+=((uint32_t)k[10])<<8;
+    case 10: c+=((uint32_t)k[9])<<16;
+    case 9 : c+=((uint32_t)k[8])<<24;
+    case 8 : b+=k[7];
+    case 7 : b+=((uint32_t)k[6])<<8;
+    case 6 : b+=((uint32_t)k[5])<<16;
+    case 5 : b+=((uint32_t)k[4])<<24;
+    case 4 : a+=k[3];
+    case 3 : a+=((uint32_t)k[2])<<8;
+    case 2 : a+=((uint32_t)k[1])<<16;
+    case 1 : a+=((uint32_t)k[0])<<24;
+             break;
+    case 0 : return c;
+    }
+  }
+
+  final(a,b,c);
+  *val2 = b;
+  return c;
+}
+
+/* I basically use hashlittle here, but use native endian within each
+ * element.  This delivers least-surprise: hash such as "int arr[] = {
+ * 1, 2 }; hash_stable(arr, 2, 0);" will be the same on big and little
+ * endian machines, even though a bytewise hash wouldn't be. */
+uint64_t hash64_stable_64(const void *key, size_t n, uint64_t base)
+{
+	const uint64_t *k = key;
+	uint32_t a,b,c;
+
+	/* Set up the internal state */
+	a = b = c = 0xdeadbeef + ((uint32_t)n*8) + (base >> 32) + base;
+
+	while (n > 3) {
+		a += (uint32_t)k[0];
+		b += (uint32_t)(k[0] >> 32);
+		c += (uint32_t)k[1];
+		mix(a,b,c);
+		a += (uint32_t)(k[1] >> 32);
+		b += (uint32_t)k[2];
+		c += (uint32_t)(k[2] >> 32);
+		mix(a,b,c);
+		n -= 3;
+		k += 3;
+	}
+	switch (n) {
+	case 2:
+		a += (uint32_t)k[0];
+		b += (uint32_t)(k[0] >> 32);
+		c += (uint32_t)k[1];
+		mix(a,b,c);
+		a += (uint32_t)(k[1] >> 32);
+		break;
+	case 1:
+		a += (uint32_t)k[0];
+		b += (uint32_t)(k[0] >> 32);
+		break;
+	case 0:
+		return c;
+	}
+	final(a,b,c);
+	return ((uint64_t)b << 32) | c;
+}
+
+uint64_t hash64_stable_32(const void *key, size_t n, uint64_t base)
+{
+	const uint32_t *k = key;
+	uint32_t a,b,c;
+
+	/* Set up the internal state */
+	a = b = c = 0xdeadbeef + ((uint32_t)n*4) + (base >> 32) + base;
+
+	while (n > 3) {
+		a += k[0];
+		b += k[1];
+		c += k[2];
+		mix(a,b,c);
+
+		n -= 3;
+		k += 3;
+	}
+	switch (n) {
+	case 2:
+		b += (uint32_t)k[1];
+	case 1:
+		a += (uint32_t)k[0];
+		break;
+	case 0:
+		return c;
+	}
+	final(a,b,c);
+	return ((uint64_t)b << 32) | c;
+}
+
+uint64_t hash64_stable_16(const void *key, size_t n, uint64_t base)
+{
+	const uint16_t *k = key;
+	uint32_t a,b,c;
+
+	/* Set up the internal state */
+	a = b = c = 0xdeadbeef + ((uint32_t)n*2) + (base >> 32) + base;
+
+	while (n > 6) {
+		a += (uint32_t)k[0] + ((uint32_t)k[1] << 16);
+		b += (uint32_t)k[2] + ((uint32_t)k[3] << 16);
+		c += (uint32_t)k[4] + ((uint32_t)k[5] << 16);
+		mix(a,b,c);
+
+		n -= 6;
+		k += 6;
+	}
+
+	switch (n) {
+	case 5:
+		c += (uint32_t)k[4];
+	case 4:
+		b += ((uint32_t)k[3] << 16);
+	case 3:
+		b += (uint32_t)k[2];
+	case 2:
+		a += ((uint32_t)k[1] << 16);
+	case 1:
+		a += (uint32_t)k[0];
+		break;
+	case 0:
+		return c;
+	}
+	final(a,b,c);
+	return ((uint64_t)b << 32) | c;
+}
+
+uint64_t hash64_stable_8(const void *key, size_t n, uint64_t base)
+{
+	uint32_t b32 = base + (base >> 32);
+	uint32_t lower = hashlittle(key, n, &b32);
+
+	return ((uint64_t)b32 << 32) | lower;
+}
+
+uint32_t hash_any(const void *key, size_t length, uint32_t base)
+{
+	if (HASH_BIG_ENDIAN)
+		return hashbig(key, length, &base);
+	else
+		return hashlittle(key, length, &base);
+}
+
+uint32_t hash_stable_64(const void *key, size_t n, uint32_t base)
+{
+	return hash64_stable_64(key, n, base);
+}
+
+uint32_t hash_stable_32(const void *key, size_t n, uint32_t base)
+{
+	return hash64_stable_32(key, n, base);
+}
+
+uint32_t hash_stable_16(const void *key, size_t n, uint32_t base)
+{
+	return hash64_stable_16(key, n, base);
+}
+
+uint32_t hash_stable_8(const void *key, size_t n, uint32_t base)
+{
+	return hashlittle(key, n, &base);
+}
+
+/* Jenkins' lookup8 is a 64 bit hash, but he says it's obsolete.  Use
+ * the plain one and recombine into 64 bits. */
+uint64_t hash64_any(const void *key, size_t length, uint64_t base)
+{
+	uint32_t b32 = base + (base >> 32);
+	uint32_t lower;
+
+	if (HASH_BIG_ENDIAN)
+		lower = hashbig(key, length, &b32);
+	else
+		lower = hashlittle(key, length, &b32);
+
+	return ((uint64_t)b32 << 32) | lower;
+}
+
+#ifdef SELF_TEST
+
+/* used for timings */
+void driver1()
+{
+  uint8_t buf[256];
+  uint32_t i;
+  uint32_t h=0;
+  time_t a,z;
+
+  time(&a);
+  for (i=0; i<256; ++i) buf[i] = 'x';
+  for (i=0; i<1; ++i)
+  {
+    h = hashlittle(&buf[0],1,h);
+  }
+  time(&z);
+  if (z-a > 0) printf("time %d %.8x\n", z-a, h);
+}
+
+/* check that every input bit changes every output bit half the time */
+#define HASHSTATE 1
+#define HASHLEN   1
+#define MAXPAIR 60
+#define MAXLEN  70
+void driver2()
+{
+  uint8_t qa[MAXLEN+1], qb[MAXLEN+2], *a = &qa[0], *b = &qb[1];
+  uint32_t c[HASHSTATE], d[HASHSTATE], i=0, j=0, k, l, m=0, z;
+  uint32_t e[HASHSTATE],f[HASHSTATE],g[HASHSTATE],h[HASHSTATE];
+  uint32_t x[HASHSTATE],y[HASHSTATE];
+  uint32_t hlen;
+
+  printf("No more than %d trials should ever be needed \n",MAXPAIR/2);
+  for (hlen=0; hlen < MAXLEN; ++hlen)
+  {
+    z=0;
+    for (i=0; i<hlen; ++i)  /*----------------------- for each input byte, */
+    {
+      for (j=0; j<8; ++j)   /*------------------------ for each input bit, */
+      {
+	for (m=1; m<8; ++m) /*------------ for several possible initvals, */
+	{
+	  for (l=0; l<HASHSTATE; ++l)
+	    e[l]=f[l]=g[l]=h[l]=x[l]=y[l]=~((uint32_t)0);
+
+	  /*---- check that every output bit is affected by that input bit */
+	  for (k=0; k<MAXPAIR; k+=2)
+	  {
+	    uint32_t finished=1;
+	    /* keys have one bit different */
+	    for (l=0; l<hlen+1; ++l) {a[l] = b[l] = (uint8_t)0;}
+	    /* have a and b be two keys differing in only one bit */
+	    a[i] ^= (k<<j);
+	    a[i] ^= (k>>(8-j));
+	     c[0] = hashlittle(a, hlen, m);
+	    b[i] ^= ((k+1)<<j);
+	    b[i] ^= ((k+1)>>(8-j));
+	     d[0] = hashlittle(b, hlen, m);
+	    /* check every bit is 1, 0, set, and not set at least once */
+	    for (l=0; l<HASHSTATE; ++l)
+	    {
+	      e[l] &= (c[l]^d[l]);
+	      f[l] &= ~(c[l]^d[l]);
+	      g[l] &= c[l];
+	      h[l] &= ~c[l];
+	      x[l] &= d[l];
+	      y[l] &= ~d[l];
+	      if (e[l]|f[l]|g[l]|h[l]|x[l]|y[l]) finished=0;
+	    }
+	    if (finished) break;
+	  }
+	  if (k>z) z=k;
+	  if (k==MAXPAIR)
+	  {
+	     printf("Some bit didn't change: ");
+	     printf("%.8x %.8x %.8x %.8x %.8x %.8x  ",
+	            e[0],f[0],g[0],h[0],x[0],y[0]);
+	     printf("i %d j %d m %d len %d\n", i, j, m, hlen);
+	  }
+	  if (z==MAXPAIR) goto done;
+	}
+      }
+    }
+   done:
+    if (z < MAXPAIR)
+    {
+      printf("Mix success  %2d bytes  %2d initvals  ",i,m);
+      printf("required  %d  trials\n", z/2);
+    }
+  }
+  printf("\n");
+}
+
+/* Check for reading beyond the end of the buffer and alignment problems */
+void driver3()
+{
+  uint8_t buf[MAXLEN+20], *b;
+  uint32_t len;
+  uint8_t q[] = "This is the time for all good men to come to the aid of their country...";
+  uint32_t h;
+  uint8_t qq[] = "xThis is the time for all good men to come to the aid of their country...";
+  uint32_t i;
+  uint8_t qqq[] = "xxThis is the time for all good men to come to the aid of their country...";
+  uint32_t j;
+  uint8_t qqqq[] = "xxxThis is the time for all good men to come to the aid of their country...";
+  uint32_t ref,x,y;
+  uint8_t *p;
+
+  printf("Endianness.  These lines should all be the same (for values filled in):\n");
+  printf("%.8x                            %.8x                            %.8x\n",
+         hash_word((const uint32_t *)q, (sizeof(q)-1)/4, 13),
+         hash_word((const uint32_t *)q, (sizeof(q)-5)/4, 13),
+         hash_word((const uint32_t *)q, (sizeof(q)-9)/4, 13));
+  p = q;
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  p = &qq[1];
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  p = &qqq[2];
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  p = &qqqq[3];
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  printf("\n");
+
+  /* check that hashlittle2 and hashlittle produce the same results */
+  i=47; j=0;
+  hashlittle2(q, sizeof(q), &i, &j);
+  if (hashlittle(q, sizeof(q), 47) != i)
+    printf("hashlittle2 and hashlittle mismatch\n");
+
+  /* check that hash_word2 and hash_word produce the same results */
+  len = 0xdeadbeef;
+  i=47, j=0;
+  hash_word2(&len, 1, &i, &j);
+  if (hash_word(&len, 1, 47) != i)
+    printf("hash_word2 and hash_word mismatch %x %x\n",
+	   i, hash_word(&len, 1, 47));
+
+  /* check hashlittle doesn't read before or after the ends of the string */
+  for (h=0, b=buf+1; h<8; ++h, ++b)
+  {
+    for (i=0; i<MAXLEN; ++i)
+    {
+      len = i;
+      for (j=0; j<i; ++j) *(b+j)=0;
+
+      /* these should all be equal */
+      ref = hashlittle(b, len, (uint32_t)1);
+      *(b+i)=(uint8_t)~0;
+      *(b-1)=(uint8_t)~0;
+      x = hashlittle(b, len, (uint32_t)1);
+      y = hashlittle(b, len, (uint32_t)1);
+      if ((ref != x) || (ref != y))
+      {
+	printf("alignment error: %.8x %.8x %.8x %d %d\n",ref,x,y,
+               h, i);
+      }
+    }
+  }
+}
+
+/* check for problems with nulls */
+ void driver4()
+{
+  uint8_t buf[1];
+  uint32_t h,i,state[HASHSTATE];
+
+
+  buf[0] = ~0;
+  for (i=0; i<HASHSTATE; ++i) state[i] = 1;
+  printf("These should all be different\n");
+  for (i=0, h=0; i<8; ++i)
+  {
+    h = hashlittle(buf, 0, h);
+    printf("%2ld  0-byte strings, hash is  %.8x\n", i, h);
+  }
+}
+
+
+int main()
+{
+  driver1();   /* test that the key is hashed: used for timings */
+  driver2();   /* test that whole key is hashed thoroughly */
+  driver3();   /* test that nothing but the key is hashed */
+  driver4();   /* test hashing multiple buffers (all buffers are null) */
+  return 1;
+}
+
+#endif  /* SELF_TEST */
diff --git a/lib/ccan/hash/hash.h b/lib/ccan/hash/hash.h
new file mode 100644
index 00000000000..5025c0d7487
--- /dev/null
+++ b/lib/ccan/hash/hash.h
@@ -0,0 +1,312 @@
+#ifndef CCAN_HASH_H
+#define CCAN_HASH_H
+#include "config.h"
+#include <stdint.h>
+#include <stdlib.h>
+#include <ccan/build_assert/build_assert.h>
+
+/* Stolen mostly from: lookup3.c, by Bob Jenkins, May 2006, Public Domain.
+ *
+ * http://burtleburtle.net/bob/c/lookup3.c
+ */
+
+/**
+ * hash - fast hash of an array for internal use
+ * @p: the array or pointer to first element
+ * @num: the number of elements to hash
+ * @base: the base number to roll into the hash (usually 0)
+ *
+ * The memory region pointed to by p is combined with the base to form
+ * a 32-bit hash.
+ *
+ * This hash will have different results on different machines, so is
+ * only useful for internal hashes (ie. not hashes sent across the
+ * network or saved to disk).
+ *
+ * It may also change with future versions: it could even detect at runtime
+ * what the fastest hash to use is.
+ *
+ * See also: hash64, hash_stable.
+ *
+ * Example:
+ *	#include <ccan/hash/hash.h>
+ *	#include <err.h>
+ *	#include <stdio.h>
+ *	#include <string.h>
+ *
+ *	// Simple demonstration: idential strings will have the same hash, but
+ *	// two different strings will probably not.
+ *	int main(int argc, char *argv[])
+ *	{
+ *		uint32_t hash1, hash2;
+ *
+ *		if (argc != 3)
+ *			err(1, "Usage: %s <string1> <string2>", argv[0]);
+ *
+ *		hash1 = hash(argv[1], strlen(argv[1]), 0);
+ *		hash2 = hash(argv[2], strlen(argv[2]), 0);
+ *		printf("Hash is %s\n", hash1 == hash2 ? "same" : "different");
+ *		return 0;
+ *	}
+ */
+#define hash(p, num, base) hash_any((p), (num)*sizeof(*(p)), (base))
+
+/**
+ * hash_stable - hash of an array for external use
+ * @p: the array or pointer to first element
+ * @num: the number of elements to hash
+ * @base: the base number to roll into the hash (usually 0)
+ *
+ * The array of simple integer types pointed to by p is combined with
+ * the base to form a 32-bit hash.
+ *
+ * This hash will have the same results on different machines, so can
+ * be used for external hashes (ie. hashes sent across the network or
+ * saved to disk).  The results will not change in future versions of
+ * this module.
+ *
+ * Note that it is only legal to hand an array of simple integer types
+ * to this hash (ie. char, uint16_t, int64_t, etc).  In these cases,
+ * the same values will have the same hash result, even though the
+ * memory representations of integers depend on the machine
+ * endianness.
+ *
+ * See also:
+ *	hash64_stable
+ *
+ * Example:
+ *	#include <ccan/hash/hash.h>
+ *	#include <err.h>
+ *	#include <stdio.h>
+ *	#include <string.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		if (argc != 2)
+ *			err(1, "Usage: %s <string-to-hash>", argv[0]);
+ *
+ *		printf("Hash stable result is %u\n",
+ *		       hash_stable(argv[1], strlen(argv[1]), 0));
+ *		return 0;
+ *	}
+ */
+#define hash_stable(p, num, base)					\
+	(BUILD_ASSERT_OR_ZERO(sizeof(*(p)) == 8 || sizeof(*(p)) == 4	\
+			      || sizeof(*(p)) == 2 || sizeof(*(p)) == 1) + \
+	 sizeof(*(p)) == 8 ? hash_stable_64((p), (num), (base))		\
+	 : sizeof(*(p)) == 4 ? hash_stable_32((p), (num), (base))	\
+	 : sizeof(*(p)) == 2 ? hash_stable_16((p), (num), (base))	\
+	 : hash_stable_8((p), (num), (base)))
+
+/**
+ * hash_u32 - fast hash an array of 32-bit values for internal use
+ * @key: the array of uint32_t
+ * @num: the number of elements to hash
+ * @base: the base number to roll into the hash (usually 0)
+ *
+ * The array of uint32_t pointed to by @key is combined with the base
+ * to form a 32-bit hash.  This is 2-3 times faster than hash() on small
+ * arrays, but the advantage vanishes over large hashes.
+ *
+ * This hash will have different results on different machines, so is
+ * only useful for internal hashes (ie. not hashes sent across the
+ * network or saved to disk).
+ */
+uint32_t hash_u32(const uint32_t *key, size_t num, uint32_t base);
+
+/**
+ * hash_string - very fast hash of an ascii string
+ * @str: the nul-terminated string
+ *
+ * The string is hashed, using a hash function optimized for ASCII and
+ * similar strings.  It's weaker than the other hash functions.
+ *
+ * This hash may have different results on different machines, so is
+ * only useful for internal hashes (ie. not hashes sent across the
+ * network or saved to disk).  The results will be different from the
+ * other hash functions in this module, too.
+ */
+static inline uint32_t hash_string(const char *string)
+{
+	/* This is Karl Nelson <kenelson@ece.ucdavis.edu>'s X31 hash.
+	 * It's a little faster than the (much better) lookup3 hash(): 56ns vs
+	 * 84ns on my 2GHz Intel Core Duo 2 laptop for a 10 char string. */
+	uint32_t ret;
+
+	for (ret = 0; *string; string++)
+		ret = (ret << 5) - ret + *string;
+
+	return ret;
+}
+
+/**
+ * hash64 - fast 64-bit hash of an array for internal use
+ * @p: the array or pointer to first element
+ * @num: the number of elements to hash
+ * @base: the 64-bit base number to roll into the hash (usually 0)
+ *
+ * The memory region pointed to by p is combined with the base to form
+ * a 64-bit hash.
+ *
+ * This hash will have different results on different machines, so is
+ * only useful for internal hashes (ie. not hashes sent across the
+ * network or saved to disk).
+ *
+ * It may also change with future versions: it could even detect at runtime
+ * what the fastest hash to use is.
+ *
+ * See also: hash.
+ *
+ * Example:
+ *	#include <ccan/hash/hash.h>
+ *	#include <err.h>
+ *	#include <stdio.h>
+ *	#include <string.h>
+ *
+ *	// Simple demonstration: idential strings will have the same hash, but
+ *	// two different strings will probably not.
+ *	int main(int argc, char *argv[])
+ *	{
+ *		uint64_t hash1, hash2;
+ *
+ *		if (argc != 3)
+ *			err(1, "Usage: %s <string1> <string2>", argv[0]);
+ *
+ *		hash1 = hash64(argv[1], strlen(argv[1]), 0);
+ *		hash2 = hash64(argv[2], strlen(argv[2]), 0);
+ *		printf("Hash is %s\n", hash1 == hash2 ? "same" : "different");
+ *		return 0;
+ *	}
+ */
+#define hash64(p, num, base) hash64_any((p), (num)*sizeof(*(p)), (base))
+
+/**
+ * hash64_stable - 64 bit hash of an array for external use
+ * @p: the array or pointer to first element
+ * @num: the number of elements to hash
+ * @base: the base number to roll into the hash (usually 0)
+ *
+ * The array of simple integer types pointed to by p is combined with
+ * the base to form a 64-bit hash.
+ *
+ * This hash will have the same results on different machines, so can
+ * be used for external hashes (ie. hashes sent across the network or
+ * saved to disk).  The results will not change in future versions of
+ * this module.
+ *
+ * Note that it is only legal to hand an array of simple integer types
+ * to this hash (ie. char, uint16_t, int64_t, etc).  In these cases,
+ * the same values will have the same hash result, even though the
+ * memory representations of integers depend on the machine
+ * endianness.
+ *
+ * See also:
+ *	hash_stable
+ *
+ * Example:
+ *	#include <ccan/hash/hash.h>
+ *	#include <err.h>
+ *	#include <stdio.h>
+ *	#include <string.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		if (argc != 2)
+ *			err(1, "Usage: %s <string-to-hash>", argv[0]);
+ *
+ *		printf("Hash stable result is %llu\n",
+ *		       (long long)hash64_stable(argv[1], strlen(argv[1]), 0));
+ *		return 0;
+ *	}
+ */
+#define hash64_stable(p, num, base)					\
+	(BUILD_ASSERT_OR_ZERO(sizeof(*(p)) == 8 || sizeof(*(p)) == 4	\
+			      || sizeof(*(p)) == 2 || sizeof(*(p)) == 1) + \
+	 sizeof(*(p)) == 8 ? hash64_stable_64((p), (num), (base))	\
+	 : sizeof(*(p)) == 4 ? hash64_stable_32((p), (num), (base))	\
+	 : sizeof(*(p)) == 2 ? hash64_stable_16((p), (num), (base))	\
+	 : hash64_stable_8((p), (num), (base)))
+
+
+/**
+ * hashl - fast 32/64-bit hash of an array for internal use
+ * @p: the array or pointer to first element
+ * @num: the number of elements to hash
+ * @base: the base number to roll into the hash (usually 0)
+ *
+ * This is either hash() or hash64(), on 32/64 bit long machines.
+ */
+#define hashl(p, num, base)						\
+	(BUILD_ASSERT_OR_ZERO(sizeof(long) == sizeof(uint32_t)		\
+			      || sizeof(long) == sizeof(uint64_t)) +	\
+	(sizeof(long) == sizeof(uint64_t)				\
+	 ? hash64((p), (num), (base)) : hash((p), (num), (base))))
+
+/* Our underlying operations. */
+uint32_t hash_any(const void *key, size_t length, uint32_t base);
+uint32_t hash_stable_64(const void *key, size_t n, uint32_t base);
+uint32_t hash_stable_32(const void *key, size_t n, uint32_t base);
+uint32_t hash_stable_16(const void *key, size_t n, uint32_t base);
+uint32_t hash_stable_8(const void *key, size_t n, uint32_t base);
+uint64_t hash64_any(const void *key, size_t length, uint64_t base);
+uint64_t hash64_stable_64(const void *key, size_t n, uint64_t base);
+uint64_t hash64_stable_32(const void *key, size_t n, uint64_t base);
+uint64_t hash64_stable_16(const void *key, size_t n, uint64_t base);
+uint64_t hash64_stable_8(const void *key, size_t n, uint64_t base);
+
+/**
+ * hash_pointer - hash a pointer for internal use
+ * @p: the pointer value to hash
+ * @base: the base number to roll into the hash (usually 0)
+ *
+ * The pointer p (not what p points to!) is combined with the base to form
+ * a 32-bit hash.
+ *
+ * This hash will have different results on different machines, so is
+ * only useful for internal hashes (ie. not hashes sent across the
+ * network or saved to disk).
+ *
+ * Example:
+ *	#include <ccan/hash/hash.h>
+ *
+ *	// Code to keep track of memory regions.
+ *	struct region {
+ *		struct region *chain;
+ *		void *start;
+ *		unsigned int size;
+ *	};
+ *	// We keep a simple hash table.
+ *	static struct region *region_hash[128];
+ *
+ *	static void add_region(struct region *r)
+ *	{
+ *		unsigned int h = hash_pointer(r->start, 0);
+ *
+ *		r->chain = region_hash[h];
+ *		region_hash[h] = r->chain;
+ *	}
+ *
+ *	static struct region *find_region(const void *start)
+ *	{
+ *		struct region *r;
+ *
+ *		for (r = region_hash[hash_pointer(start, 0)]; r; r = r->chain)
+ *			if (r->start == start)
+ *				return r;
+ *		return NULL;
+ *	}
+ */
+static inline uint32_t hash_pointer(const void *p, uint32_t base)
+{
+	if (sizeof(p) % sizeof(uint32_t) == 0) {
+		/* This convoluted union is the right way of aliasing. */
+		union {
+			uint32_t u32[sizeof(p) / sizeof(uint32_t)];
+			const void *p;
+		} u;
+		u.p = p;
+		return hash_u32(u.u32, sizeof(p) / sizeof(uint32_t), base);
+	} else
+		return hash(&p, 1, base);
+}
+#endif /* HASH_H */
diff --git a/lib/ccan/hash/test/api-hash_stable.c b/lib/ccan/hash/test/api-hash_stable.c
new file mode 100644
index 00000000000..bb58d16b18a
--- /dev/null
+++ b/lib/ccan/hash/test/api-hash_stable.c
@@ -0,0 +1,300 @@
+#include <ccan/hash/hash.h>
+#include <ccan/tap/tap.h>
+#include <stdbool.h>
+#include <string.h>
+
+#define ARRAY_WORDS 5
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	uint8_t u8array[ARRAY_WORDS];
+	uint16_t u16array[ARRAY_WORDS];
+	uint32_t u32array[ARRAY_WORDS];
+	uint64_t u64array[ARRAY_WORDS];
+
+	/* Initialize arrays. */
+	for (i = 0; i < ARRAY_WORDS; i++) {
+		u8array[i] = i;
+		u16array[i] = i;
+		u32array[i] = i;
+		u64array[i] = i;
+	}
+
+	plan_tests(264);
+
+	/* hash_stable is API-guaranteed. */
+	ok1(hash_stable(u8array, ARRAY_WORDS, 0) == 0x1d4833cc);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 1) == 0x37125e2 );
+	ok1(hash_stable(u8array, ARRAY_WORDS, 2) == 0x330a007a);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 4) == 0x7b0df29b);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 8) == 0xe7e5d741);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 16) == 0xaae57471);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 32) == 0xc55399e5);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 64) == 0x67f21f7 );
+	ok1(hash_stable(u8array, ARRAY_WORDS, 128) == 0x1d795b71);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 256) == 0xeb961671);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 512) == 0xc2597247);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 1024) == 0x3f5c4d75);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 2048) == 0xe65cf4f9);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 4096) == 0xf2cd06cb);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 8192) == 0x443041e1);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 16384) == 0xdfc618f5);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 32768) == 0x5e3d5b97);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 65536) == 0xd5f64730);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 131072) == 0x372bbecc);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 262144) == 0x7c194c8d);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 524288) == 0x16cbb416);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 1048576) == 0x53e99222);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 2097152) == 0x6394554a);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 4194304) == 0xd83a506d);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 8388608) == 0x7619d9a4);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 16777216) == 0xfe98e5f6);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 33554432) == 0x6c262927);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 67108864) == 0x3f0106fd);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 134217728) == 0xc91e3a28);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 268435456) == 0x14229579);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 536870912) == 0x9dbefa76);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 1073741824) == 0xb05c0c78);
+	ok1(hash_stable(u8array, ARRAY_WORDS, 2147483648U) == 0x88f24d81);
+
+	ok1(hash_stable(u16array, ARRAY_WORDS, 0) == 0xecb5f507);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 1) == 0xadd666e6);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 2) == 0xea0f214c);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 4) == 0xae4051ba);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 8) == 0x6ed28026);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 16) == 0xa3917a19);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 32) == 0xf370f32b);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 64) == 0x807af460);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 128) == 0xb4c8cd83);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 256) == 0xa10cb5b0);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 512) == 0x8b7d7387);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 1024) == 0x9e49d1c );
+	ok1(hash_stable(u16array, ARRAY_WORDS, 2048) == 0x288830d1);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 4096) == 0xbe078a43);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 8192) == 0xa16d5d88);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 16384) == 0x46839fcd);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 32768) == 0x9db9bd4f);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 65536) == 0xedff58f8);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 131072) == 0x95ecef18);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 262144) == 0x23c31b7d);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 524288) == 0x1d85c7d0);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 1048576) == 0x25218842);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 2097152) == 0x711d985c);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 4194304) == 0x85470eca);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 8388608) == 0x99ed4ceb);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 16777216) == 0x67b3710c);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 33554432) == 0x77f1ab35);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 67108864) == 0x81f688aa);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 134217728) == 0x27b56ca5);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 268435456) == 0xf21ba203);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 536870912) == 0xd48d1d1 );
+	ok1(hash_stable(u16array, ARRAY_WORDS, 1073741824) == 0xa542b62d);
+	ok1(hash_stable(u16array, ARRAY_WORDS, 2147483648U) == 0xa04c7058);
+
+	ok1(hash_stable(u32array, ARRAY_WORDS, 0) == 0x13305f8c);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 1) == 0x171abf74);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 2) == 0x7646fcc7);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 4) == 0xa758ed5);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 8) == 0x2dedc2e4);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 16) == 0x28e2076b);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 32) == 0xb73091c5);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 64) == 0x87daf5db);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 128) == 0xa16dfe20);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 256) == 0x300c63c3);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 512) == 0x255c91fc);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 1024) == 0x6357b26);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 2048) == 0x4bc5f339);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 4096) == 0x1301617c);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 8192) == 0x506792c9);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 16384) == 0xcd596705);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 32768) == 0xa8713cac);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 65536) == 0x94d9794);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 131072) == 0xac753e8);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 262144) == 0xcd8bdd20);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 524288) == 0xd44faf80);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 1048576) == 0x2547ccbe);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 2097152) == 0xbab06dbc);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 4194304) == 0xaac0e882);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 8388608) == 0x443f48d0);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 16777216) == 0xdff49fcc);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 33554432) == 0x9ce0fd65);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 67108864) == 0x9ddb1def);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 134217728) == 0x86096f25);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 268435456) == 0xe713b7b5);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 536870912) == 0x5baeffc5);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 1073741824) == 0xde874f52);
+	ok1(hash_stable(u32array, ARRAY_WORDS, 2147483648U) == 0xeca13b4e);
+
+	ok1(hash_stable(u64array, ARRAY_WORDS, 0) == 0x12ef6302);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 1) == 0xe9aeb406);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 2) == 0xc4218ceb);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 4) == 0xb3d11412);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 8) == 0xdafbd654);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 16) == 0x9c336cba);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 32) == 0x65059721);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 64) == 0x95b5bbe6);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 128) == 0xe7596b84);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 256) == 0x503622a2);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 512) == 0xecdcc5ca);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 1024) == 0xc40d0513);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 2048) == 0xaab25e4d);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 4096) == 0xcc353fb9);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 8192) == 0x18e2319f);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 16384) == 0xfddaae8d);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 32768) == 0xef7976f2);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 65536) == 0x86359fc9);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 131072) == 0x8b5af385);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 262144) == 0x80d4ee31);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 524288) == 0x42f5f85b);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 1048576) == 0x9a6920e1);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 2097152) == 0x7b7c9850);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 4194304) == 0x69573e09);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 8388608) == 0xc942bc0e);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 16777216) == 0x7a89f0f1);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 33554432) == 0x2dd641ca);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 67108864) == 0x89bbd391);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 134217728) == 0xbcf88e31);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 268435456) == 0xfa7a3460);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 536870912) == 0x49a37be0);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 1073741824) == 0x1b346394);
+	ok1(hash_stable(u64array, ARRAY_WORDS, 2147483648U) == 0x6c3a1592);
+
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 0) == 16887282882572727244ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 1) == 12032777473133454818ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 2) == 18183407363221487738ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 4) == 17860764172704150171ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 8) == 18076051600675559233ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 16) == 9909361918431556721ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 32) == 12937969888744675813ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 64) == 5245669057381736951ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 128) == 4376874646406519665ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 256) == 14219974419871569521ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 512) == 2263415354134458951ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 1024) == 4953859694526221685ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 2048) == 3432228642067641593ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 4096) == 1219647244417697483ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 8192) == 7629939424585859553ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 16384) == 10041660531376789749ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 32768) == 13859885793922603927ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 65536) == 15069060338344675120ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 131072) == 818163430835601100ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 262144) == 14914314323019517069ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 524288) == 17518437749769352214ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 1048576) == 14920048004901212706ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 2097152) == 8758567366332536138ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 4194304) == 6226655736088907885ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 8388608) == 13716650013685832100ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 16777216) == 305325651636315638ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 33554432) == 16784147606583781671ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 67108864) == 16509467555140798205ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 134217728) == 8717281234694060584ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 268435456) == 8098476701725660537ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 536870912) == 16345871539461094006ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 1073741824) == 3755557000429964408ULL);
+	ok1(hash64_stable(u8array, ARRAY_WORDS, 2147483648U) == 15017348801959710081ULL);
+
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 0) == 1038028831307724039ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 1) == 10155473272642627302ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 2) == 5714751190106841420ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 4) == 3923885607767527866ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 8) == 3931017318293995558ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 16) == 1469696588339313177ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 32) == 11522218526952715051ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 64) == 6953517591561958496ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 128) == 7406689491740052867ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 256) == 10101844489704093104ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 512) == 12511348870707245959ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 1024) == 1614019938016861468ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 2048) == 5294796182374592721ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 4096) == 16089570706643716675ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 8192) == 1689302638424579464ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 16384) == 1446340172370386893ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 32768) == 16535503506744393039ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 65536) == 3496794142527150328ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 131072) == 6568245367474548504ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 262144) == 9487676460765485949ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 524288) == 4519762130966530000ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 1048576) == 15623412069215340610ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 2097152) == 544013388676438108ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 4194304) == 5594904760290840266ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 8388608) == 18098755780041592043ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 16777216) == 6389168672387330316ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 33554432) == 896986127732419381ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 67108864) == 13232626471143901354ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 134217728) == 53378562890493093ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 268435456) == 10072361400297824771ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 536870912) == 14511948118285144529ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 1073741824) == 6981033484844447277ULL);
+	ok1(hash64_stable(u16array, ARRAY_WORDS, 2147483648U) == 5619339091684126808ULL);
+
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 0) == 3037571077312110476ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 1) == 14732398743825071988ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 2) == 14949132158206672071ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 4) == 1291370080511561429ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 8) == 10792665964172133092ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 16) == 14250138032054339435ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 32) == 17136741522078732741ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 64) == 3260193403318236635ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 128) == 10526616652205653536ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 256) == 9019690373358576579ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 512) == 6997491436599677436ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 1024) == 18302783371416533798ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 2048) == 10149320644446516025ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 4096) == 7073759949410623868ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 8192) == 17442399482223760073ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 16384) == 2983906194216281861ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 32768) == 4975845419129060524ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 65536) == 594019910205413268ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 131072) == 11903010186073691112ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 262144) == 7339636527154847008ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 524288) == 15243305400579108736ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 1048576) == 16737926245392043198ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 2097152) == 15725083267699862972ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 4194304) == 12527834265678833794ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 8388608) == 13908436455987824848ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 16777216) == 9672773345173872588ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 33554432) == 2305314279896710501ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 67108864) == 1866733780381408751ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 134217728) == 11906263969465724709ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 268435456) == 5501594918093830069ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 536870912) == 15823785789276225477ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 1073741824) == 17353000723889475410ULL);
+	ok1(hash64_stable(u32array, ARRAY_WORDS, 2147483648U) == 7494736910655503182ULL);
+
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 0) == 9765419389786481410ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 1) == 11182806172127114246ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 2) == 2559155171395472619ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 4) == 3311692033324815378ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 8) == 1297175419505333844ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 16) == 617896928653569210ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 32) == 1517398559958603553ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 64) == 4504821917445110758ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 128) == 1971743331114904452ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 256) == 6177667912354374306ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 512) == 15570521289777792458ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 1024) == 9204559632415917331ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 2048) == 9008982669760028237ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 4096) == 14803537660281700281ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 8192) == 2873966517448487327ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 16384) == 5859277625928363661ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 32768) == 15520461285618185970ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 65536) == 16746489793331175369ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 131072) == 514952025484227461ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 262144) == 10867212269810675249ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 524288) == 9822204377278314587ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 1048576) == 3295088921987850465ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 2097152) == 7559197431498053712ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 4194304) == 1667267269116771849ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 8388608) == 2916804068951374862ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 16777216) == 14422558383125688561ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 33554432) == 10083112683694342602ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 67108864) == 7222777647078298513ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 134217728) == 18424513674048212529ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 268435456) == 14913668581101810784ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 536870912) == 14377721174297902048ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 1073741824) == 6031715005667500948ULL);
+	ok1(hash64_stable(u64array, ARRAY_WORDS, 2147483648U) == 4827100319722378642ULL);
+
+	return exit_status();
+}
diff --git a/lib/ccan/hash/test/run.c b/lib/ccan/hash/test/run.c
new file mode 100644
index 00000000000..dad8e86b9e5
--- /dev/null
+++ b/lib/ccan/hash/test/run.c
@@ -0,0 +1,149 @@
+#include <ccan/hash/hash.h>
+#include <ccan/tap/tap.h>
+#include <ccan/hash/hash.c>
+#include <stdbool.h>
+#include <string.h>
+
+#define ARRAY_WORDS 5
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, k;
+	uint32_t array[ARRAY_WORDS], val;
+	char array2[sizeof(array) + sizeof(uint32_t)];
+	uint32_t results[256];
+
+	/* Initialize array. */
+	for (i = 0; i < ARRAY_WORDS; i++)
+		array[i] = i;
+
+	plan_tests(39);
+	/* Hash should be the same, indep of memory alignment. */
+	val = hash(array, ARRAY_WORDS, 0);
+	for (i = 0; i < sizeof(uint32_t); i++) {
+		memcpy(array2 + i, array, sizeof(array));
+		ok(hash(array2 + i, ARRAY_WORDS, 0) != val,
+		   "hash matched at offset %i", i);
+	}
+
+	/* Hash of random values should have random distribution:
+	 * check one byte at a time. */
+	for (i = 0; i < sizeof(uint32_t); i++) {
+		unsigned int lowest = -1U, highest = 0;
+
+		memset(results, 0, sizeof(results));
+
+		for (j = 0; j < 256000; j++) {
+			for (k = 0; k < ARRAY_WORDS; k++)
+				array[k] = random();
+			results[(hash(array, ARRAY_WORDS, 0) >> i*8)&0xFF]++;
+		}
+
+		for (j = 0; j < 256; j++) {
+			if (results[j] < lowest)
+				lowest = results[j];
+			if (results[j] > highest)
+				highest = results[j];
+		}
+		/* Expect within 20% */
+		ok(lowest > 800, "Byte %i lowest %i", i, lowest);
+		ok(highest < 1200, "Byte %i highest %i", i, highest);
+		diag("Byte %i, range %u-%u", i, lowest, highest);
+	}
+
+	/* Hash of random values should have random distribution:
+	 * check one byte at a time. */
+	for (i = 0; i < sizeof(uint64_t); i++) {
+		unsigned int lowest = -1U, highest = 0;
+
+		memset(results, 0, sizeof(results));
+
+		for (j = 0; j < 256000; j++) {
+			for (k = 0; k < ARRAY_WORDS; k++)
+				array[k] = random();
+			results[(hash64(array, sizeof(array)/sizeof(uint64_t),
+					0) >> i*8)&0xFF]++;
+		}
+
+		for (j = 0; j < 256; j++) {
+			if (results[j] < lowest)
+				lowest = results[j];
+			if (results[j] > highest)
+				highest = results[j];
+		}
+		/* Expect within 20% */
+		ok(lowest > 800, "Byte %i lowest %i", i, lowest);
+		ok(highest < 1200, "Byte %i highest %i", i, highest);
+		diag("Byte %i, range %u-%u", i, lowest, highest);
+	}
+
+	/* Hash of pointer values should also have random distribution. */
+	for (i = 0; i < sizeof(uint32_t); i++) {
+		unsigned int lowest = -1U, highest = 0;
+		char *p = malloc(256000);
+
+		memset(results, 0, sizeof(results));
+
+		for (j = 0; j < 256000; j++)
+			results[(hash_pointer(p + j, 0) >> i*8)&0xFF]++;
+		free(p);
+
+		for (j = 0; j < 256; j++) {
+			if (results[j] < lowest)
+				lowest = results[j];
+			if (results[j] > highest)
+				highest = results[j];
+		}
+		/* Expect within 20% */
+		ok(lowest > 800, "hash_pointer byte %i lowest %i", i, lowest);
+		ok(highest < 1200, "hash_pointer byte %i highest %i",
+		   i, highest);
+		diag("hash_pointer byte %i, range %u-%u", i, lowest, highest);
+	}
+
+	if (sizeof(long) == sizeof(uint32_t))
+		ok1(hashl(array, ARRAY_WORDS, 0)
+		    == hash(array, ARRAY_WORDS, 0));
+	else
+		ok1(hashl(array, ARRAY_WORDS, 0)
+		    == hash64(array, ARRAY_WORDS, 0));
+
+	/* String hash: weak, so only test bottom byte */
+	for (i = 0; i < 1; i++) {
+		unsigned int num = 0, cursor, lowest = -1U, highest = 0;
+		char p[5];
+
+		memset(results, 0, sizeof(results));
+
+		memset(p, 'A', sizeof(p));
+		p[sizeof(p)-1] = '\0';
+
+		for (;;) {
+			for (cursor = 0; cursor < sizeof(p)-1; cursor++) {
+				p[cursor]++;
+				if (p[cursor] <= 'z')
+					break;
+				p[cursor] = 'A';
+			}
+			if (cursor == sizeof(p)-1)
+				break;
+
+			results[(hash_string(p) >> i*8)&0xFF]++;
+			num++;
+		}
+
+		for (j = 0; j < 256; j++) {
+			if (results[j] < lowest)
+				lowest = results[j];
+			if (results[j] > highest)
+				highest = results[j];
+		}
+		/* Expect within 20% */
+		ok(lowest > 35000, "hash_pointer byte %i lowest %i", i, lowest);
+		ok(highest < 53000, "hash_pointer byte %i highest %i",
+		   i, highest);
+		diag("hash_pointer byte %i, range %u-%u", i, lowest, highest);
+	}
+
+	return exit_status();
+}
diff --git a/lib/ccan/htable/LICENSE b/lib/ccan/htable/LICENSE
new file mode 100644
index 00000000000..d511905c164
--- /dev/null
+++ b/lib/ccan/htable/LICENSE
@@ -0,0 +1,339 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/lib/ccan/htable/_info b/lib/ccan/htable/_info
new file mode 100644
index 00000000000..8dabe46a50e
--- /dev/null
+++ b/lib/ccan/htable/_info
@@ -0,0 +1,115 @@
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * htable - hash table routines
+ *
+ * A hash table is an efficient structure for looking up keys.  This version
+ * grows with usage and allows efficient deletion.
+ *
+ * Example:
+ *	#include <ccan/htable/htable.h>
+ *	#include <ccan/hash/hash.h>
+ *	#include <stdio.h>
+ *	#include <err.h>
+ *	#include <string.h>
+ *
+ *	struct name_to_digit {
+ *		const char *name;
+ *		unsigned int val;
+ *	};
+ *
+ *	static struct name_to_digit map[] = {
+ *		{ "zero", 0},
+ *		{ "one", 1 },
+ *		{ "two", 2 },
+ *		{ "three", 3 },
+ *		{ "four", 4 },
+ *		{ "five", 5 },
+ *		{ "six", 6 },
+ *		{ "seven", 7 },
+ *		{ "eight", 8 },
+ *		{ "nine", 9 }
+ *	};
+ *
+ *	// Wrapper for rehash function pointer.
+ *	static size_t rehash(const void *e, void *unused)
+ *	{
+ *		return hash_string(((struct name_to_digit *)e)->name);
+ *	}
+ *
+ *	// Comparison function.
+ *	static bool streq(const void *e, void *string)
+ *	{
+ *		return strcmp(((struct name_to_digit *)e)->name, string) == 0;
+ *	}
+ *
+ *	// We let them add their own aliases, eg. --alias=v=5
+ *	static void add_alias(struct htable *ht, const char *alias)
+ *	{
+ *		char *eq;
+ *		struct name_to_digit *n;
+ *
+ *		n = malloc(sizeof(*n));
+ *		n->name = strdup(alias);
+ *
+ *		eq = strchr(n->name, '=');
+ *		if (!eq || ((n->val = atoi(eq+1)) == 0 && !strcmp(eq+1, "0")))
+ *			errx(1, "Usage: --alias=<name>=<value>");
+ *		*eq = '\0';
+ *		htable_add(ht, hash_string(n->name), n);
+ *	}
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct htable *ht;
+ *		unsigned int i;
+ *		unsigned long val;
+ *
+ *		if (argc < 2)
+ *			errx(1, "Usage: %s [--alias=<name>=<val>]... <str>...",
+ *			     argv[0]);
+ *
+ *		// Create and populate hash table.
+ *		ht = htable_new(rehash, NULL);
+ *		for (i = 0; i < sizeof(map)/sizeof(map[0]); i++)
+ *			htable_add(ht, hash_string(map[i].name), &map[i]);
+ *
+ *		// Add any aliases to the hash table.
+ *		for (i = 1; i < argc; i++) {
+ *			if (!strncmp(argv[i], "--alias=", strlen("--alias=")))
+ *				add_alias(ht, argv[i] + strlen("--alias="));
+ *			else
+ *				break;
+ *		}
+ *
+ *		// Find the other args in the hash table.
+ *		for (val = 0; i < argc; i++) {
+ *			struct name_to_digit *n;
+ *			n = htable_get(ht, hash_string(argv[i]),
+ *				       streq, argv[i]);
+ *			if (!n)
+ *				errx(1, "Invalid digit name %s", argv[i]);
+ *			// Append it to the value we are building up.
+ *			val *= 10;
+ *			val += n->val;
+ *		}
+ *		printf("%lu\n", val);
+ *		return 0;
+ *	}
+ *
+ * License: GPLv2 (or later)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/compiler\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/htable/htable.c b/lib/ccan/htable/htable.c
new file mode 100644
index 00000000000..a15c54d7958
--- /dev/null
+++ b/lib/ccan/htable/htable.c
@@ -0,0 +1,290 @@
+#include <ccan/htable/htable.h>
+#include <ccan/compiler/compiler.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <assert.h>
+
+/* This means a struct htable takes at least 512 bytes / 1k (32/64 bits). */
+#define HTABLE_BASE_BITS 7
+
+/* We use 0x1 as deleted marker. */
+#define HTABLE_DELETED (0x1)
+
+struct htable {
+	size_t (*rehash)(const void *elem, void *priv);
+	void *priv;
+	unsigned int bits;
+	size_t elems, deleted, max, max_with_deleted;
+	/* These are the bits which are the same in all pointers. */
+	uintptr_t common_mask, common_bits;
+	uintptr_t perfect_bit;
+	uintptr_t *table;
+};
+
+/* We clear out the bits which are always the same, and put metadata there. */
+static inline uintptr_t get_extra_ptr_bits(const struct htable *ht,
+					   uintptr_t e)
+{
+	return e & ht->common_mask;
+}
+
+static inline void *get_raw_ptr(const struct htable *ht, uintptr_t e)
+{
+	return (void *)((e & ~ht->common_mask) | ht->common_bits);
+}
+
+static inline uintptr_t make_hval(const struct htable *ht,
+				  const void *p, uintptr_t bits)
+{
+	return ((uintptr_t)p & ~ht->common_mask) | bits;
+}
+
+static inline bool entry_is_valid(uintptr_t e)
+{
+	return e > HTABLE_DELETED;
+}
+
+static inline uintptr_t get_hash_ptr_bits(const struct htable *ht,
+					  size_t hash)
+{
+	/* Shuffling the extra bits (as specified in mask) down the
+	 * end is quite expensive.  But the lower bits are redundant, so
+	 * we fold the value first. */
+	return (hash ^ (hash >> ht->bits))
+		& ht->common_mask & ~ht->perfect_bit;
+}
+
+struct htable *htable_new(size_t (*rehash)(const void *elem, void *priv),
+			  void *priv)
+{
+	struct htable *ht = malloc(sizeof(struct htable));
+	if (ht) {
+		ht->bits = HTABLE_BASE_BITS;
+		ht->rehash = rehash;
+		ht->priv = priv;
+		ht->elems = 0;
+		ht->deleted = 0;
+		ht->max = ((size_t)1 << ht->bits) * 3 / 4;
+		ht->max_with_deleted = ((size_t)1 << ht->bits) * 9 / 10;
+		/* This guarantees we enter update_common first add. */
+		ht->common_mask = -1;
+		ht->common_bits = 0;
+		ht->perfect_bit = 0;
+		ht->table = calloc(1 << ht->bits, sizeof(uintptr_t));
+		if (!ht->table) {
+			free(ht);
+			ht = NULL;
+		}
+	}
+	return ht;
+}
+
+void htable_free(const struct htable *ht)
+{
+	free((void *)ht->table);
+	free((void *)ht);
+}
+
+static size_t hash_bucket(const struct htable *ht, size_t h)
+{
+	return h & ((1 << ht->bits)-1);
+}
+
+static void *htable_val(const struct htable *ht,
+			struct htable_iter *i, size_t hash, uintptr_t perfect)
+{
+	uintptr_t h2 = get_hash_ptr_bits(ht, hash) | perfect;
+
+	while (ht->table[i->off]) {
+		if (ht->table[i->off] != HTABLE_DELETED) {
+			if (get_extra_ptr_bits(ht, ht->table[i->off]) == h2)
+				return get_raw_ptr(ht, ht->table[i->off]);
+		}
+		i->off = (i->off + 1) & ((1 << ht->bits)-1);
+		h2 &= ~perfect;
+	}
+	return NULL;
+}
+
+void *htable_firstval(const struct htable *ht,
+		      struct htable_iter *i, size_t hash)
+{
+	i->off = hash_bucket(ht, hash);
+	return htable_val(ht, i, hash, ht->perfect_bit);
+}
+
+void *htable_nextval(const struct htable *ht,
+		     struct htable_iter *i, size_t hash)
+{
+	i->off = (i->off + 1) & ((1 << ht->bits)-1);
+	return htable_val(ht, i, hash, 0);
+}
+
+void *htable_first(const struct htable *ht, struct htable_iter *i)
+{
+	for (i->off = 0; i->off < (size_t)1 << ht->bits; i->off++) {
+		if (entry_is_valid(ht->table[i->off]))
+			return get_raw_ptr(ht, ht->table[i->off]);
+	}
+	return NULL;
+}
+
+void *htable_next(const struct htable *ht, struct htable_iter *i)
+{
+	for (i->off++; i->off < (size_t)1 << ht->bits; i->off++) {
+		if (entry_is_valid(ht->table[i->off]))
+			return get_raw_ptr(ht, ht->table[i->off]);
+	}
+	return NULL;
+}
+
+/* This does not expand the hash table, that's up to caller. */
+static void ht_add(struct htable *ht, const void *new, size_t h)
+{
+	size_t i;
+	uintptr_t perfect = ht->perfect_bit;
+
+	i = hash_bucket(ht, h);
+
+	while (entry_is_valid(ht->table[i])) {
+		perfect = 0;
+		i = (i + 1) & ((1 << ht->bits)-1);
+	}
+	ht->table[i] = make_hval(ht, new, get_hash_ptr_bits(ht, h)|perfect);
+}
+
+static COLD bool double_table(struct htable *ht)
+{
+	unsigned int i;
+	size_t oldnum = (size_t)1 << ht->bits;
+	uintptr_t *oldtable, e;
+
+	oldtable = ht->table;
+	ht->table = calloc(1 << (ht->bits+1), sizeof(size_t));
+	if (!ht->table) {
+		ht->table = oldtable;
+		return false;
+	}
+	ht->bits++;
+	ht->max *= 2;
+	ht->max_with_deleted *= 2;
+
+	/* If we lost our "perfect bit", get it back now. */
+	if (!ht->perfect_bit && ht->common_mask) {
+		for (i = 0; i < sizeof(ht->common_mask) * CHAR_BIT; i++) {
+			if (ht->common_mask & ((size_t)1 << i)) {
+				ht->perfect_bit = (size_t)1 << i;
+				break;
+			}
+		}
+	}
+
+	for (i = 0; i < oldnum; i++) {
+		if (entry_is_valid(e = oldtable[i])) {
+			void *p = get_raw_ptr(ht, e);
+			ht_add(ht, p, ht->rehash(p, ht->priv));
+		}
+	}
+	ht->deleted = 0;
+	free(oldtable);
+	return true;
+}
+
+static COLD void rehash_table(struct htable *ht)
+{
+	size_t start, i;
+	uintptr_t e;
+
+	/* Beware wrap cases: we need to start from first empty bucket. */
+	for (start = 0; ht->table[start]; start++);
+
+	for (i = 0; i < (size_t)1 << ht->bits; i++) {
+		size_t h = (i + start) & ((1 << ht->bits)-1);
+		e = ht->table[h];
+		if (!e)
+			continue;
+		if (e == HTABLE_DELETED)
+			ht->table[h] = 0;
+		else if (!(e & ht->perfect_bit)) {
+			void *p = get_raw_ptr(ht, e);
+			ht->table[h] = 0;
+			ht_add(ht, p, ht->rehash(p, ht->priv));
+		}
+	}
+	ht->deleted = 0;
+}
+
+/* We stole some bits, now we need to put them back... */
+static COLD void update_common(struct htable *ht, const void *p)
+{
+	unsigned int i;
+	uintptr_t maskdiff, bitsdiff;
+
+	if (ht->elems == 0) {
+		ht->common_mask = -1;
+		ht->common_bits = (uintptr_t)p;
+		ht->perfect_bit = 1;
+		return;
+	}
+
+	/* Find bits which are unequal to old common set. */
+	maskdiff = ht->common_bits ^ ((uintptr_t)p & ht->common_mask);
+
+	/* These are the bits which go there in existing entries. */
+	bitsdiff = ht->common_bits & maskdiff;
+
+	for (i = 0; i < (size_t)1 << ht->bits; i++) {
+		if (!entry_is_valid(ht->table[i]))
+			continue;
+		/* Clear the bits no longer in the mask, set them as
+		 * expected. */
+		ht->table[i] &= ~maskdiff;
+		ht->table[i] |= bitsdiff;
+	}
+
+	/* Take away those bits from our mask, bits and perfect bit. */
+	ht->common_mask &= ~maskdiff;
+	ht->common_bits &= ~maskdiff;
+	ht->perfect_bit &= ~maskdiff;
+}
+
+bool htable_add(struct htable *ht, size_t hash, const void *p)
+{
+	if (ht->elems+1 > ht->max && !double_table(ht))
+		return false;
+	if (ht->elems+1 + ht->deleted > ht->max_with_deleted)
+		rehash_table(ht);
+	assert(p);
+	if (((uintptr_t)p & ht->common_mask) != ht->common_bits)
+		update_common(ht, p);
+
+	ht_add(ht, p, hash);
+	ht->elems++;
+	return true;
+}
+
+bool htable_del(struct htable *ht, size_t h, const void *p)
+{
+	struct htable_iter i;
+	void *c;
+
+	for (c = htable_firstval(ht,&i,h); c; c = htable_nextval(ht,&i,h)) {
+		if (c == p) {
+			htable_delval(ht, &i);
+			return true;
+		}
+	}
+	return false;
+}
+
+void htable_delval(struct htable *ht, struct htable_iter *i)
+{
+	assert(i->off < (size_t)1 << ht->bits);
+	assert(entry_is_valid(ht->table[i->off]));
+
+	ht->elems--;
+	ht->table[i->off] = HTABLE_DELETED;
+	ht->deleted++;
+}
diff --git a/lib/ccan/htable/htable.h b/lib/ccan/htable/htable.h
new file mode 100644
index 00000000000..b68442972c7
--- /dev/null
+++ b/lib/ccan/htable/htable.h
@@ -0,0 +1,138 @@
+#ifndef CCAN_HTABLE_H
+#define CCAN_HTABLE_H
+#include "config.h"
+#include <stdbool.h>
+#include <stdlib.h>
+
+struct htable;
+
+/**
+ * htable_new - allocate a hash tree.
+ * @rehash: hash function to use for rehashing.
+ * @priv: private argument to @rehash function.
+ */
+struct htable *htable_new(size_t (*hash)(const void *elem, void *priv),
+			  void *priv);
+
+/**
+ * htable_free - dellocate a hash tree.
+ *
+ * This doesn't do anything to any pointers left in it.
+ */
+void htable_free(const struct htable *);
+
+/**
+ * htable_rehash - use a hashtree's rehash function
+ * @elem: the argument to rehash()
+ *
+ */
+size_t htable_rehash(const void *elem);
+
+/**
+ * htable_add - add a pointer into a hash tree.
+ * @ht: the htable
+ * @hash: the hash value of the object
+ * @p: the non-NULL pointer
+ *
+ * Also note that this can only fail due to allocation failure.  Otherwise, it
+ * returns true.
+ */
+bool htable_add(struct htable *ht, size_t hash, const void *p);
+
+/**
+ * htable_del - remove a pointer from a hash tree
+ * @ht: the htable
+ * @hash: the hash value of the object
+ * @p: the pointer
+ *
+ * Returns true if the pointer was found (and deleted).
+ */
+bool htable_del(struct htable *ht, size_t hash, const void *p);
+
+/**
+ * struct htable_iter - iterator or htable_first or htable_firstval etc.
+ *
+ * This refers to a location inside the hashtable.
+ */
+struct htable_iter {
+	size_t off;
+};
+
+/**
+ * htable_firstval - find a candidate for a given hash value
+ * @htable: the hashtable
+ * @i: the struct htable_iter to initialize
+ * @hash: the hash value
+ *
+ * You'll need to check the value is what you want; returns NULL if none.
+ * See Also:
+ *	htable_delval()
+ */
+void *htable_firstval(const struct htable *htable,
+		      struct htable_iter *i, size_t hash);
+
+/**
+ * htable_nextval - find another candidate for a given hash value
+ * @htable: the hashtable
+ * @i: the struct htable_iter to initialize
+ * @hash: the hash value
+ *
+ * You'll need to check the value is what you want; returns NULL if no more.
+ */
+void *htable_nextval(const struct htable *htable,
+		     struct htable_iter *i, size_t hash);
+
+/**
+ * htable_get - find an entry in the hash table
+ * @ht: the hashtable
+ * @h: the hash value of the entry
+ * @cmp: the comparison function
+ * @ptr: the pointer to hand to the comparison function.
+ *
+ * Convenient inline wrapper for htable_firstval/htable_nextval loop.
+ */
+static inline void *htable_get(const struct htable *ht,
+			       size_t h,
+			       bool (*cmp)(const void *candidate, void *ptr),
+			       const void *ptr)
+{
+	struct htable_iter i;
+	void *c;
+
+	for (c = htable_firstval(ht,&i,h); c; c = htable_nextval(ht,&i,h)) {
+		if (cmp(c, (void *)ptr))
+			return c;
+	}
+	return NULL;
+}
+
+/**
+ * htable_first - find an entry in the hash table
+ * @ht: the hashtable
+ * @i: the struct htable_iter to initialize
+ *
+ * Get an entry in the hashtable; NULL if empty.
+ */
+void *htable_first(const struct htable *htable, struct htable_iter *i);
+
+/**
+ * htable_next - find another entry in the hash table
+ * @ht: the hashtable
+ * @i: the struct htable_iter to use
+ *
+ * Get another entry in the hashtable; NULL if all done.
+ * This is usually used after htable_first or prior non-NULL htable_next.
+ */
+void *htable_next(const struct htable *htable, struct htable_iter *i);
+
+/**
+ * htable_delval - remove an iterated pointer from a hash tree
+ * @ht: the htable
+ * @i: the htable_iter
+ *
+ * Usually used to delete a hash entry after it has been found with
+ * htable_firstval etc.
+ */
+void htable_delval(struct htable *ht, struct htable_iter *i);
+
+#endif /* CCAN_HTABLE_H */
diff --git a/lib/ccan/htable/htable_type.h b/lib/ccan/htable/htable_type.h
new file mode 100644
index 00000000000..0d9e3fbb2dc
--- /dev/null
+++ b/lib/ccan/htable/htable_type.h
@@ -0,0 +1,97 @@
+#ifndef CCAN_HTABLE_TYPE_H
+#define CCAN_HTABLE_TYPE_H
+#include <ccan/htable/htable.h>
+#include "config.h"
+
+/**
+ * HTABLE_DEFINE_TYPE - create a set of htable ops for a type
+ * @type: a type whose pointers will be values in the hash.
+ * @keyof: a function/macro to extract a key from a @type element.
+ * @hashfn: a hash function for a @key
+ * @cmpfn: a comparison function for two keyof()s.
+ * @name: a name for all the functions to define (of form htable_<name>_*)
+ *
+ * NULL values may not be placed into the hash table.
+ *
+ * The following wrapper functions are defined; each one is a
+ * simplified version of the htable.h equivalent:
+ *
+ *	// Creating and freeing.
+ *	struct htable_@name *htable_@name_new(void);
+ *	void htable_@name_free(const struct htable_@name *ht);
+ *
+ *	// Add, delete and find.
+ *	bool htable_@name_add(struct htable_@name *ht, const type *e);
+ *	bool htable_@name_del(struct htable_@name *ht, const type *e);
+ *	bool htable_@name_delkey(struct htable_@name *ht, const ktype *k);
+ *	type *htable_@name_get(const struct htable_@name *ht, const ktype *k);
+ *
+ *	// Iteration.
+ *	struct htable_@name_iter;
+ *	type *htable_@name_first(const struct htable_@name *ht,
+ *				 struct htable_@name_iter *i);
+ *	type *htable_@name_next(const struct htable_@name *ht,
+ *				struct htable_@name_iter *i);
+ */
+#define HTABLE_DEFINE_TYPE(type, keyof, hashfn, cmpfn, name)		\
+struct htable_##name;							\
+struct htable_##name##_iter { struct htable_iter i; };			\
+static inline size_t htable_##name##_hash(const void *elem, void *priv)	\
+{									\
+	return hashfn(keyof((const type *)elem));			\
+}									\
+static inline struct htable_##name *htable_##name##_new(void)		\
+{									\
+	return (struct htable_##name *)htable_new(htable_##name##_hash,	\
+						  NULL);		\
+}									\
+static inline void htable_##name##_free(const struct htable_##name *ht)	\
+{									\
+	htable_free((const struct htable *)ht);				\
+}									\
+static inline bool htable_##name##_add(struct htable_##name *ht,	\
+				       const type *elem)		\
+{									\
+	return htable_add((struct htable *)ht, hashfn(keyof(elem)), elem); \
+}									\
+static inline bool htable_##name##_del(const struct htable_##name *ht,	\
+				       const type *elem)		\
+{									\
+	return htable_del((struct htable *)ht, hashfn(keyof(elem)), elem); \
+}									\
+static inline type *htable_##name##_get(const struct htable_##name *ht,	\
+					const HTABLE_KTYPE(keyof) k)	\
+{									\
+	/* Typecheck for cmpfn */					\
+	(void)sizeof(cmpfn((const type *)NULL,				\
+			   keyof((const type *)NULL)));			\
+	return (type *)htable_get((const struct htable *)ht,		\
+				  hashfn(k),				\
+				  (bool (*)(const void *, void *))(cmpfn), \
+				  k);					\
+}									\
+static inline bool htable_##name##_delkey(struct htable_##name *ht,	\
+					  const HTABLE_KTYPE(keyof) k) \
+{									\
+	type *elem = htable_##name##_get(ht, k);			\
+	if (elem)							\
+		return htable_##name##_del(ht, elem);			\
+	return false;							\
+}									\
+static inline type *htable_##name##_first(const struct htable_##name *ht, \
+					  struct htable_##name##_iter *iter) \
+{									\
+	return htable_first((const struct htable *)ht, &iter->i);	\
+}									\
+static inline type *htable_##name##_next(const struct htable_##name *ht, \
+					 struct htable_##name##_iter *iter) \
+{									\
+	return htable_next((const struct htable *)ht, &iter->i);	\
+}
+
+#if HAVE_TYPEOF
+#define HTABLE_KTYPE(keyof) typeof(keyof(NULL))
+#else
+#define HTABLE_KTYPE(keyof) void *
+#endif
+#endif /* CCAN_HTABLE_TYPE_H */
diff --git a/lib/ccan/htable/test/run-type.c b/lib/ccan/htable/test/run-type.c
new file mode 100644
index 00000000000..02dac29e100
--- /dev/null
+++ b/lib/ccan/htable/test/run-type.c
@@ -0,0 +1,176 @@
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/tap/tap.h>
+#include <stdbool.h>
+#include <string.h>
+
+#define NUM_VALS (1 << HTABLE_BASE_BITS)
+
+struct obj {
+	/* Makes sure we don't try to treat and obj as a key or vice versa */
+	unsigned char unused;
+	unsigned int key;
+};
+
+static const unsigned int *objkey(const struct obj *obj)
+{
+	return &obj->key;
+}
+
+/* We use the number divided by two as the hash (for lots of
+   collisions), plus set all the higher bits so we can detect if they
+   don't get masked out. */
+static size_t objhash(const unsigned int *key)
+{
+	size_t h = *key / 2;
+	h |= -1UL << HTABLE_BASE_BITS;
+	return h;
+}
+
+static bool cmp(const struct obj *obj, const unsigned int *key)
+{
+	return obj->key == *key;
+}
+
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, obj);
+
+static void add_vals(struct htable_obj *ht,
+		     struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_obj_get(ht, &i)) {
+			fail("%u already in hash", i);
+			return;
+		}
+		htable_obj_add(ht, &val[i]);
+		if (htable_obj_get(ht, &i) != &val[i]) {
+			fail("%u not added to hash", i);
+			return;
+		}
+	}
+	pass("Added %u numbers to hash", i);
+}
+
+static void find_vals(const struct htable_obj *ht,
+		      const struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_obj_get(ht, &i) != &val[i]) {
+			fail("%u not found in hash", i);
+			return;
+		}
+	}
+	pass("Found %u numbers in hash", i);
+}
+
+static void del_vals(struct htable_obj *ht,
+		     const struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (!htable_obj_delkey(ht, &val[i].key)) {
+			fail("%u not deleted from hash", i);
+			return;
+		}
+	}
+	pass("Deleted %u numbers in hash", i);
+}
+
+static void del_vals_bykey(struct htable_obj *ht,
+			   const struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (!htable_obj_delkey(ht, &i)) {
+			fail("%u not deleted by key from hash", i);
+			return;
+		}
+	}
+	pass("Deleted %u numbers by key from hash", i);
+}
+
+static bool check_mask(struct htable *ht, const struct obj val[], unsigned num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (((uintptr_t)&val[i] & ht->common_mask) != ht->common_bits)
+			return false;
+	}
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct htable_obj *ht;
+	struct obj val[NUM_VALS];
+	unsigned int dne;
+	void *p;
+	struct htable_obj_iter iter;
+
+	plan_tests(20);
+	for (i = 0; i < NUM_VALS; i++)
+		val[i].key = i;
+	dne = i;
+
+	ht = htable_obj_new();
+	ok1(((struct htable *)ht)->max < (1 << ((struct htable *)ht)->bits));
+	ok1(((struct htable *)ht)->bits == HTABLE_BASE_BITS);
+
+	/* We cannot find an entry which doesn't exist. */
+	ok1(!htable_obj_get(ht, &dne));
+
+	/* Fill it, it should increase in size (once). */
+	add_vals(ht, val, NUM_VALS);
+	ok1(((struct htable *)ht)->bits == HTABLE_BASE_BITS + 1);
+	ok1(((struct htable *)ht)->max < (1 << ((struct htable *)ht)->bits));
+
+	/* Mask should be set. */
+	ok1(((struct htable *)ht)->common_mask != 0);
+	ok1(((struct htable *)ht)->common_mask != -1);
+	ok1(check_mask((struct htable *)ht, val, NUM_VALS));
+
+	/* Find all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_obj_get(ht, &dne));
+
+	/* Walk once, should get them all. */
+	i = 0;
+	for (p = htable_obj_first(ht,&iter); p; p = htable_obj_next(ht, &iter))
+		i++;
+	ok1(i == NUM_VALS);
+
+	/* Delete all. */
+	del_vals(ht, val, NUM_VALS);
+	ok1(!htable_obj_get(ht, &val[0].key));
+
+	/* Worst case, a "pointer" which doesn't have any matching bits. */
+	htable_add((struct htable *)ht, 0,
+		   (void *)~(uintptr_t)&val[NUM_VALS-1]);
+	htable_obj_add(ht, &val[NUM_VALS-1]);
+	ok1(((struct htable *)ht)->common_mask == 0);
+	ok1(((struct htable *)ht)->common_bits == 0);
+	/* Delete the bogus one before we trip over it. */
+	htable_del((struct htable *)ht, 0,
+		   (void *)~(uintptr_t)&val[NUM_VALS-1]);
+
+	/* Add the rest. */
+	add_vals(ht, val, NUM_VALS-1);
+
+	/* Check we can find them all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_obj_get(ht, &dne));
+
+	/* Delete them all by key. */
+	del_vals_bykey(ht, val, NUM_VALS);
+	htable_obj_free(ht);
+
+	return exit_status();
+}
diff --git a/lib/ccan/htable/test/run.c b/lib/ccan/htable/test/run.c
new file mode 100644
index 00000000000..ece46a0fd7c
--- /dev/null
+++ b/lib/ccan/htable/test/run.c
@@ -0,0 +1,176 @@
+#include <ccan/htable/htable.h>
+#include <ccan/htable/htable.c>
+#include <ccan/tap/tap.h>
+#include <stdbool.h>
+#include <string.h>
+
+#define NUM_VALS (1 << HTABLE_BASE_BITS)
+
+/* We use the number divided by two as the hash (for lots of
+   collisions), plus set all the higher bits so we can detect if they
+   don't get masked out. */
+static size_t hash(const void *elem, void *unused)
+{
+	size_t h = *(uint64_t *)elem / 2;
+	h |= -1UL << HTABLE_BASE_BITS;
+	return h;
+}
+
+static bool objcmp(const void *htelem, void *cmpdata)
+{
+	return *(uint64_t *)htelem == *(uint64_t *)cmpdata;
+}
+
+static void add_vals(struct htable *ht,
+		     const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i)) {
+			fail("%llu already in hash", (long long)i);
+			return;
+		}
+		htable_add(ht, hash(&val[i], NULL), &val[i]);
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i) != &val[i]) {
+			fail("%llu not added to hash", (long long)i);
+			return;
+		}
+	}
+	pass("Added %llu numbers to hash", (long long)i);
+}
+
+#if 0
+static void refill_vals(struct htable *ht,
+			const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i))
+			continue;
+		htable_add(ht, hash(&val[i], NULL), &val[i]);
+	}
+}
+#endif
+
+static void find_vals(struct htable *ht,
+		      const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i) != &val[i]) {
+			fail("%llu not found in hash", (long long)i);
+			return;
+		}
+	}
+	pass("Found %llu numbers in hash", (long long)i);
+}
+
+static void del_vals(struct htable *ht,
+		     const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (!htable_del(ht, hash(&val[i], NULL), &val[i])) {
+			fail("%llu not deleted from hash", (long long)i);
+			return;
+		}
+	}
+	pass("Deleted %llu numbers in hash", (long long)i);
+}
+
+static bool check_mask(struct htable *ht, uint64_t val[], unsigned num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (((uintptr_t)&val[i] & ht->common_mask) != ht->common_bits)
+			return false;
+	}
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	uintptr_t perfect_bit;
+	struct htable *ht;
+	uint64_t val[NUM_VALS];
+	uint64_t dne;
+	void *p;
+	struct htable_iter iter;
+
+	plan_tests(23);
+	for (i = 0; i < NUM_VALS; i++)
+		val[i] = i;
+	dne = i;
+
+	ht = htable_new(hash, NULL);
+	ok1(ht->max < (1 << ht->bits));
+	ok1(ht->bits == HTABLE_BASE_BITS);
+
+	/* We cannot find an entry which doesn't exist. */
+	ok1(!htable_get(ht, hash(&dne, NULL), objcmp, &dne));
+
+	/* Fill it, it should increase in size (once). */
+	add_vals(ht, val, NUM_VALS);
+	ok1(ht->bits == HTABLE_BASE_BITS + 1);
+	ok1(ht->max < (1 << ht->bits));
+
+	/* Mask should be set. */
+	ok1(ht->common_mask != 0);
+	ok1(ht->common_mask != -1);
+	ok1(check_mask(ht, val, NUM_VALS));
+
+	/* Find all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_get(ht, hash(&dne, NULL), objcmp, &dne));
+
+	/* Walk once, should get them all. */
+	i = 0;
+	for (p = htable_first(ht,&iter); p; p = htable_next(ht, &iter))
+		i++;
+	ok1(i == NUM_VALS);
+
+	/* Delete all. */
+	del_vals(ht, val, NUM_VALS);
+	ok1(!htable_get(ht, hash(&val[0], NULL), objcmp, &val[0]));
+
+	/* Worst case, a "pointer" which doesn't have any matching bits. */
+	htable_add(ht, 0, (void *)~(uintptr_t)&val[NUM_VALS-1]);
+	htable_add(ht, hash(&val[NUM_VALS-1], NULL), &val[NUM_VALS-1]);
+	ok1(ht->common_mask == 0);
+	ok1(ht->common_bits == 0);
+	/* Get rid of bogus pointer before we trip over it! */
+	htable_del(ht, 0, (void *)~(uintptr_t)&val[NUM_VALS-1]);
+
+	/* Add the rest. */
+	add_vals(ht, val, NUM_VALS-1);
+
+	/* Check we can find them all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_get(ht, hash(&dne, NULL), objcmp, &dne));
+
+	htable_free(ht);
+
+	/* Corner cases: wipe out the perfect bit using bogus pointer. */
+	ht = htable_new(hash, NULL);
+	htable_add(ht, 0, (void *)((uintptr_t)&val[NUM_VALS-1]));
+	ok1(ht->perfect_bit);
+	perfect_bit = ht->perfect_bit;
+	htable_add(ht, 0, (void *)((uintptr_t)&val[NUM_VALS-1]
+				   | perfect_bit));
+	ok1(ht->perfect_bit == 0);
+	htable_del(ht, 0, (void *)((uintptr_t)&val[NUM_VALS-1] | perfect_bit));
+
+	/* Enlarging should restore it... */
+	add_vals(ht, val, NUM_VALS-1);
+
+	ok1(ht->perfect_bit != 0);
+	htable_free(ht);
+
+	return exit_status();
+}
diff --git a/lib/ccan/htable/tools/Makefile b/lib/ccan/htable/tools/Makefile
new file mode 100644
index 00000000000..001e160b78f
--- /dev/null
+++ b/lib/ccan/htable/tools/Makefile
@@ -0,0 +1,5 @@
+CFLAGS=-Wall -Werror -O3 -I../../..
+
+speed: speed.o ../../hash.o
+
+speed.o: speed.c ../htable.h ../htable.c
diff --git a/lib/ccan/htable/tools/speed.c b/lib/ccan/htable/tools/speed.c
new file mode 100644
index 00000000000..26231924a1f
--- /dev/null
+++ b/lib/ccan/htable/tools/speed.c
@@ -0,0 +1,377 @@
+/* Simple speed tests for hashtables. */
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/hash/hash.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/time.h>
+
+static size_t hashcount;
+struct object {
+	/* The key. */
+	unsigned int key;
+
+	/* Some contents. Doubles as consistency check. */
+	struct object *self;
+};
+
+static const unsigned int *objkey(const struct object *obj)
+{
+	return &obj->key;
+}
+
+static size_t hash_obj(const unsigned int *key)
+{
+	hashcount++;
+	return hashl(key, 1, 0);
+}
+
+static bool cmp(const unsigned int *key1, const unsigned int *key2)
+{
+	return *key1 == *key2;
+}
+
+HTABLE_DEFINE_TYPE(struct object, objkey, hash_obj, cmp, obj);
+
+static unsigned int popcount(unsigned long val)
+{
+#if HAVE_BUILTIN_POPCOUNTL
+	return __builtin_popcountl(val);
+#else
+	if (sizeof(long) == sizeof(u64)) {
+		u64 v = val;
+		v = (v & 0x5555555555555555ULL)
+			+ ((v >> 1) & 0x5555555555555555ULL);
+		v = (v & 0x3333333333333333ULL)
+			+ ((v >> 1) & 0x3333333333333333ULL);
+		v = (v & 0x0F0F0F0F0F0F0F0FULL)
+			+ ((v >> 1) & 0x0F0F0F0F0F0F0F0FULL);
+		v = (v & 0x00FF00FF00FF00FFULL)
+			+ ((v >> 1) & 0x00FF00FF00FF00FFULL);
+		v = (v & 0x0000FFFF0000FFFFULL)
+			+ ((v >> 1) & 0x0000FFFF0000FFFFULL);
+		v = (v & 0x00000000FFFFFFFFULL)
+			+ ((v >> 1) & 0x00000000FFFFFFFFULL);
+		return v;
+	}
+	val = (val & 0x55555555ULL) + ((val >> 1) & 0x55555555ULL);
+	val = (val & 0x33333333ULL) + ((val >> 1) & 0x33333333ULL);
+	val = (val & 0x0F0F0F0FULL) + ((val >> 1) & 0x0F0F0F0FULL);
+	val = (val & 0x00FF00FFULL) + ((val >> 1) & 0x00FF00FFULL);
+	val = (val & 0x0000FFFFULL) + ((val >> 1) & 0x0000FFFFULL);
+	return val;
+#endif
+}
+
+static size_t perfect(const struct htable *ht)
+{
+	size_t i, placed_perfect = 0;
+
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (!entry_is_valid(ht->table[i]))
+			continue;
+		if (hash_bucket(ht, ht->rehash(get_raw_ptr(ht, ht->table[i]),
+					       ht->priv)) == i) {
+			assert((ht->table[i] & ht->perfect_bit)
+			       == ht->perfect_bit);
+			placed_perfect++;
+		}
+	}
+	return placed_perfect;
+}
+
+static size_t count_deleted(const struct htable *ht)
+{
+	size_t i, delete_markers = 0;
+
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (ht->table[i] == HTABLE_DELETED)
+			delete_markers++;
+	}
+	return delete_markers;
+}
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeval *start,
+			const struct timeval *stop,
+			unsigned int num)
+{
+	struct timeval diff;
+
+	timersub(stop, start, &diff);
+
+	/* Floating point is more accurate here. */
+	return (double)(diff.tv_sec * 1000000 + diff.tv_usec)
+		/ num * 1000;
+}
+
+static size_t worst_run(struct htable *ht, size_t *deleted)
+{
+	size_t longest = 0, len = 0, this_del = 0, i;
+
+	*deleted = 0;
+	/* This doesn't take into account end-wrap, but gives an idea. */
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (ht->table[i]) {
+			len++;
+			if (ht->table[i] == HTABLE_DELETED)
+				this_del++;
+		} else {
+			if (len > longest) {
+				longest = len;
+				*deleted = this_del;
+			}
+			len = 0;
+			this_del = 0;
+		}
+	}
+	return longest;
+}
+
+int main(int argc, char *argv[])
+{
+	struct object *objs;
+	size_t i, j, num, deleted;
+	struct timeval start, stop;
+	struct htable_obj *ht;
+	struct htable *htr;
+	bool make_dumb = false;
+
+	if (argv[1] && strcmp(argv[1], "--dumb") == 0) {
+		argv++;
+		make_dumb = true;
+	}
+	num = argv[1] ? atoi(argv[1]) : 1000000;
+	objs = calloc(num, sizeof(objs[0]));
+
+	for (i = 0; i < num; i++) {
+		objs[i].key = i;
+		objs[i].self = &objs[i];
+	}
+
+	ht = htable_obj_new();
+	htr = (void *)ht;
+
+	printf("Initial insert: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+	printf("Details: hash size %u, mask bits %u, perfect %.0f%%\n",
+	       1U << htr->bits, popcount(htr->common_mask),
+	       perfect(htr) * 100.0 / htr->elems);
+
+	if (make_dumb) {
+		/* Screw with mask, to hobble us. */
+		update_common(htr, (void *)~htr->common_bits);
+		printf("Details: DUMB MODE: mask bits %u\n",
+		       popcount(htr->common_mask));
+	}
+
+	printf("Initial lookup (match): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if (htable_obj_get(ht, &i)->self != objs[i].self)
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Initial lookup (miss): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = i + num;
+		if (htable_obj_get(ht, &n))
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Lookups in order are very cache-friendly for judy; try random */
+	printf("Initial lookup (random): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num)
+		if (htable_obj_get(ht, &j)->self != &objs[j])
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Initial delete all: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+	printf("Details: rehashes %zu\n", hashcount);
+
+	printf("Initial re-inserting: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Deleting first half: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: rehashes %zu, delete markers %zu\n",
+	       hashcount, count_deleted(htr));
+
+	printf("Adding (a different) half: ");
+	fflush(stdout);
+
+	for (i = 0; i < num; i+=2)
+		objs[i].key = num+i;
+
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(htr), perfect(htr) * 100.0 / htr->elems);
+
+	printf("Lookup after half-change (match): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 1; i < num; i+=2)
+		if (htable_obj_get(ht, &i)->self != objs[i].self)
+			abort();
+	for (i = 0; i < num; i+=2) {
+		unsigned int n = i + num;
+		if (htable_obj_get(ht, &n)->self != objs[i].self)
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after half-change (miss): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = i + num * 2;
+		if (htable_obj_get(ht, &n))
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Hashtables with delete markers can fill with markers over time.
+	 * so do some changes to see how it operates in long-term. */
+	for (i = 0; i < 5; i++) {
+		if (i == 0) {
+			/* We don't measure this: jmap is different. */
+			printf("Details: initial churn\n");
+		} else {
+			printf("Churning %s time: ",
+			       i == 1 ? "second"
+			       : i == 2 ? "third"
+			       : i == 3 ? "fourth"
+			       : "fifth");
+			fflush(stdout);
+		}
+		gettimeofday(&start, NULL);
+		for (j = 0; j < num; j++) {
+			if (!htable_obj_del(ht, &objs[j]))
+				abort();
+			objs[j].key = num*i+j;
+			if (!htable_obj_add(ht, &objs[j]))
+				abort();
+		}
+		gettimeofday(&stop, NULL);
+		if (i != 0)
+			printf(" %zu ns\n", normalize(&start, &stop, num));
+	}
+
+	/* Spread out the keys more to try to make it harder. */
+	printf("Details: reinserting with spread\n");
+	for (i = 0; i < num; i++) {
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+		objs[i].key = num * 5 + i * 9;
+		if (!htable_obj_add(ht, objs[i].self))
+			abort();
+	}
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(htr), perfect(htr) * 100.0 / htr->elems);
+	i = worst_run(htr, &deleted);
+	printf("Details: worst run %zu (%zu deleted)\n", i, deleted);
+
+	printf("Lookup after churn & spread (match): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = num * 5 + i * 9;
+		if (htable_obj_get(ht, &n)->self != objs[i].self)
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after churn & spread (miss): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = num * (5 + 9) + i * 9;
+		if (htable_obj_get(ht, &n))
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after churn & spread (random): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num) {
+		unsigned int n = num * 5 + j * 9;
+		if (htable_obj_get(ht, &n)->self != &objs[j])
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Deleting half after churn & spread: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Adding (a different) half after churn & spread: ");
+	fflush(stdout);
+
+	for (i = 0; i < num; i+=2)
+		objs[i].key = num*6+i*9;
+
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(htr), perfect(htr) * 100.0 / htr->elems);
+
+	return 0;
+}
diff --git a/lib/ccan/ilog/LICENSE b/lib/ccan/ilog/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/ilog/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/ilog/_info b/lib/ccan/ilog/_info
new file mode 100644
index 00000000000..56de50d6105
--- /dev/null
+++ b/lib/ccan/ilog/_info
@@ -0,0 +1,47 @@
+/**
+ * ilog - Integer logarithm.
+ *
+ * ilog_32() and ilog_64() compute the minimum number of bits required to store
+ *  an unsigned 32-bit or 64-bit value without any leading zero bits.
+ * This can also be thought of as the location of the highest set bit, with
+ *  counting starting from one (so that 0 returns 0, 1 returns 1, and 2**31
+ *  returns 32).
+ * When the value is known to be non-zero ilog32_nz() and ilog64_nz() can
+ *  compile into as few as two instructions, one of which may get optimized out
+ *  later.
+ * STATIC_ILOG_32 and STATIC_ILOG_64 allow computation on compile-time
+ *  constants, so other compile-time constants can be derived from them.
+ *
+ * Example:
+ *  #include <stdio.h>
+ *  #include <limits.h>
+ *  #include <ccan/ilog/ilog.h>
+ *
+ *  int main(void){
+ *    int i;
+ *    printf("ilog32(0x%08X)=%i\n",0,ilog32(0));
+ *    for(i=1;i<=STATIC_ILOG_32(USHRT_MAX);i++){
+ *      uint32_t v;
+ *      v=(uint32_t)1U<<(i-1);
+ *      //Here we know v is non-zero, so we can use ilog32_nz().
+ *      printf("ilog32(0x%08X)=%i\n",v,ilog32_nz(v));
+ *    }
+ *    return 0;
+ *  }
+ *
+ * License: LGPL (v2 or later)
+ * Author: Timothy B. Terriberry <tterribe@xiph.org>
+ */
+#include <string.h>
+#include <stdio.h>
+#include "config.h"
+
+int main(int _argc,const char *_argv[]){
+  /*Expect exactly one argument.*/
+  if(_argc!=2)return 1;
+  if(strcmp(_argv[1],"depends")==0){
+    printf("ccan/compiler\n");
+    return 0;
+  }
+  return 1;
+}
diff --git a/lib/ccan/ilog/ilog.c b/lib/ccan/ilog/ilog.c
new file mode 100644
index 00000000000..40c5a6fd503
--- /dev/null
+++ b/lib/ccan/ilog/ilog.c
@@ -0,0 +1,139 @@
+/*(C) Timothy B. Terriberry (tterribe@xiph.org) 2001-2009 LGPL (v2 or later).*/
+#include "ilog.h"
+#include <limits.h>
+
+/*The fastest fallback strategy for platforms with fast multiplication appears
+   to be based on de Bruijn sequences~\cite{LP98}.
+  Tests confirmed this to be true even on an ARM11, where it is actually faster
+   than using the native clz instruction.
+  Define ILOG_NODEBRUIJN to use a simpler fallback on platforms where
+   multiplication or table lookups are too expensive.
+
+  @UNPUBLISHED{LP98,
+    author="Charles E. Leiserson and Harald Prokop",
+    title="Using de {Bruijn} Sequences to Index a 1 in a Computer Word",
+    month=Jun,
+    year=1998,
+    note="\url{http://supertech.csail.mit.edu/papers/debruijn.pdf}"
+  }*/
+static UNNEEDED const unsigned char DEBRUIJN_IDX32[32]={
+   0, 1,28, 2,29,14,24, 3,30,22,20,15,25,17, 4, 8,
+  31,27,13,23,21,19,16, 7,26,12,18, 6,11, 5,10, 9
+};
+
+/* We always compile these in, in case someone takes address of function. */
+#undef ilog32_nz
+#undef ilog32
+#undef ilog64_nz
+#undef ilog64
+
+int ilog32(uint32_t _v){
+/*On a Pentium M, this branchless version tested as the fastest version without
+   multiplications on 1,000,000,000 random 32-bit integers, edging out a
+   similar version with branches, and a 256-entry LUT version.*/
+# if defined(ILOG_NODEBRUIJN)
+  int ret;
+  int m;
+  ret=_v>0;
+  m=(_v>0xFFFFU)<<4;
+  _v>>=m;
+  ret|=m;
+  m=(_v>0xFFU)<<3;
+  _v>>=m;
+  ret|=m;
+  m=(_v>0xFU)<<2;
+  _v>>=m;
+  ret|=m;
+  m=(_v>3)<<1;
+  _v>>=m;
+  ret|=m;
+  ret+=_v>1;
+  return ret;
+/*This de Bruijn sequence version is faster if you have a fast multiplier.*/
+# else
+  int ret;
+  ret=_v>0;
+  _v|=_v>>1;
+  _v|=_v>>2;
+  _v|=_v>>4;
+  _v|=_v>>8;
+  _v|=_v>>16;
+  _v=(_v>>1)+1;
+  ret+=DEBRUIJN_IDX32[_v*0x77CB531U>>27&0x1F];
+  return ret;
+# endif
+}
+
+int ilog32_nz(uint32_t _v)
+{
+  return ilog32(_v);
+}
+
+int ilog64(uint64_t _v){
+# if defined(ILOG_NODEBRUIJN)
+  uint32_t v;
+  int      ret;
+  int      m;
+  ret=_v>0;
+  m=(_v>0xFFFFFFFFU)<<5;
+  v=(uint32_t)(_v>>m);
+  ret|=m;
+  m=(v>0xFFFFU)<<4;
+  v>>=m;
+  ret|=m;
+  m=(v>0xFFU)<<3;
+  v>>=m;
+  ret|=m;
+  m=(v>0xFU)<<2;
+  v>>=m;
+  ret|=m;
+  m=(v>3)<<1;
+  v>>=m;
+  ret|=m;
+  ret+=v>1;
+  return ret;
+# else
+/*If we don't have a 64-bit word, split it into two 32-bit halves.*/
+#  if LONG_MAX<9223372036854775807LL
+  uint32_t v;
+  int      ret;
+  int      m;
+  ret=_v>0;
+  m=(_v>0xFFFFFFFFU)<<5;
+  v=(uint32_t)(_v>>m);
+  ret|=m;
+  v|=v>>1;
+  v|=v>>2;
+  v|=v>>4;
+  v|=v>>8;
+  v|=v>>16;
+  v=(v>>1)+1;
+  ret+=DEBRUIJN_IDX32[v*0x77CB531U>>27&0x1F];
+  return ret;
+/*Otherwise do it in one 64-bit operation.*/
+#  else
+  static const unsigned char DEBRUIJN_IDX64[64]={
+     0, 1, 2, 7, 3,13, 8,19, 4,25,14,28, 9,34,20,40,
+     5,17,26,38,15,46,29,48,10,31,35,54,21,50,41,57,
+    63, 6,12,18,24,27,33,39,16,37,45,47,30,53,49,56,
+    62,11,23,32,36,44,52,55,61,22,43,51,60,42,59,58
+  };
+  int ret;
+  ret=_v>0;
+  _v|=_v>>1;
+  _v|=_v>>2;
+  _v|=_v>>4;
+  _v|=_v>>8;
+  _v|=_v>>16;
+  _v|=_v>>32;
+  _v=(_v>>1)+1;
+  ret+=DEBRUIJN_IDX64[_v*0x218A392CD3D5DBF>>58&0x3F];
+  return ret;
+#  endif
+# endif
+}
+
+int ilog64_nz(uint64_t _v)
+{
+  return ilog64(_v);
+}
diff --git a/lib/ccan/ilog/ilog.h b/lib/ccan/ilog/ilog.h
new file mode 100644
index 00000000000..55dd009885a
--- /dev/null
+++ b/lib/ccan/ilog/ilog.h
@@ -0,0 +1,150 @@
+#if !defined(_ilog_H)
+# define _ilog_H (1)
+# include "config.h"
+# include <stdint.h>
+# include <limits.h>
+# include <ccan/compiler/compiler.h>
+
+/**
+ * ilog32 - Integer binary logarithm of a 32-bit value.
+ * @_v: A 32-bit value.
+ * Returns floor(log2(_v))+1, or 0 if _v==0.
+ * This is the number of bits that would be required to represent _v in two's
+ *  complement notation with all of the leading zeros stripped.
+ * Note that many uses will resolve to the fast macro version instead.
+ *
+ * See Also:
+ *	ilog32_nz(), ilog64()
+ *
+ * Example:
+ *	// Rounds up to next power of 2 (if not a power of 2).
+ *	static uint32_t round_up32(uint32_t i)
+ *	{
+ *		assert(i != 0);
+ *		return 1U << ilog32(i-1);
+ *	}
+ */
+int ilog32(uint32_t _v) IDEMPOTENT;
+
+/**
+ * ilog32_nz - Integer binary logarithm of a non-zero 32-bit value.
+ * @_v: A 32-bit value.
+ * Returns floor(log2(_v))+1, or undefined if _v==0.
+ * This is the number of bits that would be required to represent _v in two's
+ *  complement notation with all of the leading zeros stripped.
+ * Note that many uses will resolve to the fast macro version instead.
+ * See Also:
+ *	ilog32(), ilog64_nz()
+ * Example:
+ *	// Find Last Set (ie. highest bit set, 0 to 31).
+ *	static uint32_t fls32(uint32_t i)
+ *	{
+ *		assert(i != 0);
+ *		return ilog32_nz(i) - 1;
+ *	}
+ */
+int ilog32_nz(uint32_t _v) IDEMPOTENT;
+
+/**
+ * ilog64 - Integer binary logarithm of a 64-bit value.
+ * @_v: A 64-bit value.
+ * Returns floor(log2(_v))+1, or 0 if _v==0.
+ * This is the number of bits that would be required to represent _v in two's
+ *  complement notation with all of the leading zeros stripped.
+ * Note that many uses will resolve to the fast macro version instead.
+ * See Also:
+ *	ilog64_nz(), ilog32()
+ */
+int ilog64(uint64_t _v) IDEMPOTENT;
+
+/**
+ * ilog64_nz - Integer binary logarithm of a non-zero 64-bit value.
+ * @_v: A 64-bit value.
+ * Returns floor(log2(_v))+1, or undefined if _v==0.
+ * This is the number of bits that would be required to represent _v in two's
+ *  complement notation with all of the leading zeros stripped.
+ * Note that many uses will resolve to the fast macro version instead.
+ * See Also:
+ *	ilog64(), ilog32_nz()
+ */
+int ilog64_nz(uint64_t _v) IDEMPOTENT;
+
+/**
+ * STATIC_ILOG_32 - The integer logarithm of an (unsigned, 32-bit) constant.
+ * @_v: A non-negative 32-bit constant.
+ * Returns floor(log2(_v))+1, or 0 if _v==0.
+ * This is the number of bits that would be required to represent _v in two's
+ *  complement notation with all of the leading zeros stripped.
+ * This macro should only be used when you need a compile-time constant,
+ * otherwise ilog32 or ilog32_nz are just as fast and more flexible.
+ *
+ * Example:
+ *	#define MY_PAGE_SIZE	4096
+ *	#define MY_PAGE_BITS	(STATIC_ILOG_32(PAGE_SIZE) - 1)
+ */
+#define STATIC_ILOG_32(_v) (STATIC_ILOG5((uint32_t)(_v)))
+
+/**
+ * STATIC_ILOG_64 - The integer logarithm of an (unsigned, 64-bit) constant.
+ * @_v: A non-negative 64-bit constant.
+ * Returns floor(log2(_v))+1, or 0 if _v==0.
+ * This is the number of bits that would be required to represent _v in two's
+ *  complement notation with all of the leading zeros stripped.
+ * This macro should only be used when you need a compile-time constant,
+ * otherwise ilog64 or ilog64_nz are just as fast and more flexible.
+ */
+#define STATIC_ILOG_64(_v) (STATIC_ILOG6((uint64_t)(_v)))
+
+/* Private implementation details */
+
+/*Note the casts to (int) below: this prevents "upgrading"
+   the type of an entire expression to an (unsigned) size_t.*/
+#if INT_MAX>=2147483647 && HAVE_BUILTIN_CLZ
+#define builtin_ilog32_nz(v) \
+	(((int)sizeof(unsigned)*CHAR_BIT) - __builtin_clz(v))
+#elif LONG_MAX>=2147483647L && HAVE_BUILTIN_CLZL
+#define builtin_ilog32_nz(v) \
+	(((int)sizeof(unsigned)*CHAR_BIT) - __builtin_clzl(v))
+#endif
+
+#if INT_MAX>=9223372036854775807LL && HAVE_BUILTIN_CLZ
+#define builtin_ilog64_nz(v) \
+	(((int)sizeof(unsigned)*CHAR_BIT) - __builtin_clz(v))
+#elif LONG_MAX>=9223372036854775807LL && HAVE_BUILTIN_CLZL
+#define builtin_ilog64_nz(v) \
+	(((int)sizeof(unsigned long)*CHAR_BIT) - __builtin_clzl(v))
+#elif HAVE_BUILTIN_CLZLL
+#define builtin_ilog64_nz(v) \
+	(((int)sizeof(unsigned long long)*CHAR_BIT) - __builtin_clzll(v))
+#endif
+
+#ifdef builtin_ilog32_nz
+#define ilog32(_v) (builtin_ilog32_nz(_v)&-!!(_v))
+#define ilog32_nz(_v) builtin_ilog32_nz(_v)
+#else
+#define ilog32_nz(_v) ilog32(_v)
+#define ilog32(_v) (IS_COMPILE_CONSTANT(_v) ? STATIC_ILOG_32(_v) : ilog32(_v))
+#endif /* builtin_ilog32_nz */
+
+#ifdef builtin_ilog64_nz
+#define ilog64(_v) (builtin_ilog64_nz(_v)&-!!(_v))
+#define ilog64_nz(_v) builtin_ilog64_nz(_v)
+#else
+#define ilog64_nz(_v) ilog64(_v)
+#define ilog64(_v) (IS_COMPILE_CONSTANT(_v) ? STATIC_ILOG_64(_v) : ilog64(_v))
+#endif /* builtin_ilog64_nz */
+
+/* Macros for evaluating compile-time constant ilog. */
+# define STATIC_ILOG0(_v) (!!(_v))
+# define STATIC_ILOG1(_v) (((_v)&0x2)?2:STATIC_ILOG0(_v))
+# define STATIC_ILOG2(_v) (((_v)&0xC)?2+STATIC_ILOG1((_v)>>2):STATIC_ILOG1(_v))
+# define STATIC_ILOG3(_v) \
+ (((_v)&0xF0)?4+STATIC_ILOG2((_v)>>4):STATIC_ILOG2(_v))
+# define STATIC_ILOG4(_v) \
+ (((_v)&0xFF00)?8+STATIC_ILOG3((_v)>>8):STATIC_ILOG3(_v))
+# define STATIC_ILOG5(_v) \
+ (((_v)&0xFFFF0000)?16+STATIC_ILOG4((_v)>>16):STATIC_ILOG4(_v))
+# define STATIC_ILOG6(_v) \
+ (((_v)&0xFFFFFFFF00000000ULL)?32+STATIC_ILOG5((_v)>>32):STATIC_ILOG5(_v))
+
+#endif /* _ilog_H */
diff --git a/lib/ccan/ilog/test/run-out-of-line.c b/lib/ccan/ilog/test/run-out-of-line.c
new file mode 100644
index 00000000000..48205d380ed
--- /dev/null
+++ b/lib/ccan/ilog/test/run-out-of-line.c
@@ -0,0 +1,65 @@
+#include <ccan/ilog/ilog.h>
+#include <ccan/ilog/ilog.c>
+#include <stdio.h>
+#include <ccan/tap/tap.h>
+
+/*Dead simple (but slow) versions to compare against.*/
+
+static int test_ilog32(uint32_t _v){
+  int ret;
+  for(ret=0;_v;ret++)_v>>=1;
+  return ret;
+}
+
+static int test_ilog64(uint64_t _v){
+  int ret;
+  for(ret=0;_v;ret++)_v>>=1;
+  return ret;
+}
+
+#define NTRIALS (64)
+
+int main(int _argc,const char *_argv[]){
+  int i;
+  int j;
+  int (*il32)(uint32_t) = ilog32;
+  int (*il64)(uint64_t) = ilog64;
+  int (*il32_nz)(uint32_t) = ilog32_nz;
+  int (*il64_nz)(uint64_t) = ilog64_nz;
+
+  /*This is how many tests you plan to run.*/
+  plan_tests(33 * NTRIALS * 3 + 65 * NTRIALS * 3);
+  for(i=0;i<=32;i++){
+    uint32_t v;
+    /*Test each bit in turn (and 0).*/
+    v=i?(uint32_t)1U<<(i-1):0;
+    for(j=0;j<NTRIALS;j++){
+      int l;
+      l=test_ilog32(v);
+      ok1(STATIC_ILOG_32(v)==l);
+      ok1(il32(v)==l);
+      ok1(il32_nz(v) == l || v == 0);
+      /*Also try a few more pseudo-random values with at most the same number
+         of bits.*/
+      v=(1103515245U*v+12345U)&0xFFFFFFFFU>>((33-i)>>1)>>((32-i)>>1);
+    }
+  }
+
+  for(i=0;i<=64;i++){
+    uint64_t v;
+    /*Test each bit in turn (and 0).*/
+    v=i?(uint64_t)1U<<(i-1):0;
+    for(j=0;j<NTRIALS;j++){
+      int l;
+      l=test_ilog64(v);
+      ok1(STATIC_ILOG_64(v)==l);
+      ok1(il64(v)==l);
+      ok1(il64_nz(v) == l || v == 0);
+      /*Also try a few more pseudo-random values with at most the same number
+         of bits.*/
+      v=(uint64_t)((2862933555777941757ULL*v+3037000493ULL)
+	&0xFFFFFFFFFFFFFFFFULL>>((65-i)>>1)>>((64-i)>>1));
+    }
+  }
+  return exit_status();
+}
diff --git a/lib/ccan/ilog/test/run.c b/lib/ccan/ilog/test/run.c
new file mode 100644
index 00000000000..bda59f920a3
--- /dev/null
+++ b/lib/ccan/ilog/test/run.c
@@ -0,0 +1,60 @@
+#include <ccan/ilog/ilog.h>
+#include <ccan/ilog/ilog.c>
+#include <stdio.h>
+#include <ccan/tap/tap.h>
+
+/*Dead simple (but slow) versions to compare against.*/
+
+static int test_ilog32(uint32_t _v){
+  int ret;
+  for(ret=0;_v;ret++)_v>>=1;
+  return ret;
+}
+
+static int test_ilog64(uint64_t _v){
+  int ret;
+  for(ret=0;_v;ret++)_v>>=1;
+  return ret;
+}
+
+#define NTRIALS (64)
+
+int main(int _argc,const char *_argv[]){
+  int i;
+  int j;
+  /*This is how many tests you plan to run.*/
+  plan_tests(33 * NTRIALS * 3 + 65 * NTRIALS * 3);
+  for(i=0;i<=32;i++){
+    uint32_t v;
+    /*Test each bit in turn (and 0).*/
+    v=i?(uint32_t)1U<<(i-1):0;
+    for(j=0;j<NTRIALS;j++){
+      int l;
+      l=test_ilog32(v);
+      ok1(STATIC_ILOG_32(v)==l);
+      ok1(ilog32(v)==l);
+      ok1(ilog32_nz(v) == l || v == 0);
+      /*Also try a few more pseudo-random values with at most the same number
+         of bits.*/
+      v=(1103515245U*v+12345U)&0xFFFFFFFFU>>((33-i)>>1)>>((32-i)>>1);
+    }
+  }
+
+  for(i=0;i<=64;i++){
+    uint64_t v;
+    /*Test each bit in turn (and 0).*/
+    v=i?(uint64_t)1U<<(i-1):0;
+    for(j=0;j<NTRIALS;j++){
+      int l;
+      l=test_ilog64(v);
+      ok1(STATIC_ILOG_64(v)==l);
+      ok1(ilog64(v)==l);
+      ok1(ilog64_nz(v) == l || v == 0);
+      /*Also try a few more pseudo-random values with at most the same number
+         of bits.*/
+      v=(uint64_t)((2862933555777941757ULL*v+3037000493ULL)
+	&0xFFFFFFFFFFFFFFFFULL>>((65-i)>>1)>>((64-i)>>1));
+    }
+  }
+  return exit_status();
+}
diff --git a/lib/ccan/libccan.m4 b/lib/ccan/libccan.m4
new file mode 100644
index 00000000000..92676c31842
--- /dev/null
+++ b/lib/ccan/libccan.m4
@@ -0,0 +1,315 @@
+dnl find the ccan sources.
+ccandir="../lib/ccan"
+for d in $ccanpaths; do
+	if test -f "$srcdir/$d/str/str.c"; then
+		ccandir="$d"
+		AC_SUBST(ccandir)
+		break
+	fi
+done
+if test -f "$ccandir/str/str.c"; then :; else
+   AC_MSG_ERROR([cannot find ccan source in $ccandir])
+fi
+CCAN_OBJ="$ccandir/hash/hash.o $ccandir/htable/htable.o $ccandir/ilog/ilog.o $ccandir/likely/likely.o $ccandir/str/debug.o $ccandir/str/str.o $ccandir/tally/tally.o"
+
+AC_SUBST(CCAN_OBJ)
+
+# Preferred method for including ccan modules is #include <ccan/module/...>.
+CCAN_CFLAGS="-I$ccandir/.."
+AC_SUBST(CCAN_CFLAGS)
+
+# All the configuration checks.  Regrettably, the __attribute__ checks will
+# give false positives on old GCCs, since they just cause warnings.  But that's
+# fairly harmless.
+AC_CACHE_CHECK([whether we can compile with __attribute__((cold))],
+	       samba_cv_attribute_cold,
+	       [
+	         AC_COMPILE_IFELSE(
+			[
+				static void __attribute__((cold))
+				cleanup(void) { }
+			],
+			samba_cv_attribute_cold=yes)
+		])
+
+if test x"$samba_cv_attribute_cold" = xyes ; then
+   AC_DEFINE(HAVE_ATTRIBUTE_COLD, 1,
+	     [whether we can compile with __attribute__((cold))])
+fi
+
+AC_CACHE_CHECK([whether we can compile with __attribute__((const))],
+	       samba_cv_attribute_const,
+	       [
+	         AC_COMPILE_IFELSE(
+			[
+				static void __attribute__((const))
+				cleanup(void) { }
+			],
+			samba_cv_attribute_const=yes)
+		])
+
+if test x"$samba_cv_attribute_const" = xyes ; then
+   AC_DEFINE(HAVE_ATTRIBUTE_CONST, 1,
+	     [whether we can compile with __attribute__((const))])
+fi
+
+AC_CACHE_CHECK([whether we can compile with __attribute__((noreturn))],
+	       samba_cv_attribute_noreturn,
+	       [
+	         AC_COMPILE_IFELSE(
+			[
+				static void __attribute__((noreturn))
+				cleanup(void) { exit(1); }
+			],
+			samba_cv_attribute_noreturn=yes)
+		])
+
+if test x"$samba_cv_attribute_noreturn" = xyes ; then
+   AC_DEFINE(HAVE_ATTRIBUTE_NORETURN, 1,
+	     [whether we can compile with __attribute__((noreturn))])
+fi
+
+AC_CACHE_CHECK([whether we can compile with __attribute__((printf))],
+	       samba_cv_attribute_printf,
+	       [
+	         AC_COMPILE_IFELSE(
+			[
+				static void __attribute__((format(__printf__, 1, 2)))
+				cleanup(const char *fmt, ...) { }
+			],
+			samba_cv_attribute_printf=yes)
+		])
+
+if test x"$samba_cv_attribute_printf" = xyes ; then
+   AC_DEFINE(HAVE_ATTRIBUTE_PRINTF, 1,
+	     [whether we can compile with __attribute__((format(printf)))])
+fi
+
+AC_CACHE_CHECK([whether we can compile with __attribute__((unused))],
+	       samba_cv_attribute_unused,
+	       [
+	         AC_COMPILE_IFELSE(
+			[
+				static void __attribute__((unused))
+				cleanup(void) { }
+			],
+			samba_cv_attribute_unused=yes)
+		])
+
+if test x"$samba_cv_attribute_unused" = xyes ; then
+   AC_DEFINE(HAVE_ATTRIBUTE_UNUSED, 1,
+	     [whether we can compile with __attribute__((unused))])
+fi
+
+AC_CACHE_CHECK([whether we can compile with __attribute__((used))],
+	       samba_cv_attribute_used,
+	       [
+	         AC_COMPILE_IFELSE(
+			[
+				static void __attribute__((used))
+				cleanup(void) { }
+			],
+			samba_cv_attribute_used=yes)
+		])
+
+if test x"$samba_cv_attribute_used" = xyes ; then
+   AC_DEFINE(HAVE_ATTRIBUTE_USED, 1,
+	     [whether we can compile with __attribute__((used))])
+fi
+
+# FIXME: We could use endian.h or sys/endian.h here, and __BYTE_ORDER for
+# cross-compiling.
+AC_CACHE_CHECK([whether we are big endian],samba_cv_big_endian,[
+AC_TRY_RUN([int main(void) {
+union { int i; char c[sizeof(int)]; } u;
+	  u.i = 0x01020304;
+	  return u.c[0] == 0x01 && u.c[1] == 0x02 && u.c[2] == 0x03 && u.c[3] == 0x04 ? 0 : 1;
+}],
+samba_cv_big_endian=yes,
+samba_cv_big_endian=no)])
+if test x"$samba_cv_big_endian" = xyes ; then
+   AC_DEFINE(HAVE_BIG_ENDIAN, 1,
+	     [whether we are big endian])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_clz],
+	       samba_cv_builtin_clz,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				return __builtin_clz(1) == (sizeof(int)*8 - 1) ? 0 : 1;
+			}],
+			samba_cv_builtin_clz=yes)
+		])
+
+if test x"$samba_cv_builtin_clz" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_CLZ, 1,
+	     [whether we have __builtin_clz])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_clzl],
+	       samba_cv_builtin_clzl,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				return __builtin_clzl(1) == (sizeof(int)*8 - 1) ? 0 : 1;
+			}],
+			samba_cv_builtin_clzl=yes)
+		])
+
+if test x"$samba_cv_builtin_clzl" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_CLZL, 1,
+	     [whether we have __builtin_clzl])
+fi
+AC_CACHE_CHECK([whether we have __builtin_clzll],
+	       samba_cv_builtin_clzll,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				return __builtin_clzll(1) == (sizeof(int)*8 - 1) ? 0 : 1;
+			}],
+			samba_cv_builtin_clzll=yes)
+		])
+
+if test x"$samba_cv_builtin_clzll" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_CLZLL, 1,
+	     [whether we have __builtin_clzll])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_constant_p],
+	       samba_cv_builtin_constant_p,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				return __builtin_constant_p(1) ? 0 : 1;
+			}],
+			samba_cv_builtin_constant_p=yes)
+		])
+
+if test x"$samba_cv_builtin_constant_p" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_CONSTANT_P, 1,
+	     [whether we have __builtin_constant_p])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_expect],
+	       samba_cv_builtin_expect,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				return __builtin_expect(main != 0) ? 0 : 1;
+			}],
+			samba_cv_builtin_expect=yes)
+		])
+
+if test x"$samba_cv_builtin_expect" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_EXPECT, 1,
+	     [whether we have __builtin_expect])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_popcountl],
+	       samba_cv_builtin_popcountl,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				return __builtin_popcountl(255L) == 8 ? 0 : 1;
+			}],
+			samba_cv_builtin_popcountl=yes)
+		])
+
+if test x"$samba_cv_builtin_popcountl" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_POPCOUNTL, 1,
+	     [whether we have __builtin_popcountl])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_types_compatible_p],
+	       samba_cv_builtin_types_compatible_p,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				return __builtin_types_compatible_p(char *, int) ? 1 : 0;
+			}],
+			samba_cv_builtin_types_compatible_p=yes)
+		])
+
+if test x"$samba_cv_builtin_types_compatible_p" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_TYPES_COMPATIBLE_P, 1,
+	     [whether we have __builtin_types_compatible_p])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_compound_literals],
+	       samba_cv_builtin_compound_literals,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				int *foo = (int[]) { 1, 2, 3, 4 };
+				return foo[0] == 1 ? 0 : 1;
+			}],
+			samba_cv_builtin_compound_literals=yes)
+		])
+
+if test x"$samba_cv_builtin_compound_literals" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_COMPOUND_LITERALS, 1,
+	     [whether we have __builtin_compound_literals])
+fi
+
+AC_CACHE_CHECK([whether we have __builtin_have_isblank],
+	       samba_cv_builtin_have_isblank,
+	       [
+	         AC_COMPILE_IFELSE(
+			[#include <ctype.h>
+			 int main(void) { return isblank(' ') ? 0 : 1; }
+			],
+			samba_cv_builtin_have_isblank=yes)
+		])
+
+if test x"$samba_cv_builtin_have_isblank" = xyes ; then
+   AC_DEFINE(HAVE_BUILTIN_HAVE_ISBLANK, 1,
+	     [whether we have __builtin_have_isblank])
+fi
+
+# FIXME: We could use endian.h or sys/endian.h here, and __BYTE_ORDER for
+# cross-compiling.
+AC_CACHE_CHECK([whether we are little endian],samba_cv_little_endian,[
+AC_TRY_RUN([int main(void) {
+union { int i; char c[sizeof(int)]; } u;
+	  u.i = 0x01020304;
+	  return u.c[0] == 0x04 && u.c[1] == 0x03 && u.c[2] == 0x02 && u.c[3] == 0x01 ? 0 : 1;
+}],
+samba_cv_little_endian=yes,
+samba_cv_little_endian=no)])
+if test x"$samba_cv_little_endian" = xyes ; then
+   AC_DEFINE(HAVE_LITTLE_ENDIAN, 1,
+	     [whether we are little endian])
+fi
+
+AC_CACHE_CHECK([whether we have __typeof__],
+	       samba_cv_typeof,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int main(void) {
+				int x = 1;
+				__typeof__(x) i;
+				i = x;
+				return i == x ? 0 : 1;
+			}],
+			samba_cv_typeof=yes)
+		])
+
+if test x"$samba_cv_typeof" = xyes ; then
+   AC_DEFINE(HAVE_TYPEOF, 1,
+	     [whether we have __typeof__])
+fi
+
+AC_CACHE_CHECK([whether we have __attribute__((warn_unused_result))],
+	       samba_cv_warn_unused_result,
+	       [
+	         AC_COMPILE_IFELSE(
+			[int __attribute__((warn_unused_result)) func(int x)
+			    { return x; }],
+			samba_cv_warn_unused_result=yes)
+		])
+
+if test x"$samba_cv_warn_unused_result" = xyes ; then
+   AC_DEFINE(HAVE_WARN_UNUSED_RESULT, 1,
+	     [whether we have __attribute__((warn_unused_result))])
+fi
diff --git a/lib/ccan/likely/LICENSE b/lib/ccan/likely/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/likely/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/likely/_info b/lib/ccan/likely/_info
new file mode 100644
index 00000000000..471c1ffc78c
--- /dev/null
+++ b/lib/ccan/likely/_info
@@ -0,0 +1,45 @@
+#include <string.h>
+#include <stdio.h>
+#include "config.h"
+
+/**
+ * likely - macros for annotating likely/unlikely branches in the code
+ *
+ * Inspired by Andi Kleen's macros for the Linux Kernel, these macros
+ * help you annotate rare paths in your code for the convenience of the
+ * compiler and the reader.
+ *
+ * License: LGPL (2 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Example:
+ *	#include <ccan/likely/likely.h>
+ *	#include <stdio.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		// This example is silly: the compiler knows exit() is unlikely.
+ *		if (unlikely(argc == 1)) {
+ *			fprintf(stderr, "Usage: %s <args>...\n", argv[0]);
+ *			return 1;
+ *		}
+ *		for (argc++; argv[argc]; argc++)
+ *			printf("%s\n", argv[argc]);
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/str\n");
+		printf("ccan/htable\n");
+		printf("ccan/hash\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/likely/likely.c b/lib/ccan/likely/likely.c
new file mode 100644
index 00000000000..8893d0b6d24
--- /dev/null
+++ b/lib/ccan/likely/likely.c
@@ -0,0 +1,141 @@
+#ifdef CCAN_LIKELY_DEBUG
+#include <ccan/likely/likely.h>
+#include <ccan/hash/hash.h>
+#include <ccan/htable/htable.h>
+#include <stdlib.h>
+#include <stdio.h>
+static struct htable *htable;
+
+struct trace {
+	const char *condstr;
+	const char *file;
+	unsigned int line;
+	bool expect;
+	unsigned long count, right;
+};
+
+/* We hash the pointers, which will be identical for same call. */
+static unsigned long hash_trace(const struct trace *trace)
+{
+	return hash_pointer(trace->condstr,
+			    hash_pointer(trace->file,
+					 trace->line + trace->expect));
+}
+
+static bool hash_cmp(const void *htelem, void *cmpdata)
+{
+	const struct trace *t1 = htelem, *t2 = cmpdata;
+	return t1->condstr == t2->condstr
+		&& t1->file == t2->file
+		&& t1->line == t2->line
+		&& t1->expect == t2->expect;
+}
+
+static size_t rehash(const void *elem, void *priv)
+{
+	return hash_trace(elem);
+}
+
+static void init_trace(struct trace *trace,
+		       const char *condstr, const char *file, unsigned int line,
+		       bool expect)
+{
+	trace->condstr = condstr;
+	trace->file = file;
+	trace->line = line;
+	trace->expect = expect;
+	trace->count = trace->right = 0;
+}
+
+static struct trace *add_trace(const char *condstr,
+			       const char *file, unsigned int line, bool expect)
+{
+	struct trace *trace = malloc(sizeof(*trace));
+	init_trace(trace, condstr, file, line, expect);
+	htable_add(htable, hash_trace(trace), trace);
+	return trace;
+}
+
+long _likely_trace(bool cond, bool expect,
+		   const char *condstr,
+		   const char *file, unsigned int line)
+{
+	struct trace *p, trace;
+
+	if (!htable)
+		htable = htable_new(rehash, NULL);
+
+	init_trace(&trace, condstr, file, line, expect);
+	p = htable_get(htable, hash_trace(&trace), hash_cmp, &trace);
+	if (!p)
+		p = add_trace(condstr, file, line, expect);
+
+	p->count++;
+	if (cond == expect)
+		p->right++;
+
+	return cond;
+}
+
+struct get_stats_info {
+	struct trace *worst;
+	unsigned int min_hits;
+	double worst_ratio;
+};
+
+static double right_ratio(const struct trace *t)
+{
+	return (double)t->right / t->count;
+}
+
+static void get_stats(struct trace *trace, struct get_stats_info *info)
+{
+	if (trace->count < info->min_hits)
+		return;
+
+	if (right_ratio(trace) < info->worst_ratio) {
+		info->worst = trace;
+		info->worst_ratio = right_ratio(trace);
+	}
+}
+
+const char *likely_stats(unsigned int min_hits, unsigned int percent)
+{
+	struct get_stats_info info;
+	struct htable_iter i;
+	char *ret;
+	struct trace *trace;
+
+	if (!htable)
+		return NULL;
+
+	info.min_hits = min_hits;
+	info.worst = NULL;
+	info.worst_ratio = 2;
+
+	/* This is O(n), but it's not likely called that often. */
+	for (trace = htable_first(htable, &i);
+	     trace;
+	     trace = htable_next(htable,&i)) {
+		get_stats(trace, &info);
+	}
+
+	if (info.worst_ratio * 100 > percent)
+		return NULL;
+
+	ret = malloc(strlen(info.worst->condstr) +
+		     strlen(info.worst->file) +
+		     sizeof(long int) * 8 +
+		     sizeof("%s:%u:%slikely(%s) correct %u%% (%lu/%lu)"));
+	sprintf(ret, "%s:%u:%slikely(%s) correct %u%% (%lu/%lu)",
+		info.worst->file, info.worst->line,
+		info.worst->expect ? "" : "un", info.worst->condstr,
+		(unsigned)(info.worst_ratio * 100),
+		info.worst->right, info.worst->count);
+
+	htable_del(htable, hash_trace(info.worst), info.worst);
+	free(info.worst);
+
+	return ret;
+}
+#endif /*CCAN_LIKELY_DEBUG*/
diff --git a/lib/ccan/likely/likely.h b/lib/ccan/likely/likely.h
new file mode 100644
index 00000000000..80d695c8426
--- /dev/null
+++ b/lib/ccan/likely/likely.h
@@ -0,0 +1,105 @@
+#ifndef CCAN_LIKELY_H
+#define CCAN_LIKELY_H
+#include "config.h"
+#include <ccan/str/str.h>
+#include <stdbool.h>
+
+#ifndef CCAN_LIKELY_DEBUG
+#if HAVE_BUILTIN_EXPECT
+/**
+ * likely - indicate that a condition is likely to be true.
+ * @cond: the condition
+ *
+ * This uses a compiler extension where available to indicate a likely
+ * code path and optimize appropriately; it's also useful for readers
+ * to quickly identify exceptional paths through functions.  The
+ * threshold for "likely" is usually considered to be between 90 and
+ * 99%; marginal cases should not be marked either way.
+ *
+ * See Also:
+ *	unlikely(), likely_stats()
+ *
+ * Example:
+ *	// Returns false if we overflow.
+ *	static inline bool inc_int(unsigned int *val)
+ *	{
+ *		(*val)++;
+ *		if (likely(*val))
+ *			return true;
+ *		return false;
+ *	}
+ */
+#define likely(cond) __builtin_expect(!!(cond), 1)
+
+/**
+ * unlikely - indicate that a condition is unlikely to be true.
+ * @cond: the condition
+ *
+ * This uses a compiler extension where available to indicate an unlikely
+ * code path and optimize appropriately; see likely() above.
+ *
+ * See Also:
+ *	likely(), likely_stats(), COLD (compiler.h)
+ *
+ * Example:
+ *	// Prints a warning if we overflow.
+ *	static inline void inc_int(unsigned int *val)
+ *	{
+ *		(*val)++;
+ *		if (unlikely(*val == 0))
+ *			fprintf(stderr, "Overflow!");
+ *	}
+ */
+#define unlikely(cond) __builtin_expect(!!(cond), 0)
+#else
+#define likely(cond) (!!(cond))
+#define unlikely(cond) (!!(cond))
+#endif
+#else /* CCAN_LIKELY_DEBUG versions */
+#define likely(cond) \
+	(_likely_trace(!!(cond), 1, stringify(cond), __FILE__, __LINE__))
+#define unlikely(cond) \
+	(_likely_trace(!!(cond), 0, stringify(cond), __FILE__, __LINE__))
+
+long _likely_trace(bool cond, bool expect,
+		   const char *condstr,
+		   const char *file, unsigned int line);
+#endif
+
+#ifdef CCAN_LIKELY_DEBUG
+/**
+ * likely_stats - return description of abused likely()/unlikely()
+ * @min_hits: minimum number of hits
+ * @percent: maximum percentage correct
+ *
+ * When CCAN_LIKELY_DEBUG is defined, likely() and unlikely() trace their
+ * results: this causes a significant slowdown, but allows analysis of
+ * whether the branches are labelled correctly.
+ *
+ * This function returns a malloc'ed description of the least-correct
+ * usage of likely() or unlikely().  It ignores places which have been
+ * called less than @min_hits times, and those which were predicted
+ * correctly more than @percent of the time.  It returns NULL when
+ * nothing meets those criteria.
+ *
+ * Note that this call is destructive; the returned offender is
+ * removed from the trace so that the next call to likely_stats() will
+ * return the next-worst likely()/unlikely() usage.
+ *
+ * Example:
+ *	// Print every place hit more than twice which was wrong > 5%.
+ *	static void report_stats(void)
+ *	{
+ *	#ifdef CCAN_LIKELY_DEBUG
+ *		const char *bad;
+ *
+ *		while ((bad = likely_stats(2, 95)) != NULL) {
+ *			printf("Suspicious likely: %s", bad);
+ *			free(bad);
+ *		}
+ *	#endif
+ *	}
+ */
+const char *likely_stats(unsigned int min_hits, unsigned int percent);
+#endif /* CCAN_LIKELY_DEBUG */
+#endif /* CCAN_LIKELY_H */
diff --git a/lib/ccan/likely/test/run-debug.c b/lib/ccan/likely/test/run-debug.c
new file mode 100644
index 00000000000..df78619271f
--- /dev/null
+++ b/lib/ccan/likely/test/run-debug.c
@@ -0,0 +1,87 @@
+#define CCAN_LIKELY_DEBUG 1
+#include <ccan/likely/likely.c>
+#include <ccan/likely/likely.h>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+
+static bool one_seems_likely(unsigned int val)
+{
+	if (likely(val == 1))
+		return true;
+	return false;
+}
+
+static bool one_seems_unlikely(unsigned int val)
+{
+	if (unlikely(val == 1))
+		return true;
+	return false;
+}
+
+static bool likely_one_unlikely_two(unsigned int val1, unsigned int val2)
+{
+	/* Same line, check we don't get confused! */
+	if (likely(val1 == 1) && unlikely(val2 == 2))
+		return true;
+	return false;
+}
+
+int main(int argc, char *argv[])
+{
+	const char *bad;
+
+	plan_tests(13);
+
+	/* Correct guesses. */
+	one_seems_likely(1);
+	ok1(likely_stats(0, 90) == NULL);
+	one_seems_unlikely(2);
+	ok1(likely_stats(0, 90) == NULL);
+
+	/* Incorrect guesses. */
+	one_seems_likely(0);
+	one_seems_likely(2);
+	/* Hasn't been hit 4 times, so this fails */
+	ok1(!likely_stats(4, 90));
+	bad = likely_stats(3, 90);
+	ok(strends(bad, "run-debug.c:9:likely(val == 1) correct 33% (1/3)"),
+	   "likely_stats returned %s", bad);
+
+	/* Nothing else above 90% */
+	ok1(!likely_stats(0, 90));
+
+	/* This should get everything. */
+	bad = likely_stats(0, 100);
+	ok(strends(bad, "run-debug.c:16:unlikely(val == 1) correct 100% (1/1)"),
+	   "likely_stats returned %s", bad);
+
+	/* Nothing left (table is actually cleared) */
+	ok1(!likely_stats(0, 100));
+
+	/* Make sure unlikely works */
+	one_seems_unlikely(0);
+	one_seems_unlikely(2);
+	one_seems_unlikely(1);
+
+	bad = likely_stats(0, 90);
+	ok(strends(bad, "run-debug.c:16:unlikely(val == 1) correct 66% (2/3)"),
+	   "likely_stats returned %s", bad);
+	ok1(!likely_stats(0, 100));
+
+	likely_one_unlikely_two(1, 1);
+	likely_one_unlikely_two(1, 1);
+	likely_one_unlikely_two(1, 1);
+	ok1(!likely_stats(0, 90));
+	likely_one_unlikely_two(1, 2);
+
+	bad = likely_stats(0, 90);
+	ok(strends(bad, "run-debug.c:24:unlikely(val2 == 2) correct 75% (3/4)"),
+	   "likely_stats returned %s", bad);
+	bad = likely_stats(0, 100);
+	ok(strends(bad, "run-debug.c:24:likely(val1 == 1) correct 100% (4/4)"),
+	   "likely_stats returned %s", bad);
+
+	ok1(!likely_stats(0, 100));
+
+	exit(exit_status());
+}
diff --git a/lib/ccan/likely/test/run.c b/lib/ccan/likely/test/run.c
new file mode 100644
index 00000000000..fa1dc9f6ead
--- /dev/null
+++ b/lib/ccan/likely/test/run.c
@@ -0,0 +1,30 @@
+#include <ccan/likely/likely.c>
+#include <ccan/likely/likely.h>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+
+static bool one_seems_likely(unsigned int val)
+{
+	if (likely(val == 1))
+		return true;
+	return false;
+}
+
+static bool one_seems_unlikely(unsigned int val)
+{
+	if (unlikely(val == 1))
+		return true;
+	return false;
+}
+
+int main(int argc, char *argv[])
+{
+	plan_tests(4);
+
+	/* Without debug, we can only check that it doesn't effect functions. */
+	ok1(one_seems_likely(1));
+	ok1(!one_seems_likely(2));
+	ok1(one_seems_unlikely(1));
+	ok1(!one_seems_unlikely(2));
+	exit(exit_status());
+}
diff --git a/lib/ccan/str/LICENSE b/lib/ccan/str/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/str/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/str/_info b/lib/ccan/str/_info
new file mode 100644
index 00000000000..ea314dbf799
--- /dev/null
+++ b/lib/ccan/str/_info
@@ -0,0 +1,52 @@
+#include <stdio.h>
+#include <string.h>
+#include "config.h"
+
+/**
+ * str - string helper routines
+ *
+ * This is a grab bag of functions for string operations, designed to enhance
+ * the standard string.h.
+ *
+ * Note that if you define CCAN_STR_DEBUG, you will get extra compile
+ * checks on common misuses of the following functions (they will now
+ * be out-of-line, so there is a runtime penalty!).
+ *
+ *	strstr, strchr, strrchr:
+ *		Return const char * if first argument is const (gcc only).
+ *
+ *	isalnum, isalpha, isascii, isblank, iscntrl, isdigit, isgraph,
+ *	    islower, isprint, ispunct, isspace, isupper, isxdigit:
+ *		Static and runtime check that input is EOF or an *unsigned*
+ *		char, as per C standard (really!).
+ *
+ * Example:
+ *	#include <stdio.h>
+ *	#include <ccan/str/str.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		if (argv[1] && streq(argv[1], "--verbose"))
+ *			printf("verbose set\n");
+ *		if (argv[1] && strstarts(argv[1], "--"))
+ *			printf("Some option set\n");
+ *		if (argv[1] && strends(argv[1], "cow-powers"))
+ *			printf("Magic option set\n");
+ *		return 0;
+ *	}
+ *
+ * License: LGPL (2 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/str/debug.c b/lib/ccan/str/debug.c
new file mode 100644
index 00000000000..9ef756766aa
--- /dev/null
+++ b/lib/ccan/str/debug.c
@@ -0,0 +1,104 @@
+#include "config.h"
+#include <ccan/str/str_debug.h>
+#include <assert.h>
+#include <ctype.h>
+#include <string.h>
+
+#ifdef CCAN_STR_DEBUG
+/* Because we mug the real ones with macros, we need our own wrappers. */
+int str_isalnum(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isalnum(i);
+}
+
+int str_isalpha(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isalpha(i);
+}
+
+int str_isascii(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isascii(i);
+}
+
+#if HAVE_ISBLANK
+int str_isblank(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isblank(i);
+}
+#endif
+
+int str_iscntrl(int i)
+{
+	assert(i >= -1 && i < 256);
+	return iscntrl(i);
+}
+
+int str_isdigit(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isdigit(i);
+}
+
+int str_isgraph(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isgraph(i);
+}
+
+int str_islower(int i)
+{
+	assert(i >= -1 && i < 256);
+	return islower(i);
+}
+
+int str_isprint(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isprint(i);
+}
+
+int str_ispunct(int i)
+{
+	assert(i >= -1 && i < 256);
+	return ispunct(i);
+}
+
+int str_isspace(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isspace(i);
+}
+
+int str_isupper(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isupper(i);
+}
+
+int str_isxdigit(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isxdigit(i);
+}
+
+
+char *str_strstr(const char *haystack, const char *needle)
+{
+	return strstr(haystack, needle);
+}
+
+char *str_strchr(const char *haystack, int c)
+{
+	return strchr(haystack, c);
+}
+
+char *str_strrchr(const char *haystack, int c)
+{
+	return strrchr(haystack, c);
+}
+#endif
diff --git a/lib/ccan/str/str.c b/lib/ccan/str/str.c
new file mode 100644
index 00000000000..fa9809fbd94
--- /dev/null
+++ b/lib/ccan/str/str.c
@@ -0,0 +1,12 @@
+#include <ccan/str/str.h>
+
+size_t strcount(const char *haystack, const char *needle)
+{
+	size_t i = 0, nlen = strlen(needle);
+
+	while ((haystack = strstr(haystack, needle)) != NULL) {
+		i++;
+		haystack += nlen;
+	}
+	return i;
+}
diff --git a/lib/ccan/str/str.h b/lib/ccan/str/str.h
new file mode 100644
index 00000000000..ae51cdcc998
--- /dev/null
+++ b/lib/ccan/str/str.h
@@ -0,0 +1,200 @@
+#ifndef CCAN_STR_H
+#define CCAN_STR_H
+#include "config.h"
+#include <string.h>
+#include <stdbool.h>
+#include <ctype.h>
+
+/**
+ * streq - Are two strings equal?
+ * @a: first string
+ * @b: first string
+ *
+ * This macro is arguably more readable than "!strcmp(a, b)".
+ *
+ * Example:
+ *	if (streq(somestring, ""))
+ *		printf("String is empty!\n");
+ */
+#define streq(a,b) (strcmp((a),(b)) == 0)
+
+/**
+ * strstarts - Does this string start with this prefix?
+ * @str: string to test
+ * @prefix: prefix to look for at start of str
+ *
+ * Example:
+ *	if (strstarts(somestring, "foo"))
+ *		printf("String %s begins with 'foo'!\n", somestring);
+ */
+#define strstarts(str,prefix) (strncmp((str),(prefix),strlen(prefix)) == 0)
+
+/**
+ * strends - Does this string end with this postfix?
+ * @str: string to test
+ * @postfix: postfix to look for at end of str
+ *
+ * Example:
+ *	if (strends(somestring, "foo"))
+ *		printf("String %s end with 'foo'!\n", somestring);
+ */
+static inline bool strends(const char *str, const char *postfix)
+{
+	if (strlen(str) < strlen(postfix))
+		return false;
+
+	return streq(str + strlen(str) - strlen(postfix), postfix);
+}
+
+/**
+ * stringify - Turn expression into a string literal
+ * @expr: any C expression
+ *
+ * Example:
+ *	#define PRINT_COND_IF_FALSE(cond) \
+ *		((cond) || printf("%s is false!", stringify(cond)))
+ */
+#define stringify(expr)		stringify_1(expr)
+/* Double-indirection required to stringify expansions */
+#define stringify_1(expr)	#expr
+
+/**
+ * strcount - Count number of (non-overlapping) occurrences of a substring.
+ * @haystack: a C string
+ * @needle: a substring
+ *
+ * Example:
+ *	int i;
+ *      i = strcount("aaa aaa", "a");  // i = 6;
+ *      i = strcount("aaa aaa", "ab"); // i = 0;
+ *      i = strcount("aaa aaa", "aa"); // i = 2;
+ */
+size_t strcount(const char *haystack, const char *needle);
+
+/**
+ * cisalnum - isalnum() which takes a char (and doesn't accept EOF)
+ * @c: a character
+ *
+ * Surprisingly, the standard ctype.h isalnum() takes an int, which
+ * must have the value of EOF (-1) or an unsigned char.  This variant
+ * takes a real char, and doesn't accept EOF.
+ */
+static inline bool cisalnum(char c)
+{
+	return isalnum((unsigned char)c);
+}
+static inline bool cisalpha(char c)
+{
+	return isalpha((unsigned char)c);
+}
+static inline bool cisascii(char c)
+{
+	return isascii((unsigned char)c);
+}
+#if HAVE_ISBLANK
+static inline bool cisblank(char c)
+{
+	return isblank((unsigned char)c);
+}
+#endif
+static inline bool ciscntrl(char c)
+{
+	return iscntrl((unsigned char)c);
+}
+static inline bool cisdigit(char c)
+{
+	return isdigit((unsigned char)c);
+}
+static inline bool cisgraph(char c)
+{
+	return isgraph((unsigned char)c);
+}
+static inline bool cislower(char c)
+{
+	return islower((unsigned char)c);
+}
+static inline bool cisprint(char c)
+{
+	return isprint((unsigned char)c);
+}
+static inline bool cispunct(char c)
+{
+	return ispunct((unsigned char)c);
+}
+static inline bool cisspace(char c)
+{
+	return isspace((unsigned char)c);
+}
+static inline bool cisupper(char c)
+{
+	return isupper((unsigned char)c);
+}
+static inline bool cisxdigit(char c)
+{
+	return isxdigit((unsigned char)c);
+}
+
+#include <ccan/str/str_debug.h>
+
+/* These checks force things out of line, hence they are under DEBUG. */
+#ifdef CCAN_STR_DEBUG
+#include <ccan/build_assert/build_assert.h>
+
+/* These are commonly misused: they take -1 or an *unsigned* char value. */
+#undef isalnum
+#undef isalpha
+#undef isascii
+#undef isblank
+#undef iscntrl
+#undef isdigit
+#undef isgraph
+#undef islower
+#undef isprint
+#undef ispunct
+#undef isspace
+#undef isupper
+#undef isxdigit
+
+/* You can use a char if char is unsigned. */
+#if HAVE_BUILTIN_TYPES_COMPATIBLE_P && HAVE_TYPEOF
+#define str_check_arg_(i)						\
+	((i) + BUILD_ASSERT_OR_ZERO(!__builtin_types_compatible_p(typeof(i), \
+								  char)	\
+				    || (char)255 > 0))
+#else
+#define str_check_arg_(i) (i)
+#endif
+
+#define isalnum(i) str_isalnum(str_check_arg_(i))
+#define isalpha(i) str_isalpha(str_check_arg_(i))
+#define isascii(i) str_isascii(str_check_arg_(i))
+#if HAVE_ISBLANK
+#define isblank(i) str_isblank(str_check_arg_(i))
+#endif
+#define iscntrl(i) str_iscntrl(str_check_arg_(i))
+#define isdigit(i) str_isdigit(str_check_arg_(i))
+#define isgraph(i) str_isgraph(str_check_arg_(i))
+#define islower(i) str_islower(str_check_arg_(i))
+#define isprint(i) str_isprint(str_check_arg_(i))
+#define ispunct(i) str_ispunct(str_check_arg_(i))
+#define isspace(i) str_isspace(str_check_arg_(i))
+#define isupper(i) str_isupper(str_check_arg_(i))
+#define isxdigit(i) str_isxdigit(str_check_arg_(i))
+
+#if HAVE_TYPEOF
+/* With GNU magic, we can make const-respecting standard string functions. */
+#undef strstr
+#undef strchr
+#undef strrchr
+
+/* + 0 is needed to decay array into pointer. */
+#define strstr(haystack, needle)					\
+	((typeof((haystack) + 0))str_strstr((haystack), (needle)))
+#define strchr(haystack, c)					\
+	((typeof((haystack) + 0))str_strchr((haystack), (c)))
+#define strrchr(haystack, c)					\
+	((typeof((haystack) + 0))str_strrchr((haystack), (c)))
+#endif
+#endif /* CCAN_STR_DEBUG */
+
+#endif /* CCAN_STR_H */
diff --git a/lib/ccan/str/str_debug.h b/lib/ccan/str/str_debug.h
new file mode 100644
index 00000000000..6b56477689e
--- /dev/null
+++ b/lib/ccan/str/str_debug.h
@@ -0,0 +1,29 @@
+#ifndef CCAN_STR_DEBUG_H
+#define CCAN_STR_DEBUG_H
+
+/* #define CCAN_STR_DEBUG 1 */
+
+#ifdef CCAN_STR_DEBUG
+/* Because we mug the real ones with macros, we need our own wrappers. */
+int str_isalnum(int i);
+int str_isalpha(int i);
+int str_isascii(int i);
+#if HAVE_ISBLANK
+int str_isblank(int i);
+#endif
+int str_iscntrl(int i);
+int str_isdigit(int i);
+int str_isgraph(int i);
+int str_islower(int i);
+int str_isprint(int i);
+int str_ispunct(int i);
+int str_isspace(int i);
+int str_isupper(int i);
+int str_isxdigit(int i);
+
+char *str_strstr(const char *haystack, const char *needle);
+char *str_strchr(const char *s, int c);
+char *str_strrchr(const char *s, int c);
+#endif /* CCAN_STR_DEBUG */
+
+#endif /* CCAN_STR_DEBUG_H */
diff --git a/lib/ccan/str/test/compile_fail-isalnum.c b/lib/ccan/str/test/compile_fail-isalnum.c
new file mode 100644
index 00000000000..930defffa0c
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isalnum.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isalnum.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isalnum(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isalpha.c b/lib/ccan/str/test/compile_fail-isalpha.c
new file mode 100644
index 00000000000..2005109829b
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isalpha.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isalpha.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isalpha(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isascii.c b/lib/ccan/str/test/compile_fail-isascii.c
new file mode 100644
index 00000000000..ee55e499743
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isascii.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isascii.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isascii(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isblank.c b/lib/ccan/str/test/compile_fail-isblank.c
new file mode 100644
index 00000000000..f4cb961d740
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isblank.c
@@ -0,0 +1,26 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF || !HAVE_ISBLANK
+#error We need typeof to check isblank.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+#if HAVE_ISBLANK
+	return isblank(c);
+#else
+	return c;
+#endif
+}
diff --git a/lib/ccan/str/test/compile_fail-iscntrl.c b/lib/ccan/str/test/compile_fail-iscntrl.c
new file mode 100644
index 00000000000..bc74146542f
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-iscntrl.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check iscntrl.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return iscntrl(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isdigit.c b/lib/ccan/str/test/compile_fail-isdigit.c
new file mode 100644
index 00000000000..71d1c714330
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isdigit.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isdigit.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isdigit(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-islower.c b/lib/ccan/str/test/compile_fail-islower.c
new file mode 100644
index 00000000000..ca3f9907e52
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-islower.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check islower.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return islower(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isprint.c b/lib/ccan/str/test/compile_fail-isprint.c
new file mode 100644
index 00000000000..6432e41d2b7
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isprint.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isprint.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isprint(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-ispunct.c b/lib/ccan/str/test/compile_fail-ispunct.c
new file mode 100644
index 00000000000..5d941fcba68
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-ispunct.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check ispunct.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return ispunct(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isspace.c b/lib/ccan/str/test/compile_fail-isspace.c
new file mode 100644
index 00000000000..bfee1f89f16
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isspace.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isspace.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isspace(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isupper.c b/lib/ccan/str/test/compile_fail-isupper.c
new file mode 100644
index 00000000000..4cf9fd35785
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isupper.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isupper.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isupper(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-isxdigit.c b/lib/ccan/str/test/compile_fail-isxdigit.c
new file mode 100644
index 00000000000..65e6006a885
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-isxdigit.c
@@ -0,0 +1,22 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isxdigit.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isxdigit(c);
+}
diff --git a/lib/ccan/str/test/compile_fail-strchr.c b/lib/ccan/str/test/compile_fail-strchr.c
new file mode 100644
index 00000000000..74a7314d06b
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-strchr.c
@@ -0,0 +1,18 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_TYPEOF
+	#error We need typeof to check strstr.
+#endif
+#else
+	const
+#endif
+		char *ret;
+	const char *str = "hello";
+
+	ret = strchr(str, 'l');
+	return ret ? 0 : 1;
+}
diff --git a/lib/ccan/str/test/compile_fail-strrchr.c b/lib/ccan/str/test/compile_fail-strrchr.c
new file mode 100644
index 00000000000..ba7d17e0319
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-strrchr.c
@@ -0,0 +1,18 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_TYPEOF
+	#error We need typeof to check strstr.
+#endif
+#else
+	const
+#endif
+		char *ret;
+	const char *str = "hello";
+
+	ret = strrchr(str, 'l');
+	return ret ? 0 : 1;
+}
diff --git a/lib/ccan/str/test/compile_fail-strstr.c b/lib/ccan/str/test/compile_fail-strstr.c
new file mode 100644
index 00000000000..deefef6542b
--- /dev/null
+++ b/lib/ccan/str/test/compile_fail-strstr.c
@@ -0,0 +1,18 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+#if !HAVE_TYPEOF
+	#error We need typeof to check strstr.
+#endif
+#else
+	const
+#endif
+		char *ret;
+	const char *str = "hello";
+
+	ret = strstr(str, "hell");
+	return ret ? 0 : 1;
+}
diff --git a/lib/ccan/str/test/debug.c b/lib/ccan/str/test/debug.c
new file mode 100644
index 00000000000..4bd384f2c48
--- /dev/null
+++ b/lib/ccan/str/test/debug.c
@@ -0,0 +1,5 @@
+/* We can't use the normal "#include the .c file" trick, since this is
+   contaminated by str.h's macro overrides.  So we put it in all tests
+   like this. */
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/debug.c>
diff --git a/lib/ccan/str/test/run.c b/lib/ccan/str/test/run.c
new file mode 100644
index 00000000000..a15654f8f32
--- /dev/null
+++ b/lib/ccan/str/test/run.c
@@ -0,0 +1,105 @@
+#include <ccan/str/str.h>
+#include <ccan/str/str.c>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ccan/tap/tap.h>
+
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
+
+static char *substrings[] = { "far", "bar", "baz", "b", "ba", "z", "ar", NULL };
+
+#define NUM_SUBSTRINGS (ARRAY_SIZE(substrings) - 1)
+
+static char *strdup_rev(const char *s)
+{
+	char *ret = strdup(s);
+	unsigned int i;
+
+	for (i = 0; i < strlen(s); i++)
+		ret[i] = s[strlen(s) - i - 1];
+	return ret;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, n;
+	char *strings[NUM_SUBSTRINGS * NUM_SUBSTRINGS];
+
+	n = 0;
+	for (i = 0; i < NUM_SUBSTRINGS; i++) {
+		for (j = 0; j < NUM_SUBSTRINGS; j++) {
+			strings[n] = malloc(strlen(substrings[i])
+					    + strlen(substrings[j]) + 1);
+			sprintf(strings[n++], "%s%s",
+				substrings[i], substrings[j]);
+		}
+	}
+
+	plan_tests(n * n * 5 + 16);
+	for (i = 0; i < n; i++) {
+		for (j = 0; j < n; j++) {
+			unsigned int k, identical = 0;
+			char *reva, *revb;
+
+			/* Find first difference. */
+			for (k = 0; strings[i][k]==strings[j][k]; k++) {
+				if (k == strlen(strings[i])) {
+					identical = 1;
+					break;
+				}
+			}
+
+			if (identical)
+				ok1(streq(strings[i], strings[j]));
+			else
+				ok1(!streq(strings[i], strings[j]));
+
+			/* Postfix test should be equivalent to prefix
+			 * test on reversed string. */
+			reva = strdup_rev(strings[i]);
+			revb = strdup_rev(strings[j]);
+
+			if (!strings[i][k]) {
+				ok1(strstarts(strings[j], strings[i]));
+				ok1(strends(revb, reva));
+			} else {
+				ok1(!strstarts(strings[j], strings[i]));
+				ok1(!strends(revb, reva));
+			}
+			if (!strings[j][k]) {
+				ok1(strstarts(strings[i], strings[j]));
+				ok1(strends(reva, revb));
+			} else {
+				ok1(!strstarts(strings[i], strings[j]));
+				ok1(!strends(reva, revb));
+			}
+			free(reva);
+			free(revb);
+		}
+	}
+
+	for (i = 0; i < n; i++)
+		free(strings[i]);
+
+	ok1(streq(stringify(NUM_SUBSTRINGS),
+		  "((sizeof(substrings) / sizeof(substrings[0])) - 1)"));
+	ok1(streq(stringify(ARRAY_SIZE(substrings)),
+		  "(sizeof(substrings) / sizeof(substrings[0]))"));
+	ok1(streq(stringify(i == 0), "i == 0"));
+
+	ok1(strcount("aaaaaa", "b") == 0);
+	ok1(strcount("aaaaaa", "a") == 6);
+	ok1(strcount("aaaaaa", "aa") == 3);
+	ok1(strcount("aaaaaa", "aaa") == 2);
+	ok1(strcount("aaaaaa", "aaaa") == 1);
+	ok1(strcount("aaaaaa", "aaaaa") == 1);
+	ok1(strcount("aaaaaa", "aaaaaa") == 1);
+	ok1(strcount("aaa aaa", "b") == 0);
+	ok1(strcount("aaa aaa", "a") == 6);
+	ok1(strcount("aaa aaa", "aa") == 2);
+	ok1(strcount("aaa aaa", "aaa") == 2);
+	ok1(strcount("aaa aaa", "aaaa") == 0);
+	ok1(strcount("aaa aaa", "aaaaa") == 0);
+
+	return exit_status();
+}
diff --git a/lib/ccan/tally/LICENSE b/lib/ccan/tally/LICENSE
new file mode 100644
index 00000000000..cca7fc278f5
--- /dev/null
+++ b/lib/ccan/tally/LICENSE
@@ -0,0 +1,165 @@
+		   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/lib/ccan/tally/_info b/lib/ccan/tally/_info
new file mode 100644
index 00000000000..1d67274f5c2
--- /dev/null
+++ b/lib/ccan/tally/_info
@@ -0,0 +1,58 @@
+#include <stdio.h>
+#include <string.h>
+#include "config.h"
+
+/**
+ * tally - running tally of integers
+ *
+ * The tally module implements simple analysis of a stream of integers.
+ * Numbers are fed in via tally_add(), and then the mean, median, mode and
+ * a histogram can be read out.
+ *
+ * Example:
+ *	#include <stdio.h>
+ *	#include <err.h>
+ *	#include <ccan/tally/tally.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct tally *t;
+ *		unsigned int i;
+ *		size_t err;
+ *		ssize_t val;
+ *		char *histogram;
+ *
+ *		if (argc < 2)
+ *			errx(1, "Usage: %s <number>...\n", argv[0]);
+ *
+ *		t = tally_new(100);
+ *		for (i = 1; i < argc; i++)
+ *			tally_add(t, atol(argv[i]));
+ *
+ *		printf("Mean = %zi\n", tally_mean(t));
+ *		val = tally_approx_median(t, &err);
+ *		printf("Median = %zi (+/- %zu)\n", val, err);
+ *		val = tally_approx_mode(t, &err);
+ *		printf("Mode = %zi (+/- %zu)\n", val, err);
+ *		histogram = tally_histogram(t, 50, 10);
+ *		printf("Histogram:\n%s", histogram);
+ *		free(histogram);
+ *		return 0;
+ *	}
+ *
+ * License: LGPL (3 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		printf("ccan/likely\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/tally/tally.c b/lib/ccan/tally/tally.c
new file mode 100644
index 00000000000..b1839befe3b
--- /dev/null
+++ b/lib/ccan/tally/tally.c
@@ -0,0 +1,490 @@
+#include <ccan/tally/tally.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/likely/likely.h>
+#include <stdint.h>
+#include <limits.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+
+#define SIZET_BITS (sizeof(size_t)*CHAR_BIT)
+
+/* We use power of 2 steps.  I tried being tricky, but it got buggy. */
+struct tally {
+	ssize_t min, max;
+	size_t total[2];
+	/* This allows limited frequency analysis. */
+	unsigned buckets, step_bits;
+	size_t counts[1 /* Actually: [buckets] */ ];
+};
+
+struct tally *tally_new(unsigned buckets)
+{
+	struct tally *tally;
+
+	/* There is always 1 bucket. */
+	if (buckets == 0)
+		buckets = 1;
+
+	/* Check for overflow. */
+	if (buckets && SIZE_MAX / buckets < sizeof(tally->counts[0]))
+		return NULL;
+	tally = malloc(sizeof(*tally) + sizeof(tally->counts[0])*(buckets-1));
+	if (tally) {
+		tally->max = ((size_t)1 << (SIZET_BITS - 1));
+		tally->min = ~tally->max;
+		tally->total[0] = tally->total[1] = 0;
+		tally->buckets = buckets;
+		tally->step_bits = 0;
+		memset(tally->counts, 0, sizeof(tally->counts[0])*buckets);
+	}
+	return tally;
+}
+
+static unsigned bucket_of(ssize_t min, unsigned step_bits, ssize_t val)
+{
+	/* Don't over-shift. */
+	if (step_bits == SIZET_BITS)
+		return 0;
+	assert(step_bits < SIZET_BITS);
+	return (size_t)(val - min) >> step_bits;
+}
+
+/* Return the min value in bucket b. */
+static ssize_t bucket_min(ssize_t min, unsigned step_bits, unsigned b)
+{
+	/* Don't over-shift. */
+	if (step_bits == SIZET_BITS)
+		return min;
+	assert(step_bits < SIZET_BITS);
+	return min + ((ssize_t)b << step_bits);
+}
+
+/* Does shifting by this many bits truncate the number? */
+static bool shift_overflows(size_t num, unsigned bits)
+{
+	if (bits == 0)
+		return false;
+
+	return ((num << bits) >> 1) != (num << (bits - 1));
+}
+
+/* When min or max change, we may need to shuffle the frequency counts. */
+static void renormalize(struct tally *tally,
+			ssize_t new_min, ssize_t new_max)
+{
+	size_t range, spill;
+	unsigned int i, old_min;
+
+	/* Uninitialized?  Don't do anything... */
+	if (tally->max < tally->min)
+		goto update;
+
+	/* If we don't have sufficient range, increase step bits until
+	 * buckets cover entire range of ssize_t anyway. */
+	range = (new_max - new_min) + 1;
+	while (!shift_overflows(tally->buckets, tally->step_bits)
+	       && range > ((size_t)tally->buckets << tally->step_bits)) {
+		/* Collapse down. */
+		for (i = 1; i < tally->buckets; i++) {
+			tally->counts[i/2] += tally->counts[i];
+			tally->counts[i] = 0;
+		}
+		tally->step_bits++;
+	}
+
+	/* Now if minimum has dropped, move buckets up. */
+	old_min = bucket_of(new_min, tally->step_bits, tally->min);
+	memmove(tally->counts + old_min,
+		tally->counts,
+		sizeof(tally->counts[0]) * (tally->buckets - old_min));
+	memset(tally->counts, 0, sizeof(tally->counts[0]) * old_min);
+
+	/* If we moved boundaries, adjust buckets to that ratio. */
+	spill = (tally->min - new_min) % (1 << tally->step_bits);
+	for (i = 0; i < tally->buckets-1; i++) {
+		size_t adjust = (tally->counts[i] >> tally->step_bits) * spill;
+		tally->counts[i] -= adjust;
+		tally->counts[i+1] += adjust;
+	}
+
+update:
+	tally->min = new_min;
+	tally->max = new_max;
+}
+
+void tally_add(struct tally *tally, ssize_t val)
+{
+	ssize_t new_min = tally->min, new_max = tally->max;
+	bool need_renormalize = false;
+
+	if (val < tally->min) {
+		new_min = val;
+		need_renormalize = true;
+	}
+	if (val > tally->max) {
+		new_max = val;
+		need_renormalize = true;
+	}
+	if (need_renormalize)
+		renormalize(tally, new_min, new_max);
+
+	/* 128-bit arithmetic!  If we didn't want exact mean, we could just
+	 * pull it out of counts. */
+	if (val > 0 && tally->total[0] + val < tally->total[0])
+		tally->total[1]++;
+	else if (val < 0 && tally->total[0] + val > tally->total[0])
+		tally->total[1]--;
+	tally->total[0] += val;
+	tally->counts[bucket_of(tally->min, tally->step_bits, val)]++;
+}
+
+size_t tally_num(const struct tally *tally)
+{
+	size_t i, num = 0;
+	for (i = 0; i < tally->buckets; i++)
+		num += tally->counts[i];
+	return num;
+}
+
+ssize_t tally_min(const struct tally *tally)
+{
+	return tally->min;
+}
+
+ssize_t tally_max(const struct tally *tally)
+{
+	return tally->max;
+}
+
+/* FIXME: Own ccan module please! */
+static unsigned fls64(uint64_t val)
+{
+#if HAVE_BUILTIN_CLZL
+	if (val <= ULONG_MAX) {
+		/* This is significantly faster! */
+		return val ? sizeof(long) * CHAR_BIT - __builtin_clzl(val) : 0;
+	} else {
+#endif
+	uint64_t r = 64;
+
+	if (!val)
+		return 0;
+	if (!(val & 0xffffffff00000000ull)) {
+		val <<= 32;
+		r -= 32;
+	}
+	if (!(val & 0xffff000000000000ull)) {
+		val <<= 16;
+		r -= 16;
+	}
+	if (!(val & 0xff00000000000000ull)) {
+		val <<= 8;
+		r -= 8;
+	}
+	if (!(val & 0xf000000000000000ull)) {
+		val <<= 4;
+		r -= 4;
+	}
+	if (!(val & 0xc000000000000000ull)) {
+		val <<= 2;
+		r -= 2;
+	}
+	if (!(val & 0x8000000000000000ull)) {
+		val <<= 1;
+		r -= 1;
+	}
+	return r;
+#if HAVE_BUILTIN_CLZL
+	}
+#endif
+}
+
+/* This is stolen straight from Hacker's Delight. */
+static uint64_t divlu64(uint64_t u1, uint64_t u0, uint64_t v)
+{
+	const uint64_t b = 4294967296ULL; // Number base (32 bits).
+	uint32_t un[4],		  // Dividend and divisor
+		vn[2];		  // normalized and broken
+				  // up into halfwords.
+	uint32_t q[2];		  // Quotient as halfwords.
+	uint64_t un1, un0,	  // Dividend and divisor
+		vn0;		  // as fullwords.
+	uint64_t qhat;		  // Estimated quotient digit.
+	uint64_t rhat;		  // A remainder.
+	uint64_t p;		  // Product of two digits.
+	int64_t s, i, j, t, k;
+
+	if (u1 >= v)		  // If overflow, return the largest
+		return (uint64_t)-1; // possible quotient.
+
+	s = 64 - fls64(v);		  // 0 <= s <= 63.
+	vn0 = v << s;		  // Normalize divisor.
+	vn[1] = vn0 >> 32;	  // Break divisor up into
+	vn[0] = vn0 & 0xFFFFFFFF; // two 32-bit halves.
+
+	// Shift dividend left.
+	un1 = ((u1 << s) | (u0 >> (64 - s))) & (-s >> 63);
+	un0 = u0 << s;
+	un[3] = un1 >> 32;	  // Break dividend up into
+	un[2] = un1;		  // four 32-bit halfwords
+	un[1] = un0 >> 32;	  // Note: storing into
+	un[0] = un0;		  // halfwords truncates.
+
+	for (j = 1; j >= 0; j--) {
+		// Compute estimate qhat of q[j].
+		qhat = (un[j+2]*b + un[j+1])/vn[1];
+		rhat = (un[j+2]*b + un[j+1]) - qhat*vn[1];
+	again:
+		if (qhat >= b || qhat*vn[0] > b*rhat + un[j]) {
+			qhat = qhat - 1;
+			rhat = rhat + vn[1];
+			if (rhat < b) goto again;
+		}
+
+		// Multiply and subtract.
+		k = 0;
+		for (i = 0; i < 2; i++) {
+			p = qhat*vn[i];
+			t = un[i+j] - k - (p & 0xFFFFFFFF);
+			un[i+j] = t;
+			k = (p >> 32) - (t >> 32);
+		}
+		t = un[j+2] - k;
+		un[j+2] = t;
+
+		q[j] = qhat;		  // Store quotient digit.
+		if (t < 0) {		  // If we subtracted too
+			q[j] = q[j] - 1;  // much, add back.
+			k = 0;
+			for (i = 0; i < 2; i++) {
+				t = un[i+j] + vn[i] + k;
+				un[i+j] = t;
+				k = t >> 32;
+			}
+			un[j+2] = un[j+2] + k;
+		}
+	} // End j.
+
+	return q[1]*b + q[0];
+}
+
+static int64_t divls64(int64_t u1, uint64_t u0, int64_t v)
+{
+	int64_t q, uneg, vneg, diff, borrow;
+
+	uneg = u1 >> 63;	  // -1 if u < 0.
+	if (uneg) {		  // Compute the absolute
+		u0 = -u0;	  // value of the dividend u.
+		borrow = (u0 != 0);
+		u1 = -u1 - borrow;
+	}
+
+	vneg = v >> 63;		  // -1 if v < 0.
+	v = (v ^ vneg) - vneg;	  // Absolute value of v.
+
+	if ((uint64_t)u1 >= (uint64_t)v)
+		goto overflow;
+
+	q = divlu64(u1, u0, v);
+
+	diff = uneg ^ vneg;	  // Negate q if signs of
+	q = (q ^ diff) - diff;	  // u and v differed.
+
+	if ((diff ^ q) < 0 && q != 0) {	   // If overflow, return the largest
+	overflow:			   // possible neg. quotient.
+		q = 0x8000000000000000ULL;
+	}
+	return q;
+}
+
+ssize_t tally_mean(const struct tally *tally)
+{
+	size_t count = tally_num(tally);
+	if (!count)
+		return 0;
+
+	if (sizeof(tally->total[0]) == sizeof(uint32_t)) {
+		/* Use standard 64-bit arithmetic. */
+		int64_t total = tally->total[0]
+			| (((uint64_t)tally->total[1]) << 32);
+		return total / count;
+	}
+	return divls64(tally->total[1], tally->total[0], count);
+}
+
+ssize_t tally_total(const struct tally *tally, ssize_t *overflow)
+{
+	if (overflow) {
+		*overflow = tally->total[1];
+		return tally->total[0];
+	}
+
+	/* If result is negative, make sure we can represent it. */
+	if (tally->total[1] & ((size_t)1 << (SIZET_BITS-1))) {
+		/* Must have only underflowed once, and must be able to
+		 * represent result at ssize_t. */
+		if ((~tally->total[1])+1 != 0
+		    || (ssize_t)tally->total[0] >= 0) {
+			/* Underflow, return minimum. */
+			return (ssize_t)((size_t)1 << (SIZET_BITS - 1));
+		}
+	} else {
+		/* Result is positive, must not have overflowed, and must be
+		 * able to represent as ssize_t. */
+		if (tally->total[1] || (ssize_t)tally->total[0] < 0) {
+			/* Overflow.  Return maximum. */
+			return (ssize_t)~((size_t)1 << (SIZET_BITS - 1));
+		}
+	}
+	return tally->total[0];
+}
+
+static ssize_t bucket_range(const struct tally *tally, unsigned b, size_t *err)
+{
+	ssize_t min, max;
+
+	min = bucket_min(tally->min, tally->step_bits, b);
+	if (b == tally->buckets - 1)
+		max = tally->max;
+	else
+		max = bucket_min(tally->min, tally->step_bits, b+1) - 1;
+
+	/* FIXME: Think harder about cumulative error; is this enough?. */
+	*err = (max - min + 1) / 2;
+	/* Avoid overflow. */
+	return min + (max - min) / 2;
+}
+
+ssize_t tally_approx_median(const struct tally *tally, size_t *err)
+{
+	size_t count = tally_num(tally), total = 0;
+	unsigned int i;
+
+	for (i = 0; i < tally->buckets; i++) {
+		total += tally->counts[i];
+		if (total * 2 >= count)
+			break;
+	}
+	return bucket_range(tally, i, err);
+}
+
+ssize_t tally_approx_mode(const struct tally *tally, size_t *err)
+{
+	unsigned int i, min_best = 0, max_best = 0;
+
+	for (i = 0; i < tally->buckets; i++) {
+		if (tally->counts[i] > tally->counts[min_best]) {
+			min_best = max_best = i;
+		} else if (tally->counts[i] == tally->counts[min_best]) {
+			max_best = i;
+		}
+	}
+
+	/* We can have more than one best, making our error huge. */
+	if (min_best != max_best) {
+		ssize_t min, max;
+		min = bucket_range(tally, min_best, err);
+		max = bucket_range(tally, max_best, err);
+		max += *err;
+		*err += (size_t)(max - min);
+		return min + (max - min) / 2;
+	}
+
+	return bucket_range(tally, min_best, err);
+}
+
+static unsigned get_max_bucket(const struct tally *tally)
+{
+	unsigned int i;
+
+	for (i = tally->buckets; i > 0; i--)
+		if (tally->counts[i-1])
+			break;
+	return i;
+}
+
+char *tally_histogram(const struct tally *tally,
+		      unsigned width, unsigned height)
+{
+	unsigned int i, count, max_bucket, largest_bucket;
+	struct tally *tmp;
+	char *graph, *p;
+
+	assert(width >= TALLY_MIN_HISTO_WIDTH);
+	assert(height >= TALLY_MIN_HISTO_HEIGHT);
+
+	/* Ignore unused buckets. */
+	max_bucket = get_max_bucket(tally);
+
+	/* FIXME: It'd be nice to smooth here... */
+	if (height >= max_bucket) {
+		height = max_bucket;
+		tmp = NULL;
+	} else {
+		/* We create a temporary then renormalize so < height. */
+		/* FIXME: Antialias properly! */
+		tmp = tally_new(tally->buckets);
+		if (!tmp)
+			return NULL;
+		tmp->min = tally->min;
+		tmp->max = tally->max;
+		tmp->step_bits = tally->step_bits;
+		memcpy(tmp->counts, tally->counts,
+		       sizeof(tally->counts[0]) * tmp->buckets);
+		while ((max_bucket = get_max_bucket(tmp)) >= height)
+			renormalize(tmp, tmp->min, tmp->max * 2);
+		/* Restore max */
+		tmp->max = tally->max;
+		tally = tmp;
+		height = max_bucket;
+	}
+
+	/* Figure out longest line, for scale. */
+	largest_bucket = 0;
+	for (i = 0; i < tally->buckets; i++) {
+		if (tally->counts[i] > largest_bucket)
+			largest_bucket = tally->counts[i];
+	}
+
+	p = graph = malloc(height * (width + 1) + 1);
+	if (!graph) {
+		free(tmp);
+		return NULL;
+	}
+
+	for (i = 0; i < height; i++) {
+		unsigned covered = 1, row;
+
+		/* People expect minimum at the bottom. */
+		row = height - i - 1;
+		count = (double)tally->counts[row] / largest_bucket * (width-1)+1;
+
+		if (row == 0)
+			covered = snprintf(p, width, "%zi", tally->min);
+		else if (row == height - 1)
+			covered = snprintf(p, width, "%zi", tally->max);
+		else if (row == bucket_of(tally->min, tally->step_bits, 0))
+			*p = '+';
+		else
+			*p = '|';
+
+		if (covered > width)
+			covered = width;
+		p += covered;
+
+		if (count > covered)
+			count -= covered;
+		else
+			count = 0;
+
+		memset(p, '*', count);
+		p += count;
+		*p = '\n';
+		p++;
+	}
+	*p = '\0';
+	free(tmp);
+	return graph;
+}
diff --git a/lib/ccan/tally/tally.h b/lib/ccan/tally/tally.h
new file mode 100644
index 00000000000..650e2656cd8
--- /dev/null
+++ b/lib/ccan/tally/tally.h
@@ -0,0 +1,104 @@
+#ifndef CCAN_TALLY_H
+#define CCAN_TALLY_H
+#include "config.h"
+#include <sys/types.h>
+
+struct tally;
+
+/**
+ * tally_new - allocate the tally structure.
+ * @buckets: the number of frequency buckets.
+ *
+ * This allocates a tally structure using malloc().  The greater the value
+ * of @buckets, the more accurate tally_approx_median() and tally_approx_mode()
+ * and tally_histogram() will be, but more memory is consumed.  If you want
+ * to use tally_histogram(), the optimal bucket value is the same as that
+ * @height argument.
+ */
+struct tally *tally_new(unsigned int buckets);
+
+/**
+ * tally_add - add a value.
+ * @tally: the tally structure.
+ * @val: the value to add.
+ */
+void tally_add(struct tally *tally, ssize_t val);
+
+/**
+ * tally_num - how many times as tally_add been called?
+ * @tally: the tally structure.
+ */
+size_t tally_num(const struct tally *tally);
+
+/**
+ * tally_min - the minimum value passed to tally_add.
+ * @tally: the tally structure.
+ *
+ * Undefined if tally_num() == 0.
+ */
+ssize_t tally_min(const struct tally *tally);
+
+/**
+ * tally_max - the maximum value passed to tally_add.
+ * @tally: the tally structure.
+ *
+ * Undefined if tally_num() == 0.
+ */
+ssize_t tally_max(const struct tally *tally);
+
+/**
+ * tally_mean - the mean value passed to tally_add.
+ * @tally: the tally structure.
+ *
+ * Undefined if tally_num() == 0, but will not crash.
+ */
+ssize_t tally_mean(const struct tally *tally);
+
+/**
+ * tally_total - the total value passed to tally_add.
+ * @tally: the tally structure.
+ * @overflow: the overflow value (or NULL).
+ *
+ * If your total can't overflow a ssize_t, you don't need @overflow.
+ * Otherwise, @overflow is the upper ssize_t, and the return value should
+ * be treated as the lower size_t (ie. the sign bit is in @overflow).
+ */
+ssize_t tally_total(const struct tally *tally, ssize_t *overflow);
+
+/**
+ * tally_approx_median - the approximate median value passed to tally_add.
+ * @tally: the tally structure.
+ * @err: the error in the returned value (ie. real median is +/- @err).
+ *
+ * Undefined if tally_num() == 0, but will not crash.  Because we
+ * don't reallocate, we don't store all values, so this median cannot be
+ * exact.
+ */
+ssize_t tally_approx_median(const struct tally *tally, size_t *err);
+
+/**
+ * tally_approx_mode - the approximate mode value passed to tally_add.
+ * @tally: the tally structure.
+ * @err: the error in the returned value (ie. real mode is +/- @err).
+ *
+ * Undefined if tally_num() == 0, but will not crash.  Because we
+ * don't reallocate, we don't store all values, so this mode cannot be
+ * exact.  It could well be a value which was never passed to tally_add!
+ */
+ssize_t tally_approx_mode(const struct tally *tally, size_t *err);
+
+#define TALLY_MIN_HISTO_WIDTH 8
+#define TALLY_MIN_HISTO_HEIGHT 3
+
+/**
+ * tally_graph - return an ASCII image of the tally_add distribution
+ * @tally: the tally structure.
+ * @width: the maximum string width to use (>= TALLY_MIN_HISTO_WIDTH)
+ * @height: the maximum string height to use (>= TALLY_MIN_HISTO_HEIGHT)
+ *
+ * Returns a malloc()ed string which draws a multi-line graph of the
+ * distribution of values.  On out of memory returns NULL.
+ */
+char *tally_histogram(const struct tally *tally,
+		      unsigned width, unsigned height);
+#endif /* CCAN_TALLY_H */
diff --git a/lib/ccan/tally/test/run-bucket_of.c b/lib/ccan/tally/test/run-bucket_of.c
new file mode 100644
index 00000000000..5e12725757f
--- /dev/null
+++ b/lib/ccan/tally/test/run-bucket_of.c
@@ -0,0 +1,71 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	unsigned int i, max_step;
+	ssize_t min, max;
+
+	max = (ssize_t)~(1ULL << (sizeof(max)*CHAR_BIT - 1));
+	min = (ssize_t)(1ULL << (sizeof(max)*CHAR_BIT - 1));
+	max_step = sizeof(max)*CHAR_BIT;
+
+	plan_tests(2 + 100 + 10 + 5
+		   + 2 + 100 + 5 + 4
+		   + (1 << 7) * (max_step - 7));
+
+	/* Single step, single bucket == easy. */
+	ok1(bucket_of(0, 0, 0) == 0);
+
+	/* Double step, still in first bucket. */
+	ok1(bucket_of(0, 1, 0) == 0);
+
+	/* Step 8. */
+	for (i = 0; i < 100; i++)
+		ok1(bucket_of(0, 3, i) == i >> 3);
+
+	/* 10 values in 5 buckets, step 2. */
+	for (i = 0; i < 10; i++)
+		ok1(bucket_of(0, 1, i) == i >> 1);
+
+	/* Extreme cases. */
+	ok1(bucket_of(min, 0, min) == 0);
+	ok1(bucket_of(min, max_step-1, min) == 0);
+	ok1(bucket_of(min, max_step-1, max) == 1);
+	ok1(bucket_of(min, max_step, min) == 0);
+	ok1(bucket_of(min, max_step, max) == 0);
+
+	/* Now, bucket_min() should match: */
+	ok1(bucket_min(0, 0, 0) == 0);
+
+	/* Double step, val in first bucket still 0. */
+	ok1(bucket_min(0, 1, 0) == 0);
+
+	/* Step 8. */
+	for (i = 0; i < 100; i++)
+		ok1(bucket_min(0, 3, i) == i << 3);
+
+	/* 10 values in 5 buckets, step 2. */
+	for (i = 0; i < 5; i++)
+		ok1(bucket_min(0, 1, i) == i << 1);
+
+	/* Extreme cases. */
+	ok1(bucket_min(min, 0, 0) == min);
+	ok1(bucket_min(min, max_step-1, 0) == min);
+	ok1(bucket_min(min, max_step-1, 1) == 0);
+	ok1(bucket_min(min, max_step, 0) == min);
+
+	/* Now, vary step and number of buckets, but bucket_min and bucket_of
+	 * must agree. */
+	for (i = 0; i < (1 << 7); i++) {
+		unsigned int j;
+		for (j = 0; j < max_step - 7; j++) {
+			ssize_t val;
+
+			val = bucket_min(-(ssize_t)i, j, i);
+			ok1(bucket_of(-(ssize_t)i, j, val) == i);
+		}
+	}
+
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-divlu64.c b/lib/ccan/tally/test/run-divlu64.c
new file mode 100644
index 00000000000..057e47432c5
--- /dev/null
+++ b/lib/ccan/tally/test/run-divlu64.c
@@ -0,0 +1,31 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	unsigned int i, j;
+
+	plan_tests(5985);
+	/* Simple tests. */
+	for (i = 0; i < 127; i++) {
+		uint64_t u1, u0;
+		if (i < 64) {
+			u1 = 0;
+			u0 = 1ULL << i;
+			j = 0;
+		} else {
+			u1 = 1ULL << (i - 64);
+			u0 = 0;
+			j = i - 63;
+		}
+		for (; j < 63; j++) {
+			uint64_t answer;
+			if (j > i)
+				answer = 0;
+			else
+				answer = 1ULL << (i - j);
+			ok1(divlu64(u1, u0, 1ULL << j) == answer);
+		}
+	}
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-histogram.c b/lib/ccan/tally/test/run-histogram.c
new file mode 100644
index 00000000000..a9894ecd853
--- /dev/null
+++ b/lib/ccan/tally/test/run-histogram.c
@@ -0,0 +1,108 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	int i;
+	struct tally *tally;
+	char *graph, *p;
+
+	plan_tests(100 + 1 + 10 + 1 + 100 + 1 + 10 + 1 + 10 * 2 + 1);
+
+	/* Uniform distribution, easy. */
+	tally = tally_new(100);
+	for (i = 0; i < 100; i++)
+		tally_add(tally, i);
+
+	/* 1:1 height. */
+	graph = p = tally_histogram(tally, 20, 100);
+	for (i = 0; i < 100; i++) {
+		char *eol = strchr(p, '\n');
+
+		/* We expect it filled all way to the end. */
+		ok1(eol - p == 20);
+		p = eol + 1;
+	}
+	ok1(!*p);
+	free(graph);
+
+	/* Reduced height. */
+	graph = p = tally_histogram(tally, 20, 10);
+	for (i = 0; i < 10; i++) {
+		char *eol = strchr(p, '\n');
+
+		/* First once can be truncated (bucket aliasing) */
+		if (eol) {
+			ok1(eol - p == 20 || (eol - p < 20 && i == 0));
+		} else
+			/* We should, at worst, half-fill graph */
+			ok1(i > 5);
+
+		if (eol)
+			p = eol + 1;
+	}
+	ok1(!*p);
+	free(graph);
+
+	/* Enlarged height (gets capped). */
+	graph = p = tally_histogram(tally, 20, 1000);
+	for (i = 0; i < 100; i++) {
+		char *eol = strchr(p, '\n');
+		/* We expect it filled all way to the end. */
+		ok1(eol - p == 20);
+		p = eol + 1;
+	}
+	ok1(!*p);
+	free(graph);
+	free(tally);
+
+	/* Distinctive increasing pattern. */
+	tally = tally_new(10);
+	for (i = 0; i < 10; i++) {
+		unsigned int j;
+		for (j = 0; j <= i; j++)
+			tally_add(tally, i);
+	}
+
+	graph = p = tally_histogram(tally, 10, 10);
+	for (i = 0; i < 10; i++) {
+		char *eol = strchr(p, '\n');
+		ok1(eol - p == 10 - i);
+		p = eol + 1;
+	}
+	ok1(!*p);
+	diag("Here's the pretty: %s", graph);
+	free(graph);
+	free(tally);
+
+	/* With negative values. */
+	tally = tally_new(10);
+	for (i = 0; i < 10; i++) {
+		tally_add(tally, i - 5);
+	}
+
+	graph = p = tally_histogram(tally, 10, 10);
+	for (i = 0; i < 10; i++) {
+		char *eol = strchr(p, '\n');
+
+		/* We expect it filled all way to the end. */
+		ok1(eol - p == 10);
+
+		/* Check min/max labels. */
+		if (i == 0)
+			ok1(strncmp(p, "4*", 2) == 0);
+		else if (i == 9)
+			ok1(strncmp(p, "-5*", 3) == 0);
+		else if (i == 4)
+			ok1(p[0] == '+'); /* 0 marker */
+		else
+			ok1(p[0] == '|');
+		p = eol + 1;
+	}
+	ok1(!*p);
+	diag("Here's the pretty: %s", graph);
+	free(graph);
+	free(tally);
+
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-mean.c b/lib/ccan/tally/test/run-mean.c
new file mode 100644
index 00000000000..b43dea6b286
--- /dev/null
+++ b/lib/ccan/tally/test/run-mean.c
@@ -0,0 +1,30 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	int i;
+	struct tally *tally = tally_new(0);
+	ssize_t min, max;
+
+	max = (ssize_t)~(1ULL << (sizeof(max)*CHAR_BIT - 1));
+	min = (ssize_t)(1ULL << (sizeof(max)*CHAR_BIT - 1));
+
+	plan_tests(100 + 100);
+	/* Simple mean test: should always be 0. */
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, i);
+		tally_add(tally, -i);
+		ok1(tally_mean(tally) == 0);
+	}
+
+	/* Works for big values too... */
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, max - i);
+		tally_add(tally, min + 1 + i);
+		ok1(tally_mean(tally) == 0);
+	}
+
+	free(tally);
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-median.c b/lib/ccan/tally/test/run-median.c
new file mode 100644
index 00000000000..b12fd8a021a
--- /dev/null
+++ b/lib/ccan/tally/test/run-median.c
@@ -0,0 +1,46 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	int i;
+	struct tally *tally = tally_new(100);
+	ssize_t min, max, median;
+	size_t err;
+
+	max = (ssize_t)~(1ULL << (sizeof(max)*CHAR_BIT - 1));
+	min = (ssize_t)(1ULL << (sizeof(max)*CHAR_BIT - 1));
+
+	plan_tests(100*2 + 100*2 + 100*2);
+	/* Simple median test: should always be around 0. */
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, i);
+		tally_add(tally, -i);
+		median = tally_approx_median(tally, &err);
+		ok1(err <= 4);
+		ok1(median - (ssize_t)err <= 0 && median + (ssize_t)err >= 0);
+	}
+
+	/* Works for big values too... */
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, max - i);
+		tally_add(tally, min + 1 + i);
+		median = tally_approx_median(tally, &err);
+		/* Error should be < 100th of max - min. */
+		ok1(err <= max / 100 * 2);
+		ok1(median - (ssize_t)err <= 0 && median + (ssize_t)err >= 0);
+	}
+	free(tally);
+
+	tally = tally_new(10);
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, i);
+		median = tally_approx_median(tally, &err);
+		ok1(err <= i / 10 + 1);
+		ok1(median - (ssize_t)err <= i/2
+		    && median + (ssize_t)err >= i/2);
+	}
+	free(tally);
+
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-min-max.c b/lib/ccan/tally/test/run-min-max.c
new file mode 100644
index 00000000000..c92f6d382a4
--- /dev/null
+++ b/lib/ccan/tally/test/run-min-max.c
@@ -0,0 +1,21 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	int i;
+	struct tally *tally = tally_new(0);
+
+	plan_tests(100 * 4);
+	/* Test max, min and num. */
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, i);
+		ok1(tally_num(tally) == i*2 + 1);
+		tally_add(tally, -i);
+		ok1(tally_num(tally) == i*2 + 2);
+		ok1(tally_max(tally) == i);
+		ok1(tally_min(tally) == -i);
+	}
+	free(tally);
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-mode.c b/lib/ccan/tally/test/run-mode.c
new file mode 100644
index 00000000000..cd2f2304436
--- /dev/null
+++ b/lib/ccan/tally/test/run-mode.c
@@ -0,0 +1,46 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	int i;
+	struct tally *tally = tally_new(100);
+	ssize_t min, max, mode;
+	size_t err;
+
+	max = (ssize_t)~(1ULL << (sizeof(max)*CHAR_BIT - 1));
+	min = (ssize_t)(1ULL << (sizeof(max)*CHAR_BIT - 1));
+
+	plan_tests(100 + 50 + 100 + 100 + 10);
+	/* Simple mode test: should always be around 0 (we add that twice). */
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, i);
+		tally_add(tally, -i);
+		mode = tally_approx_mode(tally, &err);
+		if (i < 50)
+			ok1(err == 0);
+		ok1(mode - (ssize_t)err <= 0 && mode + (ssize_t)err >= 0);
+	}
+
+	/* Works for big values too... */
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, max - i);
+		tally_add(tally, min + 1 + i);
+		mode = tally_approx_mode(tally, &err);
+		ok1(mode - (ssize_t)err <= 0 && mode + (ssize_t)err >= 0);
+	}
+	free(tally);
+
+	tally = tally_new(10);
+	tally_add(tally, 0);
+	for (i = 0; i < 100; i++) {
+		tally_add(tally, i);
+		mode = tally_approx_mode(tally, &err);
+		if (i < 10)
+			ok1(err == 0);
+		ok1(mode - (ssize_t)err <= 0 && mode + (ssize_t)err >= 0);
+	}
+
+	free(tally);
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-renormalize.c b/lib/ccan/tally/test/run-renormalize.c
new file mode 100644
index 00000000000..8fe9dbce329
--- /dev/null
+++ b/lib/ccan/tally/test/run-renormalize.c
@@ -0,0 +1,26 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	struct tally *tally = tally_new(2);
+
+	plan_tests(4);
+	tally->min = 0;
+	tally->max = 0;
+	tally->counts[0] = 1;
+
+	/* This renormalize should do nothing. */
+	renormalize(tally, 0, 1);
+	ok1(tally->counts[0] == 1);
+	ok1(tally->counts[1] == 0);
+	tally->counts[1]++;
+
+	/* This renormalize should collapse both into bucket 0. */
+	renormalize(tally, 0, 3);
+	ok1(tally->counts[0] == 2);
+	ok1(tally->counts[1] == 0);
+
+	free(tally);
+	return exit_status();
+}
diff --git a/lib/ccan/tally/test/run-total.c b/lib/ccan/tally/test/run-total.c
new file mode 100644
index 00000000000..d7d73e58a5d
--- /dev/null
+++ b/lib/ccan/tally/test/run-total.c
@@ -0,0 +1,56 @@
+#include <ccan/tally/tally.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	struct tally *tally;
+	ssize_t total, overflow;
+	ssize_t min, max;
+
+	max = (ssize_t)~(1ULL << (sizeof(max)*CHAR_BIT - 1));
+	min = (ssize_t)(1ULL << (sizeof(max)*CHAR_BIT - 1));
+
+	plan_tests(15);
+
+	/* Simple case. */
+	tally = tally_new(0);
+	tally_add(tally, min);
+	ok1(tally_total(tally, NULL) == min);
+	ok1(tally_total(tally, &overflow) == min);
+	ok1(overflow == -1);
+
+	/* Underflow. */
+	tally_add(tally, min);
+	total = tally_total(tally, &overflow);
+	ok1(overflow == -1);
+	ok1((size_t)total == 0);
+	ok1(tally_total(tally, NULL) == min);
+	free(tally);
+
+	/* Simple case. */
+	tally = tally_new(0);
+	tally_add(tally, max);
+	ok1(tally_total(tally, NULL) == max);
+	ok1(tally_total(tally, &overflow) == max);
+	ok1(overflow == 0);
+
+	/* Overflow into sign bit... */
+	tally_add(tally, max);
+	total = tally_total(tally, &overflow);
+	ok1(overflow == 0);
+	ok1((size_t)total == (size_t)-2);
+	ok1(tally_total(tally, NULL) == max);
+
+	/* Overflow into upper size_t. */
+	tally_add(tally, max);
+	total = tally_total(tally, &overflow);
+	ok1(overflow == 1);
+	if (sizeof(size_t) == 4)
+		ok1((size_t)total == 0x7FFFFFFD);
+	else if (sizeof(size_t) == 8)
+		ok1((size_t)total == 0x7FFFFFFFFFFFFFFDULL);
+	ok1(tally_total(tally, NULL) == max);
+	free(tally);
+
+	return exit_status();
+}
diff --git a/lib/ccan/typesafe_cb/LICENSE b/lib/ccan/typesafe_cb/LICENSE
new file mode 100644
index 00000000000..5522aa5f33e
--- /dev/null
+++ b/lib/ccan/typesafe_cb/LICENSE
@@ -0,0 +1,508 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/lib/ccan/typesafe_cb/_info b/lib/ccan/typesafe_cb/_info
new file mode 100644
index 00000000000..4f4570afc9b
--- /dev/null
+++ b/lib/ccan/typesafe_cb/_info
@@ -0,0 +1,151 @@
+#include <stdio.h>
+#include <string.h>
+#include "config.h"
+
+/**
+ * typesafe_cb - macros for safe callbacks.
+ *
+ * The basis of the typesafe_cb header is typesafe_cb_cast(): a
+ * conditional cast macro.   If an expression exactly matches a given
+ * type, it is cast to the target type, otherwise it is left alone.
+ *
+ * This allows us to create functions which take a small number of
+ * specific types, rather than being forced to use a void *.  In
+ * particular, it is useful for creating typesafe callbacks as the
+ * helpers typesafe_cb(), typesafe_cb_preargs() and
+ * typesafe_cb_postargs() demonstrate.
+ *
+ * The standard way of passing arguments to callback functions in C is
+ * to use a void pointer, which the callback then casts back to the
+ * expected type.  This unfortunately subverts the type checking the
+ * compiler would perform if it were a direct call.  Here's an example:
+ *
+ *	static void my_callback(void *_obj)
+ *	{
+ *		struct obj *obj = _obj;
+ *		...
+ *	}
+ *	...
+ *		register_callback(my_callback, &my_obj);
+ *
+ * If we wanted to use the natural type for my_callback (ie. "void
+ * my_callback(struct obj *obj)"), we could make register_callback()
+ * take a void * as its first argument, but this would subvert all
+ * type checking.  We really want register_callback() to accept only
+ * the exactly correct function type to match the argument, or a
+ * function which takes a void *.
+ *
+ * This is where typesafe_cb() comes in: it uses typesafe_cb_cast() to
+ * cast the callback function if it matches the argument type:
+ *
+ *	void _register_callback(void (*cb)(void *arg), void *arg);
+ *	#define register_callback(cb, arg)				\
+ *		_register_callback(typesafe_cb(void, void *, (cb), (arg)), \
+ *				   (arg))
+ *
+ * On compilers which don't support the extensions required
+ * typesafe_cb_cast() and friend become an unconditional cast, so your
+ * code will compile but you won't get type checking.
+ *
+ * Example:
+ *	#include <ccan/typesafe_cb/typesafe_cb.h>
+ *	#include <stdlib.h>
+ *	#include <stdio.h>
+ *
+ *	// Generic callback infrastructure.
+ *	struct callback {
+ *		struct callback *next;
+ *		int value;
+ *		int (*callback)(int value, void *arg);
+ *		void *arg;
+ *	};
+ *	static struct callback *callbacks;
+ *
+ *	static void _register_callback(int value, int (*cb)(int, void *),
+ *				       void *arg)
+ *	{
+ *		struct callback *new = malloc(sizeof(*new));
+ *		new->next = callbacks;
+ *		new->value = value;
+ *		new->callback = cb;
+ *		new->arg = arg;
+ *		callbacks = new;
+ *	}
+ *	#define register_callback(value, cb, arg)			\
+ *		_register_callback(value,				\
+ *				   typesafe_cb_preargs(int, void *,	\
+ *						       (cb), (arg), int),\
+ *				   (arg))
+ *
+ *	static struct callback *find_callback(int value)
+ *	{
+ *		struct callback *i;
+ *
+ *		for (i = callbacks; i; i = i->next)
+ *			if (i->value == value)
+ *				return i;
+ *		return NULL;
+ *	}
+ *
+ *	// Define several silly callbacks.  Note they don't use void *!
+ *	#define DEF_CALLBACK(name, op)			\
+ *		static int name(int val, int *arg)	\
+ *		{					\
+ *			printf("%s", #op);		\
+ *			return val op *arg;		\
+ *		}
+ *	DEF_CALLBACK(multiply, *);
+ *	DEF_CALLBACK(add, +);
+ *	DEF_CALLBACK(divide, /);
+ *	DEF_CALLBACK(sub, -);
+ *	DEF_CALLBACK(or, |);
+ *	DEF_CALLBACK(and, &);
+ *	DEF_CALLBACK(xor, ^);
+ *	DEF_CALLBACK(assign, =);
+ *
+ *	// Silly game to find the longest chain of values.
+ *	int main(int argc, char *argv[])
+ *	{
+ *		int i, run = 1, num = argv[1] ? atoi(argv[1]) : 0;
+ *
+ *		for (i = 1; i < 1024;) {
+ *			// Since run is an int, compiler checks "add" does too.
+ *			register_callback(i++, add, &run);
+ *			register_callback(i++, divide, &run);
+ *			register_callback(i++, sub, &run);
+ *			register_callback(i++, multiply, &run);
+ *			register_callback(i++, or, &run);
+ *			register_callback(i++, and, &run);
+ *			register_callback(i++, xor, &run);
+ *			register_callback(i++, assign, &run);
+ *		}
+ *
+ *		printf("%i ", num);
+ *		while (run < 56) {
+ *			struct callback *cb = find_callback(num % i);
+ *			if (!cb) {
+ *				printf("-> STOP\n");
+ *				return 1;
+ *			}
+ *			num = cb->callback(num, cb->arg);
+ *			printf("->%i ", num);
+ *			run++;
+ *		}
+ *		printf("-> Winner!\n");
+ *		return 0;
+ *	}
+ *
+ * License: LGPL (2 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-cast_if_any.c b/lib/ccan/typesafe_cb/test/compile_fail-cast_if_any.c
new file mode 100644
index 00000000000..dfb51167ffb
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-cast_if_any.c
@@ -0,0 +1,42 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+struct foo {
+	int x;
+};
+
+struct bar {
+	int x;
+};
+
+struct baz {
+	int x;
+};
+
+struct any {
+	int x;
+};
+
+struct other {
+	int x;
+};
+
+static void take_any(struct any *any)
+{
+}
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	struct other
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_if_type is a noop."
+#endif
+#else
+	struct foo
+#endif
+		*arg = NULL;
+	take_any(cast_if_any(struct any *, arg, arg,
+			     struct foo *, struct bar *, struct baz *));
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-cast_if_type-promotable.c b/lib/ccan/typesafe_cb/test/compile_fail-cast_if_type-promotable.c
new file mode 100644
index 00000000000..11d42f4c6b6
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-cast_if_type-promotable.c
@@ -0,0 +1,23 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdbool.h>
+
+static void _set_some_value(void *val)
+{
+}
+
+#define set_some_value(expr)						\
+	_set_some_value(typesafe_cb_cast(void *, long, (expr)))
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	bool x = 0;
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if typesafe_cb_cast is a noop."
+#endif
+#else
+	long x = 0;
+#endif
+	set_some_value(x);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-cast_if_type.c b/lib/ccan/typesafe_cb/test/compile_fail-cast_if_type.c
new file mode 100644
index 00000000000..610793514f9
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-cast_if_type.c
@@ -0,0 +1,25 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+
+void _set_some_value(void *val);
+
+void _set_some_value(void *val)
+{
+}
+
+#define set_some_value(expr)						\
+	_set_some_value(cast_if_type(void *, (expr), (expr), unsigned long))
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	int x = 0;
+	set_some_value(x);
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_if_type is a noop."
+#endif
+#else
+	void *p = 0;
+	set_some_value(p);
+#endif
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb-int.c b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb-int.c
new file mode 100644
index 00000000000..c4033364d41
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb-int.c
@@ -0,0 +1,27 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+void _callback(void (*fn)(void *arg), void *arg);
+void _callback(void (*fn)(void *arg), void *arg)
+{
+	fn(arg);
+}
+
+/* Callback is set up to warn if arg isn't a pointer (since it won't
+ * pass cleanly to _callback's second arg. */
+#define callback(fn, arg)						\
+	_callback(typesafe_cb(void, (fn), (arg)), (arg))
+
+void my_callback(int something);
+void my_callback(int something)
+{
+}
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	/* This fails due to arg, not due to cast. */
+	callback(my_callback, 100);
+#endif
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb.c b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb.c
new file mode 100644
index 00000000000..81e36d7b87b
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb.c
@@ -0,0 +1,34 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+static void _register_callback(void (*cb)(void *arg), void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb(void, void *, (cb), (arg)), (arg))
+
+static void my_callback(char *p)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	char str[] = "hello world";
+#ifdef FAIL
+	int *p;
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if typesafe_cb_cast is a noop."
+#endif
+#else
+	char *p;
+#endif
+	p = NULL;
+
+	/* This should work always. */
+	register_callback(my_callback, str);
+
+	/* This will fail with FAIL defined */
+	register_callback(my_callback, p);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast-multi.c b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast-multi.c
new file mode 100644
index 00000000000..62b5f91e180
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast-multi.c
@@ -0,0 +1,43 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+struct foo {
+	int x;
+};
+
+struct bar {
+	int x;
+};
+
+struct baz {
+	int x;
+};
+
+struct any {
+	int x;
+};
+
+struct other {
+	int x;
+};
+
+static void take_any(struct any *any)
+{
+}
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	struct other
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if typesafe_cb_cast is a noop."
+#endif
+#else
+	struct foo
+#endif
+		*arg = NULL;
+	take_any(typesafe_cb_cast3(struct any *,
+				   struct foo *, struct bar *, struct baz *,
+				   arg));
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast.c b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast.c
new file mode 100644
index 00000000000..d2e6f2ab40a
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast.c
@@ -0,0 +1,25 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+
+void _set_some_value(void *val);
+
+void _set_some_value(void *val)
+{
+}
+
+#define set_some_value(expr)						\
+	_set_some_value(typesafe_cb_cast(void *, unsigned long, (expr)))
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	int x = 0;
+	set_some_value(x);
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if typesafe_cb_cast is a noop."
+#endif
+#else
+	void *p = 0;
+	set_some_value(p);
+#endif
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_exact.c b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_exact.c
new file mode 100644
index 00000000000..0f61d5decd5
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_exact.c
@@ -0,0 +1,33 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+static void _register_callback(void (*cb)(void *arg), const void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb_exact(void, (cb), (arg)), (arg))
+
+static void my_callback(const char *p)
+{
+}
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	char *p;
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if cast_if_type is a noop."
+#endif
+#else
+	const char *p;
+#endif
+	p = NULL;
+
+	/* This should work always. */
+	register_callback(my_callback, (const char *)"hello world");
+
+	/* This will fail with FAIL defined */
+	register_callback(my_callback, p);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_postargs.c b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_postargs.c
new file mode 100644
index 00000000000..7d3530851d5
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_postargs.c
@@ -0,0 +1,27 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+static void _register_callback(void (*cb)(void *arg, int x), void *arg)
+{
+}
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb_postargs(void, void *, (cb), (arg), int), (arg))
+
+static void my_callback(char *p, int x)
+{
+}
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	int *p;
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if typesafe_cb_cast is a noop."
+#endif
+#else
+	char *p;
+#endif
+	p = NULL;
+	register_callback(my_callback, p);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_preargs.c b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_preargs.c
new file mode 100644
index 00000000000..bd55c6722c7
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_fail-typesafe_cb_preargs.c
@@ -0,0 +1,28 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+static void _register_callback(void (*cb)(int x, void *arg), void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb_preargs(void, void *, (cb), (arg), int), (arg))
+
+static void my_callback(int x, char *p)
+{
+}
+
+int main(int argc, char *argv[])
+{
+#ifdef FAIL
+	int *p;
+#if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if typesafe_cb_cast is a noop."
+#endif
+#else
+	char *p;
+#endif
+	p = NULL;
+	register_callback(my_callback, p);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-cast_if_any.c b/lib/ccan/typesafe_cb/test/compile_ok-cast_if_any.c
new file mode 100644
index 00000000000..e8f3c49406c
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-cast_if_any.c
@@ -0,0 +1,41 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+struct foo {
+	int x;
+};
+
+struct bar {
+	int x;
+};
+
+struct baz {
+	int x;
+};
+
+struct any {
+	int x;
+};
+
+static void take_any(struct any *any)
+{
+}
+
+int main(int argc, char *argv[])
+{
+#if HAVE_TYPEOF
+	/* Otherwise we get unused warnings for these. */
+	struct foo *foo = NULL;
+	struct bar *bar = NULL;
+	struct baz *baz = NULL;
+#endif
+	struct other *arg = NULL;
+
+	take_any(cast_if_any(struct any *, arg, foo,
+			     struct foo *, struct bar *, struct baz *));
+	take_any(cast_if_any(struct any *, arg, bar,
+			     struct foo *, struct bar *, struct baz *));
+	take_any(cast_if_any(struct any *, arg, baz,
+			     struct foo *, struct bar *, struct baz *));
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-NULL.c b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-NULL.c
new file mode 100644
index 00000000000..265de8b14ee
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-NULL.c
@@ -0,0 +1,17 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+/* NULL args for callback function should be OK for normal and _def. */
+
+static void _register_callback(void (*cb)(const void *arg), const void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb(void, const void *, (cb), (arg)), (arg))
+
+int main(int argc, char *argv[])
+{
+	register_callback(NULL, "hello world");
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-const.c b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-const.c
new file mode 100644
index 00000000000..7c2d62ef231
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-const.c
@@ -0,0 +1,50 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+/* const args in callbacks should be OK. */
+
+static void _register_callback(void (*cb)(void *arg), void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb(void, (cb), (arg)), (arg))
+
+#define register_callback_def(cb, arg)				\
+	_register_callback(typesafe_cb_def(void, (cb), (arg)), (arg))
+
+static void _register_callback_pre(void (*cb)(int x, void *arg), void *arg)
+{
+}
+
+#define register_callback_pre(cb, arg)					\
+	_register_callback_pre(typesafe_cb_preargs(void, (cb), (arg), int), (arg))
+
+static void _register_callback_post(void (*cb)(void *arg, int x), void *arg)
+{
+}
+
+#define register_callback_post(cb, arg)					\
+	_register_callback_post(typesafe_cb_postargs(void, (cb), (arg), int), (arg))
+
+static void my_callback(const char *p)
+{
+}
+
+static void my_callback_pre(int x, /*const*/ char *p)
+{
+}
+
+static void my_callback_post(/*const*/ char *p, int x)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	char p[] = "hello world";
+	register_callback(my_callback, p);
+	register_callback_def(my_callback, p);
+	register_callback_pre(my_callback_pre, p);
+	register_callback_post(my_callback_post, p);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-undefined.c b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-undefined.c
new file mode 100644
index 00000000000..aa50bad6a92
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-undefined.c
@@ -0,0 +1,49 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+/* const args in callbacks should be OK. */
+
+static void _register_callback(void (*cb)(void *arg), void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb(void, void *, (cb), (arg)), (arg))
+
+static void _register_callback_pre(void (*cb)(int x, void *arg), void *arg)
+{
+}
+
+#define register_callback_pre(cb, arg)					\
+	_register_callback_pre(typesafe_cb_preargs(void, void *, (cb), (arg), int), (arg))
+
+static void _register_callback_post(void (*cb)(void *arg, int x), void *arg)
+{
+}
+
+#define register_callback_post(cb, arg)					\
+	_register_callback_post(typesafe_cb_postargs(void, void *, (cb), (arg), int), (arg))
+
+struct undefined;
+
+static void my_callback(struct undefined *undef)
+{
+}
+
+static void my_callback_pre(int x, struct undefined *undef)
+{
+}
+
+static void my_callback_post(struct undefined *undef, int x)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	struct undefined *handle = NULL;
+
+	register_callback(my_callback, handle);
+	register_callback_pre(my_callback_pre, handle);
+	register_callback_post(my_callback_post, handle);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-vars.c b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-vars.c
new file mode 100644
index 00000000000..f6a2bfecbc2
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-vars.c
@@ -0,0 +1,52 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+/* const args in callbacks should be OK. */
+
+static void _register_callback(void (*cb)(void *arg), void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb(void, void *, (cb), (arg)), (arg))
+
+static void _register_callback_pre(void (*cb)(int x, void *arg), void *arg)
+{
+}
+
+#define register_callback_pre(cb, arg)					\
+	_register_callback_pre(typesafe_cb_preargs(void, void *, (cb), (arg), int), (arg))
+
+static void _register_callback_post(void (*cb)(void *arg, int x), void *arg)
+{
+}
+
+#define register_callback_post(cb, arg)					\
+	_register_callback_post(typesafe_cb_postargs(void, void *, (cb), (arg), int), (arg))
+
+struct undefined;
+
+static void my_callback(struct undefined *undef)
+{
+}
+
+static void my_callback_pre(int x, struct undefined *undef)
+{
+}
+
+static void my_callback_post(struct undefined *undef, int x)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	struct undefined *handle = NULL;
+	void (*cb)(struct undefined *undef) = my_callback;
+	void (*pre)(int x, struct undefined *undef) = my_callback_pre;
+	void (*post)(struct undefined *undef, int x) = my_callback_post;
+
+	register_callback(cb, handle);
+	register_callback_pre(pre, handle);
+	register_callback_post(post, handle);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-volatile.c b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-volatile.c
new file mode 100644
index 00000000000..3fcb1ff656b
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb-volatile.c
@@ -0,0 +1,47 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+/* volatile args in callbacks should be OK. */
+
+static void _register_callback(void (*cb)(void *arg), void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb(void, (cb), (arg)), (arg))
+
+static void _register_callback_pre(void (*cb)(int x, void *arg), void *arg)
+{
+}
+
+#define register_callback_pre(cb, arg)					\
+	_register_callback_pre(typesafe_cb_preargs(void, (cb), (arg), int), (arg))
+
+static void _register_callback_post(void (*cb)(void *arg, int x), void *arg)
+{
+}
+
+#define register_callback_post(cb, arg)					\
+	_register_callback_post(typesafe_cb_postargs(void, (cb), (arg), int), (arg))
+
+static void my_callback(volatile char *p)
+{
+}
+
+/* FIXME: Can't handle volatile for these */
+static void my_callback_pre(int x, /* volatile */ char *p)
+{
+}
+
+static void my_callback_post(/* volatile */ char *p, int x)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	char p[] = "hello world";
+	register_callback(my_callback, p);
+	register_callback_pre(my_callback_pre, p);
+	register_callback_post(my_callback_post, p);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb_cast.c b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb_cast.c
new file mode 100644
index 00000000000..b7f21dc0946
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb_cast.c
@@ -0,0 +1,41 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+struct foo {
+	int x;
+};
+
+struct bar {
+	int x;
+};
+
+struct baz {
+	int x;
+};
+
+struct any {
+	int x;
+};
+
+static void take_any(struct any *any)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	/* Otherwise we get unused warnings for these. */
+	struct foo *foo = NULL;
+	struct bar *bar = NULL;
+	struct baz *baz = NULL;
+
+	take_any(typesafe_cb_cast3(struct any *,
+				   struct foo *, struct bar *, struct baz *,
+				   foo));
+	take_any(typesafe_cb_cast3(struct any *,
+				   struct foo *, struct bar *, struct baz *,
+				   bar));
+	take_any(typesafe_cb_cast3(struct any *,
+				   struct foo *, struct bar *, struct baz *,
+				   baz));
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb_def-const.c b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb_def-const.c
new file mode 100644
index 00000000000..01e090f1dce
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/compile_ok-typesafe_cb_def-const.c
@@ -0,0 +1,46 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdlib.h>
+
+/* const args in callbacks should be OK. */
+
+static void _register_callback(void (*cb)(void *arg), void *arg)
+{
+}
+
+#define register_callback(cb, arg)				\
+	_register_callback(typesafe_cb(void, (cb), (arg)), (arg))
+
+static void _register_callback_pre(void (*cb)(int x, void *arg), void *arg)
+{
+}
+
+#define register_callback_pre(cb, arg)					\
+	_register_callback_pre(typesafe_cb_preargs(void, (cb), (arg), int), (arg))
+
+static void _register_callback_post(void (*cb)(void *arg, int x), void *arg)
+{
+}
+
+#define register_callback_post(cb, arg)					\
+	_register_callback_post(typesafe_cb_postargs(void, (cb), (arg), int), (arg))
+
+static void my_callback(const char *p)
+{
+}
+
+static void my_callback_pre(int x, /*const*/ char *p)
+{
+}
+
+static void my_callback_post(/*const*/ char *p, int x)
+{
+}
+
+int main(int argc, char *argv[])
+{
+	char p[] = "hello world";
+	register_callback(my_callback, p);
+	register_callback_pre(my_callback_pre, p);
+	register_callback_post(my_callback_post, p);
+	return 0;
+}
diff --git a/lib/ccan/typesafe_cb/test/run.c b/lib/ccan/typesafe_cb/test/run.c
new file mode 100644
index 00000000000..116e7d19465
--- /dev/null
+++ b/lib/ccan/typesafe_cb/test/run.c
@@ -0,0 +1,109 @@
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <string.h>
+#include <stdint.h>
+#include <ccan/tap/tap.h>
+
+static char dummy = 0;
+
+/* The example usage. */
+static void _set_some_value(void *val)
+{
+	ok1(val == &dummy);
+}
+
+#define set_some_value(expr)						\
+	_set_some_value(typesafe_cb_cast(void *, unsigned long, (expr)))
+
+static void _callback_onearg(void (*fn)(void *arg), void *arg)
+{
+	fn(arg);
+}
+
+static void _callback_preargs(void (*fn)(int a, int b, void *arg), void *arg)
+{
+	fn(1, 2, arg);
+}
+
+static void _callback_postargs(void (*fn)(void *arg, int a, int b), void *arg)
+{
+	fn(arg, 1, 2);
+}
+
+#define callback_onearg(cb, arg)					\
+	_callback_onearg(typesafe_cb(void, void *, (cb), (arg)), (arg))
+
+#define callback_preargs(cb, arg)					\
+	_callback_preargs(typesafe_cb_preargs(void, void *, (cb), (arg), int, int), (arg))
+
+#define callback_postargs(cb, arg)					\
+	_callback_postargs(typesafe_cb_postargs(void, void *, (cb), (arg), int, int), (arg))
+
+static void my_callback_onearg(char *p)
+{
+	ok1(strcmp(p, "hello world") == 0);
+}
+
+static void my_callback_preargs(int a, int b, char *p)
+{
+	ok1(a == 1);
+	ok1(b == 2);
+	ok1(strcmp(p, "hello world") == 0);
+}
+
+static void my_callback_postargs(char *p, int a, int b)
+{
+	ok1(a == 1);
+	ok1(b == 2);
+	ok1(strcmp(p, "hello world") == 0);
+}
+
+/* This is simply a compile test; we promised typesafe_cb_cast can be in a
+ * static initializer. */
+struct callback_onearg
+{
+	void (*fn)(void *arg);
+	const void *arg;
+};
+
+struct callback_onearg cb_onearg
+= { typesafe_cb(void, void *, my_callback_onearg, (char *)(intptr_t)"hello world"),
+    "hello world" };
+
+struct callback_preargs
+{
+	void (*fn)(int a, int b, void *arg);
+	const void *arg;
+};
+
+struct callback_preargs cb_preargs
+= { typesafe_cb_preargs(void, void *, my_callback_preargs,
+			(char *)(intptr_t)"hi", int, int), "hi" };
+
+struct callback_postargs
+{
+	void (*fn)(void *arg, int a, int b);
+	const void *arg;
+};
+
+struct callback_postargs cb_postargs
+= { typesafe_cb_postargs(void, void *, my_callback_postargs,
+			 (char *)(intptr_t)"hi", int, int), "hi" };
+
+int main(int argc, char *argv[])
+{
+	void *p = &dummy;
+	unsigned long l = (unsigned long)p;
+	char str[] = "hello world";
+
+	plan_tests(2 + 1 + 3 + 3);
+	set_some_value(p);
+	set_some_value(l);
+
+	callback_onearg(my_callback_onearg, str);
+
+	callback_preargs(my_callback_preargs, str);
+
+	callback_postargs(my_callback_postargs, str);
+
+	return exit_status();
+}
diff --git a/lib/ccan/typesafe_cb/typesafe_cb.h b/lib/ccan/typesafe_cb/typesafe_cb.h
new file mode 100644
index 00000000000..40cfa397989
--- /dev/null
+++ b/lib/ccan/typesafe_cb/typesafe_cb.h
@@ -0,0 +1,133 @@
+#ifndef CCAN_TYPESAFE_CB_H
+#define CCAN_TYPESAFE_CB_H
+#include "config.h"
+
+#if HAVE_TYPEOF && HAVE_BUILTIN_CHOOSE_EXPR && HAVE_BUILTIN_TYPES_COMPATIBLE_P
+/**
+ * typesafe_cb_cast - only cast an expression if it matches a given type
+ * @desttype: the type to cast to
+ * @oktype: the type we allow
+ * @expr: the expression to cast
+ *
+ * This macro is used to create functions which allow multiple types.
+ * The result of this macro is used somewhere that a @desttype type is
+ * expected: if @expr is exactly of type @oktype, then it will be
+ * cast to @desttype type, otherwise left alone.
+ *
+ * This macro can be used in static initializers.
+ *
+ * This is merely useful for warnings: if the compiler does not
+ * support the primitives required for typesafe_cb_cast(), it becomes an
+ * unconditional cast, and the @oktype argument is not used.  In
+ * particular, this means that @oktype can be a type which uses the
+ * "typeof": it will not be evaluated if typeof is not supported.
+ *
+ * Example:
+ *	// We can take either an unsigned long or a void *.
+ *	void _set_some_value(void *val);
+ *	#define set_some_value(e)			\
+ *		_set_some_value(typesafe_cb_cast(void *, (e), unsigned long))
+ */
+#define typesafe_cb_cast(desttype, oktype, expr)			\
+	__builtin_choose_expr(						\
+		__builtin_types_compatible_p(__typeof__(0?(expr):(expr)), \
+					     oktype),			\
+		(desttype)(expr), (expr))
+#else
+#define typesafe_cb_cast(desttype, oktype, expr) ((desttype)(expr))
+#endif
+
+/**
+ * typesafe_cb_cast3 - only cast an expression if it matches given types
+ * @desttype: the type to cast to
+ * @ok1: the first type we allow
+ * @ok2: the second type we allow
+ * @ok3: the third type we allow
+ * @expr: the expression to cast
+ *
+ * This is a convenient wrapper for multiple typesafe_cb_cast() calls.
+ * You can chain them inside each other (ie. use typesafe_cb_cast()
+ * for expr) if you need more than 3 arguments.
+ *
+ * Example:
+ *	// We can take either a long, unsigned long, void * or a const void *.
+ *	void _set_some_value(void *val);
+ *	#define set_some_value(expr)					\
+ *		_set_some_value(typesafe_cb_cast3(void *,,		\
+ *					    long, unsigned long, const void *,\
+ *					    (expr)))
+ */
+#define typesafe_cb_cast3(desttype, ok1, ok2, ok3, expr)		\
+	typesafe_cb_cast(desttype, ok1,					\
+			 typesafe_cb_cast(desttype, ok2,		\
+					  typesafe_cb_cast(desttype, ok3, \
+							   (expr))))
+
+/**
+ * typesafe_cb - cast a callback function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @atype: the (pointer) type which the callback function expects.
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument to hand to the callback function.
+ *
+ * If a callback function takes a single argument, this macro does
+ * appropriate casts to a function which takes a single atype argument if the
+ * callback provided matches the @arg.
+ *
+ * It is assumed that @arg is of pointer type: usually @arg is passed
+ * or assigned to a void * elsewhere anyway.
+ *
+ * Example:
+ *	void _register_callback(void (*fn)(void *arg), void *arg);
+ *	#define register_callback(fn, arg) \
+ *		_register_callback(typesafe_cb(void, (fn), void*, (arg)), (arg))
+ */
+#define typesafe_cb(rtype, atype, fn, arg)			\
+	typesafe_cb_cast(rtype (*)(atype),			\
+			 rtype (*)(__typeof__(arg)),		\
+			 (fn))
+
+/**
+ * typesafe_cb_preargs - cast a callback function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @atype: the (pointer) type which the callback function expects.
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument to hand to the callback function.
+ *
+ * This is a version of typesafe_cb() for callbacks that take other arguments
+ * before the @arg.
+ *
+ * Example:
+ *	void _register_callback(void (*fn)(int, void *arg), void *arg);
+ *	#define register_callback(fn, arg)				   \
+ *		_register_callback(typesafe_cb_preargs(void, (fn), void *, \
+ *				   (arg), int),				   \
+ *				   (arg))
+ */
+#define typesafe_cb_preargs(rtype, atype, fn, arg, ...)			\
+	typesafe_cb_cast(rtype (*)(__VA_ARGS__, atype),			\
+			 rtype (*)(__VA_ARGS__, __typeof__(arg)),	\
+			 (fn))
+
+/**
+ * typesafe_cb_postargs - cast a callback function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @atype: the (pointer) type which the callback function expects.
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument to hand to the callback function.
+ *
+ * This is a version of typesafe_cb() for callbacks that take other arguments
+ * after the @arg.
+ *
+ * Example:
+ *	void _register_callback(void (*fn)(void *arg, int), void *arg);
+ *	#define register_callback(fn, arg) \
+ *		_register_callback(typesafe_cb_postargs(void, (fn), void *, \
+ *				   (arg), int),				    \
+ *				   (arg))
+ */
+#define typesafe_cb_postargs(rtype, atype, fn, arg, ...)		\
+	typesafe_cb_cast(rtype (*)(atype, __VA_ARGS__),			\
+			 rtype (*)(__typeof__(arg), __VA_ARGS__),	\
+			 (fn))
+#endif /* CCAN_CAST_IF_TYPE_H */
diff --git a/lib/ccan/wscript b/lib/ccan/wscript
new file mode 100644
index 00000000000..0543a4de075
--- /dev/null
+++ b/lib/ccan/wscript
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+
+import Logs, sys
+
+def configure(conf):
+    # FIXME: if they don't have -Werror, these will all fail.  But they
+    # probably will anyway...
+    conf.CHECK_CODE('int __attribute__((cold)) func(int x) { return x; }',
+                    addmain=False, link=False, cflags="-Werror",
+                    define='HAVE_ATTRIBUTE_COLD')
+    conf.CHECK_CODE('int __attribute__((const)) func(int x) { return x; }',
+                    addmain=False, link=False, cflags="-Werror",
+                    define='HAVE_ATTRIBUTE_CONST')
+    conf.CHECK_CODE('void __attribute__((noreturn)) func(int x) { exit(x); }',
+                    addmain=False, link=False, cflags="-Werror",
+                    define='HAVE_ATTRIBUTE_NORETURN')
+    conf.CHECK_CODE('void __attribute__((format(__printf__, 1, 2))) func(const char *fmt, ...) { }',
+                    addmain=False, link=False, cflags="-Werror",
+                    define='HAVE_ATTRIBUTE_PRINTF')
+    conf.CHECK_CODE('int __attribute__((unused)) func(int x) { return x; }',
+                    addmain=False, link=False, cflags="-Werror",
+                    define='HAVE_ATTRIBUTE_UNUSED')
+    conf.CHECK_CODE('int __attribute__((used)) func(int x) { return x; }',
+                    addmain=False, link=False, cflags="-Werror",
+                    define='HAVE_ATTRIBUTE_USED')
+    # We try to use headers for a compile-time test.
+    conf.CHECK_CODE(code = """#ifdef __BYTE_ORDER
+                        #define B __BYTE_ORDER
+                        #elif defined(BYTE_ORDER)
+                        #define B BYTE_ORDER
+                        #endif
+
+                        #ifdef __LITTLE_ENDIAN
+                        #define LITTLE __LITTLE_ENDIAN
+                        #elif defined(LITTLE_ENDIAN)
+                        #define LITTLE LITTLE_ENDIAN
+                        #endif
+
+                        #if !defined(LITTLE) || !defined(B) || LITTLE != B
+                        #error Not little endian.
+                        #endif""",
+                           headers="endian.h sys/endian.h",
+                           define="HAVE_LITTLE_ENDIAN")
+    conf.CHECK_CODE(code = """#ifdef __BYTE_ORDER
+                        #define B __BYTE_ORDER
+                        #elif defined(BYTE_ORDER)
+                        #define B BYTE_ORDER
+                        #endif
+
+                        #ifdef __BIG_ENDIAN
+                        #define BIG __BIG_ENDIAN
+                        #elif defined(BIG_ENDIAN)
+                        #define BIG BIG_ENDIAN
+                        #endif
+
+                        #if !defined(BIG) || !defined(B) || BIG != B
+                        #error Not big endian.
+                        #endif""",
+                           headers="endian.h sys/endian.h",
+                           define="HAVE_BIG_ENDIAN")
+
+    if not conf.CONFIG_SET("HAVE_BIG_ENDIAN") and not conf.CONFIG_SET("HAVE_LITTLE_ENDIAN"):
+        # That didn't work!  Do runtime test.
+        conf.CHECK_CODE("""union { int i; char c[sizeof(int)]; } u;
+	  u.i = 0x01020304;
+	  return u.c[0] == 0x04 && u.c[1] == 0x03 && u.c[2] == 0x02 && u.c[3] == 0x01 ? 0 : 1;""",
+                        addmain=True, execute=True,
+                        define='HAVE_LITTLE_ENDIAN',
+                        msg="Checking for HAVE_LITTLE_ENDIAN - runtime")
+        conf.CHECK_CODE("""union { int i; char c[sizeof(int)]; } u;
+	  u.i = 0x01020304;
+	  return u.c[0] == 0x01 && u.c[1] == 0x02 && u.c[2] == 0x03 && u.c[3] == 0x04 ? 0 : 1;""",
+                        addmain=True, execute=True,
+                        define='HAVE_BIG_ENDIAN',
+                        msg="Checking for HAVE_BIG_ENDIAN - runtime")
+
+    # Extra sanity check.
+    if conf.CONFIG_SET("HAVE_BIG_ENDIAN") == conf.CONFIG_SET("HAVE_LITTLE_ENDIAN"):
+        Logs.error("Failed endian determination.  The PDP-11 is back?")
+        sys.exit(1)
+
+    conf.CHECK_CODE('return __builtin_clz(1) == (sizeof(int)*8 - 1) ? 0 : 1;',
+                    link=True,
+                    define='HAVE_BUILTIN_CLZ')
+    conf.CHECK_CODE('return __builtin_clzl(1) == (sizeof(long)*8 - 1) ? 0 : 1;',
+                    link=True,
+                    define='HAVE_BUILTIN_CLZL')
+    conf.CHECK_CODE('return __builtin_clzll(1) == (sizeof(long long)*8 - 1) ? 0 : 1;',
+                    link=True,
+                    define='HAVE_BUILTIN_CLZLL')
+    conf.CHECK_CODE('return __builtin_constant_p(1) ? 0 : 1;',
+                    link=True,
+                    define='HAVE_BUILTIN_CONSTANT_P')
+    conf.CHECK_CODE('return __builtin_expect(main != 0, 1) ? 0 : 1;',
+                    link=True,
+                    define='HAVE_BUILTIN_EXPECT')
+    conf.CHECK_CODE('return __builtin_popcountl(255L) == 8 ? 0 : 1;',
+                    link=True,
+                    define='HAVE_BUILTIN_POPCOUNTL')
+    conf.CHECK_CODE('return __builtin_types_compatible_p(char *, int) ? 1 : 0;',
+                    link=True,
+                    define='HAVE_BUILTIN_TYPES_COMPATIBLE_P')
+    conf.CHECK_CODE('int *foo = (int[]) { 1, 2, 3, 4 }; return foo[0] ? 0 : 1;',
+                    define='HAVE_COMPOUND_LITERALS')
+    conf.CHECK_CODE("""#include <ctype.h>
+	  int main(void) { return isblank(' ') ? 0 : 1; }""",
+                    link=True, addmain=False, add_headers=False,
+                    define='HAVE_ISBLANK')
+    conf.CHECK_CODE('int x = 1; __typeof__(x) i; i = x; return i == x ? 0 : 1;',
+                    link=True,
+                    define='HAVE_TYPEOF')
+    conf.CHECK_CODE('int __attribute__((warn_unused_result)) func(int x) { return x; }',
+                    addmain=False, link=False, cflags="-Werror",
+                    define='HAVE_WARN_UNUSED_RESULT')
+
+def build(bld):
+    bld.SAMBA_LIBRARY('ccan',
+                      vnum="0.1-init-1161-g661d41f",
+                      source=bld.path.ant_glob('*/*.c'),
+                      private_library=True)
diff --git a/lib/nss_wrapper/nss_wrapper.c b/lib/nss_wrapper/nss_wrapper.c
index cfa5a687121..8767fbfd892 100644
--- a/lib/nss_wrapper/nss_wrapper.c
+++ b/lib/nss_wrapper/nss_wrapper.c
@@ -36,7 +36,9 @@
 
 /* defining this gives us the posix getpwnam_r() calls on solaris
    Thanks to heimdal for this */
+#ifndef _POSIX_PTHREAD_SEMANTICS
 #define _POSIX_PTHREAD_SEMANTICS
+#endif
 
 #define NSS_WRAPPER_NOT_REPLACE
 #include "../replace/replace.h"
diff --git a/lib/replace/libreplace_network.m4 b/lib/replace/libreplace_network.m4
index f9bca40ce94..eadcc6bfc11 100644
--- a/lib/replace/libreplace_network.m4
+++ b/lib/replace/libreplace_network.m4
@@ -240,12 +240,25 @@ if test x"$libreplace_cv_HAVE_GETADDRINFO" = x"yes"; then
 		{
 			struct addrinfo hints = {0,};
 			struct addrinfo *ppres;
-			const char hostname[] = "0.0.0.0";
+			const char hostname1[] = "0.0.0.0";
+			const char hostname2[] = "127.0.0.1";
+			const char hostname3[] = "::";
 			hints.ai_socktype = SOCK_STREAM;
-			hints.ai_family = AF_INET;
+			hints.ai_family = AF_UNSPEC;
 			hints.ai_flags =
 				AI_NUMERICHOST|AI_PASSIVE|AI_ADDRCONFIG;
-			return getaddrinfo(hostname, NULL, &hints, &ppres) != 0 ? 1 : 0;
+			/* Test for broken flag combination on AIX. */
+			if (getaddrinfo(hostname1, NULL, &hints, &ppres) == EAI_BADFLAGS) {
+				/* This fails on an IPv6-only box, but not with
+				   the EAI_BADFLAGS error. */
+				return 1;
+			}
+			if (getaddrinfo(hostname2, NULL, &hints, &ppres) == 0) {
+				/* IPv4 lookup works - good enough. */
+				return 0;
+			}
+			/* Uh-oh, no IPv4. Are we IPv6-only ? */
+			return getaddrinfo(hostname3, NULL, &hints, &ppres) != 0 ? 1 : 0;
 		}],
 		libreplace_cv_HAVE_GETADDRINFO=yes,
 		libreplace_cv_HAVE_GETADDRINFO=no)
diff --git a/lib/replace/system/kerberos.h b/lib/replace/system/kerberos.h
index bb1f1b9a094..7762d4be469 100644
--- a/lib/replace/system/kerberos.h
+++ b/lib/replace/system/kerberos.h
@@ -37,5 +37,19 @@
 #include <com_err.h>
 #endif
 
+#ifdef HAVE_GSSAPI_GSSAPI_EXT_H
+#include <gssapi/gssapi_ext.h>
+#elif HAVE_GSSAPI_GSSAPI_H
+#include <gssapi/gssapi.h>
+#elif HAVE_GSSAPI_GSSAPI_GENERIC_H
+#include <gssapi/gssapi_generic.h>
+#elif HAVE_GSSAPI_H
+#include <gssapi.h>
+#endif
+
+#if HAVE_GSSAPI_GSSAPI_KRB5_H
+#include <gssapi/gssapi_krb5.h>
+#endif
+
 #endif
 #endif
diff --git a/lib/replace/system/network.h b/lib/replace/system/network.h
index f7c1bcfacbb..a4e6a7e31a6 100644
--- a/lib/replace/system/network.h
+++ b/lib/replace/system/network.h
@@ -331,8 +331,6 @@ typedef unsigned short int sa_family_t;
      * which might return 512 or bigger
      */
 #   define IOV_MAX 512
-#  else
-#   error IOV_MAX and UIO_MAXIOV undefined
 #  endif
 # endif
 #endif
diff --git a/lib/smbconf/smbconf.c b/lib/smbconf/smbconf.c
index 80fe9aac372..e0441ed9857 100644
--- a/lib/smbconf/smbconf.c
+++ b/lib/smbconf/smbconf.c
@@ -27,12 +27,13 @@
  *
  **********************************************************************/
 
-static WERROR smbconf_global_check(struct smbconf_ctx *ctx)
+static sbcErr smbconf_global_check(struct smbconf_ctx *ctx)
 {
 	if (!smbconf_share_exists(ctx, GLOBAL_NAME)) {
 		return smbconf_create_share(ctx, GLOBAL_NAME);
 	}
-	return WERR_OK;
+
+	return SBC_ERR_OK;
 }
 
 
@@ -42,6 +43,41 @@ static WERROR smbconf_global_check(struct smbconf_ctx *ctx)
  *
  **********************************************************************/
 
+const char *sbcErrorString(sbcErr error)
+{
+	switch (error) {
+		case SBC_ERR_OK:
+			return "SBC_ERR_OK";
+		case SBC_ERR_NOT_IMPLEMENTED:
+			return "SBC_ERR_NOT_IMPLEMENTED";
+		case SBC_ERR_NOT_SUPPORTED:
+			return "SBC_ERR_NOT_SUPPORTED";
+		case SBC_ERR_UNKNOWN_FAILURE:
+			return "SBC_ERR_UNKNOWN_FAILURE";
+		case SBC_ERR_NOMEM:
+			return "SBC_ERR_NOMEM";
+		case SBC_ERR_INVALID_PARAM:
+			return "SBC_ERR_INVALID_PARAM";
+		case SBC_ERR_BADFILE:
+			return "SBC_ERR_BADFILE";
+		case SBC_ERR_NO_SUCH_SERVICE:
+			return "SBC_ERR_NO_SUCH_SERVICE";
+		case SBC_ERR_IO_FAILURE:
+			return "SBC_ERR_IO_FAILURE";
+		case SBC_ERR_CAN_NOT_COMPLETE:
+			return "SBC_ERR_CAN_NOT_COMPLETE";
+		case SBC_ERR_NO_MORE_ITEMS:
+			return "SBC_ERR_NO_MORE_ITEMS";
+		case SBC_ERR_FILE_EXISTS:
+			return "SBC_ERR_FILE_EXISTS";
+		case SBC_ERR_ACCESS_DENIED:
+			return "SBC_ERR_ACCESS_DENIED";
+	}
+
+	return "unknown sbcErr value";
+}
+
+
 /**
  * Tell whether the backend requires messaging to be set up
  * for the backend to work correctly.
@@ -91,7 +127,7 @@ bool smbconf_changed(struct smbconf_ctx *ctx, struct smbconf_csn *csn,
 /**
  * Drop the whole configuration (restarting empty).
  */
-WERROR smbconf_drop(struct smbconf_ctx *ctx)
+sbcErr smbconf_drop(struct smbconf_ctx *ctx)
 {
 	return ctx->ops->drop(ctx);
 }
@@ -105,12 +141,12 @@ WERROR smbconf_drop(struct smbconf_ctx *ctx)
  *  param_names  : list of lists of parameter names for each share
  *  param_values : list of lists of parameter values for each share
  */
-WERROR smbconf_get_config(struct smbconf_ctx *ctx,
+sbcErr smbconf_get_config(struct smbconf_ctx *ctx,
 			  TALLOC_CTX *mem_ctx,
 			  uint32_t *num_shares,
 			  struct smbconf_service ***services)
 {
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	TALLOC_CTX *tmp_ctx = NULL;
 	uint32_t tmp_num_shares;
 	char **tmp_share_names;
@@ -118,36 +154,35 @@ WERROR smbconf_get_config(struct smbconf_ctx *ctx,
 	uint32_t count;
 
 	if ((num_shares == NULL) || (services == NULL)) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	tmp_ctx = talloc_stackframe();
 
-	werr = smbconf_get_share_names(ctx, tmp_ctx, &tmp_num_shares,
-				       &tmp_share_names);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_share_names(ctx, tmp_ctx, &tmp_num_shares,
+				      &tmp_share_names);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
 	tmp_services = talloc_array(tmp_ctx, struct smbconf_service *,
 				    tmp_num_shares);
-
 	if (tmp_services == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
 	for (count = 0; count < tmp_num_shares; count++) {
-		werr = smbconf_get_share(ctx, tmp_services,
-					 tmp_share_names[count],
-					 &tmp_services[count]);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_get_share(ctx, tmp_services,
+					tmp_share_names[count],
+					&tmp_services[count]);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 	}
 
-	werr = WERR_OK;
+	err = SBC_ERR_OK;
 
 	*num_shares = tmp_num_shares;
 	if (tmp_num_shares > 0) {
@@ -158,13 +193,13 @@ WERROR smbconf_get_config(struct smbconf_ctx *ctx,
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * get the list of share names defined in the configuration.
  */
-WERROR smbconf_get_share_names(struct smbconf_ctx *ctx,
+sbcErr smbconf_get_share_names(struct smbconf_ctx *ctx,
 			       TALLOC_CTX *mem_ctx,
 			       uint32_t *num_shares,
 			       char ***share_names)
@@ -185,11 +220,11 @@ bool smbconf_share_exists(struct smbconf_ctx *ctx,
 /**
  * Add a service if it does not already exist.
  */
-WERROR smbconf_create_share(struct smbconf_ctx *ctx,
+sbcErr smbconf_create_share(struct smbconf_ctx *ctx,
 			    const char *servicename)
 {
 	if ((servicename != NULL) && smbconf_share_exists(ctx, servicename)) {
-		return WERR_FILE_EXISTS;
+		return SBC_ERR_FILE_EXISTS;
 	}
 
 	return ctx->ops->create_share(ctx, servicename);
@@ -198,7 +233,7 @@ WERROR smbconf_create_share(struct smbconf_ctx *ctx,
 /**
  * get a definition of a share (service) from configuration.
  */
-WERROR smbconf_get_share(struct smbconf_ctx *ctx,
+sbcErr smbconf_get_share(struct smbconf_ctx *ctx,
 			 TALLOC_CTX *mem_ctx,
 			 const char *servicename,
 			 struct smbconf_service **service)
@@ -209,10 +244,10 @@ WERROR smbconf_get_share(struct smbconf_ctx *ctx,
 /**
  * delete a service from configuration
  */
-WERROR smbconf_delete_share(struct smbconf_ctx *ctx, const char *servicename)
+sbcErr smbconf_delete_share(struct smbconf_ctx *ctx, const char *servicename)
 {
 	if (!smbconf_share_exists(ctx, servicename)) {
-		return WERR_NO_SUCH_SERVICE;
+		return SBC_ERR_NO_SUCH_SERVICE;
 	}
 
 	return ctx->ops->delete_share(ctx, servicename);
@@ -221,7 +256,7 @@ WERROR smbconf_delete_share(struct smbconf_ctx *ctx, const char *servicename)
 /**
  * set a configuration parameter to the value provided.
  */
-WERROR smbconf_set_parameter(struct smbconf_ctx *ctx,
+sbcErr smbconf_set_parameter(struct smbconf_ctx *ctx,
 			     const char *service,
 			     const char *param,
 			     const char *valstr)
@@ -235,30 +270,31 @@ WERROR smbconf_set_parameter(struct smbconf_ctx *ctx,
  *
  * This also creates [global] when it does not exist.
  */
-WERROR smbconf_set_global_parameter(struct smbconf_ctx *ctx,
+sbcErr smbconf_set_global_parameter(struct smbconf_ctx *ctx,
 				    const char *param, const char *val)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_global_check(ctx);
-	if (W_ERROR_IS_OK(werr)) {
-		werr = smbconf_set_parameter(ctx, GLOBAL_NAME, param, val);
+	err = smbconf_global_check(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
+	err = smbconf_set_parameter(ctx, GLOBAL_NAME, param, val);
 
-	return werr;
+	return err;
 }
 
 /**
  * get the value of a configuration parameter as a string
  */
-WERROR smbconf_get_parameter(struct smbconf_ctx *ctx,
+sbcErr smbconf_get_parameter(struct smbconf_ctx *ctx,
 			     TALLOC_CTX *mem_ctx,
 			     const char *service,
 			     const char *param,
 			     char **valstr)
 {
 	if (valstr == NULL) {
-		return WERR_INVALID_PARAM;
+		return SBC_ERR_INVALID_PARAM;
 	}
 
 	return ctx->ops->get_parameter(ctx, mem_ctx, service, param, valstr);
@@ -269,26 +305,28 @@ WERROR smbconf_get_parameter(struct smbconf_ctx *ctx,
  *
  * Create [global] if it does not exist.
  */
-WERROR smbconf_get_global_parameter(struct smbconf_ctx *ctx,
+sbcErr smbconf_get_global_parameter(struct smbconf_ctx *ctx,
 				    TALLOC_CTX *mem_ctx,
 				    const char *param,
 				    char **valstr)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_global_check(ctx);
-	if (W_ERROR_IS_OK(werr)) {
-		werr = smbconf_get_parameter(ctx, mem_ctx, GLOBAL_NAME, param,
-					     valstr);
+	err = smbconf_global_check(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
 
-	return werr;
+	err = smbconf_get_parameter(ctx, mem_ctx, GLOBAL_NAME, param,
+				    valstr);
+
+	return err;
 }
 
 /**
  * delete a parameter from configuration
  */
-WERROR smbconf_delete_parameter(struct smbconf_ctx *ctx,
+sbcErr smbconf_delete_parameter(struct smbconf_ctx *ctx,
 				const char *service, const char *param)
 {
 	return ctx->ops->delete_parameter(ctx, service, param);
@@ -299,20 +337,21 @@ WERROR smbconf_delete_parameter(struct smbconf_ctx *ctx,
  *
  * Create [global] if it does not exist.
  */
-WERROR smbconf_delete_global_parameter(struct smbconf_ctx *ctx,
+sbcErr smbconf_delete_global_parameter(struct smbconf_ctx *ctx,
 				       const char *param)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_global_check(ctx);
-	if (W_ERROR_IS_OK(werr)) {
-		werr = smbconf_delete_parameter(ctx, GLOBAL_NAME, param);
+	err = smbconf_global_check(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
+	err = smbconf_delete_parameter(ctx, GLOBAL_NAME, param);
 
-	return werr;
+	return err;
 }
 
-WERROR smbconf_get_includes(struct smbconf_ctx *ctx,
+sbcErr smbconf_get_includes(struct smbconf_ctx *ctx,
 			    TALLOC_CTX *mem_ctx,
 			    const char *service,
 			    uint32_t *num_includes, char ***includes)
@@ -321,72 +360,75 @@ WERROR smbconf_get_includes(struct smbconf_ctx *ctx,
 				      includes);
 }
 
-WERROR smbconf_get_global_includes(struct smbconf_ctx *ctx,
+sbcErr smbconf_get_global_includes(struct smbconf_ctx *ctx,
 				   TALLOC_CTX *mem_ctx,
 				   uint32_t *num_includes, char ***includes)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_global_check(ctx);
-	if (W_ERROR_IS_OK(werr)) {
-		werr = smbconf_get_includes(ctx, mem_ctx, GLOBAL_NAME,
-					    num_includes, includes);
+	err = smbconf_global_check(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
+	err = smbconf_get_includes(ctx, mem_ctx, GLOBAL_NAME,
+				    num_includes, includes);
 
-	return werr;
+	return err;
 }
 
-WERROR smbconf_set_includes(struct smbconf_ctx *ctx,
+sbcErr smbconf_set_includes(struct smbconf_ctx *ctx,
 			    const char *service,
 			    uint32_t num_includes, const char **includes)
 {
 	return ctx->ops->set_includes(ctx, service, num_includes, includes);
 }
 
-WERROR smbconf_set_global_includes(struct smbconf_ctx *ctx,
+sbcErr smbconf_set_global_includes(struct smbconf_ctx *ctx,
 				   uint32_t num_includes,
 				   const char **includes)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_global_check(ctx);
-	if (W_ERROR_IS_OK(werr)) {
-		werr = smbconf_set_includes(ctx, GLOBAL_NAME,
-					    num_includes, includes);
+	err = smbconf_global_check(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
+	err = smbconf_set_includes(ctx, GLOBAL_NAME,
+				   num_includes, includes);
 
-	return werr;
+	return err;
 }
 
 
-WERROR smbconf_delete_includes(struct smbconf_ctx *ctx, const char *service)
+sbcErr smbconf_delete_includes(struct smbconf_ctx *ctx, const char *service)
 {
 	return ctx->ops->delete_includes(ctx, service);
 }
 
-WERROR smbconf_delete_global_includes(struct smbconf_ctx *ctx)
+sbcErr smbconf_delete_global_includes(struct smbconf_ctx *ctx)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_global_check(ctx);
-	if (W_ERROR_IS_OK(werr)) {
-		werr = smbconf_delete_includes(ctx, GLOBAL_NAME);
+	err = smbconf_global_check(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
+	err = smbconf_delete_includes(ctx, GLOBAL_NAME);
 
-	return werr;
+	return err;
 }
 
-WERROR smbconf_transaction_start(struct smbconf_ctx *ctx)
+sbcErr smbconf_transaction_start(struct smbconf_ctx *ctx)
 {
 	return ctx->ops->transaction_start(ctx);
 }
 
-WERROR smbconf_transaction_commit(struct smbconf_ctx *ctx)
+sbcErr smbconf_transaction_commit(struct smbconf_ctx *ctx)
 {
 	return ctx->ops->transaction_commit(ctx);
 }
 
-WERROR smbconf_transaction_cancel(struct smbconf_ctx *ctx)
+sbcErr smbconf_transaction_cancel(struct smbconf_ctx *ctx)
 {
 	return ctx->ops->transaction_cancel(ctx);
 }
diff --git a/lib/smbconf/smbconf.h b/lib/smbconf/smbconf.h
index 517302ac883..7f62b06af45 100644
--- a/lib/smbconf/smbconf.h
+++ b/lib/smbconf/smbconf.h
@@ -20,6 +20,39 @@
 #ifndef __LIBSMBCONF_H__
 #define __LIBSMBCONF_H__
 
+/**
+ * @defgroup libsmbconf The smbconf API
+ *
+ * libsmbconf is a library to read or, based on the backend, modify the Samba
+ * configuration.
+ *
+ * @{
+ */
+
+/**
+ * @brief Status codes returned from smbconf functions
+ */
+enum _sbcErrType {
+	SBC_ERR_OK = 0,          /**< Successful completion **/
+	SBC_ERR_NOT_IMPLEMENTED, /**< Function not implemented **/
+	SBC_ERR_NOT_SUPPORTED,   /**< Function not supported **/
+	SBC_ERR_UNKNOWN_FAILURE, /**< General failure **/
+	SBC_ERR_NOMEM,           /**< Memory allocation error **/
+	SBC_ERR_INVALID_PARAM,   /**< An Invalid parameter was supplied **/
+	SBC_ERR_BADFILE,         /**< A bad file was supplied **/
+	SBC_ERR_NO_SUCH_SERVICE, /**< There is no such service provided **/
+	SBC_ERR_IO_FAILURE,      /**< There was an IO error **/
+	SBC_ERR_CAN_NOT_COMPLETE,/**< Can not complete action **/
+	SBC_ERR_NO_MORE_ITEMS,   /**< No more items left **/
+	SBC_ERR_FILE_EXISTS,     /**< File already exists **/
+	SBC_ERR_ACCESS_DENIED,   /**< Access has been denied **/
+};
+
+typedef enum _sbcErrType sbcErr;
+
+#define SBC_ERROR_IS_OK(x) ((x) == SBC_ERR_OK)
+#define SBC_ERROR_EQUAL(x,y) ((x) == (y))
+
 struct smbconf_ctx;
 
 /* the change sequence number */
@@ -27,75 +60,428 @@ struct smbconf_csn {
 	uint64_t csn;
 };
 
+/** Information about a service */
 struct smbconf_service {
-	char *name;
-	uint32_t num_params;
-	char **param_names;
-	char **param_values;
+	char *name;          /**< The name of the share */
+	uint32_t num_params; /**< List of length num_shares of parameter counts for each share */
+	char **param_names;  /**< List of lists of parameter names for each share */
+	char **param_values; /**< List of lists of parameter values for each share */
 };
 
 /*
- * the smbconf API functions
+ * The smbconf API functions
+ */
+
+/**
+ * @brief Translate an error value into a string
+ *
+ * @param error
+ *
+ * @return a pointer to a static string
+ **/
+const char *sbcErrorString(sbcErr error);
+
+/**
+ * @brief Check if the backend requires messaging to be set up.
+ *
+ * Tell whether the backend requires messaging to be set up
+ * for the backend to work correctly.
+ *
+ * @param[in] ctx       The smbconf context to check.
+ *
+ * @return              True if needed, false if not.
  */
 bool smbconf_backend_requires_messaging(struct smbconf_ctx *ctx);
+
+/**
+ * @brief Tell whether the source is writeable.
+ *
+ * @param[in] ctx       The smbconf context to check.
+ *
+ * @return              True if it is writeable, false if not.
+ */
 bool smbconf_is_writeable(struct smbconf_ctx *ctx);
+
+/**
+ * @brief Close the configuration.
+ *
+ * @param[in] ctx       The smbconf context to close.
+ */
 void smbconf_shutdown(struct smbconf_ctx *ctx);
+
+/**
+ * @brief Detect changes in the configuration.
+ *
+ * Get the change sequence number of the given service/parameter. Service and
+ * parameter strings may be NULL.
+ *
+ * The given change sequence number (csn) struct is filled with the current
+ * csn. smbconf_changed() can also be used for initial retrieval of the csn.
+ *
+ * @param[in] ctx       The smbconf context to check for changes.
+ *
+ * @param[inout] csn    The smbconf csn to be filled.
+ *
+ * @param[in] service   The service name to check or NULL.
+ *
+ * @param[in] param     The param to check or NULL.
+ *
+ * @return              True if it has been changed, false if not.
+ */
 bool smbconf_changed(struct smbconf_ctx *ctx, struct smbconf_csn *csn,
 		     const char *service, const char *param);
-WERROR smbconf_drop(struct smbconf_ctx *ctx);
-WERROR smbconf_get_config(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Drop the whole configuration (restarting empty).
+ *
+ * @param[in] ctx       The smbconf context to drop the config.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_drop(struct smbconf_ctx *ctx);
+
+/**
+ * @brief Get the whole configuration as lists of strings with counts.
+ *
+ * @param[in] ctx       The smbconf context to get the lists from.
+ *
+ * @param[in] mem_ctx   The memory context to use.
+ *
+ * @param[in] num_shares A pointer to store the number of shares.
+ *
+ * @param[out] services  A pointer to store the services.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ *
+ * @see smbconf_service
+ */
+sbcErr smbconf_get_config(struct smbconf_ctx *ctx,
 			  TALLOC_CTX *mem_ctx,
 			  uint32_t *num_shares,
 			  struct smbconf_service ***services);
-WERROR smbconf_get_share_names(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Get the list of share names defined in the configuration.
+ *
+ * @param[in] ctx       The smbconf context to use.
+ *
+ * @param[in] mem_ctx   The memory context to use.
+ *
+ * @param[in] num_shares A pointer to store the number of shares.
+ *
+ * @param[in] share_names A pointer to store the share names.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_get_share_names(struct smbconf_ctx *ctx,
 			       TALLOC_CTX *mem_ctx,
 			       uint32_t *num_shares,
 			       char ***share_names);
+
+/**
+ * @brief Check if a share/service of a given name exists.
+ *
+ * @param[in] ctx       The smbconf context to use.
+ *
+ * @param[in] servicename The service name to check if it exists.
+ *
+ * @return              True if it exists, false if not.
+ */
 bool smbconf_share_exists(struct smbconf_ctx *ctx, const char *servicename);
-WERROR smbconf_create_share(struct smbconf_ctx *ctx, const char *servicename);
-WERROR smbconf_get_share(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Add a service if it does not already exist.
+ *
+ * @param[in] ctx       The smbconf context to use.
+ *
+ * @param[in] servicename The name of the service to add.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_create_share(struct smbconf_ctx *ctx, const char *servicename);
+
+/**
+ * @brief Get a definition of a share (service) from configuration.
+ *
+ * @param[in] ctx       The smbconf context to use.
+ *
+ * @param[in] mem_ctx   A memory context to allocate the result.
+ *
+ * @param[in] servicename The service name to get the information from.
+ *
+ * @param[out] service  A pointer to store the service information about the
+ *                      share.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ *
+ * @see smbconf_service
+ */
+sbcErr smbconf_get_share(struct smbconf_ctx *ctx,
 			 TALLOC_CTX *mem_ctx,
 			 const char *servicename,
 			 struct smbconf_service **service);
-WERROR smbconf_delete_share(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Delete a service from configuration.
+ *
+ * @param[in] ctx       The smbconf context to use.
+ *
+ * @param[in] servicename The service name to delete.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_delete_share(struct smbconf_ctx *ctx,
 			    const char *servicename);
-WERROR smbconf_set_parameter(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Set a configuration parameter to the value provided.
+ *
+ * @param[in] ctx       The smbconf context to use.
+ *
+ * @param[in] service   The service name to set the parameter.
+ *
+ * @param[in] param     The name of the parameter to set.
+ *
+ * @param[in] valstr    The value to set.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_set_parameter(struct smbconf_ctx *ctx,
 			     const char *service,
 			     const char *param,
 			     const char *valstr);
-WERROR smbconf_set_global_parameter(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Set a global configuration parameter to the value provided.
+ *
+ * This adds a paramet in the [global] service. It also creates [global] if it
+ * does't exist.
+ *
+ * @param[in] ctx       The smbconf context to use.
+ *
+ * @param[in] param     The name of the parameter to set.
+ *
+ * @param[in] val       The value to set.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_set_global_parameter(struct smbconf_ctx *ctx,
 				    const char *param, const char *val);
-WERROR smbconf_get_parameter(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Get the value of a configuration parameter as a string.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in]  mem_ctx  The memory context to allocate the string on.
+ *
+ * @param[in]  service  The name of the service where to find the parameter.
+ *
+ * @param[in]  param    The parameter to get.
+ *
+ * @param[out] valstr   A pointer to store the value as a string.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_get_parameter(struct smbconf_ctx *ctx,
 			     TALLOC_CTX *mem_ctx,
 			     const char *service,
 			     const char *param,
 			     char **valstr);
-WERROR smbconf_get_global_parameter(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Get the value of a global configuration parameter as a string.
+ *
+ * It also creates [global] if it does't exist.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in]  mem_ctx  The memory context to allocate the string on.
+ *
+ * @param[in]  param    The parameter to get.
+ *
+ * @param[out] valstr   A pointer to store the value as a string.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_get_global_parameter(struct smbconf_ctx *ctx,
 				    TALLOC_CTX *mem_ctx,
 				    const char *param,
 				    char **valstr);
-WERROR smbconf_delete_parameter(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Delete a parameter from the configuration.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in] service   The service where the parameter can be found.
+ *
+ * @param[in] param     The name of the parameter to delete.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_delete_parameter(struct smbconf_ctx *ctx,
 				const char *service, const char *param);
-WERROR smbconf_delete_global_parameter(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Delete a global parameter from the configuration.
+ *
+ * It also creates [global] if it does't exist.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in] param     The name of the parameter to delete.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_delete_global_parameter(struct smbconf_ctx *ctx,
 				       const char *param);
-WERROR smbconf_get_includes(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Get the list of names of included files.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in]  mem_ctx  The memory context to allocate the names.
+ *
+ * @param[in]  service  The service name to get the include files.
+ *
+ * @param[out] num_includes A pointer to store the number of included files.
+ *
+ * @param[out] includes A pointer to store the paths of the included files.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_get_includes(struct smbconf_ctx *ctx,
 			    TALLOC_CTX *mem_ctx,
 			    const char *service,
 			    uint32_t *num_includes, char ***includes);
-WERROR smbconf_get_global_includes(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Get the list of globally included files.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in]  mem_ctx  The memory context to allocate the names.
+ *
+ * @param[out] num_includes A pointer to store the number of included files.
+ *
+ * @param[out] includes A pointer to store the paths of the included files.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_get_global_includes(struct smbconf_ctx *ctx,
 				   TALLOC_CTX *mem_ctx,
 				   uint32_t *num_includes, char ***includes);
-WERROR smbconf_set_includes(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Set a list of config files to include on the given service.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in]  service  The service to add includes.
+ *
+ * @param[in]  num_includes The number of includes to set.
+ *
+ * @param[in]  includes A list of paths to include.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_set_includes(struct smbconf_ctx *ctx,
 			    const char *service,
 			    uint32_t num_includes, const char **includes);
-WERROR smbconf_set_global_includes(struct smbconf_ctx *ctx,
+
+/**
+ * @brief Set a list of config files to include globally.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in]  num_includes The number of includes to set.
+ *
+ * @param[in]  includes A list of paths to include.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_set_global_includes(struct smbconf_ctx *ctx,
 				   uint32_t num_includes,
 				   const char **includes);
-WERROR smbconf_delete_includes(struct smbconf_ctx *ctx, const char *service);
-WERROR smbconf_delete_global_includes(struct smbconf_ctx *ctx);
 
-WERROR smbconf_transaction_start(struct smbconf_ctx *ctx);
-WERROR smbconf_transaction_commit(struct smbconf_ctx *ctx);
-WERROR smbconf_transaction_cancel(struct smbconf_ctx *ctx);
+/**
+ * @brief Delete include parameter on the given service.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @param[in]  service  The name of the service to delete the includes from.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_delete_includes(struct smbconf_ctx *ctx, const char *service);
+
+/**
+ * @brief Delete include parameter from the global service.
+ *
+ * @param[in]  ctx      The smbconf context to use.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_delete_global_includes(struct smbconf_ctx *ctx);
+
+/**
+ * @brief Start a transaction on the configuration backend.
+ *
+ * This is to speed up writes to the registry based backend.
+ *
+ * @param[in] ctx       The smbconf context to start the transaction.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ */
+sbcErr smbconf_transaction_start(struct smbconf_ctx *ctx);
+
+/**
+ * @brief Commit a transaction on the configuration backend.
+ *
+ * This is to speed up writes to the registry based backend.
+ *
+ * @param[in] ctx       The smbconf context to commit the transaction.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ *
+ * @see smbconf_transaction_start()
+ */
+sbcErr smbconf_transaction_commit(struct smbconf_ctx *ctx);
+
+/**
+ * @brief Cancel a transaction on the configuration backend.
+ *
+ * @param[in] ctx       The smbconf context to cancel the transaction.
+ *
+ * @return              SBC_ERR_OK on success, a corresponding sbcErr if an
+ *                      error occured.
+ *
+ * @see smbconf_transaction_start()
+ */
+sbcErr smbconf_transaction_cancel(struct smbconf_ctx *ctx);
+
+/* @} ******************************************************************/
 
 #endif /*  _LIBSMBCONF_H_  */
diff --git a/lib/smbconf/smbconf_private.h b/lib/smbconf/smbconf_private.h
index e6998ad6392..e768c30b91e 100644
--- a/lib/smbconf/smbconf_private.h
+++ b/lib/smbconf/smbconf_private.h
@@ -27,50 +27,50 @@
 #include "lib/smbconf/smbconf.h"
 
 struct smbconf_ops {
-	WERROR (*init)(struct smbconf_ctx *ctx, const char *path);
+	sbcErr (*init)(struct smbconf_ctx *ctx, const char *path);
 	int (*shutdown)(struct smbconf_ctx *ctx);
 	bool (*requires_messaging)(struct smbconf_ctx *ctx);
 	bool (*is_writeable)(struct smbconf_ctx *ctx);
-	WERROR (*open_conf)(struct smbconf_ctx *ctx);
+	sbcErr (*open_conf)(struct smbconf_ctx *ctx);
 	int (*close_conf)(struct smbconf_ctx *ctx);
 	void (*get_csn)(struct smbconf_ctx *ctx, struct smbconf_csn *csn,
 			const char *service, const char *param);
-	WERROR (*drop)(struct smbconf_ctx *ctx);
-	WERROR (*get_share_names)(struct smbconf_ctx *ctx,
+	sbcErr (*drop)(struct smbconf_ctx *ctx);
+	sbcErr (*get_share_names)(struct smbconf_ctx *ctx,
 				  TALLOC_CTX *mem_ctx,
 				  uint32_t *num_shares,
 				  char ***share_names);
 	bool (*share_exists)(struct smbconf_ctx *ctx, const char *service);
-	WERROR (*create_share)(struct smbconf_ctx *ctx, const char *service);
-	WERROR (*get_share)(struct smbconf_ctx *ctx,
+	sbcErr (*create_share)(struct smbconf_ctx *ctx, const char *service);
+	sbcErr (*get_share)(struct smbconf_ctx *ctx,
 			    TALLOC_CTX *mem_ctx,
 			    const char *servicename,
 			    struct smbconf_service **service);
-	WERROR (*delete_share)(struct smbconf_ctx *ctx,
+	sbcErr (*delete_share)(struct smbconf_ctx *ctx,
 				    const char *servicename);
-	WERROR (*set_parameter)(struct smbconf_ctx *ctx,
+	sbcErr (*set_parameter)(struct smbconf_ctx *ctx,
 				const char *service,
 				const char *param,
 				const char *valstr);
-	WERROR (*get_parameter)(struct smbconf_ctx *ctx,
+	sbcErr (*get_parameter)(struct smbconf_ctx *ctx,
 				TALLOC_CTX *mem_ctx,
 				const char *service,
 				const char *param,
 				char **valstr);
-	WERROR (*delete_parameter)(struct smbconf_ctx *ctx,
+	sbcErr (*delete_parameter)(struct smbconf_ctx *ctx,
 				   const char *service, const char *param);
-	WERROR (*get_includes)(struct smbconf_ctx *ctx,
+	sbcErr (*get_includes)(struct smbconf_ctx *ctx,
 			       TALLOC_CTX *mem_ctx,
 			       const char *service,
 			       uint32_t *num_includes, char ***includes);
-	WERROR (*set_includes)(struct smbconf_ctx *ctx,
+	sbcErr (*set_includes)(struct smbconf_ctx *ctx,
 			       const char *service,
 			       uint32_t num_includes, const char **includes);
-	WERROR (*delete_includes)(struct smbconf_ctx *ctx,
+	sbcErr (*delete_includes)(struct smbconf_ctx *ctx,
 				  const char *service);
-	WERROR (*transaction_start)(struct smbconf_ctx *ctx);
-	WERROR (*transaction_commit)(struct smbconf_ctx *ctx);
-	WERROR (*transaction_cancel)(struct smbconf_ctx *ctx);
+	sbcErr (*transaction_start)(struct smbconf_ctx *ctx);
+	sbcErr (*transaction_commit)(struct smbconf_ctx *ctx);
+	sbcErr (*transaction_cancel)(struct smbconf_ctx *ctx);
 };
 
 struct smbconf_ctx {
@@ -79,10 +79,10 @@ struct smbconf_ctx {
 	void *data; /* private data for use in backends */
 };
 
-WERROR smbconf_init_internal(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
+sbcErr smbconf_init_internal(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 			     const char *path, struct smbconf_ops *ops);
 
-WERROR smbconf_add_string_to_array(TALLOC_CTX *mem_ctx,
+sbcErr smbconf_add_string_to_array(TALLOC_CTX *mem_ctx,
 				   char ***array,
 				   uint32_t count,
 				   const char *string);
diff --git a/lib/smbconf/smbconf_txt.c b/lib/smbconf/smbconf_txt.c
index 2114841b817..5c4bd27b9df 100644
--- a/lib/smbconf/smbconf_txt.c
+++ b/lib/smbconf/smbconf_txt.c
@@ -60,7 +60,7 @@ static struct txt_private_data *pd(struct smbconf_ctx *ctx)
 
 static bool smbconf_txt_do_section(const char *section, void *private_data)
 {
-	WERROR werr;
+	sbcErr err;
 	uint32_t idx;
 	struct txt_private_data *tpd = (struct txt_private_data *)private_data;
 	struct txt_cache *cache = tpd->cache;
@@ -72,9 +72,9 @@ static bool smbconf_txt_do_section(const char *section, void *private_data)
 		return true;
 	}
 
-	werr = smbconf_add_string_to_array(cache, &(cache->share_names),
-					   cache->num_shares, section);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_add_string_to_array(cache, &(cache->share_names),
+					  cache->num_shares, section);
+	if (!SBC_ERROR_IS_OK(err)) {
 		return false;
 	}
 	cache->current_share = cache->num_shares;
@@ -114,7 +114,7 @@ static bool smbconf_txt_do_parameter(const char *param_name,
 				     const char *param_value,
 				     void *private_data)
 {
-	WERROR werr;
+	sbcErr err;
 	char **param_names, **param_values;
 	uint32_t num_params;
 	uint32_t idx;
@@ -146,17 +146,17 @@ static bool smbconf_txt_do_parameter(const char *param_name,
 		}
 		return true;
 	}
-	werr = smbconf_add_string_to_array(cache,
+	err = smbconf_add_string_to_array(cache,
 				&(cache->param_names[cache->current_share]),
 				num_params, param_name);
-	if (!W_ERROR_IS_OK(werr)) {
+	if (!SBC_ERROR_IS_OK(err)) {
 		return false;
 	}
-	werr = smbconf_add_string_to_array(cache,
+	err = smbconf_add_string_to_array(cache,
 				&(cache->param_values[cache->current_share]),
 				num_params, param_value);
 	cache->num_params[cache->current_share]++;
-	return W_ERROR_IS_OK(werr);
+	return SBC_ERROR_IS_OK(err);
 }
 
 static void smbconf_txt_flush_cache(struct smbconf_ctx *ctx)
@@ -165,7 +165,7 @@ static void smbconf_txt_flush_cache(struct smbconf_ctx *ctx)
 	pd(ctx)->cache = NULL;
 }
 
-static WERROR smbconf_txt_init_cache(struct smbconf_ctx *ctx)
+static sbcErr smbconf_txt_init_cache(struct smbconf_ctx *ctx)
 {
 	if (pd(ctx)->cache != NULL) {
 		smbconf_txt_flush_cache(ctx);
@@ -174,40 +174,40 @@ static WERROR smbconf_txt_init_cache(struct smbconf_ctx *ctx)
 	pd(ctx)->cache = talloc_zero(pd(ctx), struct txt_cache);
 
 	if (pd(ctx)->cache == NULL) {
-		return WERR_NOMEM;
+		return SBC_ERR_NOMEM;
 	}
 
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
-static WERROR smbconf_txt_load_file(struct smbconf_ctx *ctx)
+static sbcErr smbconf_txt_load_file(struct smbconf_ctx *ctx)
 {
-	WERROR werr;
+	sbcErr err;
 	uint64_t new_csn;
 
 	if (!file_exist(ctx->path)) {
-		return WERR_BADFILE;
+		return SBC_ERR_BADFILE;
 	}
 
 	new_csn = (uint64_t)file_modtime(ctx->path);
 	if (new_csn == pd(ctx)->csn) {
-		return WERR_OK;
+		return SBC_ERR_OK;
 	}
 
-	werr = smbconf_txt_init_cache(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
-		return werr;
+	err = smbconf_txt_init_cache(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
 
 	if (!pm_process(ctx->path, smbconf_txt_do_section,
 			smbconf_txt_do_parameter, pd(ctx)))
 	{
-		return WERR_CAN_NOT_COMPLETE;
+		return SBC_ERR_CAN_NOT_COMPLETE;
 	}
 
 	pd(ctx)->csn = new_csn;
 
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
 
@@ -220,24 +220,24 @@ static WERROR smbconf_txt_load_file(struct smbconf_ctx *ctx)
 /**
  * initialize the text based smbconf backend
  */
-static WERROR smbconf_txt_init(struct smbconf_ctx *ctx, const char *path)
+static sbcErr smbconf_txt_init(struct smbconf_ctx *ctx, const char *path)
 {
 	if (path == NULL) {
-		return WERR_BADFILE;
+		return SBC_ERR_BADFILE;
 	}
 	ctx->path = talloc_strdup(ctx, path);
 	if (ctx->path == NULL) {
-		return WERR_NOMEM;
+		return SBC_ERR_NOMEM;
 	}
 
 	ctx->data = talloc_zero(ctx, struct txt_private_data);
 	if (ctx->data == NULL) {
-		return WERR_NOMEM;
+		return SBC_ERR_NOMEM;
 	}
 
 	pd(ctx)->verbatim = true;
 
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
 static int smbconf_txt_shutdown(struct smbconf_ctx *ctx)
@@ -256,7 +256,7 @@ static bool smbconf_txt_is_writeable(struct smbconf_ctx *ctx)
 	return false;
 }
 
-static WERROR smbconf_txt_open(struct smbconf_ctx *ctx)
+static sbcErr smbconf_txt_open(struct smbconf_ctx *ctx)
 {
 	return smbconf_txt_load_file(ctx);
 }
@@ -285,15 +285,15 @@ static void smbconf_txt_get_csn(struct smbconf_ctx *ctx,
 /**
  * Drop the whole configuration (restarting empty)
  */
-static WERROR smbconf_txt_drop(struct smbconf_ctx *ctx)
+static sbcErr smbconf_txt_drop(struct smbconf_ctx *ctx)
 {
-	return WERR_NOT_SUPPORTED;
+	return SBC_ERR_NOT_SUPPORTED;
 }
 
 /**
  * get the list of share names defined in the configuration.
  */
-static WERROR smbconf_txt_get_share_names(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_get_share_names(struct smbconf_ctx *ctx,
 					  TALLOC_CTX *mem_ctx,
 					  uint32_t *num_shares,
 					  char ***share_names)
@@ -301,17 +301,16 @@ static WERROR smbconf_txt_get_share_names(struct smbconf_ctx *ctx,
 	uint32_t count;
 	uint32_t added_count = 0;
 	TALLOC_CTX *tmp_ctx = NULL;
-	WERROR werr = WERR_OK;
+	sbcErr err = SBC_ERR_OK;
 	char **tmp_share_names = NULL;
 
 	if ((num_shares == NULL) || (share_names == NULL)) {
-		werr = WERR_INVALID_PARAM;
-		goto done;
+		return SBC_ERR_INVALID_PARAM;
 	}
 
-	werr = smbconf_txt_load_file(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
-		return werr;
+	err = smbconf_txt_load_file(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
 
 	tmp_ctx = talloc_stackframe();
@@ -320,18 +319,18 @@ static WERROR smbconf_txt_get_share_names(struct smbconf_ctx *ctx,
 	 * possibly after NULL section */
 
 	if (smbconf_share_exists(ctx, NULL)) {
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
-						   0, NULL);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
+						  0, NULL);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 		added_count++;
 	}
 
 	if (smbconf_share_exists(ctx, GLOBAL_NAME)) {
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
 						   added_count, GLOBAL_NAME);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 		added_count++;
@@ -344,10 +343,10 @@ static WERROR smbconf_txt_get_share_names(struct smbconf_ctx *ctx,
 			continue;
 		}
 
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
 					added_count,
 					pd(ctx)->cache->share_names[count]);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 		added_count++;
@@ -362,7 +361,7 @@ static WERROR smbconf_txt_get_share_names(struct smbconf_ctx *ctx,
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 /**
@@ -371,10 +370,10 @@ done:
 static bool smbconf_txt_share_exists(struct smbconf_ctx *ctx,
 				     const char *servicename)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_txt_load_file(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_txt_load_file(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		return false;
 	}
 
@@ -386,29 +385,29 @@ static bool smbconf_txt_share_exists(struct smbconf_ctx *ctx,
 /**
  * Add a service if it does not already exist
  */
-static WERROR smbconf_txt_create_share(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_create_share(struct smbconf_ctx *ctx,
 				       const char *servicename)
 {
-	return WERR_NOT_SUPPORTED;
+	return SBC_ERR_NOT_SUPPORTED;
 }
 
 /**
  * get a definition of a share (service) from configuration.
  */
-static WERROR smbconf_txt_get_share(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_get_share(struct smbconf_ctx *ctx,
 				    TALLOC_CTX *mem_ctx,
 				    const char *servicename,
 				    struct smbconf_service **service)
 {
-	WERROR werr;
+	sbcErr err;
 	uint32_t sidx, count;
 	bool found;
 	TALLOC_CTX *tmp_ctx = NULL;
 	struct smbconf_service *tmp_service = NULL;
 
-	werr = smbconf_txt_load_file(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
-		return werr;
+	err = smbconf_txt_load_file(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
 
 	found = smbconf_find_in_array(servicename,
@@ -416,38 +415,38 @@ static WERROR smbconf_txt_get_share(struct smbconf_ctx *ctx,
 				      pd(ctx)->cache->num_shares,
 				      &sidx);
 	if (!found) {
-		return WERR_NO_SUCH_SERVICE;
+		return SBC_ERR_NO_SUCH_SERVICE;
 	}
 
 	tmp_ctx = talloc_stackframe();
 
 	tmp_service = talloc_zero(tmp_ctx, struct smbconf_service);
 	if (tmp_service == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
 	if (servicename != NULL) {
 		tmp_service->name = talloc_strdup(tmp_service, servicename);
 		if (tmp_service->name == NULL) {
-			werr = WERR_NOMEM;
+			err = SBC_ERR_NOMEM;
 			goto done;
 		}
 	}
 
 	for (count = 0; count < pd(ctx)->cache->num_params[sidx]; count++) {
-		werr = smbconf_add_string_to_array(tmp_service,
+		err = smbconf_add_string_to_array(tmp_service,
 				&(tmp_service->param_names),
 				count,
 				pd(ctx)->cache->param_names[sidx][count]);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
-		werr = smbconf_add_string_to_array(tmp_service,
+		err = smbconf_add_string_to_array(tmp_service,
 				&(tmp_service->param_values),
 				count,
 				pd(ctx)->cache->param_values[sidx][count]);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 	}
@@ -457,45 +456,45 @@ static WERROR smbconf_txt_get_share(struct smbconf_ctx *ctx,
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * delete a service from configuration
  */
-static WERROR smbconf_txt_delete_share(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_delete_share(struct smbconf_ctx *ctx,
 				       const char *servicename)
 {
-	return WERR_NOT_SUPPORTED;
+	return SBC_ERR_NOT_SUPPORTED;
 }
 
 /**
  * set a configuration parameter to the value provided.
  */
-static WERROR smbconf_txt_set_parameter(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_set_parameter(struct smbconf_ctx *ctx,
 					const char *service,
 					const char *param,
 					const char *valstr)
 {
-	return WERR_NOT_SUPPORTED;
+	return SBC_ERR_NOT_SUPPORTED;
 }
 
 /**
  * get the value of a configuration parameter as a string
  */
-static WERROR smbconf_txt_get_parameter(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_get_parameter(struct smbconf_ctx *ctx,
 					TALLOC_CTX *mem_ctx,
 					const char *service,
 					const char *param,
 					char **valstr)
 {
-	WERROR werr;
+	sbcErr err;
 	bool found;
 	uint32_t share_index, param_index;
 
-	werr = smbconf_txt_load_file(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
-		return werr;
+	err = smbconf_txt_load_file(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
 
 	found = smbconf_find_in_array(service,
@@ -503,7 +502,7 @@ static WERROR smbconf_txt_get_parameter(struct smbconf_ctx *ctx,
 				      pd(ctx)->cache->num_shares,
 				      &share_index);
 	if (!found) {
-		return WERR_NO_SUCH_SERVICE;
+		return SBC_ERR_NO_SUCH_SERVICE;
 	}
 
 	found = smbconf_reverse_find_in_array(param,
@@ -511,45 +510,45 @@ static WERROR smbconf_txt_get_parameter(struct smbconf_ctx *ctx,
 				pd(ctx)->cache->num_params[share_index],
 				&param_index);
 	if (!found) {
-		return WERR_INVALID_PARAM;
+		return SBC_ERR_INVALID_PARAM;
 	}
 
 	*valstr = talloc_strdup(mem_ctx,
 			pd(ctx)->cache->param_values[share_index][param_index]);
 
 	if (*valstr == NULL) {
-		return WERR_NOMEM;
+		return SBC_ERR_NOMEM;
 	}
 
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
 /**
  * delete a parameter from configuration
  */
-static WERROR smbconf_txt_delete_parameter(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_delete_parameter(struct smbconf_ctx *ctx,
 					   const char *service,
 					   const char *param)
 {
-	return WERR_NOT_SUPPORTED;
+	return SBC_ERR_NOT_SUPPORTED;
 }
 
-static WERROR smbconf_txt_get_includes(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_get_includes(struct smbconf_ctx *ctx,
 				       TALLOC_CTX *mem_ctx,
 				       const char *service,
 				       uint32_t *num_includes,
 				       char ***includes)
 {
-	WERROR werr;
+	sbcErr err;
 	bool found;
 	uint32_t sidx, count;
 	TALLOC_CTX *tmp_ctx = NULL;
 	uint32_t tmp_num_includes = 0;
 	char **tmp_includes = NULL;
 
-	werr = smbconf_txt_load_file(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
-		return werr;
+	err = smbconf_txt_load_file(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
 
 	found = smbconf_find_in_array(service,
@@ -557,7 +556,7 @@ static WERROR smbconf_txt_get_includes(struct smbconf_ctx *ctx,
 				      pd(ctx)->cache->num_shares,
 				      &sidx);
 	if (!found) {
-		return WERR_NO_SUCH_SERVICE;
+		return SBC_ERR_NO_SUCH_SERVICE;
 	}
 
 	tmp_ctx = talloc_stackframe();
@@ -566,11 +565,11 @@ static WERROR smbconf_txt_get_includes(struct smbconf_ctx *ctx,
 		if (strequal(pd(ctx)->cache->param_names[sidx][count],
 			     "include"))
 		{
-			werr = smbconf_add_string_to_array(tmp_ctx,
+			err = smbconf_add_string_to_array(tmp_ctx,
 				&tmp_includes,
 				tmp_num_includes,
 				pd(ctx)->cache->param_values[sidx][count]);
-			if (!W_ERROR_IS_OK(werr)) {
+			if (!SBC_ERROR_IS_OK(err)) {
 				goto done;
 			}
 			tmp_num_includes++;
@@ -581,47 +580,47 @@ static WERROR smbconf_txt_get_includes(struct smbconf_ctx *ctx,
 	if (*num_includes > 0) {
 		*includes = talloc_move(mem_ctx, &tmp_includes);
 		if (*includes == NULL) {
-			werr = WERR_NOMEM;
+			err = SBC_ERR_NOMEM;
 			goto done;
 		}
 	} else {
 		*includes = NULL;
 	}
 
-	werr = WERR_OK;
+	err = SBC_ERR_OK;
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
-static WERROR smbconf_txt_set_includes(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_set_includes(struct smbconf_ctx *ctx,
 				       const char *service,
 				       uint32_t num_includes,
 				       const char **includes)
 {
-	return WERR_NOT_SUPPORTED;
+	return SBC_ERR_NOT_SUPPORTED;
 }
 
-static WERROR smbconf_txt_delete_includes(struct smbconf_ctx *ctx,
+static sbcErr smbconf_txt_delete_includes(struct smbconf_ctx *ctx,
 					  const char *service)
 {
-	return WERR_NOT_SUPPORTED;
+	return SBC_ERR_NOT_SUPPORTED;
 }
 
-static WERROR smbconf_txt_transaction_start(struct smbconf_ctx *ctx)
+static sbcErr smbconf_txt_transaction_start(struct smbconf_ctx *ctx)
 {
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
-static WERROR smbconf_txt_transaction_commit(struct smbconf_ctx *ctx)
+static sbcErr smbconf_txt_transaction_commit(struct smbconf_ctx *ctx)
 {
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
-static WERROR smbconf_txt_transaction_cancel(struct smbconf_ctx *ctx)
+static sbcErr smbconf_txt_transaction_cancel(struct smbconf_ctx *ctx)
 {
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
 static struct smbconf_ops smbconf_ops_txt = {
@@ -654,15 +653,15 @@ static struct smbconf_ops smbconf_ops_txt = {
  * initialize the smbconf text backend
  * the only function that is exported from this module
  */
-WERROR smbconf_init_txt(TALLOC_CTX *mem_ctx,
+sbcErr smbconf_init_txt(TALLOC_CTX *mem_ctx,
 			struct smbconf_ctx **conf_ctx,
 			const char *path)
 {
-	WERROR werr;
+	sbcErr err;
 
-	werr = smbconf_init_internal(mem_ctx, conf_ctx, path, &smbconf_ops_txt);
-	if (!W_ERROR_IS_OK(werr)) {
-		return werr;
+	err = smbconf_init_internal(mem_ctx, conf_ctx, path, &smbconf_ops_txt);
+	if (!SBC_ERROR_IS_OK(err)) {
+		return err;
 	}
 
 	return smbconf_txt_load_file(*conf_ctx);
diff --git a/lib/smbconf/smbconf_txt.h b/lib/smbconf/smbconf_txt.h
index 688bbc9d483..72d6207521c 100644
--- a/lib/smbconf/smbconf_txt.h
+++ b/lib/smbconf/smbconf_txt.h
@@ -26,7 +26,7 @@ struct smbconf_ctx;
  * initialization functions for the text/file backend modules
  */
 
-WERROR smbconf_init_txt(TALLOC_CTX *mem_ctx,
+sbcErr smbconf_init_txt(TALLOC_CTX *mem_ctx,
 			struct smbconf_ctx **conf_ctx,
 			const char *path);
 
diff --git a/lib/smbconf/smbconf_util.c b/lib/smbconf/smbconf_util.c
index b309a3454b1..86a95988f10 100644
--- a/lib/smbconf/smbconf_util.c
+++ b/lib/smbconf/smbconf_util.c
@@ -39,43 +39,43 @@ static int smbconf_destroy_ctx(struct smbconf_ctx *ctx)
  * After the work with the configuration is completed, smbconf_shutdown()
  * should be called.
  */
-WERROR smbconf_init_internal(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
+sbcErr smbconf_init_internal(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 			     const char *path, struct smbconf_ops *ops)
 {
-	WERROR werr = WERR_OK;
+	sbcErr err = SBC_ERR_OK;
 	struct smbconf_ctx *ctx;
 
 	if (conf_ctx == NULL) {
-		return WERR_INVALID_PARAM;
+		return SBC_ERR_INVALID_PARAM;
 	}
 
 	ctx = talloc_zero(mem_ctx, struct smbconf_ctx);
 	if (ctx == NULL) {
-		return WERR_NOMEM;
+		return SBC_ERR_NOMEM;
 	}
 
 	ctx->ops = ops;
 
-	werr = ctx->ops->init(ctx, path);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = ctx->ops->init(ctx, path);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto fail;
 	}
 
 	talloc_set_destructor(ctx, smbconf_destroy_ctx);
 
 	*conf_ctx = ctx;
-	return werr;
+	return err;
 
 fail:
 	talloc_free(ctx);
-	return werr;
+	return err;
 }
 
 
 /**
  * add a string to a talloced array of strings.
  */
-WERROR smbconf_add_string_to_array(TALLOC_CTX *mem_ctx,
+sbcErr smbconf_add_string_to_array(TALLOC_CTX *mem_ctx,
 				   char ***array,
 				   uint32_t count,
 				   const char *string)
@@ -83,12 +83,12 @@ WERROR smbconf_add_string_to_array(TALLOC_CTX *mem_ctx,
 	char **new_array = NULL;
 
 	if (array == NULL) {
-		return WERR_INVALID_PARAM;
+		return SBC_ERR_INVALID_PARAM;
 	}
 
 	new_array = talloc_realloc(mem_ctx, *array, char *, count + 1);
 	if (new_array == NULL) {
-		return WERR_NOMEM;
+		return SBC_ERR_NOMEM;
 	}
 
 	if (string == NULL) {
@@ -97,13 +97,13 @@ WERROR smbconf_add_string_to_array(TALLOC_CTX *mem_ctx,
 		new_array[count] = talloc_strdup(new_array, string);
 		if (new_array[count] == NULL) {
 			talloc_free(new_array);
-			return WERR_NOMEM;
+			return SBC_ERR_NOMEM;
 		}
 	}
 
 	*array = new_array;
 
-	return WERR_OK;
+	return SBC_ERR_OK;
 }
 
 bool smbconf_find_in_array(const char *string, char **list,
diff --git a/source3/lib/interfaces.c b/lib/socket/interfaces.c
index e40eaa91425..618714d1a73 100644
--- a/source3/lib/interfaces.c
+++ b/lib/socket/interfaces.c
@@ -3,6 +3,7 @@
    return a list of network interfaces
    Copyright (C) Andrew Tridgell 1998
    Copyright (C) Jeremy Allison 2007
+   Copyright (C) Jelmer Vernooij 2007
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -18,8 +19,11 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+
 #include "includes.h"
+#include "system/network.h"
 #include "interfaces.h"
+#include "lib/util/tsort.h"
 
 /****************************************************************************
  Create a struct sockaddr_storage with the netmask bits set to 1.
@@ -81,13 +85,13 @@ static void make_bcast_or_net(struct sockaddr_storage *pss_out,
 #if defined(HAVE_IPV6)
 	if (pss_in->ss_family == AF_INET6) {
 		p = (char *)&((struct sockaddr_in6 *)pss_out)->sin6_addr;
-		pmask = (char *)&((struct sockaddr_in6 *)nmask)->sin6_addr;
+		pmask = discard_const_p(char, &((struct sockaddr_in6 *)nmask)->sin6_addr);
 		len = 16;
 	}
 #endif
 	if (pss_in->ss_family == AF_INET) {
 		p = (char *)&((struct sockaddr_in *)pss_out)->sin_addr;
-		pmask = (char *)&((struct sockaddr_in *)nmask)->sin_addr;
+		pmask = discard_const_p(char, &((struct sockaddr_in *)nmask)->sin_addr);
 		len = 4;
 	}
 
@@ -115,6 +119,7 @@ void make_net(struct sockaddr_storage *pss_out,
 	make_bcast_or_net(pss_out, pss_in, nmask, false);
 }
 
+
 /****************************************************************************
  Try the "standard" getifaddrs/freeifaddrs interfaces.
  Also gets IPv6 interfaces.
@@ -296,4 +301,3 @@ int get_interfaces(TALLOC_CTX *mem_ctx, struct iface_struct **pifaces)
 	*pifaces = ifaces;
 	return total;
 }
-
diff --git a/source3/include/interfaces.h b/lib/socket/interfaces.h
index 6ba0e21f6dd..b4e113dcc82 100644
--- a/source3/include/interfaces.h
+++ b/lib/socket/interfaces.h
@@ -1,7 +1,9 @@
 /*
    Unix SMB/CIFS implementation.
-   Machine customisation and include handling
-   Copyright (C) Jeremy Allison <jra@samba.org> 2007
+
+   structures for lib/netif/
+
+   Copyright (C) Andrew Tridgell 2004
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -16,16 +18,8 @@
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-/*
-   This structure is used by lib/interfaces.c to return the list of network
-   interfaces on the machine
-*/
 
-#ifndef _INTERFACES_H
-#define _INTERFACES_H
-
-#include "../replace/replace.h"
-#include "../replace/system/network.h"
+#include "system/network.h"
 
 struct iface_struct {
 	char name[16];
@@ -35,6 +29,8 @@ struct iface_struct {
 	struct sockaddr_storage bcast;
 };
 
+struct interface;
+
 bool make_netmask(struct sockaddr_storage *pss_out,
 		  const struct sockaddr_storage *pss_in,
 		  unsigned long masklen);
@@ -44,6 +40,5 @@ void make_bcast(struct sockaddr_storage *pss_out,
 void make_net(struct sockaddr_storage *pss_out,
 	      const struct sockaddr_storage *pss_in,
 	      const struct sockaddr_storage *nmask);
-int get_interfaces(TALLOC_CTX *mem_ctx, struct iface_struct **pifaces);
 
-#endif
+int get_interfaces(TALLOC_CTX *mem_ctx, struct iface_struct **pifaces);
diff --git a/lib/socket/wscript_build b/lib/socket/wscript_build
new file mode 100644
index 00000000000..61bde129c58
--- /dev/null
+++ b/lib/socket/wscript_build
@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+bld.SAMBA_LIBRARY('interfaces',
+    source='interfaces.c',
+    deps='samba-util',
+    private_library=True
+    )
diff --git a/lib/talloc/talloc.3.xml b/lib/talloc/talloc.3.xml
index a327922dbea..99e8bcdb2f1 100644
--- a/lib/talloc/talloc.3.xml
+++ b/lib/talloc/talloc.3.xml
@@ -783,9 +783,9 @@ if (ptr) memcpy(ptr, p, strlen(p)+1);</programlisting>
     </para>
     <para>
       This program is free software; you can redistribute it and/or modify
-      it under the terms of the GNU General Public License as published by
-      the Free Software Foundation; either version 3 of the License, or (at
-      your option) any later version.
+      it under the terms of the GNU Lesser General Public License as 
+      published by the Free Software Foundation; either version 3 of the
+      License, or (at your option) any later version.
     </para>
     <para>
       This program is distributed in the hope that it will be useful, but
diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c
index 91452bfadac..4700aa99e8c 100644
--- a/lib/talloc/talloc.c
+++ b/lib/talloc/talloc.c
@@ -178,6 +178,32 @@ static struct {
 	TC_INVALIDATE_SHRINK_VALGRIND_CHUNK(_tc, _new_size); \
 } while (0)
 
+#define TC_UNDEFINE_SHRINK_FILL_CHUNK(_tc, _new_size) do { \
+	if (unlikely(talloc_fill.enabled)) { \
+		size_t _flen = (_tc)->size - (_new_size); \
+		char *_fptr = (char *)TC_PTR_FROM_CHUNK(_tc); \
+		_fptr += (_new_size); \
+		memset(_fptr, talloc_fill.fill_value, _flen); \
+	} \
+} while (0)
+
+#if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED)
+/* Mark the unused bytes as undefined */
+#define TC_UNDEFINE_SHRINK_VALGRIND_CHUNK(_tc, _new_size) do { \
+	size_t _flen = (_tc)->size - (_new_size); \
+	char *_fptr = (char *)TC_PTR_FROM_CHUNK(_tc); \
+	_fptr += (_new_size); \
+	VALGRIND_MAKE_MEM_UNDEFINED(_fptr, _flen); \
+} while (0)
+#else
+#define TC_UNDEFINE_SHRINK_VALGRIND_CHUNK(_tc, _new_size) do { } while (0)
+#endif
+
+#define TC_UNDEFINE_SHRINK_CHUNK(_tc, _new_size) do { \
+	TC_UNDEFINE_SHRINK_FILL_CHUNK(_tc, _new_size); \
+	TC_UNDEFINE_SHRINK_VALGRIND_CHUNK(_tc, _new_size); \
+} while (0)
+
 #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED)
 /* Mark the new bytes as undefined */
 #define TC_UNDEFINE_GROW_VALGRIND_CHUNK(_tc, _new_size) do { \
@@ -683,6 +709,69 @@ _PUBLIC_ void *_talloc_reference_loc(const void *context, const void *ptr, const
 
 static void *_talloc_steal_internal(const void *new_ctx, const void *ptr);
 
+static inline void _talloc_free_poolmem(struct talloc_chunk *tc,
+					const char *location)
+{
+	struct talloc_chunk *pool;
+	void *next_tc;
+	unsigned int *pool_object_count;
+
+	pool = (struct talloc_chunk *)tc->pool;
+	next_tc = TC_POOLMEM_NEXT_CHUNK(tc);
+
+	tc->flags |= TALLOC_FLAG_FREE;
+
+	/* we mark the freed memory with where we called the free
+	 * from. This means on a double free error we can report where
+	 * the first free came from
+	 */
+	tc->name = location;
+
+	TC_INVALIDATE_FULL_CHUNK(tc);
+
+	pool_object_count = talloc_pool_objectcount(pool);
+
+	if (unlikely(*pool_object_count == 0)) {
+		talloc_abort("Pool object count zero!");
+		return;
+	}
+
+	*pool_object_count -= 1;
+
+	if (unlikely(*pool_object_count == 1 && !(pool->flags & TALLOC_FLAG_FREE))) {
+		/*
+		 * if there is just one object left in the pool
+		 * and pool->flags does not have TALLOC_FLAG_FREE,
+		 * it means this is the pool itself and
+		 * the rest is available for new objects
+		 * again.
+		 */
+		pool->pool = TC_POOL_FIRST_CHUNK(pool);
+		TC_INVALIDATE_POOL(pool);
+	} else if (unlikely(*pool_object_count == 0)) {
+		/*
+		 * we mark the freed memory with where we called the free
+		 * from. This means on a double free error we can report where
+		 * the first free came from
+		 */
+		pool->name = location;
+
+		TC_INVALIDATE_FULL_CHUNK(pool);
+		free(pool);
+	} else if (pool->pool == next_tc) {
+		/*
+		 * if pool->pool still points to end of
+		 * 'tc' (which is stored in the 'next_tc' variable),
+		 * we can reclaim the memory of 'tc'.
+		 */
+		pool->pool = tc;
+	}
+}
+
+static inline void _talloc_free_children_internal(struct talloc_chunk *tc,
+						  void *ptr,
+						  const char *location);
+
 /* 
    internal talloc_free call
 */
@@ -753,41 +842,7 @@ static inline int _talloc_free_internal(void *ptr, const char *location)
 
 	tc->flags |= TALLOC_FLAG_LOOP;
 
-	while (tc->child) {
-		/* we need to work out who will own an abandoned child
-		   if it cannot be freed. In priority order, the first
-		   choice is owner of any remaining reference to this
-		   pointer, the second choice is our parent, and the
-		   final choice is the null context. */
-		void *child = TC_PTR_FROM_CHUNK(tc->child);
-		const void *new_parent = null_context;
-		struct talloc_chunk *old_parent = NULL;
-		if (unlikely(tc->child->refs)) {
-			struct talloc_chunk *p = talloc_parent_chunk(tc->child->refs);
-			if (p) new_parent = TC_PTR_FROM_CHUNK(p);
-		}
-		/* finding the parent here is potentially quite
-		   expensive, but the alternative, which is to change
-		   talloc to always have a valid tc->parent pointer,
-		   makes realloc more expensive where there are a
-		   large number of children.
-
-		   The reason we need the parent pointer here is that
-		   if _talloc_free_internal() fails due to references
-		   or a failing destructor we need to re-parent, but
-		   the free call can invalidate the prev pointer.
-		*/
-		if (new_parent == null_context && (tc->child->refs || tc->child->destructor)) {
-			old_parent = talloc_parent_chunk(ptr);
-		}
-		if (unlikely(_talloc_free_internal(child, location) == -1)) {
-			if (new_parent == null_context) {
-				struct talloc_chunk *p = old_parent;
-				if (p) new_parent = TC_PTR_FROM_CHUNK(p);
-			}
-			_talloc_steal_internal(new_parent, child);
-		}
-	}
+	_talloc_free_children_internal(tc, ptr, location);
 
 	tc->flags |= TALLOC_FLAG_FREE;
 
@@ -797,21 +852,10 @@ static inline int _talloc_free_internal(void *ptr, const char *location)
 	 */	 
 	tc->name = location;
 
-	if (tc->flags & (TALLOC_FLAG_POOL|TALLOC_FLAG_POOLMEM)) {
-		struct talloc_chunk *pool;
-		void *next_tc = NULL;
+	if (tc->flags & TALLOC_FLAG_POOL) {
 		unsigned int *pool_object_count;
 
-		if (unlikely(tc->flags & TALLOC_FLAG_POOL)) {
-			pool = tc;
-		} else {
-			pool = (struct talloc_chunk *)tc->pool;
-			next_tc = TC_POOLMEM_NEXT_CHUNK(tc);
-
-			TC_INVALIDATE_FULL_CHUNK(tc);
-		}
-
-		pool_object_count = talloc_pool_objectcount(pool);
+		pool_object_count = talloc_pool_objectcount(tc);
 
 		if (unlikely(*pool_object_count == 0)) {
 			talloc_abort("Pool object count zero!");
@@ -820,26 +864,12 @@ static inline int _talloc_free_internal(void *ptr, const char *location)
 
 		*pool_object_count -= 1;
 
-		if (unlikely(*pool_object_count == 1)) {
-			/*
-			 * if there is just object left in the pool
-			 * it means this is the pool itself and
-			 * the rest is available for new objects
-			 * again.
-			 */
-			pool->pool = TC_POOL_FIRST_CHUNK(pool);
-			TC_INVALIDATE_POOL(pool);
-		} else if (unlikely(*pool_object_count == 0)) {
-			TC_INVALIDATE_FULL_CHUNK(pool);
-			free(pool);
-		} else if (pool->pool == next_tc) {
-			/*
-			 * if pool->pool still points to end of
-			 * 'tc' (which is stored in the 'next_tc' variable),
-			 * we can reclaim the memory of 'tc'.
-			 */
-			pool->pool = tc;
+		if (unlikely(*pool_object_count == 0)) {
+			TC_INVALIDATE_FULL_CHUNK(tc);
+			free(tc);
 		}
+	} else if (tc->flags & TALLOC_FLAG_POOLMEM) {
+		_talloc_free_poolmem(tc, location);
 	} else {
 		TC_INVALIDATE_FULL_CHUNK(tc);
 		free(tc);
@@ -1204,21 +1234,10 @@ _PUBLIC_ void *talloc_init(const char *fmt, ...)
 	return ptr;
 }
 
-/*
-  this is a replacement for the Samba3 talloc_destroy_pool functionality. It
-  should probably not be used in new code. It's in here to keep the talloc
-  code consistent across Samba 3 and 4.
-*/
-_PUBLIC_ void talloc_free_children(void *ptr)
+static inline void _talloc_free_children_internal(struct talloc_chunk *tc,
+						  void *ptr,
+						  const char *location)
 {
-	struct talloc_chunk *tc;
-
-	if (unlikely(ptr == NULL)) {
-		return;
-	}
-
-	tc = talloc_chunk_from_ptr(ptr);
-
 	while (tc->child) {
 		/* we need to work out who will own an abandoned child
 		   if it cannot be freed. In priority order, the first
@@ -1227,13 +1246,28 @@ _PUBLIC_ void talloc_free_children(void *ptr)
 		   final choice is the null context. */
 		void *child = TC_PTR_FROM_CHUNK(tc->child);
 		const void *new_parent = null_context;
+		struct talloc_chunk *old_parent = NULL;
 		if (unlikely(tc->child->refs)) {
 			struct talloc_chunk *p = talloc_parent_chunk(tc->child->refs);
 			if (p) new_parent = TC_PTR_FROM_CHUNK(p);
 		}
-		if (unlikely(talloc_free(child) == -1)) {
+		/* finding the parent here is potentially quite
+		   expensive, but the alternative, which is to change
+		   talloc to always have a valid tc->parent pointer,
+		   makes realloc more expensive where there are a
+		   large number of children.
+
+		   The reason we need the parent pointer here is that
+		   if _talloc_free_internal() fails due to references
+		   or a failing destructor we need to re-parent, but
+		   the free call can invalidate the prev pointer.
+		*/
+		if (new_parent == null_context && (tc->child->refs || tc->child->destructor)) {
+			old_parent = talloc_parent_chunk(ptr);
+		}
+		if (unlikely(_talloc_free_internal(child, location) == -1)) {
 			if (new_parent == null_context) {
-				struct talloc_chunk *p = talloc_parent_chunk(ptr);
+				struct talloc_chunk *p = old_parent;
 				if (p) new_parent = TC_PTR_FROM_CHUNK(p);
 			}
 			_talloc_steal_internal(new_parent, child);
@@ -1241,6 +1275,24 @@ _PUBLIC_ void talloc_free_children(void *ptr)
 	}
 }
 
+/*
+  this is a replacement for the Samba3 talloc_destroy_pool functionality. It
+  should probably not be used in new code. It's in here to keep the talloc
+  code consistent across Samba 3 and 4.
+*/
+_PUBLIC_ void talloc_free_children(void *ptr)
+{
+	struct talloc_chunk *tc;
+
+	if (unlikely(ptr == NULL)) {
+		return;
+	}
+
+	tc = talloc_chunk_from_ptr(ptr);
+
+	_talloc_free_children_internal(tc, ptr, __location__);
+}
+
 /* 
    Allocate a bit of memory as a child of an existing pointer
 */
@@ -1365,7 +1417,16 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons
 			}
 			return ptr;
 		} else if ((tc->size - size) < 1024) {
-			TC_INVALIDATE_SHRINK_CHUNK(tc, size);
+			/*
+			 * if we call TC_INVALIDATE_SHRINK_CHUNK() here
+			 * we would need to call TC_UNDEFINE_GROW_CHUNK()
+			 * after each realloc call, which slows down
+			 * testing a lot :-(.
+			 *
+			 * That is why we only mark memory as undefined here.
+			 */
+			TC_UNDEFINE_SHRINK_CHUNK(tc, size);
+
 			/* do not shrink if we have less than 1k to gain */
 			tc->size = size;
 			return ptr;
@@ -1410,8 +1471,13 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons
 		size_t new_chunk_size = TC_ALIGN16(TC_HDR_SIZE + size);
 		size_t space_needed;
 		size_t space_left;
+		unsigned int chunk_count = *talloc_pool_objectcount(pool_tc);
+
+		if (!(pool_tc->flags & TALLOC_FLAG_FREE)) {
+			chunk_count -= 1;
+		}
 
-		if (*talloc_pool_objectcount(pool_tc) == 2) {
+		if (chunk_count == 1) {
 			/*
 			 * optimize for the case where 'tc' is the only
 			 * chunk in the pool.
@@ -1438,6 +1504,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons
 				memmove(pool_tc->pool, tc, old_used);
 				new_ptr = pool_tc->pool;
 
+				tc = (struct talloc_chunk *)new_ptr;
 				TC_UNDEFINE_GROW_CHUNK(tc, size);
 
 				/*
@@ -1481,7 +1548,6 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons
 		}
 
 		new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE);
-		*talloc_pool_objectcount(pool_tc) -= 1;
 
 		if (new_ptr == NULL) {
 			new_ptr = malloc(TC_HDR_SIZE+size);
@@ -1490,21 +1556,8 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons
 
 		if (new_ptr) {
 			memcpy(new_ptr, tc, MIN(tc->size,size) + TC_HDR_SIZE);
-			TC_INVALIDATE_FULL_CHUNK(tc);
 
-			if (*talloc_pool_objectcount(pool_tc) == 1) {
-				/*
-				 * If the pool is empty now reclaim everything.
-				 */
-				pool_tc->pool = TC_POOL_FIRST_CHUNK(pool_tc);
-				TC_INVALIDATE_POOL(pool_tc);
-			} else if (next_tc == pool_tc->pool) {
-				/*
-				 * If it was reallocated and tc was the last
-				 * chunk, we can reclaim the memory of tc.
-				 */
-				pool_tc->pool = tc;
-			}
+			_talloc_free_poolmem(tc, __location__ "_talloc_realloc");
 		}
 	}
 	else {
diff --git a/lib/talloc/testsuite.c b/lib/talloc/testsuite.c
index ba583ab84ec..90417c6ade5 100644
--- a/lib/talloc/testsuite.c
+++ b/lib/talloc/testsuite.c
@@ -1128,23 +1128,31 @@ static bool test_pool(void)
 	pool = talloc_pool(NULL, 1024);
 
 	p1 = talloc_size(pool, 80);
+	memset(p1, 0x11, talloc_get_size(p1));
 	p2 = talloc_size(pool, 20);
+	memset(p2, 0x11, talloc_get_size(p2));
 	p3 = talloc_size(p1, 50);
+	memset(p3, 0x11, talloc_get_size(p3));
 	p4 = talloc_size(p3, 1000);
+	memset(p4, 0x11, talloc_get_size(p4));
 
 #if 1 /* this relies on ALWAYS_REALLOC == 0 in talloc.c */
 	p2_2 = talloc_realloc_size(pool, p2, 20+1);
 	torture_assert("pool realloc 20+1", p2_2 == p2, "failed: pointer changed");
+	memset(p2, 0x11, talloc_get_size(p2));
 	p2_2 = talloc_realloc_size(pool, p2, 20-1);
 	torture_assert("pool realloc 20-1", p2_2 == p2, "failed: pointer changed");
+	memset(p2, 0x11, talloc_get_size(p2));
 	p2_2 = talloc_realloc_size(pool, p2, 20-1);
 	torture_assert("pool realloc 20-1", p2_2 == p2, "failed: pointer changed");
+	memset(p2, 0x11, talloc_get_size(p2));
 
 	talloc_free(p3);
 
 	/* this should reclaim the memory of p4 and p3 */
 	p2_2 = talloc_realloc_size(pool, p2, 400);
 	torture_assert("pool realloc 400", p2_2 == p2, "failed: pointer changed");
+	memset(p2, 0x11, talloc_get_size(p2));
 
 	talloc_free(p1);
 
@@ -1152,37 +1160,46 @@ static bool test_pool(void)
 	p2_2 = talloc_realloc_size(pool, p2, 800);
 	torture_assert("pool realloc 800", p2_2 == p1, "failed: pointer not changed");
 	p2 = p2_2;
+	memset(p2, 0x11, talloc_get_size(p2));
 
 	/* this should do a malloc */
 	p2_2 = talloc_realloc_size(pool, p2, 1800);
 	torture_assert("pool realloc 1800", p2_2 != p2, "failed: pointer not changed");
 	p2 = p2_2;
+	memset(p2, 0x11, talloc_get_size(p2));
 
 	/* this should reclaim the memory from the pool */
 	p3 = talloc_size(pool, 80);
 	torture_assert("pool alloc 80", p3 == p1, "failed: pointer changed");
+	memset(p3, 0x11, talloc_get_size(p3));
 
 	talloc_free(p2);
 	talloc_free(p3);
 
 	p1 = talloc_size(pool, 80);
+	memset(p1, 0x11, talloc_get_size(p1));
 	p2 = talloc_size(pool, 20);
+	memset(p2, 0x11, talloc_get_size(p2));
 
 	talloc_free(p1);
 
 	p2_2 = talloc_realloc_size(pool, p2, 20-1);
 	torture_assert("pool realloc 20-1", p2_2 == p2, "failed: pointer changed");
+	memset(p2, 0x11, talloc_get_size(p2));
 	p2_2 = talloc_realloc_size(pool, p2, 20-1);
 	torture_assert("pool realloc 20-1", p2_2 == p2, "failed: pointer changed");
+	memset(p2, 0x11, talloc_get_size(p2));
 
 	/* this should do a malloc */
 	p2_2 = talloc_realloc_size(pool, p2, 1800);
 	torture_assert("pool realloc 1800", p2_2 != p2, "failed: pointer not changed");
 	p2 = p2_2;
+	memset(p2, 0x11, talloc_get_size(p2));
 
 	/* this should reclaim the memory from the pool */
 	p3 = talloc_size(pool, 800);
 	torture_assert("pool alloc 800", p3 == p1, "failed: pointer changed");
+	memset(p3, 0x11, talloc_get_size(p3));
 
 #endif /* this relies on ALWAYS_REALLOC == 0 in talloc.c */
 
@@ -1191,6 +1208,73 @@ static bool test_pool(void)
 	return true;
 }
 
+static bool test_pool_steal(void)
+{
+	void *root;
+	void *pool;
+	void *p1, *p2;
+	void *p1_2, *p2_2;
+	size_t hdr;
+	size_t ofs1, ofs2;
+
+	root = talloc_new(NULL);
+	pool = talloc_pool(root, 1024);
+
+	p1 = talloc_size(pool, 4 * 16);
+	torture_assert("pool allocate 4 * 16", p1 != NULL, "failed ");
+	memset(p1, 0x11, talloc_get_size(p1));
+	p2 = talloc_size(pool, 4 * 16);
+	torture_assert("pool allocate 4 * 16", p2 > p1, "failed: !(p2 > p1) ");
+	memset(p2, 0x11, talloc_get_size(p2));
+
+	ofs1 = PTR_DIFF(p2, p1);
+	hdr = ofs1 - talloc_get_size(p1);
+
+	talloc_steal(root, p1);
+	talloc_steal(root, p2);
+
+	talloc_free(pool);
+
+	p1_2 = p1;
+
+#if 1 /* this relies on ALWAYS_REALLOC == 0 in talloc.c */
+	p1_2 = talloc_realloc_size(root, p1, 5 * 16);
+	torture_assert("pool realloc 5 * 16", p1_2 > p2, "failed: pointer not changed");
+	memset(p1_2, 0x11, talloc_get_size(p1_2));
+	ofs1 = PTR_DIFF(p1_2, p2);
+	ofs2 = talloc_get_size(p2) + hdr;
+
+	torture_assert("pool realloc ", ofs1 == ofs2, "failed: pointer offset unexpected");
+
+	p2_2 = talloc_realloc_size(root, p2, 3 * 16);
+	torture_assert("pool realloc 5 * 16", p2_2 == p2, "failed: pointer changed");
+	memset(p2_2, 0x11, talloc_get_size(p2_2));
+#endif /* this relies on ALWAYS_REALLOC == 0 in talloc.c */
+
+	talloc_free(p1_2);
+
+	p2_2 = p2;
+
+#if 1 /* this relies on ALWAYS_REALLOC == 0 in talloc.c */
+	/* now we should reclaim the full pool */
+	p2_2 = talloc_realloc_size(root, p2, 8 * 16);
+	torture_assert("pool realloc 8 * 16", p2_2 == p1, "failed: pointer not expected");
+	p2 = p2_2;
+	memset(p2_2, 0x11, talloc_get_size(p2_2));
+
+	/* now we malloc and free the full pool space */
+	p2_2 = talloc_realloc_size(root, p2, 2 * 1024);
+	torture_assert("pool realloc 2 * 1024", p2_2 != p1, "failed: pointer not expected");
+	memset(p2_2, 0x11, talloc_get_size(p2_2));
+
+#endif /* this relies on ALWAYS_REALLOC == 0 in talloc.c */
+
+	talloc_free(p2_2);
+
+	talloc_free(root);
+
+	return true;
+}
 
 static bool test_free_ref_null_context(void)
 {
@@ -1290,6 +1374,8 @@ bool torture_local_talloc(struct torture_context *tctx)
 	test_reset();
 	ret &= test_pool();
 	test_reset();
+	ret &= test_pool_steal();
+	test_reset();
 	ret &= test_free_ref_null_context();
 	test_reset();
 	ret &= test_rusty();
diff --git a/lib/tdb/common/hash.c b/lib/tdb/common/hash.c
index 2472ed1acec..1eed7221d2a 100644
--- a/lib/tdb/common/hash.c
+++ b/lib/tdb/common/hash.c
@@ -214,9 +214,7 @@ static uint32_t hashlittle( const void *key, size_t length )
   u.ptr = key;
   if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) {
     const uint32_t *k = (const uint32_t *)key;         /* read 32-bit chunks */
-#ifdef VALGRIND
     const uint8_t  *k8;
-#endif
 
     /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
     while (length > 12)
@@ -230,36 +228,6 @@ static uint32_t hashlittle( const void *key, size_t length )
     }
 
     /*----------------------------- handle the last (probably partial) block */
-    /*
-     * "k[2]&0xffffff" actually reads beyond the end of the string, but
-     * then masks off the part it's not allowed to read.  Because the
-     * string is aligned, the masked-off tail is in the same word as the
-     * rest of the string.  Every machine with memory protection I've seen
-     * does it on word boundaries, so is OK with this.  But VALGRIND will
-     * still catch it and complain.  The masking trick does make the hash
-     * noticably faster for short strings (like English words).
-     */
-#ifndef VALGRIND
-
-    switch(length)
-    {
-    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
-    case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
-    case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break;
-    case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break;
-    case 8 : b+=k[1]; a+=k[0]; break;
-    case 7 : b+=k[1]&0xffffff; a+=k[0]; break;
-    case 6 : b+=k[1]&0xffff; a+=k[0]; break;
-    case 5 : b+=k[1]&0xff; a+=k[0]; break;
-    case 4 : a+=k[0]; break;
-    case 3 : a+=k[0]&0xffffff; break;
-    case 2 : a+=k[0]&0xffff; break;
-    case 1 : a+=k[0]&0xff; break;
-    case 0 : return c;              /* zero length strings require no mixing */
-    }
-
-#else /* make valgrind happy */
-
     k8 = (const uint8_t *)k;
     switch(length)
     {
@@ -277,9 +245,6 @@ static uint32_t hashlittle( const void *key, size_t length )
     case 1 : a+=k8[0]; break;
     case 0 : return c;
     }
-
-#endif /* !valgrind */
-
   } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) {
     const uint16_t *k = (const uint16_t *)key;         /* read 16-bit chunks */
     const uint8_t  *k8;
diff --git a/lib/tdb/pytdb.c b/lib/tdb/pytdb.c
index 0faba562de9..3dd785e7be8 100644
--- a/lib/tdb/pytdb.c
+++ b/lib/tdb/pytdb.c
@@ -558,6 +558,7 @@ static PyMethodDef tdb_methods[] = {
 	{ NULL }
 };
 
+void inittdb(void);
 void inittdb(void)
 {
 	PyObject *m;
diff --git a/lib/tdb/python/tests/simple.py b/lib/tdb/python/tests/simple.py
index f5484a05235..2877092fe31 100644
--- a/lib/tdb/python/tests/simple.py
+++ b/lib/tdb/python/tests/simple.py
@@ -20,8 +20,13 @@ class OpenTdbTests(TestCase):
 class CloseTdbTests(TestCase):
 
     def test_double_close(self):
-        self.tdb = tdb.Tdb(tempfile.mkstemp()[1], 0, tdb.DEFAULT,
-                os.O_CREAT|os.O_RDWR)
+        # No hash size in tdb2.
+        if tdb.__version__.startswith("2"):
+            self.tdb = tdb.Tdb(tempfile.mkstemp()[1], tdb.DEFAULT,
+                               os.O_CREAT|os.O_RDWR)
+        else:
+            self.tdb = tdb.Tdb(tempfile.mkstemp()[1], 0, tdb.DEFAULT,
+                               os.O_CREAT|os.O_RDWR)
         self.assertNotEqual(None, self.tdb)
 
         # ensure that double close does not crash python
@@ -42,8 +47,12 @@ class SimpleTdbTests(TestCase):
 
     def setUp(self):
         super(SimpleTdbTests, self).setUp()
-        self.tdb = tdb.Tdb(tempfile.mkstemp()[1], 0, tdb.DEFAULT,
-                os.O_CREAT|os.O_RDWR)
+        if tdb.__version__.startswith("2"):
+            self.tdb = tdb.Tdb(tempfile.mkstemp()[1], tdb.DEFAULT,
+                               os.O_CREAT|os.O_RDWR)
+        else:
+            self.tdb = tdb.Tdb(tempfile.mkstemp()[1], 0, tdb.DEFAULT,
+                               os.O_CREAT|os.O_RDWR)
         self.assertNotEqual(None, self.tdb)
 
     def tearDown(self):
@@ -56,7 +65,8 @@ class SimpleTdbTests(TestCase):
         self.tdb.lock_all()
 
     def test_max_dead(self):
-        self.tdb.max_dead = 20
+        if not tdb.__version__.startswith("2"):
+            self.tdb.max_dead = 20
 
     def test_unlockall(self):
         self.tdb.lock_all()
@@ -67,7 +77,8 @@ class SimpleTdbTests(TestCase):
         self.tdb.read_unlock_all()
 
     def test_reopen(self):
-        self.tdb.reopen()
+        if not tdb.__version__.startswith("2"):
+            self.tdb.reopen()
 
     def test_store(self):
         self.tdb.store("bar", "bla")
@@ -75,7 +86,8 @@ class SimpleTdbTests(TestCase):
 
     def test_getitem(self):
         self.tdb["bar"] = "foo"
-        self.tdb.reopen()
+        if not tdb.__version__.startswith("2"):
+            self.tdb.reopen()
         self.assertEquals("foo", self.tdb["bar"])
 
     def test_delete(self):
@@ -91,13 +103,16 @@ class SimpleTdbTests(TestCase):
         self.assertRaises(KeyError, lambda: self.tdb["bla"])
 
     def test_hash_size(self):
-        self.tdb.hash_size
+        if not tdb.__version__.startswith("2"):
+            self.tdb.hash_size
 
     def test_map_size(self):
-        self.tdb.map_size
+        if not tdb.__version__.startswith("2"):
+            self.tdb.map_size
 
     def test_freelist_size(self):
-        self.tdb.freelist_size
+        if not tdb.__version__.startswith("2"):
+            self.tdb.freelist_size
 
     def test_name(self):
         self.tdb.filename
@@ -105,7 +120,9 @@ class SimpleTdbTests(TestCase):
     def test_iterator(self):
         self.tdb["bla"] = "1"
         self.tdb["brainslug"] = "2"
-        self.assertEquals(["bla", "brainslug"], list(self.tdb))
+        l = list(self.tdb)
+        l.sort()
+        self.assertEquals(["bla", "brainslug"], l)
 
     def test_transaction_cancel(self):
         self.tdb["bloe"] = "2"
@@ -143,17 +160,19 @@ class SimpleTdbTests(TestCase):
         self.assertEquals(0, len(list(self.tdb)))
 
     def test_repack(self):
-        self.tdb["foo"] = "abc"
-        self.tdb["bar"] = "def"
-        del self.tdb["foo"]
-        self.tdb.repack()
+        if not tdb.__version__.startswith("2"):
+            self.tdb["foo"] = "abc"
+            self.tdb["bar"] = "def"
+            del self.tdb["foo"]
+            self.tdb.repack()
 
     def test_seqnum(self):
-        self.tdb.enable_seqnum()
-        seq1 = self.tdb.seqnum
-        self.tdb.increment_seqnum_nonblock()
-        seq2 = self.tdb.seqnum
-        self.assertEquals(seq2-seq1, 1)
+        if not tdb.__version__.startswith("2"):
+            self.tdb.enable_seqnum()
+            seq1 = self.tdb.seqnum
+            self.tdb.increment_seqnum_nonblock()
+            seq2 = self.tdb.seqnum
+            self.assertEquals(seq2-seq1, 1)
 
     def test_len(self):
         self.assertEquals(0, len(list(self.tdb)))
@@ -161,8 +180,12 @@ class SimpleTdbTests(TestCase):
         self.assertEquals(1, len(list(self.tdb)))
 
     def test_add_flags(self):
-        self.tdb.add_flags(tdb.NOMMAP)
-        self.tdb.remove_flags(tdb.NOMMAP)
+        if tdb.__version__.startswith("2"):
+            self.tdb.add_flag(tdb.NOMMAP)
+            self.tdb.remove_flag(tdb.NOMMAP)
+        else:
+            self.tdb.add_flags(tdb.NOMMAP)
+            self.tdb.remove_flags(tdb.NOMMAP)
 
 
 class VersionTests(TestCase):
diff --git a/lib/tdb/tools/tdbrestore.c b/lib/tdb/tools/tdbrestore.c
index 95ee3606474..1daac63db18 100644
--- a/lib/tdb/tools/tdbrestore.c
+++ b/lib/tdb/tools/tdbrestore.c
@@ -170,7 +170,7 @@ static int read_rec(FILE *f, TDB_CONTEXT *tdb, int *eof)
 	    || (swallow(f, "}\n", NULL) == -1)) {
 		goto fail;
 	}
-	if (tdb_store(tdb, key, data, TDB_INSERT) == -1) {
+	if (tdb_store(tdb, key, data, TDB_INSERT) != 0) {
 		fprintf(stderr, "TDB error: %s\n", tdb_errorstr(tdb));
 		goto fail;
 	}
diff --git a/lib/tdb/tools/tdbtool.c b/lib/tdb/tools/tdbtool.c
index cd17f79e325..99d4841cf39 100644
--- a/lib/tdb/tools/tdbtool.c
+++ b/lib/tdb/tools/tdbtool.c
@@ -257,7 +257,7 @@ static void insert_tdb(char *keyname, size_t keylen, char* data, size_t datalen)
 	dbuf.dptr = (unsigned char *)data;
 	dbuf.dsize = datalen;
 
-	if (tdb_store(tdb, key, dbuf, TDB_INSERT) == -1) {
+	if (tdb_store(tdb, key, dbuf, TDB_INSERT) != 0) {
 		terror("insert failed");
 	}
 }
@@ -284,7 +284,7 @@ static void store_tdb(char *keyname, size_t keylen, char* data, size_t datalen)
 	printf("Storing key:\n");
 	print_rec(tdb, key, dbuf, NULL);
 
-	if (tdb_store(tdb, key, dbuf, TDB_REPLACE) == -1) {
+	if (tdb_store(tdb, key, dbuf, TDB_REPLACE) != 0) {
 		terror("store failed");
 	}
 }
@@ -363,7 +363,7 @@ static void move_rec(char *keyname, size_t keylen, char* tdbname)
 		return;
 	}
 	
-	if ( tdb_store( dst_tdb, key, dbuf, TDB_REPLACE ) == -1 ) {
+	if (tdb_store( dst_tdb, key, dbuf, TDB_REPLACE ) != 0) {
 		terror("failed to move record");
 	}
 	else
diff --git a/lib/tdb2/LICENSE b/lib/tdb2/LICENSE
new file mode 100644
index 00000000000..cca7fc278f5
--- /dev/null
+++ b/lib/tdb2/LICENSE
@@ -0,0 +1,165 @@
+		   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/lib/tdb2/TODO b/lib/tdb2/TODO
new file mode 100644
index 00000000000..0a9374f016b
--- /dev/null
+++ b/lib/tdb2/TODO
@@ -0,0 +1,4 @@
+- tdb2restore, tdb2dump, tdb2backup
+- tdb2tool man page
+- Integrate ccan testsuite
+- Integrate tdb2 testsuite
diff --git a/lib/tdb2/_info b/lib/tdb2/_info
new file mode 100644
index 00000000000..7213d67a22a
--- /dev/null
+++ b/lib/tdb2/_info
@@ -0,0 +1,91 @@
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * tdb2 - [[WORK IN PROGRESS!]] The trivial (64bit transactional) database
+ *
+ * The tdb2 module provides an efficient keyword data mapping (usually
+ * within a file).  It supports transactions, so the contents of the
+ * database is reliable even across crashes.
+ *
+ * Example:
+ *	#include <ccan/tdb2/tdb2.h>
+ *	#include <ccan/str/str.h>
+ *	#include <err.h>
+ *	#include <stdio.h>
+ *
+ *	static void usage(const char *argv0)
+ *	{
+ *		errx(1, "Usage: %s fetch <dbfile> <key>\n"
+ *		     "OR %s store <dbfile> <key> <data>", argv0, argv0);
+ *	}
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct tdb_context *tdb;
+ *		TDB_DATA key, value;
+ *		enum TDB_ERROR error;
+ *
+ *		if (argc < 4)
+ *			usage(argv[0]);
+ *
+ *		tdb = tdb_open(argv[2], TDB_DEFAULT, O_CREAT|O_RDWR,0600, NULL);
+ *		if (!tdb)
+ *			err(1, "Opening %s", argv[2]);
+ *
+ *		key.dptr = (void *)argv[3];
+ *		key.dsize = strlen(argv[3]);
+ *
+ *		if (streq(argv[1], "fetch")) {
+ *			if (argc != 4)
+ *				usage(argv[0]);
+ *			error = tdb_fetch(tdb, key, &value);
+ *			if (error)
+ *				errx(1, "fetch %s: %s",
+ *				     argv[3], tdb_errorstr(error));
+ *			printf("%.*s\n", value.dsize, (char *)value.dptr);
+ *			free(value.dptr);
+ *		} else if (streq(argv[1], "store")) {
+ *			if (argc != 5)
+ *				usage(argv[0]);
+ *			value.dptr = (void *)argv[4];
+ *			value.dsize = strlen(argv[4]);
+ *			error = tdb_store(tdb, key, value, 0);
+ *			if (error)
+ *				errx(1, "store %s: %s",
+ *				     argv[3], tdb_errorstr(error));
+ *		} else
+ *			usage(argv[0]);
+ *
+ *		return 0;
+ *	}
+ *
+ * Maintainer: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Author: Rusty Russell
+ *
+ * License: LGPLv3 (or later)
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/asprintf\n");
+		printf("ccan/hash\n");
+		printf("ccan/likely\n");
+		printf("ccan/asearch\n");
+		printf("ccan/compiler\n");
+		printf("ccan/build_assert\n");
+		printf("ccan/ilog\n");
+		printf("ccan/failtest\n");
+		printf("ccan/tally\n");
+		printf("ccan/typesafe_cb\n");
+		printf("ccan/cast\n");
+		printf("ccan/endian\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/tdb2/check.c b/lib/tdb2/check.c
new file mode 100644
index 00000000000..52fb188764d
--- /dev/null
+++ b/lib/tdb2/check.c
@@ -0,0 +1,835 @@
+ /*
+   Trivial Database 2: free list/block handling
+   Copyright (C) Rusty Russell 2010
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "private.h"
+#include <ccan/likely/likely.h>
+#include <ccan/asearch/asearch.h>
+
+/* We keep an ordered array of offsets. */
+static bool append(tdb_off_t **arr, size_t *num, tdb_off_t off)
+{
+	tdb_off_t *new = realloc(*arr, (*num + 1) * sizeof(tdb_off_t));
+	if (!new)
+		return false;
+	new[(*num)++] = off;
+	*arr = new;
+	return true;
+}
+
+static enum TDB_ERROR check_header(struct tdb_context *tdb, tdb_off_t *recovery,
+				   uint64_t *features)
+{
+	uint64_t hash_test;
+	struct tdb_header hdr;
+	enum TDB_ERROR ecode;
+
+	ecode = tdb_read_convert(tdb, 0, &hdr, sizeof(hdr));
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+	/* magic food should not be converted, so convert back. */
+	tdb_convert(tdb, hdr.magic_food, sizeof(hdr.magic_food));
+
+	hash_test = TDB_HASH_MAGIC;
+	hash_test = tdb_hash(tdb, &hash_test, sizeof(hash_test));
+	if (hdr.hash_test != hash_test) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "check: hash test %llu should be %llu",
+				  (long long)hdr.hash_test,
+				  (long long)hash_test);
+	}
+
+	if (strcmp(hdr.magic_food, TDB_MAGIC_FOOD) != 0) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "check: bad magic '%.*s'",
+				  (unsigned)sizeof(hdr.magic_food),
+				  hdr.magic_food);
+	}
+
+	/* Features which are used must be a subset of features offered. */
+	if (hdr.features_used & ~hdr.features_offered) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "check: features used (0x%llx) which"
+				  " are not offered (0x%llx)",
+				  (long long)hdr.features_used,
+				  (long long)hdr.features_offered);
+	}
+
+	*features = hdr.features_offered;
+	*recovery = hdr.recovery;
+	if (*recovery) {
+		if (*recovery < sizeof(hdr)
+		    || *recovery > tdb->file->map_size) {
+			return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+					  "tdb_check:"
+					  " invalid recovery offset %zu",
+					  (size_t)*recovery);
+		}
+	}
+
+	/* Don't check reserved: they *can* be used later. */
+	return TDB_SUCCESS;
+}
+
+static enum TDB_ERROR check_hash_tree(struct tdb_context *tdb,
+				      tdb_off_t off, unsigned int group_bits,
+				      uint64_t hprefix,
+				      unsigned hprefix_bits,
+				      tdb_off_t used[],
+				      size_t num_used,
+				      size_t *num_found,
+				      enum TDB_ERROR (*check)(TDB_DATA,
+							      TDB_DATA, void *),
+				      void *data);
+
+static enum TDB_ERROR check_hash_chain(struct tdb_context *tdb,
+				       tdb_off_t off,
+				       uint64_t hash,
+				       tdb_off_t used[],
+				       size_t num_used,
+				       size_t *num_found,
+				       enum TDB_ERROR (*check)(TDB_DATA,
+							       TDB_DATA,
+							       void *),
+				       void *data)
+{
+	struct tdb_used_record rec;
+	enum TDB_ERROR ecode;
+
+	ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec));
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (rec_magic(&rec) != TDB_CHAIN_MAGIC) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: Bad hash chain magic %llu",
+				  (long long)rec_magic(&rec));
+	}
+
+	if (rec_data_length(&rec) != sizeof(struct tdb_chain)) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check:"
+				  " Bad hash chain length %llu vs %zu",
+				  (long long)rec_data_length(&rec),
+				  sizeof(struct tdb_chain));
+	}
+	if (rec_key_length(&rec) != 0) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: Bad hash chain key length %llu",
+				  (long long)rec_key_length(&rec));
+	}
+	if (rec_hash(&rec) != 0) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: Bad hash chain hash value %llu",
+				  (long long)rec_hash(&rec));
+	}
+
+	off += sizeof(rec);
+	ecode = check_hash_tree(tdb, off, 0, hash, 64,
+				used, num_used, num_found, check, data);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	off = tdb_read_off(tdb, off + offsetof(struct tdb_chain, next));
+	if (TDB_OFF_IS_ERR(off)) {
+		return off;
+	}
+	if (off == 0)
+		return TDB_SUCCESS;
+	(*num_found)++;
+	return check_hash_chain(tdb, off, hash, used, num_used, num_found,
+				check, data);
+}
+
+static enum TDB_ERROR check_hash_record(struct tdb_context *tdb,
+					tdb_off_t off,
+					uint64_t hprefix,
+					unsigned hprefix_bits,
+					tdb_off_t used[],
+					size_t num_used,
+					size_t *num_found,
+					enum TDB_ERROR (*check)(TDB_DATA,
+								TDB_DATA,
+								void *),
+					void *data)
+{
+	struct tdb_used_record rec;
+	enum TDB_ERROR ecode;
+
+	if (hprefix_bits >= 64)
+		return check_hash_chain(tdb, off, hprefix, used, num_used,
+					num_found, check, data);
+
+	ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec));
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (rec_magic(&rec) != TDB_HTABLE_MAGIC) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: Bad hash table magic %llu",
+				  (long long)rec_magic(&rec));
+	}
+	if (rec_data_length(&rec)
+	    != sizeof(tdb_off_t) << TDB_SUBLEVEL_HASH_BITS) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check:"
+				  " Bad hash table length %llu vs %llu",
+				  (long long)rec_data_length(&rec),
+				  (long long)sizeof(tdb_off_t)
+				  << TDB_SUBLEVEL_HASH_BITS);
+	}
+	if (rec_key_length(&rec) != 0) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: Bad hash table key length %llu",
+				  (long long)rec_key_length(&rec));
+	}
+	if (rec_hash(&rec) != 0) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: Bad hash table hash value %llu",
+				  (long long)rec_hash(&rec));
+	}
+
+	off += sizeof(rec);
+	return check_hash_tree(tdb, off,
+			       TDB_SUBLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS,
+			       hprefix, hprefix_bits,
+			       used, num_used, num_found, check, data);
+}
+
+static int off_cmp(const tdb_off_t *a, const tdb_off_t *b)
+{
+	/* Can overflow an int. */
+	return *a > *b ? 1
+		: *a < *b ? -1
+		: 0;
+}
+
+static uint64_t get_bits(uint64_t h, unsigned num, unsigned *used)
+{
+	*used += num;
+
+	return (h >> (64 - *used)) & ((1U << num) - 1);
+}
+
+static enum TDB_ERROR check_hash_tree(struct tdb_context *tdb,
+				      tdb_off_t off, unsigned int group_bits,
+				      uint64_t hprefix,
+				      unsigned hprefix_bits,
+				      tdb_off_t used[],
+				      size_t num_used,
+				      size_t *num_found,
+				      enum TDB_ERROR (*check)(TDB_DATA,
+							      TDB_DATA, void *),
+				      void *data)
+{
+	unsigned int g, b;
+	const tdb_off_t *hash;
+	struct tdb_used_record rec;
+	enum TDB_ERROR ecode;
+
+	hash = tdb_access_read(tdb, off,
+			       sizeof(tdb_off_t)
+			       << (group_bits + TDB_HASH_GROUP_BITS),
+			       true);
+	if (TDB_PTR_IS_ERR(hash)) {
+		return TDB_PTR_ERR(hash);
+	}
+
+	for (g = 0; g < (1 << group_bits); g++) {
+		const tdb_off_t *group = hash + (g << TDB_HASH_GROUP_BITS);
+		for (b = 0; b < (1 << TDB_HASH_GROUP_BITS); b++) {
+			unsigned int bucket, i, used_bits;
+			uint64_t h;
+			tdb_off_t *p;
+			if (group[b] == 0)
+				continue;
+
+			off = group[b] & TDB_OFF_MASK;
+			p = asearch(&off, used, num_used, off_cmp);
+			if (!p) {
+				ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						   TDB_LOG_ERROR,
+						   "tdb_check: Invalid offset"
+						   " %llu in hash",
+						   (long long)off);
+				goto fail;
+			}
+			/* Mark it invalid. */
+			*p ^= 1;
+			(*num_found)++;
+
+			if (hprefix_bits == 64) {
+				/* Chained entries are unordered. */
+				if (is_subhash(group[b])) {
+					ecode = TDB_ERR_CORRUPT;
+					tdb_logerr(tdb, ecode,
+						   TDB_LOG_ERROR,
+						   "tdb_check: Invalid chain"
+						   " entry subhash");
+					goto fail;
+				}
+				h = hash_record(tdb, off);
+				if (h != hprefix) {
+					ecode = TDB_ERR_CORRUPT;
+					tdb_logerr(tdb, ecode,
+						   TDB_LOG_ERROR,
+						   "check: bad hash chain"
+						   " placement"
+						   " 0x%llx vs 0x%llx",
+						   (long long)h,
+						   (long long)hprefix);
+					goto fail;
+				}
+				ecode = tdb_read_convert(tdb, off, &rec,
+							 sizeof(rec));
+				if (ecode != TDB_SUCCESS) {
+					goto fail;
+				}
+				goto check;
+			}
+
+			if (is_subhash(group[b])) {
+				uint64_t subprefix;
+				subprefix = (hprefix
+				     << (group_bits + TDB_HASH_GROUP_BITS))
+					+ g * (1 << TDB_HASH_GROUP_BITS) + b;
+
+				ecode = check_hash_record(tdb,
+					       group[b] & TDB_OFF_MASK,
+					       subprefix,
+					       hprefix_bits
+						       + group_bits
+						       + TDB_HASH_GROUP_BITS,
+					       used, num_used, num_found,
+					       check, data);
+				if (ecode != TDB_SUCCESS) {
+					goto fail;
+				}
+				continue;
+			}
+			/* A normal entry */
+
+			/* Does it belong here at all? */
+			h = hash_record(tdb, off);
+			used_bits = 0;
+			if (get_bits(h, hprefix_bits, &used_bits) != hprefix
+			    && hprefix_bits) {
+				ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						   TDB_LOG_ERROR,
+						   "check: bad hash placement"
+						   " 0x%llx vs 0x%llx",
+						   (long long)h,
+						   (long long)hprefix);
+				goto fail;
+			}
+
+			/* Does it belong in this group? */
+			if (get_bits(h, group_bits, &used_bits) != g) {
+				ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						   TDB_LOG_ERROR,
+						   "check: bad group %llu"
+						   " vs %u",
+						   (long long)h, g);
+				goto fail;
+			}
+
+			/* Are bucket bits correct? */
+			bucket = group[b] & TDB_OFF_HASH_GROUP_MASK;
+			if (get_bits(h, TDB_HASH_GROUP_BITS, &used_bits)
+			    != bucket) {
+				used_bits -= TDB_HASH_GROUP_BITS;
+				ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						   TDB_LOG_ERROR,
+						   "check: bad bucket %u vs %u",
+						   (unsigned)get_bits(h,
+							TDB_HASH_GROUP_BITS,
+							&used_bits),
+						   bucket);
+				goto fail;
+			}
+
+			/* There must not be any zero entries between
+			 * the bucket it belongs in and this one! */
+			for (i = bucket;
+			     i != b;
+			     i = (i + 1) % (1 << TDB_HASH_GROUP_BITS)) {
+				if (group[i] == 0) {
+					ecode = TDB_ERR_CORRUPT;
+					tdb_logerr(tdb, ecode,
+						   TDB_LOG_ERROR,
+						   "check: bad group placement"
+						   " %u vs %u",
+						   b, bucket);
+					goto fail;
+				}
+			}
+
+			ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec));
+			if (ecode != TDB_SUCCESS) {
+				goto fail;
+			}
+
+			/* Bottom bits must match header. */
+			if ((h & ((1 << 11)-1)) != rec_hash(&rec)) {
+				ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						   TDB_LOG_ERROR,
+						   "tdb_check: Bad hash magic"
+						   " at offset %llu"
+						   " (0x%llx vs 0x%llx)",
+						   (long long)off,
+						   (long long)h,
+						   (long long)rec_hash(&rec));
+				goto fail;
+			}
+
+		check:
+			if (check) {
+				TDB_DATA k, d;
+				const unsigned char *kptr;
+
+				kptr = tdb_access_read(tdb,
+						       off + sizeof(rec),
+						       rec_key_length(&rec)
+						       + rec_data_length(&rec),
+						       false);
+				if (TDB_PTR_IS_ERR(kptr)) {
+					ecode = TDB_PTR_ERR(kptr);
+					goto fail;
+				}
+
+				k = tdb_mkdata(kptr, rec_key_length(&rec));
+				d = tdb_mkdata(kptr + k.dsize,
+					       rec_data_length(&rec));
+				ecode = check(k, d, data);
+				tdb_access_release(tdb, kptr);
+				if (ecode != TDB_SUCCESS) {
+					goto fail;
+				}
+			}
+		}
+	}
+	tdb_access_release(tdb, hash);
+	return TDB_SUCCESS;
+
+fail:
+	tdb_access_release(tdb, hash);
+	return ecode;
+}
+
+static enum TDB_ERROR check_hash(struct tdb_context *tdb,
+				 tdb_off_t used[],
+				 size_t num_used, size_t num_ftables,
+				 int (*check)(TDB_DATA, TDB_DATA, void *),
+				 void *data)
+{
+	/* Free tables also show up as used. */
+	size_t num_found = num_ftables;
+	enum TDB_ERROR ecode;
+
+	ecode = check_hash_tree(tdb, offsetof(struct tdb_header, hashtable),
+				TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS,
+				0, 0, used, num_used, &num_found,
+				check, data);
+	if (ecode == TDB_SUCCESS) {
+		if (num_found != num_used) {
+			ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+					   "tdb_check: Not all entries"
+					   " are in hash");
+		}
+	}
+	return ecode;
+}
+
+static enum TDB_ERROR check_free(struct tdb_context *tdb,
+				 tdb_off_t off,
+				 const struct tdb_free_record *frec,
+				 tdb_off_t prev, unsigned int ftable,
+				 unsigned int bucket)
+{
+	enum TDB_ERROR ecode;
+
+	if (frec_magic(frec) != TDB_FREE_MAGIC) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: offset %llu bad magic 0x%llx",
+				  (long long)off,
+				  (long long)frec->magic_and_prev);
+	}
+	if (frec_ftable(frec) != ftable) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: offset %llu bad freetable %u",
+				  (long long)off, frec_ftable(frec));
+
+	}
+
+	ecode = tdb->methods->oob(tdb, off
+				  + frec_len(frec)
+				  + sizeof(struct tdb_used_record),
+				  false);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+	if (size_to_bucket(frec_len(frec)) != bucket) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: offset %llu in wrong bucket"
+				  " (%u vs %u)",
+				  (long long)off,
+				  bucket, size_to_bucket(frec_len(frec)));
+	}
+	if (prev && prev != frec_prev(frec)) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: offset %llu bad prev"
+				  " (%llu vs %llu)",
+				  (long long)off,
+				  (long long)prev, (long long)frec_len(frec));
+	}
+	return TDB_SUCCESS;
+}
+
+static enum TDB_ERROR check_free_table(struct tdb_context *tdb,
+				       tdb_off_t ftable_off,
+				       unsigned ftable_num,
+				       tdb_off_t fr[],
+				       size_t num_free,
+				       size_t *num_found)
+{
+	struct tdb_freetable ft;
+	tdb_off_t h;
+	unsigned int i;
+	enum TDB_ERROR ecode;
+
+	ecode = tdb_read_convert(tdb, ftable_off, &ft, sizeof(ft));
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (rec_magic(&ft.hdr) != TDB_FTABLE_MAGIC
+	    || rec_key_length(&ft.hdr) != 0
+	    || rec_data_length(&ft.hdr) != sizeof(ft) - sizeof(ft.hdr)
+	    || rec_hash(&ft.hdr) != 0) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: Invalid header on free table");
+	}
+
+	for (i = 0; i < TDB_FREE_BUCKETS; i++) {
+		tdb_off_t off, prev = 0, *p, first = 0;
+		struct tdb_free_record f;
+
+		h = bucket_off(ftable_off, i);
+		for (off = tdb_read_off(tdb, h); off; off = f.next) {
+			if (TDB_OFF_IS_ERR(off)) {
+				return off;
+			}
+			if (!first) {
+				off &= TDB_OFF_MASK;
+				first = off;
+			}
+			ecode = tdb_read_convert(tdb, off, &f, sizeof(f));
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+			ecode = check_free(tdb, off, &f, prev, ftable_num, i);
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+
+			/* FIXME: Check hash bits */
+			p = asearch(&off, fr, num_free, off_cmp);
+			if (!p) {
+				return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						  TDB_LOG_ERROR,
+						  "tdb_check: Invalid offset"
+						  " %llu in free table",
+						  (long long)off);
+			}
+			/* Mark it invalid. */
+			*p ^= 1;
+			(*num_found)++;
+			prev = off;
+		}
+
+		if (first) {
+			/* Now we can check first back pointer. */
+			ecode = tdb_read_convert(tdb, first, &f, sizeof(f));
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+			ecode = check_free(tdb, first, &f, prev, ftable_num, i);
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+		}
+	}
+	return TDB_SUCCESS;
+}
+
+/* Slow, but should be very rare. */
+tdb_off_t dead_space(struct tdb_context *tdb, tdb_off_t off)
+{
+	size_t len;
+	enum TDB_ERROR ecode;
+
+	for (len = 0; off + len < tdb->file->map_size; len++) {
+		char c;
+		ecode = tdb->methods->tread(tdb, off, &c, 1);
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+		if (c != 0 && c != 0x43)
+			break;
+	}
+	return len;
+}
+
+static enum TDB_ERROR check_linear(struct tdb_context *tdb,
+				   tdb_off_t **used, size_t *num_used,
+				   tdb_off_t **fr, size_t *num_free,
+				   uint64_t features, tdb_off_t recovery)
+{
+	tdb_off_t off;
+	tdb_len_t len;
+	enum TDB_ERROR ecode;
+	bool found_recovery = false;
+
+	for (off = sizeof(struct tdb_header);
+	     off < tdb->file->map_size;
+	     off += len) {
+		union {
+			struct tdb_used_record u;
+			struct tdb_free_record f;
+			struct tdb_recovery_record r;
+		} rec;
+		/* r is larger: only get that if we need to. */
+		ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec.f));
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+
+		/* If we crash after ftruncate, we can get zeroes or fill. */
+		if (rec.r.magic == TDB_RECOVERY_INVALID_MAGIC
+		    || rec.r.magic ==  0x4343434343434343ULL) {
+			ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec.r));
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+			if (recovery == off) {
+				found_recovery = true;
+				len = sizeof(rec.r) + rec.r.max_len;
+			} else {
+				len = dead_space(tdb, off);
+				if (TDB_OFF_IS_ERR(len)) {
+					return len;
+				}
+				if (len < sizeof(rec.r)) {
+					return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+							  TDB_LOG_ERROR,
+							  "tdb_check: invalid"
+							  " dead space at %zu",
+							  (size_t)off);
+				}
+
+				tdb_logerr(tdb, TDB_SUCCESS, TDB_LOG_WARNING,
+					   "Dead space at %zu-%zu (of %zu)",
+					   (size_t)off, (size_t)(off + len),
+					   (size_t)tdb->file->map_size);
+			}
+		} else if (rec.r.magic == TDB_RECOVERY_MAGIC) {
+			ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec.r));
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+			if (recovery != off) {
+				return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						  TDB_LOG_ERROR,
+						  "tdb_check: unexpected"
+						  " recovery record at offset"
+						  " %zu",
+						  (size_t)off);
+			}
+			if (rec.r.len > rec.r.max_len) {
+				return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						  TDB_LOG_ERROR,
+						  "tdb_check: invalid recovery"
+						  " length %zu",
+						  (size_t)rec.r.len);
+			}
+			if (rec.r.eof > tdb->file->map_size) {
+				return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						  TDB_LOG_ERROR,
+						  "tdb_check: invalid old EOF"
+						  " %zu", (size_t)rec.r.eof);
+			}
+			found_recovery = true;
+			len = sizeof(rec.r) + rec.r.max_len;
+		} else if (frec_magic(&rec.f) == TDB_FREE_MAGIC) {
+			len = sizeof(rec.u) + frec_len(&rec.f);
+			if (off + len > tdb->file->map_size) {
+				return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						  TDB_LOG_ERROR,
+						  "tdb_check: free overlength"
+						  " %llu at offset %llu",
+						  (long long)len,
+						  (long long)off);
+			}
+			/* This record should be in free lists. */
+			if (frec_ftable(&rec.f) != TDB_FTABLE_NONE
+			    && !append(fr, num_free, off)) {
+				return tdb_logerr(tdb, TDB_ERR_OOM,
+						  TDB_LOG_ERROR,
+						  "tdb_check: tracking %zu'th"
+						  " free record.", *num_free);
+			}
+		} else if (rec_magic(&rec.u) == TDB_USED_MAGIC
+			   || rec_magic(&rec.u) == TDB_CHAIN_MAGIC
+			   || rec_magic(&rec.u) == TDB_HTABLE_MAGIC
+			   || rec_magic(&rec.u) == TDB_FTABLE_MAGIC) {
+			uint64_t klen, dlen, extra;
+
+			/* This record is used! */
+			if (!append(used, num_used, off)) {
+				return tdb_logerr(tdb, TDB_ERR_OOM,
+						  TDB_LOG_ERROR,
+						  "tdb_check: tracking %zu'th"
+						  " used record.", *num_used);
+			}
+
+			klen = rec_key_length(&rec.u);
+			dlen = rec_data_length(&rec.u);
+			extra = rec_extra_padding(&rec.u);
+
+			len = sizeof(rec.u) + klen + dlen + extra;
+			if (off + len > tdb->file->map_size) {
+				return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						  TDB_LOG_ERROR,
+						  "tdb_check: used overlength"
+						  " %llu at offset %llu",
+						  (long long)len,
+						  (long long)off);
+			}
+
+			if (len < sizeof(rec.f)) {
+				return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						  TDB_LOG_ERROR,
+						  "tdb_check: too short record"
+						  " %llu at %llu",
+						  (long long)len,
+						  (long long)off);
+			}
+
+			/* Check that records have correct 0 at end (but may
+			 * not in future). */
+			if (extra && !features) {
+				const char *p;
+				char c;
+				p = tdb_access_read(tdb, off + sizeof(rec.u)
+						    + klen + dlen, 1, false);
+				if (TDB_PTR_IS_ERR(p))
+					return TDB_PTR_ERR(p);
+				c = *p;
+				tdb_access_release(tdb, p);
+
+				if (c != '\0') {
+					return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+							  TDB_LOG_ERROR,
+							  "tdb_check:"
+							  " non-zero extra"
+							  " at %llu",
+							  (long long)off);
+				}
+			}
+		} else {
+			return tdb_logerr(tdb, TDB_ERR_CORRUPT,
+					  TDB_LOG_ERROR,
+					  "tdb_check: Bad magic 0x%llx"
+					  " at offset %zu",
+					  (long long)rec_magic(&rec.u),
+					  (size_t)off);
+		}
+	}
+
+	/* We must have found recovery area if there was one. */
+	if (recovery != 0 && !found_recovery) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_check: expected a recovery area at %zu",
+				  (size_t)recovery);
+	}
+
+	return TDB_SUCCESS;
+}
+
+enum TDB_ERROR tdb_check_(struct tdb_context *tdb,
+			  enum TDB_ERROR (*check)(TDB_DATA, TDB_DATA, void *),
+			  void *data)
+{
+	tdb_off_t *fr = NULL, *used = NULL, ft, recovery;
+	size_t num_free = 0, num_used = 0, num_found = 0, num_ftables = 0;
+	uint64_t features;
+	enum TDB_ERROR ecode;
+
+	ecode = tdb_allrecord_lock(tdb, F_RDLCK, TDB_LOCK_WAIT, false);
+	if (ecode != TDB_SUCCESS) {
+		return tdb->last_error = ecode;
+	}
+
+	ecode = tdb_lock_expand(tdb, F_RDLCK);
+	if (ecode != TDB_SUCCESS) {
+		tdb_allrecord_unlock(tdb, F_RDLCK);
+		return tdb->last_error = ecode;
+	}
+
+	ecode = check_header(tdb, &recovery, &features);
+	if (ecode != TDB_SUCCESS)
+		goto out;
+
+	/* First we do a linear scan, checking all records. */
+	ecode = check_linear(tdb, &used, &num_used, &fr, &num_free, features,
+			     recovery);
+	if (ecode != TDB_SUCCESS)
+		goto out;
+
+	for (ft = first_ftable(tdb); ft; ft = next_ftable(tdb, ft)) {
+		if (TDB_OFF_IS_ERR(ft)) {
+			ecode = ft;
+			goto out;
+		}
+		ecode = check_free_table(tdb, ft, num_ftables, fr, num_free,
+					 &num_found);
+		if (ecode != TDB_SUCCESS)
+			goto out;
+		num_ftables++;
+	}
+
+	/* FIXME: Check key uniqueness? */
+	ecode = check_hash(tdb, used, num_used, num_ftables, check, data);
+	if (ecode != TDB_SUCCESS)
+		goto out;
+
+	if (num_found != num_free) {
+		ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				   "tdb_check: Not all entries are in"
+				   " free table");
+	}
+
+out:
+	tdb_allrecord_unlock(tdb, F_RDLCK);
+	tdb_unlock_expand(tdb, F_RDLCK);
+	free(fr);
+	free(used);
+	return tdb->last_error = ecode;
+}
diff --git a/lib/tdb2/doc/TDB1_porting.txt b/lib/tdb2/doc/TDB1_porting.txt
new file mode 100644
index 00000000000..90ba2497382
--- /dev/null
+++ b/lib/tdb2/doc/TDB1_porting.txt
@@ -0,0 +1,44 @@
+Interface differences between TDB1 and TDB2.
+
+- tdb2 uses 'struct tdb_data', tdb1 uses 'struct TDB_DATA'.  Use the
+  TDB_DATA typedef if you want portability between the two.
+
+- tdb2 functions return 0 on success, and a negative error on failure,
+  whereas tdb1 functions returned 0 on success, and -1 on failure.
+  tdb1 then used tdb_error() to determine the error; this is also
+  supported in tdb2 to ease backwards compatibility, though the other
+  form is preferred.
+
+- tdb2's tdb_fetch() returns an error, tdb1's returned the data directly
+  (or tdb_null, and you were supposed to check tdb_error() to find out why).
+
+- tdb2's tdb_nextkey() frees the old key's dptr, in tdb2 you needed to do
+  this manually.
+
+- tdb1's tdb_open/tdb_open_ex took an explicit hash size.  tdb2's hash table
+  resizes as required.
+
+- tdb2 uses a linked list of attribute structures to implement logging and
+  alternate hashes.  tdb1 used tdb_open_ex, which was not extensible.
+
+- tdb2 does locking on read-only databases (ie. O_RDONLY passed to tdb_open).
+  tdb1 did not: use the TDB_NOLOCK flag if you want to suppress locking.
+
+- tdb2's log function is simpler than tdb1's log function.  The string is
+  already formatted, and it takes an enum tdb_log_level not a tdb_debug_level,
+  and which has only three values: TDB_LOG_ERROR, TDB_LOG_USE_ERROR and
+  TDB_LOG_WARNING.
+
+- tdb2 provides tdb_deq() for comparing two struct tdb_data.
+
+- tdb2's tdb_name() returns a copy of the name even for TDB_INTERNAL dbs.
+
+- tdb2 does not need tdb_reopen() or tdb_reopen_all().  If you call
+  fork() after during certain operations the child should close the
+  tdb, or complete the operations before continuing to use the tdb:
+
+	tdb_transaction_start(): child must tdb_transaction_cancel()
+	tdb_lockall(): child must call tdb_unlockall()
+	tdb_lockall_read(): child must call tdb_unlockall_read()
+	tdb_chainlock(): child must call tdb_chainunlock()
+	tdb_parse() callback: child must return from tdb_parse()
diff --git a/lib/tdb2/doc/design-1.3.txt b/lib/tdb2/doc/design-1.3.txt
new file mode 100644
index 00000000000..f81ecf78855
--- /dev/null
+++ b/lib/tdb2/doc/design-1.3.txt
@@ -0,0 +1,1049 @@
+TDB2: A Redesigning The Trivial DataBase
+
+Rusty Russell, IBM Corporation
+
+27-April-2010
+
+Abstract
+
+The Trivial DataBase on-disk format is 32 bits; with usage cases
+heading towards the 4G limit, that must change. This required
+breakage provides an opportunity to revisit TDB's other design
+decisions and reassess them.
+
+1 Introduction
+
+The Trivial DataBase was originally written by Andrew Tridgell as
+a simple key/data pair storage system with the same API as dbm,
+but allowing multiple readers and writers while being small
+enough (< 1000 lines of C) to include in SAMBA. The simple design
+created in 1999 has proven surprisingly robust and performant,
+used in Samba versions 3 and 4 as well as numerous other
+projects. Its useful life was greatly increased by the
+(backwards-compatible!) addition of transaction support in 2005.
+
+The wider variety and greater demands of TDB-using code has lead
+to some organic growth of the API, as well as some compromises on
+the implementation. None of these, by themselves, are seen as
+show-stoppers, but the cumulative effect is to a loss of elegance
+over the initial, simple TDB implementation. Here is a table of
+the approximate number of lines of implementation code and number
+of API functions at the end of each year:
+
+
++-----------+----------------+--------------------------------+
+| Year End  | API Functions  | Lines of C Code Implementation |
++-----------+----------------+--------------------------------+
++-----------+----------------+--------------------------------+
+|   1999    |      13        |              1195              |
++-----------+----------------+--------------------------------+
+|   2000    |      24        |              1725              |
++-----------+----------------+--------------------------------+
+|   2001    |      32        |              2228              |
++-----------+----------------+--------------------------------+
+|   2002    |      35        |              2481              |
++-----------+----------------+--------------------------------+
+|   2003    |      35        |              2552              |
++-----------+----------------+--------------------------------+
+|   2004    |      40        |              2584              |
++-----------+----------------+--------------------------------+
+|   2005    |      38        |              2647              |
++-----------+----------------+--------------------------------+
+|   2006    |      52        |              3754              |
++-----------+----------------+--------------------------------+
+|   2007    |      66        |              4398              |
++-----------+----------------+--------------------------------+
+|   2008    |      71        |              4768              |
++-----------+----------------+--------------------------------+
+|   2009    |      73        |              5715              |
++-----------+----------------+--------------------------------+
+
+
+This review is an attempt to catalog and address all the known
+issues with TDB and create solutions which address the problems
+without significantly increasing complexity; all involved are far
+too aware of the dangers of second system syndrome in rewriting a
+successful project like this.
+
+2 API Issues
+
+2.1 tdb_open_ex Is Not Expandable
+
+The tdb_open() call was expanded to tdb_open_ex(), which added an
+optional hashing function and an optional logging function
+argument. Additional arguments to open would require the
+introduction of a tdb_open_ex2 call etc.
+
+2.1.1 Proposed Solution
+
+tdb_open() will take a linked-list of attributes:
+
+enum tdb_attribute {
+
+    TDB_ATTRIBUTE_LOG = 0,
+
+    TDB_ATTRIBUTE_HASH = 1
+
+};
+
+struct tdb_attribute_base {
+
+    enum tdb_attribute attr;
+
+    union tdb_attribute *next;
+
+};
+
+struct tdb_attribute_log {
+
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_LOG
+*/
+
+    tdb_log_func log_fn;
+
+    void *log_private;
+
+};
+
+struct tdb_attribute_hash {
+
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_HASH
+*/
+
+    tdb_hash_func hash_fn;
+
+    void *hash_private;
+
+};
+
+union tdb_attribute {
+
+    struct tdb_attribute_base base;
+
+    struct tdb_attribute_log log;
+
+    struct tdb_attribute_hash hash;
+
+};
+
+This allows future attributes to be added, even if this expands
+the size of the union.
+
+2.2 tdb_traverse Makes Impossible Guarantees
+
+tdb_traverse (and tdb_firstkey/tdb_nextkey) predate transactions,
+and it was thought that it was important to guarantee that all
+records which exist at the start and end of the traversal would
+be included, and no record would be included twice.
+
+This adds complexity (see[Reliable-Traversal-Adds]) and does not
+work anyway for records which are altered (in particular, those
+which are expanded may be effectively deleted and re-added behind
+the traversal).
+
+2.2.1 <traverse-Proposed-Solution>Proposed Solution
+
+Abandon the guarantee. You will see every record if no changes
+occur during your traversal, otherwise you will see some subset.
+You can prevent changes by using a transaction or the locking
+API.
+
+2.3 Nesting of Transactions Is Fraught
+
+TDB has alternated between allowing nested transactions and not
+allowing them. Various paths in the Samba codebase assume that
+transactions will nest, and in a sense they can: the operation is
+only committed to disk when the outer transaction is committed.
+There are two problems, however:
+
+1. Canceling the inner transaction will cause the outer
+  transaction commit to fail, and will not undo any operations
+  since the inner transaction began. This problem is soluble with
+  some additional internal code.
+
+2. An inner transaction commit can be cancelled by the outer
+  transaction. This is desirable in the way which Samba's
+  database initialization code uses transactions, but could be a
+  surprise to any users expecting a successful transaction commit
+  to expose changes to others.
+
+The current solution is to specify the behavior at tdb_open(),
+with the default currently that nested transactions are allowed.
+This flag can also be changed at runtime.
+
+2.3.1 Proposed Solution
+
+Given the usage patterns, it seems that the “least-surprise”
+behavior of disallowing nested transactions should become the
+default. Additionally, it seems the outer transaction is the only
+code which knows whether inner transactions should be allowed, so
+a flag to indicate this could be added to tdb_transaction_start.
+However, this behavior can be simulated with a wrapper which uses
+tdb_add_flags() and tdb_remove_flags(), so the API should not be
+expanded for this relatively-obscure case.
+
+2.4 Incorrect Hash Function is Not Detected
+
+tdb_open_ex() allows the calling code to specify a different hash
+function to use, but does not check that all other processes
+accessing this tdb are using the same hash function. The result
+is that records are missing from tdb_fetch().
+
+2.4.1 Proposed Solution
+
+The header should contain an example hash result (eg. the hash of
+0xdeadbeef), and tdb_open_ex() should check that the given hash
+function produces the same answer, or fail the tdb_open call.
+
+2.5 tdb_set_max_dead/TDB_VOLATILE Expose Implementation
+
+In response to scalability issues with the free list ([TDB-Freelist-Is]
+) two API workarounds have been incorporated in TDB:
+tdb_set_max_dead() and the TDB_VOLATILE flag to tdb_open. The
+latter actually calls the former with an argument of “5”.
+
+This code allows deleted records to accumulate without putting
+them in the free list. On delete we iterate through each chain
+and free them in a batch if there are more than max_dead entries.
+These are never otherwise recycled except as a side-effect of a
+tdb_repack.
+
+2.5.1 Proposed Solution
+
+With the scalability problems of the freelist solved, this API
+can be removed. The TDB_VOLATILE flag may still be useful as a
+hint that store and delete of records will be at least as common
+as fetch in order to allow some internal tuning, but initially
+will become a no-op.
+
+2.6 <TDB-Files-Cannot>TDB Files Cannot Be Opened Multiple Times
+  In The Same Process
+
+No process can open the same TDB twice; we check and disallow it.
+This is an unfortunate side-effect of fcntl locks, which operate
+on a per-file rather than per-file-descriptor basis, and do not
+nest. Thus, closing any file descriptor on a file clears all the
+locks obtained by this process, even if they were placed using a
+different file descriptor!
+
+Note that even if this were solved, deadlock could occur if
+operations were nested: this is a more manageable programming
+error in most cases.
+
+2.6.1 Proposed Solution
+
+We could lobby POSIX to fix the perverse rules, or at least lobby
+Linux to violate them so that the most common implementation does
+not have this restriction. This would be a generally good idea
+for other fcntl lock users.
+
+Samba uses a wrapper which hands out the same tdb_context to
+multiple callers if this happens, and does simple reference
+counting. We should do this inside the tdb library, which already
+emulates lock nesting internally; it would need to recognize when
+deadlock occurs within a single process. This would create a new
+failure mode for tdb operations (while we currently handle
+locking failures, they are impossible in normal use and a process
+encountering them can do little but give up).
+
+I do not see benefit in an additional tdb_open flag to indicate
+whether re-opening is allowed, as though there may be some
+benefit to adding a call to detect when a tdb_context is shared,
+to allow other to create such an API.
+
+2.7 TDB API Is Not POSIX Thread-safe
+
+The TDB API uses an error code which can be queried after an
+operation to determine what went wrong. This programming model
+does not work with threads, unless specific additional guarantees
+are given by the implementation. In addition, even
+otherwise-independent threads cannot open the same TDB (as in [TDB-Files-Cannot]
+).
+
+2.7.1 Proposed Solution
+
+Reachitecting the API to include a tdb_errcode pointer would be a
+great deal of churn; we are better to guarantee that the
+tdb_errcode is per-thread so the current programming model can be
+maintained.
+
+This requires dynamic per-thread allocations, which is awkward
+with POSIX threads (pthread_key_create space is limited and we
+cannot simply allocate a key for every TDB).
+
+Internal locking is required to make sure that fcntl locks do not
+overlap between threads, and also that the global list of tdbs is
+maintained.
+
+The aim is that building tdb with -DTDB_PTHREAD will result in a
+pthread-safe version of the library, and otherwise no overhead
+will exist.
+
+2.8 *_nonblock Functions And *_mark Functions Expose
+  Implementation
+
+CTDB[footnote:
+Clustered TDB, see http://ctdb.samba.org
+] wishes to operate on TDB in a non-blocking manner. This is
+currently done as follows:
+
+1. Call the _nonblock variant of an API function (eg.
+  tdb_lockall_nonblock). If this fails:
+
+2. Fork a child process, and wait for it to call the normal
+  variant (eg. tdb_lockall).
+
+3. If the child succeeds, call the _mark variant to indicate we
+  already have the locks (eg. tdb_lockall_mark).
+
+4. Upon completion, tell the child to release the locks (eg.
+  tdb_unlockall).
+
+5. Indicate to tdb that it should consider the locks removed (eg.
+  tdb_unlockall_mark).
+
+There are several issues with this approach. Firstly, adding two
+new variants of each function clutters the API for an obscure
+use, and so not all functions have three variants. Secondly, it
+assumes that all paths of the functions ask for the same locks,
+otherwise the parent process will have to get a lock which the
+child doesn't have under some circumstances. I don't believe this
+is currently the case, but it constrains the implementation.
+
+2.8.1 <Proposed-Solution-locking-hook>Proposed Solution
+
+Implement a hook for locking methods, so that the caller can
+control the calls to create and remove fcntl locks. In this
+scenario, ctdbd would operate as follows:
+
+1. Call the normal API function, eg tdb_lockall().
+
+2. When the lock callback comes in, check if the child has the
+  lock. Initially, this is always false. If so, return 0.
+  Otherwise, try to obtain it in non-blocking mode. If that
+  fails, return EWOULDBLOCK.
+
+3. Release locks in the unlock callback as normal.
+
+4. If tdb_lockall() fails, see if we recorded a lock failure; if
+  so, call the child to repeat the operation.
+
+5. The child records what locks it obtains, and returns that
+  information to the parent.
+
+6. When the child has succeeded, goto 1.
+
+This is flexible enough to handle any potential locking scenario,
+even when lock requirements change. It can be optimized so that
+the parent does not release locks, just tells the child which
+locks it doesn't need to obtain.
+
+It also keeps the complexity out of the API, and in ctdbd where
+it is needed.
+
+2.9 tdb_chainlock Functions Expose Implementation
+
+tdb_chainlock locks some number of records, including the record
+indicated by the given key. This gave atomicity guarantees;
+no-one can start a transaction, alter, read or delete that key
+while the lock is held.
+
+It also makes the same guarantee for any other key in the chain,
+which is an internal implementation detail and potentially a
+cause for deadlock.
+
+2.9.1 Proposed Solution
+
+None. It would be nice to have an explicit single entry lock
+which effected no other keys. Unfortunately, this won't work for
+an entry which doesn't exist. Thus while chainlock may be
+implemented more efficiently for the existing case, it will still
+have overlap issues with the non-existing case. So it is best to
+keep the current (lack of) guarantee about which records will be
+effected to avoid constraining our implementation.
+
+2.10 Signal Handling is Not Race-Free
+
+The tdb_setalarm_sigptr() call allows the caller's signal handler
+to indicate that the tdb locking code should return with a
+failure, rather than trying again when a signal is received (and
+errno == EAGAIN). This is usually used to implement timeouts.
+
+Unfortunately, this does not work in the case where the signal is
+received before the tdb code enters the fcntl() call to place the
+lock: the code will sleep within the fcntl() code, unaware that
+the signal wants it to exit. In the case of long timeouts, this
+does not happen in practice.
+
+2.10.1 Proposed Solution
+
+The locking hooks proposed in[Proposed-Solution-locking-hook]
+would allow the user to decide on whether to fail the lock
+acquisition on a signal. This allows the caller to choose their
+own compromise: they could narrow the race by checking
+immediately before the fcntl call.[footnote:
+It may be possible to make this race-free in some implementations
+by having the signal handler alter the struct flock to make it
+invalid. This will cause the fcntl() lock call to fail with
+EINVAL if the signal occurs before the kernel is entered,
+otherwise EAGAIN.
+]
+
+2.11 The API Uses Gratuitous Typedefs, Capitals
+
+typedefs are useful for providing source compatibility when types
+can differ across implementations, or arguably in the case of
+function pointer definitions which are hard for humans to parse.
+Otherwise it is simply obfuscation and pollutes the namespace.
+
+Capitalization is usually reserved for compile-time constants and
+macros.
+
+  TDB_CONTEXT There is no reason to use this over 'struct
+  tdb_context'; the definition isn't visible to the API user
+  anyway.
+
+  TDB_DATA There is no reason to use this over struct TDB_DATA;
+  the struct needs to be understood by the API user.
+
+  struct TDB_DATA This would normally be called 'struct
+  tdb_data'.
+
+  enum TDB_ERROR Similarly, this would normally be enum
+  tdb_error.
+
+2.11.1 Proposed Solution
+
+None. Introducing lower case variants would please pedants like
+myself, but if it were done the existing ones should be kept.
+There is little point forcing a purely cosmetic change upon tdb
+users.
+
+2.12 <tdb_log_func-Doesnt-Take>tdb_log_func Doesn't Take The
+  Private Pointer
+
+For API compatibility reasons, the logging function needs to call
+tdb_get_logging_private() to retrieve the pointer registered by
+the tdb_open_ex for logging.
+
+2.12.1 Proposed Solution
+
+It should simply take an extra argument, since we are prepared to
+break the API/ABI.
+
+2.13 Various Callback Functions Are Not Typesafe
+
+The callback functions in tdb_set_logging_function (after [tdb_log_func-Doesnt-Take]
+ is resolved), tdb_parse_record, tdb_traverse, tdb_traverse_read
+and tdb_check all take void * and must internally convert it to
+the argument type they were expecting.
+
+If this type changes, the compiler will not produce warnings on
+the callers, since it only sees void *.
+
+2.13.1 Proposed Solution
+
+With careful use of macros, we can create callback functions
+which give a warning when used on gcc and the types of the
+callback and its private argument differ. Unsupported compilers
+will not give a warning, which is no worse than now. In addition,
+the callbacks become clearer, as they need not use void * for
+their parameter.
+
+See CCAN's typesafe_cb module at
+http://ccan.ozlabs.org/info/typesafe_cb.html
+
+2.14 TDB_CLEAR_IF_FIRST Must Be Specified On All Opens,
+  tdb_reopen_all Problematic
+
+The TDB_CLEAR_IF_FIRST flag to tdb_open indicates that the TDB
+file should be cleared if the caller discovers it is the only
+process with the TDB open. However, if any caller does not
+specify TDB_CLEAR_IF_FIRST it will not be detected, so will have
+the TDB erased underneath them (usually resulting in a crash).
+
+There is a similar issue on fork(); if the parent exits (or
+otherwise closes the tdb) before the child calls tdb_reopen_all()
+to establish the lock used to indicate the TDB is opened by
+someone, a TDB_CLEAR_IF_FIRST opener at that moment will believe
+it alone has opened the TDB and will erase it.
+
+2.14.1 Proposed Solution
+
+Remove TDB_CLEAR_IF_FIRST. Other workarounds are possible, but
+see [TDB_CLEAR_IF_FIRST-Imposes-Performance].
+
+3 Performance And Scalability Issues
+
+3.1 <TDB_CLEAR_IF_FIRST-Imposes-Performance>TDB_CLEAR_IF_FIRST
+  Imposes Performance Penalty
+
+When TDB_CLEAR_IF_FIRST is specified, a 1-byte read lock is
+placed at offset 4 (aka. the ACTIVE_LOCK). While these locks
+never conflict in normal tdb usage, they do add substantial
+overhead for most fcntl lock implementations when the kernel
+scans to detect if a lock conflict exists. This is often a single
+linked list, making the time to acquire and release a fcntl lock
+O(N) where N is the number of processes with the TDB open, not
+the number actually doing work.
+
+In a Samba server it is common to have huge numbers of clients
+sitting idle, and thus they have weaned themselves off the
+TDB_CLEAR_IF_FIRST flag.[footnote:
+There is a flag to tdb_reopen_all() which is used for this
+optimization: if the parent process will outlive the child, the
+child does not need the ACTIVE_LOCK. This is a workaround for
+this very performance issue.
+]
+
+3.1.1 Proposed Solution
+
+Remove the flag. It was a neat idea, but even trivial servers
+tend to know when they are initializing for the first time and
+can simply unlink the old tdb at that point.
+
+3.2 TDB Files Have a 4G Limit
+
+This seems to be becoming an issue (so much for “trivial”!),
+particularly for ldb.
+
+3.2.1 Proposed Solution
+
+A new, incompatible TDB format which uses 64 bit offsets
+internally rather than 32 bit as now. For simplicity of endian
+conversion (which TDB does on the fly if required), all values
+will be 64 bit on disk. In practice, some upper bits may be used
+for other purposes, but at least 56 bits will be available for
+file offsets.
+
+tdb_open() will automatically detect the old version, and even
+create them if TDB_VERSION6 is specified to tdb_open.
+
+32 bit processes will still be able to access TDBs larger than 4G
+(assuming that their off_t allows them to seek to 64 bits), they
+will gracefully fall back as they fail to mmap. This can happen
+already with large TDBs.
+
+Old versions of tdb will fail to open the new TDB files (since 28
+August 2009, commit 398d0c29290: prior to that any unrecognized
+file format would be erased and initialized as a fresh tdb!)
+
+3.3 TDB Records Have a 4G Limit
+
+This has not been a reported problem, and the API uses size_t
+which can be 64 bit on 64 bit platforms. However, other limits
+may have made such an issue moot.
+
+3.3.1 Proposed Solution
+
+Record sizes will be 64 bit, with an error returned on 32 bit
+platforms which try to access such records (the current
+implementation would return TDB_ERR_OOM in a similar case). It
+seems unlikely that 32 bit keys will be a limitation, so the
+implementation may not support this (see [sub:Records-Incur-A]).
+
+3.4 Hash Size Is Determined At TDB Creation Time
+
+TDB contains a number of hash chains in the header; the number is
+specified at creation time, and defaults to 131. This is such a
+bottleneck on large databases (as each hash chain gets quite
+long), that LDB uses 10,000 for this hash. In general it is
+impossible to know what the 'right' answer is at database
+creation time.
+
+3.4.1 Proposed Solution
+
+After comprehensive performance testing on various scalable hash
+variants[footnote:
+http://rusty.ozlabs.org/?p=89 and http://rusty.ozlabs.org/?p=94
+This was annoying because I was previously convinced that an
+expanding tree of hashes would be very close to optimal.
+], it became clear that it is hard to beat a straight linear hash
+table which doubles in size when it reaches saturation. There are
+three details which become important:
+
+1. On encountering a full bucket, we use the next bucket.
+
+2. Extra hash bits are stored with the offset, to reduce
+  comparisons.
+
+3. A marker entry is used on deleting an entry.
+
+The doubling of the table must be done under a transaction; we
+will not reduce it on deletion, so it will be an unusual case. It
+will either be placed at the head (other entries will be moved
+out the way so we can expand). We could have a pointer in the
+header to the current hashtable location, but that pointer would
+have to be read frequently to check for hashtable moves.
+
+The locking for this is slightly more complex than the chained
+case; we currently have one lock per bucket, and that means we
+would need to expand the lock if we overflow to the next bucket.
+The frequency of such collisions will effect our locking
+heuristics: we can always lock more buckets than we need.
+
+One possible optimization is to only re-check the hash size on an
+insert or a lookup miss.
+
+3.5 <TDB-Freelist-Is>TDB Freelist Is Highly Contended
+
+TDB uses a single linked list for the free list. Allocation
+occurs as follows, using heuristics which have evolved over time:
+
+1. Get the free list lock for this whole operation.
+
+2. Multiply length by 1.25, so we always over-allocate by 25%.
+
+3. Set the slack multiplier to 1.
+
+4. Examine the current freelist entry: if it is > length but <
+  the current best case, remember it as the best case.
+
+5. Multiply the slack multiplier by 1.05.
+
+6. If our best fit so far is less than length * slack multiplier,
+  return it. The slack will be turned into a new free record if
+  it's large enough.
+
+7. Otherwise, go onto the next freelist entry.
+
+Deleting a record occurs as follows:
+
+1. Lock the hash chain for this whole operation.
+
+2. Walk the chain to find the record, keeping the prev pointer
+  offset.
+
+3. If max_dead is non-zero:
+
+  (a) Walk the hash chain again and count the dead records.
+
+  (b) If it's more than max_dead, bulk free all the dead ones
+    (similar to steps 4 and below, but the lock is only obtained
+    once).
+
+  (c) Simply mark this record as dead and return.
+
+4. Get the free list lock for the remainder of this operation.
+
+5. <right-merging>Examine the following block to see if it is
+  free; if so, enlarge the current block and remove that block
+  from the free list. This was disabled, as removal from the free
+  list was O(entries-in-free-list).
+
+6. Examine the preceeding block to see if it is free: for this
+  reason, each block has a 32-bit tailer which indicates its
+  length. If it is free, expand it to cover our new block and
+  return.
+
+7. Otherwise, prepend ourselves to the free list.
+
+Disabling right-merging (step [right-merging]) causes
+fragmentation; the other heuristics proved insufficient to
+address this, so the final answer to this was that when we expand
+the TDB file inside a transaction commit, we repack the entire
+tdb.
+
+The single list lock limits our allocation rate; due to the other
+issues this is not currently seen as a bottleneck.
+
+3.5.1 Proposed Solution
+
+The first step is to remove all the current heuristics, as they
+obviously interact, then examine them once the lock contention is
+addressed.
+
+The free list must be split to reduce contention. Assuming
+perfect free merging, we can at most have 1 free list entry for
+each entry. This implies that the number of free lists is related
+to the size of the hash table, but as it is rare to walk a large
+number of free list entries we can use far fewer, say 1/32 of the
+number of hash buckets.
+
+There are various benefits in using per-size free lists (see [sub:TDB-Becomes-Fragmented]
+) but it's not clear this would reduce contention in the common
+case where all processes are allocating/freeing the same size.
+Thus we almost certainly need to divide in other ways: the most
+obvious is to divide the file into zones, and using a free list
+(or set of free lists) for each. This approximates address
+ordering.
+
+Note that this means we need to split the free lists when we
+expand the file; this is probably acceptable when we double the
+hash table size, since that is such an expensive operation
+already. In the case of increasing the file size, there is an
+optimization we can use: if we use M in the formula above as the
+file size rounded up to the next power of 2, we only need
+reshuffle free lists when the file size crosses a power of 2
+boundary, and reshuffling the free lists is trivial: we simply
+merge every consecutive pair of free lists.
+
+The basic algorithm is as follows. Freeing is simple:
+
+1. Identify the correct zone.
+
+2. Lock the corresponding list.
+
+3. Re-check the zone (we didn't have a lock, sizes could have
+  changed): relock if necessary.
+
+4. Place the freed entry in the list for that zone.
+
+Allocation is a little more complicated, as we perform delayed
+coalescing at this point:
+
+1. Pick a zone either the zone we last freed into, or based on a “
+  random” number.
+
+2. Lock the corresponding list.
+
+3. Re-check the zone: relock if necessary.
+
+4. If the top entry is -large enough, remove it from the list and
+  return it.
+
+5. Otherwise, coalesce entries in the list.
+
+  (a)
+
+  (b)
+
+  (c)
+
+  (d)
+
+6. If there was no entry large enough, unlock the list and try
+  the next zone.
+
+7.
+
+8.
+
+9. If no zone satisfies, expand the file.
+
+This optimizes rapid insert/delete of free list entries by not
+coalescing them all the time.. First-fit address ordering
+ordering seems to be fairly good for keeping fragmentation low
+(see [sub:TDB-Becomes-Fragmented]). Note that address ordering
+does not need a tailer to coalesce, though if we needed one we
+could have one cheaply: see [sub:Records-Incur-A].
+
+
+
+I anticipate that the number of entries in each free zone would
+be small, but it might be worth using one free entry to hold
+pointers to the others for cache efficiency.
+
+3.6 <sub:TDB-Becomes-Fragmented>TDB Becomes Fragmented
+
+Much of this is a result of allocation strategy[footnote:
+The Memory Fragmentation Problem: Solved? Johnstone & Wilson 1995
+ftp://ftp.cs.utexas.edu/pub/garbage/malloc/ismm98.ps
+] and deliberate hobbling of coalescing; internal fragmentation
+(aka overallocation) is deliberately set at 25%, and external
+fragmentation is only cured by the decision to repack the entire
+db when a transaction commit needs to enlarge the file.
+
+3.6.1 Proposed Solution
+
+The 25% overhead on allocation works in practice for ldb because
+indexes tend to expand by one record at a time. This internal
+fragmentation can be resolved by having an “expanded” bit in the
+header to note entries that have previously expanded, and
+allocating more space for them.
+
+There are is a spectrum of possible solutions for external
+fragmentation: one is to use a fragmentation-avoiding allocation
+strategy such as best-fit address-order allocator. The other end
+of the spectrum would be to use a bump allocator (very fast and
+simple) and simply repack the file when we reach the end.
+
+There are three problems with efficient fragmentation-avoiding
+allocators: they are non-trivial, they tend to use a single free
+list for each size, and there's no evidence that tdb allocation
+patterns will match those recorded for general allocators (though
+it seems likely).
+
+Thus we don't spend too much effort on external fragmentation; we
+will be no worse than the current code if we need to repack on
+occasion. More effort is spent on reducing freelist contention,
+and reducing overhead.
+
+3.7 <sub:Records-Incur-A>Records Incur A 28-Byte Overhead
+
+Each TDB record has a header as follows:
+
+struct tdb_record {
+
+        tdb_off_t next; /* offset of the next record in the list
+*/
+
+        tdb_len_t rec_len; /* total byte length of record */
+
+        tdb_len_t key_len; /* byte length of key */
+
+        tdb_len_t data_len; /* byte length of data */
+
+        uint32_t full_hash; /* the full 32 bit hash of the key */
+
+        uint32_t magic;   /* try to catch errors */
+
+        /* the following union is implied:
+
+                union {
+
+                        char record[rec_len];
+
+                        struct {
+
+                                char key[key_len];
+
+                                char data[data_len];
+
+                        }
+
+                        uint32_t totalsize; (tailer)
+
+                }
+
+        */
+
+};
+
+Naively, this would double to a 56-byte overhead on a 64 bit
+implementation.
+
+3.7.1 Proposed Solution
+
+We can use various techniques to reduce this for an allocated
+block:
+
+1. The 'next' pointer is not required, as we are using a flat
+  hash table.
+
+2. 'rec_len' can instead be expressed as an addition to key_len
+  and data_len (it accounts for wasted or overallocated length in
+  the record). Since the record length is always a multiple of 8,
+  we can conveniently fit it in 32 bits (representing up to 35
+  bits).
+
+3. 'key_len' and 'data_len' can be reduced. I'm unwilling to
+  restrict 'data_len' to 32 bits, but instead we can combine the
+  two into one 64-bit field and using a 5 bit value which
+  indicates at what bit to divide the two. Keys are unlikely to
+  scale as fast as data, so I'm assuming a maximum key size of 32
+  bits.
+
+4. 'full_hash' is used to avoid a memcmp on the “miss” case, but
+  this is diminishing returns after a handful of bits (at 10
+  bits, it reduces 99.9% of false memcmp). As an aside, as the
+  lower bits are already incorporated in the hash table
+  resolution, the upper bits should be used here.
+
+5. 'magic' does not need to be enlarged: it currently reflects
+  one of 5 values (used, free, dead, recovery, and
+  unused_recovery). It is useful for quick sanity checking
+  however, and should not be eliminated.
+
+6. 'tailer' is only used to coalesce free blocks (so a block to
+  the right can find the header to check if this block is free).
+  This can be replaced by a single 'free' bit in the header of
+  the following block (and the tailer only exists in free
+  blocks).[footnote:
+This technique from Thomas Standish. Data Structure Techniques.
+Addison-Wesley, Reading, Massachusetts, 1980.
+] The current proposed coalescing algorithm doesn't need this,
+  however.
+
+This produces a 16 byte used header like this:
+
+struct tdb_used_record {
+
+        uint32_t magic : 16,
+
+                 prev_is_free: 1,
+
+                 key_data_divide: 5,
+
+                 top_hash: 10;
+
+        uint32_t extra_octets;
+
+        uint64_t key_and_data_len;
+
+};
+
+And a free record like this:
+
+struct tdb_free_record {
+
+        uint32_t free_magic;
+
+        uint64_t total_length;
+
+        ...
+
+        uint64_t tailer;
+
+};
+
+
+
+3.8 Transaction Commit Requires 4 fdatasync
+
+The current transaction algorithm is:
+
+1. write_recovery_data();
+
+2. sync();
+
+3. write_recovery_header();
+
+4. sync();
+
+5. overwrite_with_new_data();
+
+6. sync();
+
+7. remove_recovery_header();
+
+8. sync();
+
+On current ext3, each sync flushes all data to disk, so the next
+3 syncs are relatively expensive. But this could become a
+performance bottleneck on other filesystems such as ext4.
+
+3.8.1 Proposed Solution
+
+
+
+
+
+
+
+
+
+Neil Brown points out that this is overzealous, and only one sync
+is needed:
+
+1. Bundle the recovery data, a transaction counter and a strong
+  checksum of the new data.
+
+2. Strong checksum that whole bundle.
+
+3. Store the bundle in the database.
+
+4. Overwrite the oldest of the two recovery pointers in the
+  header (identified using the transaction counter) with the
+  offset of this bundle.
+
+5. sync.
+
+6. Write the new data to the file.
+
+Checking for recovery means identifying the latest bundle with a
+valid checksum and using the new data checksum to ensure that it
+has been applied. This is more expensive than the current check,
+but need only be done at open. For running databases, a separate
+header field can be used to indicate a transaction in progress;
+we need only check for recovery if this is set.
+
+3.9 TDB Does Not Have Snapshot Support
+
+3.9.1 Proposed Solution
+
+None. At some point you say “use a real database”.
+
+But as a thought experiment, if we implemented transactions to
+only overwrite free entries (this is tricky: there must not be a
+header in each entry which indicates whether it is free, but use
+of presence in metadata elsewhere), and a pointer to the hash
+table, we could create an entirely new commit without destroying
+existing data. Then it would be easy to implement snapshots in a
+similar way.
+
+This would not allow arbitrary changes to the database, such as
+tdb_repack does, and would require more space (since we have to
+preserve the current and future entries at once). If we used hash
+trees rather than one big hash table, we might only have to
+rewrite some sections of the hash, too.
+
+We could then implement snapshots using a similar method, using
+multiple different hash tables/free tables.
+
+3.10 Transactions Cannot Operate in Parallel
+
+This would be useless for ldb, as it hits the index records with
+just about every update. It would add significant complexity in
+resolving clashes, and cause the all transaction callers to write
+their code to loop in the case where the transactions spuriously
+failed.
+
+3.10.1 Proposed Solution
+
+We could solve a small part of the problem by providing read-only
+transactions. These would allow one write transaction to begin,
+but it could not commit until all r/o transactions are done. This
+would require a new RO_TRANSACTION_LOCK, which would be upgraded
+on commit.
+
+3.11 Default Hash Function Is Suboptimal
+
+The Knuth-inspired multiplicative hash used by tdb is fairly slow
+(especially if we expand it to 64 bits), and works best when the
+hash bucket size is a prime number (which also means a slow
+modulus). In addition, it is highly predictable which could
+potentially lead to a Denial of Service attack in some TDB uses.
+
+3.11.1 Proposed Solution
+
+The Jenkins lookup3 hash[footnote:
+http://burtleburtle.net/bob/c/lookup3.c
+] is a fast and superbly-mixing hash. It's used by the Linux
+kernel and almost everything else. This has the particular
+properties that it takes an initial seed, and produces two 32 bit
+hash numbers, which we can combine into a 64-bit hash.
+
+The seed should be created at tdb-creation time from some random
+source, and placed in the header. This is far from foolproof, but
+adds a little bit of protection against hash bombing.
+
+3.12 <Reliable-Traversal-Adds>Reliable Traversal Adds Complexity
+
+We lock a record during traversal iteration, and try to grab that
+lock in the delete code. If that grab on delete fails, we simply
+mark it deleted and continue onwards; traversal checks for this
+condition and does the delete when it moves off the record.
+
+If traversal terminates, the dead record may be left
+indefinitely.
+
+3.12.1 Proposed Solution
+
+Remove reliability guarantees; see [traverse-Proposed-Solution].
+
+3.13 Fcntl Locking Adds Overhead
+
+Placing a fcntl lock means a system call, as does removing one.
+This is actually one reason why transactions can be faster
+(everything is locked once at transaction start). In the
+uncontended case, this overhead can theoretically be eliminated.
+
+3.13.1 Proposed Solution
+
+None.
+
+We tried this before with spinlock support, in the early days of
+TDB, and it didn't make much difference except in manufactured
+benchmarks.
+
+We could use spinlocks (with futex kernel support under Linux),
+but it means that we lose automatic cleanup when a process dies
+with a lock. There is a method of auto-cleanup under Linux, but
+it's not supported by other operating systems. We could
+reintroduce a clear-if-first-style lock and sweep for dead
+futexes on open, but that wouldn't help the normal case of one
+concurrent opener dying. Increasingly elaborate repair schemes
+could be considered, but they require an ABI change (everyone
+must use them) anyway, so there's no need to do this at the same
+time as everything else.
diff --git a/lib/tdb2/doc/design.lyx b/lib/tdb2/doc/design.lyx
new file mode 100644
index 00000000000..0a1d6a14bca
--- /dev/null
+++ b/lib/tdb2/doc/design.lyx
@@ -0,0 +1,2689 @@
+#LyX 1.6.7 created this file. For more info see http://www.lyx.org/
+\lyxformat 345
+\begin_document
+\begin_header
+\textclass article
+\use_default_options true
+\language english
+\inputencoding auto
+\font_roman default
+\font_sans default
+\font_typewriter default
+\font_default_family default
+\font_sc false
+\font_osf false
+\font_sf_scale 100
+\font_tt_scale 100
+
+\graphics default
+\paperfontsize default
+\use_hyperref false
+\papersize default
+\use_geometry false
+\use_amsmath 1
+\use_esint 1
+\cite_engine basic
+\use_bibtopic false
+\paperorientation portrait
+\secnumdepth 3
+\tocdepth 3
+\paragraph_separation indent
+\defskip medskip
+\quotes_language english
+\papercolumns 1
+\papersides 1
+\paperpagestyle default
+\tracking_changes true
+\output_changes true
+\author ""
+\author ""
+\end_header
+
+\begin_body
+
+\begin_layout Title
+TDB2: A Redesigning The Trivial DataBase
+\end_layout
+
+\begin_layout Author
+Rusty Russell, IBM Corporation
+\end_layout
+
+\begin_layout Date
+17-March-2011
+\end_layout
+
+\begin_layout Abstract
+The Trivial DataBase on-disk format is 32 bits; with usage cases heading
+ towards the 4G limit, that must change.
+ This required breakage provides an opportunity to revisit TDB's other design
+ decisions and reassess them.
+\end_layout
+
+\begin_layout Section
+Introduction
+\end_layout
+
+\begin_layout Standard
+The Trivial DataBase was originally written by Andrew Tridgell as a simple
+ key/data pair storage system with the same API as dbm, but allowing multiple
+ readers and writers while being small enough (< 1000 lines of C) to include
+ in SAMBA.
+ The simple design created in 1999 has proven surprisingly robust and performant
+, used in Samba versions 3 and 4 as well as numerous other projects.
+ Its useful life was greatly increased by the (backwards-compatible!) addition
+ of transaction support in 2005.
+\end_layout
+
+\begin_layout Standard
+The wider variety and greater demands of TDB-using code has lead to some
+ organic growth of the API, as well as some compromises on the implementation.
+ None of these, by themselves, are seen as show-stoppers, but the cumulative
+ effect is to a loss of elegance over the initial, simple TDB implementation.
+ Here is a table of the approximate number of lines of implementation code
+ and number of API functions at the end of each year:
+\end_layout
+
+\begin_layout Standard
+\begin_inset Tabular
+<lyxtabular version="3" rows="12" columns="3">
+<features>
+<column alignment="center" valignment="top" width="0">
+<column alignment="center" valignment="top" width="0">
+<column alignment="center" valignment="top" width="0">
+<row>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+Year End
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+API Functions
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+Lines of C Code Implementation
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+1999
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+13
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+1195
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2000
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+24
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+1725
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2001
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+32
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2228
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2002
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+35
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2481
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2003
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+35
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2552
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2004
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+40
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2584
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2005
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+38
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2647
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2006
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+52
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+3754
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2007
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+66
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+4398
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2008
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+71
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+4768
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2009
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+73
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+5715
+\end_layout
+
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+This review is an attempt to catalog and address all the known issues with
+ TDB and create solutions which address the problems without significantly
+ increasing complexity; all involved are far too aware of the dangers of
+ second system syndrome in rewriting a successful project like this.
+\end_layout
+
+\begin_layout Section
+API Issues
+\end_layout
+
+\begin_layout Subsection
+tdb_open_ex Is Not Expandable
+\end_layout
+
+\begin_layout Standard
+The tdb_open() call was expanded to tdb_open_ex(), which added an optional
+ hashing function and an optional logging function argument.
+ Additional arguments to open would require the introduction of a tdb_open_ex2
+ call etc.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\begin_inset CommandInset label
+LatexCommand label
+name "attributes"
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+tdb_open() will take a linked-list of attributes:
+\end_layout
+
+\begin_layout LyX-Code
+enum tdb_attribute {
+\end_layout
+
+\begin_layout LyX-Code
+    TDB_ATTRIBUTE_LOG = 0,
+\end_layout
+
+\begin_layout LyX-Code
+    TDB_ATTRIBUTE_HASH = 1
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_attribute_base {
+\end_layout
+
+\begin_layout LyX-Code
+    enum tdb_attribute attr;
+\end_layout
+
+\begin_layout LyX-Code
+    union tdb_attribute *next;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_attribute_log {
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_LOG */
+\end_layout
+
+\begin_layout LyX-Code
+    tdb_log_func log_fn;
+\end_layout
+
+\begin_layout LyX-Code
+    void *log_private;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_attribute_hash {
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_HASH */
+\end_layout
+
+\begin_layout LyX-Code
+    tdb_hash_func hash_fn;
+\end_layout
+
+\begin_layout LyX-Code
+    void *hash_private;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+union tdb_attribute {
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_base base;
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_log log;
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_hash hash;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+This allows future attributes to be added, even if this expands the size
+ of the union.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+tdb_traverse Makes Impossible Guarantees
+\end_layout
+
+\begin_layout Standard
+tdb_traverse (and tdb_firstkey/tdb_nextkey) predate transactions, and it
+ was thought that it was important to guarantee that all records which exist
+ at the start and end of the traversal would be included, and no record
+ would be included twice.
+\end_layout
+
+\begin_layout Standard
+This adds complexity (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "Reliable-Traversal-Adds"
+
+\end_inset
+
+) and does not work anyway for records which are altered (in particular,
+ those which are expanded may be effectively deleted and re-added behind
+ the traversal).
+\end_layout
+
+\begin_layout Subsubsection
+\begin_inset CommandInset label
+LatexCommand label
+name "traverse-Proposed-Solution"
+
+\end_inset
+
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Abandon the guarantee.
+ You will see every record if no changes occur during your traversal, otherwise
+ you will see some subset.
+ You can prevent changes by using a transaction or the locking API.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+ Delete-during-traverse will still delete every record, too (assuming no
+ other changes).
+\end_layout
+
+\begin_layout Subsection
+Nesting of Transactions Is Fraught
+\end_layout
+
+\begin_layout Standard
+TDB has alternated between allowing nested transactions and not allowing
+ them.
+ Various paths in the Samba codebase assume that transactions will nest,
+ and in a sense they can: the operation is only committed to disk when the
+ outer transaction is committed.
+ There are two problems, however:
+\end_layout
+
+\begin_layout Enumerate
+Canceling the inner transaction will cause the outer transaction commit
+ to fail, and will not undo any operations since the inner transaction began.
+ This problem is soluble with some additional internal code.
+\end_layout
+
+\begin_layout Enumerate
+An inner transaction commit can be cancelled by the outer transaction.
+ This is desirable in the way which Samba's database initialization code
+ uses transactions, but could be a surprise to any users expecting a successful
+ transaction commit to expose changes to others.
+\end_layout
+
+\begin_layout Standard
+The current solution is to specify the behavior at tdb_open(), with the
+ default currently that nested transactions are allowed.
+ This flag can also be changed at runtime.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Given the usage patterns, it seems that the
+\begin_inset Quotes eld
+\end_inset
+
+least-surprise
+\begin_inset Quotes erd
+\end_inset
+
+ behavior of disallowing nested transactions should become the default.
+ Additionally, it seems the outer transaction is the only code which knows
+ whether inner transactions should be allowed, so a flag to indicate this
+ could be added to tdb_transaction_start.
+ However, this behavior can be simulated with a wrapper which uses tdb_add_flags
+() and tdb_remove_flags(), so the API should not be expanded for this relatively
+-obscure case.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete; the nesting flag has been removed.
+\end_layout
+
+\begin_layout Subsection
+Incorrect Hash Function is Not Detected
+\end_layout
+
+\begin_layout Standard
+tdb_open_ex() allows the calling code to specify a different hash function
+ to use, but does not check that all other processes accessing this tdb
+ are using the same hash function.
+ The result is that records are missing from tdb_fetch().
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The header should contain an example hash result (eg.
+ the hash of 0xdeadbeef), and tdb_open_ex() should check that the given
+ hash function produces the same answer, or fail the tdb_open call.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+tdb_set_max_dead/TDB_VOLATILE Expose Implementation
+\end_layout
+
+\begin_layout Standard
+In response to scalability issues with the free list (
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "TDB-Freelist-Is"
+
+\end_inset
+
+) two API workarounds have been incorporated in TDB: tdb_set_max_dead()
+ and the TDB_VOLATILE flag to tdb_open.
+ The latter actually calls the former with an argument of
+\begin_inset Quotes eld
+\end_inset
+
+5
+\begin_inset Quotes erd
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Standard
+This code allows deleted records to accumulate without putting them in the
+ free list.
+ On delete we iterate through each chain and free them in a batch if there
+ are more than max_dead entries.
+ These are never otherwise recycled except as a side-effect of a tdb_repack.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+With the scalability problems of the freelist solved, this API can be removed.
+ The TDB_VOLATILE flag may still be useful as a hint that store and delete
+ of records will be at least as common as fetch in order to allow some internal
+ tuning, but initially will become a no-op.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+ Unknown flags cause tdb_open() to fail as well, so they can be detected
+ at runtime.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "TDB-Files-Cannot"
+
+\end_inset
+
+TDB Files Cannot Be Opened Multiple Times In The Same Process
+\end_layout
+
+\begin_layout Standard
+No process can open the same TDB twice; we check and disallow it.
+ This is an unfortunate side-effect of fcntl locks, which operate on a per-file
+ rather than per-file-descriptor basis, and do not nest.
+ Thus, closing any file descriptor on a file clears all the locks obtained
+ by this process, even if they were placed using a different file descriptor!
+\end_layout
+
+\begin_layout Standard
+Note that even if this were solved, deadlock could occur if operations were
+ nested: this is a more manageable programming error in most cases.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+We could lobby POSIX to fix the perverse rules, or at least lobby Linux
+ to violate them so that the most common implementation does not have this
+ restriction.
+ This would be a generally good idea for other fcntl lock users.
+\end_layout
+
+\begin_layout Standard
+Samba uses a wrapper which hands out the same tdb_context to multiple callers
+ if this happens, and does simple reference counting.
+ We should do this inside the tdb library, which already emulates lock nesting
+ internally; it would need to recognize when deadlock occurs within a single
+ process.
+ This would create a new failure mode for tdb operations (while we currently
+ handle locking failures, they are impossible in normal use and a process
+ encountering them can do little but give up).
+\end_layout
+
+\begin_layout Standard
+I do not see benefit in an additional tdb_open flag to indicate whether
+ re-opening is allowed, as though there may be some benefit to adding a
+ call to detect when a tdb_context is shared, to allow other to create such
+ an API.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+TDB API Is Not POSIX Thread-safe
+\end_layout
+
+\begin_layout Standard
+The TDB API uses an error code which can be queried after an operation to
+ determine what went wrong.
+ This programming model does not work with threads, unless specific additional
+ guarantees are given by the implementation.
+ In addition, even otherwise-independent threads cannot open the same TDB
+ (as in
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "TDB-Files-Cannot"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Reachitecting the API to include a tdb_errcode pointer would be a great
+ deal of churn, but fortunately most functions return 0 on success and -1
+ on error: we can change these to return 0 on success and a negative error
+ code on error, and the API remains similar to previous.
+ The tdb_fetch, tdb_firstkey and tdb_nextkey functions need to take a TDB_DATA
+ pointer and return an error code.
+ It is also simpler to have tdb_nextkey replace its key argument in place,
+ freeing up any old .dptr.
+\end_layout
+
+\begin_layout Standard
+Internal locking is required to make sure that fcntl locks do not overlap
+ between threads, and also that the global list of tdbs is maintained.
+\end_layout
+
+\begin_layout Standard
+The aim is that building tdb with -DTDB_PTHREAD will result in a pthread-safe
+ version of the library, and otherwise no overhead will exist.
+ Alternatively, a hooking mechanism similar to that proposed for
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "Proposed-Solution-locking-hook"
+
+\end_inset
+
+ could be used to enable pthread locking at runtime.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete; API has been changed but thread safety has not been implemented.
+\end_layout
+
+\begin_layout Subsection
+*_nonblock Functions And *_mark Functions Expose Implementation
+\end_layout
+
+\begin_layout Standard
+CTDB
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+Clustered TDB, see http://ctdb.samba.org
+\end_layout
+
+\end_inset
+
+ wishes to operate on TDB in a non-blocking manner.
+ This is currently done as follows:
+\end_layout
+
+\begin_layout Enumerate
+Call the _nonblock variant of an API function (eg.
+ tdb_lockall_nonblock).
+ If this fails:
+\end_layout
+
+\begin_layout Enumerate
+Fork a child process, and wait for it to call the normal variant (eg.
+ tdb_lockall).
+\end_layout
+
+\begin_layout Enumerate
+If the child succeeds, call the _mark variant to indicate we already have
+ the locks (eg.
+ tdb_lockall_mark).
+\end_layout
+
+\begin_layout Enumerate
+Upon completion, tell the child to release the locks (eg.
+ tdb_unlockall).
+\end_layout
+
+\begin_layout Enumerate
+Indicate to tdb that it should consider the locks removed (eg.
+ tdb_unlockall_mark).
+\end_layout
+
+\begin_layout Standard
+There are several issues with this approach.
+ Firstly, adding two new variants of each function clutters the API for
+ an obscure use, and so not all functions have three variants.
+ Secondly, it assumes that all paths of the functions ask for the same locks,
+ otherwise the parent process will have to get a lock which the child doesn't
+ have under some circumstances.
+ I don't believe this is currently the case, but it constrains the implementatio
+n.
+
+\end_layout
+
+\begin_layout Subsubsection
+\begin_inset CommandInset label
+LatexCommand label
+name "Proposed-Solution-locking-hook"
+
+\end_inset
+
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Implement a hook for locking methods, so that the caller can control the
+ calls to create and remove fcntl locks.
+ In this scenario, ctdbd would operate as follows:
+\end_layout
+
+\begin_layout Enumerate
+Call the normal API function, eg tdb_lockall().
+\end_layout
+
+\begin_layout Enumerate
+When the lock callback comes in, check if the child has the lock.
+ Initially, this is always false.
+ If so, return 0.
+ Otherwise, try to obtain it in non-blocking mode.
+ If that fails, return EWOULDBLOCK.
+\end_layout
+
+\begin_layout Enumerate
+Release locks in the unlock callback as normal.
+\end_layout
+
+\begin_layout Enumerate
+If tdb_lockall() fails, see if we recorded a lock failure; if so, call the
+ child to repeat the operation.
+\end_layout
+
+\begin_layout Enumerate
+The child records what locks it obtains, and returns that information to
+ the parent.
+\end_layout
+
+\begin_layout Enumerate
+When the child has succeeded, goto 1.
+\end_layout
+
+\begin_layout Standard
+This is flexible enough to handle any potential locking scenario, even when
+ lock requirements change.
+ It can be optimized so that the parent does not release locks, just tells
+ the child which locks it doesn't need to obtain.
+\end_layout
+
+\begin_layout Standard
+It also keeps the complexity out of the API, and in ctdbd where it is needed.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+tdb_chainlock Functions Expose Implementation
+\end_layout
+
+\begin_layout Standard
+tdb_chainlock locks some number of records, including the record indicated
+ by the given key.
+ This gave atomicity guarantees; no-one can start a transaction, alter,
+ read or delete that key while the lock is held.
+\end_layout
+
+\begin_layout Standard
+It also makes the same guarantee for any other key in the chain, which is
+ an internal implementation detail and potentially a cause for deadlock.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+ It would be nice to have an explicit single entry lock which effected no
+ other keys.
+ Unfortunately, this won't work for an entry which doesn't exist.
+ Thus while chainlock may be implemented more efficiently for the existing
+ case, it will still have overlap issues with the non-existing case.
+ So it is best to keep the current (lack of) guarantee about which records
+ will be effected to avoid constraining our implementation.
+\end_layout
+
+\begin_layout Subsection
+Signal Handling is Not Race-Free
+\end_layout
+
+\begin_layout Standard
+The tdb_setalarm_sigptr() call allows the caller's signal handler to indicate
+ that the tdb locking code should return with a failure, rather than trying
+ again when a signal is received (and errno == EAGAIN).
+ This is usually used to implement timeouts.
+\end_layout
+
+\begin_layout Standard
+Unfortunately, this does not work in the case where the signal is received
+ before the tdb code enters the fcntl() call to place the lock: the code
+ will sleep within the fcntl() code, unaware that the signal wants it to
+ exit.
+ In the case of long timeouts, this does not happen in practice.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The locking hooks proposed in
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "Proposed-Solution-locking-hook"
+
+\end_inset
+
+ would allow the user to decide on whether to fail the lock acquisition
+ on a signal.
+ This allows the caller to choose their own compromise: they could narrow
+ the race by checking immediately before the fcntl call.
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+It may be possible to make this race-free in some implementations by having
+ the signal handler alter the struct flock to make it invalid.
+ This will cause the fcntl() lock call to fail with EINVAL if the signal
+ occurs before the kernel is entered, otherwise EAGAIN.
+\end_layout
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+The API Uses Gratuitous Typedefs, Capitals
+\end_layout
+
+\begin_layout Standard
+typedefs are useful for providing source compatibility when types can differ
+ across implementations, or arguably in the case of function pointer definitions
+ which are hard for humans to parse.
+ Otherwise it is simply obfuscation and pollutes the namespace.
+\end_layout
+
+\begin_layout Standard
+Capitalization is usually reserved for compile-time constants and macros.
+\end_layout
+
+\begin_layout Description
+TDB_CONTEXT There is no reason to use this over 'struct tdb_context'; the
+ definition isn't visible to the API user anyway.
+\end_layout
+
+\begin_layout Description
+TDB_DATA There is no reason to use this over struct TDB_DATA; the struct
+ needs to be understood by the API user.
+\end_layout
+
+\begin_layout Description
+struct
+\begin_inset space ~
+\end_inset
+
+TDB_DATA This would normally be called 'struct tdb_data'.
+\end_layout
+
+\begin_layout Description
+enum
+\begin_inset space ~
+\end_inset
+
+TDB_ERROR Similarly, this would normally be enum tdb_error.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+ Introducing lower case variants would please pedants like myself, but if
+ it were done the existing ones should be kept.
+ There is little point forcing a purely cosmetic change upon tdb users.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "tdb_log_func-Doesnt-Take"
+
+\end_inset
+
+tdb_log_func Doesn't Take The Private Pointer
+\end_layout
+
+\begin_layout Standard
+For API compatibility reasons, the logging function needs to call tdb_get_loggin
+g_private() to retrieve the pointer registered by the tdb_open_ex for logging.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+It should simply take an extra argument, since we are prepared to break
+ the API/ABI.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Various Callback Functions Are Not Typesafe
+\end_layout
+
+\begin_layout Standard
+The callback functions in tdb_set_logging_function (after
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "tdb_log_func-Doesnt-Take"
+
+\end_inset
+
+ is resolved), tdb_parse_record, tdb_traverse, tdb_traverse_read and tdb_check
+ all take void * and must internally convert it to the argument type they
+ were expecting.
+\end_layout
+
+\begin_layout Standard
+If this type changes, the compiler will not produce warnings on the callers,
+ since it only sees void *.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+With careful use of macros, we can create callback functions which give
+ a warning when used on gcc and the types of the callback and its private
+ argument differ.
+ Unsupported compilers will not give a warning, which is no worse than now.
+ In addition, the callbacks become clearer, as they need not use void *
+ for their parameter.
+\end_layout
+
+\begin_layout Standard
+See CCAN's typesafe_cb module at http://ccan.ozlabs.org/info/typesafe_cb.html
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+TDB_CLEAR_IF_FIRST Must Be Specified On All Opens, tdb_reopen_all Problematic
+\end_layout
+
+\begin_layout Standard
+The TDB_CLEAR_IF_FIRST flag to tdb_open indicates that the TDB file should
+ be cleared if the caller discovers it is the only process with the TDB
+ open.
+ However, if any caller does not specify TDB_CLEAR_IF_FIRST it will not
+ be detected, so will have the TDB erased underneath them (usually resulting
+ in a crash).
+\end_layout
+
+\begin_layout Standard
+There is a similar issue on fork(); if the parent exits (or otherwise closes
+ the tdb) before the child calls tdb_reopen_all() to establish the lock
+ used to indicate the TDB is opened by someone, a TDB_CLEAR_IF_FIRST opener
+ at that moment will believe it alone has opened the TDB and will erase
+ it.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Remove TDB_CLEAR_IF_FIRST.
+ Other workarounds are possible, but see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "TDB_CLEAR_IF_FIRST-Imposes-Performance"
+
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Extending The Header Is Difficult
+\end_layout
+
+\begin_layout Standard
+We have reserved (zeroed) words in the TDB header, which can be used for
+ future features.
+ If the future features are compulsory, the version number must be updated
+ to prevent old code from accessing the database.
+ But if the future feature is optional, we have no way of telling if older
+ code is accessing the database or not.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The header should contain a
+\begin_inset Quotes eld
+\end_inset
+
+format variant
+\begin_inset Quotes erd
+\end_inset
+
+ value (64-bit).
+ This is divided into two 32-bit parts:
+\end_layout
+
+\begin_layout Enumerate
+The lower part reflects the format variant understood by code accessing
+ the database.
+\end_layout
+
+\begin_layout Enumerate
+The upper part reflects the format variant you must understand to write
+ to the database (otherwise you can only open for reading).
+\end_layout
+
+\begin_layout Standard
+The latter field can only be written at creation time, the former should
+ be written under the OPEN_LOCK when opening the database for writing, if
+ the variant of the code is lower than the current lowest variant.
+\end_layout
+
+\begin_layout Standard
+This should allow backwards-compatible features to be added, and detection
+ if older code (which doesn't understand the feature) writes to the database.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Record Headers Are Not Expandible
+\end_layout
+
+\begin_layout Standard
+If we later want to add (say) checksums on keys and data, it would require
+ another format change, which we'd like to avoid.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+We often have extra padding at the tail of a record.
+ If we ensure that the first byte (if any) of this padding is zero, we will
+ have a way for future changes to detect code which doesn't understand a
+ new format: the new code would write (say) a 1 at the tail, and thus if
+ there is no tail or the first byte is 0, we would know the extension is
+ not present on that record.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+TDB Does Not Use Talloc
+\end_layout
+
+\begin_layout Standard
+Many users of TDB (particularly Samba) use the talloc allocator, and thus
+ have to wrap TDB in a talloc context to use it conveniently.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The allocation within TDB is not complicated enough to justify the use of
+ talloc, and I am reluctant to force another (excellent) library on TDB
+ users.
+ Nonetheless a compromise is possible.
+ An attribute (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "attributes"
+
+\end_inset
+
+) can be added later to tdb_open() to provide an alternate allocation mechanism,
+ specifically for talloc but usable by any other allocator (which would
+ ignore the
+\begin_inset Quotes eld
+\end_inset
+
+context
+\begin_inset Quotes erd
+\end_inset
+
+ argument).
+\end_layout
+
+\begin_layout Standard
+This would form a talloc heirarchy as expected, but the caller would still
+ have to attach a destructor to the tdb context returned from tdb_open to
+ close it.
+ All TDB_DATA fields would be children of the tdb_context, and the caller
+ would still have to manage them (using talloc_free() or talloc_steal()).
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Section
+Performance And Scalability Issues
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "TDB_CLEAR_IF_FIRST-Imposes-Performance"
+
+\end_inset
+
+TDB_CLEAR_IF_FIRST Imposes Performance Penalty
+\end_layout
+
+\begin_layout Standard
+When TDB_CLEAR_IF_FIRST is specified, a 1-byte read lock is placed at offset
+ 4 (aka.
+ the ACTIVE_LOCK).
+ While these locks never conflict in normal tdb usage, they do add substantial
+ overhead for most fcntl lock implementations when the kernel scans to detect
+ if a lock conflict exists.
+ This is often a single linked list, making the time to acquire and release
+ a fcntl lock O(N) where N is the number of processes with the TDB open,
+ not the number actually doing work.
+\end_layout
+
+\begin_layout Standard
+In a Samba server it is common to have huge numbers of clients sitting idle,
+ and thus they have weaned themselves off the TDB_CLEAR_IF_FIRST flag.
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+There is a flag to tdb_reopen_all() which is used for this optimization:
+ if the parent process will outlive the child, the child does not need the
+ ACTIVE_LOCK.
+ This is a workaround for this very performance issue.
+\end_layout
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Remove the flag.
+ It was a neat idea, but even trivial servers tend to know when they are
+ initializing for the first time and can simply unlink the old tdb at that
+ point.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+TDB Files Have a 4G Limit
+\end_layout
+
+\begin_layout Standard
+This seems to be becoming an issue (so much for
+\begin_inset Quotes eld
+\end_inset
+
+trivial
+\begin_inset Quotes erd
+\end_inset
+
+!), particularly for ldb.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+A new, incompatible TDB format which uses 64 bit offsets internally rather
+ than 32 bit as now.
+ For simplicity of endian conversion (which TDB does on the fly if required),
+ all values will be 64 bit on disk.
+ In practice, some upper bits may be used for other purposes, but at least
+ 56 bits will be available for file offsets.
+\end_layout
+
+\begin_layout Standard
+tdb_open() will automatically detect the old version, and even create them
+ if TDB_VERSION6 is specified to tdb_open.
+\end_layout
+
+\begin_layout Standard
+32 bit processes will still be able to access TDBs larger than 4G (assuming
+ that their off_t allows them to seek to 64 bits), they will gracefully
+ fall back as they fail to mmap.
+ This can happen already with large TDBs.
+\end_layout
+
+\begin_layout Standard
+Old versions of tdb will fail to open the new TDB files (since 28 August
+ 2009, commit 398d0c29290: prior to that any unrecognized file format would
+ be erased and initialized as a fresh tdb!)
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+TDB Records Have a 4G Limit
+\end_layout
+
+\begin_layout Standard
+This has not been a reported problem, and the API uses size_t which can
+ be 64 bit on 64 bit platforms.
+ However, other limits may have made such an issue moot.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Record sizes will be 64 bit, with an error returned on 32 bit platforms
+ which try to access such records (the current implementation would return
+ TDB_ERR_OOM in a similar case).
+ It seems unlikely that 32 bit keys will be a limitation, so the implementation
+ may not support this (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:Records-Incur-A"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Hash Size Is Determined At TDB Creation Time
+\end_layout
+
+\begin_layout Standard
+TDB contains a number of hash chains in the header; the number is specified
+ at creation time, and defaults to 131.
+ This is such a bottleneck on large databases (as each hash chain gets quite
+ long), that LDB uses 10,000 for this hash.
+ In general it is impossible to know what the 'right' answer is at database
+ creation time.
+\end_layout
+
+\begin_layout Subsubsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:Hash-Size-Solution"
+
+\end_inset
+
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+After comprehensive performance testing on various scalable hash variants
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+http://rusty.ozlabs.org/?p=89 and http://rusty.ozlabs.org/?p=94 This was annoying
+ because I was previously convinced that an expanding tree of hashes would
+ be very close to optimal.
+\end_layout
+
+\end_inset
+
+, it became clear that it is hard to beat a straight linear hash table which
+ doubles in size when it reaches saturation.
+ Unfortunately, altering the hash table introduces serious locking complications
+: the entire hash table needs to be locked to enlarge the hash table, and
+ others might be holding locks.
+ Particularly insidious are insertions done under tdb_chainlock.
+\end_layout
+
+\begin_layout Standard
+Thus an expanding layered hash will be used: an array of hash groups, with
+ each hash group exploding into pointers to lower hash groups once it fills,
+ turning into a hash tree.
+ This has implications for locking: we must lock the entire group in case
+ we need to expand it, yet we don't know how deep the tree is at that point.
+\end_layout
+
+\begin_layout Standard
+Note that bits from the hash table entries should be stolen to hold more
+ hash bits to reduce the penalty of collisions.
+ We can use the otherwise-unused lower 3 bits.
+ If we limit the size of the database to 64 exabytes, we can use the top
+ 8 bits of the hash entry as well.
+ These 11 bits would reduce false positives down to 1 in 2000 which is more
+ than we need: we can use one of the bits to indicate that the extra hash
+ bits are valid.
+ This means we can choose not to re-hash all entries when we expand a hash
+ group; simply use the next bits we need and mark them invalid.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "TDB-Freelist-Is"
+
+\end_inset
+
+TDB Freelist Is Highly Contended
+\end_layout
+
+\begin_layout Standard
+TDB uses a single linked list for the free list.
+ Allocation occurs as follows, using heuristics which have evolved over
+ time:
+\end_layout
+
+\begin_layout Enumerate
+Get the free list lock for this whole operation.
+\end_layout
+
+\begin_layout Enumerate
+Multiply length by 1.25, so we always over-allocate by 25%.
+\end_layout
+
+\begin_layout Enumerate
+Set the slack multiplier to 1.
+\end_layout
+
+\begin_layout Enumerate
+Examine the current freelist entry: if it is > length but < the current
+ best case, remember it as the best case.
+\end_layout
+
+\begin_layout Enumerate
+Multiply the slack multiplier by 1.05.
+\end_layout
+
+\begin_layout Enumerate
+If our best fit so far is less than length * slack multiplier, return it.
+ The slack will be turned into a new free record if it's large enough.
+\end_layout
+
+\begin_layout Enumerate
+Otherwise, go onto the next freelist entry.
+\end_layout
+
+\begin_layout Standard
+Deleting a record occurs as follows:
+\end_layout
+
+\begin_layout Enumerate
+Lock the hash chain for this whole operation.
+\end_layout
+
+\begin_layout Enumerate
+Walk the chain to find the record, keeping the prev pointer offset.
+\end_layout
+
+\begin_layout Enumerate
+If max_dead is non-zero:
+\end_layout
+
+\begin_deeper
+\begin_layout Enumerate
+Walk the hash chain again and count the dead records.
+\end_layout
+
+\begin_layout Enumerate
+If it's more than max_dead, bulk free all the dead ones (similar to steps
+ 4 and below, but the lock is only obtained once).
+\end_layout
+
+\begin_layout Enumerate
+Simply mark this record as dead and return.
+
+\end_layout
+
+\end_deeper
+\begin_layout Enumerate
+Get the free list lock for the remainder of this operation.
+\end_layout
+
+\begin_layout Enumerate
+\begin_inset CommandInset label
+LatexCommand label
+name "right-merging"
+
+\end_inset
+
+Examine the following block to see if it is free; if so, enlarge the current
+ block and remove that block from the free list.
+ This was disabled, as removal from the free list was O(entries-in-free-list).
+\end_layout
+
+\begin_layout Enumerate
+Examine the preceeding block to see if it is free: for this reason, each
+ block has a 32-bit tailer which indicates its length.
+ If it is free, expand it to cover our new block and return.
+\end_layout
+
+\begin_layout Enumerate
+Otherwise, prepend ourselves to the free list.
+\end_layout
+
+\begin_layout Standard
+Disabling right-merging (step
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "right-merging"
+
+\end_inset
+
+) causes fragmentation; the other heuristics proved insufficient to address
+ this, so the final answer to this was that when we expand the TDB file
+ inside a transaction commit, we repack the entire tdb.
+\end_layout
+
+\begin_layout Standard
+The single list lock limits our allocation rate; due to the other issues
+ this is not currently seen as a bottleneck.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The first step is to remove all the current heuristics, as they obviously
+ interact, then examine them once the lock contention is addressed.
+\end_layout
+
+\begin_layout Standard
+The free list must be split to reduce contention.
+ Assuming perfect free merging, we can at most have 1 free list entry for
+ each entry.
+ This implies that the number of free lists is related to the size of the
+ hash table, but as it is rare to walk a large number of free list entries
+ we can use far fewer, say 1/32 of the number of hash buckets.
+\end_layout
+
+\begin_layout Standard
+It seems tempting to try to reuse the hash implementation which we use for
+ records here, but we have two ways of searching for free entries: for allocatio
+n we search by size (and possibly zone) which produces too many clashes
+ for our hash table to handle well, and for coalescing we search by address.
+ Thus an array of doubly-linked free lists seems preferable.
+\end_layout
+
+\begin_layout Standard
+There are various benefits in using per-size free lists (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:TDB-Becomes-Fragmented"
+
+\end_inset
+
+) but it's not clear this would reduce contention in the common case where
+ all processes are allocating/freeing the same size.
+ Thus we almost certainly need to divide in other ways: the most obvious
+ is to divide the file into zones, and using a free list (or table of free
+ lists) for each.
+ This approximates address ordering.
+\end_layout
+
+\begin_layout Standard
+Unfortunately it is difficult to know what heuristics should be used to
+ determine zone sizes, and our transaction code relies on being able to
+ create a
+\begin_inset Quotes eld
+\end_inset
+
+recovery area
+\begin_inset Quotes erd
+\end_inset
+
+ by simply appending to the file (difficult if it would need to create a
+ new zone header).
+ Thus we use a linked-list of free tables; currently we only ever create
+ one, but if there is more than one we choose one at random to use.
+ In future we may use heuristics to add new free tables on contention.
+ We only expand the file when all free tables are exhausted.
+\end_layout
+
+\begin_layout Standard
+The basic algorithm is as follows.
+ Freeing is simple:
+\end_layout
+
+\begin_layout Enumerate
+Identify the correct free list.
+\end_layout
+
+\begin_layout Enumerate
+Lock the corresponding list.
+\end_layout
+
+\begin_layout Enumerate
+Re-check the list (we didn't have a lock, sizes could have changed): relock
+ if necessary.
+\end_layout
+
+\begin_layout Enumerate
+Place the freed entry in the list.
+\end_layout
+
+\begin_layout Standard
+Allocation is a little more complicated, as we perform delayed coalescing
+ at this point:
+\end_layout
+
+\begin_layout Enumerate
+Pick a free table; usually the previous one.
+\end_layout
+
+\begin_layout Enumerate
+Lock the corresponding list.
+\end_layout
+
+\begin_layout Enumerate
+If the top entry is -large enough, remove it from the list and return it.
+\end_layout
+
+\begin_layout Enumerate
+Otherwise, coalesce entries in the list.If there was no entry large enough,
+ unlock the list and try the next largest list
+\end_layout
+
+\begin_layout Enumerate
+If no list has an entry which meets our needs, try the next free table.
+\end_layout
+
+\begin_layout Enumerate
+If no zone satisfies, expand the file.
+\end_layout
+
+\begin_layout Standard
+This optimizes rapid insert/delete of free list entries by not coalescing
+ them all the time..
+ First-fit address ordering ordering seems to be fairly good for keeping
+ fragmentation low (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:TDB-Becomes-Fragmented"
+
+\end_inset
+
+).
+ Note that address ordering does not need a tailer to coalesce, though if
+ we needed one we could have one cheaply: see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:Records-Incur-A"
+
+\end_inset
+
+.
+
+\end_layout
+
+\begin_layout Standard
+Each free entry has the free table number in the header: less than 255.
+ It also contains a doubly-linked list for easy deletion.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:TDB-Becomes-Fragmented"
+
+\end_inset
+
+TDB Becomes Fragmented
+\end_layout
+
+\begin_layout Standard
+Much of this is a result of allocation strategy
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+The Memory Fragmentation Problem: Solved? Johnstone & Wilson 1995 ftp://ftp.cs.ute
+xas.edu/pub/garbage/malloc/ismm98.ps
+\end_layout
+
+\end_inset
+
+ and deliberate hobbling of coalescing; internal fragmentation (aka overallocati
+on) is deliberately set at 25%, and external fragmentation is only cured
+ by the decision to repack the entire db when a transaction commit needs
+ to enlarge the file.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The 25% overhead on allocation works in practice for ldb because indexes
+ tend to expand by one record at a time.
+ This internal fragmentation can be resolved by having an
+\begin_inset Quotes eld
+\end_inset
+
+expanded
+\begin_inset Quotes erd
+\end_inset
+
+ bit in the header to note entries that have previously expanded, and allocating
+ more space for them.
+\end_layout
+
+\begin_layout Standard
+There are is a spectrum of possible solutions for external fragmentation:
+ one is to use a fragmentation-avoiding allocation strategy such as best-fit
+ address-order allocator.
+ The other end of the spectrum would be to use a bump allocator (very fast
+ and simple) and simply repack the file when we reach the end.
+\end_layout
+
+\begin_layout Standard
+There are three problems with efficient fragmentation-avoiding allocators:
+ they are non-trivial, they tend to use a single free list for each size,
+ and there's no evidence that tdb allocation patterns will match those recorded
+ for general allocators (though it seems likely).
+\end_layout
+
+\begin_layout Standard
+Thus we don't spend too much effort on external fragmentation; we will be
+ no worse than the current code if we need to repack on occasion.
+ More effort is spent on reducing freelist contention, and reducing overhead.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:Records-Incur-A"
+
+\end_inset
+
+Records Incur A 28-Byte Overhead
+\end_layout
+
+\begin_layout Standard
+Each TDB record has a header as follows:
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_record {
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_off_t next; /* offset of the next record in the list */
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_len_t rec_len; /* total byte length of record */
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_len_t key_len; /* byte length of key */
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_len_t data_len; /* byte length of data */
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t full_hash; /* the full 32 bit hash of the key */
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t magic;   /* try to catch errors */
+\end_layout
+
+\begin_layout LyX-Code
+        /* the following union is implied:
+\end_layout
+
+\begin_layout LyX-Code
+                union {
+\end_layout
+
+\begin_layout LyX-Code
+                        char record[rec_len];
+\end_layout
+
+\begin_layout LyX-Code
+                        struct {
+\end_layout
+
+\begin_layout LyX-Code
+                                char key[key_len];
+\end_layout
+
+\begin_layout LyX-Code
+                                char data[data_len];
+\end_layout
+
+\begin_layout LyX-Code
+                        }
+\end_layout
+
+\begin_layout LyX-Code
+                        uint32_t totalsize; (tailer)
+\end_layout
+
+\begin_layout LyX-Code
+                }
+\end_layout
+
+\begin_layout LyX-Code
+        */
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+Naively, this would double to a 56-byte overhead on a 64 bit implementation.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+We can use various techniques to reduce this for an allocated block:
+\end_layout
+
+\begin_layout Enumerate
+The 'next' pointer is not required, as we are using a flat hash table.
+\end_layout
+
+\begin_layout Enumerate
+'rec_len' can instead be expressed as an addition to key_len and data_len
+ (it accounts for wasted or overallocated length in the record).
+ Since the record length is always a multiple of 8, we can conveniently
+ fit it in 32 bits (representing up to 35 bits).
+\end_layout
+
+\begin_layout Enumerate
+'key_len' and 'data_len' can be reduced.
+ I'm unwilling to restrict 'data_len' to 32 bits, but instead we can combine
+ the two into one 64-bit field and using a 5 bit value which indicates at
+ what bit to divide the two.
+ Keys are unlikely to scale as fast as data, so I'm assuming a maximum key
+ size of 32 bits.
+\end_layout
+
+\begin_layout Enumerate
+'full_hash' is used to avoid a memcmp on the
+\begin_inset Quotes eld
+\end_inset
+
+miss
+\begin_inset Quotes erd
+\end_inset
+
+ case, but this is diminishing returns after a handful of bits (at 10 bits,
+ it reduces 99.9% of false memcmp).
+ As an aside, as the lower bits are already incorporated in the hash table
+ resolution, the upper bits should be used here.
+ Note that it's not clear that these bits will be a win, given the extra
+ bits in the hash table itself (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:Hash-Size-Solution"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Enumerate
+'magic' does not need to be enlarged: it currently reflects one of 5 values
+ (used, free, dead, recovery, and unused_recovery).
+ It is useful for quick sanity checking however, and should not be eliminated.
+\end_layout
+
+\begin_layout Enumerate
+'tailer' is only used to coalesce free blocks (so a block to the right can
+ find the header to check if this block is free).
+ This can be replaced by a single 'free' bit in the header of the following
+ block (and the tailer only exists in free blocks).
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+This technique from Thomas Standish.
+ Data Structure Techniques.
+ Addison-Wesley, Reading, Massachusetts, 1980.
+\end_layout
+
+\end_inset
+
+ The current proposed coalescing algorithm doesn't need this, however.
+\end_layout
+
+\begin_layout Standard
+This produces a 16 byte used header like this:
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_used_record {
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t used_magic : 16,
+\end_layout
+
+\begin_layout LyX-Code
+
+\end_layout
+
+\begin_layout LyX-Code
+                 key_data_divide: 5,
+\end_layout
+
+\begin_layout LyX-Code
+                 top_hash: 11;
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t extra_octets;
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t key_and_data_len;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+And a free record like this:
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_free_record {
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t free_magic: 8,
+\end_layout
+
+\begin_layout LyX-Code
+                   prev : 56;
+\end_layout
+
+\begin_layout LyX-Code
+
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t free_table: 8,
+\end_layout
+
+\begin_layout LyX-Code
+                 total_length : 56
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t next;;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+Note that by limiting valid offsets to 56 bits, we can pack everything we
+ need into 3 64-byte words, meaning our minimum record size is 8 bytes.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Transaction Commit Requires 4 fdatasync
+\end_layout
+
+\begin_layout Standard
+The current transaction algorithm is:
+\end_layout
+
+\begin_layout Enumerate
+write_recovery_data();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Enumerate
+write_recovery_header();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Enumerate
+overwrite_with_new_data();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Enumerate
+remove_recovery_header();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Standard
+On current ext3, each sync flushes all data to disk, so the next 3 syncs
+ are relatively expensive.
+ But this could become a performance bottleneck on other filesystems such
+ as ext4.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Neil Brown points out that this is overzealous, and only one sync is needed:
+\end_layout
+
+\begin_layout Enumerate
+Bundle the recovery data, a transaction counter and a strong checksum of
+ the new data.
+\end_layout
+
+\begin_layout Enumerate
+Strong checksum that whole bundle.
+\end_layout
+
+\begin_layout Enumerate
+Store the bundle in the database.
+\end_layout
+
+\begin_layout Enumerate
+Overwrite the oldest of the two recovery pointers in the header (identified
+ using the transaction counter) with the offset of this bundle.
+\end_layout
+
+\begin_layout Enumerate
+sync.
+\end_layout
+
+\begin_layout Enumerate
+Write the new data to the file.
+\end_layout
+
+\begin_layout Standard
+Checking for recovery means identifying the latest bundle with a valid checksum
+ and using the new data checksum to ensure that it has been applied.
+ This is more expensive than the current check, but need only be done at
+ open.
+ For running databases, a separate header field can be used to indicate
+ a transaction in progress; we need only check for recovery if this is set.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:TDB-Does-Not"
+
+\end_inset
+
+TDB Does Not Have Snapshot Support
+\end_layout
+
+\begin_layout Subsubsection
+Proposed SolutionNone.
+ At some point you say
+\begin_inset Quotes eld
+\end_inset
+
+use a real database
+\begin_inset Quotes erd
+\end_inset
+
+ (but see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "replay-attribute"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Standard
+But as a thought experiment, if we implemented transactions to only overwrite
+ free entries (this is tricky: there must not be a header in each entry
+ which indicates whether it is free, but use of presence in metadata elsewhere),
+ and a pointer to the hash table, we could create an entirely new commit
+ without destroying existing data.
+ Then it would be easy to implement snapshots in a similar way.
+\end_layout
+
+\begin_layout Standard
+This would not allow arbitrary changes to the database, such as tdb_repack
+ does, and would require more space (since we have to preserve the current
+ and future entries at once).
+ If we used hash trees rather than one big hash table, we might only have
+ to rewrite some sections of the hash, too.
+\end_layout
+
+\begin_layout Standard
+We could then implement snapshots using a similar method, using multiple
+ different hash tables/free tables.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Subsection
+Transactions Cannot Operate in Parallel
+\end_layout
+
+\begin_layout Standard
+This would be useless for ldb, as it hits the index records with just about
+ every update.
+ It would add significant complexity in resolving clashes, and cause the
+ all transaction callers to write their code to loop in the case where the
+ transactions spuriously failed.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None (but see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "replay-attribute"
+
+\end_inset
+
+).
+ We could solve a small part of the problem by providing read-only transactions.
+ These would allow one write transaction to begin, but it could not commit
+ until all r/o transactions are done.
+ This would require a new RO_TRANSACTION_LOCK, which would be upgraded on
+ commit.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Subsection
+Default Hash Function Is Suboptimal
+\end_layout
+
+\begin_layout Standard
+The Knuth-inspired multiplicative hash used by tdb is fairly slow (especially
+ if we expand it to 64 bits), and works best when the hash bucket size is
+ a prime number (which also means a slow modulus).
+ In addition, it is highly predictable which could potentially lead to a
+ Denial of Service attack in some TDB uses.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The Jenkins lookup3 hash
+\begin_inset Foot
+status open
+
+\begin_layout Plain Layout
+http://burtleburtle.net/bob/c/lookup3.c
+\end_layout
+
+\end_inset
+
+ is a fast and superbly-mixing hash.
+ It's used by the Linux kernel and almost everything else.
+ This has the particular properties that it takes an initial seed, and produces
+ two 32 bit hash numbers, which we can combine into a 64-bit hash.
+\end_layout
+
+\begin_layout Standard
+The seed should be created at tdb-creation time from some random source,
+ and placed in the header.
+ This is far from foolproof, but adds a little bit of protection against
+ hash bombing.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "Reliable-Traversal-Adds"
+
+\end_inset
+
+Reliable Traversal Adds Complexity
+\end_layout
+
+\begin_layout Standard
+We lock a record during traversal iteration, and try to grab that lock in
+ the delete code.
+ If that grab on delete fails, we simply mark it deleted and continue onwards;
+ traversal checks for this condition and does the delete when it moves off
+ the record.
+\end_layout
+
+\begin_layout Standard
+If traversal terminates, the dead record may be left indefinitely.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Remove reliability guarantees; see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "traverse-Proposed-Solution"
+
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Fcntl Locking Adds Overhead
+\end_layout
+
+\begin_layout Standard
+Placing a fcntl lock means a system call, as does removing one.
+ This is actually one reason why transactions can be faster (everything
+ is locked once at transaction start).
+ In the uncontended case, this overhead can theoretically be eliminated.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+\end_layout
+
+\begin_layout Standard
+We tried this before with spinlock support, in the early days of TDB, and
+ it didn't make much difference except in manufactured benchmarks.
+\end_layout
+
+\begin_layout Standard
+We could use spinlocks (with futex kernel support under Linux), but it means
+ that we lose automatic cleanup when a process dies with a lock.
+ There is a method of auto-cleanup under Linux, but it's not supported by
+ other operating systems.
+ We could reintroduce a clear-if-first-style lock and sweep for dead futexes
+ on open, but that wouldn't help the normal case of one concurrent opener
+ dying.
+ Increasingly elaborate repair schemes could be considered, but they require
+ an ABI change (everyone must use them) anyway, so there's no need to do
+ this at the same time as everything else.
+\end_layout
+
+\begin_layout Subsection
+Some Transactions Don't Require Durability
+\end_layout
+
+\begin_layout Standard
+Volker points out that gencache uses a CLEAR_IF_FIRST tdb for normal (fast)
+ usage, and occasionally empties the results into a transactional TDB.
+ This kind of usage prioritizes performance over durability: as long as
+ we are consistent, data can be lost.
+\end_layout
+
+\begin_layout Standard
+This would be more neatly implemented inside tdb: a
+\begin_inset Quotes eld
+\end_inset
+
+soft
+\begin_inset Quotes erd
+\end_inset
+
+ transaction commit (ie.
+ syncless) which meant that data may be reverted on a crash.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+\end_layout
+
+\begin_layout Standard
+Unfortunately any transaction scheme which overwrites old data requires
+ a sync before that overwrite to avoid the possibility of corruption.
+\end_layout
+
+\begin_layout Standard
+It seems possible to use a scheme similar to that described in
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:TDB-Does-Not"
+
+\end_inset
+
+,where transactions are committed without overwriting existing data, and
+ an array of top-level pointers were available in the header.
+ If the transaction is
+\begin_inset Quotes eld
+\end_inset
+
+soft
+\begin_inset Quotes erd
+\end_inset
+
+ then we would not need a sync at all: existing processes would pick up
+ the new hash table and free list and work with that.
+\end_layout
+
+\begin_layout Standard
+At some later point, a sync would allow recovery of the old data into the
+ free lists (perhaps when the array of top-level pointers filled).
+ On crash, tdb_open() would examine the array of top levels, and apply the
+ transactions until it encountered an invalid checksum.
+\end_layout
+
+\begin_layout Subsection
+Tracing Is Fragile, Replay Is External
+\end_layout
+
+\begin_layout Standard
+The current TDB has compile-time-enabled tracing code, but it often breaks
+ as it is not enabled by default.
+ In a similar way, the ctdb code has an external wrapper which does replay
+ tracing so it can coordinate cluster-wide transactions.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\begin_inset CommandInset label
+LatexCommand label
+name "replay-attribute"
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+Tridge points out that an attribute can be later added to tdb_open (see
+
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "attributes"
+
+\end_inset
+
+) to provide replay/trace hooks, which could become the basis for this and
+ future parallel transactions and snapshot support.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\end_body
+\end_document
diff --git a/lib/tdb2/doc/design.lyx,v b/lib/tdb2/doc/design.lyx,v
new file mode 100644
index 00000000000..13e6387f7fa
--- /dev/null
+++ b/lib/tdb2/doc/design.lyx,v
@@ -0,0 +1,4679 @@
+head	1.13;
+access;
+symbols;
+locks; strict;
+comment	@# @;
+
+
+1.13
+date	2011.03.01.11.46.54;	author rusty;	state Exp;
+branches;
+next	1.12;
+
+1.12
+date	2010.12.01.12.20.49;	author rusty;	state Exp;
+branches;
+next	1.11;
+
+1.11
+date	2010.12.01.11.55.20;	author rusty;	state Exp;
+branches;
+next	1.10;
+
+1.10
+date	2010.09.14.00.33.57;	author rusty;	state Exp;
+branches;
+next	1.9;
+
+1.9
+date	2010.09.09.07.25.12;	author rusty;	state Exp;
+branches;
+next	1.8;
+
+1.8
+date	2010.09.02.02.29.05;	author rusty;	state Exp;
+branches;
+next	1.7;
+
+1.7
+date	2010.09.01.10.58.12;	author rusty;	state Exp;
+branches;
+next	1.6;
+
+1.6
+date	2010.08.02.00.21.43;	author rusty;	state Exp;
+branches;
+next	1.5;
+
+1.5
+date	2010.08.02.00.21.16;	author rusty;	state Exp;
+branches;
+next	1.4;
+
+1.4
+date	2010.05.10.13.09.11;	author rusty;	state Exp;
+branches;
+next	1.3;
+
+1.3
+date	2010.05.10.11.58.37;	author rusty;	state Exp;
+branches;
+next	1.2;
+
+1.2
+date	2010.05.10.05.35.13;	author rusty;	state Exp;
+branches;
+next	1.1;
+
+1.1
+date	2010.05.04.02.29.16;	author rusty;	state Exp;
+branches;
+next	;
+
+
+desc
+@First draft
+@
+
+
+1.13
+log
+@Thread-safe API
+@
+text
+@#LyX 1.6.7 created this file. For more info see http://www.lyx.org/
+\lyxformat 345
+\begin_document
+\begin_header
+\textclass article
+\use_default_options true
+\language english
+\inputencoding auto
+\font_roman default
+\font_sans default
+\font_typewriter default
+\font_default_family default
+\font_sc false
+\font_osf false
+\font_sf_scale 100
+\font_tt_scale 100
+
+\graphics default
+\paperfontsize default
+\use_hyperref false
+\papersize default
+\use_geometry false
+\use_amsmath 1
+\use_esint 1
+\cite_engine basic
+\use_bibtopic false
+\paperorientation portrait
+\secnumdepth 3
+\tocdepth 3
+\paragraph_separation indent
+\defskip medskip
+\quotes_language english
+\papercolumns 1
+\papersides 1
+\paperpagestyle default
+\tracking_changes true
+\output_changes true
+\author "Rusty Russell,,,"
+\author ""
+\end_header
+
+\begin_body
+
+\begin_layout Title
+TDB2: A Redesigning The Trivial DataBase
+\end_layout
+
+\begin_layout Author
+Rusty Russell, IBM Corporation
+\end_layout
+
+\begin_layout Date
+1-December-2010
+\end_layout
+
+\begin_layout Abstract
+The Trivial DataBase on-disk format is 32 bits; with usage cases heading
+ towards the 4G limit, that must change.
+ This required breakage provides an opportunity to revisit TDB's other design
+ decisions and reassess them.
+\end_layout
+
+\begin_layout Section
+Introduction
+\end_layout
+
+\begin_layout Standard
+The Trivial DataBase was originally written by Andrew Tridgell as a simple
+ key/data pair storage system with the same API as dbm, but allowing multiple
+ readers and writers while being small enough (< 1000 lines of C) to include
+ in SAMBA.
+ The simple design created in 1999 has proven surprisingly robust and performant
+, used in Samba versions 3 and 4 as well as numerous other projects.
+ Its useful life was greatly increased by the (backwards-compatible!) addition
+ of transaction support in 2005.
+\end_layout
+
+\begin_layout Standard
+The wider variety and greater demands of TDB-using code has lead to some
+ organic growth of the API, as well as some compromises on the implementation.
+ None of these, by themselves, are seen as show-stoppers, but the cumulative
+ effect is to a loss of elegance over the initial, simple TDB implementation.
+ Here is a table of the approximate number of lines of implementation code
+ and number of API functions at the end of each year:
+\end_layout
+
+\begin_layout Standard
+\begin_inset Tabular
+<lyxtabular version="3" rows="12" columns="3">
+<features>
+<column alignment="center" valignment="top" width="0">
+<column alignment="center" valignment="top" width="0">
+<column alignment="center" valignment="top" width="0">
+<row>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+Year End
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+API Functions
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+Lines of C Code Implementation
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+1999
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+13
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+1195
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2000
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+24
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+1725
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2001
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+32
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2228
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2002
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+35
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2481
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2003
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+35
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2552
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2004
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+40
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2584
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2005
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+38
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2647
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2006
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+52
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+3754
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2007
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+66
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+4398
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2008
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+71
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+4768
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+2009
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+73
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+5715
+\end_layout
+
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+This review is an attempt to catalog and address all the known issues with
+ TDB and create solutions which address the problems without significantly
+ increasing complexity; all involved are far too aware of the dangers of
+ second system syndrome in rewriting a successful project like this.
+\end_layout
+
+\begin_layout Section
+API Issues
+\end_layout
+
+\begin_layout Subsection
+tdb_open_ex Is Not Expandable
+\end_layout
+
+\begin_layout Standard
+The tdb_open() call was expanded to tdb_open_ex(), which added an optional
+ hashing function and an optional logging function argument.
+ Additional arguments to open would require the introduction of a tdb_open_ex2
+ call etc.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\begin_inset CommandInset label
+LatexCommand label
+name "attributes"
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+tdb_open() will take a linked-list of attributes:
+\end_layout
+
+\begin_layout LyX-Code
+enum tdb_attribute {
+\end_layout
+
+\begin_layout LyX-Code
+    TDB_ATTRIBUTE_LOG = 0,
+\end_layout
+
+\begin_layout LyX-Code
+    TDB_ATTRIBUTE_HASH = 1
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_attribute_base {
+\end_layout
+
+\begin_layout LyX-Code
+    enum tdb_attribute attr;
+\end_layout
+
+\begin_layout LyX-Code
+    union tdb_attribute *next;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_attribute_log {
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_LOG */
+\end_layout
+
+\begin_layout LyX-Code
+    tdb_log_func log_fn;
+\end_layout
+
+\begin_layout LyX-Code
+    void *log_private;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_attribute_hash {
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_HASH */
+\end_layout
+
+\begin_layout LyX-Code
+    tdb_hash_func hash_fn;
+\end_layout
+
+\begin_layout LyX-Code
+    void *hash_private;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout LyX-Code
+union tdb_attribute {
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_base base;
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_log log;
+\end_layout
+
+\begin_layout LyX-Code
+    struct tdb_attribute_hash hash;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+This allows future attributes to be added, even if this expands the size
+ of the union.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+tdb_traverse Makes Impossible Guarantees
+\end_layout
+
+\begin_layout Standard
+tdb_traverse (and tdb_firstkey/tdb_nextkey) predate transactions, and it
+ was thought that it was important to guarantee that all records which exist
+ at the start and end of the traversal would be included, and no record
+ would be included twice.
+\end_layout
+
+\begin_layout Standard
+This adds complexity (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "Reliable-Traversal-Adds"
+
+\end_inset
+
+) and does not work anyway for records which are altered (in particular,
+ those which are expanded may be effectively deleted and re-added behind
+ the traversal).
+\end_layout
+
+\begin_layout Subsubsection
+\begin_inset CommandInset label
+LatexCommand label
+name "traverse-Proposed-Solution"
+
+\end_inset
+
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Abandon the guarantee.
+ You will see every record if no changes occur during your traversal, otherwise
+ you will see some subset.
+ You can prevent changes by using a transaction or the locking API.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+ Delete-during-traverse will still delete every record, too (assuming no
+ other changes).
+\end_layout
+
+\begin_layout Subsection
+Nesting of Transactions Is Fraught
+\end_layout
+
+\begin_layout Standard
+TDB has alternated between allowing nested transactions and not allowing
+ them.
+ Various paths in the Samba codebase assume that transactions will nest,
+ and in a sense they can: the operation is only committed to disk when the
+ outer transaction is committed.
+ There are two problems, however:
+\end_layout
+
+\begin_layout Enumerate
+Canceling the inner transaction will cause the outer transaction commit
+ to fail, and will not undo any operations since the inner transaction began.
+ This problem is soluble with some additional internal code.
+\end_layout
+
+\begin_layout Enumerate
+An inner transaction commit can be cancelled by the outer transaction.
+ This is desirable in the way which Samba's database initialization code
+ uses transactions, but could be a surprise to any users expecting a successful
+ transaction commit to expose changes to others.
+\end_layout
+
+\begin_layout Standard
+The current solution is to specify the behavior at tdb_open(), with the
+ default currently that nested transactions are allowed.
+ This flag can also be changed at runtime.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Given the usage patterns, it seems that the
+\begin_inset Quotes eld
+\end_inset
+
+least-surprise
+\begin_inset Quotes erd
+\end_inset
+
+ behavior of disallowing nested transactions should become the default.
+ Additionally, it seems the outer transaction is the only code which knows
+ whether inner transactions should be allowed, so a flag to indicate this
+ could be added to tdb_transaction_start.
+ However, this behavior can be simulated with a wrapper which uses tdb_add_flags
+() and tdb_remove_flags(), so the API should not be expanded for this relatively
+-obscure case.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1298979572
+Incomplete; nesting flag is still defined as per tdb1.
+\change_inserted 0 1298979584
+Complete; the nesting flag has been removed.
+\change_unchanged
+
+\end_layout
+
+\begin_layout Subsection
+Incorrect Hash Function is Not Detected
+\end_layout
+
+\begin_layout Standard
+tdb_open_ex() allows the calling code to specify a different hash function
+ to use, but does not check that all other processes accessing this tdb
+ are using the same hash function.
+ The result is that records are missing from tdb_fetch().
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The header should contain an example hash result (eg.
+ the hash of 0xdeadbeef), and tdb_open_ex() should check that the given
+ hash function produces the same answer, or fail the tdb_open call.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+tdb_set_max_dead/TDB_VOLATILE Expose Implementation
+\end_layout
+
+\begin_layout Standard
+In response to scalability issues with the free list (
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "TDB-Freelist-Is"
+
+\end_inset
+
+) two API workarounds have been incorporated in TDB: tdb_set_max_dead()
+ and the TDB_VOLATILE flag to tdb_open.
+ The latter actually calls the former with an argument of
+\begin_inset Quotes eld
+\end_inset
+
+5
+\begin_inset Quotes erd
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Standard
+This code allows deleted records to accumulate without putting them in the
+ free list.
+ On delete we iterate through each chain and free them in a batch if there
+ are more than max_dead entries.
+ These are never otherwise recycled except as a side-effect of a tdb_repack.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+With the scalability problems of the freelist solved, this API can be removed.
+ The TDB_VOLATILE flag may still be useful as a hint that store and delete
+ of records will be at least as common as fetch in order to allow some internal
+ tuning, but initially will become a no-op.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+ TDB_VOLATILE still defined, but implementation should fail on unknown flags
+ to be future-proof.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "TDB-Files-Cannot"
+
+\end_inset
+
+TDB Files Cannot Be Opened Multiple Times In The Same Process
+\end_layout
+
+\begin_layout Standard
+No process can open the same TDB twice; we check and disallow it.
+ This is an unfortunate side-effect of fcntl locks, which operate on a per-file
+ rather than per-file-descriptor basis, and do not nest.
+ Thus, closing any file descriptor on a file clears all the locks obtained
+ by this process, even if they were placed using a different file descriptor!
+\end_layout
+
+\begin_layout Standard
+Note that even if this were solved, deadlock could occur if operations were
+ nested: this is a more manageable programming error in most cases.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+We could lobby POSIX to fix the perverse rules, or at least lobby Linux
+ to violate them so that the most common implementation does not have this
+ restriction.
+ This would be a generally good idea for other fcntl lock users.
+\end_layout
+
+\begin_layout Standard
+Samba uses a wrapper which hands out the same tdb_context to multiple callers
+ if this happens, and does simple reference counting.
+ We should do this inside the tdb library, which already emulates lock nesting
+ internally; it would need to recognize when deadlock occurs within a single
+ process.
+ This would create a new failure mode for tdb operations (while we currently
+ handle locking failures, they are impossible in normal use and a process
+ encountering them can do little but give up).
+\end_layout
+
+\begin_layout Standard
+I do not see benefit in an additional tdb_open flag to indicate whether
+ re-opening is allowed, as though there may be some benefit to adding a
+ call to detect when a tdb_context is shared, to allow other to create such
+ an API.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+TDB API Is Not POSIX Thread-safe
+\end_layout
+
+\begin_layout Standard
+The TDB API uses an error code which can be queried after an operation to
+ determine what went wrong.
+ This programming model does not work with threads, unless specific additional
+ guarantees are given by the implementation.
+ In addition, even otherwise-independent threads cannot open the same TDB
+ (as in
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "TDB-Files-Cannot"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Reachitecting the API to include a tdb_errcode pointer would be a great
+ deal of churn
+\change_inserted 0 1298979557
+, but fortunately most functions return 0 on success and -1 on error: we
+ can change these to return 0 on success and a negative error code on error,
+ and the API remains similar to previous.
+ The tdb_fetch, tdb_firstkey and tdb_nextkey functions need to take a TDB_DATA
+ pointer and return an error code.
+ It is also simpler to have tdb_nextkey replace its key argument in place,
+ freeing up any old .dptr.
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1298979438
+; we are better to guarantee that the tdb_errcode is per-thread so the current
+ programming model can be maintained.
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1298979438
+This requires dynamic per-thread allocations, which is awkward with POSIX
+ threads (pthread_key_create space is limited and we cannot simply allocate
+ a key for every TDB).
+\change_unchanged
+
+\end_layout
+
+\begin_layout Standard
+Internal locking is required to make sure that fcntl locks do not overlap
+ between threads, and also that the global list of tdbs is maintained.
+\end_layout
+
+\begin_layout Standard
+The aim is that building tdb with -DTDB_PTHREAD will result in a pthread-safe
+ version of the library, and otherwise no overhead will exist.
+ Alternatively, a hooking mechanism similar to that proposed for
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "Proposed-Solution-locking-hook"
+
+\end_inset
+
+ could be used to enable pthread locking at runtime.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete
+\change_inserted 0 1298979681
+; API has been changed but thread safety has not been implemented.
+\change_deleted 0 1298979669
+.
+\change_unchanged
+
+\end_layout
+
+\begin_layout Subsection
+*_nonblock Functions And *_mark Functions Expose Implementation
+\end_layout
+
+\begin_layout Standard
+CTDB
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+Clustered TDB, see http://ctdb.samba.org
+\end_layout
+
+\end_inset
+
+ wishes to operate on TDB in a non-blocking manner.
+ This is currently done as follows:
+\end_layout
+
+\begin_layout Enumerate
+Call the _nonblock variant of an API function (eg.
+ tdb_lockall_nonblock).
+ If this fails:
+\end_layout
+
+\begin_layout Enumerate
+Fork a child process, and wait for it to call the normal variant (eg.
+ tdb_lockall).
+\end_layout
+
+\begin_layout Enumerate
+If the child succeeds, call the _mark variant to indicate we already have
+ the locks (eg.
+ tdb_lockall_mark).
+\end_layout
+
+\begin_layout Enumerate
+Upon completion, tell the child to release the locks (eg.
+ tdb_unlockall).
+\end_layout
+
+\begin_layout Enumerate
+Indicate to tdb that it should consider the locks removed (eg.
+ tdb_unlockall_mark).
+\end_layout
+
+\begin_layout Standard
+There are several issues with this approach.
+ Firstly, adding two new variants of each function clutters the API for
+ an obscure use, and so not all functions have three variants.
+ Secondly, it assumes that all paths of the functions ask for the same locks,
+ otherwise the parent process will have to get a lock which the child doesn't
+ have under some circumstances.
+ I don't believe this is currently the case, but it constrains the implementatio
+n.
+
+\end_layout
+
+\begin_layout Subsubsection
+\begin_inset CommandInset label
+LatexCommand label
+name "Proposed-Solution-locking-hook"
+
+\end_inset
+
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Implement a hook for locking methods, so that the caller can control the
+ calls to create and remove fcntl locks.
+ In this scenario, ctdbd would operate as follows:
+\end_layout
+
+\begin_layout Enumerate
+Call the normal API function, eg tdb_lockall().
+\end_layout
+
+\begin_layout Enumerate
+When the lock callback comes in, check if the child has the lock.
+ Initially, this is always false.
+ If so, return 0.
+ Otherwise, try to obtain it in non-blocking mode.
+ If that fails, return EWOULDBLOCK.
+\end_layout
+
+\begin_layout Enumerate
+Release locks in the unlock callback as normal.
+\end_layout
+
+\begin_layout Enumerate
+If tdb_lockall() fails, see if we recorded a lock failure; if so, call the
+ child to repeat the operation.
+\end_layout
+
+\begin_layout Enumerate
+The child records what locks it obtains, and returns that information to
+ the parent.
+\end_layout
+
+\begin_layout Enumerate
+When the child has succeeded, goto 1.
+\end_layout
+
+\begin_layout Standard
+This is flexible enough to handle any potential locking scenario, even when
+ lock requirements change.
+ It can be optimized so that the parent does not release locks, just tells
+ the child which locks it doesn't need to obtain.
+\end_layout
+
+\begin_layout Standard
+It also keeps the complexity out of the API, and in ctdbd where it is needed.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+tdb_chainlock Functions Expose Implementation
+\end_layout
+
+\begin_layout Standard
+tdb_chainlock locks some number of records, including the record indicated
+ by the given key.
+ This gave atomicity guarantees; no-one can start a transaction, alter,
+ read or delete that key while the lock is held.
+\end_layout
+
+\begin_layout Standard
+It also makes the same guarantee for any other key in the chain, which is
+ an internal implementation detail and potentially a cause for deadlock.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+ It would be nice to have an explicit single entry lock which effected no
+ other keys.
+ Unfortunately, this won't work for an entry which doesn't exist.
+ Thus while chainlock may be implemented more efficiently for the existing
+ case, it will still have overlap issues with the non-existing case.
+ So it is best to keep the current (lack of) guarantee about which records
+ will be effected to avoid constraining our implementation.
+\end_layout
+
+\begin_layout Subsection
+Signal Handling is Not Race-Free
+\end_layout
+
+\begin_layout Standard
+The tdb_setalarm_sigptr() call allows the caller's signal handler to indicate
+ that the tdb locking code should return with a failure, rather than trying
+ again when a signal is received (and errno == EAGAIN).
+ This is usually used to implement timeouts.
+\end_layout
+
+\begin_layout Standard
+Unfortunately, this does not work in the case where the signal is received
+ before the tdb code enters the fcntl() call to place the lock: the code
+ will sleep within the fcntl() code, unaware that the signal wants it to
+ exit.
+ In the case of long timeouts, this does not happen in practice.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The locking hooks proposed in
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "Proposed-Solution-locking-hook"
+
+\end_inset
+
+ would allow the user to decide on whether to fail the lock acquisition
+ on a signal.
+ This allows the caller to choose their own compromise: they could narrow
+ the race by checking immediately before the fcntl call.
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+It may be possible to make this race-free in some implementations by having
+ the signal handler alter the struct flock to make it invalid.
+ This will cause the fcntl() lock call to fail with EINVAL if the signal
+ occurs before the kernel is entered, otherwise EAGAIN.
+\end_layout
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+The API Uses Gratuitous Typedefs, Capitals
+\end_layout
+
+\begin_layout Standard
+typedefs are useful for providing source compatibility when types can differ
+ across implementations, or arguably in the case of function pointer definitions
+ which are hard for humans to parse.
+ Otherwise it is simply obfuscation and pollutes the namespace.
+\end_layout
+
+\begin_layout Standard
+Capitalization is usually reserved for compile-time constants and macros.
+\end_layout
+
+\begin_layout Description
+TDB_CONTEXT There is no reason to use this over 'struct tdb_context'; the
+ definition isn't visible to the API user anyway.
+\end_layout
+
+\begin_layout Description
+TDB_DATA There is no reason to use this over struct TDB_DATA; the struct
+ needs to be understood by the API user.
+\end_layout
+
+\begin_layout Description
+struct
+\begin_inset space ~
+\end_inset
+
+TDB_DATA This would normally be called 'struct tdb_data'.
+\end_layout
+
+\begin_layout Description
+enum
+\begin_inset space ~
+\end_inset
+
+TDB_ERROR Similarly, this would normally be enum tdb_error.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+ Introducing lower case variants would please pedants like myself, but if
+ it were done the existing ones should be kept.
+ There is little point forcing a purely cosmetic change upon tdb users.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "tdb_log_func-Doesnt-Take"
+
+\end_inset
+
+tdb_log_func Doesn't Take The Private Pointer
+\end_layout
+
+\begin_layout Standard
+For API compatibility reasons, the logging function needs to call tdb_get_loggin
+g_private() to retrieve the pointer registered by the tdb_open_ex for logging.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+It should simply take an extra argument, since we are prepared to break
+ the API/ABI.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Various Callback Functions Are Not Typesafe
+\end_layout
+
+\begin_layout Standard
+The callback functions in tdb_set_logging_function (after
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "tdb_log_func-Doesnt-Take"
+
+\end_inset
+
+ is resolved), tdb_parse_record, tdb_traverse, tdb_traverse_read and tdb_check
+ all take void * and must internally convert it to the argument type they
+ were expecting.
+\end_layout
+
+\begin_layout Standard
+If this type changes, the compiler will not produce warnings on the callers,
+ since it only sees void *.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+With careful use of macros, we can create callback functions which give
+ a warning when used on gcc and the types of the callback and its private
+ argument differ.
+ Unsupported compilers will not give a warning, which is no worse than now.
+ In addition, the callbacks become clearer, as they need not use void *
+ for their parameter.
+\end_layout
+
+\begin_layout Standard
+See CCAN's typesafe_cb module at http://ccan.ozlabs.org/info/typesafe_cb.html
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+TDB_CLEAR_IF_FIRST Must Be Specified On All Opens, tdb_reopen_all Problematic
+\end_layout
+
+\begin_layout Standard
+The TDB_CLEAR_IF_FIRST flag to tdb_open indicates that the TDB file should
+ be cleared if the caller discovers it is the only process with the TDB
+ open.
+ However, if any caller does not specify TDB_CLEAR_IF_FIRST it will not
+ be detected, so will have the TDB erased underneath them (usually resulting
+ in a crash).
+\end_layout
+
+\begin_layout Standard
+There is a similar issue on fork(); if the parent exits (or otherwise closes
+ the tdb) before the child calls tdb_reopen_all() to establish the lock
+ used to indicate the TDB is opened by someone, a TDB_CLEAR_IF_FIRST opener
+ at that moment will believe it alone has opened the TDB and will erase
+ it.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Remove TDB_CLEAR_IF_FIRST.
+ Other workarounds are possible, but see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "TDB_CLEAR_IF_FIRST-Imposes-Performance"
+
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1298979699
+Incomplete, TDB_CLEAR_IF_FIRST still defined, but not implemented.
+\change_inserted 0 1298979700
+Complete.
+\change_unchanged
+
+\end_layout
+
+\begin_layout Subsection
+Extending The Header Is Difficult
+\end_layout
+
+\begin_layout Standard
+We have reserved (zeroed) words in the TDB header, which can be used for
+ future features.
+ If the future features are compulsory, the version number must be updated
+ to prevent old code from accessing the database.
+ But if the future feature is optional, we have no way of telling if older
+ code is accessing the database or not.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The header should contain a
+\begin_inset Quotes eld
+\end_inset
+
+format variant
+\begin_inset Quotes erd
+\end_inset
+
+ value (64-bit).
+ This is divided into two 32-bit parts:
+\end_layout
+
+\begin_layout Enumerate
+The lower part reflects the format variant understood by code accessing
+ the database.
+\end_layout
+
+\begin_layout Enumerate
+The upper part reflects the format variant you must understand to write
+ to the database (otherwise you can only open for reading).
+\end_layout
+
+\begin_layout Standard
+The latter field can only be written at creation time, the former should
+ be written under the OPEN_LOCK when opening the database for writing, if
+ the variant of the code is lower than the current lowest variant.
+\end_layout
+
+\begin_layout Standard
+This should allow backwards-compatible features to be added, and detection
+ if older code (which doesn't understand the feature) writes to the database.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+Record Headers Are Not Expandible
+\end_layout
+
+\begin_layout Standard
+If we later want to add (say) checksums on keys and data, it would require
+ another format change, which we'd like to avoid.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+We often have extra padding at the tail of a record.
+ If we ensure that the first byte (if any) of this padding is zero, we will
+ have a way for future changes to detect code which doesn't understand a
+ new format: the new code would write (say) a 1 at the tail, and thus if
+ there is no tail or the first byte is 0, we would know the extension is
+ not present on that record.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Incomplete.
+\end_layout
+
+\begin_layout Subsection
+TDB Does Not Use Talloc
+\end_layout
+
+\begin_layout Standard
+Many users of TDB (particularly Samba) use the talloc allocator, and thus
+ have to wrap TDB in a talloc context to use it conveniently.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The allocation within TDB is not complicated enough to justify the use of
+ talloc, and I am reluctant to force another (excellent) library on TDB
+ users.
+ Nonetheless a compromise is possible.
+ An attribute (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "attributes"
+
+\end_inset
+
+) can be added later to tdb_open() to provide an alternate allocation mechanism,
+ specifically for talloc but usable by any other allocator (which would
+ ignore the
+\begin_inset Quotes eld
+\end_inset
+
+context
+\begin_inset Quotes erd
+\end_inset
+
+ argument).
+\end_layout
+
+\begin_layout Standard
+This would form a talloc heirarchy as expected, but the caller would still
+ have to attach a destructor to the tdb context returned from tdb_open to
+ close it.
+ All TDB_DATA fields would be children of the tdb_context, and the caller
+ would still have to manage them (using talloc_free() or talloc_steal()).
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Section
+Performance And Scalability Issues
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "TDB_CLEAR_IF_FIRST-Imposes-Performance"
+
+\end_inset
+
+TDB_CLEAR_IF_FIRST Imposes Performance Penalty
+\end_layout
+
+\begin_layout Standard
+When TDB_CLEAR_IF_FIRST is specified, a 1-byte read lock is placed at offset
+ 4 (aka.
+ the ACTIVE_LOCK).
+ While these locks never conflict in normal tdb usage, they do add substantial
+ overhead for most fcntl lock implementations when the kernel scans to detect
+ if a lock conflict exists.
+ This is often a single linked list, making the time to acquire and release
+ a fcntl lock O(N) where N is the number of processes with the TDB open,
+ not the number actually doing work.
+\end_layout
+
+\begin_layout Standard
+In a Samba server it is common to have huge numbers of clients sitting idle,
+ and thus they have weaned themselves off the TDB_CLEAR_IF_FIRST flag.
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+There is a flag to tdb_reopen_all() which is used for this optimization:
+ if the parent process will outlive the child, the child does not need the
+ ACTIVE_LOCK.
+ This is a workaround for this very performance issue.
+\end_layout
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Remove the flag.
+ It was a neat idea, but even trivial servers tend to know when they are
+ initializing for the first time and can simply unlink the old tdb at that
+ point.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1298979837
+Incomplete; TDB_CLEAR_IF_FIRST still defined, but does nothing.
+\change_inserted 0 1298979837
+Complete.
+\change_unchanged
+
+\end_layout
+
+\begin_layout Subsection
+TDB Files Have a 4G Limit
+\end_layout
+
+\begin_layout Standard
+This seems to be becoming an issue (so much for
+\begin_inset Quotes eld
+\end_inset
+
+trivial
+\begin_inset Quotes erd
+\end_inset
+
+!), particularly for ldb.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+A new, incompatible TDB format which uses 64 bit offsets internally rather
+ than 32 bit as now.
+ For simplicity of endian conversion (which TDB does on the fly if required),
+ all values will be 64 bit on disk.
+ In practice, some upper bits may be used for other purposes, but at least
+ 56 bits will be available for file offsets.
+\end_layout
+
+\begin_layout Standard
+tdb_open() will automatically detect the old version, and even create them
+ if TDB_VERSION6 is specified to tdb_open.
+\end_layout
+
+\begin_layout Standard
+32 bit processes will still be able to access TDBs larger than 4G (assuming
+ that their off_t allows them to seek to 64 bits), they will gracefully
+ fall back as they fail to mmap.
+ This can happen already with large TDBs.
+\end_layout
+
+\begin_layout Standard
+Old versions of tdb will fail to open the new TDB files (since 28 August
+ 2009, commit 398d0c29290: prior to that any unrecognized file format would
+ be erased and initialized as a fresh tdb!)
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+TDB Records Have a 4G Limit
+\end_layout
+
+\begin_layout Standard
+This has not been a reported problem, and the API uses size_t which can
+ be 64 bit on 64 bit platforms.
+ However, other limits may have made such an issue moot.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Record sizes will be 64 bit, with an error returned on 32 bit platforms
+ which try to access such records (the current implementation would return
+ TDB_ERR_OOM in a similar case).
+ It seems unlikely that 32 bit keys will be a limitation, so the implementation
+ may not support this (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:Records-Incur-A"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Hash Size Is Determined At TDB Creation Time
+\end_layout
+
+\begin_layout Standard
+TDB contains a number of hash chains in the header; the number is specified
+ at creation time, and defaults to 131.
+ This is such a bottleneck on large databases (as each hash chain gets quite
+ long), that LDB uses 10,000 for this hash.
+ In general it is impossible to know what the 'right' answer is at database
+ creation time.
+\end_layout
+
+\begin_layout Subsubsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:Hash-Size-Solution"
+
+\end_inset
+
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+After comprehensive performance testing on various scalable hash variants
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+http://rusty.ozlabs.org/?p=89 and http://rusty.ozlabs.org/?p=94 This was annoying
+ because I was previously convinced that an expanding tree of hashes would
+ be very close to optimal.
+\end_layout
+
+\end_inset
+
+, it became clear that it is hard to beat a straight linear hash table which
+ doubles in size when it reaches saturation.
+ Unfortunately, altering the hash table introduces serious locking complications
+: the entire hash table needs to be locked to enlarge the hash table, and
+ others might be holding locks.
+ Particularly insidious are insertions done under tdb_chainlock.
+\end_layout
+
+\begin_layout Standard
+Thus an expanding layered hash will be used: an array of hash groups, with
+ each hash group exploding into pointers to lower hash groups once it fills,
+ turning into a hash tree.
+ This has implications for locking: we must lock the entire group in case
+ we need to expand it, yet we don't know how deep the tree is at that point.
+\end_layout
+
+\begin_layout Standard
+Note that bits from the hash table entries should be stolen to hold more
+ hash bits to reduce the penalty of collisions.
+ We can use the otherwise-unused lower 3 bits.
+ If we limit the size of the database to 64 exabytes, we can use the top
+ 8 bits of the hash entry as well.
+ These 11 bits would reduce false positives down to 1 in 2000 which is more
+ than we need: we can use one of the bits to indicate that the extra hash
+ bits are valid.
+ This means we can choose not to re-hash all entries when we expand a hash
+ group; simply use the next bits we need and mark them invalid.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "TDB-Freelist-Is"
+
+\end_inset
+
+TDB Freelist Is Highly Contended
+\end_layout
+
+\begin_layout Standard
+TDB uses a single linked list for the free list.
+ Allocation occurs as follows, using heuristics which have evolved over
+ time:
+\end_layout
+
+\begin_layout Enumerate
+Get the free list lock for this whole operation.
+\end_layout
+
+\begin_layout Enumerate
+Multiply length by 1.25, so we always over-allocate by 25%.
+\end_layout
+
+\begin_layout Enumerate
+Set the slack multiplier to 1.
+\end_layout
+
+\begin_layout Enumerate
+Examine the current freelist entry: if it is > length but < the current
+ best case, remember it as the best case.
+\end_layout
+
+\begin_layout Enumerate
+Multiply the slack multiplier by 1.05.
+\end_layout
+
+\begin_layout Enumerate
+If our best fit so far is less than length * slack multiplier, return it.
+ The slack will be turned into a new free record if it's large enough.
+\end_layout
+
+\begin_layout Enumerate
+Otherwise, go onto the next freelist entry.
+\end_layout
+
+\begin_layout Standard
+Deleting a record occurs as follows:
+\end_layout
+
+\begin_layout Enumerate
+Lock the hash chain for this whole operation.
+\end_layout
+
+\begin_layout Enumerate
+Walk the chain to find the record, keeping the prev pointer offset.
+\end_layout
+
+\begin_layout Enumerate
+If max_dead is non-zero:
+\end_layout
+
+\begin_deeper
+\begin_layout Enumerate
+Walk the hash chain again and count the dead records.
+\end_layout
+
+\begin_layout Enumerate
+If it's more than max_dead, bulk free all the dead ones (similar to steps
+ 4 and below, but the lock is only obtained once).
+\end_layout
+
+\begin_layout Enumerate
+Simply mark this record as dead and return.
+
+\end_layout
+
+\end_deeper
+\begin_layout Enumerate
+Get the free list lock for the remainder of this operation.
+\end_layout
+
+\begin_layout Enumerate
+\begin_inset CommandInset label
+LatexCommand label
+name "right-merging"
+
+\end_inset
+
+Examine the following block to see if it is free; if so, enlarge the current
+ block and remove that block from the free list.
+ This was disabled, as removal from the free list was O(entries-in-free-list).
+\end_layout
+
+\begin_layout Enumerate
+Examine the preceeding block to see if it is free: for this reason, each
+ block has a 32-bit tailer which indicates its length.
+ If it is free, expand it to cover our new block and return.
+\end_layout
+
+\begin_layout Enumerate
+Otherwise, prepend ourselves to the free list.
+\end_layout
+
+\begin_layout Standard
+Disabling right-merging (step
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "right-merging"
+
+\end_inset
+
+) causes fragmentation; the other heuristics proved insufficient to address
+ this, so the final answer to this was that when we expand the TDB file
+ inside a transaction commit, we repack the entire tdb.
+\end_layout
+
+\begin_layout Standard
+The single list lock limits our allocation rate; due to the other issues
+ this is not currently seen as a bottleneck.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The first step is to remove all the current heuristics, as they obviously
+ interact, then examine them once the lock contention is addressed.
+\end_layout
+
+\begin_layout Standard
+The free list must be split to reduce contention.
+ Assuming perfect free merging, we can at most have 1 free list entry for
+ each entry.
+ This implies that the number of free lists is related to the size of the
+ hash table, but as it is rare to walk a large number of free list entries
+ we can use far fewer, say 1/32 of the number of hash buckets.
+\end_layout
+
+\begin_layout Standard
+It seems tempting to try to reuse the hash implementation which we use for
+ records here, but we have two ways of searching for free entries: for allocatio
+n we search by size (and possibly zone) which produces too many clashes
+ for our hash table to handle well, and for coalescing we search by address.
+ Thus an array of doubly-linked free lists seems preferable.
+\end_layout
+
+\begin_layout Standard
+There are various benefits in using per-size free lists (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:TDB-Becomes-Fragmented"
+
+\end_inset
+
+) but it's not clear this would reduce contention in the common case where
+ all processes are allocating/freeing the same size.
+ Thus we almost certainly need to divide in other ways: the most obvious
+ is to divide the file into zones, and using a free list (or table of free
+ lists) for each.
+ This approximates address ordering.
+\end_layout
+
+\begin_layout Standard
+Unfortunately it is difficult to know what heuristics should be used to
+ determine zone sizes, and our transaction code relies on being able to
+ create a
+\begin_inset Quotes eld
+\end_inset
+
+recovery area
+\begin_inset Quotes erd
+\end_inset
+
+ by simply appending to the file (difficult if it would need to create a
+ new zone header).
+ Thus we use a linked-list of free tables; currently we only ever create
+ one, but if there is more than one we choose one at random to use.
+ In future we may use heuristics to add new free tables on contention.
+ We only expand the file when all free tables are exhausted.
+\end_layout
+
+\begin_layout Standard
+The basic algorithm is as follows.
+ Freeing is simple:
+\end_layout
+
+\begin_layout Enumerate
+Identify the correct free list.
+\end_layout
+
+\begin_layout Enumerate
+Lock the corresponding list.
+\end_layout
+
+\begin_layout Enumerate
+Re-check the list (we didn't have a lock, sizes could have changed): relock
+ if necessary.
+\end_layout
+
+\begin_layout Enumerate
+Place the freed entry in the list.
+\end_layout
+
+\begin_layout Standard
+Allocation is a little more complicated, as we perform delayed coalescing
+ at this point:
+\end_layout
+
+\begin_layout Enumerate
+Pick a free table; usually the previous one.
+\end_layout
+
+\begin_layout Enumerate
+Lock the corresponding list.
+\end_layout
+
+\begin_layout Enumerate
+If the top entry is -large enough, remove it from the list and return it.
+\end_layout
+
+\begin_layout Enumerate
+Otherwise, coalesce entries in the list.If there was no entry large enough,
+ unlock the list and try the next largest list
+\end_layout
+
+\begin_layout Enumerate
+If no list has an entry which meets our needs, try the next free table.
+\end_layout
+
+\begin_layout Enumerate
+If no zone satisfies, expand the file.
+\end_layout
+
+\begin_layout Standard
+This optimizes rapid insert/delete of free list entries by not coalescing
+ them all the time..
+ First-fit address ordering ordering seems to be fairly good for keeping
+ fragmentation low (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:TDB-Becomes-Fragmented"
+
+\end_inset
+
+).
+ Note that address ordering does not need a tailer to coalesce, though if
+ we needed one we could have one cheaply: see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:Records-Incur-A"
+
+\end_inset
+
+.
+
+\end_layout
+
+\begin_layout Standard
+Each free entry has the free table number in the header: less than 255.
+ It also contains a doubly-linked list for easy deletion.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:TDB-Becomes-Fragmented"
+
+\end_inset
+
+TDB Becomes Fragmented
+\end_layout
+
+\begin_layout Standard
+Much of this is a result of allocation strategy
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+The Memory Fragmentation Problem: Solved? Johnstone & Wilson 1995 ftp://ftp.cs.ute
+xas.edu/pub/garbage/malloc/ismm98.ps
+\end_layout
+
+\end_inset
+
+ and deliberate hobbling of coalescing; internal fragmentation (aka overallocati
+on) is deliberately set at 25%, and external fragmentation is only cured
+ by the decision to repack the entire db when a transaction commit needs
+ to enlarge the file.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The 25% overhead on allocation works in practice for ldb because indexes
+ tend to expand by one record at a time.
+ This internal fragmentation can be resolved by having an
+\begin_inset Quotes eld
+\end_inset
+
+expanded
+\begin_inset Quotes erd
+\end_inset
+
+ bit in the header to note entries that have previously expanded, and allocating
+ more space for them.
+\end_layout
+
+\begin_layout Standard
+There are is a spectrum of possible solutions for external fragmentation:
+ one is to use a fragmentation-avoiding allocation strategy such as best-fit
+ address-order allocator.
+ The other end of the spectrum would be to use a bump allocator (very fast
+ and simple) and simply repack the file when we reach the end.
+\end_layout
+
+\begin_layout Standard
+There are three problems with efficient fragmentation-avoiding allocators:
+ they are non-trivial, they tend to use a single free list for each size,
+ and there's no evidence that tdb allocation patterns will match those recorded
+ for general allocators (though it seems likely).
+\end_layout
+
+\begin_layout Standard
+Thus we don't spend too much effort on external fragmentation; we will be
+ no worse than the current code if we need to repack on occasion.
+ More effort is spent on reducing freelist contention, and reducing overhead.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:Records-Incur-A"
+
+\end_inset
+
+Records Incur A 28-Byte Overhead
+\end_layout
+
+\begin_layout Standard
+Each TDB record has a header as follows:
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_record {
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_off_t next; /* offset of the next record in the list */
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_len_t rec_len; /* total byte length of record */
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_len_t key_len; /* byte length of key */
+\end_layout
+
+\begin_layout LyX-Code
+        tdb_len_t data_len; /* byte length of data */
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t full_hash; /* the full 32 bit hash of the key */
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t magic;   /* try to catch errors */
+\end_layout
+
+\begin_layout LyX-Code
+        /* the following union is implied:
+\end_layout
+
+\begin_layout LyX-Code
+                union {
+\end_layout
+
+\begin_layout LyX-Code
+                        char record[rec_len];
+\end_layout
+
+\begin_layout LyX-Code
+                        struct {
+\end_layout
+
+\begin_layout LyX-Code
+                                char key[key_len];
+\end_layout
+
+\begin_layout LyX-Code
+                                char data[data_len];
+\end_layout
+
+\begin_layout LyX-Code
+                        }
+\end_layout
+
+\begin_layout LyX-Code
+                        uint32_t totalsize; (tailer)
+\end_layout
+
+\begin_layout LyX-Code
+                }
+\end_layout
+
+\begin_layout LyX-Code
+        */
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+Naively, this would double to a 56-byte overhead on a 64 bit implementation.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+We can use various techniques to reduce this for an allocated block:
+\end_layout
+
+\begin_layout Enumerate
+The 'next' pointer is not required, as we are using a flat hash table.
+\end_layout
+
+\begin_layout Enumerate
+'rec_len' can instead be expressed as an addition to key_len and data_len
+ (it accounts for wasted or overallocated length in the record).
+ Since the record length is always a multiple of 8, we can conveniently
+ fit it in 32 bits (representing up to 35 bits).
+\end_layout
+
+\begin_layout Enumerate
+'key_len' and 'data_len' can be reduced.
+ I'm unwilling to restrict 'data_len' to 32 bits, but instead we can combine
+ the two into one 64-bit field and using a 5 bit value which indicates at
+ what bit to divide the two.
+ Keys are unlikely to scale as fast as data, so I'm assuming a maximum key
+ size of 32 bits.
+\end_layout
+
+\begin_layout Enumerate
+'full_hash' is used to avoid a memcmp on the
+\begin_inset Quotes eld
+\end_inset
+
+miss
+\begin_inset Quotes erd
+\end_inset
+
+ case, but this is diminishing returns after a handful of bits (at 10 bits,
+ it reduces 99.9% of false memcmp).
+ As an aside, as the lower bits are already incorporated in the hash table
+ resolution, the upper bits should be used here.
+ Note that it's not clear that these bits will be a win, given the extra
+ bits in the hash table itself (see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:Hash-Size-Solution"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Enumerate
+'magic' does not need to be enlarged: it currently reflects one of 5 values
+ (used, free, dead, recovery, and unused_recovery).
+ It is useful for quick sanity checking however, and should not be eliminated.
+\end_layout
+
+\begin_layout Enumerate
+'tailer' is only used to coalesce free blocks (so a block to the right can
+ find the header to check if this block is free).
+ This can be replaced by a single 'free' bit in the header of the following
+ block (and the tailer only exists in free blocks).
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+This technique from Thomas Standish.
+ Data Structure Techniques.
+ Addison-Wesley, Reading, Massachusetts, 1980.
+\end_layout
+
+\end_inset
+
+ The current proposed coalescing algorithm doesn't need this, however.
+\end_layout
+
+\begin_layout Standard
+This produces a 16 byte used header like this:
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_used_record {
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t used_magic : 16,
+\end_layout
+
+\begin_layout LyX-Code
+
+\end_layout
+
+\begin_layout LyX-Code
+                 key_data_divide: 5,
+\end_layout
+
+\begin_layout LyX-Code
+                 top_hash: 11;
+\end_layout
+
+\begin_layout LyX-Code
+        uint32_t extra_octets;
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t key_and_data_len;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+And a free record like this:
+\end_layout
+
+\begin_layout LyX-Code
+struct tdb_free_record {
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t free_magic: 8,
+\end_layout
+
+\begin_layout LyX-Code
+                   prev : 56;
+\end_layout
+
+\begin_layout LyX-Code
+
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t free_table: 8,
+\end_layout
+
+\begin_layout LyX-Code
+                 total_length : 56
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t next;;
+\end_layout
+
+\begin_layout LyX-Code
+};
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1291206079
+
+\change_unchanged
+Note that by limiting valid offsets to 56 bits, we can pack everything we
+ need into 3 64-byte words, meaning our minimum record size is 8 bytes.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Transaction Commit Requires 4 fdatasync
+\end_layout
+
+\begin_layout Standard
+The current transaction algorithm is:
+\end_layout
+
+\begin_layout Enumerate
+write_recovery_data();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Enumerate
+write_recovery_header();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Enumerate
+overwrite_with_new_data();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Enumerate
+remove_recovery_header();
+\end_layout
+
+\begin_layout Enumerate
+sync();
+\end_layout
+
+\begin_layout Standard
+On current ext3, each sync flushes all data to disk, so the next 3 syncs
+ are relatively expensive.
+ But this could become a performance bottleneck on other filesystems such
+ as ext4.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Neil Brown points out that this is overzealous, and only one sync is needed:
+\end_layout
+
+\begin_layout Enumerate
+Bundle the recovery data, a transaction counter and a strong checksum of
+ the new data.
+\end_layout
+
+\begin_layout Enumerate
+Strong checksum that whole bundle.
+\end_layout
+
+\begin_layout Enumerate
+Store the bundle in the database.
+\end_layout
+
+\begin_layout Enumerate
+Overwrite the oldest of the two recovery pointers in the header (identified
+ using the transaction counter) with the offset of this bundle.
+\end_layout
+
+\begin_layout Enumerate
+sync.
+\end_layout
+
+\begin_layout Enumerate
+Write the new data to the file.
+\end_layout
+
+\begin_layout Standard
+Checking for recovery means identifying the latest bundle with a valid checksum
+ and using the new data checksum to ensure that it has been applied.
+ This is more expensive than the current check, but need only be done at
+ open.
+ For running databases, a separate header field can be used to indicate
+ a transaction in progress; we need only check for recovery if this is set.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "sub:TDB-Does-Not"
+
+\end_inset
+
+TDB Does Not Have Snapshot Support
+\end_layout
+
+\begin_layout Subsubsection
+Proposed SolutionNone.
+ At some point you say
+\begin_inset Quotes eld
+\end_inset
+
+use a real database
+\begin_inset Quotes erd
+\end_inset
+
+ (but see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "replay-attribute"
+
+\end_inset
+
+).
+\end_layout
+
+\begin_layout Standard
+But as a thought experiment, if we implemented transactions to only overwrite
+ free entries (this is tricky: there must not be a header in each entry
+ which indicates whether it is free, but use of presence in metadata elsewhere),
+ and a pointer to the hash table, we could create an entirely new commit
+ without destroying existing data.
+ Then it would be easy to implement snapshots in a similar way.
+\end_layout
+
+\begin_layout Standard
+This would not allow arbitrary changes to the database, such as tdb_repack
+ does, and would require more space (since we have to preserve the current
+ and future entries at once).
+ If we used hash trees rather than one big hash table, we might only have
+ to rewrite some sections of the hash, too.
+\end_layout
+
+\begin_layout Standard
+We could then implement snapshots using a similar method, using multiple
+ different hash tables/free tables.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Subsection
+Transactions Cannot Operate in Parallel
+\end_layout
+
+\begin_layout Standard
+This would be useless for ldb, as it hits the index records with just about
+ every update.
+ It would add significant complexity in resolving clashes, and cause the
+ all transaction callers to write their code to loop in the case where the
+ transactions spuriously failed.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None (but see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "replay-attribute"
+
+\end_inset
+
+).
+ We could solve a small part of the problem by providing read-only transactions.
+ These would allow one write transaction to begin, but it could not commit
+ until all r/o transactions are done.
+ This would require a new RO_TRANSACTION_LOCK, which would be upgraded on
+ commit.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\begin_layout Subsection
+Default Hash Function Is Suboptimal
+\end_layout
+
+\begin_layout Standard
+The Knuth-inspired multiplicative hash used by tdb is fairly slow (especially
+ if we expand it to 64 bits), and works best when the hash bucket size is
+ a prime number (which also means a slow modulus).
+ In addition, it is highly predictable which could potentially lead to a
+ Denial of Service attack in some TDB uses.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+The Jenkins lookup3 hash
+\begin_inset Foot
+status open
+
+\begin_layout Plain Layout
+http://burtleburtle.net/bob/c/lookup3.c
+\end_layout
+
+\end_inset
+
+ is a fast and superbly-mixing hash.
+ It's used by the Linux kernel and almost everything else.
+ This has the particular properties that it takes an initial seed, and produces
+ two 32 bit hash numbers, which we can combine into a 64-bit hash.
+\end_layout
+
+\begin_layout Standard
+The seed should be created at tdb-creation time from some random source,
+ and placed in the header.
+ This is far from foolproof, but adds a little bit of protection against
+ hash bombing.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+\begin_inset CommandInset label
+LatexCommand label
+name "Reliable-Traversal-Adds"
+
+\end_inset
+
+Reliable Traversal Adds Complexity
+\end_layout
+
+\begin_layout Standard
+We lock a record during traversal iteration, and try to grab that lock in
+ the delete code.
+ If that grab on delete fails, we simply mark it deleted and continue onwards;
+ traversal checks for this condition and does the delete when it moves off
+ the record.
+\end_layout
+
+\begin_layout Standard
+If traversal terminates, the dead record may be left indefinitely.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+Remove reliability guarantees; see
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "traverse-Proposed-Solution"
+
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Complete.
+\end_layout
+
+\begin_layout Subsection
+Fcntl Locking Adds Overhead
+\end_layout
+
+\begin_layout Standard
+Placing a fcntl lock means a system call, as does removing one.
+ This is actually one reason why transactions can be faster (everything
+ is locked once at transaction start).
+ In the uncontended case, this overhead can theoretically be eliminated.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+\end_layout
+
+\begin_layout Standard
+We tried this before with spinlock support, in the early days of TDB, and
+ it didn't make much difference except in manufactured benchmarks.
+\end_layout
+
+\begin_layout Standard
+We could use spinlocks (with futex kernel support under Linux), but it means
+ that we lose automatic cleanup when a process dies with a lock.
+ There is a method of auto-cleanup under Linux, but it's not supported by
+ other operating systems.
+ We could reintroduce a clear-if-first-style lock and sweep for dead futexes
+ on open, but that wouldn't help the normal case of one concurrent opener
+ dying.
+ Increasingly elaborate repair schemes could be considered, but they require
+ an ABI change (everyone must use them) anyway, so there's no need to do
+ this at the same time as everything else.
+\end_layout
+
+\begin_layout Subsection
+Some Transactions Don't Require Durability
+\end_layout
+
+\begin_layout Standard
+Volker points out that gencache uses a CLEAR_IF_FIRST tdb for normal (fast)
+ usage, and occasionally empties the results into a transactional TDB.
+ This kind of usage prioritizes performance over durability: as long as
+ we are consistent, data can be lost.
+\end_layout
+
+\begin_layout Standard
+This would be more neatly implemented inside tdb: a
+\begin_inset Quotes eld
+\end_inset
+
+soft
+\begin_inset Quotes erd
+\end_inset
+
+ transaction commit (ie.
+ syncless) which meant that data may be reverted on a crash.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\end_layout
+
+\begin_layout Standard
+None.
+\end_layout
+
+\begin_layout Standard
+Unfortunately any transaction scheme which overwrites old data requires
+ a sync before that overwrite to avoid the possibility of corruption.
+\end_layout
+
+\begin_layout Standard
+It seems possible to use a scheme similar to that described in
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "sub:TDB-Does-Not"
+
+\end_inset
+
+,where transactions are committed without overwriting existing data, and
+ an array of top-level pointers were available in the header.
+ If the transaction is
+\begin_inset Quotes eld
+\end_inset
+
+soft
+\begin_inset Quotes erd
+\end_inset
+
+ then we would not need a sync at all: existing processes would pick up
+ the new hash table and free list and work with that.
+\end_layout
+
+\begin_layout Standard
+At some later point, a sync would allow recovery of the old data into the
+ free lists (perhaps when the array of top-level pointers filled).
+ On crash, tdb_open() would examine the array of top levels, and apply the
+ transactions until it encountered an invalid checksum.
+\end_layout
+
+\begin_layout Subsection
+Tracing Is Fragile, Replay Is External
+\end_layout
+
+\begin_layout Standard
+The current TDB has compile-time-enabled tracing code, but it often breaks
+ as it is not enabled by default.
+ In a similar way, the ctdb code has an external wrapper which does replay
+ tracing so it can coordinate cluster-wide transactions.
+\end_layout
+
+\begin_layout Subsubsection
+Proposed Solution
+\begin_inset CommandInset label
+LatexCommand label
+name "replay-attribute"
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+Tridge points out that an attribute can be later added to tdb_open (see
+
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "attributes"
+
+\end_inset
+
+) to provide replay/trace hooks, which could become the basis for this and
+ future parallel transactions and snapshot support.
+\end_layout
+
+\begin_layout Subsubsection
+Status
+\end_layout
+
+\begin_layout Standard
+Deferred.
+\end_layout
+
+\end_body
+\end_document
+@
+
+
+1.12
+log
+@Add status, some fixes, linked freelists.
+@
+text
+@d53 1
+a53 7
+
+\change_deleted 0 1291204535
+14-September
+\change_inserted 0 1291204533
+1-December
+\change_unchanged
+-2010
+a580 2
+\change_inserted 0 1291204563
+
+a583 2
+
+\change_inserted 0 1291204572
+a587 2
+
+\change_inserted 0 1291204573
+a588 2
+\change_unchanged
+
+a629 2
+\change_inserted 0 1291204588
+
+a632 2
+
+\change_inserted 0 1291204588
+a636 2
+
+\change_inserted 0 1291204631
+a639 2
+\change_unchanged
+
+a693 2
+\change_inserted 0 1291204639
+
+a696 2
+
+\change_inserted 0 1291204640
+d702 1
+a702 1
+\change_inserted 0 1291204665
+d704 2
+a728 2
+\change_inserted 0 1291204671
+
+a731 2
+
+\change_inserted 0 1291204671
+a735 2
+
+\change_inserted 0 1291204673
+a736 2
+\change_unchanged
+
+a780 2
+\change_inserted 0 1291204731
+
+a783 2
+
+\change_inserted 0 1291204732
+a787 2
+
+\change_inserted 0 1291204779
+a790 2
+\change_unchanged
+
+a842 2
+\change_inserted 0 1291204830
+
+a845 2
+
+\change_inserted 0 1291204831
+a849 2
+
+\change_inserted 0 1291204834
+a850 2
+\change_unchanged
+
+d879 9
+a887 2
+ deal of churn; we are better to guarantee that the tdb_errcode is per-thread
+ so the current programming model can be maintained.
+d891 9
+d903 2
+a922 2
+\change_inserted 0 1291204847
+
+a925 2
+
+\change_inserted 0 1291204847
+d930 5
+a934 3
+
+\change_inserted 0 1291204852
+Incomplete.
+a1051 2
+\change_inserted 0 1291204881
+
+a1054 2
+
+\change_inserted 0 1291204881
+a1058 2
+
+\change_inserted 0 1291204885
+a1059 2
+\change_unchanged
+
+a1140 2
+\change_inserted 0 1291204898
+
+a1143 2
+
+\change_inserted 0 1291204898
+a1147 2
+
+\change_inserted 0 1291204901
+a1148 2
+\change_unchanged
+
+a1224 2
+\change_inserted 0 1291204908
+
+a1227 2
+
+\change_inserted 0 1291204908
+a1231 2
+
+\change_inserted 0 1291204908
+a1232 2
+\change_unchanged
+
+a1271 2
+\change_inserted 0 1291204917
+
+a1274 2
+
+\change_inserted 0 1291204917
+a1278 2
+
+\change_inserted 0 1291204920
+a1279 2
+\change_unchanged
+
+a1316 2
+\change_inserted 0 1291204927
+
+a1319 2
+
+\change_inserted 0 1291204928
+d1325 1
+a1325 1
+\change_inserted 0 1291204942
+d1327 2
+a1381 2
+\change_inserted 0 1291205003
+
+a1384 2
+
+\change_inserted 0 1291205004
+a1388 2
+
+\change_inserted 0 1291205007
+a1411 2
+\change_inserted 0 1291205019
+
+a1414 2
+
+\change_inserted 0 1291205019
+a1418 2
+
+\change_inserted 0 1291205023
+a1419 2
+\change_unchanged
+
+a1465 2
+\change_inserted 0 1291205029
+
+a1468 2
+
+\change_inserted 0 1291205029
+a1472 2
+
+\change_inserted 0 1291206020
+a1473 2
+\change_unchanged
+
+a1528 2
+\change_inserted 0 1291205043
+
+a1531 2
+
+\change_inserted 0 1291205043
+d1537 1
+a1537 1
+\change_inserted 0 1291205057
+d1539 2
+a1589 2
+\change_inserted 0 1291205062
+
+a1592 2
+
+\change_inserted 0 1291205062
+a1596 2
+
+\change_inserted 0 1291205062
+a1597 2
+\change_unchanged
+
+a1626 2
+\change_inserted 0 1291205072
+
+a1629 2
+
+\change_inserted 0 1291205073
+a1633 2
+
+\change_inserted 0 1291205073
+a1634 2
+\change_unchanged
+
+a1674 4
+
+\change_deleted 0 1291204504
+
+\change_unchanged
+a1699 2
+\change_inserted 0 1291205079
+
+a1702 2
+
+\change_inserted 0 1291205080
+a1706 2
+
+\change_inserted 0 1291205080
+a1707 2
+\change_unchanged
+
+a1833 2
+\change_inserted 0 1291205090
+
+d1869 2
+a1870 7
+ is to divide the file into zones, and using a free list (or
+\change_inserted 0 1291205498
+table
+\change_deleted 0 1291205497
+set
+\change_unchanged
+ of free lists) for each.
+a1871 2
+\change_inserted 0 1291205203
+
+a1874 2
+
+\change_inserted 0 1291205358
+a1890 21
+\change_unchanged
+
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1291205198
+Note that this means we need to split the free lists when we expand the
+ file; this is probably acceptable when we double the hash table size, since
+ that is such an expensive operation already.
+ In the case of increasing the file size, there is an optimization we can
+ use: if we use M in the formula above as the file size rounded up to the
+ next power of 2, we only need reshuffle free lists when the file size crosses
+ a power of 2 boundary,
+\emph on
+and
+\emph default
+reshuffling the free lists is trivial: we simply merge every consecutive
+ pair of free lists.
+\change_unchanged
+
+d1899 1
+a1899 7
+Identify the correct
+\change_inserted 0 1291205366
+free list
+\change_deleted 0 1291205364
+zone
+\change_unchanged
+.
+d1907 2
+a1908 7
+Re-check the
+\change_inserted 0 1291205372
+list
+\change_deleted 0 1291205371
+zone
+\change_unchanged
+ (we didn't have a lock, sizes could have changed): relock if necessary.
+d1912 1
+a1912 5
+Place the freed entry in the list
+\change_deleted 0 1291205382
+ for that zone
+\change_unchanged
+.
+d1921 1
+a1921 15
+Pick a
+\change_deleted 0 1291205403
+zone either the zone we last freed into, or based on a
+\begin_inset Quotes eld
+\end_inset
+
+random
+\begin_inset Quotes erd
+\end_inset
+
+ number.
+\change_inserted 0 1291205411
+free table; usually the previous one.
+\change_unchanged
+
+a1925 10
+\change_deleted 0 1291205432
+
+\end_layout
+
+\begin_layout Enumerate
+
+\change_deleted 0 1291205428
+Re-check the zone: relock if necessary.
+\change_unchanged
+
+d1934 1
+a1934 7
+ unlock the list and try the next
+\change_inserted 0 1291205455
+largest list
+\change_deleted 0 1291205452
+zone.
+\change_inserted 0 1291205457
+
+a1937 2
+
+\change_inserted 0 1291205476
+a1938 2
+\change_unchanged
+
+a1966 2
+\change_inserted 0 1291205542
+
+a1969 2
+
+\change_inserted 0 1291205591
+a1971 70
+\change_unchanged
+
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1291205539
+I anticipate that the number of entries in each free zone would be small,
+ but it might be worth using one free entry to hold pointers to the others
+ for cache efficiency.
+\change_unchanged
+
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1291205534
+\begin_inset CommandInset label
+LatexCommand label
+name "freelist-in-zone"
+
+\end_inset
+
+If we want to avoid locking complexity (enlarging the free lists when we
+ enlarge the file) we could place the array of free lists at the beginning
+ of each zone.
+ This means existing array lists never move, but means that a record cannot
+ be larger than a zone.
+ That in turn implies that zones should be variable sized (say, power of
+ 2), which makes the question
+\begin_inset Quotes eld
+\end_inset
+
+what zone is this record in?
+\begin_inset Quotes erd
+\end_inset
+
+ much harder (and
+\begin_inset Quotes eld
+\end_inset
+
+pick a random zone
+\begin_inset Quotes erd
+\end_inset
+
+, but that's less common).
+ It could be done with as few as 4 bits from the record header.
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+Using
+\begin_inset Formula $2^{16+N*3}$
+\end_inset
+
+means 0 gives a minimal 65536-byte zone, 15 gives the maximal
+\begin_inset Formula $2^{61}$
+\end_inset
+
+ byte zone.
+ Zones range in factor of 8 steps.
+ Given the zone size for the zone the current record is in, we can determine
+ the start of the zone.
+\end_layout
+
+\end_inset
+
+
+\change_inserted 0 1291205139
+
+d2218 1
+a2218 5
+        uint32_t
+\change_inserted 0 1291205758
+used_
+\change_unchanged
+magic : 16,
+a2222 4
+\change_deleted 0 1291205693
+                 prev_is_free: 1,
+\change_unchanged
+
+d2230 1
+a2230 7
+                 top_hash: 1
+\change_inserted 0 1291205704
+1
+\change_deleted 0 1291205704
+0
+\change_unchanged
+;
+d2254 1
+a2254 9
+        uint
+\change_inserted 0 1291205725
+64
+\change_deleted 0 1291205723
+32
+\change_unchanged
+_t
+\change_inserted 0 1291205753
+free_magic: 8,
+a2257 2
+
+\change_inserted 0 1291205746
+a2262 24
+\change_deleted 0 1291205749
+free_magic;
+\change_unchanged
+
+\end_layout
+
+\begin_layout LyX-Code
+        uint64_t
+\change_inserted 0 1291205786
+free_table: 8,
+\end_layout
+
+\begin_layout LyX-Code
+
+\change_inserted 0 1291205788
+
+\change_unchanged
+total_length
+\change_inserted 0 1291205792
+ : 56
+\change_deleted 0 1291205790
+;
+\change_unchanged
+
+d2266 1
+a2266 7
+        uint64_t
+\change_deleted 0 1291205801
+prev,
+\change_unchanged
+next;
+\change_deleted 0 1291205811
+
+d2270 1
+a2270 3
+
+\change_deleted 0 1291205811
+        ...
+d2274 1
+a2274 5
+
+\change_deleted 0 1291205808
+        uint64_t tailer
+\change_unchanged
+;
+d2283 5
+a2287 16
+\change_deleted 0 1291205827
+We might want to take some bits from the used record's top_hash (and the
+ free record which has 32 bits of padding to spare anyway) if we use variable
+ sized zones.
+ See
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "freelist-in-zone"
+
+\end_inset
+
+.
+
+\change_inserted 0 1291205885
+ Note that by limiting valid offsets to 56 bits, we can pack everything
+ we need into 3 64-byte words, meaning our minimum record size is 8 bytes.
+a2290 2
+
+\change_inserted 0 1291205886
+a2294 2
+
+\change_inserted 0 1291205886
+a2295 2
+\change_unchanged
+
+a2385 2
+\change_inserted 0 1291205894
+
+a2388 2
+
+\change_inserted 0 1291205894
+a2392 2
+
+\change_inserted 0 1291205902
+a2393 2
+\change_unchanged
+
+a2415 4
+
+\change_deleted 0 1291204504
+
+\change_unchanged
+a2445 2
+\change_inserted 0 1291205910
+
+a2448 2
+
+\change_inserted 0 1291205910
+a2452 2
+
+\change_inserted 0 1291205914
+a2453 2
+\change_unchanged
+
+a2485 2
+\change_inserted 0 1291205919
+
+a2488 2
+
+\change_inserted 0 1291205919
+a2492 2
+
+\change_inserted 0 1291205922
+a2493 2
+\change_unchanged
+
+a2533 2
+\change_inserted 0 1291205929
+
+a2536 2
+
+\change_inserted 0 1291205929
+a2540 2
+
+\change_inserted 0 1291205929
+a2541 2
+\change_unchanged
+
+a2578 2
+\change_inserted 0 1291205932
+
+a2581 2
+
+\change_inserted 0 1291205933
+a2585 2
+
+\change_inserted 0 1291205933
+a2586 2
+\change_unchanged
+
+a2724 2
+\change_inserted 0 1291205944
+
+a2727 2
+
+\change_inserted 0 1291205945
+a2731 2
+
+\change_inserted 0 1291205948
+a2732 2
+\change_unchanged
+
+@
+
+
+1.11
+log
+@Merge changes
+@
+text
+@d53 7
+a59 1
+14-September-2010
+d587 16
+d644 18
+d716 16
+d753 16
+d813 18
+d883 16
+d953 16
+d1084 16
+d1181 16
+d1273 16
+d1328 16
+d1381 16
+d1447 19
+a1465 2
+ if older code (which doesn't understand the feature) writes to the database.Reco
+rd Headers Are Not Expandible
+d1484 16
+d1546 16
+d1617 16
+d1680 16
+d1725 16
+d1810 16
+d1951 8
+a1958 3
+Proposed SolutionThe first step is to remove all the current heuristics,
+ as they obviously interact, then examine them once the lock contention
+ is addressed.
+d1989 7
+a1995 2
+ is to divide the file into zones, and using a free list (or set of free
+ lists) for each.
+d1997 2
+d2002 25
+d2039 2
+d2049 7
+a2055 1
+Identify the correct zone.
+d2063 7
+a2069 2
+Re-check the zone (we didn't have a lock, sizes could have changed): relock
+ if necessary.
+d2073 5
+a2077 1
+Place the freed entry in the list for that zone.
+d2086 3
+a2088 1
+Pick a zone either the zone we last freed into, or based on a
+d2097 4
+d2105 2
+d2110 2
+d2113 2
+d2123 15
+a2137 1
+ unlock the list and try the next zone.
+d2166 11
+d2180 2
+d2185 2
+d2190 2
+d2223 1
+a2223 1
+status open
+d2243 2
+d2491 5
+a2495 1
+        uint32_t magic : 16,
+d2499 2
+d2502 2
+d2511 7
+a2517 1
+                 top_hash: 10;
+d2541 29
+a2569 1
+        uint32_t free_magic;
+d2573 11
+a2583 1
+        uint64_t total_length;
+d2587 7
+a2593 1
+        uint64_t prev, next;
+d2597 2
+d2603 5
+a2607 1
+        uint64_t tailer;
+d2615 2
+d2628 18
+d2736 16
+d2808 16
+d2856 16
+d2912 16
+d2965 16
+d3119 16
+@
+
+
+1.10
+log
+@Tracing attribute, talloc support.
+@
+text
+@d1 1
+a1 1
+#LyX 1.6.5 created this file. For more info see http://www.lyx.org/
+d53 1
+a53 7
+
+\change_deleted 0 1283307542
+26-July
+\change_inserted 0 1284423485
+14-September
+\change_unchanged
+-2010
+a472 2
+\change_inserted 0 1284422789
+
+a479 2
+\change_unchanged
+
+a838 2
+
+\change_inserted 0 1284016998
+a846 2
+\change_unchanged
+
+a1194 2
+\change_inserted 0 1284015637
+
+a1197 2
+
+\change_inserted 0 1284015716
+a1201 2
+
+\change_inserted 0 1284015906
+a1210 2
+
+\change_inserted 0 1284015637
+a1214 2
+
+\change_inserted 0 1284016114
+a1227 2
+
+\change_inserted 0 1284016149
+a1232 2
+
+\change_inserted 0 1284016639
+a1237 2
+
+\change_inserted 0 1284016821
+a1243 2
+
+\change_inserted 0 1284016803
+d1245 2
+a1246 9
+ if older code (which doesn't understand the feature) writes to the database.
+\change_deleted 0 1284016101
+
+\end_layout
+
+\begin_layout Subsection
+
+\change_inserted 0 1284015634
+Record Headers Are Not Expandible
+a1249 2
+
+\change_inserted 0 1284015634
+a1254 2
+
+\change_inserted 0 1284015634
+a1258 2
+
+\change_inserted 0 1284422552
+a1267 2
+
+\change_inserted 0 1284422568
+a1271 2
+
+\change_inserted 0 1284422646
+a1276 2
+
+\change_inserted 0 1284422656
+a1280 2
+
+\change_inserted 0 1284423065
+a1305 2
+
+\change_inserted 0 1284423042
+a1310 2
+\change_unchanged
+
+a1457 2
+
+\change_inserted 0 1283336713
+a1463 2
+
+\change_unchanged
+d1482 2
+d1485 1
+a1485 51
+\change_deleted 0 1283307675
+There are three details which become important:
+\end_layout
+
+\begin_layout Enumerate
+
+\change_deleted 0 1283307675
+On encountering a full bucket, we use the next bucket.
+\end_layout
+
+\begin_layout Enumerate
+
+\change_deleted 0 1283307675
+Extra hash bits are stored with the offset, to reduce comparisons.
+\end_layout
+
+\begin_layout Enumerate
+
+\change_deleted 0 1283307675
+A marker entry is used on deleting an entry.
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1283307675
+The doubling of the table must be done under a transaction; we will not
+ reduce it on deletion, so it will be an unusual case.
+ It will either be placed at the head (other entries will be moved out the
+ way so we can expand).
+ We could have a pointer in the header to the current hashtable location,
+ but that pointer would have to be read frequently to check for hashtable
+ moves.
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1283307675
+The locking for this is slightly more complex than the chained case; we
+ currently have one lock per bucket, and that means we would need to expand
+ the lock if we overflow to the next bucket.
+ The frequency of such collisions will effect our locking heuristics: we
+ can always lock more buckets than we need.
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1283307675
+One possible optimization is to only re-check the hash size on an insert
+ or a lookup miss.
+
+\change_inserted 0 1283307770
+a1492 2
+
+\change_inserted 0 1283336187
+a1500 2
+
+\change_inserted 0 1283336586
+a1510 2
+\change_unchanged
+
+d1636 3
+a1638 8
+Proposed Solution
+\change_deleted 0 1283336858
+
+\end_layout
+
+\begin_layout Standard
+The first step is to remove all the current heuristics, as they obviously
+ interact, then examine them once the lock contention is addressed.
+a1647 2
+\change_inserted 0 1283336910
+
+a1650 2
+
+\change_inserted 0 1283337052
+a1655 2
+\change_unchanged
+
+a1776 2
+\change_inserted 0 1283309850
+
+a1779 2
+
+\change_inserted 0 1283337216
+a1813 2
+
+\change_inserted 0 1284424151
+a1825 2
+\change_unchanged
+
+a1830 2
+\change_unchanged
+
+a2031 2
+
+\change_inserted 0 1283336739
+a2040 2
+\change_unchanged
+
+a2117 2
+\change_inserted 0 1283337133
+
+a2120 2
+
+\change_inserted 0 1283337139
+a2121 2
+\change_unchanged
+
+a2136 2
+
+\change_inserted 0 1283337235
+a2147 2
+\change_unchanged
+
+d2251 1
+a2251 7
+Proposed Solution
+\change_deleted 0 1284423472
+
+\end_layout
+
+\begin_layout Standard
+None.
+d2261 1
+a2261 1
+\change_inserted 0 1284423891
+d2263 1
+a2263 4
+\change_deleted 0 1284423891
+.
+
+\change_inserted 0 1284423901
+a2271 2
+\change_unchanged
+
+a2293 2
+\change_inserted 0 1284423495
+
+a2312 2
+
+\change_inserted 0 1284424201
+d2321 1
+a2321 3
+
+\change_unchanged
+We could solve a small part of the problem by providing read-only transactions.
+a2505 2
+\change_inserted 0 1284423555
+
+a2508 2
+
+\change_inserted 0 1284423617
+a2512 2
+
+\change_inserted 0 1284423719
+a2519 2
+
+\change_inserted 0 1284423864
+a2530 2
+
+\change_inserted 0 1284423850
+a2540 2
+\change_unchanged
+
+@
+
+
+1.9
+log
+@Extension mechanism.
+@
+text
+@d56 2
+a57 2
+\change_inserted 0 1284016854
+9-September
+d479 11
+d1303 1
+a1303 1
+\change_inserted 0 1284016847
+d1310 56
+d1945 1
+a1945 1
+\change_inserted 0 1283310945
+d1956 2
+d2402 2
+d2416 4
+d2421 12
+d2455 2
+d2476 12
+d2673 47
+@
+
+
+1.8
+log
+@Remove bogus footnote
+@
+text
+@d56 2
+a57 2
+\change_inserted 0 1283307544
+1-September
+d838 12
+d1198 103
+@
+
+
+1.7
+log
+@Moving hash table does not work.
+@
+text
+@a1436 12
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted 0 1283336450
+If we make the hash offsets zone-relative, then this only restricts the
+ zone size, not the overall database size.
+\end_layout
+
+\end_inset
+
+@
+
+
+1.6
+log
+@Commit changes
+@
+text
+@d38 1
+a38 1
+\author ""
+d53 7
+a59 1
+26-July-2010
+d1333 10
+d1361 3
+a1363 1
+ There are three details which become important:
+d1367 2
+d1373 2
+d1379 2
+d1385 2
+d1397 2
+d1407 2
+d1411 45
+d1582 2
+d1598 14
+d1733 62
+d1996 13
+d2086 10
+d2110 15
+a2124 1
+\begin_layout LyX-Code
+@
+
+
+1.5
+log
+@Soft transaction commit
+@
+text
+@d38 1
+a38 1
+\author "Rusty Russell,,,"
+a52 4
+
+\change_deleted 0 1280141199
+10-May-2010
+\change_inserted 0 1280141202
+a53 2
+\change_unchanged
+
+a2028 2
+
+\change_inserted 0 1280140902
+a2034 2
+
+\change_unchanged
+a2212 2
+\change_inserted 0 1280140661
+
+a2215 2
+
+\change_inserted 0 1280140703
+a2219 2
+
+\change_inserted 0 1280708312
+a2226 2
+
+\change_inserted 0 1280708400
+a2239 2
+
+\change_inserted 0 1280140836
+a2243 2
+
+\change_inserted 0 1280708255
+a2247 2
+
+\change_inserted 0 1280708374
+a2252 2
+
+\change_inserted 0 1280141181
+a2274 2
+
+\change_inserted 0 1280141345
+@
+
+
+1.4
+log
+@Merge changes
+@
+text
+@d38 1
+a38 1
+\author ""
+d53 2
+d56 4
+d2035 10
+d2223 84
+@
+
+
+1.3
+log
+@Transaction and freelist rethink.
+@
+text
+@d38 1
+a38 1
+\author "Rusty Russell,,,"
+d53 1
+a53 1
+27-April-2010
+d662 1
+a662 5
+ behavior of disallowing
+\change_inserted 0 1272940179
+nested
+\change_unchanged
+transactions should become the default.
+a1210 2
+\change_inserted 0 1272944650
+
+a1214 2
+
+\change_inserted 0 1272944763
+a1218 2
+\change_unchanged
+
+a1223 2
+\change_unchanged
+
+a1301 2
+
+\change_inserted 0 1273478114
+a1310 2
+\change_unchanged
+
+d1515 1
+a1515 11
+The free list
+\change_deleted 0 1273469807
+should
+\change_inserted 0 1273469810
+must
+\change_unchanged
+ be split
+\change_deleted 0 1273469815
+into multiple lists
+\change_unchanged
+to reduce contention.
+a1520 2
+\change_inserted 0 1273470006
+
+a1523 2
+
+\change_inserted 0 1273492055
+a1539 2
+
+\change_inserted 0 1273483888
+a1551 2
+\change_unchanged
+
+a1554 8
+
+\change_deleted 0 1272942055
+There are various ways to organize these lisys, but because we want to be
+ able to quickly identify which free list an entry is in, and reduce the
+ number of locks required for merging, we will use zoning (eg.
+ each free list covers some fixed fraction of the file).
+
+\change_inserted 0 1273484187
+d1556 1
+a1556 7
+
+\change_deleted 0 1273484194
+The algorithm for f
+\change_inserted 0 1273484194
+F
+\change_unchanged
+reeing is simple:
+d1560 1
+a1560 7
+Identify the correct
+\change_deleted 0 1273482856
+free list
+\change_inserted 0 1273482857
+zone
+\change_unchanged
+.
+d1564 1
+a1564 7
+Lock the
+\change_inserted 0 1273482895
+corresponding
+\change_unchanged
+list
+\change_inserted 0 1273482863
+.
+a1567 2
+
+\change_inserted 0 1273482909
+d1573 1
+a1573 13
+
+\change_deleted 0 1273482885
+, and p
+\change_inserted 0 1273482888
+P
+\change_unchanged
+lace the freed entry
+\change_deleted 0 1273492415
+at the head
+\change_inserted 0 1273492415
+in the list for that zone
+\change_unchanged
+.
+d1577 2
+a1578 7
+Allocation is a little more complicated, as we
+\change_deleted 0 1273483240
+merge entries as we walk the list:
+\change_inserted 0 1273484250
+perform delayed coalescing at this point:
+\change_unchanged
+
+d1582 1
+a1582 19
+Pick a
+\change_deleted 0 1273482955
+free list;
+\change_inserted 0 1273482957
+zone
+\change_unchanged
+ either the
+\change_deleted 0 1273482962
+list
+\change_inserted 0 1273482962
+zone
+\change_unchanged
+ we last freed
+\change_deleted 0 1273482966
+o
+\change_inserted 0 1273482966
+i
+\change_unchanged
+nto, or based on a
+d1594 1
+a1594 9
+Lock th
+\change_inserted 0 1273482980
+e corresponding
+\change_deleted 0 1273482973
+at
+\change_unchanged
+ list.
+\change_inserted 0 1273482982
+
+a1597 2
+
+\change_inserted 0 1273483084
+a1598 53
+\change_unchanged
+
+\end_layout
+
+\begin_layout Enumerate
+If the top entry is
+\change_deleted 0 1273492155
+well-sized,
+\change_inserted 0 1273492159
+-large enough,
+\change_unchanged
+remove it from the list and return it.
+\end_layout
+
+\begin_layout Enumerate
+Otherwise,
+\change_inserted 0 1273492206
+coalesce entries in the list.
+\change_deleted 0 1273492200
+examine the entry to the right of it in the file.
+ If it is free:
+\end_layout
+
+\begin_deeper
+\begin_layout Enumerate
+
+\change_deleted 0 1273492200
+If that entry is in a different list, lock that list too.
+\end_layout
+
+\begin_layout Enumerate
+
+\change_deleted 0 1273492200
+If we had to place a new lock, re-check that the entry is free.
+\end_layout
+
+\begin_layout Enumerate
+
+\change_deleted 0 1273492200
+Remove that entry from its free list and expand this entry to cover it.
+\end_layout
+
+\begin_layout Enumerate
+
+\change_deleted 0 1273485554
+Goto step 3.
+\end_layout
+
+\end_deeper
+\begin_layout Enumerate
+
+\change_inserted 0 1273485311
+If there was no entry large enough, unlock the list and try the next zone.
+d1602 1
+a1602 5
+
+\change_deleted 0 1273483646
+Repeat step 3 with each entry in the list.
+\change_unchanged
+
+d1606 2
+a1607 5
+
+\change_deleted 0 1273483668
+Unlock the list and repeat step 2 with the next list.
+\change_unchanged
+
+d1611 1
+a1611 7
+If no
+\change_deleted 0 1273483671
+list
+\change_inserted 0 1273483671
+zone
+\change_unchanged
+ satisfies, expand the file.
+d1615 2
+a1616 9
+This optimizes rapid insert/delete of free list entries
+\change_inserted 0 1273485794
+ by not coalescing them all the time.
+\change_deleted 0 1273483685
+, and allows us to get rid of the tailer altogether
+\change_unchanged
+.
+
+\change_inserted 0 1273492299
+a1638 39
+
+\change_deleted 0 1273476840
+The question of
+\begin_inset Quotes eld
+\end_inset
+
+well-sized
+\begin_inset Quotes erd
+\end_inset
+
+ free entries is more difficult: the 25% overhead works in practice for
+ ldb because indexes tend to expand by one record at a time.
+ This can be resolved by having an
+\begin_inset Quotes eld
+\end_inset
+
+expanded
+\begin_inset Quotes erd
+\end_inset
+
+ bit in the header to note entries that have previously expanded, and allocating
+ more space for them.
+ Whether the
+\begin_inset Quotes eld
+\end_inset
+
+increasing slack
+\begin_inset Quotes erd
+\end_inset
+
+ algorithm should be implemented or first-fit used is still unknown: we
+ will determine this once these other ideas are implemented.
+\change_inserted 0 1273483750
+
+\end_layout
+
+\begin_layout Standard
+
+\change_inserted 0 1273492450
+a1644 2
+
+\change_inserted 0 1273470441
+a1654 2
+
+\change_inserted 0 1273476556
+a1659 2
+
+\change_inserted 0 1273470423
+a1661 2
+\change_unchanged
+
+a1672 2
+
+\change_inserted 0 1273476847
+a1676 2
+
+\change_inserted 0 1273476886
+a1691 2
+
+\change_inserted 0 1273477233
+a1699 2
+
+\change_inserted 0 1273477534
+a1706 2
+
+\change_inserted 0 1273482700
+a1712 2
+
+\change_inserted 0 1273478079
+a1722 2
+
+\change_inserted 0 1273477839
+a1726 2
+
+\change_inserted 0 1273477925
+a1730 2
+
+\change_inserted 0 1273477925
+a1734 2
+
+\change_inserted 0 1273477925
+a1738 2
+
+\change_inserted 0 1273477925
+a1742 2
+
+\change_inserted 0 1273477925
+a1746 2
+
+\change_inserted 0 1273477925
+a1750 2
+
+\change_inserted 0 1273477925
+a1754 2
+
+\change_inserted 0 1273477925
+a1758 2
+
+\change_inserted 0 1273477925
+a1762 2
+
+\change_inserted 0 1273477925
+a1766 2
+
+\change_inserted 0 1273477925
+a1770 2
+
+\change_inserted 0 1273477925
+a1774 2
+
+\change_inserted 0 1273477925
+a1778 2
+
+\change_inserted 0 1273477925
+a1782 2
+
+\change_inserted 0 1273477925
+a1786 2
+
+\change_inserted 0 1273477925
+a1790 2
+
+\change_inserted 0 1273477925
+a1794 2
+
+\change_inserted 0 1273477925
+a1798 2
+
+\change_inserted 0 1273492522
+a1802 2
+
+\change_inserted 0 1273492530
+a1806 2
+
+\change_inserted 0 1273492546
+a1810 2
+
+\change_inserted 0 1273478239
+a1814 2
+
+\change_inserted 0 1273479960
+a1821 2
+
+\change_inserted 0 1273480265
+a1830 2
+
+\change_inserted 0 1273480354
+a1845 2
+
+\change_inserted 0 1273478968
+a1851 2
+
+\change_inserted 0 1273492604
+a1859 2
+
+\change_inserted 0 1273479572
+a1862 2
+\change_unchanged
+
+a1870 2
+
+\change_inserted 0 1273480282
+a1874 2
+
+\change_inserted 0 1273478931
+a1878 2
+
+\change_inserted 0 1273481549
+a1882 2
+
+\change_inserted 0 1273481557
+a1886 2
+
+\change_inserted 0 1273480307
+a1890 2
+
+\change_inserted 0 1273480335
+a1894 2
+
+\change_inserted 0 1273479897
+a1898 2
+
+\change_inserted 0 1273479653
+a1902 2
+
+\change_inserted 0 1273480371
+a1906 2
+
+\change_inserted 0 1273480464
+a1910 2
+
+\change_inserted 0 1273480399
+a1914 2
+
+\change_inserted 0 1273480425
+a1918 2
+
+\change_inserted 0 1273480453
+a1922 2
+
+\change_inserted 0 1273480455
+a1926 2
+
+\change_inserted 0 1273480450
+a1930 2
+
+\change_inserted 0 1273480452
+a1935 2
+\change_inserted 0 1273478830
+
+a1942 5
+
+\change_deleted 0 1273481604
+In theory, we could get away with 2: one after we write the new data, and
+ one to somehow atomically change over to it.
+\change_inserted 0 1273481632
+a1946 2
+
+\change_inserted 0 1273481724
+a1950 2
+
+\change_inserted 0 1273481713
+a1954 2
+
+\change_inserted 0 1273481717
+a1958 2
+
+\change_inserted 0 1273481730
+a1962 2
+
+\change_inserted 0 1273481736
+a1966 2
+
+\change_inserted 0 1273481744
+a1970 2
+
+\change_inserted 0 1273481748
+a1974 2
+
+\change_inserted 0 1273482185
+a1978 2
+
+\change_inserted 0 1273482259
+a1989 50
+
+\change_deleted 0 1273481848
+None.
+ Trying to rewrite the transaction code is a separate experiment, which
+ I encourage someone else to do.
+ At some point you say
+\begin_inset Quotes eld
+\end_inset
+
+use a real database
+\begin_inset Quotes erd
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1273481848
+But as a thought experiment:
+\change_unchanged
+
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1273481788
+Say there was a pointer in the header which said where the hash table and
+ free list tables were, and that no blocks were labeled with whether they
+ were free or not (it had to be derived from what list they were in).
+ We could create new hash table and free list in some free space, and populate
+ it as we want the post-committed state to look.
+ Then we sync, then we switch the offset in the header, then we sync again.
+\end_layout
+
+\begin_layout Standard
+
+\change_deleted 0 1273481788
+This would not allow arbitrary changes to the database, such as tdb_repack
+ does, and would require more space (since we have to preserve the current
+ and future entries at once).
+ If we used hash trees rather than one big hash table, we might only have
+ to rewrite some sections of the hash, too.
+\change_inserted 0 1273481854
+
+\end_layout
+
+\begin_layout Standard
+
+\change_inserted 0 1273482102
+a1993 2
+
+\change_inserted 0 1273482061
+a1998 2
+
+\change_inserted 0 1273482063
+a2002 2
+
+\change_inserted 0 1273482072
+a2006 2
+
+\change_inserted 0 1273482139
+a2011 2
+
+\change_inserted 0 1273482364
+a2015 2
+
+\change_inserted 0 1273482163
+a2019 2
+
+\change_inserted 0 1273482493
+a2037 2
+
+\change_inserted 0 1273482536
+a2046 2
+\change_unchanged
+
+a2049 2
+
+\change_inserted 0 1273482641
+a2058 2
+
+\change_inserted 0 1273481827
+d2067 2
+a2068 11
+We could
+\change_inserted 0 1273481829
+then
+\change_unchanged
+implement snapshots using a similar method
+\change_deleted 0 1273481838
+ to the above, only
+\change_inserted 0 1273481840
+,
+\change_unchanged
+ using multiple different hash tables/free tables.
+@
+
+
+1.2
+log
+@After first feedback (Ronnie & Volker)
+@
+text
+@d1314 13
+d1531 11
+a1541 1
+The free list should be split into multiple lists to reduce contention.
+d1547 39
+d1596 7
+d1604 1
+a1604 1
+The algorithm for freeing is simple:
+d1608 7
+a1614 1
+Identify the correct free list.
+d1618 30
+a1647 1
+Lock the list, and place the freed entry at the head.
+d1651 7
+a1657 2
+Allocation is a little more complicated, as we merge entries as we walk
+ the list:
+d1661 19
+a1679 1
+Pick a free list; either the list we last freed onto, or based on a
+d1691 17
+a1707 1
+Lock that list.
+d1711 7
+a1717 1
+If the top entry is well-sized, remove it from the list and return it.
+d1721 5
+a1725 1
+Otherwise, examine the entry to the right of it in the file.
+d1731 2
+d1737 2
+d1743 2
+d1749 2
+d1756 8
+d1765 2
+d1770 2
+d1773 2
+d1778 7
+a1784 1
+If no list satisfies, expand the file.
+d1788 28
+a1815 2
+This optimizes rapid insert/delete of free list entries, and allows us to
+ get rid of the tailer altogether.
+d1819 2
+d1851 1
+a1851 1
+\change_inserted 0 1272941474
+d1857 303
+a2159 18
+\change_inserted 0 1272942759
+There are various ways to organize these lists, but because we want to be
+ able to quickly identify which free list an entry is in, and reduce the
+ number of locks required for merging, we will use zoning (eg.
+ each of the N free lists in a tdb file of size M covers a fixed fraction
+ M/N).
+ Note that this means we need to reshuffle the free lists when we expand
+ the file; this is probably acceptable when we double the hash table size,
+ since that is such an expensive operation already.
+ In the case of increasing the file size, there is an optimization we can
+ use: if we use M in the formula above as the file size rounded up to the
+ next power of 2, we only need reshuffle free lists when the file size crosses
+ a power of 2 boundary,
+\emph on
+and
+\emph default
+reshuffling the free lists is trivial: we simply merge every consecutive
+ pair of free lists.
+d2164 107
+d2276 2
+d2280 59
+d2346 2
+d2363 2
+d2366 2
+d2371 2
+d2382 2
+d2389 57
+d2458 13
+d2474 32
+a2505 2
+We could implement snapshots using a similar method to the above, only using
+ multiple different hash tables/free tables.
+@
+
+
+1.1
+log
+@Initial revision
+@
+text
+@d1 1
+a1 1
+#LyX 1.6.4 created this file. For more info see http://www.lyx.org/
+d36 3
+a38 3
+\tracking_changes false
+\output_changes false
+\author ""
+d662 5
+a666 1
+ behavior of disallowing transactions should become the default.
+d1215 21
+d1527 2
+d1533 3
+a1535 1
+ The algorithm for freeing is simple:
+d1642 26
+@
diff --git a/lib/tdb2/doc/design.pdf b/lib/tdb2/doc/design.pdf
new file mode 100644
index 00000000000..558dc1f8c2e
--- /dev/null
+++ b/lib/tdb2/doc/design.pdf
diff --git a/lib/tdb2/doc/design.txt b/lib/tdb2/doc/design.txt
new file mode 100644
index 00000000000..bd2ffde4db6
--- /dev/null
+++ b/lib/tdb2/doc/design.txt
@@ -0,0 +1,1258 @@
+TDB2: A Redesigning The Trivial DataBase
+
+Rusty Russell, IBM Corporation
+
+1-December-2010
+
+Abstract
+
+The Trivial DataBase on-disk format is 32 bits; with usage cases
+heading towards the 4G limit, that must change. This required
+breakage provides an opportunity to revisit TDB's other design
+decisions and reassess them.
+
+1 Introduction
+
+The Trivial DataBase was originally written by Andrew Tridgell as
+a simple key/data pair storage system with the same API as dbm,
+but allowing multiple readers and writers while being small
+enough (< 1000 lines of C) to include in SAMBA. The simple design
+created in 1999 has proven surprisingly robust and performant,
+used in Samba versions 3 and 4 as well as numerous other
+projects. Its useful life was greatly increased by the
+(backwards-compatible!) addition of transaction support in 2005.
+
+The wider variety and greater demands of TDB-using code has lead
+to some organic growth of the API, as well as some compromises on
+the implementation. None of these, by themselves, are seen as
+show-stoppers, but the cumulative effect is to a loss of elegance
+over the initial, simple TDB implementation. Here is a table of
+the approximate number of lines of implementation code and number
+of API functions at the end of each year:
+
+
++-----------+----------------+--------------------------------+
+| Year End  | API Functions  | Lines of C Code Implementation |
++-----------+----------------+--------------------------------+
++-----------+----------------+--------------------------------+
+|   1999    |      13        |              1195              |
++-----------+----------------+--------------------------------+
+|   2000    |      24        |              1725              |
++-----------+----------------+--------------------------------+
+|   2001    |      32        |              2228              |
++-----------+----------------+--------------------------------+
+|   2002    |      35        |              2481              |
++-----------+----------------+--------------------------------+
+|   2003    |      35        |              2552              |
++-----------+----------------+--------------------------------+
+|   2004    |      40        |              2584              |
++-----------+----------------+--------------------------------+
+|   2005    |      38        |              2647              |
++-----------+----------------+--------------------------------+
+|   2006    |      52        |              3754              |
++-----------+----------------+--------------------------------+
+|   2007    |      66        |              4398              |
++-----------+----------------+--------------------------------+
+|   2008    |      71        |              4768              |
++-----------+----------------+--------------------------------+
+|   2009    |      73        |              5715              |
++-----------+----------------+--------------------------------+
+
+
+This review is an attempt to catalog and address all the known
+issues with TDB and create solutions which address the problems
+without significantly increasing complexity; all involved are far
+too aware of the dangers of second system syndrome in rewriting a
+successful project like this.
+
+2 API Issues
+
+2.1 tdb_open_ex Is Not Expandable
+
+The tdb_open() call was expanded to tdb_open_ex(), which added an
+optional hashing function and an optional logging function
+argument. Additional arguments to open would require the
+introduction of a tdb_open_ex2 call etc.
+
+2.1.1 Proposed Solution<attributes>
+
+tdb_open() will take a linked-list of attributes:
+
+enum tdb_attribute {
+
+    TDB_ATTRIBUTE_LOG = 0,
+
+    TDB_ATTRIBUTE_HASH = 1
+
+};
+
+struct tdb_attribute_base {
+
+    enum tdb_attribute attr;
+
+    union tdb_attribute *next;
+
+};
+
+struct tdb_attribute_log {
+
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_LOG
+*/
+
+    tdb_log_func log_fn;
+
+    void *log_private;
+
+};
+
+struct tdb_attribute_hash {
+
+    struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_HASH
+*/
+
+    tdb_hash_func hash_fn;
+
+    void *hash_private;
+
+};
+
+union tdb_attribute {
+
+    struct tdb_attribute_base base;
+
+    struct tdb_attribute_log log;
+
+    struct tdb_attribute_hash hash;
+
+};
+
+This allows future attributes to be added, even if this expands
+the size of the union.
+
+2.1.2 Status
+
+Complete.
+
+2.2 tdb_traverse Makes Impossible Guarantees
+
+tdb_traverse (and tdb_firstkey/tdb_nextkey) predate transactions,
+and it was thought that it was important to guarantee that all
+records which exist at the start and end of the traversal would
+be included, and no record would be included twice.
+
+This adds complexity (see[Reliable-Traversal-Adds]) and does not
+work anyway for records which are altered (in particular, those
+which are expanded may be effectively deleted and re-added behind
+the traversal).
+
+2.2.1 <traverse-Proposed-Solution>Proposed Solution
+
+Abandon the guarantee. You will see every record if no changes
+occur during your traversal, otherwise you will see some subset.
+You can prevent changes by using a transaction or the locking
+API.
+
+2.2.2 Status
+
+Complete. Delete-during-traverse will still delete every record,
+too (assuming no other changes).
+
+2.3 Nesting of Transactions Is Fraught
+
+TDB has alternated between allowing nested transactions and not
+allowing them. Various paths in the Samba codebase assume that
+transactions will nest, and in a sense they can: the operation is
+only committed to disk when the outer transaction is committed.
+There are two problems, however:
+
+1. Canceling the inner transaction will cause the outer
+  transaction commit to fail, and will not undo any operations
+  since the inner transaction began. This problem is soluble with
+  some additional internal code.
+
+2. An inner transaction commit can be cancelled by the outer
+  transaction. This is desirable in the way which Samba's
+  database initialization code uses transactions, but could be a
+  surprise to any users expecting a successful transaction commit
+  to expose changes to others.
+
+The current solution is to specify the behavior at tdb_open(),
+with the default currently that nested transactions are allowed.
+This flag can also be changed at runtime.
+
+2.3.1 Proposed Solution
+
+Given the usage patterns, it seems that the “least-surprise”
+behavior of disallowing nested transactions should become the
+default. Additionally, it seems the outer transaction is the only
+code which knows whether inner transactions should be allowed, so
+a flag to indicate this could be added to tdb_transaction_start.
+However, this behavior can be simulated with a wrapper which uses
+tdb_add_flags() and tdb_remove_flags(), so the API should not be
+expanded for this relatively-obscure case.
+
+2.3.2 Status
+
+Incomplete; nesting flag is still defined as per tdb1.
+
+2.4 Incorrect Hash Function is Not Detected
+
+tdb_open_ex() allows the calling code to specify a different hash
+function to use, but does not check that all other processes
+accessing this tdb are using the same hash function. The result
+is that records are missing from tdb_fetch().
+
+2.4.1 Proposed Solution
+
+The header should contain an example hash result (eg. the hash of
+0xdeadbeef), and tdb_open_ex() should check that the given hash
+function produces the same answer, or fail the tdb_open call.
+
+2.4.2 Status
+
+Complete.
+
+2.5 tdb_set_max_dead/TDB_VOLATILE Expose Implementation
+
+In response to scalability issues with the free list ([TDB-Freelist-Is]
+) two API workarounds have been incorporated in TDB:
+tdb_set_max_dead() and the TDB_VOLATILE flag to tdb_open. The
+latter actually calls the former with an argument of “5”.
+
+This code allows deleted records to accumulate without putting
+them in the free list. On delete we iterate through each chain
+and free them in a batch if there are more than max_dead entries.
+These are never otherwise recycled except as a side-effect of a
+tdb_repack.
+
+2.5.1 Proposed Solution
+
+With the scalability problems of the freelist solved, this API
+can be removed. The TDB_VOLATILE flag may still be useful as a
+hint that store and delete of records will be at least as common
+as fetch in order to allow some internal tuning, but initially
+will become a no-op.
+
+2.5.2 Status
+
+Incomplete. TDB_VOLATILE still defined, but implementation should
+fail on unknown flags to be future-proof.
+
+2.6 <TDB-Files-Cannot>TDB Files Cannot Be Opened Multiple Times
+  In The Same Process
+
+No process can open the same TDB twice; we check and disallow it.
+This is an unfortunate side-effect of fcntl locks, which operate
+on a per-file rather than per-file-descriptor basis, and do not
+nest. Thus, closing any file descriptor on a file clears all the
+locks obtained by this process, even if they were placed using a
+different file descriptor!
+
+Note that even if this were solved, deadlock could occur if
+operations were nested: this is a more manageable programming
+error in most cases.
+
+2.6.1 Proposed Solution
+
+We could lobby POSIX to fix the perverse rules, or at least lobby
+Linux to violate them so that the most common implementation does
+not have this restriction. This would be a generally good idea
+for other fcntl lock users.
+
+Samba uses a wrapper which hands out the same tdb_context to
+multiple callers if this happens, and does simple reference
+counting. We should do this inside the tdb library, which already
+emulates lock nesting internally; it would need to recognize when
+deadlock occurs within a single process. This would create a new
+failure mode for tdb operations (while we currently handle
+locking failures, they are impossible in normal use and a process
+encountering them can do little but give up).
+
+I do not see benefit in an additional tdb_open flag to indicate
+whether re-opening is allowed, as though there may be some
+benefit to adding a call to detect when a tdb_context is shared,
+to allow other to create such an API.
+
+2.6.2 Status
+
+Incomplete.
+
+2.7 TDB API Is Not POSIX Thread-safe
+
+The TDB API uses an error code which can be queried after an
+operation to determine what went wrong. This programming model
+does not work with threads, unless specific additional guarantees
+are given by the implementation. In addition, even
+otherwise-independent threads cannot open the same TDB (as in [TDB-Files-Cannot]
+).
+
+2.7.1 Proposed Solution
+
+Reachitecting the API to include a tdb_errcode pointer would be a
+great deal of churn; we are better to guarantee that the
+tdb_errcode is per-thread so the current programming model can be
+maintained.
+
+This requires dynamic per-thread allocations, which is awkward
+with POSIX threads (pthread_key_create space is limited and we
+cannot simply allocate a key for every TDB).
+
+Internal locking is required to make sure that fcntl locks do not
+overlap between threads, and also that the global list of tdbs is
+maintained.
+
+The aim is that building tdb with -DTDB_PTHREAD will result in a
+pthread-safe version of the library, and otherwise no overhead
+will exist. Alternatively, a hooking mechanism similar to that
+proposed for [Proposed-Solution-locking-hook] could be used to
+enable pthread locking at runtime.
+
+2.7.2 Status
+
+Incomplete.
+
+2.8 *_nonblock Functions And *_mark Functions Expose
+  Implementation
+
+CTDB[footnote:
+Clustered TDB, see http://ctdb.samba.org
+] wishes to operate on TDB in a non-blocking manner. This is
+currently done as follows:
+
+1. Call the _nonblock variant of an API function (eg.
+  tdb_lockall_nonblock). If this fails:
+
+2. Fork a child process, and wait for it to call the normal
+  variant (eg. tdb_lockall).
+
+3. If the child succeeds, call the _mark variant to indicate we
+  already have the locks (eg. tdb_lockall_mark).
+
+4. Upon completion, tell the child to release the locks (eg.
+  tdb_unlockall).
+
+5. Indicate to tdb that it should consider the locks removed (eg.
+  tdb_unlockall_mark).
+
+There are several issues with this approach. Firstly, adding two
+new variants of each function clutters the API for an obscure
+use, and so not all functions have three variants. Secondly, it
+assumes that all paths of the functions ask for the same locks,
+otherwise the parent process will have to get a lock which the
+child doesn't have under some circumstances. I don't believe this
+is currently the case, but it constrains the implementation.
+
+2.8.1 <Proposed-Solution-locking-hook>Proposed Solution
+
+Implement a hook for locking methods, so that the caller can
+control the calls to create and remove fcntl locks. In this
+scenario, ctdbd would operate as follows:
+
+1. Call the normal API function, eg tdb_lockall().
+
+2. When the lock callback comes in, check if the child has the
+  lock. Initially, this is always false. If so, return 0.
+  Otherwise, try to obtain it in non-blocking mode. If that
+  fails, return EWOULDBLOCK.
+
+3. Release locks in the unlock callback as normal.
+
+4. If tdb_lockall() fails, see if we recorded a lock failure; if
+  so, call the child to repeat the operation.
+
+5. The child records what locks it obtains, and returns that
+  information to the parent.
+
+6. When the child has succeeded, goto 1.
+
+This is flexible enough to handle any potential locking scenario,
+even when lock requirements change. It can be optimized so that
+the parent does not release locks, just tells the child which
+locks it doesn't need to obtain.
+
+It also keeps the complexity out of the API, and in ctdbd where
+it is needed.
+
+2.8.2 Status
+
+Incomplete.
+
+2.9 tdb_chainlock Functions Expose Implementation
+
+tdb_chainlock locks some number of records, including the record
+indicated by the given key. This gave atomicity guarantees;
+no-one can start a transaction, alter, read or delete that key
+while the lock is held.
+
+It also makes the same guarantee for any other key in the chain,
+which is an internal implementation detail and potentially a
+cause for deadlock.
+
+2.9.1 Proposed Solution
+
+None. It would be nice to have an explicit single entry lock
+which effected no other keys. Unfortunately, this won't work for
+an entry which doesn't exist. Thus while chainlock may be
+implemented more efficiently for the existing case, it will still
+have overlap issues with the non-existing case. So it is best to
+keep the current (lack of) guarantee about which records will be
+effected to avoid constraining our implementation.
+
+2.10 Signal Handling is Not Race-Free
+
+The tdb_setalarm_sigptr() call allows the caller's signal handler
+to indicate that the tdb locking code should return with a
+failure, rather than trying again when a signal is received (and
+errno == EAGAIN). This is usually used to implement timeouts.
+
+Unfortunately, this does not work in the case where the signal is
+received before the tdb code enters the fcntl() call to place the
+lock: the code will sleep within the fcntl() code, unaware that
+the signal wants it to exit. In the case of long timeouts, this
+does not happen in practice.
+
+2.10.1 Proposed Solution
+
+The locking hooks proposed in[Proposed-Solution-locking-hook]
+would allow the user to decide on whether to fail the lock
+acquisition on a signal. This allows the caller to choose their
+own compromise: they could narrow the race by checking
+immediately before the fcntl call.[footnote:
+It may be possible to make this race-free in some implementations
+by having the signal handler alter the struct flock to make it
+invalid. This will cause the fcntl() lock call to fail with
+EINVAL if the signal occurs before the kernel is entered,
+otherwise EAGAIN.
+]
+
+2.10.2 Status
+
+Incomplete.
+
+2.11 The API Uses Gratuitous Typedefs, Capitals
+
+typedefs are useful for providing source compatibility when types
+can differ across implementations, or arguably in the case of
+function pointer definitions which are hard for humans to parse.
+Otherwise it is simply obfuscation and pollutes the namespace.
+
+Capitalization is usually reserved for compile-time constants and
+macros.
+
+  TDB_CONTEXT There is no reason to use this over 'struct
+  tdb_context'; the definition isn't visible to the API user
+  anyway.
+
+  TDB_DATA There is no reason to use this over struct TDB_DATA;
+  the struct needs to be understood by the API user.
+
+  struct TDB_DATA This would normally be called 'struct
+  tdb_data'.
+
+  enum TDB_ERROR Similarly, this would normally be enum
+  tdb_error.
+
+2.11.1 Proposed Solution
+
+None. Introducing lower case variants would please pedants like
+myself, but if it were done the existing ones should be kept.
+There is little point forcing a purely cosmetic change upon tdb
+users.
+
+2.12 <tdb_log_func-Doesnt-Take>tdb_log_func Doesn't Take The
+  Private Pointer
+
+For API compatibility reasons, the logging function needs to call
+tdb_get_logging_private() to retrieve the pointer registered by
+the tdb_open_ex for logging.
+
+2.12.1 Proposed Solution
+
+It should simply take an extra argument, since we are prepared to
+break the API/ABI.
+
+2.12.2 Status
+
+Complete.
+
+2.13 Various Callback Functions Are Not Typesafe
+
+The callback functions in tdb_set_logging_function (after [tdb_log_func-Doesnt-Take]
+ is resolved), tdb_parse_record, tdb_traverse, tdb_traverse_read
+and tdb_check all take void * and must internally convert it to
+the argument type they were expecting.
+
+If this type changes, the compiler will not produce warnings on
+the callers, since it only sees void *.
+
+2.13.1 Proposed Solution
+
+With careful use of macros, we can create callback functions
+which give a warning when used on gcc and the types of the
+callback and its private argument differ. Unsupported compilers
+will not give a warning, which is no worse than now. In addition,
+the callbacks become clearer, as they need not use void * for
+their parameter.
+
+See CCAN's typesafe_cb module at
+http://ccan.ozlabs.org/info/typesafe_cb.html
+
+2.13.2 Status
+
+Incomplete.
+
+2.14 TDB_CLEAR_IF_FIRST Must Be Specified On All Opens,
+  tdb_reopen_all Problematic
+
+The TDB_CLEAR_IF_FIRST flag to tdb_open indicates that the TDB
+file should be cleared if the caller discovers it is the only
+process with the TDB open. However, if any caller does not
+specify TDB_CLEAR_IF_FIRST it will not be detected, so will have
+the TDB erased underneath them (usually resulting in a crash).
+
+There is a similar issue on fork(); if the parent exits (or
+otherwise closes the tdb) before the child calls tdb_reopen_all()
+to establish the lock used to indicate the TDB is opened by
+someone, a TDB_CLEAR_IF_FIRST opener at that moment will believe
+it alone has opened the TDB and will erase it.
+
+2.14.1 Proposed Solution
+
+Remove TDB_CLEAR_IF_FIRST. Other workarounds are possible, but
+see [TDB_CLEAR_IF_FIRST-Imposes-Performance].
+
+2.14.2 Status
+
+Incomplete, TDB_CLEAR_IF_FIRST still defined, but not
+implemented.
+
+2.15 Extending The Header Is Difficult
+
+We have reserved (zeroed) words in the TDB header, which can be
+used for future features. If the future features are compulsory,
+the version number must be updated to prevent old code from
+accessing the database. But if the future feature is optional, we
+have no way of telling if older code is accessing the database or
+not.
+
+2.15.1 Proposed Solution
+
+The header should contain a “format variant” value (64-bit). This
+is divided into two 32-bit parts:
+
+1. The lower part reflects the format variant understood by code
+  accessing the database.
+
+2. The upper part reflects the format variant you must understand
+  to write to the database (otherwise you can only open for
+  reading).
+
+The latter field can only be written at creation time, the former
+should be written under the OPEN_LOCK when opening the database
+for writing, if the variant of the code is lower than the current
+lowest variant.
+
+This should allow backwards-compatible features to be added, and
+detection if older code (which doesn't understand the feature)
+writes to the database.
+
+2.15.2 Status
+
+Incomplete.
+
+2.16 Record Headers Are Not Expandible
+
+If we later want to add (say) checksums on keys and data, it
+would require another format change, which we'd like to avoid.
+
+2.16.1 Proposed Solution
+
+We often have extra padding at the tail of a record. If we ensure
+that the first byte (if any) of this padding is zero, we will
+have a way for future changes to detect code which doesn't
+understand a new format: the new code would write (say) a 1 at
+the tail, and thus if there is no tail or the first byte is 0, we
+would know the extension is not present on that record.
+
+2.16.2 Status
+
+Incomplete.
+
+2.17 TDB Does Not Use Talloc
+
+Many users of TDB (particularly Samba) use the talloc allocator,
+and thus have to wrap TDB in a talloc context to use it
+conveniently.
+
+2.17.1 Proposed Solution
+
+The allocation within TDB is not complicated enough to justify
+the use of talloc, and I am reluctant to force another
+(excellent) library on TDB users. Nonetheless a compromise is
+possible. An attribute (see [attributes]) can be added later to
+tdb_open() to provide an alternate allocation mechanism,
+specifically for talloc but usable by any other allocator (which
+would ignore the “context” argument).
+
+This would form a talloc heirarchy as expected, but the caller
+would still have to attach a destructor to the tdb context
+returned from tdb_open to close it. All TDB_DATA fields would be
+children of the tdb_context, and the caller would still have to
+manage them (using talloc_free() or talloc_steal()).
+
+2.17.2 Status
+
+Deferred.
+
+3 Performance And Scalability Issues
+
+3.1 <TDB_CLEAR_IF_FIRST-Imposes-Performance>TDB_CLEAR_IF_FIRST
+  Imposes Performance Penalty
+
+When TDB_CLEAR_IF_FIRST is specified, a 1-byte read lock is
+placed at offset 4 (aka. the ACTIVE_LOCK). While these locks
+never conflict in normal tdb usage, they do add substantial
+overhead for most fcntl lock implementations when the kernel
+scans to detect if a lock conflict exists. This is often a single
+linked list, making the time to acquire and release a fcntl lock
+O(N) where N is the number of processes with the TDB open, not
+the number actually doing work.
+
+In a Samba server it is common to have huge numbers of clients
+sitting idle, and thus they have weaned themselves off the
+TDB_CLEAR_IF_FIRST flag.[footnote:
+There is a flag to tdb_reopen_all() which is used for this
+optimization: if the parent process will outlive the child, the
+child does not need the ACTIVE_LOCK. This is a workaround for
+this very performance issue.
+]
+
+3.1.1 Proposed Solution
+
+Remove the flag. It was a neat idea, but even trivial servers
+tend to know when they are initializing for the first time and
+can simply unlink the old tdb at that point.
+
+3.1.2 Status
+
+Incomplete; TDB_CLEAR_IF_FIRST still defined, but does nothing.
+
+3.2 TDB Files Have a 4G Limit
+
+This seems to be becoming an issue (so much for “trivial”!),
+particularly for ldb.
+
+3.2.1 Proposed Solution
+
+A new, incompatible TDB format which uses 64 bit offsets
+internally rather than 32 bit as now. For simplicity of endian
+conversion (which TDB does on the fly if required), all values
+will be 64 bit on disk. In practice, some upper bits may be used
+for other purposes, but at least 56 bits will be available for
+file offsets.
+
+tdb_open() will automatically detect the old version, and even
+create them if TDB_VERSION6 is specified to tdb_open.
+
+32 bit processes will still be able to access TDBs larger than 4G
+(assuming that their off_t allows them to seek to 64 bits), they
+will gracefully fall back as they fail to mmap. This can happen
+already with large TDBs.
+
+Old versions of tdb will fail to open the new TDB files (since 28
+August 2009, commit 398d0c29290: prior to that any unrecognized
+file format would be erased and initialized as a fresh tdb!)
+
+3.2.2 Status
+
+Complete.
+
+3.3 TDB Records Have a 4G Limit
+
+This has not been a reported problem, and the API uses size_t
+which can be 64 bit on 64 bit platforms. However, other limits
+may have made such an issue moot.
+
+3.3.1 Proposed Solution
+
+Record sizes will be 64 bit, with an error returned on 32 bit
+platforms which try to access such records (the current
+implementation would return TDB_ERR_OOM in a similar case). It
+seems unlikely that 32 bit keys will be a limitation, so the
+implementation may not support this (see [sub:Records-Incur-A]).
+
+3.3.2 Status
+
+Complete.
+
+3.4 Hash Size Is Determined At TDB Creation Time
+
+TDB contains a number of hash chains in the header; the number is
+specified at creation time, and defaults to 131. This is such a
+bottleneck on large databases (as each hash chain gets quite
+long), that LDB uses 10,000 for this hash. In general it is
+impossible to know what the 'right' answer is at database
+creation time.
+
+3.4.1 <sub:Hash-Size-Solution>Proposed Solution
+
+After comprehensive performance testing on various scalable hash
+variants[footnote:
+http://rusty.ozlabs.org/?p=89 and http://rusty.ozlabs.org/?p=94
+This was annoying because I was previously convinced that an
+expanding tree of hashes would be very close to optimal.
+], it became clear that it is hard to beat a straight linear hash
+table which doubles in size when it reaches saturation.
+Unfortunately, altering the hash table introduces serious locking
+complications: the entire hash table needs to be locked to
+enlarge the hash table, and others might be holding locks.
+Particularly insidious are insertions done under tdb_chainlock.
+
+Thus an expanding layered hash will be used: an array of hash
+groups, with each hash group exploding into pointers to lower
+hash groups once it fills, turning into a hash tree. This has
+implications for locking: we must lock the entire group in case
+we need to expand it, yet we don't know how deep the tree is at
+that point.
+
+Note that bits from the hash table entries should be stolen to
+hold more hash bits to reduce the penalty of collisions. We can
+use the otherwise-unused lower 3 bits. If we limit the size of
+the database to 64 exabytes, we can use the top 8 bits of the
+hash entry as well. These 11 bits would reduce false positives
+down to 1 in 2000 which is more than we need: we can use one of
+the bits to indicate that the extra hash bits are valid. This
+means we can choose not to re-hash all entries when we expand a
+hash group; simply use the next bits we need and mark them
+invalid.
+
+3.4.2 Status
+
+Complete.
+
+3.5 <TDB-Freelist-Is>TDB Freelist Is Highly Contended
+
+TDB uses a single linked list for the free list. Allocation
+occurs as follows, using heuristics which have evolved over time:
+
+1. Get the free list lock for this whole operation.
+
+2. Multiply length by 1.25, so we always over-allocate by 25%.
+
+3. Set the slack multiplier to 1.
+
+4. Examine the current freelist entry: if it is > length but <
+  the current best case, remember it as the best case.
+
+5. Multiply the slack multiplier by 1.05.
+
+6. If our best fit so far is less than length * slack multiplier,
+  return it. The slack will be turned into a new free record if
+  it's large enough.
+
+7. Otherwise, go onto the next freelist entry.
+
+Deleting a record occurs as follows:
+
+1. Lock the hash chain for this whole operation.
+
+2. Walk the chain to find the record, keeping the prev pointer
+  offset.
+
+3. If max_dead is non-zero:
+
+  (a) Walk the hash chain again and count the dead records.
+
+  (b) If it's more than max_dead, bulk free all the dead ones
+    (similar to steps 4 and below, but the lock is only obtained
+    once).
+
+  (c) Simply mark this record as dead and return.
+
+4. Get the free list lock for the remainder of this operation.
+
+5. <right-merging>Examine the following block to see if it is
+  free; if so, enlarge the current block and remove that block
+  from the free list. This was disabled, as removal from the free
+  list was O(entries-in-free-list).
+
+6. Examine the preceeding block to see if it is free: for this
+  reason, each block has a 32-bit tailer which indicates its
+  length. If it is free, expand it to cover our new block and
+  return.
+
+7. Otherwise, prepend ourselves to the free list.
+
+Disabling right-merging (step [right-merging]) causes
+fragmentation; the other heuristics proved insufficient to
+address this, so the final answer to this was that when we expand
+the TDB file inside a transaction commit, we repack the entire
+tdb.
+
+The single list lock limits our allocation rate; due to the other
+issues this is not currently seen as a bottleneck.
+
+3.5.1 Proposed Solution
+
+The first step is to remove all the current heuristics, as they
+obviously interact, then examine them once the lock contention is
+addressed.
+
+The free list must be split to reduce contention. Assuming
+perfect free merging, we can at most have 1 free list entry for
+each entry. This implies that the number of free lists is related
+to the size of the hash table, but as it is rare to walk a large
+number of free list entries we can use far fewer, say 1/32 of the
+number of hash buckets.
+
+It seems tempting to try to reuse the hash implementation which
+we use for records here, but we have two ways of searching for
+free entries: for allocation we search by size (and possibly
+zone) which produces too many clashes for our hash table to
+handle well, and for coalescing we search by address. Thus an
+array of doubly-linked free lists seems preferable.
+
+There are various benefits in using per-size free lists (see [sub:TDB-Becomes-Fragmented]
+) but it's not clear this would reduce contention in the common
+case where all processes are allocating/freeing the same size.
+Thus we almost certainly need to divide in other ways: the most
+obvious is to divide the file into zones, and using a free list
+(or table of free lists) for each. This approximates address
+ordering.
+
+Unfortunately it is difficult to know what heuristics should be
+used to determine zone sizes, and our transaction code relies on
+being able to create a “recovery area” by simply appending to the
+file (difficult if it would need to create a new zone header).
+Thus we use a linked-list of free tables; currently we only ever
+create one, but if there is more than one we choose one at random
+to use. In future we may use heuristics to add new free tables on
+contention. We only expand the file when all free tables are
+exhausted.
+
+The basic algorithm is as follows. Freeing is simple:
+
+1. Identify the correct free list.
+
+2. Lock the corresponding list.
+
+3. Re-check the list (we didn't have a lock, sizes could have
+  changed): relock if necessary.
+
+4. Place the freed entry in the list.
+
+Allocation is a little more complicated, as we perform delayed
+coalescing at this point:
+
+1. Pick a free table; usually the previous one.
+
+2. Lock the corresponding list.
+
+3. If the top entry is -large enough, remove it from the list and
+  return it.
+
+4. Otherwise, coalesce entries in the list.If there was no entry
+  large enough, unlock the list and try the next largest list
+
+5. If no list has an entry which meets our needs, try the next
+  free table.
+
+6. If no zone satisfies, expand the file.
+
+This optimizes rapid insert/delete of free list entries by not
+coalescing them all the time.. First-fit address ordering
+ordering seems to be fairly good for keeping fragmentation low
+(see [sub:TDB-Becomes-Fragmented]). Note that address ordering
+does not need a tailer to coalesce, though if we needed one we
+could have one cheaply: see [sub:Records-Incur-A].
+
+Each free entry has the free table number in the header: less
+than 255. It also contains a doubly-linked list for easy
+deletion.
+
+3.6 <sub:TDB-Becomes-Fragmented>TDB Becomes Fragmented
+
+Much of this is a result of allocation strategy[footnote:
+The Memory Fragmentation Problem: Solved? Johnstone & Wilson 1995
+ftp://ftp.cs.utexas.edu/pub/garbage/malloc/ismm98.ps
+] and deliberate hobbling of coalescing; internal fragmentation
+(aka overallocation) is deliberately set at 25%, and external
+fragmentation is only cured by the decision to repack the entire
+db when a transaction commit needs to enlarge the file.
+
+3.6.1 Proposed Solution
+
+The 25% overhead on allocation works in practice for ldb because
+indexes tend to expand by one record at a time. This internal
+fragmentation can be resolved by having an “expanded” bit in the
+header to note entries that have previously expanded, and
+allocating more space for them.
+
+There are is a spectrum of possible solutions for external
+fragmentation: one is to use a fragmentation-avoiding allocation
+strategy such as best-fit address-order allocator. The other end
+of the spectrum would be to use a bump allocator (very fast and
+simple) and simply repack the file when we reach the end.
+
+There are three problems with efficient fragmentation-avoiding
+allocators: they are non-trivial, they tend to use a single free
+list for each size, and there's no evidence that tdb allocation
+patterns will match those recorded for general allocators (though
+it seems likely).
+
+Thus we don't spend too much effort on external fragmentation; we
+will be no worse than the current code if we need to repack on
+occasion. More effort is spent on reducing freelist contention,
+and reducing overhead.
+
+3.7 <sub:Records-Incur-A>Records Incur A 28-Byte Overhead
+
+Each TDB record has a header as follows:
+
+struct tdb_record {
+
+        tdb_off_t next; /* offset of the next record in the list
+*/
+
+        tdb_len_t rec_len; /* total byte length of record */
+
+        tdb_len_t key_len; /* byte length of key */
+
+        tdb_len_t data_len; /* byte length of data */
+
+        uint32_t full_hash; /* the full 32 bit hash of the key */
+
+        uint32_t magic;   /* try to catch errors */
+
+        /* the following union is implied:
+
+                union {
+
+                        char record[rec_len];
+
+                        struct {
+
+                                char key[key_len];
+
+                                char data[data_len];
+
+                        }
+
+                        uint32_t totalsize; (tailer)
+
+                }
+
+        */
+
+};
+
+Naively, this would double to a 56-byte overhead on a 64 bit
+implementation.
+
+3.7.1 Proposed Solution
+
+We can use various techniques to reduce this for an allocated
+block:
+
+1. The 'next' pointer is not required, as we are using a flat
+  hash table.
+
+2. 'rec_len' can instead be expressed as an addition to key_len
+  and data_len (it accounts for wasted or overallocated length in
+  the record). Since the record length is always a multiple of 8,
+  we can conveniently fit it in 32 bits (representing up to 35
+  bits).
+
+3. 'key_len' and 'data_len' can be reduced. I'm unwilling to
+  restrict 'data_len' to 32 bits, but instead we can combine the
+  two into one 64-bit field and using a 5 bit value which
+  indicates at what bit to divide the two. Keys are unlikely to
+  scale as fast as data, so I'm assuming a maximum key size of 32
+  bits.
+
+4. 'full_hash' is used to avoid a memcmp on the “miss” case, but
+  this is diminishing returns after a handful of bits (at 10
+  bits, it reduces 99.9% of false memcmp). As an aside, as the
+  lower bits are already incorporated in the hash table
+  resolution, the upper bits should be used here. Note that it's
+  not clear that these bits will be a win, given the extra bits
+  in the hash table itself (see [sub:Hash-Size-Solution]).
+
+5. 'magic' does not need to be enlarged: it currently reflects
+  one of 5 values (used, free, dead, recovery, and
+  unused_recovery). It is useful for quick sanity checking
+  however, and should not be eliminated.
+
+6. 'tailer' is only used to coalesce free blocks (so a block to
+  the right can find the header to check if this block is free).
+  This can be replaced by a single 'free' bit in the header of
+  the following block (and the tailer only exists in free
+  blocks).[footnote:
+This technique from Thomas Standish. Data Structure Techniques.
+Addison-Wesley, Reading, Massachusetts, 1980.
+] The current proposed coalescing algorithm doesn't need this,
+  however.
+
+This produces a 16 byte used header like this:
+
+struct tdb_used_record {
+
+        uint32_t used_magic : 16,
+
+
+
+                 key_data_divide: 5,
+
+                 top_hash: 11;
+
+        uint32_t extra_octets;
+
+        uint64_t key_and_data_len;
+
+};
+
+And a free record like this:
+
+struct tdb_free_record {
+
+        uint64_t free_magic: 8,
+
+                   prev : 56;
+
+
+
+        uint64_t free_table: 8,
+
+                 total_length : 56
+
+        uint64_t next;;
+
+};
+
+Note that by limiting valid offsets to 56 bits, we can pack
+everything we need into 3 64-byte words, meaning our minimum
+record size is 8 bytes.
+
+3.7.2 Status
+
+Complete.
+
+3.8 Transaction Commit Requires 4 fdatasync
+
+The current transaction algorithm is:
+
+1. write_recovery_data();
+
+2. sync();
+
+3. write_recovery_header();
+
+4. sync();
+
+5. overwrite_with_new_data();
+
+6. sync();
+
+7. remove_recovery_header();
+
+8. sync();
+
+On current ext3, each sync flushes all data to disk, so the next
+3 syncs are relatively expensive. But this could become a
+performance bottleneck on other filesystems such as ext4.
+
+3.8.1 Proposed Solution
+
+Neil Brown points out that this is overzealous, and only one sync
+is needed:
+
+1. Bundle the recovery data, a transaction counter and a strong
+  checksum of the new data.
+
+2. Strong checksum that whole bundle.
+
+3. Store the bundle in the database.
+
+4. Overwrite the oldest of the two recovery pointers in the
+  header (identified using the transaction counter) with the
+  offset of this bundle.
+
+5. sync.
+
+6. Write the new data to the file.
+
+Checking for recovery means identifying the latest bundle with a
+valid checksum and using the new data checksum to ensure that it
+has been applied. This is more expensive than the current check,
+but need only be done at open. For running databases, a separate
+header field can be used to indicate a transaction in progress;
+we need only check for recovery if this is set.
+
+3.8.2 Status
+
+Deferred.
+
+3.9 <sub:TDB-Does-Not>TDB Does Not Have Snapshot Support
+
+3.9.1 Proposed SolutionNone. At some point you say “use a real
+  database” (but see [replay-attribute]).
+
+But as a thought experiment, if we implemented transactions to
+only overwrite free entries (this is tricky: there must not be a
+header in each entry which indicates whether it is free, but use
+of presence in metadata elsewhere), and a pointer to the hash
+table, we could create an entirely new commit without destroying
+existing data. Then it would be easy to implement snapshots in a
+similar way.
+
+This would not allow arbitrary changes to the database, such as
+tdb_repack does, and would require more space (since we have to
+preserve the current and future entries at once). If we used hash
+trees rather than one big hash table, we might only have to
+rewrite some sections of the hash, too.
+
+We could then implement snapshots using a similar method, using
+multiple different hash tables/free tables.
+
+3.9.2 Status
+
+Deferred.
+
+3.10 Transactions Cannot Operate in Parallel
+
+This would be useless for ldb, as it hits the index records with
+just about every update. It would add significant complexity in
+resolving clashes, and cause the all transaction callers to write
+their code to loop in the case where the transactions spuriously
+failed.
+
+3.10.1 Proposed Solution
+
+None (but see [replay-attribute]). We could solve a small part of
+the problem by providing read-only transactions. These would
+allow one write transaction to begin, but it could not commit
+until all r/o transactions are done. This would require a new
+RO_TRANSACTION_LOCK, which would be upgraded on commit.
+
+3.10.2 Status
+
+Deferred.
+
+3.11 Default Hash Function Is Suboptimal
+
+The Knuth-inspired multiplicative hash used by tdb is fairly slow
+(especially if we expand it to 64 bits), and works best when the
+hash bucket size is a prime number (which also means a slow
+modulus). In addition, it is highly predictable which could
+potentially lead to a Denial of Service attack in some TDB uses.
+
+3.11.1 Proposed Solution
+
+The Jenkins lookup3 hash[footnote:
+http://burtleburtle.net/bob/c/lookup3.c
+] is a fast and superbly-mixing hash. It's used by the Linux
+kernel and almost everything else. This has the particular
+properties that it takes an initial seed, and produces two 32 bit
+hash numbers, which we can combine into a 64-bit hash.
+
+The seed should be created at tdb-creation time from some random
+source, and placed in the header. This is far from foolproof, but
+adds a little bit of protection against hash bombing.
+
+3.11.2 Status
+
+Complete.
+
+3.12 <Reliable-Traversal-Adds>Reliable Traversal Adds Complexity
+
+We lock a record during traversal iteration, and try to grab that
+lock in the delete code. If that grab on delete fails, we simply
+mark it deleted and continue onwards; traversal checks for this
+condition and does the delete when it moves off the record.
+
+If traversal terminates, the dead record may be left
+indefinitely.
+
+3.12.1 Proposed Solution
+
+Remove reliability guarantees; see [traverse-Proposed-Solution].
+
+3.12.2 Status
+
+Complete.
+
+3.13 Fcntl Locking Adds Overhead
+
+Placing a fcntl lock means a system call, as does removing one.
+This is actually one reason why transactions can be faster
+(everything is locked once at transaction start). In the
+uncontended case, this overhead can theoretically be eliminated.
+
+3.13.1 Proposed Solution
+
+None.
+
+We tried this before with spinlock support, in the early days of
+TDB, and it didn't make much difference except in manufactured
+benchmarks.
+
+We could use spinlocks (with futex kernel support under Linux),
+but it means that we lose automatic cleanup when a process dies
+with a lock. There is a method of auto-cleanup under Linux, but
+it's not supported by other operating systems. We could
+reintroduce a clear-if-first-style lock and sweep for dead
+futexes on open, but that wouldn't help the normal case of one
+concurrent opener dying. Increasingly elaborate repair schemes
+could be considered, but they require an ABI change (everyone
+must use them) anyway, so there's no need to do this at the same
+time as everything else.
+
+3.14 Some Transactions Don't Require Durability
+
+Volker points out that gencache uses a CLEAR_IF_FIRST tdb for
+normal (fast) usage, and occasionally empties the results into a
+transactional TDB. This kind of usage prioritizes performance
+over durability: as long as we are consistent, data can be lost.
+
+This would be more neatly implemented inside tdb: a “soft”
+transaction commit (ie. syncless) which meant that data may be
+reverted on a crash.
+
+3.14.1 Proposed Solution
+
+None.
+
+Unfortunately any transaction scheme which overwrites old data
+requires a sync before that overwrite to avoid the possibility of
+corruption.
+
+It seems possible to use a scheme similar to that described in [sub:TDB-Does-Not]
+,where transactions are committed without overwriting existing
+data, and an array of top-level pointers were available in the
+header. If the transaction is “soft” then we would not need a
+sync at all: existing processes would pick up the new hash table
+and free list and work with that.
+
+At some later point, a sync would allow recovery of the old data
+into the free lists (perhaps when the array of top-level pointers
+filled). On crash, tdb_open() would examine the array of top
+levels, and apply the transactions until it encountered an
+invalid checksum.
+
+3.15 Tracing Is Fragile, Replay Is External
+
+The current TDB has compile-time-enabled tracing code, but it
+often breaks as it is not enabled by default. In a similar way,
+the ctdb code has an external wrapper which does replay tracing
+so it can coordinate cluster-wide transactions.
+
+3.15.1 Proposed Solution<replay-attribute>
+
+Tridge points out that an attribute can be later added to
+tdb_open (see [attributes]) to provide replay/trace hooks, which
+could become the basis for this and future parallel transactions
+and snapshot support.
+
+3.15.2 Status
+
+Deferred.
diff --git a/lib/tdb2/free.c b/lib/tdb2/free.c
new file mode 100644
index 00000000000..a770751dc02
--- /dev/null
+++ b/lib/tdb2/free.c
@@ -0,0 +1,968 @@
+ /*
+   Trivial Database 2: free list/block handling
+   Copyright (C) Rusty Russell 2010
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "private.h"
+#include <ccan/likely/likely.h>
+#include <ccan/ilog/ilog.h>
+#include <time.h>
+#include <assert.h>
+#include <limits.h>
+
+static unsigned fls64(uint64_t val)
+{
+	return ilog64(val);
+}
+
+/* In which bucket would we find a particular record size? (ignoring header) */
+unsigned int size_to_bucket(tdb_len_t data_len)
+{
+	unsigned int bucket;
+
+	/* We can't have records smaller than this. */
+	assert(data_len >= TDB_MIN_DATA_LEN);
+
+	/* Ignoring the header... */
+	if (data_len - TDB_MIN_DATA_LEN <= 64) {
+		/* 0 in bucket 0, 8 in bucket 1... 64 in bucket 8. */
+		bucket = (data_len - TDB_MIN_DATA_LEN) / 8;
+	} else {
+		/* After that we go power of 2. */
+		bucket = fls64(data_len - TDB_MIN_DATA_LEN) + 2;
+	}
+
+	if (unlikely(bucket >= TDB_FREE_BUCKETS))
+		bucket = TDB_FREE_BUCKETS - 1;
+	return bucket;
+}
+
+tdb_off_t first_ftable(struct tdb_context *tdb)
+{
+	return tdb_read_off(tdb, offsetof(struct tdb_header, free_table));
+}
+
+tdb_off_t next_ftable(struct tdb_context *tdb, tdb_off_t ftable)
+{
+	return tdb_read_off(tdb, ftable + offsetof(struct tdb_freetable,next));
+}
+
+enum TDB_ERROR tdb_ftable_init(struct tdb_context *tdb)
+{
+	/* Use reservoir sampling algorithm to select a free list at random. */
+	unsigned int rnd, max = 0, count = 0;
+	tdb_off_t off;
+
+	tdb->ftable_off = off = first_ftable(tdb);
+	tdb->ftable = 0;
+
+	while (off) {
+		if (TDB_OFF_IS_ERR(off)) {
+			return off;
+		}
+
+		rnd = random();
+		if (rnd >= max) {
+			tdb->ftable_off = off;
+			tdb->ftable = count;
+			max = rnd;
+		}
+
+		off = next_ftable(tdb, off);
+		count++;
+	}
+	return TDB_SUCCESS;
+}
+
+/* Offset of a given bucket. */
+tdb_off_t bucket_off(tdb_off_t ftable_off, unsigned bucket)
+{
+	return ftable_off + offsetof(struct tdb_freetable, buckets)
+		+ bucket * sizeof(tdb_off_t);
+}
+
+/* Returns free_buckets + 1, or list number to search, or -ve error. */
+static tdb_off_t find_free_head(struct tdb_context *tdb,
+				tdb_off_t ftable_off,
+				tdb_off_t bucket)
+{
+	/* Speculatively search for a non-zero bucket. */
+	return tdb_find_nonzero_off(tdb, bucket_off(ftable_off, 0),
+				    bucket, TDB_FREE_BUCKETS);
+}
+
+static void check_list(struct tdb_context *tdb, tdb_off_t b_off)
+{
+#ifdef CCAN_TDB2_DEBUG
+	tdb_off_t off, prev = 0, first;
+	struct tdb_free_record r;
+
+	first = off = (tdb_read_off(tdb, b_off) & TDB_OFF_MASK);
+	while (off != 0) {
+		tdb_read_convert(tdb, off, &r, sizeof(r));
+		if (frec_magic(&r) != TDB_FREE_MAGIC)
+			abort();
+		if (prev && frec_prev(&r) != prev)
+			abort();
+		prev = off;
+		off = r.next;
+	}
+
+	if (first) {
+		tdb_read_convert(tdb, first, &r, sizeof(r));
+		if (frec_prev(&r) != prev)
+			abort();
+	}
+#endif
+}
+
+/* Remove from free bucket. */
+static enum TDB_ERROR remove_from_list(struct tdb_context *tdb,
+				       tdb_off_t b_off, tdb_off_t r_off,
+				       const struct tdb_free_record *r)
+{
+	tdb_off_t off, prev_next, head;
+	enum TDB_ERROR ecode;
+
+	/* Is this only element in list?  Zero out bucket, and we're done. */
+	if (frec_prev(r) == r_off)
+		return tdb_write_off(tdb, b_off, 0);
+
+	/* off = &r->prev->next */
+	off = frec_prev(r) + offsetof(struct tdb_free_record, next);
+
+	/* Get prev->next */
+	prev_next = tdb_read_off(tdb, off);
+	if (TDB_OFF_IS_ERR(prev_next))
+		return prev_next;
+
+	/* If prev->next == 0, we were head: update bucket to point to next. */
+	if (prev_next == 0) {
+		/* We must preserve upper bits. */
+		head = tdb_read_off(tdb, b_off);
+		if (TDB_OFF_IS_ERR(head))
+			return head;
+
+		if ((head & TDB_OFF_MASK) != r_off) {
+			return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+					  "remove_from_list:"
+					  " %llu head %llu on list %llu",
+					  (long long)r_off,
+					  (long long)head,
+					  (long long)b_off);
+		}
+		head = ((head & ~TDB_OFF_MASK) | r->next);
+		ecode = tdb_write_off(tdb, b_off, head);
+		if (ecode != TDB_SUCCESS)
+			return ecode;
+	} else {
+		/* r->prev->next = r->next */
+		ecode = tdb_write_off(tdb, off, r->next);
+		if (ecode != TDB_SUCCESS)
+			return ecode;
+	}
+
+	/* If we were the tail, off = &head->prev. */
+	if (r->next == 0) {
+		head = tdb_read_off(tdb, b_off);
+		if (TDB_OFF_IS_ERR(head))
+			return head;
+		head &= TDB_OFF_MASK;
+		off = head + offsetof(struct tdb_free_record, magic_and_prev);
+	} else {
+		/* off = &r->next->prev */
+		off = r->next + offsetof(struct tdb_free_record,
+					 magic_and_prev);
+	}
+
+#ifdef CCAN_TDB2_DEBUG
+	/* *off == r */
+	if ((tdb_read_off(tdb, off) & TDB_OFF_MASK) != r_off) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "remove_from_list:"
+				  " %llu bad prev in list %llu",
+				  (long long)r_off, (long long)b_off);
+	}
+#endif
+	/* r->next->prev = r->prev */
+	return tdb_write_off(tdb, off, r->magic_and_prev);
+}
+
+/* Enqueue in this free bucket: sets coalesce if we've added 128
+ * entries to it. */
+static enum TDB_ERROR enqueue_in_free(struct tdb_context *tdb,
+				      tdb_off_t b_off,
+				      tdb_off_t off,
+				      tdb_len_t len,
+				      bool *coalesce)
+{
+	struct tdb_free_record new;
+	enum TDB_ERROR ecode;
+	tdb_off_t prev, head;
+	uint64_t magic = (TDB_FREE_MAGIC << (64 - TDB_OFF_UPPER_STEAL));
+
+	head = tdb_read_off(tdb, b_off);
+	if (TDB_OFF_IS_ERR(head))
+		return head;
+
+	/* We only need to set ftable_and_len; rest is set in enqueue_in_free */
+	new.ftable_and_len = ((uint64_t)tdb->ftable << (64 - TDB_OFF_UPPER_STEAL))
+		| len;
+
+	/* new->next = head. */
+	new.next = (head & TDB_OFF_MASK);
+
+	/* First element?  Prev points to ourselves. */
+	if (!new.next) {
+		new.magic_and_prev = (magic | off);
+	} else {
+		/* new->prev = next->prev */
+		prev = tdb_read_off(tdb,
+				    new.next + offsetof(struct tdb_free_record,
+							magic_and_prev));
+		new.magic_and_prev = prev;
+		if (frec_magic(&new) != TDB_FREE_MAGIC) {
+			return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+					  "enqueue_in_free: %llu bad head"
+					  " prev %llu",
+					  (long long)new.next,
+					  (long long)prev);
+		}
+		/* next->prev = new. */
+		ecode = tdb_write_off(tdb, new.next
+				      + offsetof(struct tdb_free_record,
+						 magic_and_prev),
+				      off | magic);
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+
+#ifdef CCAN_TDB2_DEBUG
+		prev = tdb_read_off(tdb, frec_prev(&new)
+				    + offsetof(struct tdb_free_record, next));
+		if (prev != 0) {
+			return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+					  "enqueue_in_free:"
+					  " %llu bad tail next ptr %llu",
+					  (long long)frec_prev(&new)
+					  + offsetof(struct tdb_free_record,
+						     next),
+					  (long long)prev);
+		}
+#endif
+	}
+
+	/* Update enqueue count, but don't set high bit: see TDB_OFF_IS_ERR */
+	if (*coalesce)
+		head += (1ULL << (64 - TDB_OFF_UPPER_STEAL));
+	head &= ~(TDB_OFF_MASK | (1ULL << 63));
+	head |= off;
+
+	ecode = tdb_write_off(tdb, b_off, head);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	/* It's time to coalesce if counter wrapped. */
+	if (*coalesce)
+		*coalesce = ((head & ~TDB_OFF_MASK) == 0);
+
+	return tdb_write_convert(tdb, off, &new, sizeof(new));
+}
+
+static tdb_off_t ftable_offset(struct tdb_context *tdb, unsigned int ftable)
+{
+	tdb_off_t off;
+	unsigned int i;
+
+	if (likely(tdb->ftable == ftable))
+		return tdb->ftable_off;
+
+	off = first_ftable(tdb);
+	for (i = 0; i < ftable; i++) {
+		if (TDB_OFF_IS_ERR(off)) {
+			break;
+		}
+		off = next_ftable(tdb, off);
+	}
+	return off;
+}
+
+/* Note: we unlock the current bucket if fail (-ve), or coalesce (+ve) and
+ * need to blatt the *protect record (which is set to an error). */
+static tdb_len_t coalesce(struct tdb_context *tdb,
+			  tdb_off_t off, tdb_off_t b_off,
+			  tdb_len_t data_len,
+			  tdb_off_t *protect)
+{
+	tdb_off_t end;
+	struct tdb_free_record rec;
+	enum TDB_ERROR ecode;
+
+	tdb->stats.alloc_coalesce_tried++;
+	end = off + sizeof(struct tdb_used_record) + data_len;
+
+	while (end < tdb->file->map_size) {
+		const struct tdb_free_record *r;
+		tdb_off_t nb_off;
+		unsigned ftable, bucket;
+
+		r = tdb_access_read(tdb, end, sizeof(*r), true);
+		if (TDB_PTR_IS_ERR(r)) {
+			ecode = TDB_PTR_ERR(r);
+			goto err;
+		}
+
+		if (frec_magic(r) != TDB_FREE_MAGIC
+		    || frec_ftable(r) == TDB_FTABLE_NONE) {
+			tdb_access_release(tdb, r);
+			break;
+		}
+
+		ftable = frec_ftable(r);
+		bucket = size_to_bucket(frec_len(r));
+		nb_off = ftable_offset(tdb, ftable);
+		if (TDB_OFF_IS_ERR(nb_off)) {
+			tdb_access_release(tdb, r);
+			ecode = nb_off;
+			goto err;
+		}
+		nb_off = bucket_off(nb_off, bucket);
+		tdb_access_release(tdb, r);
+
+		/* We may be violating lock order here, so best effort. */
+		if (tdb_lock_free_bucket(tdb, nb_off, TDB_LOCK_NOWAIT)
+		    != TDB_SUCCESS) {
+			tdb->stats.alloc_coalesce_lockfail++;
+			break;
+		}
+
+		/* Now we have lock, re-check. */
+		ecode = tdb_read_convert(tdb, end, &rec, sizeof(rec));
+		if (ecode != TDB_SUCCESS) {
+			tdb_unlock_free_bucket(tdb, nb_off);
+			goto err;
+		}
+
+		if (unlikely(frec_magic(&rec) != TDB_FREE_MAGIC)) {
+			tdb->stats.alloc_coalesce_race++;
+			tdb_unlock_free_bucket(tdb, nb_off);
+			break;
+		}
+
+		if (unlikely(frec_ftable(&rec) != ftable)
+		    || unlikely(size_to_bucket(frec_len(&rec)) != bucket)) {
+			tdb->stats.alloc_coalesce_race++;
+			tdb_unlock_free_bucket(tdb, nb_off);
+			break;
+		}
+
+		/* Did we just mess up a record you were hoping to use? */
+		if (end == *protect) {
+			tdb->stats.alloc_coalesce_iterate_clash++;
+			*protect = TDB_ERR_NOEXIST;
+		}
+
+		ecode = remove_from_list(tdb, nb_off, end, &rec);
+		check_list(tdb, nb_off);
+		if (ecode != TDB_SUCCESS) {
+			tdb_unlock_free_bucket(tdb, nb_off);
+			goto err;
+		}
+
+		end += sizeof(struct tdb_used_record) + frec_len(&rec);
+		tdb_unlock_free_bucket(tdb, nb_off);
+		tdb->stats.alloc_coalesce_num_merged++;
+	}
+
+	/* Didn't find any adjacent free? */
+	if (end == off + sizeof(struct tdb_used_record) + data_len)
+		return 0;
+
+	/* Before we expand, check this isn't one you wanted protected? */
+	if (off == *protect) {
+		*protect = TDB_ERR_EXISTS;
+		tdb->stats.alloc_coalesce_iterate_clash++;
+	}
+
+	/* OK, expand initial record */
+	ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec));
+	if (ecode != TDB_SUCCESS) {
+		goto err;
+	}
+
+	if (frec_len(&rec) != data_len) {
+		ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				   "coalesce: expected data len %zu not %zu",
+				   (size_t)data_len, (size_t)frec_len(&rec));
+		goto err;
+	}
+
+	ecode = remove_from_list(tdb, b_off, off, &rec);
+	check_list(tdb, b_off);
+	if (ecode != TDB_SUCCESS) {
+		goto err;
+	}
+
+	/* Try locking violation first.  We don't allow coalesce recursion! */
+	ecode = add_free_record(tdb, off, end - off, TDB_LOCK_NOWAIT, false);
+	if (ecode != TDB_SUCCESS) {
+		/* Need to drop lock.  Can't rely on anything stable. */
+		tdb->stats.alloc_coalesce_lockfail++;
+		*protect = TDB_ERR_CORRUPT;
+
+		/* We have to drop this to avoid deadlocks, so make sure record
+		 * doesn't get coalesced by someone else! */
+		rec.ftable_and_len = (TDB_FTABLE_NONE
+				      << (64 - TDB_OFF_UPPER_STEAL))
+			| (end - off - sizeof(struct tdb_used_record));
+		ecode = tdb_write_off(tdb,
+				      off + offsetof(struct tdb_free_record,
+						     ftable_and_len),
+				      rec.ftable_and_len);
+		if (ecode != TDB_SUCCESS) {
+			goto err;
+		}
+
+		tdb_unlock_free_bucket(tdb, b_off);
+
+		ecode = add_free_record(tdb, off, end - off, TDB_LOCK_WAIT,
+					false);
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+	} else if (TDB_OFF_IS_ERR(*protect)) {
+		/* For simplicity, we always drop lock if they can't continue */
+		tdb_unlock_free_bucket(tdb, b_off);
+	}
+	tdb->stats.alloc_coalesce_succeeded++;
+
+	/* Return usable length. */
+	return end - off - sizeof(struct tdb_used_record);
+
+err:
+	/* To unify error paths, we *always* unlock bucket on error. */
+	tdb_unlock_free_bucket(tdb, b_off);
+	return ecode;
+}
+
+/* List is locked: we unlock it. */
+static enum TDB_ERROR coalesce_list(struct tdb_context *tdb,
+				    tdb_off_t ftable_off,
+				    tdb_off_t b_off,
+				    unsigned int limit)
+{
+	enum TDB_ERROR ecode;
+	tdb_off_t off;
+
+	off = tdb_read_off(tdb, b_off);
+	if (TDB_OFF_IS_ERR(off)) {
+		ecode = off;
+		goto unlock_err;
+	}
+	/* A little bit of paranoia: counter should be 0. */
+	off &= TDB_OFF_MASK;
+
+	while (off && limit--) {
+		struct tdb_free_record rec;
+		tdb_len_t coal;
+		tdb_off_t next;
+
+		ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		next = rec.next;
+		coal = coalesce(tdb, off, b_off, frec_len(&rec), &next);
+		if (TDB_OFF_IS_ERR(coal)) {
+			/* This has already unlocked on error. */
+			return coal;
+		}
+		if (TDB_OFF_IS_ERR(next)) {
+			/* Coalescing had to unlock, so stop. */
+			return TDB_SUCCESS;
+		}
+		/* Keep going if we're doing well... */
+		limit += size_to_bucket(coal / 16 + TDB_MIN_DATA_LEN);
+		off = next;
+	}
+
+	/* Now, move those elements to the tail of the list so we get something
+	 * else next time. */
+	if (off) {
+		struct tdb_free_record oldhrec, newhrec, oldtrec, newtrec;
+		tdb_off_t oldhoff, oldtoff, newtoff;
+
+		/* The record we were up to is the new head. */
+		ecode = tdb_read_convert(tdb, off, &newhrec, sizeof(newhrec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		/* Get the new tail. */
+		newtoff = frec_prev(&newhrec);
+		ecode = tdb_read_convert(tdb, newtoff, &newtrec,
+					 sizeof(newtrec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		/* Get the old head. */
+		oldhoff = tdb_read_off(tdb, b_off);
+		if (TDB_OFF_IS_ERR(oldhoff)) {
+			ecode = oldhoff;
+			goto unlock_err;
+		}
+
+		/* This could happen if they all coalesced away. */
+		if (oldhoff == off)
+			goto out;
+
+		ecode = tdb_read_convert(tdb, oldhoff, &oldhrec,
+					 sizeof(oldhrec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		/* Get the old tail. */
+		oldtoff = frec_prev(&oldhrec);
+		ecode = tdb_read_convert(tdb, oldtoff, &oldtrec,
+					 sizeof(oldtrec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		/* Old tail's next points to old head. */
+		oldtrec.next = oldhoff;
+
+		/* Old head's prev points to old tail. */
+		oldhrec.magic_and_prev
+			= (TDB_FREE_MAGIC << (64 - TDB_OFF_UPPER_STEAL))
+			| oldtoff;
+
+		/* New tail's next is 0. */
+		newtrec.next = 0;
+
+		/* Write out the modified versions. */
+		ecode = tdb_write_convert(tdb, oldtoff, &oldtrec,
+					  sizeof(oldtrec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		ecode = tdb_write_convert(tdb, oldhoff, &oldhrec,
+					  sizeof(oldhrec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		ecode = tdb_write_convert(tdb, newtoff, &newtrec,
+					  sizeof(newtrec));
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+
+		/* And finally link in new head. */
+		ecode = tdb_write_off(tdb, b_off, off);
+		if (ecode != TDB_SUCCESS)
+			goto unlock_err;
+	}
+out:
+	tdb_unlock_free_bucket(tdb, b_off);
+	return TDB_SUCCESS;
+
+unlock_err:
+	tdb_unlock_free_bucket(tdb, b_off);
+	return ecode;
+}
+
+/* List must not be locked if coalesce_ok is set. */
+enum TDB_ERROR add_free_record(struct tdb_context *tdb,
+			       tdb_off_t off, tdb_len_t len_with_header,
+			       enum tdb_lock_flags waitflag,
+			       bool coalesce)
+{
+	tdb_off_t b_off;
+	tdb_len_t len;
+	enum TDB_ERROR ecode;
+
+	assert(len_with_header >= sizeof(struct tdb_free_record));
+
+	len = len_with_header - sizeof(struct tdb_used_record);
+
+	b_off = bucket_off(tdb->ftable_off, size_to_bucket(len));
+	ecode = tdb_lock_free_bucket(tdb, b_off, waitflag);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	ecode = enqueue_in_free(tdb, b_off, off, len, &coalesce);
+	check_list(tdb, b_off);
+
+	/* Coalescing unlocks free list. */
+	if (!ecode && coalesce)
+		ecode = coalesce_list(tdb, tdb->ftable_off, b_off, 2);
+	else
+		tdb_unlock_free_bucket(tdb, b_off);
+	return ecode;
+}
+
+static size_t adjust_size(size_t keylen, size_t datalen)
+{
+	size_t size = keylen + datalen;
+
+	if (size < TDB_MIN_DATA_LEN)
+		size = TDB_MIN_DATA_LEN;
+
+	/* Round to next uint64_t boundary. */
+	return (size + (sizeof(uint64_t) - 1ULL)) & ~(sizeof(uint64_t) - 1ULL);
+}
+
+/* If we have enough left over to be useful, split that off. */
+static size_t record_leftover(size_t keylen, size_t datalen,
+			      bool want_extra, size_t total_len)
+{
+	ssize_t leftover;
+
+	if (want_extra)
+		datalen += datalen / 2;
+	leftover = total_len - adjust_size(keylen, datalen);
+
+	if (leftover < (ssize_t)sizeof(struct tdb_free_record))
+		return 0;
+
+	return leftover;
+}
+
+/* We need size bytes to put our key and data in. */
+static tdb_off_t lock_and_alloc(struct tdb_context *tdb,
+				tdb_off_t ftable_off,
+				tdb_off_t bucket,
+				size_t keylen, size_t datalen,
+				bool want_extra,
+				unsigned magic,
+				unsigned hashlow)
+{
+	tdb_off_t off, b_off,best_off;
+	struct tdb_free_record best = { 0 };
+	double multiplier;
+	size_t size = adjust_size(keylen, datalen);
+	enum TDB_ERROR ecode;
+
+	tdb->stats.allocs++;
+	b_off = bucket_off(ftable_off, bucket);
+
+	/* FIXME: Try non-blocking wait first, to measure contention. */
+	/* Lock this bucket. */
+	ecode = tdb_lock_free_bucket(tdb, b_off, TDB_LOCK_WAIT);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	best.ftable_and_len = -1ULL;
+	best_off = 0;
+
+	/* Get slack if we're after extra. */
+	if (want_extra)
+		multiplier = 1.5;
+	else
+		multiplier = 1.0;
+
+	/* Walk the list to see if any are large enough, getting less fussy
+	 * as we go. */
+	off = tdb_read_off(tdb, b_off);
+	if (TDB_OFF_IS_ERR(off)) {
+		ecode = off;
+		goto unlock_err;
+	}
+	off &= TDB_OFF_MASK;
+
+	while (off) {
+		const struct tdb_free_record *r;
+		tdb_len_t len;
+		tdb_off_t next;
+
+		r = tdb_access_read(tdb, off, sizeof(*r), true);
+		if (TDB_PTR_IS_ERR(r)) {
+			ecode = TDB_PTR_ERR(r);
+			goto unlock_err;
+		}
+
+		if (frec_magic(r) != TDB_FREE_MAGIC) {
+			ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+					   "lock_and_alloc:"
+					   " %llu non-free 0x%llx",
+					   (long long)off,
+					   (long long)r->magic_and_prev);
+			tdb_access_release(tdb, r);
+			goto unlock_err;
+		}
+
+		if (frec_len(r) >= size && frec_len(r) < frec_len(&best)) {
+			best_off = off;
+			best = *r;
+		}
+
+		if (frec_len(&best) <= size * multiplier && best_off) {
+			tdb_access_release(tdb, r);
+			break;
+		}
+
+		multiplier *= 1.01;
+
+		next = r->next;
+		len = frec_len(r);
+		tdb_access_release(tdb, r);
+		off = next;
+	}
+
+	/* If we found anything at all, use it. */
+	if (best_off) {
+		struct tdb_used_record rec;
+		size_t leftover;
+
+		/* We're happy with this size: take it. */
+		ecode = remove_from_list(tdb, b_off, best_off, &best);
+		check_list(tdb, b_off);
+		if (ecode != TDB_SUCCESS) {
+			goto unlock_err;
+		}
+
+		leftover = record_leftover(keylen, datalen, want_extra,
+					   frec_len(&best));
+
+		assert(keylen + datalen + leftover <= frec_len(&best));
+		/* We need to mark non-free before we drop lock, otherwise
+		 * coalesce() could try to merge it! */
+		ecode = set_header(tdb, &rec, magic, keylen, datalen,
+				   frec_len(&best) - leftover, hashlow);
+		if (ecode != TDB_SUCCESS) {
+			goto unlock_err;
+		}
+
+		ecode = tdb_write_convert(tdb, best_off, &rec, sizeof(rec));
+		if (ecode != TDB_SUCCESS) {
+			goto unlock_err;
+		}
+
+		/* For futureproofing, we put a 0 in any unused space. */
+		if (rec_extra_padding(&rec)) {
+			ecode = tdb->methods->twrite(tdb, best_off + sizeof(rec)
+						     + keylen + datalen, "", 1);
+			if (ecode != TDB_SUCCESS) {
+				goto unlock_err;
+			}
+		}
+
+		/* Bucket of leftover will be <= current bucket, so nested
+		 * locking is allowed. */
+		if (leftover) {
+			tdb->stats.alloc_leftover++;
+			ecode = add_free_record(tdb,
+						best_off + sizeof(rec)
+						+ frec_len(&best) - leftover,
+						leftover, TDB_LOCK_WAIT, false);
+			if (ecode != TDB_SUCCESS) {
+				best_off = ecode;
+			}
+		}
+		tdb_unlock_free_bucket(tdb, b_off);
+
+		return best_off;
+	}
+
+	tdb_unlock_free_bucket(tdb, b_off);
+	return 0;
+
+unlock_err:
+	tdb_unlock_free_bucket(tdb, b_off);
+	return ecode;
+}
+
+/* Get a free block from current free list, or 0 if none, -ve on error. */
+static tdb_off_t get_free(struct tdb_context *tdb,
+			  size_t keylen, size_t datalen, bool want_extra,
+			  unsigned magic, unsigned hashlow)
+{
+	tdb_off_t off, ftable_off;
+	tdb_off_t start_b, b, ftable;
+	bool wrapped = false;
+
+	/* If they are growing, add 50% to get to higher bucket. */
+	if (want_extra)
+		start_b = size_to_bucket(adjust_size(keylen,
+						     datalen + datalen / 2));
+	else
+		start_b = size_to_bucket(adjust_size(keylen, datalen));
+
+	ftable_off = tdb->ftable_off;
+	ftable = tdb->ftable;
+	while (!wrapped || ftable_off != tdb->ftable_off) {
+		/* Start at exact size bucket, and search up... */
+		for (b = find_free_head(tdb, ftable_off, start_b);
+		     b < TDB_FREE_BUCKETS;
+		     b = find_free_head(tdb, ftable_off, b + 1)) {
+			/* Try getting one from list. */
+			off = lock_and_alloc(tdb, ftable_off,
+					     b, keylen, datalen, want_extra,
+					     magic, hashlow);
+			if (TDB_OFF_IS_ERR(off))
+				return off;
+			if (off != 0) {
+				if (b == start_b)
+					tdb->stats.alloc_bucket_exact++;
+				if (b == TDB_FREE_BUCKETS - 1)
+					tdb->stats.alloc_bucket_max++;
+				/* Worked?  Stay using this list. */
+				tdb->ftable_off = ftable_off;
+				tdb->ftable = ftable;
+				return off;
+			}
+			/* Didn't work.  Try next bucket. */
+		}
+
+		if (TDB_OFF_IS_ERR(b)) {
+			return b;
+		}
+
+		/* Hmm, try next table. */
+		ftable_off = next_ftable(tdb, ftable_off);
+		if (TDB_OFF_IS_ERR(ftable_off)) {
+			return ftable_off;
+		}
+		ftable++;
+
+		if (ftable_off == 0) {
+			wrapped = true;
+			ftable_off = first_ftable(tdb);
+			if (TDB_OFF_IS_ERR(ftable_off)) {
+				return ftable_off;
+			}
+			ftable = 0;
+		}
+	}
+
+	return 0;
+}
+
+enum TDB_ERROR set_header(struct tdb_context *tdb,
+			  struct tdb_used_record *rec,
+			  unsigned magic, uint64_t keylen, uint64_t datalen,
+			  uint64_t actuallen, unsigned hashlow)
+{
+	uint64_t keybits = (fls64(keylen) + 1) / 2;
+
+	/* Use bottom bits of hash, so it's independent of hash table size. */
+	rec->magic_and_meta = (hashlow & ((1 << 11)-1))
+		| ((actuallen - (keylen + datalen)) << 11)
+		| (keybits << 43)
+		| ((uint64_t)magic << 48);
+	rec->key_and_data_len = (keylen | (datalen << (keybits*2)));
+
+	/* Encoding can fail on big values. */
+	if (rec_key_length(rec) != keylen
+	    || rec_data_length(rec) != datalen
+	    || rec_extra_padding(rec) != actuallen - (keylen + datalen)) {
+		return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				  "Could not encode k=%llu,d=%llu,a=%llu",
+				  (long long)keylen, (long long)datalen,
+				  (long long)actuallen);
+	}
+	return TDB_SUCCESS;
+}
+
+/* Expand the database. */
+static enum TDB_ERROR tdb_expand(struct tdb_context *tdb, tdb_len_t size)
+{
+	uint64_t old_size, rec_size, map_size;
+	tdb_len_t wanted;
+	enum TDB_ERROR ecode;
+
+	/* Need to hold a hash lock to expand DB: transactions rely on it. */
+	if (!(tdb->flags & TDB_NOLOCK)
+	    && !tdb->file->allrecord_lock.count && !tdb_has_hash_locks(tdb)) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_expand: must hold lock during expand");
+	}
+
+	/* Only one person can expand file at a time. */
+	ecode = tdb_lock_expand(tdb, F_WRLCK);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	/* Someone else may have expanded the file, so retry. */
+	old_size = tdb->file->map_size;
+	tdb->methods->oob(tdb, tdb->file->map_size + 1, true);
+	if (tdb->file->map_size != old_size) {
+		tdb_unlock_expand(tdb, F_WRLCK);
+		return TDB_SUCCESS;
+	}
+
+	/* limit size in order to avoid using up huge amounts of memory for
+	 * in memory tdbs if an oddball huge record creeps in */
+	if (size > 100 * 1024) {
+		rec_size = size * 2;
+	} else {
+		rec_size = size * 100;
+	}
+
+	/* always make room for at least rec_size more records, and at
+	   least 25% more space. if the DB is smaller than 100MiB,
+	   otherwise grow it by 10% only. */
+	if (old_size > 100 * 1024 * 1024) {
+		map_size = old_size / 10;
+	} else {
+		map_size = old_size / 4;
+	}
+
+	if (map_size > rec_size) {
+		wanted = map_size;
+	} else {
+		wanted = rec_size;
+	}
+
+	/* We need room for the record header too. */
+	wanted = adjust_size(0, sizeof(struct tdb_used_record) + wanted);
+
+	ecode = tdb->methods->expand_file(tdb, wanted);
+	if (ecode != TDB_SUCCESS) {
+		tdb_unlock_expand(tdb, F_WRLCK);
+		return ecode;
+	}
+
+	/* We need to drop this lock before adding free record. */
+	tdb_unlock_expand(tdb, F_WRLCK);
+
+	tdb->stats.expands++;
+	return add_free_record(tdb, old_size, wanted, TDB_LOCK_WAIT, true);
+}
+
+/* This won't fail: it will expand the database if it has to. */
+tdb_off_t alloc(struct tdb_context *tdb, size_t keylen, size_t datalen,
+		uint64_t hash, unsigned magic, bool growing)
+{
+	tdb_off_t off;
+
+	/* We can't hold pointers during this: we could unmap! */
+	assert(!tdb->direct_access);
+
+	for (;;) {
+		enum TDB_ERROR ecode;
+		off = get_free(tdb, keylen, datalen, growing, magic, hash);
+		if (likely(off != 0))
+			break;
+
+		ecode = tdb_expand(tdb, adjust_size(keylen, datalen));
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+	}
+
+	return off;
+}
diff --git a/lib/tdb2/hash.c b/lib/tdb2/hash.c
new file mode 100644
index 00000000000..1359cfecd66
--- /dev/null
+++ b/lib/tdb2/hash.c
@@ -0,0 +1,881 @@
+ /*
+   Trivial Database 2: hash handling
+   Copyright (C) Rusty Russell 2010
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "private.h"
+#include <assert.h>
+
+uint64_t tdb_hash(struct tdb_context *tdb, const void *ptr, size_t len)
+{
+	return tdb->hash_fn(ptr, len, tdb->hash_seed, tdb->hash_data);
+}
+
+uint64_t hash_record(struct tdb_context *tdb, tdb_off_t off)
+{
+	const struct tdb_used_record *r;
+	const void *key;
+	uint64_t klen, hash;
+
+	r = tdb_access_read(tdb, off, sizeof(*r), true);
+	if (TDB_PTR_IS_ERR(r)) {
+		/* FIXME */
+		return 0;
+	}
+
+	klen = rec_key_length(r);
+	tdb_access_release(tdb, r);
+
+	key = tdb_access_read(tdb, off + sizeof(*r), klen, false);
+	if (TDB_PTR_IS_ERR(key)) {
+		return 0;
+	}
+
+	hash = tdb_hash(tdb, key, klen);
+	tdb_access_release(tdb, key);
+	return hash;
+}
+
+/* Get bits from a value. */
+static uint32_t bits_from(uint64_t val, unsigned start, unsigned num)
+{
+	assert(num <= 32);
+	return (val >> start) & ((1U << num) - 1);
+}
+
+/* We take bits from the top: that way we can lock whole sections of the hash
+ * by using lock ranges. */
+static uint32_t use_bits(struct hash_info *h, unsigned num)
+{
+	h->hash_used += num;
+	return bits_from(h->h, 64 - h->hash_used, num);
+}
+
+static tdb_bool_err key_matches(struct tdb_context *tdb,
+				const struct tdb_used_record *rec,
+				tdb_off_t off,
+				const struct tdb_data *key)
+{
+	tdb_bool_err ret = false;
+	const char *rkey;
+
+	if (rec_key_length(rec) != key->dsize) {
+		tdb->stats.compare_wrong_keylen++;
+		return ret;
+	}
+
+	rkey = tdb_access_read(tdb, off + sizeof(*rec), key->dsize, false);
+	if (TDB_PTR_IS_ERR(rkey)) {
+		return TDB_PTR_ERR(rkey);
+	}
+	if (memcmp(rkey, key->dptr, key->dsize) == 0)
+		ret = true;
+	else
+		tdb->stats.compare_wrong_keycmp++;
+	tdb_access_release(tdb, rkey);
+	return ret;
+}
+
+/* Does entry match? */
+static tdb_bool_err match(struct tdb_context *tdb,
+			  struct hash_info *h,
+			  const struct tdb_data *key,
+			  tdb_off_t val,
+			  struct tdb_used_record *rec)
+{
+	tdb_off_t off;
+	enum TDB_ERROR ecode;
+
+	tdb->stats.compares++;
+	/* Desired bucket must match. */
+	if (h->home_bucket != (val & TDB_OFF_HASH_GROUP_MASK)) {
+		tdb->stats.compare_wrong_bucket++;
+		return false;
+	}
+
+	/* Top bits of offset == next bits of hash. */
+	if (bits_from(val, TDB_OFF_HASH_EXTRA_BIT, TDB_OFF_UPPER_STEAL_EXTRA)
+	    != bits_from(h->h, 64 - h->hash_used - TDB_OFF_UPPER_STEAL_EXTRA,
+		    TDB_OFF_UPPER_STEAL_EXTRA)) {
+		tdb->stats.compare_wrong_offsetbits++;
+		return false;
+	}
+
+	off = val & TDB_OFF_MASK;
+	ecode = tdb_read_convert(tdb, off, rec, sizeof(*rec));
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if ((h->h & ((1 << 11)-1)) != rec_hash(rec)) {
+		tdb->stats.compare_wrong_rechash++;
+		return false;
+	}
+
+	return key_matches(tdb, rec, off, key);
+}
+
+static tdb_off_t hbucket_off(tdb_off_t group_start, unsigned bucket)
+{
+	return group_start
+		+ (bucket % (1 << TDB_HASH_GROUP_BITS)) * sizeof(tdb_off_t);
+}
+
+bool is_subhash(tdb_off_t val)
+{
+	return (val >> TDB_OFF_UPPER_STEAL_SUBHASH_BIT) & 1;
+}
+
+/* FIXME: Guess the depth, don't over-lock! */
+static tdb_off_t hlock_range(tdb_off_t group, tdb_off_t *size)
+{
+	*size = 1ULL << (64 - (TDB_TOPLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS));
+	return group << (64 - (TDB_TOPLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS));
+}
+
+static tdb_off_t COLD find_in_chain(struct tdb_context *tdb,
+				    struct tdb_data key,
+				    tdb_off_t chain,
+				    struct hash_info *h,
+				    struct tdb_used_record *rec,
+				    struct traverse_info *tinfo)
+{
+	tdb_off_t off, next;
+	enum TDB_ERROR ecode;
+
+	/* In case nothing is free, we set these to zero. */
+	h->home_bucket = h->found_bucket = 0;
+
+	for (off = chain; off; off = next) {
+		unsigned int i;
+
+		h->group_start = off;
+		ecode = tdb_read_convert(tdb, off, h->group, sizeof(h->group));
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+
+		for (i = 0; i < (1 << TDB_HASH_GROUP_BITS); i++) {
+			tdb_off_t recoff;
+			if (!h->group[i]) {
+				/* Remember this empty bucket. */
+				h->home_bucket = h->found_bucket = i;
+				continue;
+			}
+
+			/* We can insert extra bits via add_to_hash
+			 * empty bucket logic. */
+			recoff = h->group[i] & TDB_OFF_MASK;
+			ecode = tdb_read_convert(tdb, recoff, rec,
+						 sizeof(*rec));
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+
+			ecode = key_matches(tdb, rec, recoff, &key);
+			if (ecode < 0) {
+				return ecode;
+			}
+			if (ecode == 1) {
+				h->home_bucket = h->found_bucket = i;
+
+				if (tinfo) {
+					tinfo->levels[tinfo->num_levels]
+						.hashtable = off;
+					tinfo->levels[tinfo->num_levels]
+						.total_buckets
+						= 1 << TDB_HASH_GROUP_BITS;
+					tinfo->levels[tinfo->num_levels].entry
+						= i;
+					tinfo->num_levels++;
+				}
+				return recoff;
+			}
+		}
+		next = tdb_read_off(tdb, off
+				    + offsetof(struct tdb_chain, next));
+		if (TDB_OFF_IS_ERR(next)) {
+			return next;
+		}
+		if (next)
+			next += sizeof(struct tdb_used_record);
+	}
+	return 0;
+}
+
+/* This is the core routine which searches the hashtable for an entry.
+ * On error, no locks are held and -ve is returned.
+ * Otherwise, hinfo is filled in (and the optional tinfo).
+ * If not found, the return value is 0.
+ * If found, the return value is the offset, and *rec is the record. */
+tdb_off_t find_and_lock(struct tdb_context *tdb,
+			struct tdb_data key,
+			int ltype,
+			struct hash_info *h,
+			struct tdb_used_record *rec,
+			struct traverse_info *tinfo)
+{
+	uint32_t i, group;
+	tdb_off_t hashtable;
+	enum TDB_ERROR ecode;
+
+	h->h = tdb_hash(tdb, key.dptr, key.dsize);
+	h->hash_used = 0;
+	group = use_bits(h, TDB_TOPLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS);
+	h->home_bucket = use_bits(h, TDB_HASH_GROUP_BITS);
+
+	h->hlock_start = hlock_range(group, &h->hlock_range);
+	ecode = tdb_lock_hashes(tdb, h->hlock_start, h->hlock_range, ltype,
+				TDB_LOCK_WAIT);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	hashtable = offsetof(struct tdb_header, hashtable);
+	if (tinfo) {
+		tinfo->toplevel_group = group;
+		tinfo->num_levels = 1;
+		tinfo->levels[0].entry = 0;
+		tinfo->levels[0].hashtable = hashtable
+			+ (group << TDB_HASH_GROUP_BITS) * sizeof(tdb_off_t);
+		tinfo->levels[0].total_buckets = 1 << TDB_HASH_GROUP_BITS;
+	}
+
+	while (h->hash_used <= 64) {
+		/* Read in the hash group. */
+		h->group_start = hashtable
+			+ group * (sizeof(tdb_off_t) << TDB_HASH_GROUP_BITS);
+
+		ecode = tdb_read_convert(tdb, h->group_start, &h->group,
+					 sizeof(h->group));
+		if (ecode != TDB_SUCCESS) {
+			goto fail;
+		}
+
+		/* Pointer to another hash table?  Go down... */
+		if (is_subhash(h->group[h->home_bucket])) {
+			hashtable = (h->group[h->home_bucket] & TDB_OFF_MASK)
+				+ sizeof(struct tdb_used_record);
+			if (tinfo) {
+				/* When we come back, use *next* bucket */
+				tinfo->levels[tinfo->num_levels-1].entry
+					+= h->home_bucket + 1;
+			}
+			group = use_bits(h, TDB_SUBLEVEL_HASH_BITS
+					 - TDB_HASH_GROUP_BITS);
+			h->home_bucket = use_bits(h, TDB_HASH_GROUP_BITS);
+			if (tinfo) {
+				tinfo->levels[tinfo->num_levels].hashtable
+					= hashtable;
+				tinfo->levels[tinfo->num_levels].total_buckets
+					= 1 << TDB_SUBLEVEL_HASH_BITS;
+				tinfo->levels[tinfo->num_levels].entry
+					= group << TDB_HASH_GROUP_BITS;
+				tinfo->num_levels++;
+			}
+			continue;
+		}
+
+		/* It's in this group: search (until 0 or all searched) */
+		for (i = 0, h->found_bucket = h->home_bucket;
+		     i < (1 << TDB_HASH_GROUP_BITS);
+		     i++, h->found_bucket = ((h->found_bucket+1)
+					     % (1 << TDB_HASH_GROUP_BITS))) {
+			tdb_bool_err berr;
+			if (is_subhash(h->group[h->found_bucket]))
+				continue;
+
+			if (!h->group[h->found_bucket])
+				break;
+
+			berr = match(tdb, h, &key, h->group[h->found_bucket],
+				     rec);
+			if (berr < 0) {
+				ecode = berr;
+				goto fail;
+			}
+			if (berr) {
+				if (tinfo) {
+					tinfo->levels[tinfo->num_levels-1].entry
+						+= h->found_bucket;
+				}
+				return h->group[h->found_bucket] & TDB_OFF_MASK;
+			}
+		}
+		/* Didn't find it: h indicates where it would go. */
+		return 0;
+	}
+
+	return find_in_chain(tdb, key, hashtable, h, rec, tinfo);
+
+fail:
+	tdb_unlock_hashes(tdb, h->hlock_start, h->hlock_range, ltype);
+	return ecode;
+}
+
+/* I wrote a simple test, expanding a hash to 2GB, for the following
+ * cases:
+ * 1) Expanding all the buckets at once,
+ * 2) Expanding the bucket we wanted to place the new entry into.
+ * 3) Expanding the most-populated bucket,
+ *
+ * I measured the worst/average/best density during this process.
+ * 1) 3%/16%/30%
+ * 2) 4%/20%/38%
+ * 3) 6%/22%/41%
+ *
+ * So we figure out the busiest bucket for the moment.
+ */
+static unsigned fullest_bucket(struct tdb_context *tdb,
+			       const tdb_off_t *group,
+			       unsigned new_bucket)
+{
+	unsigned counts[1 << TDB_HASH_GROUP_BITS] = { 0 };
+	unsigned int i, best_bucket;
+
+	/* Count the new entry. */
+	counts[new_bucket]++;
+	best_bucket = new_bucket;
+
+	for (i = 0; i < (1 << TDB_HASH_GROUP_BITS); i++) {
+		unsigned this_bucket;
+
+		if (is_subhash(group[i]))
+			continue;
+		this_bucket = group[i] & TDB_OFF_HASH_GROUP_MASK;
+		if (++counts[this_bucket] > counts[best_bucket])
+			best_bucket = this_bucket;
+	}
+
+	return best_bucket;
+}
+
+static bool put_into_group(tdb_off_t *group,
+			   unsigned bucket, tdb_off_t encoded)
+{
+	unsigned int i;
+
+	for (i = 0; i < (1 << TDB_HASH_GROUP_BITS); i++) {
+		unsigned b = (bucket + i) % (1 << TDB_HASH_GROUP_BITS);
+
+		if (group[b] == 0) {
+			group[b] = encoded;
+			return true;
+		}
+	}
+	return false;
+}
+
+static void force_into_group(tdb_off_t *group,
+			     unsigned bucket, tdb_off_t encoded)
+{
+	if (!put_into_group(group, bucket, encoded))
+		abort();
+}
+
+static tdb_off_t encode_offset(tdb_off_t new_off, struct hash_info *h)
+{
+	return h->home_bucket
+		| new_off
+		| ((uint64_t)bits_from(h->h,
+				  64 - h->hash_used - TDB_OFF_UPPER_STEAL_EXTRA,
+				  TDB_OFF_UPPER_STEAL_EXTRA)
+		   << TDB_OFF_HASH_EXTRA_BIT);
+}
+
+/* Simply overwrite the hash entry we found before. */
+enum TDB_ERROR replace_in_hash(struct tdb_context *tdb,
+			       struct hash_info *h,
+			       tdb_off_t new_off)
+{
+	return tdb_write_off(tdb, hbucket_off(h->group_start, h->found_bucket),
+			     encode_offset(new_off, h));
+}
+
+/* We slot in anywhere that's empty in the chain. */
+static enum TDB_ERROR COLD add_to_chain(struct tdb_context *tdb,
+					tdb_off_t subhash,
+					tdb_off_t new_off)
+{
+	tdb_off_t entry;
+	enum TDB_ERROR ecode;
+
+	entry = tdb_find_zero_off(tdb, subhash, 1<<TDB_HASH_GROUP_BITS);
+	if (TDB_OFF_IS_ERR(entry)) {
+		return entry;
+	}
+
+	if (entry == 1 << TDB_HASH_GROUP_BITS) {
+		tdb_off_t next;
+
+		next = tdb_read_off(tdb, subhash
+				    + offsetof(struct tdb_chain, next));
+		if (TDB_OFF_IS_ERR(next)) {
+			return next;
+		}
+
+		if (!next) {
+			next = alloc(tdb, 0, sizeof(struct tdb_chain), 0,
+				     TDB_CHAIN_MAGIC, false);
+			if (TDB_OFF_IS_ERR(next))
+				return next;
+			ecode = zero_out(tdb,
+					 next+sizeof(struct tdb_used_record),
+					 sizeof(struct tdb_chain));
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+			ecode = tdb_write_off(tdb, subhash
+					      + offsetof(struct tdb_chain,
+							 next),
+					      next);
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+		}
+		return add_to_chain(tdb, next, new_off);
+	}
+
+	return tdb_write_off(tdb, subhash + entry * sizeof(tdb_off_t),
+			     new_off);
+}
+
+/* Add into a newly created subhash. */
+static enum TDB_ERROR add_to_subhash(struct tdb_context *tdb, tdb_off_t subhash,
+				     unsigned hash_used, tdb_off_t val)
+{
+	tdb_off_t off = (val & TDB_OFF_MASK), *group;
+	struct hash_info h;
+	unsigned int gnum;
+
+	h.hash_used = hash_used;
+
+	if (hash_used + TDB_SUBLEVEL_HASH_BITS > 64)
+		return add_to_chain(tdb, subhash, off);
+
+	h.h = hash_record(tdb, off);
+	gnum = use_bits(&h, TDB_SUBLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS);
+	h.group_start = subhash
+		+ gnum * (sizeof(tdb_off_t) << TDB_HASH_GROUP_BITS);
+	h.home_bucket = use_bits(&h, TDB_HASH_GROUP_BITS);
+
+	group = tdb_access_write(tdb, h.group_start,
+				 sizeof(*group) << TDB_HASH_GROUP_BITS, true);
+	if (TDB_PTR_IS_ERR(group)) {
+		return TDB_PTR_ERR(group);
+	}
+	force_into_group(group, h.home_bucket, encode_offset(off, &h));
+	return tdb_access_commit(tdb, group);
+}
+
+static enum TDB_ERROR expand_group(struct tdb_context *tdb, struct hash_info *h)
+{
+	unsigned bucket, num_vals, i, magic;
+	size_t subsize;
+	tdb_off_t subhash;
+	tdb_off_t vals[1 << TDB_HASH_GROUP_BITS];
+	enum TDB_ERROR ecode;
+
+	/* Attach new empty subhash under fullest bucket. */
+	bucket = fullest_bucket(tdb, h->group, h->home_bucket);
+
+	if (h->hash_used == 64) {
+		tdb->stats.alloc_chain++;
+		subsize = sizeof(struct tdb_chain);
+		magic = TDB_CHAIN_MAGIC;
+	} else {
+		tdb->stats.alloc_subhash++;
+		subsize = (sizeof(tdb_off_t) << TDB_SUBLEVEL_HASH_BITS);
+		magic = TDB_HTABLE_MAGIC;
+	}
+
+	subhash = alloc(tdb, 0, subsize, 0, magic, false);
+	if (TDB_OFF_IS_ERR(subhash)) {
+		return subhash;
+	}
+
+	ecode = zero_out(tdb, subhash + sizeof(struct tdb_used_record),
+			 subsize);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	/* Remove any which are destined for bucket or are in wrong place. */
+	num_vals = 0;
+	for (i = 0; i < (1 << TDB_HASH_GROUP_BITS); i++) {
+		unsigned home_bucket = h->group[i] & TDB_OFF_HASH_GROUP_MASK;
+		if (!h->group[i] || is_subhash(h->group[i]))
+			continue;
+		if (home_bucket == bucket || home_bucket != i) {
+			vals[num_vals++] = h->group[i];
+			h->group[i] = 0;
+		}
+	}
+	/* FIXME: This assert is valid, but we do this during unit test :( */
+	/* assert(num_vals); */
+
+	/* Overwrite expanded bucket with subhash pointer. */
+	h->group[bucket] = subhash | (1ULL << TDB_OFF_UPPER_STEAL_SUBHASH_BIT);
+
+	/* Point to actual contents of record. */
+	subhash += sizeof(struct tdb_used_record);
+
+	/* Put values back. */
+	for (i = 0; i < num_vals; i++) {
+		unsigned this_bucket = vals[i] & TDB_OFF_HASH_GROUP_MASK;
+
+		if (this_bucket == bucket) {
+			ecode = add_to_subhash(tdb, subhash, h->hash_used,
+					       vals[i]);
+			if (ecode != TDB_SUCCESS)
+				return ecode;
+		} else {
+			/* There should be room to put this back. */
+			force_into_group(h->group, this_bucket, vals[i]);
+		}
+	}
+	return TDB_SUCCESS;
+}
+
+enum TDB_ERROR delete_from_hash(struct tdb_context *tdb, struct hash_info *h)
+{
+	unsigned int i, num_movers = 0;
+	tdb_off_t movers[1 << TDB_HASH_GROUP_BITS];
+
+	h->group[h->found_bucket] = 0;
+	for (i = 1; i < (1 << TDB_HASH_GROUP_BITS); i++) {
+		unsigned this_bucket;
+
+		this_bucket = (h->found_bucket+i) % (1 << TDB_HASH_GROUP_BITS);
+		/* Empty bucket?  We're done. */
+		if (!h->group[this_bucket])
+			break;
+
+		/* Ignore subhashes. */
+		if (is_subhash(h->group[this_bucket]))
+			continue;
+
+		/* If this one is not happy where it is, we'll move it. */
+		if ((h->group[this_bucket] & TDB_OFF_HASH_GROUP_MASK)
+		    != this_bucket) {
+			movers[num_movers++] = h->group[this_bucket];
+			h->group[this_bucket] = 0;
+		}
+	}
+
+	/* Put back the ones we erased. */
+	for (i = 0; i < num_movers; i++) {
+		force_into_group(h->group, movers[i] & TDB_OFF_HASH_GROUP_MASK,
+				 movers[i]);
+	}
+
+	/* Now we write back the hash group */
+	return tdb_write_convert(tdb, h->group_start,
+				 h->group, sizeof(h->group));
+}
+
+enum TDB_ERROR add_to_hash(struct tdb_context *tdb, struct hash_info *h,
+			   tdb_off_t new_off)
+{
+	enum TDB_ERROR ecode;
+
+	/* We hit an empty bucket during search?  That's where it goes. */
+	if (!h->group[h->found_bucket]) {
+		h->group[h->found_bucket] = encode_offset(new_off, h);
+		/* Write back the modified group. */
+		return tdb_write_convert(tdb, h->group_start,
+					 h->group, sizeof(h->group));
+	}
+
+	if (h->hash_used > 64)
+		return add_to_chain(tdb, h->group_start, new_off);
+
+	/* We're full.  Expand. */
+	ecode = expand_group(tdb, h);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (is_subhash(h->group[h->home_bucket])) {
+		/* We were expanded! */
+		tdb_off_t hashtable;
+		unsigned int gnum;
+
+		/* Write back the modified group. */
+		ecode = tdb_write_convert(tdb, h->group_start, h->group,
+					  sizeof(h->group));
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+
+		/* Move hashinfo down a level. */
+		hashtable = (h->group[h->home_bucket] & TDB_OFF_MASK)
+			+ sizeof(struct tdb_used_record);
+		gnum = use_bits(h,TDB_SUBLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS);
+		h->home_bucket = use_bits(h, TDB_HASH_GROUP_BITS);
+		h->group_start = hashtable
+			+ gnum * (sizeof(tdb_off_t) << TDB_HASH_GROUP_BITS);
+		ecode = tdb_read_convert(tdb, h->group_start, &h->group,
+					 sizeof(h->group));
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+	}
+
+	/* Expanding the group must have made room if it didn't choose this
+	 * bucket. */
+	if (put_into_group(h->group, h->home_bucket, encode_offset(new_off,h))){
+		return tdb_write_convert(tdb, h->group_start,
+					 h->group, sizeof(h->group));
+	}
+
+	/* This can happen if all hashes in group (and us) dropped into same
+	 * group in subhash. */
+	return add_to_hash(tdb, h, new_off);
+}
+
+/* Traverse support: returns offset of record, or 0 or -ve error. */
+static tdb_off_t iterate_hash(struct tdb_context *tdb,
+			      struct traverse_info *tinfo)
+{
+	tdb_off_t off, val, i;
+	struct traverse_level *tlevel;
+
+	tlevel = &tinfo->levels[tinfo->num_levels-1];
+
+again:
+	for (i = tdb_find_nonzero_off(tdb, tlevel->hashtable,
+				      tlevel->entry, tlevel->total_buckets);
+	     i != tlevel->total_buckets;
+	     i = tdb_find_nonzero_off(tdb, tlevel->hashtable,
+				      i+1, tlevel->total_buckets)) {
+		if (TDB_OFF_IS_ERR(i)) {
+			return i;
+		}
+
+		val = tdb_read_off(tdb, tlevel->hashtable+sizeof(tdb_off_t)*i);
+		if (TDB_OFF_IS_ERR(val)) {
+			return val;
+		}
+
+		off = val & TDB_OFF_MASK;
+
+		/* This makes the delete-all-in-traverse case work
+		 * (and simplifies our logic a little). */
+		if (off == tinfo->prev)
+			continue;
+
+		tlevel->entry = i;
+
+		if (!is_subhash(val)) {
+			/* Found one. */
+			tinfo->prev = off;
+			return off;
+		}
+
+		/* When we come back, we want the next one */
+		tlevel->entry++;
+		tinfo->num_levels++;
+		tlevel++;
+		tlevel->hashtable = off + sizeof(struct tdb_used_record);
+		tlevel->entry = 0;
+		/* Next level is a chain? */
+		if (unlikely(tinfo->num_levels == TDB_MAX_LEVELS + 1))
+			tlevel->total_buckets = (1 << TDB_HASH_GROUP_BITS);
+		else
+			tlevel->total_buckets = (1 << TDB_SUBLEVEL_HASH_BITS);
+		goto again;
+	}
+
+	/* Nothing there? */
+	if (tinfo->num_levels == 1)
+		return 0;
+
+	/* Handle chained entries. */
+	if (unlikely(tinfo->num_levels == TDB_MAX_LEVELS + 1)) {
+		tlevel->hashtable = tdb_read_off(tdb, tlevel->hashtable
+						 + offsetof(struct tdb_chain,
+							    next));
+		if (TDB_OFF_IS_ERR(tlevel->hashtable)) {
+			return tlevel->hashtable;
+		}
+		if (tlevel->hashtable) {
+			tlevel->hashtable += sizeof(struct tdb_used_record);
+			tlevel->entry = 0;
+			goto again;
+		}
+	}
+
+	/* Go back up and keep searching. */
+	tinfo->num_levels--;
+	tlevel--;
+	goto again;
+}
+
+/* Return success if we find something, TDB_ERR_NOEXIST if none. */
+enum TDB_ERROR next_in_hash(struct tdb_context *tdb,
+			    struct traverse_info *tinfo,
+			    TDB_DATA *kbuf, size_t *dlen)
+{
+	const unsigned group_bits = TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS;
+	tdb_off_t hl_start, hl_range, off;
+	enum TDB_ERROR ecode;
+
+	while (tinfo->toplevel_group < (1 << group_bits)) {
+		hl_start = (tdb_off_t)tinfo->toplevel_group
+			<< (64 - group_bits);
+		hl_range = 1ULL << group_bits;
+		ecode = tdb_lock_hashes(tdb, hl_start, hl_range, F_RDLCK,
+					TDB_LOCK_WAIT);
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+
+		off = iterate_hash(tdb, tinfo);
+		if (off) {
+			struct tdb_used_record rec;
+
+			if (TDB_OFF_IS_ERR(off)) {
+				ecode = off;
+				goto fail;
+			}
+
+			ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec));
+			if (ecode != TDB_SUCCESS) {
+				goto fail;
+			}
+			if (rec_magic(&rec) != TDB_USED_MAGIC) {
+				ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT,
+						   TDB_LOG_ERROR,
+						   "next_in_hash:"
+						   " corrupt record at %llu",
+						   (long long)off);
+				goto fail;
+			}
+
+			kbuf->dsize = rec_key_length(&rec);
+
+			/* They want data as well? */
+			if (dlen) {
+				*dlen = rec_data_length(&rec);
+				kbuf->dptr = tdb_alloc_read(tdb,
+							    off + sizeof(rec),
+							    kbuf->dsize
+							    + *dlen);
+			} else {
+				kbuf->dptr = tdb_alloc_read(tdb,
+							    off + sizeof(rec),
+							    kbuf->dsize);
+			}
+			tdb_unlock_hashes(tdb, hl_start, hl_range, F_RDLCK);
+			if (TDB_PTR_IS_ERR(kbuf->dptr)) {
+				return TDB_PTR_ERR(kbuf->dptr);
+			}
+			return TDB_SUCCESS;
+		}
+
+		tdb_unlock_hashes(tdb, hl_start, hl_range, F_RDLCK);
+
+		tinfo->toplevel_group++;
+		tinfo->levels[0].hashtable
+			+= (sizeof(tdb_off_t) << TDB_HASH_GROUP_BITS);
+		tinfo->levels[0].entry = 0;
+	}
+	return TDB_ERR_NOEXIST;
+
+fail:
+	tdb_unlock_hashes(tdb, hl_start, hl_range, F_RDLCK);
+	return ecode;
+
+}
+
+enum TDB_ERROR first_in_hash(struct tdb_context *tdb,
+			     struct traverse_info *tinfo,
+			     TDB_DATA *kbuf, size_t *dlen)
+{
+	tinfo->prev = 0;
+	tinfo->toplevel_group = 0;
+	tinfo->num_levels = 1;
+	tinfo->levels[0].hashtable = offsetof(struct tdb_header, hashtable);
+	tinfo->levels[0].entry = 0;
+	tinfo->levels[0].total_buckets = (1 << TDB_HASH_GROUP_BITS);
+
+	return next_in_hash(tdb, tinfo, kbuf, dlen);
+}
+
+/* Even if the entry isn't in this hash bucket, you'd have to lock this
+ * bucket to find it. */
+static enum TDB_ERROR chainlock(struct tdb_context *tdb, const TDB_DATA *key,
+				int ltype, enum tdb_lock_flags waitflag,
+				const char *func)
+{
+	enum TDB_ERROR ecode;
+	uint64_t h = tdb_hash(tdb, key->dptr, key->dsize);
+	tdb_off_t lockstart, locksize;
+	unsigned int group, gbits;
+
+	gbits = TDB_TOPLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS;
+	group = bits_from(h, 64 - gbits, gbits);
+
+	lockstart = hlock_range(group, &locksize);
+
+	ecode = tdb_lock_hashes(tdb, lockstart, locksize, ltype, waitflag);
+	tdb_trace_1rec(tdb, func, *key);
+	return ecode;
+}
+
+/* lock/unlock one hash chain. This is meant to be used to reduce
+   contention - it cannot guarantee how many records will be locked */
+enum TDB_ERROR tdb_chainlock(struct tdb_context *tdb, TDB_DATA key)
+{
+	return tdb->last_error = chainlock(tdb, &key, F_WRLCK, TDB_LOCK_WAIT,
+					   "tdb_chainlock");
+}
+
+void tdb_chainunlock(struct tdb_context *tdb, TDB_DATA key)
+{
+	uint64_t h = tdb_hash(tdb, key.dptr, key.dsize);
+	tdb_off_t lockstart, locksize;
+	unsigned int group, gbits;
+
+	gbits = TDB_TOPLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS;
+	group = bits_from(h, 64 - gbits, gbits);
+
+	lockstart = hlock_range(group, &locksize);
+
+	tdb_trace_1rec(tdb, "tdb_chainunlock", key);
+	tdb_unlock_hashes(tdb, lockstart, locksize, F_WRLCK);
+}
+
+enum TDB_ERROR tdb_chainlock_read(struct tdb_context *tdb, TDB_DATA key)
+{
+	return tdb->last_error = chainlock(tdb, &key, F_RDLCK, TDB_LOCK_WAIT,
+					   "tdb_chainlock_read");
+}
+
+void tdb_chainunlock_read(struct tdb_context *tdb, TDB_DATA key)
+{
+	uint64_t h = tdb_hash(tdb, key.dptr, key.dsize);
+	tdb_off_t lockstart, locksize;
+	unsigned int group, gbits;
+
+	gbits = TDB_TOPLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS;
+	group = bits_from(h, 64 - gbits, gbits);
+
+	lockstart = hlock_range(group, &locksize);
+
+	tdb_trace_1rec(tdb, "tdb_chainunlock_read", key);
+	tdb_unlock_hashes(tdb, lockstart, locksize, F_RDLCK);
+}
diff --git a/lib/tdb2/io.c b/lib/tdb2/io.c
new file mode 100644
index 00000000000..8c5f45f3082
--- /dev/null
+++ b/lib/tdb2/io.c
@@ -0,0 +1,615 @@
+ /*
+   Unix SMB/CIFS implementation.
+
+   trivial database library
+
+   Copyright (C) Andrew Tridgell              1999-2005
+   Copyright (C) Paul `Rusty' Russell		   2000
+   Copyright (C) Jeremy Allison			   2000-2003
+   Copyright (C) Rusty Russell			   2010
+
+     ** NOTE! The following LGPL license applies to the tdb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "private.h"
+#include <assert.h>
+#include <ccan/likely/likely.h>
+
+void tdb_munmap(struct tdb_file *file)
+{
+	if (file->fd == -1)
+		return;
+
+	if (file->map_ptr) {
+		munmap(file->map_ptr, file->map_size);
+		file->map_ptr = NULL;
+	}
+}
+
+void tdb_mmap(struct tdb_context *tdb)
+{
+	if (tdb->flags & TDB_INTERNAL)
+		return;
+
+	if (tdb->flags & TDB_NOMMAP)
+		return;
+
+	/* size_t can be smaller than off_t. */
+	if ((size_t)tdb->file->map_size == tdb->file->map_size) {
+		tdb->file->map_ptr = mmap(NULL, tdb->file->map_size,
+					  tdb->mmap_flags,
+					  MAP_SHARED, tdb->file->fd, 0);
+	} else
+		tdb->file->map_ptr = MAP_FAILED;
+
+	/*
+	 * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!!
+	 */
+	if (tdb->file->map_ptr == MAP_FAILED) {
+		tdb->file->map_ptr = NULL;
+		tdb_logerr(tdb, TDB_SUCCESS, TDB_LOG_WARNING,
+			   "tdb_mmap failed for size %lld (%s)",
+			   (long long)tdb->file->map_size, strerror(errno));
+	}
+}
+
+/* check for an out of bounds access - if it is out of bounds then
+   see if the database has been expanded by someone else and expand
+   if necessary
+   note that "len" is the minimum length needed for the db
+*/
+static enum TDB_ERROR tdb_oob(struct tdb_context *tdb, tdb_off_t len,
+			      bool probe)
+{
+	struct stat st;
+	enum TDB_ERROR ecode;
+
+	/* We can't hold pointers during this: we could unmap! */
+	assert(!tdb->direct_access
+	       || (tdb->flags & TDB_NOLOCK)
+	       || tdb_has_expansion_lock(tdb));
+
+	if (len <= tdb->file->map_size)
+		return 0;
+	if (tdb->flags & TDB_INTERNAL) {
+		if (!probe) {
+			tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				 "tdb_oob len %lld beyond internal"
+				 " malloc size %lld",
+				 (long long)len,
+				 (long long)tdb->file->map_size);
+		}
+		return TDB_ERR_IO;
+	}
+
+	ecode = tdb_lock_expand(tdb, F_RDLCK);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (fstat(tdb->file->fd, &st) != 0) {
+		tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+			   "Failed to fstat file: %s", strerror(errno));
+		tdb_unlock_expand(tdb, F_RDLCK);
+		return TDB_ERR_IO;
+	}
+
+	tdb_unlock_expand(tdb, F_RDLCK);
+
+	if (st.st_size < (size_t)len) {
+		if (!probe) {
+			tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				   "tdb_oob len %zu beyond eof at %zu",
+				   (size_t)len, st.st_size);
+		}
+		return TDB_ERR_IO;
+	}
+
+	/* Unmap, update size, remap */
+	tdb_munmap(tdb->file);
+
+	tdb->file->map_size = st.st_size;
+	tdb_mmap(tdb);
+	return TDB_SUCCESS;
+}
+
+/* Endian conversion: we only ever deal with 8 byte quantities */
+void *tdb_convert(const struct tdb_context *tdb, void *buf, tdb_len_t size)
+{
+	assert(size % 8 == 0);
+	if (unlikely((tdb->flags & TDB_CONVERT)) && buf) {
+		uint64_t i, *p = (uint64_t *)buf;
+		for (i = 0; i < size / 8; i++)
+			p[i] = bswap_64(p[i]);
+	}
+	return buf;
+}
+
+/* Return first non-zero offset in offset array, or end, or -ve error. */
+/* FIXME: Return the off? */
+uint64_t tdb_find_nonzero_off(struct tdb_context *tdb,
+			      tdb_off_t base, uint64_t start, uint64_t end)
+{
+	uint64_t i;
+	const uint64_t *val;
+
+	/* Zero vs non-zero is the same unconverted: minor optimization. */
+	val = tdb_access_read(tdb, base + start * sizeof(tdb_off_t),
+			      (end - start) * sizeof(tdb_off_t), false);
+	if (TDB_PTR_IS_ERR(val)) {
+		return TDB_PTR_ERR(val);
+	}
+
+	for (i = 0; i < (end - start); i++) {
+		if (val[i])
+			break;
+	}
+	tdb_access_release(tdb, val);
+	return start + i;
+}
+
+/* Return first zero offset in num offset array, or num, or -ve error. */
+uint64_t tdb_find_zero_off(struct tdb_context *tdb, tdb_off_t off,
+			   uint64_t num)
+{
+	uint64_t i;
+	const uint64_t *val;
+
+	/* Zero vs non-zero is the same unconverted: minor optimization. */
+	val = tdb_access_read(tdb, off, num * sizeof(tdb_off_t), false);
+	if (TDB_PTR_IS_ERR(val)) {
+		return TDB_PTR_ERR(val);
+	}
+
+	for (i = 0; i < num; i++) {
+		if (!val[i])
+			break;
+	}
+	tdb_access_release(tdb, val);
+	return i;
+}
+
+enum TDB_ERROR zero_out(struct tdb_context *tdb, tdb_off_t off, tdb_len_t len)
+{
+	char buf[8192] = { 0 };
+	void *p = tdb->methods->direct(tdb, off, len, true);
+	enum TDB_ERROR ecode = TDB_SUCCESS;
+
+	assert(!tdb->read_only);
+	if (TDB_PTR_IS_ERR(p)) {
+		return TDB_PTR_ERR(p);
+	}
+	if (p) {
+		memset(p, 0, len);
+		return ecode;
+	}
+	while (len) {
+		unsigned todo = len < sizeof(buf) ? len : sizeof(buf);
+		ecode = tdb->methods->twrite(tdb, off, buf, todo);
+		if (ecode != TDB_SUCCESS) {
+			break;
+		}
+		len -= todo;
+		off += todo;
+	}
+	return ecode;
+}
+
+tdb_off_t tdb_read_off(struct tdb_context *tdb, tdb_off_t off)
+{
+	tdb_off_t ret;
+	enum TDB_ERROR ecode;
+
+	if (likely(!(tdb->flags & TDB_CONVERT))) {
+		tdb_off_t *p = tdb->methods->direct(tdb, off, sizeof(*p),
+						    false);
+		if (TDB_PTR_IS_ERR(p)) {
+			return TDB_PTR_ERR(p);
+		}
+		if (p)
+			return *p;
+	}
+
+	ecode = tdb_read_convert(tdb, off, &ret, sizeof(ret));
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+	return ret;
+}
+
+/* write a lump of data at a specified offset */
+static enum TDB_ERROR tdb_write(struct tdb_context *tdb, tdb_off_t off,
+				const void *buf, tdb_len_t len)
+{
+	enum TDB_ERROR ecode;
+
+	if (tdb->read_only) {
+		return tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
+				  "Write to read-only database");
+	}
+
+	ecode = tdb->methods->oob(tdb, off + len, 0);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (tdb->file->map_ptr) {
+		memcpy(off + (char *)tdb->file->map_ptr, buf, len);
+	} else {
+		ssize_t ret;
+		ret = pwrite(tdb->file->fd, buf, len, off);
+		if (ret != len) {
+			/* This shouldn't happen: we avoid sparse files. */
+			if (ret >= 0)
+				errno = ENOSPC;
+
+			return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+					  "tdb_write: %zi at %zu len=%zu (%s)",
+					  ret, (size_t)off, (size_t)len,
+					  strerror(errno));
+		}
+	}
+	return TDB_SUCCESS;
+}
+
+/* read a lump of data at a specified offset */
+static enum TDB_ERROR tdb_read(struct tdb_context *tdb, tdb_off_t off,
+			       void *buf, tdb_len_t len)
+{
+	enum TDB_ERROR ecode;
+
+	ecode = tdb->methods->oob(tdb, off + len, 0);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (tdb->file->map_ptr) {
+		memcpy(buf, off + (char *)tdb->file->map_ptr, len);
+	} else {
+		ssize_t r = pread(tdb->file->fd, buf, len, off);
+		if (r != len) {
+			return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+					  "tdb_read failed with %zi at %zu "
+					  "len=%zu (%s) map_size=%zu",
+					  r, (size_t)off, (size_t)len,
+					  strerror(errno),
+					  (size_t)tdb->file->map_size);
+		}
+	}
+	return TDB_SUCCESS;
+}
+
+enum TDB_ERROR tdb_write_convert(struct tdb_context *tdb, tdb_off_t off,
+				 const void *rec, size_t len)
+{
+	enum TDB_ERROR ecode;
+
+	if (unlikely((tdb->flags & TDB_CONVERT))) {
+		void *conv = malloc(len);
+		if (!conv) {
+			return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+					  "tdb_write: no memory converting"
+					  " %zu bytes", len);
+		}
+		memcpy(conv, rec, len);
+		ecode = tdb->methods->twrite(tdb, off,
+					   tdb_convert(tdb, conv, len), len);
+		free(conv);
+	} else {
+		ecode = tdb->methods->twrite(tdb, off, rec, len);
+	}
+	return ecode;
+}
+
+enum TDB_ERROR tdb_read_convert(struct tdb_context *tdb, tdb_off_t off,
+				void *rec, size_t len)
+{
+	enum TDB_ERROR ecode = tdb->methods->tread(tdb, off, rec, len);
+	tdb_convert(tdb, rec, len);
+	return ecode;
+}
+
+enum TDB_ERROR tdb_write_off(struct tdb_context *tdb,
+			     tdb_off_t off, tdb_off_t val)
+{
+	if (tdb->read_only) {
+		return tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
+				  "Write to read-only database");
+	}
+
+	if (likely(!(tdb->flags & TDB_CONVERT))) {
+		tdb_off_t *p = tdb->methods->direct(tdb, off, sizeof(*p),
+						    true);
+		if (TDB_PTR_IS_ERR(p)) {
+			return TDB_PTR_ERR(p);
+		}
+		if (p) {
+			*p = val;
+			return TDB_SUCCESS;
+		}
+	}
+	return tdb_write_convert(tdb, off, &val, sizeof(val));
+}
+
+static void *_tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset,
+			     tdb_len_t len, unsigned int prefix)
+{
+	unsigned char *buf;
+	enum TDB_ERROR ecode;
+
+	/* some systems don't like zero length malloc */
+	buf = malloc(prefix + len ? prefix + len : 1);
+	if (!buf) {
+		tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_USE_ERROR,
+			   "tdb_alloc_read malloc failed len=%zu",
+			   (size_t)(prefix + len));
+		return TDB_ERR_PTR(TDB_ERR_OOM);
+	} else {
+		ecode = tdb->methods->tread(tdb, offset, buf+prefix, len);
+		if (unlikely(ecode != TDB_SUCCESS)) {
+			free(buf);
+			return TDB_ERR_PTR(ecode);
+		}
+	}
+	return buf;
+}
+
+/* read a lump of data, allocating the space for it */
+void *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len_t len)
+{
+	return _tdb_alloc_read(tdb, offset, len, 0);
+}
+
+static enum TDB_ERROR fill(struct tdb_context *tdb,
+			   const void *buf, size_t size,
+			   tdb_off_t off, tdb_len_t len)
+{
+	while (len) {
+		size_t n = len > size ? size : len;
+		ssize_t ret = pwrite(tdb->file->fd, buf, n, off);
+		if (ret != n) {
+			if (ret >= 0)
+				errno = ENOSPC;
+
+			return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+					  "fill failed:"
+					  " %zi at %zu len=%zu (%s)",
+					  ret, (size_t)off, (size_t)len,
+					  strerror(errno));
+		}
+		len -= n;
+		off += n;
+	}
+	return TDB_SUCCESS;
+}
+
+/* expand a file.  we prefer to use ftruncate, as that is what posix
+  says to use for mmap expansion */
+static enum TDB_ERROR tdb_expand_file(struct tdb_context *tdb,
+				      tdb_len_t addition)
+{
+	char buf[8192];
+	enum TDB_ERROR ecode;
+
+	if (tdb->read_only) {
+		return tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
+				  "Expand on read-only database");
+	}
+
+	if (tdb->flags & TDB_INTERNAL) {
+		char *new = realloc(tdb->file->map_ptr,
+				    tdb->file->map_size + addition);
+		if (!new) {
+			return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+					  "No memory to expand database");
+		}
+		tdb->file->map_ptr = new;
+		tdb->file->map_size += addition;
+	} else {
+		/* Unmap before trying to write; old TDB claimed OpenBSD had
+		 * problem with this otherwise. */
+		tdb_munmap(tdb->file);
+
+		/* If this fails, we try to fill anyway. */
+		if (ftruncate(tdb->file->fd, tdb->file->map_size + addition))
+			;
+
+		/* now fill the file with something. This ensures that the
+		   file isn't sparse, which would be very bad if we ran out of
+		   disk. This must be done with write, not via mmap */
+		memset(buf, 0x43, sizeof(buf));
+		ecode = fill(tdb, buf, sizeof(buf), tdb->file->map_size,
+			     addition);
+		if (ecode != TDB_SUCCESS)
+			return ecode;
+		tdb->file->map_size += addition;
+		tdb_mmap(tdb);
+	}
+	return TDB_SUCCESS;
+}
+
+const void *tdb_access_read(struct tdb_context *tdb,
+			    tdb_off_t off, tdb_len_t len, bool convert)
+{
+	void *ret = NULL;
+
+	if (likely(!(tdb->flags & TDB_CONVERT))) {
+		ret = tdb->methods->direct(tdb, off, len, false);
+
+		if (TDB_PTR_IS_ERR(ret)) {
+			return ret;
+		}
+	}
+	if (!ret) {
+		struct tdb_access_hdr *hdr;
+		hdr = _tdb_alloc_read(tdb, off, len, sizeof(*hdr));
+		if (TDB_PTR_IS_ERR(hdr)) {
+			return hdr;
+		}
+		hdr->next = tdb->access;
+		tdb->access = hdr;
+		ret = hdr + 1;
+		if (convert) {
+			tdb_convert(tdb, (void *)ret, len);
+		}
+	} else
+		tdb->direct_access++;
+
+	return ret;
+}
+
+void *tdb_access_write(struct tdb_context *tdb,
+		       tdb_off_t off, tdb_len_t len, bool convert)
+{
+	void *ret = NULL;
+
+	if (tdb->read_only) {
+		tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
+			   "Write to read-only database");
+		return TDB_ERR_PTR(TDB_ERR_RDONLY);
+	}
+
+	if (likely(!(tdb->flags & TDB_CONVERT))) {
+		ret = tdb->methods->direct(tdb, off, len, true);
+
+		if (TDB_PTR_IS_ERR(ret)) {
+			return ret;
+		}
+	}
+
+	if (!ret) {
+		struct tdb_access_hdr *hdr;
+		hdr = _tdb_alloc_read(tdb, off, len, sizeof(*hdr));
+		if (TDB_PTR_IS_ERR(hdr)) {
+			return hdr;
+		}
+		hdr->next = tdb->access;
+		tdb->access = hdr;
+		hdr->off = off;
+		hdr->len = len;
+		hdr->convert = convert;
+		ret = hdr + 1;
+		if (convert)
+			tdb_convert(tdb, (void *)ret, len);
+	} else
+		tdb->direct_access++;
+
+	return ret;
+}
+
+static struct tdb_access_hdr **find_hdr(struct tdb_context *tdb, const void *p)
+{
+	struct tdb_access_hdr **hp;
+
+	for (hp = &tdb->access; *hp; hp = &(*hp)->next) {
+		if (*hp + 1 == p)
+			return hp;
+	}
+	return NULL;
+}
+
+void tdb_access_release(struct tdb_context *tdb, const void *p)
+{
+	struct tdb_access_hdr *hdr, **hp = find_hdr(tdb, p);
+
+	if (hp) {
+		hdr = *hp;
+		*hp = hdr->next;
+		free(hdr);
+	} else
+		tdb->direct_access--;
+}
+
+enum TDB_ERROR tdb_access_commit(struct tdb_context *tdb, void *p)
+{
+	struct tdb_access_hdr *hdr, **hp = find_hdr(tdb, p);
+	enum TDB_ERROR ecode;
+
+	if (hp) {
+		hdr = *hp;
+		if (hdr->convert)
+			ecode = tdb_write_convert(tdb, hdr->off, p, hdr->len);
+		else
+			ecode = tdb_write(tdb, hdr->off, p, hdr->len);
+		*hp = hdr->next;
+		free(hdr);
+	} else {
+		tdb->direct_access--;
+		ecode = TDB_SUCCESS;
+	}
+
+	return ecode;
+}
+
+static void *tdb_direct(struct tdb_context *tdb, tdb_off_t off, size_t len,
+			bool write_mode)
+{
+	enum TDB_ERROR ecode;
+
+	if (unlikely(!tdb->file->map_ptr))
+		return NULL;
+
+	ecode = tdb_oob(tdb, off + len, true);
+	if (unlikely(ecode != TDB_SUCCESS))
+		return TDB_ERR_PTR(ecode);
+	return (char *)tdb->file->map_ptr + off;
+}
+
+void tdb_inc_seqnum(struct tdb_context *tdb)
+{
+	tdb_off_t seq;
+
+	if (likely(!(tdb->flags & TDB_CONVERT))) {
+		int64_t *direct;
+
+		direct = tdb->methods->direct(tdb,
+					      offsetof(struct tdb_header,
+						       seqnum),
+					      sizeof(*direct), true);
+		if (likely(direct)) {
+			/* Don't let it go negative, even briefly */
+			if (unlikely((*direct) + 1) < 0)
+				*direct = 0;
+			(*direct)++;
+			return;
+		}
+	}
+
+	seq = tdb_read_off(tdb, offsetof(struct tdb_header, seqnum));
+	if (!TDB_OFF_IS_ERR(seq)) {
+		seq++;
+		if (unlikely((int64_t)seq < 0))
+			seq = 0;
+		tdb_write_off(tdb, offsetof(struct tdb_header, seqnum), seq);
+	}
+}
+
+static const struct tdb_methods io_methods = {
+	tdb_read,
+	tdb_write,
+	tdb_oob,
+	tdb_expand_file,
+	tdb_direct,
+};
+
+/*
+  initialise the default methods table
+*/
+void tdb_io_init(struct tdb_context *tdb)
+{
+	tdb->methods = &io_methods;
+}
diff --git a/lib/tdb2/lock.c b/lib/tdb2/lock.c
new file mode 100644
index 00000000000..76b8bc31579
--- /dev/null
+++ b/lib/tdb2/lock.c
@@ -0,0 +1,875 @@
+ /*
+   Unix SMB/CIFS implementation.
+
+   trivial database library
+
+   Copyright (C) Andrew Tridgell              1999-2005
+   Copyright (C) Paul `Rusty' Russell		   2000
+   Copyright (C) Jeremy Allison			   2000-2003
+
+     ** NOTE! The following LGPL license applies to the tdb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "private.h"
+#include <assert.h>
+#include <ccan/build_assert/build_assert.h>
+
+/* If we were threaded, we could wait for unlock, but we're not, so fail. */
+static enum TDB_ERROR owner_conflict(struct tdb_context *tdb, const char *call)
+{
+	return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_USE_ERROR,
+			  "%s: lock owned by another tdb in this process.",
+			  call);
+}
+
+/* If we fork, we no longer really own locks. */
+static bool check_lock_pid(struct tdb_context *tdb,
+			   const char *call, bool log)
+{
+	/* No locks?  No problem! */
+	if (tdb->file->allrecord_lock.count == 0
+	    && tdb->file->num_lockrecs == 0) {
+		return true;
+	}
+
+	/* No fork?  No problem! */
+	if (tdb->file->locker == getpid()) {
+		return true;
+	}
+
+	if (log) {
+		tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_USE_ERROR,
+			   "%s: fork() detected after lock acquisition!"
+			   " (%u vs %u)", call, tdb->file->locker, getpid());
+	}
+	return false;
+}
+
+int tdb_fcntl_lock(int fd, int rw, off_t off, off_t len, bool waitflag,
+		   void *unused)
+{
+	struct flock fl;
+	int ret;
+
+	do {
+		fl.l_type = rw;
+		fl.l_whence = SEEK_SET;
+		fl.l_start = off;
+		fl.l_len = len;
+
+		if (waitflag)
+			ret = fcntl(fd, F_SETLKW, &fl);
+		else
+			ret = fcntl(fd, F_SETLK, &fl);
+	} while (ret != 0 && errno == EINTR);
+	return ret;
+}
+
+int tdb_fcntl_unlock(int fd, int rw, off_t off, off_t len, void *unused)
+{
+	struct flock fl;
+	int ret;
+
+	do {
+		fl.l_type = F_UNLCK;
+		fl.l_whence = SEEK_SET;
+		fl.l_start = off;
+		fl.l_len = len;
+
+		ret = fcntl(fd, F_SETLKW, &fl);
+	} while (ret != 0 && errno == EINTR);
+	return ret;
+}
+
+static int lock(struct tdb_context *tdb,
+		      int rw, off_t off, off_t len, bool waitflag)
+{
+	int ret;
+	if (tdb->file->allrecord_lock.count == 0
+	    && tdb->file->num_lockrecs == 0) {
+		tdb->file->locker = getpid();
+	}
+
+	tdb->stats.lock_lowlevel++;
+	ret = tdb->lock_fn(tdb->file->fd, rw, off, len, waitflag,
+			   tdb->lock_data);
+	if (!waitflag) {
+		tdb->stats.lock_nonblock++;
+		if (ret != 0)
+			tdb->stats.lock_nonblock_fail++;
+	}
+	return ret;
+}
+
+static int unlock(struct tdb_context *tdb, int rw, off_t off, off_t len)
+{
+#if 0 /* Check they matched up locks and unlocks correctly. */
+	char line[80];
+	FILE *locks;
+	bool found = false;
+
+	locks = fopen("/proc/locks", "r");
+
+	while (fgets(line, 80, locks)) {
+		char *p;
+		int type, start, l;
+
+		/* eg. 1: FLOCK  ADVISORY  WRITE 2440 08:01:2180826 0 EOF */
+		p = strchr(line, ':') + 1;
+		if (strncmp(p, " POSIX  ADVISORY  ", strlen(" POSIX  ADVISORY  ")))
+			continue;
+		p += strlen(" FLOCK  ADVISORY  ");
+		if (strncmp(p, "READ  ", strlen("READ  ")) == 0)
+			type = F_RDLCK;
+		else if (strncmp(p, "WRITE ", strlen("WRITE ")) == 0)
+			type = F_WRLCK;
+		else
+			abort();
+		p += 6;
+		if (atoi(p) != getpid())
+			continue;
+		p = strchr(strchr(p, ' ') + 1, ' ') + 1;
+		start = atoi(p);
+		p = strchr(p, ' ') + 1;
+		if (strncmp(p, "EOF", 3) == 0)
+			l = 0;
+		else
+			l = atoi(p) - start + 1;
+
+		if (off == start) {
+			if (len != l) {
+				fprintf(stderr, "Len %u should be %u: %s",
+					(int)len, l, line);
+				abort();
+			}
+			if (type != rw) {
+				fprintf(stderr, "Type %s wrong: %s",
+					rw == F_RDLCK ? "READ" : "WRITE", line);
+				abort();
+			}
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		fprintf(stderr, "Unlock on %u@%u not found!",
+			(int)off, (int)len);
+		abort();
+	}
+
+	fclose(locks);
+#endif
+
+	return tdb->unlock_fn(tdb->file->fd, rw, off, len, tdb->lock_data);
+}
+
+/* a byte range locking function - return 0 on success
+   this functions locks len bytes at the specified offset.
+
+   note that a len of zero means lock to end of file
+*/
+static enum TDB_ERROR tdb_brlock(struct tdb_context *tdb,
+				 int rw_type, tdb_off_t offset, tdb_off_t len,
+				 enum tdb_lock_flags flags)
+{
+	int ret;
+
+	if (tdb->flags & TDB_NOLOCK) {
+		return TDB_SUCCESS;
+	}
+
+	if (rw_type == F_WRLCK && tdb->read_only) {
+		return tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
+				  "Write lock attempted on read-only database");
+	}
+
+	/* A 32 bit system cannot open a 64-bit file, but it could have
+	 * expanded since then: check here. */
+	if ((size_t)(offset + len) != offset + len) {
+		return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				  "tdb_brlock: lock on giant offset %llu",
+				  (long long)(offset + len));
+	}
+
+	ret = lock(tdb, rw_type, offset, len, flags & TDB_LOCK_WAIT);
+	if (ret != 0) {
+		/* Generic lock error. errno set by fcntl.
+		 * EAGAIN is an expected return from non-blocking
+		 * locks. */
+		if (!(flags & TDB_LOCK_PROBE)
+		    && (errno != EAGAIN && errno != EINTR)) {
+			tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				   "tdb_brlock failed (fd=%d) at"
+				   " offset %zu rw_type=%d flags=%d len=%zu:"
+				   " %s",
+				   tdb->file->fd, (size_t)offset, rw_type,
+				   flags, (size_t)len, strerror(errno));
+		}
+		return TDB_ERR_LOCK;
+	}
+	return TDB_SUCCESS;
+}
+
+static enum TDB_ERROR tdb_brunlock(struct tdb_context *tdb,
+				   int rw_type, tdb_off_t offset, size_t len)
+{
+	if (tdb->flags & TDB_NOLOCK) {
+		return TDB_SUCCESS;
+	}
+
+	if (!check_lock_pid(tdb, "tdb_brunlock", true))
+		return TDB_ERR_LOCK;
+
+	if (unlock(tdb, rw_type, offset, len) == -1) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_brunlock failed (fd=%d) at offset %zu"
+				  " rw_type=%d len=%zu: %s",
+				  tdb->file->fd, (size_t)offset, rw_type,
+				  (size_t)len, strerror(errno));
+	}
+	return TDB_SUCCESS;
+}
+
+/*
+  upgrade a read lock to a write lock. This needs to be handled in a
+  special way as some OSes (such as solaris) have too conservative
+  deadlock detection and claim a deadlock when progress can be
+  made. For those OSes we may loop for a while.
+*/
+enum TDB_ERROR tdb_allrecord_upgrade(struct tdb_context *tdb)
+{
+	int count = 1000;
+
+	if (!check_lock_pid(tdb, "tdb_transaction_prepare_commit", true))
+		return TDB_ERR_LOCK;
+
+	if (tdb->file->allrecord_lock.count != 1) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_allrecord_upgrade failed:"
+				  " count %u too high",
+				  tdb->file->allrecord_lock.count);
+	}
+
+	if (tdb->file->allrecord_lock.off != 1) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_allrecord_upgrade failed:"
+				  " already upgraded?");
+	}
+
+	if (tdb->file->allrecord_lock.owner != tdb) {
+		return owner_conflict(tdb, "tdb_allrecord_upgrade");
+	}
+
+	while (count--) {
+		struct timeval tv;
+		if (tdb_brlock(tdb, F_WRLCK,
+			       TDB_HASH_LOCK_START, 0,
+			       TDB_LOCK_WAIT|TDB_LOCK_PROBE) == TDB_SUCCESS) {
+			tdb->file->allrecord_lock.ltype = F_WRLCK;
+			tdb->file->allrecord_lock.off = 0;
+			return TDB_SUCCESS;
+		}
+		if (errno != EDEADLK) {
+			break;
+		}
+		/* sleep for as short a time as we can - more portable than usleep() */
+		tv.tv_sec = 0;
+		tv.tv_usec = 1;
+		select(0, NULL, NULL, NULL, &tv);
+	}
+
+	if (errno != EAGAIN && errno != EINTR)
+		tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+			   "tdb_allrecord_upgrade failed");
+	return TDB_ERR_LOCK;
+}
+
+static struct tdb_lock *find_nestlock(struct tdb_context *tdb, tdb_off_t offset,
+				      const struct tdb_context *owner)
+{
+	unsigned int i;
+
+	for (i=0; i<tdb->file->num_lockrecs; i++) {
+		if (tdb->file->lockrecs[i].off == offset) {
+			if (owner && tdb->file->lockrecs[i].owner != owner)
+				return NULL;
+			return &tdb->file->lockrecs[i];
+		}
+	}
+	return NULL;
+}
+
+enum TDB_ERROR tdb_lock_and_recover(struct tdb_context *tdb)
+{
+	enum TDB_ERROR ecode;
+
+	if (!check_lock_pid(tdb, "tdb_transaction_prepare_commit", true))
+		return TDB_ERR_LOCK;
+
+	ecode = tdb_allrecord_lock(tdb, F_WRLCK, TDB_LOCK_WAIT|TDB_LOCK_NOCHECK,
+				   false);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	ecode = tdb_lock_open(tdb, F_WRLCK, TDB_LOCK_WAIT|TDB_LOCK_NOCHECK);
+	if (ecode != TDB_SUCCESS) {
+		tdb_allrecord_unlock(tdb, F_WRLCK);
+		return ecode;
+	}
+	ecode = tdb_transaction_recover(tdb);
+	tdb_unlock_open(tdb, F_WRLCK);
+	tdb_allrecord_unlock(tdb, F_WRLCK);
+
+	return ecode;
+}
+
+/* lock an offset in the database. */
+static enum TDB_ERROR tdb_nest_lock(struct tdb_context *tdb,
+				    tdb_off_t offset, int ltype,
+				    enum tdb_lock_flags flags)
+{
+	struct tdb_lock *new_lck;
+	enum TDB_ERROR ecode;
+
+	if (offset > (TDB_HASH_LOCK_START + TDB_HASH_LOCK_RANGE
+		      + tdb->file->map_size / 8)) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_nest_lock: invalid offset %zu ltype=%d",
+				  (size_t)offset, ltype);
+	}
+
+	if (tdb->flags & TDB_NOLOCK)
+		return TDB_SUCCESS;
+
+	if (!check_lock_pid(tdb, "tdb_nest_lock", true)) {
+		return TDB_ERR_LOCK;
+	}
+
+	tdb->stats.locks++;
+
+	new_lck = find_nestlock(tdb, offset, NULL);
+	if (new_lck) {
+		if (new_lck->owner != tdb) {
+			return owner_conflict(tdb, "tdb_nest_lock");
+		}
+
+		if (new_lck->ltype == F_RDLCK && ltype == F_WRLCK) {
+			return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+					  "tdb_nest_lock:"
+					  " offset %zu has read lock",
+					  (size_t)offset);
+		}
+		/* Just increment the struct, posix locks don't stack. */
+		new_lck->count++;
+		return TDB_SUCCESS;
+	}
+
+#if 0
+	if (tdb->file->num_lockrecs
+	    && offset >= TDB_HASH_LOCK_START
+	    && offset < TDB_HASH_LOCK_START + TDB_HASH_LOCK_RANGE) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_nest_lock: already have a hash lock?");
+	}
+#endif
+
+	new_lck = (struct tdb_lock *)realloc(
+		tdb->file->lockrecs,
+		sizeof(*tdb->file->lockrecs) * (tdb->file->num_lockrecs+1));
+	if (new_lck == NULL) {
+		return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+				  "tdb_nest_lock:"
+				  " unable to allocate %zu lock struct",
+				  tdb->file->num_lockrecs + 1);
+	}
+	tdb->file->lockrecs = new_lck;
+
+	/* Since fcntl locks don't nest, we do a lock for the first one,
+	   and simply bump the count for future ones */
+	ecode = tdb_brlock(tdb, ltype, offset, 1, flags);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	/* First time we grab a lock, perhaps someone died in commit? */
+	if (!(flags & TDB_LOCK_NOCHECK)
+	    && tdb->file->num_lockrecs == 0) {
+		tdb_bool_err berr = tdb_needs_recovery(tdb);
+		if (berr != false) {
+			tdb_brunlock(tdb, ltype, offset, 1);
+
+			if (berr < 0)
+				return berr;
+			ecode = tdb_lock_and_recover(tdb);
+			if (ecode == TDB_SUCCESS) {
+				ecode = tdb_brlock(tdb, ltype, offset, 1,
+						   flags);
+			}
+			if (ecode != TDB_SUCCESS) {
+				return ecode;
+			}
+		}
+	}
+
+	tdb->file->lockrecs[tdb->file->num_lockrecs].owner = tdb;
+	tdb->file->lockrecs[tdb->file->num_lockrecs].off = offset;
+	tdb->file->lockrecs[tdb->file->num_lockrecs].count = 1;
+	tdb->file->lockrecs[tdb->file->num_lockrecs].ltype = ltype;
+	tdb->file->num_lockrecs++;
+
+	return TDB_SUCCESS;
+}
+
+static enum TDB_ERROR tdb_nest_unlock(struct tdb_context *tdb,
+				      tdb_off_t off, int ltype)
+{
+	struct tdb_lock *lck;
+	enum TDB_ERROR ecode;
+
+	if (tdb->flags & TDB_NOLOCK)
+		return TDB_SUCCESS;
+
+	lck = find_nestlock(tdb, off, tdb);
+	if ((lck == NULL) || (lck->count == 0)) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_nest_unlock: no lock for %zu",
+				  (size_t)off);
+	}
+
+	if (lck->count > 1) {
+		lck->count--;
+		return TDB_SUCCESS;
+	}
+
+	/*
+	 * This lock has count==1 left, so we need to unlock it in the
+	 * kernel. We don't bother with decrementing the in-memory array
+	 * element, we're about to overwrite it with the last array element
+	 * anyway.
+	 */
+	ecode = tdb_brunlock(tdb, ltype, off, 1);
+
+	/*
+	 * Shrink the array by overwriting the element just unlocked with the
+	 * last array element.
+	 */
+	*lck = tdb->file->lockrecs[--tdb->file->num_lockrecs];
+
+	return ecode;
+}
+
+/*
+  get the transaction lock
+ */
+enum TDB_ERROR tdb_transaction_lock(struct tdb_context *tdb, int ltype)
+{
+	return tdb_nest_lock(tdb, TDB_TRANSACTION_LOCK, ltype, TDB_LOCK_WAIT);
+}
+
+/*
+  release the transaction lock
+ */
+void tdb_transaction_unlock(struct tdb_context *tdb, int ltype)
+{
+	tdb_nest_unlock(tdb, TDB_TRANSACTION_LOCK, ltype);
+}
+
+/* We only need to lock individual bytes, but Linux merges consecutive locks
+ * so we lock in contiguous ranges. */
+static enum TDB_ERROR tdb_lock_gradual(struct tdb_context *tdb,
+				       int ltype, enum tdb_lock_flags flags,
+				       tdb_off_t off, tdb_off_t len)
+{
+	enum TDB_ERROR ecode;
+	enum tdb_lock_flags nb_flags = (flags & ~TDB_LOCK_WAIT);
+
+	if (len <= 1) {
+		/* 0 would mean to end-of-file... */
+		assert(len != 0);
+		/* Single hash.  Just do blocking lock. */
+		return tdb_brlock(tdb, ltype, off, len, flags);
+	}
+
+	/* First we try non-blocking. */
+	if (tdb_brlock(tdb, ltype, off, len, nb_flags) == TDB_SUCCESS) {
+		return TDB_SUCCESS;
+	}
+
+	/* Try locking first half, then second. */
+	ecode = tdb_lock_gradual(tdb, ltype, flags, off, len / 2);
+	if (ecode != TDB_SUCCESS)
+		return ecode;
+
+	ecode = tdb_lock_gradual(tdb, ltype, flags,
+				 off + len / 2, len - len / 2);
+	if (ecode != TDB_SUCCESS) {
+		tdb_brunlock(tdb, ltype, off, len / 2);
+	}
+	return ecode;
+}
+
+/* lock/unlock entire database.  It can only be upgradable if you have some
+ * other way of guaranteeing exclusivity (ie. transaction write lock). */
+enum TDB_ERROR tdb_allrecord_lock(struct tdb_context *tdb, int ltype,
+				  enum tdb_lock_flags flags, bool upgradable)
+{
+	enum TDB_ERROR ecode;
+	tdb_bool_err berr;
+
+	if (tdb->flags & TDB_NOLOCK)
+		return TDB_SUCCESS;
+
+	if (!check_lock_pid(tdb, "tdb_allrecord_lock", true)) {
+		return TDB_ERR_LOCK;
+	}
+
+	if (tdb->file->allrecord_lock.count) {
+		if (tdb->file->allrecord_lock.owner != tdb) {
+			return owner_conflict(tdb, "tdb_allrecord_lock");
+		}
+
+		if (ltype == F_RDLCK
+		    || tdb->file->allrecord_lock.ltype == F_WRLCK) {
+			tdb->file->allrecord_lock.count++;
+			return TDB_SUCCESS;
+		}
+
+		/* a global lock of a different type exists */
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_USE_ERROR,
+				  "tdb_allrecord_lock: already have %s lock",
+				  tdb->file->allrecord_lock.ltype == F_RDLCK
+				  ? "read" : "write");
+	}
+
+	if (tdb_has_hash_locks(tdb)) {
+		/* can't combine global and chain locks */
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_USE_ERROR,
+				  "tdb_allrecord_lock:"
+				  " already have chain lock");
+	}
+
+	if (upgradable && ltype != F_RDLCK) {
+		/* tdb error: you can't upgrade a write lock! */
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_allrecord_lock:"
+				  " can't upgrade a write lock");
+	}
+
+	tdb->stats.locks++;
+again:
+	/* Lock hashes, gradually. */
+	ecode = tdb_lock_gradual(tdb, ltype, flags, TDB_HASH_LOCK_START,
+				 TDB_HASH_LOCK_RANGE);
+	if (ecode != TDB_SUCCESS)
+		return ecode;
+
+	/* Lock free tables: there to end of file. */
+	ecode = tdb_brlock(tdb, ltype,
+			   TDB_HASH_LOCK_START + TDB_HASH_LOCK_RANGE,
+			   0, flags);
+	if (ecode != TDB_SUCCESS) {
+		tdb_brunlock(tdb, ltype, TDB_HASH_LOCK_START,
+			     TDB_HASH_LOCK_RANGE);
+		return ecode;
+	}
+
+	tdb->file->allrecord_lock.owner = tdb;
+	tdb->file->allrecord_lock.count = 1;
+	/* If it's upgradable, it's actually exclusive so we can treat
+	 * it as a write lock. */
+	tdb->file->allrecord_lock.ltype = upgradable ? F_WRLCK : ltype;
+	tdb->file->allrecord_lock.off = upgradable;
+
+	/* Now check for needing recovery. */
+	if (flags & TDB_LOCK_NOCHECK)
+		return TDB_SUCCESS;
+
+	berr = tdb_needs_recovery(tdb);
+	if (likely(berr == false))
+		return TDB_SUCCESS;
+
+	tdb_allrecord_unlock(tdb, ltype);
+	if (berr < 0)
+		return berr;
+	ecode = tdb_lock_and_recover(tdb);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+	goto again;
+}
+
+enum TDB_ERROR tdb_lock_open(struct tdb_context *tdb,
+			     int ltype, enum tdb_lock_flags flags)
+{
+	return tdb_nest_lock(tdb, TDB_OPEN_LOCK, ltype, flags);
+}
+
+void tdb_unlock_open(struct tdb_context *tdb, int ltype)
+{
+	tdb_nest_unlock(tdb, TDB_OPEN_LOCK, ltype);
+}
+
+bool tdb_has_open_lock(struct tdb_context *tdb)
+{
+	return !(tdb->flags & TDB_NOLOCK)
+		&& find_nestlock(tdb, TDB_OPEN_LOCK, tdb) != NULL;
+}
+
+enum TDB_ERROR tdb_lock_expand(struct tdb_context *tdb, int ltype)
+{
+	/* Lock doesn't protect data, so don't check (we recurse if we do!) */
+	return tdb_nest_lock(tdb, TDB_EXPANSION_LOCK, ltype,
+			     TDB_LOCK_WAIT | TDB_LOCK_NOCHECK);
+}
+
+void tdb_unlock_expand(struct tdb_context *tdb, int ltype)
+{
+	tdb_nest_unlock(tdb, TDB_EXPANSION_LOCK, ltype);
+}
+
+/* unlock entire db */
+void tdb_allrecord_unlock(struct tdb_context *tdb, int ltype)
+{
+	if (tdb->flags & TDB_NOLOCK)
+		return;
+
+	if (tdb->file->allrecord_lock.count == 0) {
+		tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_USE_ERROR,
+			   "tdb_allrecord_unlock: not locked!");
+		return;
+	}
+
+	if (tdb->file->allrecord_lock.owner != tdb) {
+		tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_USE_ERROR,
+			   "tdb_allrecord_unlock: not locked by us!");
+		return;
+	}
+
+	/* Upgradable locks are marked as write locks. */
+	if (tdb->file->allrecord_lock.ltype != ltype
+	    && (!tdb->file->allrecord_lock.off || ltype != F_RDLCK)) {
+		tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+			   "tdb_allrecord_unlock: have %s lock",
+			   tdb->file->allrecord_lock.ltype == F_RDLCK
+			   ? "read" : "write");
+		return;
+	}
+
+	if (tdb->file->allrecord_lock.count > 1) {
+		tdb->file->allrecord_lock.count--;
+		return;
+	}
+
+	tdb->file->allrecord_lock.count = 0;
+	tdb->file->allrecord_lock.ltype = 0;
+
+	tdb_brunlock(tdb, ltype, TDB_HASH_LOCK_START, 0);
+}
+
+bool tdb_has_expansion_lock(struct tdb_context *tdb)
+{
+	return find_nestlock(tdb, TDB_EXPANSION_LOCK, tdb) != NULL;
+}
+
+bool tdb_has_hash_locks(struct tdb_context *tdb)
+{
+	unsigned int i;
+
+	for (i=0; i<tdb->file->num_lockrecs; i++) {
+		if (tdb->file->lockrecs[i].off >= TDB_HASH_LOCK_START
+		    && tdb->file->lockrecs[i].off < (TDB_HASH_LOCK_START
+						     + TDB_HASH_LOCK_RANGE))
+			return true;
+	}
+	return false;
+}
+
+static bool tdb_has_free_lock(struct tdb_context *tdb)
+{
+	unsigned int i;
+
+	if (tdb->flags & TDB_NOLOCK)
+		return false;
+
+	for (i=0; i<tdb->file->num_lockrecs; i++) {
+		if (tdb->file->lockrecs[i].off
+		    > TDB_HASH_LOCK_START + TDB_HASH_LOCK_RANGE)
+			return true;
+	}
+	return false;
+}
+
+enum TDB_ERROR tdb_lock_hashes(struct tdb_context *tdb,
+			       tdb_off_t hash_lock,
+			       tdb_len_t hash_range,
+			       int ltype, enum tdb_lock_flags waitflag)
+{
+	/* FIXME: Do this properly, using hlock_range */
+	unsigned l = TDB_HASH_LOCK_START
+		+ (hash_lock >> (64 - TDB_HASH_LOCK_RANGE_BITS));
+
+	/* a allrecord lock allows us to avoid per chain locks */
+	if (tdb->file->allrecord_lock.count) {
+		if (!check_lock_pid(tdb, "tdb_lock_hashes", true))
+			return TDB_ERR_LOCK;
+
+		if (tdb->file->allrecord_lock.owner != tdb)
+			return owner_conflict(tdb, "tdb_lock_hashes");
+		if (ltype == tdb->file->allrecord_lock.ltype
+		    || ltype == F_RDLCK) {
+			return TDB_SUCCESS;
+		}
+
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_USE_ERROR,
+				  "tdb_lock_hashes:"
+				  " already have %s allrecordlock",
+				  tdb->file->allrecord_lock.ltype == F_RDLCK
+				  ? "read" : "write");
+	}
+
+	if (tdb_has_free_lock(tdb)) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_lock_hashes: already have free lock");
+	}
+
+	if (tdb_has_expansion_lock(tdb)) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_lock_hashes:"
+				  " already have expansion lock");
+	}
+
+	return tdb_nest_lock(tdb, l, ltype, waitflag);
+}
+
+enum TDB_ERROR tdb_unlock_hashes(struct tdb_context *tdb,
+				 tdb_off_t hash_lock,
+				 tdb_len_t hash_range, int ltype)
+{
+	unsigned l = TDB_HASH_LOCK_START
+		+ (hash_lock >> (64 - TDB_HASH_LOCK_RANGE_BITS));
+
+	if (tdb->flags & TDB_NOLOCK)
+		return 0;
+
+	/* a allrecord lock allows us to avoid per chain locks */
+	if (tdb->file->allrecord_lock.count) {
+		if (tdb->file->allrecord_lock.ltype == F_RDLCK
+		    && ltype == F_WRLCK) {
+			return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+					  "tdb_unlock_hashes RO allrecord!");
+		}
+		return TDB_SUCCESS;
+	}
+
+	return tdb_nest_unlock(tdb, l, ltype);
+}
+
+/* Hash locks use TDB_HASH_LOCK_START + the next 30 bits.
+ * Then we begin; bucket offsets are sizeof(tdb_len_t) apart, so we divide.
+ * The result is that on 32 bit systems we don't use lock values > 2^31 on
+ * files that are less than 4GB.
+ */
+static tdb_off_t free_lock_off(tdb_off_t b_off)
+{
+	return TDB_HASH_LOCK_START + TDB_HASH_LOCK_RANGE
+		+ b_off / sizeof(tdb_off_t);
+}
+
+enum TDB_ERROR tdb_lock_free_bucket(struct tdb_context *tdb, tdb_off_t b_off,
+				    enum tdb_lock_flags waitflag)
+{
+	assert(b_off >= sizeof(struct tdb_header));
+
+	if (tdb->flags & TDB_NOLOCK)
+		return 0;
+
+	/* a allrecord lock allows us to avoid per chain locks */
+	if (tdb->file->allrecord_lock.count) {
+		if (!check_lock_pid(tdb, "tdb_lock_free_bucket", true))
+			return TDB_ERR_LOCK;
+
+		if (tdb->file->allrecord_lock.ltype == F_WRLCK)
+			return 0;
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_lock_free_bucket with"
+				  " read-only allrecordlock!");
+	}
+
+#if 0 /* FIXME */
+	if (tdb_has_expansion_lock(tdb)) {
+		return tdb_logerr(tdb, TDB_ERR_LOCK, TDB_LOG_ERROR,
+				  "tdb_lock_free_bucket:"
+				  " already have expansion lock");
+	}
+#endif
+
+	return tdb_nest_lock(tdb, free_lock_off(b_off), F_WRLCK, waitflag);
+}
+
+void tdb_unlock_free_bucket(struct tdb_context *tdb, tdb_off_t b_off)
+{
+	if (tdb->file->allrecord_lock.count)
+		return;
+
+	tdb_nest_unlock(tdb, free_lock_off(b_off), F_WRLCK);
+}
+
+enum TDB_ERROR tdb_lockall(struct tdb_context *tdb)
+{
+	return tdb_allrecord_lock(tdb, F_WRLCK, TDB_LOCK_WAIT, false);
+}
+
+void tdb_unlockall(struct tdb_context *tdb)
+{
+	tdb_allrecord_unlock(tdb, F_WRLCK);
+}
+
+enum TDB_ERROR tdb_lockall_read(struct tdb_context *tdb)
+{
+	return tdb_allrecord_lock(tdb, F_RDLCK, TDB_LOCK_WAIT, false);
+}
+
+void tdb_unlockall_read(struct tdb_context *tdb)
+{
+	tdb_allrecord_unlock(tdb, F_RDLCK);
+}
+
+void tdb_lock_cleanup(struct tdb_context *tdb)
+{
+	unsigned int i;
+
+	/* We don't want to warn: they're allowed to close tdb after fork. */
+	if (!check_lock_pid(tdb, "tdb_close", false))
+		return;
+
+	while (tdb->file->allrecord_lock.count
+	       && tdb->file->allrecord_lock.owner == tdb) {
+		tdb_allrecord_unlock(tdb, tdb->file->allrecord_lock.ltype);
+	}
+
+	for (i=0; i<tdb->file->num_lockrecs; i++) {
+		if (tdb->file->lockrecs[i].owner == tdb) {
+			tdb_nest_unlock(tdb,
+					tdb->file->lockrecs[i].off,
+					tdb->file->lockrecs[i].ltype);
+			i--;
+		}
+	}
+}
diff --git a/lib/tdb2/open.c b/lib/tdb2/open.c
new file mode 100644
index 00000000000..c35598cdcce
--- /dev/null
+++ b/lib/tdb2/open.c
@@ -0,0 +1,661 @@
+#include "private.h"
+#include <ccan/hash/hash.h>
+#include <assert.h>
+
+/* all lock info, to detect double-opens (fcntl file don't nest!) */
+static struct tdb_file *files = NULL;
+
+static struct tdb_file *find_file(dev_t device, ino_t ino)
+{
+	struct tdb_file *i;
+
+	for (i = files; i; i = i->next) {
+		if (i->device == device && i->inode == ino) {
+			i->refcnt++;
+			break;
+		}
+	}
+	return i;
+}
+
+static bool read_all(int fd, void *buf, size_t len)
+{
+	while (len) {
+		ssize_t ret;
+		ret = read(fd, buf, len);
+		if (ret < 0)
+			return false;
+		if (ret == 0) {
+			/* ETOOSHORT? */
+			errno = EWOULDBLOCK;
+			return false;
+		}
+		buf = (char *)buf + ret;
+		len -= ret;
+	}
+	return true;
+}
+
+static uint64_t random_number(struct tdb_context *tdb)
+{
+	int fd;
+	uint64_t ret = 0;
+	struct timeval now;
+
+	fd = open("/dev/urandom", O_RDONLY);
+	if (fd >= 0) {
+		if (read_all(fd, &ret, sizeof(ret))) {
+			close(fd);
+			return ret;
+		}
+		close(fd);
+	}
+	/* FIXME: Untested!  Based on Wikipedia protocol description! */
+	fd = open("/dev/egd-pool", O_RDWR);
+	if (fd >= 0) {
+		/* Command is 1, next byte is size we want to read. */
+		char cmd[2] = { 1, sizeof(uint64_t) };
+		if (write(fd, cmd, sizeof(cmd)) == sizeof(cmd)) {
+			char reply[1 + sizeof(uint64_t)];
+			int r = read(fd, reply, sizeof(reply));
+			if (r > 1) {
+				/* Copy at least some bytes. */
+				memcpy(&ret, reply+1, r - 1);
+				if (reply[0] == sizeof(uint64_t)
+				    && r == sizeof(reply)) {
+					close(fd);
+					return ret;
+				}
+			}
+		}
+		close(fd);
+	}
+
+	/* Fallback: pid and time. */
+	gettimeofday(&now, NULL);
+	ret = getpid() * 100132289ULL + now.tv_sec * 1000000ULL + now.tv_usec;
+	tdb_logerr(tdb, TDB_SUCCESS, TDB_LOG_WARNING,
+		   "tdb_open: random from getpid and time");
+	return ret;
+}
+
+struct new_database {
+	struct tdb_header hdr;
+	struct tdb_freetable ftable;
+};
+
+/* initialise a new database */
+static enum TDB_ERROR tdb_new_database(struct tdb_context *tdb,
+				       struct tdb_attribute_seed *seed,
+				       struct tdb_header *hdr)
+{
+	/* We make it up in memory, then write it out if not internal */
+	struct new_database newdb;
+	unsigned int magic_len;
+	ssize_t rlen;
+	enum TDB_ERROR ecode;
+
+	/* Fill in the header */
+	newdb.hdr.version = TDB_VERSION;
+	if (seed)
+		newdb.hdr.hash_seed = seed->seed;
+	else
+		newdb.hdr.hash_seed = random_number(tdb);
+	newdb.hdr.hash_test = TDB_HASH_MAGIC;
+	newdb.hdr.hash_test = tdb->hash_fn(&newdb.hdr.hash_test,
+					   sizeof(newdb.hdr.hash_test),
+					   newdb.hdr.hash_seed,
+					   tdb->hash_data);
+	newdb.hdr.recovery = 0;
+	newdb.hdr.features_used = newdb.hdr.features_offered = TDB_FEATURE_MASK;
+	newdb.hdr.seqnum = 0;
+	memset(newdb.hdr.reserved, 0, sizeof(newdb.hdr.reserved));
+	/* Initial hashes are empty. */
+	memset(newdb.hdr.hashtable, 0, sizeof(newdb.hdr.hashtable));
+
+	/* Free is empty. */
+	newdb.hdr.free_table = offsetof(struct new_database, ftable);
+	memset(&newdb.ftable, 0, sizeof(newdb.ftable));
+	ecode = set_header(NULL, &newdb.ftable.hdr, TDB_FTABLE_MAGIC, 0,
+			   sizeof(newdb.ftable) - sizeof(newdb.ftable.hdr),
+			   sizeof(newdb.ftable) - sizeof(newdb.ftable.hdr),
+			   0);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	/* Magic food */
+	memset(newdb.hdr.magic_food, 0, sizeof(newdb.hdr.magic_food));
+	strcpy(newdb.hdr.magic_food, TDB_MAGIC_FOOD);
+
+	/* This creates an endian-converted database, as if read from disk */
+	magic_len = sizeof(newdb.hdr.magic_food);
+	tdb_convert(tdb,
+		    (char *)&newdb.hdr + magic_len, sizeof(newdb) - magic_len);
+
+	*hdr = newdb.hdr;
+
+	if (tdb->flags & TDB_INTERNAL) {
+		tdb->file->map_size = sizeof(newdb);
+		tdb->file->map_ptr = malloc(tdb->file->map_size);
+		if (!tdb->file->map_ptr) {
+			return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+					  "tdb_new_database:"
+					  " failed to allocate");
+		}
+		memcpy(tdb->file->map_ptr, &newdb, tdb->file->map_size);
+		return TDB_SUCCESS;
+	}
+	if (lseek(tdb->file->fd, 0, SEEK_SET) == -1) {
+		return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				  "tdb_new_database:"
+				  " failed to seek: %s", strerror(errno));
+	}
+
+	if (ftruncate(tdb->file->fd, 0) == -1) {
+		return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				  "tdb_new_database:"
+				  " failed to truncate: %s", strerror(errno));
+	}
+
+	rlen = write(tdb->file->fd, &newdb, sizeof(newdb));
+	if (rlen != sizeof(newdb)) {
+		if (rlen >= 0)
+			errno = ENOSPC;
+		return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				  "tdb_new_database: %zi writing header: %s",
+				  rlen, strerror(errno));
+	}
+	return TDB_SUCCESS;
+}
+
+static enum TDB_ERROR tdb_new_file(struct tdb_context *tdb)
+{
+	tdb->file = malloc(sizeof(*tdb->file));
+	if (!tdb->file)
+		return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+				  "tdb_open: cannot alloc tdb_file structure");
+	tdb->file->num_lockrecs = 0;
+	tdb->file->lockrecs = NULL;
+	tdb->file->allrecord_lock.count = 0;
+	tdb->file->refcnt = 1;
+	return TDB_SUCCESS;
+}
+
+enum TDB_ERROR tdb_set_attribute(struct tdb_context *tdb,
+				 const union tdb_attribute *attr)
+{
+	switch (attr->base.attr) {
+	case TDB_ATTRIBUTE_LOG:
+		tdb->log_fn = attr->log.fn;
+		tdb->log_data = attr->log.data;
+		break;
+	case TDB_ATTRIBUTE_HASH:
+	case TDB_ATTRIBUTE_SEED:
+	case TDB_ATTRIBUTE_OPENHOOK:
+		return tdb->last_error
+			= tdb_logerr(tdb, TDB_ERR_EINVAL,
+				     TDB_LOG_USE_ERROR,
+				     "tdb_set_attribute:"
+				     " cannot set %s after opening",
+				     attr->base.attr == TDB_ATTRIBUTE_HASH
+				     ? "TDB_ATTRIBUTE_HASH"
+				     : attr->base.attr == TDB_ATTRIBUTE_SEED
+				     ? "TDB_ATTRIBUTE_SEED"
+				     : "TDB_ATTRIBUTE_OPENHOOK");
+	case TDB_ATTRIBUTE_STATS:
+		return tdb->last_error
+			= tdb_logerr(tdb, TDB_ERR_EINVAL,
+				     TDB_LOG_USE_ERROR,
+				     "tdb_set_attribute:"
+				     " cannot set TDB_ATTRIBUTE_STATS");
+	case TDB_ATTRIBUTE_FLOCK:
+		tdb->lock_fn = attr->flock.lock;
+		tdb->unlock_fn = attr->flock.unlock;
+		tdb->lock_data = attr->flock.data;
+		break;
+	default:
+		return tdb->last_error
+			= tdb_logerr(tdb, TDB_ERR_EINVAL,
+				     TDB_LOG_USE_ERROR,
+				     "tdb_set_attribute:"
+				     " unknown attribute type %u",
+				     attr->base.attr);
+	}
+	return TDB_SUCCESS;
+}
+
+static uint64_t jenkins_hash(const void *key, size_t length, uint64_t seed,
+			     void *unused)
+{
+	uint64_t ret;
+	/* hash64_stable assumes lower bits are more important; they are a
+	 * slightly better hash.  We use the upper bits first, so swap them. */
+	ret = hash64_stable((const unsigned char *)key, length, seed);
+	return (ret >> 32) | (ret << 32);
+}
+
+enum TDB_ERROR tdb_get_attribute(struct tdb_context *tdb,
+				 union tdb_attribute *attr)
+{
+	switch (attr->base.attr) {
+	case TDB_ATTRIBUTE_LOG:
+		if (!tdb->log_fn)
+			return tdb->last_error = TDB_ERR_NOEXIST;
+		attr->log.fn = tdb->log_fn;
+		attr->log.data = tdb->log_data;
+		break;
+	case TDB_ATTRIBUTE_HASH:
+		attr->hash.fn = tdb->hash_fn;
+		attr->hash.data = tdb->hash_data;
+		break;
+	case TDB_ATTRIBUTE_SEED:
+		attr->seed.seed = tdb->hash_seed;
+		break;
+	case TDB_ATTRIBUTE_OPENHOOK:
+		return tdb->last_error
+			= tdb_logerr(tdb, TDB_ERR_EINVAL,
+				     TDB_LOG_USE_ERROR,
+				     "tdb_get_attribute:"
+				     " cannot get TDB_ATTRIBUTE_OPENHOOK");
+	case TDB_ATTRIBUTE_STATS: {
+		size_t size = attr->stats.size;
+		if (size > tdb->stats.size)
+			size = tdb->stats.size;
+		memcpy(&attr->stats, &tdb->stats, size);
+		break;
+	}
+	case TDB_ATTRIBUTE_FLOCK:
+		attr->flock.lock = tdb->lock_fn;
+		attr->flock.unlock = tdb->unlock_fn;
+		attr->flock.data = tdb->lock_data;
+		break;
+	default:
+		return tdb->last_error
+			= tdb_logerr(tdb, TDB_ERR_EINVAL,
+				     TDB_LOG_USE_ERROR,
+				     "tdb_get_attribute:"
+				     " unknown attribute type %u",
+				     attr->base.attr);
+	}
+	attr->base.next = NULL;
+	return TDB_SUCCESS;
+}
+
+void tdb_unset_attribute(struct tdb_context *tdb,
+			 enum tdb_attribute_type type)
+{
+	switch (type) {
+	case TDB_ATTRIBUTE_LOG:
+		tdb->log_fn = NULL;
+		break;
+	case TDB_ATTRIBUTE_HASH:
+	case TDB_ATTRIBUTE_SEED:
+	case TDB_ATTRIBUTE_OPENHOOK:
+		tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_USE_ERROR,
+			   "tdb_unset_attribute: cannot unset %s after opening",
+			   type == TDB_ATTRIBUTE_HASH
+			   ? "TDB_ATTRIBUTE_HASH"
+			   : type == TDB_ATTRIBUTE_SEED
+			   ? "TDB_ATTRIBUTE_SEED"
+			   : "TDB_ATTRIBUTE_OPENHOOK");
+		break;
+	case TDB_ATTRIBUTE_STATS:
+		tdb_logerr(tdb, TDB_ERR_EINVAL,
+			   TDB_LOG_USE_ERROR,
+			   "tdb_unset_attribute:"
+			   "cannot unset TDB_ATTRIBUTE_STATS");
+		break;
+	case TDB_ATTRIBUTE_FLOCK:
+		tdb->lock_fn = tdb_fcntl_lock;
+		tdb->unlock_fn = tdb_fcntl_unlock;
+		break;
+	default:
+		tdb_logerr(tdb, TDB_ERR_EINVAL,
+			   TDB_LOG_USE_ERROR,
+			   "tdb_unset_attribute: unknown attribute type %u",
+			   type);
+	}
+}
+
+struct tdb_context *tdb_open(const char *name, int tdb_flags,
+			     int open_flags, mode_t mode,
+			     union tdb_attribute *attr)
+{
+	struct tdb_context *tdb;
+	struct stat st;
+	int saved_errno = 0;
+	uint64_t hash_test;
+	unsigned v;
+	ssize_t rlen;
+	struct tdb_header hdr;
+	struct tdb_attribute_seed *seed = NULL;
+	struct tdb_attribute_openhook *openhook = NULL;
+	tdb_bool_err berr;
+	enum TDB_ERROR ecode;
+	int openlock;
+
+	tdb = malloc(sizeof(*tdb) + (name ? strlen(name) + 1 : 0));
+	if (!tdb) {
+		/* Can't log this */
+		errno = ENOMEM;
+		return NULL;
+	}
+	/* Set name immediately for logging functions. */
+	if (name) {
+		tdb->name = strcpy((char *)(tdb + 1), name);
+	} else {
+		tdb->name = NULL;
+	}
+	tdb->direct_access = 0;
+	tdb->flags = tdb_flags;
+	tdb->log_fn = NULL;
+	tdb->transaction = NULL;
+	tdb->access = NULL;
+	tdb->last_error = TDB_SUCCESS;
+	tdb->file = NULL;
+	tdb->lock_fn = tdb_fcntl_lock;
+	tdb->unlock_fn = tdb_fcntl_unlock;
+	tdb->hash_fn = jenkins_hash;
+	memset(&tdb->stats, 0, sizeof(tdb->stats));
+	tdb->stats.base.attr = TDB_ATTRIBUTE_STATS;
+	tdb->stats.size = sizeof(tdb->stats);
+	tdb_io_init(tdb);
+
+	while (attr) {
+		switch (attr->base.attr) {
+		case TDB_ATTRIBUTE_HASH:
+			tdb->hash_fn = attr->hash.fn;
+			tdb->hash_data = attr->hash.data;
+			break;
+		case TDB_ATTRIBUTE_SEED:
+			seed = &attr->seed;
+			break;
+		case TDB_ATTRIBUTE_OPENHOOK:
+			openhook = &attr->openhook;
+			break;
+		default:
+			/* These are set as normal. */
+			ecode = tdb_set_attribute(tdb, attr);
+			if (ecode != TDB_SUCCESS)
+				goto fail;
+		}
+		attr = attr->base.next;
+	}
+
+	if (tdb_flags & ~(TDB_INTERNAL | TDB_NOLOCK | TDB_NOMMAP | TDB_CONVERT
+			  | TDB_NOSYNC | TDB_SEQNUM | TDB_ALLOW_NESTING)) {
+		ecode = tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_USE_ERROR,
+				   "tdb_open: unknown flags %u", tdb_flags);
+		goto fail;
+	}
+
+	if ((open_flags & O_ACCMODE) == O_WRONLY) {
+		ecode = tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_USE_ERROR,
+				   "tdb_open: can't open tdb %s write-only",
+				   name);
+		goto fail;
+	}
+
+	if ((open_flags & O_ACCMODE) == O_RDONLY) {
+		tdb->read_only = true;
+		tdb->mmap_flags = PROT_READ;
+		openlock = F_RDLCK;
+	} else {
+		tdb->read_only = false;
+		tdb->mmap_flags = PROT_READ | PROT_WRITE;
+		openlock = F_WRLCK;
+	}
+
+	/* internal databases don't need any of the rest. */
+	if (tdb->flags & TDB_INTERNAL) {
+		tdb->flags |= (TDB_NOLOCK | TDB_NOMMAP);
+		ecode = tdb_new_file(tdb);
+		if (ecode != TDB_SUCCESS) {
+			goto fail;
+		}
+		tdb->file->fd = -1;
+		ecode = tdb_new_database(tdb, seed, &hdr);
+		if (ecode != TDB_SUCCESS) {
+			goto fail;
+		}
+		tdb_convert(tdb, &hdr.hash_seed, sizeof(hdr.hash_seed));
+		tdb->hash_seed = hdr.hash_seed;
+		tdb_ftable_init(tdb);
+		return tdb;
+	}
+
+	if (stat(name, &st) != -1)
+		tdb->file = find_file(st.st_dev, st.st_ino);
+
+	if (!tdb->file) {
+		int fd;
+
+		if ((fd = open(name, open_flags, mode)) == -1) {
+			/* errno set by open(2) */
+			saved_errno = errno;
+			tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				   "tdb_open: could not open file %s: %s",
+				   name, strerror(errno));
+			goto fail_errno;
+		}
+
+		/* on exec, don't inherit the fd */
+		v = fcntl(fd, F_GETFD, 0);
+		fcntl(fd, F_SETFD, v | FD_CLOEXEC);
+
+		if (fstat(fd, &st) == -1) {
+			saved_errno = errno;
+			tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				   "tdb_open: could not stat open %s: %s",
+				   name, strerror(errno));
+			close(fd);
+			goto fail_errno;
+		}
+
+		ecode = tdb_new_file(tdb);
+		if (ecode != TDB_SUCCESS) {
+			close(fd);
+			goto fail;
+		}
+
+		tdb->file->next = files;
+		tdb->file->fd = fd;
+		tdb->file->device = st.st_dev;
+		tdb->file->inode = st.st_ino;
+		tdb->file->map_ptr = NULL;
+		tdb->file->map_size = sizeof(struct tdb_header);
+	}
+
+	/* ensure there is only one process initialising at once */
+	ecode = tdb_lock_open(tdb, openlock, TDB_LOCK_WAIT|TDB_LOCK_NOCHECK);
+	if (ecode != TDB_SUCCESS) {
+		saved_errno = errno;
+		goto fail_errno;
+	}
+
+	/* call their open hook if they gave us one. */
+	if (openhook) {
+		ecode = openhook->fn(tdb->file->fd, openhook->data);
+		if (ecode != TDB_SUCCESS) {
+			tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				   "tdb_open: open hook failed");
+			goto fail;
+		}
+		open_flags |= O_CREAT;
+	}
+
+	/* If they used O_TRUNC, read will return 0. */
+	rlen = pread(tdb->file->fd, &hdr, sizeof(hdr), 0);
+	if (rlen == 0 && (open_flags & O_CREAT)) {
+		ecode = tdb_new_database(tdb, seed, &hdr);
+		if (ecode != TDB_SUCCESS) {
+			goto fail;
+		}
+	} else if (rlen < 0) {
+		ecode = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				   "tdb_open: error %s reading %s",
+				   strerror(errno), name);
+		goto fail;
+	} else if (rlen < sizeof(hdr)
+		   || strcmp(hdr.magic_food, TDB_MAGIC_FOOD) != 0) {
+		ecode = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				   "tdb_open: %s is not a tdb file", name);
+		goto fail;
+	}
+
+	if (hdr.version != TDB_VERSION) {
+		if (hdr.version == bswap_64(TDB_VERSION))
+			tdb->flags |= TDB_CONVERT;
+		else {
+			/* wrong version */
+			ecode = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+					   "tdb_open:"
+					   " %s is unknown version 0x%llx",
+					   name, (long long)hdr.version);
+			goto fail;
+		}
+	}
+
+	tdb_convert(tdb, &hdr, sizeof(hdr));
+	tdb->hash_seed = hdr.hash_seed;
+	hash_test = TDB_HASH_MAGIC;
+	hash_test = tdb_hash(tdb, &hash_test, sizeof(hash_test));
+	if (hdr.hash_test != hash_test) {
+		/* wrong hash variant */
+		ecode = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				   "tdb_open:"
+				   " %s uses a different hash function",
+				   name);
+		goto fail;
+	}
+
+	/* Clear any features we don't understand. */
+	if ((open_flags & O_ACCMODE) != O_RDONLY) {
+		hdr.features_used &= TDB_FEATURE_MASK;
+		if (tdb_write_convert(tdb, offsetof(struct tdb_header,
+						    features_used),
+				      &hdr.features_used,
+				      sizeof(hdr.features_used)) == -1)
+			goto fail;
+	}
+
+	tdb_unlock_open(tdb, openlock);
+
+	/* This make sure we have current map_size and mmap. */
+	tdb->methods->oob(tdb, tdb->file->map_size + 1, true);
+
+	/* Now it's fully formed, recover if necessary. */
+	berr = tdb_needs_recovery(tdb);
+	if (unlikely(berr != false)) {
+		if (berr < 0) {
+			ecode = berr;
+			goto fail;
+		}
+		ecode = tdb_lock_and_recover(tdb);
+		if (ecode != TDB_SUCCESS) {
+			goto fail;
+		}
+	}
+
+	ecode = tdb_ftable_init(tdb);
+	if (ecode != TDB_SUCCESS) {
+		goto fail;
+	}
+
+	/* Add to linked list if we're new. */
+	if (tdb->file->refcnt == 1)
+		files = tdb->file;
+	return tdb;
+
+ fail:
+	/* Map ecode to some logical errno. */
+	switch (ecode) {
+	case TDB_ERR_CORRUPT:
+	case TDB_ERR_IO:
+		saved_errno = EIO;
+		break;
+	case TDB_ERR_LOCK:
+		saved_errno = EWOULDBLOCK;
+		break;
+	case TDB_ERR_OOM:
+		saved_errno = ENOMEM;
+		break;
+	case TDB_ERR_EINVAL:
+		saved_errno = EINVAL;
+		break;
+	default:
+		saved_errno = EINVAL;
+		break;
+	}
+
+fail_errno:
+#ifdef TDB_TRACE
+	close(tdb->tracefd);
+#endif
+	if (tdb->file) {
+		tdb_lock_cleanup(tdb);
+		if (--tdb->file->refcnt == 0) {
+			assert(tdb->file->num_lockrecs == 0);
+			if (tdb->file->map_ptr) {
+				if (tdb->flags & TDB_INTERNAL) {
+					free(tdb->file->map_ptr);
+				} else
+					tdb_munmap(tdb->file);
+			}
+			if (close(tdb->file->fd) != 0)
+				tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+					   "tdb_open: failed to close tdb fd"
+					   " on error: %s", strerror(errno));
+			free(tdb->file->lockrecs);
+			free(tdb->file);
+		}
+	}
+
+	free(tdb);
+	errno = saved_errno;
+	return NULL;
+}
+
+int tdb_close(struct tdb_context *tdb)
+{
+	int ret = 0;
+
+	tdb_trace(tdb, "tdb_close");
+
+	if (tdb->transaction) {
+		tdb_transaction_cancel(tdb);
+	}
+
+	if (tdb->file->map_ptr) {
+		if (tdb->flags & TDB_INTERNAL)
+			free(tdb->file->map_ptr);
+		else
+			tdb_munmap(tdb->file);
+	}
+	if (tdb->file) {
+		struct tdb_file **i;
+
+		tdb_lock_cleanup(tdb);
+		if (--tdb->file->refcnt == 0) {
+			ret = close(tdb->file->fd);
+
+			/* Remove from files list */
+			for (i = &files; *i; i = &(*i)->next) {
+				if (*i == tdb->file) {
+					*i = tdb->file->next;
+					break;
+				}
+			}
+			free(tdb->file->lockrecs);
+			free(tdb->file);
+		}
+	}
+
+#ifdef TDB_TRACE
+	close(tdb->tracefd);
+#endif
+	free(tdb);
+
+	return ret;
+}
diff --git a/lib/tdb2/private.h b/lib/tdb2/private.h
new file mode 100644
index 00000000000..0c3e4416575
--- /dev/null
+++ b/lib/tdb2/private.h
@@ -0,0 +1,624 @@
+#ifndef TDB_PRIVATE_H
+#define TDB_PRIVATE_H
+ /*
+   Trivial Database 2: private types and prototypes
+   Copyright (C) Rusty Russell 2010
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "config.h"
+#if HAVE_FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#include <ccan/likely/likely.h>
+#include <ccan/compiler/compiler.h>
+#include <ccan/endian/endian.h>
+#include "tdb2.h"
+
+#ifdef _SAMBA_BUILD_
+#include "replace.h"
+#include "system/filesys.h"
+#include "system/time.h"
+#include "system/shmem.h"
+#include "system/select.h"
+#include "system/wait.h"
+#else
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+#include <stdio.h>
+#include <utime.h>
+#include <unistd.h>
+#endif
+
+#ifndef TEST_IT
+#define TEST_IT(cond)
+#endif
+
+/* #define TDB_TRACE 1 */
+
+#ifndef __STRING
+#define __STRING(x)    #x
+#endif
+
+#ifndef __STRINGSTRING
+#define __STRINGSTRING(x) __STRING(x)
+#endif
+
+#ifndef __location__
+#define __location__ __FILE__ ":" __STRINGSTRING(__LINE__)
+#endif
+
+typedef uint64_t tdb_len_t;
+typedef uint64_t tdb_off_t;
+
+#define TDB_MAGIC_FOOD "TDB file\n"
+#define TDB_VERSION ((uint64_t)(0x26011967 + 7))
+#define TDB_USED_MAGIC ((uint64_t)0x1999)
+#define TDB_HTABLE_MAGIC ((uint64_t)0x1888)
+#define TDB_CHAIN_MAGIC ((uint64_t)0x1777)
+#define TDB_FTABLE_MAGIC ((uint64_t)0x1666)
+#define TDB_FREE_MAGIC ((uint64_t)0xFE)
+#define TDB_HASH_MAGIC (0xA1ABE11A01092008ULL)
+#define TDB_RECOVERY_MAGIC (0xf53bc0e7ad124589ULL)
+#define TDB_RECOVERY_INVALID_MAGIC (0x0ULL)
+
+#define TDB_OFF_IS_ERR(off) unlikely(off >= (tdb_off_t)TDB_ERR_LAST)
+
+/* Packing errors into pointers and v.v. */
+#define TDB_PTR_IS_ERR(ptr) \
+	unlikely((unsigned long)(ptr) >= (unsigned long)TDB_ERR_LAST)
+#define TDB_PTR_ERR(p) ((enum TDB_ERROR)(long)(p))
+#define TDB_ERR_PTR(err) ((void *)(long)(err))
+
+/* Common case of returning true, false or -ve error. */
+typedef int tdb_bool_err;
+
+/* Prevent others from opening the file. */
+#define TDB_OPEN_LOCK 0
+/* Doing a transaction. */
+#define TDB_TRANSACTION_LOCK 1
+/* Expanding file. */
+#define TDB_EXPANSION_LOCK 2
+/* Hash chain locks. */
+#define TDB_HASH_LOCK_START 64
+
+/* Range for hash locks. */
+#define TDB_HASH_LOCK_RANGE_BITS 30
+#define TDB_HASH_LOCK_RANGE (1 << TDB_HASH_LOCK_RANGE_BITS)
+
+/* We have 1024 entries in the top level. */
+#define TDB_TOPLEVEL_HASH_BITS 10
+/* And 64 entries in each sub-level: thus 64 bits exactly after 9 levels. */
+#define TDB_SUBLEVEL_HASH_BITS 6
+/* And 8 entries in each group, ie 8 groups per sublevel. */
+#define TDB_HASH_GROUP_BITS 3
+/* This is currently 10: beyond this we chain. */
+#define TDB_MAX_LEVELS (1+(64-TDB_TOPLEVEL_HASH_BITS) / TDB_SUBLEVEL_HASH_BITS)
+
+/* Extend file by least 100 times larger than needed. */
+#define TDB_EXTENSION_FACTOR 100
+
+/* We steal bits from the offsets to store hash info. */
+#define TDB_OFF_HASH_GROUP_MASK ((1ULL << TDB_HASH_GROUP_BITS) - 1)
+/* We steal this many upper bits, giving a maximum offset of 64 exabytes. */
+#define TDB_OFF_UPPER_STEAL 8
+#define   TDB_OFF_UPPER_STEAL_EXTRA 7
+/* The bit number where we store extra hash bits. */
+#define TDB_OFF_HASH_EXTRA_BIT 57
+#define TDB_OFF_UPPER_STEAL_SUBHASH_BIT 56
+
+/* Additional features we understand.  Currently: none. */
+#define TDB_FEATURE_MASK ((uint64_t)0)
+
+/* The bit number where we store the extra hash bits. */
+/* Convenience mask to get actual offset. */
+#define TDB_OFF_MASK \
+	(((1ULL << (64 - TDB_OFF_UPPER_STEAL)) - 1) - TDB_OFF_HASH_GROUP_MASK)
+
+/* How many buckets in a free list: see size_to_bucket(). */
+#define TDB_FREE_BUCKETS (64 - TDB_OFF_UPPER_STEAL)
+
+/* We have to be able to fit a free record here. */
+#define TDB_MIN_DATA_LEN	\
+	(sizeof(struct tdb_free_record) - sizeof(struct tdb_used_record))
+
+/* Indicates this entry is not on an flist (can happen during coalescing) */
+#define TDB_FTABLE_NONE ((1ULL << TDB_OFF_UPPER_STEAL) - 1)
+
+struct tdb_used_record {
+	/* For on-disk compatibility, we avoid bitfields:
+	   magic: 16,        (highest)
+	   key_len_bits: 5,
+	   extra_padding: 32
+	   hash_bits: 11
+	*/
+        uint64_t magic_and_meta;
+	/* The bottom key_len_bits*2 are key length, rest is data length. */
+        uint64_t key_and_data_len;
+};
+
+static inline unsigned rec_key_bits(const struct tdb_used_record *r)
+{
+	return ((r->magic_and_meta >> 43) & ((1 << 5)-1)) * 2;
+}
+
+static inline uint64_t rec_key_length(const struct tdb_used_record *r)
+{
+	return r->key_and_data_len & ((1ULL << rec_key_bits(r)) - 1);
+}
+
+static inline uint64_t rec_data_length(const struct tdb_used_record *r)
+{
+	return r->key_and_data_len >> rec_key_bits(r);
+}
+
+static inline uint64_t rec_extra_padding(const struct tdb_used_record *r)
+{
+	return (r->magic_and_meta >> 11) & 0xFFFFFFFF;
+}
+
+static inline uint32_t rec_hash(const struct tdb_used_record *r)
+{
+	return r->magic_and_meta & ((1 << 11) - 1);
+}
+
+static inline uint16_t rec_magic(const struct tdb_used_record *r)
+{
+	return (r->magic_and_meta >> 48);
+}
+
+struct tdb_free_record {
+        uint64_t magic_and_prev; /* TDB_OFF_UPPER_STEAL bits magic, then prev */
+        uint64_t ftable_and_len; /* Len not counting these two fields. */
+	/* This is why the minimum record size is 8 bytes.  */
+	uint64_t next;
+};
+
+static inline uint64_t frec_prev(const struct tdb_free_record *f)
+{
+	return f->magic_and_prev & ((1ULL << (64 - TDB_OFF_UPPER_STEAL)) - 1);
+}
+
+static inline uint64_t frec_magic(const struct tdb_free_record *f)
+{
+	return f->magic_and_prev >> (64 - TDB_OFF_UPPER_STEAL);
+}
+
+static inline uint64_t frec_len(const struct tdb_free_record *f)
+{
+	return f->ftable_and_len & ((1ULL << (64 - TDB_OFF_UPPER_STEAL))-1);
+}
+
+static inline unsigned frec_ftable(const struct tdb_free_record *f)
+{
+	return f->ftable_and_len >> (64 - TDB_OFF_UPPER_STEAL);
+}
+
+struct tdb_recovery_record {
+	uint64_t magic;
+	/* Length of record (add this header to get total length). */
+	uint64_t max_len;
+	/* Length used. */
+	uint64_t len;
+	/* Old length of file before transaction. */
+	uint64_t eof;
+};
+
+/* If we bottom out of the subhashes, we chain. */
+struct tdb_chain {
+	tdb_off_t rec[1 << TDB_HASH_GROUP_BITS];
+	tdb_off_t next;
+};
+
+/* this is stored at the front of every database */
+struct tdb_header {
+	char magic_food[64]; /* for /etc/magic */
+	/* FIXME: Make me 32 bit? */
+	uint64_t version; /* version of the code */
+	uint64_t hash_test; /* result of hashing HASH_MAGIC. */
+	uint64_t hash_seed; /* "random" seed written at creation time. */
+	tdb_off_t free_table; /* (First) free table. */
+	tdb_off_t recovery; /* Transaction recovery area. */
+
+	uint64_t features_used; /* Features all writers understand */
+	uint64_t features_offered; /* Features offered */
+
+	uint64_t seqnum; /* Sequence number for TDB_SEQNUM */
+
+	tdb_off_t reserved[23];
+
+	/* Top level hash table. */
+	tdb_off_t hashtable[1ULL << TDB_TOPLEVEL_HASH_BITS];
+};
+
+struct tdb_freetable {
+	struct tdb_used_record hdr;
+	tdb_off_t next;
+	tdb_off_t buckets[TDB_FREE_BUCKETS];
+};
+
+/* Information about a particular (locked) hash entry. */
+struct hash_info {
+	/* Full hash value of entry. */
+	uint64_t h;
+	/* Start and length of lock acquired. */
+	tdb_off_t hlock_start;
+	tdb_len_t hlock_range;
+	/* Start of hash group. */
+	tdb_off_t group_start;
+	/* Bucket we belong in. */
+	unsigned int home_bucket;
+	/* Bucket we (or an empty space) were found in. */
+	unsigned int found_bucket;
+	/* How many bits of the hash are already used. */
+	unsigned int hash_used;
+	/* Current working group. */
+	tdb_off_t group[1 << TDB_HASH_GROUP_BITS];
+};
+
+struct traverse_info {
+	struct traverse_level {
+		tdb_off_t hashtable;
+		/* We ignore groups here, and treat it as a big array. */
+		unsigned entry;
+		unsigned int total_buckets;
+	} levels[TDB_MAX_LEVELS + 1];
+	unsigned int num_levels;
+	unsigned int toplevel_group;
+	/* This makes delete-everything-inside-traverse work as expected. */
+	tdb_off_t prev;
+};
+
+enum tdb_lock_flags {
+	/* WAIT == F_SETLKW, NOWAIT == F_SETLK */
+	TDB_LOCK_NOWAIT = 0,
+	TDB_LOCK_WAIT = 1,
+	/* If set, don't log an error on failure. */
+	TDB_LOCK_PROBE = 2,
+	/* If set, don't check for recovery (used by recovery code). */
+	TDB_LOCK_NOCHECK = 4,
+};
+
+struct tdb_lock {
+	struct tdb_context *owner;
+	uint32_t off;
+	uint32_t count;
+	uint32_t ltype;
+};
+
+/* This is only needed for tdb_access_commit, but used everywhere to
+ * simplify. */
+struct tdb_access_hdr {
+	struct tdb_access_hdr *next;
+	tdb_off_t off;
+	tdb_len_t len;
+	bool convert;
+};
+
+struct tdb_file {
+	/* Single list of all TDBs, to detect multiple opens. */
+	struct tdb_file *next;
+
+	/* How many are sharing us? */
+	unsigned int refcnt;
+
+	/* Mmap (if any), or malloc (for TDB_INTERNAL). */
+	void *map_ptr;
+
+	/* How much space has been mapped (<= current file size) */
+	tdb_len_t map_size;
+
+	/* The file descriptor (-1 for TDB_INTERNAL). */
+	int fd;
+
+	/* Lock information */
+	pid_t locker;
+	struct tdb_lock allrecord_lock;
+	size_t num_lockrecs;
+	struct tdb_lock *lockrecs;
+
+	/* Identity of this file. */
+	dev_t device;
+	ino_t inode;
+};
+
+struct tdb_context {
+	/* Filename of the database. */
+	const char *name;
+
+	/* Are we accessing directly? (debugging check). */
+	int direct_access;
+
+	/* Operating read-only? (Opened O_RDONLY, or in traverse_read) */
+	bool read_only;
+
+	/* mmap read only? */
+	int mmap_flags;
+
+	/* the flags passed to tdb_open, for tdb_reopen. */
+	uint32_t flags;
+
+	/* Logging function */
+	void (*log_fn)(struct tdb_context *tdb,
+		       enum tdb_log_level level,
+		       const char *message,
+		       void *data);
+	void *log_data;
+
+	/* Hash function. */
+	uint64_t (*hash_fn)(const void *key, size_t len, uint64_t seed, void *);
+	void *hash_data;
+	uint64_t hash_seed;
+
+	/* low level (fnctl) lock functions. */
+	int (*lock_fn)(int fd, int rw, off_t off, off_t len, bool w, void *);
+	int (*unlock_fn)(int fd, int rw, off_t off, off_t len, void *);
+	void *lock_data;
+
+	/* Set if we are in a transaction. */
+	struct tdb_transaction *transaction;
+
+	/* What free table are we using? */
+	tdb_off_t ftable_off;
+	unsigned int ftable;
+
+	/* IO methods: changes for transactions. */
+	const struct tdb_methods *methods;
+
+	/* Our statistics. */
+	struct tdb_attribute_stats stats;
+
+	/* Direct access information */
+	struct tdb_access_hdr *access;
+
+	/* Last error we returned. */
+	enum TDB_ERROR last_error;
+
+	/* The actual file information */
+	struct tdb_file *file;
+};
+
+struct tdb_methods {
+	enum TDB_ERROR (*tread)(struct tdb_context *, tdb_off_t, void *,
+				tdb_len_t);
+	enum TDB_ERROR (*twrite)(struct tdb_context *, tdb_off_t, const void *,
+				 tdb_len_t);
+	enum TDB_ERROR (*oob)(struct tdb_context *, tdb_off_t, bool);
+	enum TDB_ERROR (*expand_file)(struct tdb_context *, tdb_len_t);
+	void *(*direct)(struct tdb_context *, tdb_off_t, size_t, bool);
+};
+
+/*
+  internal prototypes
+*/
+/* hash.c: */
+tdb_bool_err first_in_hash(struct tdb_context *tdb,
+			   struct traverse_info *tinfo,
+			   TDB_DATA *kbuf, size_t *dlen);
+
+tdb_bool_err next_in_hash(struct tdb_context *tdb,
+			  struct traverse_info *tinfo,
+			  TDB_DATA *kbuf, size_t *dlen);
+
+/* Hash random memory. */
+uint64_t tdb_hash(struct tdb_context *tdb, const void *ptr, size_t len);
+
+/* Hash on disk. */
+uint64_t hash_record(struct tdb_context *tdb, tdb_off_t off);
+
+/* Find and lock a hash entry (or where it would be). */
+tdb_off_t find_and_lock(struct tdb_context *tdb,
+			struct tdb_data key,
+			int ltype,
+			struct hash_info *h,
+			struct tdb_used_record *rec,
+			struct traverse_info *tinfo);
+
+enum TDB_ERROR replace_in_hash(struct tdb_context *tdb,
+			       struct hash_info *h,
+			       tdb_off_t new_off);
+
+enum TDB_ERROR add_to_hash(struct tdb_context *tdb, struct hash_info *h,
+			   tdb_off_t new_off);
+
+enum TDB_ERROR delete_from_hash(struct tdb_context *tdb, struct hash_info *h);
+
+/* For tdb_check */
+bool is_subhash(tdb_off_t val);
+
+/* free.c: */
+enum TDB_ERROR tdb_ftable_init(struct tdb_context *tdb);
+
+/* check.c needs these to iterate through free lists. */
+tdb_off_t first_ftable(struct tdb_context *tdb);
+tdb_off_t next_ftable(struct tdb_context *tdb, tdb_off_t ftable);
+
+/* This returns space or -ve error number. */
+tdb_off_t alloc(struct tdb_context *tdb, size_t keylen, size_t datalen,
+		uint64_t hash, unsigned magic, bool growing);
+
+/* Put this record in a free list. */
+enum TDB_ERROR add_free_record(struct tdb_context *tdb,
+			       tdb_off_t off, tdb_len_t len_with_header,
+			       enum tdb_lock_flags waitflag,
+			       bool coalesce_ok);
+
+/* Set up header for a used/ftable/htable/chain record. */
+enum TDB_ERROR set_header(struct tdb_context *tdb,
+			  struct tdb_used_record *rec,
+			  unsigned magic, uint64_t keylen, uint64_t datalen,
+			  uint64_t actuallen, unsigned hashlow);
+
+/* Used by tdb_check to verify. */
+unsigned int size_to_bucket(tdb_len_t data_len);
+tdb_off_t bucket_off(tdb_off_t ftable_off, unsigned bucket);
+
+/* Used by tdb_summary */
+tdb_off_t dead_space(struct tdb_context *tdb, tdb_off_t off);
+
+/* io.c: */
+/* Initialize tdb->methods. */
+void tdb_io_init(struct tdb_context *tdb);
+
+/* Convert endian of the buffer if required. */
+void *tdb_convert(const struct tdb_context *tdb, void *buf, tdb_len_t size);
+
+/* Unmap and try to map the tdb. */
+void tdb_munmap(struct tdb_file *file);
+void tdb_mmap(struct tdb_context *tdb);
+
+/* Either alloc a copy, or give direct access.  Release frees or noop. */
+const void *tdb_access_read(struct tdb_context *tdb,
+			    tdb_off_t off, tdb_len_t len, bool convert);
+void *tdb_access_write(struct tdb_context *tdb,
+		       tdb_off_t off, tdb_len_t len, bool convert);
+
+/* Release result of tdb_access_read/write. */
+void tdb_access_release(struct tdb_context *tdb, const void *p);
+/* Commit result of tdb_acces_write. */
+enum TDB_ERROR tdb_access_commit(struct tdb_context *tdb, void *p);
+
+/* Convenience routine to get an offset. */
+tdb_off_t tdb_read_off(struct tdb_context *tdb, tdb_off_t off);
+
+/* Write an offset at an offset. */
+enum TDB_ERROR tdb_write_off(struct tdb_context *tdb, tdb_off_t off,
+			     tdb_off_t val);
+
+/* Clear an ondisk area. */
+enum TDB_ERROR zero_out(struct tdb_context *tdb, tdb_off_t off, tdb_len_t len);
+
+/* Return a non-zero offset between >= start < end in this array (or end). */
+tdb_off_t tdb_find_nonzero_off(struct tdb_context *tdb,
+			       tdb_off_t base,
+			       uint64_t start,
+			       uint64_t end);
+
+/* Return a zero offset in this array, or num. */
+tdb_off_t tdb_find_zero_off(struct tdb_context *tdb, tdb_off_t off,
+			    uint64_t num);
+
+/* Allocate and make a copy of some offset. */
+void *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len_t len);
+
+/* Writes a converted copy of a record. */
+enum TDB_ERROR tdb_write_convert(struct tdb_context *tdb, tdb_off_t off,
+				 const void *rec, size_t len);
+
+/* Reads record and converts it */
+enum TDB_ERROR tdb_read_convert(struct tdb_context *tdb, tdb_off_t off,
+				void *rec, size_t len);
+
+/* Bump the seqnum (caller checks for tdb->flags & TDB_SEQNUM) */
+void tdb_inc_seqnum(struct tdb_context *tdb);
+
+/* lock.c: */
+/* Lock/unlock a range of hashes. */
+enum TDB_ERROR tdb_lock_hashes(struct tdb_context *tdb,
+			       tdb_off_t hash_lock, tdb_len_t hash_range,
+			       int ltype, enum tdb_lock_flags waitflag);
+enum TDB_ERROR tdb_unlock_hashes(struct tdb_context *tdb,
+				 tdb_off_t hash_lock,
+				 tdb_len_t hash_range, int ltype);
+
+/* For closing the file. */
+void tdb_lock_cleanup(struct tdb_context *tdb);
+
+/* Lock/unlock a particular free bucket. */
+enum TDB_ERROR tdb_lock_free_bucket(struct tdb_context *tdb, tdb_off_t b_off,
+				    enum tdb_lock_flags waitflag);
+void tdb_unlock_free_bucket(struct tdb_context *tdb, tdb_off_t b_off);
+
+/* Serialize transaction start. */
+enum TDB_ERROR tdb_transaction_lock(struct tdb_context *tdb, int ltype);
+void tdb_transaction_unlock(struct tdb_context *tdb, int ltype);
+
+/* Do we have any hash locks (ie. via tdb_chainlock) ? */
+bool tdb_has_hash_locks(struct tdb_context *tdb);
+
+/* Lock entire database. */
+enum TDB_ERROR tdb_allrecord_lock(struct tdb_context *tdb, int ltype,
+				  enum tdb_lock_flags flags, bool upgradable);
+void tdb_allrecord_unlock(struct tdb_context *tdb, int ltype);
+enum TDB_ERROR tdb_allrecord_upgrade(struct tdb_context *tdb);
+
+/* Serialize db open. */
+enum TDB_ERROR tdb_lock_open(struct tdb_context *tdb,
+			     int ltype, enum tdb_lock_flags flags);
+void tdb_unlock_open(struct tdb_context *tdb, int ltype);
+bool tdb_has_open_lock(struct tdb_context *tdb);
+
+/* Serialize db expand. */
+enum TDB_ERROR tdb_lock_expand(struct tdb_context *tdb, int ltype);
+void tdb_unlock_expand(struct tdb_context *tdb, int ltype);
+bool tdb_has_expansion_lock(struct tdb_context *tdb);
+
+/* If it needs recovery, grab all the locks and do it. */
+enum TDB_ERROR tdb_lock_and_recover(struct tdb_context *tdb);
+
+/* Default lock and unlock functions. */
+int tdb_fcntl_lock(int fd, int rw, off_t off, off_t len, bool waitflag, void *);
+int tdb_fcntl_unlock(int fd, int rw, off_t off, off_t len, void *);
+
+/* transaction.c: */
+enum TDB_ERROR tdb_transaction_recover(struct tdb_context *tdb);
+tdb_bool_err tdb_needs_recovery(struct tdb_context *tdb);
+
+/* tdb.c: */
+enum TDB_ERROR COLD tdb_logerr(struct tdb_context *tdb,
+			       enum TDB_ERROR ecode,
+			       enum tdb_log_level level,
+			       const char *fmt, ...);
+
+#ifdef TDB_TRACE
+void tdb_trace(struct tdb_context *tdb, const char *op);
+void tdb_trace_seqnum(struct tdb_context *tdb, uint32_t seqnum, const char *op);
+void tdb_trace_open(struct tdb_context *tdb, const char *op,
+		    unsigned hash_size, unsigned tdb_flags, unsigned open_flags);
+void tdb_trace_ret(struct tdb_context *tdb, const char *op, int ret);
+void tdb_trace_retrec(struct tdb_context *tdb, const char *op, TDB_DATA ret);
+void tdb_trace_1rec(struct tdb_context *tdb, const char *op,
+		    TDB_DATA rec);
+void tdb_trace_1rec_ret(struct tdb_context *tdb, const char *op,
+			TDB_DATA rec, int ret);
+void tdb_trace_1rec_retrec(struct tdb_context *tdb, const char *op,
+			   TDB_DATA rec, TDB_DATA ret);
+void tdb_trace_2rec_flag_ret(struct tdb_context *tdb, const char *op,
+			     TDB_DATA rec1, TDB_DATA rec2, unsigned flag,
+			     int ret);
+void tdb_trace_2rec_retrec(struct tdb_context *tdb, const char *op,
+			   TDB_DATA rec1, TDB_DATA rec2, TDB_DATA ret);
+#else
+#define tdb_trace(tdb, op)
+#define tdb_trace_seqnum(tdb, seqnum, op)
+#define tdb_trace_open(tdb, op, hash_size, tdb_flags, open_flags)
+#define tdb_trace_ret(tdb, op, ret)
+#define tdb_trace_retrec(tdb, op, ret)
+#define tdb_trace_1rec(tdb, op, rec)
+#define tdb_trace_1rec_ret(tdb, op, rec, ret)
+#define tdb_trace_1rec_retrec(tdb, op, rec, ret)
+#define tdb_trace_2rec_flag_ret(tdb, op, rec1, rec2, flag, ret)
+#define tdb_trace_2rec_retrec(tdb, op, rec1, rec2, ret)
+#endif /* !TDB_TRACE */
+
+#endif
diff --git a/lib/tdb2/pytdb.c b/lib/tdb2/pytdb.c
new file mode 100644
index 00000000000..c760045508c
--- /dev/null
+++ b/lib/tdb2/pytdb.c
@@ -0,0 +1,586 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   Python interface to tdb2.  Simply modified from tdb1 version.
+
+   Copyright (C) 2004-2006 Tim Potter <tpot@samba.org>
+   Copyright (C) 2007-2008 Jelmer Vernooij <jelmer@samba.org>
+   Copyright (C) 2011 Rusty Russell <rusty@rustcorp.com.au>
+
+     ** NOTE! The following LGPL license applies to the tdb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <Python.h>
+#include "replace.h"
+#include "system/filesys.h"
+
+#ifndef Py_RETURN_NONE
+#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
+#endif
+
+/* Include tdb headers */
+#include <tdb2.h>
+
+typedef struct {
+	PyObject_HEAD
+	struct tdb_context *ctx;
+	bool closed;
+} PyTdbObject;
+
+staticforward PyTypeObject PyTdb;
+
+static void PyErr_SetTDBError(enum TDB_ERROR e)
+{
+	PyErr_SetObject(PyExc_RuntimeError,
+		Py_BuildValue("(i,s)", e, tdb_errorstr(e)));
+}
+
+static TDB_DATA PyString_AsTDB_DATA(PyObject *data)
+{
+	TDB_DATA ret;
+	ret.dptr = (unsigned char *)PyString_AsString(data);
+	ret.dsize = PyString_Size(data);
+	return ret;
+}
+
+static PyObject *PyString_FromTDB_DATA(TDB_DATA data)
+{
+	PyObject *ret = PyString_FromStringAndSize((const char *)data.dptr,
+						   data.dsize);
+	free(data.dptr);
+	return ret;
+}
+
+#define PyErr_TDB_ERROR_IS_ERR_RAISE(ret) \
+	if (ret != TDB_SUCCESS) { \
+		PyErr_SetTDBError(ret); \
+		return NULL; \
+	}
+
+static void stderr_log(struct tdb_context *tdb,
+		       enum tdb_log_level level,
+		       const char *message,
+		       void *data)
+{
+	fprintf(stderr, "%s:%s\n", tdb_name(tdb), message);
+}
+
+static PyObject *py_tdb_open(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+	char *name = NULL;
+	int tdb_flags = TDB_DEFAULT, flags = O_RDWR, mode = 0600;
+	struct tdb_context *ctx;
+	PyTdbObject *ret;
+	union tdb_attribute logattr;
+	const char *kwnames[] = { "name", "tdb_flags", "flags", "mode", NULL };
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", (char **)kwnames, &name, &tdb_flags, &flags, &mode))
+		return NULL;
+
+	if (name == NULL) {
+		tdb_flags |= TDB_INTERNAL;
+	}
+
+	logattr.log.base.attr = TDB_ATTRIBUTE_LOG;
+	logattr.log.base.next = NULL;
+	logattr.log.fn = stderr_log;
+	ctx = tdb_open(name, tdb_flags, flags, mode, &logattr);
+	if (ctx == NULL) {
+		PyErr_SetFromErrno(PyExc_IOError);
+		return NULL;
+	}
+
+	ret = PyObject_New(PyTdbObject, &PyTdb);
+	if (!ret) {
+		tdb_close(ctx);
+		return NULL;
+	}
+
+	ret->ctx = ctx;
+	ret->closed = false;
+	return (PyObject *)ret;
+}
+
+static PyObject *obj_transaction_cancel(PyTdbObject *self)
+{
+	tdb_transaction_cancel(self->ctx);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_transaction_commit(PyTdbObject *self)
+{
+	enum TDB_ERROR ret = tdb_transaction_commit(self->ctx);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_transaction_prepare_commit(PyTdbObject *self)
+{
+	enum TDB_ERROR ret = tdb_transaction_prepare_commit(self->ctx);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_transaction_start(PyTdbObject *self)
+{
+	enum TDB_ERROR ret = tdb_transaction_start(self->ctx);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_lockall(PyTdbObject *self)
+{
+	enum TDB_ERROR ret = tdb_lockall(self->ctx);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_unlockall(PyTdbObject *self)
+{
+	tdb_unlockall(self->ctx);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_lockall_read(PyTdbObject *self)
+{
+	enum TDB_ERROR ret = tdb_lockall_read(self->ctx);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_unlockall_read(PyTdbObject *self)
+{
+	tdb_unlockall_read(self->ctx);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_close(PyTdbObject *self)
+{
+	enum TDB_ERROR ret;
+	if (self->closed)
+		Py_RETURN_NONE;
+	ret = tdb_close(self->ctx);
+	self->closed = true;
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_get(PyTdbObject *self, PyObject *args)
+{
+	TDB_DATA key, data;
+	PyObject *py_key;
+	enum TDB_ERROR ret;
+	if (!PyArg_ParseTuple(args, "O", &py_key))
+		return NULL;
+
+	key = PyString_AsTDB_DATA(py_key);
+	ret = tdb_fetch(self->ctx, key, &data);
+	if (ret == TDB_ERR_NOEXIST)
+		Py_RETURN_NONE;
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	return PyString_FromTDB_DATA(data);
+}
+
+static PyObject *obj_append(PyTdbObject *self, PyObject *args)
+{
+	TDB_DATA key, data;
+	PyObject *py_key, *py_data;
+	enum TDB_ERROR ret;
+	if (!PyArg_ParseTuple(args, "OO", &py_key, &py_data))
+		return NULL;
+
+	key = PyString_AsTDB_DATA(py_key);
+	data = PyString_AsTDB_DATA(py_data);
+
+	ret = tdb_append(self->ctx, key, data);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_firstkey(PyTdbObject *self)
+{
+	enum TDB_ERROR ret;
+	TDB_DATA key;
+
+	ret = tdb_firstkey(self->ctx, &key);
+	if (ret == TDB_ERR_NOEXIST)
+		Py_RETURN_NONE;
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+
+	return PyString_FromTDB_DATA(key);
+}
+
+static PyObject *obj_nextkey(PyTdbObject *self, PyObject *args)
+{
+	TDB_DATA key;
+	PyObject *py_key;
+	enum TDB_ERROR ret;
+	if (!PyArg_ParseTuple(args, "O", &py_key))
+		return NULL;
+
+	/* Malloc here, since tdb_nextkey frees. */
+	key.dsize = PyString_Size(py_key);
+	key.dptr = malloc(key.dsize);
+	memcpy(key.dptr, PyString_AsString(py_key), key.dsize);
+
+	ret = tdb_nextkey(self->ctx, &key);
+	if (ret == TDB_ERR_NOEXIST)
+		Py_RETURN_NONE;
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+
+	return PyString_FromTDB_DATA(key);
+}
+
+static PyObject *obj_delete(PyTdbObject *self, PyObject *args)
+{
+	TDB_DATA key;
+	PyObject *py_key;
+	enum TDB_ERROR ret;
+	if (!PyArg_ParseTuple(args, "O", &py_key))
+		return NULL;
+
+	key = PyString_AsTDB_DATA(py_key);
+	ret = tdb_delete(self->ctx, key);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_has_key(PyTdbObject *self, PyObject *args)
+{
+	TDB_DATA key;
+	enum TDB_ERROR ret;
+	PyObject *py_key;
+	if (!PyArg_ParseTuple(args, "O", &py_key))
+		return NULL;
+
+	key = PyString_AsTDB_DATA(py_key);
+	ret = tdb_exists(self->ctx, key);
+	if (ret == TDB_ERR_NOEXIST)
+		return Py_False;
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	return Py_True;
+}
+
+static PyObject *obj_store(PyTdbObject *self, PyObject *args)
+{
+	TDB_DATA key, value;
+	enum TDB_ERROR ret;
+	int flag = TDB_REPLACE;
+	PyObject *py_key, *py_value;
+
+	if (!PyArg_ParseTuple(args, "OO|i", &py_key, &py_value, &flag))
+		return NULL;
+
+	key = PyString_AsTDB_DATA(py_key);
+	value = PyString_AsTDB_DATA(py_value);
+
+	ret = tdb_store(self->ctx, key, value, flag);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_add_flag(PyTdbObject *self, PyObject *args)
+{
+	unsigned flag;
+
+	if (!PyArg_ParseTuple(args, "I", &flag))
+		return NULL;
+
+	tdb_add_flag(self->ctx, flag);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_remove_flag(PyTdbObject *self, PyObject *args)
+{
+	unsigned flag;
+
+	if (!PyArg_ParseTuple(args, "I", &flag))
+		return NULL;
+
+	tdb_remove_flag(self->ctx, flag);
+	Py_RETURN_NONE;
+}
+
+typedef struct {
+	PyObject_HEAD
+	TDB_DATA current;
+	bool end;
+	PyTdbObject *iteratee;
+} PyTdbIteratorObject;
+
+static PyObject *tdb_iter_next(PyTdbIteratorObject *self)
+{
+	enum TDB_ERROR e;
+	PyObject *ret;
+	if (self->end)
+		return NULL;
+	ret = PyString_FromStringAndSize((const char *)self->current.dptr,
+					 self->current.dsize);
+	e = tdb_nextkey(self->iteratee->ctx, &self->current);
+	if (e == TDB_ERR_NOEXIST)
+		self->end = true;
+	else
+		PyErr_TDB_ERROR_IS_ERR_RAISE(e);
+	return ret;
+}
+
+static void tdb_iter_dealloc(PyTdbIteratorObject *self)
+{
+	Py_DECREF(self->iteratee);
+	PyObject_Del(self);
+}
+
+PyTypeObject PyTdbIterator = {
+	.tp_name = "Iterator",
+	.tp_basicsize = sizeof(PyTdbIteratorObject),
+	.tp_iternext = (iternextfunc)tdb_iter_next,
+	.tp_dealloc = (destructor)tdb_iter_dealloc,
+	.tp_flags = Py_TPFLAGS_DEFAULT,
+	.tp_iter = PyObject_SelfIter,
+};
+
+static PyObject *tdb_object_iter(PyTdbObject *self)
+{
+	PyTdbIteratorObject *ret;
+	enum TDB_ERROR e;
+
+	ret = PyObject_New(PyTdbIteratorObject, &PyTdbIterator);
+	if (!ret)
+		return NULL;
+	e = tdb_firstkey(self->ctx, &ret->current);
+	if (e == TDB_ERR_NOEXIST) {
+		ret->end = true;
+	} else {
+		PyErr_TDB_ERROR_IS_ERR_RAISE(e);
+		ret->end = false;
+	}
+	ret->iteratee = self;
+	Py_INCREF(self);
+	return (PyObject *)ret;
+}
+
+static PyObject *obj_clear(PyTdbObject *self)
+{
+	enum TDB_ERROR ret = tdb_wipe_all(self->ctx);
+	PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+	Py_RETURN_NONE;
+}
+
+static PyObject *obj_enable_seqnum(PyTdbObject *self)
+{
+	tdb_add_flag(self->ctx, TDB_SEQNUM);
+	Py_RETURN_NONE;
+}
+
+static PyMethodDef tdb_object_methods[] = {
+	{ "transaction_cancel", (PyCFunction)obj_transaction_cancel, METH_NOARGS,
+		"S.transaction_cancel() -> None\n"
+		"Cancel the currently active transaction." },
+	{ "transaction_commit", (PyCFunction)obj_transaction_commit, METH_NOARGS,
+		"S.transaction_commit() -> None\n"
+		"Commit the currently active transaction." },
+	{ "transaction_prepare_commit", (PyCFunction)obj_transaction_prepare_commit, METH_NOARGS,
+		"S.transaction_prepare_commit() -> None\n"
+		"Prepare to commit the currently active transaction" },
+	{ "transaction_start", (PyCFunction)obj_transaction_start, METH_NOARGS,
+		"S.transaction_start() -> None\n"
+		"Start a new transaction." },
+	{ "lock_all", (PyCFunction)obj_lockall, METH_NOARGS, NULL },
+	{ "unlock_all", (PyCFunction)obj_unlockall, METH_NOARGS, NULL },
+	{ "read_lock_all", (PyCFunction)obj_lockall_read, METH_NOARGS, NULL },
+	{ "read_unlock_all", (PyCFunction)obj_unlockall_read, METH_NOARGS, NULL },
+	{ "close", (PyCFunction)obj_close, METH_NOARGS, NULL },
+	{ "get", (PyCFunction)obj_get, METH_VARARGS, "S.get(key) -> value\n"
+		"Fetch a value." },
+	{ "append", (PyCFunction)obj_append, METH_VARARGS, "S.append(key, value) -> None\n"
+		"Append data to an existing key." },
+	{ "firstkey", (PyCFunction)obj_firstkey, METH_NOARGS, "S.firstkey() -> data\n"
+		"Return the first key in this database." },
+	{ "nextkey", (PyCFunction)obj_nextkey, METH_NOARGS, "S.nextkey(key) -> data\n"
+		"Return the next key in this database." },
+	{ "delete", (PyCFunction)obj_delete, METH_VARARGS, "S.delete(key) -> None\n"
+		"Delete an entry." },
+	{ "has_key", (PyCFunction)obj_has_key, METH_VARARGS, "S.has_key(key) -> None\n"
+		"Check whether key exists in this database." },
+	{ "store", (PyCFunction)obj_store, METH_VARARGS, "S.store(key, data, flag=REPLACE) -> None"
+		"Store data." },
+	{ "add_flag", (PyCFunction)obj_add_flag, METH_VARARGS, "S.add_flag(flag) -> None" },
+	{ "remove_flag", (PyCFunction)obj_remove_flag, METH_VARARGS, "S.remove_flag(flag) -> None" },
+	{ "iterkeys", (PyCFunction)tdb_object_iter, METH_NOARGS, "S.iterkeys() -> iterator" },
+	{ "clear", (PyCFunction)obj_clear, METH_NOARGS, "S.clear() -> None\n"
+		"Wipe the entire database." },
+	{ "enable_seqnum", (PyCFunction)obj_enable_seqnum, METH_NOARGS,
+		"S.enable_seqnum() -> None" },
+	{ NULL }
+};
+
+static PyObject *obj_get_flags(PyTdbObject *self, void *closure)
+{
+	return PyInt_FromLong(tdb_get_flags(self->ctx));
+}
+
+static PyObject *obj_get_filename(PyTdbObject *self, void *closure)
+{
+	return PyString_FromString(tdb_name(self->ctx));
+}
+
+static PyObject *obj_get_seqnum(PyTdbObject *self, void *closure)
+{
+	return PyInt_FromLong(tdb_get_seqnum(self->ctx));
+}
+
+
+static PyGetSetDef tdb_object_getsetters[] = {
+	{ (char *)"flags", (getter)obj_get_flags, NULL, NULL },
+	{ (char *)"filename", (getter)obj_get_filename, NULL, (char *)"The filename of this TDB file."},
+	{ (char *)"seqnum", (getter)obj_get_seqnum, NULL, NULL },
+	{ NULL }
+};
+
+static PyObject *tdb_object_repr(PyTdbObject *self)
+{
+	if (tdb_get_flags(self->ctx) & TDB_INTERNAL) {
+		return PyString_FromString("Tdb(<internal>)");
+	} else {
+		return PyString_FromFormat("Tdb('%s')", tdb_name(self->ctx));
+	}
+}
+
+static void tdb_object_dealloc(PyTdbObject *self)
+{
+	if (!self->closed)
+		tdb_close(self->ctx);
+	self->ob_type->tp_free(self);
+}
+
+static PyObject *obj_getitem(PyTdbObject *self, PyObject *key)
+{
+	TDB_DATA tkey, val;
+	enum TDB_ERROR ret;
+
+	if (!PyString_Check(key)) {
+		PyErr_SetString(PyExc_TypeError, "Expected string as key");
+		return NULL;
+	}
+
+	tkey.dptr = (unsigned char *)PyString_AsString(key);
+	tkey.dsize = PyString_Size(key);
+
+	ret = tdb_fetch(self->ctx, tkey, &val);
+	if (ret == TDB_ERR_NOEXIST) {
+		PyErr_SetString(PyExc_KeyError, "No such TDB entry");
+		return NULL;
+	} else {
+		PyErr_TDB_ERROR_IS_ERR_RAISE(ret);
+		return PyString_FromTDB_DATA(val);
+	}
+}
+
+static int obj_setitem(PyTdbObject *self, PyObject *key, PyObject *value)
+{
+	TDB_DATA tkey, tval;
+	enum TDB_ERROR ret;
+	if (!PyString_Check(key)) {
+		PyErr_SetString(PyExc_TypeError, "Expected string as key");
+		return -1;
+	}
+
+	tkey = PyString_AsTDB_DATA(key);
+
+	if (value == NULL) {
+		ret = tdb_delete(self->ctx, tkey);
+	} else {
+		if (!PyString_Check(value)) {
+			PyErr_SetString(PyExc_TypeError, "Expected string as value");
+			return -1;
+		}
+
+		tval = PyString_AsTDB_DATA(value);
+
+		ret = tdb_store(self->ctx, tkey, tval, TDB_REPLACE);
+	}
+
+	if (ret != TDB_SUCCESS) {
+		PyErr_SetTDBError(ret);
+		return -1;
+	}
+
+	return ret;
+}
+
+static PyMappingMethods tdb_object_mapping = {
+	.mp_subscript = (binaryfunc)obj_getitem,
+	.mp_ass_subscript = (objobjargproc)obj_setitem,
+};
+static PyTypeObject PyTdb = {
+	.tp_name = "Tdb",
+	.tp_basicsize = sizeof(PyTdbObject),
+	.tp_methods = tdb_object_methods,
+	.tp_getset = tdb_object_getsetters,
+	.tp_new = py_tdb_open,
+	.tp_doc = "A TDB file",
+	.tp_repr = (reprfunc)tdb_object_repr,
+	.tp_dealloc = (destructor)tdb_object_dealloc,
+	.tp_as_mapping = &tdb_object_mapping,
+	.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_ITER,
+	.tp_iter = (getiterfunc)tdb_object_iter,
+};
+
+static PyMethodDef tdb_methods[] = {
+	{ "open", (PyCFunction)py_tdb_open, METH_VARARGS|METH_KEYWORDS, "open(name, hash_size=0, tdb_flags=TDB_DEFAULT, flags=O_RDWR, mode=0600)\n"
+		"Open a TDB file." },
+	{ NULL }
+};
+
+void inittdb(void);
+void inittdb(void)
+{
+	PyObject *m;
+
+	if (PyType_Ready(&PyTdb) < 0)
+		return;
+
+	if (PyType_Ready(&PyTdbIterator) < 0)
+		return;
+
+	m = Py_InitModule3("tdb", tdb_methods, "TDB is a simple key-value database similar to GDBM that supports multiple writers.");
+	if (m == NULL)
+		return;
+
+	PyModule_AddObject(m, "REPLACE", PyInt_FromLong(TDB_REPLACE));
+	PyModule_AddObject(m, "INSERT", PyInt_FromLong(TDB_INSERT));
+	PyModule_AddObject(m, "MODIFY", PyInt_FromLong(TDB_MODIFY));
+
+	PyModule_AddObject(m, "DEFAULT", PyInt_FromLong(TDB_DEFAULT));
+	PyModule_AddObject(m, "INTERNAL", PyInt_FromLong(TDB_INTERNAL));
+	PyModule_AddObject(m, "NOLOCK", PyInt_FromLong(TDB_NOLOCK));
+	PyModule_AddObject(m, "NOMMAP", PyInt_FromLong(TDB_NOMMAP));
+	PyModule_AddObject(m, "CONVERT", PyInt_FromLong(TDB_CONVERT));
+	PyModule_AddObject(m, "NOSYNC", PyInt_FromLong(TDB_NOSYNC));
+	PyModule_AddObject(m, "SEQNUM", PyInt_FromLong(TDB_SEQNUM));
+	PyModule_AddObject(m, "ALLOW_NESTING", PyInt_FromLong(TDB_ALLOW_NESTING));
+
+	PyModule_AddObject(m, "__docformat__", PyString_FromString("restructuredText"));
+
+	PyModule_AddObject(m, "__version__", PyString_FromString(PACKAGE_VERSION));
+
+	Py_INCREF(&PyTdb);
+	PyModule_AddObject(m, "Tdb", (PyObject *)&PyTdb);
+
+	Py_INCREF(&PyTdbIterator);
+}
diff --git a/lib/tdb2/summary.c b/lib/tdb2/summary.c
new file mode 100644
index 00000000000..26cdd3e4fe2
--- /dev/null
+++ b/lib/tdb2/summary.c
@@ -0,0 +1,282 @@
+ /*
+   Trivial Database 2: human-readable summary code
+   Copyright (C) Rusty Russell 2010
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "private.h"
+#include <assert.h>
+#include <ccan/tally/tally.h>
+
+static tdb_off_t count_hash(struct tdb_context *tdb,
+			    tdb_off_t hash_off, unsigned bits)
+{
+	const tdb_off_t *h;
+	tdb_off_t count = 0;
+	unsigned int i;
+
+	h = tdb_access_read(tdb, hash_off, sizeof(*h) << bits, true);
+	if (TDB_PTR_IS_ERR(h)) {
+		return TDB_PTR_ERR(h);
+	}
+	for (i = 0; i < (1 << bits); i++)
+		count += (h[i] != 0);
+
+	tdb_access_release(tdb, h);
+	return count;
+}
+
+static enum TDB_ERROR summarize(struct tdb_context *tdb,
+				struct tally *hashes,
+				struct tally *ftables,
+				struct tally *fr,
+				struct tally *keys,
+				struct tally *data,
+				struct tally *extra,
+				struct tally *uncoal,
+				struct tally *chains)
+{
+	tdb_off_t off;
+	tdb_len_t len;
+	tdb_len_t unc = 0;
+
+	for (off = sizeof(struct tdb_header);
+	     off < tdb->file->map_size;
+	     off += len) {
+		const union {
+			struct tdb_used_record u;
+			struct tdb_free_record f;
+			struct tdb_recovery_record r;
+		} *p;
+		/* We might not be able to get the whole thing. */
+		p = tdb_access_read(tdb, off, sizeof(p->f), true);
+		if (TDB_PTR_IS_ERR(p)) {
+			return TDB_PTR_ERR(p);
+		}
+		if (frec_magic(&p->f) != TDB_FREE_MAGIC) {
+			if (unc > 1) {
+				tally_add(uncoal, unc);
+				unc = 0;
+			}
+		}
+
+		if (p->r.magic == TDB_RECOVERY_INVALID_MAGIC
+		    || p->r.magic == TDB_RECOVERY_MAGIC) {
+			len = sizeof(p->r) + p->r.max_len;
+		} else if (frec_magic(&p->f) == TDB_FREE_MAGIC) {
+			len = frec_len(&p->f);
+			tally_add(fr, len);
+			len += sizeof(p->u);
+			unc++;
+		} else if (rec_magic(&p->u) == TDB_USED_MAGIC) {
+			len = sizeof(p->u)
+				+ rec_key_length(&p->u)
+				+ rec_data_length(&p->u)
+				+ rec_extra_padding(&p->u);
+
+			tally_add(keys, rec_key_length(&p->u));
+			tally_add(data, rec_data_length(&p->u));
+			tally_add(extra, rec_extra_padding(&p->u));
+		} else if (rec_magic(&p->u) == TDB_HTABLE_MAGIC) {
+			tdb_off_t count = count_hash(tdb,
+						     off + sizeof(p->u),
+						     TDB_SUBLEVEL_HASH_BITS);
+			if (TDB_OFF_IS_ERR(count)) {
+				return count;
+			}
+			tally_add(hashes, count);
+			tally_add(extra, rec_extra_padding(&p->u));
+			len = sizeof(p->u)
+				+ rec_data_length(&p->u)
+				+ rec_extra_padding(&p->u);
+		} else if (rec_magic(&p->u) == TDB_FTABLE_MAGIC) {
+			len = sizeof(p->u)
+				+ rec_data_length(&p->u)
+				+ rec_extra_padding(&p->u);
+			tally_add(ftables, rec_data_length(&p->u));
+			tally_add(extra, rec_extra_padding(&p->u));
+		} else if (rec_magic(&p->u) == TDB_CHAIN_MAGIC) {
+			len = sizeof(p->u)
+				+ rec_data_length(&p->u)
+				+ rec_extra_padding(&p->u);
+			tally_add(chains, 1);
+			tally_add(extra, rec_extra_padding(&p->u));
+		} else {
+			len = dead_space(tdb, off);
+			if (TDB_OFF_IS_ERR(len)) {
+				return len;
+			}
+		}
+		tdb_access_release(tdb, p);
+	}
+	if (unc)
+		tally_add(uncoal, unc);
+	return TDB_SUCCESS;
+}
+
+#define SUMMARY_FORMAT \
+	"Size of file/data: %zu/%zu\n" \
+	"Number of records: %zu\n" \
+	"Smallest/average/largest keys: %zu/%zu/%zu\n%s" \
+	"Smallest/average/largest data: %zu/%zu/%zu\n%s" \
+	"Smallest/average/largest padding: %zu/%zu/%zu\n%s" \
+	"Number of free records: %zu\n" \
+	"Smallest/average/largest free records: %zu/%zu/%zu\n%s" \
+	"Number of uncoalesced records: %zu\n" \
+	"Smallest/average/largest uncoalesced runs: %zu/%zu/%zu\n%s" \
+	"Toplevel hash used: %u of %u\n" \
+	"Number of chains: %zu\n" \
+	"Number of subhashes: %zu\n" \
+	"Smallest/average/largest subhash entries: %zu/%zu/%zu\n%s" \
+	"Percentage keys/data/padding/free/rechdrs/freehdrs/hashes: %.0f/%.0f/%.0f/%.0f/%.0f/%.0f/%.0f\n"
+
+#define BUCKET_SUMMARY_FORMAT_A					\
+	"Free bucket %zu: total entries %zu.\n"			\
+	"Smallest/average/largest length: %zu/%zu/%zu\n%s"
+#define BUCKET_SUMMARY_FORMAT_B					\
+	"Free bucket %zu-%zu: total entries %zu.\n"		\
+	"Smallest/average/largest length: %zu/%zu/%zu\n%s"
+
+#define HISTO_WIDTH 70
+#define HISTO_HEIGHT 20
+
+enum TDB_ERROR tdb_summary(struct tdb_context *tdb,
+			   enum tdb_summary_flags flags,
+			   char **summary)
+{
+	tdb_len_t len;
+	struct tally *ftables, *hashes, *freet, *keys, *data, *extra, *uncoal,
+		*chains;
+	char *hashesg, *freeg, *keysg, *datag, *extrag, *uncoalg;
+	enum TDB_ERROR ecode;
+
+	hashesg = freeg = keysg = datag = extrag = uncoalg = NULL;
+
+	ecode = tdb_allrecord_lock(tdb, F_RDLCK, TDB_LOCK_WAIT, false);
+	if (ecode != TDB_SUCCESS) {
+		return tdb->last_error = ecode;
+	}
+
+	ecode = tdb_lock_expand(tdb, F_RDLCK);
+	if (ecode != TDB_SUCCESS) {
+		tdb_allrecord_unlock(tdb, F_RDLCK);
+		return tdb->last_error = ecode;
+	}
+
+	/* Start stats off empty. */
+	ftables = tally_new(HISTO_HEIGHT);
+	hashes = tally_new(HISTO_HEIGHT);
+	freet = tally_new(HISTO_HEIGHT);
+	keys = tally_new(HISTO_HEIGHT);
+	data = tally_new(HISTO_HEIGHT);
+	extra = tally_new(HISTO_HEIGHT);
+	uncoal = tally_new(HISTO_HEIGHT);
+	chains = tally_new(HISTO_HEIGHT);
+	if (!ftables || !hashes || !freet || !keys || !data || !extra
+	    || !uncoal || !chains) {
+		ecode = tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+				   "tdb_summary: failed to allocate"
+				   " tally structures");
+		goto unlock;
+	}
+
+	ecode = summarize(tdb, hashes, ftables, freet, keys, data, extra,
+			  uncoal, chains);
+	if (ecode != TDB_SUCCESS) {
+		goto unlock;
+	}
+
+	if (flags & TDB_SUMMARY_HISTOGRAMS) {
+		hashesg = tally_histogram(hashes, HISTO_WIDTH, HISTO_HEIGHT);
+		freeg = tally_histogram(freet, HISTO_WIDTH, HISTO_HEIGHT);
+		keysg = tally_histogram(keys, HISTO_WIDTH, HISTO_HEIGHT);
+		datag = tally_histogram(data, HISTO_WIDTH, HISTO_HEIGHT);
+		extrag = tally_histogram(extra, HISTO_WIDTH, HISTO_HEIGHT);
+		uncoalg = tally_histogram(uncoal, HISTO_WIDTH, HISTO_HEIGHT);
+	}
+
+	/* 20 is max length of a %llu. */
+	len = strlen(SUMMARY_FORMAT) + 33*20 + 1
+		+ (hashesg ? strlen(hashesg) : 0)
+		+ (freeg ? strlen(freeg) : 0)
+		+ (keysg ? strlen(keysg) : 0)
+		+ (datag ? strlen(datag) : 0)
+		+ (extrag ? strlen(extrag) : 0)
+		+ (uncoalg ? strlen(uncoalg) : 0);
+
+	*summary = malloc(len);
+	if (!*summary) {
+		ecode = tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+				   "tdb_summary: failed to allocate string");
+		goto unlock;
+	}
+
+	sprintf(*summary, SUMMARY_FORMAT,
+		(size_t)tdb->file->map_size,
+		tally_total(keys, NULL) + tally_total(data, NULL),
+		tally_num(keys),
+		tally_min(keys), tally_mean(keys), tally_max(keys),
+		keysg ? keysg : "",
+		tally_min(data), tally_mean(data), tally_max(data),
+		datag ? datag : "",
+		tally_min(extra), tally_mean(extra), tally_max(extra),
+		extrag ? extrag : "",
+		tally_num(freet),
+		tally_min(freet), tally_mean(freet), tally_max(freet),
+		freeg ? freeg : "",
+		tally_total(uncoal, NULL),
+		tally_min(uncoal), tally_mean(uncoal), tally_max(uncoal),
+		uncoalg ? uncoalg : "",
+		(unsigned)count_hash(tdb, offsetof(struct tdb_header,
+						   hashtable),
+				     TDB_TOPLEVEL_HASH_BITS),
+		1 << TDB_TOPLEVEL_HASH_BITS,
+		tally_num(chains),
+		tally_num(hashes),
+		tally_min(hashes), tally_mean(hashes), tally_max(hashes),
+		hashesg ? hashesg : "",
+		tally_total(keys, NULL) * 100.0 / tdb->file->map_size,
+		tally_total(data, NULL) * 100.0 / tdb->file->map_size,
+		tally_total(extra, NULL) * 100.0 / tdb->file->map_size,
+		tally_total(freet, NULL) * 100.0 / tdb->file->map_size,
+		(tally_num(keys) + tally_num(freet) + tally_num(hashes))
+		* sizeof(struct tdb_used_record) * 100.0 / tdb->file->map_size,
+		tally_num(ftables) * sizeof(struct tdb_freetable)
+		* 100.0 / tdb->file->map_size,
+		(tally_num(hashes)
+		 * (sizeof(tdb_off_t) << TDB_SUBLEVEL_HASH_BITS)
+		 + (sizeof(tdb_off_t) << TDB_TOPLEVEL_HASH_BITS)
+		 + sizeof(struct tdb_chain) * tally_num(chains))
+		* 100.0 / tdb->file->map_size);
+
+unlock:
+	free(hashesg);
+	free(freeg);
+	free(keysg);
+	free(datag);
+	free(extrag);
+	free(uncoalg);
+	free(hashes);
+	free(freet);
+	free(keys);
+	free(data);
+	free(extra);
+	free(uncoal);
+	free(ftables);
+	free(chains);
+
+	tdb_allrecord_unlock(tdb, F_RDLCK);
+	tdb_unlock_expand(tdb, F_RDLCK);
+	return tdb->last_error = ecode;
+}
diff --git a/lib/tdb2/tdb.c b/lib/tdb2/tdb.c
new file mode 100644
index 00000000000..753ccb0c8b2
--- /dev/null
+++ b/lib/tdb2/tdb.c
@@ -0,0 +1,486 @@
+#include "private.h"
+#ifndef _SAMBA_BUILD_
+#include <ccan/asprintf/asprintf.h>
+#include <stdarg.h>
+#endif
+
+static enum TDB_ERROR update_rec_hdr(struct tdb_context *tdb,
+				     tdb_off_t off,
+				     tdb_len_t keylen,
+				     tdb_len_t datalen,
+				     struct tdb_used_record *rec,
+				     uint64_t h)
+{
+	uint64_t dataroom = rec_data_length(rec) + rec_extra_padding(rec);
+	enum TDB_ERROR ecode;
+
+	ecode = set_header(tdb, rec, TDB_USED_MAGIC, keylen, datalen,
+			   keylen + dataroom, h);
+	if (ecode == TDB_SUCCESS) {
+		ecode = tdb_write_convert(tdb, off, rec, sizeof(*rec));
+	}
+	return ecode;
+}
+
+static enum TDB_ERROR replace_data(struct tdb_context *tdb,
+				   struct hash_info *h,
+				   struct tdb_data key, struct tdb_data dbuf,
+				   tdb_off_t old_off, tdb_len_t old_room,
+				   bool growing)
+{
+	tdb_off_t new_off;
+	enum TDB_ERROR ecode;
+
+	/* Allocate a new record. */
+	new_off = alloc(tdb, key.dsize, dbuf.dsize, h->h, TDB_USED_MAGIC,
+			growing);
+	if (TDB_OFF_IS_ERR(new_off)) {
+		return new_off;
+	}
+
+	/* We didn't like the existing one: remove it. */
+	if (old_off) {
+		tdb->stats.frees++;
+		ecode = add_free_record(tdb, old_off,
+					sizeof(struct tdb_used_record)
+					+ key.dsize + old_room,
+					TDB_LOCK_WAIT, true);
+		if (ecode == TDB_SUCCESS)
+			ecode = replace_in_hash(tdb, h, new_off);
+	} else {
+		ecode = add_to_hash(tdb, h, new_off);
+	}
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	new_off += sizeof(struct tdb_used_record);
+	ecode = tdb->methods->twrite(tdb, new_off, key.dptr, key.dsize);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	new_off += key.dsize;
+	ecode = tdb->methods->twrite(tdb, new_off, dbuf.dptr, dbuf.dsize);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	if (tdb->flags & TDB_SEQNUM)
+		tdb_inc_seqnum(tdb);
+
+	return TDB_SUCCESS;
+}
+
+static enum TDB_ERROR update_data(struct tdb_context *tdb,
+				  tdb_off_t off,
+				  struct tdb_data dbuf,
+				  tdb_len_t extra)
+{
+	enum TDB_ERROR ecode;
+
+	ecode = tdb->methods->twrite(tdb, off, dbuf.dptr, dbuf.dsize);
+	if (ecode == TDB_SUCCESS && extra) {
+		/* Put a zero in; future versions may append other data. */
+		ecode = tdb->methods->twrite(tdb, off + dbuf.dsize, "", 1);
+	}
+	if (tdb->flags & TDB_SEQNUM)
+		tdb_inc_seqnum(tdb);
+
+	return ecode;
+}
+
+enum TDB_ERROR tdb_store(struct tdb_context *tdb,
+			 struct tdb_data key, struct tdb_data dbuf, int flag)
+{
+	struct hash_info h;
+	tdb_off_t off;
+	tdb_len_t old_room = 0;
+	struct tdb_used_record rec;
+	enum TDB_ERROR ecode;
+
+	off = find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL);
+	if (TDB_OFF_IS_ERR(off)) {
+		return tdb->last_error = off;
+	}
+
+	/* Now we have lock on this hash bucket. */
+	if (flag == TDB_INSERT) {
+		if (off) {
+			ecode = TDB_ERR_EXISTS;
+			goto out;
+		}
+	} else {
+		if (off) {
+			old_room = rec_data_length(&rec)
+				+ rec_extra_padding(&rec);
+			if (old_room >= dbuf.dsize) {
+				/* Can modify in-place.  Easy! */
+				ecode = update_rec_hdr(tdb, off,
+						       key.dsize, dbuf.dsize,
+						       &rec, h.h);
+				if (ecode != TDB_SUCCESS) {
+					goto out;
+				}
+				ecode = update_data(tdb,
+						    off + sizeof(rec)
+						    + key.dsize, dbuf,
+						    old_room - dbuf.dsize);
+				if (ecode != TDB_SUCCESS) {
+					goto out;
+				}
+				tdb_unlock_hashes(tdb, h.hlock_start,
+						  h.hlock_range, F_WRLCK);
+				return tdb->last_error = TDB_SUCCESS;
+			}
+		} else {
+			if (flag == TDB_MODIFY) {
+				/* if the record doesn't exist and we
+				   are in TDB_MODIFY mode then we should fail
+				   the store */
+				ecode = TDB_ERR_NOEXIST;
+				goto out;
+			}
+		}
+	}
+
+	/* If we didn't use the old record, this implies we're growing. */
+	ecode = replace_data(tdb, &h, key, dbuf, off, old_room, off);
+out:
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_WRLCK);
+	return tdb->last_error = ecode;
+}
+
+enum TDB_ERROR tdb_append(struct tdb_context *tdb,
+			  struct tdb_data key, struct tdb_data dbuf)
+{
+	struct hash_info h;
+	tdb_off_t off;
+	struct tdb_used_record rec;
+	tdb_len_t old_room = 0, old_dlen;
+	unsigned char *newdata;
+	struct tdb_data new_dbuf;
+	enum TDB_ERROR ecode;
+
+	off = find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL);
+	if (TDB_OFF_IS_ERR(off)) {
+		return tdb->last_error = off;
+	}
+
+	if (off) {
+		old_dlen = rec_data_length(&rec);
+		old_room = old_dlen + rec_extra_padding(&rec);
+
+		/* Fast path: can append in place. */
+		if (rec_extra_padding(&rec) >= dbuf.dsize) {
+			ecode = update_rec_hdr(tdb, off, key.dsize,
+					       old_dlen + dbuf.dsize, &rec,
+					       h.h);
+			if (ecode != TDB_SUCCESS) {
+				goto out;
+			}
+
+			off += sizeof(rec) + key.dsize + old_dlen;
+			ecode = update_data(tdb, off, dbuf,
+					    rec_extra_padding(&rec));
+			goto out;
+		}
+
+		/* Slow path. */
+		newdata = malloc(key.dsize + old_dlen + dbuf.dsize);
+		if (!newdata) {
+			ecode = tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+					   "tdb_append:"
+					   " failed to allocate %zu bytes",
+					   (size_t)(key.dsize + old_dlen
+						    + dbuf.dsize));
+			goto out;
+		}
+		ecode = tdb->methods->tread(tdb, off + sizeof(rec) + key.dsize,
+					    newdata, old_dlen);
+		if (ecode != TDB_SUCCESS) {
+			goto out_free_newdata;
+		}
+		memcpy(newdata + old_dlen, dbuf.dptr, dbuf.dsize);
+		new_dbuf.dptr = newdata;
+		new_dbuf.dsize = old_dlen + dbuf.dsize;
+	} else {
+		newdata = NULL;
+		new_dbuf = dbuf;
+	}
+
+	/* If they're using tdb_append(), it implies they're growing record. */
+	ecode = replace_data(tdb, &h, key, new_dbuf, off, old_room, true);
+
+out_free_newdata:
+	free(newdata);
+out:
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_WRLCK);
+	return tdb->last_error = ecode;
+}
+
+enum TDB_ERROR tdb_fetch(struct tdb_context *tdb, struct tdb_data key,
+			 struct tdb_data *data)
+{
+	tdb_off_t off;
+	struct tdb_used_record rec;
+	struct hash_info h;
+	enum TDB_ERROR ecode;
+
+	off = find_and_lock(tdb, key, F_RDLCK, &h, &rec, NULL);
+	if (TDB_OFF_IS_ERR(off)) {
+		return tdb->last_error = off;
+	}
+
+	if (!off) {
+		ecode = TDB_ERR_NOEXIST;
+	} else {
+		data->dsize = rec_data_length(&rec);
+		data->dptr = tdb_alloc_read(tdb, off + sizeof(rec) + key.dsize,
+					    data->dsize);
+		if (TDB_PTR_IS_ERR(data->dptr)) {
+			ecode = TDB_PTR_ERR(data->dptr);
+		} else
+			ecode = TDB_SUCCESS;
+	}
+
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_RDLCK);
+	return tdb->last_error = ecode;
+}
+
+bool tdb_exists(struct tdb_context *tdb, TDB_DATA key)
+{
+	tdb_off_t off;
+	struct tdb_used_record rec;
+	struct hash_info h;
+
+	off = find_and_lock(tdb, key, F_RDLCK, &h, &rec, NULL);
+	if (TDB_OFF_IS_ERR(off)) {
+		tdb->last_error = off;
+		return false;
+	}
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_RDLCK);
+
+	tdb->last_error = TDB_SUCCESS;
+	return off ? true : false;
+}
+
+enum TDB_ERROR tdb_delete(struct tdb_context *tdb, struct tdb_data key)
+{
+	tdb_off_t off;
+	struct tdb_used_record rec;
+	struct hash_info h;
+	enum TDB_ERROR ecode;
+
+	off = find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL);
+	if (TDB_OFF_IS_ERR(off)) {
+		return tdb->last_error = off;
+	}
+
+	if (!off) {
+		ecode = TDB_ERR_NOEXIST;
+		goto unlock;
+	}
+
+	ecode = delete_from_hash(tdb, &h);
+	if (ecode != TDB_SUCCESS) {
+		goto unlock;
+	}
+
+	/* Free the deleted entry. */
+	tdb->stats.frees++;
+	ecode = add_free_record(tdb, off,
+				sizeof(struct tdb_used_record)
+				+ rec_key_length(&rec)
+				+ rec_data_length(&rec)
+				+ rec_extra_padding(&rec),
+				TDB_LOCK_WAIT, true);
+
+	if (tdb->flags & TDB_SEQNUM)
+		tdb_inc_seqnum(tdb);
+
+unlock:
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_WRLCK);
+	return tdb->last_error = ecode;
+}
+
+unsigned int tdb_get_flags(struct tdb_context *tdb)
+{
+	return tdb->flags;
+}
+
+void tdb_add_flag(struct tdb_context *tdb, unsigned flag)
+{
+	if (tdb->flags & TDB_INTERNAL) {
+		tdb->last_error = tdb_logerr(tdb, TDB_ERR_EINVAL,
+					     TDB_LOG_USE_ERROR,
+					     "tdb_add_flag: internal db");
+		return;
+	}
+	switch (flag) {
+	case TDB_NOLOCK:
+		tdb->flags |= TDB_NOLOCK;
+		break;
+	case TDB_NOMMAP:
+		tdb->flags |= TDB_NOMMAP;
+		tdb_munmap(tdb->file);
+		break;
+	case TDB_NOSYNC:
+		tdb->flags |= TDB_NOSYNC;
+		break;
+	case TDB_SEQNUM:
+		tdb->flags |= TDB_SEQNUM;
+		break;
+	case TDB_ALLOW_NESTING:
+		tdb->flags |= TDB_ALLOW_NESTING;
+		break;
+	default:
+		tdb->last_error = tdb_logerr(tdb, TDB_ERR_EINVAL,
+					     TDB_LOG_USE_ERROR,
+					     "tdb_add_flag: Unknown flag %u",
+					     flag);
+	}
+}
+
+void tdb_remove_flag(struct tdb_context *tdb, unsigned flag)
+{
+	if (tdb->flags & TDB_INTERNAL) {
+		tdb->last_error = tdb_logerr(tdb, TDB_ERR_EINVAL,
+					     TDB_LOG_USE_ERROR,
+					     "tdb_remove_flag: internal db");
+		return;
+	}
+	switch (flag) {
+	case TDB_NOLOCK:
+		tdb->flags &= ~TDB_NOLOCK;
+		break;
+	case TDB_NOMMAP:
+		tdb->flags &= ~TDB_NOMMAP;
+		tdb_mmap(tdb);
+		break;
+	case TDB_NOSYNC:
+		tdb->flags &= ~TDB_NOSYNC;
+		break;
+	case TDB_SEQNUM:
+		tdb->flags &= ~TDB_SEQNUM;
+		break;
+	case TDB_ALLOW_NESTING:
+		tdb->flags &= ~TDB_ALLOW_NESTING;
+		break;
+	default:
+		tdb->last_error = tdb_logerr(tdb, TDB_ERR_EINVAL,
+					     TDB_LOG_USE_ERROR,
+					     "tdb_remove_flag: Unknown flag %u",
+					     flag);
+	}
+}
+
+const char *tdb_errorstr(enum TDB_ERROR ecode)
+{
+	/* Gcc warns if you miss a case in the switch, so use that. */
+	switch (ecode) {
+	case TDB_SUCCESS: return "Success";
+	case TDB_ERR_CORRUPT: return "Corrupt database";
+	case TDB_ERR_IO: return "IO Error";
+	case TDB_ERR_LOCK: return "Locking error";
+	case TDB_ERR_OOM: return "Out of memory";
+	case TDB_ERR_EXISTS: return "Record exists";
+	case TDB_ERR_EINVAL: return "Invalid parameter";
+	case TDB_ERR_NOEXIST: return "Record does not exist";
+	case TDB_ERR_RDONLY: return "write not permitted";
+	}
+	return "Invalid error code";
+}
+
+enum TDB_ERROR tdb_error(struct tdb_context *tdb)
+{
+	return tdb->last_error;
+}
+
+enum TDB_ERROR COLD tdb_logerr(struct tdb_context *tdb,
+			       enum TDB_ERROR ecode,
+			       enum tdb_log_level level,
+			       const char *fmt, ...)
+{
+	char *message;
+	va_list ap;
+	size_t len;
+	/* tdb_open paths care about errno, so save it. */
+	int saved_errno = errno;
+
+	if (!tdb->log_fn)
+		return ecode;
+
+	va_start(ap, fmt);
+	len = vasprintf(&message, fmt, ap);
+	va_end(ap);
+
+	if (len < 0) {
+		tdb->log_fn(tdb, TDB_LOG_ERROR,
+			    "out of memory formatting message:", tdb->log_data);
+		tdb->log_fn(tdb, level, fmt, tdb->log_data);
+	} else {
+		tdb->log_fn(tdb, level, message, tdb->log_data);
+		free(message);
+	}
+	errno = saved_errno;
+	return ecode;
+}
+
+enum TDB_ERROR tdb_parse_record_(struct tdb_context *tdb,
+				 TDB_DATA key,
+				 enum TDB_ERROR (*parse)(TDB_DATA k,
+							 TDB_DATA d,
+							 void *data),
+				 void *data)
+{
+	tdb_off_t off;
+	struct tdb_used_record rec;
+	struct hash_info h;
+	enum TDB_ERROR ecode;
+
+	off = find_and_lock(tdb, key, F_RDLCK, &h, &rec, NULL);
+	if (TDB_OFF_IS_ERR(off)) {
+		return tdb->last_error = off;
+	}
+
+	if (!off) {
+		ecode = TDB_ERR_NOEXIST;
+	} else {
+		const void *dptr;
+		dptr = tdb_access_read(tdb, off + sizeof(rec) + key.dsize,
+				       rec_data_length(&rec), false);
+		if (TDB_PTR_IS_ERR(dptr)) {
+			ecode = TDB_PTR_ERR(dptr);
+		} else {
+			TDB_DATA d = tdb_mkdata(dptr, rec_data_length(&rec));
+
+			ecode = parse(key, d, data);
+			tdb_access_release(tdb, dptr);
+		}
+	}
+
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_RDLCK);
+	return tdb->last_error = ecode;
+}
+
+const char *tdb_name(const struct tdb_context *tdb)
+{
+	return tdb->name;
+}
+
+int64_t tdb_get_seqnum(struct tdb_context *tdb)
+{
+	tdb_off_t off = tdb_read_off(tdb, offsetof(struct tdb_header, seqnum));
+	if (TDB_OFF_IS_ERR(off))
+		tdb->last_error = off;
+	else
+		tdb->last_error = TDB_SUCCESS;
+	return off;
+}
+
+
+int tdb_fd(const struct tdb_context *tdb)
+{
+	return tdb->file->fd;
+}
diff --git a/lib/tdb2/tdb2.h b/lib/tdb2/tdb2.h
new file mode 100644
index 00000000000..3f80793d760
--- /dev/null
+++ b/lib/tdb2/tdb2.h
@@ -0,0 +1,848 @@
+#ifndef CCAN_TDB2_H
+#define CCAN_TDB2_H
+
+/*
+   TDB version 2: trivial database library
+
+   Copyright (C) Andrew Tridgell 1999-2004
+   Copyright (C) Rusty Russell 2010-2011
+
+     ** NOTE! The following LGPL license applies to the tdb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#ifndef _SAMBA_BUILD_
+/* For mode_t */
+#include <sys/types.h>
+/* For O_* flags. */
+#include <sys/stat.h>
+/* For sig_atomic_t. */
+#include <signal.h>
+/* For uint64_t */
+#include <stdint.h>
+/* For bool */
+#include <stdbool.h>
+/* For memcmp */
+#include <string.h>
+#else
+#include "replace.h"
+#endif
+#include <ccan/compiler/compiler.h>
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <ccan/cast/cast.h>
+
+union tdb_attribute;
+struct tdb_context;
+
+/**
+ * tdb_open - open a database file
+ * @name: the file name (can be NULL if flags contains TDB_INTERNAL)
+ * @tdb_flags: options for this database
+ * @open_flags: flags argument for tdb's open() call.
+ * @mode: mode argument for tdb's open() call.
+ * @attributes: linked list of extra attributes for this tdb.
+ *
+ * This call opens (and potentially creates) a database file.
+ * Multiple processes can have the TDB file open at once.
+ *
+ * On failure it will return NULL, and set errno: it may also call
+ * any log attribute found in @attributes.
+ *
+ * See also:
+ *	union tdb_attribute
+ */
+struct tdb_context *tdb_open(const char *name, int tdb_flags,
+			     int open_flags, mode_t mode,
+			     union tdb_attribute *attributes);
+
+
+/* flags for tdb_open() */
+#define TDB_DEFAULT 0 /* just a readability place holder */
+#define TDB_INTERNAL 2 /* don't store on disk */
+#define TDB_NOLOCK   4 /* don't do any locking */
+#define TDB_NOMMAP   8 /* don't use mmap */
+#define TDB_CONVERT 16 /* convert endian */
+#define TDB_NOSYNC   64 /* don't use synchronous transactions */
+#define TDB_SEQNUM   128 /* maintain a sequence number */
+#define TDB_ALLOW_NESTING   256 /* fake nested transactions */
+
+/**
+ * tdb_close - close and free a tdb.
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This always succeeds, in that @tdb is unusable after this call.  But if
+ * some unexpected error occurred while closing, it will return non-zero
+ * (the only clue as to cause will be via the log attribute).
+ */
+int tdb_close(struct tdb_context *tdb);
+
+/**
+ * struct tdb_data - representation of keys or values.
+ * @dptr: the data pointer
+ * @dsize: the size of the data pointed to by dptr.
+ *
+ * This is the "blob" representation of keys and data used by TDB.
+ */
+typedef struct tdb_data {
+	unsigned char *dptr;
+	size_t dsize;
+} TDB_DATA;
+
+/**
+ * enum TDB_ERROR - error returns for TDB
+ *
+ * See Also:
+ *	tdb_errorstr()
+ */
+enum TDB_ERROR {
+	TDB_SUCCESS	= 0,	/* No error. */
+	TDB_ERR_CORRUPT = -1,	/* We read the db, and it was bogus. */
+	TDB_ERR_IO	= -2,	/* We couldn't read/write the db. */
+	TDB_ERR_LOCK	= -3,	/* Locking failed. */
+	TDB_ERR_OOM	= -4,	/* Out of Memory. */
+	TDB_ERR_EXISTS	= -5,	/* The key already exists. */
+	TDB_ERR_NOEXIST	= -6,	/* The key does not exist. */
+	TDB_ERR_EINVAL	= -7,	/* You're using it wrong. */
+	TDB_ERR_RDONLY	= -8,	/* The database is read-only. */
+	TDB_ERR_LAST = TDB_ERR_RDONLY
+};
+
+/**
+ * tdb_store - store a key/value pair in a tdb.
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key
+ * @dbuf: the data to associate with the key.
+ * @flag: TDB_REPLACE, TDB_INSERT or TDB_MODIFY.
+ *
+ * This inserts (or overwrites) a key/value pair in the TDB.  If flag
+ * is TDB_REPLACE, it doesn't matter whether the key exists or not;
+ * TDB_INSERT means it must not exist (returns TDB_ERR_EXISTS otherwise),
+ * and TDB_MODIFY means it must exist (returns TDB_ERR_NOEXIST otherwise).
+ *
+ * On success, this returns TDB_SUCCESS.
+ *
+ * See also:
+ *	tdb_fetch, tdb_transaction_start, tdb_append, tdb_delete.
+ */
+enum TDB_ERROR tdb_store(struct tdb_context *tdb,
+			 struct tdb_data key,
+			 struct tdb_data dbuf,
+			 int flag);
+
+/* flags to tdb_store() */
+#define TDB_REPLACE 1		/* A readability place holder */
+#define TDB_INSERT 2 		/* Don't overwrite an existing entry */
+#define TDB_MODIFY 3		/* Don't create an existing entry    */
+
+/**
+ * tdb_fetch - fetch a value from a tdb.
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key
+ * @data: pointer to data.
+ *
+ * This looks up a key in the database and sets it in @data.
+ *
+ * If it returns TDB_SUCCESS, the key was found: it is your
+ * responsibility to call free() on @data->dptr.
+ *
+ * Otherwise, it returns an error (usually, TDB_ERR_NOEXIST) and @data is
+ * undefined.
+ */
+enum TDB_ERROR tdb_fetch(struct tdb_context *tdb, struct tdb_data key,
+			 struct tdb_data *data);
+
+/**
+ * tdb_errorstr - map the tdb error onto a constant readable string
+ * @ecode: the enum TDB_ERROR to map.
+ *
+ * This is useful for displaying errors to users.
+ */
+const char *tdb_errorstr(enum TDB_ERROR ecode);
+
+/**
+ * tdb_append - append a value to a key/value pair in a tdb.
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key
+ * @dbuf: the data to append.
+ *
+ * This is equivalent to fetching a record, reallocating .dptr to add the
+ * data, and writing it back, only it's much more efficient.  If the key
+ * doesn't exist, it's equivalent to tdb_store (with an additional hint that
+ * you expect to expand the record in future).
+ *
+ * See Also:
+ *	tdb_fetch(), tdb_store()
+ */
+enum TDB_ERROR tdb_append(struct tdb_context *tdb,
+			  struct tdb_data key, struct tdb_data dbuf);
+
+/**
+ * tdb_delete - delete a key from a tdb.
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key to delete.
+ *
+ * Returns TDB_SUCCESS on success, or an error (usually TDB_ERR_NOEXIST).
+ *
+ * See Also:
+ *	tdb_fetch(), tdb_store()
+ */
+enum TDB_ERROR tdb_delete(struct tdb_context *tdb, struct tdb_data key);
+
+/**
+ * tdb_exists - does a key exist in the database?
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key to search for.
+ *
+ * Returns true if it exists, or false if it doesn't or any other error.
+ */
+bool tdb_exists(struct tdb_context *tdb, TDB_DATA key);
+
+/**
+ * tdb_deq - are struct tdb_data equal?
+ * @a: one struct tdb_data
+ * @b: another struct tdb_data
+ */
+static inline bool tdb_deq(struct tdb_data a, struct tdb_data b)
+{
+	return a.dsize == b.dsize && memcmp(a.dptr, b.dptr, a.dsize) == 0;
+}
+
+/**
+ * tdb_mkdata - make a struct tdb_data from const data
+ * @p: the constant pointer
+ * @len: the length
+ *
+ * As the dptr member of struct tdb_data is not constant, you need to
+ * cast it.  This function keeps thost casts in one place, as well as
+ * suppressing the warning some compilers give when casting away a
+ * qualifier (eg. gcc with -Wcast-qual)
+ */
+static inline struct tdb_data tdb_mkdata(const void *p, size_t len)
+{
+	struct tdb_data d;
+	d.dptr = cast_const(void *, p);
+	d.dsize = len;
+	return d;
+}
+
+/**
+ * tdb_transaction_start - start a transaction
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This begins a series of atomic operations.  Other processes will be able
+ * to read the tdb, but not alter it (they will block), nor will they see
+ * any changes until tdb_transaction_commit() is called.
+ *
+ * Note that if the TDB_ALLOW_NESTING flag is set, a tdb_transaction_start()
+ * within a transaction will succeed, but it's not a real transaction:
+ * (1) An inner transaction which is committed is not actually committed until
+ *     the outer transaction is; if the outer transaction is cancelled, the
+ *     inner ones are discarded.
+ * (2) tdb_transaction_cancel() marks the outer transaction as having an error,
+ *     so the final tdb_transaction_commit() will fail.
+ * (3) the outer transaction will see the results of the inner transaction.
+ *
+ * See Also:
+ *	tdb_transaction_cancel, tdb_transaction_commit.
+ */
+enum TDB_ERROR tdb_transaction_start(struct tdb_context *tdb);
+
+/**
+ * tdb_transaction_cancel - abandon a transaction
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This aborts a transaction, discarding any changes which were made.
+ * tdb_close() does this implicitly.
+ */
+void tdb_transaction_cancel(struct tdb_context *tdb);
+
+/**
+ * tdb_transaction_commit - commit a transaction
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This completes a transaction, writing any changes which were made.
+ *
+ * fsync() is used to commit the transaction (unless TDB_NOSYNC is set),
+ * making it robust against machine crashes, but very slow compared to
+ * other TDB operations.
+ *
+ * A failure can only be caused by unexpected errors (eg. I/O or
+ * memory); this is no point looping on transaction failure.
+ *
+ * See Also:
+ *	tdb_transaction_prepare_commit()
+ */
+enum TDB_ERROR tdb_transaction_commit(struct tdb_context *tdb);
+
+/**
+ * tdb_transaction_prepare_commit - prepare to commit a transaction
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This ensures we have the resources to commit a transaction (using
+ * tdb_transaction_commit): if this succeeds then a transaction will only
+ * fail if the write() or fsync() calls fail.
+ *
+ * If this fails you must still call tdb_transaction_cancel() to cancel
+ * the transaction.
+ *
+ * See Also:
+ *	tdb_transaction_commit()
+ */
+enum TDB_ERROR tdb_transaction_prepare_commit(struct tdb_context *tdb);
+
+/**
+ * tdb_traverse - traverse a TDB
+ * @tdb: the tdb context returned from tdb_open()
+ * @fn: the function to call for every key/value pair (or NULL)
+ * @p: the pointer to hand to @f
+ *
+ * This walks the TDB until all they keys have been traversed, or @fn
+ * returns non-zero.  If the traverse function or other processes are
+ * changing data or adding or deleting keys, the traverse may be
+ * unreliable: keys may be skipped or (rarely) visited twice.
+ *
+ * There is one specific exception: the special case of deleting the
+ * current key does not undermine the reliability of the traversal.
+ *
+ * On success, returns the number of keys iterated.  On error returns
+ * a negative enum TDB_ERROR value.
+ */
+#define tdb_traverse(tdb, fn, p)					\
+	tdb_traverse_(tdb, typesafe_cb_preargs(int, void *, (fn), (p),	\
+					       struct tdb_context *,	\
+					       TDB_DATA, TDB_DATA), (p))
+
+int64_t tdb_traverse_(struct tdb_context *tdb,
+		      int (*fn)(struct tdb_context *,
+				TDB_DATA, TDB_DATA, void *), void *p);
+
+/**
+ * tdb_parse_record - operate directly on data in the database.
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key whose record we should hand to @parse
+ * @parse: the function to call for the data
+ * @data: the private pointer to hand to @parse (types must match).
+ *
+ * This avoids a copy for many cases, by handing you a pointer into
+ * the memory-mapped database.  It also locks the record to prevent
+ * other accesses at the same time.
+ *
+ * Do not alter the data handed to parse()!
+ */
+#define tdb_parse_record(tdb, key, parse, data)				\
+	tdb_parse_record_((tdb), (key),					\
+			  typesafe_cb_preargs(enum TDB_ERROR, void *,	\
+					      (parse), (data),		\
+					      TDB_DATA, TDB_DATA), (data))
+
+enum TDB_ERROR tdb_parse_record_(struct tdb_context *tdb,
+				 TDB_DATA key,
+				 enum TDB_ERROR (*parse)(TDB_DATA k,
+							 TDB_DATA d,
+							 void *data),
+				 void *data);
+
+/**
+ * tdb_get_seqnum - get a database sequence number
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This returns a sequence number: any change to the database from a
+ * tdb context opened with the TDB_SEQNUM flag will cause that number
+ * to increment.  Note that the incrementing is unreliable (it is done
+ * without locking), so this is only useful as an optimization.
+ *
+ * For example, you may have a regular database backup routine which
+ * does not operate if the sequence number is unchanged.  In the
+ * unlikely event of a failed increment, it will be backed up next
+ * time any way.
+ *
+ * Returns an enum TDB_ERROR (ie. negative) on error.
+ */
+int64_t tdb_get_seqnum(struct tdb_context *tdb);
+
+/**
+ * tdb_firstkey - get the "first" key in a TDB
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: pointer to key.
+ *
+ * This returns an arbitrary key in the database; with tdb_nextkey() it allows
+ * open-coded traversal of the database, though it is slightly less efficient
+ * than tdb_traverse.
+ *
+ * It is your responsibility to free @key->dptr on success.
+ *
+ * Returns TDB_ERR_NOEXIST if the database is empty.
+ */
+enum TDB_ERROR tdb_firstkey(struct tdb_context *tdb, struct tdb_data *key);
+
+/**
+ * tdb_nextkey - get the "next" key in a TDB
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: a key returned by tdb_firstkey() or tdb_nextkey().
+ *
+ * This returns another key in the database; it will free @key.dptr for
+ * your convenience.
+ *
+ * Returns TDB_ERR_NOEXIST if there are no more keys.
+ */
+enum TDB_ERROR tdb_nextkey(struct tdb_context *tdb, struct tdb_data *key);
+
+/**
+ * tdb_chainlock - lock a record in the TDB
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key to lock.
+ *
+ * This prevents any access occurring to a group of keys including @key,
+ * even if @key does not exist.  This allows primitive atomic updates of
+ * records without using transactions.
+ *
+ * You cannot begin a transaction while holding a tdb_chainlock(), nor can
+ * you do any operations on any other keys in the database.  This also means
+ * that you cannot hold more than one tdb_chainlock() at a time.
+ *
+ * See Also:
+ *	tdb_chainunlock()
+ */
+enum TDB_ERROR tdb_chainlock(struct tdb_context *tdb, TDB_DATA key);
+
+/**
+ * tdb_chainunlock - unlock a record in the TDB
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key to unlock.
+ *
+ * The key must have previously been locked by tdb_chainlock().
+ */
+void tdb_chainunlock(struct tdb_context *tdb, TDB_DATA key);
+
+/**
+ * tdb_chainlock_read - lock a record in the TDB, for reading
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key to lock.
+ *
+ * This prevents any changes from occurring to a group of keys including @key,
+ * even if @key does not exist.  This allows primitive atomic updates of
+ * records without using transactions.
+ *
+ * You cannot begin a transaction while holding a tdb_chainlock_read(), nor can
+ * you do any operations on any other keys in the database.  This also means
+ * that you cannot hold more than one tdb_chainlock()/read() at a time.
+ *
+ * See Also:
+ *	tdb_chainlock()
+ */
+enum TDB_ERROR tdb_chainlock_read(struct tdb_context *tdb, TDB_DATA key);
+
+/**
+ * tdb_chainunlock_read - unlock a record in the TDB for reading
+ * @tdb: the tdb context returned from tdb_open()
+ * @key: the key to unlock.
+ *
+ * The key must have previously been locked by tdb_chainlock_read().
+ */
+void tdb_chainunlock_read(struct tdb_context *tdb, TDB_DATA key);
+
+/**
+ * tdb_lockall - lock the entire TDB
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * You cannot hold a tdb_chainlock while calling this.  It nests, so you
+ * must call tdb_unlockall as many times as you call tdb_lockall.
+ */
+enum TDB_ERROR tdb_lockall(struct tdb_context *tdb);
+
+/**
+ * tdb_unlockall - unlock the entire TDB
+ * @tdb: the tdb context returned from tdb_open()
+ */
+void tdb_unlockall(struct tdb_context *tdb);
+
+/**
+ * tdb_lockall_read - lock the entire TDB for reading
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This prevents others writing to the database, eg. tdb_delete, tdb_store,
+ * tdb_append, but not tdb_fetch.
+ *
+ * You cannot hold a tdb_chainlock while calling this.  It nests, so you
+ * must call tdb_unlockall_read as many times as you call tdb_lockall_read.
+ */
+enum TDB_ERROR tdb_lockall_read(struct tdb_context *tdb);
+
+/**
+ * tdb_unlockall_read - unlock the entire TDB for reading
+ * @tdb: the tdb context returned from tdb_open()
+ */
+void tdb_unlockall_read(struct tdb_context *tdb);
+
+/**
+ * tdb_wipe_all - wipe the database clean
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * Completely erase the database.  This is faster than iterating through
+ * each key and doing tdb_delete.
+ */
+enum TDB_ERROR tdb_wipe_all(struct tdb_context *tdb);
+
+/**
+ * tdb_check - check a TDB for consistency
+ * @tdb: the tdb context returned from tdb_open()
+ * @check: function to check each key/data pair (or NULL)
+ * @data: argument for @check, must match type.
+ *
+ * This performs a consistency check of the open database, optionally calling
+ * a check() function on each record so you can do your own data consistency
+ * checks as well.  If check() returns an error, that is returned from
+ * tdb_check().
+ *
+ * Returns TDB_SUCCESS or an error.
+ */
+#define tdb_check(tdb, check, data)					\
+	tdb_check_((tdb), typesafe_cb_preargs(enum TDB_ERROR, void *,	\
+					      (check), (data),		\
+					      struct tdb_data,		\
+					      struct tdb_data),		\
+		   (data))
+
+enum TDB_ERROR tdb_check_(struct tdb_context *tdb,
+			  enum TDB_ERROR (*check)(struct tdb_data k,
+						  struct tdb_data d,
+						  void *data),
+			  void *data);
+
+/**
+ * tdb_error - get the last error (not threadsafe)
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * Returns the last error returned by a TDB function.
+ *
+ * This makes porting from TDB1 easier, but note that the last error is not
+ * reliable in threaded programs.
+ */
+enum TDB_ERROR tdb_error(struct tdb_context *tdb);
+
+/**
+ * enum tdb_summary_flags - flags for tdb_summary.
+ */
+enum tdb_summary_flags {
+	TDB_SUMMARY_HISTOGRAMS = 1 /* Draw graphs in the summary. */
+};
+
+/**
+ * tdb_summary - return a string describing the TDB state
+ * @tdb: the tdb context returned from tdb_open()
+ * @flags: flags to control the summary output.
+ * @summary: pointer to string to allocate.
+ *
+ * This returns a developer-readable string describing the overall
+ * state of the tdb, such as the percentage used and sizes of records.
+ * It is designed to provide information about the tdb at a glance
+ * without displaying any keys or data in the database.
+ *
+ * On success, sets @summary to point to a malloc()'ed nul-terminated
+ * multi-line string.  It is your responsibility to free() it.
+ */
+enum TDB_ERROR tdb_summary(struct tdb_context *tdb,
+			   enum tdb_summary_flags flags,
+			   char **summary);
+
+
+/**
+ * tdb_get_flags - return the flags for a tdb
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This returns the flags on the current tdb.  Some of these are caused by
+ * the flags argument to tdb_open(), others (such as TDB_CONVERT) are
+ * intuited.
+ */
+unsigned int tdb_get_flags(struct tdb_context *tdb);
+
+/**
+ * tdb_add_flag - set a flag for a tdb
+ * @tdb: the tdb context returned from tdb_open()
+ * @flag: one of TDB_NOLOCK, TDB_NOMMAP, TDB_NOSYNC or TDB_ALLOW_NESTING.
+ *
+ * You can use this to set a flag on the TDB.  You cannot set these flags
+ * on a TDB_INTERNAL tdb.
+ */
+void tdb_add_flag(struct tdb_context *tdb, unsigned flag);
+
+/**
+ * tdb_remove_flag - unset a flag for a tdb
+ * @tdb: the tdb context returned from tdb_open()
+ * @flag: one of TDB_NOLOCK, TDB_NOMMAP, TDB_NOSYNC or TDB_ALLOW_NESTING.
+ *
+ * You can use this to clear a flag on the TDB.  You cannot clear flags
+ * on a TDB_INTERNAL tdb.
+ */
+void tdb_remove_flag(struct tdb_context *tdb, unsigned flag);
+
+/**
+ * enum tdb_attribute_type - descriminator for union tdb_attribute.
+ */
+enum tdb_attribute_type {
+	TDB_ATTRIBUTE_LOG = 0,
+	TDB_ATTRIBUTE_HASH = 1,
+	TDB_ATTRIBUTE_SEED = 2,
+	TDB_ATTRIBUTE_STATS = 3,
+	TDB_ATTRIBUTE_OPENHOOK = 4,
+	TDB_ATTRIBUTE_FLOCK = 5
+};
+
+/**
+ * tdb_get_attribute - get an attribute for an existing tdb
+ * @tdb: the tdb context returned from tdb_open()
+ * @attr: the union tdb_attribute to set.
+ *
+ * This gets an attribute from a TDB which has previously been set (or
+ * may return the default values).  Set @attr.base.attr to the
+ * attribute type you want get.
+ *
+ * Currently this does not work for TDB_ATTRIBUTE_OPENHOOK.
+ */
+enum TDB_ERROR tdb_get_attribute(struct tdb_context *tdb,
+				 union tdb_attribute *attr);
+
+/**
+ * tdb_set_attribute - set an attribute for an existing tdb
+ * @tdb: the tdb context returned from tdb_open()
+ * @attr: the union tdb_attribute to set.
+ *
+ * This sets an attribute on a TDB, overriding any previous attribute
+ * of the same type.  It returns TDB_ERR_EINVAL if the attribute is
+ * unknown or invalid.
+ *
+ * Note that TDB_ATTRIBUTE_HASH, TDB_ATTRIBUTE_SEED and
+ * TDB_ATTRIBUTE_OPENHOOK cannot currently be set after tdb_open.
+ */
+enum TDB_ERROR tdb_set_attribute(struct tdb_context *tdb,
+				 const union tdb_attribute *attr);
+
+/**
+ * tdb_unset_attribute - reset an attribute for an existing tdb
+ * @tdb: the tdb context returned from tdb_open()
+ * @type: the attribute type to unset.
+ *
+ * This unsets an attribute on a TDB, returning it to the defaults
+ * (where applicable).
+ *
+ * Note that it only makes sense for TDB_ATTRIBUTE_LOG and TDB_ATTRIBUTE_FLOCK
+ * to be unset.
+ */
+void tdb_unset_attribute(struct tdb_context *tdb,
+			 enum tdb_attribute_type type);
+
+/**
+ * tdb_name - get the name of a tdb
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This returns a copy of the name string, made at tdb_open() time.  If that
+ * argument was NULL (possible for a TDB_INTERNAL db) this will return NULL.
+ *
+ * This is mostly useful for logging.
+ */
+const char *tdb_name(const struct tdb_context *tdb);
+
+/**
+ * tdb_fd - get the file descriptor of a tdb
+ * @tdb: the tdb context returned from tdb_open()
+ *
+ * This returns the file descriptor for the underlying database file, or -1
+ * for TDB_INTERNAL.
+ */
+int tdb_fd(const struct tdb_context *tdb);
+
+/**
+ * struct tdb_attribute_base - common fields for all tdb attributes.
+ */
+struct tdb_attribute_base {
+	enum tdb_attribute_type attr;
+	union tdb_attribute *next;
+};
+
+/**
+ * enum tdb_log_level - log levels for tdb_attribute_log
+ * @TDB_LOG_ERROR: used to log unrecoverable errors such as I/O errors
+ *		   or internal consistency failures.
+ * @TDB_LOG_USE_ERROR: used to log usage errors such as invalid parameters
+ *		   or writing to a read-only database.
+ * @TDB_LOG_WARNING: used for informational messages on issues which
+ *		     are unusual but handled by TDB internally, such
+ *		     as a failure to mmap or failure to open /dev/urandom.
+ */
+enum tdb_log_level {
+	TDB_LOG_ERROR,
+	TDB_LOG_USE_ERROR,
+	TDB_LOG_WARNING
+};
+
+/**
+ * struct tdb_attribute_log - log function attribute
+ *
+ * This attribute provides a hook for you to log errors.
+ */
+struct tdb_attribute_log {
+	struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_LOG */
+	void (*fn)(struct tdb_context *tdb,
+		   enum tdb_log_level level,
+		   const char *message,
+		   void *data);
+	void *data;
+};
+
+/**
+ * struct tdb_attribute_hash - hash function attribute
+ *
+ * This attribute allows you to provide an alternative hash function.
+ * This hash function will be handed keys from the database; it will also
+ * be handed the 8-byte TDB_HASH_MAGIC value for checking the header (the
+ * tdb_open() will fail if the hash value doesn't match the header).
+ *
+ * Note that if your hash function gives different results on
+ * different machine endians, your tdb will no longer work across
+ * different architectures!
+ */
+struct tdb_attribute_hash {
+	struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_HASH */
+	uint64_t (*fn)(const void *key, size_t len, uint64_t seed,
+		       void *data);
+	void *data;
+};
+
+/**
+ * struct tdb_attribute_seed - hash function seed attribute
+ *
+ * The hash function seed is normally taken from /dev/urandom (or equivalent)
+ * but can be set manually here.  This is mainly for testing purposes.
+ */
+struct tdb_attribute_seed {
+	struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_SEED */
+	uint64_t seed;
+};
+
+/**
+ * struct tdb_attribute_stats - tdb operational statistics
+ *
+ * This attribute records statistics of various low-level TDB operations.
+ * This can be used to assist performance evaluation.  This is only
+ * useful for tdb_get_attribute().
+ *
+ * New fields will be added at the end, hence the "size" argument which
+ * indicates how large your structure is: it must be filled in before
+ * calling tdb_get_attribute(), which will overwrite it with the size
+ * tdb knows about.
+ */
+struct tdb_attribute_stats {
+	struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_STATS */
+	size_t size; /* = sizeof(struct tdb_attribute_stats) */
+	uint64_t allocs;
+	uint64_t   alloc_subhash;
+	uint64_t   alloc_chain;
+	uint64_t   alloc_bucket_exact;
+	uint64_t   alloc_bucket_max;
+	uint64_t   alloc_leftover;
+	uint64_t   alloc_coalesce_tried;
+	uint64_t     alloc_coalesce_iterate_clash;
+	uint64_t     alloc_coalesce_lockfail;
+	uint64_t     alloc_coalesce_race;
+	uint64_t     alloc_coalesce_succeeded;
+	uint64_t       alloc_coalesce_num_merged;
+	uint64_t compares;
+	uint64_t   compare_wrong_bucket;
+	uint64_t   compare_wrong_offsetbits;
+	uint64_t   compare_wrong_keylen;
+	uint64_t   compare_wrong_rechash;
+	uint64_t   compare_wrong_keycmp;
+	uint64_t transactions;
+	uint64_t   transaction_cancel;
+	uint64_t   transaction_nest;
+	uint64_t   transaction_expand_file;
+	uint64_t   transaction_read_direct;
+	uint64_t      transaction_read_direct_fail;
+	uint64_t   transaction_write_direct;
+	uint64_t      transaction_write_direct_fail;
+	uint64_t expands;
+	uint64_t frees;
+	uint64_t locks;
+	uint64_t   lock_lowlevel;
+	uint64_t   lock_nonblock;
+	uint64_t     lock_nonblock_fail;
+};
+
+/**
+ * struct tdb_attribute_openhook - tdb special effects hook for open
+ *
+ * This attribute contains a function to call once we have the OPEN_LOCK
+ * for the tdb, but before we've examined its contents.  If this succeeds,
+ * the tdb will be populated if it's then zero-length.
+ *
+ * This is a hack to allow support for TDB1-style TDB_CLEAR_IF_FIRST
+ * behaviour.
+ */
+struct tdb_attribute_openhook {
+	struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_OPENHOOK */
+	enum TDB_ERROR (*fn)(int fd, void *data);
+	void *data;
+};
+
+/**
+ * struct tdb_attribute_flock - tdb special effects hook for file locking
+ *
+ * This attribute contains function to call to place locks on a file; it can
+ * be used to support non-blocking operations or lock proxying.
+ *
+ * They should return 0 on success, -1 on failure and set errno.
+ *
+ * An error will be logged on error if errno is neither EAGAIN nor EINTR
+ * (normally it would only return EAGAIN if waitflag is false, and
+ * loop internally on EINTR).
+ */
+struct tdb_attribute_flock {
+	struct tdb_attribute_base base; /* .attr = TDB_ATTRIBUTE_FLOCK */
+	int (*lock)(int fd,int rw, off_t off, off_t len, bool waitflag, void *);
+	int (*unlock)(int fd, int rw, off_t off, off_t len, void *);
+	void *data;
+};
+
+/**
+ * union tdb_attribute - tdb attributes.
+ *
+ * This represents all the known attributes.
+ *
+ * See also:
+ *	struct tdb_attribute_log, struct tdb_attribute_hash,
+ *	struct tdb_attribute_seed, struct tdb_attribute_stats,
+ *	struct tdb_attribute_openhook, struct tdb_attribute_flock.
+ */
+union tdb_attribute {
+	struct tdb_attribute_base base;
+	struct tdb_attribute_log log;
+	struct tdb_attribute_hash hash;
+	struct tdb_attribute_seed seed;
+	struct tdb_attribute_stats stats;
+	struct tdb_attribute_openhook openhook;
+	struct tdb_attribute_flock flock;
+};
+
+#ifdef  __cplusplus
+}
+#endif
+
+#endif /* tdb2.h */
diff --git a/lib/tdb2/test/external-agent.c b/lib/tdb2/test/external-agent.c
new file mode 100644
index 00000000000..055b5de736f
--- /dev/null
+++ b/lib/tdb2/test/external-agent.c
@@ -0,0 +1,250 @@
+#include "external-agent.h"
+#include "logging.h"
+#include "lock-tracking.h"
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <err.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <string.h>
+#include <errno.h>
+#include <ccan/tdb2/private.h>
+#include <ccan/tap/tap.h>
+#include <stdio.h>
+#include <stdarg.h>
+
+static struct tdb_context *tdb;
+
+static enum TDB_ERROR clear_if_first(int fd, void *arg)
+{
+/* We hold a lock offset 63 always, so we can tell if anyone is holding it. */
+	struct flock fl;
+
+	fl.l_type = F_WRLCK;
+	fl.l_whence = SEEK_SET;
+	fl.l_start = 63;
+	fl.l_len = 1;
+
+	if (fcntl(fd, F_SETLK, &fl) == 0) {
+		/* We must be first ones to open it! */
+		diag("agent truncating file!");
+		if (ftruncate(fd, 0) != 0) {
+			return TDB_ERR_IO;
+		}
+	}
+	fl.l_type = F_RDLCK;
+	if (fcntl(fd, F_SETLKW, &fl) != 0) {
+		return TDB_ERR_IO;
+	}
+	return TDB_SUCCESS;
+}
+
+static enum agent_return do_operation(enum operation op, const char *name)
+{
+	TDB_DATA k;
+	enum agent_return ret;
+	TDB_DATA data;
+	enum TDB_ERROR ecode;
+	union tdb_attribute cif;
+
+	if (op != OPEN && op != OPEN_WITH_HOOK && !tdb) {
+		diag("external: No tdb open!");
+		return OTHER_FAILURE;
+	}
+
+	diag("external: %s", operation_name(op));
+
+	k = tdb_mkdata(name, strlen(name));
+
+	locking_would_block = 0;
+	switch (op) {
+	case OPEN:
+		if (tdb) {
+			diag("Already have tdb %s open", tdb->name);
+			return OTHER_FAILURE;
+		}
+		tdb = tdb_open(name, TDB_DEFAULT, O_RDWR, 0, &tap_log_attr);
+		if (!tdb) {
+			if (!locking_would_block)
+				diag("Opening tdb gave %s", strerror(errno));
+			forget_locking();
+			ret = OTHER_FAILURE;
+		} else
+			ret = SUCCESS;
+		break;
+	case OPEN_WITH_HOOK:
+		if (tdb) {
+			diag("Already have tdb %s open", tdb->name);
+			return OTHER_FAILURE;
+		}
+		cif.openhook.base.attr = TDB_ATTRIBUTE_OPENHOOK;
+		cif.openhook.base.next = &tap_log_attr;
+		cif.openhook.fn = clear_if_first;
+		tdb = tdb_open(name, TDB_DEFAULT, O_RDWR, 0, &cif);
+		if (!tdb) {
+			if (!locking_would_block)
+				diag("Opening tdb gave %s", strerror(errno));
+			forget_locking();
+			ret = OTHER_FAILURE;
+		} else
+			ret = SUCCESS;
+		break;
+	case FETCH:
+		ecode = tdb_fetch(tdb, k, &data);
+		if (ecode == TDB_ERR_NOEXIST) {
+			ret = FAILED;
+		} else if (ecode < 0) {
+			ret = OTHER_FAILURE;
+		} else if (!tdb_deq(data, k)) {
+			ret = OTHER_FAILURE;
+			free(data.dptr);
+		} else {
+			ret = SUCCESS;
+			free(data.dptr);
+		}
+		break;
+	case STORE:
+		ret = tdb_store(tdb, k, k, 0) == 0 ? SUCCESS : OTHER_FAILURE;
+		break;
+	case TRANSACTION_START:
+		ret = tdb_transaction_start(tdb) == 0 ? SUCCESS : OTHER_FAILURE;
+		break;
+	case TRANSACTION_COMMIT:
+		ret = tdb_transaction_commit(tdb)==0 ? SUCCESS : OTHER_FAILURE;
+		break;
+	case NEEDS_RECOVERY:
+		ret = tdb_needs_recovery(tdb) ? SUCCESS : FAILED;
+		break;
+	case CHECK:
+		ret = tdb_check(tdb, NULL, NULL) == 0 ? SUCCESS : OTHER_FAILURE;
+		break;
+	case CLOSE:
+		ret = tdb_close(tdb) == 0 ? SUCCESS : OTHER_FAILURE;
+		tdb = NULL;
+		break;
+	case SEND_SIGNAL:
+		/* We do this async */
+		ret = SUCCESS;
+		break;
+	default:
+		ret = OTHER_FAILURE;
+	}
+
+	if (locking_would_block)
+		ret = WOULD_HAVE_BLOCKED;
+
+	return ret;
+}
+
+struct agent {
+	int cmdfd, responsefd;
+};
+
+/* Do this before doing any tdb stuff.  Return handle, or NULL. */
+struct agent *prepare_external_agent(void)
+{
+	int pid, ret;
+	int command[2], response[2];
+	char name[1+PATH_MAX];
+
+	if (pipe(command) != 0 || pipe(response) != 0)
+		return NULL;
+
+	pid = fork();
+	if (pid < 0)
+		return NULL;
+
+	if (pid != 0) {
+		struct agent *agent = malloc(sizeof(*agent));
+
+		close(command[0]);
+		close(response[1]);
+		agent->cmdfd = command[1];
+		agent->responsefd = response[0];
+		return agent;
+	}
+
+	close(command[1]);
+	close(response[0]);
+
+	/* We want to fail, not block. */
+	nonblocking_locks = true;
+	log_prefix = "external: ";
+	while ((ret = read(command[0], name, sizeof(name))) > 0) {
+		enum agent_return result;
+
+		result = do_operation(name[0], name+1);
+		if (write(response[1], &result, sizeof(result))
+		    != sizeof(result))
+			err(1, "Writing response");
+		if (name[0] == SEND_SIGNAL) {
+			struct timeval ten_ms;
+			ten_ms.tv_sec = 0;
+			ten_ms.tv_usec = 10000;
+			select(0, NULL, NULL, NULL, &ten_ms);
+			kill(getppid(), SIGUSR1);
+		}
+	}
+	exit(0);
+}
+
+/* Ask the external agent to try to do an operation. */
+enum agent_return external_agent_operation(struct agent *agent,
+					   enum operation op,
+					   const char *name)
+{
+	enum agent_return res;
+	unsigned int len;
+	char *string;
+
+	if (!name)
+		name = "";
+	len = 1 + strlen(name) + 1;
+	string = malloc(len);
+
+	string[0] = op;
+	strcpy(string+1, name);
+
+	if (write(agent->cmdfd, string, len) != len
+	    || read(agent->responsefd, &res, sizeof(res)) != sizeof(res))
+		res = AGENT_DIED;
+
+	free(string);
+	return res;
+}
+
+const char *agent_return_name(enum agent_return ret)
+{
+	return ret == SUCCESS ? "SUCCESS"
+		: ret == WOULD_HAVE_BLOCKED ? "WOULD_HAVE_BLOCKED"
+		: ret == AGENT_DIED ? "AGENT_DIED"
+		: ret == FAILED ? "FAILED"
+		: ret == OTHER_FAILURE ? "OTHER_FAILURE"
+		: "**INVALID**";
+}
+
+const char *operation_name(enum operation op)
+{
+	switch (op) {
+	case OPEN: return "OPEN";
+	case OPEN_WITH_HOOK: return "OPEN_WITH_HOOK";
+	case FETCH: return "FETCH";
+	case STORE: return "STORE";
+	case CHECK: return "CHECK";
+	case TRANSACTION_START: return "TRANSACTION_START";
+	case TRANSACTION_COMMIT: return "TRANSACTION_COMMIT";
+	case NEEDS_RECOVERY: return "NEEDS_RECOVERY";
+	case SEND_SIGNAL: return "SEND_SIGNAL";
+	case CLOSE: return "CLOSE";
+	}
+	return "**INVALID**";
+}
+
+void free_external_agent(struct agent *agent)
+{
+	close(agent->cmdfd);
+	close(agent->responsefd);
+	free(agent);
+}
diff --git a/lib/tdb2/test/external-agent.h b/lib/tdb2/test/external-agent.h
new file mode 100644
index 00000000000..9eada107508
--- /dev/null
+++ b/lib/tdb2/test/external-agent.h
@@ -0,0 +1,43 @@
+#ifndef TDB2_TEST_EXTERNAL_AGENT_H
+#define TDB2_TEST_EXTERNAL_AGENT_H
+
+/* For locking tests, we need a different process to try things at
+ * various times. */
+enum operation {
+	OPEN,
+	OPEN_WITH_HOOK,
+	FETCH,
+	STORE,
+	TRANSACTION_START,
+	TRANSACTION_COMMIT,
+	NEEDS_RECOVERY,
+	CHECK,
+	SEND_SIGNAL,
+	CLOSE,
+};
+
+/* Do this before doing any tdb stuff.  Return handle, or -1. */
+struct agent *prepare_external_agent(void);
+
+enum agent_return {
+	SUCCESS,
+	WOULD_HAVE_BLOCKED,
+	AGENT_DIED,
+	FAILED, /* For fetch, or NEEDS_RECOVERY */
+	OTHER_FAILURE,
+};
+
+/* Ask the external agent to try to do an operation.
+ * name == tdb name for OPEN/OPEN_WITH_CLEAR_IF_FIRST,
+ * record name for FETCH/STORE (store stores name as data too)
+ */
+enum agent_return external_agent_operation(struct agent *handle,
+					   enum operation op,
+					   const char *name);
+
+/* Mapping enum -> string. */
+const char *agent_return_name(enum agent_return ret);
+const char *operation_name(enum operation op);
+
+void free_external_agent(struct agent *agent);
+#endif /* TDB2_TEST_EXTERNAL_AGENT_H */
diff --git a/lib/tdb2/test/failtest_helper.c b/lib/tdb2/test/failtest_helper.c
new file mode 100644
index 00000000000..1358a6c6b26
--- /dev/null
+++ b/lib/tdb2/test/failtest_helper.c
@@ -0,0 +1,117 @@
+#include "failtest_helper.h"
+#include "logging.h"
+#include <string.h>
+#include <ccan/tap/tap.h>
+
+/* FIXME: From ccan/str */
+static inline bool strends(const char *str, const char *postfix)
+{
+	if (strlen(str) < strlen(postfix))
+		return false;
+
+	return !strcmp(str + strlen(str) - strlen(postfix), postfix);
+}
+
+bool failmatch(const struct failtest_call *call,
+	       const char *file, int line, enum failtest_call_type type)
+{
+	return call->type == type
+		&& call->line == line
+		&& ((strcmp(call->file, file) == 0)
+		    || (strends(call->file, file)
+			&& (call->file[strlen(call->file) - strlen(file) - 1]
+			    == '/')));
+}
+
+static const struct failtest_call *
+find_repeat(const struct failtest_call *start, const struct failtest_call *end,
+	    const struct failtest_call *call)
+{
+	const struct failtest_call *i;
+
+	for (i = start; i < end; i++) {
+		if (failmatch(i, call->file, call->line, call->type))
+			return i;
+	}
+	return NULL;
+}
+
+static bool is_nonblocking_lock(const struct failtest_call *call)
+{
+	return call->type == FAILTEST_FCNTL && call->u.fcntl.cmd == F_SETLK;
+}
+
+static bool is_unlock(const struct failtest_call *call)
+{
+	return call->type == FAILTEST_FCNTL
+		&& call->u.fcntl.arg.fl.l_type == F_UNLCK;
+}
+
+bool exit_check_log(struct failtest_call *history, unsigned num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (!history[i].fail)
+			continue;
+		/* Failing the /dev/urandom open doesn't count: we fall back. */
+		if (failmatch(&history[i], URANDOM_OPEN))
+			continue;
+
+		/* Similarly with read fail. */
+		if (failmatch(&history[i], URANDOM_READ))
+			continue;
+
+		/* Initial allocation of tdb doesn't log. */
+		if (failmatch(&history[i], INITIAL_TDB_MALLOC))
+			continue;
+
+		/* We don't block "failures" on non-blocking locks. */
+		if (is_nonblocking_lock(&history[i]))
+			continue;
+
+		if (!tap_log_messages)
+			diag("We didn't log for %u (%s:%u)",
+			     i, history[i].file, history[i].line);
+		return tap_log_messages != 0;
+	}
+	return true;
+}
+
+/* Some places we soldier on despite errors: only fail them once. */
+enum failtest_result
+block_repeat_failures(struct failtest_call *history, unsigned num)
+{
+	const struct failtest_call *i, *last = &history[num-1];
+
+	if (failmatch(last, INITIAL_TDB_MALLOC)
+	    || failmatch(last, URANDOM_OPEN)
+	    || failmatch(last, URANDOM_READ)) {
+		if (find_repeat(history, last, last))
+			return FAIL_DONT_FAIL;
+		return FAIL_PROBE;
+	}
+
+	/* Unlock or non-blocking lock is fail-once. */
+	if (is_unlock(last)) {
+		/* Find a previous unlock at this point? */
+		for (i = find_repeat(history, last, last);
+		     i;
+		     i = find_repeat(history, i, last)) {
+			if (is_unlock(i))
+				return FAIL_DONT_FAIL;
+		}
+		return FAIL_PROBE;
+	} else if (is_nonblocking_lock(last)) {
+		/* Find a previous non-blocking lock at this point? */
+		for (i = find_repeat(history, last, last);
+		     i;
+		     i = find_repeat(history, i, last)) {
+			if (is_nonblocking_lock(i))
+				return FAIL_DONT_FAIL;
+		}
+		return FAIL_PROBE;
+	}
+
+	return FAIL_OK;
+}
diff --git a/lib/tdb2/test/failtest_helper.h b/lib/tdb2/test/failtest_helper.h
new file mode 100644
index 00000000000..a62efbad58e
--- /dev/null
+++ b/lib/tdb2/test/failtest_helper.h
@@ -0,0 +1,17 @@
+#ifndef TDB2_TEST_FAILTEST_HELPER_H
+#define TDB2_TEST_FAILTEST_HELPER_H
+#include <ccan/failtest/failtest.h>
+#include <stdbool.h>
+
+/* FIXME: Check these! */
+#define INITIAL_TDB_MALLOC	"open.c", 338, FAILTEST_MALLOC
+#define URANDOM_OPEN		"open.c", 45, FAILTEST_OPEN
+#define URANDOM_READ		"open.c", 25, FAILTEST_READ
+
+bool exit_check_log(struct failtest_call *history, unsigned num);
+bool failmatch(const struct failtest_call *call,
+	       const char *file, int line, enum failtest_call_type type);
+enum failtest_result
+block_repeat_failures(struct failtest_call *history, unsigned num);
+
+#endif /* TDB2_TEST_LOGGING_H */
diff --git a/lib/tdb2/test/layout.c b/lib/tdb2/test/layout.c
new file mode 100644
index 00000000000..31889ad080c
--- /dev/null
+++ b/lib/tdb2/test/layout.c
@@ -0,0 +1,348 @@
+/* TDB tools to create various canned database layouts. */
+#include "layout.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <err.h>
+#include "logging.h"
+
+struct tdb_layout *new_tdb_layout(const char *filename)
+{
+	struct tdb_layout *layout = malloc(sizeof(*layout));
+	layout->filename = filename;
+	layout->num_elems = 0;
+	layout->elem = NULL;
+	return layout;
+}
+
+static void add(struct tdb_layout *layout, union tdb_layout_elem elem)
+{
+	layout->elem = realloc(layout->elem,
+			       sizeof(layout->elem[0])
+			       * (layout->num_elems+1));
+	layout->elem[layout->num_elems++] = elem;
+}
+
+void tdb_layout_add_freetable(struct tdb_layout *layout)
+{
+	union tdb_layout_elem elem;
+	elem.base.type = FREETABLE;
+	add(layout, elem);
+}
+
+void tdb_layout_add_free(struct tdb_layout *layout, tdb_len_t len,
+			 unsigned ftable)
+{
+	union tdb_layout_elem elem;
+	elem.base.type = FREE;
+	elem.free.len = len;
+	elem.free.ftable_num = ftable;
+	add(layout, elem);
+}
+
+static struct tdb_data dup_key(struct tdb_data key)
+{
+	struct tdb_data ret;
+	ret.dsize = key.dsize;
+	ret.dptr = malloc(ret.dsize);
+	memcpy(ret.dptr, key.dptr, ret.dsize);
+	return ret;
+}
+
+void tdb_layout_add_used(struct tdb_layout *layout,
+			 TDB_DATA key, TDB_DATA data,
+			 tdb_len_t extra)
+{
+	union tdb_layout_elem elem;
+	elem.base.type = DATA;
+	elem.used.key = dup_key(key);
+	elem.used.data = dup_key(data);
+	elem.used.extra = extra;
+	add(layout, elem);
+}
+
+static tdb_len_t free_record_len(tdb_len_t len)
+{
+	return sizeof(struct tdb_used_record) + len;
+}
+
+static tdb_len_t data_record_len(struct tle_used *used)
+{
+	tdb_len_t len;
+	len = sizeof(struct tdb_used_record)
+		+ used->key.dsize + used->data.dsize + used->extra;
+	assert(len >= sizeof(struct tdb_free_record));
+	return len;
+}
+
+static tdb_len_t hashtable_len(struct tle_hashtable *htable)
+{
+	return sizeof(struct tdb_used_record)
+		+ (sizeof(tdb_off_t) << TDB_SUBLEVEL_HASH_BITS)
+		+ htable->extra;
+}
+
+static tdb_len_t freetable_len(struct tle_freetable *ftable)
+{
+	return sizeof(struct tdb_freetable);
+}
+
+static void set_free_record(void *mem, tdb_len_t len)
+{
+	/* We do all the work in add_to_freetable */
+}
+
+static void add_zero_pad(struct tdb_used_record *u, size_t len, size_t extra)
+{
+	if (extra)
+		((char *)(u + 1))[len] = '\0';
+}
+
+static void set_data_record(void *mem, struct tdb_context *tdb,
+			    struct tle_used *used)
+{
+	struct tdb_used_record *u = mem;
+
+	set_header(tdb, u, TDB_USED_MAGIC, used->key.dsize, used->data.dsize,
+		   used->key.dsize + used->data.dsize + used->extra,
+		   tdb_hash(tdb, used->key.dptr, used->key.dsize));
+	memcpy(u + 1, used->key.dptr, used->key.dsize);
+	memcpy((char *)(u + 1) + used->key.dsize,
+	       used->data.dptr, used->data.dsize);
+	add_zero_pad(u, used->key.dsize + used->data.dsize, used->extra);
+}
+
+static void set_hashtable(void *mem, struct tdb_context *tdb,
+			  struct tle_hashtable *htable)
+{
+	struct tdb_used_record *u = mem;
+	tdb_len_t len = sizeof(tdb_off_t) << TDB_SUBLEVEL_HASH_BITS;
+
+	set_header(tdb, u, TDB_HTABLE_MAGIC, 0, len, len + htable->extra, 0);
+	memset(u + 1, 0, len);
+	add_zero_pad(u, len, htable->extra);
+}
+
+static void set_freetable(void *mem, struct tdb_context *tdb,
+			 struct tle_freetable *freetable, struct tdb_header *hdr,
+			 tdb_off_t last_ftable)
+{
+	struct tdb_freetable *ftable = mem;
+	memset(ftable, 0, sizeof(*ftable));
+	set_header(tdb, &ftable->hdr, TDB_FTABLE_MAGIC, 0,
+			sizeof(*ftable) - sizeof(ftable->hdr),
+			sizeof(*ftable) - sizeof(ftable->hdr), 0);
+
+	if (last_ftable) {
+		ftable = (struct tdb_freetable *)((char *)hdr + last_ftable);
+		ftable->next = freetable->base.off;
+	} else {
+		hdr->free_table = freetable->base.off;
+	}
+}
+
+static void add_to_freetable(struct tdb_context *tdb,
+			     tdb_off_t eoff,
+			     tdb_off_t elen,
+			     unsigned ftable,
+			     struct tle_freetable *freetable)
+{
+	tdb->ftable_off = freetable->base.off;
+	tdb->ftable = ftable;
+	add_free_record(tdb, eoff, sizeof(struct tdb_used_record) + elen,
+			TDB_LOCK_WAIT, false);
+}
+
+static tdb_off_t hbucket_off(tdb_off_t group_start, unsigned ingroup)
+{
+	return group_start
+		+ (ingroup % (1 << TDB_HASH_GROUP_BITS)) * sizeof(tdb_off_t);
+}
+
+/* Get bits from a value. */
+static uint32_t bits(uint64_t val, unsigned start, unsigned num)
+{
+	assert(num <= 32);
+	return (val >> start) & ((1U << num) - 1);
+}
+
+/* We take bits from the top: that way we can lock whole sections of the hash
+ * by using lock ranges. */
+static uint32_t use_bits(uint64_t h, unsigned num, unsigned *used)
+{
+	*used += num;
+	return bits(h, 64 - *used, num);
+}
+
+static tdb_off_t encode_offset(tdb_off_t new_off, unsigned bucket,
+			       uint64_t h)
+{
+	return bucket
+		| new_off
+		| ((uint64_t)bits(h, 64 - TDB_OFF_UPPER_STEAL_EXTRA,
+				  TDB_OFF_UPPER_STEAL_EXTRA)
+		   << TDB_OFF_HASH_EXTRA_BIT);
+}
+
+/* FIXME: Our hash table handling here is primitive: we don't expand! */
+static void add_to_hashtable(struct tdb_context *tdb,
+			     tdb_off_t eoff,
+			     struct tdb_data key)
+{
+	uint64_t h = tdb_hash(tdb, key.dptr, key.dsize);
+	tdb_off_t b_off, group_start;
+	unsigned i, group, in_group;
+	unsigned used = 0;
+
+	group = use_bits(h, TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS, &used);
+	in_group = use_bits(h, TDB_HASH_GROUP_BITS, &used);
+
+	group_start = offsetof(struct tdb_header, hashtable)
+		+ group * (sizeof(tdb_off_t) << TDB_HASH_GROUP_BITS);
+
+	for (i = 0; i < (1 << TDB_HASH_GROUP_BITS); i++) {
+		unsigned bucket = (in_group + i) % (1 << TDB_HASH_GROUP_BITS);
+
+		b_off = hbucket_off(group_start, bucket);
+		if (tdb_read_off(tdb, b_off) == 0) {
+			tdb_write_off(tdb, b_off,
+				      encode_offset(eoff, bucket, h));
+			return;
+		}
+	}
+	abort();
+}
+
+static struct tle_freetable *find_ftable(struct tdb_layout *layout, unsigned num)
+{
+	unsigned i;
+
+	for (i = 0; i < layout->num_elems; i++) {
+		if (layout->elem[i].base.type != FREETABLE)
+			continue;
+		if (num == 0)
+			return &layout->elem[i].ftable;
+		num--;
+	}
+	abort();
+}
+
+/* FIXME: Support TDB_CONVERT */
+struct tdb_context *tdb_layout_get(struct tdb_layout *layout)
+{
+	unsigned int i;
+	tdb_off_t off, len, last_ftable;
+	char *mem;
+	struct tdb_context *tdb;
+
+	off = sizeof(struct tdb_header);
+
+	/* First pass of layout: calc lengths */
+	for (i = 0; i < layout->num_elems; i++) {
+		union tdb_layout_elem *e = &layout->elem[i];
+		e->base.off = off;
+		switch (e->base.type) {
+		case FREETABLE:
+			len = freetable_len(&e->ftable);
+			break;
+		case FREE:
+			len = free_record_len(e->free.len);
+			break;
+		case DATA:
+			len = data_record_len(&e->used);
+			break;
+		case HASHTABLE:
+			len = hashtable_len(&e->hashtable);
+			break;
+		default:
+			abort();
+		}
+		off += len;
+	}
+
+	mem = malloc(off);
+	/* Fill with some weird pattern. */
+	memset(mem, 0x99, off);
+	/* Now populate our header, cribbing from a real TDB header. */
+	tdb = tdb_open(NULL, TDB_INTERNAL, O_RDWR, 0, &tap_log_attr);
+	memcpy(mem, tdb->file->map_ptr, sizeof(struct tdb_header));
+
+	/* Mug the tdb we have to make it use this. */
+	free(tdb->file->map_ptr);
+	tdb->file->map_ptr = mem;
+	tdb->file->map_size = off;
+
+	last_ftable = 0;
+	for (i = 0; i < layout->num_elems; i++) {
+		union tdb_layout_elem *e = &layout->elem[i];
+		switch (e->base.type) {
+		case FREETABLE:
+			set_freetable(mem + e->base.off, tdb, &e->ftable,
+				     (struct tdb_header *)mem, last_ftable);
+			last_ftable = e->base.off;
+			break;
+		case FREE:
+			set_free_record(mem + e->base.off, e->free.len);
+			break;
+		case DATA:
+			set_data_record(mem + e->base.off, tdb, &e->used);
+			break;
+		case HASHTABLE:
+			set_hashtable(mem + e->base.off, tdb, &e->hashtable);
+			break;
+		}
+	}
+	/* Must have a free table! */
+	assert(last_ftable);
+
+	/* Now fill the free and hash tables. */
+	for (i = 0; i < layout->num_elems; i++) {
+		union tdb_layout_elem *e = &layout->elem[i];
+		switch (e->base.type) {
+		case FREE:
+			add_to_freetable(tdb, e->base.off, e->free.len,
+					 e->free.ftable_num,
+					 find_ftable(layout, e->free.ftable_num));
+			break;
+		case DATA:
+			add_to_hashtable(tdb, e->base.off, e->used.key);
+			break;
+		default:
+			break;
+		}
+	}
+
+	tdb->ftable_off = find_ftable(layout, 0)->base.off;
+
+	/* Get physical if they asked for it. */
+	if (layout->filename) {
+		int fd = open(layout->filename, O_WRONLY|O_TRUNC|O_CREAT,
+			      0600);
+		if (fd < 0)
+			err(1, "opening %s for writing", layout->filename);
+		if (write(fd, tdb->file->map_ptr, tdb->file->map_size)
+		    != tdb->file->map_size)
+			err(1, "writing %s", layout->filename);
+		close(fd);
+		tdb_close(tdb);
+		/* NOMMAP is for lockcheck. */
+		tdb = tdb_open(layout->filename, TDB_NOMMAP, O_RDWR, 0,
+			       &tap_log_attr);
+	}
+
+	return tdb;
+}
+
+void tdb_layout_free(struct tdb_layout *layout)
+{
+	unsigned int i;
+
+	for (i = 0; i < layout->num_elems; i++) {
+		if (layout->elem[i].base.type == DATA) {
+			free(layout->elem[i].used.key.dptr);
+			free(layout->elem[i].used.data.dptr);
+		}
+	}
+	free(layout->elem);
+	free(layout);
+}
diff --git a/lib/tdb2/test/layout.h b/lib/tdb2/test/layout.h
new file mode 100644
index 00000000000..6e2e6657a70
--- /dev/null
+++ b/lib/tdb2/test/layout.h
@@ -0,0 +1,68 @@
+#ifndef TDB2_TEST_LAYOUT_H
+#define TDB2_TEST_LAYOUT_H
+#include <ccan/tdb2/private.h>
+
+struct tdb_layout *new_tdb_layout(const char *filename);
+void tdb_layout_add_freetable(struct tdb_layout *layout);
+void tdb_layout_add_free(struct tdb_layout *layout, tdb_len_t len,
+			 unsigned ftable);
+void tdb_layout_add_used(struct tdb_layout *layout,
+			 TDB_DATA key, TDB_DATA data,
+			 tdb_len_t extra);
+#if 0 /* FIXME: Allow allocation of subtables */
+void tdb_layout_add_hashtable(struct tdb_layout *layout,
+			      int htable_parent, /* -1 == toplevel */
+			      unsigned int bucket,
+			      tdb_len_t extra);
+#endif
+struct tdb_context *tdb_layout_get(struct tdb_layout *layout);
+void tdb_layout_free(struct tdb_layout *layout);
+
+enum layout_type {
+	FREETABLE, FREE, DATA, HASHTABLE,
+};
+
+/* Shared by all union members. */
+struct tle_base {
+	enum layout_type type;
+	tdb_off_t off;
+};
+
+struct tle_freetable {
+	struct tle_base base;
+};
+
+struct tle_free {
+	struct tle_base base;
+	tdb_len_t len;
+	unsigned ftable_num;
+};
+
+struct tle_used {
+	struct tle_base base;
+	TDB_DATA key;
+	TDB_DATA data;
+	tdb_len_t extra;
+};
+
+struct tle_hashtable {
+	struct tle_base base;
+	int parent;
+	unsigned int bucket;
+	tdb_len_t extra;
+};
+
+union tdb_layout_elem {
+	struct tle_base base;
+	struct tle_freetable ftable;
+	struct tle_free free;
+	struct tle_used used;
+	struct tle_hashtable hashtable;
+};
+
+struct tdb_layout {
+	const char *filename;
+	unsigned int num_elems;
+	union tdb_layout_elem *elem;
+};
+#endif /* TDB2_TEST_LAYOUT_H */
diff --git a/lib/tdb2/test/lock-tracking.c b/lib/tdb2/test/lock-tracking.c
new file mode 100644
index 00000000000..05dba32fd3f
--- /dev/null
+++ b/lib/tdb2/test/lock-tracking.c
@@ -0,0 +1,147 @@
+/* We save the locks so we can reaquire them. */
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <ccan/tap/tap.h>
+#include <ccan/tdb2/private.h>
+#include "lock-tracking.h"
+
+struct lock {
+	struct lock *next;
+	unsigned int off;
+	unsigned int len;
+	int type;
+};
+static struct lock *locks;
+int locking_errors = 0;
+bool suppress_lockcheck = false;
+bool nonblocking_locks;
+int locking_would_block = 0;
+void (*unlock_callback)(int fd);
+
+int fcntl_with_lockcheck(int fd, int cmd, ... /* arg */ )
+{
+	va_list ap;
+	int ret, arg3;
+	struct flock *fl;
+	bool may_block = false;
+
+	if (cmd != F_SETLK && cmd != F_SETLKW) {
+		/* This may be totally bogus, but we don't know in general. */
+		va_start(ap, cmd);
+		arg3 = va_arg(ap, int);
+		va_end(ap);
+
+		return fcntl(fd, cmd, arg3);
+	}
+
+	va_start(ap, cmd);
+	fl = va_arg(ap, struct flock *);
+	va_end(ap);
+
+	if (cmd == F_SETLKW && nonblocking_locks) {
+		cmd = F_SETLK;
+		may_block = true;
+	}
+	ret = fcntl(fd, cmd, fl);
+
+	/* Detect when we failed, but might have been OK if we waited. */
+	if (may_block && ret == -1 && (errno == EAGAIN || errno == EACCES)) {
+		locking_would_block++;
+	}
+
+	if (fl->l_type == F_UNLCK) {
+		struct lock **l;
+		struct lock *old = NULL;
+
+		for (l = &locks; *l; l = &(*l)->next) {
+			if ((*l)->off == fl->l_start
+			    && (*l)->len == fl->l_len) {
+				if (ret == 0) {
+					old = *l;
+					*l = (*l)->next;
+					free(old);
+				}
+				break;
+			}
+		}
+		if (!old && !suppress_lockcheck) {
+			diag("Unknown unlock %u@%u - %i",
+			     (int)fl->l_len, (int)fl->l_start, ret);
+			locking_errors++;
+		}
+	} else {
+		struct lock *new, *i;
+		unsigned int fl_end = fl->l_start + fl->l_len;
+		if (fl->l_len == 0)
+			fl_end = (unsigned int)-1;
+
+		/* Check for overlaps: we shouldn't do this. */
+		for (i = locks; i; i = i->next) {
+			unsigned int i_end = i->off + i->len;
+			if (i->len == 0)
+				i_end = (unsigned int)-1;
+
+			if (fl->l_start >= i->off && fl->l_start < i_end)
+				break;
+			if (fl_end > i->off && fl_end < i_end)
+				break;
+
+			/* tdb_allrecord_lock does this, handle adjacent: */
+			if (fl->l_start > TDB_HASH_LOCK_START
+			    && fl->l_start == i_end && fl->l_type == i->type) {
+				if (ret == 0) {
+					i->len = fl->l_len
+						? i->len + fl->l_len
+						: 0;
+				}
+				goto done;
+			}
+		}
+		if (i) {
+			/* Special case: upgrade of allrecord lock. */
+			if (i->type == F_RDLCK && fl->l_type == F_WRLCK
+			    && i->off == TDB_HASH_LOCK_START
+			    && fl->l_start == TDB_HASH_LOCK_START
+			    && i->len == 0
+			    && fl->l_len == 0) {
+				if (ret == 0)
+					i->type = F_WRLCK;
+				goto done;
+			}
+			if (!suppress_lockcheck) {
+				diag("%s lock %u@%u overlaps %u@%u",
+				     fl->l_type == F_WRLCK ? "write" : "read",
+				     (int)fl->l_len, (int)fl->l_start,
+				     i->len, (int)i->off);
+				locking_errors++;
+			}
+		}
+
+		if (ret == 0) {
+			new = malloc(sizeof *new);
+			new->off = fl->l_start;
+			new->len = fl->l_len;
+			new->type = fl->l_type;
+			new->next = locks;
+			locks = new;
+		}
+	}
+done:
+	if (ret == 0 && fl->l_type == F_UNLCK && unlock_callback)
+		unlock_callback(fd);
+	return ret;
+}
+
+unsigned int forget_locking(void)
+{
+	unsigned int num = 0;
+	while (locks) {
+		struct lock *next = locks->next;
+		free(locks);
+		locks = next;
+		num++;
+	}
+	return num;
+}
diff --git a/lib/tdb2/test/lock-tracking.h b/lib/tdb2/test/lock-tracking.h
new file mode 100644
index 00000000000..f2c9c44653b
--- /dev/null
+++ b/lib/tdb2/test/lock-tracking.h
@@ -0,0 +1,25 @@
+#ifndef LOCK_TRACKING_H
+#define LOCK_TRACKING_H
+#include <stdbool.h>
+
+/* Set this if you want a callback after fnctl unlock. */
+extern void (*unlock_callback)(int fd);
+
+/* Replacement fcntl. */
+int fcntl_with_lockcheck(int fd, int cmd, ... /* arg */ );
+
+/* Discard locking info: returns number of locks outstanding. */
+unsigned int forget_locking(void);
+
+/* Number of errors in locking. */
+extern int locking_errors;
+
+/* Suppress lock checking. */
+extern bool suppress_lockcheck;
+
+/* Make all locks non-blocking. */
+extern bool nonblocking_locks;
+
+/* Number of times we failed a lock because we made it non-blocking. */
+extern int locking_would_block;
+#endif /* LOCK_TRACKING_H */
diff --git a/lib/tdb2/test/logging.c b/lib/tdb2/test/logging.c
new file mode 100644
index 00000000000..d32cfa9b593
--- /dev/null
+++ b/lib/tdb2/test/logging.c
@@ -0,0 +1,24 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+unsigned tap_log_messages;
+const char *log_prefix = "";
+bool suppress_logging;
+
+union tdb_attribute tap_log_attr = {
+	.log = { .base = { .attr = TDB_ATTRIBUTE_LOG },
+		 .fn = tap_log_fn }
+};
+
+void tap_log_fn(struct tdb_context *tdb,
+		enum tdb_log_level level,
+		const char *message, void *priv)
+{
+	if (suppress_logging)
+		return;
+
+	diag("tdb log level %u: %s%s", level, log_prefix, message);
+	tap_log_messages++;
+}
diff --git a/lib/tdb2/test/logging.h b/lib/tdb2/test/logging.h
new file mode 100644
index 00000000000..d172f867fd2
--- /dev/null
+++ b/lib/tdb2/test/logging.h
@@ -0,0 +1,15 @@
+#ifndef TDB2_TEST_LOGGING_H
+#define TDB2_TEST_LOGGING_H
+#include <ccan/tdb2/tdb2.h>
+#include <stdbool.h>
+#include <string.h>
+
+extern bool suppress_logging;
+extern const char *log_prefix;
+extern unsigned tap_log_messages;
+extern union tdb_attribute tap_log_attr;
+
+void tap_log_fn(struct tdb_context *tdb,
+		enum tdb_log_level level,
+		const char *message, void *priv);
+#endif /* TDB2_TEST_LOGGING_H */
diff --git a/lib/tdb2/test/run-001-encode.c b/lib/tdb2/test/run-001-encode.c
new file mode 100644
index 00000000000..ffa4b93c02f
--- /dev/null
+++ b/lib/tdb2/test/run-001-encode.c
@@ -0,0 +1,48 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_used_record rec;
+	struct tdb_context tdb = { .log_fn = tap_log_fn };
+
+	plan_tests(64 + 32 + 48*6 + 1);
+
+	/* We should be able to encode any data value. */
+	for (i = 0; i < 64; i++)
+		ok1(set_header(&tdb, &rec, TDB_USED_MAGIC, 0, 1ULL << i,
+			       1ULL << i, 0) == 0);
+
+	/* And any key and data with < 64 bits between them. */
+	for (i = 0; i < 32; i++) {
+		tdb_len_t dlen = 1ULL >> (63 - i), klen = 1ULL << i;
+		ok1(set_header(&tdb, &rec, TDB_USED_MAGIC, klen, dlen,
+			       klen + dlen, 0)  == 0);
+	}
+
+	/* We should neatly encode all values. */
+	for (i = 0; i < 48; i++) {
+		uint64_t h = 1ULL << (i < 5 ? i : 4);
+		uint64_t klen = 1ULL << (i < 16 ? i : 15);
+		uint64_t dlen = 1ULL << i;
+		uint64_t xlen = 1ULL << (i < 32 ? i : 31);
+		ok1(set_header(&tdb, &rec, TDB_USED_MAGIC, klen, dlen,
+			       klen+dlen+xlen, h) == 0);
+		ok1(rec_key_length(&rec) == klen);
+		ok1(rec_data_length(&rec) == dlen);
+		ok1(rec_extra_padding(&rec) == xlen);
+		ok1((uint64_t)rec_hash(&rec) == h);
+		ok1(rec_magic(&rec) == TDB_USED_MAGIC);
+	}
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-001-fls.c b/lib/tdb2/test/run-001-fls.c
new file mode 100644
index 00000000000..d54cad1d1c4
--- /dev/null
+++ b/lib/tdb2/test/run-001-fls.c
@@ -0,0 +1,40 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+
+static unsigned int dumb_fls(uint64_t num)
+{
+	int i;
+
+	for (i = 63; i >= 0; i--) {
+		if (num & (1ULL << i))
+			break;
+	}
+	return i + 1;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+
+	plan_tests(64 * 64 + 2);
+
+	ok1(fls64(0) == 0);
+	ok1(dumb_fls(0) == 0);
+
+	for (i = 0; i < 64; i++) {
+		for (j = 0; j < 64; j++) {
+			uint64_t val = (1ULL << i) | (1ULL << j);
+			ok(fls64(val) == dumb_fls(val),
+			   "%llu -> %u should be %u", (long long)val,
+			   fls64(val), dumb_fls(val));
+		}
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-01-new_database.c b/lib/tdb2/test/run-01-new_database.c
new file mode 100644
index 00000000000..32ebaf09c10
--- /dev/null
+++ b/lib/tdb2/test/run-01-new_database.c
@@ -0,0 +1,42 @@
+#include <ccan/failtest/failtest_override.h>
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include <ccan/failtest/failtest.h>
+#include "logging.h"
+#include "failtest_helper.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	failtest_init(argc, argv);
+	failtest_hook = block_repeat_failures;
+	failtest_exit_check = exit_check_log;
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 3);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-new_database.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			failtest_exit(exit_status());
+		if (tdb) {
+			bool ok = ok1(tdb_check(tdb, NULL, NULL) == 0);
+			tdb_close(tdb);
+			if (!ok)
+				failtest_exit(exit_status());
+		}
+		if (!ok1(tap_log_messages == 0))
+			break;
+	}
+	failtest_exit(exit_status());
+}
diff --git a/lib/tdb2/test/run-02-expand.c b/lib/tdb2/test/run-02-expand.c
new file mode 100644
index 00000000000..6666ae167eb
--- /dev/null
+++ b/lib/tdb2/test/run-02-expand.c
@@ -0,0 +1,80 @@
+#include <ccan/failtest/failtest_override.h>
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tap/tap.h>
+#include <ccan/failtest/failtest.h>
+#include "logging.h"
+#include "failtest_helper.h"
+
+static bool failtest_suppress = false;
+
+/* Don't need to test everything here, just want expand testing. */
+static enum failtest_result
+suppress_failure(struct failtest_call *history, unsigned num)
+{
+	if (failtest_suppress)
+		return FAIL_DONT_FAIL;
+	return block_repeat_failures(history, num);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	uint64_t val;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 11 + 1);
+
+	failtest_init(argc, argv);
+	failtest_hook = suppress_failure;
+	failtest_exit_check = exit_check_log;
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		failtest_suppress = true;
+		tdb = tdb_open("run-expand.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			break;
+
+		val = tdb->file->map_size;
+		/* Need some hash lock for expand. */
+		ok1(tdb_lock_hashes(tdb, 0, 1, F_WRLCK, TDB_LOCK_WAIT) == 0);
+		failtest_suppress = false;
+		if (!ok1(tdb_expand(tdb, 1) == 0)) {
+			failtest_suppress = true;
+			tdb_close(tdb);
+			break;
+		}
+		failtest_suppress = true;
+
+		ok1(tdb->file->map_size >= val + 1 * TDB_EXTENSION_FACTOR);
+		ok1(tdb_unlock_hashes(tdb, 0, 1, F_WRLCK) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		val = tdb->file->map_size;
+		ok1(tdb_lock_hashes(tdb, 0, 1, F_WRLCK, TDB_LOCK_WAIT) == 0);
+		failtest_suppress = false;
+		if (!ok1(tdb_expand(tdb, 1024) == 0)) {
+			failtest_suppress = true;
+			tdb_close(tdb);
+			break;
+		}
+		failtest_suppress = true;
+		ok1(tdb_unlock_hashes(tdb, 0, 1, F_WRLCK) == 0);
+		ok1(tdb->file->map_size >= val + 1024 * TDB_EXTENSION_FACTOR);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	failtest_exit(exit_status());
+}
diff --git a/lib/tdb2/test/run-03-coalesce.c b/lib/tdb2/test/run-03-coalesce.c
new file mode 100644
index 00000000000..3fdd11c0770
--- /dev/null
+++ b/lib/tdb2/test/run-03-coalesce.c
@@ -0,0 +1,170 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+#include "layout.h"
+
+static tdb_len_t free_record_length(struct tdb_context *tdb, tdb_off_t off)
+{
+	struct tdb_free_record f;
+	enum TDB_ERROR ecode;
+
+	ecode = tdb_read_convert(tdb, off, &f, sizeof(f));
+	if (ecode != TDB_SUCCESS)
+		return ecode;
+	if (frec_magic(&f) != TDB_FREE_MAGIC)
+		return TDB_ERR_CORRUPT;
+	return frec_len(&f);
+}
+
+int main(int argc, char *argv[])
+{
+	tdb_off_t b_off, test;
+	struct tdb_context *tdb;
+	struct tdb_layout *layout;
+	struct tdb_data data, key;
+	tdb_len_t len;
+
+	/* FIXME: Test TDB_CONVERT */
+	/* FIXME: Test lock order fail. */
+
+	plan_tests(42);
+	data = tdb_mkdata("world", 5);
+	key = tdb_mkdata("hello", 5);
+
+	/* No coalescing can be done due to EOF */
+	layout = new_tdb_layout("run-03-coalesce.tdb");
+	tdb_layout_add_freetable(layout);
+	len = 1024;
+	tdb_layout_add_free(layout, len, 0);
+	tdb = tdb_layout_get(layout);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+	ok1(free_record_length(tdb, layout->elem[1].base.off) == len);
+
+	/* Figure out which bucket free entry is. */
+	b_off = bucket_off(tdb->ftable_off, size_to_bucket(len));
+	/* Lock and fail to coalesce. */
+	ok1(tdb_lock_free_bucket(tdb, b_off, TDB_LOCK_WAIT) == 0);
+	test = layout->elem[1].base.off;
+	ok1(coalesce(tdb, layout->elem[1].base.off, b_off, len, &test)
+	    == 0);
+	tdb_unlock_free_bucket(tdb, b_off);
+	ok1(free_record_length(tdb, layout->elem[1].base.off) == len);
+	ok1(test == layout->elem[1].base.off);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+	tdb_close(tdb);
+	tdb_layout_free(layout);
+
+	/* No coalescing can be done due to used record */
+	layout = new_tdb_layout("run-03-coalesce.tdb");
+	tdb_layout_add_freetable(layout);
+	tdb_layout_add_free(layout, 1024, 0);
+	tdb_layout_add_used(layout, key, data, 6);
+	tdb = tdb_layout_get(layout);
+	ok1(free_record_length(tdb, layout->elem[1].base.off) == 1024);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Figure out which bucket free entry is. */
+	b_off = bucket_off(tdb->ftable_off, size_to_bucket(1024));
+	/* Lock and fail to coalesce. */
+	ok1(tdb_lock_free_bucket(tdb, b_off, TDB_LOCK_WAIT) == 0);
+	test = layout->elem[1].base.off;
+	ok1(coalesce(tdb, layout->elem[1].base.off, b_off, 1024, &test)
+	    == 0);
+	tdb_unlock_free_bucket(tdb, b_off);
+	ok1(free_record_length(tdb, layout->elem[1].base.off) == 1024);
+	ok1(test == layout->elem[1].base.off);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+	tdb_close(tdb);
+	tdb_layout_free(layout);
+
+	/* Coalescing can be done due to two free records, then EOF */
+	layout = new_tdb_layout("run-03-coalesce.tdb");
+	tdb_layout_add_freetable(layout);
+	tdb_layout_add_free(layout, 1024, 0);
+	tdb_layout_add_free(layout, 2048, 0);
+	tdb = tdb_layout_get(layout);
+	ok1(free_record_length(tdb, layout->elem[1].base.off) == 1024);
+	ok1(free_record_length(tdb, layout->elem[2].base.off) == 2048);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Figure out which bucket (first) free entry is. */
+	b_off = bucket_off(tdb->ftable_off, size_to_bucket(1024));
+	/* Lock and coalesce. */
+	ok1(tdb_lock_free_bucket(tdb, b_off, TDB_LOCK_WAIT) == 0);
+	test = layout->elem[2].base.off;
+	ok1(coalesce(tdb, layout->elem[1].base.off, b_off, 1024, &test)
+	    == 1024 + sizeof(struct tdb_used_record) + 2048);
+	/* Should tell us it's erased this one... */
+	ok1(test == TDB_ERR_NOEXIST);
+	ok1(tdb->file->allrecord_lock.count == 0 && tdb->file->num_lockrecs == 0);
+	ok1(free_record_length(tdb, layout->elem[1].base.off)
+	    == 1024 + sizeof(struct tdb_used_record) + 2048);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+	tdb_close(tdb);
+	tdb_layout_free(layout);
+
+	/* Coalescing can be done due to two free records, then data */
+	layout = new_tdb_layout("run-03-coalesce.tdb");
+	tdb_layout_add_freetable(layout);
+	tdb_layout_add_free(layout, 1024, 0);
+	tdb_layout_add_free(layout, 512, 0);
+	tdb_layout_add_used(layout, key, data, 6);
+	tdb = tdb_layout_get(layout);
+	ok1(free_record_length(tdb, layout->elem[1].base.off) == 1024);
+	ok1(free_record_length(tdb, layout->elem[2].base.off) == 512);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Figure out which bucket free entry is. */
+	b_off = bucket_off(tdb->ftable_off, size_to_bucket(1024));
+	/* Lock and coalesce. */
+	ok1(tdb_lock_free_bucket(tdb, b_off, TDB_LOCK_WAIT) == 0);
+	test = layout->elem[2].base.off;
+	ok1(coalesce(tdb, layout->elem[1].base.off, b_off, 1024, &test)
+	    == 1024 + sizeof(struct tdb_used_record) + 512);
+	ok1(tdb->file->allrecord_lock.count == 0 && tdb->file->num_lockrecs == 0);
+	ok1(free_record_length(tdb, layout->elem[1].base.off)
+	    == 1024 + sizeof(struct tdb_used_record) + 512);
+	ok1(test == TDB_ERR_NOEXIST);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+	tdb_close(tdb);
+	tdb_layout_free(layout);
+
+	/* Coalescing can be done due to three free records, then EOF */
+	layout = new_tdb_layout("run-03-coalesce.tdb");
+	tdb_layout_add_freetable(layout);
+	tdb_layout_add_free(layout, 1024, 0);
+	tdb_layout_add_free(layout, 512, 0);
+	tdb_layout_add_free(layout, 256, 0);
+	tdb = tdb_layout_get(layout);
+	ok1(free_record_length(tdb, layout->elem[1].base.off) == 1024);
+	ok1(free_record_length(tdb, layout->elem[2].base.off) == 512);
+	ok1(free_record_length(tdb, layout->elem[3].base.off) == 256);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Figure out which bucket free entry is. */
+	b_off = bucket_off(tdb->ftable_off, size_to_bucket(1024));
+	/* Lock and coalesce. */
+	ok1(tdb_lock_free_bucket(tdb, b_off, TDB_LOCK_WAIT) == 0);
+	test = layout->elem[2].base.off;
+	ok1(coalesce(tdb, layout->elem[1].base.off, b_off, 1024, &test)
+	    == 1024 + sizeof(struct tdb_used_record) + 512
+	    + sizeof(struct tdb_used_record) + 256);
+	ok1(tdb->file->allrecord_lock.count == 0
+	    && tdb->file->num_lockrecs == 0);
+	ok1(free_record_length(tdb, layout->elem[1].base.off)
+	    == 1024 + sizeof(struct tdb_used_record) + 512
+	    + sizeof(struct tdb_used_record) + 256);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+	tdb_close(tdb);
+	tdb_layout_free(layout);
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-04-basichash.c b/lib/tdb2/test/run-04-basichash.c
new file mode 100644
index 00000000000..62031bdb40a
--- /dev/null
+++ b/lib/tdb2/test/run-04-basichash.c
@@ -0,0 +1,267 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+/* We rig the hash so adjacent-numbered records always clash. */
+static uint64_t clash(const void *key, size_t len, uint64_t seed, void *priv)
+{
+	return ((uint64_t)*(const unsigned int *)key)
+		<< (64 - TDB_TOPLEVEL_HASH_BITS - 1);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	unsigned int v;
+	struct tdb_used_record rec;
+	struct tdb_data key = { (unsigned char *)&v, sizeof(v) };
+	struct tdb_data dbuf = { (unsigned char *)&v, sizeof(v) };
+	union tdb_attribute hattr = { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+						.fn = clash } };
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT,
+	};
+
+	hattr.base.next = &tap_log_attr;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0])
+		   * (91 + (2 * ((1 << TDB_HASH_GROUP_BITS) - 1))) + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		struct hash_info h;
+		tdb_off_t new_off, off, subhash;
+
+		tdb = tdb_open("run-04-basichash.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &hattr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		v = 0;
+		/* Should not find it. */
+		ok1(find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL) == 0);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in group 0, bucket 0. */
+		ok1(h.group_start == offsetof(struct tdb_header, hashtable));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 0);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS);
+
+		/* Should have lock on bucket 0 */
+		ok1(h.hlock_start == 0);
+		ok1(h.hlock_range ==
+		    1ULL << (64-(TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS)));
+		ok1((tdb->flags & TDB_NOLOCK) || tdb->file->num_lockrecs == 1);
+		ok1((tdb->flags & TDB_NOLOCK)
+		    || tdb->file->lockrecs[0].off == TDB_HASH_LOCK_START);
+		/* FIXME: Check lock length */
+
+		/* Allocate a new record. */
+		new_off = alloc(tdb, key.dsize, dbuf.dsize, h.h,
+				TDB_USED_MAGIC, false);
+		ok1(!TDB_OFF_IS_ERR(new_off));
+
+		/* We should be able to add it now. */
+		ok1(add_to_hash(tdb, &h, new_off) == 0);
+
+		/* Make sure we fill it in for later finding. */
+		off = new_off + sizeof(struct tdb_used_record);
+		ok1(!tdb->methods->twrite(tdb, off, key.dptr, key.dsize));
+		off += key.dsize;
+		ok1(!tdb->methods->twrite(tdb, off, dbuf.dptr, dbuf.dsize));
+
+		/* We should be able to unlock that OK. */
+		ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+				      F_WRLCK) == 0);
+
+		/* Database should be consistent. */
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Now, this should give a successful lookup. */
+		ok1(find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL)
+		    == new_off);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in group 0, bucket 0. */
+		ok1(h.group_start == offsetof(struct tdb_header, hashtable));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 0);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS);
+
+		/* Should have lock on bucket 0 */
+		ok1(h.hlock_start == 0);
+		ok1(h.hlock_range ==
+		    1ULL << (64-(TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS)));
+		ok1((tdb->flags & TDB_NOLOCK) || tdb->file->num_lockrecs == 1);
+		ok1((tdb->flags & TDB_NOLOCK)
+		    || tdb->file->lockrecs[0].off == TDB_HASH_LOCK_START);
+		/* FIXME: Check lock length */
+
+		ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+				      F_WRLCK) == 0);
+
+		/* Database should be consistent. */
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Test expansion. */
+		v = 1;
+		ok1(find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL) == 0);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in group 0, bucket 1. */
+		ok1(h.group_start == offsetof(struct tdb_header, hashtable));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 1);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS);
+
+		/* Should have lock on bucket 0 */
+		ok1(h.hlock_start == 0);
+		ok1(h.hlock_range ==
+		    1ULL << (64-(TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS)));
+		ok1((tdb->flags & TDB_NOLOCK) || tdb->file->num_lockrecs == 1);
+		ok1((tdb->flags & TDB_NOLOCK)
+		    || tdb->file->lockrecs[0].off == TDB_HASH_LOCK_START);
+		/* FIXME: Check lock length */
+
+		/* Make it expand 0'th bucket. */
+		ok1(expand_group(tdb, &h) == 0);
+		/* First one should be subhash, next should be empty. */
+		ok1(is_subhash(h.group[0]));
+		subhash = (h.group[0] & TDB_OFF_MASK);
+		for (j = 1; j < (1 << TDB_HASH_GROUP_BITS); j++)
+			ok1(h.group[j] == 0);
+
+		ok1(tdb_write_convert(tdb, h.group_start,
+				      h.group, sizeof(h.group)) == 0);
+		ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+				      F_WRLCK) == 0);
+
+		/* Should be happy with expansion. */
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Should be able to find it. */
+		v = 0;
+		ok1(find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL)
+		    == new_off);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in expanded group 0, bucket 0. */
+		ok1(h.group_start == subhash + sizeof(struct tdb_used_record));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 0);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS
+		    + TDB_SUBLEVEL_HASH_BITS);
+
+		/* Should have lock on bucket 0 */
+		ok1(h.hlock_start == 0);
+		ok1(h.hlock_range ==
+		    1ULL << (64-(TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS)));
+		ok1((tdb->flags & TDB_NOLOCK) || tdb->file->num_lockrecs == 1);
+		ok1((tdb->flags & TDB_NOLOCK)
+		    || tdb->file->lockrecs[0].off == TDB_HASH_LOCK_START);
+		/* FIXME: Check lock length */
+
+		/* Simple delete should work. */
+		ok1(delete_from_hash(tdb, &h) == 0);
+		ok1(add_free_record(tdb, new_off,
+				    sizeof(struct tdb_used_record)
+				    + rec_key_length(&rec)
+				    + rec_data_length(&rec)
+				    + rec_extra_padding(&rec),
+				    TDB_LOCK_NOWAIT, false) == 0);
+		ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+				      F_WRLCK) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Test second-level expansion: should expand 0th bucket. */
+		v = 0;
+		ok1(find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL) == 0);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in group 0, bucket 0. */
+		ok1(h.group_start == subhash + sizeof(struct tdb_used_record));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 0);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS+TDB_SUBLEVEL_HASH_BITS);
+
+		/* Should have lock on bucket 0 */
+		ok1(h.hlock_start == 0);
+		ok1(h.hlock_range ==
+		    1ULL << (64-(TDB_TOPLEVEL_HASH_BITS-TDB_HASH_GROUP_BITS)));
+		ok1((tdb->flags & TDB_NOLOCK) || tdb->file->num_lockrecs == 1);
+		ok1((tdb->flags & TDB_NOLOCK)
+		    || tdb->file->lockrecs[0].off == TDB_HASH_LOCK_START);
+		/* FIXME: Check lock length */
+
+		ok1(expand_group(tdb, &h) == 0);
+		/* First one should be subhash, next should be empty. */
+		ok1(is_subhash(h.group[0]));
+		subhash = (h.group[0] & TDB_OFF_MASK);
+		for (j = 1; j < (1 << TDB_HASH_GROUP_BITS); j++)
+			ok1(h.group[j] == 0);
+		ok1(tdb_write_convert(tdb, h.group_start,
+				      h.group, sizeof(h.group)) == 0);
+		ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+				      F_WRLCK) == 0);
+
+		/* Should be happy with expansion. */
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		ok1(find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL) == 0);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in group 0, bucket 0. */
+		ok1(h.group_start == subhash + sizeof(struct tdb_used_record));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 0);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS
+		    + TDB_SUBLEVEL_HASH_BITS * 2);
+
+		/* We should be able to add it now. */
+		/* Allocate a new record. */
+		new_off = alloc(tdb, key.dsize, dbuf.dsize, h.h,
+				TDB_USED_MAGIC, false);
+		ok1(!TDB_OFF_IS_ERR(new_off));
+		ok1(add_to_hash(tdb, &h, new_off) == 0);
+
+		/* Make sure we fill it in for later finding. */
+		off = new_off + sizeof(struct tdb_used_record);
+		ok1(!tdb->methods->twrite(tdb, off, key.dptr, key.dsize));
+		off += key.dsize;
+		ok1(!tdb->methods->twrite(tdb, off, dbuf.dptr, dbuf.dsize));
+
+		/* We should be able to unlock that OK. */
+		ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+				      F_WRLCK) == 0);
+
+		/* Database should be consistent. */
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Should be able to find it. */
+		v = 0;
+		ok1(find_and_lock(tdb, key, F_WRLCK, &h, &rec, NULL)
+		    == new_off);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in expanded group 0, bucket 0. */
+		ok1(h.group_start == subhash + sizeof(struct tdb_used_record));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 0);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS
+		    + TDB_SUBLEVEL_HASH_BITS * 2);
+
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-05-readonly-open.c b/lib/tdb2/test/run-05-readonly-open.c
new file mode 100644
index 00000000000..0f1a4343d82
--- /dev/null
+++ b/lib/tdb2/test/run-05-readonly-open.c
@@ -0,0 +1,88 @@
+#include <ccan/failtest/failtest_override.h>
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include <ccan/failtest/failtest.h>
+#include "logging.h"
+#include "failtest_helper.h"
+
+static bool failtest_suppress = false;
+
+/* Don't need to test everything here, just want expand testing. */
+static enum failtest_result
+suppress_failure(struct failtest_call *history, unsigned num)
+{
+	if (failtest_suppress)
+		return FAIL_DONT_FAIL;
+	return block_repeat_failures(history, num);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4), d;
+	union tdb_attribute seed_attr;
+	unsigned int msgs = 0;
+
+	failtest_init(argc, argv);
+	failtest_hook = suppress_failure;
+	failtest_exit_check = exit_check_log;
+
+	seed_attr.base.attr = TDB_ATTRIBUTE_SEED;
+	seed_attr.base.next = &tap_log_attr;
+	seed_attr.seed.seed = 0;
+
+	failtest_suppress = true;
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 11);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-05-readonly-open.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &seed_attr);
+		ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+		tdb_close(tdb);
+
+		failtest_suppress = false;
+		tdb = tdb_open("run-05-readonly-open.tdb", flags[i],
+			       O_RDONLY, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			break;
+		ok1(tap_log_messages == msgs);
+		/* Fetch should succeed, stores should fail. */
+		if (!ok1(tdb_fetch(tdb, key, &d) == 0))
+			goto fail;
+		ok1(tdb_deq(d, data));
+		free(d.dptr);
+		if (!ok1(tdb_store(tdb, key, data, TDB_MODIFY)
+			 == TDB_ERR_RDONLY))
+			goto fail;
+		ok1(tap_log_messages == ++msgs);
+		if (!ok1(tdb_store(tdb, key, data, TDB_INSERT)
+			 == TDB_ERR_RDONLY))
+			goto fail;
+		ok1(tap_log_messages == ++msgs);
+		failtest_suppress = true;
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		tdb_close(tdb);
+		ok1(tap_log_messages == msgs);
+		/* SIGH: failtest bug, it doesn't save the tdb file because
+		 * we have it read-only.  If we go around again, it gets
+		 * changed underneath us and things get screwy. */
+		if (failtest_has_failed())
+			break;
+	}
+	failtest_exit(exit_status());
+
+fail:
+	failtest_suppress = true;
+	tdb_close(tdb);
+	failtest_exit(exit_status());
+}
diff --git a/lib/tdb2/test/run-10-simple-store.c b/lib/tdb2/test/run-10-simple-store.c
new file mode 100644
index 00000000000..35c387a3be5
--- /dev/null
+++ b/lib/tdb2/test/run-10-simple-store.c
@@ -0,0 +1,76 @@
+#include <ccan/failtest/failtest_override.h>
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include <ccan/failtest/failtest.h>
+#include "logging.h"
+#include "failtest_helper.h"
+
+static bool failtest_suppress = false;
+
+/* Don't need to test everything here, just want expand testing. */
+static enum failtest_result
+suppress_failure(struct failtest_call *history, unsigned num)
+{
+	if (failtest_suppress)
+		return FAIL_DONT_FAIL;
+	return block_repeat_failures(history, num);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4);
+
+	failtest_init(argc, argv);
+	failtest_hook = suppress_failure;
+	failtest_exit_check = exit_check_log;
+
+	failtest_suppress = true;
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 7 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-10-simple-store.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			break;
+		/* Modify should fail. */
+		failtest_suppress = false;
+		if (!ok1(tdb_store(tdb, key, data, TDB_MODIFY)
+			 == TDB_ERR_NOEXIST))
+			goto fail;
+		failtest_suppress = true;
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		/* Insert should succeed. */
+		failtest_suppress = false;
+		if (!ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0))
+			goto fail;
+		failtest_suppress = true;
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		/* Second insert should fail. */
+		failtest_suppress = false;
+		if (!ok1(tdb_store(tdb, key, data, TDB_INSERT)
+			 == TDB_ERR_EXISTS))
+			goto fail;
+		failtest_suppress = true;
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		tdb_close(tdb);
+	}
+	ok1(tap_log_messages == 0);
+	failtest_exit(exit_status());
+
+fail:
+	failtest_suppress = true;
+	tdb_close(tdb);
+	failtest_exit(exit_status());
+}
diff --git a/lib/tdb2/test/run-11-simple-fetch.c b/lib/tdb2/test/run-11-simple-fetch.c
new file mode 100644
index 00000000000..29b6bf08727
--- /dev/null
+++ b/lib/tdb2/test/run-11-simple-fetch.c
@@ -0,0 +1,76 @@
+#include <ccan/failtest/failtest_override.h>
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include <ccan/failtest/failtest.h>
+#include "logging.h"
+#include "failtest_helper.h"
+
+static bool failtest_suppress = false;
+
+/* Don't need to test everything here, just want fetch testing. */
+static enum failtest_result
+suppress_failure(struct failtest_call *history, unsigned num)
+{
+	if (failtest_suppress)
+		return FAIL_DONT_FAIL;
+	return block_repeat_failures(history, num);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4);
+
+	failtest_init(argc, argv);
+	failtest_hook = suppress_failure;
+	failtest_exit_check = exit_check_log;
+
+	failtest_suppress = true;
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 8 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-11-simple-fetch.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (tdb) {
+			struct tdb_data d = { NULL, 0 }; /* Bogus GCC warning */
+
+			/* fetch should fail. */
+			failtest_suppress = false;
+			if (!ok1(tdb_fetch(tdb, key, &d) == TDB_ERR_NOEXIST))
+				goto fail;
+			failtest_suppress = true;
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			/* Insert should succeed. */
+			ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			/* Fetch should now work. */
+			failtest_suppress = false;
+			if (!ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS))
+				goto fail;
+			failtest_suppress = true;
+			ok1(tdb_deq(d, data));
+			free(d.dptr);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			tdb_close(tdb);
+		}
+	}
+	ok1(tap_log_messages == 0);
+	return exit_status();
+
+fail:
+	failtest_suppress = true;
+	tdb_close(tdb);
+	failtest_exit(exit_status());
+}
diff --git a/lib/tdb2/test/run-12-store.c b/lib/tdb2/test/run-12-store.c
new file mode 100644
index 00000000000..ba2e4f89717
--- /dev/null
+++ b/lib/tdb2/test/run-12-store.c
@@ -0,0 +1,58 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+/* We use the same seed which we saw a failure on. */
+static uint64_t fixedhash(const void *key, size_t len, uint64_t seed, void *p)
+{
+	return hash64_stable((const unsigned char *)key, len,
+			     *(uint64_t *)p);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	uint64_t seed = 16014841315512641303ULL;
+	union tdb_attribute fixed_hattr
+		= { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+			      .fn = fixedhash,
+			      .data = &seed } };
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = { (unsigned char *)&j, sizeof(j) };
+	struct tdb_data data = { (unsigned char *)&j, sizeof(j) };
+
+	fixed_hattr.base.next = &tap_log_attr;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * (1 + 500 * 3) + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-12-store.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &fixed_hattr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* We seemed to lose some keys.
+		 * Insert and check they're in there! */
+		for (j = 0; j < 500; j++) {
+			struct tdb_data d = { NULL, 0 }; /* Bogus GCC warning */
+			ok1(tdb_store(tdb, key, data, TDB_REPLACE) == 0);
+			ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+			ok1(tdb_deq(d, data));
+			free(d.dptr);
+		}
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-13-delete.c b/lib/tdb2/test/run-13-delete.c
new file mode 100644
index 00000000000..3b464d927ef
--- /dev/null
+++ b/lib/tdb2/test/run-13-delete.c
@@ -0,0 +1,207 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+/* We rig the hash so adjacent-numbered records always clash. */
+static uint64_t clash(const void *key, size_t len, uint64_t seed, void *priv)
+{
+	return ((uint64_t)*(const unsigned int *)key)
+		<< (64 - TDB_TOPLEVEL_HASH_BITS - 1);
+}
+
+/* We use the same seed which we saw a failure on. */
+static uint64_t fixedhash(const void *key, size_t len, uint64_t seed, void *p)
+{
+	return hash64_stable((const unsigned char *)key, len,
+			     *(uint64_t *)p);
+}
+
+static bool store_records(struct tdb_context *tdb)
+{
+	int i;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data d, data = { (unsigned char *)&i, sizeof(i) };
+
+	for (i = 0; i < 1000; i++) {
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+			return false;
+		tdb_fetch(tdb, key, &d);
+		if (!tdb_deq(d, data))
+			return false;
+		free(d.dptr);
+	}
+	return true;
+}
+
+static void test_val(struct tdb_context *tdb, uint64_t val)
+{
+	uint64_t v;
+	struct tdb_data key = { (unsigned char *)&v, sizeof(v) };
+	struct tdb_data d, data = { (unsigned char *)&v, sizeof(v) };
+
+	/* Insert an entry, then delete it. */
+	v = val;
+	/* Delete should fail. */
+	ok1(tdb_delete(tdb, key) == TDB_ERR_NOEXIST);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Insert should succeed. */
+	ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Delete should succeed. */
+	ok1(tdb_delete(tdb, key) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Re-add it, then add collision. */
+	ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+	v = val + 1;
+	ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Can find both? */
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+	v = val;
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+
+	/* Delete second one. */
+	v = val + 1;
+	ok1(tdb_delete(tdb, key) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Re-add */
+	ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Now, try deleting first one. */
+	v = val;
+	ok1(tdb_delete(tdb, key) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* Can still find second? */
+	v = val + 1;
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+
+	/* Now, this will be ideally placed. */
+	v = val + 2;
+	ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	/* This will collide with both. */
+	v = val;
+	ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+
+	/* We can still find them all, right? */
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+	v = val + 1;
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+	v = val + 2;
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+
+	/* And if we delete val + 1, that val + 2 should not move! */
+	v = val + 1;
+	ok1(tdb_delete(tdb, key) == 0);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	v = val;
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+	v = val + 2;
+	ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+	ok1(d.dsize == data.dsize);
+	free(d.dptr);
+
+	/* Delete those two, so we are empty. */
+	ok1(tdb_delete(tdb, key) == 0);
+	v = val;
+	ok1(tdb_delete(tdb, key) == 0);
+
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	uint64_t seed = 16014841315512641303ULL;
+	union tdb_attribute clash_hattr
+		= { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+			      .fn = clash } };
+	union tdb_attribute fixed_hattr
+		= { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+			      .fn = fixedhash,
+			      .data = &seed } };
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	/* These two values gave trouble before. */
+	int vals[] = { 755, 837 };
+
+	clash_hattr.base.next = &tap_log_attr;
+	fixed_hattr.base.next = &tap_log_attr;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0])
+		   * (39 * 3 + 5 + sizeof(vals)/sizeof(vals[0])*2) + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-13-delete.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &clash_hattr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* Check start of hash table. */
+		test_val(tdb, 0);
+
+		/* Check end of hash table. */
+		test_val(tdb, -1ULL);
+
+		/* Check mixed bitpattern. */
+		test_val(tdb, 0x123456789ABCDEF0ULL);
+
+		ok1(!tdb->file || (tdb->file->allrecord_lock.count == 0
+				   && tdb->file->num_lockrecs == 0));
+		tdb_close(tdb);
+
+		/* Deleting these entries in the db gave problems. */
+		tdb = tdb_open("run-13-delete.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &fixed_hattr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		ok1(store_records(tdb));
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		for (j = 0; j < sizeof(vals)/sizeof(vals[0]); j++) {
+			struct tdb_data key;
+
+			key.dptr = (unsigned char *)&vals[j];
+			key.dsize = sizeof(vals[j]);
+			ok1(tdb_delete(tdb, key) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+		}
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-14-exists.c b/lib/tdb2/test/run-14-exists.c
new file mode 100644
index 00000000000..f264a6f2c98
--- /dev/null
+++ b/lib/tdb2/test/run-14-exists.c
@@ -0,0 +1,57 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static bool test_records(struct tdb_context *tdb)
+{
+	int i;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+
+	for (i = 0; i < 1000; i++) {
+		if (tdb_exists(tdb, key))
+			return false;
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+			return false;
+		if (!tdb_exists(tdb, key))
+			return false;
+	}
+
+	for (i = 0; i < 1000; i++) {
+		if (!tdb_exists(tdb, key))
+			return false;
+		if (tdb_delete(tdb, key) != 0)
+			return false;
+		if (tdb_exists(tdb, key))
+			return false;
+	}
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 2 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-14-exists.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (ok1(tdb))
+			ok1(test_records(tdb));
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-15-append.c b/lib/tdb2/test/run-15-append.c
new file mode 100644
index 00000000000..d2f9ec65989
--- /dev/null
+++ b/lib/tdb2/test/run-15-append.c
@@ -0,0 +1,135 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include <ccan/ilog/ilog.h>
+#include "logging.h"
+
+#define MAX_SIZE 13100
+#define SIZE_STEP 131
+
+static tdb_off_t tdb_offset(struct tdb_context *tdb, struct tdb_data key)
+{
+	tdb_off_t off;
+	struct tdb_used_record rec;
+	struct hash_info h;
+
+	off = find_and_lock(tdb, key, F_RDLCK, &h, &rec, NULL);
+	if (TDB_OFF_IS_ERR(off))
+		return 0;
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_RDLCK);
+	return off;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, moves;
+	struct tdb_context *tdb;
+	unsigned char *buffer;
+	tdb_off_t oldoff = 0, newoff;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data;
+
+	buffer = malloc(MAX_SIZE);
+	for (i = 0; i < MAX_SIZE; i++)
+		buffer[i] = i;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0])
+		   * ((3 + MAX_SIZE/SIZE_STEP * 5) * 2 + 7)
+		   + 1);
+
+	/* Using tdb_store. */
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-append.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		moves = 0;
+		for (j = 0; j < MAX_SIZE; j += SIZE_STEP) {
+			data.dptr = buffer;
+			data.dsize = j;
+			ok1(tdb_store(tdb, key, data, TDB_REPLACE) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			ok1(tdb_fetch(tdb, key, &data) == TDB_SUCCESS);
+			ok1(data.dsize == j);
+			ok1(memcmp(data.dptr, buffer, data.dsize) == 0);
+			free(data.dptr);
+			newoff = tdb_offset(tdb, key);
+			if (newoff != oldoff)
+				moves++;
+			oldoff = newoff;
+		}
+		ok1(!tdb->file || (tdb->file->allrecord_lock.count == 0
+				   && tdb->file->num_lockrecs == 0));
+		/* We should increase by 50% each time... */
+		ok(moves <= ilog64(j / SIZE_STEP)*2, "Moved %u times", moves);
+		tdb_close(tdb);
+	}
+
+	/* Using tdb_append. */
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		size_t prev_len = 0;
+		tdb = tdb_open("run-append.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		moves = 0;
+		for (j = 0; j < MAX_SIZE; j += SIZE_STEP) {
+			data.dptr = buffer + prev_len;
+			data.dsize = j - prev_len;
+			ok1(tdb_append(tdb, key, data) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			ok1(tdb_fetch(tdb, key, &data) == TDB_SUCCESS);
+			ok1(data.dsize == j);
+			ok1(memcmp(data.dptr, buffer, data.dsize) == 0);
+			free(data.dptr);
+			prev_len = data.dsize;
+			newoff = tdb_offset(tdb, key);
+			if (newoff != oldoff)
+				moves++;
+			oldoff = newoff;
+		}
+		ok1(!tdb->file || (tdb->file->allrecord_lock.count == 0
+				   && tdb->file->num_lockrecs == 0));
+		/* We should increase by 50% each time... */
+		ok(moves <= ilog64(j / SIZE_STEP)*2, "Moved %u times", moves);
+		tdb_close(tdb);
+	}
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-append.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* Huge initial store. */
+		data.dptr = buffer;
+		data.dsize = MAX_SIZE;
+		ok1(tdb_append(tdb, key, data) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tdb_fetch(tdb, key, &data) == TDB_SUCCESS);
+		ok1(data.dsize == MAX_SIZE);
+		ok1(memcmp(data.dptr, buffer, data.dsize) == 0);
+		free(data.dptr);
+		ok1(!tdb->file || (tdb->file->allrecord_lock.count == 0
+				   && tdb->file->num_lockrecs == 0));
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	free(buffer);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-16-wipe_all.c b/lib/tdb2/test/run-16-wipe_all.c
new file mode 100644
index 00000000000..d9c5128e0bb
--- /dev/null
+++ b/lib/tdb2/test/run-16-wipe_all.c
@@ -0,0 +1,50 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static bool add_records(struct tdb_context *tdb)
+{
+	int i;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+
+	for (i = 0; i < 1000; i++) {
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+			return false;
+	}
+	return true;
+}
+
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 4 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-16-wipe_all.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (ok1(tdb)) {
+			struct tdb_data key;
+			ok1(add_records(tdb));
+			ok1(tdb_wipe_all(tdb) == TDB_SUCCESS);
+			ok1(tdb_firstkey(tdb, &key) == TDB_ERR_NOEXIST);
+			tdb_close(tdb);
+		}
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-20-growhash.c b/lib/tdb2/test/run-20-growhash.c
new file mode 100644
index 00000000000..22a88c45043
--- /dev/null
+++ b/lib/tdb2/test/run-20-growhash.c
@@ -0,0 +1,144 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static uint64_t myhash(const void *key, size_t len, uint64_t seed, void *priv)
+{
+	return *(const uint64_t *)key;
+}
+
+static void add_bits(uint64_t *val, unsigned new, unsigned new_bits,
+		     unsigned *done)
+{
+	*done += new_bits;
+	*val |= ((uint64_t)new << (64 - *done));
+}
+
+static uint64_t make_key(unsigned topgroup, unsigned topbucket,
+			 unsigned subgroup1, unsigned subbucket1,
+			 unsigned subgroup2, unsigned subbucket2)
+{
+	uint64_t key = 0;
+	unsigned done = 0;
+
+	add_bits(&key, topgroup, TDB_TOPLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS,
+		 &done);
+	add_bits(&key, topbucket, TDB_HASH_GROUP_BITS, &done);
+	add_bits(&key, subgroup1, TDB_SUBLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS,
+		 &done);
+	add_bits(&key, subbucket1, TDB_HASH_GROUP_BITS, &done);
+	add_bits(&key, subgroup2, TDB_SUBLEVEL_HASH_BITS - TDB_HASH_GROUP_BITS,
+		 &done);
+	add_bits(&key, subbucket2, TDB_HASH_GROUP_BITS, &done);
+	return key;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	uint64_t kdata;
+	struct tdb_used_record rec;
+	struct tdb_data key = { (unsigned char *)&kdata, sizeof(kdata) };
+	struct tdb_data dbuf = { (unsigned char *)&kdata, sizeof(kdata) };
+	union tdb_attribute hattr = { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+						.fn = myhash } };
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT,
+	};
+
+	hattr.base.next = &tap_log_attr;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0])
+		   * (9 + (20 + 2 * ((1 << TDB_HASH_GROUP_BITS) - 2))
+		      * (1 << TDB_HASH_GROUP_BITS)) + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		struct hash_info h;
+
+		tdb = tdb_open("run-04-basichash.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &hattr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* Fill a group. */
+		for (j = 0; j < (1 << TDB_HASH_GROUP_BITS); j++) {
+			kdata = make_key(0, j, 0, 0, 0, 0);
+			ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+		}
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Check first still exists. */
+		kdata = make_key(0, 0, 0, 0, 0, 0);
+		ok1(find_and_lock(tdb, key, F_RDLCK, &h, &rec, NULL) != 0);
+		/* Should have created correct hash. */
+		ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+		/* Should have located space in group 0, bucket 0. */
+		ok1(h.group_start == offsetof(struct tdb_header, hashtable));
+		ok1(h.home_bucket == 0);
+		ok1(h.found_bucket == 0);
+		ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS);
+		/* Entire group should be full! */
+		for (j = 0; j < (1 << TDB_HASH_GROUP_BITS); j++)
+			ok1(h.group[j] != 0);
+
+		ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+				      F_RDLCK) == 0);
+
+		/* Now, add one more to each should expand (that) bucket. */
+		for (j = 0; j < (1 << TDB_HASH_GROUP_BITS); j++) {
+			unsigned int k;
+			kdata = make_key(0, j, 0, 1, 0, 0);
+			ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+			ok1(find_and_lock(tdb, key, F_RDLCK, &h, &rec, NULL));
+			/* Should have created correct hash. */
+			ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+			/* Should have moved to subhash */
+			ok1(h.group_start >= sizeof(struct tdb_header));
+			ok1(h.home_bucket == 1);
+			ok1(h.found_bucket == 1);
+			ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS
+			    + TDB_SUBLEVEL_HASH_BITS);
+			ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+					      F_RDLCK) == 0);
+
+			/* Keep adding, make it expand again. */
+			for (k = 2; k < (1 << TDB_HASH_GROUP_BITS); k++) {
+				kdata = make_key(0, j, 0, k, 0, 0);
+				ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+				ok1(tdb_check(tdb, NULL, NULL) == 0);
+			}
+
+			/* This should tip it over to sub-sub-hash. */
+			kdata = make_key(0, j, 0, 0, 0, 1);
+			ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+			ok1(find_and_lock(tdb, key, F_RDLCK, &h, &rec, NULL));
+			/* Should have created correct hash. */
+			ok1(h.h == tdb_hash(tdb, key.dptr, key.dsize));
+			/* Should have moved to subhash */
+			ok1(h.group_start >= sizeof(struct tdb_header));
+			ok1(h.home_bucket == 1);
+			ok1(h.found_bucket == 1);
+			ok1(h.hash_used == TDB_TOPLEVEL_HASH_BITS
+			    + TDB_SUBLEVEL_HASH_BITS + TDB_SUBLEVEL_HASH_BITS);
+			ok1(tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range,
+					      F_RDLCK) == 0);
+		}
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-21-parse_record.c b/lib/tdb2/test/run-21-parse_record.c
new file mode 100644
index 00000000000..773cdff4e0c
--- /dev/null
+++ b/lib/tdb2/test/run-21-parse_record.c
@@ -0,0 +1,70 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static enum TDB_ERROR parse(TDB_DATA key, TDB_DATA data, TDB_DATA *expected)
+{
+	if (!tdb_deq(data, *expected))
+		return TDB_ERR_EINVAL;
+	return TDB_SUCCESS;
+}
+
+static enum TDB_ERROR parse_err(TDB_DATA key, TDB_DATA data, void *unused)
+{
+	return 100;
+}
+
+static bool test_records(struct tdb_context *tdb)
+{
+	int i;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+
+	for (i = 0; i < 1000; i++) {
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+			return false;
+	}
+
+	for (i = 0; i < 1000; i++) {
+		if (tdb_parse_record(tdb, key, parse, &data) != TDB_SUCCESS)
+			return false;
+	}
+
+	if (tdb_parse_record(tdb, key, parse, &data) != TDB_ERR_NOEXIST)
+		return false;
+
+	/* Test error return from parse function. */
+	i = 0;
+	if (tdb_parse_record(tdb, key, parse_err, NULL) != 100)
+		return false;
+
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 2 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-14-exists.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (ok1(tdb))
+			ok1(test_records(tdb));
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-25-hashoverload.c b/lib/tdb2/test/run-25-hashoverload.c
new file mode 100644
index 00000000000..83f549d6b26
--- /dev/null
+++ b/lib/tdb2/test/run-25-hashoverload.c
@@ -0,0 +1,121 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static uint64_t badhash(const void *key, size_t len, uint64_t seed, void *priv)
+{
+	return 0;
+}
+
+static int trav(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, void *p)
+{
+	if (p)
+		return tdb_delete(tdb, key);
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	struct tdb_data key = { (unsigned char *)&j, sizeof(j) };
+	struct tdb_data dbuf = { (unsigned char *)&j, sizeof(j) };
+	union tdb_attribute hattr = { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+						.fn = badhash } };
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT,
+	};
+
+	hattr.base.next = &tap_log_attr;
+
+	plan_tests(6883);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		struct tdb_data d = { NULL, 0 }; /* Bogus GCC warning */
+
+		tdb = tdb_open("run-25-hashoverload.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &hattr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* Fill a group. */
+		for (j = 0; j < (1 << TDB_HASH_GROUP_BITS); j++) {
+			ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+		}
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Now store one last value: should form chain. */
+		ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Check we can find them all. */
+		for (j = 0; j < (1 << TDB_HASH_GROUP_BITS) + 1; j++) {
+			ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+			ok1(d.dsize == sizeof(j));
+			ok1(d.dptr != NULL);
+			ok1(d.dptr && memcmp(d.dptr, &j, d.dsize) == 0);
+			free(d.dptr);
+		}
+
+		/* Now add a *lot* more. */
+		for (j = (1 << TDB_HASH_GROUP_BITS) + 1;
+		     j < (16 << TDB_HASH_GROUP_BITS);
+		     j++) {
+			ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+			ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+			ok1(d.dsize == sizeof(j));
+			ok1(d.dptr != NULL);
+			ok1(d.dptr && memcmp(d.dptr, &j, d.dsize) == 0);
+			free(d.dptr);
+		}
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Traverse through them. */
+		ok1(tdb_traverse(tdb, trav, NULL) == j);
+
+		/* Empty the first chain-worth. */
+		for (j = 0; j < (1 << TDB_HASH_GROUP_BITS); j++)
+			ok1(tdb_delete(tdb, key) == 0);
+
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		for (j = (1 << TDB_HASH_GROUP_BITS);
+		     j < (16 << TDB_HASH_GROUP_BITS);
+		     j++) {
+			ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+			ok1(d.dsize == sizeof(j));
+			ok1(d.dptr != NULL);
+			ok1(d.dptr && memcmp(d.dptr, &j, d.dsize) == 0);
+			free(d.dptr);
+		}
+
+		/* Traverse through them. */
+		ok1(tdb_traverse(tdb, trav, NULL)
+		    == (15 << TDB_HASH_GROUP_BITS));
+
+		/* Re-add */
+		for (j = 0; j < (1 << TDB_HASH_GROUP_BITS); j++) {
+			ok1(tdb_store(tdb, key, dbuf, TDB_INSERT) == 0);
+		}
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Now try deleting as we go. */
+		ok1(tdb_traverse(tdb, trav, trav)
+		    == (16 << TDB_HASH_GROUP_BITS));
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tdb_traverse(tdb, trav, NULL) == 0);
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-30-exhaust-before-expand.c b/lib/tdb2/test/run-30-exhaust-before-expand.c
new file mode 100644
index 00000000000..2386f85f26f
--- /dev/null
+++ b/lib/tdb2/test/run-30-exhaust-before-expand.c
@@ -0,0 +1,79 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include <err.h>
+#include "logging.h"
+
+static bool empty_freetable(struct tdb_context *tdb)
+{
+	struct tdb_freetable ftab;
+	unsigned int i;
+
+	/* Now, free table should be completely exhausted in zone 0 */
+	if (tdb_read_convert(tdb, tdb->ftable_off, &ftab, sizeof(ftab)) != 0)
+		abort();
+
+	for (i = 0; i < sizeof(ftab.buckets)/sizeof(ftab.buckets[0]); i++) {
+		if (ftab.buckets[i])
+			return false;
+	}
+	return true;
+}
+
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 9 + 1);
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		TDB_DATA k;
+		uint64_t size;
+		bool was_empty = false;
+
+		k.dptr = (void *)&j;
+		k.dsize = sizeof(j);
+
+		tdb = tdb_open("run-30-exhaust-before-expand.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		ok1(empty_freetable(tdb));
+		/* Need some hash lock for expand. */
+		ok1(tdb_lock_hashes(tdb, 0, 1, F_WRLCK, TDB_LOCK_WAIT) == 0);
+		/* Create some free space. */
+		ok1(tdb_expand(tdb, 1) == 0);
+		ok1(tdb_unlock_hashes(tdb, 0, 1, F_WRLCK) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(!empty_freetable(tdb));
+
+		size = tdb->file->map_size;
+		/* Insert minimal-length records until we expand. */
+		for (j = 0; tdb->file->map_size == size; j++) {
+			was_empty = empty_freetable(tdb);
+			if (tdb_store(tdb, k, k, TDB_INSERT) != 0)
+				err(1, "Failed to store record %i", j);
+		}
+
+		/* Would have been empty before expansion, but no longer. */
+		ok1(was_empty);
+		ok1(!empty_freetable(tdb));
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-50-multiple-freelists.c b/lib/tdb2/test/run-50-multiple-freelists.c
new file mode 100644
index 00000000000..7a48c3e0eee
--- /dev/null
+++ b/lib/tdb2/test/run-50-multiple-freelists.c
@@ -0,0 +1,71 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include <ccan/tdb2/transaction.c>
+#include "logging.h"
+#include "layout.h"
+
+int main(int argc, char *argv[])
+{
+	tdb_off_t off;
+	struct tdb_context *tdb;
+	struct tdb_layout *layout;
+	TDB_DATA key, data;
+
+	plan_tests(11);
+	key = tdb_mkdata("Hello", 5);
+	data = tdb_mkdata("world", 5);
+
+	/* Create a TDB with three free tables. */
+	layout = new_tdb_layout(NULL);
+	tdb_layout_add_freetable(layout);
+	tdb_layout_add_freetable(layout);
+	tdb_layout_add_freetable(layout);
+	tdb_layout_add_free(layout, 80, 0);
+	/* Used record prevent coalescing. */
+	tdb_layout_add_used(layout, key, data, 6);
+	tdb_layout_add_free(layout, 160, 1);
+	key.dsize--;
+	tdb_layout_add_used(layout, key, data, 7);
+	tdb_layout_add_free(layout, 320, 2);
+	key.dsize--;
+	tdb_layout_add_used(layout, key, data, 8);
+	tdb_layout_add_free(layout, 40, 0);
+	tdb = tdb_layout_get(layout);
+	ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+	off = get_free(tdb, 0, 80 - sizeof(struct tdb_used_record), 0,
+		       TDB_USED_MAGIC, 0);
+	ok1(off == layout->elem[3].base.off);
+	ok1(tdb->ftable_off == layout->elem[0].base.off);
+
+	off = get_free(tdb, 0, 160 - sizeof(struct tdb_used_record), 0,
+		       TDB_USED_MAGIC, 0);
+	ok1(off == layout->elem[5].base.off);
+	ok1(tdb->ftable_off == layout->elem[1].base.off);
+
+	off = get_free(tdb, 0, 320 - sizeof(struct tdb_used_record), 0,
+		       TDB_USED_MAGIC, 0);
+	ok1(off == layout->elem[7].base.off);
+	ok1(tdb->ftable_off == layout->elem[2].base.off);
+
+	off = get_free(tdb, 0, 40 - sizeof(struct tdb_used_record), 0,
+		       TDB_USED_MAGIC, 0);
+	ok1(off == layout->elem[9].base.off);
+	ok1(tdb->ftable_off == layout->elem[0].base.off);
+
+	/* Now we fail. */
+	off = get_free(tdb, 0, 0, 1, TDB_USED_MAGIC, 0);
+	ok1(off == 0);
+
+	tdb_close(tdb);
+	tdb_layout_free(layout);
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-55-transaction.c b/lib/tdb2/test/run-55-transaction.c
new file mode 100644
index 00000000000..1650e40e1f3
--- /dev/null
+++ b/lib/tdb2/test/run-55-transaction.c
@@ -0,0 +1,75 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	unsigned char *buffer;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data;
+
+	buffer = malloc(1000);
+	for (i = 0; i < 1000; i++)
+		buffer[i] = i;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 20 + 1);
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-55-transaction.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		ok1(tdb_transaction_start(tdb) == 0);
+		data.dptr = buffer;
+		data.dsize = 1000;
+		ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+		ok1(tdb_fetch(tdb, key, &data) == TDB_SUCCESS);
+		ok1(data.dsize == 1000);
+		ok1(memcmp(data.dptr, buffer, data.dsize) == 0);
+		free(data.dptr);
+
+		/* Cancelling a transaction means no store */
+		tdb_transaction_cancel(tdb);
+		ok1(tdb->file->allrecord_lock.count == 0
+		    && tdb->file->num_lockrecs == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tdb_fetch(tdb, key, &data) == TDB_ERR_NOEXIST);
+
+		/* Commit the transaction. */
+		ok1(tdb_transaction_start(tdb) == 0);
+		data.dptr = buffer;
+		data.dsize = 1000;
+		ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+		ok1(tdb_fetch(tdb, key, &data) == TDB_SUCCESS);
+		ok1(data.dsize == 1000);
+		ok1(memcmp(data.dptr, buffer, data.dsize) == 0);
+		free(data.dptr);
+		ok1(tdb_transaction_commit(tdb) == 0);
+		ok1(tdb->file->allrecord_lock.count == 0
+		    && tdb->file->num_lockrecs == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tdb_fetch(tdb, key, &data) == TDB_SUCCESS);
+		ok1(data.dsize == 1000);
+		ok1(memcmp(data.dptr, buffer, data.dsize) == 0);
+		free(data.dptr);
+
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	free(buffer);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-56-open-during-transaction.c b/lib/tdb2/test/run-56-open-during-transaction.c
new file mode 100644
index 00000000000..96107d637e4
--- /dev/null
+++ b/lib/tdb2/test/run-56-open-during-transaction.c
@@ -0,0 +1,175 @@
+#include "config.h"
+#include <unistd.h>
+#include "lock-tracking.h"
+
+static ssize_t pwrite_check(int fd, const void *buf, size_t count, off_t offset);
+static ssize_t write_check(int fd, const void *buf, size_t count);
+static int ftruncate_check(int fd, off_t length);
+
+#define pwrite pwrite_check
+#define write write_check
+#define fcntl fcntl_with_lockcheck
+#define ftruncate ftruncate_check
+
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <err.h>
+#include "external-agent.h"
+#include "logging.h"
+
+static struct agent *agent;
+static bool opened;
+static int errors = 0;
+#define TEST_DBNAME "run-56-open-during-transaction.tdb"
+
+#undef write
+#undef pwrite
+#undef fcntl
+#undef ftruncate
+
+static bool is_same(const char *snapshot, const char *latest, off_t len)
+{
+	unsigned i;
+
+	for (i = 0; i < len; i++) {
+		if (snapshot[i] != latest[i])
+			return false;
+	}
+	return true;
+}
+
+static bool compare_file(int fd, const char *snapshot, off_t snapshot_len)
+{
+	char *contents;
+	bool same;
+
+	/* over-length read serves as length check. */
+	contents = malloc(snapshot_len+1);
+	same = pread(fd, contents, snapshot_len+1, 0) == snapshot_len
+		&& is_same(snapshot, contents, snapshot_len);
+	free(contents);
+	return same;
+}
+
+static void check_file_intact(int fd)
+{
+	enum agent_return ret;
+	struct stat st;
+	char *contents;
+
+	fstat(fd, &st);
+	contents = malloc(st.st_size);
+	if (pread(fd, contents, st.st_size, 0) != st.st_size) {
+		diag("Read fail");
+		errors++;
+		return;
+	}
+
+	/* Ask agent to open file. */
+	ret = external_agent_operation(agent, OPEN, TEST_DBNAME);
+
+	/* It's OK to open it, but it must not have changed! */
+	if (!compare_file(fd, contents, st.st_size)) {
+		diag("Agent changed file after opening %s",
+		     agent_return_name(ret));
+		errors++;
+	}
+
+	if (ret == SUCCESS) {
+		ret = external_agent_operation(agent, CLOSE, NULL);
+		if (ret != SUCCESS) {
+			diag("Agent failed to close tdb: %s",
+			     agent_return_name(ret));
+			errors++;
+		}
+	} else if (ret != WOULD_HAVE_BLOCKED) {
+		diag("Agent opening file gave %s",
+		     agent_return_name(ret));
+		errors++;
+	}
+
+	free(contents);
+}
+
+static void after_unlock(int fd)
+{
+	if (opened)
+		check_file_intact(fd);
+}
+
+static ssize_t pwrite_check(int fd,
+			    const void *buf, size_t count, off_t offset)
+{
+	if (opened)
+		check_file_intact(fd);
+
+	return pwrite(fd, buf, count, offset);
+}
+
+static ssize_t write_check(int fd, const void *buf, size_t count)
+{
+	if (opened)
+		check_file_intact(fd);
+
+	return write(fd, buf, count);
+}
+
+static int ftruncate_check(int fd, off_t length)
+{
+	if (opened)
+		check_file_intact(fd);
+
+	return ftruncate(fd, length);
+
+}
+
+int main(int argc, char *argv[])
+{
+	const int flags[] = { TDB_DEFAULT,
+			      TDB_NOMMAP,
+			      TDB_CONVERT,
+			      TDB_CONVERT | TDB_NOMMAP };
+	int i;
+	struct tdb_context *tdb;
+	TDB_DATA key, data;
+
+	plan_tests(20);
+	agent = prepare_external_agent();
+	if (!agent)
+		err(1, "preparing agent");
+
+	unlock_callback = after_unlock;
+	for (i = 0; i < sizeof(flags)/sizeof(flags[0]); i++) {
+		diag("Test with %s and %s\n",
+		     (flags[i] & TDB_CONVERT) ? "CONVERT" : "DEFAULT",
+		     (flags[i] & TDB_NOMMAP) ? "no mmap" : "mmap");
+		unlink(TEST_DBNAME);
+		tdb = tdb_open(TEST_DBNAME, flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+
+		opened = true;
+		ok1(tdb_transaction_start(tdb) == 0);
+		key = tdb_mkdata("hi", strlen("hi"));
+		data = tdb_mkdata("world", strlen("world"));
+
+		ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+		ok1(tdb_transaction_commit(tdb) == 0);
+		ok(!errors, "We had %u open errors", errors);
+
+		opened = false;
+		tdb_close(tdb);
+	}
+
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-57-die-during-transaction.c b/lib/tdb2/test/run-57-die-during-transaction.c
new file mode 100644
index 00000000000..84f01eb21a8
--- /dev/null
+++ b/lib/tdb2/test/run-57-die-during-transaction.c
@@ -0,0 +1,275 @@
+#include "config.h"
+#include <unistd.h>
+#include "lock-tracking.h"
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+#include <assert.h>
+static ssize_t pwrite_check(int fd, const void *buf, size_t count, off_t offset);
+static ssize_t write_check(int fd, const void *buf, size_t count);
+static int ftruncate_check(int fd, off_t length);
+
+#define pwrite pwrite_check
+#define write write_check
+#define fcntl fcntl_with_lockcheck
+#define ftruncate ftruncate_check
+
+/* There's a malloc inside transaction_setup_recovery, and valgrind complains
+ * when we longjmp and leak it. */
+#define MAX_ALLOCATIONS 200
+static void *allocated[MAX_ALLOCATIONS];
+
+static void *malloc_noleak(size_t len)
+{
+	unsigned int i;
+
+	for (i = 0; i < MAX_ALLOCATIONS; i++)
+		if (!allocated[i]) {
+			allocated[i] = malloc(len);
+			return allocated[i];
+		}
+	diag("Too many allocations!");
+	abort();
+}
+
+static void free_noleak(void *p)
+{
+	unsigned int i;
+
+	/* We don't catch realloc, so don't care if we miss one. */
+	for (i = 0; i < MAX_ALLOCATIONS; i++) {
+		if (allocated[i] == p) {
+			allocated[i] = NULL;
+			break;
+		}
+	}
+	free(p);
+}
+
+static void free_all(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < MAX_ALLOCATIONS; i++) {
+		free(allocated[i]);
+		allocated[i] = NULL;
+	}
+}
+
+#define malloc malloc_noleak
+#define free free_noleak
+
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#undef malloc
+#undef free
+#undef write
+#undef pwrite
+#undef fcntl
+#undef ftruncate
+
+#include <stdbool.h>
+#include <stdarg.h>
+#include <err.h>
+#include <setjmp.h>
+#include "external-agent.h"
+#include "logging.h"
+
+static bool in_transaction;
+static int target, current;
+static jmp_buf jmpbuf;
+#define TEST_DBNAME "run-57-die-during-transaction.tdb"
+#define KEY_STRING "helloworld"
+
+static void maybe_die(int fd)
+{
+	if (in_transaction && current++ == target) {
+		longjmp(jmpbuf, 1);
+	}
+}
+
+static ssize_t pwrite_check(int fd,
+			    const void *buf, size_t count, off_t offset)
+{
+	ssize_t ret;
+
+	maybe_die(fd);
+
+	ret = pwrite(fd, buf, count, offset);
+	if (ret != count)
+		return ret;
+
+	maybe_die(fd);
+	return ret;
+}
+
+static ssize_t write_check(int fd, const void *buf, size_t count)
+{
+	ssize_t ret;
+
+	maybe_die(fd);
+
+	ret = write(fd, buf, count);
+	if (ret != count)
+		return ret;
+
+	maybe_die(fd);
+	return ret;
+}
+
+static int ftruncate_check(int fd, off_t length)
+{
+	int ret;
+
+	maybe_die(fd);
+
+	ret = ftruncate(fd, length);
+
+	maybe_die(fd);
+	return ret;
+}
+
+static bool test_death(enum operation op, struct agent *agent)
+{
+	struct tdb_context *tdb = NULL;
+	TDB_DATA key;
+	enum agent_return ret;
+	int needed_recovery = 0;
+
+	current = target = 0;
+reset:
+	unlink(TEST_DBNAME);
+	tdb = tdb_open(TEST_DBNAME, TDB_NOMMAP,
+		       O_CREAT|O_TRUNC|O_RDWR, 0600, &tap_log_attr);
+	if (!tdb) {
+		diag("Failed opening TDB: %s", strerror(errno));
+		return false;
+	}
+
+	if (setjmp(jmpbuf) != 0) {
+		/* We're partway through.  Simulate our death. */
+		close(tdb->file->fd);
+		forget_locking();
+		in_transaction = false;
+
+		ret = external_agent_operation(agent, NEEDS_RECOVERY, "");
+		if (ret == SUCCESS)
+			needed_recovery++;
+		else if (ret != FAILED) {
+			diag("Step %u agent NEEDS_RECOVERY = %s", current,
+			     agent_return_name(ret));
+			return false;
+		}
+
+		ret = external_agent_operation(agent, op, KEY_STRING);
+		if (ret != SUCCESS) {
+			diag("Step %u op %s failed = %s", current,
+			     operation_name(op),
+			     agent_return_name(ret));
+			return false;
+		}
+
+		ret = external_agent_operation(agent, NEEDS_RECOVERY, "");
+		if (ret != FAILED) {
+			diag("Still needs recovery after step %u = %s",
+			     current, agent_return_name(ret));
+			return false;
+		}
+
+		ret = external_agent_operation(agent, CHECK, "");
+		if (ret != SUCCESS) {
+			diag("Step %u check failed = %s", current,
+			     agent_return_name(ret));
+			return false;
+		}
+
+		ret = external_agent_operation(agent, CLOSE, "");
+		if (ret != SUCCESS) {
+			diag("Step %u close failed = %s", current,
+			     agent_return_name(ret));
+			return false;
+		}
+
+		/* Suppress logging as this tries to use closed fd. */
+		suppress_logging = true;
+		suppress_lockcheck = true;
+		tdb_close(tdb);
+		suppress_logging = false;
+		suppress_lockcheck = false;
+		target++;
+		current = 0;
+		free_all();
+		goto reset;
+	}
+
+	/* Put key for agent to fetch. */
+	key = tdb_mkdata(KEY_STRING, strlen(KEY_STRING));
+	if (tdb_store(tdb, key, key, TDB_INSERT) != 0)
+		return false;
+
+	/* This is the key we insert in transaction. */
+	key.dsize--;
+
+	ret = external_agent_operation(agent, OPEN, TEST_DBNAME);
+	if (ret != SUCCESS)
+		errx(1, "Agent failed to open: %s", agent_return_name(ret));
+
+	ret = external_agent_operation(agent, FETCH, KEY_STRING);
+	if (ret != SUCCESS)
+		errx(1, "Agent failed find key: %s", agent_return_name(ret));
+
+	in_transaction = true;
+	if (tdb_transaction_start(tdb) != 0)
+		return false;
+
+	if (tdb_store(tdb, key, key, TDB_INSERT) != 0)
+		return false;
+
+	if (tdb_transaction_commit(tdb) != 0)
+		return false;
+
+	in_transaction = false;
+
+	/* We made it! */
+	diag("Completed %u runs", current);
+	tdb_close(tdb);
+	ret = external_agent_operation(agent, CLOSE, "");
+	if (ret != SUCCESS) {
+		diag("Step %u close failed = %s", current,
+		     agent_return_name(ret));
+		return false;
+	}
+
+	ok1(needed_recovery);
+	ok1(locking_errors == 0);
+	ok1(forget_locking() == 0);
+	locking_errors = 0;
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	enum operation ops[] = { FETCH, STORE, TRANSACTION_START };
+	struct agent *agent;
+	int i;
+
+	plan_tests(12);
+	unlock_callback = maybe_die;
+
+	agent = prepare_external_agent();
+	if (!agent)
+		err(1, "preparing agent");
+
+	for (i = 0; i < sizeof(ops)/sizeof(ops[0]); i++) {
+		diag("Testing %s after death", operation_name(ops[i]));
+		ok1(test_death(ops[i], agent));
+	}
+
+	free_external_agent(agent);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-64-bit-tdb.c b/lib/tdb2/test/run-64-bit-tdb.c
new file mode 100644
index 00000000000..78dadca0164
--- /dev/null
+++ b/lib/tdb2/test/run-64-bit-tdb.c
@@ -0,0 +1,80 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	if (sizeof(off_t) <= 4) {
+		plan_tests(1);
+		pass("No 64 bit off_t");
+		return exit_status();
+	}
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 14);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		off_t old_size;
+		TDB_DATA k, d;
+		struct hash_info h;
+		struct tdb_used_record rec;
+		tdb_off_t off;
+
+		tdb = tdb_open("run-64-bit-tdb.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		old_size = tdb->file->map_size;
+
+		/* This makes a sparse file */
+		ok1(ftruncate(tdb->file->fd, 0xFFFFFFF0) == 0);
+		ok1(add_free_record(tdb, old_size, 0xFFFFFFF0 - old_size,
+				    TDB_LOCK_WAIT, false) == TDB_SUCCESS);
+
+		/* Now add a little record past the 4G barrier. */
+		ok1(tdb_expand_file(tdb, 100) == TDB_SUCCESS);
+		ok1(add_free_record(tdb, 0xFFFFFFF0, 100, TDB_LOCK_WAIT, false)
+		    == TDB_SUCCESS);
+
+		ok1(tdb_check(tdb, NULL, NULL) == TDB_SUCCESS);
+
+		/* Test allocation path. */
+		k = tdb_mkdata("key", 4);
+		d = tdb_mkdata("data", 5);
+		ok1(tdb_store(tdb, k, d, TDB_INSERT) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == TDB_SUCCESS);
+
+		/* Make sure it put it at end as we expected. */
+		off = find_and_lock(tdb, k, F_RDLCK, &h, &rec, NULL);
+		ok1(off >= 0xFFFFFFF0);
+		tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_RDLCK);
+
+		ok1(tdb_fetch(tdb, k, &d) == 0);
+		ok1(d.dsize == 5);
+		ok1(strcmp((char *)d.dptr, "data") == 0);
+		free(d.dptr);
+
+		ok1(tdb_delete(tdb, k) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == TDB_SUCCESS);
+
+		tdb_close(tdb);
+	}
+
+	/* We might get messages about mmap failing, so don't test
+	 * tap_log_messages */
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-80-tdb_fd.c b/lib/tdb2/test/run-80-tdb_fd.c
new file mode 100644
index 00000000000..e8b2fae2dd6
--- /dev/null
+++ b/lib/tdb2/test/run-80-tdb_fd.c
@@ -0,0 +1,35 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 3);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-new_database.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			continue;
+
+		if (flags[i] & TDB_INTERNAL)
+			ok1(tdb_fd(tdb) == -1);
+		else
+			ok1(tdb_fd(tdb) > 2);
+		tdb_close(tdb);
+		ok1(tap_log_messages == 0);
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-81-seqnum.c b/lib/tdb2/test/run-81-seqnum.c
new file mode 100644
index 00000000000..6e8b2698b6f
--- /dev/null
+++ b/lib/tdb2/test/run-81-seqnum.c
@@ -0,0 +1,71 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	struct tdb_data d = { NULL, 0 }; /* Bogus GCC warning */
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4);
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 15 + 4 * 13);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-new_database.tdb", flags[i]|TDB_SEQNUM,
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			continue;
+
+		ok1(tdb_get_seqnum(tdb) == 0);
+		ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+		ok1(tdb_get_seqnum(tdb) == 1);
+		/* Fetch doesn't change seqnum */
+		if (ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS))
+			free(d.dptr);
+		ok1(tdb_get_seqnum(tdb) == 1);
+		ok1(tdb_append(tdb, key, data) == TDB_SUCCESS);
+		ok1(tdb_get_seqnum(tdb) == 2);
+
+		ok1(tdb_delete(tdb, key) == TDB_SUCCESS);
+		ok1(tdb_get_seqnum(tdb) == 3);
+		/* Empty append works */
+		ok1(tdb_append(tdb, key, data) == TDB_SUCCESS);
+		ok1(tdb_get_seqnum(tdb) == 4);
+
+		ok1(tdb_wipe_all(tdb) == TDB_SUCCESS);
+		ok1(tdb_get_seqnum(tdb) == 5);
+
+		if (!(flags[i] & TDB_INTERNAL)) {
+			ok1(tdb_transaction_start(tdb) == TDB_SUCCESS);
+			ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+			ok1(tdb_get_seqnum(tdb) == 6);
+			ok1(tdb_append(tdb, key, data) == TDB_SUCCESS);
+			ok1(tdb_get_seqnum(tdb) == 7);
+			ok1(tdb_delete(tdb, key) == TDB_SUCCESS);
+			ok1(tdb_get_seqnum(tdb) == 8);
+			ok1(tdb_transaction_commit(tdb) == TDB_SUCCESS);
+			ok1(tdb_get_seqnum(tdb) == 8);
+
+			ok1(tdb_transaction_start(tdb) == TDB_SUCCESS);
+			ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+			ok1(tdb_get_seqnum(tdb) == 9);
+			tdb_transaction_cancel(tdb);
+			ok1(tdb_get_seqnum(tdb) == 8);
+		}
+		tdb_close(tdb);
+		ok1(tap_log_messages == 0);
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-82-lockattr.c b/lib/tdb2/test/run-82-lockattr.c
new file mode 100644
index 00000000000..bfc2653222f
--- /dev/null
+++ b/lib/tdb2/test/run-82-lockattr.c
@@ -0,0 +1,263 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static int mylock(int fd, int rw, off_t off, off_t len, bool waitflag,
+		  void *_err)
+{
+	int *lock_err = _err;
+	struct flock fl;
+	int ret;
+
+	if (*lock_err) {
+		errno = *lock_err;
+		return -1;
+	}
+
+	do {
+		fl.l_type = rw;
+		fl.l_whence = SEEK_SET;
+		fl.l_start = off;
+		fl.l_len = len;
+
+		if (waitflag)
+			ret = fcntl(fd, F_SETLKW, &fl);
+		else
+			ret = fcntl(fd, F_SETLK, &fl);
+	} while (ret != 0 && errno == EINTR);
+
+	return ret;
+}
+
+static int myunlock(int fd, int rw, off_t off, off_t len, void *_err)
+{
+	int *lock_err = _err;
+	struct flock fl;
+	int ret;
+
+	if (*lock_err) {
+		errno = *lock_err;
+		return -1;
+	}
+
+	do {
+		fl.l_type = F_UNLCK;
+		fl.l_whence = SEEK_SET;
+		fl.l_start = off;
+		fl.l_len = len;
+
+		ret = fcntl(fd, F_SETLKW, &fl);
+	} while (ret != 0 && errno == EINTR);
+
+	return ret;
+}
+
+static int trav_err;
+static int trav(struct tdb_context *tdb, TDB_DATA k, TDB_DATA d, int *err)
+{
+	*err = trav_err;
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+	union tdb_attribute lock_attr;
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4);
+	int lock_err;
+
+	lock_attr.base.attr = TDB_ATTRIBUTE_FLOCK;
+	lock_attr.base.next = &tap_log_attr;
+	lock_attr.flock.lock = mylock;
+	lock_attr.flock.unlock = myunlock;
+	lock_attr.flock.data = &lock_err;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 80);
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		struct tdb_data d;
+
+		/* Nonblocking open; expect no error message. */
+		lock_err = EAGAIN;
+		tdb = tdb_open("run-82-lockattr.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &lock_attr);
+		ok(errno == lock_err, "Errno is %u", errno);
+		ok1(!tdb);
+		ok1(tap_log_messages == 0);
+
+		lock_err = EINTR;
+		tdb = tdb_open("run-82-lockattr.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &lock_attr);
+		ok(errno == lock_err, "Errno is %u", errno);
+		ok1(!tdb);
+		ok1(tap_log_messages == 0);
+
+		/* Forced fail open. */
+		lock_err = ENOMEM;
+		tdb = tdb_open("run-82-lockattr.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &lock_attr);
+		ok1(errno == lock_err);
+		ok1(!tdb);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		lock_err = 0;
+		tdb = tdb_open("run-82-lockattr.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &lock_attr);
+		if (!ok1(tdb))
+			continue;
+		ok1(tap_log_messages == 0);
+
+		/* Nonblocking store. */
+		lock_err = EAGAIN;
+		ok1(tdb_store(tdb, key, data, TDB_REPLACE) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_store(tdb, key, data, TDB_REPLACE) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_store(tdb, key, data, TDB_REPLACE) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		/* Nonblocking fetch. */
+		lock_err = EAGAIN;
+		ok1(!tdb_exists(tdb, key));
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(!tdb_exists(tdb, key));
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(!tdb_exists(tdb, key));
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		lock_err = EAGAIN;
+		ok1(tdb_fetch(tdb, key, &d) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_fetch(tdb, key, &d) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_fetch(tdb, key, &d) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		/* Nonblocking delete. */
+		lock_err = EAGAIN;
+		ok1(tdb_delete(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_delete(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_delete(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		/* Nonblocking locks. */
+		lock_err = EAGAIN;
+		ok1(tdb_chainlock(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_chainlock(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_chainlock(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		lock_err = EAGAIN;
+		ok1(tdb_chainlock_read(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_chainlock_read(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_chainlock_read(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		lock_err = EAGAIN;
+		ok1(tdb_lockall(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_lockall(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_lockall(tdb) == TDB_ERR_LOCK);
+		/* This actually does divide and conquer. */
+		ok1(tap_log_messages > 0);
+		tap_log_messages = 0;
+
+		lock_err = EAGAIN;
+		ok1(tdb_lockall_read(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_lockall_read(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_lockall_read(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages > 0);
+		tap_log_messages = 0;
+
+		/* Nonblocking traverse; go nonblock partway through. */
+		lock_err = 0;
+		ok1(tdb_store(tdb, key, data, TDB_REPLACE) == 0);
+		trav_err = EAGAIN;
+		ok1(tdb_traverse(tdb, trav, &lock_err) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		trav_err = EINTR;
+		lock_err = 0;
+		ok1(tdb_traverse(tdb, trav, &lock_err) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		trav_err = ENOMEM;
+		lock_err = 0;
+		ok1(tdb_traverse(tdb, trav, &lock_err) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		/* Nonblocking transactions. */
+		lock_err = EAGAIN;
+		ok1(tdb_transaction_start(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = EINTR;
+		ok1(tdb_transaction_start(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+		lock_err = ENOMEM;
+		ok1(tdb_transaction_start(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		tap_log_messages = 0;
+
+		/* Nonblocking transaction prepare. */
+		lock_err = 0;
+		ok1(tdb_transaction_start(tdb) == 0);
+		ok1(tdb_delete(tdb, key) == 0);
+
+		lock_err = EAGAIN;
+		ok1(tdb_transaction_prepare_commit(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+
+		lock_err = 0;
+		ok1(tdb_transaction_prepare_commit(tdb) == 0);
+		ok1(tdb_transaction_commit(tdb) == 0);
+
+		/* And the transaction was committed, right? */
+		ok1(!tdb_exists(tdb, key));
+		tdb_close(tdb);
+		ok1(tap_log_messages == 0);
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-83-openhook.c b/lib/tdb2/test/run-83-openhook.c
new file mode 100644
index 00000000000..320be7d4da1
--- /dev/null
+++ b/lib/tdb2/test/run-83-openhook.c
@@ -0,0 +1,98 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <err.h>
+#include "external-agent.h"
+#include "logging.h"
+
+static enum TDB_ERROR clear_if_first(int fd, void *arg)
+{
+/* We hold a lock offset 63 always, so we can tell if anyone is holding it. */
+	struct flock fl;
+
+	if (arg != clear_if_first)
+		return TDB_ERR_CORRUPT;
+
+	fl.l_type = F_WRLCK;
+	fl.l_whence = SEEK_SET;
+	fl.l_start = 63;
+	fl.l_len = 1;
+
+	if (fcntl(fd, F_SETLK, &fl) == 0) {
+		/* We must be first ones to open it! */
+		diag("truncating file!");
+		if (ftruncate(fd, 0) != 0) {
+			return TDB_ERR_IO;
+		}
+	}
+	fl.l_type = F_RDLCK;
+	if (fcntl(fd, F_SETLKW, &fl) != 0) {
+		return TDB_ERR_IO;
+	}
+	return TDB_SUCCESS;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	struct agent *agent;
+	union tdb_attribute cif;
+	struct tdb_data key = tdb_mkdata("key", 3);
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+
+	cif.openhook.base.attr = TDB_ATTRIBUTE_OPENHOOK;
+	cif.openhook.base.next = &tap_log_attr;
+	cif.openhook.fn = clear_if_first;
+	cif.openhook.data = clear_if_first;
+
+	agent = prepare_external_agent();
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 13);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		/* Create it */
+		tdb = tdb_open("run-83-openhook.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, NULL);
+		ok1(tdb);
+		ok1(tdb_store(tdb, key, key, TDB_REPLACE) == 0);
+		tdb_close(tdb);
+
+		/* Now, open with CIF, should clear it. */
+		tdb = tdb_open("run-83-openhook.tdb", flags[i],
+			       O_RDWR, 0, &cif);
+		ok1(tdb);
+		ok1(!tdb_exists(tdb, key));
+		ok1(tdb_store(tdb, key, key, TDB_REPLACE) == 0);
+
+		/* Agent should not clear it, since it's still open. */
+		ok1(external_agent_operation(agent, OPEN_WITH_HOOK,
+					     "run-83-openhook.tdb") == SUCCESS);
+		ok1(external_agent_operation(agent, FETCH, "key") == SUCCESS);
+		ok1(external_agent_operation(agent, CLOSE, "") == SUCCESS);
+
+		/* Still exists for us too. */
+		ok1(tdb_exists(tdb, key));
+
+		/* Close it, now agent should clear it. */
+		tdb_close(tdb);
+
+		ok1(external_agent_operation(agent, OPEN_WITH_HOOK,
+					     "run-83-openhook.tdb") == SUCCESS);
+		ok1(external_agent_operation(agent, FETCH, "key") == FAILED);
+		ok1(external_agent_operation(agent, CLOSE, "") == SUCCESS);
+
+		ok1(tap_log_messages == 0);
+	}
+
+	free_external_agent(agent);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-90-get-set-attributes.c b/lib/tdb2/test/run-90-get-set-attributes.c
new file mode 100644
index 00000000000..159d8a01eaf
--- /dev/null
+++ b/lib/tdb2/test/run-90-get-set-attributes.c
@@ -0,0 +1,165 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static int mylock(int fd, int rw, off_t off, off_t len, bool waitflag,
+		  void *unused)
+{
+	return 0;
+}
+
+static int myunlock(int fd, int rw, off_t off, off_t len, void *unused)
+{
+	return 0;
+}
+
+static uint64_t hash_fn(const void *key, size_t len, uint64_t seed,
+			void *priv)
+{
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+	union tdb_attribute seed_attr;
+	union tdb_attribute hash_attr;
+	union tdb_attribute lock_attr;
+
+	hash_attr.base.attr = TDB_ATTRIBUTE_HASH;
+	hash_attr.base.next = &seed_attr;
+	hash_attr.hash.fn = hash_fn;
+	hash_attr.hash.data = &hash_attr;
+
+	seed_attr.base.attr = TDB_ATTRIBUTE_SEED;
+	seed_attr.base.next = &lock_attr;
+	seed_attr.seed.seed = 100;
+
+	lock_attr.base.attr = TDB_ATTRIBUTE_FLOCK;
+	lock_attr.base.next = &tap_log_attr;
+	lock_attr.flock.lock = mylock;
+	lock_attr.flock.unlock = myunlock;
+	lock_attr.flock.data = &lock_attr;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 50);
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		union tdb_attribute attr;
+
+		/* First open with no attributes. */
+		tdb = tdb_open("run-90-get-set-attributes.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, NULL);
+		ok1(tdb);
+
+		/* Get log on no attributes will fail */
+		attr.base.attr = TDB_ATTRIBUTE_LOG;
+		ok1(tdb_get_attribute(tdb, &attr) == TDB_ERR_NOEXIST);
+		/* These always work. */
+		attr.base.attr = TDB_ATTRIBUTE_HASH;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_HASH);
+		ok1(attr.hash.fn == jenkins_hash);
+		attr.base.attr = TDB_ATTRIBUTE_FLOCK;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_FLOCK);
+		ok1(attr.flock.lock == tdb_fcntl_lock);
+		ok1(attr.flock.unlock == tdb_fcntl_unlock);
+		attr.base.attr = TDB_ATTRIBUTE_SEED;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_SEED);
+		/* This is possible, just astronomically unlikely. */
+		ok1(attr.seed.seed != 0);
+
+		/* Unset attributes. */
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_LOG);
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_FLOCK);
+
+		/* Set them. */
+		ok1(tdb_set_attribute(tdb, &tap_log_attr) == 0);
+		ok1(tdb_set_attribute(tdb, &lock_attr) == 0);
+		/* These should fail. */
+		ok1(tdb_set_attribute(tdb, &seed_attr) == TDB_ERR_EINVAL);
+		ok1(tap_log_messages == 1);
+		ok1(tdb_set_attribute(tdb, &hash_attr) == TDB_ERR_EINVAL);
+		ok1(tap_log_messages == 2);
+		tap_log_messages = 0;
+
+		/* Getting them should work as expected. */
+		attr.base.attr = TDB_ATTRIBUTE_LOG;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_LOG);
+		ok1(attr.log.fn == tap_log_attr.log.fn);
+		ok1(attr.log.data == tap_log_attr.log.data);
+
+		attr.base.attr = TDB_ATTRIBUTE_FLOCK;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_FLOCK);
+		ok1(attr.flock.lock == mylock);
+		ok1(attr.flock.unlock == myunlock);
+		ok1(attr.flock.data == &lock_attr);
+
+		/* Unset them again. */
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_FLOCK);
+		ok1(tap_log_messages == 0);
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_LOG);
+		ok1(tap_log_messages == 0);
+
+		tdb_close(tdb);
+		ok1(tap_log_messages == 0);
+
+		/* Now open with all attributes. */
+		tdb = tdb_open("run-90-get-set-attributes.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &hash_attr);
+		ok1(tdb);
+
+		/* Get will succeed */
+		attr.base.attr = TDB_ATTRIBUTE_LOG;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_LOG);
+		ok1(attr.log.fn == tap_log_attr.log.fn);
+		ok1(attr.log.data == tap_log_attr.log.data);
+
+		attr.base.attr = TDB_ATTRIBUTE_HASH;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_HASH);
+		ok1(attr.hash.fn == hash_fn);
+		ok1(attr.hash.data == &hash_attr);
+
+		attr.base.attr = TDB_ATTRIBUTE_FLOCK;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_FLOCK);
+		ok1(attr.flock.lock == mylock);
+		ok1(attr.flock.unlock == myunlock);
+		ok1(attr.flock.data == &lock_attr);
+
+		attr.base.attr = TDB_ATTRIBUTE_SEED;
+		ok1(tdb_get_attribute(tdb, &attr) == 0);
+		ok1(attr.base.attr == TDB_ATTRIBUTE_SEED);
+		ok1(attr.seed.seed == seed_attr.seed.seed);
+
+		/* Unset attributes. */
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_HASH);
+		ok1(tap_log_messages == 1);
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_SEED);
+		ok1(tap_log_messages == 2);
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_FLOCK);
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_LOG);
+		ok1(tap_log_messages == 2);
+		tap_log_messages = 0;
+
+		tdb_close(tdb);
+
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-91-get-stats.c b/lib/tdb2/test/run-91-get-stats.c
new file mode 100644
index 00000000000..795dfd6602c
--- /dev/null
+++ b/lib/tdb2/test/run-91-get-stats.c
@@ -0,0 +1,59 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 11);
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		union tdb_attribute *attr;
+		struct tdb_data key = tdb_mkdata("key", 3);
+
+		tdb = tdb_open("run-91-get-stats.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		ok1(tdb_store(tdb, key, key, TDB_REPLACE) == 0);
+
+		/* Use malloc so valgrind will catch overruns. */
+		attr = malloc(sizeof *attr);
+		attr->stats.base.attr = TDB_ATTRIBUTE_STATS;
+		attr->stats.size = sizeof(*attr);
+
+		ok1(tdb_get_attribute(tdb, attr) == 0);
+		ok1(attr->stats.size == sizeof(*attr));
+		ok1(attr->stats.allocs > 0);
+		ok1(attr->stats.expands > 0);
+		ok1(attr->stats.locks > 0);
+		free(attr);
+
+		/* Try short one. */
+		attr = malloc(offsetof(struct tdb_attribute_stats, allocs)
+			      + sizeof(attr->stats.allocs));
+		attr->stats.base.attr = TDB_ATTRIBUTE_STATS;
+		attr->stats.size = offsetof(struct tdb_attribute_stats, allocs)
+			+ sizeof(attr->stats.allocs);
+		ok1(tdb_get_attribute(tdb, attr) == 0);
+		ok1(attr->stats.size == sizeof(*attr));
+		ok1(attr->stats.allocs > 0);
+		free(attr);
+		ok1(tap_log_messages == 0);
+
+		tdb_close(tdb);
+
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-add-remove-flags.c b/lib/tdb2/test/run-add-remove-flags.c
new file mode 100644
index 00000000000..1dc84636628
--- /dev/null
+++ b/lib/tdb2/test/run-add-remove-flags.c
@@ -0,0 +1,93 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(87);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-add-remove-flags.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		ok1(tdb_get_flags(tdb) == tdb->flags);
+		tap_log_messages = 0;
+		tdb_add_flag(tdb, TDB_NOLOCK);
+		if (flags[i] & TDB_INTERNAL)
+			ok1(tap_log_messages == 1);
+		else {
+			ok1(tap_log_messages == 0);
+			ok1(tdb_get_flags(tdb) & TDB_NOLOCK);
+		}
+
+		tap_log_messages = 0;
+		tdb_add_flag(tdb, TDB_NOMMAP);
+		if (flags[i] & TDB_INTERNAL)
+			ok1(tap_log_messages == 1);
+		else {
+			ok1(tap_log_messages == 0);
+			ok1(tdb_get_flags(tdb) & TDB_NOMMAP);
+			ok1(tdb->file->map_ptr == NULL);
+		}
+
+		tap_log_messages = 0;
+		tdb_add_flag(tdb, TDB_NOSYNC);
+		if (flags[i] & TDB_INTERNAL)
+			ok1(tap_log_messages == 1);
+		else {
+			ok1(tap_log_messages == 0);
+			ok1(tdb_get_flags(tdb) & TDB_NOSYNC);
+		}
+
+		ok1(tdb_get_flags(tdb) == tdb->flags);
+
+		tap_log_messages = 0;
+		tdb_remove_flag(tdb, TDB_NOLOCK);
+		if (flags[i] & TDB_INTERNAL)
+			ok1(tap_log_messages == 1);
+		else {
+			ok1(tap_log_messages == 0);
+			ok1(!(tdb_get_flags(tdb) & TDB_NOLOCK));
+		}
+
+		tap_log_messages = 0;
+		tdb_remove_flag(tdb, TDB_NOMMAP);
+		if (flags[i] & TDB_INTERNAL)
+			ok1(tap_log_messages == 1);
+		else {
+			ok1(tap_log_messages == 0);
+			ok1(!(tdb_get_flags(tdb) & TDB_NOMMAP));
+			ok1(tdb->file->map_ptr != NULL);
+		}
+
+		tap_log_messages = 0;
+		tdb_remove_flag(tdb, TDB_NOSYNC);
+		if (flags[i] & TDB_INTERNAL)
+			ok1(tap_log_messages == 1);
+		else {
+			ok1(tap_log_messages == 0);
+			ok1(!(tdb_get_flags(tdb) & TDB_NOSYNC));
+		}
+
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-check-callback.c b/lib/tdb2/test/run-check-callback.c
new file mode 100644
index 00000000000..1e87436717f
--- /dev/null
+++ b/lib/tdb2/test/run-check-callback.c
@@ -0,0 +1,90 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+#define NUM_RECORDS 1000
+
+static bool store_records(struct tdb_context *tdb)
+{
+	int i;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+
+	for (i = 0; i < NUM_RECORDS; i++)
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+			return false;
+	return true;
+}
+
+static enum TDB_ERROR check(struct tdb_data key,
+			    struct tdb_data data,
+			    bool *array)
+{
+	int val;
+
+	if (key.dsize != sizeof(val)) {
+		diag("Wrong key size: %u\n", key.dsize);
+		return TDB_ERR_CORRUPT;
+	}
+
+	if (key.dsize != data.dsize
+	    || memcmp(key.dptr, data.dptr, sizeof(val)) != 0) {
+		diag("Key and data differ\n");
+		return TDB_ERR_CORRUPT;
+	}
+
+	memcpy(&val, key.dptr, sizeof(val));
+	if (val >= NUM_RECORDS || val < 0) {
+		diag("check value %i\n", val);
+		return TDB_ERR_CORRUPT;
+	}
+
+	if (array[val]) {
+		diag("Value %i already seen\n", val);
+		return TDB_ERR_CORRUPT;
+	}
+
+	array[val] = true;
+	return TDB_SUCCESS;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 4 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		bool array[NUM_RECORDS];
+
+		tdb = tdb_open("run-check-callback.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		ok1(store_records(tdb));
+		for (j = 0; j < NUM_RECORDS; j++)
+			array[j] = false;
+		ok1(tdb_check(tdb, check, array) == TDB_SUCCESS);
+		for (j = 0; j < NUM_RECORDS; j++)
+			if (!array[j])
+				break;
+		ok1(j == NUM_RECORDS);
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-expand-in-transaction.c b/lib/tdb2/test/run-expand-in-transaction.c
new file mode 100644
index 00000000000..49ba03c924a
--- /dev/null
+++ b/lib/tdb2/test/run-expand-in-transaction.c
@@ -0,0 +1,45 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT,
+			TDB_CONVERT|TDB_NOSYNC,
+			TDB_NOMMAP|TDB_CONVERT|TDB_NOSYNC };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4);
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 7 + 1);
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		size_t size;
+		tdb = tdb_open("run-expand-in-transaction.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		size = tdb->file->map_size;
+		ok1(tdb_transaction_start(tdb) == 0);
+		ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+		ok1(tdb->file->map_size > size);
+		ok1(tdb_transaction_commit(tdb) == 0);
+		ok1(tdb->file->map_size > size);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-features.c b/lib/tdb2/test/run-features.c
new file mode 100644
index 00000000000..6d82dc308c9
--- /dev/null
+++ b/lib/tdb2/test/run-features.c
@@ -0,0 +1,70 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/summary.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = { (unsigned char *)&j, sizeof(j) };
+	struct tdb_data data = { (unsigned char *)&j, sizeof(j) };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 8 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		uint64_t features;
+		tdb = tdb_open("run-features.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* Put some stuff in there. */
+		for (j = 0; j < 100; j++) {
+			if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+				fail("Storing in tdb");
+		}
+
+		/* Mess with features fields in hdr. */
+		features = (~TDB_FEATURE_MASK ^ 1);
+		ok1(tdb_write_convert(tdb, offsetof(struct tdb_header,
+						    features_used),
+				      &features, sizeof(features)) == 0);
+		ok1(tdb_write_convert(tdb, offsetof(struct tdb_header,
+						    features_offered),
+				      &features, sizeof(features)) == 0);
+		tdb_close(tdb);
+
+		tdb = tdb_open("run-features.tdb", flags[i], O_RDWR, 0,
+			       &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* Should not have changed features offered. */
+		ok1(tdb_read_convert(tdb, offsetof(struct tdb_header,
+						   features_offered),
+				     &features, sizeof(features)) == 0);
+		ok1(features == (~TDB_FEATURE_MASK ^ 1));
+
+		/* Should have cleared unknown bits in features_used. */
+		ok1(tdb_read_convert(tdb, offsetof(struct tdb_header,
+						   features_used),
+				     &features, sizeof(features)) == 0);
+		ok1(features == (1 & TDB_FEATURE_MASK));
+
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-firstkey-nextkey.c b/lib/tdb2/test/run-firstkey-nextkey.c
new file mode 100644
index 00000000000..65a6090a96b
--- /dev/null
+++ b/lib/tdb2/test/run-firstkey-nextkey.c
@@ -0,0 +1,162 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+#define NUM_RECORDS 1000
+
+static bool store_records(struct tdb_context *tdb)
+{
+	int i;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+
+	for (i = 0; i < NUM_RECORDS; i++)
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+			return false;
+	return true;
+}
+
+struct trav_data {
+	unsigned int records[NUM_RECORDS];
+	unsigned int calls;
+};
+
+static int trav(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, void *p)
+{
+	struct trav_data *td = p;
+	int val;
+
+	memcpy(&val, dbuf.dptr, dbuf.dsize);
+	td->records[td->calls++] = val;
+	return 0;
+}
+
+/* Since tdb_nextkey frees dptr, we need to clone it. */
+static TDB_DATA dup_key(TDB_DATA key)
+{
+	void *p = malloc(key.dsize);
+	memcpy(p, key.dptr, key.dsize);
+	key.dptr = p;
+	return key;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	int num;
+	struct trav_data td;
+	TDB_DATA k;
+	struct tdb_context *tdb;
+	union tdb_attribute seed_attr;
+	enum TDB_ERROR ecode;
+
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	seed_attr.base.attr = TDB_ATTRIBUTE_SEED;
+	seed_attr.base.next = &tap_log_attr;
+	seed_attr.seed.seed = 6334326220117065685ULL;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0])
+		   * (NUM_RECORDS*6 + (NUM_RECORDS-1)*3 + 22) + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-traverse.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &seed_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		ok1(tdb_firstkey(tdb, &k) == TDB_ERR_NOEXIST);
+
+		/* One entry... */
+		k.dptr = (unsigned char *)&num;
+		k.dsize = sizeof(num);
+		num = 0;
+		ok1(tdb_store(tdb, k, k, TDB_INSERT) == 0);
+		ok1(tdb_firstkey(tdb, &k) == TDB_SUCCESS);
+		ok1(k.dsize == sizeof(num));
+		ok1(memcmp(k.dptr, &num, sizeof(num)) == 0);
+		ok1(tdb_nextkey(tdb, &k) == TDB_ERR_NOEXIST);
+
+		/* Two entries. */
+		k.dptr = (unsigned char *)&num;
+		k.dsize = sizeof(num);
+		num = 1;
+		ok1(tdb_store(tdb, k, k, TDB_INSERT) == 0);
+		ok1(tdb_firstkey(tdb, &k) == TDB_SUCCESS);
+		ok1(k.dsize == sizeof(num));
+		memcpy(&num, k.dptr, sizeof(num));
+		ok1(num == 0 || num == 1);
+		ok1(tdb_nextkey(tdb, &k) == TDB_SUCCESS);
+		ok1(k.dsize == sizeof(j));
+		memcpy(&j, k.dptr, sizeof(j));
+		ok1(j == 0 || j == 1);
+		ok1(j != num);
+		ok1(tdb_nextkey(tdb, &k) == TDB_ERR_NOEXIST);
+
+		/* Clean up. */
+		k.dptr = (unsigned char *)&num;
+		k.dsize = sizeof(num);
+		num = 0;
+		ok1(tdb_delete(tdb, k) == 0);
+		num = 1;
+		ok1(tdb_delete(tdb, k) == 0);
+
+		/* Now lots of records. */
+		ok1(store_records(tdb));
+		td.calls = 0;
+
+		num = tdb_traverse(tdb, trav, &td);
+		ok1(num == NUM_RECORDS);
+		ok1(td.calls == NUM_RECORDS);
+
+		/* Simple loop should match tdb_traverse */
+		for (j = 0, ecode = tdb_firstkey(tdb, &k); j < td.calls; j++) {
+			int val;
+
+			ok1(ecode == TDB_SUCCESS);
+			ok1(k.dsize == sizeof(val));
+			memcpy(&val, k.dptr, k.dsize);
+			ok1(td.records[j] == val);
+			ecode = tdb_nextkey(tdb, &k);
+		}
+
+		/* But arbitrary orderings should work too. */
+		for (j = td.calls-1; j > 0; j--) {
+			k.dptr = (unsigned char *)&td.records[j-1];
+			k.dsize = sizeof(td.records[j-1]);
+			k = dup_key(k);
+			ok1(tdb_nextkey(tdb, &k) == TDB_SUCCESS);
+			ok1(k.dsize == sizeof(td.records[j]));
+			ok1(memcmp(k.dptr, &td.records[j], k.dsize) == 0);
+			free(k.dptr);
+		}
+
+		/* Even delete should work. */
+		for (j = 0, ecode = tdb_firstkey(tdb, &k);
+		     ecode != TDB_ERR_NOEXIST;
+		     j++) {
+			ok1(ecode == TDB_SUCCESS);
+			ok1(k.dsize == 4);
+			ok1(tdb_delete(tdb, k) == 0);
+			ecode = tdb_nextkey(tdb, &k);
+		}
+
+		diag("delete using first/nextkey gave %u of %u records",
+		     j, NUM_RECORDS);
+		ok1(j == NUM_RECORDS);
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-fork-test.c b/lib/tdb2/test/run-fork-test.c
new file mode 100644
index 00000000000..e9813e0a0f0
--- /dev/null
+++ b/lib/tdb2/test/run-fork-test.c
@@ -0,0 +1,180 @@
+/* Test forking while holding lock.
+ *
+ * There are only five ways to do this currently:
+ * (1) grab a tdb_chainlock, then fork.
+ * (2) grab a tdb_lockall, then fork.
+ * (3) grab a tdb_lockall_read, then fork.
+ * (4) start a transaction, then fork.
+ * (5) fork from inside a tdb_parse() callback.
+ *
+ * Note that we don't hold a lock across tdb_traverse callbacks, so
+ * that doesn't matter.
+ */
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include "logging.h"
+
+static enum TDB_ERROR fork_in_parse(TDB_DATA key, TDB_DATA data,
+				    struct tdb_context *tdb)
+{
+	int status;
+
+	if (fork() == 0) {
+		/* We expect this to fail. */
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != TDB_ERR_LOCK)
+			exit(1);
+
+		if (tdb_fetch(tdb, key, &data) != TDB_ERR_LOCK)
+			exit(1);
+
+		if (tap_log_messages != 2)
+			exit(2);
+
+		tdb_close(tdb);
+		if (tap_log_messages != 2)
+			exit(3);
+		exit(0);
+	}
+	wait(&status);
+	ok1(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+	return TDB_SUCCESS;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4);
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 14);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		int status;
+
+		tap_log_messages = 0;
+
+		tdb = tdb_open("run-fork-test.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			continue;
+
+		/* Put a record in here. */
+		ok1(tdb_store(tdb, key, data, TDB_REPLACE) == TDB_SUCCESS);
+
+		ok1(tdb_chainlock(tdb, key) == TDB_SUCCESS);
+		if (fork() == 0) {
+			/* We expect this to fail. */
+			if (tdb_store(tdb, key, data, TDB_REPLACE) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tdb_fetch(tdb, key, &data) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tap_log_messages != 2)
+				return 2;
+
+			tdb_chainunlock(tdb, key);
+			if (tap_log_messages != 3)
+				return 3;
+			tdb_close(tdb);
+			if (tap_log_messages != 3)
+				return 4;
+			return 0;
+		}
+		wait(&status);
+		ok1(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+		tdb_chainunlock(tdb, key);
+
+		ok1(tdb_lockall(tdb) == TDB_SUCCESS);
+		if (fork() == 0) {
+			/* We expect this to fail. */
+			if (tdb_store(tdb, key, data, TDB_REPLACE) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tdb_fetch(tdb, key, &data) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tap_log_messages != 2)
+				return 2;
+
+			tdb_unlockall(tdb);
+			if (tap_log_messages != 2)
+				return 3;
+			tdb_close(tdb);
+			if (tap_log_messages != 2)
+				return 4;
+			return 0;
+		}
+		wait(&status);
+		ok1(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+		tdb_unlockall(tdb);
+
+		ok1(tdb_lockall_read(tdb) == TDB_SUCCESS);
+		if (fork() == 0) {
+			/* We expect this to fail. */
+			/* This would always fail anyway... */
+			if (tdb_store(tdb, key, data, TDB_REPLACE) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tdb_fetch(tdb, key, &data) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tap_log_messages != 2)
+				return 2;
+
+			tdb_unlockall_read(tdb);
+			if (tap_log_messages != 2)
+				return 3;
+			tdb_close(tdb);
+			if (tap_log_messages != 2)
+				return 4;
+			return 0;
+		}
+		wait(&status);
+		ok1(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+		tdb_unlockall_read(tdb);
+
+		ok1(tdb_transaction_start(tdb) == TDB_SUCCESS);
+		/* If transactions is empty, noop "commit" succeeds. */
+		ok1(tdb_delete(tdb, key) == TDB_SUCCESS);
+		if (fork() == 0) {
+			/* We expect this to fail. */
+			if (tdb_store(tdb, key, data, TDB_REPLACE) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tdb_fetch(tdb, key, &data) != TDB_ERR_LOCK)
+				return 1;
+
+			if (tap_log_messages != 2)
+				return 2;
+
+			if (tdb_transaction_commit(tdb) != TDB_ERR_LOCK)
+				return 3;
+
+			tdb_close(tdb);
+			if (tap_log_messages < 3)
+				return 4;
+			return 0;
+		}
+		wait(&status);
+		ok1(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+		tdb_transaction_cancel(tdb);
+
+		ok1(tdb_parse_record(tdb, key, fork_in_parse, tdb)
+		    == TDB_SUCCESS);
+		tdb_close(tdb);
+		ok1(tap_log_messages == 0);
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-lockall.c b/lib/tdb2/test/run-lockall.c
new file mode 100644
index 00000000000..4aedf59743d
--- /dev/null
+++ b/lib/tdb2/test/run-lockall.c
@@ -0,0 +1,80 @@
+#include "config.h"
+#include <unistd.h>
+#include "lock-tracking.h"
+
+#define fcntl fcntl_with_lockcheck
+
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <err.h>
+#include "external-agent.h"
+#include "logging.h"
+
+#define TEST_DBNAME "run-lockall.tdb"
+
+#undef fcntl
+
+int main(int argc, char *argv[])
+{
+	struct agent *agent;
+	const int flags[] = { TDB_DEFAULT,
+			      TDB_NOMMAP,
+			      TDB_CONVERT,
+			      TDB_CONVERT | TDB_NOMMAP };
+	int i;
+
+	plan_tests(13 * sizeof(flags)/sizeof(flags[0]) + 1);
+	agent = prepare_external_agent();
+	if (!agent)
+		err(1, "preparing agent");
+
+	for (i = 0; i < sizeof(flags)/sizeof(flags[0]); i++) {
+		enum agent_return ret;
+		struct tdb_context *tdb;
+
+		tdb = tdb_open(TEST_DBNAME, flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+
+		ret = external_agent_operation(agent, OPEN, TEST_DBNAME);
+		ok1(ret == SUCCESS);
+
+		ok1(tdb_lockall(tdb) == TDB_SUCCESS);
+		ok1(external_agent_operation(agent, STORE, "key")
+		    == WOULD_HAVE_BLOCKED);
+		ok1(external_agent_operation(agent, FETCH, "key")
+		    == WOULD_HAVE_BLOCKED);
+		/* Test nesting. */
+		ok1(tdb_lockall(tdb) == TDB_SUCCESS);
+		tdb_unlockall(tdb);
+		tdb_unlockall(tdb);
+
+		ok1(external_agent_operation(agent, STORE, "key") == SUCCESS);
+
+		ok1(tdb_lockall_read(tdb) == TDB_SUCCESS);
+		ok1(external_agent_operation(agent, STORE, "key")
+		    == WOULD_HAVE_BLOCKED);
+		ok1(external_agent_operation(agent, FETCH, "key") == SUCCESS);
+		ok1(tdb_lockall_read(tdb) == TDB_SUCCESS);
+		tdb_unlockall_read(tdb);
+		tdb_unlockall_read(tdb);
+
+		ok1(external_agent_operation(agent, STORE, "key") == SUCCESS);
+		ok1(external_agent_operation(agent, CLOSE, NULL) == SUCCESS);
+		tdb_close(tdb);
+	}
+
+	free_external_agent(agent);
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-locktimeout.c b/lib/tdb2/test/run-locktimeout.c
new file mode 100644
index 00000000000..bb5b5db29b4
--- /dev/null
+++ b/lib/tdb2/test/run-locktimeout.c
@@ -0,0 +1,192 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+#include "external-agent.h"
+
+#undef alarm
+#define alarm fast_alarm
+
+/* Speed things up by doing things in milliseconds. */
+static unsigned int fast_alarm(unsigned int milli_seconds)
+{
+	struct itimerval it;
+
+	it.it_interval.tv_sec = it.it_interval.tv_usec = 0;
+	it.it_value.tv_sec = milli_seconds / 1000;
+	it.it_value.tv_usec = milli_seconds * 1000;
+	setitimer(ITIMER_REAL, &it, NULL);
+	return 0;
+}
+
+#define CatchSignal(sig, handler) signal((sig), (handler))
+
+static void do_nothing(int signum)
+{
+}
+
+/* This example code is taken from SAMBA, so try not to change it. */
+static struct flock flock_struct;
+
+/* Return a value which is none of v1, v2 or v3. */
+static inline short int invalid_value(short int v1, short int v2, short int v3)
+{
+	short int try = (v1+v2+v3)^((v1+v2+v3) << 16);
+	while (try == v1 || try == v2 || try == v3)
+		try++;
+	return try;
+}
+
+/* We invalidate in as many ways as we can, so the OS rejects it */
+static void invalidate_flock_struct(int signum)
+{
+	flock_struct.l_type = invalid_value(F_RDLCK, F_WRLCK, F_UNLCK);
+	flock_struct.l_whence = invalid_value(SEEK_SET, SEEK_CUR, SEEK_END);
+	flock_struct.l_start = -1;
+	/* A large negative. */
+	flock_struct.l_len = (((off_t)1 << (sizeof(off_t)*CHAR_BIT - 1)) + 1);
+}
+
+static int timeout_lock(int fd, int rw, off_t off, off_t len, bool waitflag,
+			void *_timeout)
+{
+	int ret, saved_errno = errno;
+	unsigned int timeout = *(unsigned int *)_timeout;
+
+	flock_struct.l_type = rw;
+	flock_struct.l_whence = SEEK_SET;
+	flock_struct.l_start = off;
+	flock_struct.l_len = len;
+
+	CatchSignal(SIGALRM, invalidate_flock_struct);
+	alarm(timeout);
+
+	for (;;) {
+		if (waitflag)
+			ret = fcntl(fd, F_SETLKW, &flock_struct);
+		else
+			ret = fcntl(fd, F_SETLK, &flock_struct);
+
+		if (ret == 0)
+			break;
+
+		/* Not signalled?  Something else went wrong. */
+		if (flock_struct.l_len == len) {
+			if (errno == EAGAIN || errno == EINTR)
+				continue;
+			saved_errno = errno;
+			break;
+		} else {
+			saved_errno = EINTR;
+			break;
+		}
+	}
+
+	alarm(0);
+	errno = saved_errno;
+	return ret;
+}
+
+static int tdb_chainlock_with_timeout_internal(struct tdb_context *tdb,
+					       TDB_DATA key,
+					       unsigned int timeout,
+					       int rw_type)
+{
+	union tdb_attribute locking;
+	enum TDB_ERROR ecode;
+
+	if (timeout) {
+		locking.base.attr = TDB_ATTRIBUTE_FLOCK;
+		ecode = tdb_get_attribute(tdb, &locking);
+		if (ecode != TDB_SUCCESS)
+			return ecode;
+
+		/* Replace locking function with our own. */
+		locking.flock.data = &timeout;
+		locking.flock.lock = timeout_lock;
+
+		ecode = tdb_set_attribute(tdb, &locking);
+		if (ecode != TDB_SUCCESS)
+			return ecode;
+	}
+	if (rw_type == F_RDLCK)
+		ecode = tdb_chainlock_read(tdb, key);
+	else
+		ecode = tdb_chainlock(tdb, key);
+
+	if (timeout) {
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_FLOCK);
+	}
+	return ecode;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	TDB_DATA key = tdb_mkdata("hello", 5);
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+	struct agent *agent;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 15);
+
+	agent = prepare_external_agent();
+
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		enum TDB_ERROR ecode;
+		tdb = tdb_open("run-locktimeout.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		if (!ok1(tdb))
+			break;
+
+		/* Simple cases: should succeed. */
+		ecode = tdb_chainlock_with_timeout_internal(tdb, key, 20,
+							    F_RDLCK);
+		ok1(ecode == TDB_SUCCESS);
+		ok1(tap_log_messages == 0);
+
+		tdb_chainunlock_read(tdb, key);
+		ok1(tap_log_messages == 0);
+
+		ecode = tdb_chainlock_with_timeout_internal(tdb, key, 20,
+							    F_WRLCK);
+		ok1(ecode == TDB_SUCCESS);
+		ok1(tap_log_messages == 0);
+
+		tdb_chainunlock(tdb, key);
+		ok1(tap_log_messages == 0);
+
+		/* OK, get agent to start transaction, then we should time out. */
+		ok1(external_agent_operation(agent, OPEN, "run-locktimeout.tdb")
+		    == SUCCESS);
+		ok1(external_agent_operation(agent, TRANSACTION_START, "")
+		    == SUCCESS);
+		ecode = tdb_chainlock_with_timeout_internal(tdb, key, 20,
+							    F_WRLCK);
+		ok1(ecode == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+
+		/* Even if we get a different signal, should be fine. */
+		CatchSignal(SIGUSR1, do_nothing);
+		external_agent_operation(agent, SEND_SIGNAL, "");
+		ecode = tdb_chainlock_with_timeout_internal(tdb, key, 20,
+							    F_WRLCK);
+		ok1(ecode == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 0);
+
+		ok1(external_agent_operation(agent, TRANSACTION_COMMIT, "")
+		    == SUCCESS);
+		ok1(external_agent_operation(agent, CLOSE, "")
+		    == SUCCESS);
+		tdb_close(tdb);
+	}
+	free_external_agent(agent);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-missing-entries.c b/lib/tdb2/test/run-missing-entries.c
new file mode 100644
index 00000000000..e99572f64c1
--- /dev/null
+++ b/lib/tdb2/test/run-missing-entries.c
@@ -0,0 +1,48 @@
+/* Another test revealed that we lost an entry.  This reproduces it. */
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+#define NUM_RECORDS 1189
+
+/* We use the same seed which we saw this failure on. */
+static uint64_t failhash(const void *key, size_t len, uint64_t seed, void *p)
+{
+	seed = 699537674708983027ULL;
+	return hash64_stable((const unsigned char *)key, len, seed);
+}
+
+int main(int argc, char *argv[])
+{
+	int i;
+	struct tdb_context *tdb;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+	union tdb_attribute hattr = { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+						.fn = failhash } };
+
+	hattr.base.next = &tap_log_attr;
+	plan_tests(1 + 2 * NUM_RECORDS + 1);
+
+	tdb = tdb_open("run-missing-entries.tdb", TDB_INTERNAL,
+		       O_RDWR|O_CREAT|O_TRUNC, 0600, &hattr);
+	ok1(tdb);
+	if (tdb) {
+		for (i = 0; i < NUM_RECORDS; i++) {
+			ok1(tdb_store(tdb, key, data, TDB_REPLACE) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+		}
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-open-multiple-times.c b/lib/tdb2/test/run-open-multiple-times.c
new file mode 100644
index 00000000000..240828df16b
--- /dev/null
+++ b/lib/tdb2/test/run-open-multiple-times.c
@@ -0,0 +1,84 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb, *tdb2;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data d = { NULL, 0 }; /* Bogus GCC warning */
+	int flags[] = { TDB_DEFAULT, TDB_NOMMAP,
+			TDB_CONVERT, TDB_NOMMAP|TDB_CONVERT };
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 28);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-open-multiple-times.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+		tdb2 = tdb_open("run-open-multiple-times.tdb", flags[i],
+				O_RDWR|O_CREAT, 0600, &tap_log_attr);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tdb_check(tdb2, NULL, NULL) == 0);
+
+		/* Store in one, fetch in the other. */
+		ok1(tdb_store(tdb, key, data, TDB_REPLACE) == 0);
+		ok1(tdb_fetch(tdb2, key, &d) == TDB_SUCCESS);
+		ok1(tdb_deq(d, data));
+		free(d.dptr);
+
+		/* Vice versa, with delete. */
+		ok1(tdb_delete(tdb2, key) == 0);
+		ok1(tdb_fetch(tdb, key, &d) == TDB_ERR_NOEXIST);
+
+		/* OK, now close first one, check second still good. */
+		ok1(tdb_close(tdb) == 0);
+
+		ok1(tdb_store(tdb2, key, data, TDB_REPLACE) == 0);
+		ok1(tdb_fetch(tdb2, key, &d) == TDB_SUCCESS);
+		ok1(tdb_deq(d, data));
+		free(d.dptr);
+
+		/* Reopen */
+		tdb = tdb_open("run-open-multiple-times.tdb", flags[i],
+			       O_RDWR|O_CREAT, 0600, &tap_log_attr);
+		ok1(tdb);
+
+		ok1(tdb_transaction_start(tdb2) == 0);
+
+		/* Anything in the other one should fail. */
+		ok1(tdb_fetch(tdb, key, &d) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 1);
+		ok1(tdb_store(tdb, key, data, TDB_REPLACE) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 2);
+		ok1(tdb_transaction_start(tdb) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 3);
+		ok1(tdb_chainlock(tdb, key) == TDB_ERR_LOCK);
+		ok1(tap_log_messages == 4);
+
+		/* Transaciton should work as normal. */
+		ok1(tdb_store(tdb2, key, data, TDB_REPLACE) == TDB_SUCCESS);
+
+		/* Now... try closing with locks held. */
+		ok1(tdb_close(tdb2) == 0);
+
+		ok1(tdb_fetch(tdb, key, &d) == TDB_SUCCESS);
+		ok1(tdb_deq(d, data));
+		free(d.dptr);
+		ok1(tdb_close(tdb) == 0);
+		ok1(tap_log_messages == 4);
+		tap_log_messages = 0;
+	}
+
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-record-expand.c b/lib/tdb2/test/run-record-expand.c
new file mode 100644
index 00000000000..109a099278b
--- /dev/null
+++ b/lib/tdb2/test/run-record-expand.c
@@ -0,0 +1,53 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+#define MAX_SIZE 10000
+#define SIZE_STEP 131
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data;
+
+	data.dptr = malloc(MAX_SIZE);
+	memset(data.dptr, 0x24, MAX_SIZE);
+
+	plan_tests(sizeof(flags) / sizeof(flags[0])
+		   * (3 + (1 + (MAX_SIZE/SIZE_STEP)) * 2) + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-record-expand.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		data.dsize = 0;
+		ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		for (data.dsize = 0;
+		     data.dsize < MAX_SIZE;
+		     data.dsize += SIZE_STEP) {
+			memset(data.dptr, data.dsize, data.dsize);
+			ok1(tdb_store(tdb, key, data, TDB_MODIFY) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+		}
+		tdb_close(tdb);
+	}
+	ok1(tap_log_messages == 0);
+	free(data.dptr);
+
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-remap-in-read_traverse.c b/lib/tdb2/test/run-remap-in-read_traverse.c
new file mode 100644
index 00000000000..d784ca34074
--- /dev/null
+++ b/lib/tdb2/test/run-remap-in-read_traverse.c
@@ -0,0 +1,65 @@
+/* We had a bug where we marked the tdb read-only for a tdb_traverse_read.
+ * If we then expanded the tdb, we would remap read-only, and later SEGV. */
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "external-agent.h"
+#include "logging.h"
+
+static bool file_larger(int fd, tdb_len_t size)
+{
+	struct stat st;
+
+	fstat(fd, &st);
+	return st.st_size != size;
+}
+
+static unsigned add_records_to_grow(struct agent *agent, int fd, tdb_len_t size)
+{
+	unsigned int i;
+
+	for (i = 0; !file_larger(fd, size); i++) {
+		char data[20];
+		sprintf(data, "%i", i);
+		if (external_agent_operation(agent, STORE, data) != SUCCESS)
+			return 0;
+	}
+	diag("Added %u records to grow file", i);
+	return i;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct agent *agent;
+	struct tdb_context *tdb;
+	struct tdb_data d = tdb_mkdata("hello", 5);
+	const char filename[] = "run-remap-in-read_traverse.tdb";
+
+	plan_tests(4);
+
+	agent = prepare_external_agent();
+
+	tdb = tdb_open(filename, TDB_DEFAULT,
+		       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+
+	ok1(external_agent_operation(agent, OPEN, filename) == SUCCESS);
+	i = add_records_to_grow(agent, tdb->file->fd, tdb->file->map_size);
+
+	/* Do a traverse. */
+	ok1(tdb_traverse(tdb, NULL, NULL) == i);
+
+	/* Now store something! */
+	ok1(tdb_store(tdb, d, d, TDB_INSERT) == 0);
+	ok1(tap_log_messages == 0);
+	tdb_close(tdb);
+	free_external_agent(agent);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-seed.c b/lib/tdb2/test/run-seed.c
new file mode 100644
index 00000000000..a9b370b6e55
--- /dev/null
+++ b/lib/tdb2/test/run-seed.c
@@ -0,0 +1,67 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+static int log_count = 0;
+
+/* Normally we get a log when setting random seed. */
+static void my_log_fn(struct tdb_context *tdb,
+		      enum tdb_log_level level,
+		      const char *message, void *priv)
+{
+	log_count++;
+}
+
+static union tdb_attribute log_attr = {
+	.log = { .base = { .attr = TDB_ATTRIBUTE_LOG },
+		 .fn = my_log_fn }
+};
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	union tdb_attribute attr;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+
+	attr.seed.base.attr = TDB_ATTRIBUTE_SEED;
+	attr.seed.base.next = &log_attr;
+	attr.seed.seed = 42;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 4 + 4 * 3);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		struct tdb_header hdr;
+		int fd;
+		tdb = tdb_open("run-seed.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tdb->hash_seed == 42);
+		ok1(log_count == 0);
+		tdb_close(tdb);
+
+		if (flags[i] & TDB_INTERNAL)
+			continue;
+
+		fd = open("run-seed.tdb", O_RDONLY);
+		ok1(fd >= 0);
+		ok1(read(fd, &hdr, sizeof(hdr)) == sizeof(hdr));
+		if (flags[i] & TDB_CONVERT)
+			ok1(bswap_64(hdr.hash_seed) == 42);
+		else
+			ok1(hdr.hash_seed == 42);
+		close(fd);
+	}
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-simple-delete.c b/lib/tdb2/test/run-simple-delete.c
new file mode 100644
index 00000000000..d06bf2d2bd1
--- /dev/null
+++ b/lib/tdb2/test/run-simple-delete.c
@@ -0,0 +1,42 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = tdb_mkdata("key", 3);
+	struct tdb_data data = tdb_mkdata("data", 4);
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 7 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-simple-delete.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (tdb) {
+			/* Delete should fail. */
+			ok1(tdb_delete(tdb, key) == TDB_ERR_NOEXIST);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			/* Insert should succeed. */
+			ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			/* Delete should now work. */
+			ok1(tdb_delete(tdb, key) == 0);
+			ok1(tdb_check(tdb, NULL, NULL) == 0);
+			tdb_close(tdb);
+		}
+	}
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-summary.c b/lib/tdb2/test/run-summary.c
new file mode 100644
index 00000000000..c92e7593738
--- /dev/null
+++ b/lib/tdb2/test/run-summary.c
@@ -0,0 +1,60 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/summary.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct tdb_context *tdb;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	struct tdb_data key = { (unsigned char *)&j, sizeof(j) };
+	struct tdb_data data = { (unsigned char *)&j, sizeof(j) };
+	char *summary;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * (1 + 2 * 5) + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-summary.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		/* Put some stuff in there. */
+		for (j = 0; j < 500; j++) {
+			/* Make sure padding varies to we get some graphs! */
+			data.dsize = j % (sizeof(j) + 1);
+			if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+				fail("Storing in tdb");
+		}
+
+		for (j = 0;
+		     j <= TDB_SUMMARY_HISTOGRAMS;
+		     j += TDB_SUMMARY_HISTOGRAMS) {
+			ok1(tdb_summary(tdb, j, &summary) == TDB_SUCCESS);
+			ok1(strstr(summary, "Number of records: 500\n"));
+			ok1(strstr(summary, "Smallest/average/largest keys: 4/4/4\n"));
+			ok1(strstr(summary, "Smallest/average/largest data: 0/2/4\n"));
+			if (j == TDB_SUMMARY_HISTOGRAMS)
+				ok1(strstr(summary, "|")
+				    && strstr(summary, "*"));
+			else
+				ok1(!strstr(summary, "|")
+				    && !strstr(summary, "*"));
+			free(summary);
+		}
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-tdb_errorstr.c b/lib/tdb2/test/run-tdb_errorstr.c
new file mode 100644
index 00000000000..27bdfcd67cf
--- /dev/null
+++ b/lib/tdb2/test/run-tdb_errorstr.c
@@ -0,0 +1,59 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+
+int main(int argc, char *argv[])
+{
+	enum TDB_ERROR err;
+	plan_tests(TDB_ERR_RDONLY*-1 + 2);
+
+	for (err = TDB_SUCCESS; err >= TDB_ERR_RDONLY; err--) {
+		switch (err) {
+		case TDB_SUCCESS:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "Success"));
+			break;
+		case TDB_ERR_IO:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "IO Error"));
+			break;
+		case TDB_ERR_LOCK:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "Locking error"));
+			break;
+		case TDB_ERR_OOM:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "Out of memory"));
+			break;
+		case TDB_ERR_EXISTS:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "Record exists"));
+			break;
+		case TDB_ERR_EINVAL:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "Invalid parameter"));
+			break;
+		case TDB_ERR_NOEXIST:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "Record does not exist"));
+			break;
+		case TDB_ERR_RDONLY:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "write not permitted"));
+			break;
+		case TDB_ERR_CORRUPT:
+			ok1(!strcmp(tdb_errorstr(err),
+				    "Corrupt database"));
+			break;
+		}
+	}
+	ok1(!strcmp(tdb_errorstr(err), "Invalid error code"));
+
+	return exit_status();
+}
diff --git a/lib/tdb2/test/run-traverse.c b/lib/tdb2/test/run-traverse.c
new file mode 100644
index 00000000000..f973d95d0f6
--- /dev/null
+++ b/lib/tdb2/test/run-traverse.c
@@ -0,0 +1,211 @@
+#include <ccan/tdb2/tdb.c>
+#include <ccan/tdb2/open.c>
+#include <ccan/tdb2/free.c>
+#include <ccan/tdb2/lock.c>
+#include <ccan/tdb2/io.c>
+#include <ccan/tdb2/hash.c>
+#include <ccan/tdb2/check.c>
+#include <ccan/tdb2/traverse.c>
+#include <ccan/tdb2/transaction.c>
+#include <ccan/tap/tap.h>
+#include "logging.h"
+
+#define NUM_RECORDS 1000
+
+/* We use the same seed which we saw a failure on. */
+static uint64_t fixedhash(const void *key, size_t len, uint64_t seed, void *p)
+{
+	return hash64_stable((const unsigned char *)key, len,
+			     *(uint64_t *)p);
+}
+
+static bool store_records(struct tdb_context *tdb)
+{
+	int i;
+	struct tdb_data key = { (unsigned char *)&i, sizeof(i) };
+	struct tdb_data data = { (unsigned char *)&i, sizeof(i) };
+
+	for (i = 0; i < NUM_RECORDS; i++)
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
+			return false;
+	return true;
+}
+
+struct trav_data {
+	unsigned int calls, call_limit;
+	int low, high;
+	bool mismatch;
+	bool delete;
+	enum TDB_ERROR delete_error;
+};
+
+static int trav(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf,
+		struct trav_data *td)
+{
+	int val;
+
+	td->calls++;
+	if (key.dsize != sizeof(val) || dbuf.dsize != sizeof(val)
+	    || memcmp(key.dptr, dbuf.dptr, key.dsize) != 0) {
+		td->mismatch = true;
+		return -1;
+	}
+	memcpy(&val, dbuf.dptr, dbuf.dsize);
+	if (val < td->low)
+		td->low = val;
+	if (val > td->high)
+		td->high = val;
+
+	if (td->delete) {
+		td->delete_error = tdb_delete(tdb, key);
+		if (td->delete_error != TDB_SUCCESS) {
+			return -1;
+		}
+	}
+
+	if (td->calls == td->call_limit)
+		return 1;
+	return 0;
+}
+
+struct trav_grow_data {
+	unsigned int calls;
+	unsigned int num_large;
+	bool mismatch;
+	enum TDB_ERROR error;
+};
+
+static int trav_grow(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf,
+		     struct trav_grow_data *tgd)
+{
+	int val;
+	unsigned char buffer[128] = { 0 };
+
+	tgd->calls++;
+	if (key.dsize != sizeof(val) || dbuf.dsize < sizeof(val)
+	    || memcmp(key.dptr, dbuf.dptr, key.dsize) != 0) {
+		tgd->mismatch = true;
+		return -1;
+	}
+
+	if (dbuf.dsize > sizeof(val))
+		/* We must have seen this before! */
+		tgd->num_large++;
+
+	/* Make a big difference to the database. */
+	dbuf.dptr = buffer;
+	dbuf.dsize = sizeof(buffer);
+	tgd->error = tdb_append(tdb, key, dbuf);
+	if (tgd->error != TDB_SUCCESS) {
+		return -1;
+	}
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	int num;
+	struct trav_data td;
+	struct trav_grow_data tgd;
+	struct tdb_context *tdb;
+	uint64_t seed = 16014841315512641303ULL;
+	int flags[] = { TDB_INTERNAL, TDB_DEFAULT, TDB_NOMMAP,
+			TDB_INTERNAL|TDB_CONVERT, TDB_CONVERT,
+			TDB_NOMMAP|TDB_CONVERT };
+	union tdb_attribute hattr = { .hash = { .base = { TDB_ATTRIBUTE_HASH },
+						.fn = fixedhash,
+						.data = &seed } };
+
+	hattr.base.next = &tap_log_attr;
+
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 32 + 1);
+	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+		tdb = tdb_open("run-traverse.tdb", flags[i],
+			       O_RDWR|O_CREAT|O_TRUNC, 0600, &hattr);
+		ok1(tdb);
+		if (!tdb)
+			continue;
+
+		ok1(tdb_traverse(tdb, NULL, NULL) == 0);
+
+		ok1(store_records(tdb));
+		num = tdb_traverse(tdb, NULL, NULL);
+		ok1(num == NUM_RECORDS);
+
+		/* Full traverse. */
+		td.calls = 0;
+		td.call_limit = UINT_MAX;
+		td.low = INT_MAX;
+		td.high = INT_MIN;
+		td.mismatch = false;
+		td.delete = false;
+
+		num = tdb_traverse(tdb, trav, &td);
+		ok1(num == NUM_RECORDS);
+		ok1(!td.mismatch);
+		ok1(td.calls == NUM_RECORDS);
+		ok1(td.low == 0);
+		ok1(td.high == NUM_RECORDS-1);
+
+		/* Short traverse. */
+		td.calls = 0;
+		td.call_limit = NUM_RECORDS / 2;
+		td.low = INT_MAX;
+		td.high = INT_MIN;
+		td.mismatch = false;
+		td.delete = false;
+
+		num = tdb_traverse(tdb, trav, &td);
+		ok1(num == NUM_RECORDS / 2);
+		ok1(!td.mismatch);
+		ok1(td.calls == NUM_RECORDS / 2);
+		ok1(td.low <= NUM_RECORDS / 2);
+		ok1(td.high > NUM_RECORDS / 2);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tap_log_messages == 0);
+
+		/* Deleting traverse (delete everything). */
+		td.calls = 0;
+		td.call_limit = UINT_MAX;
+		td.low = INT_MAX;
+		td.high = INT_MIN;
+		td.mismatch = false;
+		td.delete = true;
+		td.delete_error = TDB_SUCCESS;
+		num = tdb_traverse(tdb, trav, &td);
+		ok1(num == NUM_RECORDS);
+		ok1(td.delete_error == TDB_SUCCESS);
+		ok1(!td.mismatch);
+		ok1(td.calls == NUM_RECORDS);
+		ok1(td.low == 0);
+		ok1(td.high == NUM_RECORDS - 1);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Now it's empty! */
+		ok1(tdb_traverse(tdb, NULL, NULL) == 0);
+
+		/* Re-add. */
+		ok1(store_records(tdb));
+		ok1(tdb_traverse(tdb, NULL, NULL) == NUM_RECORDS);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+
+		/* Grow.  This will cause us to be reshuffled. */
+		tgd.calls = 0;
+		tgd.num_large = 0;
+		tgd.mismatch = false;
+		tgd.error = TDB_SUCCESS;
+		ok1(tdb_traverse(tdb, trav_grow, &tgd) > 1);
+		ok1(tgd.error == 0);
+		ok1(!tgd.mismatch);
+		ok1(tdb_check(tdb, NULL, NULL) == 0);
+		ok1(tgd.num_large < tgd.calls);
+		diag("growing db: %u calls, %u repeats",
+		     tgd.calls, tgd.num_large);
+
+		tdb_close(tdb);
+	}
+
+	ok1(tap_log_messages == 0);
+	return exit_status();
+}
diff --git a/lib/tdb2/tools/Makefile b/lib/tdb2/tools/Makefile
new file mode 100644
index 00000000000..11188c3baf6
--- /dev/null
+++ b/lib/tdb2/tools/Makefile
@@ -0,0 +1,16 @@
+OBJS:=../../tdb2.o ../../hash.o ../../tally.o
+CFLAGS:=-I../../.. -I.. -Wall -g -O3 #-g -pg
+LDFLAGS:=-L../../..
+
+default: tdb2torture tdb2tool tdb2dump tdb2restore mktdb2 speed growtdb-bench
+
+tdb2dump: tdb2dump.c $(OBJS)
+tdb2restore: tdb2restore.c $(OBJS)
+tdb2torture: tdb2torture.c $(OBJS)
+tdb2tool: tdb2tool.c $(OBJS)
+mktdb2: mktdb2.c $(OBJS)
+speed: speed.c $(OBJS)
+growtdb-bench: growtdb-bench.c $(OBJS)
+
+clean:
+	rm -f tdb2torture tdb2dump tdb2restore tdb2tool mktdb2 speed growtdb-bench
diff --git a/lib/tdb2/tools/growtdb-bench.c b/lib/tdb2/tools/growtdb-bench.c
new file mode 100644
index 00000000000..f7f6845a8a4
--- /dev/null
+++ b/lib/tdb2/tools/growtdb-bench.c
@@ -0,0 +1,112 @@
+#include "tdb2.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <err.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+static void logfn(struct tdb_context *tdb,
+		  enum tdb_log_level level,
+		  const char *message,
+		  void *data)
+{
+	fprintf(stderr, "tdb:%s:%s\n", tdb_name(tdb), message);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, users, groups;
+	TDB_DATA idxkey, idxdata;
+	TDB_DATA k, d, gk;
+	char cmd[100];
+	struct tdb_context *tdb;
+	enum TDB_ERROR ecode;
+	union tdb_attribute log;
+
+	if (argc != 3) {
+		printf("Usage: growtdb-bench <users> <groups>\n");
+		exit(1);
+	}
+	users = atoi(argv[1]);
+	groups = atoi(argv[2]);
+
+	sprintf(cmd, "cat /proc/%i/statm", getpid());
+
+	log.base.attr = TDB_ATTRIBUTE_LOG;
+	log.base.next = NULL;
+	log.log.fn = logfn;
+
+	tdb = tdb_open("/tmp/growtdb.tdb", TDB_DEFAULT,
+		       O_RDWR|O_CREAT|O_TRUNC, 0600, &log);
+
+	idxkey.dptr = (unsigned char *)"User index";
+	idxkey.dsize = strlen("User index");
+	idxdata.dsize = 51;
+	idxdata.dptr = calloc(idxdata.dsize, 1);
+
+	/* Create users. */
+	k.dsize = 48;
+	k.dptr = calloc(k.dsize, 1);
+	d.dsize = 64;
+	d.dptr = calloc(d.dsize, 1);
+
+	tdb_transaction_start(tdb);
+	for (i = 0; i < users; i++) {
+		memcpy(k.dptr, &i, sizeof(i));
+		ecode = tdb_store(tdb, k, d, TDB_INSERT);
+		if (ecode != TDB_SUCCESS)
+			errx(1, "tdb insert failed: %s", tdb_errorstr(ecode));
+
+		/* This simulates a growing index record. */
+		ecode = tdb_append(tdb, idxkey, idxdata);
+		if (ecode != TDB_SUCCESS)
+			errx(1, "tdb append failed: %s", tdb_errorstr(ecode));
+	}
+	if ((ecode = tdb_transaction_commit(tdb)) != 0)
+		errx(1, "tdb commit1 failed: %s", tdb_errorstr(ecode));
+
+	if ((ecode = tdb_check(tdb, NULL, NULL)) != 0)
+		errx(1, "tdb_check failed after initial insert!");
+
+	system(cmd);
+
+	/* Now put them all in groups: add 32 bytes to each record for
+	 * a group. */
+	gk.dsize = 48;
+	gk.dptr = calloc(k.dsize, 1);
+	gk.dptr[gk.dsize-1] = 1;
+
+	d.dsize = 32;
+	for (i = 0; i < groups; i++) {
+		tdb_transaction_start(tdb);
+		/* Create the "group". */
+		memcpy(gk.dptr, &i, sizeof(i));
+		ecode = tdb_store(tdb, gk, d, TDB_INSERT);
+		if (ecode != TDB_SUCCESS)
+			errx(1, "tdb insert failed: %s", tdb_errorstr(ecode));
+
+		/* Now populate it. */
+		for (j = 0; j < users; j++) {
+			/* Append to the user. */
+			memcpy(k.dptr, &j, sizeof(j));
+			if ((ecode = tdb_append(tdb, k, d)) != 0)
+				errx(1, "tdb append failed: %s",
+				     tdb_errorstr(ecode));
+
+			/* Append to the group. */
+			if ((ecode = tdb_append(tdb, gk, d)) != 0)
+				errx(1, "tdb append failed: %s",
+				     tdb_errorstr(ecode));
+		}
+		if ((ecode = tdb_transaction_commit(tdb)) != 0)
+			errx(1, "tdb commit2 failed: %s", tdb_errorstr(ecode));
+		if ((ecode = tdb_check(tdb, NULL, NULL)) != 0)
+			errx(1, "tdb_check failed after iteration %i!", i);
+		system(cmd);
+	}
+
+	return 0;
+}
diff --git a/lib/tdb2/tools/mktdb2.c b/lib/tdb2/tools/mktdb2.c
new file mode 100644
index 00000000000..c8c280349e6
--- /dev/null
+++ b/lib/tdb2/tools/mktdb2.c
@@ -0,0 +1,29 @@
+#include "tdb2.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <err.h>
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, num_recs;
+	struct tdb_context *tdb;
+
+	if (argc != 3 || (num_recs = atoi(argv[2])) == 0)
+		errx(1, "Usage: mktdb <tdbfile> <numrecords>");
+
+	tdb = tdb_open(argv[1], TDB_DEFAULT, O_CREAT|O_TRUNC|O_RDWR, 0600,NULL);
+	if (!tdb)
+		err(1, "Opening %s", argv[1]);
+
+	for (i = 0; i < num_recs; i++) {
+		TDB_DATA d;
+
+		d.dptr = (void *)&i;
+		d.dsize = sizeof(i);
+		if (tdb_store(tdb, d, d, TDB_INSERT) != 0)
+			err(1, "Failed to store record %i", i);
+	}
+	printf("Done\n");
+	return 0;
+}
diff --git a/lib/tdb2/tools/speed.c b/lib/tdb2/tools/speed.c
new file mode 100644
index 00000000000..3222465a712
--- /dev/null
+++ b/lib/tdb2/tools/speed.c
@@ -0,0 +1,440 @@
+/* Simple speed test for TDB */
+#include <err.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include "tdb2.h"
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeval *start,
+			const struct timeval *stop,
+			unsigned int num)
+{
+	struct timeval diff;
+
+	timersub(stop, start, &diff);
+
+	/* Floating point is more accurate here. */
+	return (double)(diff.tv_sec * 1000000 + diff.tv_usec)
+		/ num * 1000;
+}
+
+static size_t file_size(void)
+{
+	struct stat st;
+
+	if (stat("/tmp/speed.tdb", &st) != 0)
+		return -1;
+	return st.st_size;
+}
+
+static int count_record(struct tdb_context *tdb,
+			TDB_DATA key, TDB_DATA data, void *p)
+{
+	int *total = p;
+	*total += *(int *)data.dptr;
+	return 0;
+}
+
+static void dump_and_clear_stats(struct tdb_context **tdb,
+				 int flags,
+				 union tdb_attribute *attr)
+{
+	union tdb_attribute stats;
+	enum TDB_ERROR ecode;
+
+	stats.base.attr = TDB_ATTRIBUTE_STATS;
+	stats.stats.size = sizeof(stats.stats);
+	ecode = tdb_get_attribute(*tdb, &stats);
+	if (ecode != TDB_SUCCESS)
+		errx(1, "Getting stats: %s", tdb_errorstr(ecode));
+
+	printf("allocs = %llu\n",
+	       (unsigned long long)stats.stats.allocs);
+	printf("  alloc_subhash = %llu\n",
+	       (unsigned long long)stats.stats.alloc_subhash);
+	printf("  alloc_chain = %llu\n",
+	       (unsigned long long)stats.stats.alloc_chain);
+	printf("  alloc_bucket_exact = %llu\n",
+	       (unsigned long long)stats.stats.alloc_bucket_exact);
+	printf("  alloc_bucket_max = %llu\n",
+	       (unsigned long long)stats.stats.alloc_bucket_max);
+	printf("  alloc_leftover = %llu\n",
+	       (unsigned long long)stats.stats.alloc_leftover);
+	printf("  alloc_coalesce_tried = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_tried);
+	printf("    alloc_coalesce_iterate_clash = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_iterate_clash);
+	printf("    alloc_coalesce_lockfail = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_lockfail);
+	printf("    alloc_coalesce_race = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_race);
+	printf("    alloc_coalesce_succeeded = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_succeeded);
+	printf("      alloc_coalesce_num_merged = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_num_merged);
+	printf("compares = %llu\n",
+	       (unsigned long long)stats.stats.compares);
+	printf("  compare_wrong_bucket = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_bucket);
+	printf("  compare_wrong_offsetbits = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_offsetbits);
+	printf("  compare_wrong_keylen = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_keylen);
+	printf("  compare_wrong_rechash = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_rechash);
+	printf("  compare_wrong_keycmp = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_keycmp);
+	printf("transactions = %llu\n",
+	       (unsigned long long)stats.stats.transactions);
+	printf("  transaction_cancel = %llu\n",
+	       (unsigned long long)stats.stats.transaction_cancel);
+	printf("  transaction_nest = %llu\n",
+	       (unsigned long long)stats.stats.transaction_nest);
+	printf("  transaction_expand_file = %llu\n",
+	       (unsigned long long)stats.stats.transaction_expand_file);
+	printf("  transaction_read_direct = %llu\n",
+	       (unsigned long long)stats.stats.transaction_read_direct);
+	printf("    transaction_read_direct_fail = %llu\n",
+	       (unsigned long long)stats.stats.transaction_read_direct_fail);
+	printf("  transaction_write_direct = %llu\n",
+	       (unsigned long long)stats.stats.transaction_write_direct);
+	printf("    transaction_write_direct_fail = %llu\n",
+	       (unsigned long long)stats.stats.transaction_write_direct_fail);
+	printf("expands = %llu\n",
+	       (unsigned long long)stats.stats.expands);
+	printf("frees = %llu\n",
+	       (unsigned long long)stats.stats.frees);
+	printf("locks = %llu\n",
+	       (unsigned long long)stats.stats.locks);
+	printf("  lock_lowlevel = %llu\n",
+	       (unsigned long long)stats.stats.lock_lowlevel);
+	printf("  lock_nonblock = %llu\n",
+	       (unsigned long long)stats.stats.lock_nonblock);
+	printf("    lock_nonblock_fail = %llu\n",
+	       (unsigned long long)stats.stats.lock_nonblock_fail);
+
+	/* Now clear. */
+	tdb_close(*tdb);
+	*tdb = tdb_open("/tmp/speed.tdb", flags, O_RDWR, 0, attr);
+}
+
+static void tdb_log(struct tdb_context *tdb, enum tdb_log_level level,
+		    const char *message, void *data)
+{
+	fputs(message, stderr);
+	putc('\n', stderr);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, num = 1000, stage = 0, stopat = -1;
+	int flags = TDB_DEFAULT;
+	bool transaction = false, summary = false;
+	TDB_DATA key, data;
+	struct tdb_context *tdb;
+	struct timeval start, stop;
+	union tdb_attribute seed, log;
+	bool do_stats = false;
+	enum TDB_ERROR ecode;
+
+	/* Try to keep benchmarks even. */
+	seed.base.attr = TDB_ATTRIBUTE_SEED;
+	seed.base.next = NULL;
+	seed.seed.seed = 0;
+
+	log.base.attr = TDB_ATTRIBUTE_LOG;
+	log.base.next = &seed;
+	log.log.fn = tdb_log;
+
+	if (argv[1] && strcmp(argv[1], "--internal") == 0) {
+		flags = TDB_INTERNAL;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--transaction") == 0) {
+		transaction = true;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--no-sync") == 0) {
+		flags |= TDB_NOSYNC;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--summary") == 0) {
+		summary = true;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--stats") == 0) {
+		do_stats = true;
+		argc--;
+		argv++;
+	}
+
+	tdb = tdb_open("/tmp/speed.tdb", flags, O_RDWR|O_CREAT|O_TRUNC,
+		       0600, &log);
+	if (!tdb)
+		err(1, "Opening /tmp/speed.tdb");
+
+	key.dptr = (void *)&i;
+	key.dsize = sizeof(i);
+	data = key;
+
+	if (argv[1]) {
+		num = atoi(argv[1]);
+		argv++;
+		argc--;
+	}
+
+	if (argv[1]) {
+		stopat = atoi(argv[1]);
+		argv++;
+		argc--;
+	}
+
+	/* Add 1000 records. */
+	printf("Adding %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if ((ecode = tdb_store(tdb, key, data, TDB_INSERT)) != 0)
+			errx(1, "Inserting key %u in tdb: %s",
+			     i, tdb_errorstr(ecode));
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&tdb, flags, &log);
+
+	if (++stage == stopat)
+		exit(0);
+
+	/* Finding 1000 records. */
+	printf("Finding %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		struct tdb_data dbuf;
+		if ((ecode = tdb_fetch(tdb, key, &dbuf)) != TDB_SUCCESS
+		    || *(int *)dbuf.dptr != i) {
+			errx(1, "Fetching key %u in tdb gave %u",
+			     i, ecode ? ecode : *(int *)dbuf.dptr);
+		}
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&tdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Missing 1000 records. */
+	printf("Missing %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (i = num; i < num*2; i++) {
+		struct tdb_data dbuf;
+		ecode = tdb_fetch(tdb, key, &dbuf);
+		if (ecode != TDB_ERR_NOEXIST)
+			errx(1, "Fetching key %u in tdb gave %s",
+			     i, tdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&tdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Traverse 1000 records. */
+	printf("Traversing %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	i = 0;
+	gettimeofday(&start, NULL);
+	if (tdb_traverse(tdb, count_record, &i) != num)
+		errx(1, "Traverse returned wrong number of records");
+	if (i != (num - 1) * (num / 2))
+		errx(1, "Traverse tallied to %u", i);
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&tdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Delete 1000 records (not in order). */
+	printf("Deleting %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (j = 0; j < num; j++) {
+		i = (j + 100003) % num;
+		if ((ecode = tdb_delete(tdb, key)) != TDB_SUCCESS)
+			errx(1, "Deleting key %u in tdb: %s",
+			     i, tdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&tdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Re-add 1000 records (not in order). */
+	printf("Re-adding %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (j = 0; j < num; j++) {
+		i = (j + 100003) % num;
+		if ((ecode = tdb_store(tdb, key, data, TDB_INSERT)) != 0)
+			errx(1, "Inserting key %u in tdb: %s",
+			     i, tdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&tdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Append 1000 records. */
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	printf("Appending %u records: ", num); fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if ((ecode = tdb_append(tdb, key, data)) != TDB_SUCCESS)
+			errx(1, "Appending key %u in tdb: %s",
+			     i, tdb_errorstr(ecode));
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (++stage == stopat)
+		exit(0);
+
+	/* Churn 1000 records: not in order! */
+	if (transaction && (ecode = tdb_transaction_start(tdb)))
+		errx(1, "starting transaction: %s", tdb_errorstr(ecode));
+	printf("Churning %u records: ", num); fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (j = 0; j < num; j++) {
+		i = (j + 1000019) % num;
+		if ((ecode = tdb_delete(tdb, key)) != TDB_SUCCESS)
+			errx(1, "Deleting key %u in tdb: %s",
+			     i, tdb_errorstr(ecode));
+		i += num;
+		if ((ecode = tdb_store(tdb, key, data, TDB_INSERT)) != 0)
+			errx(1, "Inserting key %u in tdb: %s",
+			     i, tdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = tdb_transaction_commit(tdb)))
+		errx(1, "committing transaction: %s", tdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+
+	if (tdb_check(tdb, NULL, NULL))
+		errx(1, "tdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&tdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	return 0;
+}
diff --git a/lib/tdb2/tools/tdb2dump.c b/lib/tdb2/tools/tdb2dump.c
new file mode 100644
index 00000000000..abe1d9b8710
--- /dev/null
+++ b/lib/tdb2/tools/tdb2dump.c
@@ -0,0 +1,115 @@
+/*
+   simple tdb2 dump util
+   Copyright (C) Andrew Tridgell              2001
+   Copyright (C) Rusty Russell                2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "tdb2.h"
+#include <ctype.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+static void print_data(TDB_DATA d)
+{
+	unsigned char *p = (unsigned char *)d.dptr;
+	int len = d.dsize;
+	while (len--) {
+		if (isprint(*p) && !strchr("\"\\", *p)) {
+			fputc(*p, stdout);
+		} else {
+			printf("\\%02X", *p);
+		}
+		p++;
+	}
+}
+
+static int traverse_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+{
+	printf("{\n");
+	printf("key(%d) = \"", (int)key.dsize);
+	print_data(key);
+	printf("\"\n");
+	printf("data(%d) = \"", (int)dbuf.dsize);
+	print_data(dbuf);
+	printf("\"\n");
+	printf("}\n");
+	return 0;
+}
+
+static int dump_tdb(const char *fname, const char *keyname)
+{
+	struct tdb_context *tdb;
+	TDB_DATA key, value;
+
+	tdb = tdb_open(fname, 0, O_RDONLY, 0, NULL);
+	if (!tdb) {
+		printf("Failed to open %s\n", fname);
+		return 1;
+	}
+
+	if (!keyname) {
+		tdb_traverse(tdb, traverse_fn, NULL);
+	} else {
+		key = tdb_mkdata(keyname, strlen(keyname));
+		if (tdb_fetch(tdb, key, &value) != 0) {
+			return 1;
+		} else {
+			print_data(value);
+			free(value.dptr);
+		}
+	}
+
+	return 0;
+}
+
+static void usage( void)
+{
+	printf( "Usage: tdb2dump [options] <filename>\n\n");
+	printf( "   -h          this help message\n");
+	printf( "   -k keyname  dumps value of keyname\n");
+}
+
+ int main(int argc, char *argv[])
+{
+	char *fname, *keyname=NULL;
+	int c;
+
+	if (argc < 2) {
+		printf("Usage: tdb2dump <fname>\n");
+		exit(1);
+	}
+
+	while ((c = getopt( argc, argv, "hk:")) != -1) {
+		switch (c) {
+		case 'h':
+			usage();
+			exit( 0);
+		case 'k':
+			keyname = optarg;
+			break;
+		default:
+			usage();
+			exit( 1);
+		}
+	}
+
+	fname = argv[optind];
+
+	return dump_tdb(fname, keyname);
+}
diff --git a/lib/tdb2/tools/tdb2restore.c b/lib/tdb2/tools/tdb2restore.c
new file mode 100644
index 00000000000..658215a16c4
--- /dev/null
+++ b/lib/tdb2/tools/tdb2restore.c
@@ -0,0 +1,227 @@
+/*
+   tdb2restore -- construct a tdb from tdbdump output.
+   Copyright (C) Volker Lendecke		2010
+   Copyright (C) Simon McVittie			2005
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "tdb2.h"
+#include <assert.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define debug_fprintf(file, fmt, ...) do {/*nothing*/} while (0)
+
+static int read_linehead(FILE *f)
+{
+	int i, c;
+	int num_bytes;
+	char prefix[128];
+
+	while (1) {
+		c = getc(f);
+		if (c == EOF) {
+			return -1;
+		}
+		if (c == '(') {
+			break;
+		}
+	}
+	for (i=0; i<sizeof(prefix); i++) {
+		c = getc(f);
+		if (c == EOF) {
+			return -1;
+		}
+		prefix[i] = c;
+		if (c == '"') {
+			break;
+		}
+	}
+	if (i == sizeof(prefix)) {
+		return -1;
+	}
+	prefix[i] = '\0';
+
+	if (sscanf(prefix, "%d) = ", &num_bytes) != 1) {
+		return -1;
+	}
+	return num_bytes;
+}
+
+static int read_hex(void) {
+	int c;
+	c = getchar();
+	if (c == EOF) {
+		fprintf(stderr, "Unexpected EOF in data\n");
+		return -1;
+	} else if (c == '"') {
+		fprintf(stderr, "Unexpected \\\" sequence\n");
+		return -1;
+	} else if ('0' <= c && c <= '9')  {
+		return c - '0';
+	} else if ('A' <= c && c <= 'F')  {
+		return c - 'A' + 10;
+	} else if ('a' <= c && c <= 'f')  {
+		return c - 'a' + 10;
+	} else {
+		fprintf(stderr, "Invalid hex: %c\n", c);
+		return -1;
+	}
+}
+
+static int read_data(FILE *f, struct tdb_data *d, size_t size) {
+	int c, low, high;
+	int i;
+
+	d->dptr = (unsigned char *)malloc(size);
+	if (d->dptr == NULL) {
+		return -1;
+	}
+	d->dsize = size;
+
+	for (i=0; i<size; i++) {
+		c = getc(f);
+		if (c == EOF) {
+			fprintf(stderr, "Unexpected EOF in data\n");
+			return 1;
+		} else if (c == '"') {
+			return 0;
+		} else if (c == '\\') {
+			high = read_hex();
+			if (high < 0) {
+				return -1;
+			}
+			high = high << 4;
+			assert(high == (high & 0xf0));
+			low = read_hex();
+			if (low < 0) {
+				return -1;
+			}
+			assert(low == (low & 0x0f));
+			d->dptr[i] = (low|high);
+		} else {
+			d->dptr[i] = c;
+		}
+	}
+	return 0;
+}
+
+static int swallow(FILE *f, const char *s, int *eof)
+{
+	char line[128];
+
+	if (fgets(line, sizeof(line), f) == NULL) {
+		if (eof != NULL) {
+			*eof = 1;
+		}
+		return -1;
+	}
+	if (strcmp(line, s) != 0) {
+		return -1;
+	}
+	return 0;
+}
+
+static bool read_rec(FILE *f, struct tdb_context *tdb, int *eof)
+{
+	int length;
+	struct tdb_data key, data;
+	bool ret = false;
+	enum TDB_ERROR e;
+
+	key.dptr = NULL;
+	data.dptr = NULL;
+
+	if (swallow(f, "{\n", eof) == -1) {
+		goto fail;
+	}
+	length = read_linehead(f);
+	if (length == -1) {
+		goto fail;
+	}
+	if (read_data(f, &key, length) == -1) {
+		goto fail;
+	}
+	if (swallow(f, "\"\n", NULL) == -1) {
+		goto fail;
+	}
+	length = read_linehead(f);
+	if (length == -1) {
+		goto fail;
+	}
+	if (read_data(f, &data, length) == -1) {
+		goto fail;
+	}
+	if ((swallow(f, "\"\n", NULL) == -1)
+	    || (swallow(f, "}\n", NULL) == -1)) {
+		goto fail;
+	}
+	e = tdb_store(tdb, key, data, TDB_INSERT);
+	if (e != TDB_SUCCESS) {
+		fprintf(stderr, "TDB error: %s\n", tdb_errorstr(e));
+		goto fail;
+	}
+
+	ret = true;
+fail:
+	free(key.dptr);
+	free(data.dptr);
+	return ret;
+}
+
+static int restore_tdb(const char *fname)
+{
+	struct tdb_context *tdb;
+
+	tdb = tdb_open(fname, 0, O_RDWR|O_CREAT|O_EXCL, 0666, NULL);
+	if (!tdb) {
+		perror("tdb_open");
+		fprintf(stderr, "Failed to open %s\n", fname);
+		return 1;
+	}
+
+	while (1) {
+		int eof = 0;
+		if (!read_rec(stdin, tdb, &eof)) {
+			if (eof) {
+				break;
+			}
+			return 1;
+		}
+	}
+	if (tdb_close(tdb)) {
+		fprintf(stderr, "Error closing tdb\n");
+		return 1;
+	}
+	fprintf(stderr, "EOF\n");
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	char *fname;
+
+	if (argc < 2) {
+		printf("Usage: %s dbname < tdbdump_output\n", argv[0]);
+		exit(1);
+	}
+
+	fname = argv[1];
+
+	return restore_tdb(fname);
+}
diff --git a/lib/tdb2/tools/tdb2tool.c b/lib/tdb2/tools/tdb2tool.c
new file mode 100644
index 00000000000..cd301c80b78
--- /dev/null
+++ b/lib/tdb2/tools/tdb2tool.c
@@ -0,0 +1,798 @@
+/*
+   Unix SMB/CIFS implementation.
+   Samba database functions
+   Copyright (C) Andrew Tridgell              1999-2000
+   Copyright (C) Paul `Rusty' Russell		   2000
+   Copyright (C) Jeremy Allison			   2000
+   Copyright (C) Andrew Esh                        2001
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "tdb2.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <stdarg.h>
+
+static int do_command(void);
+const char *cmdname;
+char *arg1, *arg2;
+size_t arg1len, arg2len;
+int bIterate = 0;
+char *line;
+TDB_DATA iterate_kbuf;
+char cmdline[1024];
+static int disable_mmap;
+
+enum commands {
+	CMD_CREATE_TDB,
+	CMD_OPEN_TDB,
+	CMD_TRANSACTION_START,
+	CMD_TRANSACTION_COMMIT,
+	CMD_TRANSACTION_CANCEL,
+	CMD_ERASE,
+	CMD_DUMP,
+	CMD_INSERT,
+	CMD_MOVE,
+	CMD_STORE,
+	CMD_SHOW,
+	CMD_KEYS,
+	CMD_HEXKEYS,
+	CMD_DELETE,
+#if 0
+	CMD_LIST_HASH_FREE,
+	CMD_LIST_FREE,
+#endif
+	CMD_INFO,
+	CMD_MMAP,
+	CMD_SPEED,
+	CMD_FIRST,
+	CMD_NEXT,
+	CMD_SYSTEM,
+	CMD_CHECK,
+	CMD_QUIT,
+	CMD_HELP
+};
+
+typedef struct {
+	const char *name;
+	enum commands cmd;
+} COMMAND_TABLE;
+
+COMMAND_TABLE cmd_table[] = {
+	{"create",	CMD_CREATE_TDB},
+	{"open",	CMD_OPEN_TDB},
+#if 0
+	{"transaction_start",	CMD_TRANSACTION_START},
+	{"transaction_commit",	CMD_TRANSACTION_COMMIT},
+	{"transaction_cancel",	CMD_TRANSACTION_CANCEL},
+#endif
+	{"erase",	CMD_ERASE},
+	{"dump",	CMD_DUMP},
+	{"insert",	CMD_INSERT},
+	{"move",	CMD_MOVE},
+	{"store",	CMD_STORE},
+	{"show",	CMD_SHOW},
+	{"keys",	CMD_KEYS},
+	{"hexkeys",	CMD_HEXKEYS},
+	{"delete",	CMD_DELETE},
+#if 0
+	{"list",	CMD_LIST_HASH_FREE},
+	{"free",	CMD_LIST_FREE},
+#endif
+	{"info",	CMD_INFO},
+	{"speed",	CMD_SPEED},
+	{"mmap",	CMD_MMAP},
+	{"first",	CMD_FIRST},
+	{"1",		CMD_FIRST},
+	{"next",	CMD_NEXT},
+	{"n",		CMD_NEXT},
+	{"check",	CMD_CHECK},
+	{"quit",	CMD_QUIT},
+	{"q",		CMD_QUIT},
+	{"!",		CMD_SYSTEM},
+	{NULL,		CMD_HELP}
+};
+
+struct timeval tp1,tp2;
+
+static void _start_timer(void)
+{
+	gettimeofday(&tp1,NULL);
+}
+
+static double _end_timer(void)
+{
+	gettimeofday(&tp2,NULL);
+	return((tp2.tv_sec - tp1.tv_sec) +
+	       (tp2.tv_usec - tp1.tv_usec)*1.0e-6);
+}
+
+static void tdb_log(struct tdb_context *tdb, enum tdb_log_level level,
+		    const char *message, void *priv)
+{
+	fputs(message, stderr);
+}
+
+/* a tdb tool for manipulating a tdb database */
+
+static struct tdb_context *tdb;
+
+static int print_rec(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state);
+static int print_key(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state);
+static int print_hexkey(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state);
+
+static void print_asc(const char *buf,int len)
+{
+	int i;
+
+	/* We're probably printing ASCII strings so don't try to display
+	   the trailing NULL character. */
+
+	if (buf[len - 1] == 0)
+	        len--;
+
+	for (i=0;i<len;i++)
+		printf("%c",isprint(buf[i])?buf[i]:'.');
+}
+
+static void print_data(const char *buf,int len)
+{
+	int i=0;
+	if (len<=0) return;
+	printf("[%03X] ",i);
+	for (i=0;i<len;) {
+		printf("%02X ",(int)((unsigned char)buf[i]));
+		i++;
+		if (i%8 == 0) printf(" ");
+		if (i%16 == 0) {
+			print_asc(&buf[i-16],8); printf(" ");
+			print_asc(&buf[i-8],8); printf("\n");
+			if (i<len) printf("[%03X] ",i);
+		}
+	}
+	if (i%16) {
+		int n;
+
+		n = 16 - (i%16);
+		printf(" ");
+		if (n>8) printf(" ");
+		while (n--) printf("   ");
+
+		n = i%16;
+		if (n > 8) n = 8;
+		print_asc(&buf[i-(i%16)],n); printf(" ");
+		n = (i%16) - n;
+		if (n>0) print_asc(&buf[i-n],n);
+		printf("\n");
+	}
+}
+
+static void help(void)
+{
+	printf("\n"
+"tdbtool: \n"
+"  create    dbname     : create a database\n"
+"  open      dbname     : open an existing database\n"
+"  openjh    dbname     : open an existing database (jenkins hash)\n"
+"  transaction_start    : start a transaction\n"
+"  transaction_commit   : commit a transaction\n"
+"  transaction_cancel   : cancel a transaction\n"
+"  erase                : erase the database\n"
+"  dump                 : dump the database as strings\n"
+"  keys                 : dump the database keys as strings\n"
+"  hexkeys              : dump the database keys as hex values\n"
+"  info                 : print summary info about the database\n"
+"  insert    key  data  : insert a record\n"
+"  move      key  file  : move a record to a destination tdb\n"
+"  store     key  data  : store a record (replace)\n"
+"  show      key        : show a record by key\n"
+"  delete    key        : delete a record by key\n"
+#if 0
+"  list                 : print the database hash table and freelist\n"
+"  free                 : print the database freelist\n"
+#endif
+"  check                : check the integrity of an opened database\n"
+"  speed                : perform speed tests on the database\n"
+"  ! command            : execute system command\n"
+"  1 | first            : print the first record\n"
+"  n | next             : print the next record\n"
+"  q | quit             : terminate\n"
+"  \\n                   : repeat 'next' command\n"
+"\n");
+}
+
+static void terror(enum TDB_ERROR err, const char *why)
+{
+	if (err != TDB_SUCCESS)
+		printf("%s:%s\n", tdb_errorstr(err), why);
+	else
+		printf("%s\n", why);
+}
+
+static void create_tdb(const char *tdbname)
+{
+	union tdb_attribute log_attr;
+	log_attr.base.attr = TDB_ATTRIBUTE_LOG;
+	log_attr.base.next = NULL;
+	log_attr.log.fn = tdb_log;
+
+	if (tdb) tdb_close(tdb);
+	tdb = tdb_open(tdbname, (disable_mmap?TDB_NOMMAP:0),
+		       O_RDWR | O_CREAT | O_TRUNC, 0600, &log_attr);
+	if (!tdb) {
+		printf("Could not create %s: %s\n", tdbname, strerror(errno));
+	}
+}
+
+static void open_tdb(const char *tdbname)
+{
+	union tdb_attribute log_attr;
+	log_attr.base.attr = TDB_ATTRIBUTE_LOG;
+	log_attr.base.next = NULL;
+	log_attr.log.fn = tdb_log;
+
+	if (tdb) tdb_close(tdb);
+	tdb = tdb_open(tdbname, disable_mmap?TDB_NOMMAP:0, O_RDWR, 0600,
+		       &log_attr);
+	if (!tdb) {
+		printf("Could not open %s: %s\n", tdbname, strerror(errno));
+	}
+}
+
+static void insert_tdb(char *keyname, size_t keylen, char* data, size_t datalen)
+{
+	TDB_DATA key, dbuf;
+	enum TDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(TDB_SUCCESS, "need key");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+	dbuf.dptr = (unsigned char *)data;
+	dbuf.dsize = datalen;
+
+	ecode = tdb_store(tdb, key, dbuf, TDB_INSERT);
+	if (ecode) {
+		terror(ecode, "insert failed");
+	}
+}
+
+static void store_tdb(char *keyname, size_t keylen, char* data, size_t datalen)
+{
+	TDB_DATA key, dbuf;
+	enum TDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(TDB_SUCCESS, "need key");
+		return;
+	}
+
+	if ((data == NULL) || (datalen == 0)) {
+		terror(TDB_SUCCESS, "need data");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+	dbuf.dptr = (unsigned char *)data;
+	dbuf.dsize = datalen;
+
+	printf("Storing key:\n");
+	print_rec(tdb, key, dbuf, NULL);
+
+	ecode = tdb_store(tdb, key, dbuf, TDB_REPLACE);
+	if (ecode) {
+		terror(ecode, "store failed");
+	}
+}
+
+static void show_tdb(char *keyname, size_t keylen)
+{
+	TDB_DATA key, dbuf;
+	enum TDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(TDB_SUCCESS, "need key");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+
+	ecode = tdb_fetch(tdb, key, &dbuf);
+	if (ecode) {
+		terror(ecode, "fetch failed");
+		return;
+	}
+
+	print_rec(tdb, key, dbuf, NULL);
+
+	free( dbuf.dptr );
+}
+
+static void delete_tdb(char *keyname, size_t keylen)
+{
+	TDB_DATA key;
+	enum TDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(TDB_SUCCESS, "need key");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+
+	ecode = tdb_delete(tdb, key);
+	if (ecode) {
+		terror(ecode, "delete failed");
+	}
+}
+
+static void move_rec(char *keyname, size_t keylen, char* tdbname)
+{
+	TDB_DATA key, dbuf;
+	struct tdb_context *dst_tdb;
+	enum TDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(TDB_SUCCESS, "need key");
+		return;
+	}
+
+	if ( !tdbname ) {
+		terror(TDB_SUCCESS, "need destination tdb name");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+
+	ecode = tdb_fetch(tdb, key, &dbuf);
+	if (ecode) {
+		terror(ecode, "fetch failed");
+		return;
+	}
+
+	print_rec(tdb, key, dbuf, NULL);
+
+	dst_tdb = tdb_open(tdbname, 0, O_RDWR, 0600, NULL);
+	if ( !dst_tdb ) {
+		terror(TDB_SUCCESS, "unable to open destination tdb");
+		return;
+	}
+
+	ecode = tdb_store( dst_tdb, key, dbuf, TDB_REPLACE);
+	if (ecode)
+		terror(ecode, "failed to move record");
+	else
+		printf("record moved\n");
+
+	tdb_close( dst_tdb );
+}
+
+static int print_rec(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+{
+	printf("\nkey %d bytes\n", (int)key.dsize);
+	print_asc((const char *)key.dptr, key.dsize);
+	printf("\ndata %d bytes\n", (int)dbuf.dsize);
+	print_data((const char *)dbuf.dptr, dbuf.dsize);
+	return 0;
+}
+
+static int print_key(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+{
+	printf("key %d bytes: ", (int)key.dsize);
+	print_asc((const char *)key.dptr, key.dsize);
+	printf("\n");
+	return 0;
+}
+
+static int print_hexkey(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+{
+	printf("key %d bytes\n", (int)key.dsize);
+	print_data((const char *)key.dptr, key.dsize);
+	printf("\n");
+	return 0;
+}
+
+static int total_bytes;
+
+static int traverse_fn(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+{
+	total_bytes += dbuf.dsize;
+	return 0;
+}
+
+static void info_tdb(void)
+{
+	enum TDB_ERROR ecode;
+	char *summary;
+
+	ecode = tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &summary);
+
+	if (ecode) {
+		terror(ecode, "Getting summary");
+	} else {
+		printf("%s", summary);
+		free(summary);
+	}
+}
+
+static void speed_tdb(const char *tlimit)
+{
+	unsigned timelimit = tlimit?atoi(tlimit):0;
+	double t;
+	int ops;
+	if (timelimit == 0) timelimit = 5;
+
+	ops = 0;
+	printf("Testing store speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		long int r = random();
+		TDB_DATA key, dbuf;
+		key = tdb_mkdata("store test", strlen("store test"));
+		dbuf.dptr = (unsigned char *)&r;
+		dbuf.dsize = sizeof(r);
+		tdb_store(tdb, key, dbuf, TDB_REPLACE);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+
+	ops = 0;
+	printf("Testing fetch speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		long int r = random();
+		TDB_DATA key, dbuf;
+		key = tdb_mkdata("store test", strlen("store test"));
+		dbuf.dptr = (unsigned char *)&r;
+		dbuf.dsize = sizeof(r);
+		tdb_fetch(tdb, key, &dbuf);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+
+	ops = 0;
+	printf("Testing transaction speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		long int r = random();
+		TDB_DATA key, dbuf;
+		key = tdb_mkdata("transaction test", strlen("transaction test"));
+		dbuf.dptr = (unsigned char *)&r;
+		dbuf.dsize = sizeof(r);
+		tdb_transaction_start(tdb);
+		tdb_store(tdb, key, dbuf, TDB_REPLACE);
+		tdb_transaction_commit(tdb);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+
+	ops = 0;
+	printf("Testing traverse speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		tdb_traverse(tdb, traverse_fn, NULL);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+}
+
+static void toggle_mmap(void)
+{
+	disable_mmap = !disable_mmap;
+	if (disable_mmap) {
+		printf("mmap is disabled\n");
+	} else {
+		printf("mmap is enabled\n");
+	}
+}
+
+static char *tdb_getline(const char *prompt)
+{
+	static char thisline[1024];
+	char *p;
+	fputs(prompt, stdout);
+	thisline[0] = 0;
+	p = fgets(thisline, sizeof(thisline)-1, stdin);
+	if (p) p = strchr(p, '\n');
+	if (p) *p = 0;
+	return p?thisline:NULL;
+}
+
+static int do_delete_fn(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf,
+                     void *state)
+{
+    return tdb_delete(the_tdb, key);
+}
+
+static void first_record(struct tdb_context *the_tdb, TDB_DATA *pkey)
+{
+	TDB_DATA dbuf;
+	enum TDB_ERROR ecode;
+	ecode = tdb_firstkey(the_tdb, pkey);
+	if (!ecode)
+		ecode = tdb_fetch(the_tdb, *pkey, &dbuf);
+	if (ecode) terror(ecode, "fetch failed");
+	else {
+		print_rec(the_tdb, *pkey, dbuf, NULL);
+	}
+}
+
+static void next_record(struct tdb_context *the_tdb, TDB_DATA *pkey)
+{
+	TDB_DATA dbuf;
+	enum TDB_ERROR ecode;
+	ecode = tdb_nextkey(the_tdb, pkey);
+
+	if (!ecode)
+		ecode = tdb_fetch(the_tdb, *pkey, &dbuf);
+	if (ecode)
+		terror(ecode, "fetch failed");
+	else
+		print_rec(the_tdb, *pkey, dbuf, NULL);
+}
+
+static void check_db(struct tdb_context *the_tdb)
+{
+	if (!the_tdb) {
+		printf("Error: No database opened!\n");
+	} else {
+		if (tdb_check(the_tdb, NULL, NULL) != 0)
+			printf("Integrity check for the opened database failed.\n");
+		else
+			printf("Database integrity is OK.\n");
+	}
+}
+
+static int do_command(void)
+{
+	COMMAND_TABLE *ctp = cmd_table;
+	enum commands mycmd = CMD_HELP;
+	int cmd_len;
+
+	if (cmdname && strlen(cmdname) == 0) {
+		mycmd = CMD_NEXT;
+	} else {
+		while (ctp->name) {
+			cmd_len = strlen(ctp->name);
+			if (strncmp(ctp->name,cmdname,cmd_len) == 0) {
+				mycmd = ctp->cmd;
+				break;
+			}
+			ctp++;
+		}
+	}
+
+	switch (mycmd) {
+	case CMD_CREATE_TDB:
+		bIterate = 0;
+		create_tdb(arg1);
+		return 0;
+	case CMD_OPEN_TDB:
+		bIterate = 0;
+		open_tdb(arg1);
+		return 0;
+	case CMD_SYSTEM:
+		/* Shell command */
+		if (system(arg1) == -1) {
+			terror(TDB_SUCCESS, "system() call failed\n");
+		}
+		return 0;
+	case CMD_QUIT:
+		return 1;
+	default:
+		/* all the rest require a open database */
+		if (!tdb) {
+			bIterate = 0;
+			terror(TDB_SUCCESS, "database not open");
+			help();
+			return 0;
+		}
+		switch (mycmd) {
+		case CMD_TRANSACTION_START:
+			bIterate = 0;
+			tdb_transaction_start(tdb);
+			return 0;
+		case CMD_TRANSACTION_COMMIT:
+			bIterate = 0;
+			tdb_transaction_commit(tdb);
+			return 0;
+		case CMD_TRANSACTION_CANCEL:
+			bIterate = 0;
+			tdb_transaction_cancel(tdb);
+			return 0;
+		case CMD_ERASE:
+			bIterate = 0;
+			tdb_traverse(tdb, do_delete_fn, NULL);
+			return 0;
+		case CMD_DUMP:
+			bIterate = 0;
+			tdb_traverse(tdb, print_rec, NULL);
+			return 0;
+		case CMD_INSERT:
+			bIterate = 0;
+			insert_tdb(arg1, arg1len,arg2,arg2len);
+			return 0;
+		case CMD_MOVE:
+			bIterate = 0;
+			move_rec(arg1,arg1len,arg2);
+			return 0;
+		case CMD_STORE:
+			bIterate = 0;
+			store_tdb(arg1,arg1len,arg2,arg2len);
+			return 0;
+		case CMD_SHOW:
+			bIterate = 0;
+			show_tdb(arg1, arg1len);
+			return 0;
+		case CMD_KEYS:
+			tdb_traverse(tdb, print_key, NULL);
+			return 0;
+		case CMD_HEXKEYS:
+			tdb_traverse(tdb, print_hexkey, NULL);
+			return 0;
+		case CMD_DELETE:
+			bIterate = 0;
+			delete_tdb(arg1,arg1len);
+			return 0;
+#if 0
+		case CMD_LIST_HASH_FREE:
+			tdb_dump_all(tdb);
+			return 0;
+		case CMD_LIST_FREE:
+			tdb_printfreelist(tdb);
+			return 0;
+#endif
+		case CMD_INFO:
+			info_tdb();
+			return 0;
+		case CMD_SPEED:
+			speed_tdb(arg1);
+			return 0;
+		case CMD_MMAP:
+			toggle_mmap();
+			return 0;
+		case CMD_FIRST:
+			bIterate = 1;
+			first_record(tdb, &iterate_kbuf);
+			return 0;
+		case CMD_NEXT:
+			if (bIterate)
+				next_record(tdb, &iterate_kbuf);
+			return 0;
+		case CMD_CHECK:
+			check_db(tdb);
+			return 0;
+		case CMD_HELP:
+			help();
+			return 0;
+		case CMD_CREATE_TDB:
+		case CMD_OPEN_TDB:
+		case CMD_SYSTEM:
+		case CMD_QUIT:
+			/*
+			 * unhandled commands.  cases included here to avoid compiler
+			 * warnings.
+			 */
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+static char *convert_string(char *instring, size_t *sizep)
+{
+	size_t length = 0;
+	char *outp, *inp;
+	char temp[3];
+
+	outp = inp = instring;
+
+	while (*inp) {
+		if (*inp == '\\') {
+			inp++;
+			if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+				temp[0] = *inp++;
+				temp[1] = '\0';
+				if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+					temp[1] = *inp++;
+					temp[2] = '\0';
+				}
+				*outp++ = (char)strtol((const char *)temp,NULL,16);
+			} else {
+				*outp++ = *inp++;
+			}
+		} else {
+			*outp++ = *inp++;
+		}
+		length++;
+	}
+	*sizep = length;
+	return instring;
+}
+
+int main(int argc, char *argv[])
+{
+	cmdname = "";
+	arg1 = NULL;
+	arg1len = 0;
+	arg2 = NULL;
+	arg2len = 0;
+
+	if (argv[1]) {
+		cmdname = "open";
+		arg1 = argv[1];
+		do_command();
+		cmdname =  "";
+		arg1 = NULL;
+	}
+
+	switch (argc) {
+	case 1:
+	case 2:
+		/* Interactive mode */
+		while ((cmdname = tdb_getline("tdb> "))) {
+			arg2 = arg1 = NULL;
+			if ((arg1 = strchr((const char *)cmdname,' ')) != NULL) {
+				arg1++;
+				arg2 = arg1;
+				while (*arg2) {
+					if (*arg2 == ' ') {
+						*arg2++ = '\0';
+						break;
+					}
+					if ((*arg2++ == '\\') && (*arg2 == ' ')) {
+						arg2++;
+					}
+				}
+			}
+			if (arg1) arg1 = convert_string(arg1,&arg1len);
+			if (arg2) arg2 = convert_string(arg2,&arg2len);
+			if (do_command()) break;
+		}
+		break;
+	case 5:
+		arg2 = convert_string(argv[4],&arg2len);
+	case 4:
+		arg1 = convert_string(argv[3],&arg1len);
+	case 3:
+		cmdname = argv[2];
+	default:
+		do_command();
+		break;
+	}
+
+	if (tdb) tdb_close(tdb);
+
+	return 0;
+}
diff --git a/lib/tdb2/tools/tdb2torture.c b/lib/tdb2/tools/tdb2torture.c
new file mode 100644
index 00000000000..f6a7a5064a3
--- /dev/null
+++ b/lib/tdb2/tools/tdb2torture.c
@@ -0,0 +1,494 @@
+/* this tests tdb by doing lots of ops from several simultaneous
+   writers - that stresses the locking code.
+*/
+
+#include "tdb2.h"
+#include <stdlib.h>
+#include <err.h>
+#include <getopt.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <time.h>
+#include <sys/wait.h>
+
+//#define REOPEN_PROB 30
+#define DELETE_PROB 8
+#define STORE_PROB 4
+#define APPEND_PROB 6
+#define TRANSACTION_PROB 10
+#define TRANSACTION_PREPARE_PROB 2
+#define LOCKSTORE_PROB 5
+#define TRAVERSE_PROB 20
+#define TRAVERSE_MOD_PROB 100
+#define TRAVERSE_ABORT_PROB 500
+#define CULL_PROB 100
+#define KEYLEN 3
+#define DATALEN 100
+
+static struct tdb_context *db;
+static int in_transaction;
+static int in_traverse;
+static int error_count;
+#if TRANSACTION_PROB
+static int always_transaction = 0;
+#endif
+static int loopnum;
+static int count_pipe;
+static union tdb_attribute log_attr;
+static union tdb_attribute seed_attr;
+
+static void tdb_log(struct tdb_context *tdb, enum tdb_log_level level,
+		    const char *message, void *data)
+{
+	fputs(message, stdout);
+	fflush(stdout);
+#if 0
+	{
+		char str[200];
+		signal(SIGUSR1, SIG_IGN);
+		sprintf(str,"xterm -e gdb /proc/%d/exe %d", getpid(), getpid());
+		system(str);
+	}
+#endif
+}
+
+#include "../private.h"
+
+static void segv_handler(int sig, siginfo_t *info, void *p)
+{
+	char string[100];
+
+	sprintf(string, "%u: death at %p (map_ptr %p, map_size %zu)\n",
+		getpid(), info->si_addr, db->file->map_ptr,
+		(size_t)db->file->map_size);
+	if (write(2, string, strlen(string)) > 0)
+		sleep(60);
+	_exit(11);
+}
+
+static void fatal(struct tdb_context *tdb, const char *why)
+{
+	fprintf(stderr, "%u:%s:%s\n", getpid(), why,
+		tdb ? tdb_errorstr(tdb_error(tdb)) : "(no tdb)");
+	error_count++;
+}
+
+static char *randbuf(int len)
+{
+	char *buf;
+	int i;
+	buf = (char *)malloc(len+1);
+
+	for (i=0;i<len;i++) {
+		buf[i] = 'a' + (rand() % 26);
+	}
+	buf[i] = 0;
+	return buf;
+}
+
+static void addrec_db(void);
+static int modify_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf,
+			   void *state)
+{
+#if CULL_PROB
+	if (random() % CULL_PROB == 0) {
+		tdb_delete(tdb, key);
+	}
+#endif
+
+#if TRAVERSE_MOD_PROB
+	if (random() % TRAVERSE_MOD_PROB == 0) {
+		addrec_db();
+	}
+#endif
+
+#if TRAVERSE_ABORT_PROB
+	if (random() % TRAVERSE_ABORT_PROB == 0)
+		return 1;
+#endif
+
+	return 0;
+}
+
+static void addrec_db(void)
+{
+	int klen, dlen;
+	char *k, *d;
+	TDB_DATA key, data;
+
+	klen = 1 + (rand() % KEYLEN);
+	dlen = 1 + (rand() % DATALEN);
+
+	k = randbuf(klen);
+	d = randbuf(dlen);
+
+	key.dptr = (unsigned char *)k;
+	key.dsize = klen+1;
+
+	data.dptr = (unsigned char *)d;
+	data.dsize = dlen+1;
+
+#if REOPEN_PROB
+	if (in_traverse == 0 && in_transaction == 0 && random() % REOPEN_PROB == 0) {
+		tdb_reopen_all(0);
+		goto next;
+	}
+#endif
+
+#if TRANSACTION_PROB
+	if (in_traverse == 0 && in_transaction == 0 && (always_transaction || random() % TRANSACTION_PROB == 0)) {
+		if (tdb_transaction_start(db) != 0) {
+			fatal(db, "tdb_transaction_start failed");
+		}
+		in_transaction++;
+		goto next;
+	}
+	if (in_traverse == 0 && in_transaction && random() % TRANSACTION_PROB == 0) {
+		if (random() % TRANSACTION_PREPARE_PROB == 0) {
+			if (tdb_transaction_prepare_commit(db) != 0) {
+				fatal(db, "tdb_transaction_prepare_commit failed");
+			}
+		}
+		if (tdb_transaction_commit(db) != 0) {
+			fatal(db, "tdb_transaction_commit failed");
+		}
+		in_transaction--;
+		goto next;
+	}
+
+	if (in_traverse == 0 && in_transaction && random() % TRANSACTION_PROB == 0) {
+		tdb_transaction_cancel(db);
+		in_transaction--;
+		goto next;
+	}
+#endif
+
+#if DELETE_PROB
+	if (random() % DELETE_PROB == 0) {
+		tdb_delete(db, key);
+		goto next;
+	}
+#endif
+
+#if STORE_PROB
+	if (random() % STORE_PROB == 0) {
+		if (tdb_store(db, key, data, TDB_REPLACE) != 0) {
+			fatal(db, "tdb_store failed");
+		}
+		goto next;
+	}
+#endif
+
+#if APPEND_PROB
+	if (random() % APPEND_PROB == 0) {
+		if (tdb_append(db, key, data) != 0) {
+			fatal(db, "tdb_append failed");
+		}
+		goto next;
+	}
+#endif
+
+#if LOCKSTORE_PROB
+	if (random() % LOCKSTORE_PROB == 0) {
+		tdb_chainlock(db, key);
+		if (tdb_fetch(db, key, &data) != TDB_SUCCESS) {
+			data.dsize = 0;
+			data.dptr = NULL;
+		}
+		if (tdb_store(db, key, data, TDB_REPLACE) != 0) {
+			fatal(db, "tdb_store failed");
+		}
+		if (data.dptr) free(data.dptr);
+		tdb_chainunlock(db, key);
+		goto next;
+	}
+#endif
+
+#if TRAVERSE_PROB
+	/* FIXME: recursive traverses break transactions? */
+	if (in_traverse == 0 && random() % TRAVERSE_PROB == 0) {
+		in_traverse++;
+		tdb_traverse(db, modify_traverse, NULL);
+		in_traverse--;
+		goto next;
+	}
+#endif
+
+	if (tdb_fetch(db, key, &data) == TDB_SUCCESS)
+		free(data.dptr);
+
+next:
+	free(k);
+	free(d);
+}
+
+static int traverse_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf,
+                       void *state)
+{
+	tdb_delete(tdb, key);
+	return 0;
+}
+
+static void usage(void)
+{
+	printf("Usage: tdbtorture"
+#if TRANSACTION_PROB
+	       " [-t]"
+#endif
+	       " [-k] [-n NUM_PROCS] [-l NUM_LOOPS] [-s SEED] [-S]\n");
+	exit(0);
+}
+
+static void send_count_and_suicide(int sig)
+{
+	/* This ensures our successor can continue where we left off. */
+	if (write(count_pipe, &loopnum, sizeof(loopnum)) != sizeof(loopnum))
+		exit(2);
+	/* This gives a unique signature. */
+	kill(getpid(), SIGUSR2);
+}
+
+static int run_child(int i, int seed, unsigned num_loops, unsigned start,
+		     int tdb_flags)
+{
+	struct sigaction act = { .sa_sigaction = segv_handler,
+				 .sa_flags = SA_SIGINFO };
+	sigaction(11, &act, NULL);
+
+	db = tdb_open("torture.tdb", tdb_flags, O_RDWR | O_CREAT, 0600,
+		      &log_attr);
+	if (!db) {
+		fatal(NULL, "db open failed");
+	}
+
+#if 0
+	if (i == 0) {
+		printf("pid %i\n", getpid());
+		sleep(9);
+	} else
+		sleep(10);
+#endif
+
+	srand(seed + i);
+	srandom(seed + i);
+
+	/* Set global, then we're ready to handle being killed. */
+	loopnum = start;
+	signal(SIGUSR1, send_count_and_suicide);
+
+	for (;loopnum<num_loops && error_count == 0;loopnum++) {
+		addrec_db();
+	}
+
+	if (error_count == 0) {
+		tdb_traverse(db, NULL, NULL);
+#if TRANSACTION_PROB
+		if (always_transaction) {
+			while (in_transaction) {
+				tdb_transaction_cancel(db);
+				in_transaction--;
+			}
+			if (tdb_transaction_start(db) != 0)
+				fatal(db, "tdb_transaction_start failed");
+		}
+#endif
+		tdb_traverse(db, traverse_fn, NULL);
+		tdb_traverse(db, traverse_fn, NULL);
+
+#if TRANSACTION_PROB
+		if (always_transaction) {
+			if (tdb_transaction_commit(db) != 0)
+				fatal(db, "tdb_transaction_commit failed");
+		}
+#endif
+	}
+
+	tdb_close(db);
+
+	return (error_count < 100 ? error_count : 100);
+}
+
+int main(int argc, char * const *argv)
+{
+	int i, seed = -1;
+	int num_loops = 5000;
+	int num_procs = 3;
+	int c, pfds[2];
+	extern char *optarg;
+	pid_t *pids;
+	int kill_random = 0;
+	int *done;
+	int tdb_flags = TDB_DEFAULT;
+
+	log_attr.base.attr = TDB_ATTRIBUTE_LOG;
+	log_attr.base.next = &seed_attr;
+	log_attr.log.fn = tdb_log;
+	seed_attr.base.attr = TDB_ATTRIBUTE_SEED;
+
+	while ((c = getopt(argc, argv, "n:l:s:thkS")) != -1) {
+		switch (c) {
+		case 'n':
+			num_procs = strtol(optarg, NULL, 0);
+			break;
+		case 'l':
+			num_loops = strtol(optarg, NULL, 0);
+			break;
+		case 's':
+			seed = strtol(optarg, NULL, 0);
+			break;
+		case 'S':
+			tdb_flags = TDB_NOSYNC;
+			break;
+		case 't':
+#if TRANSACTION_PROB
+			always_transaction = 1;
+#else
+			fprintf(stderr, "Transactions not supported\n");
+			usage();
+#endif
+			break;
+		case 'k':
+			kill_random = 1;
+			break;
+		default:
+			usage();
+		}
+	}
+
+	unlink("torture.tdb");
+
+	if (seed == -1) {
+		seed = (getpid() + time(NULL)) & 0x7FFFFFFF;
+	}
+	seed_attr.seed.seed = (((uint64_t)seed) << 32) | seed;
+
+	if (num_procs == 1 && !kill_random) {
+		/* Don't fork for this case, makes debugging easier. */
+		error_count = run_child(0, seed, num_loops, 0, tdb_flags);
+		goto done;
+	}
+
+	pids = (pid_t *)calloc(sizeof(pid_t), num_procs);
+	done = (int *)calloc(sizeof(int), num_procs);
+
+	if (pipe(pfds) != 0) {
+		perror("Creating pipe");
+		exit(1);
+	}
+	count_pipe = pfds[1];
+
+	for (i=0;i<num_procs;i++) {
+		if ((pids[i]=fork()) == 0) {
+			close(pfds[0]);
+			if (i == 0) {
+				printf("testing with %d processes, %d loops, seed=%d%s\n",
+				       num_procs, num_loops, seed,
+#if TRANSACTION_PROB
+				       always_transaction ? " (all within transactions)" : ""
+#else
+				       ""
+#endif
+					);
+			}
+			exit(run_child(i, seed, num_loops, 0, tdb_flags));
+		}
+	}
+
+	while (num_procs) {
+		int status, j;
+		pid_t pid;
+
+		if (error_count != 0) {
+			/* try and stop the test on any failure */
+			for (j=0;j<num_procs;j++) {
+				if (pids[j] != 0) {
+					kill(pids[j], SIGTERM);
+				}
+			}
+		}
+
+		pid = waitpid(-1, &status, kill_random ? WNOHANG : 0);
+		if (pid == 0) {
+			struct timespec ts;
+
+			/* Sleep for 1/10 second. */
+			ts.tv_sec = 0;
+			ts.tv_nsec = 100000000;
+			nanosleep(&ts, NULL);
+
+			/* Kill someone. */
+			kill(pids[random() % num_procs], SIGUSR1);
+			continue;
+		}
+
+		if (pid == -1) {
+			perror("failed to wait for child\n");
+			exit(1);
+		}
+
+		for (j=0;j<num_procs;j++) {
+			if (pids[j] == pid) break;
+		}
+		if (j == num_procs) {
+			printf("unknown child %d exited!?\n", (int)pid);
+			exit(1);
+		}
+		if (WIFSIGNALED(status)) {
+			if (WTERMSIG(status) == SIGUSR2
+			    || WTERMSIG(status) == SIGUSR1) {
+				/* SIGUSR2 means they wrote to pipe. */
+				if (WTERMSIG(status) == SIGUSR2) {
+					if (read(pfds[0], &done[j],
+						 sizeof(done[j]))
+					    != sizeof(done[j]))
+						err(1,
+						    "Short read from child?");
+				}
+				pids[j] = fork();
+				if (pids[j] == 0)
+					exit(run_child(j, seed, num_loops,
+						       done[j], tdb_flags));
+				printf("Restarting child %i for %u-%u\n",
+				       j, done[j], num_loops);
+				continue;
+			}
+			printf("child %d exited with signal %d\n",
+			       (int)pid, WTERMSIG(status));
+			error_count++;
+		} else {
+			if (WEXITSTATUS(status) != 0) {
+				printf("child %d exited with status %d\n",
+				       (int)pid, WEXITSTATUS(status));
+				error_count++;
+			}
+		}
+		memmove(&pids[j], &pids[j+1],
+			(num_procs - j - 1)*sizeof(pids[0]));
+		num_procs--;
+	}
+
+	free(pids);
+
+done:
+	if (error_count == 0) {
+		db = tdb_open("torture.tdb", TDB_DEFAULT, O_RDWR | O_CREAT,
+			      0600, &log_attr);
+		if (!db) {
+			fatal(db, "db open failed");
+			exit(1);
+		}
+		if (tdb_check(db, NULL, NULL) != 0) {
+			fatal(db, "db check failed");
+			exit(1);
+		}
+		tdb_close(db);
+		printf("OK\n");
+	}
+
+	return error_count;
+}
diff --git a/lib/tdb2/transaction.c b/lib/tdb2/transaction.c
new file mode 100644
index 00000000000..b13223bc2e1
--- /dev/null
+++ b/lib/tdb2/transaction.c
@@ -0,0 +1,1308 @@
+ /*
+   Unix SMB/CIFS implementation.
+
+   trivial database library
+
+   Copyright (C) Andrew Tridgell              2005
+   Copyright (C) Rusty Russell                2010
+
+     ** NOTE! The following LGPL license applies to the tdb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "private.h"
+#define SAFE_FREE(x) do { if ((x) != NULL) {free(x); (x)=NULL;} } while(0)
+
+/*
+  transaction design:
+
+  - only allow a single transaction at a time per database. This makes
+    using the transaction API simpler, as otherwise the caller would
+    have to cope with temporary failures in transactions that conflict
+    with other current transactions
+
+  - keep the transaction recovery information in the same file as the
+    database, using a special 'transaction recovery' record pointed at
+    by the header. This removes the need for extra journal files as
+    used by some other databases
+
+  - dynamically allocated the transaction recover record, re-using it
+    for subsequent transactions. If a larger record is needed then
+    tdb_free() the old record to place it on the normal tdb freelist
+    before allocating the new record
+
+  - during transactions, keep a linked list of writes all that have
+    been performed by intercepting all tdb_write() calls. The hooked
+    transaction versions of tdb_read() and tdb_write() check this
+    linked list and try to use the elements of the list in preference
+    to the real database.
+
+  - don't allow any locks to be held when a transaction starts,
+    otherwise we can end up with deadlock (plus lack of lock nesting
+    in POSIX locks would mean the lock is lost)
+
+  - if the caller gains a lock during the transaction but doesn't
+    release it then fail the commit
+
+  - allow for nested calls to tdb_transaction_start(), re-using the
+    existing transaction record. If the inner transaction is canceled
+    then a subsequent commit will fail
+
+  - keep a mirrored copy of the tdb hash chain heads to allow for the
+    fast hash heads scan on traverse, updating the mirrored copy in
+    the transaction version of tdb_write
+
+  - allow callers to mix transaction and non-transaction use of tdb,
+    although once a transaction is started then an exclusive lock is
+    gained until the transaction is committed or canceled
+
+  - the commit stategy involves first saving away all modified data
+    into a linearised buffer in the transaction recovery area, then
+    marking the transaction recovery area with a magic value to
+    indicate a valid recovery record. In total 4 fsync/msync calls are
+    needed per commit to prevent race conditions. It might be possible
+    to reduce this to 3 or even 2 with some more work.
+
+  - check for a valid recovery record on open of the tdb, while the
+    open lock is held. Automatically recover from the transaction
+    recovery area if needed, then continue with the open as
+    usual. This allows for smooth crash recovery with no administrator
+    intervention.
+
+  - if TDB_NOSYNC is passed to flags in tdb_open then transactions are
+    still available, but no transaction recovery area is used and no
+    fsync/msync calls are made.
+*/
+
+/*
+  hold the context of any current transaction
+*/
+struct tdb_transaction {
+	/* the original io methods - used to do IOs to the real db */
+	const struct tdb_methods *io_methods;
+
+	/* the list of transaction blocks. When a block is first
+	   written to, it gets created in this list */
+	uint8_t **blocks;
+	size_t num_blocks;
+	size_t last_block_size; /* number of valid bytes in the last block */
+
+	/* non-zero when an internal transaction error has
+	   occurred. All write operations will then fail until the
+	   transaction is ended */
+	int transaction_error;
+
+	/* when inside a transaction we need to keep track of any
+	   nested tdb_transaction_start() calls, as these are allowed,
+	   but don't create a new transaction */
+	unsigned int nesting;
+
+	/* set when a prepare has already occurred */
+	bool prepared;
+	tdb_off_t magic_offset;
+
+	/* old file size before transaction */
+	tdb_len_t old_map_size;
+};
+
+/* This doesn't really need to be pagesize, but we use it for similar reasons. */
+#define PAGESIZE 65536
+
+/*
+  read while in a transaction. We need to check first if the data is in our list
+  of transaction elements, then if not do a real read
+*/
+static enum TDB_ERROR transaction_read(struct tdb_context *tdb, tdb_off_t off,
+				       void *buf, tdb_len_t len)
+{
+	size_t blk;
+	enum TDB_ERROR ecode;
+
+	/* break it down into block sized ops */
+	while (len + (off % PAGESIZE) > PAGESIZE) {
+		tdb_len_t len2 = PAGESIZE - (off % PAGESIZE);
+		ecode = transaction_read(tdb, off, buf, len2);
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+		len -= len2;
+		off += len2;
+		buf = (void *)(len2 + (char *)buf);
+	}
+
+	if (len == 0) {
+		return TDB_SUCCESS;
+	}
+
+	blk = off / PAGESIZE;
+
+	/* see if we have it in the block list */
+	if (tdb->transaction->num_blocks <= blk ||
+	    tdb->transaction->blocks[blk] == NULL) {
+		/* nope, do a real read */
+		ecode = tdb->transaction->io_methods->tread(tdb, off, buf, len);
+		if (ecode != TDB_SUCCESS) {
+			goto fail;
+		}
+		return 0;
+	}
+
+	/* it is in the block list. Now check for the last block */
+	if (blk == tdb->transaction->num_blocks-1) {
+		if (len > tdb->transaction->last_block_size) {
+			ecode = TDB_ERR_IO;
+			goto fail;
+		}
+	}
+
+	/* now copy it out of this block */
+	memcpy(buf, tdb->transaction->blocks[blk] + (off % PAGESIZE), len);
+	return TDB_SUCCESS;
+
+fail:
+	tdb->transaction->transaction_error = 1;
+	return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+			  "transaction_read: failed at off=%zu len=%zu",
+			  (size_t)off, (size_t)len);
+}
+
+
+/*
+  write while in a transaction
+*/
+static enum TDB_ERROR transaction_write(struct tdb_context *tdb, tdb_off_t off,
+					const void *buf, tdb_len_t len)
+{
+	size_t blk;
+	enum TDB_ERROR ecode;
+
+	/* Only a commit is allowed on a prepared transaction */
+	if (tdb->transaction->prepared) {
+		ecode = tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_ERROR,
+				   "transaction_write: transaction already"
+				   " prepared, write not allowed");
+		goto fail;
+	}
+
+	/* break it up into block sized chunks */
+	while (len + (off % PAGESIZE) > PAGESIZE) {
+		tdb_len_t len2 = PAGESIZE - (off % PAGESIZE);
+		ecode = transaction_write(tdb, off, buf, len2);
+		if (ecode != TDB_SUCCESS) {
+			return -1;
+		}
+		len -= len2;
+		off += len2;
+		if (buf != NULL) {
+			buf = (const void *)(len2 + (const char *)buf);
+		}
+	}
+
+	if (len == 0) {
+		return TDB_SUCCESS;
+	}
+
+	blk = off / PAGESIZE;
+	off = off % PAGESIZE;
+
+	if (tdb->transaction->num_blocks <= blk) {
+		uint8_t **new_blocks;
+		/* expand the blocks array */
+		if (tdb->transaction->blocks == NULL) {
+			new_blocks = (uint8_t **)malloc(
+				(blk+1)*sizeof(uint8_t *));
+		} else {
+			new_blocks = (uint8_t **)realloc(
+				tdb->transaction->blocks,
+				(blk+1)*sizeof(uint8_t *));
+		}
+		if (new_blocks == NULL) {
+			ecode = tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+					   "transaction_write:"
+					   " failed to allocate");
+			goto fail;
+		}
+		memset(&new_blocks[tdb->transaction->num_blocks], 0,
+		       (1+(blk - tdb->transaction->num_blocks))*sizeof(uint8_t *));
+		tdb->transaction->blocks = new_blocks;
+		tdb->transaction->num_blocks = blk+1;
+		tdb->transaction->last_block_size = 0;
+	}
+
+	/* allocate and fill a block? */
+	if (tdb->transaction->blocks[blk] == NULL) {
+		tdb->transaction->blocks[blk] = (uint8_t *)calloc(PAGESIZE, 1);
+		if (tdb->transaction->blocks[blk] == NULL) {
+			ecode = tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+					   "transaction_write:"
+					   " failed to allocate");
+			goto fail;
+		}
+		if (tdb->transaction->old_map_size > blk * PAGESIZE) {
+			tdb_len_t len2 = PAGESIZE;
+			if (len2 + (blk * PAGESIZE) > tdb->transaction->old_map_size) {
+				len2 = tdb->transaction->old_map_size - (blk * PAGESIZE);
+			}
+			ecode = tdb->transaction->io_methods->tread(tdb,
+					blk * PAGESIZE,
+					tdb->transaction->blocks[blk],
+					len2);
+			if (ecode != TDB_SUCCESS) {
+				ecode = tdb_logerr(tdb, ecode,
+						   TDB_LOG_ERROR,
+						   "transaction_write:"
+						   " failed to"
+						   " read old block: %s",
+						   strerror(errno));
+				SAFE_FREE(tdb->transaction->blocks[blk]);
+				goto fail;
+			}
+			if (blk == tdb->transaction->num_blocks-1) {
+				tdb->transaction->last_block_size = len2;
+			}
+		}
+	}
+
+	/* overwrite part of an existing block */
+	if (buf == NULL) {
+		memset(tdb->transaction->blocks[blk] + off, 0, len);
+	} else {
+		memcpy(tdb->transaction->blocks[blk] + off, buf, len);
+	}
+	if (blk == tdb->transaction->num_blocks-1) {
+		if (len + off > tdb->transaction->last_block_size) {
+			tdb->transaction->last_block_size = len + off;
+		}
+	}
+
+	return TDB_SUCCESS;
+
+fail:
+	tdb->transaction->transaction_error = 1;
+	return ecode;
+}
+
+
+/*
+  write while in a transaction - this variant never expands the transaction blocks, it only
+  updates existing blocks. This means it cannot change the recovery size
+*/
+static void transaction_write_existing(struct tdb_context *tdb, tdb_off_t off,
+				       const void *buf, tdb_len_t len)
+{
+	size_t blk;
+
+	/* break it up into block sized chunks */
+	while (len + (off % PAGESIZE) > PAGESIZE) {
+		tdb_len_t len2 = PAGESIZE - (off % PAGESIZE);
+		transaction_write_existing(tdb, off, buf, len2);
+		len -= len2;
+		off += len2;
+		if (buf != NULL) {
+			buf = (const void *)(len2 + (const char *)buf);
+		}
+	}
+
+	if (len == 0) {
+		return;
+	}
+
+	blk = off / PAGESIZE;
+	off = off % PAGESIZE;
+
+	if (tdb->transaction->num_blocks <= blk ||
+	    tdb->transaction->blocks[blk] == NULL) {
+		return;
+	}
+
+	if (blk == tdb->transaction->num_blocks-1 &&
+	    off + len > tdb->transaction->last_block_size) {
+		if (off >= tdb->transaction->last_block_size) {
+			return;
+		}
+		len = tdb->transaction->last_block_size - off;
+	}
+
+	/* overwrite part of an existing block */
+	memcpy(tdb->transaction->blocks[blk] + off, buf, len);
+}
+
+
+/*
+  out of bounds check during a transaction
+*/
+static enum TDB_ERROR transaction_oob(struct tdb_context *tdb, tdb_off_t len,
+				      bool probe)
+{
+	if (len <= tdb->file->map_size) {
+		return TDB_SUCCESS;
+	}
+	if (!probe) {
+		tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+			   "tdb_oob len %lld beyond transaction size %lld",
+			   (long long)len,
+			   (long long)tdb->file->map_size);
+	}
+	return TDB_ERR_IO;
+}
+
+/*
+  transaction version of tdb_expand().
+*/
+static enum TDB_ERROR transaction_expand_file(struct tdb_context *tdb,
+					      tdb_off_t addition)
+{
+	enum TDB_ERROR ecode;
+
+	/* add a write to the transaction elements, so subsequent
+	   reads see the zero data */
+	ecode = transaction_write(tdb, tdb->file->map_size, NULL, addition);
+	if (ecode == TDB_SUCCESS) {
+		tdb->file->map_size += addition;
+	}
+	return ecode;
+}
+
+static void *transaction_direct(struct tdb_context *tdb, tdb_off_t off,
+				size_t len, bool write_mode)
+{
+	size_t blk = off / PAGESIZE, end_blk;
+
+	/* This is wrong for zero-length blocks, but will fail gracefully */
+	end_blk = (off + len - 1) / PAGESIZE;
+
+	/* Can only do direct if in single block and we've already copied. */
+	if (write_mode) {
+		tdb->stats.transaction_write_direct++;
+		if (blk != end_blk
+		    || blk >= tdb->transaction->num_blocks
+		    || tdb->transaction->blocks[blk] == NULL) {
+			tdb->stats.transaction_write_direct_fail++;
+			return NULL;
+		}
+		return tdb->transaction->blocks[blk] + off % PAGESIZE;
+	}
+
+	tdb->stats.transaction_read_direct++;
+	/* Single which we have copied? */
+	if (blk == end_blk
+	    && blk < tdb->transaction->num_blocks
+	    && tdb->transaction->blocks[blk])
+		return tdb->transaction->blocks[blk] + off % PAGESIZE;
+
+	/* Otherwise must be all not copied. */
+	while (blk <= end_blk) {
+		if (blk >= tdb->transaction->num_blocks)
+			break;
+		if (tdb->transaction->blocks[blk]) {
+			tdb->stats.transaction_read_direct_fail++;
+			return NULL;
+		}
+		blk++;
+	}
+	return tdb->transaction->io_methods->direct(tdb, off, len, false);
+}
+
+static const struct tdb_methods transaction_methods = {
+	transaction_read,
+	transaction_write,
+	transaction_oob,
+	transaction_expand_file,
+	transaction_direct,
+};
+
+/*
+  sync to disk
+*/
+static enum TDB_ERROR transaction_sync(struct tdb_context *tdb,
+				       tdb_off_t offset, tdb_len_t length)
+{
+	if (tdb->flags & TDB_NOSYNC) {
+		return TDB_SUCCESS;
+	}
+
+	if (fsync(tdb->file->fd) != 0) {
+		return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+				  "tdb_transaction: fsync failed: %s",
+				  strerror(errno));
+	}
+#ifdef MS_SYNC
+	if (tdb->file->map_ptr) {
+		tdb_off_t moffset = offset & ~(getpagesize()-1);
+		if (msync(moffset + (char *)tdb->file->map_ptr,
+			  length + (offset - moffset), MS_SYNC) != 0) {
+			return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
+					  "tdb_transaction: msync failed: %s",
+					  strerror(errno));
+		}
+	}
+#endif
+	return TDB_SUCCESS;
+}
+
+
+static void _tdb_transaction_cancel(struct tdb_context *tdb)
+{
+	int i;
+	enum TDB_ERROR ecode;
+
+	if (tdb->transaction == NULL) {
+		tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_USE_ERROR,
+			   "tdb_transaction_cancel: no transaction");
+		return;
+	}
+
+	if (tdb->transaction->nesting != 0) {
+		tdb->transaction->transaction_error = 1;
+		tdb->transaction->nesting--;
+		return;
+	}
+
+	tdb->file->map_size = tdb->transaction->old_map_size;
+
+	/* free all the transaction blocks */
+	for (i=0;i<tdb->transaction->num_blocks;i++) {
+		if (tdb->transaction->blocks[i] != NULL) {
+			free(tdb->transaction->blocks[i]);
+		}
+	}
+	SAFE_FREE(tdb->transaction->blocks);
+
+	if (tdb->transaction->magic_offset) {
+		const struct tdb_methods *methods = tdb->transaction->io_methods;
+		uint64_t invalid = TDB_RECOVERY_INVALID_MAGIC;
+
+		/* remove the recovery marker */
+		ecode = methods->twrite(tdb, tdb->transaction->magic_offset,
+					&invalid, sizeof(invalid));
+		if (ecode == TDB_SUCCESS)
+			ecode = transaction_sync(tdb,
+						 tdb->transaction->magic_offset,
+						 sizeof(invalid));
+		if (ecode != TDB_SUCCESS) {
+			tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				   "tdb_transaction_cancel: failed to remove"
+				   " recovery magic");
+		}
+	}
+
+	if (tdb->file->allrecord_lock.count)
+		tdb_allrecord_unlock(tdb, tdb->file->allrecord_lock.ltype);
+
+	/* restore the normal io methods */
+	tdb->methods = tdb->transaction->io_methods;
+
+	tdb_transaction_unlock(tdb, F_WRLCK);
+
+	if (tdb_has_open_lock(tdb))
+		tdb_unlock_open(tdb, F_WRLCK);
+
+	SAFE_FREE(tdb->transaction);
+}
+
+/*
+  start a tdb transaction. No token is returned, as only a single
+  transaction is allowed to be pending per tdb_context
+*/
+enum TDB_ERROR tdb_transaction_start(struct tdb_context *tdb)
+{
+	enum TDB_ERROR ecode;
+
+	tdb->stats.transactions++;
+	/* some sanity checks */
+	if (tdb->read_only || (tdb->flags & TDB_INTERNAL)) {
+		return tdb->last_error = tdb_logerr(tdb, TDB_ERR_EINVAL,
+						    TDB_LOG_USE_ERROR,
+						    "tdb_transaction_start:"
+						    " cannot start a"
+						    " transaction on a "
+						    "read-only or internal db");
+	}
+
+	/* cope with nested tdb_transaction_start() calls */
+	if (tdb->transaction != NULL) {
+		if (!(tdb->flags & TDB_ALLOW_NESTING)) {
+			return tdb->last_error
+				= tdb_logerr(tdb, TDB_ERR_IO,
+					     TDB_LOG_USE_ERROR,
+					     "tdb_transaction_start:"
+					     " already inside transaction");
+		}
+		tdb->transaction->nesting++;
+		tdb->stats.transaction_nest++;
+		return 0;
+	}
+
+	if (tdb_has_hash_locks(tdb)) {
+		/* the caller must not have any locks when starting a
+		   transaction as otherwise we'll be screwed by lack
+		   of nested locks in POSIX */
+		return tdb->last_error = tdb_logerr(tdb, TDB_ERR_LOCK,
+						    TDB_LOG_USE_ERROR,
+						    "tdb_transaction_start:"
+						    " cannot start a"
+						    " transaction with locks"
+						    " held");
+	}
+
+	tdb->transaction = (struct tdb_transaction *)
+		calloc(sizeof(struct tdb_transaction), 1);
+	if (tdb->transaction == NULL) {
+		return tdb->last_error = tdb_logerr(tdb, TDB_ERR_OOM,
+						    TDB_LOG_ERROR,
+						    "tdb_transaction_start:"
+						    " cannot allocate");
+	}
+
+	/* get the transaction write lock. This is a blocking lock. As
+	   discussed with Volker, there are a number of ways we could
+	   make this async, which we will probably do in the future */
+	ecode = tdb_transaction_lock(tdb, F_WRLCK);
+	if (ecode != TDB_SUCCESS) {
+		SAFE_FREE(tdb->transaction->blocks);
+		SAFE_FREE(tdb->transaction);
+		return tdb->last_error = ecode;
+	}
+
+	/* get a read lock over entire file. This is upgraded to a write
+	   lock during the commit */
+	ecode = tdb_allrecord_lock(tdb, F_RDLCK, TDB_LOCK_WAIT, true);
+	if (ecode != TDB_SUCCESS) {
+		goto fail_allrecord_lock;
+	}
+
+	/* make sure we know about any file expansions already done by
+	   anyone else */
+	tdb->methods->oob(tdb, tdb->file->map_size + 1, true);
+	tdb->transaction->old_map_size = tdb->file->map_size;
+
+	/* finally hook the io methods, replacing them with
+	   transaction specific methods */
+	tdb->transaction->io_methods = tdb->methods;
+	tdb->methods = &transaction_methods;
+	return tdb->last_error = TDB_SUCCESS;
+
+fail_allrecord_lock:
+	tdb_transaction_unlock(tdb, F_WRLCK);
+	SAFE_FREE(tdb->transaction->blocks);
+	SAFE_FREE(tdb->transaction);
+	return tdb->last_error = ecode;
+}
+
+
+/*
+  cancel the current transaction
+*/
+void tdb_transaction_cancel(struct tdb_context *tdb)
+{
+	tdb->stats.transaction_cancel++;
+	_tdb_transaction_cancel(tdb);
+}
+
+/*
+  work out how much space the linearised recovery data will consume (worst case)
+*/
+static tdb_len_t tdb_recovery_size(struct tdb_context *tdb)
+{
+	tdb_len_t recovery_size = 0;
+	int i;
+
+	recovery_size = 0;
+	for (i=0;i<tdb->transaction->num_blocks;i++) {
+		if (i * PAGESIZE >= tdb->transaction->old_map_size) {
+			break;
+		}
+		if (tdb->transaction->blocks[i] == NULL) {
+			continue;
+		}
+		recovery_size += 2*sizeof(tdb_off_t);
+		if (i == tdb->transaction->num_blocks-1) {
+			recovery_size += tdb->transaction->last_block_size;
+		} else {
+			recovery_size += PAGESIZE;
+		}
+	}
+
+	return recovery_size;
+}
+
+static enum TDB_ERROR tdb_recovery_area(struct tdb_context *tdb,
+					const struct tdb_methods *methods,
+					tdb_off_t *recovery_offset,
+					struct tdb_recovery_record *rec)
+{
+	enum TDB_ERROR ecode;
+
+	*recovery_offset = tdb_read_off(tdb,
+					offsetof(struct tdb_header, recovery));
+	if (TDB_OFF_IS_ERR(*recovery_offset)) {
+		return *recovery_offset;
+	}
+
+	if (*recovery_offset == 0) {
+		rec->max_len = 0;
+		return TDB_SUCCESS;
+	}
+
+	ecode = methods->tread(tdb, *recovery_offset, rec, sizeof(*rec));
+	if (ecode != TDB_SUCCESS)
+		return ecode;
+
+	tdb_convert(tdb, rec, sizeof(*rec));
+	/* ignore invalid recovery regions: can happen in crash */
+	if (rec->magic != TDB_RECOVERY_MAGIC &&
+	    rec->magic != TDB_RECOVERY_INVALID_MAGIC) {
+		*recovery_offset = 0;
+		rec->max_len = 0;
+	}
+	return TDB_SUCCESS;
+}
+
+static unsigned int same(const unsigned char *new,
+			 const unsigned char *old,
+			 unsigned int length)
+{
+	unsigned int i;
+
+	for (i = 0; i < length; i++) {
+		if (new[i] != old[i])
+			break;
+	}
+	return i;
+}
+
+static unsigned int different(const unsigned char *new,
+			      const unsigned char *old,
+			      unsigned int length,
+			      unsigned int min_same,
+			      unsigned int *samelen)
+{
+	unsigned int i;
+
+	*samelen = 0;
+	for (i = 0; i < length; i++) {
+		if (new[i] == old[i]) {
+			(*samelen)++;
+		} else {
+			if (*samelen >= min_same) {
+				return i - *samelen;
+			}
+			*samelen = 0;
+		}
+	}
+
+	if (*samelen < min_same)
+		*samelen = 0;
+	return length - *samelen;
+}
+
+/* Allocates recovery blob, without tdb_recovery_record at head set up. */
+static struct tdb_recovery_record *alloc_recovery(struct tdb_context *tdb,
+						  tdb_len_t *len)
+{
+	struct tdb_recovery_record *rec;
+	size_t i;
+	enum TDB_ERROR ecode;
+	unsigned char *p;
+	const struct tdb_methods *old_methods = tdb->methods;
+
+	rec = malloc(sizeof(*rec) + tdb_recovery_size(tdb));
+	if (!rec) {
+		tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+			   "transaction_setup_recovery:"
+			   " cannot allocate");
+		return TDB_ERR_PTR(TDB_ERR_OOM);
+	}
+
+	/* We temporarily revert to the old I/O methods, so we can use
+	 * tdb_access_read */
+	tdb->methods = tdb->transaction->io_methods;
+
+	/* build the recovery data into a single blob to allow us to do a single
+	   large write, which should be more efficient */
+	p = (unsigned char *)(rec + 1);
+	for (i=0;i<tdb->transaction->num_blocks;i++) {
+		tdb_off_t offset;
+		tdb_len_t length;
+		unsigned int off;
+		const unsigned char *buffer;
+
+		if (tdb->transaction->blocks[i] == NULL) {
+			continue;
+		}
+
+		offset = i * PAGESIZE;
+		length = PAGESIZE;
+		if (i == tdb->transaction->num_blocks-1) {
+			length = tdb->transaction->last_block_size;
+		}
+
+		if (offset >= tdb->transaction->old_map_size) {
+			continue;
+		}
+
+		if (offset + length > tdb->file->map_size) {
+			ecode = tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+					   "tdb_transaction_setup_recovery:"
+					   " transaction data over new region"
+					   " boundary");
+			goto fail;
+		}
+		if (offset + length > tdb->transaction->old_map_size) {
+			/* Short read at EOF. */
+			length = tdb->transaction->old_map_size - offset;
+		}
+		buffer = tdb_access_read(tdb, offset, length, false);
+		if (TDB_PTR_IS_ERR(buffer)) {
+			ecode = TDB_PTR_ERR(buffer);
+			goto fail;
+		}
+
+		/* Skip over anything the same at the start. */
+		off = same(tdb->transaction->blocks[i], buffer, length);
+		offset += off;
+
+		while (off < length) {
+			tdb_len_t len;
+			unsigned int samelen;
+
+			len = different(tdb->transaction->blocks[i] + off,
+					buffer + off, length - off,
+					sizeof(offset) + sizeof(len) + 1,
+					&samelen);
+
+			memcpy(p, &offset, sizeof(offset));
+			memcpy(p + sizeof(offset), &len, sizeof(len));
+			tdb_convert(tdb, p, sizeof(offset) + sizeof(len));
+			p += sizeof(offset) + sizeof(len);
+			memcpy(p, buffer + off, len);
+			p += len;
+			off += len + samelen;
+			offset += len + samelen;
+		}
+		tdb_access_release(tdb, buffer);
+	}
+
+	*len = p - (unsigned char *)(rec + 1);
+	tdb->methods = old_methods;
+	return rec;
+
+fail:
+	free(rec);
+	tdb->methods = old_methods;
+	return TDB_ERR_PTR(ecode);
+}
+
+static tdb_off_t create_recovery_area(struct tdb_context *tdb,
+				      tdb_len_t rec_length,
+				      struct tdb_recovery_record *rec)
+{
+	tdb_off_t off, recovery_off;
+	tdb_len_t addition;
+	enum TDB_ERROR ecode;
+	const struct tdb_methods *methods = tdb->transaction->io_methods;
+
+	/* round up to a multiple of page size. Overallocate, since each
+	 * such allocation forces us to expand the file. */
+	rec->max_len
+		= (((sizeof(*rec) + rec_length + rec_length / 2)
+		    + PAGESIZE-1) & ~(PAGESIZE-1))
+		- sizeof(*rec);
+	off = tdb->file->map_size;
+
+	/* Restore ->map_size before calling underlying expand_file.
+	   Also so that we don't try to expand the file again in the
+	   transaction commit, which would destroy the recovery
+	   area */
+	addition = (tdb->file->map_size - tdb->transaction->old_map_size) +
+		sizeof(*rec) + rec->max_len;
+	tdb->file->map_size = tdb->transaction->old_map_size;
+	tdb->stats.transaction_expand_file++;
+	ecode = methods->expand_file(tdb, addition);
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_recovery_allocate:"
+				  " failed to create recovery area");
+	}
+
+	/* we have to reset the old map size so that we don't try to
+	   expand the file again in the transaction commit, which
+	   would destroy the recovery area */
+	tdb->transaction->old_map_size = tdb->file->map_size;
+
+	/* write the recovery header offset and sync - we can sync without a race here
+	   as the magic ptr in the recovery record has not been set */
+	recovery_off = off;
+	tdb_convert(tdb, &recovery_off, sizeof(recovery_off));
+	ecode = methods->twrite(tdb, offsetof(struct tdb_header, recovery),
+				&recovery_off, sizeof(tdb_off_t));
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_recovery_allocate:"
+				  " failed to write recovery head");
+	}
+	transaction_write_existing(tdb, offsetof(struct tdb_header, recovery),
+				   &recovery_off,
+				   sizeof(tdb_off_t));
+	return off;
+}
+
+/*
+  setup the recovery data that will be used on a crash during commit
+*/
+static enum TDB_ERROR transaction_setup_recovery(struct tdb_context *tdb)
+{
+	tdb_len_t recovery_size = 0;
+	tdb_off_t recovery_off = 0;
+	tdb_off_t old_map_size = tdb->transaction->old_map_size;
+	struct tdb_recovery_record *recovery;
+	const struct tdb_methods *methods = tdb->transaction->io_methods;
+	uint64_t magic;
+	enum TDB_ERROR ecode;
+
+	recovery = alloc_recovery(tdb, &recovery_size);
+	if (TDB_PTR_IS_ERR(recovery))
+		return TDB_PTR_ERR(recovery);
+
+	ecode = tdb_recovery_area(tdb, methods, &recovery_off, recovery);
+	if (ecode) {
+		free(recovery);
+		return ecode;
+	}
+
+	if (recovery->max_len < recovery_size) {
+		/* Not large enough. Free up old recovery area. */
+		if (recovery_off) {
+			tdb->stats.frees++;
+			ecode = add_free_record(tdb, recovery_off,
+						sizeof(*recovery)
+						+ recovery->max_len,
+						TDB_LOCK_WAIT, true);
+			free(recovery);
+			if (ecode != TDB_SUCCESS) {
+				return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+						  "tdb_recovery_allocate:"
+						  " failed to free previous"
+						  " recovery area");
+			}
+
+			/* Refresh recovery after add_free_record above. */
+			recovery = alloc_recovery(tdb, &recovery_size);
+			if (TDB_PTR_IS_ERR(recovery))
+				return TDB_PTR_ERR(recovery);
+		}
+
+		recovery_off = create_recovery_area(tdb, recovery_size,
+						    recovery);
+		if (TDB_OFF_IS_ERR(recovery_off)) {
+			free(recovery);
+			return recovery_off;
+		}
+	}
+
+	/* Now we know size, convert rec header. */
+	recovery->magic = TDB_RECOVERY_INVALID_MAGIC;
+	recovery->len = recovery_size;
+	recovery->eof = old_map_size;
+	tdb_convert(tdb, recovery, sizeof(*recovery));
+
+	/* write the recovery data to the recovery area */
+	ecode = methods->twrite(tdb, recovery_off, recovery, recovery_size);
+	if (ecode != TDB_SUCCESS) {
+		free(recovery);
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_transaction_setup_recovery:"
+				  " failed to write recovery data");
+	}
+	transaction_write_existing(tdb, recovery_off, recovery, recovery_size);
+
+	free(recovery);
+
+	/* as we don't have ordered writes, we have to sync the recovery
+	   data before we update the magic to indicate that the recovery
+	   data is present */
+	ecode = transaction_sync(tdb, recovery_off, recovery_size);
+	if (ecode != TDB_SUCCESS)
+		return ecode;
+
+	magic = TDB_RECOVERY_MAGIC;
+	tdb_convert(tdb, &magic, sizeof(magic));
+
+	tdb->transaction->magic_offset
+		= recovery_off + offsetof(struct tdb_recovery_record, magic);
+
+	ecode = methods->twrite(tdb, tdb->transaction->magic_offset,
+				&magic, sizeof(magic));
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_transaction_setup_recovery:"
+				  " failed to write recovery magic");
+	}
+	transaction_write_existing(tdb, tdb->transaction->magic_offset,
+				   &magic, sizeof(magic));
+
+	/* ensure the recovery magic marker is on disk */
+	return transaction_sync(tdb, tdb->transaction->magic_offset,
+				sizeof(magic));
+}
+
+static enum TDB_ERROR _tdb_transaction_prepare_commit(struct tdb_context *tdb)
+{
+	const struct tdb_methods *methods;
+	enum TDB_ERROR ecode;
+
+	if (tdb->transaction == NULL) {
+		return tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_USE_ERROR,
+				  "tdb_transaction_prepare_commit:"
+				  " no transaction");
+	}
+
+	if (tdb->transaction->prepared) {
+		_tdb_transaction_cancel(tdb);
+		return tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_USE_ERROR,
+				  "tdb_transaction_prepare_commit:"
+				  " transaction already prepared");
+	}
+
+	if (tdb->transaction->transaction_error) {
+		_tdb_transaction_cancel(tdb);
+		return tdb_logerr(tdb, TDB_ERR_EINVAL, TDB_LOG_ERROR,
+				  "tdb_transaction_prepare_commit:"
+				  " transaction error pending");
+	}
+
+
+	if (tdb->transaction->nesting != 0) {
+		return TDB_SUCCESS;
+	}
+
+	/* check for a null transaction */
+	if (tdb->transaction->blocks == NULL) {
+		return TDB_SUCCESS;
+	}
+
+	methods = tdb->transaction->io_methods;
+
+	/* upgrade the main transaction lock region to a write lock */
+	ecode = tdb_allrecord_upgrade(tdb);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	/* get the open lock - this prevents new users attaching to the database
+	   during the commit */
+	ecode = tdb_lock_open(tdb, F_WRLCK, TDB_LOCK_WAIT|TDB_LOCK_NOCHECK);
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	/* Since we have whole db locked, we don't need the expansion lock. */
+	if (!(tdb->flags & TDB_NOSYNC)) {
+		/* Sets up tdb->transaction->recovery and
+		 * tdb->transaction->magic_offset. */
+		ecode = transaction_setup_recovery(tdb);
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+	}
+
+	tdb->transaction->prepared = true;
+
+	/* expand the file to the new size if needed */
+	if (tdb->file->map_size != tdb->transaction->old_map_size) {
+		tdb_len_t add;
+
+		add = tdb->file->map_size - tdb->transaction->old_map_size;
+		/* Restore original map size for tdb_expand_file */
+		tdb->file->map_size = tdb->transaction->old_map_size;
+		ecode = methods->expand_file(tdb, add);
+		if (ecode != TDB_SUCCESS) {
+			return ecode;
+		}
+	}
+
+	/* Keep the open lock until the actual commit */
+	return TDB_SUCCESS;
+}
+
+/*
+   prepare to commit the current transaction
+*/
+enum TDB_ERROR tdb_transaction_prepare_commit(struct tdb_context *tdb)
+{
+	return _tdb_transaction_prepare_commit(tdb);
+}
+
+/*
+  commit the current transaction
+*/
+enum TDB_ERROR tdb_transaction_commit(struct tdb_context *tdb)
+{
+	const struct tdb_methods *methods;
+	int i;
+	enum TDB_ERROR ecode;
+
+	if (tdb->transaction == NULL) {
+		return tdb->last_error = tdb_logerr(tdb, TDB_ERR_EINVAL,
+						    TDB_LOG_USE_ERROR,
+						    "tdb_transaction_commit:"
+						    " no transaction");
+	}
+
+	tdb_trace(tdb, "tdb_transaction_commit");
+
+	if (tdb->transaction->nesting != 0) {
+		tdb->transaction->nesting--;
+		return tdb->last_error = TDB_SUCCESS;
+	}
+
+	/* check for a null transaction */
+	if (tdb->transaction->blocks == NULL) {
+		_tdb_transaction_cancel(tdb);
+		return tdb->last_error = TDB_SUCCESS;
+	}
+
+	if (!tdb->transaction->prepared) {
+		ecode = _tdb_transaction_prepare_commit(tdb);
+		if (ecode != TDB_SUCCESS) {
+			_tdb_transaction_cancel(tdb);
+			return tdb->last_error = ecode;
+		}
+	}
+
+	methods = tdb->transaction->io_methods;
+
+	/* perform all the writes */
+	for (i=0;i<tdb->transaction->num_blocks;i++) {
+		tdb_off_t offset;
+		tdb_len_t length;
+
+		if (tdb->transaction->blocks[i] == NULL) {
+			continue;
+		}
+
+		offset = i * PAGESIZE;
+		length = PAGESIZE;
+		if (i == tdb->transaction->num_blocks-1) {
+			length = tdb->transaction->last_block_size;
+		}
+
+		ecode = methods->twrite(tdb, offset,
+					tdb->transaction->blocks[i], length);
+		if (ecode != TDB_SUCCESS) {
+			/* we've overwritten part of the data and
+			   possibly expanded the file, so we need to
+			   run the crash recovery code */
+			tdb->methods = methods;
+			tdb_transaction_recover(tdb);
+
+			_tdb_transaction_cancel(tdb);
+
+			return tdb->last_error = ecode;
+		}
+		SAFE_FREE(tdb->transaction->blocks[i]);
+	}
+
+	SAFE_FREE(tdb->transaction->blocks);
+	tdb->transaction->num_blocks = 0;
+
+	/* ensure the new data is on disk */
+	ecode = transaction_sync(tdb, 0, tdb->file->map_size);
+	if (ecode != TDB_SUCCESS) {
+		return tdb->last_error = ecode;
+	}
+
+	/*
+	  TODO: maybe write to some dummy hdr field, or write to magic
+	  offset without mmap, before the last sync, instead of the
+	  utime() call
+	*/
+
+	/* on some systems (like Linux 2.6.x) changes via mmap/msync
+	   don't change the mtime of the file, this means the file may
+	   not be backed up (as tdb rounding to block sizes means that
+	   file size changes are quite rare too). The following forces
+	   mtime changes when a transaction completes */
+#if HAVE_UTIME
+	utime(tdb->name, NULL);
+#endif
+
+	/* use a transaction cancel to free memory and remove the
+	   transaction locks: it "restores" map_size, too. */
+	tdb->transaction->old_map_size = tdb->file->map_size;
+	_tdb_transaction_cancel(tdb);
+
+	return tdb->last_error = TDB_SUCCESS;
+}
+
+
+/*
+  recover from an aborted transaction. Must be called with exclusive
+  database write access already established (including the open
+  lock to prevent new processes attaching)
+*/
+enum TDB_ERROR tdb_transaction_recover(struct tdb_context *tdb)
+{
+	tdb_off_t recovery_head, recovery_eof;
+	unsigned char *data, *p;
+	struct tdb_recovery_record rec;
+	enum TDB_ERROR ecode;
+
+	/* find the recovery area */
+	recovery_head = tdb_read_off(tdb, offsetof(struct tdb_header,recovery));
+	if (TDB_OFF_IS_ERR(recovery_head)) {
+		return tdb_logerr(tdb, recovery_head, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " failed to read recovery head");
+	}
+
+	if (recovery_head == 0) {
+		/* we have never allocated a recovery record */
+		return TDB_SUCCESS;
+	}
+
+	/* read the recovery record */
+	ecode = tdb_read_convert(tdb, recovery_head, &rec, sizeof(rec));
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " failed to read recovery record");
+	}
+
+	if (rec.magic != TDB_RECOVERY_MAGIC) {
+		/* there is no valid recovery data */
+		return TDB_SUCCESS;
+	}
+
+	if (tdb->read_only) {
+		return tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " attempt to recover read only database");
+	}
+
+	recovery_eof = rec.eof;
+
+	data = (unsigned char *)malloc(rec.len);
+	if (data == NULL) {
+		return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " failed to allocate recovery data");
+	}
+
+	/* read the full recovery data */
+	ecode = tdb->methods->tread(tdb, recovery_head + sizeof(rec), data,
+				    rec.len);
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " failed to read recovery data");
+	}
+
+	/* recover the file data */
+	p = data;
+	while (p+sizeof(tdb_off_t)+sizeof(tdb_len_t) < data + rec.len) {
+		tdb_off_t ofs;
+		tdb_len_t len;
+		tdb_convert(tdb, p, sizeof(ofs) + sizeof(len));
+		memcpy(&ofs, p, sizeof(ofs));
+		memcpy(&len, p + sizeof(ofs), sizeof(len));
+		p += sizeof(ofs) + sizeof(len);
+
+		ecode = tdb->methods->twrite(tdb, ofs, p, len);
+		if (ecode != TDB_SUCCESS) {
+			free(data);
+			return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+					  "tdb_transaction_recover:"
+					  " failed to recover %zu bytes"
+					  " at offset %zu",
+					  (size_t)len, (size_t)ofs);
+		}
+		p += len;
+	}
+
+	free(data);
+
+	ecode = transaction_sync(tdb, 0, tdb->file->map_size);
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " failed to sync recovery");
+	}
+
+	/* if the recovery area is after the recovered eof then remove it */
+	if (recovery_eof <= recovery_head) {
+		ecode = tdb_write_off(tdb, offsetof(struct tdb_header,
+						    recovery),
+				      0);
+		if (ecode != TDB_SUCCESS) {
+			return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+					  "tdb_transaction_recover:"
+					  " failed to remove recovery head");
+		}
+	}
+
+	/* remove the recovery magic */
+	ecode = tdb_write_off(tdb,
+			      recovery_head
+			      + offsetof(struct tdb_recovery_record, magic),
+			      TDB_RECOVERY_INVALID_MAGIC);
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " failed to remove recovery magic");
+	}
+
+	ecode = transaction_sync(tdb, 0, recovery_eof);
+	if (ecode != TDB_SUCCESS) {
+		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,
+				  "tdb_transaction_recover:"
+				  " failed to sync2 recovery");
+	}
+
+	tdb_logerr(tdb, TDB_SUCCESS, TDB_LOG_WARNING,
+		   "tdb_transaction_recover: recovered %zu byte database",
+		   (size_t)recovery_eof);
+
+	/* all done */
+	return TDB_SUCCESS;
+}
+
+tdb_bool_err tdb_needs_recovery(struct tdb_context *tdb)
+{
+	tdb_off_t recovery_head;
+	struct tdb_recovery_record rec;
+	enum TDB_ERROR ecode;
+
+	/* find the recovery area */
+	recovery_head = tdb_read_off(tdb, offsetof(struct tdb_header,recovery));
+	if (TDB_OFF_IS_ERR(recovery_head)) {
+		return recovery_head;
+	}
+
+	if (recovery_head == 0) {
+		/* we have never allocated a recovery record */
+		return false;
+	}
+
+	/* read the recovery record */
+	ecode = tdb_read_convert(tdb, recovery_head, &rec, sizeof(rec));
+	if (ecode != TDB_SUCCESS) {
+		return ecode;
+	}
+
+	return (rec.magic == TDB_RECOVERY_MAGIC);
+}
diff --git a/lib/tdb2/traverse.c b/lib/tdb2/traverse.c
new file mode 100644
index 00000000000..179e095142a
--- /dev/null
+++ b/lib/tdb2/traverse.c
@@ -0,0 +1,99 @@
+ /*
+   Trivial Database 2: traverse function.
+   Copyright (C) Rusty Russell 2010
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "private.h"
+#include <ccan/likely/likely.h>
+
+int64_t tdb_traverse_(struct tdb_context *tdb,
+		      int (*fn)(struct tdb_context *,
+				TDB_DATA, TDB_DATA, void *),
+		      void *p)
+{
+	enum TDB_ERROR ecode;
+	struct traverse_info tinfo;
+	struct tdb_data k, d;
+	int64_t count = 0;
+
+	k.dptr = NULL;
+	for (ecode = first_in_hash(tdb, &tinfo, &k, &d.dsize);
+	     ecode == TDB_SUCCESS;
+	     ecode = next_in_hash(tdb, &tinfo, &k, &d.dsize)) {
+		d.dptr = k.dptr + k.dsize;
+
+		count++;
+		if (fn && fn(tdb, k, d, p)) {
+			free(k.dptr);
+			tdb->last_error = TDB_SUCCESS;
+			return count;
+		}
+		free(k.dptr);
+	}
+
+	if (ecode != TDB_ERR_NOEXIST) {
+		return tdb->last_error = ecode;
+	}
+	tdb->last_error = TDB_SUCCESS;
+	return count;
+}
+
+enum TDB_ERROR tdb_firstkey(struct tdb_context *tdb, struct tdb_data *key)
+{
+	struct traverse_info tinfo;
+
+	return tdb->last_error = first_in_hash(tdb, &tinfo, key, NULL);
+}
+
+/* We lock twice, not very efficient.  We could keep last key & tinfo cached. */
+enum TDB_ERROR tdb_nextkey(struct tdb_context *tdb, struct tdb_data *key)
+{
+	struct traverse_info tinfo;
+	struct hash_info h;
+	struct tdb_used_record rec;
+
+	tinfo.prev = find_and_lock(tdb, *key, F_RDLCK, &h, &rec, &tinfo);
+	free(key->dptr);
+	if (TDB_OFF_IS_ERR(tinfo.prev)) {
+		return tdb->last_error = tinfo.prev;
+	}
+	tdb_unlock_hashes(tdb, h.hlock_start, h.hlock_range, F_RDLCK);
+
+	return tdb->last_error = next_in_hash(tdb, &tinfo, key, NULL);
+}
+
+static int wipe_one(struct tdb_context *tdb,
+		    TDB_DATA key, TDB_DATA data, enum TDB_ERROR *ecode)
+{
+	*ecode = tdb_delete(tdb, key);
+	return (*ecode != TDB_SUCCESS);
+}
+
+enum TDB_ERROR tdb_wipe_all(struct tdb_context *tdb)
+{
+	enum TDB_ERROR ecode;
+	int64_t count;
+
+	ecode = tdb_allrecord_lock(tdb, F_WRLCK, TDB_LOCK_WAIT, false);
+	if (ecode != TDB_SUCCESS)
+		return tdb->last_error = ecode;
+
+	/* FIXME: Be smarter. */
+	count = tdb_traverse(tdb, wipe_one, &ecode);
+	if (count < 0)
+		ecode = count;
+	tdb_allrecord_unlock(tdb, F_WRLCK);
+	return tdb->last_error = ecode;
+}
diff --git a/lib/tdb2/wscript b/lib/tdb2/wscript
new file mode 100644
index 00000000000..386768f0fcb
--- /dev/null
+++ b/lib/tdb2/wscript
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+APPNAME = 'tdb'
+VERSION = '2.0-alpha'
+
+blddir = 'bin'
+
+import sys, os
+
+# find the buildtools directory
+srcdir = '.'
+while not os.path.exists(srcdir+'/buildtools') and len(srcdir.split('/')) < 5:
+    srcdir = '../' + srcdir
+sys.path.insert(0, srcdir + '/buildtools/wafsamba')
+
+import wafsamba, samba_dist, Options, Logs
+
+samba_dist.DIST_DIRS('lib/tdb2:. lib/replace:lib/replace buildtools:buildtools')
+
+def set_options(opt):
+    opt.BUILTIN_DEFAULT('replace')
+    opt.PRIVATE_EXTENSION_DEFAULT('tdb2', noextension='tdb2')
+    opt.RECURSE('lib/replace')
+    opt.add_option('--enable-tdb2-breaks-compat',
+                   help=("Build tdb2 instead of tdb1 (BREAKS TDB1!) [False]"),
+                   action="store_true", dest='BUILD_TDB2', default=False)
+    if opt.IN_LAUNCH_DIR():
+        opt.add_option('--disable-python',
+                       help=("disable the pytdb module"),
+                       action="store_true", dest='disable_python', default=False)
+
+def configure(conf):
+    if conf.env.BUILD_TDB2:
+        conf.DEFINE('BUILD_TDB2', 1)
+        conf.RECURSE('lib/replace')
+        conf.RECURSE('lib/ccan')
+
+        conf.env.standalone_tdb2 = conf.IN_LAUNCH_DIR()
+        conf.env.disable_python = getattr(Options.options, 'disable_python', False)
+
+#        if not conf.env.standalone_tdb2:
+#            if conf.CHECK_BUNDLED_SYSTEM('tdb', minversion=VERSION,
+#                                         implied_deps='replace'):
+#                conf.define('USING_SYSTEM_TDB2', 1)
+
+        conf.SAMBA_CONFIG_H()
+
+def build(bld):
+    if bld.env.BUILD_TDB2:
+        bld.RECURSE('lib/replace')
+
+        if bld.env.standalone_tdb2:
+            bld.env.PKGCONFIGDIR = '${LIBDIR}/pkgconfig'
+            bld.PKG_CONFIG_FILES('tdb2.pc', vnum=VERSION)
+            bld.INSTALL_FILES('${INCLUDEDIR}', 'tdb2.h', flat=True)
+            private_library = False
+        else:
+            private_library = True
+
+        if not bld.CONFIG_SET('USING_SYSTEM_TDB2'):
+            # FIXME: hide_symbols=True, abi_directory='ABI', abi_match='tdb_*',  vnum=VERSION,
+            bld.SAMBA_LIBRARY('tdb',
+                              '''check.c free.c hash.c io.c lock.c open.c
+                                 summary.c tdb.c transaction.c traverse.c''',
+                              deps='replace ccan',
+                              private_library=private_library)
+
+            bld.SAMBA_BINARY('tdb2torture',
+                             'tools/tdb2torture.c',
+                             'tdb',
+                             install=False)
+
+            bld.SAMBA_BINARY('tdb2tool',
+                             'tools/tdb2tool.c',
+                             'tdb')
+
+            bld.SAMBA_BINARY('tdb2dump',
+                             'tools/tdb2dump.c',
+                             'tdb')
+
+            bld.SAMBA_BINARY('tdb2restore',
+                             'tools/tdb2restore.c',
+                             'tdb')
+
+        bld.SAMBA_PYTHON('pytdb',
+                         'pytdb.c',
+                         deps='tdb',
+                         enabled=not bld.env.disable_python,
+                         realname='tdb.so',
+                         cflags='-DPACKAGE_VERSION=\"%s\"' % VERSION)
+
+def dist():
+    '''makes a tarball for distribution'''
+    samba_dist.dist()
+
+def reconfigure(ctx):
+    '''reconfigure if config scripts have changed'''
+    import samba_utils
+    samba_utils.reconfigure(ctx)
diff --git a/lib/tdb_compat/tdb_compat.c b/lib/tdb_compat/tdb_compat.c
new file mode 100644
index 00000000000..a9173fc33d0
--- /dev/null
+++ b/lib/tdb_compat/tdb_compat.c
@@ -0,0 +1,102 @@
+#include <tdb_compat.h>
+
+/* Note: for the moment, we only need this file for TDB2, so we can
+ * assume waf. */
+#if BUILD_TDB2
+TDB_DATA tdb_null = { NULL, 0 };
+
+/* Proxy which sets waitflag to false so we never block. */
+static int lock_nonblock(int fd, int rw, off_t off, off_t len, bool waitflag,
+			 void *_orig)
+{
+	struct tdb_attribute_flock *orig = _orig;
+
+	return orig->lock(fd, rw, off, len, false, orig->data);
+}
+
+enum TDB_ERROR tdb_transaction_start_nonblock(struct tdb_context *tdb)
+{
+	union tdb_attribute locking, orig;
+	enum TDB_ERROR ecode;
+
+	orig.base.attr = TDB_ATTRIBUTE_FLOCK;
+	ecode = tdb_get_attribute(tdb, &orig);
+	if (ecode != TDB_SUCCESS)
+		return ecode;
+
+	/* Replace locking function with our own. */
+	locking = orig;
+	locking.flock.data = &orig;
+	locking.flock.lock = lock_nonblock;
+
+	ecode = tdb_set_attribute(tdb, &locking);
+	if (ecode != TDB_SUCCESS)
+		return ecode;
+
+	ecode = tdb_transaction_start(tdb);
+	tdb_unset_attribute(tdb, TDB_ATTRIBUTE_FLOCK);
+	return ecode;
+}
+
+/*
+ * This handles TDB_CLEAR_IF_FIRST.
+ */
+static enum TDB_ERROR clear_if_first(int fd, void *unused)
+{
+	/* We hold a lock offset 63 always, so we can tell if anyone else is. */
+	struct flock fl;
+
+	fl.l_type = F_WRLCK;
+	fl.l_whence = SEEK_SET;
+	fl.l_start = 63;
+	fl.l_len = 1;
+
+	if (fcntl(fd, F_SETLK, &fl) == 0) {
+		/* We must be first ones to open it w/ TDB_CLEAR_IF_FIRST! */
+		if (ftruncate(fd, 0) != 0) {
+			return TDB_ERR_IO;
+		}
+	}
+	fl.l_type = F_RDLCK;
+	if (fcntl(fd, F_SETLKW, &fl) != 0) {
+		return TDB_ERR_IO;
+	}
+	return TDB_SUCCESS;
+}
+
+struct tdb_context *
+tdb_open_compat_(const char *name, int hash_size_unused,
+		 int tdb_flags, int open_flags, mode_t mode,
+		 void (*log_fn)(struct tdb_context *,
+				enum tdb_log_level,
+				const char *message,
+				void *data),
+		 void *log_data)
+{
+	union tdb_attribute cif, log, *attr = NULL;
+
+	if (log_fn) {
+		log.log.base.attr = TDB_ATTRIBUTE_LOG;
+		log.log.base.next = NULL;
+		log.log.fn = log_fn;
+		log.log.data = log_data;
+		attr = &log;
+	}
+
+	if (tdb_flags & TDB_CLEAR_IF_FIRST) {
+		cif.openhook.base.attr = TDB_ATTRIBUTE_OPENHOOK;
+		cif.openhook.base.next = attr;
+		cif.openhook.fn = clear_if_first;
+		attr = &cif;
+		tdb_flags &= ~TDB_CLEAR_IF_FIRST;
+	}
+
+	/* Testsuite uses this to speed things up. */
+	if (getenv("TDB_NO_FSYNC")) {
+		tdb_flags |= TDB_NOSYNC;
+	}
+
+	return tdb_open(name, tdb_flags|TDB_ALLOW_NESTING, open_flags, mode,
+			attr);
+}
+#endif
diff --git a/lib/tdb_compat/tdb_compat.h b/lib/tdb_compat/tdb_compat.h
new file mode 100644
index 00000000000..ea401cba498
--- /dev/null
+++ b/lib/tdb_compat/tdb_compat.h
@@ -0,0 +1,136 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   Compatibility layer for TDB1 vs TDB2.
+
+   Copyright (C) Rusty Russell 2011
+
+     ** NOTE! The following LGPL license applies to the tdb_compat
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef TDB_COMPAT_H
+#define TDB_COMPAT_H
+
+#include "replace.h"
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#if BUILD_TDB2
+#include <tdb2.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+extern TDB_DATA tdb_null;
+
+/* Old-style tdb_fetch. */
+static inline TDB_DATA tdb_fetch_compat(struct tdb_context *tdb, TDB_DATA k)
+{
+	TDB_DATA dbuf;
+	if (tdb_fetch(tdb, k, &dbuf) != TDB_SUCCESS) {
+		return tdb_null;
+	}
+	return dbuf;
+}
+
+static inline TDB_DATA tdb_firstkey_compat(struct tdb_context *tdb)
+{
+	TDB_DATA k;
+	if (tdb_firstkey(tdb, &k) != TDB_SUCCESS) {
+		return tdb_null;
+	}
+	return k;
+}
+
+/* Note: this frees the old key.dptr. */
+static inline TDB_DATA tdb_nextkey_compat(struct tdb_context *tdb, TDB_DATA k)
+{
+	if (tdb_nextkey(tdb, &k) != TDB_SUCCESS) {
+		return tdb_null;
+	}
+	return k;
+}
+
+/* tdb_traverse_read and tdb_traverse are equal: both only take read locks. */
+#define tdb_traverse_read tdb_traverse
+
+/* Old-style tdb_errorstr */
+#define tdb_errorstr_compat(tdb) tdb_errorstr(tdb_error(tdb))
+
+/* This typedef doesn't exist in TDB2. */
+typedef struct tdb_context TDB_CONTEXT;
+
+/* We don't need these any more. */
+#define tdb_reopen_all(flag) 0
+#define tdb_reopen(tdb) 0
+
+/* These no longer exist in tdb2. */
+#define TDB_CLEAR_IF_FIRST 1048576
+#define TDB_INCOMPATIBLE_HASH 0
+#define TDB_VOLATILE 0
+
+/* tdb2 does nonblocking functions via attibutes. */
+enum TDB_ERROR tdb_transaction_start_nonblock(struct tdb_context *tdb);
+
+/* Convenient (typesafe) wrapper for tdb open with logging */
+#define tdb_open_compat(name, hsize, tdb_fl, open_fl, mode, log_fn, log_data) \
+	tdb_open_compat_((name), (hsize), (tdb_fl), (open_fl), (mode),	\
+			 typesafe_cb_preargs(void, void *,		\
+					     (log_fn), (log_data),	\
+					     struct tdb_context *,	\
+					     enum tdb_log_level,	\
+					     const char *),		\
+			 (log_data))
+
+struct tdb_context *
+tdb_open_compat_(const char *name, int hash_size_unused,
+		 int tdb_flags, int open_flags, mode_t mode,
+		 void (*log_fn)(struct tdb_context *,
+				enum tdb_log_level,
+				const char *message,
+				void *data),
+		 void *log_data);
+#else
+#include <tdb.h>
+
+/* FIXME: Inlining this is a bit lazy, but eases S3 build. */
+static inline struct tdb_context *
+tdb_open_compat(const char *name, int hash_size,
+		int tdb_flags, int open_flags, mode_t mode,
+		tdb_log_func log_fn, void *log_private)
+{
+	struct tdb_logging_context lctx;
+	lctx.log_fn = log_fn;
+	lctx.log_private = log_private;
+
+	if (log_fn)
+		return tdb_open_ex(name, hash_size, tdb_flags, open_flags,
+				   mode, &lctx, NULL);
+	else
+		return tdb_open(name, hash_size, tdb_flags, open_flags, mode);
+}
+
+#define tdb_firstkey_compat tdb_firstkey
+/* Note: this frees the old key.dptr. */
+static inline TDB_DATA tdb_nextkey_compat(struct tdb_context *tdb, TDB_DATA k)
+{
+	TDB_DATA next = tdb_nextkey(tdb, k);
+	free(k.dptr);
+	return next;
+}
+#define tdb_errorstr_compat(tdb) tdb_errorstr(tdb)
+#define tdb_fetch_compat tdb_fetch
+#endif
+
+#endif /* TDB_COMPAT_H */
diff --git a/lib/tdb_compat/wscript b/lib/tdb_compat/wscript
new file mode 100644
index 00000000000..574e67e8ef4
--- /dev/null
+++ b/lib/tdb_compat/wscript
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import Options
+
+def set_options(opt):
+    opt.RECURSE('lib/tdb2')
+    opt.RECURSE('lib/tdb')
+
+def configure(conf):
+    conf.env.BUILD_TDB2 = getattr(Options.options, 'BUILD_TDB2', False)
+
+    if conf.env.BUILD_TDB2:
+        conf.RECURSE('lib/tdb2')
+    else:
+        conf.RECURSE('lib/tdb')
+    conf.RECURSE('lib/ccan')
+
+def build(bld):
+    bld.RECURSE('lib/ccan')
+    if bld.env.BUILD_TDB2:
+        bld.RECURSE('lib/tdb2')
+    else:
+        bld.RECURSE('lib/tdb')
+    bld.SAMBA_LIBRARY('tdb_compat',
+                      source='tdb_compat.c',
+		      deps='replace tdb ccan',
+                      private_library=True)
diff --git a/lib/tevent/ABI/tevent-0.9.12.sigs b/lib/tevent/ABI/tevent-0.9.12.sigs
new file mode 100644
index 00000000000..df9b08dfd50
--- /dev/null
+++ b/lib/tevent/ABI/tevent-0.9.12.sigs
@@ -0,0 +1,74 @@
+_tevent_add_fd: struct tevent_fd *(struct tevent_context *, TALLOC_CTX *, int, uint16_t, tevent_fd_handler_t, void *, const char *, const char *)
+_tevent_add_signal: struct tevent_signal *(struct tevent_context *, TALLOC_CTX *, int, int, tevent_signal_handler_t, void *, const char *, const char *)
+_tevent_add_timer: struct tevent_timer *(struct tevent_context *, TALLOC_CTX *, struct timeval, tevent_timer_handler_t, void *, const char *, const char *)
+_tevent_create_immediate: struct tevent_immediate *(TALLOC_CTX *, const char *)
+_tevent_loop_once: int (struct tevent_context *, const char *)
+_tevent_loop_until: int (struct tevent_context *, bool (*)(void *), void *, const char *)
+_tevent_loop_wait: int (struct tevent_context *, const char *)
+_tevent_queue_create: struct tevent_queue *(TALLOC_CTX *, const char *, const char *)
+_tevent_req_callback_data: void *(struct tevent_req *)
+_tevent_req_cancel: bool (struct tevent_req *, const char *)
+_tevent_req_create: struct tevent_req *(TALLOC_CTX *, void *, size_t, const char *, const char *)
+_tevent_req_data: void *(struct tevent_req *)
+_tevent_req_done: void (struct tevent_req *, const char *)
+_tevent_req_error: bool (struct tevent_req *, uint64_t, const char *)
+_tevent_req_nomem: bool (const void *, struct tevent_req *, const char *)
+_tevent_req_notify_callback: void (struct tevent_req *, const char *)
+_tevent_req_oom: void (struct tevent_req *, const char *)
+_tevent_schedule_immediate: void (struct tevent_immediate *, struct tevent_context *, tevent_immediate_handler_t, void *, const char *, const char *)
+tevent_backend_list: const char **(TALLOC_CTX *)
+tevent_cleanup_pending_signal_handlers: void (struct tevent_signal *)
+tevent_common_add_fd: struct tevent_fd *(struct tevent_context *, TALLOC_CTX *, int, uint16_t, tevent_fd_handler_t, void *, const char *, const char *)
+tevent_common_add_signal: struct tevent_signal *(struct tevent_context *, TALLOC_CTX *, int, int, tevent_signal_handler_t, void *, const char *, const char *)
+tevent_common_add_timer: struct tevent_timer *(struct tevent_context *, TALLOC_CTX *, struct timeval, tevent_timer_handler_t, void *, const char *, const char *)
+tevent_common_check_signal: int (struct tevent_context *)
+tevent_common_context_destructor: int (struct tevent_context *)
+tevent_common_fd_destructor: int (struct tevent_fd *)
+tevent_common_fd_get_flags: uint16_t (struct tevent_fd *)
+tevent_common_fd_set_close_fn: void (struct tevent_fd *, tevent_fd_close_fn_t)
+tevent_common_fd_set_flags: void (struct tevent_fd *, uint16_t)
+tevent_common_loop_immediate: bool (struct tevent_context *)
+tevent_common_loop_timer_delay: struct timeval (struct tevent_context *)
+tevent_common_loop_wait: int (struct tevent_context *, const char *)
+tevent_common_schedule_immediate: void (struct tevent_immediate *, struct tevent_context *, tevent_immediate_handler_t, void *, const char *, const char *)
+tevent_context_init: struct tevent_context *(TALLOC_CTX *)
+tevent_context_init_byname: struct tevent_context *(TALLOC_CTX *, const char *)
+tevent_debug: void (struct tevent_context *, enum tevent_debug_level, const char *, ...)
+tevent_fd_get_flags: uint16_t (struct tevent_fd *)
+tevent_fd_set_auto_close: void (struct tevent_fd *)
+tevent_fd_set_close_fn: void (struct tevent_fd *, tevent_fd_close_fn_t)
+tevent_fd_set_flags: void (struct tevent_fd *, uint16_t)
+tevent_loop_allow_nesting: void (struct tevent_context *)
+tevent_loop_set_nesting_hook: void (struct tevent_context *, tevent_nesting_hook, void *)
+tevent_queue_add: bool (struct tevent_queue *, struct tevent_context *, struct tevent_req *, tevent_queue_trigger_fn_t, void *)
+tevent_queue_length: size_t (struct tevent_queue *)
+tevent_queue_start: void (struct tevent_queue *)
+tevent_queue_stop: void (struct tevent_queue *)
+tevent_re_initialise: int (struct tevent_context *)
+tevent_register_backend: bool (const char *, const struct tevent_ops *)
+tevent_req_default_print: char *(struct tevent_req *, TALLOC_CTX *)
+tevent_req_is_error: bool (struct tevent_req *, enum tevent_req_state *, uint64_t *)
+tevent_req_is_in_progress: bool (struct tevent_req *)
+tevent_req_poll: bool (struct tevent_req *, struct tevent_context *)
+tevent_req_post: struct tevent_req *(struct tevent_req *, struct tevent_context *)
+tevent_req_print: char *(TALLOC_CTX *, struct tevent_req *)
+tevent_req_received: void (struct tevent_req *)
+tevent_req_set_callback: void (struct tevent_req *, tevent_req_fn, void *)
+tevent_req_set_cancel_fn: void (struct tevent_req *, tevent_req_cancel_fn)
+tevent_req_set_endtime: bool (struct tevent_req *, struct tevent_context *, struct timeval)
+tevent_req_set_print_fn: void (struct tevent_req *, tevent_req_print_fn)
+tevent_set_abort_fn: void (void (*)(const char *))
+tevent_set_debug: int (struct tevent_context *, void (*)(void *, enum tevent_debug_level, const char *, va_list), void *)
+tevent_set_debug_stderr: int (struct tevent_context *)
+tevent_set_default_backend: void (const char *)
+tevent_signal_support: bool (struct tevent_context *)
+tevent_timeval_add: struct timeval (const struct timeval *, uint32_t, uint32_t)
+tevent_timeval_compare: int (const struct timeval *, const struct timeval *)
+tevent_timeval_current: struct timeval (void)
+tevent_timeval_current_ofs: struct timeval (uint32_t, uint32_t)
+tevent_timeval_is_zero: bool (const struct timeval *)
+tevent_timeval_set: struct timeval (uint32_t, uint32_t)
+tevent_timeval_until: struct timeval (const struct timeval *, const struct timeval *)
+tevent_timeval_zero: struct timeval (void)
+tevent_wakeup_recv: bool (struct tevent_req *)
+tevent_wakeup_send: struct tevent_req *(TALLOC_CTX *, struct tevent_context *, struct timeval)
diff --git a/lib/tevent/tevent.h b/lib/tevent/tevent.h
index 665c491ebb3..8204a28fbed 100644
--- a/lib/tevent/tevent.h
+++ b/lib/tevent/tevent.h
@@ -136,7 +136,7 @@ struct tevent_context *tevent_context_init_byname(TALLOC_CTX *mem_ctx, const cha
 const char **tevent_backend_list(TALLOC_CTX *mem_ctx);
 
 /**
- * @brief Set the default tevent backent.
+ * @brief Set the default tevent backend.
  *
  * @param[in]  backend  The name of the backend to set.
  */
@@ -995,6 +995,20 @@ bool _tevent_req_nomem(const void *p,
 	_tevent_req_nomem(p, req, __location__)
 #endif
 
+#ifdef DOXYGEN
+/**
+ * @brief Indicate out of memory to a request
+ *
+ * @param[in]  req      The request being processed.
+ */
+void tevent_req_oom(struct tevent_req *req);
+#else
+void _tevent_req_oom(struct tevent_req *req,
+		     const char *location);
+#define tevent_req_oom(req) \
+	_tevent_req_oom(req, __location__)
+#endif
+
 /**
  * @brief Finish a request before the caller had the change to set the callback.
  *
@@ -1218,7 +1232,7 @@ struct timeval tevent_timeval_current(void);
  *
  * @param[in]  secs     The seconds to set.
  *
- * @param[in]  usecs    The milliseconds to set.
+ * @param[in]  usecs    The microseconds to set.
  *
  * @return              A timeval structure with the given values.
  */
@@ -1253,7 +1267,7 @@ bool tevent_timeval_is_zero(const struct timeval *tv);
  *
  * @param[in]  secs      The seconds to add to the timeval.
  *
- * @param[in]  usecs     The milliseconds to add to the timeval.
+ * @param[in]  usecs     The microseconds to add to the timeval.
  *
  * @return               The timeval structure with the new time.
  */
@@ -1265,7 +1279,7 @@ struct timeval tevent_timeval_add(const struct timeval *tv, uint32_t secs,
  *
  * @param[in]  secs     The seconds of the offset from now.
  *
- * @param[in]  usecs    The milliseconds of the offset from now.
+ * @param[in]  usecs    The microseconds of the offset from now.
  *
  * @return              A timval with the given offset in the future.
  */
diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c
index 712255b3732..0b782e99bb8 100644
--- a/lib/tevent/tevent_poll.c
+++ b/lib/tevent/tevent_poll.c
@@ -233,7 +233,19 @@ static int poll_event_loop_poll(struct tevent_context *ev,
 
 			pfd = &poll_ev->fds[pfd_idx];
 
-			if (pfd->revents & (POLLIN|POLLHUP|POLLERR)) {
+			if (pfd->revents & (POLLHUP|POLLERR)) {
+				/* If we only wait for TEVENT_FD_WRITE, we
+				   should not tell the event handler about it,
+				   and remove the writable flag, as we only
+				   report errors when waiting for read events
+				   to match the select behavior. */
+				if (!(fde->flags & TEVENT_FD_READ)) {
+					TEVENT_FD_NOT_WRITEABLE(fde);
+					continue;
+				}
+				flags |= TEVENT_FD_READ;
+			}
+			if (pfd->revents & POLLIN) {
 				flags |= TEVENT_FD_READ;
 			}
 			if (pfd->revents & POLLOUT) {
diff --git a/lib/tevent/tevent_req.c b/lib/tevent/tevent_req.c
index b0c9c57ddec..92697b7df95 100644
--- a/lib/tevent/tevent_req.c
+++ b/lib/tevent/tevent_req.c
@@ -123,6 +123,11 @@ bool _tevent_req_error(struct tevent_req *req,
 	return true;
 }
 
+void _tevent_req_oom(struct tevent_req *req, const char *location)
+{
+	tevent_req_finish(req, TEVENT_REQ_NO_MEMORY, location);
+}
+
 bool _tevent_req_nomem(const void *p,
 		       struct tevent_req *req,
 		       const char *location)
@@ -130,7 +135,7 @@ bool _tevent_req_nomem(const void *p,
 	if (p != NULL) {
 		return false;
 	}
-	tevent_req_finish(req, TEVENT_REQ_NO_MEMORY, location);
+	_tevent_req_oom(req, location);
 	return true;
 }
 
diff --git a/lib/tevent/wscript b/lib/tevent/wscript
index 75c44c29622..5dcd18814b2 100644
--- a/lib/tevent/wscript
+++ b/lib/tevent/wscript
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 APPNAME = 'tevent'
-VERSION = '0.9.11'
+VERSION = '0.9.12'
 
 blddir = 'bin'
 
diff --git a/lib/torture/torture.c b/lib/torture/torture.c
index a0b35bfe7ca..cee6bdb934d 100644
--- a/lib/torture/torture.c
+++ b/lib/torture/torture.c
@@ -88,7 +88,7 @@ _PUBLIC_ NTSTATUS torture_temp_dir(struct torture_context *tctx,
 	NT_STATUS_HAVE_NO_MEMORY(*tempdir);
 
 	if (mkdtemp(*tempdir) == NULL) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return NT_STATUS_OK;
@@ -154,7 +154,7 @@ _PUBLIC_ NTSTATUS torture_deltree_outputdir(struct torture_context *tctx)
 
 	if (local_deltree(tctx->outputdir) == -1) {
 		if (errno != 0) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 		return NT_STATUS_UNSUCCESSFUL;
 	}
diff --git a/lib/tsocket/tsocket_helpers.c b/lib/tsocket/tsocket_helpers.c
index 3a41a3efc3d..db6b6148e95 100644
--- a/lib/tsocket/tsocket_helpers.c
+++ b/lib/tsocket/tsocket_helpers.c
@@ -73,7 +73,7 @@ struct tevent_req *tdgram_sendto_queue_send(TALLOC_CTX *mem_ctx,
 			      tdgram_sendto_queue_trigger,
 			      NULL);
 	if (!ok) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		goto post;
 	}
 
@@ -346,7 +346,7 @@ struct tevent_req *tstream_readv_pdu_queue_send(TALLOC_CTX *mem_ctx,
 			      tstream_readv_pdu_queue_trigger,
 			      NULL);
 	if (!ok) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		goto post;
 	}
 
@@ -453,7 +453,7 @@ struct tevent_req *tstream_writev_queue_send(TALLOC_CTX *mem_ctx,
 			      tstream_writev_queue_trigger,
 			      NULL);
 	if (!ok) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		goto post;
 	}
 
diff --git a/lib/util/asn1.c b/lib/util/asn1.c
index b716da63c05..c23bf65b8d5 100644
--- a/lib/util/asn1.c
+++ b/lib/util/asn1.c
@@ -885,10 +885,19 @@ bool asn1_read_ContextSimple(struct asn1_data *data, uint8_t num, DATA_BLOB *blo
 bool asn1_read_implicit_Integer(struct asn1_data *data, int *i)
 {
 	uint8_t b;
+	bool first_byte = true;
 	*i = 0;
 
 	while (!data->has_error && asn1_tag_remaining(data)>0) {
 		if (!asn1_read_uint8(data, &b)) return false;
+		if (first_byte) {
+			if (b & 0x80) {
+				/* Number is negative.
+				   Set i to -1 for sign extend. */
+				*i = -1;
+			}
+			first_byte = false;
+		}
 		*i = (*i << 8) + b;
 	}
 	return !data->has_error;	
diff --git a/lib/util/byteorder.h b/lib/util/byteorder.h
index 59ad8371e4d..6bcf71e83bd 100644
--- a/lib/util/byteorder.h
+++ b/lib/util/byteorder.h
@@ -201,18 +201,29 @@ static __inline__ void st_le32(uint32_t *addr, const uint32_t val)
 
 #endif /* not CAREFUL_ALIGNMENT */
 
+/* 64 bit macros */
+#define BVAL(p, ofs) (IVAL(p,ofs) | (((uint64_t)IVAL(p,(ofs)+4)) << 32))
+#define BVALS(p, ofs) ((int64_t)BVAL(p,ofs))
+#define SBVAL(p, ofs, v) (SIVAL(p,ofs,(v)&0xFFFFFFFF), SIVAL(p,(ofs)+4,((uint64_t)(v))>>32))
+#define SBVALS(p, ofs, v) (SBVAL(p,ofs,(uint64_t)v))
+
 /* now the reverse routines - these are used in nmb packets (mostly) */
 #define SREV(x) ((((x)&0xFF)<<8) | (((x)>>8)&0xFF))
 #define IREV(x) ((SREV(x)<<16) | (SREV((x)>>16)))
+#define BREV(x) ((IREV(x)<<32) | (IREV((x)>>32)))
 
 #define RSVAL(buf,pos) SREV(SVAL(buf,pos))
 #define RSVALS(buf,pos) SREV(SVALS(buf,pos))
 #define RIVAL(buf,pos) IREV(IVAL(buf,pos))
 #define RIVALS(buf,pos) IREV(IVALS(buf,pos))
+#define RBVAL(buf,pos) BREV(BVAL(buf,pos))
+#define RBVALS(buf,pos) BREV(BVALS(buf,pos))
 #define RSSVAL(buf,pos,val) SSVAL(buf,pos,SREV(val))
 #define RSSVALS(buf,pos,val) SSVALS(buf,pos,SREV(val))
 #define RSIVAL(buf,pos,val) SIVAL(buf,pos,IREV(val))
 #define RSIVALS(buf,pos,val) SIVALS(buf,pos,IREV(val))
+#define RSBVAL(buf,pos,val) SBVAL(buf,pos,BREV(val))
+#define RSBVALS(buf,pos,val) SBVALS(buf,pos,BREV(val))
 
 /* Alignment macros. */
 #define ALIGN4(p,base) ((p) + ((4 - (PTR_DIFF((p), (base)) & 3)) & 3))
@@ -222,10 +233,4 @@ static __inline__ void st_le32(uint32_t *addr, const uint32_t val)
 /* macros for accessing SMB protocol elements */
 #define VWV(vwv) ((vwv)*2)
 
-/* 64 bit macros */
-#define BVAL(p, ofs) (IVAL(p,ofs) | (((uint64_t)IVAL(p,(ofs)+4)) << 32))
-#define BVALS(p, ofs) ((int64_t)BVAL(p,ofs))
-#define SBVAL(p, ofs, v) (SIVAL(p,ofs,(v)&0xFFFFFFFF), SIVAL(p,(ofs)+4,((uint64_t)(v))>>32))
-#define SBVALS(p, ofs, v) (SBVAL(p,ofs,(uint64_t)v))
-
 #endif /* _BYTEORDER_H */
diff --git a/source3/modules/CP437.c b/lib/util/charset/CP437.c
index 96d14b1f023..1e478d678f9 100644
--- a/source3/modules/CP437.c
+++ b/lib/util/charset/CP437.c
@@ -22,7 +22,7 @@
 
 #include "includes.h"
 
-static const uint16 to_ucs2[256] = {
+static const uint16_t to_ucs2[256] = {
  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
  0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
diff --git a/source3/modules/CP850.c b/lib/util/charset/CP850.c
index 40730c0d8d6..87a76f4cdf9 100644
--- a/source3/modules/CP850.c
+++ b/lib/util/charset/CP850.c
@@ -22,7 +22,7 @@
 
 #include "includes.h"
 
-static const uint16 to_ucs2[256] = {
+static const uint16_t to_ucs2[256] = {
  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
  0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
diff --git a/lib/util/charset/charcnv.c b/lib/util/charset/charcnv.c
index 998bb08fd71..076795a0b29 100644
--- a/lib/util/charset/charcnv.c
+++ b/lib/util/charset/charcnv.c
@@ -113,138 +113,3 @@ convert:
 	return destlen;
 
 }
-
-/**
- * Convert string from one encoding to another, making error checking etc
- *
- * @param src pointer to source string (multibyte or singlebyte)
- * @param srclen length of the source string in bytes
- * @param dest pointer to destination string (multibyte or singlebyte)
- * @param destlen maximal length allowed for string
- * @returns the number of bytes occupied in the destination
- * on error, returns -1, and sets errno
- **/
-_PUBLIC_ bool convert_string_error_handle(struct smb_iconv_handle *ic,
-					  charset_t from, charset_t to,
-					  void const *src, size_t srclen,
-					  void *dest, size_t destlen,
-					  size_t *converted_size)
-{
-	size_t i_len, o_len;
-	ssize_t retval;
-	const char* inbuf = (const char*)src;
-	char* outbuf = (char*)dest;
-	smb_iconv_t descriptor;
-
-	if (srclen == (size_t)-1)
-		srclen = strlen(inbuf)+1;
-
-	descriptor = get_conv_handle(ic, from, to);
-	if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
-		if (converted_size) {
-			*converted_size = 0;
-		}
-		errno = EINVAL;
-		return -1;
-	}
-
-	i_len=srclen;
-	o_len=destlen;
-
-	retval = smb_iconv(descriptor,  &inbuf, &i_len, &outbuf, &o_len);
-
-	if (converted_size != NULL)
-		*converted_size = destlen-o_len;
-	return (retval != (ssize_t)-1);
-}
-
-
-/**
- * Convert string from one encoding to another, making error checking etc
- *
- * @param src pointer to source string (multibyte or singlebyte)
- * @param srclen length of the source string in bytes
- * @param dest pointer to destination string (multibyte or singlebyte)
- * @param destlen maximal length allowed for string
- * @returns the number of bytes occupied in the destination
- **/
-_PUBLIC_ bool convert_string_handle(struct smb_iconv_handle *ic,
-					 charset_t from, charset_t to,
-					 void const *src, size_t srclen,
-					 void *dest, size_t destlen, size_t *converted_size)
-{
-	bool retval;
-
-	retval = convert_string_error_handle(ic, from, to, src, srclen, dest, destlen, converted_size);
-	if(retval==false) {
-	    	const char *reason;
-		switch(errno) {
-		case EINVAL:
-			reason="Incomplete multibyte sequence";
-			return false;
-		case E2BIG:
-			reason="No more room";
-			if (from == CH_UNIX) {
-				DEBUG(0,("E2BIG: convert_string_handle(%s,%s): srclen=%d destlen=%d - '%s'\n",
-					 charset_name(ic, from), charset_name(ic, to),
-					 (int)srclen, (int)destlen,
-					 (const char *)src));
-			} else {
-				DEBUG(0,("E2BIG: convert_string_handle(%s,%s): srclen=%d destlen=%d\n",
-					 charset_name(ic, from), charset_name(ic, to),
-					 (int)srclen, (int)destlen));
-			}
-			return false;
-		case EILSEQ:
-			reason="Illegal multibyte sequence";
-			return false;
-		default:
-			return false;
-		}
-	}
-	return true;
-}
-	
-/**
- * Convert between character sets, allocating a new buffer using talloc for the result.
- *
- * @param srclen length of source buffer.
- * @param dest always set at least to NULL
- * @note -1 is not accepted for srclen.
- *
- * @returns Size in bytes of the converted string; or -1 in case of error.
- **/
-
-_PUBLIC_ bool convert_string_talloc_handle(TALLOC_CTX *ctx,
-						struct smb_iconv_handle *ic,
-						charset_t from, charset_t to, 
-						void const *src, size_t srclen, 
-						void *dst, size_t *converted_size)
-{
-	void **dest = (void **)dst;
-	smb_iconv_t descriptor;
-	ssize_t ret;
-
-	*dest = NULL;
-
-	if (src == NULL || srclen == (size_t)-1 || srclen == 0)
-		return false;
-
-	descriptor = get_conv_handle(ic, from, to);
-
-	if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
-		/* conversion not supported, return -1*/
-		DEBUG(3, ("convert_string_talloc_handle: conversion from %s to %s not supported!\n",
-			  charset_name(ic, from), 
-			  charset_name(ic, to)));
-		return false;
-	}
-
-	ret = iconv_talloc(ctx, descriptor, src, srclen, dest);
-	if (ret == -1)
-		return false;
-	if (converted_size != NULL)
-		*converted_size = ret;
-	return true;
-}
-
diff --git a/lib/util/charset/charset.h b/lib/util/charset/charset.h
index 1078035592a..b36c461003b 100644
--- a/lib/util/charset/charset.h
+++ b/lib/util/charset/charset.h
@@ -28,7 +28,7 @@
 #include <talloc.h>
 
 /* this defines the charset types used in samba */
-typedef enum {CH_UTF16LE=0, CH_UTF16=0, CH_UNIX, CH_DISPLAY, CH_DOS, CH_UTF8, CH_UTF16BE, CH_UTF16MUNGED} charset_t;
+typedef enum {CH_UTF16LE=0, CH_UTF16=0, CH_UNIX, CH_DOS, CH_UTF8, CH_UTF16BE, CH_UTF16MUNGED} charset_t;
 
 #define NUM_CHARSETS 7
 
@@ -105,11 +105,6 @@ typedef struct smb_iconv_s {
 struct loadparm_context;
 struct smb_iconv_handle;
 
-/* replace some string functions with multi-byte
-   versions */
-#define strlower(s) strlower_m(s)
-#define strupper(s) strupper_m(s)
-
 char *strchr_m(const char *s, char c);
 /**
  * Calculate the number of units (8 or 16-bit, depending on the
@@ -137,8 +132,6 @@ int strcasecmp_m_handle(struct smb_iconv_handle *iconv_handle,
 			const char *s1, const char *s2);
 int strcasecmp_m(const char *s1, const char *s2);
 size_t count_chars_m(const char *s, char c);
-void strupper_m(char *s);
-void strlower_m(char *s);
 char *strupper_talloc(TALLOC_CTX *ctx, const char *src);
 char *talloc_strdup_upper(TALLOC_CTX *ctx, const char *src);
 char *strupper_talloc_n_handle(struct smb_iconv_handle *iconv_handle,
@@ -155,6 +148,7 @@ bool strhasupper_handle(struct smb_iconv_handle *ic,
 			const char *string);
 char *strrchr_m(const char *s, char c);
 char *strchr_m(const char *s, char c);
+char *strstr_m(const char *src, const char *findstr);
 
 bool push_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src, size_t *converted_size);
 bool push_ucs2_talloc(TALLOC_CTX *ctx, smb_ucs2_t **dest, const char *src, size_t *converted_size);
@@ -188,8 +182,7 @@ extern struct smb_iconv_handle *global_iconv_handle;
 struct smb_iconv_handle *get_iconv_handle(void);
 struct smb_iconv_handle *get_iconv_testing_handle(TALLOC_CTX *mem_ctx, 
 						  const char *dos_charset, 
-						  const char *unix_charset, 
-						  const char *display_charset);
+						  const char *unix_charset);
 smb_iconv_t get_conv_handle(struct smb_iconv_handle *ic,
 			    charset_t from, charset_t to);
 const char *charset_name(struct smb_iconv_handle *ic, charset_t ch);
@@ -218,7 +211,6 @@ int codepoint_cmpi(codepoint_t c1, codepoint_t c2);
 struct smb_iconv_handle *smb_iconv_handle_reinit(TALLOC_CTX *mem_ctx,
 							   const char *dos_charset,
 							   const char *unix_charset,
-							   const char *display_charset,
 							   bool native_iconv,
 							   struct smb_iconv_handle *old_ic);
 
@@ -285,7 +277,7 @@ static size_t CHARSETNAME ## _push(void *cd, const char **inbuf, size_t *inbytes
 		int i; 										\
 		int done = 0; 									\
 												\
-		uint16 ch = SVAL(*inbuf,0); 							\
+		uint16_t ch = SVAL(*inbuf,0); 							\
 												\
 		for (i=0; from_idx[i].start != 0xffff; i++) {					\
 			if ((from_idx[i].start <= ch) && (from_idx[i].end >= ch)) {		\
diff --git a/source3/modules/charset_macosxfs.c b/lib/util/charset/charset_macosxfs.c
index 8c2fdc7776b..4d2ba5b6ffd 100644
--- a/source3/modules/charset_macosxfs.c
+++ b/lib/util/charset/charset_macosxfs.c
@@ -30,6 +30,7 @@
  */
 
 #include "includes.h"
+#undef realloc
 
 /*
  * Include OS frameworks.  These are only needed in this module.
@@ -61,7 +62,7 @@ static inline void *resize_buffer (void *buffer, size_t *size, size_t newsize)
 {
 	if (newsize > *size) {
 		*size = newsize + 128;
-		buffer = SMB_REALLOC(buffer, *size);
+		buffer = realloc(buffer, *size);
 	}
 	return buffer;
 }
@@ -372,7 +373,7 @@ static size_t macosxfs_encoding_push(
 	cfsize = CFStringGetLength(cfstring);
 	charsconverted = CFStringGetBytes(
 		cfstring, CFRangeMake(0,cfsize),
-		script_code, 0, False,
+		script_code, 0, false,
 		*outbuf, *outbytesleft, &outsize);
 
 	if (0 == charsconverted) {
diff --git a/lib/util/charset/codepoints.c b/lib/util/charset/codepoints.c
index cd54420e8e2..8cc33a9782a 100644
--- a/lib/util/charset/codepoints.c
+++ b/lib/util/charset/codepoints.c
@@ -23,7 +23,7 @@
 #include "includes.h"
 #include "lib/util/charset/charset.h"
 #include "system/locale.h"
-#include "dynconfig.h"
+#include "dynconfig/dynconfig.h"
 
 #ifdef strcasecmp
 #undef strcasecmp
@@ -168,17 +168,16 @@ struct smb_iconv_handle *get_iconv_handle(void)
 {
 	if (global_iconv_handle == NULL)
 		global_iconv_handle = smb_iconv_handle_reinit(talloc_autofree_context(),
-									"ASCII", "UTF-8", "ASCII", true, NULL);
+							      "ASCII", "UTF-8", true, NULL);
 	return global_iconv_handle;
 }
 
 struct smb_iconv_handle *get_iconv_testing_handle(TALLOC_CTX *mem_ctx, 
 						  const char *dos_charset, 
-						  const char *unix_charset, 
-						  const char *display_charset)
+						  const char *unix_charset)
 {
 	return smb_iconv_handle_reinit(mem_ctx,
-				       dos_charset, unix_charset, display_charset, true, NULL);
+				       dos_charset, unix_charset, true, NULL);
 }
 
 /**
@@ -190,7 +189,6 @@ const char *charset_name(struct smb_iconv_handle *ic, charset_t ch)
 	case CH_UTF16: return "UTF-16LE";
 	case CH_UNIX: return ic->unix_charset;
 	case CH_DOS: return ic->dos_charset;
-	case CH_DISPLAY: return ic->display_charset;
 	case CH_UTF8: return "UTF8";
 	case CH_UTF16BE: return "UTF-16BE";
 	case CH_UTF16MUNGED: return "UTF16_MUNGED";
@@ -219,37 +217,6 @@ static int close_iconv_handle(struct smb_iconv_handle *data)
 	return 0;
 }
 
-static const char *map_locale(const char *charset)
-{
-	if (strcmp(charset, "LOCALE") != 0) {
-		return charset;
-	}
-#if defined(HAVE_NL_LANGINFO) && defined(CODESET)
-	{
-		const char *ln;
-		smb_iconv_t handle;
-
-		ln = nl_langinfo(CODESET);
-		if (ln == NULL) {
-			DEBUG(1,("Unable to determine charset for LOCALE - using ASCII\n"));
-			return "ASCII";
-		}
-		/* Check whether the charset name is supported
-		   by iconv */
-		handle = smb_iconv_open(ln, "UCS-2LE");
-		if (handle == (smb_iconv_t) -1) {
-			DEBUG(5,("Locale charset '%s' unsupported, using ASCII instead\n", ln));
-			return "ASCII";
-		} else {
-			DEBUG(5,("Substituting charset '%s' for LOCALE\n", ln));
-			smb_iconv_close(handle);
-		}
-		return ln;
-	}
-#endif
-	return "ASCII";
-}
-
 /*
   the old_ic is passed in here as the smb_iconv_handle structure
   is used as a global pointer in some places (eg. python modules). We
@@ -261,14 +228,11 @@ static const char *map_locale(const char *charset)
 _PUBLIC_ struct smb_iconv_handle *smb_iconv_handle_reinit(TALLOC_CTX *mem_ctx,
 								    const char *dos_charset,
 								    const char *unix_charset,
-								    const char *display_charset,
 								    bool native_iconv,
 								    struct smb_iconv_handle *old_ic)
 {
 	struct smb_iconv_handle *ret;
 
-	display_charset = map_locale(display_charset);
-
 	if (old_ic != NULL) {
 		ret = old_ic;
 		close_iconv_handle(ret);
@@ -290,9 +254,13 @@ _PUBLIC_ struct smb_iconv_handle *smb_iconv_handle_reinit(TALLOC_CTX *mem_ctx,
 
 	talloc_set_destructor(ret, close_iconv_handle);
 
+	if (strcasecmp(dos_charset, "UTF8") == 0 || strcasecmp(dos_charset, "UTF-8") == 0) {
+		DEBUG(0,("ERROR: invalid DOS charset: 'dos charset' must not be UTF8, using (default value) CP850 instead\n"));
+		dos_charset = "CP850";
+	}
+
 	ret->dos_charset = talloc_strdup(ret->child_ctx, dos_charset);
 	ret->unix_charset = talloc_strdup(ret->child_ctx, unix_charset);
-	ret->display_charset = talloc_strdup(ret->child_ctx, display_charset);
 	ret->native_iconv = native_iconv;
 
 	return ret;
diff --git a/lib/util/charset/convert_string.c b/lib/util/charset/convert_string.c
index e51add2aaf9..51f9fec137a 100644
--- a/lib/util/charset/convert_string.c
+++ b/lib/util/charset/convert_string.c
@@ -2,7 +2,8 @@
    Unix SMB/CIFS implementation.
    Character set conversion Extensions
    Copyright (C) Igor Vergeichik <iverg@mail.ru> 2001
-   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Andrew Tridgell 2001-2011
+   Copyright (C) Andrew Bartlett 2011
    Copyright (C) Simo Sorce 2001
    Copyright (C) Martin Pool 2003
 
@@ -21,6 +22,7 @@
 
 */
 #include "includes.h"
+#include "system/iconv.h"
 
 /**
  * @file
@@ -177,28 +179,29 @@ bool convert_string_error_handle(struct smb_iconv_handle *ic,
 		size_t slen = srclen;
 		size_t dlen = destlen;
 		unsigned char lastp = '\0';
+		bool ret;
 
-		/* If all characters are ascii, fast path here. */
-		while (((slen == (size_t)-1) || (slen >= 2)) && dlen) {
-			if (((lastp = *p) <= 0x7f) && (p[1] == 0)) {
+		if (slen == (size_t)-1) {
+			while (dlen &&
+			       ((lastp = *p) <= 0x7f) && (p[1] == 0)) {
 				*q++ = *p;
-				if (slen != (size_t)-1) {
-					slen -= 2;
-				}
 				p += 2;
 				dlen--;
 				retval++;
 				if (!lastp)
 					break;
-			} else {
-#ifdef BROKEN_UNICODE_COMPOSE_CHARACTERS
-				goto general_case;
-#else
-				bool ret = convert_string_internal(ic, from, to, p, slen, q, dlen, converted_size);
-				*converted_size += retval;
-				return ret;
-#endif
 			}
+			if (lastp != 0) goto slow_path;
+		} else {
+			while (slen >= 2 && dlen &&
+			       (*p <= 0x7f) && (p[1] == 0)) {
+				*q++ = *p;
+				slen -= 2;
+				p += 2;
+				dlen--;
+				retval++;
+			}
+			if (slen != 0) goto slow_path;
 		}
 
 		*converted_size = retval;
@@ -212,6 +215,19 @@ bool convert_string_error_handle(struct smb_iconv_handle *ic,
 			}
 		}
 		return true;
+
+	slow_path:
+		/* come here when we hit a character we can't deal
+		 * with in the fast path
+		 */
+#ifdef BROKEN_UNICODE_COMPOSE_CHARACTERS
+		goto general_case;
+#else
+		ret = convert_string_internal(ic, from, to, p, slen, q, dlen, converted_size);
+		*converted_size += retval;
+		return ret;
+#endif
+
 	} else if (from != CH_UTF16LE && from != CH_UTF16BE && to == CH_UTF16LE) {
 		const unsigned char *p = (const unsigned char *)src;
 		unsigned char *q = (unsigned char *)dest;
@@ -221,8 +237,8 @@ bool convert_string_error_handle(struct smb_iconv_handle *ic,
 		unsigned char lastp = '\0';
 
 		/* If all characters are ascii, fast path here. */
-		while (slen && (dlen >= 2)) {
-			if ((lastp = *p) <= 0x7F) {
+		while (slen && (dlen >= 1)) {
+			if (dlen >=2 && (lastp = *p) <= 0x7F) {
 				*q++ = *p++;
 				*q++ = '\0';
 				if (slen != (size_t)-1) {
@@ -387,7 +403,7 @@ bool convert_string_talloc_handle(TALLOC_CTX *ctx, struct smb_iconv_handle *ic,
 	}
 
 	/* +2 is for ucs2 null termination. */
-	ob = (char *)TALLOC_REALLOC(ctx, ob, destlen + 2);
+	ob = talloc_realloc(ctx, ob, char, destlen + 2);
 
 	if (!ob) {
 		DEBUG(0, ("convert_string_talloc: realloc failed!\n"));
@@ -428,7 +444,7 @@ bool convert_string_talloc_handle(TALLOC_CTX *ctx, struct smb_iconv_handle *ic,
 	 */
 	if (o_len > 1024) {
 		/* We're shrinking here so we know the +2 is safe from wrap. */
-		ob = (char *)TALLOC_REALLOC(ctx,ob,destlen + 2);
+		ob = talloc_realloc(ctx,ob, char, destlen + 2);
 	}
 
 	if (destlen && !ob) {
diff --git a/lib/util/charset/pull_push.c b/lib/util/charset/pull_push.c
new file mode 100644
index 00000000000..b7a5bcdc65b
--- /dev/null
+++ b/lib/util/charset/pull_push.c
@@ -0,0 +1,150 @@
+/*
+   Unix SMB/CIFS implementation.
+   Character set conversion Extensions
+   Copyright (C) Igor Vergeichik <iverg@mail.ru> 2001
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Simo Sorce 2001
+   Copyright (C) Martin Pool 2003
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "includes.h"
+#include "system/locale.h"
+
+/**
+ * Copy a string from a unix char* src to a UCS2 destination,
+ * allocating a buffer using talloc().
+ *
+ * @param dest always set at least to NULL
+ * @parm converted_size set to the number of bytes occupied by the string in
+ * the destination on success.
+ *
+ * @return true if new buffer was correctly allocated, and string was
+ * converted.
+ **/
+bool push_ucs2_talloc(TALLOC_CTX *ctx, smb_ucs2_t **dest, const char *src,
+		      size_t *converted_size)
+{
+	size_t src_len = strlen(src)+1;
+
+	*dest = NULL;
+	return convert_string_talloc(ctx, CH_UNIX, CH_UTF16LE, src, src_len,
+				     (void **)dest, converted_size);
+}
+
+/**
+ * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer using talloc
+ *
+ * @param dest always set at least to NULL
+ * @parm converted_size set to the number of bytes occupied by the string in
+ * the destination on success.
+ *
+ * @return true if new buffer was correctly allocated, and string was
+ * converted.
+ **/
+
+bool push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
+		      size_t *converted_size)
+{
+	size_t src_len = strlen(src)+1;
+
+	*dest = NULL;
+	return convert_string_talloc(ctx, CH_UNIX, CH_UTF8, src, src_len,
+				     (void**)dest, converted_size);
+}
+
+/**
+ * Copy a string from a unix char* src to an ASCII destination,
+ * allocating a buffer using talloc().
+ *
+ * @param dest always set at least to NULL
+ *
+ * @param converted_size The number of bytes occupied by the string in the destination
+ * @returns boolean indicating if the conversion was successful
+ **/
+bool push_ascii_talloc(TALLOC_CTX *mem_ctx, char **dest, const char *src, size_t *converted_size)
+{
+	size_t src_len = strlen(src)+1;
+
+	*dest = NULL;
+	return convert_string_talloc(mem_ctx, CH_UNIX, CH_DOS, src, src_len,
+				     (void **)dest, converted_size);
+}
+
+/**
+ * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer using talloc
+ *
+ * @param dest always set at least to NULL
+ * @parm converted_size set to the number of bytes occupied by the string in
+ * the destination on success.
+ *
+ * @return true if new buffer was correctly allocated, and string was
+ * converted.
+ **/
+
+bool pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const smb_ucs2_t *src,
+		      size_t *converted_size)
+{
+	size_t src_len = (strlen_w(src)+1) * sizeof(smb_ucs2_t);
+
+	*dest = NULL;
+	return convert_string_talloc(ctx, CH_UTF16LE, CH_UNIX, src, src_len,
+				     (void **)dest, converted_size);
+}
+
+
+/**
+ * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer using talloc
+ *
+ * @param dest always set at least to NULL
+ * @parm converted_size set to the number of bytes occupied by the string in
+ * the destination on success.
+ *
+ * @return true if new buffer was correctly allocated, and string was
+ * converted.
+ **/
+
+bool pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
+		      size_t *converted_size)
+{
+	size_t src_len = strlen(src)+1;
+
+	*dest = NULL;
+	return convert_string_talloc(ctx, CH_UTF8, CH_UNIX, src, src_len,
+				     (void **)dest, converted_size);
+}
+
+
+/**
+ * Copy a string from a DOS src to a unix char * destination, allocating a buffer using talloc
+ *
+ * @param dest always set at least to NULL
+ * @parm converted_size set to the number of bytes occupied by the string in
+ * the destination on success.
+ *
+ * @return true if new buffer was correctly allocated, and string was
+ * converted.
+ **/
+
+bool pull_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
+		       size_t *converted_size)
+{
+	size_t src_len = strlen(src)+1;
+
+	*dest = NULL;
+	return convert_string_talloc(ctx, CH_DOS, CH_UNIX, src, src_len,
+				     (void **)dest, converted_size);
+}
diff --git a/lib/util/charset/tests/convert_string.c b/lib/util/charset/tests/convert_string.c
index 32fc11f527f..9a5d974fe35 100644
--- a/lib/util/charset/tests/convert_string.c
+++ b/lib/util/charset/tests/convert_string.c
@@ -105,7 +105,7 @@ static bool test_gd_iso8859_cp850_handle(struct torture_context *tctx)
 	talloc_steal(tctx, gd_iso8859_1.data);
 	talloc_steal(tctx, gd_utf16le.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "ISO8859-1", "CP850", "UTF8");
+	iconv_handle = get_iconv_testing_handle(tctx, "ISO8859-1", "CP850");
 	torture_assert(tctx, iconv_handle, "getting iconv handle");
 		
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
@@ -199,11 +199,11 @@ static bool test_gd_iso8859_cp850_handle(struct torture_context *tctx)
 	torture_assert_data_blob_equal(tctx, gd_output, gd_cp850, "conversion from UTF8 to (unix charset) CP850 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_UTF8, CH_DISPLAY, 
+						    CH_UTF8, CH_UTF8, 
 						    gd_utf8.data, gd_utf8.length, 
 						    (void *)&gd_output.data, &gd_output.length), 
-		       "conversion from UTF8 to (display charset) UTF8");
-	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8, "conversion from UTF8 to (display charset) UTF8 incorrect");
+		       "conversion from UTF8 to UTF8");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8, "conversion from UTF8 to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_UTF16LE, CH_DOS, 
@@ -227,11 +227,11 @@ static bool test_gd_iso8859_cp850_handle(struct torture_context *tctx)
 	torture_assert_data_blob_equal(tctx, gd_output, gd_cp850, "conversion from UTF16LE to (unix charset) CP850 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_UTF16LE, CH_DISPLAY, 
+						    CH_UTF16LE, CH_UTF8, 
 						    gd_utf16le.data, gd_utf16le.length, 
 						    (void *)&gd_output.data, &gd_output.length), 
-		       "conversion from UTF16LE to (display charset) UTF8");
-	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8, "conversion from UTF16LE to (display charset) UTF8 incorrect");
+		       "conversion from UTF16LE to UTF8");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8, "conversion from UTF16LE to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_DOS, CH_DOS, 
@@ -248,11 +248,11 @@ static bool test_gd_iso8859_cp850_handle(struct torture_context *tctx)
 	torture_assert_data_blob_equal(tctx, gd_output, gd_cp850, "conversion from UTF16LE to (unix charset) CP850 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_DOS, CH_DISPLAY, 
+						    CH_DOS, CH_UTF8, 
 						    gd_iso8859_1.data, gd_iso8859_1.length, 
 						    (void *)&gd_output.data, &gd_output.length), 
-		       "conversion from (dos charset) ISO8859-1 to (display charset) UTF8");
-	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8, "conversion from UTF16LE to (display charset) UTF8 incorrect");
+		       "conversion from (dos charset) ISO8859-1 to UTF8");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8, "conversion from UTF16LE to UTF8 incorrect");
 
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_DOS, CH_UTF16LE, 
@@ -265,7 +265,7 @@ static bool test_gd_iso8859_cp850_handle(struct torture_context *tctx)
 						     (const char *)gd_iso8859_1.data,
 						     CH_DOS, CH_UTF16LE),
 				 gd_output.length / 2,
-				 "checking strlen_m_ext of round trip conversion of UTF16 latin charset greek to display charset UTF8 and back again");
+				 "checking strlen_m_ext of round trip conversion of UTF16 latin charset greek to UTF8 and back again");
 
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle,
 						    CH_DOS, CH_UTF8,
@@ -282,6 +282,191 @@ static bool test_gd_iso8859_cp850_handle(struct torture_context *tctx)
 	return true;
 }
 
+static bool test_gd_minus_1_handle(struct torture_context *tctx)
+{
+	struct smb_iconv_handle *iconv_handle;
+	DATA_BLOB gd_utf8 = base64_decode_data_blob(gd_utf8_base64);
+	DATA_BLOB gd_cp850 = base64_decode_data_blob(gd_cp850_base64);
+	DATA_BLOB gd_utf16le = base64_decode_data_blob(gd_utf16le_base64);
+	DATA_BLOB gd_output;
+	DATA_BLOB gd_utf8_terminated;
+	DATA_BLOB gd_cp850_terminated;
+	DATA_BLOB gd_utf16le_terminated;
+	
+	talloc_steal(tctx, gd_utf8.data);
+	talloc_steal(tctx, gd_cp850.data);
+	talloc_steal(tctx, gd_utf16le.data);
+
+	iconv_handle = get_iconv_testing_handle(tctx, "CP850", "CP850");
+	torture_assert(tctx, iconv_handle, "getting iconv handle");
+
+	gd_utf8_terminated = data_blob_talloc(tctx, NULL, gd_utf8.length + 1);
+	memcpy(gd_utf8_terminated.data, gd_utf8.data, gd_utf8.length);
+	gd_utf8_terminated.data[gd_utf8.length] = '\0';
+
+	gd_cp850_terminated = data_blob_talloc(tctx, NULL, gd_cp850.length + 1);
+	memcpy(gd_cp850_terminated.data, gd_cp850.data, gd_cp850.length);
+	gd_cp850_terminated.data[gd_cp850.length] = '\0';
+
+	gd_utf16le_terminated = data_blob_talloc(tctx, NULL, gd_utf16le.length + 2);
+	memcpy(gd_utf16le_terminated.data, gd_utf16le.data, gd_utf16le.length);
+	gd_utf16le_terminated.data[gd_utf16le.length] = '\0';
+	gd_utf16le_terminated.data[gd_utf16le.length + 1] = '\0';
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  gd_utf8_terminated.data, -1,
+							 (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf16le.length + 10);
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  gd_utf8_terminated.data, -1,
+							  (void *)gd_output.data, gd_utf16le.length, &gd_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf16le, "conversion from UTF8 to UTF16LE null terminated");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf16le.length + 10);
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  gd_utf8_terminated.data, -1,
+							  (void *)gd_output.data, gd_utf16le.length - 1, &gd_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf16le.length + 10);
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  gd_utf8_terminated.data, -1,
+							  (void *)gd_output.data, gd_utf16le.length - 2, &gd_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf8.length + 10);
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_utf8.length, &gd_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8, "conversion from UTF16LE to UTF8 null terminated");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_utf8.length - 1, &gd_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_utf8.length - 2, &gd_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_cp850.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_DOS,
+							 gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF16LE to CP850 (dos) null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_cp850_terminated, "conversion from UTF16LE to CP850 (dos) null terminated");
+
+	/* Now null terminate the string early, the confirm we don't skip the NULL and convert any further */
+	gd_utf8_terminated.data[3] = '\0';
+	gd_utf8_terminated.length = 4; /* used for the comparison only */
+
+	gd_cp850_terminated.data[2] = '\0';
+	gd_cp850_terminated.length = 3; /* used for the comparison only */
+
+	gd_utf16le_terminated.data[4] = '\0';
+	gd_utf16le_terminated.data[5] = '\0';
+	gd_utf16le_terminated.length = 6; /* used for the comparison only */
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  gd_utf8_terminated.data, -1,
+							  (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated early");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF16LE, CH_UTF8,
+							  gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated early");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_DOS, CH_UTF16LE,
+							  gd_cp850_terminated.data, -1,
+							  (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from CP850 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated early");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_cp850.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF16LE, CH_DOS,
+							  gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_cp850_terminated, "conversion from UTF16LE to UTF8 null terminated early");
+	
+	/* Now null terminate the string particularly early, the confirm we don't skip the NULL and convert any further */
+	gd_utf8_terminated.data[1] = '\0';
+	gd_utf8_terminated.length = 2; /* used for the comparison only */
+	
+	gd_utf16le_terminated.data[2] = '\0';
+	gd_utf16le_terminated.data[3] = '\0';
+	gd_utf16le_terminated.length = 4; /* used for the comparison only */
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle, CH_UTF8, CH_UTF16LE,
+							  gd_utf8_terminated.data, -1,
+							 (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated very early");
+
+	gd_output = data_blob_talloc(tctx, NULL, gd_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF16LE, CH_UTF8,
+							  gd_utf16le_terminated.data, -1,
+							 (void *)gd_output.data, gd_output.length, &gd_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, gd_output, gd_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated very early");
+
+	return true;
+}
+
 static bool test_gd_ascii_handle(struct torture_context *tctx)
 {
 	struct smb_iconv_handle *iconv_handle;
@@ -296,7 +481,7 @@ static bool test_gd_ascii_handle(struct torture_context *tctx)
 	talloc_steal(tctx, gd_iso8859_1.data);
 	talloc_steal(tctx, gd_utf16le.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "UTF8", "UTF8");
+	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "UTF8");
 	torture_assert(tctx, iconv_handle, "getting iconv handle");
 
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle,
@@ -365,7 +550,7 @@ static bool test_plato_english_iso8859_cp850_handle(struct torture_context *tctx
 	
 	talloc_steal(tctx, plato_english_utf16le.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "ISO8859-1", "CP850", "UTF8");
+	iconv_handle = get_iconv_testing_handle(tctx, "ISO8859-1", "CP850");
 	torture_assert(tctx, iconv_handle, "getting iconv handle");
 		
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
@@ -383,11 +568,11 @@ static bool test_plato_english_iso8859_cp850_handle(struct torture_context *tctx
 	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_cp850, "conversion from UTF8 to (unix charset) CP850 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_UTF8, CH_DISPLAY, 
+						    CH_UTF8, CH_UTF8, 
 						    plato_english_utf8.data, plato_english_utf8.length, 
 						    (void *)&plato_english_output.data, &plato_english_output.length), 
-		       "conversion from UTF8 to (display charset) UTF8");
-	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8, "conversion from UTF8 to (display charset) UTF8 incorrect");
+		       "conversion from UTF8 to UTF8");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8, "conversion from UTF8 to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_UTF16LE, CH_DOS, 
@@ -436,11 +621,11 @@ static bool test_plato_english_iso8859_cp850_handle(struct torture_context *tctx
 	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_cp850, "conversion from UTF16LE to (unix charset) CP850 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_UTF16LE, CH_DISPLAY, 
+						    CH_UTF16LE, CH_UTF8, 
 						    plato_english_utf16le.data, plato_english_utf16le.length, 
 						    (void *)&plato_english_output.data, &plato_english_output.length), 
-		       "conversion from UTF16LE to (display charset) UTF8");
-	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8, "conversion from UTF16LE to (display charset) UTF8 incorrect");
+		       "conversion from UTF16LE to UTF8");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8, "conversion from UTF16LE to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_DOS, CH_DOS, 
@@ -457,11 +642,11 @@ static bool test_plato_english_iso8859_cp850_handle(struct torture_context *tctx
 	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_cp850, "conversion from UTF16LE to (unix charset) CP850 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_DOS, CH_DISPLAY, 
+						    CH_DOS, CH_UTF8, 
 						    plato_english_iso8859_1.data, plato_english_iso8859_1.length, 
 						    (void *)&plato_english_output.data, &plato_english_output.length), 
-		       "conversion from (dos charset) ISO8859-1 to (display charset) UTF8");
-	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8, "conversion from UTF16LE to (display charset) UTF8 incorrect");
+		       "conversion from (dos charset) ISO8859-1 to UTF8");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8, "conversion from UTF16LE to UTF8 incorrect");
 
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_DOS, CH_UTF16LE, 
@@ -472,6 +657,261 @@ static bool test_plato_english_iso8859_cp850_handle(struct torture_context *tctx
 	return true;
 }
 
+static bool test_plato_english_minus_1_handle(struct torture_context *tctx)
+{
+	struct smb_iconv_handle *iconv_handle;
+	DATA_BLOB plato_english_utf8 = data_blob_string_const(plato_english_ascii);
+	DATA_BLOB plato_english_utf16le = base64_decode_data_blob(plato_english_utf16le_base64);
+	DATA_BLOB plato_english_output;
+	DATA_BLOB plato_english_utf8_terminated;
+	DATA_BLOB plato_english_utf16le_terminated;
+	
+	talloc_steal(tctx, plato_english_utf16le.data);
+
+	iconv_handle = get_iconv_testing_handle(tctx, "ISO8859-1", "CP850");
+	torture_assert(tctx, iconv_handle, "getting iconv handle");
+
+	plato_english_utf8_terminated = data_blob_talloc(tctx, NULL, plato_english_utf8.length + 1);
+	memcpy(plato_english_utf8_terminated.data, plato_english_utf8.data, plato_english_utf8.length);
+	plato_english_utf8_terminated.data[plato_english_utf8.length] = '\0';
+
+	plato_english_utf16le_terminated = data_blob_talloc(tctx, NULL, plato_english_utf16le.length + 2);
+	memcpy(plato_english_utf16le_terminated.data, plato_english_utf16le.data, plato_english_utf16le.length);
+	plato_english_utf16le_terminated.data[plato_english_utf16le.length] = '\0';
+	plato_english_utf16le_terminated.data[plato_english_utf16le.length + 1] = '\0';
+		
+	plato_english_output = data_blob_talloc(tctx, NULL, plato_english_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_english_utf8_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_output.length, &plato_english_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_english_utf8_terminated.data, -1,
+							  (void *)plato_english_output.data, plato_english_utf16le.length, &plato_english_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf16le, "conversion from UTF8 to UTF16LE null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_english_utf8_terminated.data, -1,
+							  (void *)plato_english_output.data, plato_english_utf16le.length - 1, &plato_english_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_english_utf8_terminated.data, -1,
+							  (void *)plato_english_output.data, plato_english_utf16le.length - 2, &plato_english_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+
+	plato_english_output = data_blob_talloc(tctx, NULL, plato_english_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_english_utf16le_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_output.length, &plato_english_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_english_utf16le_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_utf8.length, &plato_english_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8, "conversion from UTF16LE to UTF8 null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_english_utf16le_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_utf8.length - 1, &plato_english_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_english_utf16le_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_utf8.length - 2, &plato_english_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+
+	/* Now null terminate the string early, the confirm we don't skip the NULL and convert any further */
+	plato_english_utf8_terminated.data[3] = '\0';
+	plato_english_utf8_terminated.length = 4; /* used for the comparison only */
+
+	plato_english_utf16le_terminated.data[6] = '\0';
+	plato_english_utf16le_terminated.data[7] = '\0';
+	plato_english_utf16le_terminated.length = 8; /* used for the comparison only */
+
+	plato_english_output = data_blob_talloc(tctx, NULL, plato_english_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_english_utf8_terminated.data, -1,
+							  (void *)plato_english_output.data, plato_english_output.length, &plato_english_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated early");
+
+	plato_english_output = data_blob_talloc(tctx, NULL, plato_english_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF16LE, CH_UTF8,
+							  plato_english_utf16le_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_output.length, &plato_english_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated early");
+
+	
+	/* Now null terminate the string particularly early, the confirm we don't skip the NULL and convert any further */
+	plato_english_utf8_terminated.data[1] = '\0';
+	plato_english_utf8_terminated.length = 2; /* used for the comparison only */
+	
+	plato_english_utf16le_terminated.data[2] = '\0';
+	plato_english_utf16le_terminated.data[3] = '\0';
+	plato_english_utf16le_terminated.length = 4; /* used for the comparison only */
+
+	plato_english_output = data_blob_talloc(tctx, NULL, plato_english_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle, CH_UTF8, CH_UTF16LE,
+							  plato_english_utf8_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_output.length, &plato_english_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated very early");
+
+	plato_english_output = data_blob_talloc(tctx, NULL, plato_english_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF16LE, CH_UTF8,
+							  plato_english_utf16le_terminated.data, -1,
+							 (void *)plato_english_output.data, plato_english_output.length, &plato_english_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, plato_english_output, plato_english_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated very early");
+
+	return true;
+}
+
+static bool test_plato_minus_1_handle(struct torture_context *tctx)
+{
+	struct smb_iconv_handle *iconv_handle;
+	DATA_BLOB plato_utf8 = base64_decode_data_blob(plato_utf8_base64);
+	DATA_BLOB plato_utf16le = base64_decode_data_blob(plato_utf16le_base64);
+	DATA_BLOB plato_output;
+	DATA_BLOB plato_utf8_terminated;
+	DATA_BLOB plato_utf16le_terminated;
+	
+	talloc_steal(tctx, plato_utf8.data);
+	talloc_steal(tctx, plato_utf16le.data);
+
+	iconv_handle = get_iconv_testing_handle(tctx, "ISO8859-1", "CP850");
+	torture_assert(tctx, iconv_handle, "getting iconv handle");
+
+	plato_utf8_terminated = data_blob_talloc(tctx, NULL, plato_utf8.length + 1);
+	memcpy(plato_utf8_terminated.data, plato_utf8.data, plato_utf8.length);
+	plato_utf8_terminated.data[plato_utf8.length] = '\0';
+
+	plato_utf16le_terminated = data_blob_talloc(tctx, NULL, plato_utf16le.length + 2);
+	memcpy(plato_utf16le_terminated.data, plato_utf16le.data, plato_utf16le.length);
+	plato_utf16le_terminated.data[plato_utf16le.length] = '\0';
+	plato_utf16le_terminated.data[plato_utf16le.length + 1] = '\0';
+
+	plato_output = data_blob_talloc(tctx, NULL, plato_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_utf8_terminated.data, -1,
+							 (void *)plato_output.data, plato_output.length, &plato_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_utf8_terminated.data, -1,
+							  (void *)plato_output.data, plato_utf16le.length, &plato_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf16le, "conversion from UTF8 to UTF16LE null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_utf8_terminated.data, -1,
+							  (void *)plato_output.data, plato_utf16le.length - 1, &plato_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_utf8_terminated.data, -1,
+							  (void *)plato_output.data, plato_utf16le.length - 2, &plato_output.length) == false,
+		       "conversion from UTF8 to UTF16LE null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF8 to UTF16LE should fail E2BIG");
+
+	plato_output = data_blob_talloc(tctx, NULL, plato_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_utf16le_terminated.data, -1,
+							 (void *)plato_output.data, plato_output.length, &plato_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_utf16le_terminated.data, -1,
+							 (void *)plato_output.data, plato_utf8.length, &plato_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8, "conversion from UTF16LE to UTF8 null terminated");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_utf16le_terminated.data, -1,
+							 (void *)plato_output.data, plato_utf8.length - 1, &plato_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							 CH_UTF16LE, CH_UTF8,
+							 plato_utf16le_terminated.data, -1,
+							 (void *)plato_output.data, plato_utf8.length - 2, &plato_output.length) == false,
+		       "conversion from UTF16LE to UTF8 null terminated should fail");
+	torture_assert_errno_equal(tctx, E2BIG, "conversion from UTF16LE to UTF8 should fail E2BIG");
+
+	/* Now null terminate the string early, the confirm we don't skip the NULL and convert any further */
+	plato_utf8_terminated.data[5] = '\0';
+	plato_utf8_terminated.length = 6; /* used for the comparison only */
+
+	plato_utf16le_terminated.data[4] = '\0';
+	plato_utf16le_terminated.data[5] = '\0';
+	plato_utf16le_terminated.length = 6; /* used for the comparison only */
+
+	plato_output = data_blob_talloc(tctx, NULL, plato_utf16le.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF8, CH_UTF16LE,
+							  plato_utf8_terminated.data, -1,
+							  (void *)plato_output.data, plato_output.length, &plato_output.length),
+		       "conversion from UTF8 to UTF16LE null terminated");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf16le_terminated, "conversion from UTF8 to UTF16LE null terminated early");
+
+	plato_output = data_blob_talloc(tctx, NULL, plato_utf8.length + 10);
+
+	torture_assert(tctx, convert_string_error_handle(iconv_handle,
+							  CH_UTF16LE, CH_UTF8,
+							  plato_utf16le_terminated.data, -1,
+							 (void *)plato_output.data, plato_output.length, &plato_output.length),
+		       "conversion from UTF16LE to UTF8 null terminated");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8_terminated, "conversion from UTF16LE to UTF8 null terminated early");
+	
+	return true;
+}
+
 static bool test_plato_cp850_utf8_handle(struct torture_context *tctx)
 {
 	struct smb_iconv_handle *iconv_handle;
@@ -483,7 +923,7 @@ static bool test_plato_cp850_utf8_handle(struct torture_context *tctx)
 	talloc_steal(tctx, plato_utf8.data);
 	talloc_steal(tctx, plato_utf16le.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "CP850", "UTF8", "UTF8");
+	iconv_handle = get_iconv_testing_handle(tctx, "CP850", "UTF8");
 	torture_assert(tctx, iconv_handle, "creating iconv handle");
 		
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
@@ -568,11 +1008,11 @@ static bool test_plato_cp850_utf8_handle(struct torture_context *tctx)
 	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8, "conversion from UTF8 to (unix charset) UTF8 incorrect");
 
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_UTF8, CH_DISPLAY, 
+						    CH_UTF8, CH_UTF8, 
 						    plato_utf8.data, plato_utf8.length, 
 						    (void *)&plato_output.data, &plato_output.length),
 		       "conversion of UTF16 ancient greek to unix charset UTF8 failed");
-	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8, "conversion from UTF8 to (display charset) UTF8 incorrect");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8, "conversion from UTF8 to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_UTF16LE, CH_DOS, 
@@ -627,39 +1067,39 @@ static bool test_plato_cp850_utf8_handle(struct torture_context *tctx)
 		       "conversion of UTF16 ancient greek to UTF8 failed");
 	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8, "conversion from UTF16LE to UTF8 incorrect");
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-							  CH_UTF16LE, CH_DISPLAY, 
+							  CH_UTF16LE, CH_UTF8, 
 							  plato_utf16le.data, plato_utf16le.length, 
 							  (void *)&plato_output.data, &plato_output.length),
-		       "conversion of UTF16 ancient greek to display charset UTF8 failed");
-	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8, "conversion from UTF16LE to (display charset) UTF8 incorrect");
+		       "conversion of UTF16 ancient greek to UTF8 failed");
+	torture_assert_data_blob_equal(tctx, plato_output, plato_utf8, "conversion from UTF16LE to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-							  CH_DISPLAY, CH_UTF16LE, 
+							  CH_UTF8, CH_UTF16LE, 
 							  plato_output.data, plato_output.length, 
 							  (void *)&plato_output2.data, &plato_output2.length),
-		       "round trip conversion of UTF16 ancient greek to display charset UTF8 and back again failed");
+		       "round trip conversion of UTF16 ancient greek to UTF8 and back again failed");
 	torture_assert_data_blob_equal(tctx, plato_output2, plato_utf16le,
-				       "round trip conversion of UTF16 ancient greek to display charset UTF8 and back again failed");
+				       "round trip conversion of UTF16 ancient greek to UTF8 and back again failed");
 	torture_assert_int_equal(tctx,
 				 strlen_m_ext_handle(iconv_handle,
 						     (const char *)plato_output.data,
-						     CH_DISPLAY, CH_UTF16LE),
+						     CH_UTF8, CH_UTF16LE),
 				 plato_output2.length / 2,
-				 "checking strlen_m_ext of round trip conversion of UTF16 latin charset greek to display charset UTF8 and back again");
+				 "checking strlen_m_ext of round trip conversion of UTF16 latin charset greek to UTF8 and back again");
 
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle,
-							  CH_DISPLAY, CH_UTF8,
+							  CH_UTF8, CH_UTF8,
 							  plato_output.data, plato_output.length,
 							  (void *)&plato_output2.data, &plato_output2.length),
-		       "conversion of display charset UTF8 to UTF8");
+		       "conversion of UTF8 to UTF8");
 	torture_assert_data_blob_equal(tctx, plato_output2, plato_utf8,
-				       "conversion of display charset UTF8 to UTF8");
+				       "conversion of UTF8 to UTF8");
 	torture_assert_int_equal(tctx,
 				 strlen_m_ext_handle(iconv_handle,
 						     (const char *)plato_output.data,
-						     CH_DISPLAY, CH_UTF8),
+						     CH_UTF8, CH_UTF8),
 				 plato_output2.length,
-				 "checking strlen_m_ext of conversion of display charset UTF8 to UTF8");
+				 "checking strlen_m_ext of conversion of UTF8 to UTF8");
 	return true;
 }
 
@@ -674,7 +1114,7 @@ static bool test_plato_latin_cp850_utf8_handle(struct torture_context *tctx)
 	talloc_steal(tctx, plato_latin_utf8.data);
 	talloc_steal(tctx, plato_latin_utf16le.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "CP850", "UTF8", "UTF8");
+	iconv_handle = get_iconv_testing_handle(tctx, "CP850", "UTF8");
 	torture_assert(tctx, iconv_handle, "creating iconv handle");
 		
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
@@ -691,11 +1131,11 @@ static bool test_plato_latin_cp850_utf8_handle(struct torture_context *tctx)
 	torture_assert_data_blob_equal(tctx, plato_latin_output, plato_latin_utf8, "conversion from UTF8 to (unix charset) UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-						    CH_UTF8, CH_DISPLAY, 
+						    CH_UTF8, CH_UTF8, 
 						    plato_latin_utf8.data, plato_latin_utf8.length, 
 						    (void *)&plato_latin_output.data, &plato_latin_output.length),
 		       "conversion of UTF16 latin charset greek to unix charset UTF8 failed");
-	torture_assert_data_blob_equal(tctx, plato_latin_output, plato_latin_utf8, "conversion from UTF8 to (display charset) UTF8 incorrect");
+	torture_assert_data_blob_equal(tctx, plato_latin_output, plato_latin_utf8, "conversion from UTF8 to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
 						    CH_UTF16LE, CH_DOS, 
@@ -711,25 +1151,25 @@ static bool test_plato_latin_cp850_utf8_handle(struct torture_context *tctx)
 	torture_assert_data_blob_equal(tctx, plato_latin_output, plato_latin_utf8, "conversion from UTF16LE to (unix charset) CP850 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-							  CH_UTF16LE, CH_DISPLAY, 
+							  CH_UTF16LE, CH_UTF8, 
 							  plato_latin_utf16le.data, plato_latin_utf16le.length, 
 							  (void *)&plato_latin_output.data, &plato_latin_output.length),
-		       "conversion of UTF16 latin charset greek to display charset UTF8 failed");
-	torture_assert_data_blob_equal(tctx, plato_latin_output, plato_latin_utf8, "conversion from UTF16LE to (display charset) UTF8 incorrect");
+		       "conversion of UTF16 latin charset greek to UTF8 failed");
+	torture_assert_data_blob_equal(tctx, plato_latin_output, plato_latin_utf8, "conversion from UTF16LE to UTF8 incorrect");
 	
 	torture_assert(tctx, convert_string_talloc_handle(tctx, iconv_handle, 
-							  CH_DISPLAY, CH_UTF16LE, 
+							  CH_UTF8, CH_UTF16LE, 
 							  plato_latin_output.data, plato_latin_output.length, 
 							  (void *)&plato_latin_output2.data, &plato_latin_output2.length),
-		       "round trip conversion of UTF16 latin charset greek to display charset UTF8 and back again failed");
+		       "round trip conversion of UTF16 latin charset greek to UTF8 and back again failed");
 	torture_assert_data_blob_equal(tctx, plato_latin_output2, plato_latin_utf16le,
-				       "round trip conversion of UTF16 latin charset greek to display charset UTF8 and back again failed");
+				       "round trip conversion of UTF16 latin charset greek to UTF8 and back again failed");
 	torture_assert_int_equal(tctx,
 				 strlen_m_ext_handle(iconv_handle,
 						     (const char *)plato_latin_output.data,
-						     CH_DISPLAY, CH_UTF16LE),
+						     CH_UTF8, CH_UTF16LE),
 				 plato_latin_output2.length / 2,
-				 "checking strlen_m_ext of round trip conversion of UTF16 latin charset greek to display charset UTF8 and back again");
+				 "checking strlen_m_ext of round trip conversion of UTF16 latin charset greek to UTF8 and back again");
 	return true;
 }
 
@@ -742,7 +1182,7 @@ static bool test_gd_case_utf8_handle(struct torture_context *tctx)
 	char *gd_lower, *gd_upper;
 	talloc_steal(tctx, gd_utf8.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "UTF8", "UTF8");
+	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "UTF8");
 	torture_assert(tctx, iconv_handle, "getting utf8 iconv handle");
 
 	torture_assert(tctx,
@@ -805,7 +1245,7 @@ static bool test_gd_case_cp850_handle(struct torture_context *tctx)
 	char *gd_lower, *gd_upper;
 	talloc_steal(tctx, gd_cp850.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "CP850", "CP850");
+	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "CP850");
 	torture_assert(tctx, iconv_handle, "getting cp850 iconv handle");
 
 	torture_assert(tctx,
@@ -866,7 +1306,7 @@ static bool test_plato_case_utf8_handle(struct torture_context *tctx)
 	char *plato_lower, *plato_upper;
 	talloc_steal(tctx, plato_utf8.data);
 
-	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "UTF8", "UTF8");
+	iconv_handle = get_iconv_testing_handle(tctx, "ASCII", "UTF8");
 	torture_assert(tctx, iconv_handle, "getting utf8 iconv handle");
 
 	torture_assert(tctx,
@@ -1248,9 +1688,12 @@ struct torture_suite *torture_local_convert_string_handle(TALLOC_CTX *mem_ctx)
 	struct torture_suite *suite = torture_suite_create(mem_ctx, "convert_string_handle");
 
 	torture_suite_add_simple_test(suite, "gd_ascii", test_gd_ascii_handle);
+	torture_suite_add_simple_test(suite, "gd_minus_1", test_gd_minus_1_handle);
 	torture_suite_add_simple_test(suite, "gd_iso8859_cp850", test_gd_iso8859_cp850_handle);
 	torture_suite_add_simple_test(suite, "plato_english_iso8859_cp850", test_plato_english_iso8859_cp850_handle);
+	torture_suite_add_simple_test(suite, "plato_english_minus_1", test_plato_english_minus_1_handle);
 	torture_suite_add_simple_test(suite, "plato_cp850_utf8", test_plato_cp850_utf8_handle);
+	torture_suite_add_simple_test(suite, "plato_minus_1", test_plato_minus_1_handle);
 	torture_suite_add_simple_test(suite, "plato_latin_cp850_utf8", test_plato_latin_cp850_utf8_handle);
 	return suite;
 }
diff --git a/lib/util/charset/util_str.c b/lib/util/charset/util_str.c
index e8f0b788b1c..688ab5a0a1c 100644
--- a/lib/util/charset/util_str.c
+++ b/lib/util/charset/util_str.c
@@ -5,6 +5,8 @@
    Copyright (C) Simo Sorce 2001
    Copyright (C) Andrew Bartlett 2011
    Copyright (C) Jeremy Allison  1992-2007
+   Copyright (C) Martin Pool     2003
+   Copyright (C) James Peach	 2006
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -167,7 +169,6 @@ _PUBLIC_ size_t strlen_m_ext_handle(struct smb_iconv_handle *ic,
 	switch (dst_charset) {
 	case CH_DOS:
 	case CH_UNIX:
-	case CH_DISPLAY:
 		smb_panic("cannot call strlen_m_ext() with a variable dest charset (must be UTF16* or UTF8)");
 	default:
 		break;
@@ -327,7 +328,7 @@ _PUBLIC_ char *strchr_m(const char *src, char c)
 
 	for (s = src; *s && !(((unsigned char)s[0]) & 0x80); s++) {
 		if (*s == c)
-			return (char *)s;
+			return discard_const_p(char, s);
 	}
 
 	if (!*s)
@@ -395,7 +396,7 @@ _PUBLIC_ char *strrchr_m(const char *s, char c)
 					break;
 				}
 				/* No - we have a match ! */
-				return (char *)cp;
+				return discard_const_p(char , cp);
 			}
 		} while (cp-- != s);
 		if (!got_mb)
@@ -473,3 +474,84 @@ _PUBLIC_ bool strhasupper(const char *string)
 	struct smb_iconv_handle *ic = get_iconv_handle();
 	return strhasupper_handle(ic, string);
 }
+
+/***********************************************************************
+ strstr_m - We convert via ucs2 for now.
+***********************************************************************/
+
+char *strstr_m(const char *src, const char *findstr)
+{
+	smb_ucs2_t *p;
+	smb_ucs2_t *src_w, *find_w;
+	const char *s;
+	char *s2;
+	char *retp;
+	size_t converted_size, findstr_len = 0;
+
+	TALLOC_CTX *frame; /* Only set up in the iconv case */
+
+	/* for correctness */
+	if (!findstr[0]) {
+		return discard_const_p(char, src);
+	}
+
+	/* Samba does single character findstr calls a *lot*. */
+	if (findstr[1] == '\0')
+		return strchr_m(src, *findstr);
+
+	/* We optimise for the ascii case, knowing that all our
+	   supported multi-byte character sets are ascii-compatible
+	   (ie. they match for the first 128 chars) */
+
+	for (s = src; *s && !(((unsigned char)s[0]) & 0x80); s++) {
+		if (*s == *findstr) {
+			if (!findstr_len)
+				findstr_len = strlen(findstr);
+
+			if (strncmp(s, findstr, findstr_len) == 0) {
+				return discard_const_p(char, s);
+			}
+		}
+	}
+
+	if (!*s)
+		return NULL;
+
+#if 1 /* def BROKEN_UNICODE_COMPOSE_CHARACTERS */
+	/* 'make check' fails unless we do this */
+
+	/* With compose characters we must restart from the beginning. JRA. */
+	s = src;
+#endif
+
+	frame = talloc_stackframe();
+
+	if (!push_ucs2_talloc(frame, &src_w, src, &converted_size)) {
+		DEBUG(0,("strstr_m: src malloc fail\n"));
+		TALLOC_FREE(frame);
+		return NULL;
+	}
+
+	if (!push_ucs2_talloc(frame, &find_w, findstr, &converted_size)) {
+		DEBUG(0,("strstr_m: find malloc fail\n"));
+		TALLOC_FREE(frame);
+		return NULL;
+	}
+
+	p = strstr_w(src_w, find_w);
+
+	if (!p) {
+		TALLOC_FREE(frame);
+		return NULL;
+	}
+
+	*p = 0;
+	if (!pull_ucs2_talloc(frame, &s2, src_w, &converted_size)) {
+		TALLOC_FREE(frame);
+		DEBUG(0,("strstr_m: dest malloc fail\n"));
+		return NULL;
+	}
+	retp = discard_const_p(char, (s+strlen(s2)));
+	TALLOC_FREE(frame);
+	return retp;
+}
diff --git a/lib/util/charset/util_unistr.c b/lib/util/charset/util_unistr.c
index a1be501c7ce..e4ae65053c7 100644
--- a/lib/util/charset/util_unistr.c
+++ b/lib/util/charset/util_unistr.c
@@ -161,85 +161,6 @@ _PUBLIC_ char *talloc_strdup_upper(TALLOC_CTX *ctx, const char *src)
 }
 
 /**
- Convert a string to lower case.
-**/
-_PUBLIC_ void strlower_m(char *s)
-{
-	char *d;
-	struct smb_iconv_handle *iconv_handle;
-
-	/* this is quite a common operation, so we want it to be
-	   fast. We optimise for the ascii case, knowing that all our
-	   supported multi-byte character sets are ascii-compatible
-	   (ie. they match for the first 128 chars) */
-	while (*s && !(((uint8_t)*s) & 0x80)) {
-		*s = tolower((uint8_t)*s);
-		s++;
-	}
-
-	if (!*s)
-		return;
-
-	iconv_handle = get_iconv_handle();
-
-	d = s;
-
-	while (*s) {
-		size_t c_size, c_size2;
-		codepoint_t c = next_codepoint_handle(iconv_handle, s, &c_size);
-		c_size2 = push_codepoint_handle(iconv_handle, d, tolower_m(c));
-		if (c_size2 > c_size) {
-			DEBUG(0,("FATAL: codepoint 0x%x (0x%x) expanded from %d to %d bytes in strlower_m\n",
-				 c, tolower_m(c), (int)c_size, (int)c_size2));
-			smb_panic("codepoint expansion in strlower_m\n");
-		}
-		s += c_size;
-		d += c_size2;
-	}
-	*d = 0;
-}
-
-/**
- Convert a string to UPPER case.
-**/
-_PUBLIC_ void strupper_m(char *s)
-{
-	char *d;
-	struct smb_iconv_handle *iconv_handle;
-
-	/* this is quite a common operation, so we want it to be
-	   fast. We optimise for the ascii case, knowing that all our
-	   supported multi-byte character sets are ascii-compatible
-	   (ie. they match for the first 128 chars) */
-	while (*s && !(((uint8_t)*s) & 0x80)) {
-		*s = toupper((uint8_t)*s);
-		s++;
-	}
-
-	if (!*s)
-		return;
-
-	iconv_handle = get_iconv_handle();
-
-	d = s;
-
-	while (*s) {
-		size_t c_size, c_size2;
-		codepoint_t c = next_codepoint_handle(iconv_handle, s, &c_size);
-		c_size2 = push_codepoint_handle(iconv_handle, d, toupper_m(c));
-		if (c_size2 > c_size) {
-			DEBUG(0,("FATAL: codepoint 0x%x (0x%x) expanded from %d to %d bytes in strupper_m\n",
-				 c, toupper_m(c), (int)c_size, (int)c_size2));
-			smb_panic("codepoint expansion in strupper_m\n");
-		}
-		s += c_size;
-		d += c_size2;
-	}
-	*d = 0;
-}
-
-
-/**
  Find the number of 'c' chars in a string
 **/
 _PUBLIC_ size_t count_chars_m(const char *s, char c)
@@ -273,7 +194,7 @@ _PUBLIC_ size_t count_chars_m(const char *s, char c)
  * @param dest_len the maximum length in bytes allowed in the
  * destination.  If @p dest_len is -1 then no maximum is used.
  **/
-static bool push_ascii(void *dest, const char *src, size_t dest_len, int flags, size_t *converted_size)
+static bool push_ascii_string(void *dest, const char *src, size_t dest_len, int flags, size_t *converted_size)
 {
 	size_t src_len;
 	bool ret;
@@ -283,7 +204,7 @@ static bool push_ascii(void *dest, const char *src, size_t dest_len, int flags,
 		if (tmpbuf == NULL) {
 			return false;
 		}
-		ret = push_ascii(dest, tmpbuf, dest_len, flags & ~STR_UPPER, converted_size);
+		ret = push_ascii_string(dest, tmpbuf, dest_len, flags & ~STR_UPPER, converted_size);
 		talloc_free(tmpbuf);
 		return ret;
 	}
@@ -297,23 +218,6 @@ static bool push_ascii(void *dest, const char *src, size_t dest_len, int flags,
 }
 
 /**
- * Copy a string from a unix char* src to an ASCII destination,
- * allocating a buffer using talloc().
- *
- * @param dest always set at least to NULL 
- *
- * @returns The number of bytes occupied by the string in the destination
- *         or -1 in case of error.
- **/
-_PUBLIC_ bool push_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src, size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UNIX, CH_DOS, src, src_len, (void **)dest, converted_size);
-}
-
-
-/**
  * Copy a string from a dos codepage source to a unix char* destination.
  *
  * The resulting string in "dest" is always null terminated.
@@ -328,7 +232,7 @@ _PUBLIC_ bool push_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src, s
  * @param src_len is the length of the source area in bytes.
  * @returns the number of bytes occupied by the string in @p src.
  **/
-static ssize_t pull_ascii(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
+static ssize_t pull_ascii_string(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
 {
 	size_t size = 0;
 
@@ -411,38 +315,6 @@ static ssize_t push_ucs2(void *dest, const char *src, size_t dest_len, int flags
 
 
 /**
- * Copy a string from a unix char* src to a UCS2 destination,
- * allocating a buffer using talloc().
- *
- * @param dest always set at least to NULL 
- *
- * @returns The number of bytes occupied by the string in the destination
- *         or -1 in case of error.
- **/
-_PUBLIC_ bool push_ucs2_talloc(TALLOC_CTX *ctx, smb_ucs2_t **dest, const char *src, size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UNIX, CH_UTF16, src, src_len, (void **)dest, converted_size);
-}
-
-
-/**
- * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- *
- * @returns The number of bytes occupied by the string in the destination
- **/
-
-_PUBLIC_ bool push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src, size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UNIX, CH_UTF8, src, src_len, (void **)dest, converted_size);
-}
-
-/**
  Copy a string from a ucs2 source to a unix char* destination.
  Flags can have:
   STR_TERMINATE means the string in src is null terminated.
@@ -484,51 +356,6 @@ static size_t pull_ucs2(char *dest, const void *src, size_t dest_len, size_t src
 }
 
 /**
- * Copy a string from a ASCII src to a unix char * destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- *
- * @returns The number of bytes occupied by the string in the destination
- **/
-
-_PUBLIC_ bool pull_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src, size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_DOS, CH_UNIX, src, src_len, (void **)dest, converted_size);
-}
-
-/**
- * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- *
- * @returns The number of bytes occupied by the string in the destination
- **/
-
-_PUBLIC_ bool pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const smb_ucs2_t *src, size_t *converted_size)
-{
-	size_t src_len = utf16_len(src);
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UTF16, CH_UNIX, src, src_len, (void **)dest, converted_size);
-}
-
-/**
- * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- *
- * @returns The number of bytes occupied by the string in the destination
- **/
-
-_PUBLIC_ bool pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src, size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UTF8, CH_UNIX, src, src_len, (void **)dest, converted_size);
-}
-
-/**
  Copy a string from a char* src to a unicode or ascii
  dos codepage destination choosing unicode or ascii based on the 
  flags in the SMB buffer starting at base_ptr.
@@ -546,7 +373,7 @@ _PUBLIC_ ssize_t push_string(void *dest, const char *src, size_t dest_len, int f
 {
 	if (flags & STR_ASCII) {
 		size_t size = 0;
-		if (push_ascii(dest, src, dest_len, flags, &size)) {
+		if (push_ascii_string(dest, src, dest_len, flags, &size)) {
 			return (ssize_t)size;
 		} else {
 			return (ssize_t)-1;
@@ -577,7 +404,7 @@ _PUBLIC_ ssize_t push_string(void *dest, const char *src, size_t dest_len, int f
 _PUBLIC_ ssize_t pull_string(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
 {
 	if (flags & STR_ASCII) {
-		return pull_ascii(dest, src, dest_len, src_len, flags);
+		return pull_ascii_string(dest, src, dest_len, src_len, flags);
 	} else if (flags & STR_UNICODE) {
 		return pull_ucs2(dest, src, dest_len, src_len, flags);
 	} else {
@@ -585,68 +412,3 @@ _PUBLIC_ ssize_t pull_string(char *dest, const void *src, size_t dest_len, size_
 		return -1;
 	}
 }
-
-
-/**
- * Convert string from one encoding to another, making error checking etc
- *
- * @param src pointer to source string (multibyte or singlebyte)
- * @param srclen length of the source string in bytes
- * @param dest pointer to destination string (multibyte or singlebyte)
- * @param destlen maximal length allowed for string
- * @param converted_size the number of bytes occupied in the destination
- *
- * @returns true on success, false on fail.
- **/
-_PUBLIC_ bool convert_string(charset_t from, charset_t to,
-			       void const *src, size_t srclen, 
-			       void *dest, size_t destlen,
-			       size_t *converted_size)
-{
-	return convert_string_handle(get_iconv_handle(), from, to,
-					src, srclen,
-					dest, destlen, converted_size);
-}
-
-/**
- * Convert string from one encoding to another, making error checking etc
- *
- * @param src pointer to source string (multibyte or singlebyte)
- * @param srclen length of the source string in bytes
- * @param dest pointer to destination string (multibyte or singlebyte)
- * @param destlen maximal length allowed for string
- * @param converted_size the number of bytes occupied in the destination
- *
- * @returns true on success, false on fail.
- **/
-_PUBLIC_ bool convert_string_error(charset_t from, charset_t to,
-				   void const *src, size_t srclen,
-				   void *dest, size_t destlen,
-				   size_t *converted_size)
-{
-	return convert_string_error_handle(get_iconv_handle(), from, to,
-					   src, srclen,
-					   dest, destlen, converted_size);
-}
-
-/**
- * Convert between character sets, allocating a new buffer using talloc for the result.
- *
- * @param srclen length of source buffer.
- * @param dest always set at least to NULL
- * @param converted_size Size in bytes of the converted string
- * @note -1 is not accepted for srclen.
- *
- * @returns boolean indication whether the conversion succeeded
- **/
-
-_PUBLIC_ bool convert_string_talloc(TALLOC_CTX *ctx, 
-				    charset_t from, charset_t to, 
-				    void const *src, size_t srclen, 
-				    void *dest, size_t *converted_size)
-{
-	return convert_string_talloc_handle(ctx, get_iconv_handle(),
-						 from, to, src, srclen, dest,
-						 converted_size);
-}
-
diff --git a/lib/util/charset/util_unistr_w.c b/lib/util/charset/util_unistr_w.c
index a550e527763..3fbed7f67cb 100644
--- a/lib/util/charset/util_unistr_w.c
+++ b/lib/util/charset/util_unistr_w.c
@@ -22,8 +22,8 @@
 #include "includes.h"
 
 /* Copy into a smb_ucs2_t from a possibly unaligned buffer. Return the copied smb_ucs2_t */
-#define COPY_UCS2_CHAR(dest,src) (((unsigned char *)(dest))[0] = ((unsigned char *)(src))[0],\
-				((unsigned char *)(dest))[1] = ((unsigned char *)(src))[1], (dest))
+#define COPY_UCS2_CHAR(dest,src) (((unsigned char *)(dest))[0] = ((const unsigned char *)(src))[0],\
+				((unsigned char *)(dest))[1] = ((const unsigned char *)(src))[1], (dest))
 
 
 /* return an ascii version of a ucs2 character */
@@ -72,12 +72,12 @@ smb_ucs2_t *strchr_w(const smb_ucs2_t *s, smb_ucs2_t c)
 	smb_ucs2_t cp;
 	while (*(COPY_UCS2_CHAR(&cp,s))) {
 		if (c == cp) {
-			return (smb_ucs2_t *)s;
+			return discard_const_p(smb_ucs2_t, s);
 		}
 		s++;
 	}
 	if (c == cp) {
-		return (smb_ucs2_t *)s;
+		return discard_const_p(smb_ucs2_t, s);
 	}
 
 	return NULL;
@@ -104,7 +104,7 @@ smb_ucs2_t *strrchr_w(const smb_ucs2_t *s, smb_ucs2_t c)
 	p += (len - 1);
 	do {
 		if (c == *(COPY_UCS2_CHAR(&cp,p))) {
-			return (smb_ucs2_t *)p;
+			return discard_const_p(smb_ucs2_t, p);
 		}
 	} while (p-- != s);
 	return NULL;
@@ -234,38 +234,6 @@ static int strncmp_w(const smb_ucs2_t *a, const smb_ucs2_t *b, size_t len)
 	return (len - n)?(*(COPY_UCS2_CHAR(&cpa,a)) - *(COPY_UCS2_CHAR(&cpb,b))):0;
 }
 
-/*******************************************************************
- Case insensitive string comparison.
-********************************************************************/
-
-int strcasecmp_w(const smb_ucs2_t *a, const smb_ucs2_t *b)
-{
-	smb_ucs2_t cpa, cpb;
-
-	while ((*COPY_UCS2_CHAR(&cpb,b)) && toupper_m(*(COPY_UCS2_CHAR(&cpa,a))) == toupper_m(cpb)) {
-		a++;
-		b++;
-	}
-	return (tolower_m(*(COPY_UCS2_CHAR(&cpa,a))) - tolower_m(*(COPY_UCS2_CHAR(&cpb,b))));
-}
-
-/*******************************************************************
- Case insensitive string comparison, length limited.
-********************************************************************/
-
-int strncasecmp_w(const smb_ucs2_t *a, const smb_ucs2_t *b, size_t len)
-{
-	smb_ucs2_t cpa, cpb;
-	size_t n = 0;
-
-	while ((n < len) && *COPY_UCS2_CHAR(&cpb,b) && (toupper_m(*(COPY_UCS2_CHAR(&cpa,a))) == toupper_m(cpb))) {
-		a++;
-		b++;
-		n++;
-	}
-	return (len - n)?(tolower_m(*(COPY_UCS2_CHAR(&cpa,a))) - tolower_m(*(COPY_UCS2_CHAR(&cpb,b)))):0;
-}
-
 /*
   The *_wa() functions take a combination of 7 bit ascii
   and wide characters They are used so that you can use string
diff --git a/source3/modules/weird.c b/lib/util/charset/weird.c
index 5db8cdcecd1..5db8cdcecd1 100644
--- a/source3/modules/weird.c
+++ b/lib/util/charset/weird.c
diff --git a/lib/util/charset/wscript_build b/lib/util/charset/wscript_build
index 29e168dce1b..1f2c8dfa7a6 100644
--- a/lib/util/charset/wscript_build
+++ b/lib/util/charset/wscript_build
@@ -1,18 +1,44 @@
 #!/usr/bin/env python
 
-
-if bld.env._SAMBA_BUILD_ == 4:
-    bld.SAMBA_SUBSYSTEM('CHARSET',
-                        source='charcnv.c util_unistr.c',
-                        public_deps='CODEPOINTS',
-                        public_headers='charset.h',
-                        )
-
 bld.SAMBA_SUBSYSTEM('ICONV_WRAPPER',
                     source='iconv.c',
                     public_deps='iconv replace talloc')
 
-bld.SAMBA_SUBSYSTEM('CODEPOINTS',
-	source='codepoints.c util_str.c util_unistr_w.c',
-	deps='DYNCONFIG ICONV_WRAPPER'
-	)
+bld.SAMBA_SUBSYSTEM('CHARSET',
+                    public_headers='charset.h',
+                    source='codepoints.c convert_string.c util_str.c util_unistr_w.c charcnv.c pull_push.c util_unistr.c',
+                    deps='DYNCONFIG ICONV_WRAPPER',
+                    public_deps='talloc')
+
+bld.SAMBA_MODULE('charset_weird',
+                 subsystem='CHARSET',
+                 source='weird.c',
+                 init_function='',
+                 deps='samba-util',
+                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_weird'),
+                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_weird'))
+
+bld.SAMBA_MODULE('charset_CP850',
+                 subsystem='CHARSET',
+                 source='CP850.c',
+                 init_function='',
+                 deps='samba-util',
+                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_CP850'),
+                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_CP850'))
+
+bld.SAMBA_MODULE('charset_CP437',
+                 subsystem='CHARSET',
+                 source='CP437.c',
+                 init_function='',
+                 deps='samba-util',
+                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_CP437'),
+                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_CP437'))
+
+bld.SAMBA_MODULE('charset_macosxfs',
+                 subsystem='CHARSET',
+                 source='charset_macosxfs.c',
+                 init_function='',
+                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_macosxfs'),
+                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_macosxfs'))
+
+
diff --git a/lib/util/data_blob.h b/lib/util/data_blob.h
index 83e6cd5f096..558ade92482 100644
--- a/lib/util/data_blob.h
+++ b/lib/util/data_blob.h
@@ -1,7 +1,10 @@
 /* 
    Unix SMB/CIFS implementation.
    DATA BLOB
-   
+
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Andrew Bartlett 2001
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
diff --git a/lib/util/debug.c b/lib/util/debug.c
index b0a78823fcb..c1b33de6d1e 100644
--- a/lib/util/debug.c
+++ b/lib/util/debug.c
@@ -203,7 +203,7 @@ void gfree_debugsyms(void)
 
 	TALLOC_FREE(format_bufr);
 
-	debug_num_classes = DBGC_MAX_FIXED;
+	debug_num_classes = 0;
 
 	state.initialized = false;
 }
diff --git a/lib/util/debug_s3.h b/lib/util/debug_s3.h
index 96b8ed74d94..9e5211b19b9 100644
--- a/lib/util/debug_s3.h
+++ b/lib/util/debug_s3.h
@@ -17,6 +17,8 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "librpc/gen_ndr/server_id.h"
+
 struct messaging_context;
 struct server_id;
 void debug_message(struct messaging_context *msg_ctx, void *private_data, uint32_t msg_type, struct server_id src, DATA_BLOB *data);
diff --git a/lib/util/dprintf.c b/lib/util/dprintf.c
index e9a15dcbe62..90ca36c1ae5 100644
--- a/lib/util/dprintf.c
+++ b/lib/util/dprintf.c
@@ -33,58 +33,10 @@
 
 #include "includes.h"
 #include "system/locale.h"
-#include "param/param.h"
 
-static smb_iconv_t display_cd = (smb_iconv_t)-1;
-
-void d_set_iconv(smb_iconv_t cd)
+static int d_vfprintf(FILE *f, const char *format, va_list ap) 
 {
-	if (display_cd != (smb_iconv_t)-1)
-		talloc_free(display_cd);
-
-	display_cd = cd;
-}
-
-_PUBLIC_ int d_vfprintf(FILE *f, const char *format, va_list ap) 
-{
-	char *p, *p2;
-	int ret, clen;
-	va_list ap2;
-
-	/* If there's nothing to convert, take a shortcut */
-	if (display_cd == (smb_iconv_t)-1) {
-		return vfprintf(f, format, ap);
-	}
-
-	/* do any message translations */
-	va_copy(ap2, ap);
-	ret = vasprintf(&p, format, ap2);
-	va_end(ap2);
-
-	if (ret <= 0) return ret;
-
-	clen = iconv_talloc(NULL, display_cd, p, ret, (void **)&p2);
-        if (clen == -1) {
-		/* the string can't be converted - do the best we can,
-		   filling in non-printing chars with '?' */
-		int i;
-		for (i=0;i<ret;i++) {
-			if (isprint(p[i]) || isspace(p[i])) {
-				fwrite(p+i, 1, 1, f);
-			} else {
-				fwrite("?", 1, 1, f);
-			}
-		}
-		SAFE_FREE(p);
-		return ret;
-        }
-
-	/* good, its converted OK */
-	SAFE_FREE(p);
-	ret = fwrite(p2, 1, clen, f);
-	talloc_free(p2);
-
-	return ret;
+	return vfprintf(f, format, ap);
 }
 
 
@@ -100,15 +52,25 @@ _PUBLIC_ int d_fprintf(FILE *f, const char *format, ...)
 	return ret;
 }
 
-_PUBLIC_ int d_printf(const char *format, ...)
+static FILE *outfile;
+
+_PUBLIC_  int d_printf(const char *format, ...)
 {
 	int ret;
-	va_list ap;
-
-	va_start(ap, format);
-	ret = d_vfprintf(stdout, format, ap);
-	va_end(ap);
-
-	return ret;
+       va_list ap;
+       
+       if (!outfile) outfile = stdout;
+       
+       va_start(ap, format);
+       ret = d_vfprintf(outfile, format, ap);
+       va_end(ap);
+       
+       return ret;
 }
 
+/* interactive programs need a way of tell d_*() to write to stderr instead
+   of stdout */
+void display_set_stderr(void)
+{
+	outfile = stderr;
+}
diff --git a/lib/util/fault.c b/lib/util/fault.c
index 086dc33545d..708dc670d16 100644
--- a/lib/util/fault.c
+++ b/lib/util/fault.c
@@ -119,7 +119,7 @@ static void smb_panic_default(const char *why)
 	if (panic_action && *panic_action) {
 		char pidstr[20];
 		char cmdstring[200];
-		safe_strcpy(cmdstring, panic_action, sizeof(cmdstring)-1);
+		strlcpy(cmdstring, panic_action, sizeof(cmdstring));
 		snprintf(pidstr, sizeof(pidstr), "%d", (int) getpid());
 		all_string_sub(cmdstring, "%PID%", pidstr, sizeof(cmdstring));
 		DEBUG(0, ("smb_panic(): calling panic action [%s]\n", cmdstring));
diff --git a/lib/util/ms_fnmatch.c b/lib/util/ms_fnmatch.c
index 73fb0e09668..1ba5888ca09 100644
--- a/lib/util/ms_fnmatch.c
+++ b/lib/util/ms_fnmatch.c
@@ -154,7 +154,7 @@ static int ms_fnmatch_core(const char *p, const char *n,
 	return -1;
 }
 
-int ms_fnmatch(const char *pattern, const char *string, enum protocol_types protocol)
+int ms_fnmatch_protocol(const char *pattern, const char *string, int protocol)
 {
 	int ret, count, i;
 	struct max_n *max_n = NULL;
@@ -192,7 +192,7 @@ int ms_fnmatch(const char *pattern, const char *string, enum protocol_types prot
 				p[i] = '<';
 			}
 		}
-		ret = ms_fnmatch(p, string, PROTOCOL_NT1);
+		ret = ms_fnmatch_protocol(p, string, PROTOCOL_NT1);
 		talloc_free(p);
 		return ret;
 	}
@@ -217,5 +217,5 @@ int ms_fnmatch(const char *pattern, const char *string, enum protocol_types prot
 /** a generic fnmatch function - uses for non-CIFS pattern matching */
 int gen_fnmatch(const char *pattern, const char *string)
 {
-	return ms_fnmatch(pattern, string, PROTOCOL_NT1);
+	return ms_fnmatch_protocol(pattern, string, PROTOCOL_NT1);
 }
diff --git a/lib/util/parmlist.c b/lib/util/parmlist.c
index 6658fa7e332..0f2f3af8eea 100644
--- a/lib/util/parmlist.c
+++ b/lib/util/parmlist.c
@@ -20,6 +20,8 @@
 #include "../lib/util/dlinklist.h"
 #include "../lib/util/parmlist.h"
 
+#undef strcasecmp
+
 struct parmlist_entry *parmlist_get(struct parmlist *ctx, const char *name)
 {
 	struct parmlist_entry *e;
diff --git a/lib/util/server_id.c b/lib/util/server_id.c
new file mode 100644
index 00000000000..195deeac7c1
--- /dev/null
+++ b/lib/util/server_id.c
@@ -0,0 +1,41 @@
+/*
+   Unix SMB/CIFS implementation.
+   Samba utility functions
+   Copyright (C) Andrew Bartlett 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "librpc/gen_ndr/server_id.h"
+
+char *server_id_str(TALLOC_CTX *mem_ctx, const struct server_id *id)
+{
+	if (id->vnn == NONCLUSTER_VNN && id->task_id == 0) {
+		return talloc_asprintf(mem_ctx,
+				       "%llu",
+				       (unsigned long long)id->pid);
+	} else if (id->vnn == NONCLUSTER_VNN) {
+		return talloc_asprintf(mem_ctx,
+				       "%llu.%u",
+				       (unsigned long long)id->pid,
+				       (unsigned)id->task_id);
+	} else {
+		return talloc_asprintf(mem_ctx,
+				       "%u:%llu.%u",
+				       (unsigned)id->vnn,
+				       (unsigned long long)id->pid,
+				       (unsigned)id->task_id);
+	}
+}
diff --git a/lib/util/string_wrappers.h b/lib/util/string_wrappers.h
index 75718e942ba..37384fc5a32 100644
--- a/lib/util/string_wrappers.h
+++ b/lib/util/string_wrappers.h
@@ -41,28 +41,36 @@ size_t __unsafe_string_function_usage_here_size_t__(void);
 
 #endif /* HAVE_COMPILER_WILL_OPTIMIZE_OUT_FNS */
 
-#define safe_strcpy_base(dest, src, base, size) \
-    safe_strcpy(dest, src, size-PTR_DIFF(dest,base)-1)
+#define strlcpy_base(dest, src, base, size) \
+do { \
+	const char *_strlcpy_base_src = (const char *)src; \
+	strlcpy((dest), _strlcpy_base_src? _strlcpy_base_src : "", (size)-PTR_DIFF((dest),(base))); \
+} while (0)
 
 /* String copy functions - macro hell below adds 'type checking' (limited,
    but the best we can do in C) */
 
-#define fstrcpy(d,s) safe_strcpy((d),(s),sizeof(fstring)-1)
-#define fstrcat(d,s) safe_strcat((d),(s),sizeof(fstring)-1)
-#define nstrcpy(d,s) safe_strcpy((d), (s),sizeof(nstring)-1)
-#define unstrcpy(d,s) safe_strcpy((d), (s),sizeof(unstring)-1)
-
-/* the addition of the DEVELOPER checks in safe_strcpy means we must
- * update a lot of code. To make this a little easier here are some
- * functions that provide the lengths with less pain */
-
-/* overmalloc_safe_strcpy: DEPRECATED!  Used when you know the
- * destination buffer is longer than maxlength, but you don't know how
- * long.  This is not a good situation, because we can't do the normal
- * sanity checks. Don't use in new code! */
-
-#define overmalloc_safe_strcpy(dest,src,maxlength) \
-	safe_strcpy_fn(dest,src,maxlength)
+#define fstrcpy(d,s) \
+do { \
+	const char *_fstrcpy_src = (const char *)(s); \
+	strlcpy((d),_fstrcpy_src ? _fstrcpy_src : "",sizeof(fstring)); \
+} while (0)
+
+#define fstrcat(d,s) \
+do { \
+	const char *_fstrcat_src = (const char *)(s); \
+	strlcat((d),_fstrcat_src ? _fstrcat_src : "",sizeof(fstring)); \
+} while (0)
+#define nstrcpy(d,s) \
+do { \
+	const char *_nstrcpy_src = (const char *)(s); \
+	strlcpy((d),_nstrcpy_src ? _nstrcpy_src : "",sizeof(fstring)); \
+} while (0)
+#define unstrcpy(d,s) \
+do { \
+	const char *_unstrcpy_src = (const char *)(s); \
+	strlcpy((d),_unstrcpy_src ? _unstrcpy_src : "",sizeof(fstring)); \
+} while (0)
 
 #ifdef HAVE_COMPILER_WILL_OPTIMIZE_OUT_FNS
 
@@ -70,16 +78,6 @@ size_t __unsafe_string_function_usage_here_size_t__(void);
    have the correct types (this works only where sizeof() returns the size of the buffer, not
    the size of the pointer). */
 
-#define safe_strcpy(d, s, max_len) \
-    (CHECK_STRING_SIZE(d, max_len+1) \
-    ? __unsafe_string_function_usage_here__() \
-    : safe_strcpy_fn((d), (s), (max_len)))
-
-#define safe_strcat(d, s, max_len) \
-    (CHECK_STRING_SIZE(d, max_len+1) \
-    ? __unsafe_string_function_usage_here__() \
-    : safe_strcat_fn((d), (s), (max_len)))
-
 #define push_string_check(dest, src, dest_len, flags) \
     (CHECK_STRING_SIZE(dest, dest_len) \
     ? __unsafe_string_function_usage_here_size_t__() \
@@ -113,8 +111,6 @@ size_t __unsafe_string_function_usage_here_size_t__(void);
 
 #else
 
-#define safe_strcpy safe_strcpy_fn
-#define safe_strcat safe_strcat_fn
 #define push_string_check push_string_check_fn
 #define clistr_push clistr_push_fn
 #define clistr_pull clistr_pull_fn
diff --git a/lib/util/substitute.c b/lib/util/substitute.c
index 32945a72134..500d12777f9 100644
--- a/lib/util/substitute.c
+++ b/lib/util/substitute.c
@@ -29,18 +29,20 @@
  **/
 
 /**
- Substitute a string for a pattern in another string. Make sure there is 
+ Substitute a string for a pattern in another string. Make sure there is
  enough room!
 
- This routine looks for pattern in s and replaces it with 
- insert. It may do multiple replacements.
+ This routine looks for pattern in s and replaces it with
+ insert. It may do multiple replacements or just one.
 
  Any of " ; ' $ or ` in the insert string are replaced with _
  if len==0 then the string cannot be extended. This is different from the old
  use of len==0 which was for no length checks to be done.
 **/
 
-_PUBLIC_ void string_sub(char *s, const char *pattern, const char *insert, size_t len)
+static void string_sub2(char *s,const char *pattern, const char *insert, size_t len,
+			bool remove_unsafe_characters, bool replace_once,
+			bool allow_trailing_dollar)
 {
 	char *p;
 	ssize_t ls, lp, li, i;
@@ -55,9 +57,10 @@ _PUBLIC_ void string_sub(char *s, const char *pattern, const char *insert, size_
 	if (len == 0)
 		len = ls + 1; /* len is number of *bytes* */
 
-	while (lp <= ls && (p = strstr(s, pattern))) {
+	while (lp <= ls && (p = strstr_m(s,pattern))) {
 		if (ls + (li-lp) >= len) {
-			DEBUG(0,("ERROR: string overflow by %d in string_sub(%.50s, %d)\n", 
+			DEBUG(0,("ERROR: string overflow by "
+				"%d in string_sub(%.50s, %d)\n",
 				 (int)(ls + (li-lp) - len),
 				 pattern, (int)len));
 			break;
@@ -67,25 +70,50 @@ _PUBLIC_ void string_sub(char *s, const char *pattern, const char *insert, size_
 		}
 		for (i=0;i<li;i++) {
 			switch (insert[i]) {
+			case '$':
+				/* allow a trailing $
+				 * (as in machine accounts) */
+				if (allow_trailing_dollar && (i == li - 1 )) {
+					p[i] = insert[i];
+					break;
+				}
 			case '`':
 			case '"':
 			case '\'':
 			case ';':
-			case '$':
 			case '%':
 			case '\r':
 			case '\n':
-				p[i] = '_';
-				break;
+				if ( remove_unsafe_characters ) {
+					p[i] = '_';
+					/* yes this break should be here
+					 * since we want to fall throw if
+					 * not replacing unsafe chars */
+					break;
+				}
 			default:
 				p[i] = insert[i];
 			}
 		}
 		s = p + li;
 		ls += (li-lp);
+
+		if (replace_once)
+			break;
 	}
 }
 
+void string_sub_once(char *s, const char *pattern,
+		const char *insert, size_t len)
+{
+	string_sub2( s, pattern, insert, len, true, true, false );
+}
+
+void string_sub(char *s,const char *pattern, const char *insert, size_t len)
+{
+	string_sub2( s, pattern, insert, len, true, false, false );
+}
+
 /**
  * Talloc'ed version of string_sub
  */
@@ -146,13 +174,14 @@ _PUBLIC_ void all_string_sub(char *s,const char *pattern,const char *insert, siz
 
 	if (!*pattern)
 		return;
-	
+
 	if (len == 0)
 		len = ls + 1; /* len is number of *bytes* */
-	
-	while (lp <= ls && (p = strstr(s,pattern))) {
+
+	while (lp <= ls && (p = strstr_m(s,pattern))) {
 		if (ls + (li-lp) >= len) {
-			DEBUG(0,("ERROR: string overflow by %d in all_string_sub(%.50s, %d)\n", 
+			DEBUG(0,("ERROR: string overflow by "
+				"%d in all_string_sub(%.50s, %d)\n",
 				 (int)(ls + (li-lp) - len),
 				 pattern, (int)len));
 			break;
diff --git a/lib/util/system.c b/lib/util/system.c
index 9bf5de1a83e..1e80f1a88a3 100644
--- a/lib/util/system.c
+++ b/lib/util/system.c
@@ -22,6 +22,8 @@
 #include "system/network.h"
 #include "system/filesys.h"
 
+#undef malloc
+
 /*
    The idea is that this file will eventually have wrappers around all
    important system calls in samba. The aims are:
@@ -37,6 +39,42 @@
      expansions/etc make sense to the OS should be acceptable to Samba.
 */
 
+/*******************************************************************
+ A wrapper for memalign
+********************************************************************/
+
+void *sys_memalign( size_t align, size_t size )
+{
+#if defined(HAVE_POSIX_MEMALIGN)
+	void *p = NULL;
+	int ret = posix_memalign( &p, align, size );
+	if ( ret == 0 )
+		return p;
+
+	return NULL;
+#elif defined(HAVE_MEMALIGN)
+	return memalign( align, size );
+#else
+	/* On *BSD systems memaligns doesn't exist, but memory will
+	 * be aligned on allocations of > pagesize. */
+#if defined(SYSCONF_SC_PAGESIZE)
+	size_t pagesize = (size_t)sysconf(_SC_PAGESIZE);
+#elif defined(HAVE_GETPAGESIZE)
+	size_t pagesize = (size_t)getpagesize();
+#else
+	size_t pagesize = (size_t)-1;
+#endif
+	if (pagesize == (size_t)-1) {
+		DEBUG(0,("memalign functionalaity not available on this platform!\n"));
+		return NULL;
+	}
+	if (size < pagesize) {
+		size = pagesize;
+	}
+	return malloc(size);
+#endif
+}
+
 /**************************************************************************
 A wrapper for gethostbyname() that tries avoids looking up hostnames 
 in the root domain, which can cause dial-on-demand links to come up for no
@@ -117,3 +155,76 @@ _PUBLIC_ pid_t sys_getpid(void)
 
 	return mypid;
 }
+
+
+_PUBLIC_ int sys_getpeereid( int s, uid_t *uid)
+{
+#if defined(HAVE_PEERCRED)
+	struct ucred cred;
+	socklen_t cred_len = sizeof(struct ucred);
+	int ret;
+
+	ret = getsockopt(s, SOL_SOCKET, SO_PEERCRED, (void *)&cred, &cred_len);
+	if (ret != 0) {
+		return -1;
+	}
+
+	if (cred_len != sizeof(struct ucred)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	*uid = cred.uid;
+	return 0;
+#else
+#if defined(HAVE_GETPEEREID)
+	gid_t gid;
+	return getpeereid(s, uid, &gid);
+#endif
+	errno = ENOSYS;
+	return -1;
+#endif
+}
+
+_PUBLIC_ int sys_getnameinfo(const struct sockaddr *psa,
+			     int salen,
+			     char *host,
+			     size_t hostlen,
+			     char *service,
+			     size_t servlen,
+			     int flags)
+{
+	/*
+	 * For Solaris we must make sure salen is the
+	 * correct length for the incoming sa_family.
+	 */
+
+	if (salen == sizeof(struct sockaddr_storage)) {
+		salen = sizeof(struct sockaddr_in);
+#if defined(HAVE_IPV6)
+		if (psa->sa_family == AF_INET6) {
+			salen = sizeof(struct sockaddr_in6);
+		}
+#endif
+	}
+	return getnameinfo(psa, salen, host, hostlen, service, servlen, flags);
+}
+
+_PUBLIC_ int sys_connect(int fd, const struct sockaddr * addr)
+{
+	socklen_t salen = (socklen_t)-1;
+
+	if (addr->sa_family == AF_INET) {
+	    salen = sizeof(struct sockaddr_in);
+	} else if (addr->sa_family == AF_UNIX) {
+	    salen = sizeof(struct sockaddr_un);
+	}
+#if defined(HAVE_IPV6)
+	else if (addr->sa_family == AF_INET6) {
+	    salen = sizeof(struct sockaddr_in6);
+	}
+#endif
+
+	return connect(fd, addr, salen);
+}
+
diff --git a/lib/util/talloc_stack.c b/lib/util/talloc_stack.c
index 8e559cc20f4..16e9d745d34 100644
--- a/lib/util/talloc_stack.c
+++ b/lib/util/talloc_stack.c
@@ -188,3 +188,20 @@ TALLOC_CTX *talloc_tos(void)
 
 	return ts->talloc_stack[ts->talloc_stacksize-1];
 }
+
+/*
+ * return true if a talloc stackframe exists
+ * this can be used to prevent memory leaks for code that can
+ * optionally use a talloc stackframe (eg. nt_errstr())
+ */
+
+bool talloc_stackframe_exists(void)
+{
+	struct talloc_stackframe *ts =
+		(struct talloc_stackframe *)SMB_THREAD_GET_TLS(global_ts);
+
+	if (ts == NULL || ts->talloc_stacksize == 0) {
+		return false;
+	}
+	return true;
+}
diff --git a/lib/util/talloc_stack.h b/lib/util/talloc_stack.h
index 0e8fab37598..ec0c1c6f37e 100644
--- a/lib/util/talloc_stack.h
+++ b/lib/util/talloc_stack.h
@@ -53,4 +53,12 @@ TALLOC_CTX *talloc_stackframe_pool(size_t poolsize);
 
 TALLOC_CTX *talloc_tos(void);
 
+/*
+ * return true if a talloc stackframe exists
+ * this can be used to prevent memory leaks for code that can
+ * optionally use a talloc stackframe (eg. nt_errstr())
+ */
+
+bool talloc_stackframe_exists(void);
+
 #endif
diff --git a/lib/util/tdb_wrap.c b/lib/util/tdb_wrap.c
new file mode 100644
index 00000000000..71aea5e36cb
--- /dev/null
+++ b/lib/util/tdb_wrap.c
@@ -0,0 +1,215 @@
+/* 
+   Unix SMB/CIFS implementation.
+   TDB wrap functions
+
+   Copyright (C) Andrew Tridgell 2004
+   Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2007
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/util/dlinklist.h"
+#include "lib/util/tdb_wrap.h"
+
+/* FIXME: TDB2 does this internally, so no need to wrap multiple opens! */
+#if BUILD_TDB2
+static void tdb_wrap_log(struct tdb_context *tdb,
+			 enum tdb_log_level level,
+			 const char *message,
+			 void *unused)
+{
+	int dl;
+	const char *name = tdb_name(tdb);
+
+	switch (level) {
+	case TDB_LOG_USE_ERROR:
+	case TDB_LOG_ERROR:
+		dl = 0;
+		break;
+	case TDB_LOG_WARNING:
+		dl = 2;
+		break;
+	default:
+		dl = 0;
+	}
+
+	DEBUG(dl, ("tdb(%s): %s", name ? name : "unnamed", message));
+}
+#else
+/*
+ Log tdb messages via DEBUG().
+*/
+static void tdb_wrap_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, 
+			 const char *format, ...) PRINTF_ATTRIBUTE(3,4);
+
+static void tdb_wrap_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, 
+			 const char *format, ...)
+{
+	va_list ap;
+	char *ptr = NULL;
+	int debuglevel = 0;
+	int ret;
+
+	switch (level) {
+	case TDB_DEBUG_FATAL:
+		debuglevel = 0;
+		break;
+	case TDB_DEBUG_ERROR:
+		debuglevel = 1;
+		break;
+	case TDB_DEBUG_WARNING:
+		debuglevel = 2;
+		break;
+	case TDB_DEBUG_TRACE:
+		debuglevel = 5;
+		break;
+	default:
+		debuglevel = 0;
+	}		
+
+	va_start(ap, format);
+	ret = vasprintf(&ptr, format, ap);
+	va_end(ap);
+
+	if (ret != -1) {
+		const char *name = tdb_name(tdb);
+		DEBUG(debuglevel, ("tdb(%s): %s", name ? name : "unnamed", ptr));
+		free(ptr);
+	}
+}
+#endif
+
+struct tdb_wrap_private {
+	struct tdb_context *tdb;
+	const char *name;
+	struct tdb_wrap_private *next, *prev;
+};
+
+static struct tdb_wrap_private *tdb_list;
+
+/* destroy the last connection to a tdb */
+static int tdb_wrap_private_destructor(struct tdb_wrap_private *w)
+{
+	tdb_close(w->tdb);
+	DLIST_REMOVE(tdb_list, w);
+	return 0;
+}				 
+
+static struct tdb_wrap_private *tdb_wrap_private_open(TALLOC_CTX *mem_ctx,
+						      const char *name,
+						      int hash_size,
+						      int tdb_flags,
+						      int open_flags,
+						      mode_t mode)
+{
+	struct tdb_wrap_private *result;
+
+	result = talloc(mem_ctx, struct tdb_wrap_private);
+	if (result == NULL) {
+		return NULL;
+	}
+	result->name = talloc_strdup(result, name);
+	if (result->name == NULL) {
+		goto fail;
+	}
+
+#if _SAMBA_BUILD_ == 3	
+	/* This #if _SAMBA_BUILD == 3 is very unfortunate, as it means
+	 * that in the top level build, these options are not
+	 * available for these databases.  However, having two
+	 * different tdb_wrap lists is a worse fate, so this will do
+	 * for now */
+
+	if (!lp_use_mmap()) {
+		tdb_flags |= TDB_NOMMAP;
+	}
+
+	if ((hash_size == 0) && (name != NULL)) {
+		const char *base;
+		base = strrchr_m(name, '/');
+
+		if (base != NULL) {
+			base += 1;
+		} else {
+			base = name;
+		}
+		hash_size = lp_parm_int(-1, "tdb_hashsize", base, 0);
+	}
+#endif
+
+	result->tdb = tdb_open_compat(name, hash_size, tdb_flags,
+				      open_flags, mode, tdb_wrap_log, NULL);
+	if (result->tdb == NULL) {
+		goto fail;
+	}
+	talloc_set_destructor(result, tdb_wrap_private_destructor);
+	DLIST_ADD(tdb_list, result);
+	return result;
+
+fail:
+	TALLOC_FREE(result);
+	return NULL;
+}
+
+/*
+  wrapped connection to a tdb database
+  to close just talloc_free() the tdb_wrap pointer
+ */
+struct tdb_wrap *tdb_wrap_open(TALLOC_CTX *mem_ctx,
+			       const char *name, int hash_size, int tdb_flags,
+			       int open_flags, mode_t mode)
+{
+	struct tdb_wrap *result;
+	struct tdb_wrap_private *w;
+
+	result = talloc(mem_ctx, struct tdb_wrap);
+	if (result == NULL) {
+		return NULL;
+	}
+
+	for (w=tdb_list;w;w=w->next) {
+		if (strcmp(name, w->name) == 0) {
+			break;
+		}
+	}
+
+	if (w == NULL) {
+		w = tdb_wrap_private_open(result, name, hash_size, tdb_flags,
+					  open_flags, mode);
+	} else {
+		/*
+		 * Correctly use talloc_reference: The tdb will be
+		 * closed when "w" is being freed. The caller never
+		 * sees "w", so an incorrect use of talloc_free(w)
+		 * instead of calling talloc_unlink is not possible.
+		 * To avoid having to refcount ourselves, "w" will
+		 * have multiple parents that hang off all the
+		 * tdb_wrap's being returned from here. Those parents
+		 * can be freed without problem.
+		 */
+		if (talloc_reference(result, w) == NULL) {
+			goto fail;
+		}
+	}
+	if (w == NULL) {
+		goto fail;
+	}
+	result->tdb = w->tdb;
+	return result;
+fail:
+	TALLOC_FREE(result);
+	return NULL;
+}
+
diff --git a/source4/lib/tdb_wrap.h b/lib/util/tdb_wrap.h
index 94035c1bea6..6f9f3834d43 100644
--- a/source4/lib/tdb_wrap.h
+++ b/lib/util/tdb_wrap.h
@@ -29,13 +29,10 @@
 #ifndef _TDB_WRAP_H_
 #define _TDB_WRAP_H_
 
-#include <tdb.h>
+#include "tdb_compat.h"
 
 struct tdb_wrap {
 	struct tdb_context *tdb;
-
-	const char *name;
-	struct tdb_wrap *next, *prev;
 };
 
 struct tdb_wrap *tdb_wrap_open(TALLOC_CTX *mem_ctx,
diff --git a/lib/util/tests/asn1_tests.c b/lib/util/tests/asn1_tests.c
index ac8ca538f84..3ee64c3f7a8 100644
--- a/lib/util/tests/asn1_tests.c
+++ b/lib/util/tests/asn1_tests.c
@@ -4,6 +4,8 @@
    util_asn1 testing
 
    Copyright (C) Kamen Mazdrashki <kamen.mazdrashki@postpath.com> 2009
+   Copyright (C) Volker Lendecke 2004
+   Copyright (C) Andrew Bartlett 2011
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -103,6 +105,55 @@ static const struct oid_data partial_oid_data_ok[] = {
 	},
 };
 
+static const struct {
+	DATA_BLOB blob;
+	int value;
+} integer_tests[] = {
+        {
+		.blob = {"\x02\x01\x00", 3},
+		.value = 0
+	},
+	{
+		.blob = {"\x02\x01\x7f", 3},
+		.value = 127
+	},
+	{
+		.blob = {"\x02\x02\x00\x80", 4},
+		.value = 128
+	},
+	{
+		.blob = {"\x02\x02\x01\x00", 4},
+		.value = 256
+	},
+	{
+		.blob = {"\x02\x01\x80", 3},
+		.value = -128
+	},
+	{
+		.blob = {"\x02\x02\xff\x7f", 4},
+		.value = -129
+	},
+	{
+		.blob = {"\x02\x01\xff", 3},
+		.value = -1
+	},
+	{
+		.blob = {"\x02\x02\xff\x01", 4},
+		.value = -255
+	},
+	{
+		.blob = {"\x02\x02\x00\xff", 4},
+		.value = 255
+	},
+	{
+		.blob = {"\x02\x04\x80\x00\x00\x00", 6},
+		.value = 0x80000000
+	},
+	{
+		.blob = {"\x02\x04\x7f\xff\xff\xff", 6},
+		.value = 0x7fffffff
+	}
+};
 
 /* Testing ber_write_OID_String() function */
 static bool test_ber_write_OID_String(struct torture_context *tctx)
@@ -260,6 +311,46 @@ static bool test_ber_read_partial_OID_String(struct torture_context *tctx)
 	return true;
 }
 
+/*
+ * Testing asn1_read_Integer and asn1_write_Integer functions,
+ * inspired by Love Hornquist Astrand
+ */
+
+static bool test_asn1_Integer(struct torture_context *tctx)
+{
+	int i;
+	TALLOC_CTX *mem_ctx;
+
+	mem_ctx = talloc_new(tctx);
+
+	for (i = 0; i < ARRAY_SIZE(integer_tests); i++) {
+		ASN1_DATA *data;
+		DATA_BLOB blob;
+		int val;
+
+		data = asn1_init(mem_ctx);
+		if (!data) {
+			return -1;
+		}
+
+		asn1_write_Integer(data, integer_tests[i].value);
+
+		blob.data = data->data;
+		blob.length = data->length;
+		torture_assert_data_blob_equal(tctx, blob, integer_tests[i].blob, "asn1_write_Integer gave incorrect result");
+
+		asn1_load(data, blob);
+		torture_assert(tctx, asn1_read_Integer(data, &val), "asn1_write_Integer output could not be read by asn1_read_Integer()");
+
+		torture_assert_int_equal(tctx, val, integer_tests[i].value,
+			"readback of asn1_write_Integer output by asn1_read_Integer() failed");
+	}
+
+	talloc_free(mem_ctx);
+
+	return true;
+}
+
 
 /* LOCAL-ASN1 test suite creation */
 struct torture_suite *torture_local_util_asn1(TALLOC_CTX *mem_ctx)
@@ -278,5 +369,8 @@ struct torture_suite *torture_local_util_asn1(TALLOC_CTX *mem_ctx)
 	torture_suite_add_simple_test(suite, "ber_read_partial_OID_String",
 				      test_ber_read_partial_OID_String);
 
+	torture_suite_add_simple_test(suite, "asn1_Integer",
+				      test_asn1_Integer);
+
 	return suite;
 }
diff --git a/lib/util/tests/str.c b/lib/util/tests/str.c
index 6b38feaf43e..f9f3abf7316 100644
--- a/lib/util/tests/str.c
+++ b/lib/util/tests/str.c
@@ -25,7 +25,7 @@
 static bool test_string_sub_simple(struct torture_context *tctx)
 {
 	char tmp[100];
-	safe_strcpy(tmp, "foobar", sizeof(tmp));
+	strlcpy(tmp, "foobar", sizeof(tmp));
 	string_sub(tmp, "foo", "bar", sizeof(tmp));
 	torture_assert_str_equal(tctx, tmp, "barbar", "invalid sub");
 	return true;
@@ -34,7 +34,7 @@ static bool test_string_sub_simple(struct torture_context *tctx)
 static bool test_string_sub_multiple(struct torture_context *tctx)
 {
 	char tmp[100];
-	safe_strcpy(tmp, "fooblafoo", sizeof(tmp));
+	strlcpy(tmp, "fooblafoo", sizeof(tmp));
 	string_sub(tmp, "foo", "bar", sizeof(tmp));
 	torture_assert_str_equal(tctx, tmp, "barblabar", "invalid sub");
 	return true;
@@ -43,7 +43,7 @@ static bool test_string_sub_multiple(struct torture_context *tctx)
 static bool test_string_sub_longer(struct torture_context *tctx)
 {
 	char tmp[100];
-	safe_strcpy(tmp, "foobla", sizeof(tmp));
+	strlcpy(tmp, "foobla", sizeof(tmp));
 	string_sub(tmp, "foo", "blie", sizeof(tmp));
 	torture_assert_str_equal(tctx, tmp, "bliebla", "invalid sub");
 	return true;
@@ -52,7 +52,7 @@ static bool test_string_sub_longer(struct torture_context *tctx)
 static bool test_string_sub_shorter(struct torture_context *tctx)
 {
 	char tmp[100];
-	safe_strcpy(tmp, "foobla", sizeof(tmp));
+	strlcpy(tmp, "foobla", sizeof(tmp));
 	string_sub(tmp, "foo", "bl", sizeof(tmp));
 	torture_assert_str_equal(tctx, tmp, "blbla", "invalid sub");
 	return true;
@@ -61,7 +61,7 @@ static bool test_string_sub_shorter(struct torture_context *tctx)
 static bool test_string_sub_special_char(struct torture_context *tctx)
 {
 	char tmp[100];
-	safe_strcpy(tmp, "foobla", sizeof(tmp));
+	strlcpy(tmp, "foobla", sizeof(tmp));
 	string_sub(tmp, "foo", "%b;l", sizeof(tmp));
 	torture_assert_str_equal(tctx, tmp, "_b_lbla", "invalid sub");
 	return true;
diff --git a/lib/util/tests/time.c b/lib/util/tests/time.c
index 592f88f88bd..a8b26762e31 100644
--- a/lib/util/tests/time.c
+++ b/lib/util/tests/time.c
@@ -81,29 +81,11 @@ static bool test_timestring(struct torture_context *tctx)
 	return true;
 }
 
-static bool test_get_time_zone(struct torture_context *tctx)
-{
-	time_t t = time(NULL);
-	int old_extra_time_offset = extra_time_offset;
-	int old_offset, new_offset;
-	/* test that extra_time_offset works */
-
-	old_offset = get_time_zone(t);
-	extra_time_offset = 42;
-	new_offset = get_time_zone(t);
-	extra_time_offset = old_extra_time_offset;
-	torture_assert_int_equal(tctx, old_offset+60*42, new_offset,
-				 "time offset not used");
-	return true;
-}
-
-
 struct torture_suite *torture_local_util_time(TALLOC_CTX *mem_ctx)
 {
 	struct torture_suite *suite = torture_suite_create(mem_ctx, "time");
 
 	torture_suite_add_simple_test(suite, "null_time", test_null_time);
-	torture_suite_add_simple_test(suite, "get_time_zone", test_get_time_zone);
 	torture_suite_add_simple_test(suite, "null_nttime", test_null_nttime);
 	torture_suite_add_simple_test(suite, "http_timestring", 
 								  test_http_timestring);
diff --git a/lib/util/time.c b/lib/util/time.c
index 4843fc96972..31aa05cd0f5 100644
--- a/lib/util/time.c
+++ b/lib/util/time.c
@@ -580,6 +580,24 @@ _PUBLIC_ struct timeval timeval_current_ofs(uint32_t secs, uint32_t usecs)
 }
 
 /**
+  return a timeval milliseconds into the future
+*/
+_PUBLIC_ struct timeval timeval_current_ofs_msec(uint32_t msecs)
+{
+	struct timeval tv = timeval_current();
+	return timeval_add(&tv, msecs / 1000, (msecs % 1000) * 1000);
+}
+
+/**
+  return a timeval microseconds into the future
+*/
+_PUBLIC_ struct timeval timeval_current_ofs_usec(uint32_t usecs)
+{
+	struct timeval tv = timeval_current();
+	return timeval_add(&tv, usecs / 1000000, usecs % 1000000);
+}
+
+/**
   compare two timeval structures. 
   Return -1 if tv1 < tv2
   Return 0 if tv1 == tv2
@@ -720,8 +738,6 @@ static int tm_diff(struct tm *a, struct tm *b)
 }
 
 
-int extra_time_offset=0;
-
 /**
   return the UTC offset in seconds west of UTC, or 0 if it cannot be determined
  */
@@ -735,7 +751,7 @@ _PUBLIC_ int get_time_zone(time_t t)
 	tm = localtime(&t);
 	if (!tm)
 		return 0;
-	return tm_diff(&tm_utc,tm)+60*extra_time_offset;
+	return tm_diff(&tm_utc,tm);
 }
 
 struct timespec nt_time_to_unix_timespec(NTTIME *nt)
diff --git a/lib/util/time.h b/lib/util/time.h
index 3a406340f45..204c261c1d2 100644
--- a/lib/util/time.h
+++ b/lib/util/time.h
@@ -1,7 +1,12 @@
 /* 
    Unix SMB/CIFS implementation.
    time utility functions
-   
+
+   Copyright (C) Andrew Tridgell 		1992-2004
+   Copyright (C) Stefan (metze) Metzmacher	2002
+   Copyright (C) Jeremy Allison			2007
+   Copyright (C) Andrew Bartlett                2011
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
@@ -213,6 +218,16 @@ struct timeval timeval_sum(const struct timeval *tv1,
 _PUBLIC_ struct timeval timeval_current_ofs(uint32_t secs, uint32_t usecs);
 
 /**
+  return a timeval milliseconds into the future
+*/
+_PUBLIC_ struct timeval timeval_current_ofs_msec(uint32_t msecs);
+
+/**
+  return a timeval microseconds into the future
+*/
+_PUBLIC_ struct timeval timeval_current_ofs_usec(uint32_t usecs);
+
+/**
   compare two timeval structures. 
   Return -1 if tv1 < tv2
   Return 0 if tv1 == tv2
@@ -285,7 +300,4 @@ struct timespec convert_time_t_to_timespec(time_t t);
 
 bool null_timespec(struct timespec ts);
 
-/** Extra minutes to add to the normal GMT to local time conversion. */
-extern int extra_time_offset;
-
 #endif /* _SAMBA_TIME_H_ */
diff --git a/lib/util/util.c b/lib/util/util.c
index d4a936fae9f..7f30d436e8e 100644
--- a/lib/util/util.c
+++ b/lib/util/util.c
@@ -152,7 +152,8 @@ _PUBLIC_ bool directory_create_or_exist(const char *dname, uid_t uid,
 		}
 		if ((st.st_mode & 0777) != dir_perms) {
 			DEBUG(0, ("invalid permissions on directory "
-				  "%s\n", dname));
+				  "'%s': has 0%o should be 0%o\n", dname,
+				  (st.st_mode & 0777), dir_perms));
 			umask(old_umask);
 			return false;
 		}
diff --git a/lib/util/util.h b/lib/util/util.h
index 45779912f31..c7154401864 100644
--- a/lib/util/util.h
+++ b/lib/util/util.h
@@ -62,6 +62,8 @@ extern const char *panic_action;
 
 #include "lib/util/memory.h"
 
+#include "lib/util/string_wrappers.h"
+
 /**
  * Write backtrace to debug log
  */
@@ -113,6 +115,8 @@ void CatchChildLeaveStatus(void);
 
 /* The following definitions come from lib/util/system.c  */
 
+void *sys_memalign( size_t align, size_t size );
+
 /**************************************************************************
 A wrapper for gethostbyname() that tries avoids looking up hostnames 
 in the root domain, which can cause dial-on-demand links to come up for no
@@ -131,8 +135,20 @@ _PUBLIC_ pid_t sys_fork(void);
  **/
 _PUBLIC_ pid_t sys_getpid(void);
 
-/* The following definitions come from lib/util/genrand.c  */
+_PUBLIC_ int sys_getpeereid( int s, uid_t *uid);
+
+struct sockaddr;
+
+_PUBLIC_ int sys_getnameinfo(const struct sockaddr *psa,
+			     int salen,
+			     char *host,
+			     size_t hostlen,
+			     char *service,
+			     size_t servlen,
+			     int flags);
+_PUBLIC_ int sys_connect(int fd, const struct sockaddr * addr);
 
+/* The following definitions come from lib/util/genrand.c  */
 /**
  Copy any user given reseed data.
 **/
@@ -195,14 +211,10 @@ _PUBLIC_ char** generate_unique_strs(TALLOC_CTX *mem_ctx, size_t len,
                                          uint32_t num);
 
 /* The following definitions come from lib/util/dprintf.c  */
-#if _SAMBA_BUILD_ == 4
 
-_PUBLIC_ void d_set_iconv(smb_iconv_t);
-_PUBLIC_ int d_vfprintf(FILE *f, const char *format, va_list ap) PRINTF_ATTRIBUTE(2,0);
 _PUBLIC_ int d_fprintf(FILE *f, const char *format, ...) PRINTF_ATTRIBUTE(2,3);
 _PUBLIC_ int d_printf(const char *format, ...) PRINTF_ATTRIBUTE(1,2);
 _PUBLIC_ void display_set_stderr(void);
-#endif
 
 /* The following definitions come from lib/util/util_str.c  */
 
@@ -233,18 +245,6 @@ _PUBLIC_ bool trim_string(char *s, const char *front, const char *back);
 _PUBLIC_ _PURE_ size_t count_chars(const char *s, char c);
 
 /**
- Safe string copy into a known length string. maxlength does not
- include the terminating zero.
-**/
-_PUBLIC_ char *safe_strcpy(char *dest,const char *src, size_t maxlength);
-
-/**
- Safe string cat into a string. maxlength does not
- include the terminating zero.
-**/
-_PUBLIC_ char *safe_strcat(char *dest, const char *src, size_t maxlength);
-
-/**
  Routine to get hex characters and turn them into a 16 byte array.
  the array can be variable length, and any non-hex-numeric
  characters are skipped.  "0xnn" or "0Xnn" is specially catered
@@ -284,6 +284,8 @@ _PUBLIC_ char *hex_encode_talloc(TALLOC_CTX *mem_ctx, const unsigned char *buff_
 **/
 _PUBLIC_ void string_sub(char *s,const char *pattern, const char *insert, size_t len);
 
+_PUBLIC_ void string_sub_once(char *s, const char *pattern,
+			      const char *insert, size_t len);
 
 _PUBLIC_ char *string_sub_talloc(TALLOC_CTX *mem_ctx, const char *s, 
 				const char *pattern, const char *insert);
@@ -369,12 +371,10 @@ _PUBLIC_ bool set_boolean(const char *boolean_string, bool *boolean);
  */
 _PUBLIC_ bool conv_str_bool(const char * str, bool * val);
 
-#if _SAMBA_BUILD_ == 4
 /**
  * Convert a size specification like 16K into an integral number of bytes. 
  **/
-_PUBLIC_ bool conv_str_size(const char * str, uint64_t * val);
-#endif
+_PUBLIC_ bool conv_str_size_error(const char * str, uint64_t * val);
 
 /**
  * Parse a uint64_t value from a string
@@ -775,11 +775,12 @@ enum protocol_types {
 	PROTOCOL_SMB2
 };
 
-int ms_fnmatch(const char *pattern, const char *string, enum protocol_types protocol);
+#endif
+
+int ms_fnmatch_protocol(const char *pattern, const char *string, int protocol);
 
 /** a generic fnmatch function - uses for non-CIFS pattern matching */
 int gen_fnmatch(const char *pattern, const char *string);
-#endif
 
 /* The following definitions come from lib/util/idtree.c  */
 
@@ -886,4 +887,32 @@ int samba_runcmd_recv(struct tevent_req *req, int *perrno);
 void samba_start_debugger(void);
 #endif
 
+/**
+ * @brief Returns an absolute path to a file in the Samba modules directory.
+ *
+ * @param name File to find, relative to MODULESDIR.
+ *
+ * @retval Pointer to a string containing the full path.
+ **/
+char *modules_path(TALLOC_CTX *mem_ctx, const char *name);
+
+/**
+ * @brief Returns an absolute path to a file in the Samba data directory.
+ *
+ * @param name File to find, relative to CODEPAGEDIR.
+ *
+ * @retval Pointer to a talloc'ed string containing the full path.
+ **/
+char *data_path(TALLOC_CTX *mem_ctx, const char *name);
+
+/**
+ * @brief Returns the platform specific shared library extension.
+ *
+ * @retval Pointer to a const char * containing the extension.
+ **/
+const char *shlib_ext(void);
+
+struct server_id;
+char *server_id_str(TALLOC_CTX *mem_ctx, const struct server_id *id);
+
 #endif /* _SAMBA_UTIL_H_ */
diff --git a/lib/util/util_ldb.h b/lib/util/util_ldb.h
index d2bc3b0ff71..66916443c34 100644
--- a/lib/util/util_ldb.h
+++ b/lib/util/util_ldb.h
@@ -1,3 +1,26 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   common share info functions
+
+   Copyright (C) Andrew Tridgell 2004
+   Copyright (C) Tim Potter 2004
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
 #ifndef __LIB_UTIL_UTIL_LDB_H__
 #define __LIB_UTIL_UTIL_LDB_H__
 
diff --git a/lib/util/util_net.c b/lib/util/util_net.c
index 9c8f5c6d479..64aa674d8b0 100644
--- a/lib/util/util_net.c
+++ b/lib/util/util_net.c
@@ -54,6 +54,15 @@ bool interpret_string_addr_internal(struct addrinfo **ppres,
 
 	/* By default make sure it supports TCP. */
 	hints.ai_socktype = SOCK_STREAM;
+
+	/* always try as a numeric host first. This prevents unnecessary name
+	 * lookups, and also ensures we accept IPv6 addresses */
+	hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST;
+	ret = getaddrinfo(str, NULL, &hints, ppres);
+	if (ret == 0) {
+		return true;
+	}
+
 	hints.ai_flags = flags;
 
 	/* Linux man page on getaddrinfo() says port will be
@@ -297,10 +306,10 @@ bool is_ipaddress_v4(const char *str)
 }
 
 /**
- * Return true if a string could be an IPv4 or IPv6 address.
+ * Return true if a string could be a IPv6 address.
  */
 
-bool is_ipaddress(const char *str)
+bool is_ipaddress_v6(const char *str)
 {
 #if defined(HAVE_IPV6)
 	int ret = -1;
@@ -328,7 +337,16 @@ bool is_ipaddress(const char *str)
 		}
 	}
 #endif
-	return is_ipaddress_v4(str);
+	return false;
+}
+
+/**
+ * Return true if a string could be an IPv4 or IPv6 address.
+ */
+
+bool is_ipaddress(const char *str)
+{
+	return is_ipaddress_v4(str) || is_ipaddress_v6(str);
 }
 
 /**
@@ -405,7 +423,7 @@ bool is_zero_addr(const struct sockaddr_storage *pss)
  */
 void zero_ip_v4(struct in_addr *ip)
 {
-	memset(ip, '\0', sizeof(struct in_addr));
+	ZERO_STRUCTP(ip);
 }
 
 /**
@@ -415,7 +433,7 @@ void in_addr_to_sockaddr_storage(struct sockaddr_storage *ss,
 		struct in_addr ip)
 {
 	struct sockaddr_in *sa = (struct sockaddr_in *)ss;
-	memset(ss, '\0', sizeof(*ss));
+	ZERO_STRUCTP(ss);
 	sa->sin_family = AF_INET;
 	sa->sin_addr = ip;
 }
@@ -540,3 +558,319 @@ void set_sockaddr_port(struct sockaddr *psa, uint16_t port)
 }
 
 
+/****************************************************************************
+ Get a port number in host byte order from a sockaddr_storage.
+****************************************************************************/
+
+uint16_t get_sockaddr_port(const struct sockaddr_storage *pss)
+{
+	uint16_t port = 0;
+
+	if (pss->ss_family != AF_INET) {
+#if defined(HAVE_IPV6)
+		/* IPv6 */
+		const struct sockaddr_in6 *sa6 =
+			(const struct sockaddr_in6 *)pss;
+		port = ntohs(sa6->sin6_port);
+#endif
+	} else {
+		const struct sockaddr_in *sa =
+			(const struct sockaddr_in *)pss;
+		port = ntohs(sa->sin_port);
+	}
+	return port;
+}
+
+/****************************************************************************
+ Print out an IPv4 or IPv6 address from a struct sockaddr_storage.
+****************************************************************************/
+
+char *print_sockaddr_len(char *dest,
+			 size_t destlen,
+			const struct sockaddr *psa,
+			socklen_t psalen)
+{
+	if (destlen > 0) {
+		dest[0] = '\0';
+	}
+	(void)sys_getnameinfo(psa,
+			psalen,
+			dest, destlen,
+			NULL, 0,
+			NI_NUMERICHOST);
+	return dest;
+}
+
+/****************************************************************************
+ Print out an IPv4 or IPv6 address from a struct sockaddr_storage.
+****************************************************************************/
+
+char *print_sockaddr(char *dest,
+			size_t destlen,
+			const struct sockaddr_storage *psa)
+{
+	return print_sockaddr_len(dest, destlen, (const struct sockaddr *)psa,
+			sizeof(struct sockaddr_storage));
+}
+
+/****************************************************************************
+ Print out a canonical IPv4 or IPv6 address from a struct sockaddr_storage.
+****************************************************************************/
+
+char *print_canonical_sockaddr(TALLOC_CTX *ctx,
+			const struct sockaddr_storage *pss)
+{
+	char addr[INET6_ADDRSTRLEN];
+	char *dest = NULL;
+	int ret;
+
+	/* Linux getnameinfo() man pages says port is unitialized if
+	   service name is NULL. */
+
+	ret = sys_getnameinfo((const struct sockaddr *)pss,
+			sizeof(struct sockaddr_storage),
+			addr, sizeof(addr),
+			NULL, 0,
+			NI_NUMERICHOST);
+	if (ret != 0) {
+		return NULL;
+	}
+
+	if (pss->ss_family != AF_INET) {
+#if defined(HAVE_IPV6)
+		dest = talloc_asprintf(ctx, "[%s]", addr);
+#else
+		return NULL;
+#endif
+	} else {
+		dest = talloc_asprintf(ctx, "%s", addr);
+	}
+
+	return dest;
+}
+
+/****************************************************************************
+ Return the port number we've bound to on a socket.
+****************************************************************************/
+
+int get_socket_port(int fd)
+{
+	struct sockaddr_storage sa;
+	socklen_t length = sizeof(sa);
+
+	if (fd == -1) {
+		return -1;
+	}
+
+	if (getsockname(fd, (struct sockaddr *)&sa, &length) < 0) {
+		int level = (errno == ENOTCONN) ? 2 : 0;
+		DEBUG(level, ("getsockname failed. Error was %s\n",
+			       strerror(errno)));
+		return -1;
+	}
+
+#if defined(HAVE_IPV6)
+	if (sa.ss_family == AF_INET6) {
+		return ntohs(((struct sockaddr_in6 *)&sa)->sin6_port);
+	}
+#endif
+	if (sa.ss_family == AF_INET) {
+		return ntohs(((struct sockaddr_in *)&sa)->sin_port);
+	}
+	return -1;
+}
+
+/****************************************************************************
+ Return the string of an IP address (IPv4 or IPv6).
+****************************************************************************/
+
+static const char *get_socket_addr(int fd, char *addr_buf, size_t addr_len)
+{
+	struct sockaddr_storage sa;
+	socklen_t length = sizeof(sa);
+
+	/* Ok, returning a hard coded IPv4 address
+	 * is bogus, but it's just as bogus as a
+	 * zero IPv6 address. No good choice here.
+	 */
+
+	strlcpy(addr_buf, "0.0.0.0", addr_len);
+
+	if (fd == -1) {
+		return addr_buf;
+	}
+
+	if (getsockname(fd, (struct sockaddr *)&sa, &length) < 0) {
+		DEBUG(0,("getsockname failed. Error was %s\n",
+			strerror(errno) ));
+		return addr_buf;
+	}
+
+	return print_sockaddr_len(addr_buf, addr_len, (struct sockaddr *)&sa, length);
+}
+
+const char *client_socket_addr(int fd, char *addr, size_t addr_len)
+{
+	return get_socket_addr(fd, addr, addr_len);
+}
+
+
+enum SOCK_OPT_TYPES {OPT_BOOL,OPT_INT,OPT_ON};
+
+typedef struct smb_socket_option {
+	const char *name;
+	int level;
+	int option;
+	int value;
+	int opttype;
+} smb_socket_option;
+
+static const smb_socket_option socket_options[] = {
+  {"SO_KEEPALIVE", SOL_SOCKET, SO_KEEPALIVE, 0, OPT_BOOL},
+  {"SO_REUSEADDR", SOL_SOCKET, SO_REUSEADDR, 0, OPT_BOOL},
+  {"SO_BROADCAST", SOL_SOCKET, SO_BROADCAST, 0, OPT_BOOL},
+#ifdef TCP_NODELAY
+  {"TCP_NODELAY", IPPROTO_TCP, TCP_NODELAY, 0, OPT_BOOL},
+#endif
+#ifdef TCP_KEEPCNT
+  {"TCP_KEEPCNT", IPPROTO_TCP, TCP_KEEPCNT, 0, OPT_INT},
+#endif
+#ifdef TCP_KEEPIDLE
+  {"TCP_KEEPIDLE", IPPROTO_TCP, TCP_KEEPIDLE, 0, OPT_INT},
+#endif
+#ifdef TCP_KEEPINTVL
+  {"TCP_KEEPINTVL", IPPROTO_TCP, TCP_KEEPINTVL, 0, OPT_INT},
+#endif
+#ifdef IPTOS_LOWDELAY
+  {"IPTOS_LOWDELAY", IPPROTO_IP, IP_TOS, IPTOS_LOWDELAY, OPT_ON},
+#endif
+#ifdef IPTOS_THROUGHPUT
+  {"IPTOS_THROUGHPUT", IPPROTO_IP, IP_TOS, IPTOS_THROUGHPUT, OPT_ON},
+#endif
+#ifdef SO_REUSEPORT
+  {"SO_REUSEPORT", SOL_SOCKET, SO_REUSEPORT, 0, OPT_BOOL},
+#endif
+#ifdef SO_SNDBUF
+  {"SO_SNDBUF", SOL_SOCKET, SO_SNDBUF, 0, OPT_INT},
+#endif
+#ifdef SO_RCVBUF
+  {"SO_RCVBUF", SOL_SOCKET, SO_RCVBUF, 0, OPT_INT},
+#endif
+#ifdef SO_SNDLOWAT
+  {"SO_SNDLOWAT", SOL_SOCKET, SO_SNDLOWAT, 0, OPT_INT},
+#endif
+#ifdef SO_RCVLOWAT
+  {"SO_RCVLOWAT", SOL_SOCKET, SO_RCVLOWAT, 0, OPT_INT},
+#endif
+#ifdef SO_SNDTIMEO
+  {"SO_SNDTIMEO", SOL_SOCKET, SO_SNDTIMEO, 0, OPT_INT},
+#endif
+#ifdef SO_RCVTIMEO
+  {"SO_RCVTIMEO", SOL_SOCKET, SO_RCVTIMEO, 0, OPT_INT},
+#endif
+#ifdef TCP_FASTACK
+  {"TCP_FASTACK", IPPROTO_TCP, TCP_FASTACK, 0, OPT_INT},
+#endif
+#ifdef TCP_QUICKACK
+  {"TCP_QUICKACK", IPPROTO_TCP, TCP_QUICKACK, 0, OPT_BOOL},
+#endif
+#ifdef TCP_KEEPALIVE_THRESHOLD
+  {"TCP_KEEPALIVE_THRESHOLD", IPPROTO_TCP, TCP_KEEPALIVE_THRESHOLD, 0, OPT_INT},
+#endif
+#ifdef TCP_KEEPALIVE_ABORT_THRESHOLD
+  {"TCP_KEEPALIVE_ABORT_THRESHOLD", IPPROTO_TCP, TCP_KEEPALIVE_ABORT_THRESHOLD, 0, OPT_INT},
+#endif
+  {NULL,0,0,0,0}};
+
+/****************************************************************************
+ Print socket options.
+****************************************************************************/
+
+static void print_socket_options(int s)
+{
+	int value;
+	socklen_t vlen = 4;
+	const smb_socket_option *p = &socket_options[0];
+
+	/* wrapped in if statement to prevent streams
+	 * leak in SCO Openserver 5.0 */
+	/* reported on samba-technical  --jerry */
+	if ( DEBUGLEVEL >= 5 ) {
+		DEBUG(5,("Socket options:\n"));
+		for (; p->name != NULL; p++) {
+			if (getsockopt(s, p->level, p->option,
+						(void *)&value, &vlen) == -1) {
+				DEBUGADD(5,("\tCould not test socket option %s.\n",
+							p->name));
+			} else {
+				DEBUGADD(5,("\t%s = %d\n",
+							p->name,value));
+			}
+		}
+	}
+ }
+
+/****************************************************************************
+ Set user socket options.
+****************************************************************************/
+
+void set_socket_options(int fd, const char *options)
+{
+	TALLOC_CTX *ctx = talloc_new(NULL);
+	char *tok;
+
+	while (next_token_talloc(ctx, &options, &tok," \t,")) {
+		int ret=0,i;
+		int value = 1;
+		char *p;
+		bool got_value = false;
+
+		if ((p = strchr_m(tok,'='))) {
+			*p = 0;
+			value = atoi(p+1);
+			got_value = true;
+		}
+
+		for (i=0;socket_options[i].name;i++)
+			if (strequal(socket_options[i].name,tok))
+				break;
+
+		if (!socket_options[i].name) {
+			DEBUG(0,("Unknown socket option %s\n",tok));
+			continue;
+		}
+
+		switch (socket_options[i].opttype) {
+		case OPT_BOOL:
+		case OPT_INT:
+			ret = setsockopt(fd,socket_options[i].level,
+					socket_options[i].option,
+					(char *)&value,sizeof(int));
+			break;
+
+		case OPT_ON:
+			if (got_value)
+				DEBUG(0,("syntax error - %s "
+					"does not take a value\n",tok));
+
+			{
+				int on = socket_options[i].value;
+				ret = setsockopt(fd,socket_options[i].level,
+					socket_options[i].option,
+					(char *)&on,sizeof(int));
+			}
+			break;
+		}
+
+		if (ret != 0) {
+			/* be aware that some systems like Solaris return
+			 * EINVAL to a setsockopt() call when the client
+			 * sent a RST previously - no need to worry */
+			DEBUG(2,("Failed to set socket option %s (Error %s)\n",
+				tok, strerror(errno) ));
+		}
+	}
+
+	TALLOC_FREE(ctx);
+	print_socket_options(fd);
+}
diff --git a/lib/util/util_net.h b/lib/util/util_net.h
index 530311e5c86..fc2776a32b1 100644
--- a/lib/util/util_net.h
+++ b/lib/util/util_net.h
@@ -50,6 +50,15 @@ void set_sockaddr_port(struct sockaddr *psa, uint16_t port);
 **/
 _PUBLIC_ bool is_zero_ip_v4(struct in_addr ip);
 
+void in_addr_to_sockaddr_storage(struct sockaddr_storage *ss,
+				 struct in_addr ip);
+#if defined(HAVE_IPV6)
+/**
+ * Convert an IPv6 struct in_addr to a struct sockaddr_storage.
+ */
+void in6_addr_to_sockaddr_storage(struct sockaddr_storage *ss,
+				  struct in6_addr ip);
+#endif
 /**
  Are two IPs on the same subnet?
 **/
@@ -60,6 +69,11 @@ _PUBLIC_ bool same_net_v4(struct in_addr ip1,struct in_addr ip2,struct in_addr m
 **/
 _PUBLIC_ bool is_ipaddress(const char *str);
 
+bool is_broadcast_addr(const struct sockaddr *pss);
+bool is_loopback_ip_v4(struct in_addr ip);
+bool is_loopback_addr(const struct sockaddr *pss);
+bool is_zero_addr(const struct sockaddr_storage *pss);
+void zero_ip_v4(struct in_addr *ip);
 /**
  Interpret an internet address or name into an IP address in 4 byte form.
 **/
@@ -71,6 +85,30 @@ _PUBLIC_ uint32_t interpret_addr(const char *str);
 _PUBLIC_ struct in_addr interpret_addr2(const char *str);
 
 _PUBLIC_ bool is_ipaddress_v4(const char *str);
-
+_PUBLIC_ bool is_ipaddress_v6(const char *str);
+
+bool is_address_any(const struct sockaddr *psa);
+bool same_net(const struct sockaddr *ip1,
+	      const struct sockaddr *ip2,
+	      const struct sockaddr *mask);
+bool sockaddr_equal(const struct sockaddr *ip1,
+		    const struct sockaddr *ip2);
+
+bool is_address_any(const struct sockaddr *psa);
+uint16_t get_sockaddr_port(const struct sockaddr_storage *pss);
+char *print_sockaddr_len(char *dest,
+			 size_t destlen,
+			 const struct sockaddr *psa,
+			 socklen_t psalen);
+char *print_sockaddr(char *dest,
+			size_t destlen,
+			const struct sockaddr_storage *psa);
+char *print_canonical_sockaddr(TALLOC_CTX *ctx,
+			const struct sockaddr_storage *pss);
+const char *client_name(int fd);
+int get_socket_port(int fd);
+const char *client_socket_addr(int fd, char *addr, size_t addr_len);
+
+void set_socket_options(int fd, const char *options);
 
 #endif /* _SAMBA_UTIL_NET_H_ */
diff --git a/lib/util/util_paths.c b/lib/util/util_paths.c
new file mode 100644
index 00000000000..0baa6801c54
--- /dev/null
+++ b/lib/util/util_paths.c
@@ -0,0 +1,63 @@
+/* 
+   Unix SMB/CIFS implementation.
+   Samba utility functions
+   Copyright (C) Andrew Tridgell 1992-1998
+   Copyright (C) Jeremy Allison 2001-2007
+   Copyright (C) Simo Sorce 2001
+   Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2003
+   Copyright (C) James Peach 2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "dynconfig/dynconfig.h"
+
+/**
+ * @brief Returns an absolute path to a file in the Samba modules directory.
+ *
+ * @param name File to find, relative to MODULESDIR.
+ *
+ * @retval Pointer to a string containing the full path.
+ **/
+
+char *modules_path(TALLOC_CTX *mem_ctx, const char *name)
+{
+	return talloc_asprintf(mem_ctx, "%s/%s", get_dyn_MODULESDIR(), name);
+}
+
+/**
+ * @brief Returns an absolute path to a file in the Samba data directory.
+ *
+ * @param name File to find, relative to CODEPAGEDIR.
+ *
+ * @retval Pointer to a talloc'ed string containing the full path.
+ **/
+
+char *data_path(TALLOC_CTX *mem_ctx, const char *name)
+{
+	return talloc_asprintf(mem_ctx, "%s/%s", get_dyn_CODEPAGEDIR(), name);
+}
+
+/**
+ * @brief Returns the platform specific shared library extension.
+ *
+ * @retval Pointer to a const char * containing the extension.
+ **/
+
+const char *shlib_ext(void)
+{
+	return get_dyn_SHLIBEXT();
+}
+
diff --git a/lib/util/util_str.c b/lib/util/util_str.c
index cf1b07ff0fa..388d7887ef6 100644
--- a/lib/util/util_str.c
+++ b/lib/util/util_str.c
@@ -32,87 +32,6 @@
  **/
 
 /**
- Safe string copy into a known length string. maxlength does not
- include the terminating zero.
-**/
-_PUBLIC_ char *safe_strcpy(char *dest,const char *src, size_t maxlength)
-{
-	size_t len;
-
-	if (!dest) {
-		DEBUG(0,("ERROR: NULL dest in safe_strcpy\n"));
-		return NULL;
-	}
-
-#ifdef DEVELOPER
-	/* We intentionally write out at the extremity of the destination
-	 * string.  If the destination is too short (e.g. pstrcpy into mallocd
-	 * or fstring) then this should cause an error under a memory
-	 * checker. */
-	dest[maxlength] = '\0';
-	if (PTR_DIFF(&len, dest) > 0) {  /* check if destination is on the stack, ok if so */
-		log_suspicious_usage("safe_strcpy", src);
-	}
-#endif
-
-	if (!src) {
-		*dest = 0;
-		return dest;
-	}  
-
-	len = strlen(src);
-
-	if (len > maxlength) {
-		DEBUG(0,("ERROR: string overflow by %u (%u - %u) in safe_strcpy [%.50s]\n",
-			 (unsigned int)(len-maxlength), (unsigned)len, (unsigned)maxlength, src));
-		len = maxlength;
-	}
-      
-	memmove(dest, src, len);
-	dest[len] = 0;
-	return dest;
-}  
-
-/**
- Safe string cat into a string. maxlength does not
- include the terminating zero.
-**/
-_PUBLIC_ char *safe_strcat(char *dest, const char *src, size_t maxlength)
-{
-	size_t src_len, dest_len;
-
-	if (!dest) {
-		DEBUG(0,("ERROR: NULL dest in safe_strcat\n"));
-		return NULL;
-	}
-
-	if (!src)
-		return dest;
-	
-#ifdef DEVELOPER
-	if (PTR_DIFF(&src_len, dest) > 0) {  /* check if destination is on the stack, ok if so */
-		log_suspicious_usage("safe_strcat", src);
-	}
-#endif
-	src_len = strlen(src);
-	dest_len = strlen(dest);
-
-	if (src_len + dest_len > maxlength) {
-		DEBUG(0,("ERROR: string overflow by %d in safe_strcat [%.50s]\n",
-			 (int)(src_len + dest_len - maxlength), src));
-		if (maxlength > dest_len) {
-			memcpy(&dest[dest_len], src, maxlength - dest_len);
-		}
-		dest[maxlength] = 0;
-		return NULL;
-	}
-	
-	memcpy(&dest[dest_len], src, src_len);
-	dest[dest_len + src_len] = 0;
-	return dest;
-}
-
-/**
   format a string into length-prefixed dotted domain format, as used in NBT
   and in some ADS structures
 **/
@@ -175,7 +94,7 @@ _PUBLIC_ bool conv_str_bool(const char * str, bool * val)
 /**
  * Convert a size specification like 16K into an integral number of bytes. 
  **/
-_PUBLIC_ bool conv_str_size(const char * str, uint64_t * val)
+_PUBLIC_ bool conv_str_size_error(const char * str, uint64_t * val)
 {
 	char *		    end = NULL;
 	unsigned long long  lval;
@@ -246,6 +165,6 @@ _PUBLIC_ bool strequal(const char *s1, const char *s2)
 	if (!s1 || !s2)
 		return false;
   
-	return strcasecmp(s1,s2) == 0;
+	return strcasecmp_m(s1,s2) == 0;
 }
 
diff --git a/lib/util/util_tdb.c b/lib/util/util_tdb.c
index 4a81678808e..02c7095f66b 100644
--- a/lib/util/util_tdb.c
+++ b/lib/util/util_tdb.c
@@ -20,7 +20,7 @@
 */
 
 #include "includes.h"
-#include <tdb.h>
+#include "../lib/tdb_compat/tdb_compat.h"
 #include "../lib/util/util_tdb.h"
 
 /* these are little tdb utility functions that are meant to make
@@ -57,7 +57,7 @@ TDB_DATA string_term_tdb_data(const char *string)
 }
 
 /****************************************************************************
- Lock a chain by string. Return -1 if lock failed.
+ Lock a chain by string. Return non-zero if lock failed.
 ****************************************************************************/
 
 int tdb_lock_bystring(struct tdb_context *tdb, const char *keyval)
@@ -79,7 +79,7 @@ void tdb_unlock_bystring(struct tdb_context *tdb, const char *keyval)
 }
 
 /****************************************************************************
- Read lock a chain by string. Return -1 if lock failed.
+ Read lock a chain by string. Return non-zero if lock failed.
 ****************************************************************************/
 
 int tdb_read_lock_bystring(struct tdb_context *tdb, const char *keyval)
@@ -111,7 +111,7 @@ int32_t tdb_fetch_int32_byblob(struct tdb_context *tdb, TDB_DATA key)
 	TDB_DATA data;
 	int32_t ret;
 
-	data = tdb_fetch(tdb, key);
+	data = tdb_fetch_compat(tdb, key);
 	if (!data.dptr || data.dsize != sizeof(int32_t)) {
 		SAFE_FREE(data.dptr);
 		return -1;
@@ -133,7 +133,7 @@ int32_t tdb_fetch_int32(struct tdb_context *tdb, const char *keystr)
 }
 
 /****************************************************************************
- Store a int32_t value by an arbitrary blob key, return 0 on success, -1 on failure.
+ Store a int32_t value by an arbitrary blob key, return 0 on success, -ve on failure.
  Input is int32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 
@@ -150,7 +150,7 @@ int tdb_store_int32_byblob(struct tdb_context *tdb, TDB_DATA key, int32_t v)
 }
 
 /****************************************************************************
- Store a int32_t value by string key, return 0 on success, -1 on failure.
+ Store a int32_t value by string key, return 0 on success, -ve on failure.
  Input is int32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 
@@ -168,7 +168,7 @@ bool tdb_fetch_uint32_byblob(struct tdb_context *tdb, TDB_DATA key, uint32_t *va
 {
 	TDB_DATA data;
 
-	data = tdb_fetch(tdb, key);
+	data = tdb_fetch_compat(tdb, key);
 	if (!data.dptr || data.dsize != sizeof(uint32_t)) {
 		SAFE_FREE(data.dptr);
 		return false;
@@ -190,7 +190,7 @@ bool tdb_fetch_uint32(struct tdb_context *tdb, const char *keystr, uint32_t *val
 }
 
 /****************************************************************************
- Store a uint32_t value by an arbitrary blob key, return 0 on success, -1 on failure.
+ Store a uint32_t value by an arbitrary blob key, return true on success, false on failure.
  Input is uint32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 
@@ -204,14 +204,14 @@ bool tdb_store_uint32_byblob(struct tdb_context *tdb, TDB_DATA key, uint32_t val
 	data.dptr = (unsigned char *)&v_store;
 	data.dsize = sizeof(uint32_t);
 
-	if (tdb_store(tdb, key, data, TDB_REPLACE) == -1)
+	if (tdb_store(tdb, key, data, TDB_REPLACE) != 0)
 		ret = false;
 
 	return ret;
 }
 
 /****************************************************************************
- Store a uint32_t value by string key, return 0 on success, -1 on failure.
+ Store a uint32_t value by string key, return true on success, false on failure.
  Input is uint32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 
@@ -220,7 +220,7 @@ bool tdb_store_uint32(struct tdb_context *tdb, const char *keystr, uint32_t valu
 	return tdb_store_uint32_byblob(tdb, string_term_tdb_data(keystr), value);
 }
 /****************************************************************************
- Store a buffer by a null terminated string key.  Return 0 on success, -1
+ Store a buffer by a null terminated string key.  Return 0 on success, -ve
  on failure.
 ****************************************************************************/
 
@@ -240,7 +240,7 @@ TDB_DATA tdb_fetch_bystring(struct tdb_context *tdb, const char *keystr)
 {
 	TDB_DATA key = string_term_tdb_data(keystr);
 
-	return tdb_fetch(tdb, key);
+	return tdb_fetch_compat(tdb, key);
 }
 
 /****************************************************************************
@@ -263,7 +263,7 @@ int32_t tdb_change_int32_atomic(struct tdb_context *tdb, const char *keystr, int
 	int32_t val;
 	int32_t ret = -1;
 
-	if (tdb_lock_bystring(tdb, keystr) == -1)
+	if (tdb_lock_bystring(tdb, keystr) != 0)
 		return -1;
 
 	if ((val = tdb_fetch_int32(tdb, keystr)) == -1) {
@@ -284,7 +284,7 @@ int32_t tdb_change_int32_atomic(struct tdb_context *tdb, const char *keystr, int
 	/* Increment value for storage and return next time */
 	val += change_val;
 		
-	if (tdb_store_int32(tdb, keystr, val) == -1)
+	if (tdb_store_int32(tdb, keystr, val) != 0)
 		goto err_out;
 
 	ret = 0;
@@ -304,7 +304,7 @@ bool tdb_change_uint32_atomic(struct tdb_context *tdb, const char *keystr, uint3
 	uint32_t val;
 	bool ret = false;
 
-	if (tdb_lock_bystring(tdb, keystr) == -1)
+	if (tdb_lock_bystring(tdb, keystr) != 0)
 		return false;
 
 	if (!tdb_fetch_uint32(tdb, keystr, &val)) {
diff --git a/lib/util/util_tdb.h b/lib/util/util_tdb.h
index d2f6648462a..2d805d7d20c 100644
--- a/lib/util/util_tdb.h
+++ b/lib/util/util_tdb.h
@@ -1,7 +1,27 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   tdb utility functions
+
+   Copyright (C) Andrew Tridgell 1992-2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef _____LIB_UTIL_UTIL_TDB_H__
 #define _____LIB_UTIL_UTIL_TDB_H__
 
-
 /***************************************************************
  Make a TDB_DATA and keep the const warning in one place
 ****************************************************************/
@@ -11,7 +31,7 @@ TDB_DATA string_tdb_data(const char *string);
 TDB_DATA string_term_tdb_data(const char *string);
 
 /****************************************************************************
- Lock a chain by string. Return -1 if lock failed.
+ Lock a chain by string. Return non-zero if lock failed.
 ****************************************************************************/
 int tdb_lock_bystring(struct tdb_context *tdb, const char *keyval);
 
@@ -21,7 +41,7 @@ int tdb_lock_bystring(struct tdb_context *tdb, const char *keyval);
 void tdb_unlock_bystring(struct tdb_context *tdb, const char *keyval);
 
 /****************************************************************************
- Read lock a chain by string. Return -1 if lock failed.
+ Read lock a chain by string. Return non-zero if lock failed.
 ****************************************************************************/
 int tdb_read_lock_bystring(struct tdb_context *tdb, const char *keyval);
 
@@ -43,13 +63,13 @@ int32_t tdb_fetch_int32_byblob(struct tdb_context *tdb, TDB_DATA key);
 int32_t tdb_fetch_int32(struct tdb_context *tdb, const char *keystr);
 
 /****************************************************************************
- Store a int32_t value by an arbitrary blob key, return 0 on success, -1 on failure.
+ Store a int32_t value by an arbitrary blob key, return 0 on success, -ve on failure.
  Input is int32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 int tdb_store_int32_byblob(struct tdb_context *tdb, TDB_DATA key, int32_t v);
 
 /****************************************************************************
- Store a int32_t value by string key, return 0 on success, -1 on failure.
+ Store a int32_t value by string key, return 0 on success, -ve on failure.
  Input is int32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 int tdb_store_int32(struct tdb_context *tdb, const char *keystr, int32_t v);
@@ -67,19 +87,19 @@ bool tdb_fetch_uint32_byblob(struct tdb_context *tdb, TDB_DATA key, uint32_t *va
 bool tdb_fetch_uint32(struct tdb_context *tdb, const char *keystr, uint32_t *value);
 
 /****************************************************************************
- Store a uint32_t value by an arbitrary blob key, return 0 on success, -1 on failure.
+ Store a uint32_t value by an arbitrary blob key, return true on success, false on failure.
  Input is uint32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 bool tdb_store_uint32_byblob(struct tdb_context *tdb, TDB_DATA key, uint32_t value);
 
 /****************************************************************************
- Store a uint32_t value by string key, return 0 on success, -1 on failure.
+ Store a uint32_t value by string key, return true on success, false on failure.
  Input is uint32_t in native byte order. Output in tdb is in little-endian.
 ****************************************************************************/
 bool tdb_store_uint32(struct tdb_context *tdb, const char *keystr, uint32_t value);
 
 /****************************************************************************
- Store a buffer by a null terminated string key.  Return 0 on success, -1
+ Store a buffer by a null terminated string key.  Return 0 on success, -ve
  on failure.
 ****************************************************************************/
 int tdb_store_bystring(struct tdb_context *tdb, const char *keystr, TDB_DATA data, int flags);
@@ -91,7 +111,7 @@ int tdb_store_bystring(struct tdb_context *tdb, const char *keystr, TDB_DATA dat
 TDB_DATA tdb_fetch_bystring(struct tdb_context *tdb, const char *keystr);
 
 /****************************************************************************
- Delete an entry using a null terminated string key. 
+ Delete an entry using a null terminated string key.  0 on success, -ve on err.
 ****************************************************************************/
 int tdb_delete_bystring(struct tdb_context *tdb, const char *keystr);
 
diff --git a/lib/util/wrap_xattr.h b/lib/util/wrap_xattr.h
index 64b28d250cd..745b93d764b 100644
--- a/lib/util/wrap_xattr.h
+++ b/lib/util/wrap_xattr.h
@@ -1,3 +1,24 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   POSIX NTVFS backend - xattr support using filesystem xattrs
+
+   Copyright (C) Andrew Tridgell 2004
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef __LIB_UTIL_WRAP_XATTR_H__
 #define __LIB_UTIL_WRAP_XATTR_H__
 
diff --git a/lib/util/wscript_build b/lib/util/wscript_build
index aad386ef2ad..bdc9d101509 100755
--- a/lib/util/wscript_build
+++ b/lib/util/wscript_build
@@ -1,69 +1,25 @@
 #!/usr/bin/env python
 
-common_util_sources = '''talloc_stack.c smb_threads.c xfile.c data_blob.c
+bld.SAMBA_LIBRARY('samba-util',
+                  source='''talloc_stack.c smb_threads.c xfile.c data_blob.c
                     util_file.c time.c rbtree.c rfc1738.c select.c
                     genrand.c fsusage.c blocking.c become_daemon.c
                     signal.c system.c params.c util.c util_id.c util_net.c
-                    util_strlist.c idtree.c debug.c fault.c base64.c
-                    util_str_common.c'''
-
-common_util_headers = 'debug.h'
-common_util_public_deps = 'talloc pthread LIBCRYPTO'
-s4_util_sources = '''dprintf.c ms_fnmatch.c parmlist.c substitute.c util_str.c'''
-s4_util_deps = 'DYNCONFIG'
-s4_util_public_deps = 'talloc CHARSET execinfo uid_wrapper'
-s4_util_public_headers = 'attr.h byteorder.h data_blob.h memory.h safe_string.h time.h talloc_stack.h xfile.h dlinklist.h util.h'
-s4_util_header_path = [ ('dlinklist.h util.h', '.'), ('*', 'util') ]
-
-if bld.env.enable_s3build or bld.env._SAMBA_BUILD_ == 3:
-    # as we move files into common between samba-util and samba-util3, move them here.
-    # Both samba-util and samba-util3 depend on this private library
-    bld.SAMBA_LIBRARY('samba-util-common',
-                    source=common_util_sources,
-                    public_deps=common_util_public_deps,
-                    # until we get all the dependencies in this library in common
-                    # we need to allow this library to be built with unresolved symbols
-                    allow_undefined_symbols=True,
-                    local_include=False,
-                    public_headers=common_util_headers,
-                    header_path= [('*', 'util') ],
-                    private_library=True
-                    )
-
-    if bld.env._SAMBA_BUILD_ == 4:
-        bld.SAMBA_LIBRARY('samba-util',
-                        source=s4_util_sources,
-                        deps=s4_util_deps + ' samba-util-common',
-                        public_deps=s4_util_public_deps,
-                        public_headers=s4_util_public_headers,
-                        header_path= s4_util_header_path,
-                        local_include=False,
-                        vnum='0.0.1',
-                        pc_files='samba-util.pc'
-                        )
-
-else:
-    if bld.env._SAMBA_BUILD_ == 4:
-        bld.SAMBA_LIBRARY('samba-util',
-                        source=s4_util_sources + " " + common_util_sources,
-                        deps=s4_util_deps,
-                        public_deps=s4_util_public_deps + ' ' + common_util_public_deps,
-                        public_headers=s4_util_public_headers + ' ' + common_util_headers,
-                        header_path= s4_util_header_path,
-                        local_include=False,
-                        vnum='0.0.1',
-                        pc_files='samba-util.pc'
-                        )
-
-    # dummy subsystem for avoid wider deps changes.
-    bld.SAMBA_SUBSYSTEM('samba-util-common',
-                        source=[],
-                        deps='samba-util',
-                        local_include=False,)
+                    util_strlist.c util_paths.c idtree.c debug.c fault.c base64.c
+                    util_str.c util_str_common.c substitute.c ms_fnmatch.c
+                    server_id.c dprintf.c parmlist.c''',
+                  deps='DYNCONFIG',
+                  public_deps='talloc execinfo uid_wrapper pthread LIBCRYPTO CHARSET',
+                  public_headers='debug.h attr.h byteorder.h data_blob.h memory.h safe_string.h time.h talloc_stack.h xfile.h dlinklist.h util.h string_wrappers.h',
+                  header_path= [ ('dlinklist.h util.h', '.'), ('*', 'util') ],
+                  local_include=False,
+                  vnum='0.0.1',
+                  pc_files='samba-util.pc'
+                  )
 
 bld.SAMBA_LIBRARY('asn1util',
                   source='asn1.c',
-                  deps='talloc samba-util-common',
+                  deps='talloc samba-util',
                   private_library=True,
                   local_include=False)
 
@@ -88,7 +44,7 @@ bld.SAMBA_LIBRARY('wrap_xattr',
 bld.SAMBA_LIBRARY('UTIL_TDB',
 	source='util_tdb.c',
 	local_include=False,
-	public_deps='tdb talloc',
+	public_deps='tdb_compat talloc',
                   private_library=True
 	)
 
@@ -121,3 +77,12 @@ bld.SAMBA_SUBSYSTEM('UTIL_PW',
 	local_include=False,
 	public_deps='talloc'
 	)
+
+
+bld.SAMBA_LIBRARY('tdb-wrap',
+                  source='tdb_wrap.c',
+                  deps='tdb_compat talloc samba-util',
+                  private_library=True,
+                  local_include=False
+                  )
+
diff --git a/libcli/auth/krb5_wrap.c b/libcli/auth/krb5_wrap.c
index c69e3946c67..e7e071d4841 100644
--- a/libcli/auth/krb5_wrap.c
+++ b/libcli/auth/krb5_wrap.c
@@ -5,6 +5,7 @@
    Copyright (C) Luke Howard 2002-2003
    Copyright (C) Andrew Bartlett <abartlet@samba.org> 2005-2011
    Copyright (C) Guenther Deschner 2005-2009
+   Copyright (C) Simo Sorce 2010.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -119,13 +120,15 @@ int create_kerberos_key_from_string_direct(krb5_context context,
 	krb5_error_code ret;
 	char *utf8_name;
 	size_t converted_size;
+	TALLOC_CTX *frame = talloc_stackframe();
 
-	if (!push_utf8_talloc(talloc_tos(), &utf8_name, name, &converted_size)) {
+	if (!push_utf8_talloc(frame, &utf8_name, name, &converted_size)) {
+		talloc_free(frame);
 		return ENOMEM;
 	}
 
 	ret = krb5_parse_name(context, utf8_name, principal);
-	TALLOC_FREE(utf8_name);
+	TALLOC_FREE(frame);
 	return ret;
 }
 
@@ -202,8 +205,8 @@ krb5_error_code smb_krb5_unparse_name(TALLOC_CTX *mem_ctx,
 
 	for (i = 0; i < len1; i++) {
 
-		p1 = krb5_princ_component(context, CONST_DISCARD(krb5_principal, princ1), i);
-		p2 = krb5_princ_component(context, CONST_DISCARD(krb5_principal, princ2), i);
+		p1 = krb5_princ_component(context, (krb5_principal)discard_const(princ1), i);
+		p2 = krb5_princ_component(context, (krb5_principal)discard_const(princ2), i);
 
 		if (p1->length != p2->length ||	memcmp(p1->data, p2->data, p1->length))
 			return False;
@@ -307,6 +310,44 @@ krb5_error_code smb_krb5_unparse_name(TALLOC_CTX *mem_ctx,
 	return ret;
 }
 
+char *gssapi_error_string(TALLOC_CTX *mem_ctx, 
+			  OM_uint32 maj_stat, OM_uint32 min_stat, 
+			  const gss_OID mech)
+{
+	OM_uint32 disp_min_stat, disp_maj_stat;
+	gss_buffer_desc maj_error_message;
+	gss_buffer_desc min_error_message;
+	char *maj_error_string, *min_error_string;
+	OM_uint32 msg_ctx = 0;
+
+	char *ret;
+
+	maj_error_message.value = NULL;
+	min_error_message.value = NULL;
+	maj_error_message.length = 0;
+	min_error_message.length = 0;
+	
+	disp_maj_stat = gss_display_status(&disp_min_stat, maj_stat, GSS_C_GSS_CODE,
+			   mech, &msg_ctx, &maj_error_message);
+	disp_maj_stat = gss_display_status(&disp_min_stat, min_stat, GSS_C_MECH_CODE,
+			   mech, &msg_ctx, &min_error_message);
+	
+	maj_error_string = talloc_strndup(mem_ctx, (char *)maj_error_message.value, maj_error_message.length);
+
+	min_error_string = talloc_strndup(mem_ctx, (char *)min_error_message.value, min_error_message.length);
+
+	ret = talloc_asprintf(mem_ctx, "%s: %s", maj_error_string, min_error_string);
+
+	talloc_free(maj_error_string);
+	talloc_free(min_error_string);
+
+	gss_release_buffer(&disp_min_stat, &maj_error_message);
+	gss_release_buffer(&disp_min_stat, &min_error_message);
+
+	return ret;
+}
+
+
  char *smb_get_krb5_error_message(krb5_context context, krb5_error_code code, TALLOC_CTX *mem_ctx)
 {
 	char *ret;
diff --git a/libcli/auth/krb5_wrap.h b/libcli/auth/krb5_wrap.h
index 4f333cc4b0d..82769aede93 100644
--- a/libcli/auth/krb5_wrap.h
+++ b/libcli/auth/krb5_wrap.h
@@ -31,7 +31,6 @@ int create_kerberos_key_from_string_direct(krb5_context context,
 					   krb5_enctype enctype);
 void kerberos_free_data_contents(krb5_context context, krb5_data *pdata);
 krb5_error_code smb_krb5_kt_free_entry(krb5_context context, krb5_keytab_entry *kt_entry);
-char *smb_get_krb5_error_message(krb5_context context, krb5_error_code code, TALLOC_CTX *mem_ctx);
 
  krb5_error_code smb_krb5_parse_name(krb5_context context,
 				const char *name, /* in unix charset */
@@ -54,6 +53,10 @@ krb5_error_code smb_krb5_unparse_name(TALLOC_CTX *mem_ctx,
 					 krb5_checksum *cksum,
 					 uint8_t *data,
 					  size_t length);
+char *gssapi_error_string(TALLOC_CTX *mem_ctx, 
+			  OM_uint32 maj_stat, OM_uint32 min_stat, 
+			  const gss_OID mech);
+char *smb_get_krb5_error_message(krb5_context context, krb5_error_code code, TALLOC_CTX *mem_ctx);
 
 krb5_error_code check_pac_checksum(TALLOC_CTX *mem_ctx,
 				   DATA_BLOB pac_data,
@@ -69,3 +72,8 @@ NTSTATUS kerberos_decode_pac(TALLOC_CTX *mem_ctx,
 			     krb5_const_principal client_principal,
 			     time_t tgs_authtime,
 			     struct PAC_DATA **pac_data_out);
+
+NTSTATUS gssapi_obtain_pac_blob(TALLOC_CTX *mem_ctx,
+				gss_ctx_id_t gssapi_context,
+				gss_name_t gss_client_name,
+				DATA_BLOB *pac_data);
diff --git a/libcli/auth/msrpc_parse.c b/libcli/auth/msrpc_parse.c
index bdbba3d76cc..8b64e98feb2 100644
--- a/libcli/auth/msrpc_parse.c
+++ b/libcli/auth/msrpc_parse.c
@@ -78,7 +78,7 @@ NTSTATUS msrpc_gen(TALLOC_CTX *mem_ctx,
 				s, &n);
 			if (!ret) {
 				va_end(ap);
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 			pointers[i].length = n;
 			pointers[i].length -= 2;
@@ -92,7 +92,7 @@ NTSTATUS msrpc_gen(TALLOC_CTX *mem_ctx,
 				s, &n);
 			if (!ret) {
 				va_end(ap);
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 			pointers[i].length = n;
 			pointers[i].length -= 1;
@@ -108,7 +108,7 @@ NTSTATUS msrpc_gen(TALLOC_CTX *mem_ctx,
 				s, &n);
 			if (!ret) {
 				va_end(ap);
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 			pointers[i].length = n;
 			pointers[i].length -= 2;
diff --git a/libcli/auth/msrpc_parse.h b/libcli/auth/msrpc_parse.h
index d976a95b737..47529f24a95 100644
--- a/libcli/auth/msrpc_parse.h
+++ b/libcli/auth/msrpc_parse.h
@@ -1,3 +1,24 @@
+/*
+   Unix SMB/CIFS implementation.
+   simple kerberos5/SPNEGO routines
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2002
+   Copyright (C) Andrew Bartlett 2002-2003
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef _LIBCLI_AUTH_MSRPC_PARSE_H__
 #define _LIBCLI_AUTH_MSRPC_PARSE_H__
 
diff --git a/libcli/auth/schannel_state_tdb.c b/libcli/auth/schannel_state_tdb.c
index c332bae8baa..76110b89eff 100644
--- a/libcli/auth/schannel_state_tdb.c
+++ b/libcli/auth/schannel_state_tdb.c
@@ -23,13 +23,11 @@
 
 #include "includes.h"
 #include "system/filesys.h"
-#include <tdb.h>
+#include "../lib/tdb_compat/tdb_compat.h"
 #include "../lib/util/util_tdb.h"
 #include "../libcli/auth/schannel.h"
 #include "../librpc/gen_ndr/ndr_schannel.h"
-#if _SAMBA_BUILD_ == 4
-#include "tdb_wrap.h"
-#endif
+#include "lib/util/tdb_wrap.h"
 
 #define SECRETS_SCHANNEL_STATE "SECRETS/SCHANNEL"
 
@@ -102,7 +100,7 @@ NTSTATUS schannel_store_session_key_tdb(struct tdb_wrap *tdb_sc,
 	ret = tdb_store_bystring(tdb_sc->tdb, keystr, value, TDB_REPLACE);
 	if (ret != TDB_SUCCESS) {
 		DEBUG(0,("Unable to add %s to session key db - %s\n",
-			 keystr, tdb_errorstr(tdb_sc->tdb)));
+			 keystr, tdb_errorstr_compat(tdb_sc->tdb)));
 		talloc_free(keystr);
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
diff --git a/libcli/auth/smbencrypt.c b/libcli/auth/smbencrypt.c
index 366f6df3adc..66fdbd25a80 100644
--- a/libcli/auth/smbencrypt.c
+++ b/libcli/auth/smbencrypt.c
@@ -118,7 +118,7 @@ bool E_deshash(const char *passwd, uint8_t p16[16])
 {
 	bool ret;
 	uint8_t dospwd[14];
-	TALLOC_CTX *mem_ctx;
+	TALLOC_CTX *frame = talloc_stackframe();
 
 	size_t converted_size;
 
@@ -126,23 +126,19 @@ bool E_deshash(const char *passwd, uint8_t p16[16])
 
 	ZERO_STRUCT(dospwd);
 
-#if _SAMBA_BUILD_ == 3
-	mem_ctx = talloc_tos();
-#else
-	mem_ctx = NULL;
-#endif
-	tmpbuf = strupper_talloc(mem_ctx, passwd);
+	tmpbuf = strupper_talloc(frame, passwd);
 	if (tmpbuf == NULL) {
 		/* Too many callers don't check this result, we need to fill in the buffer with something */
-		safe_strcpy((char *)dospwd, passwd, sizeof(dospwd)-1);
+		strlcpy((char *)dospwd, passwd ? passwd : "", sizeof(dospwd));
 		E_P16(dospwd, p16);
+		talloc_free(frame);
 		return false;
 	}
 
 	ZERO_STRUCT(dospwd);
 
 	ret = convert_string_error(CH_UNIX, CH_DOS, tmpbuf, strlen(tmpbuf), dospwd, sizeof(dospwd), &converted_size);
-	talloc_free(tmpbuf);
+	talloc_free(frame);
 
 	/* Only the first 14 chars are considered, password need not
 	 * be null terminated.  We do this in the error and success
diff --git a/libcli/auth/wscript_build b/libcli/auth/wscript_build
index bdf52d0399e..acb1ce38f06 100644
--- a/libcli/auth/wscript_build
+++ b/libcli/auth/wscript_build
@@ -2,7 +2,7 @@
 
 bld.SAMBA_LIBRARY('cliauth',
                   source='',
-                  deps='NTLMSSP_COMMON MSRPC_PARSE LIBCLI_AUTH COMMON_SCHANNEL PAM_ERRORS SPNEGO_PARSE KRB5_WRAP',
+                  deps='NTLMSSP_COMMON MSRPC_PARSE LIBCLI_AUTH COMMON_SCHANNEL PAM_ERRORS SPNEGO_PARSE KRB5_WRAP errors',
                   private_library=True,
                   grouping_library=True)
 
@@ -40,5 +40,5 @@ bld.SAMBA_SUBSYSTEM('SPNEGO_PARSE',
                     deps='asn1util')
 
 bld.SAMBA_SUBSYSTEM('KRB5_WRAP',
-                    source='krb5_wrap.c kerberos_pac.c',
-                    deps='krb5 ndr-krb5pac com_err')
+                    source='krb5_wrap.c',
+                    deps='gssapi_krb5 krb5 ndr-krb5pac com_err KRB5_PAC')
diff --git a/libcli/cldap/cldap.c b/libcli/cldap/cldap.c
index f5585c2b730..a5a0e415988 100644
--- a/libcli/cldap/cldap.c
+++ b/libcli/cldap/cldap.c
@@ -284,7 +284,7 @@ static bool cldap_socket_recv_dgram(struct cldap_socket *c,
 nomem:
 	in->recv_errno = ENOMEM;
 error:
-	status = map_nt_error_from_unix(in->recv_errno);
+	status = map_nt_error_from_unix_common(in->recv_errno);
 nterror:
 	/* in connected mode the first pending search gets the error */
 	if (!c->connected) {
@@ -337,7 +337,7 @@ NTSTATUS cldap_socket_init(TALLOC_CTX *mem_ctx,
 							NULL, 0,
 							&any);
 		if (ret != 0) {
-			status = map_nt_error_from_unix(errno);
+			status = map_nt_error_from_unix_common(errno);
 			goto nterror;
 		}
 		local_addr = any;
@@ -351,7 +351,7 @@ NTSTATUS cldap_socket_init(TALLOC_CTX *mem_ctx,
 	ret = tdgram_inet_udp_socket(local_addr, remote_addr,
 				     c, &c->sock);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		goto nterror;
 	}
 	talloc_free(any);
@@ -626,11 +626,12 @@ struct tevent_req *cldap_search_send(TALLOC_CTX *mem_ctx,
 	now = tevent_timeval_current();
 	end = now;
 	for (i = 0; i < state->request.count; i++) {
-		end = tevent_timeval_add(&end, 0, state->request.delay);
+		end = tevent_timeval_add(&end, state->request.delay / 1000000,
+					 state->request.delay % 1000000);
 	}
 
 	if (!tevent_req_set_endtime(req, state->caller.cldap->event.ctx, end)) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		goto post;
 	}
 
@@ -668,7 +669,7 @@ static void cldap_search_state_queue_done(struct tevent_req *subreq)
 	talloc_free(subreq);
 	if (ret == -1) {
 		NTSTATUS status;
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		DLIST_REMOVE(state->caller.cldap->searches.list, state);
 		ZERO_STRUCT(state->caller.cldap);
 		tevent_req_nterror(req, status);
@@ -679,7 +680,7 @@ static void cldap_search_state_queue_done(struct tevent_req *subreq)
 
 	/* wait for incoming traffic */
 	if (!cldap_recvfrom_setup(state->caller.cldap)) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return;
 	}
 
@@ -688,7 +689,8 @@ static void cldap_search_state_queue_done(struct tevent_req *subreq)
 		return;
 	}
 
-	next = tevent_timeval_current_ofs(0, state->request.delay);
+	next = tevent_timeval_current_ofs(state->request.delay / 1000000,
+					  state->request.delay % 1000000);
 	subreq = tevent_wakeup_send(state,
 				    state->caller.cldap->event.ctx,
 				    next);
diff --git a/libcli/cldap/wscript_build b/libcli/cldap/wscript_build
index d0c00d7f3b0..9dd94c66910 100644
--- a/libcli/cldap/wscript_build
+++ b/libcli/cldap/wscript_build
@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 
 
-bld.SAMBA_SUBSYSTEM('LIBCLI_CLDAP',
-	source='cldap.c',
-	public_deps='cli-ldap',
-	deps='LIBTSOCKET samba-util UTIL_TEVENT ldb LIBCLI_NETLOGON'
-	)
+bld.SAMBA_LIBRARY('LIBCLI_CLDAP',
+                  source='cldap.c',
+                  public_deps='cli-ldap',
+                  deps='LIBTSOCKET samba-util UTIL_TEVENT ldb LIBCLI_NETLOGON',
+                  private_library=True
+                  )
 
diff --git a/libcli/dns/dns.h b/libcli/dns/dns.h
new file mode 100644
index 00000000000..01aa6c4e6c4
--- /dev/null
+++ b/libcli/dns/dns.h
@@ -0,0 +1,68 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Internal DNS query structures
+ *  Copyright (C) Gerald Carter                2006.
+ *  Copyright (C) Andrew Bartlett 2011
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* DNS query section in replies */
+
+struct dns_query {
+	const char *hostname;
+	uint16_t type;
+	uint16_t in_class;
+};
+
+/* DNS RR record in reply */
+
+struct dns_rr {
+	const char *hostname;
+	uint16_t type;
+	uint16_t in_class;
+	uint32_t ttl;
+	uint16_t rdatalen;
+	uint8_t *rdata;
+};
+
+/* SRV records */
+
+struct dns_rr_srv {
+	const char *hostname;
+	uint16_t priority;
+	uint16_t weight;
+	uint16_t port;
+	size_t num_ips;
+	struct sockaddr_storage *ss_s;	/* support multi-homed hosts */
+};
+
+/* NS records */
+
+struct dns_rr_ns {
+	const char *hostname;
+	struct sockaddr_storage ss;
+};
+
+NTSTATUS resolve_dns_hosts_file_as_sockaddr(const char *dns_hosts_file,
+					    const char *name, bool srv_lookup,
+					    TALLOC_CTX *mem_ctx,
+					    struct sockaddr_storage **return_iplist,
+					    int *return_count);
+
+NTSTATUS resolve_dns_hosts_file_as_dns_rr(const char *dns_hosts_file,
+					  const char *name, bool srv_lookup,
+					  TALLOC_CTX *mem_ctx,
+					  struct dns_rr_srv **return_rr,
+					  int *return_count);
diff --git a/libcli/nbt/dns_hosts_file.c b/libcli/dns/dns_hosts_file.c
index 801393650e0..94d1d9704a3 100644
--- a/libcli/nbt/dns_hosts_file.c
+++ b/libcli/dns/dns_hosts_file.c
@@ -5,7 +5,7 @@
 
    Copyright (C) Andrew Tridgell 1994-1998
    Copyright (C) Jeremy Allison 2007
-   Copyright (C) Andrew Bartlett 2009.
+   Copyright (C) Andrew Bartlett 2009-2011
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -29,6 +29,11 @@
 #include "system/filesys.h"
 #include "system/network.h"
 #include "libcli/nbt/libnbt.h"
+#include "libcli/dns/dns.h"
+
+#ifdef strcasecmp
+#undef strcasecmp
+#endif
 
 /********************************************************
  Start parsing the dns_hosts_file file.
@@ -51,7 +56,7 @@ static XFILE *startdns_hosts_file(const char *fname)
 *********************************************************/
 
 static bool getdns_hosts_fileent(TALLOC_CTX *ctx, XFILE *fp, char **pp_name, char **pp_name_type,
-			  char **pp_next_name, 
+			  char **pp_next_name,
 			  struct sockaddr_storage *pss, uint32_t *p_port)
 {
 	char line[1024];
@@ -176,12 +181,12 @@ static void enddns_hosts_file(XFILE *fp)
  Resolve via "dns_hosts" method.
 *********************************************************/
 
-static NTSTATUS resolve_dns_hosts_file_as_sockaddr_recurse(const char *dns_hosts_file, 
-							   const char *name, bool srv_lookup,
-							   int level, uint32_t port, 
-							   TALLOC_CTX *mem_ctx, 
-							   struct sockaddr_storage **return_iplist,
-							   int *return_count)
+static NTSTATUS resolve_dns_hosts_file_as_dns_rr_recurse(const char *dns_hosts_file,
+							 const char *name, bool srv_lookup,
+							 int level, uint32_t port,
+							 TALLOC_CTX *mem_ctx,
+							 struct dns_rr_srv **return_rr,
+							 int *return_count)
 {
 	/*
 	 * "dns_hosts" means parse the local dns_hosts file.
@@ -196,18 +201,21 @@ static NTSTATUS resolve_dns_hosts_file_as_sockaddr_recurse(const char *dns_hosts
 	NTSTATUS status = NT_STATUS_OBJECT_NAME_NOT_FOUND;
 	TALLOC_CTX *ctx = NULL;
 	TALLOC_CTX *ip_list_ctx = NULL;
+	struct dns_rr_srv *rr = NULL;
+
+	*return_rr = NULL;
 
 	/* Don't recurse forever, even on our own flat files */
 	if (level > 11) {
-
+		DEBUG(0, ("resolve_dns_hosts_file recursion limit reached looking up %s!\n", name));
+		return status;
 	}
 
-	*return_iplist = NULL;
 	*return_count = 0;
 
-	DEBUG(3,("resolve_dns_hosts: "
-		"Attempting dns_hosts lookup for name %s\n",
-		name));
+	DEBUG(3,("resolve_dns_hosts: (%d) "
+		 "Attempting %s dns_hosts lookup for name %s\n",
+		 level, srv_lookup ? "SRV" : "A", name));
 
 	fp = startdns_hosts_file(dns_hosts_file);
 
@@ -229,79 +237,179 @@ static NTSTATUS resolve_dns_hosts_file_as_sockaddr_recurse(const char *dns_hosts
 
 	while (getdns_hosts_fileent(ctx, fp, &host_name, &name_type, &next_name, &return_ss, &srv_port)) {
 		if (!strequal(name, host_name)) {
-			TALLOC_FREE(ctx);
-			ctx = talloc_new(mem_ctx);
-			if (!ctx) {
-				enddns_hosts_file(fp);
-				return NT_STATUS_NO_MEMORY;
-			}
-
-			continue;
-		}
-
-		if (srv_lookup) {
+			/* continue at the bottom of the loop */
+		} else if (srv_lookup) {
 			if (strcasecmp(name_type, "SRV") == 0) {
+				NTSTATUS status_recurse;
+				struct dns_rr_srv *tmp_rr;
+				int tmp_count = 0;
 				/* we only accept one host name per SRV entry */
-				enddns_hosts_file(fp);
-				status = resolve_dns_hosts_file_as_sockaddr_recurse(dns_hosts_file, next_name, 
-										    false, 
-										    level + 1, srv_port, 
-										    mem_ctx, return_iplist, 
-										    return_count);
-				talloc_free(ip_list_ctx);
-				return status;
-			} else {
-				continue;
+				status_recurse
+					= resolve_dns_hosts_file_as_dns_rr_recurse(dns_hosts_file, next_name,
+										     false,
+										     level + 1, srv_port,
+										     ip_list_ctx, &tmp_rr,
+										     &tmp_count);
+				if (NT_STATUS_EQUAL(status_recurse, NT_STATUS_OBJECT_NAME_NOT_FOUND)) {
+					/* Don't fail on a dangling SRV record */
+				} else if (!NT_STATUS_IS_OK(status_recurse)) {
+					enddns_hosts_file(fp);
+					talloc_free(ip_list_ctx);
+					return status_recurse;
+				} else if (tmp_count != 1) {
+					status = NT_STATUS_OBJECT_NAME_NOT_FOUND;
+				} else {
+					status = status_recurse;
+					rr = talloc_realloc(ip_list_ctx, rr, struct dns_rr_srv, (*return_count) + 1);
+					if (!rr) {
+						enddns_hosts_file(fp);
+						return NT_STATUS_NO_MEMORY;
+					}
+					talloc_steal(rr, tmp_rr);
+					rr[*return_count] = *tmp_rr;
+					*return_count = (*return_count) + 1;
+				}
 			}
 		} else if (strcasecmp(name_type, "CNAME") == 0) {
 			/* we only accept one host name per CNAME */
 			enddns_hosts_file(fp);
-			status = resolve_dns_hosts_file_as_sockaddr_recurse(dns_hosts_file, next_name, false, 
-									    level + 1, port, 
-									    mem_ctx, return_iplist, return_count);
+			status = resolve_dns_hosts_file_as_dns_rr_recurse(dns_hosts_file, next_name, false,
+									  level + 1, port,
+									  mem_ctx, return_rr, return_count);
 			talloc_free(ip_list_ctx);
 			return status;
 		} else if (strcasecmp(name_type, "A") == 0) {
+			if (*return_count == 0) {
+				/* We are happy to keep looking for other possible A record matches */
+				rr = talloc_zero(ip_list_ctx,
+						  struct dns_rr_srv);
+
+				if (rr == NULL) {
+					TALLOC_FREE(ctx);
+					enddns_hosts_file(fp);
+					DEBUG(3,("resolve_dns_hosts: talloc_realloc fail !\n"));
+					return NT_STATUS_NO_MEMORY;
+				}
+
+				rr->hostname = talloc_strdup(rr, host_name);
+
+				if (rr->hostname == NULL) {
+					TALLOC_FREE(ctx);
+					enddns_hosts_file(fp);
+					DEBUG(3,("resolve_dns_hosts: talloc_realloc fail !\n"));
+					return NT_STATUS_NO_MEMORY;
+				}
+				rr->port = port;
+
+				*return_count = 1;
+			}
+
 			/* Set the specified port (possibly from a SRV lookup) into the structure we return */
 			set_sockaddr_port((struct sockaddr *)&return_ss, port);
 
 			/* We are happy to keep looking for other possible A record matches */
-			*return_iplist = talloc_realloc(ip_list_ctx, (*return_iplist), 
-							struct sockaddr_storage,
-							(*return_count)+1);
+			rr->ss_s = talloc_realloc(rr, rr->ss_s,
+						  struct sockaddr_storage,
+						  rr->num_ips + 1);
 
-			if ((*return_iplist) == NULL) {
+			if (rr->ss_s == NULL) {
 				TALLOC_FREE(ctx);
 				enddns_hosts_file(fp);
 				DEBUG(3,("resolve_dns_hosts: talloc_realloc fail !\n"));
 				return NT_STATUS_NO_MEMORY;
 			}
-			
-			(*return_iplist)[*return_count] = return_ss;
-			*return_count += 1;
-			
+
+			rr->ss_s[rr->num_ips] = return_ss;
+			rr->num_ips += 1;
+
 			/* we found something */
 			status = NT_STATUS_OK;
 		}
+
+		TALLOC_FREE(ctx);
+		ctx = talloc_new(mem_ctx);
+		if (!ctx) {
+			enddns_hosts_file(fp);
+			return NT_STATUS_NO_MEMORY;
+		}
 	}
 
-	talloc_steal(mem_ctx, *return_iplist);
+	*return_rr = talloc_steal(mem_ctx, rr);
 	TALLOC_FREE(ip_list_ctx);
 	enddns_hosts_file(fp);
 	return status;
 }
 
 /********************************************************
- Resolve via "dns_hosts" method.
+ Resolve via "dns_hosts_file" method, returning a list of sockaddr_storage values
 *********************************************************/
 
-NTSTATUS resolve_dns_hosts_file_as_sockaddr(const char *dns_hosts_file, 
+NTSTATUS resolve_dns_hosts_file_as_sockaddr(const char *dns_hosts_file,
 					    const char *name, bool srv_lookup,
-					    TALLOC_CTX *mem_ctx, 
+					    TALLOC_CTX *mem_ctx,
 					    struct sockaddr_storage **return_iplist,
 					    int *return_count)
 {
-	return resolve_dns_hosts_file_as_sockaddr_recurse(dns_hosts_file, name, srv_lookup, 
-							  0, 0, 
-							  mem_ctx, return_iplist, return_count);
+	NTSTATUS status;
+	struct dns_rr_srv *dns_rr = NULL;
+	int i, j, rr_count = 0;
+
+	*return_iplist = NULL;
+	*return_count = 0;
+
+	status = resolve_dns_hosts_file_as_dns_rr_recurse(dns_hosts_file, name, srv_lookup,
+							  0, 0,
+							  mem_ctx, &dns_rr, &rr_count);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(3,("resolve_dns_hosts (sockaddr): "
+			 "failed to obtain %s result records for for name %s: %s\n",
+			 srv_lookup ? "SRV" : "A", name, nt_errstr(status)));
+		return status;
+	}
+
+	for (i=0; i < rr_count; i++) {
+		*return_iplist = talloc_realloc(mem_ctx, *return_iplist, struct sockaddr_storage, *return_count + dns_rr[i].num_ips);
+		if (!*return_iplist) {
+			return NT_STATUS_NO_MEMORY;
+		}
+		for (j=0; j < dns_rr[i].num_ips; j++) {
+			(*return_iplist)[*return_count] = dns_rr[i].ss_s[j];
+			*return_count = *return_count + 1;
+		}
+	}
+	DEBUG(3,("resolve_dns_hosts (sockaddr): "
+		"Found %d results for for name %s\n",
+		 *return_count, name));
+
+	return NT_STATUS_OK;
+}
+
+/********************************************************
+ Resolve via "dns_hosts_file" method, returning struct dns_rr_srv
+*********************************************************/
+
+NTSTATUS resolve_dns_hosts_file_as_dns_rr(const char *dns_hosts_file,
+					    const char *name, bool srv_lookup,
+					    TALLOC_CTX *mem_ctx,
+					    struct dns_rr_srv **return_rr,
+					    int *return_count)
+{
+	NTSTATUS status;
+	*return_rr = NULL;
+	*return_count = 0;
+
+	status = resolve_dns_hosts_file_as_dns_rr_recurse(dns_hosts_file, name, srv_lookup,
+							  0, 0,
+							  mem_ctx, return_rr, return_count);
+
+	if (NT_STATUS_IS_OK(status)) {
+		DEBUG(3,("resolve_dns_hosts (dns_rr): "
+			 "Found %d %s result records for for name %s\n",
+			 *return_count, srv_lookup ? "SRV" : "A", name));
+	} else {
+		DEBUG(3,("resolve_dns_hosts (dns_rr): "
+			 "failed to obtain %s result records for for name %s: %s\n",
+			 srv_lookup ? "SRV" : "A", name, nt_errstr(status)));
+	}
+	return status;
 }
diff --git a/libcli/echo/echo.c b/libcli/echo/echo.c
index 271f71b0ae8..b6005cb32c0 100644
--- a/libcli/echo/echo.c
+++ b/libcli/echo/echo.c
@@ -75,20 +75,20 @@ struct tevent_req *echo_request_send(TALLOC_CTX *mem_ctx,
 	ret = tsocket_address_inet_from_strings(state, "ip", NULL, 0,
 						&local_addr);
 	if (ret != 0) {
-		tevent_req_nterror(req, map_nt_error_from_unix(ret));
+		tevent_req_nterror(req, map_nt_error_from_unix_common(ret));
 		return tevent_req_post(req, ev);
 	}
 
 	ret = tsocket_address_inet_from_strings(state, "ip", server_addr_string,
 						ECHO_PORT, &server_addr);
 	if (ret != 0) {
-		tevent_req_nterror(req, map_nt_error_from_unix(ret));
+		tevent_req_nterror(req, map_nt_error_from_unix_common(ret));
 		return tevent_req_post(req, ev);
 	}
 
 	ret = tdgram_inet_udp_socket(local_addr, server_addr, state, &dgram);
 	if (ret != 0) {
-		tevent_req_nterror(req, map_nt_error_from_unix(ret));
+		tevent_req_nterror(req, map_nt_error_from_unix_common(ret));
 		return tevent_req_post(req, ev);
 	}
 
@@ -132,7 +132,7 @@ static void echo_request_get_reply(struct tevent_req *subreq)
 	TALLOC_FREE(subreq);
 
 	if (len == -1 && err != 0) {
-		tevent_req_nterror(req, map_nt_error_from_unix(err));
+		tevent_req_nterror(req, map_nt_error_from_unix_common(err));
 		return;
 	}
 
@@ -168,7 +168,7 @@ static void echo_request_done(struct tevent_req *subreq)
 	TALLOC_FREE(subreq);
 
 	if (len == -1 && err != 0) {
-		tevent_req_nterror(req, map_nt_error_from_unix(err));
+		tevent_req_nterror(req, map_nt_error_from_unix_common(err));
 		return;
 	}
 
diff --git a/libcli/echo/tests/echo.c b/libcli/echo/tests/echo.c
index c47b7d0d0fc..6424c810016 100644
--- a/libcli/echo/tests/echo.c
+++ b/libcli/echo/tests/echo.c
@@ -64,8 +64,10 @@ static bool torture_echo_udp(struct torture_context *tctx)
 
 	make_nbt_name_server(&name,
 			     torture_setting_string(tctx, "host", NULL));
-	status = resolve_name(lpcfg_resolve_context(tctx->lp_ctx), &name, tctx,
-			      &address, tctx->ev);
+	status = resolve_name_ex(lpcfg_resolve_context(tctx->lp_ctx),
+				 0, 0,
+				 &name, tctx,
+				 &address, tctx->ev);
 	if (!NT_STATUS_IS_OK(status)) {
 		printf("Failed to resolve %s - %s\n", name.name,
 		       nt_errstr(status));
diff --git a/libcli/ldap/ldap_ndr.h b/libcli/ldap/ldap_ndr.h
index df5ee478cb9..8b3c55d5735 100644
--- a/libcli/ldap/ldap_ndr.h
+++ b/libcli/ldap/ldap_ndr.h
@@ -1,3 +1,25 @@
+/*
+   Unix SMB/CIFS mplementation.
+
+   wrap/unwrap NDR encoded elements for ldap calls
+
+   Copyright (C) Andrew Tridgell  2005
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
 #ifndef __LIBCLI_LDAP_LDAP_NDR_H__
 #define __LIBCLI_LDAP_LDAP_NDR_H__
 
diff --git a/libcli/named_pipe_auth/npa_tstream.h b/libcli/named_pipe_auth/npa_tstream.h
index f2223170f0c..1d85de7d911 100644
--- a/libcli/named_pipe_auth/npa_tstream.h
+++ b/libcli/named_pipe_auth/npa_tstream.h
@@ -23,6 +23,7 @@
 struct tevent_req;
 struct tevent_context;
 struct auth_session_info_transport;
+struct tsocket_address;
 
 struct tevent_req *tstream_npa_connect_send(TALLOC_CTX *mem_ctx,
 					    struct tevent_context *ev,
diff --git a/libcli/nbt/namerefresh.c b/libcli/nbt/namerefresh.c
index 79c6c1f2ea1..b525356c747 100644
--- a/libcli/nbt/namerefresh.c
+++ b/libcli/nbt/namerefresh.c
@@ -331,7 +331,7 @@ _PUBLIC_ NTSTATUS nbt_name_refresh_wins(struct nbt_name_socket *nbtsock,
 	}
 
 	if (!tevent_req_poll(subreq, ev)) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		talloc_free(frame);
 		return status;
 	}
diff --git a/libcli/nbt/nameregister.c b/libcli/nbt/nameregister.c
index 204fb6f3284..ff5418c85e5 100644
--- a/libcli/nbt/nameregister.c
+++ b/libcli/nbt/nameregister.c
@@ -283,7 +283,7 @@ NTSTATUS nbt_name_register_bcast(struct nbt_name_socket *nbtsock,
 	}
 
 	if (!tevent_req_poll(subreq, ev)) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		talloc_free(frame);
 		return status;
 	}
@@ -498,7 +498,7 @@ _PUBLIC_ NTSTATUS nbt_name_register_wins(struct nbt_name_socket *nbtsock,
 	}
 
 	if (!tevent_req_poll(subreq, ev)) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		talloc_free(frame);
 		return status;
 	}
diff --git a/libcli/nbt/nbt_proto.h b/libcli/nbt/nbt_proto.h
index b2e8d8525a6..281ce25a864 100644
--- a/libcli/nbt/nbt_proto.h
+++ b/libcli/nbt/nbt_proto.h
@@ -1,3 +1,24 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   manipulate nbt name structures
+
+   Copyright (C) Andrew Tridgell 2005
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef _____LIBCLI_NBT_NBT_PROTO_H__
 #define _____LIBCLI_NBT_NBT_PROTO_H__
 
diff --git a/libcli/nbt/tools/nmblookup.c b/libcli/nbt/tools/nmblookup.c
index 7950ab80015..9b875b08c49 100644
--- a/libcli/nbt/tools/nmblookup.c
+++ b/libcli/nbt/tools/nmblookup.c
@@ -246,9 +246,9 @@ static bool process_one(struct loadparm_context *lp_ctx, struct tevent_context *
 	} else {
 		int i, num_interfaces;
 
-		num_interfaces = iface_count(ifaces);
+		num_interfaces = iface_list_count(ifaces);
 		for (i=0;i<num_interfaces;i++) {
-			const char *bcast = iface_n_bcast(ifaces, i);
+			const char *bcast = iface_list_n_bcast(ifaces, i);
 			if (bcast == NULL) continue;
 			status = do_node_query(nbtsock, bcast, nbt_port, 
 					       node_name, node_type, true);
@@ -357,7 +357,7 @@ int main(int argc, const char *argv[])
 		exit(1);
 	}
 
-	load_interfaces(NULL, lpcfg_interfaces(cmdline_lp_ctx), &ifaces);
+	load_interface_list(NULL, cmdline_lp_ctx, &ifaces);
 
 	ev = s4_event_context_init(talloc_autofree_context());
 
diff --git a/libcli/nbt/wscript_build b/libcli/nbt/wscript_build
index 6f7a1745b75..07d8c51b1f4 100644
--- a/libcli/nbt/wscript_build
+++ b/libcli/nbt/wscript_build
@@ -7,13 +7,13 @@ bld.SAMBA_SUBSYSTEM('NDR_NBT_BUF',
 	)
 
 bld.SAMBA_SUBSYSTEM('lmhosts',
-                    source='lmhosts.c',
+                    source='lmhosts.c ../dns/dns_hosts_file.c',
                     deps='replace talloc'
                     )
 
 if bld.env._SAMBA_BUILD_ == 4:
     bld.SAMBA_LIBRARY('cli-nbt',
-                      source='nbtsocket.c namequery.c nameregister.c namerefresh.c namerelease.c dns_hosts_file.c',
+                      source='nbtsocket.c namequery.c nameregister.c namerefresh.c namerelease.c',
                       public_deps='ndr NDR_NBT tevent UTIL_TEVENT NDR_SECURITY samba_socket samba-util lmhosts',
                       private_library=True
                       )
diff --git a/libcli/registry/wscript_build b/libcli/registry/wscript_build
index b9c73aad604..ec3fdd3b6bd 100644
--- a/libcli/registry/wscript_build
+++ b/libcli/registry/wscript_build
@@ -1,4 +1,5 @@
 
-bld.SAMBA_SUBSYSTEM('util_reg',
+bld.SAMBA_LIBRARY('util_reg',
                     source='util_reg.c',
-                    deps='ndr')
+                    deps='ndr',
+		    private_library=True)
diff --git a/libcli/security/create_descriptor.c b/libcli/security/create_descriptor.c
index 62919a19c73..92528965d1f 100644
--- a/libcli/security/create_descriptor.c
+++ b/libcli/security/create_descriptor.c
@@ -305,6 +305,7 @@ static void cr_descr_log_descriptor(struct security_descriptor *sd,
 	}
 }
 
+#if 0
 static void cr_descr_log_acl(struct security_acl *acl,
 				    const char *message,
 				    int level)
@@ -318,6 +319,7 @@ static void cr_descr_log_acl(struct security_acl *acl,
 		DEBUG(level,("%s: NULL\n", message));
 	}
 }
+#endif
 
 static bool compute_acl(struct security_descriptor *parent_sd,
 			struct security_descriptor *creator_sd,
diff --git a/libcli/security/dom_sid.c b/libcli/security/dom_sid.c
index 2f80a36ae14..16b7af92f4a 100644
--- a/libcli/security/dom_sid.c
+++ b/libcli/security/dom_sid.c
@@ -367,8 +367,7 @@ int dom_sid_string_buf(const struct dom_sid *sid, char *buf, int buflen)
 	uint32_t ia;
 
 	if (!sid) {
-		strlcpy(buf, "(NULL SID)", buflen);
-		return 10;	/* strlen("(NULL SID)") */
+		return strlcpy(buf, "(NULL SID)", buflen);
 	}
 
 	ia = (sid->id_auth[5]) +
diff --git a/source3/lib/secdesc.c b/libcli/security/secdesc.c
index 001eccb5763..2c8fbc59ca4 100644
--- a/source3/lib/secdesc.c
+++ b/libcli/security/secdesc.c
@@ -1,28 +1,28 @@
-/* 
+/*
  *  Unix SMB/Netbios implementation.
  *  SEC_DESC handling functions
  *  Copyright (C) Andrew Tridgell              1992-1998,
  *  Copyright (C) Jeremy R. Allison            1995-2003.
  *  Copyright (C) Luke Kenneth Casson Leighton 1996-1998,
  *  Copyright (C) Paul Ashton                  1997-1998.
- *  
+ *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
  *  the Free Software Foundation; either version 3 of the License, or
  *  (at your option) any later version.
- *  
+ *
  *  This program is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *  GNU General Public License for more details.
- *  
+ *
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
 #include "includes.h"
-#include "../librpc/gen_ndr/ndr_security.h"
-#include "../libcli/security/security.h"
+#include "librpc/gen_ndr/ndr_security.h"
+#include "libcli/security/security.h"
 
 #define ALL_SECURITY_INFORMATION (SECINFO_OWNER|SECINFO_GROUP|\
 					SECINFO_DACL|SECINFO_SACL|\
@@ -68,7 +68,7 @@ uint32_t get_sec_info(const struct security_descriptor *sd)
 
 
 /*******************************************************************
- Merge part of security descriptor old_sec in to the empty sections of 
+ Merge part of security descriptor old_sec in to the empty sections of
  security descriptor new_sec.
 ********************************************************************/
 
@@ -78,7 +78,7 @@ struct sec_desc_buf *sec_desc_merge_buf(TALLOC_CTX *ctx, struct sec_desc_buf *ne
 	struct sec_desc_buf *return_sdb;
 	struct security_acl *dacl, *sacl;
 	struct security_descriptor *psd = NULL;
-	uint16 secdesc_type;
+	uint16_t secdesc_type;
 	size_t secdesc_size;
 
 	/* Copy over owner and group sids.  There seems to be no flag for
@@ -89,11 +89,11 @@ struct sec_desc_buf *sec_desc_merge_buf(TALLOC_CTX *ctx, struct sec_desc_buf *ne
 
 	group_sid = new_sdb->sd->group_sid ? new_sdb->sd->group_sid :
 		old_sdb->sd->group_sid;
-	
+
 	secdesc_type = new_sdb->sd->type;
 
 	/* Ignore changes to the system ACL.  This has the effect of making
-	   changes through the security tab audit button not sticking. 
+	   changes through the security tab audit button not sticking.
 	   Perhaps in future Samba could implement these settings somehow. */
 
 	sacl = NULL;
@@ -122,7 +122,7 @@ struct security_descriptor *sec_desc_merge(TALLOC_CTX *ctx, struct security_desc
 	struct dom_sid *owner_sid, *group_sid;
 	struct security_acl *dacl, *sacl;
 	struct security_descriptor *psd = NULL;
-	uint16 secdesc_type;
+	uint16_t secdesc_type;
 	size_t secdesc_size;
 
 	/* Copy over owner and group sids.  There seems to be no flag for
@@ -162,20 +162,22 @@ struct security_descriptor *sec_desc_merge(TALLOC_CTX *ctx, struct security_desc
  Creates a struct security_descriptor structure
 ********************************************************************/
 
-#define  SEC_DESC_HEADER_SIZE (2 * sizeof(uint16) + 4 * sizeof(uint32))
+#define  SEC_DESC_HEADER_SIZE (2 * sizeof(uint16_t) + 4 * sizeof(uint32_t))
 
 struct security_descriptor *make_sec_desc(TALLOC_CTX *ctx,
 			enum security_descriptor_revision revision,
-			uint16 type,
+			uint16_t type,
 			const struct dom_sid *owner_sid, const struct dom_sid *grp_sid,
 			struct security_acl *sacl, struct security_acl *dacl, size_t *sd_size)
 {
 	struct security_descriptor *dst;
-	uint32 offset     = 0;
+	uint32_t offset     = 0;
 
-	*sd_size = 0;
+	if (sd_size != NULL) {
+		*sd_size = 0;
+	}
 
-	if(( dst = TALLOC_ZERO_P(ctx, struct security_descriptor)) == NULL)
+	if(( dst = talloc_zero(ctx, struct security_descriptor)) == NULL)
 		return NULL;
 
 	dst->revision = revision;
@@ -203,6 +205,10 @@ struct security_descriptor *make_sec_desc(TALLOC_CTX *ctx,
 	if(dacl && ((dst->dacl = dup_sec_acl(dst, dacl)) == NULL))
 		goto error_exit;
 
+	if (sd_size == NULL) {
+		return dst;
+	}
+
 	offset = SEC_DESC_HEADER_SIZE;
 
 	/*
@@ -229,7 +235,9 @@ struct security_descriptor *make_sec_desc(TALLOC_CTX *ctx,
 
 error_exit:
 
-	*sd_size = 0;
+	if (sd_size != NULL) {
+		*sd_size = 0;
+	}
 	return NULL;
 }
 
@@ -254,7 +262,7 @@ struct security_descriptor *dup_sec_desc(TALLOC_CTX *ctx, const struct security_
 ********************************************************************/
 NTSTATUS marshall_sec_desc(TALLOC_CTX *mem_ctx,
 			   struct security_descriptor *secdesc,
-			   uint8 **data, size_t *len)
+			   uint8_t **data, size_t *len)
 {
 	DATA_BLOB blob;
 	enum ndr_err_code ndr_err;
@@ -303,7 +311,7 @@ NTSTATUS marshall_sec_desc_buf(TALLOC_CTX *mem_ctx,
 /*******************************************************************
  Parse a byte stream into a secdesc
 ********************************************************************/
-NTSTATUS unmarshall_sec_desc(TALLOC_CTX *mem_ctx, uint8 *data, size_t len,
+NTSTATUS unmarshall_sec_desc(TALLOC_CTX *mem_ctx, uint8_t *data, size_t len,
 			     struct security_descriptor **psecdesc)
 {
 	DATA_BLOB blob;
@@ -314,7 +322,7 @@ NTSTATUS unmarshall_sec_desc(TALLOC_CTX *mem_ctx, uint8 *data, size_t len,
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	result = TALLOC_ZERO_P(mem_ctx, struct security_descriptor);
+	result = talloc_zero(mem_ctx, struct security_descriptor);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -350,7 +358,7 @@ NTSTATUS unmarshall_sec_desc_buf(TALLOC_CTX *mem_ctx, uint8_t *data, size_t len,
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	result = TALLOC_ZERO_P(mem_ctx, struct sec_desc_buf);
+	result = talloc_zero(mem_ctx, struct sec_desc_buf);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -391,12 +399,12 @@ struct sec_desc_buf *make_sec_desc_buf(TALLOC_CTX *ctx, size_t len, struct secur
 {
 	struct sec_desc_buf *dst;
 
-	if((dst = TALLOC_ZERO_P(ctx, struct sec_desc_buf)) == NULL)
+	if((dst = talloc_zero(ctx, struct sec_desc_buf)) == NULL)
 		return NULL;
 
 	/* max buffer size (allocated size) */
-	dst->sd_size = (uint32)len;
-	
+	dst->sd_size = (uint32_t)len;
+
 	if(sec_desc && ((dst->sd = dup_sec_desc(ctx, sec_desc)) == NULL)) {
 		return NULL;
 	}
@@ -420,7 +428,7 @@ struct sec_desc_buf *dup_sec_desc_buf(TALLOC_CTX *ctx, struct sec_desc_buf *src)
  Add a new SID with its permissions to struct security_descriptor.
 ********************************************************************/
 
-NTSTATUS sec_desc_add_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, const struct dom_sid *sid, uint32 mask, size_t *sd_size)
+NTSTATUS sec_desc_add_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, const struct dom_sid *sid, uint32_t mask, size_t *sd_size)
 {
 	struct security_descriptor *sd   = 0;
 	struct security_acl  *dacl = 0;
@@ -433,14 +441,14 @@ NTSTATUS sec_desc_add_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, con
 	*sd_size = 0;
 
 	status = sec_ace_add_sid(ctx, &ace, psd[0]->dacl->aces, &psd[0]->dacl->num_aces, sid, mask);
-	
+
 	if (!NT_STATUS_IS_OK(status))
 		return status;
 
 	if (!(dacl = make_sec_acl(ctx, psd[0]->dacl->revision, psd[0]->dacl->num_aces, ace)))
 		return NT_STATUS_UNSUCCESSFUL;
-	
-	if (!(sd = make_sec_desc(ctx, psd[0]->revision, psd[0]->type, psd[0]->owner_sid, 
+
+	if (!(sd = make_sec_desc(ctx, psd[0]->revision, psd[0]->type, psd[0]->owner_sid,
 		psd[0]->group_sid, psd[0]->sacl, dacl, sd_size)))
 		return NT_STATUS_UNSUCCESSFUL;
 
@@ -453,7 +461,7 @@ NTSTATUS sec_desc_add_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, con
  Modify a SID's permissions in a struct security_descriptor.
 ********************************************************************/
 
-NTSTATUS sec_desc_mod_sid(struct security_descriptor *sd, struct dom_sid *sid, uint32 mask)
+NTSTATUS sec_desc_mod_sid(struct security_descriptor *sd, struct dom_sid *sid, uint32_t mask)
 {
 	NTSTATUS status;
 
@@ -464,7 +472,7 @@ NTSTATUS sec_desc_mod_sid(struct security_descriptor *sd, struct dom_sid *sid, u
 
 	if (!NT_STATUS_IS_OK(status))
 		return status;
-	
+
 	return NT_STATUS_OK;
 }
 
@@ -483,7 +491,7 @@ NTSTATUS sec_desc_del_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, str
 		return NT_STATUS_INVALID_PARAMETER;
 
 	*sd_size = 0;
-	
+
 	status = sec_ace_del_sid(ctx, &ace, psd[0]->dacl->aces, &psd[0]->dacl->num_aces, sid);
 
 	if (!NT_STATUS_IS_OK(status))
@@ -491,8 +499,8 @@ NTSTATUS sec_desc_del_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, str
 
 	if (!(dacl = make_sec_acl(ctx, psd[0]->dacl->revision, psd[0]->dacl->num_aces, ace)))
 		return NT_STATUS_UNSUCCESSFUL;
-	
-	if (!(sd = make_sec_desc(ctx, psd[0]->revision, psd[0]->type, psd[0]->owner_sid, 
+
+	if (!(sd = make_sec_desc(ctx, psd[0]->revision, psd[0]->type, psd[0]->owner_sid,
 		psd[0]->group_sid, psd[0]->sacl, dacl, sd_size)))
 		return NT_STATUS_UNSUCCESSFUL;
 
@@ -560,6 +568,8 @@ NTSTATUS se_create_child_secdesc(TALLOC_CTX *ctx,
 	struct security_ace *new_ace_list = NULL;
 	unsigned int new_ace_list_ndx = 0, i;
 
+	TALLOC_CTX *frame;
+
 	*ppsd = NULL;
 	*psize = 0;
 
@@ -574,20 +584,22 @@ NTSTATUS se_create_child_secdesc(TALLOC_CTX *ctx,
 			return NT_STATUS_NO_MEMORY;
 		}
 
-		if (!(new_ace_list = TALLOC_ARRAY(ctx, struct security_ace,
-						2*the_acl->num_aces))) {
+		if (!(new_ace_list = talloc_array(ctx, struct security_ace,
+						  2*the_acl->num_aces))) {
 			return NT_STATUS_NO_MEMORY;
 		}
 	} else {
 		new_ace_list = NULL;
 	}
 
+	frame = talloc_stackframe();
+
 	for (i = 0; i < the_acl->num_aces; i++) {
 		const struct security_ace *ace = &the_acl->aces[i];
 		struct security_ace *new_ace = &new_ace_list[new_ace_list_ndx];
 		const struct dom_sid *ptrustee = &ace->trustee;
 		const struct dom_sid *creator = NULL;
-		uint8 new_flags = ace->flags;
+		uint8_t new_flags = ace->flags;
 
 		if (!is_inheritable_ace(ace, container)) {
 			continue;
@@ -621,15 +633,15 @@ NTSTATUS se_create_child_secdesc(TALLOC_CTX *ctx,
 
 			/* First add the regular ACE entry. */
 			init_sec_ace(new_ace, ptrustee, ace->type,
-			     	ace->access_mask, 0);
+				ace->access_mask, 0);
 
 			DEBUG(5,("se_create_child_secdesc(): %s:%d/0x%02x/0x%08x"
-				" inherited as %s:%d/0x%02x/0x%08x\n",
-				sid_string_dbg(&ace->trustee),
-				ace->type, ace->flags, ace->access_mask,
-				sid_string_dbg(&new_ace->trustee),
-				new_ace->type, new_ace->flags,
-				new_ace->access_mask));
+				 " inherited as %s:%d/0x%02x/0x%08x\n",
+				 dom_sid_string(frame, &ace->trustee),
+				 ace->type, ace->flags, ace->access_mask,
+				 dom_sid_string(frame, &new_ace->trustee),
+				 new_ace->type, new_ace->flags,
+				 new_ace->access_mask));
 
 			new_ace_list_ndx++;
 
@@ -638,6 +650,7 @@ NTSTATUS se_create_child_secdesc(TALLOC_CTX *ctx,
 
 			ptrustee = creator;
 			new_flags |= SEC_ACE_FLAG_INHERIT_ONLY;
+
 		} else if (container &&
 				!(ace->flags & SEC_ACE_FLAG_NO_PROPAGATE_INHERIT)) {
 			ptrustee = &ace->trustee;
@@ -648,15 +661,17 @@ NTSTATUS se_create_child_secdesc(TALLOC_CTX *ctx,
 
 		DEBUG(5, ("se_create_child_secdesc(): %s:%d/0x%02x/0x%08x "
 			  " inherited as %s:%d/0x%02x/0x%08x\n",
-			  sid_string_dbg(&ace->trustee),
+			  dom_sid_string(frame, &ace->trustee),
 			  ace->type, ace->flags, ace->access_mask,
-			  sid_string_dbg(&ace->trustee),
+			  dom_sid_string(frame, &ace->trustee),
 			  new_ace->type, new_ace->flags,
 			  new_ace->access_mask));
 
 		new_ace_list_ndx++;
 	}
 
+	talloc_free(frame);
+
 	/* Create child security descriptor to return */
 	if (new_ace_list_ndx) {
 		new_dacl = make_sec_acl(ctx,
diff --git a/libcli/security/secdesc.h b/libcli/security/secdesc.h
new file mode 100644
index 00000000000..1e6d2e574d7
--- /dev/null
+++ b/libcli/security/secdesc.h
@@ -0,0 +1,124 @@
+/*
+ *  Unix SMB/Netbios implementation.
+ *  SEC_DESC handling functions
+ *  Copyright (C) Andrew Tridgell              1992-1998,
+ *  Copyright (C) Jeremy R. Allison            1995-2003.
+ *  Copyright (C) Luke Kenneth Casson Leighton 1996-1998,
+ *  Copyright (C) Paul Ashton                  1997-1998.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _SECDESC_H_
+#define _SECDESC_H_
+
+/* The following definitions come from libcli/security/secdesc.c  */
+#include "librpc/gen_ndr/security.h"
+
+/*******************************************************************
+ Given a security_descriptor return the sec_info.
+********************************************************************/
+uint32_t get_sec_info(const struct security_descriptor *sd);
+
+/*******************************************************************
+ Merge part of security descriptor old_sec in to the empty sections of
+ security descriptor new_sec.
+********************************************************************/
+struct sec_desc_buf *sec_desc_merge_buf(TALLOC_CTX *ctx, struct sec_desc_buf *new_sdb, struct sec_desc_buf *old_sdb);
+struct security_descriptor *sec_desc_merge(TALLOC_CTX *ctx, struct security_descriptor *new_sdb, struct security_descriptor *old_sdb);
+
+/*******************************************************************
+ Creates a struct security_descriptor structure
+********************************************************************/
+struct security_descriptor *make_sec_desc(TALLOC_CTX *ctx,
+			enum security_descriptor_revision revision,
+			uint16_t type,
+			const struct dom_sid *owner_sid, const struct dom_sid *grp_sid,
+			struct security_acl *sacl, struct security_acl *dacl, size_t *sd_size);
+
+/*******************************************************************
+ Duplicate a struct security_descriptor structure.
+********************************************************************/
+struct security_descriptor *dup_sec_desc(TALLOC_CTX *ctx, const struct security_descriptor *src);
+
+/*******************************************************************
+ Convert a secdesc into a byte stream
+********************************************************************/
+NTSTATUS marshall_sec_desc(TALLOC_CTX *mem_ctx,
+			   struct security_descriptor *secdesc,
+			   uint8_t **data, size_t *len);
+
+/*******************************************************************
+ Convert a secdesc_buf into a byte stream
+********************************************************************/
+NTSTATUS marshall_sec_desc_buf(TALLOC_CTX *mem_ctx,
+			       struct sec_desc_buf *secdesc_buf,
+			       uint8_t **data, size_t *len);
+
+/*******************************************************************
+ Parse a byte stream into a secdesc
+********************************************************************/
+NTSTATUS unmarshall_sec_desc(TALLOC_CTX *mem_ctx, uint8_t *data, size_t len,
+			     struct security_descriptor **psecdesc);
+
+/*******************************************************************
+ Parse a byte stream into a sec_desc_buf
+********************************************************************/
+NTSTATUS unmarshall_sec_desc_buf(TALLOC_CTX *mem_ctx, uint8_t *data, size_t len,
+				 struct sec_desc_buf **psecdesc_buf);
+
+/*******************************************************************
+ Creates a struct security_descriptor structure with typical defaults.
+********************************************************************/
+struct security_descriptor *make_standard_sec_desc(TALLOC_CTX *ctx, const struct dom_sid *owner_sid, const struct dom_sid *grp_sid,
+				 struct security_acl *dacl, size_t *sd_size);
+
+/*******************************************************************
+ Creates a struct sec_desc_buf structure.
+********************************************************************/
+struct sec_desc_buf *make_sec_desc_buf(TALLOC_CTX *ctx, size_t len, struct security_descriptor *sec_desc);
+
+/*******************************************************************
+ Duplicates a struct sec_desc_buf structure.
+********************************************************************/
+struct sec_desc_buf *dup_sec_desc_buf(TALLOC_CTX *ctx, struct sec_desc_buf *src);
+
+/*******************************************************************
+ Add a new SID with its permissions to struct security_descriptor.
+********************************************************************/
+NTSTATUS sec_desc_add_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, const struct dom_sid *sid, uint32_t mask, size_t *sd_size);
+
+/*******************************************************************
+ Modify a SID's permissions in a struct security_descriptor.
+********************************************************************/
+NTSTATUS sec_desc_mod_sid(struct security_descriptor *sd, struct dom_sid *sid, uint32_t mask);
+
+/*******************************************************************
+ Delete a SID from a struct security_descriptor.
+********************************************************************/
+NTSTATUS sec_desc_del_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, struct dom_sid *sid, size_t *sd_size);
+bool sd_has_inheritable_components(const struct security_descriptor *parent_ctr, bool container);
+NTSTATUS se_create_child_secdesc(TALLOC_CTX *ctx,
+					struct security_descriptor **ppsd,
+					size_t *psize,
+					const struct security_descriptor *parent_ctr,
+					const struct dom_sid *owner_sid,
+					const struct dom_sid *group_sid,
+					bool container);
+NTSTATUS se_create_child_secdesc_buf(TALLOC_CTX *ctx,
+					struct sec_desc_buf **ppsdb,
+					const struct security_descriptor *parent_ctr,
+					bool container);
+
+#endif /* _SECDESC_H_ */
diff --git a/libcli/security/security.h b/libcli/security/security.h
index 1a9f4fa3866..659d34157b0 100644
--- a/libcli/security/security.h
+++ b/libcli/security/security.h
@@ -100,6 +100,7 @@ struct object_tree {
 #include "libcli/security/dom_sid.h"
 #include "libcli/security/secace.h"
 #include "libcli/security/secacl.h"
+#include "libcli/security/secdesc.h"
 #include "libcli/security/security_descriptor.h"
 #include "libcli/security/security_token.h"
 #include "libcli/security/sddl.h"
diff --git a/libcli/security/wscript_build b/libcli/security/wscript_build
index 3f2e06a908b..940ee700f26 100644
--- a/libcli/security/wscript_build
+++ b/libcli/security/wscript_build
@@ -2,7 +2,7 @@
 
 
 bld.SAMBA_LIBRARY('security',
-                  source='dom_sid.c display_sec.c secace.c secacl.c security_descriptor.c sddl.c privileges.c security_token.c access_check.c object_tree.c create_descriptor.c util_sid.c session.c',
+                  source='dom_sid.c display_sec.c secace.c secacl.c security_descriptor.c sddl.c privileges.c security_token.c access_check.c object_tree.c create_descriptor.c util_sid.c session.c secdesc.c',
                   private_library=True,
                   deps='talloc ndr NDR_SECURITY'
                   )
diff --git a/libcli/smb/smb_common.h b/libcli/smb/smb_common.h
index 8fe0623727f..d8339cbb7f3 100644
--- a/libcli/smb/smb_common.h
+++ b/libcli/smb/smb_common.h
@@ -25,5 +25,7 @@
 #include "../libcli/smb/smb2_constants.h"
 #include "../libcli/smb/smb2_create_blob.h"
 #include "../libcli/smb/smb_constants.h"
+#include "../libcli/smb/smb_util.h"
+#include "../libcli/smb/smb_unix_ext.h"
 
 #endif
diff --git a/libcli/smb/smb_constants.h b/libcli/smb/smb_constants.h
index 83e6388286e..59a6d9e388b 100644
--- a/libcli/smb/smb_constants.h
+++ b/libcli/smb/smb_constants.h
@@ -35,5 +35,28 @@
 #define FLAGS2_UNICODE_STRINGS         0x8000
 #define FLAGS2_WIN2K_SIGNATURE         0xC852 /* Hack alert ! For now... JRA. */
 
+/* FileAttributes (search attributes) field */
+#define FILE_ATTRIBUTE_READONLY		0x0001L
+#define FILE_ATTRIBUTE_HIDDEN		0x0002L
+#define FILE_ATTRIBUTE_SYSTEM		0x0004L
+#define FILE_ATTRIBUTE_VOLUME		0x0008L
+#define FILE_ATTRIBUTE_DIRECTORY	0x0010L
+#define FILE_ATTRIBUTE_ARCHIVE		0x0020L
+#define FILE_ATTRIBUTE_DEVICE		0x0040L
+#define FILE_ATTRIBUTE_NORMAL		0x0080L
+#define FILE_ATTRIBUTE_TEMPORARY	0x0100L
+#define FILE_ATTRIBUTE_SPARSE		0x0200L
+#define FILE_ATTRIBUTE_REPARSE_POINT	0x0400L
+#define FILE_ATTRIBUTE_COMPRESSED	0x0800L
+#define FILE_ATTRIBUTE_OFFLINE		0x1000L
+#define FILE_ATTRIBUTE_NONINDEXED	0x2000L
+#define FILE_ATTRIBUTE_ENCRYPTED	0x4000L
+#define FILE_ATTRIBUTE_ALL_MASK 	0x7FFFL
+
+#define SAMBA_ATTRIBUTES_MASK		(FILE_ATTRIBUTE_READONLY|\
+					FILE_ATTRIBUTE_HIDDEN|\
+					FILE_ATTRIBUTE_SYSTEM|\
+					FILE_ATTRIBUTE_DIRECTORY|\
+					FILE_ATTRIBUTE_ARCHIVE)
 
 #endif /* _SMB_CONSTANTS_H */
diff --git a/libcli/smb/smb_unix_ext.h b/libcli/smb/smb_unix_ext.h
new file mode 100644
index 00000000000..9516d310ad3
--- /dev/null
+++ b/libcli/smb/smb_unix_ext.h
@@ -0,0 +1,452 @@
+/* 
+   Unix SMB/CIFS implementation.
+   SMB transaction2 handling
+
+   Copyright (C) James Peach 2007
+   Copyright (C) Jeremy Allison 1994-2002.
+
+   Extensively modified by Andrew Tridgell, 1995
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/* UNIX CIFS Extensions - created by HP */
+/*
+ * UNIX CIFS Extensions have the range 0x200 - 0x2FF reserved.
+ * Supposedly Microsoft have agreed to this.
+ */
+
+#define MIN_UNIX_INFO_LEVEL 0x200
+#define MAX_UNIX_INFO_LEVEL 0x2FF
+
+#define INFO_LEVEL_IS_UNIX(level) (((level) >= MIN_UNIX_INFO_LEVEL) && ((level) <= MAX_UNIX_INFO_LEVEL))
+
+#define SMB_QUERY_FILE_UNIX_BASIC      0x200   /* UNIX File Info*/
+#define SMB_SET_FILE_UNIX_BASIC        0x200
+#define SMB_SET_FILE_UNIX_INFO2        0x20B   /* UNIX File Info2 */
+
+#define SMB_MODE_NO_CHANGE                 0xFFFFFFFF     /* file mode value which */
+                                              /* means "don't change it" */
+#define SMB_UID_NO_CHANGE                  0xFFFFFFFF
+#define SMB_GID_NO_CHANGE                  0xFFFFFFFF
+
+#define SMB_SIZE_NO_CHANGE_LO              0xFFFFFFFF
+#define SMB_SIZE_NO_CHANGE_HI              0xFFFFFFFF
+ 
+#define SMB_TIME_NO_CHANGE_LO              0xFFFFFFFF
+#define SMB_TIME_NO_CHANGE_HI              0xFFFFFFFF
+
+/*
+Offset Size         Name
+0      LARGE_INTEGER EndOfFile                File size
+8      LARGE_INTEGER Blocks                   Number of bytes used on disk (st_blocks).
+16     LARGE_INTEGER CreationTime             Creation time
+24     LARGE_INTEGER LastAccessTime           Last access time
+32     LARGE_INTEGER LastModificationTime     Last modification time
+40     LARGE_INTEGER Uid                      Numeric user id for the owner
+48     LARGE_INTEGER Gid                      Numeric group id of owner
+56     ULONG Type                             Enumeration specifying the pathname type:
+                                              0 -- File
+                                              1 -- Directory
+                                              2 -- Symbolic link
+                                              3 -- Character device
+                                              4 -- Block device
+                                              5 -- FIFO (named pipe)
+                                              6 -- Unix domain socket
+
+60     LARGE_INTEGER devmajor                 Major device number if type is device
+68     LARGE_INTEGER devminor                 Minor device number if type is device
+76     LARGE_INTEGER uniqueid                 This is a server-assigned unique id for the file. The client
+                                              will typically map this onto an inode number. The scope of
+                                              uniqueness is the share.
+84     LARGE_INTEGER permissions              Standard UNIX file permissions  - see below.
+92     LARGE_INTEGER nlinks                   The number of directory entries that map to this entry
+                                              (number of hard links)
+
+100 - end.
+*/
+
+#define SMB_FILE_UNIX_BASIC_SIZE 100
+
+/* UNIX filetype mappings. */
+
+#define UNIX_TYPE_FILE 0
+#define UNIX_TYPE_DIR 1
+#define UNIX_TYPE_SYMLINK 2
+#define UNIX_TYPE_CHARDEV 3
+#define UNIX_TYPE_BLKDEV 4
+#define UNIX_TYPE_FIFO 5
+#define UNIX_TYPE_SOCKET 6
+#define UNIX_TYPE_UNKNOWN 0xFFFFFFFF
+
+/*
+ * Oh this is fun. "Standard UNIX permissions" has no
+ * meaning in POSIX. We need to define the mapping onto
+ * and off the wire as this was not done in the original HP
+ * spec. JRA.
+ */
+
+#define UNIX_X_OTH			0000001
+#define UNIX_W_OTH			0000002
+#define UNIX_R_OTH			0000004
+#define UNIX_X_GRP			0000010
+#define UNIX_W_GRP                      0000020
+#define UNIX_R_GRP                      0000040
+#define UNIX_X_USR                      0000100
+#define UNIX_W_USR                      0000200
+#define UNIX_R_USR                      0000400
+#define UNIX_STICKY                     0001000
+#define UNIX_SET_GID                    0002000
+#define UNIX_SET_UID                    0004000
+
+/* Masks for the above */
+#define UNIX_OTH_MASK                   0000007
+#define UNIX_GRP_MASK                   0000070
+#define UNIX_USR_MASK                   0000700
+#define UNIX_PERM_MASK                  0000777
+#define UNIX_EXTRA_MASK                 0007000
+#define UNIX_ALL_MASK                   0007777
+
+/* Flags for chflags (CIFS_UNIX_EXTATTR_CAP capability) and
+ * SMB_QUERY_FILE_UNIX_INFO2.
+ */
+#define EXT_SECURE_DELETE               0x00000001
+#define EXT_ENABLE_UNDELETE             0x00000002
+#define EXT_SYNCHRONOUS                 0x00000004
+#define EXT_IMMUTABLE			0x00000008
+#define EXT_OPEN_APPEND_ONLY            0x00000010
+#define EXT_DO_NOT_BACKUP               0x00000020
+#define EXT_NO_UPDATE_ATIME             0x00000040
+#define EXT_HIDDEN			0x00000080
+
+#define SMB_QUERY_FILE_UNIX_LINK       0x201
+#define SMB_SET_FILE_UNIX_LINK         0x201
+#define SMB_SET_FILE_UNIX_HLINK        0x203
+/* SMB_QUERY_POSIX_ACL 0x204 see below */
+#define SMB_QUERY_XATTR                0x205 /* need for non-user XATTRs */
+#define SMB_QUERY_ATTR_FLAGS           0x206 /* chflags, chattr */
+#define SMB_SET_ATTR_FLAGS             0x206 
+#define SMB_QUERY_POSIX_PERMISSION     0x207
+/* Only valid for qfileinfo */
+#define SMB_QUERY_POSIX_LOCK	       0x208
+/* Only valid for setfileinfo */
+#define SMB_SET_POSIX_LOCK	       0x208
+
+/* The set info levels for POSIX path operations. */
+#define SMB_POSIX_PATH_OPEN	       0x209
+#define SMB_POSIX_PATH_UNLINK	       0x20A
+
+#define SMB_QUERY_FILE_UNIX_INFO2      0x20B   /* UNIX File Info2 */
+#define SMB_SET_FILE_UNIX_INFO2        0x20B
+
+/*
+SMB_QUERY_FILE_UNIX_INFO2 is SMB_QUERY_FILE_UNIX_BASIC with create
+time and file flags appended. The corresponding info level for
+findfirst/findnext is SMB_FIND_FILE_UNIX_INFO2.
+    Size    Offset  Value
+    ---------------------
+    0      LARGE_INTEGER EndOfFile  File size
+    8      LARGE_INTEGER Blocks     Number of blocks used on disk
+    16     LARGE_INTEGER ChangeTime Attribute change time
+    24     LARGE_INTEGER LastAccessTime           Last access time
+    32     LARGE_INTEGER LastModificationTime     Last modification time
+    40     LARGE_INTEGER Uid        Numeric user id for the owner
+    48     LARGE_INTEGER Gid        Numeric group id of owner
+    56     ULONG Type               Enumeration specifying the file type
+    60     LARGE_INTEGER devmajor   Major device number if type is device
+    68     LARGE_INTEGER devminor   Minor device number if type is device
+    76     LARGE_INTEGER uniqueid   This is a server-assigned unique id
+    84     LARGE_INTEGER permissions		Standard UNIX permissions
+    92     LARGE_INTEGER nlinks			Number of hard links
+    100    LARGE_INTEGER CreationTime		Create/birth time
+    108    ULONG FileFlags          File flags enumeration
+    112    ULONG FileFlagsMask      Mask of valid flags
+*/
+
+/* Transact 2 Find First levels */
+#define SMB_FIND_FILE_UNIX             0x202
+#define SMB_FIND_FILE_UNIX_INFO2       0x20B /* UNIX File Info2 */
+
+#define SMB_FILE_UNIX_INFO2_SIZE 116
+
+/*
+ Info level for TRANS2_QFSINFO - returns version of CIFS UNIX extensions, plus
+ 64-bits worth of capability fun :-).
+ Use the same info level for TRANS2_SETFSINFO
+*/
+
+#define SMB_QUERY_CIFS_UNIX_INFO      0x200
+#define SMB_SET_CIFS_UNIX_INFO        0x200
+
+/* Returns or sets the following.
+
+  UINT16             major version number
+  UINT16             minor version number
+  LARGE_INTEGER      capability bitfield
+
+*/
+
+#define CIFS_UNIX_MAJOR_VERSION 1
+#define CIFS_UNIX_MINOR_VERSION 0
+
+#define CIFS_UNIX_FCNTL_LOCKS_CAP           0x1
+#define CIFS_UNIX_POSIX_ACLS_CAP            0x2
+#define CIFS_UNIX_XATTTR_CAP	            0x4 /* for support of other xattr
+						namespaces such as system,
+						security and trusted */
+#define CIFS_UNIX_EXTATTR_CAP		    0x8 /* for support of chattr
+						(chflags) and lsattr */
+#define CIFS_UNIX_POSIX_PATHNAMES_CAP	   0x10 /* Use POSIX pathnames on the wire. */
+#define CIFS_UNIX_POSIX_PATH_OPERATIONS_CAP	   0x20 /* We can cope with POSIX open/mkdir/unlink etc. */
+#define CIFS_UNIX_LARGE_READ_CAP           0x40 /* We can cope with 24 bit reads in readX. */
+#define CIFS_UNIX_LARGE_WRITE_CAP          0x80 /* We can cope with 24 bit writes in writeX. */
+#define CIFS_UNIX_TRANSPORT_ENCRYPTION_CAP      0x100 /* We can do SPNEGO negotiations for encryption. */
+#define CIFS_UNIX_TRANSPORT_ENCRYPTION_MANDATORY_CAP    0x200 /* We *must* SPNEGO negotiations for encryption. */
+
+#define SMB_QUERY_POSIX_FS_INFO     0x201
+
+/* Returns FILE_SYSTEM_POSIX_INFO struct as follows
+      (NB   For undefined values return -1 in that field) 
+   le32 OptimalTransferSize;    bsize on some os, iosize on other os, This 
+				is a hint to the client about best size. Server
+				can return -1 if no preference, ie if SMB 
+				negotiated size is adequate for optimal
+				read/write performance
+   le32 BlockSize; (often 512 bytes) NB: BlockSize * TotalBlocks = disk space
+   le64 TotalBlocks;  redundant with other infolevels but easy to ret here
+   le64 BlocksAvail;  although redundant, easy to return
+   le64 UserBlocksAvail;      bavail 
+   le64 TotalFileNodes;
+   le64 FreeFileNodes;
+   le64 FileSysIdentifier;    fsid 
+   (NB statfs field Namelen comes from FILE_SYSTEM_ATTRIBUTE_INFO call) 
+   (NB statfs field flags can come from FILE_SYSTEM_DEVICE_INFO call)  
+*/
+
+#define SMB_QUERY_POSIX_WHO_AM_I  0x202 /* QFS Info */
+/* returns:
+        __u32 flags;  0 = Authenticated user 1 = GUEST 
+        __u32 mask;  which flags bits server understands ie 0x0001 
+        __u64 unix_user_id;
+        __u64 unix_user_gid;
+        __u32 number_of_supplementary_gids;  may be zero 
+        __u32 number_of_sids;  may be zero
+        __u32 length_of_sid_array;  in bytes - may be zero 
+        __u32 pad;  reserved - MBZ 
+        __u64 gid_array[0];  may be empty 
+        __u8 * psid_list  may be empty
+*/
+
+/* ... more as we think of them :-). */
+
+/* SMB POSIX ACL definitions. */
+/* Wire format is (all little endian) :
+
+[2 bytes]              -     Version number.
+[2 bytes]              -     Number of ACE entries to follow.
+[2 bytes]              -     Number of default ACE entries to follow.
+-------------------------------------
+^
+|
+ACE entries
+|
+v
+-------------------------------------
+^
+|
+Default ACE entries
+|
+v
+-------------------------------------
+
+Where an ACE entry looks like :
+
+[1 byte]           - Entry type.
+
+Entry types are :
+
+ACL_USER_OBJ            0x01
+ACL_USER                0x02
+ACL_GROUP_OBJ           0x04
+ACL_GROUP               0x08
+ACL_MASK                0x10
+ACL_OTHER               0x20
+
+[1 byte]          - permissions (perm_t)
+
+perm_t types are :
+
+ACL_READ                0x04
+ACL_WRITE               0x02
+ACL_EXECUTE             0x01
+
+[8 bytes]         - uid/gid to apply this permission to.
+
+In the same format as the uid/gid fields in the other
+UNIX extensions definitions. Use 0xFFFFFFFFFFFFFFFF for
+the MASK and OTHER entry types.
+
+If the Number of ACE entries for either file or default ACE's
+is set to 0xFFFF this means ignore this kind of ACE (and the
+number of entries sent will be zero.
+
+*/
+
+#define SMB_QUERY_POSIX_WHOAMI     0x202
+
+enum smb_whoami_flags {
+    SMB_WHOAMI_GUEST = 0x1 /* Logged in as (or squashed to) guest */
+};
+
+/* Mask of which WHOAMI bits are valid. This should make it easier for clients
+ * to cope with servers that have different sets of WHOAMI flags (as more get
+ * added).
+ */
+#define SMB_WHOAMI_MASK 0x00000001
+
+/*
+   SMBWhoami - Query the user mapping performed by the server for the
+   connected tree. This is a subcommand of the TRANS2_QFSINFO.
+
+   Returns:
+	4 bytes unsigned -	mapping flags (smb_whoami_flags)
+	4 bytes unsigned -	flags mask
+
+	8 bytes unsigned -	primary UID
+	8 bytes unsigned -	primary GID
+	4 bytes unsigned -	number of supplementary GIDs
+	4 bytes unsigned -	number of SIDs
+	4 bytes unsigned -	SID list byte count
+	4 bytes -		pad / reserved (must be zero)
+
+	8 bytes unsigned[] -	list of GIDs (may be empty)
+	struct dom_sid[] -		list of SIDs (may be empty)
+*/
+
+/*
+ * The following trans2 is done between client and server
+ * as a FSINFO call to set up the encryption state for transport
+ * encryption.
+ * This is a subcommand of the TRANS2_QFSINFO.
+ *
+ * The request looks like :
+ *
+ * [data block] -> SPNEGO framed GSSAPI request.
+ *
+ * The reply looks like :
+ *
+ * [data block] -> SPNEGO framed GSSAPI reply - if error
+ *                 is NT_STATUS_OK then we're done, if it's
+ *                 NT_STATUS_MORE_PROCESSING_REQUIRED then the
+ *                 client needs to keep going. If it's an
+ *                 error it can be any NT_STATUS error.
+ *
+ */
+
+#define SMB_REQUEST_TRANSPORT_ENCRYPTION     0x203 /* QFSINFO */
+
+
+/* The query/set info levels for POSIX ACLs. */
+#define SMB_QUERY_POSIX_ACL  0x204
+#define SMB_SET_POSIX_ACL  0x204
+
+/* Current on the wire ACL version. */
+#define SMB_POSIX_ACL_VERSION 1
+
+/* ACE entry type. */
+#define SMB_POSIX_ACL_USER_OBJ            0x01
+#define SMB_POSIX_ACL_USER                0x02
+#define SMB_POSIX_ACL_GROUP_OBJ           0x04
+#define SMB_POSIX_ACL_GROUP               0x08
+#define SMB_POSIX_ACL_MASK                0x10
+#define SMB_POSIX_ACL_OTHER               0x20
+
+/* perm_t types. */
+#define SMB_POSIX_ACL_READ                0x04
+#define SMB_POSIX_ACL_WRITE               0x02
+#define SMB_POSIX_ACL_EXECUTE             0x01
+
+#define SMB_POSIX_ACL_HEADER_SIZE         6
+#define SMB_POSIX_ACL_ENTRY_SIZE         10
+
+#define SMB_POSIX_IGNORE_ACE_ENTRIES	0xFFFF
+
+/* Definition of data block of SMB_SET_POSIX_LOCK */
+/*
+  [2 bytes] lock_type - 0 = Read, 1 = Write, 2 = Unlock
+  [2 bytes] lock_flags - 1 = Wait (only valid for setlock)
+  [4 bytes] pid = locking context.
+  [8 bytes] start = unsigned 64 bits.
+  [8 bytes] length = unsigned 64 bits.
+*/
+
+#define POSIX_LOCK_TYPE_OFFSET 0
+#define POSIX_LOCK_FLAGS_OFFSET 2
+#define POSIX_LOCK_PID_OFFSET 4
+#define POSIX_LOCK_START_OFFSET 8
+#define POSIX_LOCK_LEN_OFFSET 16
+#define POSIX_LOCK_DATA_SIZE 24
+
+#define POSIX_LOCK_FLAG_NOWAIT 0
+#define POSIX_LOCK_FLAG_WAIT 1
+
+#define POSIX_LOCK_TYPE_READ 0
+#define POSIX_LOCK_TYPE_WRITE 1
+#define POSIX_LOCK_TYPE_UNLOCK 2
+
+/* SMB_POSIX_PATH_OPEN "open_mode" definitions. */
+#define SMB_O_RDONLY			  0x1
+#define SMB_O_WRONLY			  0x2
+#define SMB_O_RDWR			  0x4
+
+#define SMB_ACCMODE			  0x7
+
+#define SMB_O_CREAT			 0x10
+#define SMB_O_EXCL			 0x20
+#define SMB_O_TRUNC			 0x40
+#define SMB_O_APPEND			 0x80
+#define SMB_O_SYNC			0x100
+#define SMB_O_DIRECTORY			0x200
+#define SMB_O_NOFOLLOW			0x400
+#define SMB_O_DIRECT			0x800
+
+/* Definition of request data block for SMB_POSIX_PATH_OPEN */
+/*
+  [4 bytes] flags (as smb_ntcreate_Flags).
+  [4 bytes] open_mode			- SMB_O_xxx flags above.
+  [8 bytes] mode_t (permissions)	- same encoding as "Standard UNIX permissions" above in SMB_SET_FILE_UNIX_BASIC.
+  [2 bytes] ret_info_level	- optimization. Info level to be returned.
+*/
+
+/* Definition of reply data block for SMB_POSIX_PATH_OPEN */
+
+#define SMB_NO_INFO_LEVEL_RETURNED 0xFFFF
+
+/*
+  [2 bytes] - flags field. Identical to flags reply for oplock response field in SMBNTCreateX)
+  [2 bytes] - FID returned.
+  [4 bytes] - CreateAction (same as in NTCreateX response).
+  [2 bytes] - reply info level    - as requested or 0xFFFF if not available.
+  [2 bytes] - padding (must be zero)
+  [n bytes] - info level reply  - if available.
+*/
+
+/* Definition of request data block for SMB_POSIX_UNLINK */
+/*
+  [2 bytes] flags (defined below).
+*/
+
+#define SMB_POSIX_UNLINK_FILE_TARGET 0
+#define SMB_POSIX_UNLINK_DIRECTORY_TARGET 1
+
diff --git a/libcli/smb/smb_util.h b/libcli/smb/smb_util.h
new file mode 100644
index 00000000000..b5a06524cb2
--- /dev/null
+++ b/libcli/smb/smb_util.h
@@ -0,0 +1,25 @@
+/*
+   Unix SMB/CIFS implementation.
+   client file operations
+   Copyright (C) Andrew Tridgell 1994-1998
+   Copyright (C) Jeremy Allison 2001-2002
+   Copyright (C) James Myers 2003
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+char *attrib_string(TALLOC_CTX *mem_ctx, uint32_t attrib);
+uint32_t unix_perms_to_wire(mode_t perms);
+mode_t wire_perms_to_unix(uint32_t perms);
+mode_t unix_filetype_from_wire(uint32_t wire_type);
diff --git a/libcli/smb/util.c b/libcli/smb/util.c
new file mode 100644
index 00000000000..b8c3dc5e968
--- /dev/null
+++ b/libcli/smb/util.c
@@ -0,0 +1,165 @@
+/*
+   Unix SMB/CIFS implementation.
+   client file operations
+   Copyright (C) Andrew Tridgell 1994-1998
+   Copyright (C) Jeremy Allison 2001-2002
+   Copyright (C) James Myers 2003
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "libcli/smb/smb_common.h"
+#include "system/filesys.h"
+
+/**
+ Return a string representing a CIFS attribute for a file.
+**/
+char *attrib_string(TALLOC_CTX *mem_ctx, uint32_t attrib)
+{
+	int i, len;
+	const struct {
+		char c;
+		uint16_t attr;
+	} attr_strs[] = {
+		{'V', FILE_ATTRIBUTE_VOLUME},
+		{'D', FILE_ATTRIBUTE_DIRECTORY},
+		{'A', FILE_ATTRIBUTE_ARCHIVE},
+		{'H', FILE_ATTRIBUTE_HIDDEN},
+		{'S', FILE_ATTRIBUTE_SYSTEM},
+		{'N', FILE_ATTRIBUTE_NORMAL},
+		{'R', FILE_ATTRIBUTE_READONLY},
+		{'d', FILE_ATTRIBUTE_DEVICE},
+		{'t', FILE_ATTRIBUTE_TEMPORARY},
+		{'s', FILE_ATTRIBUTE_SPARSE},
+		{'r', FILE_ATTRIBUTE_REPARSE_POINT},
+		{'c', FILE_ATTRIBUTE_COMPRESSED},
+		{'o', FILE_ATTRIBUTE_OFFLINE},
+		{'n', FILE_ATTRIBUTE_NONINDEXED},
+		{'e', FILE_ATTRIBUTE_ENCRYPTED}
+	};
+	char *ret;
+
+	ret = talloc_array(mem_ctx, char, ARRAY_SIZE(attr_strs)+1);
+	if (!ret) {
+		return NULL;
+	}
+
+	for (len=i=0; i<ARRAY_SIZE(attr_strs); i++) {
+		if (attrib & attr_strs[i].attr) {
+			ret[len++] = attr_strs[i].c;
+		}
+	}
+
+	ret[len] = 0;
+
+	talloc_set_name_const(ret, ret);
+
+	return ret;
+}
+
+/****************************************************************************
+ Map standard UNIX permissions onto wire representations.
+****************************************************************************/
+
+uint32_t unix_perms_to_wire(mode_t perms)
+{
+        unsigned int ret = 0;
+
+        ret |= ((perms & S_IXOTH) ?  UNIX_X_OTH : 0);
+        ret |= ((perms & S_IWOTH) ?  UNIX_W_OTH : 0);
+        ret |= ((perms & S_IROTH) ?  UNIX_R_OTH : 0);
+        ret |= ((perms & S_IXGRP) ?  UNIX_X_GRP : 0);
+        ret |= ((perms & S_IWGRP) ?  UNIX_W_GRP : 0);
+        ret |= ((perms & S_IRGRP) ?  UNIX_R_GRP : 0);
+        ret |= ((perms & S_IXUSR) ?  UNIX_X_USR : 0);
+        ret |= ((perms & S_IWUSR) ?  UNIX_W_USR : 0);
+        ret |= ((perms & S_IRUSR) ?  UNIX_R_USR : 0);
+#ifdef S_ISVTX
+        ret |= ((perms & S_ISVTX) ?  UNIX_STICKY : 0);
+#endif
+#ifdef S_ISGID
+        ret |= ((perms & S_ISGID) ?  UNIX_SET_GID : 0);
+#endif
+#ifdef S_ISUID
+        ret |= ((perms & S_ISUID) ?  UNIX_SET_UID : 0);
+#endif
+        return ret;
+}
+
+/****************************************************************************
+ Map wire permissions to standard UNIX.
+****************************************************************************/
+
+mode_t wire_perms_to_unix(uint32_t perms)
+{
+        mode_t ret = (mode_t)0;
+
+        ret |= ((perms & UNIX_X_OTH) ? S_IXOTH : 0);
+        ret |= ((perms & UNIX_W_OTH) ? S_IWOTH : 0);
+        ret |= ((perms & UNIX_R_OTH) ? S_IROTH : 0);
+        ret |= ((perms & UNIX_X_GRP) ? S_IXGRP : 0);
+        ret |= ((perms & UNIX_W_GRP) ? S_IWGRP : 0);
+        ret |= ((perms & UNIX_R_GRP) ? S_IRGRP : 0);
+        ret |= ((perms & UNIX_X_USR) ? S_IXUSR : 0);
+        ret |= ((perms & UNIX_W_USR) ? S_IWUSR : 0);
+        ret |= ((perms & UNIX_R_USR) ? S_IRUSR : 0);
+#ifdef S_ISVTX
+        ret |= ((perms & UNIX_STICKY) ? S_ISVTX : 0);
+#endif
+#ifdef S_ISGID
+        ret |= ((perms & UNIX_SET_GID) ? S_ISGID : 0);
+#endif
+#ifdef S_ISUID
+        ret |= ((perms & UNIX_SET_UID) ? S_ISUID : 0);
+#endif
+        return ret;
+}
+
+/****************************************************************************
+ Return the file type from the wire filetype for UNIX extensions.
+****************************************************************************/
+
+mode_t unix_filetype_from_wire(uint32_t wire_type)
+{
+	switch (wire_type) {
+		case UNIX_TYPE_FILE:
+			return S_IFREG;
+		case UNIX_TYPE_DIR:
+			return S_IFDIR;
+#ifdef S_IFLNK
+		case UNIX_TYPE_SYMLINK:
+			return S_IFLNK;
+#endif
+#ifdef S_IFCHR
+		case UNIX_TYPE_CHARDEV:
+			return S_IFCHR;
+#endif
+#ifdef S_IFBLK
+		case UNIX_TYPE_BLKDEV:
+			return S_IFBLK;
+#endif
+#ifdef S_IFIFO
+		case UNIX_TYPE_FIFO:
+			return S_IFIFO;
+#endif
+#ifdef S_IFSOCK
+		case UNIX_TYPE_SOCKET:
+			return S_IFSOCK;
+#endif
+		default:
+			return (mode_t)0;
+	}
+}
+
diff --git a/libcli/smb/wscript_build b/libcli/smb/wscript_build
index 6796b4c7fb7..0d74e261c28 100644
--- a/libcli/smb/wscript_build
+++ b/libcli/smb/wscript_build
@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 
 
-bld.SAMBA_SUBSYSTEM('LIBCLI_SMB_COMMON',
-	source='smb2_create_blob.c',
+bld.SAMBA_LIBRARY('LIBCLI_SMB_COMMON',
+	source='smb2_create_blob.c util.c',
 	autoproto='smb_common_proto.h',
-	public_deps='talloc'
+	public_deps='talloc samba-util',
+                  private_library=True
 	)
-
diff --git a/libcli/smbreadline/smbreadline.h b/libcli/smbreadline/smbreadline.h
index 102106f2626..9adc4b3a046 100644
--- a/libcli/smbreadline/smbreadline.h
+++ b/libcli/smbreadline/smbreadline.h
@@ -1,3 +1,23 @@
+/*
+   Unix SMB/CIFS implementation.
+   Samba readline wrapper implementation
+   Copyright (C) Simo Sorce 2001
+   Copyright (C) Andrew Tridgell 2001
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef __SMBREADLINE_H__
 #define __SMBREADLINE_H__
 
diff --git a/libcli/util/errmap_unix.c b/libcli/util/errmap_unix.c
new file mode 100644
index 00000000000..8f9f89835ed
--- /dev/null
+++ b/libcli/util/errmap_unix.c
@@ -0,0 +1,157 @@
+/* 
+ *  Unix SMB/CIFS implementation.
+ *  error mapping functions
+ *  Copyright (C) Andrew Tridgell 2001
+ *  Copyright (C) Andrew Bartlett 2001
+ *  Copyright (C) Tim Potter 2000
+ *  
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "includes.h"
+
+/* Mapping between Unix, and NT error numbers */
+
+static const struct {
+	int unix_error;
+	NTSTATUS nt_error;
+} unix_nt_errmap[] = {
+	{ EAGAIN,       STATUS_MORE_ENTRIES },
+	{ EINTR,        STATUS_MORE_ENTRIES },
+	{ ENOBUFS,      STATUS_MORE_ENTRIES },
+#ifdef EWOULDBLOCK
+	{ EWOULDBLOCK,  STATUS_MORE_ENTRIES },
+#endif
+	{ EINPROGRESS,  NT_STATUS_MORE_PROCESSING_REQUIRED },
+	{ EPERM,        NT_STATUS_ACCESS_DENIED },
+	{ EACCES,       NT_STATUS_ACCESS_DENIED },
+	{ ENOENT,       NT_STATUS_OBJECT_NAME_NOT_FOUND },
+	{ ENOTDIR,      NT_STATUS_NOT_A_DIRECTORY },
+	{ EIO,          NT_STATUS_IO_DEVICE_ERROR },
+	{ EBADF,        NT_STATUS_INVALID_HANDLE },
+	{ EINVAL,       NT_STATUS_INVALID_PARAMETER },
+	{ EEXIST,       NT_STATUS_OBJECT_NAME_COLLISION},
+	{ ENFILE,       NT_STATUS_TOO_MANY_OPENED_FILES },
+	{ EMFILE,       NT_STATUS_TOO_MANY_OPENED_FILES },
+	{ ENOSPC,       NT_STATUS_DISK_FULL },
+	{ ENOTSOCK,     NT_STATUS_INVALID_HANDLE },
+	{ EFAULT,       NT_STATUS_INVALID_PARAMETER },
+	{ EMSGSIZE,     NT_STATUS_INVALID_BUFFER_SIZE },
+	{ ENOMEM,       NT_STATUS_NO_MEMORY },
+	{ EISDIR,       NT_STATUS_FILE_IS_A_DIRECTORY},
+#ifdef EPIPE
+	{ EPIPE,        NT_STATUS_CONNECTION_DISCONNECTED },
+#endif
+	{ EBUSY,        NT_STATUS_SHARING_VIOLATION },
+	{ ENOSYS,	NT_STATUS_INVALID_SYSTEM_SERVICE },
+#ifdef EOPNOTSUPP
+	{ EOPNOTSUPP,   NT_STATUS_NOT_SUPPORTED},
+#endif
+	{ EMLINK,       NT_STATUS_TOO_MANY_LINKS },
+	{ ENOSYS,       NT_STATUS_NOT_SUPPORTED },
+#ifdef ELOOP
+	{ ELOOP,        NT_STATUS_OBJECT_PATH_NOT_FOUND },
+#endif
+#ifdef ENODATA
+	{ ENODATA,      NT_STATUS_NOT_FOUND },
+#endif
+#ifdef EFTYPE
+	{ EFTYPE,       NT_STATUS_OBJECT_PATH_NOT_FOUND },
+#endif
+#ifdef EDQUOT
+	{ EDQUOT,       NT_STATUS_DISK_FULL }, /* Windows apps need this, not NT_STATUS_QUOTA_EXCEEDED */
+#endif
+#ifdef ENOTEMPTY
+	{ ENOTEMPTY,    NT_STATUS_DIRECTORY_NOT_EMPTY },
+#endif
+#ifdef EXDEV
+	{ EXDEV,        NT_STATUS_NOT_SAME_DEVICE },
+#endif
+#ifdef EROFS
+	{ EROFS,        NT_STATUS_MEDIA_WRITE_PROTECTED },
+#endif
+#ifdef ENAMETOOLONG
+	{ ENAMETOOLONG, NT_STATUS_NAME_TOO_LONG },
+#endif
+#ifdef EFBIG
+	{ EFBIG,        NT_STATUS_DISK_FULL },
+#endif
+#ifdef EADDRINUSE
+	{ EADDRINUSE,   NT_STATUS_ADDRESS_ALREADY_ASSOCIATED},
+#endif
+#ifdef ENETUNREACH
+	{ ENETUNREACH,  NT_STATUS_NETWORK_UNREACHABLE},
+#endif
+#ifdef EHOSTUNREACH
+        { EHOSTUNREACH, NT_STATUS_HOST_UNREACHABLE},
+#endif
+#ifdef ECONNREFUSED
+	{ ECONNREFUSED, NT_STATUS_CONNECTION_REFUSED},
+#endif
+#ifdef EADDRNOTAVAIL
+	{ EADDRNOTAVAIL,NT_STATUS_ADDRESS_NOT_ASSOCIATED },
+#endif
+#ifdef ETIMEDOUT
+	{ ETIMEDOUT,    NT_STATUS_IO_TIMEOUT},
+#endif
+#ifdef ESOCKTNOSUPPORT
+	{ ESOCKTNOSUPPORT,NT_STATUS_INVALID_PARAMETER_MIX },
+#endif
+#ifdef EAFNOSUPPORT
+	{ EAFNOSUPPORT,	NT_STATUS_INVALID_PARAMETER_MIX },
+#endif
+#ifdef ECONNABORTED
+	{ ECONNABORTED, NT_STATUS_CONNECTION_ABORTED},
+#endif
+#ifdef ECONNRESET
+	{ ECONNRESET,   NT_STATUS_CONNECTION_RESET},
+#endif
+#ifdef ENOPROTOOPT
+	{ ENOPROTOOPT,	NT_STATUS_INVALID_PARAMETER_MIX },
+#endif
+#ifdef ENODEV
+	{ ENODEV,	NT_STATUS_NO_SUCH_DEVICE },
+#endif
+#ifdef ENOATTR
+	{ ENOATTR,      NT_STATUS_NOT_FOUND },
+#endif
+#ifdef ECANCELED
+	{ ECANCELED,    NT_STATUS_CANCELLED},
+#endif
+#ifdef ENOTSUP
+        { ENOTSUP,      NT_STATUS_NOT_SUPPORTED},
+#endif
+
+	{ 0, NT_STATUS_UNSUCCESSFUL }
+};
+
+
+/*********************************************************************
+ Map an NT error code from a Unix error code.
+*********************************************************************/
+NTSTATUS map_nt_error_from_unix_common(int unix_error)
+{
+	int i;
+
+	/* Look through list */
+	for (i=0;i<ARRAY_SIZE(unix_nt_errmap);i++) {
+		if (unix_nt_errmap[i].unix_error == unix_error) {
+			return unix_nt_errmap[i].nt_error;
+		}
+	}
+
+	/* Default return */
+	return NT_STATUS_UNSUCCESSFUL;
+}
+
diff --git a/libcli/util/error.h b/libcli/util/error.h
index 7b7de6c4f0f..5c4dc10916b 100644
--- a/libcli/util/error.h
+++ b/libcli/util/error.h
@@ -23,9 +23,6 @@
 #include "libcli/util/doserr.h"
 #include "libcli/util/ntstatus.h"
 
-/** NT error on DOS connection! (NT_STATUS_OK) */
-bool ntstatus_dos_equal(NTSTATUS status1, NTSTATUS status2);
-
 /*****************************************************************************
 convert a NT status code to a dos class/code
  *****************************************************************************/
@@ -44,12 +41,7 @@ WERROR ntstatus_to_werror(NTSTATUS error);
 /*********************************************************************
  Map an NT error code from a Unix error code.
 *********************************************************************/
-NTSTATUS map_nt_error_from_unix(int unix_error);
-
-/*********************************************************************
-convert a Unix error code to a WERROR
-*********************************************************************/
-WERROR unix_to_werror(int unix_error);
+NTSTATUS map_nt_error_from_unix_common(int unix_error);
 
 NTSTATUS nt_status_squash(NTSTATUS nt_status);
 
diff --git a/source4/libcli/util/errormap.c b/libcli/util/errormap.c
index a0154e370be..f96d83fa62a 100644
--- a/source4/libcli/util/errormap.c
+++ b/libcli/util/errormap.c
@@ -49,6 +49,10 @@ static const struct {
 	uint32_t dos_code;
 	NTSTATUS ntstatus;
 } ntstatus_to_dos_map[] = {
+/*
+ * Not an official error, as only bit 0x80000000, not bits 0xC0000000 are set.
+ */
+	{ERRDOS,	ERRmoredata,	STATUS_BUFFER_OVERFLOW},
 	{ERRDOS,	ERRnofiles,	STATUS_NO_MORE_FILES},
 	{ERRDOS,	ERRnofiles,	NT_STATUS_NO_MORE_ENTRIES},
 	{ERRDOS,	ERRgeneral,	NT_STATUS_UNSUCCESSFUL},
@@ -63,7 +67,7 @@ static const struct {
 	{ERRDOS,	193,	NT_STATUS_BAD_INITIAL_PC},
 	{ERRDOS,	87,	NT_STATUS_INVALID_CID},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_TIMER_NOT_CANCELED},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER},
+	{ERRDOS,	ERRinvalidparam,	NT_STATUS_INVALID_PARAMETER},
 	{ERRDOS,	ERRbadfile,	NT_STATUS_NO_SUCH_DEVICE},
 	{ERRDOS,	ERRbadfile,	NT_STATUS_NO_SUCH_FILE},
 	{ERRDOS,	ERRbadfunc,	NT_STATUS_INVALID_DEVICE_REQUEST},
@@ -116,13 +120,13 @@ static const struct {
 	{ERRHRD,	ERRgeneral,	NT_STATUS_DISK_CORRUPT_ERROR},
 	{ERRDOS,	ERRinvalidname,	NT_STATUS_OBJECT_NAME_INVALID},
 	{ERRDOS,	ERRbadfile,	NT_STATUS_OBJECT_NAME_NOT_FOUND},
-	{ERRDOS,	ERRfilexists,	NT_STATUS_OBJECT_NAME_COLLISION},
+	{ERRDOS,	183,	NT_STATUS_OBJECT_NAME_COLLISION},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_HANDLE_NOT_WAITABLE},
 	{ERRDOS,	ERRbadfid,	NT_STATUS_PORT_DISCONNECTED},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_DEVICE_ALREADY_ATTACHED},
-	{ERRDOS,	161,	NT_STATUS_OBJECT_PATH_INVALID},
+	{ERRDOS,	ERRinvalidpath,	NT_STATUS_OBJECT_PATH_INVALID},
 	{ERRDOS,	ERRbadpath,	NT_STATUS_OBJECT_PATH_NOT_FOUND},
-	{ERRDOS,	161,	NT_STATUS_OBJECT_PATH_SYNTAX_BAD},
+	{ERRDOS,	ERRinvalidpath,	NT_STATUS_OBJECT_PATH_SYNTAX_BAD},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_DATA_OVERRUN},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_DATA_LATE_ERROR},
 	{ERRDOS,	23,	NT_STATUS_DATA_ERROR},
@@ -142,7 +146,7 @@ static const struct {
 	{ERRDOS,	87,	NT_STATUS_BAD_WORKING_SET_LIMIT},
 	{ERRDOS,	87,	NT_STATUS_INCOMPATIBLE_FILE_MAP},
 	{ERRDOS,	87,	NT_STATUS_SECTION_PROTECTION},
-	{ERRDOS,	282,	NT_STATUS_EAS_NOT_SUPPORTED},
+	{ERRDOS,	ERReasnotsupported,	NT_STATUS_EAS_NOT_SUPPORTED},
 	{ERRDOS,	255,	NT_STATUS_EA_TOO_LARGE},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_NONEXISTENT_EA_ENTRY},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_EAS_ON_FILE},
@@ -335,7 +339,7 @@ static const struct {
 	{ERRDOS,	203,	NT_STATUS(0xc0000100)},
 	{ERRDOS,	145,	NT_STATUS_DIRECTORY_NOT_EMPTY},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_FILE_CORRUPT_ERROR},
-	{ERRDOS,	267,	NT_STATUS_NOT_A_DIRECTORY},
+	{ERRDOS,	ERRbaddirectory,	NT_STATUS_NOT_A_DIRECTORY},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_LOGON_SESSION_STATE},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_LOGON_SESSION_COLLISION},
 	{ERRDOS,	206,	NT_STATUS_NAME_TOO_LONG},
@@ -404,7 +408,7 @@ static const struct {
 	{ERRHRD,	ERRgeneral,	NT_STATUS_APP_INIT_FAILURE},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_PAGEFILE_CREATE_FAILED},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_PAGEFILE},
-	{ERRDOS,	124,	NT_STATUS_INVALID_LEVEL},
+	{ERRDOS,	ERRunknownlevel,	NT_STATUS_INVALID_LEVEL},
 	{ERRDOS,	86,	NT_STATUS_WRONG_PASSWORD_CORE},
 	{ERRHRD,	ERRgeneral,	NT_STATUS_ILLEGAL_FLOAT_CONTEXT},
 	{ERRDOS,	109,	NT_STATUS_PIPE_BROKEN},
@@ -611,17 +615,11 @@ static const struct {
 	{ERRDOS,	ERRbadfunc,	NT_STATUS(0xc000029c)},
 };
 
-
 /* errmap NTSTATUS->Win32 */
 static const struct {
 	NTSTATUS ntstatus;
 	WERROR werror;
 } ntstatus_to_werror_map[] = {
-	/*
-	 * we add this manualy here, so that W_ERROR(0x5)
-	 * gets mapped to NTSTATUS_ACCESS_DENIED
-	 */
-	{NT_STATUS_ACCESS_DENIED, WERR_ACCESS_DENIED},
 	{NT_STATUS(0x103), W_ERROR(0x3e5)},
 	{NT_STATUS(0x105), W_ERROR(0xea)},
 	{NT_STATUS(0x106), W_ERROR(0x514)},
@@ -1156,43 +1154,16 @@ static const struct {
 	{NT_STATUS(0x80000025), W_ERROR(0x962)},
 	{NT_STATUS(0x80000288), W_ERROR(0x48d)},
 	{NT_STATUS(0x80000289), W_ERROR(0x48e)},
-	{NT_STATUS_OK, WERR_OK}
-};
+	{NT_STATUS_OK, WERR_OK}};
 
-bool ntstatus_check_dos_mapping = true;
-
-/*
-  check if a DOS encoded NTSTATUS code maps to the given NTSTATUS code
-*/
-bool ntstatus_dos_equal(NTSTATUS status1, NTSTATUS status2)
-{
-	/* when we negotiate nt status support, we don't want to consider
-	   the mapping of dos codes, as we want to catch the cases where
-	   a forced dos code is needed
-	*/
-	if (ntstatus_check_dos_mapping) {
-		return NT_STATUS_V(status1) == NT_STATUS_V(status2);
-	}
+static const struct {
+	WERROR werror;
+	NTSTATUS ntstatus;
+} werror_to_ntstatus_map[] = {
+	{ W_ERROR(0x5), NT_STATUS_ACCESS_DENIED },
+	{ WERR_OK, NT_STATUS_OK }
+};
 
-	/* otherwise check if the mapping comes out right. Note that it is important
-	   that we do the mapping only from ntstatus -> dos and not from dos -> ntstatus,
-	   as that is the mapping that servers must do */
-	if (!NT_STATUS_IS_DOS(status1) && NT_STATUS_IS_DOS(status2)) {
-		uint8_t eclass;
-		uint32_t ecode;
-		ntstatus_to_dos(status1, &eclass, &ecode);
-		return eclass == NT_STATUS_DOS_CLASS(status2) &&
-			ecode == NT_STATUS_DOS_CODE(status2);
-	}
-	if (NT_STATUS_IS_DOS(status1) && !NT_STATUS_IS_DOS(status2)) {
-		uint8_t eclass;
-		uint32_t ecode;
-		ntstatus_to_dos(status2, &eclass, &ecode);
-		return eclass == NT_STATUS_DOS_CLASS(status1) &&
-			ecode == NT_STATUS_DOS_CODE(status1);
-	}
-	return NT_STATUS_V(status1) == NT_STATUS_V(status2);
-}
 
 /*****************************************************************************
 convert a NT status code to a dos class/code
@@ -1230,6 +1201,14 @@ NTSTATUS werror_to_ntstatus(WERROR error)
 {
 	int i;
 	if (W_ERROR_IS_OK(error)) return NT_STATUS_OK;
+
+	for (i=0; !W_ERROR_IS_OK(werror_to_ntstatus_map[i].werror); i++) {
+		if (W_ERROR_V(error) == 
+		    W_ERROR_V(werror_to_ntstatus_map[i].werror)) {
+			return werror_to_ntstatus_map[i].ntstatus;
+		}
+	}
+
 	for (i=0; NT_STATUS_V(ntstatus_to_werror_map[i].ntstatus); i++) {
 		if (W_ERROR_V(error) == 
 		    W_ERROR_V(ntstatus_to_werror_map[i].werror)) {
@@ -1259,131 +1238,3 @@ WERROR ntstatus_to_werror(NTSTATUS error)
 	return W_ERROR(NT_STATUS_V(error) & 0xffff);
 }
 
-/* Mapping between Unix, DOS and NT error numbers */
-
-struct unix_error_map {
-	int unix_error;
-	NTSTATUS nt_error;
-};
-
-const struct unix_error_map unix_nt_errmap[] = {
-	{ EAGAIN,       STATUS_MORE_ENTRIES },
-	{ EINTR,        STATUS_MORE_ENTRIES },
-	{ ENOBUFS,      STATUS_MORE_ENTRIES },
-#ifdef EWOULDBLOCK
-	{ EWOULDBLOCK,  STATUS_MORE_ENTRIES },
-#endif
-	{ EINPROGRESS,  NT_STATUS_MORE_PROCESSING_REQUIRED },
-	{ EPERM,        NT_STATUS_ACCESS_DENIED },
-	{ EACCES,       NT_STATUS_ACCESS_DENIED },
-	{ ENOENT,       NT_STATUS_OBJECT_NAME_NOT_FOUND },
-	{ ENOTDIR,      NT_STATUS_NOT_A_DIRECTORY },
-	{ EIO,          NT_STATUS_IO_DEVICE_ERROR },
-	{ EBADF,        NT_STATUS_INVALID_HANDLE },
-	{ EINVAL,       NT_STATUS_INVALID_PARAMETER },
-	{ EEXIST,       NT_STATUS_OBJECT_NAME_COLLISION},
-	{ ENFILE,       NT_STATUS_TOO_MANY_OPENED_FILES },
-	{ EMFILE,       NT_STATUS_TOO_MANY_OPENED_FILES },
-	{ ENOSPC,       NT_STATUS_DISK_FULL },
-	{ EISDIR,       NT_STATUS_FILE_IS_A_DIRECTORY },
-	{ ENOTSOCK,     NT_STATUS_INVALID_HANDLE },
-	{ EFAULT,       NT_STATUS_INVALID_PARAMETER },
-	{ EMSGSIZE,     NT_STATUS_INVALID_BUFFER_SIZE },
-	{ ENOMEM,       NT_STATUS_NO_MEMORY },
-	{ EPIPE,        NT_STATUS_CONNECTION_DISCONNECTED },
-	{ ECONNREFUSED, NT_STATUS_CONNECTION_REFUSED },
-#ifdef ECONNRESET
-	{ ECONNRESET,   NT_STATUS_CONNECTION_RESET },
-#endif
-	{ EBUSY,        NT_STATUS_SHARING_VIOLATION },
-#ifdef ENOTSUP
-	{ ENOTSUP,      NT_STATUS_NOT_SUPPORTED},
-#endif
-#ifdef EOPNOTSUPP
-	{ EOPNOTSUPP,   NT_STATUS_NOT_SUPPORTED},
-#endif
-#ifdef EHOSTUNREACH
-	{ EHOSTUNREACH, NT_STATUS_HOST_UNREACHABLE },
-#endif
-#ifdef ENETUNREACH
-	{ ENETUNREACH,  NT_STATUS_NETWORK_UNREACHABLE },
-#endif
-#ifdef ETIMEDOUT
-	{ ETIMEDOUT,    NT_STATUS_IO_TIMEOUT },
-#endif
-#ifdef EADDRINUSE
-	{ EADDRINUSE,   NT_STATUS_ADDRESS_ALREADY_ASSOCIATED },
-#endif
-#ifdef ENOATTR
-	{ ENOATTR,      NT_STATUS_NOT_FOUND },
-#endif
-#ifdef ENODATA
-	{ ENODATA,      NT_STATUS_NOT_FOUND },
-#endif
-#ifdef EDQUOT
-	{ EDQUOT,       NT_STATUS_DISK_FULL }, /* Windows does NOT return NT_STATUS_QUOTA_EXCEEDED */
-#endif
-#ifdef ENOTEMPTY
-	{ ENOTEMPTY,    NT_STATUS_DIRECTORY_NOT_EMPTY },
-#endif
-#ifdef EXDEV
-	{ EXDEV,        NT_STATUS_NOT_SAME_DEVICE },
-#endif
-#ifdef EROFS
-	{ EROFS,        NT_STATUS_MEDIA_WRITE_PROTECTED },
-#endif
-#ifdef ENAMETOOLONG
-	{ ENAMETOOLONG, NT_STATUS_NAME_TOO_LONG },
-#endif
-#ifdef EFBIG
-	{ EFBIG,        NT_STATUS_DISK_FULL },
-#endif
-#ifdef EADDRNOTAVAIL
-	{ EADDRNOTAVAIL,NT_STATUS_ADDRESS_NOT_ASSOCIATED },
-#endif
-#ifdef ESOCKTNOSUPPORT
-	{ ESOCKTNOSUPPORT,NT_STATUS_INVALID_PARAMETER_MIX },
-#endif
-#ifdef EAFNOSUPPORT
-	{ EAFNOSUPPORT,	NT_STATUS_INVALID_PARAMETER_MIX },
-#endif
-#ifdef ENOPROTOOPT
-	{ ENOPROTOOPT,	NT_STATUS_INVALID_PARAMETER_MIX },
-#endif
-#ifdef ENODEV
-	{ ENODEV,	NT_STATUS_NO_SUCH_DEVICE },
-#endif
-#ifdef ENOSYS
-	{ ENOSYS,	NT_STATUS_INVALID_SYSTEM_SERVICE },
-#endif
-#ifdef ECANCELED
-	{ ECANCELED, 	NT_STATUS_CANCELLED },
-#endif
-
-	{ 0, NT_STATUS_UNSUCCESSFUL }
-};
-
-
-/*********************************************************************
- Map an NT error code from a Unix error code.
-*********************************************************************/
-NTSTATUS map_nt_error_from_unix(int unix_error)
-{
-	int i;
-
-	/* Look through list */
-	for (i=0;i<ARRAY_SIZE(unix_nt_errmap);i++) {
-		if (unix_nt_errmap[i].unix_error == unix_error) {
-			return unix_nt_errmap[i].nt_error;
-		}
-	}
-
-	/* Default return */
-	return NT_STATUS_UNSUCCESSFUL;
-}
-
-/* Convert a Unix error code to WERROR */
-WERROR unix_to_werror(int unix_error)
-{
-	return ntstatus_to_werror(map_nt_error_from_unix(unix_error));
-}
diff --git a/source4/libcli/util/nterr.c b/libcli/util/nterr.c
index ca998bbf6f5..1158fddd769 100644
--- a/source4/libcli/util/nterr.c
+++ b/libcli/util/nterr.c
@@ -1,7 +1,10 @@
 /*
  *  Unix SMB/CIFS implementation.
  *  RPC Pipe client / server routines
+ *
  *  Copyright (C) Luke Kenneth Casson Leighton 1997-2001.
+ *  Copyright (C) Andrew Bartlett
+ *  Copyright (C) Andrew Tridgell
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -27,16 +30,10 @@
 #define N_(string) string
 #endif
 
-typedef struct
-{
-	const char *nt_errstr;
-	NTSTATUS nt_errcode;
-} nt_err_code_struct;
-
 #define DOS_CODE(class, code) { #class ":" #code, NT_STATUS_DOS(class, code) }
 #define LDAP_CODE(code) { #code, NT_STATUS_LDAP(code) }
 
-static const nt_err_code_struct nt_errs[] =
+const nt_err_code_struct nt_errs[] =
 {
 	{ "NT_STATUS_OK", NT_STATUS_OK },
 	{ "STATUS_NO_MORE_FILES", STATUS_NO_MORE_FILES },
@@ -569,6 +566,8 @@ static const nt_err_code_struct nt_errs[] =
 	{ "NT_STATUS_CURRENT_DOMAIN_NOT_ALLOWED", NT_STATUS_CURRENT_DOMAIN_NOT_ALLOWED },
 	{ "NT_STATUS_OBJECTID_NOT_FOUND", NT_STATUS_OBJECTID_NOT_FOUND },
 	{ "NT_STATUS_DOWNGRADE_DETECTED", NT_STATUS_DOWNGRADE_DETECTED },
+	{ "NT_STATUS_NO_S4U_PROT_SUPPORT", NT_STATUS_NO_S4U_PROT_SUPPORT },
+	{ "NT_STATUS_CROSSREALM_DELEGATION_FAILURE", NT_STATUS_CROSSREALM_DELEGATION_FAILURE },
 	{ "NT_STATUS_INVALID_LOCK_RANGE", NT_STATUS_INVALID_LOCK_RANGE },
 	{ "NT_STATUS_ERROR_DS_OBJ_STRING_NAME_EXISTS", NT_STATUS_ERROR_DS_OBJ_STRING_NAME_EXISTS },
 	{ "NT_STATUS_ERROR_DS_INCOMPATIBLE_VERSION", NT_STATUS_ERROR_DS_INCOMPATIBLE_VERSION },
@@ -743,7 +742,7 @@ static const nt_err_code_struct nt_errs[] =
 
 /* These need sorting..... */
 
-static const nt_err_code_struct nt_err_desc[] =
+const nt_err_code_struct nt_err_desc[] =
 {
 	{ N_("Success"), 				NT_STATUS_OK },
 	{ N_("Undetermined error"), 		NT_STATUS_UNSUCCESSFUL },
@@ -845,72 +844,26 @@ static const nt_err_code_struct nt_err_desc[] =
 };
 
 /*****************************************************************************
- Returns an NT error message.  not amazingly helpful, but better than a number.
- *****************************************************************************/
-
-const char *nt_errstr(NTSTATUS nt_code)
-{
-	static char msg[40];
-	int idx = 0;
-
-	while (nt_errs[idx].nt_errstr != NULL) {
-		if (NT_STATUS_V(nt_errs[idx].nt_errcode) ==
-		    NT_STATUS_V(nt_code)) {
-			return nt_errs[idx].nt_errstr;
-		}
-		idx++;
-	}
-
-	if (NT_STATUS_IS_LDAP(nt_code)) {
-		slprintf(msg, sizeof(msg), "LDAP code %u", NT_STATUS_LDAP_CODE(nt_code));
-		return msg;
-	}
-
-	slprintf(msg, sizeof(msg), "NT code 0x%08x", NT_STATUS_V(nt_code));
-
-	return msg;
-}
-
-/************************************************************************
- Print friendler version fo NT error code
- ***********************************************************************/
-
-const char *get_friendly_nt_error_msg(NTSTATUS nt_code)
-{
-	int idx = 0;
-
-	while (nt_err_desc[idx].nt_errstr != NULL) {
-		if (NT_STATUS_V(nt_err_desc[idx].nt_errcode) == NT_STATUS_V(nt_code)) {
-			return nt_err_desc[idx].nt_errstr;
-		}
-		idx++;
-	}
-
-	/* fall back to NT_STATUS_XXX string */
-
-	return nt_errstr(nt_code);
-}
-
-/*****************************************************************************
  Returns an NT_STATUS constant as a string for inclusion in autogen C code.
  *****************************************************************************/
 
-const char *get_nt_error_c_code(NTSTATUS nt_code)
+const char *get_nt_error_c_code(TALLOC_CTX *mem_ctx, NTSTATUS nt_code)
 {
-	static char out[40];
+	char *result;
 	int idx = 0;
 
 	while (nt_errs[idx].nt_errstr != NULL) {
 		if (NT_STATUS_V(nt_errs[idx].nt_errcode) ==
 		    NT_STATUS_V(nt_code)) {
-			return nt_errs[idx].nt_errstr;
+			result = talloc_strdup(mem_ctx, nt_errs[idx].nt_errstr);
+			return result;
 		}
 		idx++;
 	}
 
-	slprintf(out, sizeof(out), "NT_STATUS(0x%08x)", NT_STATUS_V(nt_code));
-
-	return out;
+	result = talloc_asprintf(mem_ctx, "NT_STATUS(0x%08x)",
+				 NT_STATUS_V(nt_code));
+	return result;
 }
 
 /*****************************************************************************
@@ -956,3 +909,58 @@ NTSTATUS nt_status_squash(NTSTATUS nt_status)
 		return nt_status;
 	}
 }
+
+/*****************************************************************************
+ Returns an NT error message.  not amazingly helpful, but better than a number.
+ *****************************************************************************/
+
+const char *nt_errstr(NTSTATUS nt_code)
+{
+	int idx = 0;
+	char *result;
+
+	while (nt_errs[idx].nt_errstr != NULL) {
+		if (NT_STATUS_V(nt_errs[idx].nt_errcode) ==
+		    NT_STATUS_V(nt_code)) {
+			return nt_errs[idx].nt_errstr;
+		}
+		idx++;
+	}
+
+	if (!talloc_stackframe_exists()) {
+		/* prevent memory leaks from talloc_tos() by using a
+		 * static area. This means the caller will overwrite
+		 * the string with subsequent calls, which can cause
+		 * display of the wrong error. If that happens the
+		 * caller should have a talloc stackframe
+		 */
+		static char msg[20];
+		snprintf(msg, sizeof(msg), "NT code 0x%08x", NT_STATUS_V(nt_code));
+		return msg;
+	}
+
+	result = talloc_asprintf(talloc_tos(), "NT code 0x%08x",
+				 NT_STATUS_V(nt_code));
+	SMB_ASSERT(result != NULL);
+	return result;
+}
+
+/************************************************************************
+ Print friendler version fo NT error code
+ ***********************************************************************/
+
+const char *get_friendly_nt_error_msg(NTSTATUS nt_code)
+{
+	int idx = 0;
+
+	while (nt_err_desc[idx].nt_errstr != NULL) {
+		if (NT_STATUS_V(nt_err_desc[idx].nt_errcode) == NT_STATUS_V(nt_code)) {
+			return nt_err_desc[idx].nt_errstr;
+		}
+		idx++;
+	}
+
+	/* fall back to NT_STATUS_XXX string */
+
+	return nt_errstr(nt_code);
+}
diff --git a/libcli/util/ntstatus.h b/libcli/util/ntstatus.h
index a7fc5799e9b..2f1635093b8 100644
--- a/libcli/util/ntstatus.h
+++ b/libcli/util/ntstatus.h
@@ -601,6 +601,8 @@ typedef uint32_t NTSTATUS;
 #define NT_STATUS_OBJECTID_NOT_FOUND NT_STATUS(0xC0000000 | 0x02F0)
 #define NT_STATUS_NO_SUCH_JOB NT_STATUS(0xC0000000 | 0xEDE) /* scheduler */
 #define NT_STATUS_DOWNGRADE_DETECTED NT_STATUS(0xC0000000 | 0x0388)
+#define NT_STATUS_NO_S4U_PROT_SUPPORT NT_STATUS(0xC0000000 | 0x040A)
+#define NT_STATUS_CROSSREALM_DELEGATION_FAILURE NT_STATUS(0xC0000000 | 0x040B)
 #define NT_STATUS_NO_SUCH_JOB NT_STATUS(0xC0000000 | 0xEDE) /* scheduler */
 #define NT_STATUS_RPC_PROTSEQ_NOT_SUPPORTED NT_STATUS(0xC0000000 | 0x20004)
 #define NT_STATUS_RPC_UNSUPPORTED_NAME_SYNTAX NT_STATUS(0xC0000000 | 0x20026)
@@ -627,6 +629,15 @@ typedef uint32_t NTSTATUS;
 #define NT_STATUS_FOOBAR NT_STATUS_UNSUCCESSFUL
 
 /*****************************************************************************
+ Returns an NT error message.  not amazingly helpful, but better than a number.
+
+ This version is const, and so neither allocates memory nor uses a
+ static variable for unknown errors.
+ *****************************************************************************/
+
+const char *nt_errstr_const(NTSTATUS nt_code);
+
+/*****************************************************************************
  returns an NT error message.  not amazingly helpful, but better than a number.
  *****************************************************************************/
 const char *nt_errstr(NTSTATUS nt_code);
@@ -639,16 +650,13 @@ const char *get_friendly_nt_error_msg(NTSTATUS nt_code);
 /*****************************************************************************
  returns an NT_STATUS constant as a string for inclusion in autogen C code
  *****************************************************************************/
-const char *get_nt_error_c_code(NTSTATUS nt_code);
+const char *get_nt_error_c_code(void *mem_ctx, NTSTATUS nt_code);
 
 /*****************************************************************************
  returns the NT_STATUS constant matching the string supplied (as an NTSTATUS)
  *****************************************************************************/
 NTSTATUS nt_status_string_to_code(const char *nt_status_str);
 
-/** Used by ntstatus_dos_equal: */
-extern bool ntstatus_check_dos_mapping;
-
 /* we need these here for openchange */
 #ifndef likely
 #define likely(x) (x)
@@ -659,13 +667,7 @@ extern bool ntstatus_check_dos_mapping;
 
 #define NT_STATUS_IS_OK(x) (likely(NT_STATUS_V(x) == 0))
 #define NT_STATUS_IS_ERR(x) (unlikely((NT_STATUS_V(x) & 0xc0000000) == 0xc0000000))
-/* checking for DOS error mapping here is ugly, but unfortunately the
-   alternative is a very intrusive rewrite of the torture code */
-#if _SAMBA_BUILD_ == 4
-#define NT_STATUS_EQUAL(x,y) (NT_STATUS_IS_DOS(x)||NT_STATUS_IS_DOS(y)?ntstatus_dos_equal(x,y):NT_STATUS_V(x) == NT_STATUS_V(y))
-#else
 #define NT_STATUS_EQUAL(x,y) (NT_STATUS_V(x) == NT_STATUS_V(y))
-#endif
 
 #define NT_STATUS_HAVE_NO_MEMORY(x) do { \
 	if (unlikely(!(x))) {		\
@@ -730,4 +732,10 @@ extern bool ntstatus_check_dos_mapping;
 	(((NT_STATUS_V(status) & 0xFFFF) == 0xC0020000) || \
 	 ((NT_STATUS_V(status) & 0xFFFF) == 0xC0030000))
 
+typedef struct
+{
+	const char *nt_errstr;
+	NTSTATUS nt_errcode;
+} nt_err_code_struct;
+
 #endif /* _NTSTATUS_H */
diff --git a/libcli/util/tstream.c b/libcli/util/tstream.c
index b287597c62c..12cef9b6ddd 100644
--- a/libcli/util/tstream.c
+++ b/libcli/util/tstream.c
@@ -106,7 +106,7 @@ static void tstream_read_pdu_blob_done(struct tevent_req *subreq)
 	ret = tstream_readv_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		tevent_req_nterror(req, status);
 		return;
 	}
diff --git a/libcli/util/wscript_build b/libcli/util/wscript_build
new file mode 100644
index 00000000000..43c6cec7873
--- /dev/null
+++ b/libcli/util/wscript_build
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+
+bld.SAMBA_LIBRARY('errors',
+                  public_headers='error.h ntstatus.h doserr.h werror.h',
+                  header_path='core',
+                  source='doserr.c errormap.c nterr.c errmap_unix.c',
+                  public_deps='talloc samba-util',
+                  private_library=True
+                  )
+
diff --git a/libds/common/flag_mapping.h b/libds/common/flag_mapping.h
index 4b1cc3b05fc..ae721da894a 100644
--- a/libds/common/flag_mapping.h
+++ b/libds/common/flag_mapping.h
@@ -1,3 +1,25 @@
+/*
+   Unix SMB/CIFS implementation.
+   helper mapping functions for the UF and ACB flags
+
+   Copyright (C) Stefan (metze) Metzmacher 2002
+   Copyright (C) Andrew Tridgell 2004
+   Copyright (C) Matthias Dieter Wallnöfer 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef __LIBDS_COMMON_FLAG_MAPPING_H__
 #define __LIBDS_COMMON_FLAG_MAPPING_H__
 
diff --git a/libds/common/roles.h b/libds/common/roles.h
new file mode 100644
index 00000000000..0ebb8771ac1
--- /dev/null
+++ b/libds/common/roles.h
@@ -0,0 +1,45 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   domain roles
+
+   Copyright (C) Andrew Tridgell 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBDS_ROLES_H_
+#define _LIBDS_ROLES_H_
+
+/* server roles. If you add new roles, please keep ensure that the
+ * existing role values match samr_Role from samr.idl
+ */
+enum server_role {
+	ROLE_STANDALONE    = 0,
+	ROLE_DOMAIN_MEMBER = 1,
+	ROLE_DOMAIN_BDC    = 2,
+	ROLE_DOMAIN_PDC    = 3
+};
+
+/* keep compatibility with the s4 'ROLE_DOMAIN_CONTROLLER' by mapping
+ * it to ROLE_DOMAIN_BDC. The PDC/BDC split is really historical from
+ * NT4 domains which were not multi-master, but even in AD there is
+ * only one machine that has the PDC FSMO role in a domain.
+*/
+#define ROLE_DOMAIN_CONTROLLER ROLE_DOMAIN_BDC
+
+/* security levels for 'security =' option */
+enum security_types {SEC_SHARE,SEC_USER,SEC_SERVER,SEC_DOMAIN,SEC_ADS};
+
+#endif /* _LIBDS_ROLES_H_ */
diff --git a/libds/common/wscript_build b/libds/common/wscript_build
index 8ee217709c3..d7f8acb73d0 100644
--- a/libds/common/wscript_build
+++ b/libds/common/wscript_build
@@ -3,4 +3,5 @@ bld.SAMBA_LIBRARY('flag_mapping',
                   public_deps='talloc replace',
                   source='flag_mapping.c',
 		  private_library=True,
-		  deps='samba-util-common')
+                  public_headers='roles.h',
+		  deps='samba-util')
diff --git a/libgpo/gpext/gpext.c b/libgpo/gpext/gpext.c
index 080bf074523..42ab6223312 100644
--- a/libgpo/gpext/gpext.c
+++ b/libgpo/gpext/gpext.c
@@ -351,7 +351,7 @@ NTSTATUS gp_ext_info_add_entry(TALLOC_CTX *mem_ctx,
 	NTSTATUS status;
 	struct gp_extension_reg_info_entry *entry = NULL;
 
-	entry = TALLOC_ZERO_P(mem_ctx, struct gp_extension_reg_info_entry);
+	entry = talloc_zero(mem_ctx, struct gp_extension_reg_info_entry);
 	NT_STATUS_HAVE_NO_MEMORY(entry);
 
 	status = GUID_from_string(ext_guid, &entry->guid);
@@ -442,7 +442,7 @@ static WERROR gp_extension_store_reg_entry(TALLOC_CTX *mem_ctx,
 	subkeyname = GUID_string2(mem_ctx, &entry->guid);
 	W_ERROR_HAVE_NO_MEMORY(subkeyname);
 
-	strupper_m(CONST_DISCARD(char *,subkeyname));
+	strupper_m(discard_const_p(char, subkeyname));
 
 	werr = gp_store_reg_subkey(mem_ctx,
 				   subkeyname,
@@ -492,9 +492,10 @@ static NTSTATUS gp_glob_ext_list(TALLOC_CTX *mem_ctx,
 	SMB_STRUCT_DIR *dir = NULL;
 	SMB_STRUCT_DIRENT *dirent = NULL;
 
-	dir = sys_opendir(modules_path(SAMBA_SUBSYSTEM_GPEXT));
+	dir = sys_opendir(modules_path(talloc_tos(), 
+				       SAMBA_SUBSYSTEM_GPEXT));
 	if (!dir) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	while ((dirent = sys_readdir(dir))) {
diff --git a/libgpo/gpo_fetch.c b/libgpo/gpo_fetch.c
index 40f8c5870de..cae11d5c0d2 100644
--- a/libgpo/gpo_fetch.c
+++ b/libgpo/gpo_fetch.c
@@ -34,6 +34,7 @@
 #include "lib/util/util.h"
 #else
 #include "libgpo/gpo_proto.h"
+#include "libsmb/libsmb.h"
 #endif
 
 /****************************************************************
@@ -142,7 +143,7 @@ static NTSTATUS gpo_connect_server(ADS_STRUCT *ads, struct loadparm_context *lp_
 
 
 	result = cli_full_connection(&cli,
-			global_myname(),
+			lp_netbios_name(),
 			server,
 			NULL, 0,
 			service, "A:",
diff --git a/libgpo/gpo_ini.c b/libgpo/gpo_ini.c
index a4bff0e86b8..494162466d0 100644
--- a/libgpo/gpo_ini.c
+++ b/libgpo/gpo_ini.c
@@ -108,7 +108,7 @@ static NTSTATUS convert_file_from_ucs2(TALLOC_CTX *mem_ctx,
 	}
 
 	if (write(tmp_fd, data_out, converted_size) != converted_size) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		goto out;
 	}
 
diff --git a/libgpo/gpo_ldap.c b/libgpo/gpo_ldap.c
index 6abaf29cf0d..9d984b5b990 100644
--- a/libgpo/gpo_ldap.c
+++ b/libgpo/gpo_ldap.c
@@ -582,7 +582,7 @@ static ADS_STATUS add_gplink_to_gpo_list(ADS_STRUCT *ads,
 			}
 		}
 
-		new_gpo = TALLOC_ZERO_P(mem_ctx, struct GROUP_POLICY_OBJECT);
+		new_gpo = talloc_zero(mem_ctx, struct GROUP_POLICY_OBJECT);
 		ADS_ERROR_HAVE_NO_MEMORY(new_gpo);
 
 		status = ads_get_gpo(ads, mem_ctx, gp_link->link_names[i],
@@ -640,7 +640,7 @@ ADS_STATUS ads_get_sid_token(ADS_STRUCT *ads,
 		return status;
 	}
 
-	token_sids = TALLOC_ARRAY(mem_ctx, struct dom_sid, 1);
+	token_sids = talloc_array(mem_ctx, struct dom_sid, 1);
 	ADS_ERROR_HAVE_NO_MEMORY(token_sids);
 
 	status = ADS_ERROR_NT(add_sid_to_array_unique(mem_ctx,
@@ -688,7 +688,7 @@ static ADS_STATUS add_local_policy_to_gpo_list(TALLOC_CTX *mem_ctx,
 
 	ADS_ERROR_HAVE_NO_MEMORY(gpo_list);
 
-	gpo = TALLOC_ZERO_P(mem_ctx, struct GROUP_POLICY_OBJECT);
+	gpo = talloc_zero(mem_ctx, struct GROUP_POLICY_OBJECT);
 	ADS_ERROR_HAVE_NO_MEMORY(gpo);
 
 	gpo->name = talloc_strdup(mem_ctx, "Local Policy");
diff --git a/libgpo/wscript_build b/libgpo/wscript_build
index e2422c566b6..f182b44df00 100644
--- a/libgpo/wscript_build
+++ b/libgpo/wscript_build
@@ -2,7 +2,7 @@
 
 bld.SAMBA_SUBSYSTEM('LIBGPO',
 	source='gpo_util.c gpo_sec.c ../libgpo/gpext/gpext.c gpo_fetch.c gpo_ini.c ../source4/libgpo/ads_convenience.c ../source3/libgpo/gpo_filesync.c ../source4/libgpo/gpo_filesync.c',
-	deps='ldb samba-net',
+	deps='ldb samba-net samba-util',
 	enabled=False
 	)
 
diff --git a/librpc/idl/dfsblobs.idl b/librpc/idl/dfsblobs.idl
index 6151c3f77b2..7b8795d29cd 100644
--- a/librpc/idl/dfsblobs.idl
+++ b/librpc/idl/dfsblobs.idl
@@ -53,7 +53,6 @@ interface dfsblobs
 		[relative_short] nstring *DFS_path;
 		[relative_short] nstring *DFS_alt_path;
 		[relative_short] nstring *netw_address;
-		/* As stated in MS DFSC 2.2.4.3.1 this array was guid but now MUST be 16 null bytes*/
 	} dfs_normal_referral;
 
 	typedef struct {
@@ -74,32 +73,20 @@ interface dfsblobs
 	} dfs_padding;
 
 	typedef [flag(NDR_NOALIGN)] struct {
+		uint16 size;
 		DFS_SERVER_TYPE server_type;
 		DFS_FLAGS_REFERRAL entry_flags;
 		uint32 ttl;
 		[switch_is(entry_flags & DFS_FLAG_REFERRAL_DOMAIN_RESP)] dfs_referral referrals;
-	} dfs_referral_v3_remaining;
-
-	typedef [flag(NDR_NOALIGN)] struct {
-		uint16 size;
-		dfs_referral_v3_remaining data;
 		/* this is either 0 or 16 bytes */
 		[switch_is(size - 18)] dfs_padding service_site_guid;
 	} dfs_referral_v3;
 
-	typedef struct {
-		uint16 size;
-		DFS_SERVER_TYPE server_type;
-		DFS_FLAGS_REFERRAL entry_flags;
-		uint32 ttl;
-		dfs_normal_referral r1;
-	} dfs_referral_v4;
-
 	typedef [nodiscriminant] union {
 		[case(1)] dfs_referral_v1 v1;
 		[case(2)] dfs_referral_v2 v2;
 		[case(3)] dfs_referral_v3 v3;
-		[case(4)] dfs_referral_v4 v4;
+		[case(4)] dfs_referral_v3 v4;
 		[default];
 	} dfs_referral_version;
 
diff --git a/librpc/idl/idl_types.h b/librpc/idl/idl_types.h
index 023c04020ea..c50eface0d1 100644
--- a/librpc/idl/idl_types.h
+++ b/librpc/idl/idl_types.h
@@ -8,6 +8,7 @@
 #define STR_CONFORMANT  LIBNDR_FLAG_STR_CONFORMANT
 #define STR_CHARLEN	LIBNDR_FLAG_STR_CHARLEN
 #define STR_UTF8	LIBNDR_FLAG_STR_UTF8
+#define STR_RAW8	LIBNDR_FLAG_STR_RAW8
 
 /*
   a null terminated UCS2 string
@@ -25,6 +26,11 @@
 #define utf8string	[flag(STR_UTF8|STR_NULLTERM)] string
 
 /*
+  a null terminated "raw" string (null terminated byte sequence)
+*/
+#define raw8string	[flag(STR_RAW8|STR_NULLTERM)] string
+
+/*
   a null terminated UCS2 string
 */
 #define nstring_array	[flag(STR_NULLTERM|NDR_ALIGN2)] string_array
diff --git a/source3/librpc/idl/notify.idl b/librpc/idl/notify.idl
index 0776ed107cc..845010601ea 100644
--- a/source3/librpc/idl/notify.idl
+++ b/librpc/idl/notify.idl
@@ -41,7 +41,7 @@ interface notify
 	  The max_mask and max_mask_subdir at each depth is the
 	  bitwise or of the filters and subdir filters for all entries
 	  at that depth. This allows a depth to be quickly skipped if
-	  no entries will match the target filter	  
+	  no entries will match the target filter
 	*/
 	typedef struct {
 		uint32 max_mask;
diff --git a/librpc/idl/printcap.idl b/librpc/idl/printcap.idl
index 5ab380ce6cb..d9c34f3fab7 100644
--- a/librpc/idl/printcap.idl
+++ b/librpc/idl/printcap.idl
@@ -7,6 +7,7 @@ interface printcap
 	typedef struct {
 		[charset(UTF8),string] uint8 *name;
 		[charset(UTF8),string] uint8 *info;
+		[charset(UTF8),string] uint8 *location;
 	} pcap_printer;
 
 	typedef [public] struct {
diff --git a/librpc/idl/rap.idl b/librpc/idl/rap.idl
index 8087e22c5b2..780951c7503 100644
--- a/librpc/idl/rap.idl
+++ b/librpc/idl/rap.idl
@@ -966,6 +966,28 @@ interface rap
 		[out] uint16 convert
 	);
 
+	typedef [public] struct {
+		uint32 TimeSinceJan11970;
+		uint32 TimeSinceBoot;
+		uint8 Hours;
+		uint8 Minutes;
+		uint8 Seconds;
+		uint8 Hundreds;
+		uint16 TimeZone;
+		uint16 ClockFrequency;
+		uint8 Day;
+		uint8 Month;
+		uint16 Year;
+		uint8 Weekday;
+	} rap_TimeOfDayInfo;
+
+	[public] void rap_NetRemoteTOD(
+		[in] uint16 bufsize,
+		[out] rap_status status,
+		[out] uint16 convert,
+		[out] rap_TimeOfDayInfo tod
+	);
+
 	/* Parameter description strings for RAP calls   */
 	/* Names are defined name for RAP call with _REQ */
 	/* appended to end.                              */
diff --git a/source3/librpc/idl/server_id.idl b/librpc/idl/server_id.idl
index ed727881c73..ac2e9ab3182 100644
--- a/source3/librpc/idl/server_id.idl
+++ b/librpc/idl/server_id.idl
@@ -4,6 +4,13 @@
 interface server_id
 {
 
+	/*
+	 * Virtual Node Numbers are identifying a node within a cluster.
+	 * Ctdbd sets this, we retrieve our vnn from it.
+	 */
+
+	const int NONCLUSTER_VNN = 0xFFFFFFFF;
+
 	/* used to look like the following, note that unique_id was not
 	 * marshalled at all...
 
@@ -18,8 +25,9 @@ interface server_id
 	*/
 
 	typedef [public] struct {
-		uint32 pid;
+		hyper pid;
+		uint32 task_id;
 		uint32 vnn;
-		udlong unique_id;
+		hyper unique_id;
 	} server_id;
 }
diff --git a/librpc/idl/wscript_build b/librpc/idl/wscript_build
index 7e1340e6f77..ffb7a9c8088 100644
--- a/librpc/idl/wscript_build
+++ b/librpc/idl/wscript_build
@@ -8,8 +8,9 @@ bld.SAMBA_PIDL_LIST('PIDL',
                        browser.idl dfs.idl dssetup.idl frsapi.idl krb5pac.idl
                        named_pipe_auth.idl orpc.idl rot.idl spoolss.idl w32time.idl xattr.idl
                        dbgidl.idl dnsserver.idl echo.idl frsrpc.idl lsa.idl nbt.idl dns.idl
-                       oxidresolver.idl samr.idl srvsvc.idl winreg.idl dcerpc.idl
+                       oxidresolver.idl samr.idl server_id.idl srvsvc.idl winreg.idl dcerpc.idl
                        drsblobs.idl efs.idl frstrans.idl mgmt.idl netlogon.idl
+                       notify.idl
                        policyagent.idl scerpc.idl svcctl.idl wkssvc.idl eventlog6.idl backupkey.idl
                        printcap.idl''',
                     options='--header --ndr-parser --samba3-ndr-server --server --client --python',
diff --git a/librpc/ndr/libndr.h b/librpc/ndr/libndr.h
index 8c59bb9bb06..ca3710bc986 100644
--- a/librpc/ndr/libndr.h
+++ b/librpc/ndr/libndr.h
@@ -122,6 +122,7 @@ struct ndr_print {
 #define LIBNDR_FLAG_STR_CONFORMANT	(1<<10)
 #define LIBNDR_FLAG_STR_CHARLEN		(1<<11)
 #define LIBNDR_FLAG_STR_UTF8		(1<<12)
+#define LIBNDR_FLAG_STR_RAW8		(1<<13)
 #define LIBNDR_STRING_FLAGS		(0x7FFC)
 
 /* set if relative pointers should *not* be marshalled in reverse order */
@@ -374,6 +375,8 @@ struct ndr_interface_list {
 	const struct ndr_interface_table *table;
 };
 
+struct sockaddr_storage;
+
 /*********************************************************************
  Map an NT error code from a NDR error code.
 *********************************************************************/
@@ -395,6 +398,7 @@ enum ndr_err_code ndr_pull_dom_sid0(struct ndr_pull *ndr, int ndr_flags, struct
 void ndr_print_dom_sid0(struct ndr_print *ndr, const char *name, const struct dom_sid *sid);
 size_t ndr_size_dom_sid0(const struct dom_sid *sid, int flags);
 void ndr_print_GUID(struct ndr_print *ndr, const char *name, const struct GUID *guid);
+void ndr_print_sockaddr_storage(struct ndr_print *ndr, const char *name, const struct sockaddr_storage *ss);
 bool ndr_syntax_id_equal(const struct ndr_syntax_id *i1, const struct ndr_syntax_id *i2); 
 enum ndr_err_code ndr_push_struct_blob(DATA_BLOB *blob, TALLOC_CTX *mem_ctx, const void *p, ndr_push_flags_fn_t fn);
 enum ndr_err_code ndr_push_union_blob(DATA_BLOB *blob, TALLOC_CTX *mem_ctx, void *p, uint32_t level, ndr_push_flags_fn_t fn);
diff --git a/librpc/ndr/ndr_backupkey.h b/librpc/ndr/ndr_backupkey.h
index c5c7c39995a..4e82f6296ad 100644
--- a/librpc/ndr/ndr_backupkey.h
+++ b/librpc/ndr/ndr_backupkey.h
@@ -1,2 +1,23 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   routines for top backup key protocol marshalling/unmarshalling
+
+   Copyright (C) Matthieu Patou 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 _PUBLIC_ enum ndr_err_code ndr_push_bkrp_access_check_v2(struct ndr_push *ndr, int ndr_flags, const struct bkrp_access_check_v2 *r);
 _PUBLIC_ enum ndr_err_code ndr_pull_bkrp_access_check_v2(struct ndr_pull *ndr, int ndr_flags, struct bkrp_access_check_v2 *r);
diff --git a/librpc/ndr/ndr_compression.h b/librpc/ndr/ndr_compression.h
index b939f6a42ba..1c22fbf7595 100644
--- a/librpc/ndr/ndr_compression.h
+++ b/librpc/ndr/ndr_compression.h
@@ -1,3 +1,25 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   libndr compression support
+
+   Copyright (C) Stefan Metzmacher 2005
+   Copyright (C) Matthieu Suiche 2008
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef __LIBRPC_NDR_NDR_COMPRESSION_H__
 #define __LIBRPC_NDR_NDR_COMPRESSION_H__
 
diff --git a/librpc/ndr/ndr_dns.h b/librpc/ndr/ndr_dns.h
index d97d57fdacd..16bf11ba87f 100644
--- a/librpc/ndr/ndr_dns.h
+++ b/librpc/ndr/ndr_dns.h
@@ -1,3 +1,28 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   manipulate dns name structures
+
+   Copyright (C) 2010 Kai Blin  <kai@samba.org>
+
+   Heavily based on nbtname.c which is:
+
+   Copyright (C) Andrew Tridgell 2005
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 void ndr_print_dns_string(struct ndr_print *ndr,
 			  const char *name,
 			  const char *s);
diff --git a/librpc/ndr/ndr_spoolss_buf.h b/librpc/ndr/ndr_spoolss_buf.h
index c5938c00c86..77053cb13d7 100644
--- a/librpc/ndr/ndr_spoolss_buf.h
+++ b/librpc/ndr/ndr_spoolss_buf.h
@@ -1,3 +1,26 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   routines for marshalling/unmarshalling spoolss subcontext buffer structures
+
+   Copyright (C) Andrew Tridgell 2003
+   Copyright (C) Tim Potter 2003
+   Copyright (C) Guenther Deschner 2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef ___SPACE_SRC_SAMBA_SOURCES_SAMBA_GIT_SOURCE3____SOURCE4_LIBRPC_NDR_NDR_SPOOLSS_BUF_H__
 #define ___SPACE_SRC_SAMBA_SOURCES_SAMBA_GIT_SOURCE3____SOURCE4_LIBRPC_NDR_NDR_SPOOLSS_BUF_H__
 
diff --git a/librpc/ndr/ndr_string.c b/librpc/ndr/ndr_string.c
index 6e20333f043..840776fa3a9 100644
--- a/librpc/ndr/ndr_string.c
+++ b/librpc/ndr/ndr_string.c
@@ -30,8 +30,8 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
 	char *as=NULL;
 	uint32_t len1, ofs, len2;
 	uint16_t len3;
-	size_t converted_size;
-	int chset = CH_UTF16;
+	size_t conv_src_len = 0, converted_size;
+	int do_convert = 1, chset = CH_UTF16;
 	unsigned byte_mul = 2;
 	unsigned flags = ndr->flags;
 	unsigned c_len_term = 0;
@@ -56,6 +56,12 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
 		flags &= ~LIBNDR_FLAG_STR_UTF8;
 	}
 
+	if (flags & LIBNDR_FLAG_STR_RAW8) {
+		do_convert = 0;
+		byte_mul = 1;
+		flags &= ~LIBNDR_FLAG_STR_RAW8;
+	}
+
 	flags &= ~LIBNDR_FLAG_STR_CONFORMANT;
 	if (flags & LIBNDR_FLAG_STR_CHARLEN) {
 		c_len_term = 1;
@@ -73,77 +79,19 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
 		}
 		NDR_CHECK(ndr_pull_uint32(ndr, NDR_SCALARS, &len2));
 		if (len2 > len1) {
-			return ndr_pull_error(ndr, NDR_ERR_STRING, 
-					      "Bad string lengths len1=%u ofs=%u len2=%u\n", 
+			return ndr_pull_error(ndr, NDR_ERR_STRING,
+					      "Bad string lengths len1=%u ofs=%u len2=%u\n",
 					      len1, ofs, len2);
-		}
-		NDR_PULL_NEED_BYTES(ndr, (len2 + c_len_term)*byte_mul);
-		if (len2 == 0) {
-			as = talloc_strdup(ndr->current_mem_ctx, "");
-		} else {
-			if (!convert_string_talloc(ndr->current_mem_ctx, chset,
-						   CH_UNIX,
-						   ndr->data+ndr->offset,
-						   (len2 + c_len_term)*byte_mul,
-						   (void **)(void *)&as,
-						   &converted_size))
-			{
-				return ndr_pull_error(ndr, NDR_ERR_CHARCNV,
-						      "Bad character conversion with flags 0x%x", flags);
-			}
-		}
-		NDR_CHECK(ndr_pull_advance(ndr, (len2 + c_len_term)*byte_mul));
-
-		if (len1 != len2) {
+		} else if (len1 != len2) {
 			DEBUG(6,("len1[%u] != len2[%u] '%s'\n", len1, len2, as));
 		}
-
-		/* this is a way of detecting if a string is sent with the wrong
-		   termination */
-		if (ndr->flags & LIBNDR_FLAG_STR_NOTERM) {
-			if (strlen(as) < (len2 + c_len_term)) {
-				DEBUG(6,("short string '%s'\n", as));
-			}
-		} else {
-			if (strlen(as) == (len2 + c_len_term)) {
-				DEBUG(6,("long string '%s'\n", as));
-			}
-		}
-		*s = as;
+		conv_src_len = len2 + c_len_term;
 		break;
 
 	case LIBNDR_FLAG_STR_SIZE4:
 	case LIBNDR_FLAG_STR_SIZE4|LIBNDR_FLAG_STR_NOTERM:
 		NDR_CHECK(ndr_pull_uint32(ndr, NDR_SCALARS, &len1));
-		NDR_PULL_NEED_BYTES(ndr, (len1 + c_len_term)*byte_mul);
-		if (len1 == 0) {
-			as = talloc_strdup(ndr->current_mem_ctx, "");
-		} else {
-			if (!convert_string_talloc(ndr->current_mem_ctx, chset,
-						   CH_UNIX,
-						   ndr->data+ndr->offset,
-						   (len1 + c_len_term)*byte_mul,
-						   (void **)(void *)&as,
-						   &converted_size))
-			{
-				return ndr_pull_error(ndr, NDR_ERR_CHARCNV, 
-						      "Bad character conversion with flags 0x%x", flags);
-			}
-		}
-		NDR_CHECK(ndr_pull_advance(ndr, (len1 + c_len_term)*byte_mul));
-
-		/* this is a way of detecting if a string is sent with the wrong
-		   termination */
-		if (ndr->flags & LIBNDR_FLAG_STR_NOTERM) {
-			if (strlen(as) < (len1 + c_len_term)) {
-				DEBUG(6,("short string '%s'\n", as));
-			}
-		} else {
-			if (strlen(as) == (len1 + c_len_term)) {
-				DEBUG(6,("long string '%s'\n", as));
-			}
-		}
-		*s = as;
+		conv_src_len = len1 + c_len_term;
 		break;
 
 	case LIBNDR_FLAG_STR_LEN4:
@@ -154,108 +102,28 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
 					      ndr->flags & LIBNDR_STRING_FLAGS);
 		}
 		NDR_CHECK(ndr_pull_uint32(ndr, NDR_SCALARS, &len1));
-		NDR_PULL_NEED_BYTES(ndr, (len1 + c_len_term)*byte_mul);
-		if (len1 == 0) {
-			as = talloc_strdup(ndr->current_mem_ctx, "");
-		} else {
-			if (!convert_string_talloc(ndr->current_mem_ctx, chset,
-						   CH_UNIX,
-						   ndr->data+ndr->offset,
-						   (len1 + c_len_term)*byte_mul,
-						   (void **)(void *)&as,
-						   &converted_size))
-			{
-				return ndr_pull_error(ndr, NDR_ERR_CHARCNV, 
-						      "Bad character conversion with flags 0x%x", flags);
-			}
-		}
-		NDR_CHECK(ndr_pull_advance(ndr, (len1 + c_len_term)*byte_mul));
-
-		/* this is a way of detecting if a string is sent with the wrong
-		   termination */
-		if (ndr->flags & LIBNDR_FLAG_STR_NOTERM) {
-			if (strlen(as) < (len1 + c_len_term)) {
-				DEBUG(6,("short string '%s'\n", as));
-			}
-		} else {
-			if (strlen(as) == (len1 + c_len_term)) {
-				DEBUG(6,("long string '%s'\n", as));
-			}
-		}
-		*s = as;
+		conv_src_len = len1 + c_len_term;
 		break;
 
-
 	case LIBNDR_FLAG_STR_SIZE2:
 	case LIBNDR_FLAG_STR_SIZE2|LIBNDR_FLAG_STR_NOTERM:
 		NDR_CHECK(ndr_pull_uint16(ndr, NDR_SCALARS, &len3));
-		NDR_PULL_NEED_BYTES(ndr, (len3 + c_len_term)*byte_mul);
-		if (len3 == 0) {
-			as = talloc_strdup(ndr->current_mem_ctx, "");
-		} else {
-			if (!convert_string_talloc(ndr->current_mem_ctx, chset,
-						   CH_UNIX,
-						   ndr->data+ndr->offset,
-						   (len3 + c_len_term)*byte_mul,
-						   (void **)(void *)&as,
-						   &converted_size))
-			{
-				return ndr_pull_error(ndr, NDR_ERR_CHARCNV, 
-						      "Bad character conversion with flags 0x%x", flags);
-			}
-		}
-		NDR_CHECK(ndr_pull_advance(ndr, (len3 + c_len_term)*byte_mul));
-
-		/* this is a way of detecting if a string is sent with the wrong
-		   termination */
-		if (ndr->flags & LIBNDR_FLAG_STR_NOTERM) {
-			if (strlen(as) < (len3 + c_len_term)) {
-				DEBUG(6,("short string '%s'\n", as));
-			}
-		} else {
-			if (strlen(as) == (len3 + c_len_term)) {
-				DEBUG(6,("long string '%s'\n", as));
-			}
-		}
-		*s = as;
+		conv_src_len = len3 + c_len_term;
 		break;
 
 	case LIBNDR_FLAG_STR_SIZE2|LIBNDR_FLAG_STR_NOTERM|LIBNDR_FLAG_STR_BYTESIZE:
 		NDR_CHECK(ndr_pull_uint16(ndr, NDR_SCALARS, &len3));
-		NDR_PULL_NEED_BYTES(ndr, len3);
-		if (len3 == 0) {
-			as = talloc_strdup(ndr->current_mem_ctx, "");
-		} else {
-			if (!convert_string_talloc(ndr->current_mem_ctx, chset,
-						   CH_UNIX,
-						   ndr->data+ndr->offset, len3,
-						   (void **)(void *)&as,
-						   &converted_size))
-			{
-				return ndr_pull_error(ndr, NDR_ERR_CHARCNV, 
-						      "Bad character conversion with flags 0x%x", flags);
-			}
-		}
-		NDR_CHECK(ndr_pull_advance(ndr, len3));
-		*s = as;
+		conv_src_len = len3;
+		byte_mul = 1; /* the length is now absolute */
 		break;
 
 	case LIBNDR_FLAG_STR_NULLTERM:
 		if (byte_mul == 1) {
-			len1 = ascii_len_n((const char *)(ndr->data+ndr->offset), ndr->data_size - ndr->offset);
+			conv_src_len = ascii_len_n((const char *)(ndr->data+ndr->offset), ndr->data_size - ndr->offset);
 		} else {
-			len1 = utf16_len_n(ndr->data+ndr->offset, ndr->data_size - ndr->offset);
-		}
-		if (!convert_string_talloc(ndr->current_mem_ctx, chset, CH_UNIX,
-					   ndr->data+ndr->offset, len1,
-					   (void **)(void *)&as,
-					   &converted_size))
-		{
-			return ndr_pull_error(ndr, NDR_ERR_CHARCNV, 
-					      "Bad character conversion with flags 0x%x", flags);
+			conv_src_len = utf16_len_n(ndr->data+ndr->offset, ndr->data_size - ndr->offset);
 		}
-		NDR_CHECK(ndr_pull_advance(ndr, len1));
-		*s = as;
+		byte_mul = 1; /* the length is now absolute */
 		break;
 
 	case LIBNDR_FLAG_STR_NOTERM:
@@ -263,26 +131,8 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
 			return ndr_pull_error(ndr, NDR_ERR_STRING, "Bad string flags 0x%x (missing NDR_REMAINING)\n",
 					      ndr->flags & LIBNDR_STRING_FLAGS);
 		}
-
-		len1 = ndr->data_size - ndr->offset;
-
-		NDR_PULL_NEED_BYTES(ndr, len1);
-		if (len1 == 0) {
-			as = talloc_strdup(ndr->current_mem_ctx, "");
-		} else {
-			if (!convert_string_talloc(ndr->current_mem_ctx, chset,
-						   CH_UNIX,
-						   ndr->data+ndr->offset, len1,
-						   (void **)(void *)&as,
-						   &converted_size))
-			{
-				return ndr_pull_error(ndr, NDR_ERR_CHARCNV, 
-						      "Bad character conversion with flags 0x%x", flags);
-			}
-		}
-		NDR_CHECK(ndr_pull_advance(ndr, len1));
-
-		*s = as;
+		conv_src_len = ndr->data_size - ndr->offset;
+		byte_mul = 1; /* the length is now absolute */
 		break;
 
 	default:
@@ -290,6 +140,45 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
 				      ndr->flags & LIBNDR_STRING_FLAGS);
 	}
 
+	NDR_PULL_NEED_BYTES(ndr, conv_src_len * byte_mul);
+	if (conv_src_len == 0) {
+		as = talloc_strdup(ndr->current_mem_ctx, "");
+		converted_size = 0;
+	} else {
+		if (!do_convert) {
+			as = talloc_strndup(ndr->current_mem_ctx,
+			                    (char *)ndr->data + ndr->offset,
+					    conv_src_len);
+			if (!as) {
+				return ndr_pull_error(ndr, NDR_ERR_ALLOC,
+						      "Failed to talloc_strndup() in RAW8 ndr_string_pull()");
+			}
+			converted_size = MIN(strlen(as)+1, conv_src_len);
+		} else if (!convert_string_talloc(ndr->current_mem_ctx, chset,
+					   CH_UNIX, ndr->data + ndr->offset,
+					   conv_src_len * byte_mul,
+					   (void **)(void *)&as,
+					   &converted_size)) {
+			return ndr_pull_error(ndr, NDR_ERR_CHARCNV,
+					      "Bad character conversion with flags 0x%x", flags);
+		}
+	}
+
+	/* this is a way of detecting if a string is sent with the wrong
+	   termination */
+	if (ndr->flags & LIBNDR_FLAG_STR_NOTERM) {
+		if (as && converted_size > 0 && as[converted_size-1] == '\0') {
+			DEBUG(6,("short string '%s', sent with NULL termination despite NOTERM flag in IDL\n", as));
+		}
+	} else {
+		if (as && converted_size > 0 && as[converted_size-1] != '\0') {
+			DEBUG(6,("long string '%s', send without NULL termination (which was expected)\n", as));
+		}
+	}
+
+	NDR_CHECK(ndr_pull_advance(ndr, conv_src_len * byte_mul));
+	*s = as;
+
 	return NDR_ERR_SUCCESS;
 }
 
@@ -301,7 +190,7 @@ _PUBLIC_ enum ndr_err_code ndr_push_string(struct ndr_push *ndr, int ndr_flags,
 {
 	ssize_t s_len, c_len;
 	size_t d_len;
-	int chset = CH_UTF16;
+	int do_convert = 1, chset = CH_UTF16;
 	unsigned flags = ndr->flags;
 	unsigned byte_mul = 2;
 	uint8_t *dest = NULL;
@@ -328,12 +217,22 @@ _PUBLIC_ enum ndr_err_code ndr_push_string(struct ndr_push *ndr, int ndr_flags,
 		flags &= ~LIBNDR_FLAG_STR_UTF8;
 	}
 
+	if (flags & LIBNDR_FLAG_STR_RAW8) {
+		do_convert = 0;
+		byte_mul = 1;
+		flags &= ~LIBNDR_FLAG_STR_RAW8;
+	}
+
 	flags &= ~LIBNDR_FLAG_STR_CONFORMANT;
 
 	if (!(flags & LIBNDR_FLAG_STR_NOTERM)) {
 		s_len++;
 	}
-	if (!convert_string_talloc(ndr, CH_UNIX, chset, s, s_len,
+
+	if (!do_convert) {
+		d_len = s_len;
+		dest = (uint8_t *)talloc_strndup(ndr, s, s_len);
+	} else if (!convert_string_talloc(ndr, CH_UNIX, chset, s, s_len,
 				   (void **)(void *)&dest, &d_len))
 	{
 		return ndr_push_error(ndr, NDR_ERR_CHARCNV, 
@@ -403,9 +302,13 @@ _PUBLIC_ size_t ndr_string_array_size(struct ndr_push *ndr, const char *s)
 	unsigned byte_mul = 2;
 	unsigned c_len_term = 1;
 
-	c_len = s?strlen_m(s):0;
+	if (flags & LIBNDR_FLAG_STR_RAW8) {
+		c_len = s?strlen(s):0;
+	} else {
+		c_len = s?strlen_m(s):0;
+	}
 
-	if (flags & (LIBNDR_FLAG_STR_ASCII|LIBNDR_FLAG_STR_UTF8)) {
+	if (flags & (LIBNDR_FLAG_STR_ASCII|LIBNDR_FLAG_STR_RAW8|LIBNDR_FLAG_STR_UTF8)) {
 		byte_mul = 1;
 	}
 
@@ -611,16 +514,22 @@ _PUBLIC_ size_t ndr_size_string_array(const char **a, uint32_t count, int flags)
 {
 	uint32_t i;
 	size_t size = 0;
+	int rawbytes = 0;
+
+	if (flags & LIBNDR_FLAG_STR_RAW8) {
+		rawbytes = 1;
+		flags &= ~LIBNDR_FLAG_STR_RAW8;
+	}
 
 	switch (flags & LIBNDR_STRING_FLAGS) {
 	case LIBNDR_FLAG_STR_NULLTERM:
 		for (i = 0; i < count; i++) {
-			size += strlen_m_term(a[i]);
+			size += rawbytes?strlen(a[i]) + 1:strlen_m_term(a[i]);
 		}
 		break;
 	case LIBNDR_FLAG_STR_NOTERM:
 		for (i = 0; i < count; i++) {
-			size += strlen_m(a[i]);
+			size += rawbytes?strlen(a[i]):strlen_m(a[i]);
 		}
 		break;
 	default:
@@ -736,7 +645,6 @@ _PUBLIC_ uint32_t ndr_charset_length(const void *var, charset_t chset)
 	case CH_UTF16MUNGED:
 	case CH_UTF8:
 		return strlen_m_ext_term((const char *)var, CH_UNIX, chset);
-	case CH_DISPLAY:
 	case CH_DOS:
 	case CH_UNIX:
 		return strlen((const char *)var)+1;
diff --git a/librpc/ndr/ndr_table.h b/librpc/ndr/ndr_table.h
index 9e8fea1db6a..207618f644e 100644
--- a/librpc/ndr/ndr_table.h
+++ b/librpc/ndr/ndr_table.h
@@ -1,3 +1,25 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   dcerpc utility functions
+
+   Copyright (C) Andrew Tridgell 2003
+   Copyright (C) Jelmer Vernooij 2004
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef _NDR_TABLE_PROTO_H_
 #define _NDR_TABLE_PROTO_H_
 
diff --git a/librpc/ndr/ndr_wmi.c b/librpc/ndr/ndr_wmi.c
index 87f3e9690ad..47488883890 100644
--- a/librpc/ndr/ndr_wmi.c
+++ b/librpc/ndr/ndr_wmi.c
@@ -23,6 +23,7 @@
 #include "includes.h"
 #include "librpc/gen_ndr/ndr_dcom.h"
 #include "librpc/gen_ndr/ndr_wmi.h"
+#include "librpc/ndr/ndr_wmi.h"
 
 // Just for debugging
 int NDR_CHECK_depth = 0;
diff --git a/librpc/ndr/ndr_wmi.h b/librpc/ndr/ndr_wmi.h
new file mode 100644
index 00000000000..62d2c535cd8
--- /dev/null
+++ b/librpc/ndr/ndr_wmi.h
@@ -0,0 +1,24 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   routines for marshalling/unmarshalling DCOM string arrays
+
+   Copyright (C) Jelmer Vernooij 2004
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+enum ndr_err_code ndr_push_BSTR(struct ndr_push *ndr, int ndr_flags, const struct BSTR *r);
+enum ndr_err_code ndr_pull_BSTR(struct ndr_pull *ndr, int ndr_flags, struct BSTR *r);
diff --git a/source3/librpc/ndr/util.c b/librpc/ndr/util.c
index 6bbe0549590..b2df28dde09 100644
--- a/source3/librpc/ndr/util.c
+++ b/librpc/ndr/util.c
@@ -1,27 +1,28 @@
-/* 
+/*
    Unix SMB/CIFS implementation.
 
    libndr interface
 
    Copyright (C) Andrew Tridgell 2003
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
 #include "../librpc/ndr/libndr.h"
-#include "librpc/ndr/util.h"
+#include "system/network.h"
+#include "lib/util/util_net.h"
 
 _PUBLIC_ void ndr_print_sockaddr_storage(struct ndr_print *ndr, const char *name, const struct sockaddr_storage *ss)
 {
diff --git a/librpc/rpc/binding_handle.c b/librpc/rpc/binding_handle.c
index 6c2d8a8b12b..1e90827dbe1 100644
--- a/librpc/rpc/binding_handle.c
+++ b/librpc/rpc/binding_handle.c
@@ -227,7 +227,7 @@ NTSTATUS dcerpc_binding_handle_raw_call(struct dcerpc_binding_handle *h,
 	}
 
 	if (!tevent_req_poll(subreq, ev)) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		talloc_free(frame);
 		return status;
 	}
@@ -540,7 +540,7 @@ NTSTATUS dcerpc_binding_handle_call(struct dcerpc_binding_handle *h,
 	}
 
 	if (!tevent_req_poll(subreq, ev)) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		talloc_free(frame);
 		return status;
 	}
diff --git a/librpc/rpc/dcerpc_util.c b/librpc/rpc/dcerpc_util.c
index a405ca8966a..de292c83669 100644
--- a/librpc/rpc/dcerpc_util.c
+++ b/librpc/rpc/dcerpc_util.c
@@ -266,7 +266,7 @@ static void dcerpc_read_ncacn_packet_done(struct tevent_req *subreq)
 	ret = tstream_readv_pdu_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		tevent_req_nterror(req, status);
 		return;
 	}
diff --git a/librpc/wscript_build b/librpc/wscript_build
index d6f0f6dd41c..a68ce477dd2 100644
--- a/librpc/wscript_build
+++ b/librpc/wscript_build
@@ -175,6 +175,13 @@ bld.SAMBA_SUBSYSTEM('NDR_SCERPC',
 	public_deps='ndr'
 	)
 
+bld.SAMBA_SUBSYSTEM('NDR_SERVER_ID',
+	source='gen_ndr/ndr_server_id.c',
+	deps='ndr',
+	public_headers='gen_ndr/server_id.h',
+	header_path='gen_ndr'
+	)
+
 bld.SAMBA_SUBSYSTEM('NDR_TRKWKS',
 	source='gen_ndr/ndr_trkwks.c',
 	public_deps='ndr'
@@ -195,6 +202,11 @@ bld.SAMBA_SUBSYSTEM('NDR_IDMAP',
 	public_deps='ndr ndr-standard'
 	)
 
+bld.SAMBA_SUBSYSTEM('NDR_NOTIFY',
+	source='gen_ndr/ndr_notify.c',
+	public_deps='ndr ndr-standard NDR_SERVER_ID NDR_FILE_ID'
+	)
+
 bld.SAMBA_SUBSYSTEM('NDR_NTLMSSP',
 	source='ndr/ndr_ntlmssp.c gen_ndr/ndr_ntlmssp.c',
 	public_deps='ndr ndr-standard'
@@ -305,7 +317,8 @@ bld.SAMBA_LIBRARY('ndr-standard',
 	pc_files='ndr_standard.pc',
 	deps='''NDR_SECURITY NDR_LSA NDR_SAMR NDR_NETLOGON NDR_EVENTLOG NDR_DFS
 	NDR_NTSVCS NDR_SVCCTL NDR_INITSHUTDOWN NDR_WKSSVC NDR_SRVSVC NDR_WINREG
-	NDR_ECHO security NDR_DNS NDR_ATSVC''',
+	NDR_ECHO security NDR_DNS NDR_ATSVC NDR_SPOOLSS NDR_DSSETUP
+        NDR_SERVER_ID NDR_NOTIFY''',
 	public_deps='ndr',
 	public_headers='gen_ndr/samr.h gen_ndr/ndr_samr.h gen_ndr/lsa.h gen_ndr/netlogon.h gen_ndr/atsvc.h gen_ndr/ndr_atsvc.h gen_ndr/ndr_svcctl.h gen_ndr/svcctl.h',
 	header_path='gen_ndr'
@@ -345,114 +358,114 @@ bld.SAMBA_SUBSYSTEM('NDR_FILE_ID',
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_XATTR',
 	source='gen_ndr/ndr_xattr_c.c',
-	public_deps='NDR_XATTR dcerpc'
+	public_deps='NDR_XATTR rpccommon'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_ROT',
 	source='gen_ndr/ndr_rot_c.c',
-	public_deps='NDR_ROT dcerpc'
+	public_deps='NDR_ROT rpccommon'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_ATSVC',
 	source='gen_ndr/ndr_atsvc_c.c',
-	public_deps='dcerpc NDR_ATSVC'
+	public_deps='rpccommon NDR_ATSVC'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_AUDIOSRV',
 	source='gen_ndr/ndr_audiosrv_c.c',
-	public_deps='NDR_AUDIOSRV dcerpc'
+	public_deps='NDR_AUDIOSRV rpccommon'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_ECHO',
 	source='gen_ndr/ndr_echo_c.c',
-	public_deps='dcerpc NDR_ECHO'
+	public_deps='rpccommon NDR_ECHO'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_DSBACKUP',
 	source='gen_ndr/ndr_dsbackup_c.c',
-	public_deps='dcerpc NDR_DSBACKUP'
+	public_deps='rpccommon NDR_DSBACKUP'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_EFS',
 	source='gen_ndr/ndr_efs_c.c',
-	public_deps='dcerpc NDR_EFS'
+	public_deps='rpccommon NDR_EFS'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_LSA',
 	source='gen_ndr/ndr_lsa_c.c',
-	public_deps='dcerpc ndr-standard'
+	public_deps='rpccommon ndr-standard'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_SAMR',
 	source='gen_ndr/ndr_samr_c.c',
-	public_deps='dcerpc NDR_SAMR'
+	public_deps='rpccommon NDR_SAMR'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_DFS',
 	source='gen_ndr/ndr_dfs_c.c',
-	public_deps='dcerpc ndr-standard'
+	public_deps='rpccommon ndr-standard'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_FRSAPI',
 	source='gen_ndr/ndr_frsapi_c.c',
-	public_deps='dcerpc NDR_FRSAPI'
+	public_deps='rpccommon NDR_FRSAPI'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_DRSUAPI',
 	source='gen_ndr/ndr_drsuapi_c.c',
-	public_deps='dcerpc NDR_DRSUAPI'
+	public_deps='rpccommon NDR_DRSUAPI'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_DRSBLOBS',
 	source='gen_ndr/ndr_drsblobs_c.c',
-	public_deps='dcerpc NDR_DRSBLOBS'
+	public_deps='rpccommon NDR_DRSBLOBS'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_POLICYAGENT',
 	source='gen_ndr/ndr_policyagent_c.c',
-	public_deps='dcerpc NDR_POLICYAGENT'
+	public_deps='rpccommon NDR_POLICYAGENT'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_UNIXINFO',
 	source='gen_ndr/ndr_unixinfo_c.c',
-	public_deps='dcerpc NDR_UNIXINFO'
+	public_deps='rpccommon NDR_UNIXINFO'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_BROWSER',
 	source='gen_ndr/ndr_browser_c.c',
-	public_deps='dcerpc NDR_BROWSER'
+	public_deps='rpccommon NDR_BROWSER'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_SPOOLSS',
 	source='gen_ndr/ndr_spoolss_c.c',
-	public_deps='dcerpc NDR_SPOOLSS'
+	public_deps='rpccommon NDR_SPOOLSS'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_NBT',
 	source='gen_ndr/ndr_nbt_c.c',
-	public_deps='dcerpc NDR_NBT'
+	public_deps='rpccommon NDR_NBT'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_WKSSVC',
 	source='gen_ndr/ndr_wkssvc_c.c',
-	public_deps='dcerpc ndr-standard'
+	public_deps='rpccommon ndr-standard'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_SRVSVC',
 	source='gen_ndr/ndr_srvsvc_c.c',
-	public_deps='dcerpc NDR_SRVSVC'
+	public_deps='rpccommon NDR_SRVSVC'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_SVCCTL',
 	source='gen_ndr/ndr_svcctl_c.c',
-	public_deps='dcerpc ndr-standard',
+	public_deps='rpccommon ndr-standard',
 	public_headers='gen_ndr/ndr_svcctl_c.h ndr/ndr_svcctl.h',
         header_path=[ ('gen_ndr*', 'gen_ndr'), ('ndr*', 'ndr')]
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_EVENTLOG',
 	source='gen_ndr/ndr_eventlog_c.c',
-	public_deps='dcerpc ndr-standard'
+	public_deps='rpccommon ndr-standard'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_EPMAPPER',
@@ -462,27 +475,27 @@ bld.SAMBA_SUBSYSTEM('RPC_NDR_EPMAPPER',
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_DBGIDL',
 	source='gen_ndr/ndr_dbgidl_c.c',
-	public_deps='dcerpc NDR_DBGIDL'
+	public_deps='rpccommon NDR_DBGIDL'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_DSSETUP',
 	source='gen_ndr/ndr_dssetup_c.c',
-	public_deps='dcerpc NDR_DSSETUP'
+	public_deps='rpccommon NDR_DSSETUP'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_MSGSVC',
 	source='gen_ndr/ndr_msgsvc_c.c',
-	public_deps='dcerpc NDR_MSGSVC'
+	public_deps='rpccommon NDR_MSGSVC'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_WINREG',
 	source='gen_ndr/ndr_winreg_c.c',
-	public_deps='dcerpc ndr-standard'
+	public_deps='rpccommon ndr-standard'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_INITSHUTDOWN',
 	source='gen_ndr/ndr_initshutdown_c.c',
-	public_deps='dcerpc ndr-standard'
+	public_deps='rpccommon ndr-standard'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_MGMT',
@@ -492,32 +505,32 @@ bld.SAMBA_SUBSYSTEM('RPC_NDR_MGMT',
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_OXIDRESOLVER',
 	source='gen_ndr/ndr_oxidresolver_c.c',
-	public_deps='dcerpc NDR_OXIDRESOLVER'
+	public_deps='rpccommon NDR_OXIDRESOLVER'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_REMACT',
 	source='gen_ndr/ndr_remact_c.c',
-	public_deps='dcerpc NDR_REMACT'
+	public_deps='rpccommon NDR_REMACT'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_WZCSVC',
 	source='gen_ndr/ndr_wzcsvc_c.c',
-	public_deps='dcerpc NDR_WZCSVC'
+	public_deps='rpccommon NDR_WZCSVC'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_W32TIME',
 	source='gen_ndr/ndr_w32time_c.c',
-	public_deps='dcerpc NDR_W32TIME'
+	public_deps='rpccommon NDR_W32TIME'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_SCERPC',
 	source='gen_ndr/ndr_scerpc_c.c',
-	public_deps='dcerpc NDR_SCERPC'
+	public_deps='rpccommon NDR_SCERPC'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_NTSVCS',
 	source='gen_ndr/ndr_ntsvcs_c.c',
-	public_deps='dcerpc ndr-standard'
+	public_deps='rpccommon ndr-standard'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_NETLOGON',
@@ -527,22 +540,22 @@ bld.SAMBA_SUBSYSTEM('RPC_NDR_NETLOGON',
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_TRKWKS',
 	source='gen_ndr/ndr_trkwks_c.c',
-	public_deps='dcerpc NDR_TRKWKS'
+	public_deps='rpccommon NDR_TRKWKS'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_KEYSVC',
 	source='gen_ndr/ndr_keysvc_c.c',
-	public_deps='dcerpc NDR_KEYSVC'
+	public_deps='rpccommon NDR_KEYSVC'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_BACKUPKEY',
 	source='gen_ndr/ndr_backupkey_c.c',
-	public_deps='dcerpc NDR_BACKUPKEY'
+	public_deps='rpccommon NDR_BACKUPKEY'
 	)
 
 bld.SAMBA_SUBSYSTEM('RPC_NDR_DNSP',
 	source='gen_ndr/ndr_dnsp_c.c',
-	public_deps='dcerpc NDR_DNSP'
+	public_deps='rpccommon NDR_DNSP'
 	)
 
 # a grouping library for NDR subsystems that may be used by more than one target
@@ -555,14 +568,17 @@ bld.SAMBA_LIBRARY('ndr-samba',
 
 # a grouping library for RPC_NDR subsystems that may be used by more than one target
 bld.SAMBA_LIBRARY('dcerpc-samba',
-	source=[],
-	deps='RPC_NDR_DRSUAPI RPC_NDR_LSA RPC_NDR_WINREG',
-	private_library=True,
-	grouping_library=True
-	)
+                  source='',
+                  deps='''RPC_NDR_LSA RPC_NDR_SAMR RPC_NDR_NETLOGON RPC_NDR_EVENTLOG RPC_NDR_DFS
+	          RPC_NDR_NTSVCS RPC_NDR_SVCCTL RPC_NDR_INITSHUTDOWN RPC_NDR_WKSSVC RPC_NDR_SRVSVC RPC_NDR_WINREG
+	          RPC_NDR_ECHO RPC_NDR_EPMAPPER RPC_NDR_ATSVC RPC_NDR_SPOOLSS''',
+                  public_deps='ndr-standard',
+                  private_library=True,
+                  grouping_library=True
+                  )
 
 bld.SAMBA_LIBRARY('ndr',
-	source='ndr/ndr_string.c ndr/ndr_basic.c ndr/uuid.c ndr/ndr.c ndr/ndr_misc.c gen_ndr/ndr_misc.c',
+	source='ndr/ndr_string.c ndr/ndr_basic.c ndr/uuid.c ndr/ndr.c ndr/ndr_misc.c gen_ndr/ndr_misc.c ndr/util.c',
 	pc_files='ndr.pc',
 	public_deps='errors talloc samba-util',
 	public_headers='gen_ndr/misc.h gen_ndr/ndr_misc.h ndr/libndr.h:ndr.h',
diff --git a/nsswitch/libwbclient/tests/wbclient.c b/nsswitch/libwbclient/tests/wbclient.c
index c6ee5314817..12fc31ea5e3 100644
--- a/nsswitch/libwbclient/tests/wbclient.c
+++ b/nsswitch/libwbclient/tests/wbclient.c
@@ -395,7 +395,7 @@ static bool test_wbc_lookup_rids(struct torture_context *tctx)
 	torture_assert_str_equal(
 		tctx, names[1], "Users", "S-1-5-32-545 not mapped to 'Users'");
 
-	wbcFreeMemory((char *)domain_name);
+	wbcFreeMemory(discard_const_p(char ,domain_name));
 	wbcFreeMemory(names);
 	wbcFreeMemory(types);
 
diff --git a/nsswitch/libwbclient/wbc_util.c b/nsswitch/libwbclient/wbc_util.c
index d783ba36d87..af134ba7e5b 100644
--- a/nsswitch/libwbclient/wbc_util.c
+++ b/nsswitch/libwbclient/wbc_util.c
@@ -623,13 +623,13 @@ static void wbcDomainControllerInfoExDestructor(void *ptr)
 {
 	struct wbcDomainControllerInfoEx *i =
 		(struct wbcDomainControllerInfoEx *)ptr;
-	free((char *)(i->dc_unc));
-	free((char *)(i->dc_address));
-	free((char *)(i->domain_guid));
-	free((char *)(i->domain_name));
-	free((char *)(i->forest_name));
-	free((char *)(i->dc_site_name));
-	free((char *)(i->client_site_name));
+	free(discard_const_p(char, i->dc_unc));
+	free(discard_const_p(char, i->dc_address));
+	free(discard_const_p(char, i->domain_guid));
+	free(discard_const_p(char, i->domain_name));
+	free(discard_const_p(char, i->forest_name));
+	free(discard_const_p(char, i->dc_site_name));
+	free(discard_const_p(char, i->client_site_name));
 }
 
 static wbcErr wbc_create_domain_controller_info_ex(const struct winbindd_response *resp,
@@ -758,7 +758,7 @@ static void wbcNamedBlobDestructor(void *ptr)
 	struct wbcNamedBlob *b = (struct wbcNamedBlob *)ptr;
 
 	while (b->name != NULL) {
-		free((char *)(b->name));
+		free(discard_const_p(char, b->name));
 		free(b->blob.data);
 		b += 1;
 	}
diff --git a/nsswitch/pam_winbind.c b/nsswitch/pam_winbind.c
index 7005c25fae4..e59a6d8ea16 100644
--- a/nsswitch/pam_winbind.c
+++ b/nsswitch/pam_winbind.c
@@ -11,8 +11,6 @@
 */
 
 #include "pam_winbind.h"
-#define CONST_DISCARD(type,ptr) ((type)(void *)ptr)
-
 
 static int wbc_error_to_pam_error(wbcErr status)
 {
@@ -412,51 +410,51 @@ static int _pam_parse(const pam_handle_t *pamh,
 		config_file = PAM_WINBIND_CONFIG_FILE;
 	}
 
-	d = iniparser_load(CONST_DISCARD(char *, config_file));
+	d = iniparser_load(discard_const_p(char, config_file));
 	if (d == NULL) {
 		goto config_from_pam;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:debug"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:debug"), false)) {
 		ctrl |= WINBIND_DEBUG_ARG;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:debug_state"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:debug_state"), false)) {
 		ctrl |= WINBIND_DEBUG_STATE;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:cached_login"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:cached_login"), false)) {
 		ctrl |= WINBIND_CACHED_LOGIN;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:krb5_auth"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:krb5_auth"), false)) {
 		ctrl |= WINBIND_KRB5_AUTH;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:silent"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:silent"), false)) {
 		ctrl |= WINBIND_SILENT;
 	}
 
-	if (iniparser_getstr(d, CONST_DISCARD(char *, "global:krb5_ccache_type")) != NULL) {
+	if (iniparser_getstr(d, discard_const_p(char, "global:krb5_ccache_type")) != NULL) {
 		ctrl |= WINBIND_KRB5_CCACHE_TYPE;
 	}
 
-	if ((iniparser_getstr(d, CONST_DISCARD(char *, "global:require-membership-of"))
+	if ((iniparser_getstr(d, discard_const_p(char, "global:require-membership-of"))
 	     != NULL) ||
-	    (iniparser_getstr(d, CONST_DISCARD(char *, "global:require_membership_of"))
+	    (iniparser_getstr(d, discard_const_p(char, "global:require_membership_of"))
 	     != NULL)) {
 		ctrl |= WINBIND_REQUIRED_MEMBERSHIP;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:try_first_pass"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:try_first_pass"), false)) {
 		ctrl |= WINBIND_TRY_FIRST_PASS_ARG;
 	}
 
-	if (iniparser_getint(d, CONST_DISCARD(char *, "global:warn_pwd_expire"), 0)) {
+	if (iniparser_getint(d, discard_const_p(char, "global:warn_pwd_expire"), 0)) {
 		ctrl |= WINBIND_WARN_PWD_EXPIRE;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:mkhomedir"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:mkhomedir"), false)) {
 		ctrl |= WINBIND_MKHOMEDIR;
 	}
 
@@ -538,7 +536,7 @@ static int _pam_winbind_init_context(pam_handle_t *pamh,
 	textdomain_init();
 #endif
 
-	r = TALLOC_ZERO_P(NULL, struct pwb_context);
+	r = talloc_zero(NULL, struct pwb_context);
 	if (!r) {
 		return PAM_BUF_ERR;
 	}
@@ -1214,7 +1212,7 @@ out:
 static void _pam_setup_krb5_env(struct pwb_context *ctx,
 				struct wbcLogonUserInfo *info)
 {
-	char var[PATH_MAX];
+	char *var = NULL;
 	int ret;
 	uint32_t i;
 	const char *krb5ccname = NULL;
@@ -1241,7 +1239,7 @@ static void _pam_setup_krb5_env(struct pwb_context *ctx,
 	_pam_log_debug(ctx, LOG_DEBUG,
 		       "request returned KRB5CCNAME: %s", krb5ccname);
 
-	if (snprintf(var, sizeof(var), "KRB5CCNAME=%s", krb5ccname) == -1) {
+	if (asprintf(&var, "KRB5CCNAME=%s", krb5ccname) == -1) {
 		return;
 	}
 
@@ -1251,6 +1249,7 @@ static void _pam_setup_krb5_env(struct pwb_context *ctx,
 			 "failed to set KRB5CCNAME to %s: %s",
 			 var, pam_strerror(ctx->pamh, ret));
 	}
+	free(var);
 }
 
 /**
@@ -1764,7 +1763,7 @@ static int winbind_auth_request(struct pwb_context *ctx,
 					     &logon.blobs,
 					     "krb5_cc_type",
 					     0,
-					     (uint8_t *)cctype,
+					     discard_const_p(uint8_t, cctype),
 					     strlen(cctype)+1);
 		if (!WBC_ERROR_IS_OK(wbc_status)) {
 			goto done;
@@ -2514,7 +2513,7 @@ static int _pam_delete_cred(pam_handle_t *pamh, int flags,
 						     &logoff.blobs,
 						     "ccfilename",
 						     0,
-						     (uint8_t *)ccname,
+						     discard_const_p(uint8_t, ccname),
 						     strlen(ccname)+1);
 			if (!WBC_ERROR_IS_OK(wbc_status)) {
 				goto out;
diff --git a/nsswitch/pam_winbind.h b/nsswitch/pam_winbind.h
index ed924794c48..0d9529e726a 100644
--- a/nsswitch/pam_winbind.h
+++ b/nsswitch/pam_winbind.h
@@ -1,9 +1,55 @@
+/*
+ * Copyright (c) Andrew Tridgell  <tridge@samba.org>   2000
+ * Copyright (c) Tim Potter       <tpot@samba.org>     2000
+ * Copyright (c) Andrew Bartlettt <abartlet@samba.org> 2002
+ * Copyright (c) Guenther Deschner <gd@samba.org>      2005-2008
+ * Copyright (c) Jan Rêkorajski 1999.
+ * Copyright (c) Andrew G. Morgan 1996-8.
+ * Copyright (c) Alex O. Yuriev, 1996.
+ * Copyright (c) Cristian Gafton 1996.
+ * Copyright (C) Elliot Lee <sopwith@redhat.com> 1996, Red Hat Software.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, and the entire permission notice in its entirety,
+ *    including the disclaimer of warranties.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote
+ *    products derived from this software without specific prior
+ *    written permission.
+ *
+ * ALTERNATIVELY, this product may be distributed under the terms of
+ * the GNU Public License, in which case the provisions of the GPL are
+ * required INSTEAD OF the above restrictions.  (This clause is
+ * necessary due to a potential bad interaction between the GPL and
+ * the restrictions contained in a BSD-style copyright.)
+ *
+ * THIS SOFTWARE IS PROVIDED `AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
 /* pam_winbind header file
    (Solaris needs some macros from Linux for common PAM code)
 
    Shirish Kalele 2000
 */
 
+#ifndef _NSSWITCH_PAM_WINBIND_H_
+#define _NSSWITCH_PAM_WINBIND_H_
+
 #include "../lib/replace/replace.h"
 #include "system/syslog.h"
 #include "system/time.h"
@@ -175,3 +221,5 @@ struct pwb_context {
 #endif
 #define TALLOC_ZERO_P(ctx, type) (type *)_talloc_zero(ctx, sizeof(type), #type)
 #define TALLOC_P(ctx, type) (type *)talloc_named_const(ctx, sizeof(type), #type)
+
+#endif /* _NSSWITCH_PAM_WINBIND_H_ */
diff --git a/nsswitch/wb_common.c b/nsswitch/wb_common.c
index dcfc8a51568..80c729a3b6b 100644
--- a/nsswitch/wb_common.c
+++ b/nsswitch/wb_common.c
@@ -369,13 +369,14 @@ static int winbind_open_pipe_sock(int recursing, int need_priv)
 static int winbind_write_sock(void *buffer, int count, int recursing,
 			      int need_priv)
 {
-	int result, nwritten;
+	int fd, result, nwritten;
 
 	/* Open connection to winbind daemon */
 
  restart:
 
-	if (winbind_open_pipe_sock(recursing, need_priv) == -1) {
+	fd = winbind_open_pipe_sock(recursing, need_priv);
+	if (fd == -1) {
 		errno = ENOENT;
 		return -1;
 	}
@@ -391,7 +392,7 @@ static int winbind_write_sock(void *buffer, int count, int recursing,
 		/* Catch pipe close on other end by checking if a read()
 		   call would not block by calling poll(). */
 
-		pfd.fd = winbindd_fd;
+		pfd.fd = fd;
 		pfd.events = POLLIN|POLLHUP;
 
 		ret = poll(&pfd, 1, 0);
@@ -412,8 +413,7 @@ static int winbind_write_sock(void *buffer, int count, int recursing,
 
 		/* Do the write */
 
-		result = write(winbindd_fd,
-			       (char *)buffer + nwritten,
+		result = write(fd, (char *)buffer + nwritten,
 			       count - nwritten);
 
 		if ((result == -1) || (result == 0)) {
@@ -434,10 +434,12 @@ static int winbind_write_sock(void *buffer, int count, int recursing,
 
 static int winbind_read_sock(void *buffer, int count)
 {
+	int fd;
 	int nread = 0;
 	int total_time = 0;
 
-	if (winbindd_fd == -1) {
+	fd = winbind_open_pipe_sock(false, false);
+	if (fd == -1) {
 		return -1;
 	}
 
@@ -449,7 +451,7 @@ static int winbind_read_sock(void *buffer, int count)
 		/* Catch pipe close on other end by checking if a read()
 		   call would not block by calling poll(). */
 
-		pfd.fd = winbindd_fd;
+		pfd.fd = fd;
 		pfd.events = POLLIN|POLLHUP;
 
 		/* Wait for 5 seconds for a reply. May need to parameterise this... */
@@ -475,7 +477,7 @@ static int winbind_read_sock(void *buffer, int count)
 
 			/* Do the Read */
 
-			int result = read(winbindd_fd, (char *)buffer + nread,
+			int result = read(fd, (char *)buffer + nread,
 			      count - nread);
 
 			if ((result == -1) || (result == 0)) {
diff --git a/nsswitch/wbinfo.c b/nsswitch/wbinfo.c
index ac071750228..30e23b6a8fb 100644
--- a/nsswitch/wbinfo.c
+++ b/nsswitch/wbinfo.c
@@ -134,7 +134,6 @@ static bool parse_wbinfo_domain_user(const char *domuser, fstring domain,
 	fstrcpy(user, p+1);
 	fstrcpy(domain, domuser);
 	domain[PTR_DIFF(p, domuser)] = 0;
-	strupper_m(domain);
 
 	return true;
 }
diff --git a/nsswitch/winbind_client.h b/nsswitch/winbind_client.h
index ed1c81ddb2b..905a189c820 100644
--- a/nsswitch/winbind_client.h
+++ b/nsswitch/winbind_client.h
@@ -1,3 +1,30 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   winbind client common code
+
+   Copyright (C) Tim Potter 2000
+   Copyright (C) Andrew Tridgell 2000
+   Copyright (C) Andrew Bartlett 2002
+
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _NSSWITCH_WINBIND_CLIENT_H_
+#define _NSSWITCH_WINBIND_CLIENT_H_
+
 #include "winbind_nss_config.h"
 #include "winbind_struct_protocol.h"
 
@@ -19,3 +46,5 @@ NSS_STATUS winbindd_priv_request_response(int req_type,
 
 #define winbind_on() \
 	(setenv(WINBINDD_DONT_ENV, "0", 1) == 0)
+
+#endif /* _NSSWITCH_WINBIND_CLIENT_H_ */
diff --git a/nsswitch/winbind_nss_config.h b/nsswitch/winbind_nss_config.h
index 3e2ce68252a..e1ad3f64635 100644
--- a/nsswitch/winbind_nss_config.h
+++ b/nsswitch/winbind_nss_config.h
@@ -54,7 +54,13 @@
 #ifndef FSTRING_LEN
 #define FSTRING_LEN 256
 typedef char fstring[FSTRING_LEN];
-#define fstrcpy(d,s) safe_strcpy((d),(s),sizeof(fstring)-1)
+#ifndef fstrcpy
+#define fstrcpy(d,s) \
+do { \
+        const char *_fstrcpy_src = (const char *)(s); \
+        strlcpy((d),_fstrcpy_src ? _fstrcpy_src : "",sizeof(fstring)); \
+} while (0)
+#endif
 #endif
 
 /* Some systems (SCO) treat UNIX domain sockets as FIFOs */
diff --git a/nsswitch/wins.c b/nsswitch/wins.c
index e0aa2ad8f55..d63968b2bc5 100644
--- a/nsswitch/wins.c
+++ b/nsswitch/wins.c
@@ -59,8 +59,10 @@ static void nss_wins_init(void)
 
 static struct in_addr *lookup_byname_backend(const char *name, int *count)
 {
-	struct ip_service *address = NULL;
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct sockaddr_storage *address = NULL;
 	struct in_addr *ret = NULL;
+	NTSTATUS status;
 	int j;
 
 	if (!initialised) {
@@ -70,19 +72,21 @@ static struct in_addr *lookup_byname_backend(const char *name, int *count)
 	*count = 0;
 
 	/* always try with wins first */
-	if (NT_STATUS_IS_OK(resolve_wins(name,0x00,&address,count))) {
+	status = resolve_wins(name, 0x00, talloc_tos(),
+			      &address, count);
+	if (NT_STATUS_IS_OK(status)) {
 		if ( (ret = SMB_MALLOC_P(struct in_addr)) == NULL ) {
-			free( address );
+			TALLOC_FREE(frame);
 			return NULL;
 		}
-		if (address[0].ss.ss_family != AF_INET) {
-			free(address);
+		if (address[0].ss_family != AF_INET) {
 			free(ret);
+			TALLOC_FREE(frame);
 			return NULL;
 		}
-		*ret = ((struct sockaddr_in *)(void *)&address[0].ss)
+		*ret = ((struct sockaddr_in *)(void *)address)
 			->sin_addr;
-		free( address );
+		TALLOC_FREE(frame);
 		return ret;
 	}
 
@@ -91,24 +95,23 @@ static struct in_addr *lookup_byname_backend(const char *name, int *count)
 		const struct in_addr *bcast = iface_n_bcast_v4(j);
 		struct sockaddr_storage ss;
 		struct sockaddr_storage *pss;
-		NTSTATUS status;
 
 		if (!bcast) {
 			continue;
 		}
 		in_addr_to_sockaddr_storage(&ss, *bcast);
 		status = name_query(name, 0x00, True, True, &ss,
-				    NULL, &pss, count, NULL);
-		if (pss) {
+				    talloc_tos(), &pss, count, NULL);
+		if (NT_STATUS_IS_OK(status) && (*count > 0)) {
 			if ((ret = SMB_MALLOC_P(struct in_addr)) == NULL) {
+				TALLOC_FREE(frame);
 				return NULL;
 			}
 			*ret = ((struct sockaddr_in *)pss)->sin_addr;
-			TALLOC_FREE(pss);
 			break;
 		}
 	}
-
+	TALLOC_FREE(frame);
 	return ret;
 }
 
@@ -180,7 +183,7 @@ int lookup(nsd_file_t *rq)
 	 * response needs to be a string of the following format
 	 * ip_address[ ip_address]*\tname[ alias]*
 	 */
-	if (StrCaseCmp(map,"hosts.byaddr") == 0) {
+	if (strcasecmp_m(map,"hosts.byaddr") == 0) {
 		if ( status = lookup_byaddr_backend(key, &count)) {
 		    size = strlen(key) + 1;
 		    if (size > len) {
@@ -208,7 +211,7 @@ int lookup(nsd_file_t *rq)
 		    response[strlen(response)-1] = '\n';
 		    talloc_free(status);
 		}
-	} else if (StrCaseCmp(map,"hosts.byname") == 0) {
+	} else if (strcasecmp_m(map,"hosts.byname") == 0) {
 	    if (ip_list = lookup_byname_backend(key, &count)) {
 		for (i = count; i ; i--) {
 		    addr = inet_ntoa(ip_list[i-1]);
diff --git a/nsswitch/wscript_build b/nsswitch/wscript_build
index 83b10a7969f..47f6fbcc23c 100644
--- a/nsswitch/wscript_build
+++ b/nsswitch/wscript_build
@@ -21,7 +21,7 @@ bld.SAMBA_LIBRARY('nss_winbind',
 	vnum='2')
 
 
-if bld.CONFIG_SET('WITH_PAM_MODULES') or bld.CONFIG_SET('HAVE_PAM_START'):
+if bld.CONFIG_SET('WITH_PAM_MODULES') and bld.CONFIG_SET('HAVE_PAM_START'):
 	bld.SAMBA_LIBRARY('pamwinbind',
 		source='pam_winbind.c',
 		deps='intl talloc wbclient winbind-client LIBINIPARSER pam',
diff --git a/packaging/RHEL-CTDB/configure.rpm b/packaging/RHEL-CTDB/configure.rpm
index 616336ea3a3..fafad81fc6e 100755
--- a/packaging/RHEL-CTDB/configure.rpm
+++ b/packaging/RHEL-CTDB/configure.rpm
@@ -51,7 +51,7 @@ CC="$CC" CFLAGS="-Wall -g -D_GNU_SOURCE -O3" ./configure -C \
 	--without-smbwrapper \
 	--with-pam \
 	--with-quotas \
-	--with-shared-modules=idmap_rid,idmap_ad,idmap_tdb2,vfs_gpfs \
+	--with-shared-modules=idmap_rid,idmap_ad,idmap_tdb2,vfs_gpfs,vfs_tsmsm,vfs_gpfs_hsm_notify \
 	--with-syslog \
 	--with-utmp \
 	--with-cluster-support \
@@ -60,6 +60,9 @@ CC="$CC" CFLAGS="-Wall -g -D_GNU_SOURCE -O3" ./configure -C \
 	--without-dnsupdate \
 	--with-aio-support \
 	--disable-merged-build \
+	--disable-smbtorture4 \
+	--disable-external-libtalloc \
+	--disable-external-libtdb \
 	$*
 
 make showlayout
diff --git a/packaging/RHEL-CTDB/makerpms.sh b/packaging/RHEL-CTDB/makerpms.sh
index c1a4b70151f..d23d14ed8ab 100755
--- a/packaging/RHEL-CTDB/makerpms.sh
+++ b/packaging/RHEL-CTDB/makerpms.sh
@@ -42,6 +42,14 @@ case $RPMVER in
        ;;
 esac
 
+mkdir -p `rpm --eval %_specdir`
+mkdir -p `rpm --eval %_sourcedir`
+mkdir -p `rpm --eval %_builddir`
+mkdir -p `rpm --eval %_srcrpmdir`
+mkdir -p `rpm --eval %_rpmdir`/noarch
+mkdir -p `rpm --eval %_rpmdir`/i386
+mkdir -p `rpm --eval %_rpmdir`/x86_64
+
 ##
 ## Delete the old debuginfo remnants:
 ##
@@ -131,8 +139,17 @@ fi
 ##
 echo "$(basename $0): Getting Ready to build release package"
 
+case ${EXTRA_OPTIONS} in
+	*-b*)
+		BUILD_TARGET=""
+		;;
+	*)
+		BUILD_TARGET="-ba"
+		;;
+esac
+
 pushd ${RPMSPECDIR}
-${RPM} -ba $EXTRA_OPTIONS $SPECFILE
+${RPM} ${BUILD_TARGET} ${EXTRA_OPTIONS} ${SPECFILE}
 popd
 
 echo "$(basename $0): Done."
diff --git a/packaging/RHEL-CTDB/makespec.sh b/packaging/RHEL-CTDB/makespec.sh
index 29dcbeb30e4..946ce8ebf98 100755
--- a/packaging/RHEL-CTDB/makespec.sh
+++ b/packaging/RHEL-CTDB/makespec.sh
@@ -45,8 +45,21 @@ else
 	echo "GITHASH: ${GITHASH}"
 fi
 
-sed -e s/PVERSION/${VERSION}/g \
+
+#
+# get the versions of libtdb and libtalloc we provide
+#
+#LIBTDBVERSION=1.2.9
+LIBTDBVERSION=$(grep ^VERSION ${DIRNAME}/../../lib/tdb/wscript | sed -e "s/'//g" -e 's/.* //')
+
+#LIBTALLOCVERSION=2.0.1
+LIBTALLOCVERSION=$(grep ^VERSION ${DIRNAME}/../../lib/talloc/wscript | sed -e "s/'//g" -e 's/.* //')
+
+sed \
+	-e s/PVERSION/${VERSION}/g \
 	-e s/GITHASH/${GITHASH}/g \
+	-e s/LIBTDBVERSION/${LIBTDBVERSION}/g \
+	-e s/LIBTALLOCVERSION/${LIBTALLOCVERSION}/g \
 	< ${SPECFILE}.tmpl \
 	> ${SPECFILE}
 
diff --git a/packaging/RHEL-CTDB/samba.spec.tmpl b/packaging/RHEL-CTDB/samba.spec.tmpl
index 5e808407158..fae9ffe4526 100644
--- a/packaging/RHEL-CTDB/samba.spec.tmpl
+++ b/packaging/RHEL-CTDB/samba.spec.tmpl
@@ -77,6 +77,8 @@ shares and printing to SMB printers.
 Summary: Files used by both Samba servers and clients.
 Group: Applications/System
 Provides: samba-common = %{version}-%{release}
+Requires: libtalloc >= 2.0.1
+Requires: libtdb >= 1.2.6
 
 %description common
 Samba-common provides files necessary for both the server and client
@@ -84,6 +86,28 @@ packages of Samba.
 
 
 #######################################################################
+%package libtdb
+Summary: the tdb library
+Group: Applications/System
+Provides: libtdb = LIBTDBVERSION-%{release}
+Obsoletes: libtdb
+#Conflicts: libtdb < LIBTALLOCVERSION
+
+%description libtdb
+Samba's tdb library.
+
+#######################################################################
+%package libtalloc
+Summary: the talloc library
+Group: Applications/System
+Provides: libtalloc = LIBTALLOCVERSION-%{release}
+Obsoletes: libtalloc
+#Conflicts: libtalloc < LIBTALLOCVERSION
+
+%description libtalloc
+Samba's talloc library
+
+#######################################################################
 %package swat
 Summary: The Samba SMB server configuration program.
 Group: Applications/System
@@ -95,15 +119,6 @@ The samba-swat package includes the new SWAT (Samba Web Administration
 Tool), for remotely managing Samba's smb.conf file using your favorite
 Web browser.
 
-%ifarch x86_64 ppc64
-%package winbind-32bit
-Summary:        Samba winbind compatibility package for 32bit apps on 64bit archs
-Group:          Applications/System
-
-%description winbind-32bit
-Compatibility package for 32 bit apps on 64 bit architecures
-%endif
-
 
 #######################################################################
 %package doc
@@ -146,81 +161,6 @@ export CC
 ./autogen.sh
 
 
-##
-## build the files for the winbind-32bit compat package
-## and copy them to a safe location
-##
-%ifarch x86_64 ppc64
-
-# a directory to store the 32bit compatibility modules for later install
-%define _32bit_tmp_dir %{_tmppath}/%{name}-%{version}-32bit
-
-CC_SAVE="$CC"
-CC="$CC -m32"
-
-CFLAGS="$RPM_OPT_FLAGS -O3 -D_GNU_SOURCE -m32" ./configure \
-	--prefix=%{_prefix} \
-	--localstatedir=/var \
-        --with-configdir=%{_sysconfdir}/samba \
-        --with-libdir=/usr/lib/samba \
-	--with-pammodulesdir=/lib/security \
-        --with-lockdir=/var/lib/samba \
-        --with-logfilebase=/var/log/samba \
-        --with-mandir=%{_mandir} \
-        --with-piddir=/var/run \
-	--with-privatedir=%{_sysconfdir}/samba \
-	--disable-cups \
-        --with-acl-support \
-	--with-ads \
-        --with-automount \
-        --with-fhs \
-	--with-pam_smbpass \
-	--with-libsmbclient \
-	--with-libsmbsharemodes \
-        --without-smbwrapper \
-	--with-pam \
-	--with-quotas \
-	--with-syslog \
-	--with-utmp \
-	--with-cluster-support \
-	--with-ctdb=/usr/include \
-	--without-ldb \
-	--without-dnsupdate \
-	--with-aio-support \
-	--disable-merged-build
-
-make showlayout
-
-make samba3-idl
-
-## check for gcc 3.4 or later
-CC_VERSION=`${CC} --version | head -1 | awk '{print $3}'`
-CC_MAJOR=`echo ${CC_VERSION} | cut -d. -f 1`
-CC_MINOR=`echo ${CC_VERSION} | cut -d. -f 2`
-if [ ${CC_MAJOR} -ge 3 ]; then
-        if [ ${CC_MAJOR} -gt 3 -o ${CC_MINOR} -ge 4 ]; then
-                make pch
-        fi
-fi
-
-make -j%{numcpu} %{?_smp_mflags} \
-	nss_modules pam_modules
-
-rm -rf %{_32bit_tmp_dir}
-mkdir %{_32bit_tmp_dir}
-
-mv ../nsswitch/libnss_winbind.so %{_32bit_tmp_dir}/
-mv bin/pam_winbind.so %{_32bit_tmp_dir}/
-mv bin/libtalloc.so* %{_32bit_tmp_dir}/
-mv bin/libtdb.so* %{_32bit_tmp_dir}/
-mv bin/libwbclient.so* %{_32bit_tmp_dir}/
-
-make clean
-
-CC="$CC_SAVE"
-
-%endif
-
 CFLAGS="$RPM_OPT_FLAGS $EXTRA -D_GNU_SOURCE" ./configure \
 	--prefix=%{_prefix} \
 	--localstatedir=/var \
@@ -246,7 +186,7 @@ CFLAGS="$RPM_OPT_FLAGS $EXTRA -D_GNU_SOURCE" ./configure \
         --without-smbwrapper \
 	--with-pam \
 	--with-quotas \
-	--with-shared-modules=idmap_rid,idmap_ad,idmap_tdb2,vfs_gpfs,vfs_tsmsm \
+	--with-shared-modules=idmap_rid,idmap_ad,idmap_tdb2,vfs_gpfs,vfs_tsmsm,vfs_gpfs_hsm_notify \
 	--with-syslog \
 	--with-utmp \
 	--with-cluster-support \
@@ -254,7 +194,10 @@ CFLAGS="$RPM_OPT_FLAGS $EXTRA -D_GNU_SOURCE" ./configure \
 	--without-ldb \
 	--without-dnsupdate \
 	--with-aio-support\
-	--disable-merged-build
+	--disable-merged-build \
+	--disable-smbtorture4 \
+	--disable-external-libtalloc \
+	--disable-external-libtdb
 
 make showlayout
 
@@ -335,24 +278,15 @@ install -m 755 nsswitch/libnss_winbind.so $RPM_BUILD_ROOT/%{_libarch}/libnss_win
 
 cp -p source3/bin/winbind_krb5_locator.so ${RPM_BUILD_ROOT}/%{_libarchdir}/krb5/plugins/libkrb5
 
-# install files for winbind-32bit package
-%ifarch x86_64 ppc64
-
-install -m 755 %{_32bit_tmp_dir}/libnss_winbind.so ${RPM_BUILD_ROOT}/lib/libnss_winbind.so.2
-( cd ${RPM_BUILD_ROOT}/lib; ln -sf libnss_winbind.so.2  libnss_winbind.so )
-
-mv %{_32bit_tmp_dir}/libtalloc* ${RPM_BUILD_ROOT}/usr/lib
-mv %{_32bit_tmp_dir}/libtdb* ${RPM_BUILD_ROOT}/usr/lib
-mv %{_32bit_tmp_dir}/libwbclient* ${RPM_BUILD_ROOT}/usr/lib
-mv %{_32bit_tmp_dir}/pam_winbind.so ${RPM_BUILD_ROOT}/lib/security
-
-rm -rf %{_32bit_tmp_dir}
-
-%endif
 
 ## cleanup
 /bin/rm -rf $RPM_BUILD_ROOT/usr/lib*/samba/security
 
+# remove installed but unpackaged files:
+/bin/rm -f $RPM_BUILD_ROOT/usr/lib*/libtalloc.so
+/bin/rm -f $RPM_BUILD_ROOT/usr/lib*/libtdb.so
+
+
 # Install the miscellany
 echo 127.0.0.1 localhost > $RPM_BUILD_ROOT%{_sysconfdir}/samba/lmhosts
 
@@ -482,6 +416,7 @@ exit 0
 %{_libarchdir}/samba/vfs/fileid.so
 %{_libarchdir}/samba/vfs/full_audit.so
 %{_libarchdir}/samba/vfs/gpfs.so
+%{_libarchdir}/samba/vfs/gpfs_hsm_notify.so
 %{_libarchdir}/samba/vfs/linux_xfs_sgid.so
 %{_libarchdir}/samba/vfs/netatalk.so
 %{_libarchdir}/samba/vfs/preopen.so
@@ -579,6 +514,14 @@ exit 0
 
 ##########
 
+%files libtalloc
+%{_libarchdir}/libtalloc.so.LIBTALLOCVERSION
+%{_libarchdir}/libtalloc.so.2
+
+%files libtdb
+%{_libarchdir}/libtdb.so.LIBTDBVERSION
+%{_libarchdir}/libtdb.so.1
+
 %files common
 %defattr(-,root,root)
 %dir %{_sysconfdir}/samba
@@ -620,10 +563,6 @@ exit 0
 %{_includedir}/tdb.h
 %{_libarchdir}/libnetapi.so
 %{_libarchdir}/libnetapi.so.0
-%{_libarchdir}/libtalloc.so
-%{_libarchdir}/libtalloc.so.2
-%{_libarchdir}/libtdb.so
-%{_libarchdir}/libtdb.so.1
 %{_libarchdir}/libwbclient.so
 %{_libarchdir}/libwbclient.so.0
 
@@ -664,19 +603,6 @@ exit 0
 %{_mandir}/man7/winbind_krb5_locator.7*
 %{_mandir}/man8/idmap_*.8*
 
-%ifarch x86_64 ppc64
-%files winbind-32bit
-%attr(755,root,root) /lib/libnss_winbind.so
-%attr(755,root,root) /lib/libnss_winbind.so.2
-%attr(755,root,root) /usr/lib/libtalloc.so
-%attr(755,root,root) /usr/lib/libtalloc.so.2
-%attr(755,root,root) /usr/lib/libtdb.so
-%attr(755,root,root) /usr/lib/libtdb.so.1
-%attr(755,root,root) /usr/lib/libwbclient.so
-%attr(755,root,root) /usr/lib/libwbclient.so.0
-%attr(755,root,root) /lib/security/pam_winbind.so
-%endif
-
 
 
 %changelog
diff --git a/script/autobuild.py b/script/autobuild.py
index 47751435e38..545e47ed489 100755
--- a/script/autobuild.py
+++ b/script/autobuild.py
@@ -14,8 +14,6 @@ samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/g
 
 cleanup_list = []
 
-os.environ['CC'] = "ccache gcc"
-
 builddirs = {
     "samba3"  : "source3",
     "samba3-waf": "source3",
@@ -50,7 +48,7 @@ tasks = {
                  ("clean", "make clean", "text/plain") ],
 
     # We have 'test' before 'install' because, 'test' should work without 'install'
-    "samba4" : [ ("configure", "./configure.developer ${PREFIX}", "text/plain"),
+    "samba4" : [ ("configure", "./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab", "text/plain"),
                  ("make", "make -j", "text/plain"),
                  ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
                  ("install", "make install", "text/plain"),
diff --git a/selftest/README b/selftest/README
index 752b07bf640..206f864ab9d 100644
--- a/selftest/README
+++ b/selftest/README
@@ -15,7 +15,7 @@ format-subunit, which is used by default as part of "make test".
 Available testsuites
 ====================
 The available testsuites are obtained from a script, usually 
-source{3,4}/selftest/tests.sh. This script should for each testsuite output 
+source{3,4}/selftest/tests.py. This script should for each testsuite output
 the name of the test, the command to run and the environment that should be 
 provided. Use the included "plantest" function to generate the required output.
 
@@ -77,10 +77,10 @@ Tests often need to run against a server with particular things set up,
 a "environment". This environment is provided by the test "target": Samba 3, 
 Samba 4 or Windows.
 
-The following environments are currently available:
+The environments are currently available include
 
  - none: No server set up, no variables set.
- - dc: Domain controller set up. The following environment variables will 
+ - dc,s3dc: Domain controller set up. The following environment variables will
    be set:
 
      * USERNAME: Administrator user name
@@ -92,7 +92,7 @@ The following environments are currently available:
 	 * NETBIOSNAME: DC NetBIOS name
 	 * NETIOSALIAS: DC NetBIOS alias
 
- - member: Domain controller and member server that is joined to it set up. The
+ - member,s4member,s3member: Domain controller and member server that is joined to it set up. The
    following environment variables will be set:
 
      * USERNAME: Domain administrator user name
@@ -101,6 +101,7 @@ The following environments are currently available:
 	 * REALM: Realm name
 	 * SERVER: Name of the member server
 
+See Samba.pm, Samba3.pm and Samba4.pm for the full list.
 
 Running tests
 =============
@@ -109,7 +110,7 @@ To run all the tests use::
 
    make test
 
-To run a quick subset (aiming for about 1 minute of testing) run::
+To run a quicker subset run::
 
    make quicktest
 
diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index 9f937f16556..23434fc86ce 100755
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -399,8 +399,14 @@ $prefix =~ s+/$++;
 
 die("using an empty prefix isn't allowed") unless $prefix ne "";
 
-#Ensure we have the test prefix around
-mkdir($prefix, 0777) unless -d $prefix;
+# Ensure we have the test prefix around.
+#
+# We need restrictive
+# permissions on this as some subdirectories in this tree will have
+# wider permissions (ie 0777) and this would allow other users on the
+# host to subvert the test process.
+mkdir($prefix, 0700) unless -d $prefix;
+chmod 0700, $prefix;
 
 my $prefix_abs = abs_path($prefix);
 my $tmpdir_abs = abs_path("$prefix/tmp");
@@ -468,19 +474,6 @@ if ($opt_binary_mapping) {
 
 $ENV{BINARY_MAPPING} = $opt_binary_mapping;
 
-sub bindir_path($$) {
-	my ($self, $path) = @_;
-
-	if (defined($self->{binary_mapping}->{$path})) {
-	    $path = $self->{binary_mapping}->{$path};
-	}
-
-	my $valpath = "$self->{bindir}/$path$self->{exeext}";
-
-	return $valpath if (-f $valpath);
-	return $path;
-}
-
 # After this many seconds, the server will self-terminate.  All tests
 # must terminate in this time, and testenv will only stay alive this
 # long
@@ -496,18 +489,18 @@ if ($opt_target eq "samba") {
 	}
 	$testenv_default = "all";
 	require target::Samba;
-	$target = new Samba($bindir, \%binary_mapping, \&bindir_path, $ldap, $srcdir, $exeext, $server_maxtime);
+	$target = new Samba($bindir, \%binary_mapping, $ldap, $srcdir, $exeext, $server_maxtime);
 } elsif ($opt_target eq "samba4") {
 	$testenv_default = "all";
 	require target::Samba4;
-	$target = new Samba4($bindir, \%binary_mapping, \&bindir_path, $ldap, $srcdir, $exeext, $server_maxtime);
+	$target = new Samba4($bindir, \%binary_mapping, $ldap, $srcdir, $exeext, $server_maxtime);
 } elsif ($opt_target eq "samba3") {
 	if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
 		die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 	}
 	$testenv_default = "member";
 	require target::Samba3;
-	$target = new Samba3($bindir, \%binary_mapping, \&bindir_path, $srcdir_abs, $exeext, $server_maxtime);
+	$target = new Samba3($bindir, \%binary_mapping, $srcdir_abs, $exeext, $server_maxtime);
 } elsif ($opt_target eq "win") {
 	die("Windows tests will not run with socket wrapper enabled.") 
 		if ($opt_socket_wrapper);
@@ -604,19 +597,23 @@ sub write_clientconf($$$)
 	        mkdir("$clientdir/lockdir", 0777);
 	}
 
+	# this is ugly, but the ncalrpcdir needs exactly 0755
+	# otherwise tests fail.
+	my $mask = umask;
+	umask 0022;
+	if ( -d "$clientdir/ncalrpcdir/np" ) {
+	        unlink <$clientdir/ncalrpcdir/np/*>;
+		rmdir <$clientdir/ncalrpcdir/np>;
+	}
 	if ( -d "$clientdir/ncalrpcdir" ) {
 	        unlink <$clientdir/ncalrpcdir/*>;
-	} else {
-	        mkdir("$clientdir/ncalrpcdir", 0777);
+		rmdir <$clientdir/ncalrpcdir>;
 	}
+	mkdir("$clientdir/ncalrpcdir", 0755);
+	umask $mask;
 
 	open(CF, ">$conffile");
 	print CF "[global]\n";
-	if (defined($ENV{VALGRIND})) {
-		print CF "\ticonv:native = true\n";
-	} else {
-		print CF "\ticonv:native = false\n";
-	}
 	print CF "\tnetbios name = client\n";
 	if (defined($vars->{DOMAIN})) {
 		print CF "\tworkgroup = $vars->{DOMAIN}\n";
diff --git a/selftest/target/Samba.pm b/selftest/target/Samba.pm
index fc7e68dfde7..1b1eb539336 100644
--- a/selftest/target/Samba.pm
+++ b/selftest/target/Samba.pm
@@ -10,11 +10,11 @@ use target::Samba3;
 use target::Samba4;
 
 sub new($$$$$) {
-	my ($classname, $bindir, $binary_mapping, $bindir_path, $ldap, $srcdir, $exeext, $server_maxtime) = @_;
+	my ($classname, $bindir, $binary_mapping,$ldap, $srcdir, $exeext, $server_maxtime) = @_;
 
 	my $self = {
-	    samba3 => new Samba3($bindir,$binary_mapping, $bindir_path, $srcdir, $exeext, $server_maxtime),
-	    samba4 => new Samba4($bindir,$binary_mapping, $bindir_path, $ldap, $srcdir, $exeext, $server_maxtime),
+	    samba3 => new Samba3($bindir,$binary_mapping, $srcdir, $exeext, $server_maxtime),
+	    samba4 => new Samba4($bindir,$binary_mapping, $ldap, $srcdir, $exeext, $server_maxtime),
 	};
 	bless $self;
 	return $self;
@@ -28,11 +28,15 @@ sub setup_env($$$)
 
 	my $env = $self->{samba4}->setup_env($envname, $path);
 	if (defined($env)) {
-	    $env->{target} = $self->{samba4};
+	    if (not defined($env->{target})) {
+		$env->{target} = $self->{samba4};
+	    }
 	} else {
 	   	$env = $self->{samba3}->setup_env($envname, $path);
 		if (defined($env)) {
-		    $env->{target} = $self->{samba3};
+		    if (not defined($env->{target})) {
+			$env->{target} = $self->{samba3};
+		    }
 		}
 	}
 	if (not defined $env) {
@@ -42,4 +46,73 @@ sub setup_env($$$)
 	return $env;
 }
 
+sub bindir_path($$) {
+	my ($object, $path) = @_;
+
+	if (defined($object->{binary_mapping}->{$path})) {
+	    $path = $object->{binary_mapping}->{$path};
+	}
+
+	my $valpath = "$object->{bindir}/$path$object->{exeext}";
+
+	return $valpath if (-f $valpath);
+	return $path;
+}
+
+sub mk_krb5_conf($)
+{
+	my ($ctx) = @_;
+
+	unless (open(KRB5CONF, ">$ctx->{krb5_conf}")) {
+	        warn("can't open $ctx->{krb5_conf}$?");
+		return undef;
+	}
+	print KRB5CONF "
+#Generated krb5.conf for $ctx->{realm}
+
+[libdefaults]
+ default_realm = $ctx->{realm}
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ forwardable = yes
+ allow_weak_crypto = yes
+
+[realms]
+ $ctx->{realm} = {
+  kdc = $ctx->{kdc_ipv4}:88
+  admin_server = $ctx->{kdc_ipv4}:88
+  default_domain = $ctx->{dnsname}
+ }
+ $ctx->{dnsname} = {
+  kdc = $ctx->{kdc_ipv4}:88
+  admin_server = $ctx->{kdc_ipv4}:88
+  default_domain = $ctx->{dnsname}
+ }
+ $ctx->{domain} = {
+  kdc = $ctx->{kdc_ipv4}:88
+  admin_server = $ctx->{kdc_ipv4}:88
+  default_domain = $ctx->{dnsname}
+ }
+
+[domain_realm]
+ .$ctx->{dnsname} = $ctx->{realm}
+";
+
+        if (defined($ctx->{tlsdir})) {
+	       print KRB5CONF "
+
+[appdefaults]
+	pkinit_anchors = FILE:$ctx->{tlsdir}/ca.pem
+
+[kdc]
+	enable-pkinit = true
+	pkinit_identity = FILE:$ctx->{tlsdir}/kdc.pem,$ctx->{tlsdir}/key.pem
+	pkinit_anchors = FILE:$ctx->{tlsdir}/ca.pem
+
+";
+        }
+	close(KRB5CONF);
+}
+
 1;
diff --git a/selftest/target/Samba3.pm b/selftest/target/Samba3.pm
index 3c0dd0918a3..505130f5962 100644
--- a/selftest/target/Samba3.pm
+++ b/selftest/target/Samba3.pm
@@ -9,14 +9,14 @@ use strict;
 use Cwd qw(abs_path);
 use FindBin qw($RealBin);
 use POSIX;
+use target::Samba;
 
 sub new($$) {
-	my ($classname, $bindir, $binary_mapping, $bindir_path, $srcdir, $exeext, $server_maxtime) = @_;
+	my ($classname, $bindir, $binary_mapping, $srcdir, $exeext, $server_maxtime) = @_;
 	$exeext = "" unless defined($exeext);
 	my $self = { vars => {},
 		     bindir => $bindir,
 		     binary_mapping => $binary_mapping,
-		     bindir_path => $bindir_path,
 		     srcdir => $srcdir,
 		     exeext => $exeext,
 		     server_maxtime => $server_maxtime
@@ -172,13 +172,16 @@ sub setup_member($$$)
 
 	$ret or return undef;
 
-	my $net = $self->{bindir_path}->($self, "net");
+	my $net = Samba::bindir_path($self, "net");
 	my $cmd = "";
 	$cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$ret->{SOCKET_WRAPPER_DEFAULT_IFACE}\" ";
 	$cmd .= "$net join $ret->{CONFIGURATION} $s3dcvars->{DOMAIN} member";
 	$cmd .= " -U$s3dcvars->{USERNAME}\%$s3dcvars->{PASSWORD}";
 
-	system($cmd) == 0 or die("Join failed\n$cmd");
+	if (system($cmd) != 0) {
+	    warn("Join failed\n$cmd");
+	    return undef;
+	}
 
 	$self->check_or_start($ret, "yes", "yes", "yes");
 
@@ -195,6 +198,125 @@ sub setup_member($$$)
 	return $ret;
 }
 
+sub setup_admember($$$$)
+{
+	my ($self, $prefix, $dcvars, $iface) = @_;
+
+	print "PROVISIONING S3 AD MEMBER$iface...";
+
+	my $member_options = "
+	security = ads
+	server signing = on
+        workgroup = $dcvars->{DOMAIN}
+        realm = $dcvars->{REALM}
+";
+
+	my $ret = $self->provision($prefix,
+				   "LOCALADMEMBER$iface",
+				   $iface,
+				   "loCalMember${iface}Pass",
+				   $member_options);
+
+	$ret or return undef;
+
+	close(USERMAP);
+	$ret->{DOMAIN} = $dcvars->{DOMAIN};
+	$ret->{REALM} = $dcvars->{REALM};
+
+	my $ctx;
+	my $prefix_abs = abs_path($prefix);
+	$ctx = {};
+	$ctx->{krb5_conf} = "$prefix_abs/lib/krb5.conf";
+	$ctx->{domain} = $dcvars->{DOMAIN};
+	$ctx->{realm} = $dcvars->{REALM};
+	$ctx->{dnsname} = lc($dcvars->{REALM});
+	$ctx->{kdc_ipv4} = $dcvars->{SERVER_IP};
+	Samba::mk_krb5_conf($ctx);
+
+	$ret->{KRB5_CONFIG} = $ctx->{krb5_conf};
+
+	my $net = Samba::bindir_path($self, "net");
+	my $cmd = "";
+	$cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$ret->{SOCKET_WRAPPER_DEFAULT_IFACE}\" ";
+	$cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" ";
+	$cmd .= "$net join $ret->{CONFIGURATION}";
+	$cmd .= " -U$dcvars->{USERNAME}\%$dcvars->{PASSWORD}";
+
+	if (system($cmd) != 0) {
+	    warn("Join failed\n$cmd");
+	    return undef;
+	}
+
+	# We need world access to this share, as otherwise the domain
+	# administrator from the AD domain provided by Samba4 can't
+	# access the share for tests.
+	chmod 0777, "$prefix/share";
+
+	$self->check_or_start($ret,
+			      "yes", "yes", "yes");
+
+	$self->wait_for_start($ret);
+
+	$ret->{DC_SERVER} = $dcvars->{SERVER};
+	$ret->{DC_SERVER_IP} = $dcvars->{SERVER_IP};
+	$ret->{DC_NETBIOSNAME} = $dcvars->{NETBIOSNAME};
+	$ret->{DC_USERNAME} = $dcvars->{USERNAME};
+	$ret->{DC_PASSWORD} = $dcvars->{PASSWORD};
+
+	# Special case, this is called from Samba4.pm but needs to use the Samba3 check_env and get_log_env
+	$ret->{target} = $self;
+
+	return $ret;
+}
+
+sub setup_plugin_s4_dc($$$$)
+{
+	my ($self, $prefix, $dcvars, $iface) = @_;
+
+	print "PROVISIONING S4 PLUGIN AD DC$iface...";
+
+	my $plugin_s4_dc_options = "
+        workgroup = $dcvars->{DOMAIN}
+        realm = $dcvars->{REALM}
+        security=ads
+        passdb backend = samba4
+        auth methods = guest samba4
+        domain logons = yes
+        rpc_server:lsarpc = external
+        rpc_server:netlogon = external
+        rpc_server:samr = external
+	server signing = on
+";
+
+	my $ret = $self->provision($prefix,
+				   "plugindc",
+				   $iface,
+				   "pluGin${iface}Pass",
+				   $plugin_s4_dc_options, 1);
+
+	$ret or return undef;
+
+	close(USERMAP);
+	$ret->{DOMAIN} = $dcvars->{DOMAIN};
+	$ret->{REALM} = $dcvars->{REALM};
+	$ret->{KRB5_CONFIG} = $dcvars->{KRB5_CONFIG};
+
+	# We need world access to this share, as otherwise the domain
+	# administrator from the AD domain provided by Samba4 can't
+	# access the share for tests.
+	chmod 0777, "$prefix/share";
+
+	$self->check_or_start($ret,
+			      "no", "yes", "yes");
+
+	$self->wait_for_start($ret);
+
+	# Special case, this is called from Samba4.pm but needs to use the Samba3 check_env and get_log_env
+	$ret->{target} = $self;
+
+	return $ret;
+}
+
 sub setup_secshare($$)
 {
 	my ($self, $path) = @_;
@@ -261,7 +383,7 @@ sub setup_secserver($$$)
 
 sub setup_ktest($$$)
 {
-	my ($self, $prefix, $s3dcvars) = @_;
+	my ($self, $prefix) = @_;
 
 	print "PROVISIONING server with security=ads...";
 
@@ -280,6 +402,18 @@ sub setup_ktest($$$)
 
 	$ret or return undef;
 
+	my $ctx;
+	my $prefix_abs = abs_path($prefix);
+	$ctx = {};
+	$ctx->{krb5_conf} = "$prefix_abs/lib/krb5.conf";
+	$ctx->{domain} = "KTEST";
+	$ctx->{realm} = "KTEST.SAMBA.EXAMPLE.COM";
+	$ctx->{dnsname} = lc($ctx->{realm});
+	$ctx->{kdc_ipv4} = "0.0.0.0";
+	Samba::mk_krb5_conf($ctx);
+
+	$ret->{KRB5_CONFIG} = $ctx->{krb5_conf};
+
 	open(USERMAP, ">$prefix/lib/username.map") or die("Unable to open $prefix/lib/username.map");
 	print USERMAP "
 $ret->{USERNAME} = KTEST\\Administrator
@@ -290,7 +424,11 @@ $ret->{USERNAME} = KTEST\\Administrator
 #Samba4 DC with the same parameters as are being used here.  The
 #domain SID is S-1-5-21-1071277805-689288055-3486227160
 
-	system("cp $self->{srcdir}/source3/selftest/ktest-secrets.tdb $prefix/private/secrets.tdb");
+	if (defined($ENV{BUILD_TDB2})) {
+	    system("cp $self->{srcdir}/source3/selftest/ktest-secrets.tdb2 $prefix/private/secrets.tdb");
+	} else {
+	    system("cp $self->{srcdir}/source3/selftest/ktest-secrets.tdb $prefix/private/secrets.tdb");
+	}
 	chmod 0600, "$prefix/private/secrets.tdb";
 
 #This uses a pre-calculated krb5 credentials cache, obtained by running Samba4 with:
@@ -373,6 +511,7 @@ sub check_or_start($$$$) {
 
 		SocketWrapper::set_default_iface($env_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
 
+		$ENV{KRB5_CONFIG} = $env_vars->{KRB5_CONFIG};
 		$ENV{WINBINDD_SOCKET_DIR} = $env_vars->{WINBINDD_SOCKET_DIR};
 		$ENV{NMBD_SOCKET_DIR} = $env_vars->{NMBD_SOCKET_DIR};
 
@@ -395,14 +534,14 @@ sub check_or_start($$$$) {
 			@optargs = split(/ /, $ENV{NMBD_OPTIONS});
 		}
 
-		$ENV{MAKE_TEST_BINARY} = $self->{bindir_path}->($self, "nmbd");
+		$ENV{MAKE_TEST_BINARY} = Samba::bindir_path($self, "nmbd");
 
-		my @preargs = ($self->{bindir_path}->($self, "timelimit"), $self->{server_maxtime});
+		my @preargs = (Samba::bindir_path($self, "timelimit"), $self->{server_maxtime});
 		if(defined($ENV{NMBD_VALGRIND})) { 
 			@preargs = split(/ /, $ENV{NMBD_VALGRIND});
 		}
 
-		exec(@preargs, $self->{bindir_path}->($self, "nmbd"), "-F", "--no-process-group", "-S", "-s", $env_vars->{SERVERCONFFILE}, @optargs) or die("Unable to start nmbd: $!");
+		exec(@preargs, Samba::bindir_path($self, "nmbd"), "-F", "--no-process-group", "--log-stdout", "-s", $env_vars->{SERVERCONFFILE}, @optargs) or die("Unable to start nmbd: $!");
 	}
 	write_pid($env_vars, "nmbd", $pid);
 	print "DONE\n";
@@ -416,6 +555,7 @@ sub check_or_start($$$$) {
 
 		SocketWrapper::set_default_iface($env_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
 
+		$ENV{KRB5_CONFIG} = $env_vars->{KRB5_CONFIG};
 		$ENV{WINBINDD_SOCKET_DIR} = $env_vars->{WINBINDD_SOCKET_DIR};
 		$ENV{NMBD_SOCKET_DIR} = $env_vars->{NMBD_SOCKET_DIR};
 
@@ -438,14 +578,16 @@ sub check_or_start($$$$) {
 			@optargs = split(/ /, $ENV{WINBINDD_OPTIONS});
 		}
 
-		$ENV{MAKE_TEST_BINARY} = $self->{bindir_path}->($self, "winbindd");
+		$ENV{MAKE_TEST_BINARY} = Samba::bindir_path($self, "winbindd");
 
-		my @preargs = ($self->{bindir_path}->($self, "timelimit"), $self->{server_maxtime});
+		my @preargs = (Samba::bindir_path($self, "timelimit"), $self->{server_maxtime});
 		if(defined($ENV{WINBINDD_VALGRIND})) {
 			@preargs = split(/ /, $ENV{WINBINDD_VALGRIND});
 		}
 
-		exec(@preargs, $self->{bindir_path}->($self, "winbindd"), "-F", "--no-process-group", "-s", $env_vars->{SERVERCONFFILE}, @optargs) or die("Unable to start winbindd: $!");
+		print "Starting winbindd with config $env_vars->{SERVERCONFFILE})\n";
+
+		exec(@preargs, Samba::bindir_path($self, "winbindd"), "-F", "--no-process-group", "--stdout", "-s", $env_vars->{SERVERCONFFILE}, @optargs) or die("Unable to start winbindd: $!");
 	}
 	write_pid($env_vars, "winbindd", $pid);
 	print "DONE\n";
@@ -459,6 +601,7 @@ sub check_or_start($$$$) {
 
 		SocketWrapper::set_default_iface($env_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
 
+		$ENV{KRB5_CONFIG} = $env_vars->{KRB5_CONFIG};
 		$ENV{WINBINDD_SOCKET_DIR} = $env_vars->{WINBINDD_SOCKET_DIR};
 		$ENV{NMBD_SOCKET_DIR} = $env_vars->{NMBD_SOCKET_DIR};
 
@@ -476,16 +619,16 @@ sub check_or_start($$$$) {
 			exit 0;
 		}
 
-		$ENV{MAKE_TEST_BINARY} = $self->{bindir_path}->($self, "smbd");
+		$ENV{MAKE_TEST_BINARY} = Samba::bindir_path($self, "smbd");
 		my @optargs = ("-d0");
 		if (defined($ENV{SMBD_OPTIONS})) {
 			@optargs = split(/ /, $ENV{SMBD_OPTIONS});
 		}
-		my @preargs = ($self->{bindir_path}->($self, "timelimit"), $self->{server_maxtime});
+		my @preargs = (Samba::bindir_path($self, "timelimit"), $self->{server_maxtime});
 		if(defined($ENV{SMBD_VALGRIND})) {
 			@preargs = split(/ /,$ENV{SMBD_VALGRIND});
 		}
-		exec(@preargs, $self->{bindir_path}->($self, "smbd"), "-F", "--no-process-group", "-s", $env_vars->{SERVERCONFFILE}, @optargs) or die("Unable to start smbd: $!");
+		exec(@preargs, Samba::bindir_path($self, "smbd"), "-F", "--no-process-group", "--log-stdout", "-s", $env_vars->{SERVERCONFFILE}, @optargs) or die("Unable to start smbd: $!");
 	}
 	write_pid($env_vars, "smbd", $pid);
 	print "DONE\n";
@@ -493,9 +636,9 @@ sub check_or_start($$$$) {
 	return 0;
 }
 
-sub provision($$$$$$)
+sub provision($$$$$$$)
 {
-	my ($self, $prefix, $server, $swiface, $password, $extra_options) = @_;
+	my ($self, $prefix, $server, $swiface, $password, $extra_options, $no_delete_prefix) = @_;
 
 	##
 	## setup the various environment variables we need
@@ -515,6 +658,8 @@ sub provision($$$$$$)
 	my $bindir_abs = abs_path($self->{bindir});
 	my $vfs_modulesdir_abs = ($ENV{VFSLIBDIR} or $bindir_abs);
 
+	my $dns_host_file = "$ENV{SELFTEST_PREFIX}/dns_host_file";
+
 	my @dirs = ();
 
 	my $shrdir="$prefix_abs/share";
@@ -571,7 +716,9 @@ sub provision($$$$$$)
 
 	mkdir($prefix_abs, 0777);
 	print "CREATE TEST ENVIRONMENT IN '$prefix'...";
-	system("rm -rf $prefix_abs/*");
+	if (not defined($no_delete_prefix) or not $no_delete_prefix) {
+	    system("rm -rf $prefix_abs/*");
+	}
 	mkdir($_, 0777) foreach(@dirs);
 
 	##
@@ -580,12 +727,18 @@ sub provision($$$$$$)
 
 	chmod 0755, $ro_shrdir;
 	my $unreadable_file = "$ro_shrdir/unreadable_file";
-	open(UNREADABLE_FILE, ">$unreadable_file") or die("Unable to open $unreadable_file");
+	unless (open(UNREADABLE_FILE, ">$unreadable_file")) {
+	        warn("Unable to open $unreadable_file");
+		return undef;
+	}
 	close(UNREADABLE_FILE);
 	chmod 0600, $unreadable_file;
 
 	my $msdfs_target = "$ro_shrdir/msdfs-target";
-	open(MSDFS_TARGET, ">$msdfs_target") or die("Unable to open $msdfs_target");
+	unless (open(MSDFS_TARGET, ">$msdfs_target")) {
+	        warn("Unable to open $msdfs_target");
+		return undef;
+	}
 	close(MSDFS_TARGET);
 	chmod 0666, $msdfs_target;
 	symlink "msdfs:$server_ip\\ro-tmp", "$msdfs_shrdir/msdfs-src1";
@@ -633,7 +786,10 @@ sub provision($$$$$$)
 	## create conffile
 	##
 
-	open(CONF, ">$conffile") or die("Unable to open $conffile");
+	unless (open(CONF, ">$conffile")) {
+	        warn("Unable to open $conffile");
+		return undef;
+	}
 	print CONF "
 [global]
 	netbios name = $server
@@ -651,8 +807,6 @@ sub provision($$$$$$)
 	debug pid = yes
         max log size = 0
 
-	name resolve order = bcast
-
 	state directory = $lockdir
 	cache directory = $lockdir
 
@@ -711,9 +865,13 @@ sub provision($$$$$$)
 	queue resume command = $bindir_abs/vlp tdbfile=$lockdir/vlp.tdb queueresume %p
 	lpq cache time = 0
 
-	ncalrpc dir = $lockdir/ncalrpc
+	ncalrpc dir = $prefix_abs/ncalrpc
 	rpc_server:epmapper = embedded
 
+        resolv:host file = $dns_host_file
+
+        message command = mv %s $shrdir/message.%m
+
 	# Begin extra options
 	$extra_options
 	# End extra options
@@ -778,7 +936,10 @@ sub provision($$$$$$)
 	## create a test account
 	##
 
-	open(PASSWD, ">$nss_wrapper_passwd") or die("Unable to open $nss_wrapper_passwd");
+	unless (open(PASSWD, ">$nss_wrapper_passwd")) {
+           warn("Unable to open $nss_wrapper_passwd");
+           return undef;
+        } 
 	print PASSWD "nobody:x:$uid_nobody:$gid_nobody:nobody gecos:$prefix_abs:/bin/false
 $unix_name:x:$unix_uid:$unix_gids[0]:$unix_name gecos:$prefix_abs:/bin/false
 ";
@@ -787,7 +948,10 @@ $unix_name:x:$unix_uid:$unix_gids[0]:$unix_name gecos:$prefix_abs:/bin/false
 	}
 	close(PASSWD);
 
-	open(GROUP, ">$nss_wrapper_group") or die("Unable to open $nss_wrapper_group");
+	unless (open(GROUP, ">$nss_wrapper_group")) {
+             warn("Unable to open $nss_wrapper_group");
+             return undef;
+        }
 	print GROUP "nobody:x:$gid_nobody:
 nogroup:x:$gid_nogroup:nobody
 $unix_name-group:x:$unix_gids[0]:
@@ -808,10 +972,16 @@ domusers:X:$gid_domusers:
 	$ENV{NSS_WRAPPER_PASSWD} = $nss_wrapper_passwd;
 	$ENV{NSS_WRAPPER_GROUP} = $nss_wrapper_group;
 
-	open(PWD, "|".$self->{bindir_path}->($self, "smbpasswd")." -c $conffile -L -s -a $unix_name >/dev/null");
+        my $cmd = Samba::bindir_path($self, "smbpasswd")." -c $conffile -L -s -a $unix_name > /dev/null";
+	unless (open(PWD, "|$cmd")) {
+             warn("Unable to set password for test account\n$cmd");
+             return undef;
+        }
 	print PWD "$password\n$password\n";
-	close(PWD) or die("Unable to set password for test account");
-
+	unless (close(PWD)) {
+             warn("Unable to set password for test account\n$cmd");
+             return undef; 
+        }
 	print "DONE\n";
 
 	open(HOSTS, ">>$ENV{SELFTEST_PREFIX}/dns_host_file") or die("Unable to open $ENV{SELFTEST_PREFIX}/dns_host_file");
@@ -843,7 +1013,7 @@ domusers:X:$gid_domusers:
 	$ret{NSS_WRAPPER_GROUP} = $nss_wrapper_group;
 	$ret{NSS_WRAPPER_WINBIND_SO_PATH} = $ENV{NSS_WRAPPER_WINBIND_SO_PATH};
         if (not defined($ret{NSS_WRAPPER_WINBIND_SO_PATH})) {
-	        $ret{NSS_WRAPPER_WINBIND_SO_PATH} = $self->{bindir_path}->($self, "default/nsswitch/libnss-winbind.so");
+	        $ret{NSS_WRAPPER_WINBIND_SO_PATH} = Samba::bindir_path($self, "default/nsswitch/libnss-winbind.so");
         }
 	$ret{LOCAL_PATH} = "$shrdir";
 
@@ -858,11 +1028,11 @@ sub wait_for_start($$)
 	print "delaying for nbt name registration\n";
 	sleep(10);
 	# This will return quickly when things are up, but be slow if we need to wait for (eg) SSL init 
-	system($self->{bindir_path}->($self, "nmblookup3") ." $envvars->{CONFIGURATION} -U $envvars->{SERVER_IP} __SAMBA__");
-	system($self->{bindir_path}->($self, "nmblookup3") ." $envvars->{CONFIGURATION} __SAMBA__");
-	system($self->{bindir_path}->($self, "nmblookup3") ." $envvars->{CONFIGURATION} -U 127.255.255.255 __SAMBA__");
-	system($self->{bindir_path}->($self, "nmblookup3") ." $envvars->{CONFIGURATION} -U $envvars->{SERVER_IP} $envvars->{SERVER}");
-	system($self->{bindir_path}->($self, "nmblookup3") ." $envvars->{CONFIGURATION} $envvars->{SERVER}");
+	system(Samba::bindir_path($self, "nmblookup3") ." $envvars->{CONFIGURATION} -U $envvars->{SERVER_IP} __SAMBA__");
+	system(Samba::bindir_path($self, "nmblookup3") ." $envvars->{CONFIGURATION} __SAMBA__");
+	system(Samba::bindir_path($self, "nmblookup3") ." $envvars->{CONFIGURATION} -U 127.255.255.255 __SAMBA__");
+	system(Samba::bindir_path($self, "nmblookup3") ." $envvars->{CONFIGURATION} -U $envvars->{SERVER_IP} $envvars->{SERVER}");
+	system(Samba::bindir_path($self, "nmblookup3") ." $envvars->{CONFIGURATION} $envvars->{SERVER}");
 
 	# make sure smbd is also up set
 	print "wait for smbd\n";
@@ -870,7 +1040,7 @@ sub wait_for_start($$)
 	my $count = 0;
 	my $ret;
 	do {
-	    $ret = system($self->{bindir_path}->($self, "smbclient3") ." $envvars->{CONFIGURATION} -L $envvars->{SERVER} -U% -p 139");
+	    $ret = system(Samba::bindir_path($self, "smbclient3") ." $envvars->{CONFIGURATION} -L $envvars->{SERVER} -U% -p 139");
 	    if ($ret != 0) {
 		sleep(2);
 	    }
@@ -882,7 +1052,7 @@ sub wait_for_start($$)
 	    return 0;
 	}
 	# Ensure we have domain users mapped.
-	$ret = system($self->{bindir_path}->($self, "net") ." $envvars->{CONFIGURATION} groupmap add rid=513 unixgroup=domusers type=domain");
+	$ret = system(Samba::bindir_path($self, "net") ." $envvars->{CONFIGURATION} groupmap add rid=513 unixgroup=domusers type=domain");
 	if ($ret != 0) {
 	    return 1;
 	}
diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm
index 69d5c3bc686..49bab371fc9 100644
--- a/selftest/target/Samba4.pm
+++ b/selftest/target/Samba4.pm
@@ -10,9 +10,11 @@ use Cwd qw(abs_path);
 use FindBin qw($RealBin);
 use POSIX;
 use SocketWrapper;
+use target::Samba;
+use target::Samba3;
 
 sub new($$$$$) {
-	my ($classname, $bindir, $binary_mapping, $bindir_path, $ldap, $srcdir, $exeext, $server_maxtime) = @_;
+	my ($classname, $bindir, $binary_mapping, $ldap, $srcdir, $exeext, $server_maxtime) = @_;
 	$exeext = "" unless defined($exeext);
 
 	my $self = {
@@ -20,10 +22,10 @@ sub new($$$$$) {
 		ldap => $ldap,
 		bindir => $bindir,
 		binary_mapping => $binary_mapping,
-		bindir_path => $bindir_path,
 		srcdir => $srcdir,
 		exeext => $exeext,
-		server_maxtime => $server_maxtime
+		server_maxtime => $server_maxtime,
+		target3 => new Samba3($bindir, $binary_mapping, $srcdir, $exeext, $server_maxtime)
 	};
 	bless $self;
 	return $self;
@@ -41,7 +43,7 @@ sub slapd_start($$)
 {
 	my $count = 0;
 	my ($self, $env_vars) = @_;
-	my $ldbsearch = $self->bindir_path($self, "ldbsearch");
+	my $ldbsearch = Samba::bindir_path($self, "ldbsearch");
 
 	my $uri = $env_vars->{LDAP_URI};
 
@@ -131,7 +133,7 @@ sub check_or_start($$)
 		if (defined($ENV{SAMBA_OPTIONS})) {
 			$optarg.= " $ENV{SAMBA_OPTIONS}";
 		}
-		my $samba = $self->{bindir_path}->($self, "samba");
+		my $samba =  Samba::bindir_path($self, "samba");
 
 		# allow selection of the process model using
 		# the environment varibale SAMBA_PROCESS_MODEL
@@ -179,7 +181,7 @@ sub wait_for_start($$)
 
 	# This will return quickly when things are up, but be slow if we
 	# need to wait for (eg) SSL init
-	my $nmblookup = $self->{bindir_path}->($self, "nmblookup");
+	my $nmblookup =  Samba::bindir_path($self, "nmblookup");
 	system("$nmblookup $testenv_vars->{CONFIGURATION} $testenv_vars->{SERVER}");
 	system("$nmblookup $testenv_vars->{CONFIGURATION} -U $testenv_vars->{SERVER_IP} $testenv_vars->{SERVER}");
 	system("$nmblookup $testenv_vars->{CONFIGURATION} $testenv_vars->{NETBIOSNAME}");
@@ -200,7 +202,7 @@ sub write_ldb_file($$$)
 {
 	my ($self, $file, $ldif) = @_;
 
-	my $ldbadd = $self->{bindir_path}->($self, "ldbadd");
+	my $ldbadd =  Samba::bindir_path($self, "ldbadd");
 	open(LDIF, "|$ldbadd -H $file >/dev/null");
 	print LDIF $ldif;
 	return(close(LDIF));
@@ -452,56 +454,6 @@ Wfz/8alZ5aMezCQzXJyIaJsCLeKABosSwHcpAFmxlQ==
 EOF
 }
 
-sub mk_krb5_conf($$)
-{
-	my ($self, $ctx) = @_;
-
-	unless (open(KRB5CONF, ">$ctx->{krb5_conf}")) {
-		warn("can't open $ctx->{krb5_conf}$?");
-		return undef;
-	}
-	print KRB5CONF "
-#Generated krb5.conf for $ctx->{realm}
-
-[libdefaults]
- default_realm = $ctx->{realm}
- dns_lookup_realm = false
- dns_lookup_kdc = false
- ticket_lifetime = 24h
- forwardable = yes
- allow_weak_crypto = yes
-
-[realms]
- $ctx->{realm} = {
-  kdc = $ctx->{kdc_ipv4}:88
-  admin_server = $ctx->{kdc_ipv4}:88
-  default_domain = $ctx->{dnsname}
- }
- $ctx->{dnsname} = {
-  kdc = $ctx->{kdc_ipv4}:88
-  admin_server = $ctx->{kdc_ipv4}:88
-  default_domain = $ctx->{dnsname}
- }
- $ctx->{domain} = {
-  kdc = $ctx->{kdc_ipv4}:88
-  admin_server = $ctx->{kdc_ipv4}:88
-  default_domain = $ctx->{dnsname}
- }
-
-[appdefaults]
-	pkinit_anchors = FILE:$ctx->{tlsdir}/ca.pem
-
-[kdc]
-	enable-pkinit = true
-	pkinit_identity = FILE:$ctx->{tlsdir}/kdc.pem,$ctx->{tlsdir}/key.pem
-	pkinit_anchors = FILE:$ctx->{tlsdir}/ca.pem
-
-[domain_realm]
- .$ctx->{dnsname} = $ctx->{realm}
-";
-	close(KRB5CONF);
-}
-
 sub provision_raw_prepare($$$$$$$$$$)
 {
 	my ($self, $prefix, $server_role, $netbiosname, 
@@ -570,7 +522,6 @@ sub provision_raw_prepare($$$$$$$$$$)
 	push(@{$ctx->{directories}}, $ctx->{privatedir});
 	push(@{$ctx->{directories}}, $ctx->{etcdir});
 	push(@{$ctx->{directories}}, $ctx->{piddir});
-	push(@{$ctx->{directories}}, $ctx->{ncalrpcdir});
 	push(@{$ctx->{directories}}, $ctx->{lockdir});
 
 	$ctx->{smb_conf_extra_options} = "";
@@ -627,8 +578,11 @@ sub provision_raw_step1($$)
 		warn("can't open $ctx->{smb_conf}$?");
 		return undef;
 	}
+	my $acl = "false";
+	$acl = "true" if (defined $ENV{WITH_ACL});
 	print CONFFILE "
 [global]
+	acl:search = $acl
 	netbios name = $ctx->{netbiosname}
 	posix:eadb = $ctx->{lockdir}/eadb.tdb
 	workgroup = $ctx->{domain}
@@ -681,7 +635,7 @@ sub provision_raw_step1($$)
              $ctx->{kdc_ipv4} = $ctx->{ipv4};
         }
 
-	$self->mk_krb5_conf($ctx);
+	Samba::mk_krb5_conf($ctx);
 
 	open(PWD, ">$ctx->{nsswrap_passwd}");
 	print PWD "
@@ -704,7 +658,7 @@ nogroup:x:65534:nobody
 	my $configuration = "--configfile=$ctx->{smb_conf}";
 
 #Ensure the config file is valid before we start
-	my $testparm = $self->scriptdir_path("bin/testparm");
+	my $testparm = Samba::bindir_path($self, "samba-tool") . " testparm";
 	if (system("$testparm $configuration -v --suppress-prompt >/dev/null 2>&1") != 0) {
 		system("$testparm -v --suppress-prompt $configuration >&2");
 		warn("Failed to create a valid smb.conf configuration $testparm!");
@@ -889,7 +843,7 @@ sub provision_member($$$)
 		return undef;
 	}
 
-	my $samba_tool = $self->{bindir_path}->($self, "samba-tool");
+	my $samba_tool =  Samba::bindir_path($self, "samba-tool");
 	my $cmd = "";
 	$cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$ret->{SOCKET_WRAPPER_DEFAULT_IFACE}\" ";
 	$cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" ";
@@ -921,7 +875,7 @@ sub provision_rpc_proxy($$$)
 	my ($self, $prefix, $dcvars) = @_;
 	print "PROVISIONING RPC PROXY...";
 
-	my $extra_smbconf_options = "dcerpc_remote:binding = ncacn_ip_tcp:localdc
+	my $extra_smbconf_options = "dcerpc_remote:binding = ncacn_ip_tcp:$dcvars->{SERVER}
        dcerpc endpoint servers = epmapper, remote
        dcerpc_remote:interfaces = rpcecho
 ";
@@ -941,7 +895,7 @@ sub provision_rpc_proxy($$$)
 		return undef;
 	}
 
-	my $samba_tool = $self->{bindir_path}->($self, "samba-tool");
+	my $samba_tool =  Samba::bindir_path($self, "samba-tool");
 	my $cmd = "";
 	$cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$ret->{SOCKET_WRAPPER_DEFAULT_IFACE}\" ";
 	$cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" ";
@@ -1001,7 +955,7 @@ sub provision_vampire_dc($$$)
 		return undef;
 	}
 
-	my $samba_tool = $self->{bindir_path}->($self, "samba-tool");
+	my $samba_tool =  Samba::bindir_path($self, "samba-tool");
 	my $cmd = "";
 	$cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$ret->{SOCKET_WRAPPER_DEFAULT_IFACE}\" ";
 	$cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" ";
@@ -1173,7 +1127,7 @@ sub provision_rodc($$$)
 		return undef;
 	}
 
-	my $samba_tool = $self->{bindir_path}->($self, "samba-tool");
+	my $samba_tool =  Samba::bindir_path($self, "samba-tool");
 	my $cmd = "";
 	$cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$ret->{SOCKET_WRAPPER_DEFAULT_IFACE}\" ";
 	$cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" ";
@@ -1190,7 +1144,7 @@ sub provision_rodc($$$)
 	# so that use the RODC as kdc and test
 	# the proxy code
 	$ctx->{kdc_ipv4} = $ret->{SERVER_IP};
-	$self->mk_krb5_conf($ctx);
+	Samba::mk_krb5_conf($ctx);
 
 	$ret->{RODC_DC_SERVER} = $ret->{SERVER};
 	$ret->{RODC_DC_SERVER_IP} = $ret->{SERVER_IP};
@@ -1205,6 +1159,40 @@ sub provision_rodc($$$)
 	return $ret;
 }
 
+sub provision_plugin_s4_dc($$)
+{
+	my ($self, $prefix) = @_;
+
+	my $extra_smbconf_options = "
+server services = -winbind, -smb
+";
+
+	print "PROVISIONING PLUGIN S4 DC...";
+	my $ret = $self->provision($prefix,
+				   "domain controller",
+				   "plugindc",
+				   "PLUGINDOMAIN",
+				   "plugin.samba.example.com",
+				   "2008",
+				   30,
+				   "locDCpass1",
+				   undef, $extra_smbconf_options);
+
+	return undef unless(defined $ret);
+	unless($self->add_wins_config("$prefix/private")) {
+		warn("Unable to add wins configuration");
+		return undef;
+	}
+
+	$ret->{DC_SERVER} = $ret->{SERVER};
+	$ret->{DC_SERVER_IP} = $ret->{SERVER_IP};
+	$ret->{DC_NETBIOSNAME} = $ret->{NETBIOSNAME};
+	$ret->{DC_USERNAME} = $ret->{USERNAME};
+	$ret->{DC_PASSWORD} = $ret->{PASSWORD};
+
+	return $ret;
+}
+
 sub teardown_env($$)
 {
 	my ($self, $envvars) = @_;
@@ -1272,6 +1260,7 @@ sub check_env($$)
 sub setup_env($$$)
 {
 	my ($self, $envname, $path) = @_;
+	my $target3 = $self->{target3};
 
 	$ENV{ENVNAME} = $envname;
 
@@ -1303,6 +1292,13 @@ sub setup_env($$$)
 			$self->setup_dc("$path/dc");
 		}
 		return $self->setup_rodc("$path/rodc", $self->{vars}->{dc});
+	} elsif ($envname eq "s3member") {
+		if (not defined($self->{vars}->{dc})) {
+			$self->setup_dc("$path/dc");
+		}
+		return $target3->setup_admember("$path/s3member", $self->{vars}->{dc}, 29);
+	} elsif ($envname eq "plugin_s4_dc") {
+		return $self->setup_plugin_s4_dc("$path/plugin_s4_dc");
 	} elsif ($envname eq "all") {
 		if (not defined($self->{vars}->{dc})) {
 			$ENV{ENVNAME} = "dc";
@@ -1349,6 +1345,18 @@ sub setup_env($$$)
 			$ret->{FL2008R2DC_USERNAME} = $fl2008r2dc_ret->{USERNAME};
 			$ret->{FL2008R2DC_PASSWORD} = $fl2008r2dc_ret->{PASSWORD};
 		}
+		if (not defined($self->{vars}->{s3member})) {
+			$ENV{ENVNAME} = "s3member";
+			my $s3member_ret = $target3->setup_admember("$path/s3member", $self->{vars}->{dc}, 29);
+			$self->{vars}->{s3member} = $s3member_ret;
+
+			$ret->{S3MEMBER_SERVER} = $s3member_ret->{SERVER};
+			$ret->{S3MEMBER_SERVER_IP} = $s3member_ret->{SERVER_IP};
+			$ret->{S3MEMBER_NETBIOSNAME} = $s3member_ret->{NETBIOSNAME};
+			$ret->{S3MEMBER_NETBIOSALIAS} = $s3member_ret->{NETBIOSALIAS};
+			$ret->{S3MEMBER_USERNAME} = $s3member_ret->{USERNAME};
+			$ret->{S3MEMBER_PASSWORD} = $s3member_ret->{PASSWORD};
+		}
 		return $ret;
 	} else {
 		return undef;
@@ -1467,7 +1475,7 @@ sub setup_vampire_dc($$$)
 
 		# force replicated DC to update repsTo/repsFrom
 		# for vampired partitions
-		my $samba_tool = $self->{bindir_path}->($self, "samba-tool");
+		my $samba_tool =  Samba::bindir_path($self, "samba-tool");
 		my $cmd = "";
 		$cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\"";
 		$cmd .= " KRB5_CONFIG=\"$env->{KRB5_CONFIG}\"";
@@ -1521,4 +1529,24 @@ sub setup_rodc($$$)
 	return $env;
 }
 
+sub setup_plugin_s4_dc($$)
+{
+	my ($self, $path) = @_;
+
+	my $env = $self->provision_plugin_s4_dc($path);
+	if (defined $env) {
+		$self->check_or_start($env);
+
+		$self->wait_for_start($env);
+
+		my $s3_part_env = $self->{target3}->setup_plugin_s4_dc($path, $env, 30);
+		if (not defined($s3_part_env)) {
+		    return undef;
+		}
+
+		$self->{vars}->{plugin_s4_dc} = $s3_part_env;
+	}
+	return $env;
+}
+
 1;
diff --git a/selftest/wscript b/selftest/wscript
index 25ea823ff7d..558cb2abd2e 100644
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -73,6 +73,19 @@ def set_options(opt):
 def configure(conf):
     conf.env.SELFTEST_PREFIX = Options.options.SELFTEST_PREFIX
 
+def combine_files(file1, file2, outfile):
+
+    f1 = open(file1)
+    f2 = open(file2)
+    of = open(outfile, mode='w')
+    for line in f1:
+        of.write(line)
+    for line in f2:
+        of.write(line)
+    f1.close()
+    f2.close()
+    of.close()
+
 def cmd_testonly(opt):
     '''run tests without doing a build first'''
     env = LOAD_ENVIRONMENT()
@@ -161,6 +174,9 @@ def cmd_testonly(opt):
     if env.USING_SYSTEM_LDB:
         os.environ['LDB_MODULES_PATH'] = 'bin/modules/ldb'
 
+    if env.BUILD_TDB2:
+        os.environ['BUILD_TDB2'] = '1'
+
     # tell build system where to find config.h
     os.environ['VFSLIBDIR'] = os.path.abspath('bin/modules/vfs')
     os.environ['CONFIG_H'] = 'bin/default/include/config.h'
@@ -175,7 +191,7 @@ def cmd_testonly(opt):
     if Options.options.TARGET:
         env.SELFTEST_TARGET = Options.options.TARGET
     else:
-        env.SELFTEST_TARGET = "samba4"
+        env.SELFTEST_TARGET = "samba"
 
     if env.SELFTEST_TARGET == "samba4":
         env.SELFTEST_DIR = "${srcdir}/source4/selftest"
@@ -183,17 +199,19 @@ def cmd_testonly(opt):
         env.SELFTEST_DIR = "${srcdir}/source3/selftest"
 
     if env.SELFTEST_TARGET == "samba":
-        env.SELFTEST_DIR = "${srcdir}/source3/selftest"
-        xfail = ""
-        xfail += EXPAND_VARIABLES(opt, env.FILTER_XFAIL)
-        xfail += " | "
-        env.SELFTEST_DIR = "${srcdir}/source4/selftest"
-        xfail += EXPAND_VARIABLES(opt, env.FILTER_XFAIL)
-        env.FILTER_XFAIL = xfail;
-        cmd = '(${PERL} ${srcdir}/selftest/selftest.pl --target=${SELFTEST_TARGET} --prefix=${SELFTEST_PREFIX} --srcdir=${srcdir} --exclude="${srcdir}/source4/selftest/skip" --exclude="${srcdir}/source3/selftest/skip" --testlist="${PYTHON} ${srcdir}/source3/selftest/tests.py|" --testlist="${PYTHON} ${srcdir}/source4/selftest/tests.py|" ${OPTIONS} --socket-wrapper ${TESTS} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS} | tee ${SELFTEST_PREFIX}/subunit'
+        for f in ["knownfail", "slow", "quick", "skip" ]:
+            combine_files(env.srcdir + "/source4/selftest/" + f,
+                          env.srcdir + "/source3/selftest/" + f,
+                          env.SELFTEST_PREFIX + "/" + f)
+
+        env.SELFTEST_DIR = env.SELFTEST_PREFIX
+        env.TESTLISTS = ('--testlist="${PYTHON} ${srcdir}/source3/selftest/tests.py|" ' +
+                         '--testlist="${PYTHON} ${srcdir}/source4/selftest/tests.py|"')
     else:
-        # We use the full path rather than relative path because it cause problems on some plateforms (ie. solaris 8).
-        cmd = '(${PERL} ${srcdir}/selftest/selftest.pl --target=${SELFTEST_TARGET} --prefix=${SELFTEST_PREFIX} --srcdir=${srcdir} --exclude=${SELFTEST_DIR}/skip --testlist="${PYTHON} ${SELFTEST_DIR}/tests.py|" ${OPTIONS} --socket-wrapper ${TESTS} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS} | tee ${SELFTEST_PREFIX}/subunit'
+        env.TESTLISTS = '--testlist="${SELFTEST_DIR}/tests.py|"'
+
+    # We use the full path rather than relative path because it cause problems on some plateforms (ie. solaris 8).
+    cmd = '(${PERL} ${srcdir}/selftest/selftest.pl --target=${SELFTEST_TARGET} --prefix=${SELFTEST_PREFIX} --srcdir=${srcdir} --exclude=${SELFTEST_DIR}/skip ${TESTLISTS} ${OPTIONS} --socket-wrapper ${TESTS} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS} | tee ${SELFTEST_PREFIX}/subunit'
 
     if os.environ.get('RUN_FROM_BUILD_FARM') is None and not Options.options.FILTERED_SUBUNIT:
         cmd += ' | ${FORMAT_TEST_OUTPUT}'
diff --git a/source3/.clang_complete b/source3/.clang_complete
index c3f2f30dd7d..46925f9b8a2 100644
--- a/source3/.clang_complete
+++ b/source3/.clang_complete
@@ -1,2 +1,22 @@
--I. -I./.. -Iinclude -I./../lib/replace -I./../lib/tevent -I./librpc -I./../lib/talloc -I../lib/tdb/include
--Iinclude/includes.h.gch
+-I.
+-I./..
+-I./../lib
+-I./../lib/replace
+-I./../lib/talloc
+-I./../lib/tevent
+-I./../lib/popt
+-I./../lib/iniparser/src
+-I./../lib/popt
+-I./../lib/tdb/include
+-I./../lib/tdb_compat
+-I./include/autoconf
+-I./include
+-I./librpc
+-I./lib
+-DDEBUG_PASSWORD
+-DDEVELOPER
+-DHAVE_CONFIG_H
+-D_GNU_SOURCE
+-D_SAMBA_BUILD_=3
+-D_REENTRANT
+-DUSING_SMBCONTROL
diff --git a/source3/Makefile-smbtorture4 b/source3/Makefile-smbtorture4
index 4a1bed7eabf..0dc4706944f 100644
--- a/source3/Makefile-smbtorture4
+++ b/source3/Makefile-smbtorture4
@@ -6,7 +6,7 @@ SAMBA4_BINARIES="smbtorture,ndrdump"
 samba4-configure:
 	@(cd .. && \
 		CFLAGS='' $(WAF) reconfigure || \
-		CFLAGS='' $(WAF) configure --enable-socket-wrapper --enable-nss-wrapper --enable-uid-wrapper --nonshared-binary=$(SAMBA4_BINARIES) --enable-auto-reconfigure )
+		CFLAGS='' $(WAF) configure --enable-socket-wrapper --enable-nss-wrapper --enable-uid-wrapper --nonshared-binary=$(SAMBA4_BINARIES) --enable-auto-reconfigure --enable-developer)
 
 .PHONY: samba4-configure
 
diff --git a/source3/Makefile.in b/source3/Makefile.in
index e503112392a..59f9032ea2b 100644
--- a/source3/Makefile.in
+++ b/source3/Makefile.in
@@ -97,6 +97,7 @@ TDBTOOL = @TDBTOOL@
 TDBDUMP = @TDBDUMP@
 TDBRESTORE = @TDBRESTORE@
 TDBTORTURE = @TDBTORTURE@
+PTHREADPOOLTEST = @PTHREADPOOLTEST@
 
 INSTALLCMD=@INSTALL@
 INSTALLLIBCMD_SH=@INSTALLLIBCMD_SH@
@@ -139,6 +140,7 @@ CONFIGDIR = @configdir@
 VARDIR = @localstatedir@
 MANDIR = @mandir@
 DATADIR = @datadir@
+SETUPDIR = @datadir@/setup
 
 # The permissions to give the executables and other data
 INSTALLPERMS_BIN = 0755
@@ -209,7 +211,9 @@ PATH_FLAGS = -DSMB_PASSWD_FILE=\"$(SMB_PASSWD_FILE)\" \
 	-DCACHEDIR=\"$(CACHEDIR)\" \
 	-DSTATEDIR=\"$(STATEDIR)\" \
 	-DNMBDSOCKETDIR=\"$(NMBDSOCKETDIR)\" \
-	-DLOCALEDIR=\"$(LOCALEDIR)\"
+	-DLOCALEDIR=\"$(LOCALEDIR)\" \
+	-DDATADIR=\"$(DATADIR)\" \
+	-DSETUPDIR=\"$(SETUPDIR)\"
 
 # Note that all executable programs now provide for an optional executable suffix.
 
@@ -230,7 +234,7 @@ TORTURE_PROGS = bin/smbtorture@EXEEXT@ bin/msgtest@EXEEXT@ \
 	bin/masktest@EXEEXT@ bin/locktest@EXEEXT@ \
 	bin/locktest2@EXEEXT@ bin/nsstest@EXEEXT@ bin/vfstest@EXEEXT@ \
 	bin/pdbtest@EXEEXT@ $(TALLOCTORT) bin/replacetort@EXEEXT@ \
-	$(TDBTORTURE) \
+	$(TDBTORTURE) $(PTHREADPOOLTEST) \
 	bin/smbconftort@EXEEXT@ bin/vlp@EXEEXT@
 
 BIN_PROGS = @EXTRA_BIN_PROGS@ \
@@ -268,6 +272,7 @@ EXTRA_ALL_TARGETS = @EXTRA_ALL_TARGETS@
 ######################################################################
 
 TDB_LIB_OBJ = lib/util_tdb.o ../lib/util/util_tdb.o \
+	  ../lib/util/tdb_wrap.o \
 	  lib/dbwrap.o lib/dbwrap_tdb.o \
 	  lib/dbwrap_ctdb.o \
 	  lib/g_lock.o \
@@ -317,7 +322,7 @@ LIBNDR_OBJ = ../librpc/ndr/ndr_basic.o \
 	     ../librpc/ndr/ndr_sec_helper.o \
 	     ../librpc/ndr/ndr_string.o \
 	     ../librpc/ndr/uuid.o \
-	     librpc/ndr/util.o \
+	     ../librpc/ndr/util.o \
 	     librpc/gen_ndr/ndr_server_id.o \
 	     librpc/gen_ndr/ndr_dcerpc.o
 
@@ -416,7 +421,7 @@ LIBSAMBAUTIL_OBJ = \
 UTIL_OBJ = ../lib/util/rbtree.o ../lib/util/signal.o ../lib/util/time.o \
 		   ../lib/util/xfile.o ../lib/util/util_strlist.o  \
 		   ../lib/util/util_file.o ../lib/util/data_blob.o \
-		   ../lib/util/util.o ../lib/util/fsusage.o \
+		   ../lib/util/util.o ../lib/util/util_paths.o ../lib/util/fsusage.o \
 		   ../lib/util/params.o ../lib/util/talloc_stack.o \
 		   ../lib/util/genrand.o ../lib/util/util_net.o \
 		   ../lib/util/become_daemon.o ../lib/util/system.o \
@@ -424,7 +429,7 @@ UTIL_OBJ = ../lib/util/rbtree.o ../lib/util/signal.o ../lib/util/time.o \
 		   ../lib/util/tevent_werror.o \
 		   ../lib/util/smb_threads.o ../lib/util/util_id.o \
 		   ../lib/util/blocking.o ../lib/util/rfc1738.o \
-		   ../lib/util/select.o ../lib/util/util_pw.o
+		   ../lib/util/select.o ../lib/util/util_pw.o ../lib/util/server_id.o
 
 CRYPTO_OBJ = ../lib/crypto/crc32.o ../lib/crypto/md5.o \
 			 ../lib/crypto/hmacmd5.o ../lib/crypto/arcfour.o \
@@ -434,38 +439,43 @@ CRYPTO_OBJ = ../lib/crypto/crc32.o ../lib/crypto/md5.o \
 
 LIB_OBJ = $(LIBSAMBAUTIL_OBJ) $(UTIL_OBJ) $(CRYPTO_OBJ) \
 	  lib/messages.o librpc/gen_ndr/ndr_messaging.o lib/messages_local.o \
-	  lib/messages_ctdbd.o lib/packet.o lib/ctdbd_conn.o \
-	  lib/interfaces.o lib/memcache.o \
+	  lib/messages_ctdbd.o lib/ctdb_packet.o lib/ctdbd_conn.o \
+	  ../lib/socket/interfaces.o lib/memcache.o \
 	  lib/talloc_dict.o \
 	  lib/serverid.o \
 	  lib/util_sconn.o \
 	  lib/util_transfer_file.o ../lib/async_req/async_sock.o \
 	  lib/addrchange.o \
 	  $(TDB_LIB_OBJ) \
-	  $(VERSION_OBJ) lib/charcnv.o ../lib/util/charset/convert_string.o lib/fstring.o ../lib/util/debug.o ../lib/util/debug_s3.o ../lib/util/fault.o \
+	  $(VERSION_OBJ) lib/charcnv.o ../lib/util/charset/convert_string.o \
+	  ../lib/util/charset/pull_push.o \
+	  lib/fstring.o ../lib/util/debug.o ../lib/util/debug_s3.o ../lib/util/fault.o \
 	  lib/interface.o lib/pidfile.o lib/dumpcore.o \
 	  lib/system.o lib/sendfile.o lib/recvfile.o lib/time.o \
 	  lib/username.o \
 	  ../libds/common/flag_mapping.o \
 	  lib/access.o lib/smbrun.o \
-	  lib/bitmap.o lib/dprintf.o $(UTIL_REG_OBJ) \
-	  lib/wins_srv.o \
+	  lib/bitmap.o ../lib/util/dprintf.o $(UTIL_REG_OBJ) \
+	  lib/wins_srv.o lib/string_init.o \
 	  lib/util_str.o ../lib/util/util_str_common.o \
+	  ../lib/util/util_str.o \
 	  ../lib/util/base64.o lib/util_sid.o \
+	  ../lib/util/charset/util_unistr.o \
 	  ../lib/util/charset/util_unistr_w.o ../lib/util/charset/codepoints.o ../lib/util/charset/util_str.o lib/util_file.o \
-	  lib/util.o lib/util_cmdline.o lib/util_names.o \
+	  lib/util.o lib/util_malloc.o lib/namearray.o lib/util_cmdline.o lib/util_names.o \
 	  lib/util_sock.o lib/sock_exec.o lib/util_sec.o \
-	  lib/substitute.o lib/dbwrap_util.o \
-	  lib/ms_fnmatch.o lib/errmap_unix.o \
+	  lib/substitute.o lib/substitute_generic.o ../lib/util/substitute.o lib/dbwrap_util.o \
+	  lib/ms_fnmatch.o ../lib/util/ms_fnmatch.o lib/errmap_unix.o ../libcli/util/errmap_unix.o \
 	  lib/tallocmsg.o lib/dmallocmsg.o \
 	  libsmb/clisigning.o libsmb/smb_signing.o \
 	  ../lib/util/charset/iconv.o intl/lang_tdb.o \
 	  lib/conn_tdb.o lib/adt_tree.o lib/gencache.o \
 	  lib/sessionid_tdb.o \
 	  lib/module.o lib/events.o @LIBTEVENT_OBJ0@ \
+	  @CCAN_OBJ@ \
 	  lib/server_contexts.o \
 	  lib/ldap_escape.o @CHARSET_STATIC@ \
-	  lib/secdesc.o ../libcli/security/access_check.o \
+	  ../libcli/security/secdesc.o ../libcli/security/access_check.o \
 	  ../libcli/security/secace.o ../libcli/security/object_tree.o \
 	  ../libcli/security/sddl.o \
 	  ../libcli/security/secacl.o @PTHREADPOOL_OBJ@ \
@@ -473,9 +483,10 @@ LIB_OBJ = $(LIBSAMBAUTIL_OBJ) $(UTIL_OBJ) $(CRYPTO_OBJ) \
 	  libads/krb5_errs.o lib/system_smbd.o lib/audit.o $(LIBNDR_OBJ) \
 	  lib/file_id.o lib/idmap_cache.o \
 	  ../libcli/security/dom_sid.o ../libcli/security/security_descriptor.o \
-	  ../libcli/security/security_token.o ../libcli/security/util_sid.o
+	  ../libcli/security/security_token.o ../libcli/security/util_sid.o \
+	  ../libcli/smb/util.o
 
-LIB_DUMMY_OBJ = lib/dummysmbd.o lib/dummyroot.o
+LIB_DUMMY_OBJ = lib/smbd_shim.o lib/dummyparam.o
 LIB_NONSMBD_OBJ = $(LIB_OBJ) $(LIB_DUMMY_OBJ)
 
 READLINE_OBJ = ../libcli/smbreadline/smbreadline.o
@@ -484,7 +495,7 @@ READLINE_OBJ = ../libcli/smbreadline/smbreadline.o
 # Be sure to include them into your application
 POPT_LIB_OBJ = lib/popt_common.o
 
-PARAM_WITHOUT_REG_OBJ = dynconfig.o param/loadparm.o param/loadparm_server_role.o param/util.o lib/sharesec.o lib/ldap_debug_handler.o
+PARAM_WITHOUT_REG_OBJ = ../dynconfig/dynconfig.o param/loadparm.o param/loadparm_server_role.o param/util.o lib/sharesec.o lib/ldap_debug_handler.o
 PARAM_REG_ADD_OBJ = $(REG_SMBCONF_OBJ) $(LIBSMBCONF_OBJ) $(PRIVILEGES_BASIC_OBJ)
 PARAM_OBJ = $(PARAM_WITHOUT_REG_OBJ) $(PARAM_REG_ADD_OBJ)
 
@@ -511,7 +522,7 @@ LIBADS_OBJ = libads/ldap.o \
              libads/disp_sec.o libads/ldap_utils.o \
 	     libads/ldap_schema.o libads/util.o libads/ndr.o
 
-LIBADS_SERVER_OBJ = libads/kerberos_verify.o libads/authdata.o ../libcli/auth/kerberos_pac.o \
+LIBADS_SERVER_OBJ = libads/kerberos_verify.o libads/authdata.o ../auth/kerberos/kerberos_pac.o \
 		    ../librpc/ndr/ndr_krb5pac.o \
 		    librpc/gen_ndr/ndr_krb5pac.o
 
@@ -528,12 +539,12 @@ LIBNBT_OBJ = ../libcli/nbt/nbtname.o \
 	     ../librpc/ndr/ndr_svcctl.o
 
 LIBNMB_OBJ = libsmb/unexpected.o libsmb/namecache.o libsmb/nmblib.o \
-	     libsmb/namequery.o ../libcli/nbt/lmhosts.o libsmb/conncache.o \
+	     libsmb/namequery.o ../libcli/nbt/lmhosts.o ../libcli/dns/dns_hosts_file.o libsmb/conncache.o \
 	     libads/dns.o libads/sitename_cache.o
 
-NTERR_OBJ = libsmb/nterr.o libsmb/smberr.o
+NTERR_OBJ = ../libcli/util/nterr.o libsmb/smberr.o
 DOSERR_OBJ = ../libcli/util/doserr.o
-ERRORMAP_OBJ = libsmb/errormap.o
+ERRORMAP_OBJ = ../libcli/util/errormap.o libsmb/errormap.o
 DCE_RPC_ERR_OBJ = ../librpc/rpc/dcerpc_error.o
 
 LIBSMB_ERR_OBJ0 = $(NTERR_OBJ) $(DOSERR_OBJ) $(ERRORMAP_OBJ) $(DCE_RPC_ERR_OBJ)
@@ -585,6 +596,7 @@ LIBSMB_OBJ = libsmb/clientgen.o libsmb/cliconnect.o libsmb/clifile.o \
 	     libsmb/clistr.o libsmb/cliquota.o libsmb/clifsinfo.o libsmb/clidfs.o \
 	     libsmb/clioplock.o libsmb/clirap2.o \
 	     libsmb/smb_seal.o libsmb/async_smb.o \
+	     libsmb/read_smb.o \
 	     libsmb/cli_np_tstream.o \
 	     libsmb/smbsock_connect.o \
 	     $(LIBSAMBA_OBJ) \
@@ -597,6 +609,7 @@ LIBMSRPC_OBJ = $(SCHANNEL_OBJ) \
 	       rpc_client/cli_pipe.o \
 	       librpc/crypto/gse_krb5.o \
 	       librpc/crypto/gse.o \
+	       ../auth/kerberos/gssapi_pac.o \
 	       librpc/crypto/cli_spnego.o \
 	       librpc/rpc/rpc_common.o \
 	       rpc_client/rpc_transport_np.o \
@@ -715,7 +728,8 @@ NPA_TSTREAM_OBJ = ../libcli/named_pipe_auth/npa_tstream.o \
 		  ../auth/auth_sam_reply.o librpc/gen_ndr/ndr_auth.o ../librpc/ndr/ndr_auth.o
 
 RPC_NCACN_NP = rpc_server/srv_pipe_register.o rpc_server/rpc_ncacn_np.o \
-	       rpc_server/rpc_handles.o rpc_server/srv_access_check.o
+	       rpc_server/rpc_handles.o rpc_server/rpc_contexts.o \
+	       rpc_server/srv_access_check.o
 
 RPC_SERVICE = rpc_server/rpc_server.o
 
@@ -767,10 +781,10 @@ PASSDB_OBJ = $(PASSDB_GET_SET_OBJ) passdb/passdb.o passdb/pdb_interface.o \
 		lib/util_nscd.o lib/winbind_util.o $(SERVER_MUTEX_OBJ) \
 		passdb/pdb_util.o
 
-DEVEL_HELP_WEIRD_OBJ = modules/weird.o
-CP850_OBJ = modules/CP850.o
-CP437_OBJ = modules/CP437.o
-CHARSET_MACOSXFS_OBJ = modules/charset_macosxfs.o
+DEVEL_HELP_WEIRD_OBJ = ../lib/util/charset/weird.o
+CP850_OBJ = ../lib/util/charset/CP850.o
+CP437_OBJ = ../lib/util/charset/CP437.o
+CHARSET_MACOSXFS_OBJ = ../lib/util/charset/charset_macosxfs.o
 
 GROUPDB_OBJ = groupdb/mapping.o groupdb/mapping_tdb.o
 
@@ -820,6 +834,7 @@ VFS_CACHEPRIME_OBJ = modules/vfs_cacheprime.o
 VFS_PREALLOC_OBJ = modules/vfs_prealloc.o
 VFS_COMMIT_OBJ = modules/vfs_commit.o
 VFS_GPFS_OBJ = modules/vfs_gpfs.o modules/gpfs.o modules/nfs4_acls.o
+VFS_GPFS_HSM_NOTIFY_OBJ = modules/vfs_gpfs_hsm_notify.o
 VFS_NOTIFY_FAM_OBJ = modules/vfs_notify_fam.o
 VFS_READAHEAD_OBJ = modules/vfs_readahead.o
 VFS_TSMSM_OBJ = modules/vfs_tsmsm.o
@@ -877,7 +892,7 @@ BUILDOPT_OBJ = smbd/build_options.o
 SMBD_OBJ_SRV = smbd/server_reload.o \
 	       smbd/files.o smbd/connection.o \
 	       smbd/utmp.o smbd/session.o \
-               smbd/dfree.o smbd/dir.o smbd/password.o smbd/conn.o \
+               smbd/dfree.o smbd/dir.o smbd/password.o smbd/conn.o smbd/conn_idle.o smbd/conn_msg.o \
 	       smbd/share_access.o smbd/fileio.o \
                smbd/ipc.o smbd/lanman.o smbd/negprot.o \
                smbd/message.o smbd/nttrans.o smbd/pipes.o \
@@ -886,12 +901,13 @@ SMBD_OBJ_SRV = smbd/server_reload.o \
 	       smbd/blocking.o smbd/sec_ctx.o smbd/srvstr.o \
 	       smbd/vfs.o smbd/perfcount.o smbd/statcache.o smbd/seal.o \
                smbd/posix_acls.o lib/sysacls.o \
-	       smbd/process.o smbd/service.o smbd/error.o \
+	       smbd/process.o smbd/service.o param/service.o smbd/error.o \
 	       rpc_server/epmd.o \
 	       printing/printspoolss.o printing/spoolssd.o \
 	       lib/sysquotas.o lib/sysquotas_linux.o \
 	       lib/sysquotas_xfs.o lib/sysquotas_4A.o \
 	       lib/sysquotas_nfs.o \
+	       lib/smbd_shim.o \
 	       smbd/fake_file.o \
 	       smbd/quotas.o smbd/ntquotas.o $(AFS_OBJ) smbd/msdfs.o \
 	       $(AFS_SETTOKEN_OBJ) smbd/aio.o smbd/statvfs.o \
@@ -1023,6 +1039,7 @@ PDBEDIT_OBJ = utils/pdbedit.o $(PASSWD_UTIL_OBJ) $(PARAM_OBJ) $(PASSDB_OBJ) \
 		$(LIBSAMBA_OBJ) $(LIBTSOCKET_OBJ) \
 		$(LIB_NONSMBD_OBJ) $(GROUPDB_OBJ) \
 		$(LIBCLI_LDAP_NDR_OBJ) \
+		$(DRSUAPI_OBJ) $(LIBNDR_GEN_OBJ0) \
 		$(POPT_LIB_OBJ) $(SMBLDAP_OBJ) ../lib/util/asn1.o
 
 SMBGET_OBJ = utils/smbget.o $(POPT_LIB_OBJ) $(LIBSMBCLIENT_OBJ1)
@@ -1129,6 +1146,9 @@ SMBCONFTORT_OBJ = $(SMBCONFTORT_OBJ0) \
 		  $(LIBSMB_ERR_OBJ) \
 		  $(POPT_LIB_OBJ)
 
+PTHREADPOOLTEST_OBJ = lib/pthreadpool/pthreadpool.o \
+		lib/pthreadpool/tests.o
+
 LIBNET_OBJ = libnet/libnet_join.o \
 	     libnet/libnet_keytab.o \
 	     librpc/gen_ndr/ndr_libnet_join.o
@@ -1214,6 +1234,8 @@ SMBTORTURE_OBJ1 = torture/torture.o torture/nbio.o torture/scanner.o torture/uta
 		torture/nbench.o \
 		torture/test_async_echo.o \
 		torture/test_smbsock_any_connect.o \
+		torture/test_nttrans_create.o \
+		torture/test_notify_online.o \
 		torture/test_addrchange.o \
 		torture/test_case_insensitive.o \
 		torture/test_posix_append.o
@@ -1302,6 +1324,7 @@ PAM_SMBPASS_OBJ_0 = pam_smbpass/pam_smb_auth.o pam_smbpass/pam_smb_passwd.o \
 		pam_smbpass/pam_smb_acct.o pam_smbpass/support.o ../lib/util/asn1.o
 PAM_SMBPASS_OBJ = $(PAM_SMBPASS_OBJ_0) $(PARAM_OBJ) $(LIB_NONSMBD_OBJ) $(PASSDB_OBJ) $(GROUPDB_OBJ) \
 		$(SMBLDAP_OBJ) $(LIBSAMBA_OBJ) \
+		$(DRSUAPI_OBJ) $(LIBNDR_GEN_OBJ0) \
 		$(LIBTSOCKET_OBJ) $(PAM_ERRORS_OBJ)
 
 IDMAP_RW_OBJ = winbindd/idmap_rw.o
@@ -1330,7 +1353,8 @@ WINBINDD_OBJ1 = \
 		winbindd/winbindd_pam.o   \
 		winbindd/winbindd_misc.o  \
 		winbindd/winbindd_cm.o    \
-		winbindd/winbindd_wins.o  \
+		winbindd/winbindd_wins_byip.o  \
+		winbindd/winbindd_wins_byname.o  \
 		winbindd/winbindd_msrpc.o \
 		winbindd/winbindd_rpc.o   \
 		winbindd/winbindd_reconnect.o \
@@ -1483,6 +1507,7 @@ NTLM_AUTH_OBJ = ${NTLM_AUTH_OBJ1} $(LIBSAMBA_OBJ) $(POPT_LIB_OBJ) \
 		$(SMBLDAP_OBJ) $(LIBNMB_OBJ) \
 		$(WBCOMMON_OBJ) \
 		$(LIBCLI_LDAP_NDR_OBJ) \
+		$(DRSUAPI_OBJ) \
 		$(LIBNDR_GEN_OBJ0) $(LIBNDR_NETLOGON_OBJ) @BUILD_INIPARSER@
 
 
@@ -1567,6 +1592,8 @@ replacetort : SHOWFLAGS bin/replacetort@EXEEXT@
 
 smbconftort : SHOWFLAGS bin/smbconftort@EXEEXT@
 
+pthreadpooltest : SHOWFLAGS bin/pthreadpooltest@EXEEXT@
+
 timelimit : SHOWFLAGS bin/timelimit@EXEEXT@
 
 nsswitch : SHOWFLAGS bin/winbindd@EXEEXT@ bin/wbinfo@EXEEXT@ @WINBIND_NSS@ \
@@ -1583,12 +1610,10 @@ modules:: SHOWFLAGS $(MODULES)
 
 IDL_FILES = librpc/idl/messaging.idl \
 	    librpc/idl/libnetapi.idl \
-	    librpc/idl/notify.idl \
 	    librpc/idl/wbint.idl \
 	    librpc/idl/perfcount.idl \
 	    librpc/idl/secrets.idl \
-	    librpc/idl/libnet_join.idl \
-	    librpc/idl/server_id.idl
+	    librpc/idl/libnet_join.idl
 
 samba3-idl::
 	@PIDL_OUTPUTDIR="librpc/gen_ndr" PIDL_ARGS="$(PIDL_ARGS)" CPP="$(CPP)" PIDL="../pidl/pidl" \
@@ -1690,7 +1715,7 @@ BINARY_PREREQS = bin/.dummy
 # but since we also require "make install prefix=/opt/samba" *not* to
 # rebuild it's a bit hard.
 
-dynconfig.o: dynconfig.c Makefile
+../dynconfig/dynconfig.o: ../dynconfig/dynconfig.c Makefile
 	@echo Compiling $*.c
 	@$(COMPILE_CC_PATH) && exit 0;\
 		echo "The following command failed:" 1>&2;\
@@ -1748,16 +1773,16 @@ lib/version.o: lib/version.c include/version.h
 		echo "$(COMPILE_CC_PATH)" 1>&2;\
 		$(COMPILE_CC_PATH) >/dev/null 2>&1
 
-smbd/build_options.o: smbd/build_options.c Makefile include/config.h include/build_env.h include/proto.h
+smbd/build_options.o: smbd/build_options.c Makefile include/autoconf/config.h include/build_env.h include/proto.h
 	@echo Compiling $*.c
 	@$(COMPILE_CC_PATH) && exit 0;\
 		echo "The following command failed:" 1>&2;\
 		echo "$(COMPILE_CC_PATH)" 1>&2;\
 		$(COMPILE_CC_PATH) >/dev/null 2>&1
 
-smbd/build_options.c: include/config.h.in script/mkbuildoptions.awk
+smbd/build_options.c: include/autoconf/config.h.in script/mkbuildoptions.awk
 	@echo Generating $@
-	@dir=smbd $(MAKEDIR) && $(AWK) -f $(srcdir)/script/mkbuildoptions.awk > $(builddir)/smbd/build_options.c < $(srcdir)/include/config.h.in
+	@dir=smbd $(MAKEDIR) && $(AWK) -f $(srcdir)/script/mkbuildoptions.awk > $(builddir)/smbd/build_options.c < $(srcdir)/include/autoconf/config.h.in
 
 bin/.dummy:
 	@if (: >> $@ || : > $@) >/dev/null 2>&1; then :; else \
@@ -1852,11 +1877,11 @@ bin/smbpasswd@EXEEXT@: $(BINARY_PREREQS) $(SMBPASSWD_OBJ) @BUILD_POPT@ $(LIBTALL
 		$(DYNEXP) $(LIBS) $(POPT_LIBS) $(KRB5LIBS) $(LDAP_LIBS) \
 		$(LIBTALLOC_LIBS) $(LIBTDB_LIBS) $(LIBWBCLIENT_LIBS) $(ZLIB_LIBS)
 
-bin/pdbedit@EXEEXT@: $(BINARY_PREREQS) $(PDBEDIT_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(LIBTDB) $(LIBWBCLIENT)
+bin/pdbedit@EXEEXT@: $(BINARY_PREREQS) $(PDBEDIT_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(LIBTDB) $(LIBWBCLIENT) $(ZLIB_LIBS)
 	@echo Linking $@
 	@$(CC) -o $@ $(PDBEDIT_OBJ) $(LDFLAGS) $(DYNEXP) $(LIBS) \
 		$(POPT_LIBS) $(PASSDB_LIBS) $(LDAP_LIBS) $(LIBTALLOC_LIBS) \
-		$(LIBTDB_LIBS) $(LIBWBCLIENT_LIBS)
+		$(LIBTDB_LIBS) $(LIBWBCLIENT_LIBS) $(ZLIB_LIBS)
 
 bin/smbget@EXEEXT@: $(BINARY_PREREQS) $(SMBGET_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(LIBTDB) $(LIBWBCLIENT)
 	@echo Linking $@
@@ -1892,6 +1917,10 @@ bin/smbconftort@EXEEXT@: $(SMBCONFTORT_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(LIBTDB)
 		$(DYNEXP) $(LIBS) $(LDAP_LIBS) $(POPT_LIBS) \
 		$(LIBTALLOC_LIBS) $(LIBTDB_LIBS)
 
+bin/pthreadpooltest@EXEEXT@: $(PTHREADPOOLTEST_OBJ)
+	@echo Linking $@
+	@$(CC) -o $@ $(PTHREADPOOLTEST_OBJ) $(LDFLAGS) -lpthread
+
 bin/masktest@EXEEXT@: $(BINARY_PREREQS) $(MASKTEST_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(LIBTDB) $(LIBWBCLIENT)
 	@echo Linking $@
 	@$(CC) -o $@ $(MASKTEST_OBJ) $(LDFLAGS) $(DYNEXP) \
@@ -2008,7 +2037,9 @@ LIBTALLOC_OBJ = $(LIBTALLOC_OBJ0) $(LIBREPLACE_OBJ)
 
 LIBTALLOC_SHARED_TARGET=@LIBTALLOC_SHARED_TARGET@
 LIBTALLOC_SOVER=@LIBTALLOC_SOVER@
+LIBTALLOC_FULLVER=@LIBTALLOC_FULLVER@
 LIBTALLOC_SHARED_TARGET_SONAME=$(LIBTALLOC_SHARED_TARGET).$(LIBTALLOC_SOVER)
+LIBTALLOC_SHARED_TARGET_FULLVER=$(LIBTALLOC_SHARED_TARGET).$(LIBTALLOC_FULLVER)
 LIBTALLOC_STATIC_TARGET=@LIBTALLOC_STATIC_TARGET@
 LIBTALLOC_SYMS=$(srcdir)/exports/libtalloc.@SYMSEXT@
 LIBTALLOC_HEADERS=$(srcdir)/@tallocdir@/talloc.h
@@ -2020,7 +2051,12 @@ $(LIBTALLOC_SHARED_TARGET_SONAME): $(BINARY_PREREQS) $(LIBTALLOC_OBJ) $(LIBTALLO
 	@echo Linking shared library $@
 	@$(SHLD_DSO) $(LIBTALLOC_OBJ) $(LIBREPLACE_LIBS) @SONAMEFLAG@`basename $@`
 
-$(LIBTALLOC_SHARED_TARGET): $(LIBTALLOC_SHARED_TARGET_SONAME)
+$(LIBTALLOC_SHARED_TARGET_FULLVER): $(LIBTALLOC_SHARED_TARGET_SONAME)
+	@rm -f $@
+	@mv -f $(LIBTALLOC_SHARED_TARGET_SONAME) $@
+	@ln -f -s `basename $@` $(LIBTALLOC_SHARED_TARGET_SONAME)
+
+$(LIBTALLOC_SHARED_TARGET): $(LIBTALLOC_SHARED_TARGET_FULLVER)
 	@rm -f $@
 	@ln -f -s `basename $(LIBTALLOC_SHARED_TARGET_SONAME)` $@
 
@@ -2038,8 +2074,13 @@ cleanlibtalloc::
 
 installlibtalloc:: installdirs libtalloc
 	@$(SHELL) $(srcdir)/script/installdirs.sh $(INSTALLPERMS_BIN) $(DESTDIR) $(LIBDIR)
-	-$(INSTALLLIBCMD_SH) $(LIBTALLOC_SHARED_TARGET_SONAME) $(DESTDIR)$(LIBDIR)
+	-$(INSTALLLIBCMD_SH) $(LIBTALLOC_SHARED_TARGET_FULLVER) $(DESTDIR)$(LIBDIR)
 	@rm -f $(DESTDIR)$(LIBDIR)/`basename $(LIBTALLOC_SHARED_TARGET)`
+	@rm -f $(DESTDIR)$(LIBDIR)/`basename $(LIBTALLOC_SHARED_TARGET_SONAME)`
+	-if test -r $(LIBTALLOC_SHARED_TARGET_FULLVER) ; then \
+		ln -f -s `basename $(LIBTALLOC_SHARED_TARGET_FULLVER)` \
+			$(DESTDIR)$(LIBDIR)/`basename $(LIBTALLOC_SHARED_TARGET_SONAME)` ; \
+	fi
 	-if test -r $(LIBTALLOC_SHARED_TARGET_SONAME) ; then \
 		ln -f -s `basename $(LIBTALLOC_SHARED_TARGET_SONAME)` \
 			$(DESTDIR)$(LIBDIR)/`basename $(LIBTALLOC_SHARED_TARGET)` ; \
@@ -2062,15 +2103,17 @@ shlibs test_shlibs: @LIBTALLOC_SHARED@
 
 showlibtalloc:
 	@echo  ""
-	@echo "LIBTALLOC_SHARED               = @LIBTALLOC_SHARED@"
-	@echo "LIBTALLOC_STATIC               = @LIBTALLOC_STATIC@"
-	@echo "LIBTALLOC_LIBS                 = $(LIBTALLOC_LIBS)"
+	@echo "LIBTALLOC_SHARED                = @LIBTALLOC_SHARED@"
+	@echo "LIBTALLOC_STATIC                = @LIBTALLOC_STATIC@"
+	@echo "LIBTALLOC_LIBS                  = $(LIBTALLOC_LIBS)"
 	@echo ""
-	@echo "LIBTALLOC_SHARED_TARGET        = $(LIBTALLOC_SHARED_TARGET)"
-	@echo "LIBTALLOC_SOVER                = $(LIBTALLOC_SOVER)"
-	@echo "LIBTALLOC_SHARED TARGET_SONAME = $(LIBTALLOC_SHARED_TARGET_SONAME)"
-	@echo "LIBTALLOC_STATIC_TARGET        = $(LIBTALLOC_STATIC_TARGET)"
-	@echo "LIBTALLOC                      = $(LIBTALLOC)"
+	@echo "LIBTALLOC_SHARED_TARGET         = $(LIBTALLOC_SHARED_TARGET)"
+	@echo "LIBTALLOC_SOVER                 = $(LIBTALLOC_SOVER)"
+	@echo "LIBTALLOC_FULLVER               = $(LIBTALLOC_FULLVER)"
+	@echo "LIBTALLOC_SHARED TARGET_SONAME  = $(LIBTALLOC_SHARED_TARGET_SONAME)"
+	@echo "LIBTALLOC_SHARED TARGET_FULLVER = $(LIBTALLOC_SHARED_TARGET_FULLVER)"
+	@echo "LIBTALLOC_STATIC_TARGET         = $(LIBTALLOC_STATIC_TARGET)"
+	@echo "LIBTALLOC                       = $(LIBTALLOC)"
 
 showlibs:: showlibtalloc
 
@@ -2085,7 +2128,9 @@ LIBTDB_OBJ = $(LIBTDB_OBJ0) $(LIBREPLACE_OBJ)
 
 LIBTDB_SHARED_TARGET=@LIBTDB_SHARED_TARGET@
 LIBTDB_SOVER=@LIBTDB_SOVER@
+LIBTDB_FULLVER=@LIBTDB_FULLVER@
 LIBTDB_SHARED_TARGET_SONAME=$(LIBTDB_SHARED_TARGET).$(LIBTDB_SOVER)
+LIBTDB_SHARED_TARGET_FULLVER=$(LIBTDB_SHARED_TARGET).$(LIBTDB_FULLVER)
 LIBTDB_STATIC_TARGET=@LIBTDB_STATIC_TARGET@
 LIBTDB_SYMS=$(srcdir)/exports/libtdb.@SYMSEXT@
 LIBTDB_HEADERS=$(srcdir)/@tdbdir@/include/tdb.h
@@ -2093,12 +2138,18 @@ LIBTDB_HEADERS=$(srcdir)/@tdbdir@/include/tdb.h
 $(LIBTDB_SYMS): $(LIBTDB_HEADERS)
 	@$(MKSYMS_SH) $(AWK) $@ $(LIBTDB_HEADERS)
 
+
 $(LIBTDB_SHARED_TARGET_SONAME): $(BINARY_PREREQS) $(LIBTDB_OBJ) $(LIBTDB_SYMS)
 	@echo Linking shared library $@
 	@$(SHLD_DSO) $(LIBTDB_OBJ) $(LIBREPLACE_LIBS) $(TDB_DEPS) \
 		@SONAMEFLAG@`basename $@`
 
-$(LIBTDB_SHARED_TARGET): $(LIBTDB_SHARED_TARGET_SONAME)
+$(LIBTDB_SHARED_TARGET_FULLVER): $(LIBTDB_SHARED_TARGET_SONAME)
+	@rm -f $@
+	@mv -f $(LIBTDB_SHARED_TARGET_SONAME) $(LIBTDB_SHARED_TARGET_FULLVER)
+	@ln -f -s `basename $(LIBTDB_SHARED_TARGET_FULLVER)` $(LIBTDB_SHARED_TARGET_SONAME)
+
+$(LIBTDB_SHARED_TARGET): $(LIBTDB_SHARED_TARGET_FULLVER)
 	@rm -f $@
 	@ln -f -s `basename $(LIBTDB_SHARED_TARGET_SONAME)` $@
 
@@ -2118,8 +2169,13 @@ cleanlibtdb::
 
 installlibtdb:: installdirs libtdb
 	@$(SHELL) $(srcdir)/script/installdirs.sh $(INSTALLPERMS_BIN) $(DESTDIR) $(LIBDIR)
-	-$(INSTALLLIBCMD_SH) $(LIBTDB_SHARED_TARGET_SONAME) $(DESTDIR)$(LIBDIR)
+	-$(INSTALLLIBCMD_SH) $(LIBTDB_SHARED_TARGET_FULLVER) $(DESTDIR)$(LIBDIR)
 	@rm -f $(DESTDIR)$(LIBDIR)/`basename $(LIBTDB_SHARED_TARGET)`
+	@rm -f $(DESTDIR)$(LIBDIR)/`basename $(LIBTDB_SHARED_TARGET_SONAME)`
+	-if test -r $(LIBTDB_SHARED_TARGET_FULLVER) ; then \
+		ln -f -s `basename $(LIBTDB_SHARED_TARGET_FULLVER)` \
+			$(DESTDIR)$(LIBDIR)/`basename $(LIBTDB_SHARED_TARGET_SONAME)` ; \
+	fi
 	-if test -r $(LIBTDB_SHARED_TARGET_SONAME) ; then \
 		ln -f -s `basename $(LIBTDB_SHARED_TARGET_SONAME)` \
 			$(DESTDIR)$(LIBDIR)/`basename $(LIBTDB_SHARED_TARGET)` ; \
@@ -2142,15 +2198,17 @@ shlibs test_shlibs: @LIBTDB_SHARED@
 
 showlibtdb:
 	@echo  ""
-	@echo "LIBTDB_SHARED               = @LIBTDB_SHARED@"
-	@echo "LIBTDB_STATIC               = @LIBTDB_STATIC@"
-	@echo "LIBTDB_LIBS                 = $(LIBTDB_LIBS)"
+	@echo "LIBTDB_SHARED                = @LIBTDB_SHARED@"
+	@echo "LIBTDB_STATIC                = @LIBTDB_STATIC@"
+	@echo "LIBTDB_LIBS                  = $(LIBTDB_LIBS)"
 	@echo ""
-	@echo "LIBTDB_SHARED_TARGET        = $(LIBTDB_SHARED_TARGET)"
-	@echo "LIBTDB_SOVER                = $(LIBTDB_SOVER)"
-	@echo "LIBTDB_SHARED TARGET_SONAME = $(LIBTDB_SHARED_TARGET_SONAME)"
-	@echo "LIBTDB_STATIC_TARGET        = $(LIBTDB_STATIC_TARGET)"
-	@echo "LIBTDB                      = $(LIBTDB)"
+	@echo "LIBTDB_SHARED_TARGET         = $(LIBTDB_SHARED_TARGET)"
+	@echo "LIBTDB_SOVER                 = $(LIBTDB_SOVER)"
+	@echo "LIBTDB_FULLVER               = $(LIBTDB_FULLVER)"
+	@echo "LIBTDB_SHARED TARGET_SONAME  = $(LIBTDB_SHARED_TARGET_SONAME)"
+	@echo "LIBTDB_SHARED TARGET_FULLVER = $(LIBTDB_SHARED_TARGET_FULLVER)"
+	@echo "LIBTDB_STATIC_TARGET         = $(LIBTDB_STATIC_TARGET)"
+	@echo "LIBTDB                       = $(LIBTDB)"
 
 showlibs:: showlibtdb
 
@@ -2176,6 +2234,7 @@ LIBWBCLIENT_OBJ = $(LIBWBCLIENT_OBJ0) \
 
 LIBWBCLIENT_SHARED_TARGET=@LIBWBCLIENT_SHARED_TARGET@
 LIBWBCLIENT_SOVER=@LIBWBCLIENT_SOVER@
+LIBWBCLIENT_FULLVER=@LIBWBCLIENT_FULLVER@
 LIBWBCLIENT_SHARED_TARGET_SONAME=$(LIBWBCLIENT_SHARED_TARGET).$(LIBWBCLIENT_SOVER)
 LIBWBCLIENT_STATIC_TARGET=@LIBWBCLIENT_STATIC_TARGET@
 LIBWBCLIENT_SYMS=$(srcdir)/exports/libwbclient.@SYMSEXT@
@@ -2238,6 +2297,7 @@ showlibwbclient:
 	@echo ""
 	@echo "LIBWBCLIENT_SHARED_TARGET        = $(LIBWBCLIENT_SHARED_TARGET)"
 	@echo "LIBWBCLIENT_SOVER                = $(LIBWBCLIENT_SOVER)"
+	@echo "LIBWBCLIENT_FULLVER              = $(LIBWBCLIENT_FULLVER)"
 	@echo "LIBWBCLIENT_SHARED TARGET_SONAME = $(LIBWBCLIENT_SHARED_TARGET_SONAME)"
 	@echo "LIBWBCLIENT_STATIC_TARGET        = $(LIBWBCLIENT_STATIC_TARGET)"
 	@echo "LIBWBCLIENT                      = $(LIBWBCLIENT)"
@@ -2254,6 +2314,7 @@ showlibs:: showlibwbclient
 
 LIBADDNS_SHARED_TARGET=@LIBADDNS_SHARED_TARGET@
 LIBADDNS_SOVER=@LIBADDNS_SOVER@
+LIBADDNS_FULLVER=@LIBADDNS_FULLVER@
 LIBADDNS_SHARED_TARGET_SONAME=$(LIBADDNS_SHARED_TARGET).$(LIBADDNS_SOVER)
 LIBADDNS_STATIC_TARGET=@LIBADDNS_STATIC_TARGET@
 LIBADDNS=@LIBADDNS_STATIC@ @LIBADDNS_SHARED@
@@ -2311,6 +2372,7 @@ showlibaddns:
 	@echo ""
 	@echo "LIBADDNS_SHARED_TARGET        = $(LIBADDNS_SHARED_TARGET)"
 	@echo "LIBADDNS_SOVER                = $(LIBADDNS_SOVER)"
+	@echo "LIBADDNS_FULLVER              = $(LIBADDNS_FULLVER)"
 	@echo "LIBADDNS_SHARED TARGET_SONAME = $(LIBADDNS_SHARED_TARGET_SONAME)"
 	@echo "LIBADDNS_STATIC_TARGET        = $(LIBADDNS_STATIC_TARGET)"
 	@echo "LIBADDNS                      = $(LIBADDNS)"
@@ -2361,6 +2423,7 @@ LIBNETAPI_OBJ  = $(LIBNETAPI_OBJ0) $(LIBNET_OBJ) \
 
 LIBNETAPI_SHARED_TARGET=@LIBNETAPI_SHARED_TARGET@
 LIBNETAPI_SOVER=@LIBNETAPI_SOVER@
+LIBNETAPI_FULLVER=@LIBNETAPI_FULLVER@
 LIBNETAPI_SHARED_TARGET_SONAME=$(LIBNETAPI_SHARED_TARGET).$(LIBNETAPI_SOVER)
 LIBNETAPI_STATIC_TARGET=@LIBNETAPI_STATIC_TARGET@
 LIBNETAPI_SYMS=$(srcdir)/exports/libnetapi.@SYMSEXT@
@@ -2425,6 +2488,7 @@ showlibnetapi:
 	@echo ""
 	@echo "LIBNETAPI_SHARED_TARGET        = $(LIBNETAPI_SHARED_TARGET)"
 	@echo "LIBNETAPI_SOVER                = $(LIBNETAPI_SOVER)"
+	@echo "LIBNETAPI_FULLVER              = $(LIBNETAPI_FULLVER)"
 	@echo "LIBNETAPI_SHARED TARGET_SONAME = $(LIBNETAPI_SHARED_TARGET_SONAME)"
 	@echo "LIBNETAPI_STATIC_TARGET        = $(LIBNETAPI_STATIC_TARGET)"
 	@echo "LIBNETAPI                      = $(LIBNETAPI)"
@@ -2439,6 +2503,7 @@ showlibs:: showlibnetapi
 
 LIBSMBCLIENT_SHARED_TARGET=@LIBSMBCLIENT_SHARED_TARGET@
 LIBSMBCLIENT_SOVER=@LIBSMBCLIENT_SOVER@
+LIBSMBCLIENT_FULLVER=@LIBSMBCLIENT_FULLVER@
 LIBSMBCLIENT_SHARED_TARGET_SONAME=$(LIBSMBCLIENT_SHARED_TARGET).$(LIBSMBCLIENT_SOVER)
 LIBSMBCLIENT_STATIC_TARGET=@LIBSMBCLIENT_STATIC_TARGET@
 LIBSMBCLIENT=@LIBSMBCLIENT_STATIC@ @LIBSMBCLIENT_SHARED@
@@ -2507,6 +2572,7 @@ showlibsmbclient:
 	@echo ""
 	@echo "LIBSMBCLIENT_SHARED_TARGET        = $(LIBSMBCLIENT_SHARED_TARGET)"
 	@echo "LIBSMBCLIENT_SOVER                = $(LIBSMBCLIENT_SOVER)"
+	@echo "LIBSMBCLIENT_FULLVER              = $(LIBSMBCLIENT_FULLVER)"
 	@echo "LIBSMBCLIENT_SHARED TARGET_SONAME = $(LIBSMBCLIENT_SHARED_TARGET_SONAME)"
 	@echo "LIBSMBCLIENT_STATIC_TARGET        = $(LIBSMBCLIENT_STATIC_TARGET)"
 	@echo "LIBSMBCLIENT                      = $(LIBSMBCLIENT)"
@@ -2519,12 +2585,13 @@ showlibs:: showlibsmbclient
 #
 #-------------------------------------------------------------------
 
-LIBSMBSHAREMODES_OBJ0 = libsmb/smb_share_modes.o
+LIBSMBSHAREMODES_OBJ0 = libsmb/smb_share_modes.o ../lib/ccan/hash/hash.o
 
 LIBSMBSHAREMODES_OBJ = $(LIBSMBSHAREMODES_OBJ0)
 
 LIBSMBSHAREMODES_SHARED_TARGET=@LIBSMBSHAREMODES_SHARED_TARGET@
 LIBSMBSHAREMODES_SOVER=@LIBSMBSHAREMODES_SOVER@
+LIBSMBSHAREMODES_FULLVER=@LIBSMBSHAREMODES_FULLVER@
 LIBSMBSHAREMODES_SHARED_TARGET_SONAME=$(LIBSMBSHAREMODES_SHARED_TARGET).$(LIBSMBSHAREMODES_SOVER)
 LIBSMBSHAREMODES_STATIC_TARGET=@LIBSMBSHAREMODES_STATIC_TARGET@
 LIBSMBSHAREMODES=@LIBSMBSHAREMODES_STATIC@ @LIBSMBSHAREMODES_SHARED@
@@ -2589,6 +2656,7 @@ showlibsmbsharemodes:
 	@echo ""
 	@echo "LIBSMBSHAREMODES_SHARED_TARGET        = $(LIBSMBSHAREMODES_SHARED_TARGET)"
 	@echo "LIBSMBSHAREMODES_SOVER                = $(LIBSMBSHAREMODES_SOVER)"
+	@echo "LIBSMBSHAREMODES_FULLVER              = $(LIBSMBSHAREMODES_FULLVER)"
 	@echo "LIBSMBSHAREMODES_SHARED TARGET_SONAME = $(LIBSMBSHAREMODES_SHARED_TARGET_SONAME)"
 	@echo "LIBSMBSHAREMODES_STATIC_TARGET        = $(LIBSMBSHAREMODES_STATIC_TARGET)"
 	@echo "LIBSMBSHAREMODES                      = $(LIBSMBSHAREMODES)"
@@ -2952,6 +3020,10 @@ bin/gpfs.@SHLIBEXT@: $(BINARY_PREREQS) $(VFS_GPFS_OBJ)
 	@echo "Building plugin $@"
 	@$(SHLD_MODULE) $(VFS_GPFS_OBJ)
 
+bin/gpfs_hsm_notify.@SHLIBEXT@: $(BINARY_PREREQS) $(VFS_GPFS_HSM_NOTIFY_OBJ)
+	@echo "Building plugin $@"
+	@$(SHLD_MODULE) $(VFS_GPFS_HSM_NOTIFY_OBJ)
+
 bin/notify_fam.@SHLIBEXT@: $(BINARY_PREREQS) $(VFS_NOTIFY_FAM_OBJ)
 	@echo "Building plugin $@"
 	@$(SHLD_MODULE) $(VFS_NOTIFY_FAM_OBJ) @SMB_FAM_LIBS@
@@ -3049,17 +3121,17 @@ bin/wbinfo@EXEEXT@: $(BINARY_PREREQS) $(WBINFO_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(
 		$(LIBWBCLIENT_LIBS)
 
 bin/ntlm_auth@EXEEXT@: $(BINARY_PREREQS) $(NTLM_AUTH_OBJ) $(PARAM_OBJ) \
-	$(LIB_NONSMBD_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(LIBTDB) $(LIBWBCLIENT)
+	$(LIB_NONSMBD_OBJ) @BUILD_POPT@ $(LIBTALLOC) $(LIBTDB) $(LIBWBCLIENT) $(ZLIB_LIBS)
 	@echo Linking $@
 	@$(CC) -o $@ $(LDFLAGS) $(DYNEXP) $(NTLM_AUTH_OBJ) \
-		$(PARAM_OBJ) $(LIB_NONSMBD_OBJ) $(LIBS) \
+		$(PARAM_OBJ) $(LIB_NONSMBD_OBJ) $(LIBS) $(ZLIB_LIBS) \
 		$(POPT_LIBS) $(KRB5LIBS) $(LDAP_LIBS) $(NSCD_LIBS) \
 		$(LIBTALLOC_LIBS) $(LIBTDB_LIBS) $(LIBWBCLIENT_LIBS) @INIPARSERLIBS@
 
-bin/pam_smbpass.@SHLIBEXT@: $(BINARY_PREREQS) $(PAM_SMBPASS_OBJ) $(LIBCLI_LDAP_NDR_OBJ) $(LIBTALLOC) $(LIBWBCLIENT) $(LIBTDB)
+bin/pam_smbpass.@SHLIBEXT@: $(BINARY_PREREQS) $(PAM_SMBPASS_OBJ) $(LIBCLI_LDAP_NDR_OBJ) $(LIBTALLOC) $(LIBWBCLIENT) $(LIBTDB) $(ZLIB_LIBS)
 	@echo "Linking shared library $@"
 	@$(SHLD) $(LDSHFLAGS) -o $@ $(PAM_SMBPASS_OBJ) $(LIBCLI_LDAP_NDR_OBJ) -lpam $(DYNEXP) \
-		$(LIBS) $(LDAP_LIBS) $(NSCD_LIBS) \
+		$(LIBS) $(LDAP_LIBS) $(NSCD_LIBS) $(ZLIB_LIBS) \
 		$(LIBTALLOC_LIBS) $(LIBTDB_LIBS) $(LIBWBCLIENT_LIBS)
 
 bin/tdbbackup@EXEEXT@: $(BINARY_PREREQS) $(TDBBACKUP_OBJ) $(LIBTALLOC) $(LIBTDB)
@@ -3291,16 +3363,13 @@ uninstallpammodules::
 		rm -f "$(DESTDIR)/$(PAMMODULESDIR)/$${module}.@SHLIBEXT@"; \
 	done
 
-# Toplevel clean files
-TOPFILES=dynconfig.o localedir.o
-
 cleanlibs::
 	-rm -f ../lib/*/*.o ../lib/*/*/*.o \
 		../lib/*/*.ho \
 		../libcli/*.o ../libcli/*/*.o ../libcli/*/*/*.o \
 		../librpc/*/*.o \
 		../libgpo/*.o ../libgpo/*/*.o \
-		../libds/*.o ../libds/*/*.o
+		../libds/*.o ../libds/*/*.o ../dynconfig/*.o
 
 clean:: cleanlibs
 	-rm -f include/build_env.h
@@ -3339,12 +3408,12 @@ realclean:: clean
 
 distclean:: realclean
 	-rm -f smbadduser
-	-rm -f include/config.h Makefile
+	-rm -f include/autoconf/config.h Makefile
 	-rm -f config.status config.cache so_locations
 	-rm -rf .deps TAGS
 
 realdistclean:: distclean
-	-rm -f include/config.h.in
+	-rm -f include/autoconf/config.h.in
 	-rm -f include/version.h
 	-rm -f configure
 
diff --git a/source3/auth/auth_builtin.c b/source3/auth/auth_builtin.c
index afe71ee2a60..cfe89495a01 100644
--- a/source3/auth/auth_builtin.c
+++ b/source3/auth/auth_builtin.c
@@ -57,7 +57,7 @@ static NTSTATUS auth_init_guest(struct auth_context *auth_context, const char *o
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -117,7 +117,7 @@ static NTSTATUS auth_init_name_to_ntstatus(struct auth_context *auth_context, co
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -171,7 +171,7 @@ static NTSTATUS auth_init_fixed_challenge(struct auth_context *auth_context, con
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_domain.c b/source3/auth/auth_domain.c
index 5824d91a0a2..c18a81ab579 100644
--- a/source3/auth/auth_domain.c
+++ b/source3/auth/auth_domain.c
@@ -26,6 +26,7 @@
 #include "rpc_client/cli_netlogon.h"
 #include "secrets.h"
 #include "passdb.h"
+#include "libsmb/libsmb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_AUTH
@@ -68,7 +69,7 @@ void attempt_machine_password_change(void)
 		DEBUG(0,("attempt_machine_password_change: unable to lock "
 			"the machine account password for machine %s in "
 			"domain %s.\n",
-			global_myname(), lp_workgroup() ));
+			lp_netbios_name(), lp_workgroup() ));
 		return;
 	}
 
@@ -76,7 +77,7 @@ void attempt_machine_password_change(void)
 			trust_passwd_hash, &lct, NULL)) {
 		DEBUG(0,("attempt_machine_password_change: unable to read the "
 			"machine account password for %s in domain %s.\n",
-			global_myname(), lp_workgroup()));
+			lp_netbios_name(), lp_workgroup()));
 		TALLOC_FREE(lock);
 		return;
 	}
@@ -145,7 +146,7 @@ static NTSTATUS connect_to_domain_password_server(struct cli_state **cli,
 	}
 
 	/* Attempt connection */
-	result = cli_full_connection(cli, global_myname(), dc_name, dc_ss, 0, 
+	result = cli_full_connection(cli, lp_netbios_name(), dc_name, dc_ss, 0,
 		"IPC$", "IPC", "", "", "", 0, Undefined);
 
 	if (!NT_STATUS_IS_OK(result)) {
@@ -218,7 +219,7 @@ machine %s. Error was : %s.\n", dc_name, nt_errstr(result)));
 		result = rpccli_netlogon_setup_creds(netlogon_pipe,
 					dc_name, /* server name */
 					domain, /* domain */
-					global_myname(), /* client name */
+					lp_netbios_name(), /* client name */
 					account_name, /* machine account name */
 					machine_pwd,
 					sec_chan_type,
@@ -418,7 +419,7 @@ static NTSTATUS auth_init_ntdomain(struct auth_context *auth_context, const char
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -524,7 +525,7 @@ static NTSTATUS auth_init_trustdomain(struct auth_context *auth_context, const c
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_netlogond.c b/source3/auth/auth_netlogond.c
index 8e332c8e5aa..7fb037457e7 100644
--- a/source3/auth/auth_netlogond.c
+++ b/source3/auth/auth_netlogond.c
@@ -133,7 +133,7 @@ static NTSTATUS netlogond_validate(TALLOC_CTX *mem_ctx,
 		p, p,
 		user_info->logon_parameters,           /* flags such as 'allow
 					                * workstation logon' */
-		global_myname(),                       /* server name */
+		lp_netbios_name(),                       /* server name */
 		user_info->client.account_name,        /* user name logging on. */
 		user_info->client.domain_name,         /* domain name */
 		user_info->workstation_name,           /* workstation name */
@@ -234,7 +234,7 @@ static NTSTATUS mymachinepw(uint8_t pwd[16])
 	DEBUG(10, ("default_nc = %s\n", default_nc));
 
 	myname = talloc_asprintf_strupper_m(talloc_tos(), "%s$",
-					    global_myname());
+					    lp_netbios_name());
 	if (myname == NULL) {
 		DEBUG(10, ("talloc failed\n"));
 		status = NT_STATUS_NO_MEMORY;
@@ -375,8 +375,8 @@ static NTSTATUS check_netlogond_security(const struct auth_context *auth_context
 	dump_data(10, machine_password, 16);
 
 	status = rpccli_netlogon_setup_creds(
-		p, global_myname(), lp_workgroup(), global_myname(),
-		global_myname(), machine_password, SEC_CHAN_BDC, &neg_flags);
+		p, lp_netbios_name(), lp_workgroup(), lp_netbios_name(),
+		lp_netbios_name(), machine_password, SEC_CHAN_BDC, &neg_flags);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(10, ("rpccli_netlogon_setup_creds failed: %s\n",
@@ -429,7 +429,7 @@ static NTSTATUS auth_init_netlogond(struct auth_context *auth_context,
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_ntlmssp.c b/source3/auth/auth_ntlmssp.c
index 58974210dd9..1fecc0988ea 100644
--- a/source3/auth/auth_ntlmssp.c
+++ b/source3/auth/auth_ntlmssp.c
@@ -184,13 +184,13 @@ NTSTATUS auth_ntlmssp_start(struct auth_ntlmssp_state **auth_ntlmssp_state)
 	struct auth_ntlmssp_state *ans;
 	struct auth_context *auth_context;
 
-	if ((enum server_types)lp_server_role() == ROLE_STANDALONE) {
+	if ((enum server_role)lp_server_role() == ROLE_STANDALONE) {
 		is_standalone = true;
 	} else {
 		is_standalone = false;
 	}
 
-	netbios_name = global_myname();
+	netbios_name = lp_netbios_name();
 	netbios_domain = lp_workgroup();
 	/* This should be a 'netbios domain -> DNS domain' mapping */
 	dns_domain = get_mydnsdomname(talloc_tos());
diff --git a/source3/auth/auth_sam.c b/source3/auth/auth_sam.c
index 8d2300dda74..7faa8de0275 100644
--- a/source3/auth/auth_sam.c
+++ b/source3/auth/auth_sam.c
@@ -22,7 +22,6 @@
 
 #include "includes.h"
 #include "auth.h"
-#include "../libcli/auth/libcli_auth.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_AUTH
@@ -45,7 +44,7 @@ static NTSTATUS auth_init_sam_ignoredomain(struct auth_context *auth_context, co
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -109,7 +108,7 @@ static NTSTATUS auth_init_sam(struct auth_context *auth_context, const char *par
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_samba4.c b/source3/auth/auth_samba4.c
new file mode 100644
index 00000000000..03b7884068d
--- /dev/null
+++ b/source3/auth/auth_samba4.c
@@ -0,0 +1,118 @@
+/*
+   Unix SMB/CIFS implementation.
+   Authenticate against Samba4's auth subsystem
+   Copyright (C) Volker Lendecke 2008
+   Copyright (C) Andrew Bartlett 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "source3/include/auth.h"
+#include "source4/auth/auth.h"
+#include "auth/auth_sam_reply.h"
+#include "param/param.h"
+#include "source4/lib/events/events.h"
+
+#undef DBGC_CLASS
+#define DBGC_CLASS DBGC_AUTH
+
+static NTSTATUS check_samba4_security(const struct auth_context *auth_context,
+				      void *my_private_data,
+				      TALLOC_CTX *mem_ctx,
+				      const struct auth_usersupplied_info *user_info,
+				      struct auth_serversupplied_info **server_info)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct netr_SamInfo3 *info3 = NULL;
+	NTSTATUS nt_status;
+	struct auth_user_info_dc *user_info_dc;
+	struct auth4_context *auth4_context;
+	struct loadparm_context *lp_ctx;
+
+	lp_ctx = loadparm_init_s3(frame, loadparm_s3_context());
+	if (lp_ctx == NULL) {
+		DEBUG(10, ("loadparm_init_s3 failed\n"));
+		talloc_free(frame);
+		return NT_STATUS_INVALID_SERVER_STATE;
+	}
+
+	/* We create a private tevent context here to avoid nested loops in
+	 * the s3 one, as that may not be expected */
+	nt_status = auth_context_create(mem_ctx,
+					s4_event_context_init(frame), NULL, 
+					lp_ctx,
+					&auth4_context);
+	NT_STATUS_NOT_OK_RETURN(nt_status);
+		
+	nt_status = auth_context_set_challenge(auth4_context, auth_context->challenge.data, "auth_samba4");
+	NT_STATUS_NOT_OK_RETURN_AND_FREE(nt_status, auth4_context);
+
+	nt_status = auth_check_password(auth4_context, auth4_context, user_info, &user_info_dc);
+	NT_STATUS_NOT_OK_RETURN_AND_FREE(nt_status, auth4_context);
+	
+	nt_status = auth_convert_user_info_dc_saminfo3(mem_ctx,
+						       user_info_dc,
+						       &info3);
+	if (NT_STATUS_IS_OK(nt_status)) {
+		/* We need the strings from the server_info to be valid as long as the info3 is around */
+		talloc_steal(info3, user_info_dc);
+	}
+	talloc_free(auth4_context);
+
+	if (!NT_STATUS_IS_OK(nt_status)) {
+		goto done;
+	}
+
+	nt_status = make_server_info_info3(mem_ctx, user_info->client.account_name,
+					   user_info->mapped.domain_name, server_info,
+					info3);
+	if (!NT_STATUS_IS_OK(nt_status)) {
+		DEBUG(10, ("make_server_info_info3 failed: %s\n",
+			   nt_errstr(nt_status)));
+		TALLOC_FREE(frame);
+		return nt_status;
+	}
+
+	nt_status = NT_STATUS_OK;
+
+ done:
+	TALLOC_FREE(frame);
+	return nt_status;
+}
+
+/* module initialisation */
+static NTSTATUS auth_init_samba4(struct auth_context *auth_context,
+				    const char *param,
+				    auth_methods **auth_method)
+{
+	struct auth_methods *result;
+
+	result = talloc_zero(auth_context, struct auth_methods);
+	if (result == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+	result->name = "samba4";
+	result->auth = check_samba4_security;
+
+        *auth_method = result;
+	return NT_STATUS_OK;
+}
+
+NTSTATUS auth_samba4_init(void)
+{
+	smb_register_auth(AUTH_INTERFACE_VERSION, "samba4",
+			  auth_init_samba4);
+	return NT_STATUS_OK;
+}
diff --git a/source3/auth/auth_script.c b/source3/auth/auth_script.c
index ad973106b1c..4432ff4aecc 100644
--- a/source3/auth/auth_script.c
+++ b/source3/auth/auth_script.c
@@ -74,32 +74,32 @@ static NTSTATUS script_check_user_credentials(const struct auth_context *auth_co
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	safe_strcpy( secret_str, user_info->mapped.domain_name, secret_str_len - 1);
-	safe_strcat( secret_str, "\n", secret_str_len - 1);
-	safe_strcat( secret_str, user_info->client.account_name, secret_str_len - 1);
-	safe_strcat( secret_str, "\n", secret_str_len - 1);
+	strlcpy( secret_str, user_info->mapped.domain_name, secret_str_len);
+	strlcat( secret_str, "\n", secret_str_len);
+	strlcat( secret_str, user_info->client.account_name, secret_str_len);
+	strlcat( secret_str, "\n", secret_str_len);
 
 	for (i = 0; i < 8; i++) {
 		slprintf(&hex_str[i*2], 3, "%02X", auth_context->challenge.data[i]);
 	}
-	safe_strcat( secret_str, hex_str, secret_str_len - 1);
-	safe_strcat( secret_str, "\n", secret_str_len - 1);
+	strlcat( secret_str, hex_str, secret_str_len);
+	strlcat( secret_str, "\n", secret_str_len);
 
 	if (user_info->password.response.lanman.data) {
 		for (i = 0; i < 24; i++) {
 			slprintf(&hex_str[i*2], 3, "%02X", user_info->password.response.lanman.data[i]);
 		}
-		safe_strcat( secret_str, hex_str, secret_str_len - 1);
+		strlcat( secret_str, hex_str, secret_str_len);
 	}
-	safe_strcat( secret_str, "\n", secret_str_len - 1);
+	strlcat( secret_str, "\n", secret_str_len);
 
 	if (user_info->password.response.nt.data) {
 		for (i = 0; i < 24; i++) {
 			slprintf(&hex_str[i*2], 3, "%02X", user_info->password.response.nt.data[i]);
 		}
-		safe_strcat( secret_str, hex_str, secret_str_len - 1);
+		strlcat( secret_str, hex_str, secret_str_len);
 	}
-	safe_strcat( secret_str, "\n", secret_str_len - 1);
+	strlcat( secret_str, "\n", secret_str_len);
 
 	DEBUG(10,("script_check_user_credentials: running %s with parameters:\n%s\n",
 		script, secret_str ));
@@ -124,7 +124,7 @@ static NTSTATUS auth_init_script(struct auth_context *auth_context, const char *
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_server.c b/source3/auth/auth_server.c
index 5dc8ea7f04b..da380db01ad 100644
--- a/source3/auth/auth_server.c
+++ b/source3/auth/auth_server.c
@@ -22,6 +22,7 @@
 #include "auth.h"
 #include "system/passwd.h"
 #include "smbd/smbd.h"
+#include "libsmb/libsmb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_AUTH
@@ -43,12 +44,6 @@ static struct cli_state *server_cryptkey(TALLOC_CTX *mem_ctx)
 	struct named_mutex *mutex = NULL;
 	NTSTATUS status;
 
-	if (!(cli = cli_initialise()))
-		return NULL;
-
-	/* security = server just can't function with spnego */
-	cli->use_spnego = False;
-
         pserver = talloc_strdup(mem_ctx, lp_passwordserver());
 	p = pserver;
 
@@ -63,12 +58,18 @@ static struct cli_state *server_cryptkey(TALLOC_CTX *mem_ctx)
 		}
 		strupper_m(desthost);
 
+		if (strequal(desthost, myhostname())) {
+			DEBUG(1,("Password server loop - disabling "
+				 "password server %s\n", desthost));
+			continue;
+		}
+
 		if(!resolve_name( desthost, &dest_ss, 0x20, false)) {
 			DEBUG(1,("server_cryptkey: Can't resolve address for %s\n",desthost));
 			continue;
 		}
 
-		if (ismyaddr((struct sockaddr *)&dest_ss)) {
+		if (ismyaddr((struct sockaddr *)(void *)&dest_ss)) {
 			DEBUG(1,("Password server loop - disabling password server %s\n",desthost));
 			continue;
 		}
@@ -80,11 +81,11 @@ static struct cli_state *server_cryptkey(TALLOC_CTX *mem_ctx)
 
 		mutex = grab_named_mutex(talloc_tos(), desthost, 10);
 		if (mutex == NULL) {
-			cli_shutdown(cli);
 			return NULL;
 		}
 
-		status = cli_connect(cli, desthost, &dest_ss);
+		status = cli_connect_nb(desthost, &dest_ss, 0, 0x20,
+					lp_netbios_name(), Undefined, &cli);
 		if (NT_STATUS_IS_OK(status)) {
 			DEBUG(3,("connected to password server %s\n",desthost));
 			connected_ok = True;
@@ -97,21 +98,11 @@ static struct cli_state *server_cryptkey(TALLOC_CTX *mem_ctx)
 
 	if (!connected_ok) {
 		DEBUG(0,("password server not available\n"));
-		cli_shutdown(cli);
 		return NULL;
 	}
 
-	if (!attempt_netbios_session_request(&cli, global_myname(),
-					     desthost, &dest_ss)) {
-		TALLOC_FREE(mutex);
-		DEBUG(1,("password server fails session request\n"));
-		cli_shutdown(cli);
-		return NULL;
-	}
-
-	if (strequal(desthost,myhostname())) {
-		exit_server_cleanly("Password server loop!");
-	}
+	/* security = server just can't function with spnego */
+	cli->use_spnego = False;
 
 	DEBUG(3,("got session\n"));
 
@@ -258,7 +249,7 @@ static DATA_BLOB auth_get_challenge_server(const struct auth_context *auth_conte
 
 		/* The return must be allocated on the caller's mem_ctx, as our own will be
 		   destoyed just after the call. */
-		return data_blob_talloc((TALLOC_CTX *)auth_context, cli->secblob.data,8);
+		return data_blob_talloc(discard_const_p(TALLOC_CTX, auth_context), cli->secblob.data,8);
 	} else {
 		return data_blob_null;
 	}
@@ -341,7 +332,7 @@ static NTSTATUS check_smbserver_security(const struct auth_context *auth_context
 		baduser = talloc_asprintf(mem_ctx,
 					"%s%s",
 					INVALID_USER_PREFIX,
-					global_myname());
+					lp_netbios_name());
 		if (!baduser) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -426,7 +417,7 @@ use this machine as the password server.\n"));
 	}
 
 	/* if logged in as guest then reject */
-	if ((SVAL(cli->inbuf,smb_vwv2) & 1) != 0) {
+	if (cli->is_guestlogin) {
 		DEBUG(1,("password server %s gave us guest only\n", cli->desthost));
 		nt_status = NT_STATUS_LOGON_FAILURE;
 	}
@@ -461,7 +452,7 @@ static NTSTATUS auth_init_smbserver(struct auth_context *auth_context, const cha
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_unix.c b/source3/auth/auth_unix.c
index 8f9eacf76b3..36956986c59 100644
--- a/source3/auth/auth_unix.c
+++ b/source3/auth/auth_unix.c
@@ -20,7 +20,6 @@
 #include "includes.h"
 #include "auth.h"
 #include "system/passwd.h"
-#include "../librpc/gen_ndr/samr.h"
 #include "smbd/globals.h"
 
 #undef DBGC_CLASS
@@ -74,7 +73,7 @@ static NTSTATUS auth_init_unix(struct auth_context *auth_context, const char* pa
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_util.c b/source3/auth/auth_util.c
index b7ec9fff704..64c290eb048 100644
--- a/source3/auth/auth_util.c
+++ b/source3/auth/auth_util.c
@@ -23,7 +23,6 @@
 
 #include "includes.h"
 #include "auth.h"
-#include "smbd/globals.h"
 #include "../libcli/auth/libcli_auth.h"
 #include "../lib/crypto/arcfour.h"
 #include "rpc_client/init_lsa.h"
@@ -520,7 +519,7 @@ NTSTATUS create_local_token(TALLOC_CTX *mem_ctx,
 
 	t = session_info->security_token;
 
-	ids = TALLOC_ARRAY(talloc_tos(), struct wbcUnixId,
+	ids = talloc_array(talloc_tos(), struct wbcUnixId,
 			   t->num_sids);
 	if (ids == NULL) {
 		return NT_STATUS_NO_MEMORY;
@@ -669,7 +668,7 @@ NTSTATUS make_server_info_pw(struct auth_serversupplied_info **server_info,
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	status = samu_to_SamInfo3(result, sampass, global_myname(),
+	status = samu_to_SamInfo3(result, sampass, lp_netbios_name(),
 				  &result->info3, &result->extra);
 	TALLOC_FREE(sampass);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -759,7 +758,7 @@ static NTSTATUS make_new_server_info_guest(struct auth_serversupplied_info **ses
 	struct auth_serversupplied_info *server_info;
 	static const char zeros[16] = {0};
 	const char *guest_account = lp_guestaccount();
-	const char *domain = global_myname();
+	const char *domain = lp_netbios_name();
 	struct netr_SamInfo3 info3;
 	TALLOC_CTX *tmp_ctx;
 	NTSTATUS status;
@@ -774,6 +773,8 @@ static NTSTATUS make_new_server_info_guest(struct auth_serversupplied_info **ses
 
 	status = get_guest_info3(tmp_ctx, &info3);
 	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(0, ("get_guest_info3 failed with %s\n",
+			  nt_errstr(status)));
 		goto done;
 	}
 
@@ -783,6 +784,8 @@ static NTSTATUS make_new_server_info_guest(struct auth_serversupplied_info **ses
 					&server_info,
 					&info3);
 	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(0, ("make_server_info_info3 failed with %s\n",
+			  nt_errstr(status)));
 		goto done;
 	}
 
@@ -794,8 +797,8 @@ static NTSTATUS make_new_server_info_guest(struct auth_serversupplied_info **ses
 	status = create_local_token(tmp_ctx, server_info, NULL, session_info);
 	TALLOC_FREE(server_info);
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(10, ("create_local_token failed: %s\n",
-			   nt_errstr(status)));
+		DEBUG(0, ("create_local_token failed: %s\n",
+			  nt_errstr(status)));
 		goto done;
 	}
 	talloc_steal(NULL, *session_info);
@@ -812,7 +815,7 @@ static NTSTATUS make_new_server_info_guest(struct auth_serversupplied_info **ses
 	status = NT_STATUS_OK;
 done:
 	TALLOC_FREE(tmp_ctx);
-	return NT_STATUS_OK;
+	return status;
 }
 
 /***************************************************************************
@@ -908,7 +911,7 @@ struct auth_serversupplied_info *copy_serverinfo(TALLOC_CTX *mem_ctx,
 	dst->utok.gid = src->utok.gid;
 	dst->utok.ngroups = src->utok.ngroups;
 	if (src->utok.ngroups != 0) {
-		dst->utok.groups = (gid_t *)TALLOC_MEMDUP(
+		dst->utok.groups = (gid_t *)talloc_memdup(
 			dst, src->utok.groups,
 			sizeof(gid_t)*dst->utok.ngroups);
 	} else {
diff --git a/source3/auth/auth_wbc.c b/source3/auth/auth_wbc.c
index 61500402293..1b70042d909 100644
--- a/source3/auth/auth_wbc.c
+++ b/source3/auth/auth_wbc.c
@@ -182,7 +182,7 @@ static NTSTATUS auth_init_wbc(struct auth_context *auth_context, const char *par
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/auth_winbind.c b/source3/auth/auth_winbind.c
index 21433535415..d4ace2c9193 100644
--- a/source3/auth/auth_winbind.c
+++ b/source3/auth/auth_winbind.c
@@ -142,7 +142,7 @@ static NTSTATUS auth_init_winbind(struct auth_context *auth_context, const char
 {
 	struct auth_methods *result;
 
-	result = TALLOC_ZERO_P(auth_context, struct auth_methods);
+	result = talloc_zero(auth_context, struct auth_methods);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/auth/check_samsec.c b/source3/auth/check_samsec.c
index c4bcda4b1b7..2d3cb657859 100644
--- a/source3/auth/check_samsec.c
+++ b/source3/auth/check_samsec.c
@@ -531,7 +531,7 @@ NTSTATUS check_sam_security_info3(const DATA_BLOB *challenge,
 		goto done;
 	}
 
-	info3 = TALLOC_ZERO_P(mem_ctx, struct netr_SamInfo3);
+	info3 = talloc_zero(mem_ctx, struct netr_SamInfo3);
 	if (info3 == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
diff --git a/source3/auth/pass_check.c b/source3/auth/pass_check.c
index a7a1c3d8f5d..714cc968a78 100644
--- a/source3/auth/pass_check.c
+++ b/source3/auth/pass_check.c
@@ -502,8 +502,8 @@ it assumes the string starts lowercased
 ****************************************************************************/
 static NTSTATUS string_combinations2(char *s, int offset,
 				     NTSTATUS (*fn)(const char *s,
-						    void *private_data),
-				     int N, void *private_data)
+						    const void *private_data),
+				     int N, const void *private_data)
 {
 	int len = strlen(s);
 	int i;
@@ -540,8 +540,8 @@ it assumes the string starts lowercased
 ****************************************************************************/
 static NTSTATUS string_combinations(char *s,
 				    NTSTATUS (*fn)(const char *s,
-						   void *private_data),
-				    int N, void *private_data)
+						   const void *private_data),
+				    int N, const void *private_data)
 {
 	int n;
 	NTSTATUS nt_status;
@@ -558,7 +558,7 @@ static NTSTATUS string_combinations(char *s,
 /****************************************************************************
 core of password checking routine
 ****************************************************************************/
-static NTSTATUS password_check(const char *password, void *private_data)
+static NTSTATUS password_check(const char *password, const void *private_data)
 {
 #ifdef WITH_PAM
 	const char *rhost = (const char *)private_data;
@@ -840,7 +840,7 @@ NTSTATUS pass_check(const struct passwd *pass,
 #endif /* defined(WITH_PAM) */
 
 	/* try it as it came to us */
-	nt_status = password_check(password, (void *)rhost);
+	nt_status = password_check(password, (const void *)rhost);
         if NT_STATUS_IS_OK(nt_status) {
 		return (nt_status);
 	} else if (!NT_STATUS_EQUAL(nt_status, NT_STATUS_WRONG_PASSWORD)) {
@@ -868,7 +868,7 @@ NTSTATUS pass_check(const struct passwd *pass,
 	/* try all lowercase if it's currently all uppercase */
 	if (strhasupper(pass2)) {
 		strlower_m(pass2);
-		nt_status = password_check(pass2, (void *)rhost);
+		nt_status = password_check(pass2, (const void *)rhost);
 		if (NT_STATUS_IS_OK(nt_status)) {
 			return (nt_status);
 		}
@@ -883,7 +883,7 @@ NTSTATUS pass_check(const struct passwd *pass,
 	strlower_m(pass2);
 
 	nt_status = string_combinations(pass2, password_check, level,
-					(void *)rhost);
+					(const void *)rhost);
         if (NT_STATUS_IS_OK(nt_status)) {
 		return nt_status;
 	}
diff --git a/source3/auth/proto.h b/source3/auth/proto.h
index b88bb8c6265..46846ac9074 100644
--- a/source3/auth/proto.h
+++ b/source3/auth/proto.h
@@ -1,3 +1,37 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Password and authentication handling
+ *
+ *  Copyright (C) Andrew Tridgell		1992-2001
+ *  Copyright (C) Luke Kenneth Casson Leighton	1996-2000
+ *  Copyright (C) Jeremy Allison		1997-2001
+ *  Copyright (C) John H Terpsta		1999-2001
+ *  Copyright (C) Tim Potter			2000
+ *  Copyright (C) Andrew Bartlett		2001-2003
+ *  Copyright (C) Jelmer Vernooij		2002
+ *  Copyright (C) Rafal Szczesniak		2002
+ *  Copyright (C) Gerald Carter			2003
+ *  Copyright (C) Volker Lendecke		2006,2010
+ *  Copyright (C) Michael Adam			2007
+ *  Copyright (C) Dan Sledz			2009
+ *  Copyright (C) Simo Sorce			2010
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _AUTH_PROTO_H_
+#define _AUTH_PROTO_H_
 
 /* The following definitions come from auth/auth.c  */
 
@@ -270,3 +304,5 @@ NTSTATUS make_session_info_krb5(TALLOC_CTX *mem_ctx,
 				bool mapped_to_guest, bool username_was_mapped,
 				DATA_BLOB *session_key,
 				struct auth_serversupplied_info **session_info);
+
+#endif /* _AUTH_PROTO_H_ */
diff --git a/source3/auth/server_info.c b/source3/auth/server_info.c
index 440e45c8e5c..a53e556d283 100644
--- a/source3/auth/server_info.c
+++ b/source3/auth/server_info.c
@@ -45,7 +45,7 @@ struct auth_serversupplied_info *make_server_info(TALLOC_CTX *mem_ctx)
 {
 	struct auth_serversupplied_info *result;
 
-	result = TALLOC_ZERO_P(mem_ctx, struct auth_serversupplied_info);
+	result = talloc_zero(mem_ctx, struct auth_serversupplied_info);
 	if (result == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
diff --git a/source3/auth/server_info_sam.c b/source3/auth/server_info_sam.c
index 5fd78a30fa7..261e3de8f2d 100644
--- a/source3/auth/server_info_sam.c
+++ b/source3/auth/server_info_sam.c
@@ -23,8 +23,6 @@
 
 #include "includes.h"
 #include "auth.h"
-#include "smbd/globals.h"
-#include "../libcli/auth/libcli_auth.h"
 #include "nsswitch/winbind_client.h"
 #include "passdb.h"
 
@@ -51,7 +49,7 @@ static bool is_our_machine_account(const char *username)
 		return false;
 	}
 	truncname[ulen-1] = '\0';
-	ret = strequal(truncname, global_myname());
+	ret = strequal(truncname, lp_netbios_name());
 	SAFE_FREE(truncname);
 	return ret;
 }
@@ -79,7 +77,7 @@ NTSTATUS make_server_info_sam(struct auth_serversupplied_info **server_info,
 		return NT_STATUS_NO_SUCH_USER;
 	}
 
-	status = samu_to_SamInfo3(result, sampass, global_myname(),
+	status = samu_to_SamInfo3(result, sampass, lp_netbios_name(),
 				  &result->info3, &result->extra);
 	if (!NT_STATUS_IS_OK(status)) {
 		TALLOC_FREE(result);
diff --git a/source3/auth/token_util.c b/source3/auth/token_util.c
index da4745d8339..22df21f5ed4 100644
--- a/source3/auth/token_util.c
+++ b/source3/auth/token_util.c
@@ -31,7 +31,6 @@
 #include "../librpc/gen_ndr/netlogon.h"
 #include "../libcli/security/security.h"
 #include "../lib/util/util_pw.h"
-#include "lib/winbind_util.h"
 #include "passdb.h"
 #include "lib/privileges.h"
 
@@ -339,7 +338,7 @@ struct security_token *create_local_nt_token(TALLOC_CTX *mem_ctx,
 	DEBUG(10, ("Create local NT token for %s\n",
 		   sid_string_dbg(user_sid)));
 
-	if (!(result = TALLOC_ZERO_P(mem_ctx, struct security_token))) {
+	if (!(result = talloc_zero(mem_ctx, struct security_token))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
@@ -694,9 +693,9 @@ NTSTATUS create_token_from_username(TALLOC_CTX *mem_ctx, const char *username,
 		num_group_sids = getgroups_num_group_sids;
 
 		if (num_group_sids) {
-			group_sids = TALLOC_ARRAY(tmp_ctx, struct dom_sid, num_group_sids);
+			group_sids = talloc_array(tmp_ctx, struct dom_sid, num_group_sids);
 			if (group_sids == NULL) {
-				DEBUG(1, ("TALLOC_ARRAY failed\n"));
+				DEBUG(1, ("talloc_array failed\n"));
 				result = NT_STATUS_NO_MEMORY;
 				goto done;
 			}
@@ -733,9 +732,9 @@ NTSTATUS create_token_from_username(TALLOC_CTX *mem_ctx, const char *username,
 		}
 
 		num_group_sids = 1;
-		group_sids = TALLOC_ARRAY(tmp_ctx, struct dom_sid, num_group_sids);
+		group_sids = talloc_array(tmp_ctx, struct dom_sid, num_group_sids);
 		if (group_sids == NULL) {
-			DEBUG(1, ("TALLOC_ARRAY failed\n"));
+			DEBUG(1, ("talloc_array failed\n"));
 			result = NT_STATUS_NO_MEMORY;
 			goto done;
 		}
diff --git a/source3/auth/wscript_build b/source3/auth/wscript_build
index 8bf20515261..74d870fc1b5 100644
--- a/source3/auth/wscript_build
+++ b/source3/auth/wscript_build
@@ -16,16 +16,20 @@ AUTH_SRC = '''auth.c
 
 bld.SAMBA3_SUBSYSTEM('TOKEN_UTIL',
                     source='token_util.c',
+                    deps='samba-util',
                     vars=locals())
 
+bld.SAMBA3_SUBSYSTEM('USER_UTIL',
+                     source='user_util.c',
+                     deps='TOKEN_UTIL')
+
 bld.SAMBA3_SUBSYSTEM('AUTH_COMMON',
                     source='''auth_util.c
-			      user_util.c
 			      check_samsec.c
 			      server_info.c
 			      server_info_sam.c
-			      user_info.c
-			      user_util.c''',
+			      user_info.c''',
+                     deps='TOKEN_UTIL DCUTIL USER_UTIL',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('auth',
@@ -36,6 +40,7 @@ bld.SAMBA3_SUBSYSTEM('auth',
 bld.SAMBA3_MODULE('auth_sam',
                  subsystem='auth',
                  source=AUTH_SAM_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_sam'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_sam'))
@@ -43,6 +48,7 @@ bld.SAMBA3_MODULE('auth_sam',
 bld.SAMBA3_MODULE('auth_unix',
                  subsystem='auth',
                  source=AUTH_UNIX_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_unix'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_unix'))
@@ -50,6 +56,7 @@ bld.SAMBA3_MODULE('auth_unix',
 bld.SAMBA3_MODULE('auth_winbind',
                  subsystem='auth',
                  source=AUTH_WINBIND_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_winbind'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_winbind'))
@@ -57,6 +64,7 @@ bld.SAMBA3_MODULE('auth_winbind',
 bld.SAMBA3_MODULE('auth_wbc',
                  subsystem='auth',
                  source=AUTH_WBC_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_wbc'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_wbc'))
@@ -64,6 +72,7 @@ bld.SAMBA3_MODULE('auth_wbc',
 bld.SAMBA3_MODULE('auth_server',
                  subsystem='auth',
                  source=AUTH_SERVER_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_server'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_server'))
@@ -79,6 +88,7 @@ bld.SAMBA3_MODULE('auth_domain',
 bld.SAMBA3_MODULE('auth_builtin',
                  subsystem='auth',
                  source=AUTH_BUILTIN_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_builtin'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_builtin'))
@@ -96,3 +106,12 @@ bld.SAMBA3_MODULE('auth_script',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_script'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_script'))
+
+bld.SAMBA3_MODULE('auth_samba4',
+                  subsystem='auth',
+                  source='auth_samba4.c',
+                  init_function='',
+                  deps='auth4',
+                  allow_undefined_symbols=False,
+                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('auth_samba4'),
+                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('auth_samba4'))
diff --git a/source3/build/charset.py b/source3/build/charset.py
index 03ba2493677..44852a6c52a 100644
--- a/source3/build/charset.py
+++ b/source3/build/charset.py
@@ -8,7 +8,6 @@ def CHECK_SAMBA3_CHARSET(conf, crossbuild=False):
     '''
     if conf.CHECK_ICONV(define='HAVE_NATIVE_ICONV'):
         default_dos_charset=False
-        default_display_charset=False
         default_unix_charset=False
 
         # check for default dos charset name
@@ -17,12 +16,6 @@ def CHECK_SAMBA3_CHARSET(conf, crossbuild=False):
                 default_dos_charset=charset
                 break
 
-        # check for default display charset name
-        for charset in ['ASCII', '646']:
-            if conf.CHECK_CHARSET_EXISTS(charset, headers='iconv.h'):
-                default_display_charset=charset
-                break
-
         # check for default unix charset name
         for charset in ['UTF-8', 'UTF8']:
             if conf.CHECK_CHARSET_EXISTS(charset, headers='iconv.h'):
@@ -37,16 +30,13 @@ def CHECK_SAMBA3_CHARSET(conf, crossbuild=False):
 	# match the results we get at runtime anyway.
 	if crossbuild:
 	    default_dos_charset="CP850"
-	    default_display_charset="ASCII"
 	    default_unix_charset="UTF-8"
             # TODO: this used to warn about the set charset on cross builds
 
         conf.DEFINE('DEFAULT_DOS_CHARSET', default_dos_charset, quote=True)
-        conf.DEFINE('DEFAULT_DISPLAY_CHARSET', default_display_charset, quote=True)
         conf.DEFINE('DEFAULT_UNIX_CHARSET', default_unix_charset, quote=True)
 
     else:
         conf.DEFINE('DEFAULT_DOS_CHARSET', "ASCII", quote=True)
-        conf.DEFINE('DEFAULT_DISPLAY_CHARSET', "ASCII", quote=True)
         conf.DEFINE('DEFAULT_UNIX_CHARSET', "UTF8", quote=True)
 
diff --git a/source3/build/wscript b/source3/build/wscript
index d370daf8996..b9c6bd264c8 100644
--- a/source3/build/wscript
+++ b/source3/build/wscript
@@ -42,7 +42,7 @@ def configure(conf):
 def build(bld):
     cflags = dynconfig_cflags(bld)
     bld.SAMBA3_SUBSYSTEM('DYNCONFIG',
-                        '../dynconfig.c',
+                        '../../dynconfig/dynconfig.c',
                         deps='replace talloc tdb popt',
                         cflags=cflags)
 
diff --git a/source3/client/client.c b/source3/client/client.c
index a12f8a29653..b485df7741a 100644
--- a/source3/client/client.c
+++ b/source3/client/client.c
@@ -32,6 +32,7 @@
 #include "../libcli/smbreadline/smbreadline.h"
 #include "../libcli/security/security.h"
 #include "system/select.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "trans2.h"
 #include "libsmb/nmblib.h"
@@ -48,7 +49,6 @@ extern char tar_type;
 static int port = 0;
 static char *service;
 static char *desthost;
-static char *calling_name;
 static bool grepable = false;
 static char *cmdstr = NULL;
 const char *cmd_ptr = NULL;
@@ -505,7 +505,7 @@ static bool do_this_one(struct file_info *finfo)
 		return false;
 	}
 
-	if (finfo->mode & aDIR) {
+	if (finfo->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		return true;
 	}
 
@@ -520,7 +520,7 @@ static bool do_this_one(struct file_info *finfo)
 		return false;
 	}
 
-	if ((archive_level==1 || archive_level==2) && !(finfo->mode & aARCH)) {
+	if ((archive_level==1 || archive_level==2) && !(finfo->mode & FILE_ATTRIBUTE_ARCHIVE)) {
 		DEBUG(3,("archive %s failed\n", finfo->name));
 		return false;
 	}
@@ -547,7 +547,7 @@ static NTSTATUS display_finfo(struct cli_state *cli_state, struct file_info *fin
 	if (!showacls) {
 		d_printf("  %-30s%7.7s %8.0f  %s",
 			 finfo->name,
-			 attrib_string(finfo->mode),
+			 attrib_string(talloc_tos(), finfo->mode),
 		 	(double)finfo->size,
 			time_to_asc(t));
 		dir_total += finfo->size;
@@ -569,7 +569,7 @@ static NTSTATUS display_finfo(struct cli_state *cli_state, struct file_info *fin
 		}
 		/* print file meta date header */
 		d_printf( "FILENAME:%s\n", finfo->name);
-		d_printf( "MODE:%s\n", attrib_string(finfo->mode));
+		d_printf( "MODE:%s\n", attrib_string(talloc_tos(), finfo->mode));
 		d_printf( "SIZE:%.0f\n", (double)finfo->size);
 		d_printf( "MTIME:%s", time_to_asc(t));
 		status = cli_ntcreate(cli_state, afname, 0,
@@ -702,7 +702,7 @@ static void add_to_do_list_queue(const char *entry)
 		}
 	}
 	if (do_list_queue) {
-		safe_strcpy_base(do_list_queue + do_list_queue_end,
+		strlcpy_base(do_list_queue + do_list_queue_end,
 				 entry, do_list_queue, do_list_queue_size);
 		do_list_queue_end = new_end;
 		DEBUG(4,("added %s to do_list_queue (start=%d, end=%d)\n",
@@ -752,7 +752,7 @@ static NTSTATUS do_list_helper(const char *mntpoint, struct file_info *f,
 		*dir_end = '\0';
 	}
 
-	if (f->mode & aDIR) {
+	if (f->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		if (do_list_dirs && do_this_one(f)) {
 			status = do_list_fn(cli_state, f, dir);
 			if (!NT_STATUS_IS_OK(status)) {
@@ -924,7 +924,7 @@ NTSTATUS do_list(const char *mask,
 static int cmd_dir(void)
 {
 	TALLOC_CTX *ctx = talloc_tos();
-	uint16 attribute = aDIR | aSYSTEM | aHIDDEN;
+	uint16 attribute = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN;
 	char *mask = NULL;
 	char *buf = NULL;
 	int rc = 1;
@@ -974,7 +974,7 @@ static int cmd_dir(void)
 static int cmd_du(void)
 {
 	TALLOC_CTX *ctx = talloc_tos();
-	uint16 attribute = aDIR | aSYSTEM | aHIDDEN;
+	uint16 attribute = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN;
 	char *mask = NULL;
 	char *buf = NULL;
 	NTSTATUS status;
@@ -1147,8 +1147,8 @@ static int do_get(const char *rname, const char *lname_in, bool reget)
 		close(handle);
 	}
 
-	if (archive_level >= 2 && (attr & aARCH)) {
-		cli_setatr(cli, rname, attr & ~(uint16)aARCH, 0);
+	if (archive_level >= 2 && (attr & FILE_ATTRIBUTE_ARCHIVE)) {
+		cli_setatr(cli, rname, attr & ~(uint16)FILE_ATTRIBUTE_ARCHIVE, 0);
 	}
 
 	{
@@ -1233,7 +1233,7 @@ static NTSTATUS do_mget(struct cli_state *cli_state, struct file_info *finfo,
 		return NT_STATUS_UNSUCCESSFUL;
 	}
 
-	if (finfo->mode & aDIR) {
+	if (finfo->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		if (asprintf(&quest,
 			 "Get directory %s? ",finfo->name) < 0) {
 			return NT_STATUS_NO_MEMORY;
@@ -1251,7 +1251,7 @@ static NTSTATUS do_mget(struct cli_state *cli_state, struct file_info *finfo,
 	}
 	SAFE_FREE(quest);
 
-	if (!(finfo->mode & aDIR)) {
+	if (!(finfo->mode & FILE_ATTRIBUTE_DIRECTORY)) {
 		rname = talloc_asprintf(ctx,
 				"%s%s",
 				client_get_cur_dir(),
@@ -1306,7 +1306,7 @@ static NTSTATUS do_mget(struct cli_state *cli_state, struct file_info *finfo,
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	status = do_list(mget_mask, aSYSTEM | aHIDDEN | aDIR,do_mget,false, true);
+	status = do_list(mget_mask, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY,do_mget,false, true);
 	if (!NT_STATUS_IS_OK(status)) {
 		return status;
 	}
@@ -1395,13 +1395,13 @@ static int cmd_more(void)
 static int cmd_mget(void)
 {
 	TALLOC_CTX *ctx = talloc_tos();
-	uint16 attribute = aSYSTEM | aHIDDEN;
+	uint16 attribute = FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN;
 	char *mget_mask = NULL;
 	char *buf = NULL;
 	NTSTATUS status = NT_STATUS_OK;
 
 	if (recurse) {
-		attribute |= aDIR;
+		attribute |= FILE_ATTRIBUTE_DIRECTORY;
 	}
 
 	abort_mget = false;
@@ -1605,7 +1605,7 @@ static int cmd_altname(void)
 
 static char *attr_str(TALLOC_CTX *mem_ctx, uint16_t mode)
 {
-	char *attrs = TALLOC_ZERO_ARRAY(mem_ctx, char, 17);
+	char *attrs = talloc_zero_array(mem_ctx, char, 17);
 	int i = 0;
 
 	if (!(mode & FILE_ATTRIBUTE_NORMAL)) {
@@ -2306,12 +2306,12 @@ static NTSTATUS do_del(struct cli_state *cli_state, struct file_info *finfo,
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	if (finfo->mode & aDIR) {
+	if (finfo->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		TALLOC_FREE(mask);
 		return NT_STATUS_OK;
 	}
 
-	status = cli_unlink(cli_state, mask, aSYSTEM | aHIDDEN);
+	status = cli_unlink(cli_state, mask, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	if (!NT_STATUS_IS_OK(status)) {
 		d_printf("%s deleting remote file %s\n",
 			 nt_errstr(status), mask);
@@ -2330,10 +2330,10 @@ static int cmd_del(void)
 	char *mask = NULL;
 	char *buf = NULL;
 	NTSTATUS status = NT_STATUS_OK;
-	uint16 attribute = aSYSTEM | aHIDDEN;
+	uint16 attribute = FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN;
 
 	if (recurse) {
-		attribute |= aDIR;
+		attribute |= FILE_ATTRIBUTE_DIRECTORY;
 	}
 
 	mask = talloc_strdup(ctx, client_get_cur_dir());
@@ -3875,11 +3875,12 @@ static int cmd_lcd(void)
 				buf, strerror(errno));
 		}
 	}
-	d = TALLOC_ARRAY(ctx, char, PATH_MAX+1);
+	d = sys_getwd();
 	if (!d) {
 		return 1;
 	}
-	DEBUG(2,("the local directory is now %s\n",sys_getwd(d)));
+	DEBUG(2,("the local directory is now %s\n",d));
+	SAFE_FREE(d);
 	return 0;
 }
 
@@ -4516,7 +4517,7 @@ static NTSTATUS completion_remote_filter(const char *mnt,
 		return NT_STATUS_OK;
 	}
 
-	if ((info->dirmask[0] == 0) && !(f->mode & aDIR))
+	if ((info->dirmask[0] == 0) && !(f->mode & FILE_ATTRIBUTE_DIRECTORY))
 		info->matches[info->count] = SMB_STRDUP(f->name);
 	else {
 		TALLOC_CTX *ctx = talloc_stackframe();
@@ -4532,7 +4533,7 @@ static NTSTATUS completion_remote_filter(const char *mnt,
 			TALLOC_FREE(ctx);
 			return NT_STATUS_NO_MEMORY;
 		}
-		if (f->mode & aDIR) {
+		if (f->mode & FILE_ATTRIBUTE_DIRECTORY) {
 			tmp = talloc_asprintf_append(tmp, "%s",
 						     CLI_DIRSEP_STR);
 		}
@@ -4546,7 +4547,7 @@ static NTSTATUS completion_remote_filter(const char *mnt,
 	if (info->matches[info->count] == NULL) {
 		return NT_STATUS_OK;
 	}
-	if (f->mode & aDIR) {
+	if (f->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		smb_readline_ca_char(0);
 	}
 	if (info->count == 1) {
@@ -4629,7 +4630,7 @@ static char **remote_completion(const char *text, int len)
 	if (!cli_resolve_path(ctx, "", auth_info, cli, dirmask, &targetcli, &targetpath)) {
 		goto cleanup;
 	}
-	status = cli_list(targetcli, targetpath, aDIR | aSYSTEM | aHIDDEN,
+	status = cli_list(targetcli, targetpath, FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN,
 			  completion_remote_filter, (void *)&info);
 	if (!NT_STATUS_IS_OK(status)) {
 		goto cleanup;
@@ -5036,46 +5037,16 @@ static int do_tar_op(const char *base_directory)
 
 static int do_message_op(struct user_auth_info *a_info)
 {
-	struct sockaddr_storage ss;
-	struct nmb_name called, calling;
-	fstring server_name;
-	char name_type_hex[10];
-	int msg_port;
 	NTSTATUS status;
 
-	make_nmb_name(&calling, calling_name, 0x0);
-	make_nmb_name(&called , desthost, name_type);
-
-	fstrcpy(server_name, desthost);
-	snprintf(name_type_hex, sizeof(name_type_hex), "#%X", name_type);
-	fstrcat(server_name, name_type_hex);
-
-        zero_sockaddr(&ss);
-	if (have_ip)
-		ss = dest_ss;
-
-	/* we can only do messages over port 139 (to windows clients at least) */
-
-	msg_port = port ? port : 139;
-
-	if (!(cli=cli_initialise())) {
-		d_printf("Connection to %s failed\n", desthost);
-		return 1;
-	}
-	cli_set_port(cli, msg_port);
-
-	status = cli_connect(cli, server_name, &ss);
+	status = cli_connect_nb(desthost, have_ip ? &dest_ss : NULL,
+				port ? port : 139, name_type,
+				lp_netbios_name(), Undefined, &cli);
 	if (!NT_STATUS_IS_OK(status)) {
 		d_printf("Connection to %s failed. Error %s\n", desthost, nt_errstr(status));
 		return 1;
 	}
 
-	if (!cli_session_request(cli, &calling, &called)) {
-		d_printf("session request failed\n");
-		cli_shutdown(cli);
-		return 1;
-	}
-
 	send_message(get_cmdline_auth_info_username(a_info));
 	cli_shutdown(cli);
 
@@ -5096,7 +5067,6 @@ static int do_message_op(struct user_auth_info *a_info)
 	poptContext pc;
 	char *p;
 	int rc = 0;
-	fstring new_workgroup;
 	bool tar_opt = false;
 	bool service_opt = false;
 	struct poptOption long_options[] = {
@@ -5126,12 +5096,6 @@ static int do_message_op(struct user_auth_info *a_info)
 		exit(ENOMEM);
 	}
 
-	/* initialize the workgroup name so we can determine whether or
-	   not it was set by a command line option */
-
-	set_global_myworkgroup( "" );
-	set_global_myname( "" );
-
         /* set default debug level to 1 regardless of what smb.conf sets */
 	setup_logging( "smbclient", DEBUG_DEFAULT_STDERR );
 	load_case_tables();
@@ -5277,18 +5241,6 @@ static int do_message_op(struct user_auth_info *a_info)
 					       poptGetArg(pc));
 	}
 
-	/* save the workgroup...
-
-	   FIXME!! do we need to do this for other options as well
-	   (or maybe a generic way to keep lp_load() from overwriting
-	   everything)?  */
-
-	fstrcpy( new_workgroup, lp_workgroup() );
-	calling_name = talloc_strdup(frame, global_myname() );
-	if (!calling_name) {
-		exit(ENOMEM);
-	}
-
 	if ( override_logfile )
 		setup_logging( lp_logfile(), DEBUG_FILE );
 
@@ -5322,17 +5274,6 @@ static int do_message_op(struct user_auth_info *a_info)
 		}
 	}
 
-	if ( strlen(new_workgroup) != 0 ) {
-		set_global_myworkgroup( new_workgroup );
-	}
-
-	if ( strlen(calling_name) != 0 ) {
-		set_global_myname( calling_name );
-	} else {
-		TALLOC_FREE(calling_name);
-		calling_name = talloc_strdup(frame, global_myname() );
-	}
-
 	smb_encrypt = get_cmdline_auth_info_smb_encrypt(auth_info);
 	if (!init_names()) {
 		fprintf(stderr, "init_names() failed\n");
diff --git a/source3/client/client_proto.h b/source3/client/client_proto.h
index fa3ca40d6b7..d119014abdc 100644
--- a/source3/client/client_proto.h
+++ b/source3/client/client_proto.h
@@ -23,6 +23,8 @@
 #ifndef _CLIENT_PROTO_H_
 #define _CLIENT_PROTO_H_
 
+struct cli_state;
+struct file_info;
 
 /* The following definitions come from client/client.c  */
 
diff --git a/source3/client/clitar.c b/source3/client/clitar.c
index b46115becf3..4dc202e17ea 100644
--- a/source3/client/clitar.c
+++ b/source3/client/clitar.c
@@ -38,6 +38,7 @@
 #include "system/filesys.h"
 #include "clitar.h"
 #include "client/client_proto.h"
+#include "libsmb/libsmb.h"
 
 static int clipfind(char **aret, int ret, char *tok);
 
@@ -70,7 +71,7 @@ extern struct cli_state *cli;
 #define ATTRSET 1
 #define ATTRRESET 0
 
-static uint16 attribute = aDIR | aSYSTEM | aHIDDEN;
+static uint16 attribute = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN;
 
 #ifndef CLIENT_TIMEOUT
 #define CLIENT_TIMEOUT (30*1000)
@@ -101,7 +102,7 @@ char tar_type='\0';
 static char **cliplist=NULL;
 static int clipn=0;
 static bool must_free_cliplist = False;
-extern const char *cmd_ptr;
+extern char *cmd_ptr;
 
 extern bool lowercase;
 extern uint16 cnum;
@@ -136,23 +137,6 @@ static void unfixtarname(char *tptr, char *fp, int l, bool first);
  * tar specific utitlities
  */
 
-/*******************************************************************
-Create  a string of size size+1 (for the null)
-*******************************************************************/
-
-static char *string_create_s(int size)
-{
-	char *tmp;
-
-	tmp = (char *)SMB_MALLOC(size+1);
-
-	if (tmp == NULL) {
-		DEBUG(0, ("Out of memory in string_create_s\n"));
-	}
-
-	return(tmp);
-}
-
 /****************************************************************************
 Write a tar header to buffer
 ****************************************************************************/
@@ -197,7 +181,7 @@ static void writetarheader(int f, const char *aname, uint64_t size, time_t mtime
 	/* write out a "standard" tar format header */
 
 	hb.dbuf.name[NAMSIZ-1]='\0';
-	safe_strcpy(hb.dbuf.mode, amode, sizeof(hb.dbuf.mode)-1);
+	strlcpy(hb.dbuf.mode, amode ? amode : "", sizeof(hb.dbuf.mode));
 	oct_it((uint64_t)0, 8, hb.dbuf.uid);
 	oct_it((uint64_t)0, 8, hb.dbuf.gid);
 	oct_it((uint64_t) size, 13, hb.dbuf.size);
@@ -263,12 +247,12 @@ static long readtarheader(union hblock *hb, file_info2 *finfo, const char *prefi
 		return -1;
 	}
 
-	if ((finfo->name = string_create_s(strlen(prefix) + strlen(hb -> dbuf.name) + 3)) == NULL) {
+	if ((finfo->name = SMB_MALLOC(strlen(prefix) + strlen(hb -> dbuf.name) + 4)) == NULL) {
 		DEBUG(0, ("Out of space creating file_info2 for %s\n", hb -> dbuf.name));
 		return(-1);
 	}
 
-	safe_strcpy(finfo->name, prefix, strlen(prefix) + strlen(hb -> dbuf.name) + 3);
+	strlcpy(finfo->name, prefix, strlen(prefix) + strlen(hb -> dbuf.name) + 4);
 
 	/* use l + 1 to do the null too; do prefix - prefcnt to zap leading slash */
 	unfixtarname(finfo->name + strlen(prefix), hb->dbuf.name,
@@ -294,7 +278,7 @@ of link other than a GNUtar Longlink - ignoring\n"));
 
 	if ((unoct(hb->dbuf.mode, sizeof(hb->dbuf.mode)) & S_IFDIR) ||
 				(*(finfo->name+strlen(finfo->name)-1) == '\\')) {
-		finfo->mode=aDIR;
+		finfo->mode=FILE_ATTRIBUTE_DIRECTORY;
 	} else {
 		finfo->mode=0; /* we don't care about mode at the moment, we'll
 				* just make it a regular file */
@@ -521,14 +505,16 @@ static bool ensurepath(const char *fname)
 	/* ensures path exists */
 
 	char *partpath, *ffname;
+	size_t fnamelen = strlen(fname)+1;
 	const char *p=fname;
 	char *basehack;
 	char *saveptr;
+	NTSTATUS status;
 
 	DEBUG(5, ( "Ensurepath called with: %s\n", fname));
 
-	partpath = string_create_s(strlen(fname));
-	ffname = string_create_s(strlen(fname));
+	partpath = SMB_MALLOC(fnamelen);
+	ffname = SMB_MALLOC(fnamelen);
 
 	if ((partpath == NULL) || (ffname == NULL)){
 		DEBUG(0, ("Out of memory in ensurepath: %s\n", fname));
@@ -541,7 +527,7 @@ static bool ensurepath(const char *fname)
 
 	/* fname copied to ffname so can strtok_r */
 
-	safe_strcpy(ffname, fname, strlen(fname));
+	strlcpy(ffname, fname, fnamelen);
 
 	/* do a `basename' on ffname, so don't try and make file name directory */
 	if ((basehack=strrchr_m(ffname, '\\')) == NULL) {
@@ -555,20 +541,22 @@ static bool ensurepath(const char *fname)
 	p=strtok_r(ffname, "\\", &saveptr);
 
 	while (p) {
-		safe_strcat(partpath, p, strlen(fname) + 1);
+		strlcat(partpath, p, fnamelen);
 
-		if (!NT_STATUS_IS_OK(cli_chkpath(cli, partpath))) {
-			if (!NT_STATUS_IS_OK(cli_mkdir(cli, partpath))) {
+		status = cli_chkpath(cli, partpath);
+		if (!NT_STATUS_IS_OK(status)) {
+			status = cli_mkdir(cli, partpath);
+			if (!NT_STATUS_IS_OK(status)) {
 				SAFE_FREE(partpath);
 				SAFE_FREE(ffname);
-				DEBUG(0, ("Error mkdir %s\n", cli_errstr(cli)));
+				DEBUG(0, ("Error mkdir %s\n", nt_errstr(status)));
 				return False;
 			} else {
 				DEBUG(3, ("mkdirhiering %s\n", partpath));
 			}
 		}
 
-		safe_strcat(partpath, "\\", strlen(fname) + 1);
+		strlcat(partpath, "\\", fnamelen);
 		p = strtok_r(NULL, "/\\", &saveptr);
 	}
 
@@ -596,6 +584,7 @@ static int padit(char *buf, uint64_t bufsize, uint64_t padsize)
 static void do_setrattr(char *name, uint16 attr, int set)
 {
 	uint16 oldattr;
+	NTSTATUS status;
 
 	if (!NT_STATUS_IS_OK(cli_getatr(cli, name, &oldattr, NULL, NULL))) {
 		return;
@@ -607,8 +596,9 @@ static void do_setrattr(char *name, uint16 attr, int set)
 		attr = oldattr & ~attr;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_setatr(cli, name, attr, 0))) {
-		DEBUG(1,("setatr failed: %s\n", cli_errstr(cli)));
+	status = cli_setatr(cli, name, attr, 0);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(1, ("setatr failed: %s\n", nt_errstr(status)));
 	}
 }
 
@@ -671,28 +661,26 @@ static NTSTATUS do_atar(const char *rname_in, char *lname,
 	status = cli_open(cli, rname, O_RDONLY, DENY_NONE, &fnum);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0,("%s opening remote file %s (%s)\n",
-				cli_errstr(cli),rname, client_get_cur_dir()));
+				nt_errstr(status),rname, client_get_cur_dir()));
 		goto cleanup;
 	}
 
-	finfo.name = string_create_s(strlen(rname));
+	finfo.name = smb_xstrdup(rname);
 	if (finfo.name == NULL) {
 		DEBUG(0, ("Unable to allocate space for finfo.name in do_atar\n"));
 		status = NT_STATUS_NO_MEMORY;
 		goto cleanup;
 	}
 
-	safe_strcpy(finfo.name,rname, strlen(rname));
-
 	DEBUG(3,("file %s attrib 0x%X\n",finfo.name,finfo.mode));
 
-	if (tar_inc && !(finfo.mode & aARCH)) {
+	if (tar_inc && !(finfo.mode & FILE_ATTRIBUTE_ARCHIVE)) {
 		DEBUG(4, ("skipping %s - archive bit not set\n", finfo.name));
 		shallitime=0;
-	} else if (!tar_system && (finfo.mode & aSYSTEM)) {
+	} else if (!tar_system && (finfo.mode & FILE_ATTRIBUTE_SYSTEM)) {
 		DEBUG(4, ("skipping %s - system bit is set\n", finfo.name));
 		shallitime=0;
-	} else if (!tar_hidden && (finfo.mode & aHIDDEN)) {
+	} else if (!tar_hidden && (finfo.mode & FILE_ATTRIBUTE_HIDDEN)) {
 		DEBUG(4, ("skipping %s - hidden bit is set\n", finfo.name));
 		shallitime=0;
 	} else {
@@ -784,7 +772,7 @@ static NTSTATUS do_atar(const char *rname_in, char *lname,
 
 		/* if shallitime is true then we didn't skip */
 		if (tar_reset && !dry_run)
-			(void) do_setrattr(finfo.name, aARCH, ATTRRESET);
+			(void) do_setrattr(finfo.name, FILE_ATTRIBUTE_ARCHIVE, ATTRRESET);
 
 		clock_gettime_mono(&tp_end);
 		this_time = (tp_end.tv_sec - tp_start.tv_sec)*1000 + (tp_end.tv_nsec - tp_start.tv_nsec)/1000000;
@@ -852,7 +840,7 @@ static NTSTATUS do_tar(struct cli_state *cli_state, struct file_info *finfo,
 		TALLOC_FREE(exclaim);
 	}
 
-	if (finfo->mode & aDIR) {
+	if (finfo->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		char *saved_curdir = NULL;
 		char *new_cd = NULL;
 		char *mtar_mask = NULL;
@@ -934,9 +922,12 @@ static void unfixtarname(char *tptr, char *fp, int l, bool first)
 			fp++;
 			l--;
 		}
+		if (l <= 0) {
+			return;
+		}
 	}
 
-	safe_strcpy(tptr, fp, l);
+	strlcpy(tptr, fp, l);
 	string_replace(tptr, '/', '\\');
 }
 
@@ -1045,8 +1036,12 @@ static int get_file(file_info2 finfo)
 		dsize = MIN(dsize, rsize);  /* Should be only what is left */
 		DEBUG(5, ("writing %i bytes, bpos = %i ...\n", dsize, bpos));
 
-		if (cli_write(cli, fnum, 0, buffer_p + bpos, pos, dsize) != dsize) {
-			DEBUG(0, ("Error writing remote file\n"));
+		status = cli_writeall(cli, fnum, 0,
+				      (uint8_t *)(buffer_p + bpos), pos,
+				      dsize, NULL);
+		if (!NT_STATUS_IS_OK(status)) {
+			DEBUG(0, ("Error writing remote file: %s\n",
+				  nt_errstr(status)));
 			return 0;
 		}
 
@@ -1092,10 +1087,10 @@ static int get_file(file_info2 finfo)
 	}
 
 	/* Now close the file ... */
-
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum))) {
+	status = cli_close(cli, fnum);
+	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0, ("Error %s closing remote file\n",
-			cli_errstr(cli)));
+			nt_errstr(status)));
 		return(False);
 	}
 
@@ -1166,7 +1161,8 @@ static char *get_longfilename(file_info2 finfo)
 			return(NULL);
 		}
 
-		unfixtarname(longname + offset, buffer_p, MIN(TBLOCK, finfo.size), first--);
+		unfixtarname(longname + offset, buffer_p,
+			namesize - offset, first--);
 		DEBUG(5, ("UnfixedName: %s, buffer: %s\n", longname, buffer_p));
 
 		offset += TBLOCK;
@@ -1301,7 +1297,7 @@ int cmd_block(void)
 	char *buf;
 	int block;
 
-	if (!next_token_talloc(ctx, &cmd_ptr,&buf,NULL)) {
+	if (!next_token_talloc(ctx, (const char **)&cmd_ptr,&buf,NULL)) {
 		DEBUG(0, ("blocksize <n>\n"));
 		return 1;
 	}
@@ -1326,7 +1322,7 @@ int cmd_tarmode(void)
 	TALLOC_CTX *ctx = talloc_tos();
 	char *buf;
 
-	while (next_token_talloc(ctx, &cmd_ptr,&buf,NULL)) {
+	while (next_token_talloc(ctx, (const char **)&cmd_ptr,&buf,NULL)) {
 		if (strequal(buf, "full"))
 			tar_inc=False;
 		else if (strequal(buf, "inc"))
@@ -1376,7 +1372,7 @@ int cmd_setmode(void)
 
 	attra[0] = attra[1] = 0;
 
-	if (!next_token_talloc(ctx, &cmd_ptr,&buf,NULL)) {
+	if (!next_token_talloc(ctx, (const char **)&cmd_ptr,&buf,NULL)) {
 		DEBUG(0, ("setmode <filename> <[+|-]rsha>\n"));
 		return 1;
 	}
@@ -1389,7 +1385,7 @@ int cmd_setmode(void)
 		return 1;
 	}
 
-	while (next_token_talloc(ctx, &cmd_ptr,&buf,NULL)) {
+	while (next_token_talloc(ctx, (const char **)&cmd_ptr,&buf,NULL)) {
 		q=buf;
 
 		while(*q) {
@@ -1401,16 +1397,16 @@ int cmd_setmode(void)
 					direct=0;
 					break;
 				case 'r':
-					attra[direct]|=aRONLY;
+					attra[direct]|=FILE_ATTRIBUTE_READONLY;
 					break;
 				case 'h':
-					attra[direct]|=aHIDDEN;
+					attra[direct]|=FILE_ATTRIBUTE_HIDDEN;
 					break;
 				case 's':
-					attra[direct]|=aSYSTEM;
+					attra[direct]|=FILE_ATTRIBUTE_SYSTEM;
 					break;
 				case 'a':
-					attra[direct]|=aARCH;
+					attra[direct]|=FILE_ATTRIBUTE_ARCHIVE;
 					break;
 				default:
 					DEBUG(0, ("setmode <filename> <perm=[+|-]rsha>\n"));
@@ -1491,7 +1487,7 @@ int cmd_tar(void)
 	int argcl = 0;
 	int ret;
 
-	if (!next_token_talloc(ctx, &cmd_ptr,&buf,NULL)) {
+	if (!next_token_talloc(ctx, (const char **)&cmd_ptr,&buf,NULL)) {
 		DEBUG(0,("tar <c|x>[IXbgan] <filename>\n"));
 		return 1;
 	}
@@ -1717,7 +1713,7 @@ static int read_inclusion_file(char *filename)
 			}
 		}
 
-		safe_strcpy(inclusion_buffer + inclusion_buffer_sofar, buf, inclusion_buffer_size - inclusion_buffer_sofar);
+		strlcpy(inclusion_buffer + inclusion_buffer_sofar, buf, inclusion_buffer_size - inclusion_buffer_sofar);
 		inclusion_buffer_sofar += strlen(buf) + 1;
 		clipn++;
 	}
diff --git a/source3/client/dnsbrowse.c b/source3/client/dnsbrowse.c
index c252b253438..03f87afe140 100644
--- a/source3/client/dnsbrowse.c
+++ b/source3/client/dnsbrowse.c
@@ -124,7 +124,7 @@ do_smb_browse_reply(DNSServiceRef sdRef, DNSServiceFlags flags,
 		return;
 	}
 
-	bresult = TALLOC_ARRAY(talloc_tos(), struct mdns_smbsrv_result, 1);
+	bresult = talloc_array(talloc_tos(), struct mdns_smbsrv_result, 1);
 	if (bresult == NULL) {
 		return;
 	}
diff --git a/source3/client/smbspool.c b/source3/client/smbspool.c
index 98c350d7ddb..e16086dd679 100644
--- a/source3/client/smbspool.c
+++ b/source3/client/smbspool.c
@@ -25,6 +25,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "system/passwd.h"
+#include "libsmb/libsmb.h"
 
 /*
  * Starting with CUPS 1.3, Kerberos support is provided by cupsd including
@@ -595,14 +596,19 @@ smb_print(struct cli_state * cli,	/* I - SMB connection */
 	tbytes = 0;
 
 	while ((nbytes = fread(buffer, 1, sizeof(buffer), fp)) > 0) {
-		if (cli_write(cli, fnum, 0, buffer, tbytes, nbytes) != nbytes) {
-			int status = get_exit_code(cli, cli_nt_error(cli));
-
-			fprintf(stderr, "ERROR: Error writing spool: %s\n", cli_errstr(cli));
-			fprintf(stderr, "DEBUG: Returning status %d...\n", status);
+		NTSTATUS status;
+
+		status = cli_writeall(cli, fnum, 0, (uint8_t *)buffer,
+				      tbytes, nbytes, NULL);
+		if (!NT_STATUS_IS_OK(status)) {
+			int ret = get_exit_code(cli, status);
+			fprintf(stderr, "ERROR: Error writing spool: %s\n",
+				nt_errstr(status));
+			fprintf(stderr, "DEBUG: Returning status %d...\n",
+				ret);
 			cli_close(cli, fnum);
 
-			return (status);
+			return (ret);
 		}
 		tbytes += nbytes;
 	}
diff --git a/source3/configure.in b/source3/configure.in
index b2c1856becc..814c7356d2a 100644
--- a/source3/configure.in
+++ b/source3/configure.in
@@ -6,7 +6,7 @@ AC_PREREQ(2.54)
 AC_INIT([Samba],[3],[samba-technical@samba.org])
 
 AC_CONFIG_SRCDIR([include/includes.h])
-AC_CONFIG_HEADER(include/config.h)
+AC_CONFIG_HEADER(include/autoconf/config.h)
 AC_DEFINE(CONFIG_H_IS_FROM_SAMBA,1,[Marker for samba's config.h])
 
 case "$PATH" in
@@ -26,6 +26,7 @@ AC_LIBREPLACE_CC_CHECKS
 AC_LIBREPLACE_RUNTIME_LIB_PATH_VAR
 
 m4_include(../lib/tevent/libtevent.m4)
+m4_include(../lib/ccan/libccan.m4)
 
 LIBTEVENT_OBJ0=""
 for obj in ${TEVENT_OBJ}; do
@@ -34,11 +35,13 @@ done
 AC_SUBST(LIBTEVENT_OBJ0)
 LIBS="${LIBS} ${TEVENT_LIBS}"
 
-SAMBA_CPPFLAGS="-Iinclude -I${srcdir-.}/include  -I. -I${srcdir-.}"
+SAMBA_CPPFLAGS="-Iinclude/autoconf -Iinclude -I${srcdir-.}/include  -I. -I${srcdir-.}"
 SAMBA_CPPFLAGS="${SAMBA_CPPFLAGS} -I${srcdir-.}/../lib/replace"
 SAMBA_CPPFLAGS="${SAMBA_CPPFLAGS} ${TEVENT_CFLAGS}"
+SAMBA_CPPFLAGS="${SAMBA_CPPFLAGS} ${CCAN_CFLAGS}"
 SAMBA_CPPFLAGS="${SAMBA_CPPFLAGS} -I${srcdir-.}/librpc"
 SAMBA_CPPFLAGS="${SAMBA_CPPFLAGS} -I${srcdir-.}/.."
+SAMBA_CPPFLAGS="${SAMBA_CPPFLAGS} -I${srcdir-.}/../lib/tdb_compat"
 
 SAMBA_CONFIGURE_CPPFLAGS="${SAMBA_CPPFLAGS} -I${srcdir-.}/../lib/popt"
 
@@ -73,6 +76,7 @@ AC_SUBST(LIBWBCLIENT_SHARED)
 AC_SUBST(LIBWBCLIENT_STATIC_TARGET)
 AC_SUBST(LIBWBCLIENT_STATIC)
 AC_SUBST(LIBWBCLIENT_SOVER)
+AC_SUBST(LIBWBCLIENT_FULLVER)
 AC_SUBST(LIBWBCLIENT)
 AC_SUBST(LIBWBCLIENT_LIBS)
 
@@ -196,7 +200,14 @@ AC_ARG_WITH(profiling-data,
 )
 
 dnl Checks for programs.
-smbtorture4_possible=yes
+AC_ARG_ENABLE(smbtorture4,
+	[AS_HELP_STRING([--enable-smbtorture4], [Enable building smbtorture4 (default=auto)])])
+
+if test x$enable_smbtorture4 != xno; then
+	smbtorture4_possible=yes
+else
+	smbtorture4_possible=no
+fi
 
 AC_PROG_INSTALL
 AC_PROG_AWK
@@ -518,6 +529,8 @@ case "$host_os" in
 	  			AC_MSG_RESULT([no large file support])
 				;;
 			5.*)
+			LDFLAGS="$LDFLAGS -lthread"
+			CPPFLAGS="$CPPFLAGS -D_REENTRANT"
 			AC_MSG_RESULT([enabling large file support])
 			if test "$ac_cv_prog_gcc" = yes; then
 				${CC-cc} -v >conftest.c 2>&1
@@ -525,20 +538,17 @@ case "$host_os" in
 				rm -fr conftest.c
 				case "$ac_cv_gcc_compiler_version_number" in
 					*"gcc version 2.6"*|*"gcc version 2.7"*)
-						CPPFLAGS="$CPPFLAGS -D_LARGEFILE64_SOURCE -D_REENTRANT"
-						LDFLAGS="$LDFLAGS -lthread"
+						CPPFLAGS="$CPPFLAGS -D_LARGEFILE64_SOURCE"
 						AC_DEFINE(_LARGEFILE64_SOURCE, 1, [Whether to enable large file support])
 						;;
 					*)
-						CPPFLAGS="$CPPFLAGS -D_LARGEFILE_SOURCE -D_REENTRANT -D_FILE_OFFSET_BITS=64"
-						LDFLAGS="$LDFLAGS -lthread"
+						CPPFLAGS="$CPPFLAGS -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64"
 						AC_DEFINE(_LARGEFILE64_SOURCE, 1, [Whether to enable large file support])
 						AC_DEFINE(_FILE_OFFSET_BITS, 64, [File offset bits])
 						;;
 				esac
 			else
-				CPPFLAGS="$CPPFLAGS -D_LARGEFILE_SOURCE -D_REENTRANT -D_FILE_OFFSET_BITS=64"
-				LDFLAGS="$LDFLAGS -lthread"
+				CPPFLAGS="$CPPFLAGS -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64"
 				AC_DEFINE(_LARGEFILE64_SOURCE, 1, [Whether to enable large file support])
 				AC_DEFINE(_FILE_OFFSET_BITS, 64, [File offset bits])
 			fi
@@ -1081,7 +1091,7 @@ AC_CHECK_FUNCS(getpwent_r)
 AC_CHECK_FUNCS(getdents64)
 AC_CHECK_FUNCS(setenv strcasecmp fcvt fcvtl)
 AC_CHECK_FUNCS(syslog vsyslog timegm)
-AC_CHECK_FUNCS(setlocale nl_langinfo)
+AC_CHECK_FUNCS(setlocale)
 AC_CHECK_FUNCS(nanosleep,,[AC_CHECK_LIB_EXT(rt, LIBS, nanosleep)])
 AC_CHECK_FUNCS(lutimes futimes utimensat futimens)
 AC_CHECK_FUNCS(mlock munlock mlockall munlockall)
@@ -1090,7 +1100,7 @@ AC_CHECK_HEADERS(sys/mman.h)
 # setbuffer, shmget, shm_open are needed for smbtorture
 AC_CHECK_FUNCS(shmget shm_open)
 AC_CHECK_FUNCS(gettext dgettext)
-AC_CHECK_FUNCS(bindtextdomain textdomain)
+AC_CHECK_FUNCS(bindtextdomain textdomain bind_textdomain_codeset)
 AC_CHECK_FUNCS(strupr)
 
 # Find a method of generating a stack trace
@@ -1118,6 +1128,7 @@ AC_CHECK_HEADERS(gpfs_gpl.h)
 if test x"$ac_cv_header_gpfs_gpl_h" = x"yes"; then
     AC_DEFINE(HAVE_GPFS,1,[Whether GPFS GPL headers are available])
     default_shared_modules="$default_shared_modules vfs_gpfs"
+    default_shared_modules="$default_shared_modules vfs_gpfs_hsm_notify"
 fi
 
 #############################################
@@ -1411,6 +1422,14 @@ AC_CHECK_LIB(resolv, dn_expand)
 AC_CHECK_LIB(resolv, _dn_expand)
 AC_CHECK_LIB(resolv, __dn_expand)
 
+AC_TRY_COMPILE([#include <resolv.h>],
+	       [struct __res_state s; res_ninit(&s);],
+	       ac_cv_have_res_ninit=yes,ac_cv_have_res_ninit=no)
+
+if test x"$ac_cv_have_res_ninit" = x"yes"; then
+	AC_DEFINE(HAVE_RES_NINIT, 1, [Whether we have res_ninit])
+fi
+
 #
 # Check for the functions putprpwnam, set_auth_parameters,
 # getspnam, bigcrypt and getprpwnam in -lsec and -lsecurity
@@ -2072,8 +2091,11 @@ LINK_LIBSMBCLIENT=STATIC
 #  TODO: for talloc and tdb (at least), these should
 #  be extracted from their respective source directories
 #
-AC_ARG_ENABLE(external_libtalloc, [AS_HELP_STRING([--enable-external-libtalloc], [Enable external talloc [default=auto]])],
-[ enable_external_libtalloc=$enableval ], [ enable_external_libtalloc=auto ])
+AC_ARG_ENABLE(external_libtalloc,
+	[AS_HELP_STRING([--enable-external-libtalloc],
+		[Enable external talloc [default=auto]])],
+	[ enable_external_libtalloc=$enableval ],
+	[ enable_external_libtalloc=auto ])
 
 if test "x$enable_external_libtalloc" != xno
 then
@@ -2090,8 +2112,11 @@ fi
 if test "x$enable_external_libtalloc" = xno
 then
 	m4_include(../lib/talloc/libtalloc.m4)
-	LINK_LIBTALLOC=STATIC
-	SMB_LIBRARY(talloc, 2)
+	if test x"$USESHARED" == x"no" ; then
+		LINK_LIBTALLOC=STATIC
+	fi
+	LIBTALLOCVERSION=`grep ^VERSION ${tallocdir}/wscript | sed -e "s/'//g" -e 's/.* //'`
+	SMB_LIBRARY(talloc, 2, ${LIBTALLOCVERSION})
 	LIBTALLOC_OBJ0=""
 	for obj in ${TALLOC_OBJ}; do
 		LIBTALLOC_OBJ0="${LIBTALLOC_OBJ0} ${tallocdir}/${obj}"
@@ -2108,8 +2133,8 @@ fi
 AC_ARG_ENABLE(external_libtdb,
 	[AS_HELP_STRING([--enable-external-libtdb],
 		[Enable external tdb [default=auto]])],
-		[ enable_external_libtalloc=$enableval ],
-		[ enable_external_libtalloc=auto ])
+		[ enable_external_libtdb=$enableval ],
+		[ enable_external_libtdb=auto ])
 
 if test "x$enable_external_libtdb" != xno
 then
@@ -2128,8 +2153,11 @@ AC_SUBST(LIBTDB_OBJ0)
 if test "x$enable_external_libtdb" = xno
 then
 	m4_include(../lib/tdb/libtdb.m4)
-	LINK_LIBTDB=STATIC
-	SMB_LIBRARY(tdb, 1)
+	if test x"$USESHARED" == x"no" ; then
+		LINK_LIBTDB=STATIC
+	fi
+	LIBTDBVERSION=`grep ^VERSION ${tdbdir}/wscript | sed -e "s/'//g" -e 's/.* //'`
+	SMB_LIBRARY(tdb, 1, ${LIBTDBVERSION})
 	LIBTDB_OBJ0=""
 	LIBTDB_LIBS="$LIBTDB_LIBS $TDB_DEPS"
 	for obj in ${TDB_OBJ}; do
@@ -2164,7 +2192,7 @@ fi
 SMB_LIBRARY(netapi, 0)
 SMB_LIBRARY(smbclient, 0)
 SMB_LIBRARY(smbsharemodes, 0)
-SMB_LIBRARY(addns, 0, no, [undefined API])
+SMB_LIBRARY(addns, 0, [], no, [undefined API])
 
 
 
@@ -2577,6 +2605,17 @@ if test x"$samba_cv_HAVE_LINUX_FALLOCATE64" = x"yes" && test x"$ac_cv_func_fallo
     AC_DEFINE(HAVE_LINUX_FALLOCATE64,1,[Whether the Linux 'fallocate64' function is available])
 fi
 
+AC_CACHE_CHECK([for getcwd takes NULL],samba_cv_GETCWD_TAKES_NULL,[
+AC_TRY_RUN([
+#if defined(HAVE_UNISTD_H)
+#include <unistd.h>
+#endif
+main() { char *s = getcwd(NULL,0); if (s) exit(0); exit(1); }],
+samba_cv_GETCWD_TAKES_NULL=yes,samba_cv_GETCWD_TAKES_NULL=no,samba_cv_GETCWD_TAKES_NULL=cross)])
+if test x"$samba_cv_GETCWD_TAKES_NULL" = x"yes"; then
+    AC_DEFINE(GETCWD_TAKES_NULL,1,[Whether the getcwd function takes NULL as an argument])
+fi
+
 ICONV_LOOK_DIRS="/usr /usr/local /sw /opt"
 AC_ARG_WITH(libiconv,
 [AS_HELP_STRING([--with-libiconv=BASEDIR], [Use libiconv in BASEDIR/lib and BASEDIR/include (default=auto)])],
@@ -2654,7 +2693,6 @@ for i in $ICONV_LOOK_DIRS ; do
         export LDFLAGS LIBS CPPFLAGS
 
 	default_dos_charset=no
-	default_display_charset=no
 	default_unix_charset=no
 
 	# check for default dos charset name
@@ -2666,15 +2704,6 @@ for i in $ICONV_LOOK_DIRS ; do
 	    fi
 	done
 
-	# check for default display charset name
-	for j in ASCII 646 ; do
-	    rjs_CHARSET($j)
-	    default_display_charset="$ICONV_CHARSET"
-	    if test x"$default_display_charset" = x"$j"; then
-		break
-	    fi
-	done
-
 	# check for default unix charset name
 	for j in UTF-8 UTF8 ; do
 	    rjs_CHARSET($j)
@@ -2686,15 +2715,12 @@ for i in $ICONV_LOOK_DIRS ; do
 
 	if test "$default_dos_charset" != "no" -a \
 		"$default_dos_charset" != "cross" -a \
-		"$default_display_charset" != "no" -a \
-		"$default_display_charset" != "cross" -a \
 		"$default_unix_charset" != "no" -a \
 		"$default_unix_charset" != "cross"
 	then
 		samba_cv_HAVE_NATIVE_ICONV=yes
 	else
 	    if test "$default_dos_charset" = "cross" -o \
-		     "$default_display_charset" = "cross" -o \
 		     "$default_unix_charset" = "cross"
 	    then
 		    samba_cv_HAVE_NATIVE_ICONV=cross
@@ -2711,11 +2737,10 @@ for i in $ICONV_LOOK_DIRS ; do
 	# match the results we get at runtime anyway.
 	if test x"$samba_cv_HAVE_NATIVE_ICONV" = x"cross" ; then
 	    default_dos_charset="CP850"
-	    default_display_charset="ASCII"
 	    default_unix_charset="UTF-8"
 	    samba_cv_HAVE_NATIVE_ICONV=yes
 	    AC_MSG_WARN(assuming the libiconv in $iconv_current_LDFLAGS can convert)
-	    AC_MSG_WARN([$default_dos_charset, $default_display_charset and $default_unix_charset to UCS-16LE])
+	    AC_MSG_WARN([$default_dos_charset and $default_unix_charset to UCS-16LE])
 	fi
 
         if test x"$samba_cv_HAVE_NATIVE_ICONV" = x"yes" ; then
@@ -2734,12 +2759,10 @@ for i in $ICONV_LOOK_DIRS ; do
 
 	    # Turn the #defines into string literals
 	    default_dos_charset="\"$default_dos_charset\""
-	    default_display_charset="\"$default_display_charset\""
 	    default_unix_charset="\"$default_unix_charset\""
 
 	    AC_DEFINE(HAVE_NATIVE_ICONV,1,[Whether to use native iconv])
 	    AC_DEFINE_UNQUOTED(DEFAULT_DOS_CHARSET,$default_dos_charset,[Default dos charset name])
-	    AC_DEFINE_UNQUOTED(DEFAULT_DISPLAY_CHARSET,$default_display_charset,[Default display charset name])
 	    AC_DEFINE_UNQUOTED(DEFAULT_UNIX_CHARSET,$default_unix_charset,[Default unix charset name])
 
            break
@@ -2761,7 +2784,6 @@ if test x"$ICONV_FOUND" = x"no" -o x"$samba_cv_HAVE_NATIVE_ICONV" != x"yes" ; th
     AC_MSG_WARN([Sufficient support for iconv function was not found.
     Install libiconv from http://freshmeat.net/projects/libiconv/ for better charset compatibility!])
    AC_DEFINE_UNQUOTED(DEFAULT_DOS_CHARSET,"ASCII",[Default dos charset name])
-   AC_DEFINE_UNQUOTED(DEFAULT_DISPLAY_CHARSET,"ASCII",[Default display charset name])
    AC_DEFINE_UNQUOTED(DEFAULT_UNIX_CHARSET,"UTF8",[Default unix charset name])
 fi
 
@@ -3571,16 +3593,16 @@ else
     # Check to see whether there is enough LDAP functionality to be able
     # to build AD support.
 
-# HPUX only has ldap_init; ok, we take care of this in smbldap.c
+# HPUX and Solaris only has ldap_init; ok, we take care of this in smbldap.c
 case "$host_os" in
-	*hpux*)
+	*hpux* | *solaris*)
     AC_CHECK_FUNC_EXT(ldap_init,$LDAP_LIBS)
 
     if test x"$ac_cv_func_ext_ldap_init" != x"yes"; then
 	if test x"$with_ads_support" = x"yes"; then
-	    AC_MSG_ERROR(Active Directory support on HPUX requires ldap_init)
+	    AC_MSG_ERROR(Active Directory support on HPUX or Solaris requires ldap_init)
 	elif test x"$with_ads_support" = x"auto"; then
-	    AC_MSG_WARN(Disabling Active Directory support (requires ldap_init on HPUX))
+	    AC_MSG_WARN(Disabling Active Directory support (requires ldap_init on HPUX or Solaris))
 	    with_ads_support=no
 	fi
     fi
@@ -3630,7 +3652,6 @@ if test x"$with_ads_support" != x"no"; then
         ;;
       yes)
         AC_MSG_RESULT(/usr)
-        FOUND_KRB5=yes
         ;;
       *)
         AC_MSG_RESULT($withval)
@@ -3719,6 +3740,21 @@ if test x"$with_ads_support" != x"no"; then
     fi
   fi
 
+  if test x$FOUND_KRB5 = x"no"; then
+    #################################################
+    # see if this box has Solaris MIT kerberos implementation
+    AC_MSG_CHECKING(for Solaris MIT kerberos)
+    if test -x "$KRB5CONFIG" && $KRB5CONFIG --version | grep -s Solaris | grep -s MIT > /dev/null ; then
+       FOUND_KRB5=yes
+       KRB5_LIBS="-lgss -lkrb5"
+       KRB5_CFLAGS="`$KRB5CONFIG --cflags`"
+       KRB5_CPPFLAGS="`$KRB5CONFIG --cflags`"
+       AC_MSG_RESULT(yes)
+    else
+      AC_MSG_RESULT(no)
+    fi
+  fi
+
   ac_save_CFLAGS=$CFLAGS
   ac_save_CPPFLAGS=$CPPFLAGS
   ac_save_LDFLAGS=$LDFLAGS
@@ -3783,7 +3819,7 @@ if test x"$with_ads_support" != x"no"; then
 
   # now check for gssapi headers.  This is also done here to allow for
   # different kerberos include paths
-  AC_CHECK_HEADERS(gssapi.h gssapi/gssapi_generic.h gssapi/gssapi.h gssapi/gssapi_ext.h com_err.h)
+  AC_CHECK_HEADERS(gssapi.h gssapi/gssapi_generic.h gssapi/gssapi.h gssapi/gssapi_ext.h gssapi/gssapi_krb5.h com_err.h)
 
   ##################################################################
   # we might need the k5crypto and com_err libraries on some systems
@@ -3808,6 +3844,7 @@ if test x"$with_ads_support" != x"no"; then
   # now see if we can find the gssapi libs in standard paths
   if test x"$have_gssapi" != x"yes"; then
      AC_CHECK_LIB_EXT(gssapi_krb5, KRB5_LIBS,gss_display_status,[],[],have_gssapi=yes)
+     AC_CHECK_LIB_EXT(gss, KRB5_LIBS,gss_display_status,[],[],have_gssapi=yes)
      AC_CHECK_FUNC_EXT(gss_wrap_iov, $KRB5_LIBS)
   fi
 
@@ -3861,6 +3898,15 @@ if test x"$with_ads_support" != x"no"; then
   AC_CHECK_FUNC_EXT(krb5_get_host_realm, $KRB5_LIBS)
   AC_CHECK_FUNC_EXT(krb5_free_host_realm, $KRB5_LIBS)
   AC_CHECK_FUNC_EXT(gss_krb5_import_cred, $KRB5_LIBS)
+  AC_CHECK_FUNC_EXT(gss_get_name_attribute, $KRB5_LIBS)
+  AC_CHECK_FUNC_EXT(gss_mech_krb5, $KRB5_LIBS)
+  AC_CHECK_FUNC_EXT(gss_oid_equal, $KRB5_LIBS)
+  AC_CHECK_FUNC_EXT(gss_inquire_sec_context_by_oid, $KRB5_LIBS)
+
+  # This is for FreeBSD (and possibly others). gss_mech_krb5 is a
+  # #define to GSS_KRB5_MECHANISM, which is defined in -lgssapi_krb5
+  AC_CHECK_LIB_EXT(gssapi_krb5, KRB5_LIBS, GSS_KRB5_MECHANISM,
+  		   [KRB5_LIBS="$KRB5_LIBS -lgssapi_krb5"])
 
   # MIT krb5 1.8 does not expose this call (yet)
   AC_CHECK_DECLS(krb5_get_credentials_for_user, [], [], [#include <krb5.h>])
@@ -6364,6 +6410,7 @@ else
 	LIBWBCLIENT_SHARED_TARGET=bin/libwbclient.$SHLIBEXT
 	LIBWBCLIENT_STATIC_TARGET=bin/libwbclient.a
 	LIBWBCLIENT_SOVER=0
+	LIBWBCLIENT_FULLVER=0
 	if test $BLDSHARED = true -a x"$HAVE_WINBIND" = x"yes" -a x"$BUILD_LIBWBCLIENT_SHARED" = x"yes"; then
 		NSS_MODULES="${WINBIND_NSS} ${WINBIND_WINS_NSS}"
 		## Only worry about libwbclient if we have shared
@@ -6647,7 +6694,9 @@ if test x"$enable_pthreadpool" = x"yes" -a x"$samba_cv_HAVE_PTHREAD" = x"yes"; t
     LIBS="$LIBS $PTHREAD_LDFLAGS"
     CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
     AC_DEFINE(WITH_PTHREADPOOL, 1, [Whether to include pthreadpool helpers])
-    AC_SUBST(PTHREADPOOL_OBJ, "lib/pthreadpool.o")
+    AC_SUBST(PTHREADPOOL_OBJ, "lib/pthreadpool/pthreadpool.o")
+    PTHREADPOOLTEST="bin/pthreadpooltest\$(EXEEXT)"
+    AC_SUBST(PTHREADPOOLTEST)
 fi
 
 #################################################
@@ -6870,6 +6919,7 @@ SMB_MODULE(vfs_cacheprime, \$(VFS_CACHEPRIME_OBJ), "bin/cacheprime.$SHLIBEXT", V
 SMB_MODULE(vfs_prealloc, \$(VFS_PREALLOC_OBJ), "bin/prealloc.$SHLIBEXT", VFS)
 SMB_MODULE(vfs_commit, \$(VFS_COMMIT_OBJ), "bin/commit.$SHLIBEXT", VFS)
 SMB_MODULE(vfs_gpfs, \$(VFS_GPFS_OBJ), "bin/gpfs.$SHLIBEXT", VFS)
+SMB_MODULE(vfs_gpfs_hsm_notify, \$(VFS_GPFS_PREFETCH_OBJ), "bin/gpfs_hsm_notify.$SHLIBEXT", VFS)
 SMB_MODULE(vfs_readahead, \$(VFS_READAHEAD_OBJ), "bin/readahead.$SHLIBEXT", VFS)
 SMB_MODULE(vfs_tsmsm, \$(VFS_TSMSM_OBJ), "bin/tsmsm.$SHLIBEXT", VFS)
 SMB_MODULE(vfs_fileid, \$(VFS_FILEID_OBJ), "bin/fileid.$SHLIBEXT", VFS)
diff --git a/source3/dynconfig.c b/source3/dynconfig.c
deleted file mode 100644
index 32f64a6e22c..00000000000
--- a/source3/dynconfig.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
-   Unix SMB/CIFS implementation.
-   Copyright (C) 2001 by Martin Pool <mbp@samba.org>
-   Copyright (C) 2003 by Jim McDonough <jmcd@us.ibm.com>
-   Copyright (C) 2007 by Jeremy Allison <jra@samba.org>
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-
-/**
- * @file dynconfig.c
- *
- * @brief Global configurations, initialized to configured defaults.
- *
- * This file should be the only file that depends on path
- * configuration (--prefix, etc), so that if ./configure is re-run,
- * all programs will be appropriately updated.  Everything else in
- * Samba should import extern variables from here, rather than relying
- * on preprocessor macros.
- *
- * Eventually some of these may become even more variable, so that
- * they can for example consistently be set across the whole of Samba
- * by command-line parameters, config file entries, or environment
- * variables.
- *
- * @todo Perhaps eventually these should be merged into the parameter
- * table?  There's kind of a chicken-and-egg situation there...
- **/
-
-#define DEFINE_DYN_CONFIG_PARAM(name) \
-static char *dyn_##name; \
-\
- const char *get_dyn_##name(void) \
-{\
-	if (dyn_##name == NULL) {\
-		return name;\
-	}\
-	return dyn_##name;\
-}\
-\
- const char *set_dyn_##name(const char *newpath) \
-{\
-	if (dyn_##name) {\
-		SAFE_FREE(dyn_##name);\
-	}\
-	dyn_##name = SMB_STRDUP(newpath);\
-	return dyn_##name;\
-}\
-\
- bool is_default_dyn_##name(void) \
-{\
-	return (dyn_##name == NULL);\
-}
-
-DEFINE_DYN_CONFIG_PARAM(SBINDIR)
-DEFINE_DYN_CONFIG_PARAM(BINDIR)
-DEFINE_DYN_CONFIG_PARAM(SWATDIR)
-DEFINE_DYN_CONFIG_PARAM(CONFIGFILE) /**< Location of smb.conf file. **/
-DEFINE_DYN_CONFIG_PARAM(LOGFILEBASE) /** Log file directory. **/
-DEFINE_DYN_CONFIG_PARAM(LMHOSTSFILE) /** Statically configured LanMan hosts. **/
-DEFINE_DYN_CONFIG_PARAM(CODEPAGEDIR)
-DEFINE_DYN_CONFIG_PARAM(LIBDIR)
-DEFINE_DYN_CONFIG_PARAM(MODULESDIR)
-DEFINE_DYN_CONFIG_PARAM(SHLIBEXT)
-DEFINE_DYN_CONFIG_PARAM(LOCKDIR)
-DEFINE_DYN_CONFIG_PARAM(STATEDIR) /** Persistent state files. Default LOCKDIR */
-DEFINE_DYN_CONFIG_PARAM(CACHEDIR) /** Temporary cache files. Default LOCKDIR */
-DEFINE_DYN_CONFIG_PARAM(PIDDIR)
-DEFINE_DYN_CONFIG_PARAM(NMBDSOCKETDIR)
-DEFINE_DYN_CONFIG_PARAM(NCALRPCDIR)
-DEFINE_DYN_CONFIG_PARAM(SMB_PASSWD_FILE)
-DEFINE_DYN_CONFIG_PARAM(PRIVATE_DIR)
-DEFINE_DYN_CONFIG_PARAM(LOCALEDIR)
diff --git a/source3/groupdb/mapping.c b/source3/groupdb/mapping.c
index 3646e04d0a3..70c96f186ce 100644
--- a/source3/groupdb/mapping.c
+++ b/source3/groupdb/mapping.c
@@ -26,6 +26,7 @@
 #include "groupdb/mapping.h"
 #include "../libcli/security/security.h"
 #include "lib/winbind_util.h"
+#include "tdb_compat.h"
 
 static const struct mapping_backend *backend;
 
@@ -634,7 +635,7 @@ NTSTATUS pdb_default_alias_memberships(struct pdb_methods *methods,
 		return NT_STATUS_OK;
 	}
 
-	*pp_alias_rids = TALLOC_ARRAY(mem_ctx, uint32, num_alias_sids);
+	*pp_alias_rids = talloc_array(mem_ctx, uint32, num_alias_sids);
 	if (*pp_alias_rids == NULL)
 		return NT_STATUS_NO_MEMORY;
 
@@ -777,8 +778,8 @@ NTSTATUS pdb_create_builtin_alias(uint32 rid)
 	map.gid = gid;
 	sid_copy(&map.sid, &sid);
 	map.sid_name_use = SID_NAME_ALIAS;
-	fstrcpy(map.nt_name, groupname);
-	fstrcpy(map.comment, "");
+	strlcpy(map.nt_name, groupname, sizeof(map.nt_name));
+	strlcpy(map.comment, "", sizeof(map.comment));
 
 	status = pdb_add_group_mapping_entry(&map);
 
diff --git a/source3/groupdb/mapping_tdb.c b/source3/groupdb/mapping_tdb.c
index 1137b8f6587..fc195cb3487 100644
--- a/source3/groupdb/mapping_tdb.c
+++ b/source3/groupdb/mapping_tdb.c
@@ -25,6 +25,7 @@
 #include "passdb.h"
 #include "groupdb/mapping.h"
 #include "dbwrap.h"
+#include "util_tdb.h"
 #include "../libcli/security/security.h"
 
 static struct db_context *db; /* used for driver files */
@@ -144,7 +145,7 @@ static bool add_mapping_entry(GROUP_MAP *map, int flag)
 	len = tdb_pack(NULL, 0, "ddff",
 		map->gid, map->sid_name_use, map->nt_name, map->comment);
 
-	buf = TALLOC_ARRAY(key, char, len);
+	buf = talloc_array(key, char, len);
 	if (!buf) {
 		TALLOC_FREE(key);
 		return false;
@@ -848,7 +849,7 @@ static int convert_ldb_record(TDB_CONTEXT *ltdb, TDB_DATA key,
 		p += len + 1;
 
 		num_vals = pull_uint32(p, 0);
-		if (StrCaseCmp(name, "member") == 0) {
+		if (strcasecmp_m(name, "member") == 0) {
 			num_mem = num_vals;
 			members = talloc_array(tmp_ctx, struct dom_sid, num_mem);
 			if (members == NULL) {
@@ -881,30 +882,30 @@ static int convert_ldb_record(TDB_CONTEXT *ltdb, TDB_DATA key,
 
 			/* we ignore unknown or uninteresting attributes
 			 * (objectclass, etc.) */
-			if (StrCaseCmp(name, "gidNumber") == 0) {
+			if (strcasecmp_m(name, "gidNumber") == 0) {
 				map.gid = strtoul(val, &q, 10);
 				if (*q) {
 					errno = EIO;
 					goto failed;
 				}
-			} else if (StrCaseCmp(name, "sid") == 0) {
+			} else if (strcasecmp_m(name, "sid") == 0) {
 				if (!string_to_sid(&map.sid, val)) {
 					errno = EIO;
 					goto failed;
 				}
-			} else if (StrCaseCmp(name, "sidNameUse") == 0) {
+			} else if (strcasecmp_m(name, "sidNameUse") == 0) {
 				map.sid_name_use = strtoul(val, &q, 10);
 				if (*q) {
 					errno = EIO;
 					goto failed;
 				}
-			} else if (StrCaseCmp(name, "ntname") == 0) {
+			} else if (strcasecmp_m(name, "ntname") == 0) {
 				strlcpy(map.nt_name, val,
-					sizeof(map.nt_name) -1);
-			} else if (StrCaseCmp(name, "comment") == 0) {
+					sizeof(map.nt_name));
+			} else if (strcasecmp_m(name, "comment") == 0) {
 				strlcpy(map.comment, val,
-					sizeof(map.comment) -1);
-			} else if (StrCaseCmp(name, "member") == 0) {
+					sizeof(map.comment));
+			} else if (strcasecmp_m(name, "member") == 0) {
 				if (!string_to_sid(&members[j], val)) {
 					errno = EIO;
 					goto failed;
@@ -959,7 +960,7 @@ static bool mapping_switch(const char *ldb_path)
 	/* ldb is just a very fancy tdb, read out raw data and perform
 	 * conversion */
 	ret = tdb_traverse(ltdb, convert_ldb_record, NULL);
-	if (ret == -1) goto failed;
+	if (ret < 0) goto failed;
 
 	if (ltdb) {
 		tdb_close(ltdb);
diff --git a/source3/groupdb/proto.h b/source3/groupdb/proto.h
index 11162e6d3c7..db07d4d64f5 100644
--- a/source3/groupdb/proto.h
+++ b/source3/groupdb/proto.h
@@ -1,3 +1,28 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Group Mapping Database
+ *
+ *  Copyright (C) Andrew Tridgell              1992-2006
+ *  Copyright (C) Jean François Micouleau      1998-2001
+ *  Copyright (C) Gerald Carter                2006
+ *  Copyright (C) Volker Lendecke              2006
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _GROUPDB_PROTO_H_
+#define _GROUPDB_PROTO_H_
 
 /* The following definitions come from groupdb/mapping.c  */
 
@@ -73,3 +98,5 @@ NTSTATUS pdb_create_builtin_alias(uint32 rid);
 /* The following definitions come from groupdb/mapping_tdb.c  */
 
 const struct mapping_backend *groupdb_tdb_init(void);
+
+#endif /* _GROUPDB_PROTO_H_ */
diff --git a/source3/include/ads.h b/source3/include/ads.h
index ee6e5b8649b..8a8ce28e530 100644
--- a/source3/include/ads.h
+++ b/source3/include/ads.h
@@ -6,6 +6,7 @@
   basically this is a wrapper around ldap
 */
 
+#include "libads/ads_status.h"
 #include "smb_ldap.h"
 
 struct ads_struct;
diff --git a/source3/include/async_smb.h b/source3/include/async_smb.h
index d2303cc7036..1685d4985d4 100644
--- a/source3/include/async_smb.h
+++ b/source3/include/async_smb.h
@@ -20,7 +20,7 @@
 #ifndef __ASYNC_SMB_H__
 #define __ASYNC_SMB_H__
 
-#include "includes.h"
+struct cli_state;
 
 /*
  * Fetch an error out of a NBT packet
@@ -51,6 +51,8 @@ void cli_smb_req_unset_pending(struct tevent_req *req);
 bool cli_smb_req_set_pending(struct tevent_req *req);
 uint16_t cli_smb_req_mid(struct tevent_req *req);
 void cli_smb_req_set_mid(struct tevent_req *req, uint16_t mid);
+uint32_t cli_smb_req_seqnum(struct tevent_req *req);
+void cli_smb_req_set_seqnum(struct tevent_req *req, uint32_t seqnum);
 struct tevent_req *cli_smb_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
 				struct cli_state *cli,
 				uint8_t smb_command, uint8_t additional_flags,
diff --git a/source3/include/autoconf/README b/source3/include/autoconf/README
new file mode 100644
index 00000000000..21afbc70eab
--- /dev/null
+++ b/source3/include/autoconf/README
@@ -0,0 +1,5 @@
+This directory added so that the autoconf build can generate a
+config.h in a location that the recursive waf build for smbtorture
+won't find.
+
+Andrew Bartlett
\ No newline at end of file
diff --git a/source3/include/client.h b/source3/include/client.h
index 9e4a61dbacb..a853e90af64 100644
--- a/source3/include/client.h
+++ b/source3/include/client.h
@@ -46,13 +46,6 @@ struct print_job_info {
 	time_t t;
 };
 
-struct cli_state_seqnum {
-	struct cli_state_seqnum *prev, *next;
-	uint16_t mid;
-	uint32_t seqnum;
-	bool persistent;
-};
-
 struct cli_state {
 	/**
 	 * A list of subsidiary connections for DFS.
@@ -71,7 +64,7 @@ struct cli_state {
 	int rap_error;
 	int privileges;
 
-	fstring desthost;
+	char *desthost;
 
 	/* The credentials used to open the cli_state connection. */
 	char *domain;
@@ -83,12 +76,12 @@ struct cli_state {
 	 * ones returned by the server if
 	 * the protocol > NT1.
 	 */
-	fstring server_type;
-	fstring server_os;
-	fstring server_domain;
+	char *server_type;
+	char *server_os;
+	char *server_domain;
 
-	fstring share;
-	fstring dev;
+	char *share;
+	char *dev;
 	struct nmb_name called;
 	struct nmb_name calling;
 	struct sockaddr_storage dest_ss;
@@ -103,12 +96,12 @@ struct cli_state {
 	size_t max_xmit;
 	size_t max_mux;
 	char *outbuf;
-	struct cli_state_seqnum *seqnum;
 	char *inbuf;
 	unsigned int bufsize;
 	int initialised;
 	int win95;
 	bool is_samba;
+	bool is_guestlogin;
 	uint32 capabilities;
 	/* What the server offered. */
 	uint32_t server_posix_capabilities;
diff --git a/source3/include/packet.h b/source3/include/ctdb_packet.h
index c96a8017f89..026b23f90e4 100644
--- a/source3/include/packet.h
+++ b/source3/include/ctdb_packet.h
@@ -1,6 +1,6 @@
-/* 
+/*
    Unix SMB/CIFS implementation.
-   Packet handling
+   CTDB Packet handling
    Copyright (C) Volker Lendecke 2007
 
    This program is free software; you can redistribute it and/or modify
@@ -18,35 +18,35 @@
 */
 
 /*
- * A packet context is a wrapper around a bidirectional file descriptor,
+ * A ctdb_packet context is a wrapper around a bidirectional file descriptor,
  * hiding the handling of individual requests.
  */
 
-struct packet_context;
+struct ctdb_packet_context;
 
 /*
- * Initialize a packet context. The fd is given to the packet context, meaning
- * that it is automatically closed when the packet context is freed.
+ * Initialize a ctdb_packet context. The fd is given to the ctdb_packet context, meaning
+ * that it is automatically closed when the ctdb_packet context is freed.
  */
-struct packet_context *packet_init(TALLOC_CTX *mem_ctx, int fd);
+struct ctdb_packet_context *ctdb_packet_init(TALLOC_CTX *mem_ctx, int fd);
 
 /*
  * Pull data from the fd
  */
-NTSTATUS packet_fd_read(struct packet_context *ctx);
+NTSTATUS ctdb_packet_fd_read(struct ctdb_packet_context *ctx);
 
 /*
  * Sync read, wait for the next chunk
  */
-NTSTATUS packet_fd_read_sync(struct packet_context *ctx, int timeout);
+NTSTATUS ctdb_packet_fd_read_sync_timeout(struct ctdb_packet_context *ctx, int timeout);
 
 /*
- * Handle an incoming packet:
+ * Handle an incoming ctdb_packet:
  * Return False if none is available
  * Otherwise return True and store the callback result in *status
  * Callback must either talloc_move or talloc_free buf
  */
-bool packet_handler(struct packet_context *ctx,
+bool ctdb_packet_handler(struct ctdb_packet_context *ctx,
 		    bool (*full_req)(const uint8_t *buf,
 				     size_t available,
 				     size_t *length,
@@ -59,27 +59,27 @@ bool packet_handler(struct packet_context *ctx,
 /*
  * How many bytes of outgoing data do we have pending?
  */
-size_t packet_outgoing_bytes(struct packet_context *ctx);
+size_t ctdb_packet_outgoing_bytes(struct ctdb_packet_context *ctx);
 
 /*
  * Push data to the fd
  */
-NTSTATUS packet_fd_write(struct packet_context *ctx);
+NTSTATUS ctdb_packet_fd_write(struct ctdb_packet_context *ctx);
 
 /*
  * Sync flush all outgoing bytes
  */
-NTSTATUS packet_flush(struct packet_context *ctx);
+NTSTATUS ctdb_packet_flush(struct ctdb_packet_context *ctx);
 
 /*
  * Send a list of DATA_BLOBs
  *
- * Example:  packet_send(ctx, 2, data_blob_const(&size, sizeof(size)),
+ * Example:  ctdb_packet_send(ctx, 2, data_blob_const(&size, sizeof(size)),
  *			 data_blob_const(buf, size));
  */
-NTSTATUS packet_send(struct packet_context *ctx, int num_blobs, ...);
+NTSTATUS ctdb_packet_send(struct ctdb_packet_context *ctx, int num_blobs, ...);
 
 /*
- * Get the packet context's file descriptor
+ * Get the ctdb_packet context's file descriptor
  */
-int packet_get_fd(struct packet_context *ctx);
+int ctdb_packet_get_fd(struct ctdb_packet_context *ctx);
diff --git a/source3/include/ctdbd_conn.h b/source3/include/ctdbd_conn.h
index ef291fb4ad6..3d712674e01 100644
--- a/source3/include/ctdbd_conn.h
+++ b/source3/include/ctdbd_conn.h
@@ -20,6 +20,8 @@
 #ifndef _CTDBD_CONN_H
 #define _CTDBD_CONN_H
 
+#include "tdb_compat.h"
+
 struct ctdbd_connection;
 struct messaging_context;
 struct messaging_rec;
diff --git a/source3/include/dbwrap.h b/source3/include/dbwrap.h
index ef35b0071d0..303a5549381 100644
--- a/source3/include/dbwrap.h
+++ b/source3/include/dbwrap.h
@@ -20,6 +20,8 @@
 #ifndef __DBWRAP_H__
 #define __DBWRAP_H__
 
+#include "tdb_compat.h"
+
 struct db_record {
 	TDB_DATA key, value;
 	NTSTATUS (*store)(struct db_record *rec, TDB_DATA data, int flag);
diff --git a/source3/include/dynconfig.h b/source3/include/dynconfig.h
deleted file mode 100644
index d0e42dc9f51..00000000000
--- a/source3/include/dynconfig.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/* 
-   Unix SMB/CIFS implementation.
-   Copyright (C) 2001 by Martin Pool <mbp@samba.org>
-   Copyright (C) 2003 by Jim McDonough <jmcd@us.ibm.com>
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * @file dynconfig.h
- *
- * @brief Exported global configurations.
- **/
-
-const char *get_dyn_SBINDIR(void);
-const char *set_dyn_SBINDIR(const char *newpath);
-bool is_default_dyn_SBINDIR(void);
-
-const char *get_dyn_BINDIR(void);
-const char *set_dyn_BINDIR(const char *newpath);
-bool is_default_dyn_BINDIR(void);
-
-const char *get_dyn_SWATDIR(void);
-const char *set_dyn_SWATDIR(const char *newpath);
-bool is_default_dyn_SWATDIR(void);
-
-const char *get_dyn_CONFIGFILE(void);
-const char *set_dyn_CONFIGFILE(const char *newpath);
-bool is_default_dyn_CONFIGFILE(void);
-
-const char *get_dyn_LOGFILEBASE(void);
-const char *set_dyn_LOGFILEBASE(const char *newpath);
-bool is_default_dyn_LOGFILEBASE(void);
-
-const char *get_dyn_LMHOSTSFILE(void);
-const char *set_dyn_LMHOSTSFILE(const char *newpath);
-bool is_default_dyn_LMHOSTSFILE(void);
-
-const char *get_dyn_CODEPAGEDIR(void);
-const char *set_dyn_CODEPAGEDIR(const char *newpath);
-bool is_default_dyn_CODEPAGEDIR(void);
-
-const char *get_dyn_LIBDIR(void);
-const char *set_dyn_LIBDIR(const char *newpath);
-bool is_default_dyn_LIBDIR(void);
-
-const char *get_dyn_MODULESDIR(void);
-const char *set_dyn_MODULESDIR(const char *newpath);
-bool is_default_dyn_MODULESDIR(void);
-
-const char *get_dyn_SHLIBEXT(void);
-const char *set_dyn_SHLIBEXT(const char *newpath);
-bool is_default_dyn_SHLIBEXT(void);
-
-const char *get_dyn_LOCKDIR(void);
-const char *set_dyn_LOCKDIR(const char *newpath);
-bool is_default_dyn_LOCKDIR(void);
-
-const char *get_dyn_STATEDIR(void);
-const char *set_dyn_STATEDIR(const char *newpath);
-bool is_default_dyn_STATEDIR(void);
-
-const char *get_dyn_CACHEDIR(void);
-const char *set_dyn_CACHEDIR(const char *newpath);
-bool is_default_dyn_CACHEDIR(void);
-
-const char *get_dyn_PIDDIR(void);
-const char *set_dyn_PIDDIR(const char *newpath);
-bool is_default_dyn_PIDDIR(void);
-
-const char *get_dyn_NMBDSOCKETDIR(void);
-const char *set_dyn_NMBDSOCKETDIR(const char *newpath);
-bool is_default_dyn_NMBDSOCKETDIR(void);
-
-const char *get_dyn_NCALRPCDIR(void);
-const char *set_dyn_NCALRPCDIR(const char *newpath);
-bool is_default_dyn_NCALRPCDIR(void);
-
-const char *get_dyn_SMB_PASSWD_FILE(void);
-const char *set_dyn_SMB_PASSWD_FILE(const char *newpath);
-bool is_default_dyn_SMB_PASSWD_FILE(void);
-
-const char *get_dyn_PRIVATE_DIR(void);
-const char *set_dyn_PRIVATE_DIR(const char *newpath);
-bool is_default_dyn_PRIVATE_DIR(void);
-
-const char *get_dyn_LOCALEDIR(void);
-const char *set_dyn_LOCALEDIR(const char *newpath);
-bool is_default_dyn_LOCALEDIR(void);
diff --git a/source3/include/includes.h b/source3/include/includes.h
index a76942faad5..08d5d96ab55 100644
--- a/source3/include/includes.h
+++ b/source3/include/includes.h
@@ -173,10 +173,6 @@
 #include <aio.h>
 #endif
 
-#ifdef WITH_MADVISE_PROTECTED
-#include <sys/mman.h>
-#endif
-
 /* Special macros that are no-ops except when run under Valgrind on
  * x86.  They've moved a little bit from valgrind 1.0.4 to 1.9.4 */
 #if HAVE_VALGRIND_MEMCHECK_H
@@ -308,10 +304,10 @@ typedef sig_atomic_t volatile SIG_ATOMIC_T;
 #endif
 
 #ifdef LARGE_SMB_INO_T
-#define SINO_T_VAL(p, ofs, v) (SIVAL((p),(ofs),(v)&0xFFFFFFFF), SIVAL((p),(ofs)+4,(v)>>32))
-#define INO_T_VAL(p, ofs) ((SMB_INO_T)(((uint64_t)(IVAL(p,ofs)))| (((uint64_t)(IVAL(p,(ofs)+4))) << 32)))
+#define SINO_T_VAL(p, ofs, v) SBVAL(p, ofs, v)
+#define INO_T_VAL(p, ofs) ((SMB_INO_T)BVAL(p, ofs))
 #else 
-#define SINO_T_VAL(p, ofs, v) (SIVAL(p,ofs,v),SIVAL(p,(ofs)+4,0))
+#define SINO_T_VAL(p, ofs, v) SBVAL(p, ofs, ((uint64_t)(v)) & UINT32_MAX)
 #define INO_T_VAL(p, ofs) ((SMB_INO_T)(IVAL((p),(ofs))))
 #endif
 
@@ -323,11 +319,10 @@ typedef sig_atomic_t volatile SIG_ATOMIC_T;
 #  endif
 #endif
 
-#define SBIG_UINT(p, ofs, v) (SIVAL(p,ofs,(v)&0xFFFFFFFF), SIVAL(p,(ofs)+4,(v)>>32))
-#define BIG_UINT(p, ofs) ((((uint64_t) IVAL(p,(ofs)+4))<<32)|IVAL(p,ofs))
-#define IVAL2_TO_SMB_BIG_UINT(buf,off) ( (((uint64_t)(IVAL((buf),(off)))) & ((uint64_t)0xFFFFFFFF)) | \
-		(( ((uint64_t)(IVAL((buf),(off+4)))) & ((uint64_t)0xFFFFFFFF) ) << 32 ) )
-
+/* TODO: remove this macros */
+#define SBIG_UINT(p, ofs, v) SBVAL(p, ofs, v)
+#define BIG_UINT(p, ofs) BVAL(p, ofs)
+#define IVAL2_TO_SMB_BIG_UINT(p, ofs) BVAL(p, ofs)
 
 /* this should really be a 64 bit type if possible */
 typedef uint64_t br_off;
@@ -515,53 +510,30 @@ typedef char fstring[FSTRING_LEN];
 #include "../lib/util/attr.h"
 #include "../lib/util/tsort.h"
 #include "../lib/util/dlinklist.h"
-#include <tdb.h>
-#include "util_tdb.h"
 
 #include <talloc.h>
 
 #include "event.h"
-#include "../lib/util/tevent_unix.h"
-#include "../lib/util/tevent_ntstatus.h"
-#include "../lib/tsocket/tsocket.h"
 
 #include "../lib/util/data_blob.h"
 #include "../lib/util/time.h"
 #include "../lib/util/debug.h"
 #include "../lib/util/debug_s3.h"
 
-#include "libads/ads_status.h"
+#include "../libcli/util/ntstatus.h"
 #include "../libcli/util/error.h"
 #include "../lib/util/charset/charset.h"
-#include "dynconfig.h"
+#include "dynconfig/dynconfig.h"
 #include "locking.h"
 #include "smb_perfcount.h"
 #include "smb.h"
 #include "../lib/util/byteorder.h"
 
-#include "client.h"
-
 #include "module.h"
 #include "../lib/util/talloc_stack.h"
 #include "../lib/util/smb_threads.h"
 #include "../lib/util/smb_threads_internal.h"
 
-/*
- * Reasons for cache flush.
- */
-
-enum flush_reason_enum {
-    SEEK_FLUSH,
-    READ_FLUSH,
-    WRITE_FLUSH,
-    READRAW_FLUSH,
-    OPLOCK_RELEASE_FLUSH,
-    CLOSE_FLUSH,
-    SYNC_FLUSH,
-    SIZECHANGE_FLUSH,
-    /* NUM_FLUSH_REASONS must remain the last value in the enumeration. */
-    NUM_FLUSH_REASONS};
-
 /***** prototypes *****/
 #ifndef NO_PROTO_H
 #include "proto.h"
@@ -606,15 +578,6 @@ enum flush_reason_enum {
 #endif
 
 
-#if HAVE_KERNEL_SHARE_MODES
-#ifndef LOCK_MAND 
-#define LOCK_MAND	32	/* This is a mandatory flock */
-#define LOCK_READ	64	/* ... Which allows concurrent read operations */
-#define LOCK_WRITE	128	/* ... Which allows concurrent write operations */
-#define LOCK_RW		192	/* ... Which allows concurrent read & write ops */
-#endif
-#endif
-
 #define MAX_SEC_CTX_DEPTH 8    /* Maximum number of security contexts */
 
 
@@ -645,8 +608,6 @@ void sys_adminlog(int priority, const char *format_str, ...) PRINTF_ATTRIBUTE(2,
 /* PRINTFLIKE2 */
 int fstr_sprintf(fstring s, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3);
 
-int d_vfprintf(FILE *f, const char *format, va_list ap) PRINTF_ATTRIBUTE(2,0);
-
 int smb_xvasprintf(char **ptr, const char *format, va_list ap) PRINTF_ATTRIBUTE(2,0);
 
 int asprintf_strupper_m(char **strp, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3);
@@ -685,20 +646,11 @@ char *talloc_asprintf_strupper_m(TALLOC_CTX *t, const char *fmt, ...) PRINTF_ATT
 #undef HAVE_MMAP
 #endif
 
-#ifndef CONST_DISCARD
-#define CONST_DISCARD(type, ptr)      ((type) ((void *) (ptr)))
-#endif
-
 void dump_core(void) _NORETURN_;
 void exit_server(const char *const reason) _NORETURN_;
 void exit_server_cleanly(const char *const reason) _NORETURN_;
 void exit_server_fault(void) _NORETURN_;
 
-#if defined(HAVE_IPV6)
-void in6_addr_to_sockaddr_storage(struct sockaddr_storage *ss,
-				  struct in6_addr ip);
-#endif
-
 /* samba3 doesn't use uwrap yet */
 #define uwrap_enabled() 0
 
diff --git a/source3/include/krb5_env.h b/source3/include/krb5_env.h
index aa967953928..0022da839c1 100644
--- a/source3/include/krb5_env.h
+++ b/source3/include/krb5_env.h
@@ -1,2 +1,26 @@
+/*
+   Samba Unix/Linux SMB client library
+
+   Copyright (C) 2010 Günther Deschner
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _INCLUDE_KRB5_ENV_H_
+#define _INCLUDE_KRB5_ENV_H_
+
 /* Kerberos environment variable names */
 #define KRB5_ENV_CCNAME "KRB5CCNAME"
+
+#endif /* _INCLUDE_KRB5_ENV_H_ */
diff --git a/source3/include/krb5_protos.h b/source3/include/krb5_protos.h
index d80e77d2eb6..37fc1c6cd51 100644
--- a/source3/include/krb5_protos.h
+++ b/source3/include/krb5_protos.h
@@ -1,3 +1,28 @@
+/*
+   Unix SMB/CIFS implementation.
+   simple kerberos5 routines for active directory
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Luke Howard 2002-2003
+   Copyright (C) Andrew Bartlett <abartlet@samba.org> 2005
+   Copyright (C) Guenther Deschner 2005-2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _INCLUDE_KRB5_PROTOS_H_
+#define _INCLUDE_KRB5_PROTOS_H_
+
 struct PAC_DATA;
 struct PAC_SIGNATURE_DATA;
 
@@ -39,6 +64,8 @@ void krb5_free_unparsed_name(krb5_context ctx, char *val);
 #define initialize_krb5_error_table()
 #endif
 
+/* The following definitions come from libsmb/clikrb5.c  */
+
 /* Samba wrapper function for krb5 functionality. */
 bool setup_kaddr( krb5_address *pkaddr, struct sockaddr_storage *paddr);
 int create_kerberos_key_from_string(krb5_context context, krb5_principal host_princ, krb5_data *password, krb5_keyblock *key, krb5_enctype enctype, bool no_salt);
@@ -131,3 +158,9 @@ int cli_krb5_get_ticket(TALLOC_CTX *mem_ctx,
 			time_t *tgs_expire,
 			const char *impersonate_princ_s);
 
+bool unwrap_edata_ntstatus(TALLOC_CTX *mem_ctx,
+			   DATA_BLOB *edata,
+			   DATA_BLOB *edata_out);
+bool unwrap_pac(TALLOC_CTX *mem_ctx, DATA_BLOB *auth_data, DATA_BLOB *unwrapped_pac_data);
+
+#endif /* _INCLUDE_KRB5_PROTOS_H_ */
diff --git a/source3/include/libsmb_internal.h b/source3/include/libsmb_internal.h
index 61dc95e0ad6..7f16ab6859b 100644
--- a/source3/include/libsmb_internal.h
+++ b/source3/include/libsmb_internal.h
@@ -366,7 +366,7 @@ SMBC_close_ctx(SMBCCTX *context,
 bool
 SMBC_getatr(SMBCCTX * context,
             SMBCSRV *srv,
-            char *path,
+            const char *path,
             uint16 *mode,
             SMB_OFF_T *size,
             struct timespec *create_time_ts,
diff --git a/source3/include/mangle.h b/source3/include/mangle.h
index c07b852453e..7c4602b4eba 100644
--- a/source3/include/mangle.h
+++ b/source3/include/mangle.h
@@ -1,3 +1,22 @@
+/*
+   Unix SMB/CIFS implementation.
+   Name mangling interface
+   Copyright (C) Andrew Tridgell 2002
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef _MANGLE_H_
 #define _MANGLE_H_
 /*
diff --git a/source3/include/messages.h b/source3/include/messages.h
index a78599055d2..41d5bb19fdb 100644
--- a/source3/include/messages.h
+++ b/source3/include/messages.h
@@ -49,13 +49,6 @@
 
 
 /*
- * Virtual Node Numbers are identifying a node within a cluster. Ctdbd sets
- * this, we retrieve our vnn from it.
- */
-
-#define NONCLUSTER_VNN (0xFFFFFFFF)
-
-/*
  * ctdb gives us 64-bit server ids for messaging_send. This is done to avoid
  * pid clashes and to be able to register for special messages like "all
  * smbds".
diff --git a/source3/include/nss_info.h b/source3/include/nss_info.h
index bf66ef0a8cc..f92937e990d 100644
--- a/source3/include/nss_info.h
+++ b/source3/include/nss_info.h
@@ -95,5 +95,12 @@ NTSTATUS nss_map_from_alias( TALLOC_CTX *mem_ctx, const char *domain,
 
 NTSTATUS nss_close( const char *parameters );
 
+/* The following definitions come from winbindd/nss_info.c  */
+
+
+/* The following definitions come from winbindd/nss_info_template.c  */
+
+NTSTATUS nss_info_template_init( void );
+
 #endif /* _IDMAP_NSS_H_ */
 
diff --git a/source3/include/nt_printing.h b/source3/include/nt_printing.h
index 02a72b32f19..ad6262ecff0 100644
--- a/source3/include/nt_printing.h
+++ b/source3/include/nt_printing.h
@@ -23,7 +23,7 @@
 #define NT_PRINTING_H_
 
 #include "client.h"
-#include "../librpc/gen_ndr/srv_spoolss.h"
+#include "../librpc/gen_ndr/spoolss.h"
 
 #ifndef SAMBA_PRINTER_PORT_NAME
 #define SAMBA_PRINTER_PORT_NAME "Samba Printer Port"
@@ -169,11 +169,11 @@ bool printer_driver_files_in_use(TALLOC_CTX *mem_ctx,
 bool delete_driver_files(const struct auth_serversupplied_info *server_info,
 			 const struct spoolss_DriverInfo8 *r);
 
-WERROR move_driver_to_download_area(struct pipes_struct *p,
+WERROR move_driver_to_download_area(struct auth_serversupplied_info *session_info,
 				    struct spoolss_AddDriverInfoCtr *r);
 
 WERROR clean_up_driver_struct(TALLOC_CTX *mem_ctx,
-			      struct pipes_struct *rpc_pipe,
+			      struct auth_serversupplied_info *session_info,
 			      struct spoolss_AddDriverInfoCtr *r);
 
 void map_printer_permissions(struct security_descriptor *sd);
diff --git a/source3/include/ntdomain.h b/source3/include/ntdomain.h
index 077924d6f48..ff728736d48 100644
--- a/source3/include/ntdomain.h
+++ b/source3/include/ntdomain.h
@@ -86,7 +86,8 @@ typedef struct pipe_rpc_fns {
 
 	const struct api_struct *cmds;
 	int n_cmds;
-	uint32 context_id;
+	uint32_t context_id;
+	struct ndr_syntax_id syntax;
 
 } PIPE_RPC_FNS;
 
@@ -97,20 +98,6 @@ typedef struct pipe_rpc_fns {
 
 struct gse_context;
 
-/* auth state for all bind types. */
-
-struct pipe_auth_data {
-	enum dcerpc_AuthType auth_type;
-	enum dcerpc_AuthLevel auth_level;
-
-	void *auth_ctx;
-
-	/* Only the client code uses these 3 for now */
-	char *domain;
-	char *user_name;
-	DATA_BLOB user_session_key;
-};
-
 struct dcesrv_ep_entry_list;
 
 /*
@@ -129,7 +116,6 @@ struct pipes_struct {
 	struct auth_serversupplied_info *session_info;
 	struct messaging_context *msg_ctx;
 
-	struct ndr_syntax_id syntax;
 	struct dcesrv_ep_entry_list *ep_entries;
 
 	/* linked list of rpc dispatch tables associated 
@@ -207,4 +193,33 @@ struct api_struct {
 	bool (*fn) (struct pipes_struct *);
 };
 
+/* The following definitions come from rpc_server/rpc_handles.c  */
+
+size_t num_pipe_handles(struct pipes_struct *p);
+bool init_pipe_handles(struct pipes_struct *p, const struct ndr_syntax_id *syntax);
+bool create_policy_hnd(struct pipes_struct *p, struct policy_handle *hnd, void *data_ptr);
+bool find_policy_by_hnd(struct pipes_struct *p, const struct policy_handle *hnd,
+			void **data_p);
+bool close_policy_hnd(struct pipes_struct *p, struct policy_handle *hnd);
+void close_policy_by_pipe(struct pipes_struct *p);
+bool pipe_access_check(struct pipes_struct *p);
+
+void *_policy_handle_create(struct pipes_struct *p, struct policy_handle *hnd,
+			    uint32_t access_granted, size_t data_size,
+			    const char *type, NTSTATUS *pstatus);
+#define policy_handle_create(_p, _hnd, _access, _type, _pstatus) \
+	(_type *)_policy_handle_create((_p), (_hnd), (_access), sizeof(_type), #_type, \
+				       (_pstatus))
+
+void *_policy_handle_find(struct pipes_struct *p,
+			  const struct policy_handle *hnd,
+			  uint32_t access_required, uint32_t *paccess_granted,
+			  const char *name, const char *location,
+			  NTSTATUS *pstatus);
+#define policy_handle_find(_p, _hnd, _access_required, _access_granted, _type, _pstatus) \
+	(_type *)_policy_handle_find((_p), (_hnd), (_access_required), \
+				     (_access_granted), #_type, __location__, (_pstatus))
+
+#include "rpc_server/srv_pipe_register.h"
+
 #endif /* _NT_DOMAIN_H */
diff --git a/source3/include/ntioctl.h b/source3/include/ntioctl.h
index 41b1dcefa25..131d596dedf 100644
--- a/source3/include/ntioctl.h
+++ b/source3/include/ntioctl.h
@@ -2,17 +2,17 @@
    Unix SMB/CIFS implementation.
    NT ioctl code constants
    Copyright (C) Andrew Tridgell              2002
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
@@ -69,6 +69,7 @@
 
 #endif
 
+#define IO_REPARSE_TAG_SYMLINK	     0xA000000C
 #define IO_REPARSE_TAG_MOUNT_POINT   0xA0000003
 #define IO_REPARSE_TAG_HSM           0xC0000004
 #define IO_REPARSE_TAG_SIS           0x80000007
@@ -77,13 +78,12 @@
 /* For FSCTL_GET_SHADOW_COPY_DATA ...*/
 typedef char SHADOW_COPY_LABEL[25];
 
-typedef struct shadow_copy_data {
-	TALLOC_CTX *mem_ctx;
+struct shadow_copy_data {
 	/* Total number of shadow volumes currently mounted */
 	uint32 num_volumes;
 	/* Concatenated list of labels */
 	SHADOW_COPY_LABEL *labels;
-} SHADOW_COPY_DATA;
+};
 
 
 #endif /* _NTIOCTL_H */
diff --git a/source3/include/popt_common.h b/source3/include/popt_common.h
index 70d7278f01d..10dd2f4ebee 100644
--- a/source3/include/popt_common.h
+++ b/source3/include/popt_common.h
@@ -42,7 +42,7 @@ extern const struct poptOption popt_common_dynconfig[];
 #define POPT_COMMON_CONFIGFILE { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_configfile, 0, "Common samba config:", NULL },
 #define POPT_COMMON_CREDENTIALS { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_credentials, 0, "Authentication options:", NULL },
 #define POPT_COMMON_DYNCONFIG { NULL, 0, POPT_ARG_INCLUDE_TABLE, \
-    CONST_DISCARD(poptOption *, popt_common_dynconfig), 0, \
+    discard_const_p(poptOption, popt_common_dynconfig), 0, \
     "Build-time configuration overrides:", NULL },
 #define POPT_COMMON_DEBUGLEVEL { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_debuglevel, 0, "Common samba debugging:", NULL },
 #define POPT_COMMON_OPTION { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_option, 0, "Common samba commandline config:", NULL },
diff --git a/source3/include/printing.h b/source3/include/printing.h
index 084c699c5bd..1bbb72c7be6 100644
--- a/source3/include/printing.h
+++ b/source3/include/printing.h
@@ -24,6 +24,7 @@
    This file defines the low-level printing system interfaces used by the
    SAMBA printing subsystem.
 */
+#include "tdb_compat.h"
 
 /* Extra fields above "LPQ_PRINTING" are used to map extra NT status codes. */
 
@@ -160,7 +161,7 @@ extern struct printif	iprint_printif;
 
 struct tdb_print_db {
 	struct tdb_print_db *next, *prev;
-	TDB_CONTEXT *tdb;
+	struct tdb_context *tdb;
 	int ref_count;
 	fstring printer_name;
 };
@@ -249,7 +250,7 @@ uint32_t print_parse_jobid(const char *fname);
 struct tdb_print_db *get_print_db_byname(const char *printername);
 void release_print_db( struct tdb_print_db *pdb);
 void close_all_print_db(void);
-TDB_DATA get_printer_notify_pid_list(TDB_CONTEXT *tdb, const char *printer_name, bool cleanlist);
+TDB_DATA get_printer_notify_pid_list(struct tdb_context *tdb, const char *printer_name, bool cleanlist);
 
 void print_queue_receive(struct messaging_context *msg,
 				void *private_data,
diff --git a/source3/include/proto.h b/source3/include/proto.h
index 36f0235b02f..90dbafc96c9 100644
--- a/source3/include/proto.h
+++ b/source3/include/proto.h
@@ -73,19 +73,6 @@ int bitmap_find(struct bitmap *bm, unsigned ofs);
 /* The following definitions come from lib/charcnv.c  */
 
 void gfree_charcnv(void);
-void init_iconv(void);
-bool convert_string(charset_t from, charset_t to,
-		      void const *src, size_t srclen, 
-		      void *dest, size_t destlen,
-		      size_t *converted_size);
-bool convert_string_error(charset_t from, charset_t to,
-			    void const *src, size_t srclen,
-			    void *dest, size_t destlen,
-			    size_t *converted_size);
-char *talloc_strdup_upper(TALLOC_CTX *ctx, const char *s);
-char *strupper_talloc(TALLOC_CTX *ctx, const char *s);
-char *talloc_strdup_lower(TALLOC_CTX *ctx, const char *s);
-char *strlower_talloc(TALLOC_CTX *ctx, const char *s);
 size_t ucs2_align(const void *base_ptr, const void *p, int flags);
 size_t push_ascii(void *dest, const char *src, size_t dest_len, int flags);
 size_t push_ascii_fstring(void *dest, const char *src);
@@ -93,17 +80,6 @@ size_t push_ascii_nstring(void *dest, const char *src);
 size_t pull_ascii(char *dest, const void *src, size_t dest_len, size_t src_len, int flags);
 size_t pull_ascii_fstring(char *dest, const void *src);
 size_t pull_ascii_nstring(char *dest, size_t dest_len, const void *src);
-size_t push_ucs2(const void *base_ptr, void *dest, const char *src, size_t dest_len, int flags);
-bool push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
-		      size_t *converted_size);
-bool push_ucs2_talloc(TALLOC_CTX *ctx, smb_ucs2_t **dest, const char *src,
-		      size_t *converted_size);
-bool pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
-		      size_t *converted_size);
-bool pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const smb_ucs2_t *src,
-		      size_t *converted_size);
-bool pull_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
-		       size_t *converted_size);
 size_t push_string_check_fn(void *dest, const char *src,
 			    size_t dest_len, int flags);
 size_t push_string_base(const char *base, uint16 flags2,
@@ -232,7 +208,6 @@ void init_modules(void);
 
 int ms_fnmatch(const char *pattern, const char *string, bool translate_pattern,
 	       bool is_case_sensitive);
-int gen_fnmatch(const char *pattern, const char *string);
 
 /* The following definitions come from lib/pidfile.c  */
 
@@ -252,47 +227,6 @@ ssize_t sys_recvfile(int fromfd,
 			size_t count);
 ssize_t drain_socket(int sockfd, size_t count);
 
-/* The following definitions come from lib/secdesc.c  */
-
-uint32_t get_sec_info(const struct security_descriptor *sd);
-struct security_descriptor *sec_desc_merge(TALLOC_CTX *ctx, struct security_descriptor *new_sdb, struct security_descriptor *old_sdb);
-struct sec_desc_buf *sec_desc_merge_buf(TALLOC_CTX *ctx, struct sec_desc_buf *new_sdb, struct sec_desc_buf *old_sdb);
-struct security_descriptor *make_sec_desc(TALLOC_CTX *ctx,
-			enum security_descriptor_revision revision,
-			uint16 type,
-			const struct dom_sid *owner_sid, const struct dom_sid *grp_sid,
-			struct security_acl *sacl, struct security_acl *dacl, size_t *sd_size);
-struct security_descriptor *dup_sec_desc(TALLOC_CTX *ctx, const struct security_descriptor *src);
-NTSTATUS marshall_sec_desc(TALLOC_CTX *mem_ctx,
-			   struct security_descriptor *secdesc,
-			   uint8 **data, size_t *len);
-NTSTATUS marshall_sec_desc_buf(TALLOC_CTX *mem_ctx,
-			       struct sec_desc_buf *secdesc_buf,
-			       uint8_t **data, size_t *len);
-NTSTATUS unmarshall_sec_desc(TALLOC_CTX *mem_ctx, uint8 *data, size_t len,
-			     struct security_descriptor **psecdesc);
-NTSTATUS unmarshall_sec_desc_buf(TALLOC_CTX *mem_ctx, uint8_t *data, size_t len,
-				 struct sec_desc_buf **psecdesc_buf);
-struct security_descriptor *make_standard_sec_desc(TALLOC_CTX *ctx, const struct dom_sid *owner_sid, const struct dom_sid *grp_sid,
-				 struct security_acl *dacl, size_t *sd_size);
-struct sec_desc_buf *make_sec_desc_buf(TALLOC_CTX *ctx, size_t len, struct security_descriptor *sec_desc);
-struct sec_desc_buf *dup_sec_desc_buf(TALLOC_CTX *ctx, struct sec_desc_buf *src);
-NTSTATUS sec_desc_add_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, const struct dom_sid *sid, uint32 mask, size_t *sd_size);
-NTSTATUS sec_desc_mod_sid(struct security_descriptor *sd, struct dom_sid *sid, uint32 mask);
-NTSTATUS sec_desc_del_sid(TALLOC_CTX *ctx, struct security_descriptor **psd, struct dom_sid *sid, size_t *sd_size);
-bool sd_has_inheritable_components(const struct security_descriptor *parent_ctr, bool container);
-NTSTATUS se_create_child_secdesc(TALLOC_CTX *ctx,
-                                        struct security_descriptor **ppsd,
-					size_t *psize,
-                                        const struct security_descriptor *parent_ctr,
-                                        const struct dom_sid *owner_sid,
-                                        const struct dom_sid *group_sid,
-                                        bool container);
-NTSTATUS se_create_child_secdesc_buf(TALLOC_CTX *ctx,
-					struct sec_desc_buf **ppsdb,
-					const struct security_descriptor *parent_ctr,
-					bool container);
-
 /* The following definitions come from lib/sendfile.c  */
 
 ssize_t sys_sendfile(int tofd, int fromfd, const DATA_BLOB *header, SMB_OFF_T offset, size_t count);
@@ -386,60 +320,6 @@ int sys_acl_set_fd(vfs_handle_struct *handle, files_struct *fsp,
 		   SMB_ACL_T acl_d);
 int sys_acl_delete_def_file(vfs_handle_struct *handle,
 			    const char *path);
-SMB_ACL_T sys_acl_get_file(vfs_handle_struct *handle,
-			   const char *path_p, SMB_ACL_TYPE_T type);
-SMB_ACL_T sys_acl_get_fd(vfs_handle_struct *handle, files_struct *fsp);
-int sys_acl_set_file(vfs_handle_struct *handle,
-		     const char *name, SMB_ACL_TYPE_T type, SMB_ACL_T acl_d);
-int sys_acl_set_fd(vfs_handle_struct *handle, files_struct *fsp,
-		   SMB_ACL_T acl_d);
-int sys_acl_delete_def_file(vfs_handle_struct *handle,
-			    const char *path);
-SMB_ACL_T sys_acl_get_file(vfs_handle_struct *handle,
-			   const char *path_p, SMB_ACL_TYPE_T type);
-SMB_ACL_T sys_acl_get_fd(vfs_handle_struct *handle, files_struct *fsp);
-int sys_acl_set_file(vfs_handle_struct *handle,
-		     const char *name, SMB_ACL_TYPE_T type, SMB_ACL_T acl_d);
-int sys_acl_set_fd(vfs_handle_struct *handle, files_struct *fsp,
-		   SMB_ACL_T acl_d);
-int sys_acl_delete_def_file(vfs_handle_struct *handle,
-			    const char *path);
-SMB_ACL_T sys_acl_get_file(vfs_handle_struct *handle,
-			   const char *path_p, SMB_ACL_TYPE_T type);
-SMB_ACL_T sys_acl_get_fd(vfs_handle_struct *handle, files_struct *fsp);
-int sys_acl_set_file(vfs_handle_struct *handle,
-		     const char *name, SMB_ACL_TYPE_T type, SMB_ACL_T acl_d);
-int sys_acl_set_fd(vfs_handle_struct *handle, files_struct *fsp,
-		   SMB_ACL_T acl_d);
-int sys_acl_delete_def_file(vfs_handle_struct *handle,
-			    const char *path);
-SMB_ACL_T sys_acl_get_file(vfs_handle_struct *handle,
-			   const char *path_p, SMB_ACL_TYPE_T type);
-SMB_ACL_T sys_acl_get_fd(vfs_handle_struct *handle, files_struct *fsp);
-int sys_acl_set_file(vfs_handle_struct *handle,
-		     const char *name, SMB_ACL_TYPE_T type, SMB_ACL_T acl_d);
-int sys_acl_set_fd(vfs_handle_struct *handle, files_struct *fsp,
-		   SMB_ACL_T acl_d);
-int sys_acl_delete_def_file(vfs_handle_struct *handle,
-			    const char *path);
-SMB_ACL_T sys_acl_get_file(vfs_handle_struct *handle,
-			   const char *path_p, SMB_ACL_TYPE_T type);
-SMB_ACL_T sys_acl_get_fd(vfs_handle_struct *handle, files_struct *fsp);
-int sys_acl_set_file(vfs_handle_struct *handle,
-		     const char *name, SMB_ACL_TYPE_T type, SMB_ACL_T acl_d);
-int sys_acl_set_fd(vfs_handle_struct *handle, files_struct *fsp,
-		   SMB_ACL_T acl_d);
-int sys_acl_delete_def_file(vfs_handle_struct *handle,
-			    const char *path);
-SMB_ACL_T sys_acl_get_file(vfs_handle_struct *handle,
-			   const char *path_p, SMB_ACL_TYPE_T type);
-SMB_ACL_T sys_acl_get_fd(vfs_handle_struct *handle, files_struct *fsp);
-int sys_acl_set_file(vfs_handle_struct *handle,
-		     const char *name, SMB_ACL_TYPE_T type, SMB_ACL_T acl_d);
-int sys_acl_set_fd(vfs_handle_struct *handle, files_struct *fsp,
-		   SMB_ACL_T acl_d);
-int sys_acl_delete_def_file(vfs_handle_struct *handle,
-			    const char *path);
 int no_acl_syscall_error(int err);
 
 /* The following definitions come from lib/sysquotas.c  */
@@ -464,7 +344,6 @@ int sys_set_nfs_quota(const char *path, const char *bdev,
 
 /* The following definitions come from lib/system.c  */
 
-void *sys_memalign( size_t align, size_t size );
 int sys_usleep(long usecs);
 ssize_t sys_read(int fd, void *buf, size_t count);
 ssize_t sys_write(int fd, const void *buf, size_t count);
@@ -504,7 +383,7 @@ void sys_rewinddir(SMB_STRUCT_DIR *dirp);
 int sys_closedir(SMB_STRUCT_DIR *dirp);
 int sys_mknod(const char *path, mode_t mode, SMB_DEV_T dev);
 int sys_waitpid(pid_t pid,int *status,int options);
-char *sys_getwd(char *s);
+char *sys_getwd(void);
 void set_effective_capability(enum smbd_capability capability);
 void drop_effective_capability(enum smbd_capability capability);
 long sys_random(void);
@@ -542,15 +421,18 @@ int sys_aio_cancel(int fd, SMB_STRUCT_AIOCB *aiocb);
 int sys_aio_error(const SMB_STRUCT_AIOCB *aiocb);
 int sys_aio_fsync(int op, SMB_STRUCT_AIOCB *aiocb);
 int sys_aio_suspend(const SMB_STRUCT_AIOCB * const cblist[], int n, const struct timespec *timeout);
-int sys_getpeereid( int s, uid_t *uid);
-int sys_getnameinfo(const struct sockaddr *psa,
-			socklen_t salen,
-			char *host,
-			size_t hostlen,
-			char *service,
-			size_t servlen,
-			int flags);
-int sys_connect(int fd, const struct sockaddr * addr);
+
+#if defined(HAVE_EXPLICIT_LARGEFILE_SUPPORT) && defined(HAVE_OFF64_T) && defined(HAVE_STAT64)
+struct stat64;
+void init_stat_ex_from_stat (struct stat_ex *dst,
+			    const struct stat64 *src,
+			    bool fake_dir_create_times);
+#else
+struct stat;
+void init_stat_ex_from_stat (struct stat_ex *dst,
+			    const struct stat *src,
+			    bool fake_dir_create_times);
+#endif
 
 /* The following definitions come from lib/system_smbd.c  */
 
@@ -616,11 +498,6 @@ char *get_user_home_dir(TALLOC_CTX *mem_ctx, const char *user);
 struct passwd *Get_Pwnam_alloc(TALLOC_CTX *mem_ctx, const char *user);
 
 /* The following definitions come from lib/util_names.c  */
-void gfree_netbios_names(void);
-bool set_global_myname(const char *myname);
-const char *global_myname(void);
-bool set_global_myworkgroup(const char *myworkgroup);
-const char *lp_workgroup(void);
 const char *get_global_sam_name(void);
 
 /* The following definitions come from lib/util.c  */
@@ -628,8 +505,6 @@ const char *get_global_sam_name(void);
 enum protocol_types get_Protocol(void);
 void set_Protocol(enum protocol_types  p);
 bool all_zero(const uint8_t *ptr, size_t size);
-bool set_global_scope(const char *scope);
-const char *global_scope(void);
 void gfree_names(void);
 void gfree_all( void );
 const char *my_netbios_names(int i);
@@ -640,8 +515,7 @@ bool file_exist_stat(const char *fname,SMB_STRUCT_STAT *sbuf,
 bool socket_exist(const char *fname);
 uint64_t get_file_size_stat(const SMB_STRUCT_STAT *sbuf);
 SMB_OFF_T get_file_size(char *file_name);
-char *attrib_string(uint16 mode);
-void show_msg(char *buf);
+void show_msg(const char *buf);
 void smb_set_enclen(char *buf,int len,uint16 enc_ctx_num);
 void smb_setlen(char *buf,int len);
 int set_message_bcc(char *buf,int num_bytes);
@@ -692,14 +566,11 @@ int set_maxfiles(int requested_max);
 int smb_mkstemp(char *name_template);
 void *smb_xmalloc_array(size_t size, unsigned int count);
 char *myhostname(void);
+char *myhostname_upper(void);
 char *lock_path(const char *name);
 char *pid_path(const char *name);
-char *lib_path(const char *name);
-char *modules_path(const char *name);
-char *data_path(const char *name);
 char *state_path(const char *name);
 char *cache_path(const char *name);
-const char *shlib_ext(void);
 bool parent_dirname(TALLOC_CTX *mem_ctx, const char *dir, char **parent,
 		    const char **name);
 bool ms_has_wild(const char *s);
@@ -722,7 +593,6 @@ bool cluster_id_equal(const struct server_id *id1,
 		      const struct server_id *id2);
 bool procid_is_me(const struct server_id *pid);
 struct server_id interpret_pid(const char *pid_string);
-char *procid_str(TALLOC_CTX *mem_ctx, const struct server_id *pid);
 char *procid_str_static(const struct server_id *pid);
 bool procid_valid(const struct server_id *pid);
 bool procid_is_local(const struct server_id *pid);
@@ -736,17 +606,13 @@ void split_domain_user(TALLOC_CTX *mem_ctx,
 		       const char *full_name,
 		       char **domain,
 		       char **user);
-void *_talloc_zero_zeronull(const void *ctx, size_t size, const char *name);
-void *_talloc_memdup_zeronull(const void *t, const void *p, size_t size, const char *name);
-void *_talloc_array_zeronull(const void *ctx, size_t el_size, unsigned count, const char *name);
-void *_talloc_zero_array_zeronull(const void *ctx, size_t el_size, unsigned count, const char *name);
-void *talloc_zeronull(const void *context, size_t size, const char *name);
 const char *strip_hostname(const char *s);
 bool tevent_req_poll_ntstatus(struct tevent_req *req,
 			      struct tevent_context *ev,
 			      NTSTATUS *status);
 bool any_nt_status_not_ok(NTSTATUS err1, NTSTATUS err2, NTSTATUS *result);
 int timeval_to_msec(struct timeval t);
+char *valid_share_pathname(TALLOC_CTX *ctx, const char *dos_pathname);
 
 /* The following definitions come from lib/util_cmdline.c  */
 
@@ -936,6 +802,7 @@ struct tevent_req *getaddrinfo_send(TALLOC_CTX *mem_ctx,
 int getaddrinfo_recv(struct tevent_req *req, struct addrinfo **res);
 int poll_one_fd(int fd, int events, int timeout, int *revents);
 int poll_intr_one_fd(int fd, int events, int timeout, int *revents);
+struct tstream_context;
 struct tevent_req *tstream_read_packet_send(TALLOC_CTX *mem_ctx,
 					    struct tevent_context *ev,
 					    struct tstream_context *stream,
@@ -950,8 +817,6 @@ ssize_t tstream_read_packet_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
 /* The following definitions come from lib/util_str.c  */
 
 bool next_token(const char **ptr, char *buff, const char *sep, size_t bufsize);
-int StrCaseCmp(const char *s, const char *t);
-int StrnCaseCmp(const char *s, const char *t, size_t len);
 bool strnequal(const char *s1,const char *s2,size_t n);
 bool strcsequal(const char *s1,const char *s2);
 void strnorm(char *s, int case_default);
@@ -961,22 +826,10 @@ size_t str_charnum(const char *s);
 bool trim_char(char *s,char cfront,char cback);
 bool strhasupper(const char *s);
 bool strhaslower(const char *s);
-char *safe_strcpy_fn(char *dest,
-		const char *src,
-		size_t maxlength);
-char *safe_strcat_fn(char *dest,
-		const char *src,
-		size_t maxlength);
 char *StrnCpy(char *dest,const char *src,size_t n);
 bool in_list(const char *s, const char *list, bool casesensitive);
 void string_free(char **s);
 bool string_set(char **dest,const char *src);
-void string_sub2(char *s,const char *pattern, const char *insert, size_t len,
-		 bool remove_unsafe_characters, bool replace_once,
-		 bool allow_trailing_dollar);
-void string_sub_once(char *s, const char *pattern,
-		const char *insert, size_t len);
-void string_sub(char *s,const char *pattern, const char *insert, size_t len);
 void fstring_sub(char *s,const char *pattern,const char *insert);
 char *realloc_string_sub2(char *string,
 			const char *pattern,
@@ -1023,7 +876,7 @@ char *ipstr_list_make(char **ipstr_list,
 int ipstr_list_parse(const char *ipstr_list, struct ip_service **ip_list);
 void ipstr_list_free(char* ipstr_list);
 uint64_t STR_TO_SMB_BIG_UINT(const char *nptr, const char **entptr);
-SMB_OFF_T conv_str_size(const char * str);
+uint64_t conv_str_size(const char * str);
 bool add_string_to_array(TALLOC_CTX *mem_ctx,
 			 const char *str, const char ***strings,
 			 int *num);
@@ -1032,7 +885,6 @@ void sprintf_append(TALLOC_CTX *mem_ctx, char **string, ssize_t *len,
 int asprintf_strupper_m(char **strp, const char *fmt, ...);
 char *talloc_asprintf_strupper_m(TALLOC_CTX *t, const char *fmt, ...);
 char *talloc_asprintf_strlower_m(TALLOC_CTX *t, const char *fmt, ...);
-char *sstring_sub(const char *src, char front, char back);
 bool validate_net_name( const char *name,
 		const char *invalid_chars,
 		int max_len);
@@ -1053,746 +905,10 @@ unsigned wins_srv_count(void);
 char **wins_srv_tags(void);
 void wins_srv_tags_free(char **list);
 struct in_addr wins_srv_ip_tag(const char *tag, struct in_addr src_ip);
+bool wins_server_tag_ips(const char *tag, TALLOC_CTX *mem_ctx,
+			 struct in_addr **pservers, int *pnum_servers);
 unsigned wins_srv_count_tag(const char *tag);
 
-/* The following definitions come from libsmb/cliconnect.c  */
-
-ADS_STATUS cli_session_setup_spnego(struct cli_state *cli, const char *user, 
-			      const char *pass, const char *user_domain,
-				    const char * dest_realm);
-
-NTSTATUS cli_session_setup(struct cli_state *cli,
-			   const char *user,
-			   const char *pass, int passlen,
-			   const char *ntpass, int ntpasslen,
-			   const char *workgroup);
-struct tevent_req *cli_session_setup_guest_create(TALLOC_CTX *mem_ctx,
-						  struct event_context *ev,
-						  struct cli_state *cli,
-						  struct tevent_req **psmbreq);
-struct tevent_req *cli_session_setup_guest_send(TALLOC_CTX *mem_ctx,
-						struct event_context *ev,
-						struct cli_state *cli);
-NTSTATUS cli_session_setup_guest_recv(struct tevent_req *req);
-struct tevent_req *cli_ulogoff_send(TALLOC_CTX *mem_ctx,
-				    struct tevent_context *ev,
-				    struct cli_state *cli);
-NTSTATUS cli_ulogoff_recv(struct tevent_req *req);
-NTSTATUS cli_ulogoff(struct cli_state *cli);
-struct tevent_req *cli_tcon_andx_create(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *share, const char *dev,
-					const char *pass, int passlen,
-					struct tevent_req **psmbreq);
-struct tevent_req *cli_tcon_andx_send(TALLOC_CTX *mem_ctx,
-				      struct event_context *ev,
-				      struct cli_state *cli,
-				      const char *share, const char *dev,
-				      const char *pass, int passlen);
-NTSTATUS cli_tcon_andx_recv(struct tevent_req *req);
-NTSTATUS cli_tcon_andx(struct cli_state *cli, const char *share,
-		       const char *dev, const char *pass, int passlen);
-struct tevent_req *cli_tdis_send(TALLOC_CTX *mem_ctx,
-                                 struct tevent_context *ev,
-                                 struct cli_state *cli);
-NTSTATUS cli_tdis_recv(struct tevent_req *req);
-NTSTATUS cli_tdis(struct cli_state *cli);
-NTSTATUS cli_negprot(struct cli_state *cli);
-struct tevent_req *cli_negprot_send(TALLOC_CTX *mem_ctx,
-				    struct event_context *ev,
-				    struct cli_state *cli);
-NTSTATUS cli_negprot_recv(struct tevent_req *req);
-bool cli_session_request(struct cli_state *cli,
-			 struct nmb_name *calling, struct nmb_name *called);
-NTSTATUS cli_connect(struct cli_state *cli,
-		const char *host,
-		struct sockaddr_storage *dest_ss);
-NTSTATUS cli_start_connection(struct cli_state **output_cli, 
-			      const char *my_name, 
-			      const char *dest_host, 
-			      struct sockaddr_storage *dest_ss, int port,
-			      int signing_state, int flags);
-NTSTATUS cli_full_connection(struct cli_state **output_cli, 
-			     const char *my_name, 
-			     const char *dest_host, 
-			     struct sockaddr_storage *dest_ss, int port,
-			     const char *service, const char *service_type,
-			     const char *user, const char *domain, 
-			     const char *password, int flags,
-			     int signing_state);
-bool attempt_netbios_session_request(struct cli_state **ppcli, const char *srchost, const char *desthost,
-                                     struct sockaddr_storage *pdest_ss);
-NTSTATUS cli_raw_tcon(struct cli_state *cli, 
-		      const char *service, const char *pass, const char *dev,
-		      uint16 *max_xmit, uint16 *tid);
-struct cli_state *get_ipc_connect(char *server,
-				struct sockaddr_storage *server_ss,
-				const struct user_auth_info *user_info);
-struct cli_state *get_ipc_connect_master_ip(TALLOC_CTX *ctx,
-				struct ip_service *mb_ip,
-				const struct user_auth_info *user_info,
-				char **pp_workgroup_out);
-struct cli_state *get_ipc_connect_master_ip_bcast(TALLOC_CTX *ctx,
-					const struct user_auth_info *user_info,
-					char **pp_workgroup_out);
-
-/* The following definitions come from libsmb/clidfs.c  */
-
-NTSTATUS cli_cm_force_encryption(struct cli_state *c,
-			const char *username,
-			const char *password,
-			const char *domain,
-			const char *sharename);
-struct cli_state *cli_cm_open(TALLOC_CTX *ctx,
-				struct cli_state *referring_cli,
-				const char *server,
-				const char *share,
-				const struct user_auth_info *auth_info,
-				bool show_hdr,
-				bool force_encrypt,
-				int max_protocol,
-				int port,
-				int name_type);
-void cli_cm_display(const struct cli_state *c);
-struct client_dfs_referral;
-NTSTATUS cli_dfs_get_referral(TALLOC_CTX *ctx,
-			struct cli_state *cli,
-			const char *path,
-			struct client_dfs_referral **refs,
-			size_t *num_refs,
-			size_t *consumed);
-bool cli_resolve_path(TALLOC_CTX *ctx,
-			const char *mountpt,
-			const struct user_auth_info *dfs_auth_info,
-			struct cli_state *rootcli,
-			const char *path,
-			struct cli_state **targetcli,
-			char **pp_targetpath);
-
-bool cli_check_msdfs_proxy(TALLOC_CTX *ctx,
-			struct cli_state *cli,
-			const char *sharename,
-			char **pp_newserver,
-			char **pp_newshare,
-			bool force_encrypt,
-			const char *username,
-			const char *password,
-			const char *domain);
-
-/* The following definitions come from libsmb/clientgen.c  */
-
-int cli_set_message(char *buf,int num_words,int num_bytes,bool zero);
-unsigned int cli_set_timeout(struct cli_state *cli, unsigned int timeout);
-void cli_set_port(struct cli_state *cli, int port);
-bool cli_state_seqnum_persistent(struct cli_state *cli,
-				 uint16_t mid);
-bool cli_state_seqnum_remove(struct cli_state *cli,
-			     uint16_t mid);
-bool cli_receive_smb(struct cli_state *cli);
-bool cli_send_smb(struct cli_state *cli);
-bool cli_send_smb_direct_writeX(struct cli_state *cli,
-				const char *p,
-				size_t extradata);
-void cli_setup_packet_buf(struct cli_state *cli, char *buf);
-void cli_setup_packet(struct cli_state *cli);
-void cli_setup_bcc(struct cli_state *cli, void *p);
-NTSTATUS cli_set_domain(struct cli_state *cli, const char *domain);
-NTSTATUS cli_set_username(struct cli_state *cli, const char *username);
-NTSTATUS cli_set_password(struct cli_state *cli, const char *password);
-NTSTATUS cli_init_creds(struct cli_state *cli, const char *username, const char *domain, const char *password);
-struct cli_state *cli_initialise(void);
-struct cli_state *cli_initialise_ex(int signing_state);
-void cli_nt_pipes_close(struct cli_state *cli);
-void cli_shutdown(struct cli_state *cli);
-void cli_sockopt(struct cli_state *cli, const char *options);
-uint16 cli_setpid(struct cli_state *cli, uint16 pid);
-bool cli_set_case_sensitive(struct cli_state *cli, bool case_sensitive);
-struct tevent_req *cli_echo_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
-				 struct cli_state *cli, uint16_t num_echos,
-				 DATA_BLOB data);
-NTSTATUS cli_echo_recv(struct tevent_req *req);
-NTSTATUS cli_echo(struct cli_state *cli, uint16_t num_echos, DATA_BLOB data);
-bool cli_ucs2(struct cli_state *cli);
-bool is_andx_req(uint8_t cmd);
-NTSTATUS cli_smb(TALLOC_CTX *mem_ctx, struct cli_state *cli,
-		 uint8_t smb_command, uint8_t additional_flags,
-		 uint8_t wct, uint16_t *vwv,
-		 uint32_t num_bytes, const uint8_t *bytes,
-		 struct tevent_req **result_parent,
-		 uint8_t min_wct, uint8_t *pwct, uint16_t **pvwv,
-		 uint32_t *pnum_bytes, uint8_t **pbytes);
-
-/* The following definitions come from libsmb/clierror.c  */
-
-const char *cli_errstr(struct cli_state *cli);
-NTSTATUS cli_nt_error(struct cli_state *cli);
-void cli_dos_error(struct cli_state *cli, uint8 *eclass, uint32 *ecode);
-int cli_errno(struct cli_state *cli);
-bool cli_is_error(struct cli_state *cli);
-bool cli_is_nt_error(struct cli_state *cli);
-bool cli_is_dos_error(struct cli_state *cli);
-NTSTATUS cli_get_nt_error(struct cli_state *cli);
-void cli_set_nt_error(struct cli_state *cli, NTSTATUS status);
-void cli_reset_error(struct cli_state *cli);
-bool cli_state_is_connected(struct cli_state *cli);
-
-/* The following definitions come from libsmb/clifile.c  */
-
-struct tevent_req *cli_setpathinfo_send(TALLOC_CTX *mem_ctx,
-					struct tevent_context *ev,
-					struct cli_state *cli,
-					uint16_t level,
-					const char *path,
-					uint8_t *data,
-					size_t data_len);
-NTSTATUS cli_setpathinfo_recv(struct tevent_req *req);
-NTSTATUS cli_setpathinfo(struct cli_state *cli,
-			 uint16_t level,
-			 const char *path,
-			 uint8_t *data,
-			 size_t data_len);
-
-struct tevent_req *cli_posix_symlink_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *oldname,
-					const char *newname);
-NTSTATUS cli_posix_symlink_recv(struct tevent_req *req);
-NTSTATUS cli_posix_symlink(struct cli_state *cli,
-			const char *oldname,
-			const char *newname);
-struct tevent_req *cli_posix_readlink_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname,
-					size_t len);
-NTSTATUS cli_posix_readlink_recv(struct tevent_req *req, struct cli_state *cli,
-				char *retpath, size_t len);
-NTSTATUS cli_posix_readlink(struct cli_state *cli, const char *fname,
-			char *linkpath, size_t len);
-struct tevent_req *cli_posix_hardlink_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *oldname,
-					const char *newname);
-NTSTATUS cli_posix_hardlink_recv(struct tevent_req *req);
-NTSTATUS cli_posix_hardlink(struct cli_state *cli,
-			const char *oldname,
-			const char *newname);
-uint32_t unix_perms_to_wire(mode_t perms);
-mode_t wire_perms_to_unix(uint32_t perms);
-struct tevent_req *cli_posix_getfacl_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname);
-NTSTATUS cli_posix_getfacl_recv(struct tevent_req *req,
-				TALLOC_CTX *mem_ctx,
-				size_t *prb_size,
-				char **retbuf);
-NTSTATUS cli_posix_getfacl(struct cli_state *cli,
-			const char *fname,
-			TALLOC_CTX *mem_ctx,
-			size_t *prb_size,
-			char **retbuf);
-struct tevent_req *cli_posix_stat_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname);
-NTSTATUS cli_posix_stat_recv(struct tevent_req *req,
-				SMB_STRUCT_STAT *sbuf);
-NTSTATUS cli_posix_stat(struct cli_state *cli,
-			const char *fname,
-			SMB_STRUCT_STAT *sbuf);
-struct tevent_req *cli_posix_chmod_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname,
-					mode_t mode);
-NTSTATUS cli_posix_chmod_recv(struct tevent_req *req);
-NTSTATUS cli_posix_chmod(struct cli_state *cli, const char *fname, mode_t mode);
-struct tevent_req *cli_posix_chown_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname,
-					uid_t uid,
-					gid_t gid);
-NTSTATUS cli_posix_chown_recv(struct tevent_req *req);
-NTSTATUS cli_posix_chown(struct cli_state *cli,
-			const char *fname,
-			uid_t uid,
-			gid_t gid);
-struct tevent_req *cli_rename_send(TALLOC_CTX *mem_ctx,
-                                struct event_context *ev,
-                                struct cli_state *cli,
-                                const char *fname_src,
-                                const char *fname_dst);
-NTSTATUS cli_rename_recv(struct tevent_req *req);
-NTSTATUS cli_rename(struct cli_state *cli, const char *fname_src, const char *fname_dst);
-struct tevent_req *cli_ntrename_send(TALLOC_CTX *mem_ctx,
-                                struct event_context *ev,
-                                struct cli_state *cli,
-                                const char *fname_src,
-                                const char *fname_dst);
-NTSTATUS cli_ntrename_recv(struct tevent_req *req);
-NTSTATUS cli_ntrename(struct cli_state *cli, const char *fname_src, const char *fname_dst);
-
-struct tevent_req *cli_nt_hardlink_send(TALLOC_CTX *mem_ctx,
-                                struct event_context *ev,
-                                struct cli_state *cli,
-                                const char *fname_src,
-                                const char *fname_dst);
-NTSTATUS cli_nt_hardlink_recv(struct tevent_req *req);
-NTSTATUS cli_nt_hardlink(struct cli_state *cli, const char *fname_src, const char *fname_dst);
-
-struct tevent_req *cli_unlink_send(TALLOC_CTX *mem_ctx,
-                                struct event_context *ev,
-                                struct cli_state *cli,
-                                const char *fname,
-                                uint16_t mayhave_attrs);
-NTSTATUS cli_unlink_recv(struct tevent_req *req);
-NTSTATUS cli_unlink(struct cli_state *cli, const char *fname, uint16_t mayhave_attrs);
-
-struct tevent_req *cli_mkdir_send(TALLOC_CTX *mem_ctx,
-				  struct event_context *ev,
-				  struct cli_state *cli,
-				  const char *dname);
-NTSTATUS cli_mkdir_recv(struct tevent_req *req);
-NTSTATUS cli_mkdir(struct cli_state *cli, const char *dname);
-struct tevent_req *cli_rmdir_send(TALLOC_CTX *mem_ctx,
-				  struct event_context *ev,
-				  struct cli_state *cli,
-				  const char *dname);
-NTSTATUS cli_rmdir_recv(struct tevent_req *req);
-NTSTATUS cli_rmdir(struct cli_state *cli, const char *dname);
-struct tevent_req *cli_nt_delete_on_close_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					uint16_t fnum,
-					bool flag);
-NTSTATUS cli_nt_delete_on_close_recv(struct tevent_req *req);
-NTSTATUS cli_nt_delete_on_close(struct cli_state *cli, uint16_t fnum, bool flag);
-struct tevent_req *cli_ntcreate_send(TALLOC_CTX *mem_ctx,
-				     struct event_context *ev,
-				     struct cli_state *cli,
-				     const char *fname,
-				     uint32_t CreatFlags,
-				     uint32_t DesiredAccess,
-				     uint32_t FileAttributes,
-				     uint32_t ShareAccess,
-				     uint32_t CreateDisposition,
-				     uint32_t CreateOptions,
-				     uint8_t SecurityFlags);
-NTSTATUS cli_ntcreate_recv(struct tevent_req *req, uint16_t *pfnum);
-NTSTATUS cli_ntcreate(struct cli_state *cli,
-		      const char *fname,
-		      uint32_t CreatFlags,
-		      uint32_t DesiredAccess,
-		      uint32_t FileAttributes,
-		      uint32_t ShareAccess,
-		      uint32_t CreateDisposition,
-		      uint32_t CreateOptions,
-		      uint8_t SecurityFlags,
-		      uint16_t *pfid);
-uint8_t *smb_bytes_push_str(uint8_t *buf, bool ucs2, const char *str,
-			    size_t str_len, size_t *pconverted_size);
-uint8_t *smb_bytes_push_bytes(uint8_t *buf, uint8_t prefix,
-			      const uint8_t *bytes, size_t num_bytes);
-struct tevent_req *cli_open_create(TALLOC_CTX *mem_ctx,
-				   struct event_context *ev,
-				   struct cli_state *cli, const char *fname,
-				   int flags, int share_mode,
-				   struct tevent_req **psmbreq);
-struct tevent_req *cli_open_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
-				 struct cli_state *cli, const char *fname,
-				 int flags, int share_mode);
-NTSTATUS cli_open_recv(struct tevent_req *req, uint16_t *fnum);
-NTSTATUS cli_open(struct cli_state *cli, const char *fname, int flags, int share_mode, uint16_t *pfnum);
-struct tevent_req *cli_close_create(TALLOC_CTX *mem_ctx,
-				    struct event_context *ev,
-				    struct cli_state *cli, uint16_t fnum,
-				    struct tevent_req **psubreq);
-struct tevent_req *cli_close_send(TALLOC_CTX *mem_ctx,
-				  struct event_context *ev,
-				  struct cli_state *cli, uint16_t fnum);
-NTSTATUS cli_close_recv(struct tevent_req *req);
-NTSTATUS cli_close(struct cli_state *cli, uint16_t fnum);
-struct tevent_req *cli_ftruncate_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					uint16_t fnum,
-					uint64_t size);
-NTSTATUS cli_ftruncate_recv(struct tevent_req *req);
-NTSTATUS cli_ftruncate(struct cli_state *cli, uint16_t fnum, uint64_t size);
-NTSTATUS cli_locktype(struct cli_state *cli, uint16_t fnum,
-		      uint32_t offset, uint32_t len,
-		      int timeout, unsigned char locktype);
-bool cli_lock(struct cli_state *cli, uint16_t fnum,
-	      uint32_t offset, uint32_t len, int timeout, enum brl_type lock_type);
-struct tevent_req *cli_unlock_send(TALLOC_CTX *mem_ctx,
-                                struct event_context *ev,
-                                struct cli_state *cli,
-                                uint16_t fnum,
-                                uint64_t offset,
-                                uint64_t len);
-NTSTATUS cli_unlock_recv(struct tevent_req *req);
-NTSTATUS cli_unlock(struct cli_state *cli, uint16_t fnum, uint32_t offset, uint32_t len);
-bool cli_lock64(struct cli_state *cli, uint16_t fnum,
-		uint64_t offset, uint64_t len, int timeout, enum brl_type lock_type);
-struct tevent_req *cli_unlock64_send(TALLOC_CTX *mem_ctx,
-                                struct event_context *ev,
-                                struct cli_state *cli,
-                                uint16_t fnum,
-                                uint64_t offset,
-                                uint64_t len);
-NTSTATUS cli_unlock64_recv(struct tevent_req *req);
-NTSTATUS cli_unlock64(struct cli_state *cli, uint16_t fnum, uint64_t offset, uint64_t len);
-struct tevent_req *cli_posix_lock_send(TALLOC_CTX *mem_ctx,
-                                        struct event_context *ev,
-                                        struct cli_state *cli,
-                                        uint16_t fnum,
-                                        uint64_t offset,
-                                        uint64_t len,
-                                        bool wait_lock,
-                                        enum brl_type lock_type);
-NTSTATUS cli_posix_lock_recv(struct tevent_req *req);
-NTSTATUS cli_posix_lock(struct cli_state *cli, uint16_t fnum,
-			uint64_t offset, uint64_t len,
-			bool wait_lock, enum brl_type lock_type);
-struct tevent_req *cli_posix_unlock_send(TALLOC_CTX *mem_ctx,
-                                        struct event_context *ev,
-                                        struct cli_state *cli,
-                                        uint16_t fnum,
-                                        uint64_t offset,
-                                        uint64_t len);
-NTSTATUS cli_posix_unlock_recv(struct tevent_req *req);
-NTSTATUS cli_posix_unlock(struct cli_state *cli, uint16_t fnum, uint64_t offset, uint64_t len);
-struct tevent_req *cli_getattrE_send(TALLOC_CTX *mem_ctx,
-				struct event_context *ev,
-				struct cli_state *cli,
-                                uint16_t fnum);
-NTSTATUS cli_getattrE_recv(struct tevent_req *req,
-                        uint16_t *attr,
-                        SMB_OFF_T *size,
-                        time_t *change_time,
-                        time_t *access_time,
-                        time_t *write_time);
-NTSTATUS cli_getattrE(struct cli_state *cli,
-			uint16_t fnum,
-			uint16_t *attr,
-			SMB_OFF_T *size,
-			time_t *change_time,
-			time_t *access_time,
-			time_t *write_time);
-struct tevent_req *cli_setattrE_send(TALLOC_CTX *mem_ctx,
-				struct event_context *ev,
-				struct cli_state *cli,
-				uint16_t fnum,
-				time_t change_time,
-				time_t access_time,
-				time_t write_time);
-NTSTATUS cli_setattrE_recv(struct tevent_req *req);
-NTSTATUS cli_setattrE(struct cli_state *cli,
-			uint16_t fnum,
-			time_t change_time,
-			time_t access_time,
-			time_t write_time);
-struct tevent_req *cli_getatr_send(TALLOC_CTX *mem_ctx,
-				struct event_context *ev,
-				struct cli_state *cli,
-				const char *fname);
-NTSTATUS cli_getatr_recv(struct tevent_req *req,
-				uint16_t *attr,
-				SMB_OFF_T *size,
-				time_t *write_time);
-NTSTATUS cli_getatr(struct cli_state *cli,
-			const char *fname,
-			uint16_t *attr,
-			SMB_OFF_T *size,
-			time_t *write_time);
-struct tevent_req *cli_setatr_send(TALLOC_CTX *mem_ctx,
-				struct event_context *ev,
-				struct cli_state *cli,
-				const char *fname,
-				uint16_t attr,
-				time_t mtime);
-NTSTATUS cli_setatr_recv(struct tevent_req *req);
-NTSTATUS cli_setatr(struct cli_state *cli,
-                const char *fname,
-                uint16_t attr,
-                time_t mtime);
-struct tevent_req *cli_chkpath_send(TALLOC_CTX *mem_ctx,
-				  struct event_context *ev,
-				  struct cli_state *cli,
-				  const char *fname);
-NTSTATUS cli_chkpath_recv(struct tevent_req *req);
-NTSTATUS cli_chkpath(struct cli_state *cli, const char *path);
-struct tevent_req *cli_dskattr_send(TALLOC_CTX *mem_ctx,
-				  struct event_context *ev,
-				  struct cli_state *cli);
-NTSTATUS cli_dskattr_recv(struct tevent_req *req, int *bsize, int *total,
-			  int *avail);
-NTSTATUS cli_dskattr(struct cli_state *cli, int *bsize, int *total, int *avail);
-struct tevent_req *cli_ctemp_send(TALLOC_CTX *mem_ctx,
-				struct event_context *ev,
-				struct cli_state *cli,
-				const char *path);
-NTSTATUS cli_ctemp_recv(struct tevent_req *req,
-			TALLOC_CTX *ctx,
-			uint16_t *pfnum,
-			char **outfile);
-NTSTATUS cli_ctemp(struct cli_state *cli,
-			TALLOC_CTX *ctx,
-			const char *path,
-			uint16_t *pfnum,
-			char **out_path);
-NTSTATUS cli_raw_ioctl(struct cli_state *cli, uint16_t fnum, uint32_t code, DATA_BLOB *blob);
-NTSTATUS cli_set_ea_path(struct cli_state *cli, const char *path,
-			 const char *ea_name, const char *ea_val,
-			 size_t ea_len);
-NTSTATUS cli_set_ea_fnum(struct cli_state *cli, uint16_t fnum,
-			 const char *ea_name, const char *ea_val,
-			 size_t ea_len);
-struct tevent_req *cli_get_ea_list_path_send(TALLOC_CTX *mem_ctx,
-					     struct tevent_context *ev,
-					     struct cli_state *cli,
-					     const char *fname);
-NTSTATUS cli_get_ea_list_path_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
-				   size_t *pnum_eas, struct ea_struct **peas);
-NTSTATUS cli_get_ea_list_path(struct cli_state *cli, const char *path,
-		TALLOC_CTX *ctx,
-		size_t *pnum_eas,
-		struct ea_struct **pea_list);
-struct tevent_req *cli_posix_open_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname,
-					int flags,
-					mode_t mode);
-NTSTATUS cli_posix_open_recv(struct tevent_req *req, uint16_t *pfnum);
-NTSTATUS cli_posix_open(struct cli_state *cli, const char *fname,
-			int flags, mode_t mode, uint16_t *fnum);
-struct tevent_req *cli_posix_mkdir_send(TALLOC_CTX *mem_ctx,
-                                        struct event_context *ev,
-                                        struct cli_state *cli,
-                                        const char *fname,
-                                        mode_t mode);
-NTSTATUS cli_posix_mkdir_recv(struct tevent_req *req);
-NTSTATUS cli_posix_mkdir(struct cli_state *cli, const char *fname, mode_t mode);
-
-struct tevent_req *cli_posix_unlink_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname);
-NTSTATUS cli_posix_unlink_recv(struct tevent_req *req);
-NTSTATUS cli_posix_unlink(struct cli_state *cli, const char *fname);
-
-struct tevent_req *cli_posix_rmdir_send(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli,
-					const char *fname);
-NTSTATUS cli_posix_rmdir_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx);
-NTSTATUS cli_posix_rmdir(struct cli_state *cli, const char *fname);
-struct tevent_req *cli_notify_send(TALLOC_CTX *mem_ctx,
-				   struct tevent_context *ev,
-				   struct cli_state *cli, uint16_t fnum,
-				   uint32_t buffer_size,
-				   uint32_t completion_filter, bool recursive);
-NTSTATUS cli_notify_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
-			 uint32_t *pnum_changes,
-			 struct notify_change **pchanges);
-
-/* The following definitions come from libsmb/clifsinfo.c  */
-
-struct tevent_req *cli_unix_extensions_version_send(TALLOC_CTX *mem_ctx,
-						    struct tevent_context *ev,
-						    struct cli_state *cli);
-NTSTATUS cli_unix_extensions_version_recv(struct tevent_req *req,
-					  uint16_t *pmajor, uint16_t *pminor,
-					  uint32_t *pcaplow,
-					  uint32_t *pcaphigh);
-NTSTATUS cli_unix_extensions_version(struct cli_state *cli, uint16 *pmajor,
-				     uint16 *pminor, uint32 *pcaplow,
-				     uint32 *pcaphigh);
-struct tevent_req *cli_set_unix_extensions_capabilities_send(
-	TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct cli_state *cli,
-	uint16_t major, uint16_t minor, uint32_t caplow, uint32_t caphigh);
-NTSTATUS cli_set_unix_extensions_capabilities_recv(struct tevent_req *req);
-NTSTATUS cli_set_unix_extensions_capabilities(struct cli_state *cli,
-					      uint16 major, uint16 minor,
-					      uint32 caplow, uint32 caphigh);
-struct tevent_req *cli_get_fs_attr_info_send(TALLOC_CTX *mem_ctx,
-					     struct tevent_context *ev,
-					     struct cli_state *cli);
-NTSTATUS cli_get_fs_attr_info_recv(struct tevent_req *req, uint32_t *fs_attr);
-NTSTATUS cli_get_fs_attr_info(struct cli_state *cli, uint32_t *fs_attr);
-NTSTATUS cli_get_fs_volume_info(struct cli_state *cli, fstring volume_name,
-				uint32 *pserial_number, time_t *pdate);
-NTSTATUS cli_get_fs_full_size_info(struct cli_state *cli,
-				   uint64_t *total_allocation_units,
-				   uint64_t *caller_allocation_units,
-				   uint64_t *actual_allocation_units,
-				   uint64_t *sectors_per_allocation_unit,
-				   uint64_t *bytes_per_sector);
-NTSTATUS cli_get_posix_fs_info(struct cli_state *cli,
-			       uint32 *optimal_transfer_size,
-			       uint32 *block_size,
-			       uint64_t *total_blocks,
-			       uint64_t *blocks_available,
-			       uint64_t *user_blocks_available,
-			       uint64_t *total_file_nodes,
-			       uint64_t *free_file_nodes,
-			       uint64_t *fs_identifier);
-NTSTATUS cli_raw_ntlm_smb_encryption_start(struct cli_state *cli, 
-				const char *user,
-				const char *pass,
-				const char *domain);
-NTSTATUS cli_gss_smb_encryption_start(struct cli_state *cli);
-NTSTATUS cli_force_encryption(struct cli_state *c,
-			const char *username,
-			const char *password,
-			const char *domain);
-
-/* The following definitions come from libsmb/clikrb5.c  */
-
-bool unwrap_edata_ntstatus(TALLOC_CTX *mem_ctx, 
-			   DATA_BLOB *edata, 
-			   DATA_BLOB *edata_out);
-bool unwrap_pac(TALLOC_CTX *mem_ctx, DATA_BLOB *auth_data, DATA_BLOB *unwrapped_pac_data);
-
-/* The following definitions come from libsmb/clilist.c  */
-
-NTSTATUS cli_list_old(struct cli_state *cli,const char *Mask,uint16 attribute,
-		      NTSTATUS (*fn)(const char *, struct file_info *,
-				 const char *, void *), void *state);
-NTSTATUS cli_list_trans(struct cli_state *cli, const char *mask,
-			uint16_t attribute, int info_level,
-			NTSTATUS (*fn)(const char *mnt, struct file_info *finfo,
-				   const char *mask, void *private_data),
-			void *private_data);
-struct tevent_req *cli_list_send(TALLOC_CTX *mem_ctx,
-				 struct tevent_context *ev,
-				 struct cli_state *cli,
-				 const char *mask,
-				 uint16_t attribute,
-				 uint16_t info_level);
-NTSTATUS cli_list_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
-		       struct file_info **finfo, size_t *num_finfo);
-NTSTATUS cli_list(struct cli_state *cli,const char *Mask,uint16 attribute,
-		  NTSTATUS (*fn)(const char *, struct file_info *, const char *,
-			     void *), void *state);
-
-/* The following definitions come from libsmb/climessage.c  */
-
-struct tevent_req *cli_message_send(TALLOC_CTX *mem_ctx,
-				    struct tevent_context *ev,
-				    struct cli_state *cli,
-				    const char *host, const char *username,
-				    const char *message);
-NTSTATUS cli_message_recv(struct tevent_req *req);
-NTSTATUS cli_message(struct cli_state *cli, const char *host,
-		     const char *username, const char *message);
-
-/* The following definitions come from libsmb/clioplock.c  */
-
-struct tevent_req *cli_oplock_ack_send(TALLOC_CTX *mem_ctx,
-				       struct tevent_context *ev,
-				       struct cli_state *cli,
-				       uint16_t fnum, uint8_t level);
-NTSTATUS cli_oplock_ack_recv(struct tevent_req *req);
-NTSTATUS cli_oplock_ack(struct cli_state *cli, uint16_t fnum, unsigned char level);
-void cli_oplock_handler(struct cli_state *cli, 
-			NTSTATUS (*handler)(struct cli_state *, uint16_t, unsigned char));
-
-/* The following definitions come from libsmb/cliprint.c  */
-
-int cli_print_queue(struct cli_state *cli,
-		    void (*fn)(struct print_job_info *));
-int cli_printjob_del(struct cli_state *cli, int job);
-
-/* The following definitions come from libsmb/cliquota.c  */
-
-NTSTATUS cli_get_quota_handle(struct cli_state *cli, uint16_t *quota_fnum);
-void free_ntquota_list(SMB_NTQUOTA_LIST **qt_list);
-NTSTATUS cli_get_user_quota(struct cli_state *cli, int quota_fnum,
-			    SMB_NTQUOTA_STRUCT *pqt);
-NTSTATUS cli_set_user_quota(struct cli_state *cli, int quota_fnum,
-			    SMB_NTQUOTA_STRUCT *pqt);
-NTSTATUS cli_list_user_quota(struct cli_state *cli, int quota_fnum,
-			     SMB_NTQUOTA_LIST **pqt_list);
-NTSTATUS cli_get_fs_quota_info(struct cli_state *cli, int quota_fnum,
-			       SMB_NTQUOTA_STRUCT *pqt);
-NTSTATUS cli_set_fs_quota_info(struct cli_state *cli, int quota_fnum,
-			       SMB_NTQUOTA_STRUCT *pqt);
-
-/* The following definitions come from libsmb/clireadwrite.c  */
-
-struct tevent_req *cli_read_andx_create(TALLOC_CTX *mem_ctx,
-					struct event_context *ev,
-					struct cli_state *cli, uint16_t fnum,
-					off_t offset, size_t size,
-					struct tevent_req **psmbreq);
-struct tevent_req *cli_read_andx_send(TALLOC_CTX *mem_ctx,
-				      struct event_context *ev,
-				      struct cli_state *cli, uint16_t fnum,
-				      off_t offset, size_t size);
-NTSTATUS cli_read_andx_recv(struct tevent_req *req, ssize_t *received,
-			    uint8_t **rcvbuf);
-struct tevent_req *cli_pull_send(TALLOC_CTX *mem_ctx,
-				 struct event_context *ev,
-				 struct cli_state *cli,
-				 uint16_t fnum, off_t start_offset,
-				 SMB_OFF_T size, size_t window_size,
-				 NTSTATUS (*sink)(char *buf, size_t n,
-						  void *priv),
-				 void *priv);
-NTSTATUS cli_pull_recv(struct tevent_req *req, SMB_OFF_T *received);
-NTSTATUS cli_pull(struct cli_state *cli, uint16_t fnum,
-		  off_t start_offset, SMB_OFF_T size, size_t window_size,
-		  NTSTATUS (*sink)(char *buf, size_t n, void *priv),
-		  void *priv, SMB_OFF_T *received);
-ssize_t cli_read(struct cli_state *cli, uint16_t fnum, char *buf,
-		 off_t offset, size_t size);
-ssize_t cli_write(struct cli_state *cli,
-    	         uint16_t fnum, uint16 write_mode,
-		 const char *buf, off_t offset, size_t size);
-NTSTATUS cli_smbwrite(struct cli_state *cli, uint16_t fnum, char *buf,
-		      off_t offset, size_t size1, size_t *ptotal);
-struct tevent_req *cli_write_andx_create(TALLOC_CTX *mem_ctx,
-					 struct event_context *ev,
-					 struct cli_state *cli, uint16_t fnum,
-					 uint16_t mode, const uint8_t *buf,
-					 off_t offset, size_t size,
-					 struct tevent_req **reqs_before,
-					 int num_reqs_before,
-					 struct tevent_req **psmbreq);
-struct tevent_req *cli_write_andx_send(TALLOC_CTX *mem_ctx,
-				       struct event_context *ev,
-				       struct cli_state *cli, uint16_t fnum,
-				       uint16_t mode, const uint8_t *buf,
-				       off_t offset, size_t size);
-NTSTATUS cli_write_andx_recv(struct tevent_req *req, size_t *pwritten);
-
-struct tevent_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
-				 struct cli_state *cli,
-				 uint16_t fnum, uint16_t mode,
-				 off_t start_offset, size_t window_size,
-				 size_t (*source)(uint8_t *buf, size_t n,
-						  void *priv),
-				 void *priv);
-NTSTATUS cli_push_recv(struct tevent_req *req);
-NTSTATUS cli_push(struct cli_state *cli, uint16_t fnum, uint16_t mode,
-		  off_t start_offset, size_t window_size,
-		  size_t (*source)(uint8_t *buf, size_t n, void *priv),
-		  void *priv);
-
-/* The following definitions come from libsmb/clisecdesc.c  */
-
-struct security_descriptor *cli_query_secdesc(struct cli_state *cli, uint16_t fnum,
-			    TALLOC_CTX *mem_ctx);
-NTSTATUS cli_set_secdesc(struct cli_state *cli, uint16_t fnum,
-			 struct security_descriptor *sd);
-
 /* The following definitions come from libsmb/clispnego.c  */
 
 DATA_BLOB spnego_gen_negTokenInit(TALLOC_CTX *ctx,
@@ -1836,57 +952,6 @@ DATA_BLOB spnego_gen_auth_response_and_mic(TALLOC_CTX *ctx,
 bool spnego_mech_list_blob(TALLOC_CTX *mem_ctx,
 			   char **oid_list, DATA_BLOB *data);
 
-/* The following definitions come from libsmb/clistr.c  */
-
-size_t clistr_push_fn(struct cli_state *cli,
-			void *dest,
-			const char *src,
-			int dest_len,
-			int flags);
-size_t clistr_pull_fn(const char *inbuf,
-			char *dest,
-			const void *src,
-			int dest_len,
-			int src_len,
-			int flags);
-size_t clistr_pull_talloc(TALLOC_CTX *ctx,
-			  const char *base,
-			  uint16_t flags2,
-			  char **pp_dest,
-			  const void *src,
-			  int src_len,
-			  int flags);
-size_t clistr_align_out(struct cli_state *cli, const void *p, int flags);
-
-/* The following definitions come from libsmb/clitrans.c  */
-
-struct tevent_req *cli_trans_send(
-	TALLOC_CTX *mem_ctx, struct event_context *ev,
-	struct cli_state *cli, uint8_t cmd,
-	const char *pipe_name, uint16_t fid, uint16_t function, int flags,
-	uint16_t *setup, uint8_t num_setup, uint8_t max_setup,
-	uint8_t *param, uint32_t num_param, uint32_t max_param,
-	uint8_t *data, uint32_t num_data, uint32_t max_data);
-NTSTATUS cli_trans_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
-			uint16_t *recv_flags2,
-			uint16_t **setup, uint8_t min_setup,
-			uint8_t *num_setup,
-			uint8_t **param, uint32_t min_param,
-			uint32_t *num_param,
-			uint8_t **data, uint32_t min_data,
-			uint32_t *num_data);
-NTSTATUS cli_trans(TALLOC_CTX *mem_ctx, struct cli_state *cli,
-		   uint8_t trans_cmd,
-		   const char *pipe_name, uint16_t fid, uint16_t function,
-		   int flags,
-		   uint16_t *setup, uint8_t num_setup, uint8_t max_setup,
-		   uint8_t *param, uint32_t num_param, uint32_t max_param,
-		   uint8_t *data, uint32_t num_data, uint32_t max_data,
-		   uint16_t *recv_flags2,
-		   uint16_t **rsetup, uint8_t min_rsetup, uint8_t *num_rsetup,
-		   uint8_t **rparam, uint32_t min_rparam, uint32_t *num_rparam,
-		   uint8_t **rdata, uint32_t min_rdata, uint32_t *num_rdata);
-
 /* The following definitions come from libsmb/conncache.c  */
 
 NTSTATUS check_negative_conn_cache( const char *domain, const char *server);
@@ -1909,9 +974,6 @@ NTSTATUS dsgetdcname(TALLOC_CTX *mem_ctx,
 /* The following definitions come from libsmb/errormap.c  */
 
 NTSTATUS dos_to_ntstatus(uint8 eclass, uint32 ecode);
-void ntstatus_to_dos(NTSTATUS ntstatus, uint8 *eclass, uint32 *ecode);
-NTSTATUS werror_to_ntstatus(WERROR error);
-WERROR ntstatus_to_werror(NTSTATUS error);
 NTSTATUS map_nt_error_from_gss(uint32 gss_maj, uint32 minor);
 
 /* The following definitions come from libsmb/namecache.c  */
@@ -1974,13 +1036,29 @@ NTSTATUS name_query(const char *name, int name_type,
 		    TALLOC_CTX *mem_ctx,
 		    struct sockaddr_storage **addrs,
 		    int *num_addrs, uint8_t *flags);
+struct tevent_req *name_resolve_bcast_send(TALLOC_CTX *mem_ctx,
+					   struct tevent_context *ev,
+					   const char *name,
+					   int name_type);
+NTSTATUS name_resolve_bcast_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+				 struct sockaddr_storage **addrs,
+				 int *num_addrs);
 NTSTATUS name_resolve_bcast(const char *name,
 			int name_type,
-			struct ip_service **return_iplist,
+			TALLOC_CTX *mem_ctx,
+			struct sockaddr_storage **return_iplist,
 			int *return_count);
+struct tevent_req *resolve_wins_send(TALLOC_CTX *mem_ctx,
+				     struct tevent_context *ev,
+				     const char *name,
+				     int name_type);
+NTSTATUS resolve_wins_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+			   struct sockaddr_storage **addrs,
+			   int *num_addrs, uint8_t *flags);
 NTSTATUS resolve_wins(const char *name,
 		int name_type,
-		struct ip_service **return_iplist,
+		TALLOC_CTX *mem_ctx,
+		struct sockaddr_storage **return_iplist,
 		int *return_count);
 NTSTATUS internal_resolve_name(const char *name,
 			        int name_type,
@@ -2016,14 +1094,6 @@ bool get_dc_name(const char *domain,
 		fstring srv_name,
 		struct sockaddr_storage *ss_out);
 
-/* The following definitions come from libsmb/nterr.c  */
-
-const char *nt_errstr(NTSTATUS nt_code);
-const char *get_friendly_nt_error_msg(NTSTATUS nt_code);
-const char *get_nt_error_c_code(NTSTATUS nt_code);
-NTSTATUS nt_status_string_to_code(const char *nt_status_str);
-NTSTATUS nt_status_squash(NTSTATUS nt_status);
-
 /* The following definitions come from libsmb/ntlmssp.c  */
 struct ntlmssp_state;
 NTSTATUS ntlmssp_set_username(struct ntlmssp_state *ntlmssp_state, const char *user) ;
@@ -2064,38 +1134,6 @@ bool netsamlogon_cache_store(const char *username, struct netr_SamInfo3 *info3);
 struct netr_SamInfo3 *netsamlogon_cache_get(TALLOC_CTX *mem_ctx, const struct dom_sid *user_sid);
 bool netsamlogon_cache_have(const struct dom_sid *user_sid);
 
-/* The following definitions come from libsmb/smb_seal.c  */
-
-NTSTATUS get_enc_ctx_num(const uint8_t *buf, uint16 *p_enc_ctx_num);
-bool common_encryption_on(struct smb_trans_enc_state *es);
-NTSTATUS common_ntlm_decrypt_buffer(struct ntlmssp_state *ntlmssp_state, char *buf);
-NTSTATUS common_ntlm_encrypt_buffer(struct ntlmssp_state *ntlmssp_state,
-				uint16 enc_ctx_num,
-				char *buf,
-				char **ppbuf_out);
-NTSTATUS common_encrypt_buffer(struct smb_trans_enc_state *es, char *buffer, char **buf_out);
-NTSTATUS common_decrypt_buffer(struct smb_trans_enc_state *es, char *buf);
-void common_free_encryption_state(struct smb_trans_enc_state **pp_es);
-void common_free_enc_buffer(struct smb_trans_enc_state *es, char *buf);
-bool cli_encryption_on(struct cli_state *cli);
-void cli_free_encryption_context(struct cli_state *cli);
-void cli_free_enc_buffer(struct cli_state *cli, char *buf);
-NTSTATUS cli_decrypt_message(struct cli_state *cli);
-NTSTATUS cli_encrypt_message(struct cli_state *cli, char *buf, char **buf_out);
-
-/* The following definitions come from libsmb/clisigning.c  */
-
-bool cli_simple_set_signing(struct cli_state *cli,
-			    const DATA_BLOB user_session_key,
-			    const DATA_BLOB response);
-bool cli_temp_set_signing(struct cli_state *cli);
-void cli_calculate_sign_mac(struct cli_state *cli, char *buf, uint32_t *seqnum);
-bool cli_check_sign_mac(struct cli_state *cli, const char *buf, uint32_t seqnum);
-bool client_is_signing_on(struct cli_state *cli);
-bool client_is_signing_allowed(struct cli_state *cli);
-bool client_is_signing_mandatory(struct cli_state *cli);
-void cli_set_signing_negotiated(struct cli_state *cli);
-
 /* The following definitions come from libsmb/smberr.c  */
 
 const char *smb_dos_err_name(uint8 e_class, uint16 num);
@@ -2131,22 +1169,15 @@ bool enumerate_domain_trusts( TALLOC_CTX *mem_ctx, const char *domain,
 				     struct dom_sid **sids );
 NTSTATUS change_trust_account_password( const char *domain, const char *remote_machine);
 
-/* The following definitions come from modules/vfs_default.c  */
-
-ssize_t vfswrap_llistxattr(struct vfs_handle_struct *handle, const char *path, char *list, size_t size);
-ssize_t vfswrap_flistxattr(struct vfs_handle_struct *handle, struct files_struct *fsp, char *list, size_t size);
-NTSTATUS vfs_default_init(void);
-
 /* The following definitions come from param/loadparm.c  */
 
-char *lp_smb_ports(void);
-char *lp_dos_charset(void);
-char *lp_unix_charset(void);
-char *lp_display_charset(void);
+const char *lp_smb_ports(void);
+const char *lp_dos_charset(void);
+const char *lp_unix_charset(void);
 char *lp_logfile(void);
 char *lp_configfile(void);
-char *lp_smb_passwd_file(void);
-char *lp_private_dir(void);
+const char *lp_smb_passwd_file(void);
+const char *lp_private_dir(void);
 char *lp_serverstring(void);
 int lp_printcap_cache_time(void);
 char *lp_addport_cmd(void);
@@ -2154,14 +1185,14 @@ char *lp_enumports_cmd(void);
 char *lp_addprinter_cmd(void);
 char *lp_deleteprinter_cmd(void);
 char *lp_os2_driver_map(void);
-char *lp_lockdir(void);
-char *lp_statedir(void);
-char *lp_cachedir(void);
-char *lp_piddir(void);
+const char *lp_lockdir(void);
+const char *lp_statedir(void);
+const char *lp_cachedir(void);
+const char *lp_piddir(void);
 char *lp_mangling_method(void);
 int lp_mangle_prefix(void);
-char *lp_utmpdir(void);
-char *lp_wtmpdir(void);
+const char *lp_utmpdir(void);
+const char *lp_wtmpdir(void);
 bool lp_utmp(void);
 char *lp_rootdir(void);
 char *lp_defaultservice(void);
@@ -2171,9 +1202,13 @@ char *lp_set_quota_command(void);
 char *lp_auto_services(void);
 char *lp_passwd_program(void);
 char *lp_passwd_chat(void);
-char *lp_passwordserver(void);
-char *lp_name_resolve_order(void);
-char *lp_realm(void);
+const char *lp_passwordserver(void);
+const char *lp_name_resolve_order(void);
+const char *lp_netbios_scope(void);
+const char *lp_netbios_name(void);
+const char *lp_workgroup(void);
+const char *lp_realm(void);
+const char *lp_dnsdomain(void);
 const char *lp_afs_username_map(void);
 int lp_afs_token_lifetime(void);
 char *lp_log_nt_token_command(void);
@@ -2316,7 +1351,7 @@ bool lp_send_spnego_principal(void);
 bool lp_hostname_lookups(void);
 bool lp_change_notify(const struct share_params *p );
 bool lp_kernel_change_notify(const struct share_params *p );
-char * lp_dedicated_keytab_file(void);
+const char * lp_dedicated_keytab_file(void);
 int lp_kerberos_method(void);
 bool lp_defer_sharing_violations(void);
 bool lp_enable_privileges(void);
@@ -2552,9 +1587,11 @@ bool dump_a_parameter(int snum, char *parm_name, FILE * f, bool isGlobal);
 struct parm_struct *lp_get_parameter(const char *param_name);
 struct parm_struct *lp_next_parameter(int snum, int *i, int allparameters);
 bool lp_snum_ok(int iService);
-void lp_add_one_printer(const char *name, const char *comment, void *pdata);
+void lp_add_one_printer(const char *name, const char *comment,
+			const char *location, void *pdata);
 bool lp_loaded(void);
-void lp_killunused(bool (*snumused) (int));
+void lp_killunused(struct smbd_server_connection *sconn,
+		   bool (*snumused) (struct smbd_server_connection *, int));
 void lp_kill_all_services(void);
 void lp_killservice(int iServiceIn);
 const char* server_role_str(uint32 role);
@@ -2570,7 +1607,7 @@ enum usershare_err parse_usershare_file(TALLOC_CTX *ctx,
 			struct security_descriptor **ppsd,
 			bool *pallow_guest);
 int load_usershare_service(const char *servicename);
-int load_usershare_shares(void);
+int load_usershare_shares(struct smbd_server_connection *sconn);
 void gfree_loadparm(void);
 void lp_set_in_client(bool b);
 bool lp_is_in_client(void);
@@ -2592,10 +1629,6 @@ int lp_servicenumber(const char *pszServiceName);
 bool share_defined(const char *service_name);
 struct share_params *get_share_params(TALLOC_CTX *mem_ctx,
 				      const char *sharename);
-struct share_iterator *share_list_all(TALLOC_CTX *mem_ctx);
-struct share_params *next_share(struct share_iterator *list);
-struct share_params *next_printer(struct share_iterator *list);
-struct share_params *snum2params_static(int snum);
 const char *volume_label(int snum);
 bool lp_domain_master(void);
 bool lp_domain_master_true_or_auto(void);
@@ -2603,8 +1636,6 @@ bool lp_preferred_master(void);
 void lp_remove_service(int snum);
 void lp_copy_service(int snum, const char *new_name);
 int lp_default_server_announce(void);
-int lp_major_announce_version(void);
-int lp_minor_announce_version(void);
 void lp_set_name_resolve_order(const char *new_order);
 const char *lp_printername(int snum);
 void lp_set_logfile(const char *name);
@@ -2613,6 +1644,7 @@ const char *lp_printcapname(void);
 bool lp_disable_spoolss( void );
 void lp_set_spoolss_state( uint32 state );
 uint32 lp_get_spoolss_state( void );
+struct smb_signing_state;
 bool lp_use_sendfile(int snum, struct smb_signing_state *signing_state);
 void set_use_sendfile(int snum, bool val);
 void set_store_dos_attributes(int snum, bool val);
@@ -2625,7 +1657,11 @@ int lp_min_receive_file_size(void);
 char* lp_perfcount_module(void);
 void lp_set_passdb_backend(const char *backend);
 void widelinks_warning(int snum);
-char *lp_ncalrpc_dir(void);
+const char *lp_ncalrpc_dir(void);
+
+/* The following definitions come from param/loadparm_ctx.c  */
+
+const struct loadparm_s3_context *loadparm_s3_context(void);
 
 /* The following definitions come from param/loadparm_server_role.c  */
 
@@ -2637,145 +1673,6 @@ void set_server_role(void);
 uint32 get_int_param( const char* param );
 char* get_string_param( const char* param );
 
-/* The following definitions come from librpc/rpc/rpc_common.c  */
-struct ndr_interface_table;
-bool smb_register_ndr_interface(const struct ndr_interface_table *interface);
-const struct ndr_interface_table *get_iface_from_syntax(
-	const struct ndr_syntax_id *syntax);
-const char *get_pipe_name_from_syntax(TALLOC_CTX *mem_ctx,
-                                     const struct ndr_syntax_id *syntax);
-
-/* The following definitions come from rpc_server/rpc_ncacn_np.c  */
-struct auth_serversupplied_info;
-struct pipes_struct *make_internal_rpc_pipe_p(TALLOC_CTX *mem_ctx,
-					      const struct ndr_syntax_id *syntax,
-					      struct client_address *client_id,
-					      const struct auth_serversupplied_info *session_info,
-					      struct messaging_context *msg_ctx);
-struct dcerpc_binding_handle;
-NTSTATUS rpcint_binding_handle(TALLOC_CTX *mem_ctx,
-			       const struct ndr_interface_table *ndr_table,
-			       struct client_address *client_id,
-			       const struct auth_serversupplied_info *session_info,
-			       struct messaging_context *msg_ctx,
-			       struct dcerpc_binding_handle **binding_handle);
-NTSTATUS rpc_pipe_open_internal(TALLOC_CTX *mem_ctx,
-				const struct ndr_syntax_id *abstract_syntax,
-				const struct auth_serversupplied_info *serversupplied_info,
-				struct client_address *client_id,
-				struct messaging_context *msg_ctx,
-				struct rpc_pipe_client **presult);
-NTSTATUS rpc_pipe_open_interface(TALLOC_CTX *mem_ctx,
-				 const struct ndr_syntax_id *syntax,
-				 const struct auth_serversupplied_info *session_info,
-				 struct client_address *client_id,
-				 struct messaging_context *msg_ctx,
-				 struct rpc_pipe_client **cli_pipe);
-
-/* The following definitions come from rpc_server/rpc_handles.c  */
-
-size_t num_pipe_handles(struct pipes_struct *p);
-bool init_pipe_handles(struct pipes_struct *p, const struct ndr_syntax_id *syntax);
-bool create_policy_hnd(struct pipes_struct *p, struct policy_handle *hnd, void *data_ptr);
-bool find_policy_by_hnd(struct pipes_struct *p, const struct policy_handle *hnd,
-			void **data_p);
-bool close_policy_hnd(struct pipes_struct *p, struct policy_handle *hnd);
-void close_policy_by_pipe(struct pipes_struct *p);
-bool pipe_access_check(struct pipes_struct *p);
-
-void *_policy_handle_create(struct pipes_struct *p, struct policy_handle *hnd,
-			    uint32_t access_granted, size_t data_size,
-			    const char *type, NTSTATUS *pstatus);
-#define policy_handle_create(_p, _hnd, _access, _type, _pstatus) \
-	(_type *)_policy_handle_create((_p), (_hnd), (_access), sizeof(_type), #_type, \
-				       (_pstatus))
-
-void *_policy_handle_find(struct pipes_struct *p,
-			  const struct policy_handle *hnd,
-			  uint32_t access_required, uint32_t *paccess_granted,
-			  const char *name, const char *location,
-			  NTSTATUS *pstatus);
-#define policy_handle_find(_p, _hnd, _access_required, _access_granted, _type, _pstatus) \
-	(_type *)_policy_handle_find((_p), (_hnd), (_access_required), \
-				     (_access_granted), #_type, __location__, (_pstatus))
-
-
-/* The following definitions come from rpc_server/srv_rpc_register.c  */
-
-struct rpc_srv_callbacks {
-	bool (*init)(void *private_data);
-	bool (*shutdown)(void *private_data);
-	void *private_data;
-};
-
-struct api_struct;
-NTSTATUS rpc_srv_register(int version, const char *clnt,
-			  const char *srv,
-			  const struct ndr_interface_table *iface,
-			  const struct api_struct *cmds, int size,
-			  const struct rpc_srv_callbacks *rpc_srv_cb);
-
-NTSTATUS rpc_srv_unregister(const struct ndr_interface_table *iface);
-
-/* The following definitions come from rpc_server/srv_pipe.c  */
-struct ncacn_packet;
-bool create_next_pdu(struct pipes_struct *p);
-bool api_pipe_bind_auth3(struct pipes_struct *p, struct ncacn_packet *pkt);
-bool setup_fault_pdu(struct pipes_struct *p, NTSTATUS status);
-NTSTATUS rpc_pipe_register_commands(int version, const char *clnt,
-				    const char *srv,
-				    const struct ndr_syntax_id *interface,
-				    const struct api_struct *cmds, int size);
-bool is_known_pipename(const char *cli_filename, struct ndr_syntax_id *syntax);
-
-/* The following definitions come from rpc_server/srv_pipe_hnd.c  */
-
-struct pipes_struct *get_first_internal_pipe(void);
-struct pipes_struct *get_next_internal_pipe(struct pipes_struct *p);
-bool check_open_pipes(void);
-int close_internal_rpc_pipe_hnd(struct pipes_struct *p);
-
-bool fsp_is_np(struct files_struct *fsp);
-struct tsocket_address;
-NTSTATUS np_open(TALLOC_CTX *mem_ctx, const char *name,
-		 const struct tsocket_address *local_address,
-		 const struct tsocket_address *remote_address,
-		 struct client_address *client_id,
-		 struct auth_serversupplied_info *session_info,
-		 struct messaging_context *msg_ctx,
-		 struct fake_file_handle **phandle);
-bool np_read_in_progress(struct fake_file_handle *handle);
-struct tevent_req *np_write_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
-				 struct fake_file_handle *handle,
-				 const uint8_t *data, size_t len);
-NTSTATUS np_write_recv(struct tevent_req *req, ssize_t *pnwritten);
-struct tevent_req *np_read_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
-				struct fake_file_handle *handle,
-				uint8_t *data, size_t len);
-NTSTATUS np_read_recv(struct tevent_req *req, ssize_t *nread,
-		      bool *is_data_outstanding);
-
-ssize_t process_incoming_data(struct pipes_struct *p, char *data, size_t n);
-
-/* The following definitions come from rpc_server/srv_spoolss_nt.c  */
-void srv_spoolss_cleanup(void);
-
-void do_drv_upgrade_printer(struct messaging_context *msg,
-			    void *private_data,
-			    uint32_t msg_type,
-			    struct server_id server_id,
-			    DATA_BLOB *data);
-void update_monitored_printq_cache(struct messaging_context *msg_ctx);
-
-/* The following definitions come from rpc_server/srv_srvsvc_nt.c  */
-
-char *valid_share_pathname(TALLOC_CTX *ctx, const char *dos_pathname);
-
-/* The following definitions come from rpc_server/srv_svcctl_nt.c  */
-
-bool init_service_op_table( void );
-bool shutdown_service_op_table(void);
-
 /* The following definitions come from lib/server_contexts.c  */
 struct tevent_context *server_event_context(void);
 void server_event_context_free(void);
@@ -2800,13 +1697,6 @@ int sessionid_traverse_read(int (*fn)(const char *key,
 char *stdin_new_passwd( void);
 char *get_pass( const char *prompt, bool stdin_get);
 
-/* The following definitions come from winbindd/nss_info.c  */
-
-
-/* The following definitions come from winbindd/nss_info_template.c  */
-
-NTSTATUS nss_info_template_init( void );
-
 /* The following definitions come from lib/avahi.c */
 
 struct AvahiPoll *tevent_avahi_poll(TALLOC_CTX *mem_ctx,
@@ -2837,7 +1727,7 @@ NTSTATUS smbsock_connect_recv(struct tevent_req *req, int *sock,
 NTSTATUS smbsock_connect(const struct sockaddr_storage *addr, uint16_t port,
 			 const char *called_name, int called_type,
 			 const char *calling_name, int calling_type,
-			 int *pfd, uint16_t *ret_port);
+			 int *pfd, uint16_t *ret_port, int sec_timeout);
 
 struct tevent_req *smbsock_any_connect_send(TALLOC_CTX *mem_ctx,
 					    struct tevent_context *ev,
@@ -2856,6 +1746,7 @@ NTSTATUS smbsock_any_connect(const struct sockaddr_storage *addrs,
 			     int *calling_types,
 			     size_t num_addrs,
 			     uint16_t port,
+			     int sec_timeout,
 			     int *pfd, size_t *chosen_index,
 			     uint16_t *chosen_port);
 
@@ -2909,7 +1800,7 @@ void unbecome_root(void);
 /* The following definitions come from lib/dummysmbd.c */
 
 int find_service(TALLOC_CTX *ctx, const char *service_in, char **p_service_out);
-bool conn_snum_used(int snum);
+bool conn_snum_used(struct smbd_server_connection *sconn, int snum);
 void cancel_pending_lock_requests_by_fid(files_struct *fsp,
 			struct byte_range_lock *br_lck,
 			enum file_close_type close_type);
@@ -2918,7 +1809,7 @@ void send_stat_cache_delete_message(struct messaging_context *msg_ctx,
 NTSTATUS can_delete_directory(struct connection_struct *conn,
 				const char *dirname);
 bool change_to_root_user(void);
-struct event_context *smbd_event_context(void);
+void smbd_set_server_fd(int fd);
 void contend_level2_oplocks_begin(files_struct *fsp,
 				  enum level2_contention_type type);
 void contend_level2_oplocks_end(files_struct *fsp,
diff --git a/source3/include/pthreadpool.h b/source3/include/pthreadpool.h
deleted file mode 100644
index 7ef7ddf4192..00000000000
--- a/source3/include/pthreadpool.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Unix SMB/CIFS implementation.
- * threadpool implementation based on pthreads
- * Copyright (C) Volker Lendecke 2009
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __PTHREADPOOL_H__
-#define __PTHREADPOOL_H__
-
-struct pthreadpool;
-
-int pthreadpool_init(unsigned max_threads, struct pthreadpool **presult);
-int pthreadpool_destroy(struct pthreadpool *pool);
-
-/*
- * Add a job to a pthreadpool.
- */
-int pthreadpool_add_job(struct pthreadpool *pool, int job_id,
-			void (*fn)(void *private_data), void *private_data);
-
-/*
- * Get the signalling fd out of a thread pool. This fd will become readable
- * when a job is finished. The job that finished can be retrieved via
- * pthreadpool_finished_job().
- */
-int pthreadpool_sig_fd(struct pthreadpool *pool);
-int pthreadpool_finished_job(struct pthreadpool *pool);
-
-#endif
diff --git a/source3/include/smb.h b/source3/include/smb.h
index 8ebde2f4adc..589dfd78c3c 100644
--- a/source3/include/smb.h
+++ b/source3/include/smb.h
@@ -27,6 +27,7 @@
 #define _SMB_H
 
 #include "libcli/smb/smb_common.h"
+#include "libds/common/roles.h"
 
 /* logged when starting the various Samba daemons */
 #define COPYRIGHT_STARTUP_MESSAGE	"Copyright Andrew Tridgell and the Samba Team 1992-2011"
@@ -85,14 +86,6 @@ enum smb_read_errors {
 
 #define DIR_STRUCT_SIZE 43
 
-/* these define the attribute byte as seen by DOS */
-#define aRONLY (1L<<0)		/* 0x01 */
-#define aHIDDEN (1L<<1)		/* 0x02 */
-#define aSYSTEM (1L<<2)		/* 0x04 */
-#define aVOLID (1L<<3)		/* 0x08 */
-#define aDIR (1L<<4)		/* 0x10 */
-#define aARCH (1L<<5)		/* 0x20 */
-
 /* deny modes */
 #define DENY_DOS 0
 #define DENY_ALL 1
@@ -404,10 +397,6 @@ struct share_params {
 	int service;
 };
 
-struct share_iterator {
-	int next_id;
-};
-
 typedef struct connection_struct {
 	struct connection_struct *next, *prev;
 	struct smbd_server_connection *sconn; /* can be NULL */
@@ -1084,26 +1073,6 @@ struct bitmap {
 #define FILE_SHARE_WRITE 2
 #define FILE_SHARE_DELETE 4
 
-/* FileAttributesField */
-#define FILE_ATTRIBUTE_READONLY		0x001L
-#define FILE_ATTRIBUTE_HIDDEN		0x002L
-#define FILE_ATTRIBUTE_SYSTEM		0x004L
-#define FILE_ATTRIBUTE_DIRECTORY	0x010L
-#define FILE_ATTRIBUTE_ARCHIVE		0x020L
-#define FILE_ATTRIBUTE_NORMAL		0x080L
-#define FILE_ATTRIBUTE_TEMPORARY	0x100L
-#define FILE_ATTRIBUTE_SPARSE		0x200L
-#define FILE_ATTRIBUTE_REPARSE_POINT    0x400L
-#define FILE_ATTRIBUTE_COMPRESSED	0x800L
-#define FILE_ATTRIBUTE_OFFLINE          0x1000L
-#define FILE_ATTRIBUTE_NONINDEXED	0x2000L
-#define FILE_ATTRIBUTE_ENCRYPTED        0x4000L
-#define SAMBA_ATTRIBUTES_MASK		(FILE_ATTRIBUTE_READONLY|\
-					FILE_ATTRIBUTE_HIDDEN|\
-					FILE_ATTRIBUTE_SYSTEM|\
-					FILE_ATTRIBUTE_DIRECTORY|\
-					FILE_ATTRIBUTE_ARCHIVE)
-
 /* Flags - combined with attributes. */
 #define FILE_FLAG_WRITE_THROUGH    0x80000000L
 #define FILE_FLAG_NO_BUFFERING     0x20000000L
@@ -1232,7 +1201,7 @@ struct bitmap {
 
 
 /* where to find the base of the SMB packet proper */
-#define smb_base(buf) (((char *)(buf))+4)
+#define smb_base(buf) (((const char *)(buf))+4)
 
 /* we don't allow server strings to be longer than 48 characters as
    otherwise NT will not honour the announce packets */
@@ -1267,8 +1236,8 @@ char *strdup(char *s);
  *        This may change again in Samba-3.0 after further testing. JHT
  */
  
-#define DEFAULT_MAJOR_VERSION 0x04
-#define DEFAULT_MINOR_VERSION 0x09
+#define SAMBA_MAJOR_NBT_ANNOUNCE_VERSION 0x04
+#define SAMBA_MINOR_NBT_ANNOUNCE_VERSION 0x09
 
 /* Browser Election Values */
 #define BROWSER_ELECTION_VERSION	0x010f
@@ -1323,17 +1292,6 @@ enum protocol_types {
 	PROTOCOL_SMB2
 };
 
-/* security levels */
-enum security_types {SEC_SHARE,SEC_USER,SEC_SERVER,SEC_DOMAIN,SEC_ADS};
-
-/* server roles */
-enum server_types {
-	ROLE_STANDALONE,
-	ROLE_DOMAIN_MEMBER,
-	ROLE_DOMAIN_BDC,
-	ROLE_DOMAIN_PDC
-};
-
 /* printing types */
 enum printing_types {PRINT_BSD,PRINT_SYSV,PRINT_AIX,PRINT_HPUX,
 		     PRINT_QNX,PRINT_PLP,PRINT_LPRNG,PRINT_SOFTQ,
@@ -1584,13 +1542,6 @@ typedef struct user_struct {
 	struct auth_ntlmssp_state *auth_ntlmssp_state;
 } user_struct;
 
-struct unix_error_map {
-	int unix_error;
-	int dos_class;
-	int dos_code;
-	NTSTATUS nt_error;
-};
-
 /*
    Do you want session setups at user level security with a invalid
    password to be rejected or allowed in as guest? WinNT rejects them
@@ -1736,4 +1687,20 @@ struct deferred_open_record;
 /* Used inside aio code. */
 struct aio_extra;
 
+/*
+ * Reasons for cache flush.
+ */
+
+enum flush_reason_enum {
+    SEEK_FLUSH,
+    READ_FLUSH,
+    WRITE_FLUSH,
+    READRAW_FLUSH,
+    OPLOCK_RELEASE_FLUSH,
+    CLOSE_FLUSH,
+    SYNC_FLUSH,
+    SIZECHANGE_FLUSH,
+    /* NUM_FLUSH_REASONS must remain the last value in the enumeration. */
+    NUM_FLUSH_REASONS};
+
 #endif /* _SMB_H */
diff --git a/source3/include/smb_krb5.h b/source3/include/smb_krb5.h
index d87dc79ff9f..bc9996c541d 100644
--- a/source3/include/smb_krb5.h
+++ b/source3/include/smb_krb5.h
@@ -1,3 +1,25 @@
+/*
+   Unix SMB/CIFS implementation.
+   simple kerberos5 routines for active directory
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Luke Howard 2002-2003
+   Copyright (C) Andrew Bartlett <abartlet@samba.org> 2005
+   Copyright (C) Guenther Deschner 2005-2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef _HEADER_smb_krb5_h
 #define _HEADER_smb_krb5_h
 
@@ -14,14 +36,6 @@
 
 #include "libcli/auth/krb5_wrap.h"
 
-#if HAVE_GSSAPI_GSSAPI_H
-#include <gssapi/gssapi.h>
-#elif HAVE_GSSAPI_GSSAPI_GENERIC_H
-#include <gssapi/gssapi_generic.h>
-#elif HAVE_GSSAPI_H
-#include <gssapi.h>
-#endif
-
 #ifndef KRB5_ADDR_NETBIOS
 #define KRB5_ADDR_NETBIOS 0x14
 #endif
diff --git a/source3/include/smb_ldap.h b/source3/include/smb_ldap.h
index 45e586859db..594f0158629 100644
--- a/source3/include/smb_ldap.h
+++ b/source3/include/smb_ldap.h
@@ -1,3 +1,25 @@
+/*
+   Unix SMB/CIFS implementation.
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Remus Koos 2001
+   Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2002
+   Copyright (C) Guenther Deschner 2005
+   Copyright (C) Gerald Carter 2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #ifndef _SMB_LDAP_H
 #define _SMB_LDAP_H
 
@@ -22,6 +44,20 @@ typedef int ber_int_t;
 #ifndef LDAP_CONST
 #define LDAP_CONST const
 #endif
+
+#ifdef HAVE_LDAP_PVT_H
+#include <ldap_pvt.h>
+#endif /* HAVE_LDAP_PVT_H */
+
+#ifdef HAVE_LDAP_INIT_FD
+int ldap_init_fd(ber_socket_t fd, int proto, char *uri, LDAP **ldp);
+#endif
+
+/* function declarations not included in proto.h */
+LDAP *ldap_open_with_timeout(const char *server,
+			     struct sockaddr_storage *ss,
+			     int port, unsigned int to);
+
 #ifndef LDAP_OPT_SUCCESS
 #define LDAP_OPT_SUCCESS 0
 #endif
@@ -36,9 +72,6 @@ typedef int ber_int_t;
 #define LDAPS_PORT 636
 #endif
 
-/* function declarations not included in proto.h */
-LDAP *ldap_open_with_timeout(const char *server, int port, unsigned int to);
-
 #endif /* HAVE_LDAP_H */
 
 #ifndef HAVE_LDAP
diff --git a/source3/include/smb_macros.h b/source3/include/smb_macros.h
index 93bda975bf9..37ea1b42925 100644
--- a/source3/include/smb_macros.h
+++ b/source3/include/smb_macros.h
@@ -28,11 +28,11 @@
 #define BITSETW(ptr,bit) ((SVAL(ptr,0) & (1<<(bit)))!=0)
 
 /* for readability... */
-#define IS_DOS_READONLY(test_mode) (((test_mode) & aRONLY) != 0)
-#define IS_DOS_DIR(test_mode)      (((test_mode) & aDIR) != 0)
-#define IS_DOS_ARCHIVE(test_mode)  (((test_mode) & aARCH) != 0)
-#define IS_DOS_SYSTEM(test_mode)   (((test_mode) & aSYSTEM) != 0)
-#define IS_DOS_HIDDEN(test_mode)   (((test_mode) & aHIDDEN) != 0)
+#define IS_DOS_READONLY(test_mode) (((test_mode) & FILE_ATTRIBUTE_READONLY) != 0)
+#define IS_DOS_DIR(test_mode)      (((test_mode) & FILE_ATTRIBUTE_DIRECTORY) != 0)
+#define IS_DOS_ARCHIVE(test_mode)  (((test_mode) & FILE_ATTRIBUTE_ARCHIVE) != 0)
+#define IS_DOS_SYSTEM(test_mode)   (((test_mode) & FILE_ATTRIBUTE_SYSTEM) != 0)
+#define IS_DOS_HIDDEN(test_mode)   (((test_mode) & FILE_ATTRIBUTE_HIDDEN) != 0)
 
 #define SMB_WARN(condition, message) \
     ((condition) ? (void)0 : \
@@ -123,6 +123,7 @@
 
 /* Extra macros added by Ying Chen at IBM - speed increase by inlining. */
 #define smb_buf(buf) (((char *)(buf)) + smb_size + CVAL(buf,smb_wct)*2)
+#define smb_buf_const(buf) (((const char *)(buf)) + smb_size + CVAL(buf,smb_wct)*2)
 #define smb_buflen(buf) (SVAL(buf,smb_vwv0 + (int)CVAL(buf, smb_wct)*2))
 
 /* the remaining number of bytes in smb buffer 'buf' from pointer 'p'. */
@@ -217,39 +218,12 @@ copy an IP address from one buffer to another
 #define SMB_XMALLOC_P(type) (type *)smb_xmalloc_array(sizeof(type),1)
 #define SMB_XMALLOC_ARRAY(type,count) (type *)smb_xmalloc_array(sizeof(type),(count))
 
-/* The new talloc is paranoid malloc checker safe. */
-
-#if 0
-
-Disable these now we have checked all code paths and ensured
-NULL returns on zero request. JRA.
-
-#define TALLOC(ctx, size) talloc_zeronull(ctx, size, __location__)
-#define TALLOC_P(ctx, type) (type *)talloc_zeronull(ctx, sizeof(type), #type)
-#define TALLOC_ARRAY(ctx, type, count) (type *)_talloc_array_zeronull(ctx, sizeof(type), count, #type)
-#define TALLOC_MEMDUP(ctx, ptr, size) _talloc_memdup_zeronull(ctx, ptr, size, __location__)
-#define TALLOC_ZERO(ctx, size) _talloc_zero_zeronull(ctx, size, __location__)
-#define TALLOC_ZERO_P(ctx, type) (type *)_talloc_zero_zeronull(ctx, sizeof(type), #type)
-#define TALLOC_ZERO_ARRAY(ctx, type, count) (type *)_talloc_zero_array_zeronull(ctx, sizeof(type), count, #type)
-#define TALLOC_SIZE(ctx, size) talloc_zeronull(ctx, size, __location__)
-#define TALLOC_ZERO_SIZE(ctx, size) _talloc_zero_zeronull(ctx, size, __location__)
-
-#else
-
 #define TALLOC(ctx, size) talloc_named_const(ctx, size, __location__)
-#define TALLOC_P(ctx, type) (type *)talloc_named_const(ctx, sizeof(type), #type)
-#define TALLOC_ARRAY(ctx, type, count) (type *)_talloc_array(ctx, sizeof(type), count, #type)
-#define TALLOC_MEMDUP(ctx, ptr, size) _talloc_memdup(ctx, ptr, size, __location__)
 #define TALLOC_ZERO(ctx, size) _talloc_zero(ctx, size, __location__)
-#define TALLOC_ZERO_P(ctx, type) (type *)_talloc_zero(ctx, sizeof(type), #type)
-#define TALLOC_ZERO_ARRAY(ctx, type, count) (type *)_talloc_zero_array(ctx, sizeof(type), count, #type)
 #define TALLOC_SIZE(ctx, size) talloc_named_const(ctx, size, __location__)
 #define TALLOC_ZERO_SIZE(ctx, size) _talloc_zero(ctx, size, __location__)
 
-#endif
-
 #define TALLOC_REALLOC(ctx, ptr, count) _talloc_realloc(ctx, ptr, count, __location__)
-#define TALLOC_REALLOC_ARRAY(ctx, ptr, type, count) (type *)_talloc_realloc_array(ctx, ptr, sizeof(type), count, #type)
 #define talloc_destroy(ctx) talloc_free(ctx)
 #ifndef TALLOC_FREE
 #define TALLOC_FREE(ctx) do { talloc_free(ctx); ctx=NULL; } while(0)
@@ -312,7 +286,7 @@ NULL returns on zero request. JRA.
 #define ADD_TO_ARRAY(mem_ctx, type, elem, array, num) \
 do { \
 	*(array) = ((mem_ctx) != NULL) ? \
-		TALLOC_REALLOC_ARRAY(mem_ctx, (*(array)), type, (*(num))+1) : \
+		talloc_realloc(mem_ctx, (*(array)), type, (*(num))+1) : \
 		SMB_REALLOC_ARRAY((*(array)), type, (*(num))+1); \
 	SMB_ASSERT((*(array)) != NULL); \
 	(*(array))[*(num)] = (elem); \
diff --git a/source3/include/trans2.h b/source3/include/trans2.h
index 076780dec0c..7a8c411396f 100644
--- a/source3/include/trans2.h
+++ b/source3/include/trans2.h
@@ -355,433 +355,4 @@ Byte offset   Type     name                description
 #define FLAG_TRANS2_FIND_CONTINUE       0x8
 #define FLAG_TRANS2_FIND_BACKUP_INTENT  0x10
 
-/* UNIX CIFS Extensions - created by HP */
-/*
- * UNIX CIFS Extensions have the range 0x200 - 0x2FF reserved.
- * Supposedly Microsoft have agreed to this.
- */
-
-#define MIN_UNIX_INFO_LEVEL 0x200
-#define MAX_UNIX_INFO_LEVEL 0x2FF
-
-#define INFO_LEVEL_IS_UNIX(level) (((level) >= MIN_UNIX_INFO_LEVEL) && ((level) <= MAX_UNIX_INFO_LEVEL))
-
-#define SMB_QUERY_FILE_UNIX_BASIC      0x200   /* UNIX File Info*/
-#define SMB_SET_FILE_UNIX_BASIC        0x200
-#define SMB_SET_FILE_UNIX_INFO2        0x20B   /* UNIX File Info2 */
-
-#define SMB_MODE_NO_CHANGE                 0xFFFFFFFF     /* file mode value which */
-                                              /* means "don't change it" */
-#define SMB_UID_NO_CHANGE                  0xFFFFFFFF
-#define SMB_GID_NO_CHANGE                  0xFFFFFFFF
-
-#define SMB_SIZE_NO_CHANGE_LO              0xFFFFFFFF
-#define SMB_SIZE_NO_CHANGE_HI              0xFFFFFFFF
- 
-#define SMB_TIME_NO_CHANGE_LO              0xFFFFFFFF
-#define SMB_TIME_NO_CHANGE_HI              0xFFFFFFFF
-
-/*
-Offset Size         Name
-0      LARGE_INTEGER EndOfFile                File size
-8      LARGE_INTEGER Blocks                   Number of bytes used on disk (st_blocks).
-16     LARGE_INTEGER CreationTime             Creation time
-24     LARGE_INTEGER LastAccessTime           Last access time
-32     LARGE_INTEGER LastModificationTime     Last modification time
-40     LARGE_INTEGER Uid                      Numeric user id for the owner
-48     LARGE_INTEGER Gid                      Numeric group id of owner
-56     ULONG Type                             Enumeration specifying the pathname type:
-                                              0 -- File
-                                              1 -- Directory
-                                              2 -- Symbolic link
-                                              3 -- Character device
-                                              4 -- Block device
-                                              5 -- FIFO (named pipe)
-                                              6 -- Unix domain socket
-
-60     LARGE_INTEGER devmajor                 Major device number if type is device
-68     LARGE_INTEGER devminor                 Minor device number if type is device
-76     LARGE_INTEGER uniqueid                 This is a server-assigned unique id for the file. The client
-                                              will typically map this onto an inode number. The scope of
-                                              uniqueness is the share.
-84     LARGE_INTEGER permissions              Standard UNIX file permissions  - see below.
-92     LARGE_INTEGER nlinks                   The number of directory entries that map to this entry
-                                              (number of hard links)
-
-100 - end.
-*/
-
-#define SMB_FILE_UNIX_BASIC_SIZE 100
-
-/* UNIX filetype mappings. */
-
-#define UNIX_TYPE_FILE 0
-#define UNIX_TYPE_DIR 1
-#define UNIX_TYPE_SYMLINK 2
-#define UNIX_TYPE_CHARDEV 3
-#define UNIX_TYPE_BLKDEV 4
-#define UNIX_TYPE_FIFO 5
-#define UNIX_TYPE_SOCKET 6
-#define UNIX_TYPE_UNKNOWN 0xFFFFFFFF
-
-/*
- * Oh this is fun. "Standard UNIX permissions" has no
- * meaning in POSIX. We need to define the mapping onto
- * and off the wire as this was not done in the original HP
- * spec. JRA.
- */
-
-#define UNIX_X_OTH			0000001
-#define UNIX_W_OTH			0000002
-#define UNIX_R_OTH			0000004
-#define UNIX_X_GRP			0000010
-#define UNIX_W_GRP                      0000020
-#define UNIX_R_GRP                      0000040
-#define UNIX_X_USR                      0000100
-#define UNIX_W_USR                      0000200
-#define UNIX_R_USR                      0000400
-#define UNIX_STICKY                     0001000
-#define UNIX_SET_GID                    0002000
-#define UNIX_SET_UID                    0004000
-
-/* Masks for the above */
-#define UNIX_OTH_MASK                   0000007
-#define UNIX_GRP_MASK                   0000070
-#define UNIX_USR_MASK                   0000700
-#define UNIX_PERM_MASK                  0000777
-#define UNIX_EXTRA_MASK                 0007000
-#define UNIX_ALL_MASK                   0007777
-
-/* Flags for chflags (CIFS_UNIX_EXTATTR_CAP capability) and
- * SMB_QUERY_FILE_UNIX_INFO2.
- */
-#define EXT_SECURE_DELETE               0x00000001
-#define EXT_ENABLE_UNDELETE             0x00000002
-#define EXT_SYNCHRONOUS                 0x00000004
-#define EXT_IMMUTABLE			0x00000008
-#define EXT_OPEN_APPEND_ONLY            0x00000010
-#define EXT_DO_NOT_BACKUP               0x00000020
-#define EXT_NO_UPDATE_ATIME             0x00000040
-#define EXT_HIDDEN			0x00000080
-
-#define SMB_QUERY_FILE_UNIX_LINK       0x201
-#define SMB_SET_FILE_UNIX_LINK         0x201
-#define SMB_SET_FILE_UNIX_HLINK        0x203
-/* SMB_QUERY_POSIX_ACL 0x204 see below */
-#define SMB_QUERY_XATTR                0x205 /* need for non-user XATTRs */
-#define SMB_QUERY_ATTR_FLAGS           0x206 /* chflags, chattr */
-#define SMB_SET_ATTR_FLAGS             0x206 
-#define SMB_QUERY_POSIX_PERMISSION     0x207
-/* Only valid for qfileinfo */
-#define SMB_QUERY_POSIX_LOCK	       0x208
-/* Only valid for setfileinfo */
-#define SMB_SET_POSIX_LOCK	       0x208
-
-/* The set info levels for POSIX path operations. */
-#define SMB_POSIX_PATH_OPEN	       0x209
-#define SMB_POSIX_PATH_UNLINK	       0x20A
-
-#define SMB_QUERY_FILE_UNIX_INFO2      0x20B   /* UNIX File Info2 */
-#define SMB_SET_FILE_UNIX_INFO2        0x20B
-
-/*
-SMB_QUERY_FILE_UNIX_INFO2 is SMB_QUERY_FILE_UNIX_BASIC with create
-time and file flags appended. The corresponding info level for
-findfirst/findnext is SMB_FIND_FILE_UNIX_INFO2.
-    Size    Offset  Value
-    ---------------------
-    0      LARGE_INTEGER EndOfFile  File size
-    8      LARGE_INTEGER Blocks     Number of blocks used on disk
-    16     LARGE_INTEGER ChangeTime Attribute change time
-    24     LARGE_INTEGER LastAccessTime           Last access time
-    32     LARGE_INTEGER LastModificationTime     Last modification time
-    40     LARGE_INTEGER Uid        Numeric user id for the owner
-    48     LARGE_INTEGER Gid        Numeric group id of owner
-    56     ULONG Type               Enumeration specifying the file type
-    60     LARGE_INTEGER devmajor   Major device number if type is device
-    68     LARGE_INTEGER devminor   Minor device number if type is device
-    76     LARGE_INTEGER uniqueid   This is a server-assigned unique id
-    84     LARGE_INTEGER permissions		Standard UNIX permissions
-    92     LARGE_INTEGER nlinks			Number of hard links
-    100    LARGE_INTEGER CreationTime		Create/birth time
-    108    ULONG FileFlags          File flags enumeration
-    112    ULONG FileFlagsMask      Mask of valid flags
-*/
-
-/* Transact 2 Find First levels */
-#define SMB_FIND_FILE_UNIX             0x202
-#define SMB_FIND_FILE_UNIX_INFO2       0x20B /* UNIX File Info2 */
-
-#define SMB_FILE_UNIX_INFO2_SIZE 116
-
-/*
- Info level for TRANS2_QFSINFO - returns version of CIFS UNIX extensions, plus
- 64-bits worth of capability fun :-).
- Use the same info level for TRANS2_SETFSINFO
-*/
-
-#define SMB_QUERY_CIFS_UNIX_INFO      0x200
-#define SMB_SET_CIFS_UNIX_INFO        0x200
-
-/* Returns or sets the following.
-
-  UINT16             major version number
-  UINT16             minor version number
-  LARGE_INTEGER      capability bitfield
-
-*/
-
-#define CIFS_UNIX_MAJOR_VERSION 1
-#define CIFS_UNIX_MINOR_VERSION 0
-
-#define CIFS_UNIX_FCNTL_LOCKS_CAP           0x1
-#define CIFS_UNIX_POSIX_ACLS_CAP            0x2
-#define CIFS_UNIX_XATTTR_CAP	            0x4 /* for support of other xattr
-						namespaces such as system,
-						security and trusted */
-#define CIFS_UNIX_EXTATTR_CAP		    0x8 /* for support of chattr
-						(chflags) and lsattr */
-#define CIFS_UNIX_POSIX_PATHNAMES_CAP	   0x10 /* Use POSIX pathnames on the wire. */
-#define CIFS_UNIX_POSIX_PATH_OPERATIONS_CAP	   0x20 /* We can cope with POSIX open/mkdir/unlink etc. */
-#define CIFS_UNIX_LARGE_READ_CAP           0x40 /* We can cope with 24 bit reads in readX. */
-#define CIFS_UNIX_LARGE_WRITE_CAP          0x80 /* We can cope with 24 bit writes in writeX. */
-#define CIFS_UNIX_TRANSPORT_ENCRYPTION_CAP      0x100 /* We can do SPNEGO negotiations for encryption. */
-#define CIFS_UNIX_TRANSPORT_ENCRYPTION_MANDATORY_CAP    0x200 /* We *must* SPNEGO negotiations for encryption. */
-
-#define SMB_QUERY_POSIX_FS_INFO     0x201
-
-/* Returns FILE_SYSTEM_POSIX_INFO struct as follows
-      (NB   For undefined values return -1 in that field) 
-   le32 OptimalTransferSize;    bsize on some os, iosize on other os, This 
-				is a hint to the client about best size. Server
-				can return -1 if no preference, ie if SMB 
-				negotiated size is adequate for optimal
-				read/write performance
-   le32 BlockSize; (often 512 bytes) NB: BlockSize * TotalBlocks = disk space
-   le64 TotalBlocks;  redundant with other infolevels but easy to ret here
-   le64 BlocksAvail;  although redundant, easy to return
-   le64 UserBlocksAvail;      bavail 
-   le64 TotalFileNodes;
-   le64 FreeFileNodes;
-   le64 FileSysIdentifier;    fsid 
-   (NB statfs field Namelen comes from FILE_SYSTEM_ATTRIBUTE_INFO call) 
-   (NB statfs field flags can come from FILE_SYSTEM_DEVICE_INFO call)  
-*/
-
-#define SMB_QUERY_POSIX_WHO_AM_I  0x202 /* QFS Info */
-/* returns:
-        __u32 flags;  0 = Authenticated user 1 = GUEST 
-        __u32 mask;  which flags bits server understands ie 0x0001 
-        __u64 unix_user_id;
-        __u64 unix_user_gid;
-        __u32 number_of_supplementary_gids;  may be zero 
-        __u32 number_of_sids;  may be zero
-        __u32 length_of_sid_array;  in bytes - may be zero 
-        __u32 pad;  reserved - MBZ 
-        __u64 gid_array[0];  may be empty 
-        __u8 * psid_list  may be empty
-*/
-
-/* ... more as we think of them :-). */
-
-/* SMB POSIX ACL definitions. */
-/* Wire format is (all little endian) :
-
-[2 bytes]              -     Version number.
-[2 bytes]              -     Number of ACE entries to follow.
-[2 bytes]              -     Number of default ACE entries to follow.
--------------------------------------
-^
-|
-ACE entries
-|
-v
--------------------------------------
-^
-|
-Default ACE entries
-|
-v
--------------------------------------
-
-Where an ACE entry looks like :
-
-[1 byte]           - Entry type.
-
-Entry types are :
-
-ACL_USER_OBJ            0x01
-ACL_USER                0x02
-ACL_GROUP_OBJ           0x04
-ACL_GROUP               0x08
-ACL_MASK                0x10
-ACL_OTHER               0x20
-
-[1 byte]          - permissions (perm_t)
-
-perm_t types are :
-
-ACL_READ                0x04
-ACL_WRITE               0x02
-ACL_EXECUTE             0x01
-
-[8 bytes]         - uid/gid to apply this permission to.
-
-In the same format as the uid/gid fields in the other
-UNIX extensions definitions. Use 0xFFFFFFFFFFFFFFFF for
-the MASK and OTHER entry types.
-
-If the Number of ACE entries for either file or default ACE's
-is set to 0xFFFF this means ignore this kind of ACE (and the
-number of entries sent will be zero.
-
-*/
-
-#define SMB_QUERY_POSIX_WHOAMI     0x202
-
-enum smb_whoami_flags {
-    SMB_WHOAMI_GUEST = 0x1 /* Logged in as (or squashed to) guest */
-};
-
-/* Mask of which WHOAMI bits are valid. This should make it easier for clients
- * to cope with servers that have different sets of WHOAMI flags (as more get
- * added).
- */
-#define SMB_WHOAMI_MASK 0x00000001
-
-/*
-   SMBWhoami - Query the user mapping performed by the server for the
-   connected tree. This is a subcommand of the TRANS2_QFSINFO.
-
-   Returns:
-	4 bytes unsigned -	mapping flags (smb_whoami_flags)
-	4 bytes unsigned -	flags mask
-
-	8 bytes unsigned -	primary UID
-	8 bytes unsigned -	primary GID
-	4 bytes unsigned -	number of supplementary GIDs
-	4 bytes unsigned -	number of SIDs
-	4 bytes unsigned -	SID list byte count
-	4 bytes -		pad / reserved (must be zero)
-
-	8 bytes unsigned[] -	list of GIDs (may be empty)
-	struct dom_sid[] -		list of SIDs (may be empty)
-*/
-
-/*
- * The following trans2 is done between client and server
- * as a FSINFO call to set up the encryption state for transport
- * encryption.
- * This is a subcommand of the TRANS2_QFSINFO.
- *
- * The request looks like :
- *
- * [data block] -> SPNEGO framed GSSAPI request.
- *
- * The reply looks like :
- *
- * [data block] -> SPNEGO framed GSSAPI reply - if error
- *                 is NT_STATUS_OK then we're done, if it's
- *                 NT_STATUS_MORE_PROCESSING_REQUIRED then the
- *                 client needs to keep going. If it's an
- *                 error it can be any NT_STATUS error.
- *
- */
-
-#define SMB_REQUEST_TRANSPORT_ENCRYPTION     0x203 /* QFSINFO */
-
-
-/* The query/set info levels for POSIX ACLs. */
-#define SMB_QUERY_POSIX_ACL  0x204
-#define SMB_SET_POSIX_ACL  0x204
-
-/* Current on the wire ACL version. */
-#define SMB_POSIX_ACL_VERSION 1
-
-/* ACE entry type. */
-#define SMB_POSIX_ACL_USER_OBJ            0x01
-#define SMB_POSIX_ACL_USER                0x02
-#define SMB_POSIX_ACL_GROUP_OBJ           0x04
-#define SMB_POSIX_ACL_GROUP               0x08
-#define SMB_POSIX_ACL_MASK                0x10
-#define SMB_POSIX_ACL_OTHER               0x20
-
-/* perm_t types. */
-#define SMB_POSIX_ACL_READ                0x04
-#define SMB_POSIX_ACL_WRITE               0x02
-#define SMB_POSIX_ACL_EXECUTE             0x01
-
-#define SMB_POSIX_ACL_HEADER_SIZE         6
-#define SMB_POSIX_ACL_ENTRY_SIZE         10
-
-#define SMB_POSIX_IGNORE_ACE_ENTRIES	0xFFFF
-
-/* Definition of data block of SMB_SET_POSIX_LOCK */
-/*
-  [2 bytes] lock_type - 0 = Read, 1 = Write, 2 = Unlock
-  [2 bytes] lock_flags - 1 = Wait (only valid for setlock)
-  [4 bytes] pid = locking context.
-  [8 bytes] start = unsigned 64 bits.
-  [8 bytes] length = unsigned 64 bits.
-*/
-
-#define POSIX_LOCK_TYPE_OFFSET 0
-#define POSIX_LOCK_FLAGS_OFFSET 2
-#define POSIX_LOCK_PID_OFFSET 4
-#define POSIX_LOCK_START_OFFSET 8
-#define POSIX_LOCK_LEN_OFFSET 16
-#define POSIX_LOCK_DATA_SIZE 24
-
-#define POSIX_LOCK_FLAG_NOWAIT 0
-#define POSIX_LOCK_FLAG_WAIT 1
-
-#define POSIX_LOCK_TYPE_READ 0
-#define POSIX_LOCK_TYPE_WRITE 1
-#define POSIX_LOCK_TYPE_UNLOCK 2
-
-/* SMB_POSIX_PATH_OPEN "open_mode" definitions. */
-#define SMB_O_RDONLY			  0x1
-#define SMB_O_WRONLY			  0x2
-#define SMB_O_RDWR			  0x4
-
-#define SMB_ACCMODE			  0x7
-
-#define SMB_O_CREAT			 0x10
-#define SMB_O_EXCL			 0x20
-#define SMB_O_TRUNC			 0x40
-#define SMB_O_APPEND			 0x80
-#define SMB_O_SYNC			0x100
-#define SMB_O_DIRECTORY			0x200
-#define SMB_O_NOFOLLOW			0x400
-#define SMB_O_DIRECT			0x800
-
-/* Definition of request data block for SMB_POSIX_PATH_OPEN */
-/*
-  [4 bytes] flags (as smb_ntcreate_Flags).
-  [4 bytes] open_mode			- SMB_O_xxx flags above.
-  [8 bytes] mode_t (permissions)	- same encoding as "Standard UNIX permissions" above in SMB_SET_FILE_UNIX_BASIC.
-  [2 bytes] ret_info_level	- optimization. Info level to be returned.
-*/
-
-/* Definition of reply data block for SMB_POSIX_PATH_OPEN */
-
-#define SMB_NO_INFO_LEVEL_RETURNED 0xFFFF
-
-/*
-  [2 bytes] - flags field. Identical to flags reply for oplock response field in SMBNTCreateX)
-  [2 bytes] - FID returned.
-  [4 bytes] - CreateAction (same as in NTCreateX response).
-  [2 bytes] - reply info level    - as requested or 0xFFFF if not available.
-  [2 bytes] - padding (must be zero)
-  [n bytes] - info level reply  - if available.
-*/
-
-/* Definition of request data block for SMB_POSIX_UNLINK */
-/*
-  [2 bytes] flags (defined below).
-*/
-
-#define SMB_POSIX_UNLINK_FILE_TARGET 0
-#define SMB_POSIX_UNLINK_DIRECTORY_TARGET 1
-
 #endif
diff --git a/source3/include/util_tdb.h b/source3/include/util_tdb.h
index 3bdb6977d87..f127cea1ea5 100644
--- a/source3/include/util_tdb.h
+++ b/source3/include/util_tdb.h
@@ -20,16 +20,12 @@
 #ifndef __TDBUTIL_H__
 #define __TDBUTIL_H__
 
-#include <tdb.h>
+#include "tdb_compat.h"
 
 #include <talloc.h> /* for tdb_wrap_open() */
 #include "../libcli/util/ntstatus.h" /* for map_nt_error_from_tdb() */
 #include "../../lib/util/util_tdb.h"
 
-struct tdb_wrap {
-	struct tdb_context *tdb;
-};
-
 int tdb_chainlock_with_timeout( struct tdb_context *tdb, TDB_DATA key,
 				unsigned int timeout);
 int tdb_lock_bystring_with_timeout(struct tdb_context *tdb, const char *keyval,
@@ -51,10 +47,6 @@ bool tdb_pack_append(TALLOC_CTX *mem_ctx, uint8 **buf, size_t *len,
 struct tdb_context *tdb_open_log(const char *name, int hash_size,
 				 int tdb_flags, int open_flags, mode_t mode);
 
-struct tdb_wrap *tdb_wrap_open(TALLOC_CTX *mem_ctx,
-			       const char *name, int hash_size, int tdb_flags,
-			       int open_flags, mode_t mode);
-
 NTSTATUS map_nt_error_from_tdb(enum TDB_ERROR err);
 
 int tdb_data_cmp(TDB_DATA t1, TDB_DATA t2);
diff --git a/source3/include/vfs.h b/source3/include/vfs.h
index 185bc7687c8..5c7e4e52bc4 100644
--- a/source3/include/vfs.h
+++ b/source3/include/vfs.h
@@ -134,7 +134,9 @@
 		to split out the two possible uses. JRA. */
 /* Leave at 28 - not yet released. Add fdopendir. JRA. */
 /* Leave at 28 - not yet released. Rename open function to open_fn. - gd */
-#define SMB_VFS_INTERFACE_VERSION 28
+/* Leave at 28 - not yet released. Make getwd function always return malloced memory. JRA. */
+/* Bump to version 29 - Samba 3.6.0 will ship with interface version 28. */
+#define SMB_VFS_INTERFACE_VERSION 29
 
 /*
     All intercepted VFS operations must be declared as static functions inside module source
@@ -250,7 +252,7 @@ struct vfs_fn_pointers {
 	int (*fchown)(struct vfs_handle_struct *handle, struct files_struct *fsp, uid_t uid, gid_t gid);
 	int (*lchown)(struct vfs_handle_struct *handle, const char *path, uid_t uid, gid_t gid);
 	int (*chdir)(struct vfs_handle_struct *handle, const char *path);
-	char *(*getwd)(struct vfs_handle_struct *handle, char *buf);
+	char *(*getwd)(struct vfs_handle_struct *handle);
 	int (*ntimes)(struct vfs_handle_struct *handle,
 		      const struct smb_filename *smb_fname,
 		      struct smb_file_time *ft);
@@ -613,7 +615,7 @@ int smb_vfs_call_fchown(struct vfs_handle_struct *handle,
 int smb_vfs_call_lchown(struct vfs_handle_struct *handle, const char *path,
 			uid_t uid, gid_t gid);
 int smb_vfs_call_chdir(struct vfs_handle_struct *handle, const char *path);
-char *smb_vfs_call_getwd(struct vfs_handle_struct *handle, char *buf);
+char *smb_vfs_call_getwd(struct vfs_handle_struct *handle);
 int smb_vfs_call_ntimes(struct vfs_handle_struct *handle,
 			const struct smb_filename *smb_fname,
 			struct smb_file_time *ft);
diff --git a/source3/include/vfs_macros.h b/source3/include/vfs_macros.h
index d04be9b5aae..c7686f1db52 100644
--- a/source3/include/vfs_macros.h
+++ b/source3/include/vfs_macros.h
@@ -239,10 +239,10 @@
 #define SMB_VFS_NEXT_CHDIR(handle, path) \
 	smb_vfs_call_chdir((handle)->next, (path))
 
-#define SMB_VFS_GETWD(conn, buf) \
-	smb_vfs_call_getwd((conn)->vfs_handles, (buf))
-#define SMB_VFS_NEXT_GETWD(handle, buf) \
-	smb_vfs_call_getwd((handle)->next, (buf))
+#define SMB_VFS_GETWD(conn) \
+	smb_vfs_call_getwd((conn)->vfs_handles)
+#define SMB_VFS_NEXT_GETWD(handle) \
+	smb_vfs_call_getwd((handle)->next)
 
 #define SMB_VFS_NTIMES(conn, path, ts) \
 	smb_vfs_call_ntimes((conn)->vfs_handles, (path), (ts))
@@ -364,9 +364,6 @@
 #define SMB_VFS_NEXT_TRANSLATE_NAME(handle, name, direction, mem_ctx, mapped_name) \
 	smb_vfs_call_translate_name((handle)->next, (name), (direction), (mem_ctx), (mapped_name))
 
-#define SMB_VFS_NEXT_STRICT_UNLOCK(handle, fsp, plock) \
-	smb_vfs_call_strict_unlock((handle)->next, (fsp), (plock))
-
 #define SMB_VFS_FGET_NT_ACL(fsp, security_info, ppdesc) \
 	smb_vfs_call_fget_nt_acl((fsp)->conn->vfs_handles, (fsp), (security_info), (ppdesc))
 #define SMB_VFS_NEXT_FGET_NT_ACL(handle, fsp, security_info, ppdesc) \
diff --git a/source3/intl/lang_tdb.c b/source3/intl/lang_tdb.c
index 8fcb9f2bfc8..6070e29e5ad 100644
--- a/source3/intl/lang_tdb.c
+++ b/source3/intl/lang_tdb.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "intl/lang_tdb.h"
+#include "util_tdb.h"
 
 static TDB_CONTEXT *tdb;
 
@@ -130,7 +131,7 @@ bool lang_tdb_init(const char *lang)
 		return True;
 
 	if (asprintf(&msg_path, "%s.msg",
-		     data_path((const char *)lang)) == -1) {
+		     data_path(talloc_tos(), (const char *)lang)) == -1) {
 		DEBUG(0, ("asprintf failed\n"));
 		goto done;
 	}
@@ -236,7 +237,7 @@ const char *lang_msg(const char *msgid)
 void lang_msg_free(const char *msgstr)
 {
 	if (!tdb) return;
-	free((void *)msgstr);
+	free(discard_const_p(void, msgstr));
 }
 
 /* 
diff --git a/source3/intl/lang_tdb.h b/source3/intl/lang_tdb.h
index da246846d0d..2f9c3f19e56 100644
--- a/source3/intl/lang_tdb.h
+++ b/source3/intl/lang_tdb.h
@@ -1,3 +1,24 @@
+/*
+   Unix SMB/CIFS implementation.
+   tdb based replacement for gettext
+   Copyright (C) Andrew Tridgell 2001
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _INTL_LANG_TDB_H_
+#define _INTL_LANG_TDB_H_
 
 /* The following definitions come from intl/lang_tdb.c  */
 
@@ -5,3 +26,5 @@ bool lang_tdb_init(const char *lang);
 const char *lang_msg(const char *msgid);
 void lang_msg_free(const char *msgstr);
 char *lang_tdb_current(void);
+
+#endif /* _INTL_LANG_TDB_H_ */
diff --git a/source3/lib/access.c b/source3/lib/access.c
index a7475a5edc3..044c079cc5e 100644
--- a/source3/lib/access.c
+++ b/source3/lib/access.c
@@ -12,7 +12,7 @@
 
 #include "includes.h"
 #include "memcache.h"
-#include "interfaces.h"
+#include "lib/socket/interfaces.h"
 
 #define NAME_INDEX 0
 #define ADDR_INDEX 1
@@ -182,29 +182,32 @@ static bool string_match(const char *tok,const char *s)
 bool client_match(const char *tok, const void *item)
 {
 	const char **client = (const char **)item;
+	const char *tok_addr = tok;
+	const char *cli_addr = client[ADDR_INDEX];
+
+	/*
+	 * tok and client[ADDR_INDEX] can be an IPv4 mapped to IPv6,
+	 * we try and match the IPv4 part of address only.
+	 * Bug #5311 and #7383.
+	 */
+
+	if (strnequal(tok_addr, "::ffff:",7)) {
+		tok_addr += 7;
+	}
+
+	if (strnequal(cli_addr,"::ffff:",7)) {
+		cli_addr += 7;
+	}
 
 	/*
 	 * Try to match the address first. If that fails, try to match the host
 	 * name if available.
 	 */
 
-	if (string_match(tok, client[ADDR_INDEX])) {
+	if (string_match(tok_addr, cli_addr)) {
 		return true;
 	}
 
-	if (strnequal(client[ADDR_INDEX],"::ffff:",7) &&
-			!strnequal(tok, "::ffff:",7)) {
-		/* client[ADDR_INDEX] is an IPv4 mapped to IPv6, but
- 		 * the list item is not. Try and match the IPv4 part of
- 		 * address only. This will happen a lot on IPv6 enabled
- 		 * systems with IPv4 allow/deny lists in smb.conf.
- 		 * Bug #5311. JRA.
- 		 */
-		if (string_match(tok, (client[ADDR_INDEX])+7)) {
-			return true;
-		}
-	}
-
 	if (client[NAME_INDEX][0] != 0) {
 		if (string_match(tok, client[NAME_INDEX])) {
 			return true;
diff --git a/source3/lib/addrchange.c b/source3/lib/addrchange.c
index bd4bfbaa460..a5d20419535 100644
--- a/source3/lib/addrchange.c
+++ b/source3/lib/addrchange.c
@@ -18,6 +18,7 @@
 
 #include "includes.h"
 #include "lib/addrchange.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 #if HAVE_LINUX_RTNETLINK_H
 
diff --git a/source3/lib/adt_tree.c b/source3/lib/adt_tree.c
index 6d481613c7b..de7f2d26372 100644
--- a/source3/lib/adt_tree.c
+++ b/source3/lib/adt_tree.c
@@ -134,7 +134,7 @@ static struct tree_node *pathtree_birth_child(struct tree_node *node,
 			/* the strings should never match assuming that we 
 			   have called pathtree_find_child() first */
 
-			if ( StrCaseCmp( infant->key, node->children[i-1]->key ) > 0 ) {
+			if ( strcasecmp_m( infant->key, node->children[i-1]->key ) > 0 ) {
 				DEBUG(11,("pathtree_birth_child: storing infant in i == [%d]\n", 
 					i));
 				node->children[i] = infant;
@@ -183,7 +183,7 @@ static struct tree_node *pathtree_find_child(struct tree_node *node,
 		DEBUG(11,("pathtree_find_child: child key => [%s]\n",
 			node->children[i]->key));
 
-		result = StrCaseCmp( node->children[i]->key, key );
+		result = strcasecmp_m( node->children[i]->key, key );
 
 		if ( result == 0 )
 			next = node->children[i];
diff --git a/source3/lib/bitmap.c b/source3/lib/bitmap.c
index bd56b4aad1a..5216b05c585 100644
--- a/source3/lib/bitmap.c
+++ b/source3/lib/bitmap.c
@@ -29,12 +29,12 @@ struct bitmap *bitmap_talloc(TALLOC_CTX *mem_ctx, int n)
 {
 	struct bitmap *bm;
 
-	bm = TALLOC_P(mem_ctx, struct bitmap);
+	bm = talloc(mem_ctx, struct bitmap);
 
 	if (!bm) return NULL;
 
 	bm->n = n;
-	bm->b = TALLOC_ZERO_ARRAY(bm, uint32, (n+31)/32);
+	bm->b = talloc_zero_array(bm, uint32, (n+31)/32);
 	if (!bm->b) {
 		TALLOC_FREE(bm);
 		return NULL;
diff --git a/source3/lib/charcnv.c b/source3/lib/charcnv.c
index f6fed8d1ec5..17e836dfe0f 100644
--- a/source3/lib/charcnv.c
+++ b/source3/lib/charcnv.c
@@ -31,112 +31,6 @@ void gfree_charcnv(void)
 }
 
 /**
- * Initialize iconv conversion descriptors.
- *
- * This is called the first time it is needed, and also called again
- * every time the configuration is reloaded, because the charset or
- * codepage might have changed.
- **/
-void init_iconv(void)
-{
-	global_iconv_handle = smb_iconv_handle_reinit(NULL, lp_dos_charset(),
-								lp_unix_charset(), lp_display_charset(),
-								true, global_iconv_handle);
-}
-
-/**
- talloc_strdup() a unix string to upper case.
-**/
-
-char *talloc_strdup_upper(TALLOC_CTX *ctx, const char *s)
-{
-	char *out_buffer = talloc_strdup(ctx,s);
-	const unsigned char *p = (const unsigned char *)s;
-	unsigned char *q = (unsigned char *)out_buffer;
-
-	if (!q) {
-		return NULL;
-	}
-
-	/* this is quite a common operation, so we want it to be
-	   fast. We optimise for the ascii case, knowing that all our
-	   supported multi-byte character sets are ascii-compatible
-	   (ie. they match for the first 128 chars) */
-
-	while (*p) {
-		if (*p & 0x80)
-			break;
-		*q++ = toupper_ascii_fast(*p);
-		p++;
-	}
-
-	if (*p) {
-		/* MB case. */
-		size_t converted_size, converted_size2;
-		smb_ucs2_t *ubuf = NULL;
-
-		/* We're not using the ascii buffer above. */
-		TALLOC_FREE(out_buffer);
-
-		if (!convert_string_talloc(ctx, CH_UNIX, CH_UTF16LE, s,
-					   strlen(s)+1, (void *)&ubuf,
-					   &converted_size))
-		{
-			return NULL;
-		}
-
-		strupper_w(ubuf);
-
-		if (!convert_string_talloc(ctx, CH_UTF16LE, CH_UNIX, ubuf,
-					   converted_size, (void *)&out_buffer,
-					   &converted_size2))
-		{
-			TALLOC_FREE(ubuf);
-			return NULL;
-		}
-
-		/* Don't need the intermediate buffer
- 		 * anymore.
- 		 */
-		TALLOC_FREE(ubuf);
-	}
-
-	return out_buffer;
-}
-
-char *strupper_talloc(TALLOC_CTX *ctx, const char *s) {
-	return talloc_strdup_upper(ctx, s);
-}
-
-
-char *talloc_strdup_lower(TALLOC_CTX *ctx, const char *s)
-{
-	size_t converted_size;
-	smb_ucs2_t *buffer = NULL;
-	char *out_buffer;
-
-	if (!push_ucs2_talloc(ctx, &buffer, s, &converted_size)) {
-		return NULL;
-	}
-
-	strlower_w(buffer);
-
-	if (!pull_ucs2_talloc(ctx, &out_buffer, buffer, &converted_size)) {
-		TALLOC_FREE(buffer);
-		return NULL;
-	}
-
-	TALLOC_FREE(buffer);
-
-	return out_buffer;
-}
-
-char *strlower_talloc(TALLOC_CTX *ctx, const char *s) {
-	return talloc_strdup_lower(ctx, s);
-}
-
-
-/**
  * Copy a string from a char* unix src to a dos codepage string destination.
  *
  * @return the number of bytes occupied by the string in the destination.
@@ -189,15 +83,6 @@ size_t push_ascii(void *dest, const char *src, size_t dest_len, int flags)
  Push and malloc an ascii string. src and dest null terminated.
 ********************************************************************/
 
-bool push_ascii_talloc(TALLOC_CTX *mem_ctx, char **dest, const char *src, size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-
-	*dest = NULL;
-	return convert_string_talloc(mem_ctx, CH_UNIX, CH_DOS, src, src_len,
-				     (void **)dest, converted_size);
-}
-
 /**
  * Copy a string from a dos codepage source to a unix char* destination.
  *
@@ -317,7 +202,7 @@ static size_t pull_ascii_base_talloc(TALLOC_CTX *ctx,
 			/* Have we got space to append the '\0' ? */
 			if (size <= dest_len) {
 				/* No, realloc. */
-				dest = TALLOC_REALLOC_ARRAY(ctx, dest, char,
+				dest = talloc_realloc(ctx, dest, char,
 						dest_len+1);
 				if (!dest) {
 					/* talloc fail. */
@@ -354,7 +239,7 @@ static size_t pull_ascii_base_talloc(TALLOC_CTX *ctx,
  * destination.
  **/
 
-size_t push_ucs2(const void *base_ptr, void *dest, const char *src, size_t dest_len, int flags)
+static size_t push_ucs2(const void *base_ptr, void *dest, const char *src, size_t dest_len, int flags)
 {
 	size_t len=0;
 	size_t src_len;
@@ -413,48 +298,6 @@ size_t push_ucs2(const void *base_ptr, void *dest, const char *src, size_t dest_
 }
 
 
-/**
- * Copy a string from a unix char* src to a UCS2 destination,
- * allocating a buffer using talloc().
- *
- * @param dest always set at least to NULL 
- * @parm converted_size set to the number of bytes occupied by the string in
- * the destination on success.
- *
- * @return true if new buffer was correctly allocated, and string was
- * converted.
- **/
-bool push_ucs2_talloc(TALLOC_CTX *ctx, smb_ucs2_t **dest, const char *src,
-		      size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UNIX, CH_UTF16LE, src, src_len,
-				     (void **)dest, converted_size);
-}
-
-
-/**
- * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- * @parm converted_size set to the number of bytes occupied by the string in
- * the destination on success.
- *
- * @return true if new buffer was correctly allocated, and string was
- * converted.
- **/
-
-bool push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
-		      size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UNIX, CH_UTF8, src, src_len,
-				     (void**)dest, converted_size);
-}
 
 /**
  Copy a string from a ucs2 source to a unix char* destination.
@@ -606,7 +449,7 @@ static size_t pull_ucs2_base_talloc(TALLOC_CTX *ctx,
 			/* Have we got space to append the '\0' ? */
 			if (size <= dest_len) {
 				/* No, realloc. */
-				dest = TALLOC_REALLOC_ARRAY(ctx, dest, char,
+				dest = talloc_realloc(ctx, dest, char,
 						dest_len+1);
 				if (!dest) {
 					/* talloc fail. */
@@ -627,70 +470,6 @@ static size_t pull_ucs2_base_talloc(TALLOC_CTX *ctx,
 }
 
 /**
- * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- * @parm converted_size set to the number of bytes occupied by the string in
- * the destination on success.
- *
- * @return true if new buffer was correctly allocated, and string was
- * converted.
- **/
-
-bool pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const smb_ucs2_t *src,
-		      size_t *converted_size)
-{
-	size_t src_len = (strlen_w(src)+1) * sizeof(smb_ucs2_t);
-
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UTF16LE, CH_UNIX, src, src_len,
-				     (void **)dest, converted_size);
-}
-
-/**
- * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- * @parm converted_size set to the number of bytes occupied by the string in
- * the destination on success.
- *
- * @return true if new buffer was correctly allocated, and string was
- * converted.
- **/
-
-bool pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
-		      size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_UTF8, CH_UNIX, src, src_len,
-				     (void **)dest, converted_size);
-}
-
- 
-/**
- * Copy a string from a DOS src to a unix char * destination, allocating a buffer using talloc
- *
- * @param dest always set at least to NULL 
- * @parm converted_size set to the number of bytes occupied by the string in
- * the destination on success.
- *
- * @return true if new buffer was correctly allocated, and string was
- * converted.
- **/
-
-bool pull_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src,
-		       size_t *converted_size)
-{
-	size_t src_len = strlen(src)+1;
-
-	*dest = NULL;
-	return convert_string_talloc(ctx, CH_DOS, CH_UNIX, src, src_len,
-				     (void **)dest, converted_size);
-}
-
-/**
  Copy a string from a char* src to a unicode or ascii
  dos codepage destination choosing unicode or ascii based on the 
  flags supplied
@@ -742,36 +521,6 @@ size_t push_string_base(const char *base, uint16 flags2,
 }
 
 /**
- Copy a string from a char* src to a unicode or ascii
- dos codepage destination choosing unicode or ascii based on the 
- flags supplied
- Return the number of bytes occupied by the string in the destination.
- flags can have:
-  STR_TERMINATE means include the null termination.
-  STR_UPPER     means uppercase in the destination.
-  STR_ASCII     use ascii even with unicode packet.
-  STR_NOALIGN   means don't do alignment.
- dest_len is the maximum length allowed in the destination. If dest_len
- is -1 then no maxiumum is used.
-**/
-
-ssize_t push_string(void *dest, const char *src, size_t dest_len, int flags)
-{
-	size_t ret;
-
-	if (!(flags & STR_ASCII) && \
-	    (flags & STR_UNICODE)) {
-		ret = push_ucs2(NULL, dest, src, dest_len, flags);
-	} else {
-		ret = push_ascii(dest, src, dest_len, flags);
-	}
-	if (ret == (size_t)-1) {
-		return -1;
-	}
-	return ret;
-}
-
-/**
  Copy a string from a unicode or ascii source (depending on
  the packet flags) to a char* destination.
  Flags can have:
diff --git a/source3/lib/packet.c b/source3/lib/ctdb_packet.c
index 133983d3d43..17c70ff2d67 100644
--- a/source3/lib/packet.c
+++ b/source3/lib/ctdb_packet.c
@@ -1,6 +1,6 @@
-/* 
+/*
    Unix SMB/CIFS implementation.
-   Packet handling
+   CTDB Packet handling
    Copyright (C) Volker Lendecke 2007
 
    This program is free software; you can redistribute it and/or modify
@@ -21,9 +21,9 @@
 #include "../lib/util/select.h"
 #include "system/filesys.h"
 #include "system/select.h"
-#include "packet.h"
+#include "ctdb_packet.h"
 
-struct packet_context {
+struct ctdb_packet_context {
 	int fd;
 	DATA_BLOB in, out;
 };
@@ -31,32 +31,32 @@ struct packet_context {
 /*
  * Close the underlying fd
  */
-static int packet_context_destructor(struct packet_context *ctx)
+static int ctdb_packet_context_destructor(struct ctdb_packet_context *ctx)
 {
 	return close(ctx->fd);
 }
 
 /*
- * Initialize a packet context. The fd is given to the packet context, meaning
- * that it is automatically closed when the packet context is freed.
+ * Initialize a ctdb_packet context. The fd is given to the ctdb_packet context, meaning
+ * that it is automatically closed when the ctdb_packet context is freed.
  */
-struct packet_context *packet_init(TALLOC_CTX *mem_ctx, int fd)
+struct ctdb_packet_context *ctdb_packet_init(TALLOC_CTX *mem_ctx, int fd)
 {
-	struct packet_context *result;
+	struct ctdb_packet_context *result;
 
-	if (!(result = TALLOC_ZERO_P(mem_ctx, struct packet_context))) {
+	if (!(result = talloc_zero(mem_ctx, struct ctdb_packet_context))) {
 		return NULL;
 	}
 
 	result->fd = fd;
-	talloc_set_destructor(result, packet_context_destructor);
+	talloc_set_destructor(result, ctdb_packet_context_destructor);
 	return result;
 }
 
 /*
  * Pull data from the fd
  */
-NTSTATUS packet_fd_read(struct packet_context *ctx)
+NTSTATUS ctdb_packet_fd_read(struct ctdb_packet_context *ctx)
 {
 	int res, available;
 	size_t new_size;
@@ -82,7 +82,7 @@ NTSTATUS packet_fd_read(struct packet_context *ctx)
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	if (!(in = TALLOC_REALLOC_ARRAY(ctx, ctx->in.data, uint8, new_size))) {
+	if (!(in = talloc_realloc(ctx, ctx->in.data, uint8, new_size))) {
 		DEBUG(10, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -105,7 +105,7 @@ NTSTATUS packet_fd_read(struct packet_context *ctx)
 	return NT_STATUS_OK;
 }
 
-NTSTATUS packet_fd_read_sync(struct packet_context *ctx, int timeout)
+NTSTATUS ctdb_packet_fd_read_sync_timeout(struct ctdb_packet_context *ctx, int timeout)
 {
 	int res, revents;
 
@@ -124,10 +124,10 @@ NTSTATUS packet_fd_read_sync(struct packet_context *ctx, int timeout)
 		return NT_STATUS_IO_TIMEOUT;
 	}
 
-	return packet_fd_read(ctx);
+	return ctdb_packet_fd_read(ctx);
 }
 
-bool packet_handler(struct packet_context *ctx,
+bool ctdb_packet_handler(struct ctdb_packet_context *ctx,
 		    bool (*full_req)(const uint8_t *buf,
 				     size_t available,
 				     size_t *length,
@@ -153,7 +153,7 @@ bool packet_handler(struct packet_context *ctx,
 		ctx->in.data = NULL;
 		ctx->in.length = 0;
 	} else {
-		buf = (uint8_t *)TALLOC_MEMDUP(ctx, ctx->in.data, length);
+		buf = (uint8_t *)talloc_memdup(ctx, ctx->in.data, length);
 		if (buf == NULL) {
 			*status = NT_STATUS_NO_MEMORY;
 			return true;
@@ -171,7 +171,7 @@ bool packet_handler(struct packet_context *ctx,
 /*
  * How many bytes of outgoing data do we have pending?
  */
-size_t packet_outgoing_bytes(struct packet_context *ctx)
+size_t ctdb_packet_outgoing_bytes(struct ctdb_packet_context *ctx)
 {
 	return ctx->out.length;
 }
@@ -179,7 +179,7 @@ size_t packet_outgoing_bytes(struct packet_context *ctx)
 /*
  * Push data to the fd
  */
-NTSTATUS packet_fd_write(struct packet_context *ctx)
+NTSTATUS ctdb_packet_fd_write(struct ctdb_packet_context *ctx)
 {
 	ssize_t sent;
 
@@ -200,10 +200,10 @@ NTSTATUS packet_fd_write(struct packet_context *ctx)
 /*
  * Sync flush all outgoing bytes
  */
-NTSTATUS packet_flush(struct packet_context *ctx)
+NTSTATUS ctdb_packet_flush(struct ctdb_packet_context *ctx)
 {
 	while (ctx->out.length != 0) {
-		NTSTATUS status = packet_fd_write(ctx);
+		NTSTATUS status = ctdb_packet_fd_write(ctx);
 		if (!NT_STATUS_IS_OK(status)) {
 			return status;
 		}
@@ -214,10 +214,10 @@ NTSTATUS packet_flush(struct packet_context *ctx)
 /*
  * Send a list of DATA_BLOBs
  *
- * Example:  packet_send(ctx, 2, data_blob_const(&size, sizeof(size)),
+ * Example:  ctdb_packet_send(ctx, 2, data_blob_const(&size, sizeof(size)),
  *			 data_blob_const(buf, size));
  */
-NTSTATUS packet_send(struct packet_context *ctx, int num_blobs, ...)
+NTSTATUS ctdb_packet_send(struct ctdb_packet_context *ctx, int num_blobs, ...)
 {
 	va_list ap;
 	int i;
@@ -245,7 +245,7 @@ NTSTATUS packet_send(struct packet_context *ctx, int num_blobs, ...)
 		return NT_STATUS_OK;
 	}
 
-	if (!(out = TALLOC_REALLOC_ARRAY(ctx, ctx->out.data, uint8, len))) {
+	if (!(out = talloc_realloc(ctx, ctx->out.data, uint8, len))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -266,10 +266,9 @@ NTSTATUS packet_send(struct packet_context *ctx, int num_blobs, ...)
 }
 
 /*
- * Get the packet context's file descriptor
+ * Get the ctdb_packet context's file descriptor
  */
-int packet_get_fd(struct packet_context *ctx)
+int ctdb_packet_get_fd(struct ctdb_packet_context *ctx)
 {
 	return ctx->fd;
 }
-
diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index a81691a8c9f..79dc1f246aa 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -19,11 +19,12 @@
 */
 
 #include "includes.h"
+#include "util_tdb.h"
 
 #ifdef CLUSTER_SUPPORT
 
 #include "ctdbd_conn.h"
-#include "packet.h"
+#include "ctdb_packet.h"
 #include "messages.h"
 
 /* paths to these include files come from --with-ctdb= in configure */
@@ -35,7 +36,7 @@ struct ctdbd_connection {
 	uint32 reqid;
 	uint32 our_vnn;
 	uint64 rand_srvid;
-	struct packet_context *pkt;
+	struct ctdb_packet_context *pkt;
 	struct fd_event *fde;
 
 	void (*release_ip_handler)(const char *ip_addr, void *private_data);
@@ -168,9 +169,9 @@ uint32 ctdbd_vnn(const struct ctdbd_connection *conn)
  */
 
 static NTSTATUS ctdbd_connect(TALLOC_CTX *mem_ctx,
-			      struct packet_context **presult)
+			      struct ctdb_packet_context **presult)
 {
-	struct packet_context *result;
+	struct ctdb_packet_context *result;
 	const char *sockname = lp_ctdbd_socket();
 	struct sockaddr_un addr;
 	int fd;
@@ -196,7 +197,7 @@ static NTSTATUS ctdbd_connect(TALLOC_CTX *mem_ctx,
 		return map_nt_error_from_unix(errno);
 	}
 
-	if (!(result = packet_init(mem_ctx, fd))) {
+	if (!(result = ctdb_packet_init(mem_ctx, fd))) {
 		close(fd);
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -271,7 +272,7 @@ struct req_pull_state {
 };
 
 /*
- * Pull a ctdb request out of the incoming packet queue
+ * Pull a ctdb request out of the incoming ctdb_packet queue
  */
 
 static NTSTATUS ctdb_req_pull(uint8_t *buf, size_t length,
@@ -303,7 +304,7 @@ static struct messaging_rec *ctdb_pull_messaging_rec(TALLOC_CTX *mem_ctx,
 		cluster_fatal("got invalid msg length");
 	}
 
-	if (!(result = TALLOC_P(mem_ctx, struct messaging_rec))) {
+	if (!(result = talloc(mem_ctx, struct messaging_rec))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
@@ -329,14 +330,14 @@ static struct messaging_rec *ctdb_pull_messaging_rec(TALLOC_CTX *mem_ctx,
 	return result;
 }
 
-static NTSTATUS ctdb_packet_fd_read_sync(struct packet_context *ctx)
+static NTSTATUS ctdb_packet_fd_read_sync(struct ctdb_packet_context *ctx)
 {
 	int timeout = lp_ctdb_timeout();
 
 	if (timeout == 0) {
 		timeout = -1;
 	}
-	return packet_fd_read_sync(ctx, timeout);
+	return ctdb_packet_fd_read_sync_timeout(ctx, timeout);
 }
 
 /*
@@ -364,7 +365,7 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32 reqid,
 	}
 
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(0, ("packet_fd_read failed: %s\n", nt_errstr(status)));
+		DEBUG(0, ("ctdb_packet_fd_read failed: %s\n", nt_errstr(status)));
 		cluster_fatal("ctdbd died\n");
 	}
 
@@ -373,7 +374,7 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32 reqid,
 	ZERO_STRUCT(state);
 	state.mem_ctx = mem_ctx;
 
-	if (!packet_handler(conn->pkt, ctdb_req_complete, ctdb_req_pull,
+	if (!ctdb_packet_handler(conn->pkt, ctdb_req_complete, ctdb_req_pull,
 			    &state, &status)) {
 		/*
 		 * Not enough data
@@ -383,7 +384,7 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32 reqid,
 	}
 
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(0, ("Could not read packet: %s\n", nt_errstr(status)));
+		DEBUG(0, ("Could not read ctdb_packet: %s\n", nt_errstr(status)));
 		cluster_fatal("ctdbd died\n");
 	}
 
@@ -432,7 +433,7 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32 reqid,
 			goto next_pkt;
 		}
 
-		msg_state = TALLOC_P(NULL, struct deferred_msg_state);
+		msg_state = talloc(NULL, struct deferred_msg_state);
 		if (msg_state == NULL) {
 			DEBUG(0, ("talloc failed\n"));
 			TALLOC_FREE(hdr);
@@ -493,7 +494,7 @@ static NTSTATUS ctdbd_init_connection(TALLOC_CTX *mem_ctx,
 	struct ctdbd_connection *conn;
 	NTSTATUS status;
 
-	if (!(conn = TALLOC_ZERO_P(mem_ctx, struct ctdbd_connection))) {
+	if (!(conn = talloc_zero(mem_ctx, struct ctdbd_connection))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -583,7 +584,7 @@ struct messaging_context *ctdb_conn_msg_ctx(struct ctdbd_connection *conn)
 
 int ctdbd_conn_get_fd(struct ctdbd_connection *conn)
 {
-	return packet_get_fd(conn->pkt);
+	return ctdb_packet_get_fd(conn->pkt);
 }
 
 /*
@@ -674,14 +675,14 @@ static void ctdbd_socket_handler(struct event_context *event_ctx,
 
 	NTSTATUS status;
 
-	status = packet_fd_read(conn->pkt);
+	status = ctdb_packet_fd_read(conn->pkt);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0, ("packet_fd_read failed: %s\n", nt_errstr(status)));
 		cluster_fatal("ctdbd died\n");
 	}
 
-	while (packet_handler(conn->pkt, ctdb_req_complete,
+	while (ctdb_packet_handler(conn->pkt, ctdb_req_complete,
 			      ctdb_handle_message, conn, &status)) {
 		if (!NT_STATUS_IS_OK(status)) {
 			DEBUG(10, ("could not handle incoming message: %s\n",
@@ -701,7 +702,7 @@ NTSTATUS ctdbd_register_msg_ctx(struct ctdbd_connection *conn,
 	SMB_ASSERT(conn->fde == NULL);
 
 	if (!(conn->fde = event_add_fd(msg_ctx->event_ctx, conn,
-				       packet_get_fd(conn->pkt),
+				       ctdb_packet_get_fd(conn->pkt),
 				       EVENT_FD_READ,
 				       ctdbd_socket_handler,
 				       conn))) {
@@ -758,17 +759,17 @@ NTSTATUS ctdbd_messaging_send(struct ctdbd_connection *conn,
 	DEBUG(10, ("ctdbd_messaging_send: Sending ctdb packet\n"));
 	ctdb_packet_dump(&r.hdr);
 
-	status = packet_send(
+	status = ctdb_packet_send(
 		conn->pkt, 2,
 		data_blob_const(&r, offsetof(struct ctdb_req_message, data)),
 		blob);
 
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(0, ("packet_send failed: %s\n", nt_errstr(status)));
+		DEBUG(0, ("ctdb_packet_send failed: %s\n", nt_errstr(status)));
 		goto fail;
 	}
 
-	status = packet_flush(conn->pkt);
+	status = ctdb_packet_flush(conn->pkt);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(3, ("write to ctdbd failed: %s\n", nt_errstr(status)));
@@ -823,17 +824,17 @@ static NTSTATUS ctdbd_control(struct ctdbd_connection *conn,
 	DEBUG(10, ("ctdbd_control: Sending ctdb packet\n"));
 	ctdb_packet_dump(&req.hdr);
 
-	status = packet_send(
+	status = ctdb_packet_send(
 		conn->pkt, 2,
 		data_blob_const(&req, offsetof(struct ctdb_req_control, data)),
 		data_blob_const(data.dptr, data.dsize));
 
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(3, ("packet_send failed: %s\n", nt_errstr(status)));
+		DEBUG(3, ("ctdb_packet_send failed: %s\n", nt_errstr(status)));
 		goto fail;
 	}
 
-	status = packet_flush(conn->pkt);
+	status = ctdb_packet_flush(conn->pkt);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(3, ("write to ctdbd failed: %s\n", nt_errstr(status)));
@@ -1005,17 +1006,17 @@ NTSTATUS ctdbd_migrate(struct ctdbd_connection *conn, uint32 db_id,
 	DEBUG(10, ("ctdbd_migrate: Sending ctdb packet\n"));
 	ctdb_packet_dump(&req.hdr);
 
-	status = packet_send(
+	status = ctdb_packet_send(
 		conn->pkt, 2,
 		data_blob_const(&req, offsetof(struct ctdb_req_call, data)),
 		data_blob_const(key.dptr, key.dsize));
 
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(3, ("packet_send failed: %s\n", nt_errstr(status)));
+		DEBUG(3, ("ctdb_packet_send failed: %s\n", nt_errstr(status)));
 		return status;
 	}
 
-	status = packet_flush(conn->pkt);
+	status = ctdb_packet_flush(conn->pkt);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(3, ("write to ctdbd failed: %s\n", nt_errstr(status)));
@@ -1064,17 +1065,17 @@ NTSTATUS ctdbd_fetch(struct ctdbd_connection *conn, uint32 db_id,
 	req.db_id            = db_id;
 	req.keylen           = key.dsize;
 
-	status = packet_send(
+	status = ctdb_packet_send(
 		conn->pkt, 2,
 		data_blob_const(&req, offsetof(struct ctdb_req_call, data)),
 		data_blob_const(key.dptr, key.dsize));
 
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(3, ("packet_send failed: %s\n", nt_errstr(status)));
+		DEBUG(3, ("ctdb_packet_send failed: %s\n", nt_errstr(status)));
 		return status;
 	}
 
-	status = packet_flush(conn->pkt);
+	status = ctdb_packet_flush(conn->pkt);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(3, ("write to ctdbd failed: %s\n", nt_errstr(status)));
@@ -1235,7 +1236,7 @@ NTSTATUS ctdbd_traverse(uint32 db_id,
 
 		status = NT_STATUS_OK;
 
-		if (packet_handler(conn->pkt, ctdb_req_complete,
+		if (ctdb_packet_handler(conn->pkt, ctdb_req_complete,
 				   ctdb_traverse_handler, &state, &status)) {
 
 			if (NT_STATUS_EQUAL(status, NT_STATUS_END_OF_FILE)) {
@@ -1268,7 +1269,7 @@ NTSTATUS ctdbd_traverse(uint32 db_id,
 		}
 
 		if (!NT_STATUS_IS_OK(status)) {
-			DEBUG(0, ("packet_fd_read_sync failed: %s\n", nt_errstr(status)));
+			DEBUG(0, ("ctdb_packet_fd_read_sync failed: %s\n", nt_errstr(status)));
 			cluster_fatal("ctdbd died\n");
 		}
 	}
@@ -1357,6 +1358,11 @@ NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
 	}
 
 	conn->release_ip_handler = release_ip_handler;
+	/*
+	 * store the IP address of the server socket for later
+	 * comparison in release_ip()
+	 */
+	conn->release_ip_priv = private_data;
 
 	/*
 	 * We want to be told about IP releases
diff --git a/source3/lib/dbwrap.c b/source3/lib/dbwrap.c
index 4e7346c2c4f..83fc40efacf 100644
--- a/source3/lib/dbwrap.c
+++ b/source3/lib/dbwrap.c
@@ -21,6 +21,7 @@
 
 #include "includes.h"
 #include "dbwrap.h"
+#include "util_tdb.h"
 #ifdef CLUSTER_SUPPORT
 #include "ctdb_private.h"
 #endif
@@ -156,63 +157,3 @@ struct db_context *db_open(TALLOC_CTX *mem_ctx,
 
 	return result;
 }
-
-NTSTATUS dbwrap_delete(struct db_context *db, TDB_DATA key)
-{
-	struct db_record *rec;
-	NTSTATUS status;
-
-	rec = db->fetch_locked(db, talloc_tos(), key);
-	if (rec == NULL) {
-		return NT_STATUS_NO_MEMORY;
-	}
-	status = rec->delete_rec(rec);
-	TALLOC_FREE(rec);
-	return status;
-}
-
-NTSTATUS dbwrap_store(struct db_context *db, TDB_DATA key,
-		      TDB_DATA data, int flags)
-{
-	struct db_record *rec;
-	NTSTATUS status;
-
-	rec = db->fetch_locked(db, talloc_tos(), key);
-	if (rec == NULL) {
-		return NT_STATUS_NO_MEMORY;
-	}
-
-	status = rec->store(rec, data, flags);
-	TALLOC_FREE(rec);
-	return status;
-}
-
-TDB_DATA dbwrap_fetch(struct db_context *db, TALLOC_CTX *mem_ctx,
-		      TDB_DATA key)
-{
-	TDB_DATA result;
-
-	if (db->fetch(db, mem_ctx, key, &result) == -1) {
-		return make_tdb_data(NULL, 0);
-	}
-
-	return result;
-}
-
-NTSTATUS dbwrap_delete_bystring(struct db_context *db, const char *key)
-{
-	return dbwrap_delete(db, string_term_tdb_data(key));
-}
-
-NTSTATUS dbwrap_store_bystring(struct db_context *db, const char *key,
-			       TDB_DATA data, int flags)
-{
-	return dbwrap_store(db, string_term_tdb_data(key), data, flags);
-}
-
-TDB_DATA dbwrap_fetch_bystring(struct db_context *db, TALLOC_CTX *mem_ctx,
-			       const char *key)
-{
-	return dbwrap_fetch(db, mem_ctx, string_term_tdb_data(key));
-}
-
diff --git a/source3/lib/dbwrap_ctdb.c b/source3/lib/dbwrap_ctdb.c
index f9a7dd6a6ff..255a6730203 100644
--- a/source3/lib/dbwrap_ctdb.c
+++ b/source3/lib/dbwrap_ctdb.c
@@ -20,6 +20,8 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "lib/util/tdb_wrap.h"
+#include "util_tdb.h"
 #ifdef CLUSTER_SUPPORT
 #include "ctdb.h"
 #include "ctdb_private.h"
@@ -89,7 +91,7 @@ static NTSTATUS db_ctdb_ltdb_fetch(struct db_ctdb_ctx *db,
 	TDB_DATA rec;
 	NTSTATUS status;
 
-	rec = tdb_fetch(db->wtdb->tdb, key);
+	rec = tdb_fetch_compat(db->wtdb->tdb, key);
 	if (rec.dsize < sizeof(struct ctdb_ltdb_header)) {
 		status = NT_STATUS_NOT_FOUND;
 		if (data) {
@@ -537,7 +539,7 @@ static struct db_record *db_ctdb_fetch_locked_transaction(struct db_ctdb_ctx *ct
 		return result;
 	}
 
-	ctdb_data = tdb_fetch(ctx->wtdb->tdb, key);
+	ctdb_data = tdb_fetch_compat(ctx->wtdb->tdb, key);
 	if (ctdb_data.dptr == NULL) {
 		/* create the record */
 		result->value = tdb_null;
@@ -623,7 +625,7 @@ static NTSTATUS db_ctdb_transaction_store(struct db_ctdb_transaction_handle *h,
 	if (!pull_newest_from_marshall_buffer(h->m_write, key, &header,
 					      NULL, NULL)) {
 
-		rec = tdb_fetch(h->ctx->wtdb->tdb, key);
+		rec = tdb_fetch_compat(h->ctx->wtdb->tdb, key);
 
 		if (rec.dptr != NULL) {
 			memcpy(&header, rec.dptr,
@@ -992,10 +994,7 @@ static int db_ctdb_record_destr(struct db_record* data)
 		   hex_encode_talloc(data, (unsigned char *)data->key.dptr,
 			      data->key.dsize)));
 
-	if (tdb_chainunlock(crec->ctdb_ctx->wtdb->tdb, data->key) != 0) {
-		DEBUG(0, ("tdb_chainunlock failed\n"));
-		return -1;
-	}
+	tdb_chainunlock(crec->ctdb_ctx->wtdb->tdb, data->key);
 
 	threshold = lp_ctdb_locktime_warn_threshold();
 	if (threshold != 0) {
@@ -1023,7 +1022,7 @@ static struct db_record *fetch_locked_internal(struct db_ctdb_ctx *ctx,
 		return NULL;
 	}
 
-	if (!(crec = TALLOC_ZERO_P(result, struct db_ctdb_rec))) {
+	if (!(crec = talloc_zero(result, struct db_ctdb_rec))) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(result);
 		return NULL;
@@ -1064,7 +1063,7 @@ again:
 	result->delete_rec = db_ctdb_delete;
 	talloc_set_destructor(result, db_ctdb_record_destr);
 
-	ctdb_data = tdb_fetch(ctx->wtdb->tdb, key);
+	ctdb_data = tdb_fetch_compat(ctx->wtdb->tdb, key);
 
 	/*
 	 * See if we have a valid record and we are the dmaster. If so, we can
@@ -1164,7 +1163,7 @@ static int db_ctdb_fetch(struct db_context *db, TALLOC_CTX *mem_ctx,
 	}
 
 	/* try a direct fetch */
-	ctdb_data = tdb_fetch(ctx->wtdb->tdb, key);
+	ctdb_data = tdb_fetch_compat(ctx->wtdb->tdb, key);
 
 	/*
 	 * See if we have a valid record and we are the dmaster. If so, we can
@@ -1363,13 +1362,13 @@ struct db_context *db_open_ctdb(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
-	if (!(result = TALLOC_ZERO_P(mem_ctx, struct db_context))) {
+	if (!(result = talloc_zero(mem_ctx, struct db_context))) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(result);
 		return NULL;
 	}
 
-	if (!(db_ctdb = TALLOC_P(result, struct db_ctdb_ctx))) {
+	if (!(db_ctdb = talloc(result, struct db_ctdb_ctx))) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(result);
 		return NULL;
diff --git a/source3/lib/dbwrap_file.c b/source3/lib/dbwrap_file.c
index 69ad8e4b20c..6ecd72810de 100644
--- a/source3/lib/dbwrap_file.c
+++ b/source3/lib/dbwrap_file.c
@@ -80,12 +80,12 @@ static struct db_record *db_file_fetch_locked(struct db_context *db,
 	SMB_ASSERT(ctx->locked_record == NULL);
 
  again:
-	if (!(result = TALLOC_P(mem_ctx, struct db_record))) {
+	if (!(result = talloc(mem_ctx, struct db_record))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
 
-	if (!(file = TALLOC_P(result, struct db_locked_file))) {
+	if (!(file = talloc(result, struct db_locked_file))) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(result);
 		return NULL;
@@ -168,7 +168,7 @@ static struct db_record *db_file_fetch_locked(struct db_context *db,
 
 	if (statbuf.st_size != 0) {
 		result->value.dsize = statbuf.st_size;
-		result->value.dptr = TALLOC_ARRAY(result, uint8,
+		result->value.dptr = talloc_array(result, uint8,
 						  statbuf.st_size);
 		if (result->value.dptr == NULL) {
 			DEBUG(1, ("talloc failed\n"));
@@ -348,12 +348,12 @@ struct db_context *db_open_file(TALLOC_CTX *mem_ctx,
 	struct db_context *result = NULL;
 	struct db_file_ctx *ctx;
 
-	if (!(result = TALLOC_ZERO_P(mem_ctx, struct db_context))) {
+	if (!(result = talloc_zero(mem_ctx, struct db_context))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
 
-	if (!(ctx = TALLOC_P(result, struct db_file_ctx))) {
+	if (!(ctx = talloc(result, struct db_file_ctx))) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(result);
 		return NULL;
diff --git a/source3/lib/dbwrap_rbt.c b/source3/lib/dbwrap_rbt.c
index af88c79e6a4..fd6e988864b 100644
--- a/source3/lib/dbwrap_rbt.c
+++ b/source3/lib/dbwrap_rbt.c
@@ -400,7 +400,7 @@ struct db_context *db_open_rbt(TALLOC_CTX *mem_ctx)
 		return NULL;
 	}
 
-	result->private_data = TALLOC_ZERO_P(result, struct db_rbt_ctx);
+	result->private_data = talloc_zero(result, struct db_rbt_ctx);
 
 	if (result->private_data == NULL) {
 		TALLOC_FREE(result);
diff --git a/source3/lib/dbwrap_tdb.c b/source3/lib/dbwrap_tdb.c
index 4cdc1930eed..2efb3dfe39d 100644
--- a/source3/lib/dbwrap_tdb.c
+++ b/source3/lib/dbwrap_tdb.c
@@ -19,6 +19,7 @@
 
 #include "includes.h"
 #include "dbwrap.h"
+#include "lib/util/tdb_wrap.h"
 
 struct db_tdb_ctx {
 	struct tdb_wrap *wtdb;
@@ -42,10 +43,7 @@ static int db_tdb_record_destr(struct db_record* data)
 		   hex_encode_talloc(data, (unsigned char *)data->key.dptr,
 			      data->key.dsize)));
 
-	if (tdb_chainunlock(ctx->wtdb->tdb, data->key) != 0) {
-		DEBUG(0, ("tdb_chainunlock failed\n"));
-		return -1;
-	}
+	tdb_chainunlock(ctx->wtdb->tdb, data->key);
 	return 0;
 }
 
@@ -329,7 +327,8 @@ static int db_tdb_transaction_cancel(struct db_context *db)
 {
 	struct db_tdb_ctx *db_ctx =
 		talloc_get_type_abort(db->private_data, struct db_tdb_ctx);
-	return tdb_transaction_cancel(db_ctx->wtdb->tdb);
+	tdb_transaction_cancel(db_ctx->wtdb->tdb);
+	return 0;
 }
 
 struct db_context *db_open_tdb(TALLOC_CTX *mem_ctx,
@@ -340,13 +339,13 @@ struct db_context *db_open_tdb(TALLOC_CTX *mem_ctx,
 	struct db_context *result = NULL;
 	struct db_tdb_ctx *db_tdb;
 
-	result = TALLOC_ZERO_P(mem_ctx, struct db_context);
+	result = talloc_zero(mem_ctx, struct db_context);
 	if (result == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		goto fail;
 	}
 
-	result->private_data = db_tdb = TALLOC_P(result, struct db_tdb_ctx);
+	result->private_data = db_tdb = talloc(result, struct db_tdb_ctx);
 	if (db_tdb == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		goto fail;
diff --git a/source3/lib/dbwrap_util.c b/source3/lib/dbwrap_util.c
index 35f8a14d0d4..effcf40c6b3 100644
--- a/source3/lib/dbwrap_util.c
+++ b/source3/lib/dbwrap_util.c
@@ -3,7 +3,10 @@
    Utility functions for the dbwrap API
    Copyright (C) Volker Lendecke 2007
    Copyright (C) Michael Adam 2009
-   
+   Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2006
+
+   Major code contributions from Aleksey Fedoseev (fedoseev@ru.ibm.com)
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
@@ -21,6 +24,7 @@
 
 #include "includes.h"
 #include "dbwrap.h"
+#include "util_tdb.h"
 
 int32_t dbwrap_fetch_int32(struct db_context *db, const char *keystr)
 {
@@ -429,7 +433,7 @@ static NTSTATUS dbwrap_trans_traverse_action(struct db_context* db, void* privat
 
 	int ret = db->traverse(db, ctx->f, ctx->private_data);
 
-	return (ret == -1) ? NT_STATUS_INTERNAL_DB_CORRUPTION : NT_STATUS_OK;
+	return (ret < 0) ? NT_STATUS_INTERNAL_DB_CORRUPTION : NT_STATUS_OK;
 }
 
 NTSTATUS dbwrap_trans_traverse(struct db_context *db,
@@ -448,9 +452,69 @@ NTSTATUS dbwrap_traverse(struct db_context *db,
 			 void *private_data)
 {
 	int ret = db->traverse(db, f, private_data);
-	return (ret == -1) ? NT_STATUS_INTERNAL_DB_CORRUPTION : NT_STATUS_OK;
+	return (ret < 0) ? NT_STATUS_INTERNAL_DB_CORRUPTION : NT_STATUS_OK;
+}
+
+
+
+NTSTATUS dbwrap_delete(struct db_context *db, TDB_DATA key)
+{
+	struct db_record *rec;
+	NTSTATUS status;
+
+	rec = db->fetch_locked(db, talloc_tos(), key);
+	if (rec == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+	status = rec->delete_rec(rec);
+	TALLOC_FREE(rec);
+	return status;
+}
+
+NTSTATUS dbwrap_store(struct db_context *db, TDB_DATA key,
+		      TDB_DATA data, int flags)
+{
+	struct db_record *rec;
+	NTSTATUS status;
+
+	rec = db->fetch_locked(db, talloc_tos(), key);
+	if (rec == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	status = rec->store(rec, data, flags);
+	TALLOC_FREE(rec);
+	return status;
 }
 
+TDB_DATA dbwrap_fetch(struct db_context *db, TALLOC_CTX *mem_ctx,
+		      TDB_DATA key)
+{
+	TDB_DATA result;
+
+	if (db->fetch(db, mem_ctx, key, &result) != 0) {
+		return make_tdb_data(NULL, 0);
+	}
+
+	return result;
+}
+
+NTSTATUS dbwrap_delete_bystring(struct db_context *db, const char *key)
+{
+	return dbwrap_delete(db, string_term_tdb_data(key));
+}
+
+NTSTATUS dbwrap_store_bystring(struct db_context *db, const char *key,
+			       TDB_DATA data, int flags)
+{
+	return dbwrap_store(db, string_term_tdb_data(key), data, flags);
+}
+
+TDB_DATA dbwrap_fetch_bystring(struct db_context *db, TALLOC_CTX *mem_ctx,
+			       const char *key)
+{
+	return dbwrap_fetch(db, mem_ctx, string_term_tdb_data(key));
+}
 
 
 
diff --git a/source3/lib/dprintf.c b/source3/lib/dprintf.c
deleted file mode 100644
index ad3c79a8d52..00000000000
--- a/source3/lib/dprintf.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/* 
-   Unix SMB/CIFS implementation.
-   display print functions
-   Copyright (C) Andrew Tridgell 2001
-   
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-
-/*
-  this module provides functions for printing internal strings in the "display charset"
-  This charset may be quite different from the chosen unix charset
-
-  Eventually these functions will need to take care of column count constraints
-
-  The d_ prefix on print functions in Samba refers to the display character set
-  conversion
-*/
-
-#include "includes.h"
-#include "intl/lang_tdb.h"
-
- int d_vfprintf(FILE *f, const char *format, va_list ap)
-{
-	char *p = NULL, *p2 = NULL;
-	int ret, maxlen, clen;
-	size_t size = 0;
-	const char *msgstr;
-	va_list ap2;
-
-	va_copy(ap2, ap);
-
-	/* do any message translations */
-	msgstr = lang_msg(format);
-	if (!msgstr) {
-		ret = -1;
-		goto out;
-	}
-
-	ret = vasprintf(&p, msgstr, ap2);
-
-	lang_msg_free(msgstr);
-
-	if (ret <= 0) {
-		ret = -1;
-		goto out;
-	}
-
-	/* now we have the string in unix format, convert it to the display
-	   charset, but beware of it growing */
-	maxlen = ret*2;
-again:
-	p2 = (char *)SMB_MALLOC(maxlen);
-	if (!p2) {
-		ret = -1;
-		goto out;
-	}
-
-	if (!convert_string(CH_UNIX, CH_DISPLAY, p, ret, p2, maxlen, &size)) {
-		ret = -1;
-		goto out;
-	}
-	clen = size;
-
-	if (clen >= maxlen) {
-		/* it didn't fit - try a larger buffer */
-		maxlen *= 2;
-		SAFE_FREE(p2);
-		goto again;
-	}
-
-	/* good, its converted OK */
-	ret = fwrite(p2, 1, clen, f);
-out:
-
-	SAFE_FREE(p);
-	SAFE_FREE(p2);
-	va_end(ap2);
-
-	return ret;
-}
-
-
- int d_fprintf(FILE *f, const char *format, ...)
-{
-	int ret;
-	va_list ap;
-
-	va_start(ap, format);
-	ret = d_vfprintf(f, format, ap);
-	va_end(ap);
-
-	return ret;
-}
-
-static FILE *outfile;
-
- int d_printf(const char *format, ...)
-{
-	int ret;
-	va_list ap;
-
-	if (!outfile) outfile = stdout;
-	
-	va_start(ap, format);
-	ret = d_vfprintf(outfile, format, ap);
-	va_end(ap);
-
-	return ret;
-}
-
-/* interactive programs need a way of tell d_*() to write to stderr instead
-   of stdout */
-void display_set_stderr(void)
-{
-	outfile = stderr;
-}
diff --git a/source3/lib/dummyroot.c b/source3/lib/dummyparam.c
index 64ea75814a1..bad5d56ffc5 100644
--- a/source3/lib/dummyroot.c
+++ b/source3/lib/dummyparam.c
@@ -1,19 +1,19 @@
-/* 
+/*
    Unix SMB/CIFS implementation.
    RPC pipe client
 
-   Copyright (C) Tim Potter 2003
+   Copyright (C) Gerald (Jerry) Carter          2004.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
@@ -23,12 +23,13 @@
 
 #include "includes.h"
 
-void become_root(void)
+int find_service(TALLOC_CTX *ctx, const char *service_in, char **p_service_out)
 {
-        return;
+	return -1;
 }
 
-void unbecome_root(void)
+bool conn_snum_used(struct smbd_server_connection *sconn,
+		    int snum)
 {
-        return;
+	return False;
 }
diff --git a/source3/lib/errmap_unix.c b/source3/lib/errmap_unix.c
index b4a98f96342..ea195474ef7 100644
--- a/source3/lib/errmap_unix.c
+++ b/source3/lib/errmap_unix.c
@@ -24,91 +24,93 @@
 
 /* Mapping from Unix, to NT error numbers */
 
-const struct unix_error_map unix_dos_nt_errmap[] = {
-	{ EPERM, ERRDOS, ERRnoaccess, NT_STATUS_ACCESS_DENIED },
-	{ EACCES, ERRDOS, ERRnoaccess, NT_STATUS_ACCESS_DENIED },
-	{ ENOENT, ERRDOS, ERRbadfile, NT_STATUS_OBJECT_NAME_NOT_FOUND },
-	{ ENOTDIR, ERRDOS, ERRbadpath,  NT_STATUS_NOT_A_DIRECTORY },
-	{ EIO, ERRHRD, ERRgeneral, NT_STATUS_IO_DEVICE_ERROR },
-	{ EBADF, ERRSRV, ERRsrverror, NT_STATUS_INVALID_HANDLE },
-	{ EINVAL, ERRSRV, ERRsrverror, NT_STATUS_INVALID_PARAMETER },
-	{ EEXIST, ERRDOS, ERRfilexists, NT_STATUS_OBJECT_NAME_COLLISION},
-	{ ENFILE, ERRDOS, ERRnofids, NT_STATUS_TOO_MANY_OPENED_FILES },
-	{ EMFILE, ERRDOS, ERRnofids, NT_STATUS_TOO_MANY_OPENED_FILES },
-	{ ENOSPC, ERRHRD, ERRdiskfull, NT_STATUS_DISK_FULL },
-	{ ENOMEM, ERRDOS, ERRnomem, NT_STATUS_NO_MEMORY },
-	{ EISDIR, ERRDOS, ERRnoaccess, NT_STATUS_FILE_IS_A_DIRECTORY},
-	{ EMLINK, ERRDOS, ERRgeneral, NT_STATUS_TOO_MANY_LINKS },
-	{ EINTR,  ERRHRD, ERRgeneral, NT_STATUS_RETRY },
-	{ ENOSYS, ERRDOS, ERRunsup, NT_STATUS_NOT_SUPPORTED },
+static const struct {
+	int unix_error;
+	NTSTATUS nt_error;
+} unix_nt_errmap[] = {
+	{ EAGAIN,       NT_STATUS_NETWORK_BUSY },
+	{ EINTR,        NT_STATUS_RETRY },
+#ifdef ENOBUFS
+	{ ENOBUFS,      NT_STATUS_INSUFFICIENT_RESOURCES },
+#endif
+#ifdef EWOULDBLOCK
+	{ EWOULDBLOCK,  NT_STATUS_NETWORK_BUSY },
+#endif
+	{ EPERM,        NT_STATUS_ACCESS_DENIED },
+	{ EACCES,       NT_STATUS_ACCESS_DENIED },
+	{ ENOENT,       NT_STATUS_OBJECT_NAME_NOT_FOUND },
+	{ ENOTDIR,      NT_STATUS_NOT_A_DIRECTORY },
+	{ EIO,          NT_STATUS_IO_DEVICE_ERROR },
+	{ EBADF,        NT_STATUS_INVALID_HANDLE },
+	{ EINVAL,       NT_STATUS_INVALID_PARAMETER },
+	{ EEXIST,       NT_STATUS_OBJECT_NAME_COLLISION},
+	{ ENFILE,       NT_STATUS_TOO_MANY_OPENED_FILES },
+	{ EMFILE,       NT_STATUS_TOO_MANY_OPENED_FILES },
+	{ ENOSPC,       NT_STATUS_DISK_FULL },
+	{ ENOMEM,       NT_STATUS_NO_MEMORY },
+	{ EISDIR,       NT_STATUS_FILE_IS_A_DIRECTORY},
+#ifdef EPIPE
+	{ EPIPE,        NT_STATUS_PIPE_BROKEN},
+#endif
+	{ EMLINK,       NT_STATUS_TOO_MANY_LINKS },
+	{ ENOSYS,       NT_STATUS_NOT_SUPPORTED },
 #ifdef ELOOP
-	{ ELOOP, ERRDOS, ERRbadpath, NT_STATUS_OBJECT_PATH_NOT_FOUND },
+	{ ELOOP,        NT_STATUS_OBJECT_PATH_NOT_FOUND },
 #endif
 #ifdef EFTYPE
-	{ EFTYPE, ERRDOS, ERRbadpath, NT_STATUS_OBJECT_PATH_NOT_FOUND },
+	{ EFTYPE,       NT_STATUS_OBJECT_PATH_NOT_FOUND },
 #endif
 #ifdef EDQUOT
-	{ EDQUOT, ERRHRD, ERRdiskfull, NT_STATUS_DISK_FULL }, /* Windows apps need this, not NT_STATUS_QUOTA_EXCEEDED */
+	{ EDQUOT,       NT_STATUS_DISK_FULL }, /* Windows apps need this, not NT_STATUS_QUOTA_EXCEEDED */
 #endif
 #ifdef ENOTEMPTY
-	{ ENOTEMPTY, ERRDOS, ERRnoaccess, NT_STATUS_DIRECTORY_NOT_EMPTY },
+	{ ENOTEMPTY,    NT_STATUS_DIRECTORY_NOT_EMPTY },
 #endif
 #ifdef EXDEV
-	{ EXDEV, ERRDOS, ERRdiffdevice, NT_STATUS_NOT_SAME_DEVICE },
+	{ EXDEV,        NT_STATUS_NOT_SAME_DEVICE },
 #endif
 #ifdef EROFS
-	{ EROFS, ERRHRD, ERRnowrite, NT_STATUS_ACCESS_DENIED },
+	{ EROFS,        NT_STATUS_ACCESS_DENIED },
 #endif
 #ifdef ENAMETOOLONG
-	{ ENAMETOOLONG, ERRDOS, 206, NT_STATUS_OBJECT_NAME_INVALID },
+	{ ENAMETOOLONG, NT_STATUS_OBJECT_NAME_INVALID },
 #endif
 #ifdef EFBIG
-	{ EFBIG, ERRHRD, ERRdiskfull, NT_STATUS_DISK_FULL },
-#endif
-#ifdef ENOBUFS
-	{ ENOBUFS, ERRDOS, ERRnomem, NT_STATUS_INSUFFICIENT_RESOURCES },
+	{ EFBIG,        NT_STATUS_DISK_FULL },
 #endif
-	{ EAGAIN, ERRDOS, 111, NT_STATUS_NETWORK_BUSY },
 #ifdef EADDRINUSE
-	{ EADDRINUSE, ERRDOS, 52, NT_STATUS_ADDRESS_ALREADY_ASSOCIATED},
+	{ EADDRINUSE,   NT_STATUS_ADDRESS_ALREADY_ASSOCIATED},
 #endif
 #ifdef ENETUNREACH
-	{ ENETUNREACH, ERRHRD, ERRgeneral, NT_STATUS_NETWORK_UNREACHABLE},
+	{ ENETUNREACH,  NT_STATUS_NETWORK_UNREACHABLE},
 #endif
 #ifdef EHOSTUNREACH
-		{ EHOSTUNREACH, ERRHRD, ERRgeneral, NT_STATUS_HOST_UNREACHABLE},
+        { EHOSTUNREACH, NT_STATUS_HOST_UNREACHABLE},
 #endif
 #ifdef ECONNREFUSED
-	{ ECONNREFUSED, ERRHRD, ERRgeneral, NT_STATUS_CONNECTION_REFUSED},
+	{ ECONNREFUSED, NT_STATUS_CONNECTION_REFUSED},
 #endif
 #ifdef ETIMEDOUT
-	{ ETIMEDOUT, ERRHRD, 121, NT_STATUS_IO_TIMEOUT},
+	{ ETIMEDOUT,    NT_STATUS_IO_TIMEOUT},
 #endif
 #ifdef ECONNABORTED
-	{ ECONNABORTED, ERRHRD, ERRgeneral, NT_STATUS_CONNECTION_ABORTED},
+	{ ECONNABORTED, NT_STATUS_CONNECTION_ABORTED},
 #endif
 #ifdef ECONNRESET
-	{ ECONNRESET, ERRHRD, ERRgeneral, NT_STATUS_CONNECTION_RESET},
+	{ ECONNRESET,   NT_STATUS_CONNECTION_RESET},
 #endif
 #ifdef ENODEV
-	{ ENODEV, ERRDOS, 55, NT_STATUS_DEVICE_DOES_NOT_EXIST},
-#endif
-#ifdef EPIPE
-	{ EPIPE, ERRDOS, 109, NT_STATUS_PIPE_BROKEN},
-#endif
-#ifdef EWOULDBLOCK
-	{ EWOULDBLOCK, ERRDOS, 111, NT_STATUS_NETWORK_BUSY },
+	{ ENODEV,       NT_STATUS_DEVICE_DOES_NOT_EXIST},
 #endif
 #ifdef ENOATTR
-	{ ENOATTR, ERRDOS, ERRbadfile, NT_STATUS_NOT_FOUND },
+	{ ENOATTR,      NT_STATUS_NOT_FOUND },
 #endif
 #ifdef ECANCELED
-	{ ECANCELED, ERRDOS, ERRbadfid, NT_STATUS_CANCELLED},
+	{ ECANCELED,    NT_STATUS_CANCELLED},
 #endif
 #ifdef ENOTSUP
-        { ENOTSUP, ERRSRV, ERRnosupport, NT_STATUS_NOT_SUPPORTED},
+        { ENOTSUP,      NT_STATUS_NOT_SUPPORTED},
 #endif
-	{ 0, 0, 0, NT_STATUS_OK }
 };
 
 /*********************************************************************
@@ -131,22 +133,16 @@ NTSTATUS map_nt_error_from_unix(int unix_error)
 	}
 
 	/* Look through list */
-	while(unix_dos_nt_errmap[i].unix_error != 0) {
-		if (unix_dos_nt_errmap[i].unix_error == unix_error)
-			return unix_dos_nt_errmap[i].nt_error;
-		i++;
+	for (i=0;i<ARRAY_SIZE(unix_nt_errmap);i++) {
+		if (unix_nt_errmap[i].unix_error == unix_error) {
+			return unix_nt_errmap[i].nt_error;
+		}
 	}
 
 	/* Default return */
 	return NT_STATUS_ACCESS_DENIED;
 }
 
-/* Convert a Unix error code to a WERROR. */
-WERROR unix_to_werror(int unix_error)
-{
-	return ntstatus_to_werror(map_nt_error_from_unix(unix_error));
-}
-
 /* Return a UNIX errno from a NT status code */
 static const struct {
 	NTSTATUS status;
@@ -257,7 +253,6 @@ static const struct {
 #ifdef EXDEV
 	{NT_STATUS_NOT_SAME_DEVICE, EXDEV},
 #endif
-	{NT_STATUS(0), 0}
 };
 
 int map_errno_from_nt_status(NTSTATUS status)
@@ -272,7 +267,7 @@ int map_errno_from_nt_status(NTSTATUS status)
 		return 0;
 	}
 
-	for (i=0;nt_errno_map[i].error;i++) {
+	for (i=0;i<ARRAY_SIZE(nt_errno_map);i++) {
 		if (NT_STATUS_V(nt_errno_map[i].status) ==
 			    NT_STATUS_V(status)) {
 			return nt_errno_map[i].error;
diff --git a/source3/lib/eventlog/eventlog.c b/source3/lib/eventlog/eventlog.c
index 4941199c5ea..67583b8666c 100644
--- a/source3/lib/eventlog/eventlog.c
+++ b/source3/lib/eventlog/eventlog.c
@@ -24,6 +24,7 @@
 #include "system/filesys.h"
 #include "lib/eventlog/eventlog.h"
 #include "../libcli/security/security.h"
+#include "util_tdb.h"
 
 /* maintain a list of open eventlog tdbs with reference counts */
 
@@ -202,7 +203,7 @@ static bool make_way_for_eventlogs( TDB_CONTEXT * the_tdb, int32_t needed,
 		/* read a record, add the amt to nbytes */
 		key.dsize = sizeof(int32_t);
 		key.dptr = (unsigned char *)&i;
-		ret = tdb_fetch( the_tdb, key );
+		ret = tdb_fetch_compat( the_tdb, key );
 		if ( ret.dsize == 0 ) {
 			DEBUG( 8,
 			       ( "Can't find a record for the key, record [%d]\n",
@@ -415,7 +416,7 @@ ELOG_TDB *elog_open_tdb( const char *logname, bool force_clear, bool read_only )
 			return ptr;
 		}
 
-		if ( !(tdb_node = TALLOC_ZERO_P( NULL, ELOG_TDB)) ) {
+		if ( !(tdb_node = talloc_zero( NULL, ELOG_TDB)) ) {
 			DEBUG(0,("elog_open_tdb: talloc() failure!\n"));
 			tdb_close( tdb );
 			return NULL;
@@ -678,7 +679,7 @@ struct eventlog_Record_tdb *evlog_pull_record_tdb(TALLOC_CTX *mem_ctx,
 	key.dptr = (unsigned char *)&srecno;
 	key.dsize = sizeof(int32_t);
 
-	data = tdb_fetch(tdb, key);
+	data = tdb_fetch_compat(tdb, key);
 	if (data.dsize == 0) {
 		DEBUG(8,("evlog_pull_record_tdb: "
 			"Can't find a record for the key, record %d\n",
@@ -780,7 +781,7 @@ NTSTATUS evlog_push_record_tdb(TALLOC_CTX *mem_ctx,
 
 	/* lock */
 	ret = tdb_lock_bystring_with_timeout(tdb, EVT_NEXT_RECORD, 1);
-	if (ret == -1) {
+	if (ret != 0) {
 		return NT_STATUS_LOCK_NOT_GRANTED;
 	}
 
@@ -803,13 +804,13 @@ NTSTATUS evlog_push_record_tdb(TALLOC_CTX *mem_ctx,
 	ebuf.dptr  = blob.data;
 
 	ret = tdb_store(tdb, kbuf, ebuf, 0);
-	if (ret == -1) {
+	if (ret != 0) {
 		tdb_unlock_bystring(tdb, EVT_NEXT_RECORD);
 		return NT_STATUS_EVENTLOG_FILE_CORRUPT;
 	}
 
 	ret = tdb_store_int32(tdb, EVT_NEXT_RECORD, r->record_number + 1);
-	if (ret == -1) {
+	if (ret != 0) {
 		tdb_unlock_bystring(tdb, EVT_NEXT_RECORD);
 		return NT_STATUS_EVENTLOG_FILE_CORRUPT;
 	}
diff --git a/source3/lib/eventlog/eventlog.h b/source3/lib/eventlog/eventlog.h
index 29c25c31222..694732d184a 100644
--- a/source3/lib/eventlog/eventlog.h
+++ b/source3/lib/eventlog/eventlog.h
@@ -17,6 +17,8 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tdb_compat.h"
+
 /* Defines for TDB keys */
 #define  EVT_OLDEST_ENTRY  "INFO/oldest_entry"
 #define  EVT_NEXT_RECORD   "INFO/next_record"
diff --git a/source3/lib/eventlog/proto.h b/source3/lib/eventlog/proto.h
index 21790d0795c..d3341ce16d8 100644
--- a/source3/lib/eventlog/proto.h
+++ b/source3/lib/eventlog/proto.h
@@ -1,3 +1,28 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Eventlog utility  routines
+ *
+ *  Copyright (C) Marcin Krzysztof Porwit    2005
+ *  Copyright (C) Brian Moran                2005
+ *  Copyright (C) Gerald (Jerry) Carter      2005
+ *  Copyright (C) Guenther Deschner          2009
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LIB_EVENTLOG_PROTO_H_
+#define _LIB_EVENTLOG_PROTO_H_
 
 /* The following definitions come from lib/eventlog/eventlog.c  */
 
@@ -33,3 +58,5 @@ NTSTATUS evlog_convert_tdb_to_evt(TALLOC_CTX *mem_ctx,
 				  ELOG_TDB *etdb,
 				  DATA_BLOB *blob_p,
 				  uint32_t *num_records_p);
+
+#endif /* _LIB_EVENTLOG_PROTO_H_ */
diff --git a/source3/lib/events.c b/source3/lib/events.c
index e461ebe977c..f077f585812 100644
--- a/source3/lib/events.c
+++ b/source3/lib/events.c
@@ -19,7 +19,7 @@
 */
 
 #include "includes.h"
-#include <tevent_internal.h>
+#include "lib/tevent/tevent_internal.h"
 #include "../lib/util/select.h"
 #include "system/select.h"
 
@@ -42,7 +42,7 @@ static struct tevent_poll_private *tevent_get_poll_private(
 
 	state = (struct tevent_poll_private *)ev->additional_data;
 	if (state == NULL) {
-		state = TALLOC_ZERO_P(ev, struct tevent_poll_private);
+		state = talloc_zero(ev, struct tevent_poll_private);
 		ev->additional_data = (void *)state;
 		if (state == NULL) {
 			DEBUG(10, ("talloc failed\n"));
@@ -90,7 +90,7 @@ bool event_add_to_poll_args(struct tevent_context *ev, TALLOC_CTX *mem_ctx,
 	idx_len = max_fd+1;
 
 	if (talloc_array_length(state->pollfd_idx) < idx_len) {
-		state->pollfd_idx = TALLOC_REALLOC_ARRAY(
+		state->pollfd_idx = talloc_realloc(
 			state, state->pollfd_idx, int, idx_len);
 		if (state->pollfd_idx == NULL) {
 			DEBUG(10, ("talloc_realloc failed\n"));
@@ -107,7 +107,7 @@ bool event_add_to_poll_args(struct tevent_context *ev, TALLOC_CTX *mem_ctx,
 	 */
 
 	if (talloc_array_length(fds) < num_pollfds + num_fds + 1) {
-		fds = TALLOC_REALLOC_ARRAY(mem_ctx, fds, struct pollfd,
+		fds = talloc_realloc(mem_ctx, fds, struct pollfd,
 					   num_pollfds + num_fds + 1);
 		if (fds == NULL) {
 			DEBUG(10, ("talloc_realloc failed\n"));
@@ -258,7 +258,20 @@ bool run_events_poll(struct tevent_context *ev, int pollrtn,
 			return false;
 		}
 
-		if (pfd->revents & (POLLIN|POLLHUP|POLLERR)) {
+		if (pfd->revents & (POLLHUP|POLLERR)) {
+			/* If we only wait for EVENT_FD_WRITE, we
+			   should not tell the event handler about it,
+			   and remove the writable flag, as we only
+			   report errors when waiting for read events
+			   to match the select behavior. */
+			if (!(fde->flags & EVENT_FD_READ)) {
+				EVENT_FD_NOT_WRITEABLE(fde);
+				continue;
+			}
+			flags |= EVENT_FD_READ;
+		}
+
+		if (pfd->revents & POLLIN) {
 			flags |= EVENT_FD_READ;
 		}
 		if (pfd->revents & POLLOUT) {
diff --git a/source3/lib/filename_util.c b/source3/lib/filename_util.c
index f12ca2d781b..2405183d53b 100644
--- a/source3/lib/filename_util.c
+++ b/source3/lib/filename_util.c
@@ -17,7 +17,6 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 #include "includes.h"
-#include "smbd/smbd.h"
 
 /**
  * XXX: This is temporary and there should be no callers of this outside of
@@ -61,8 +60,8 @@ NTSTATUS create_synthetic_smb_fname(TALLOC_CTX *ctx, const char *base_name,
 	ZERO_STRUCT(smb_fname_loc);
 
 	/* Setup the base_name/stream_name. */
-	smb_fname_loc.base_name = CONST_DISCARD(char *, base_name);
-	smb_fname_loc.stream_name = CONST_DISCARD(char *, stream_name);
+	smb_fname_loc.base_name = discard_const_p(char, base_name);
+	smb_fname_loc.stream_name = discard_const_p(char, stream_name);
 
 	/* Copy the psbuf if one was given. */
 	if (psbuf)
@@ -203,5 +202,5 @@ bool is_ntfs_default_stream_smb_fname(const struct smb_filename *smb_fname)
 		return false;
 	}
 
-	return StrCaseCmp(smb_fname->stream_name, "::$DATA") == 0;
+	return strcasecmp_m(smb_fname->stream_name, "::$DATA") == 0;
 }
diff --git a/source3/lib/fncall.c b/source3/lib/fncall.c
index e810b6814ed..6e6b7c92504 100644
--- a/source3/lib/fncall.c
+++ b/source3/lib/fncall.c
@@ -18,10 +18,11 @@
  */
 
 #include "includes.h"
+#include "../lib/util/tevent_unix.h"
 
 #if WITH_PTHREADPOOL
 
-#include "pthreadpool.h"
+#include "lib/pthreadpool/pthreadpool.h"
 
 struct fncall_state {
 	struct fncall_context *ctx;
@@ -86,7 +87,7 @@ struct fncall_context *fncall_context_init(TALLOC_CTX *mem_ctx,
 	}
 	talloc_set_destructor(ctx, fncall_context_destructor);
 
-	ctx->sig_fd = pthreadpool_sig_fd(ctx->pool);
+	ctx->sig_fd = pthreadpool_signal_fd(ctx->pool);
 	if (ctx->sig_fd == -1) {
 		TALLOC_FREE(ctx);
 		return NULL;
@@ -265,7 +266,7 @@ struct tevent_req *fncall_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
 		return tevent_req_post(req, ev);
 	}
 	if (!fncall_set_pending(req, state->ctx, ev)) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return tevent_req_post(req, ev);
 	}
 	return req;
diff --git a/source3/lib/g_lock.c b/source3/lib/g_lock.c
index 184da9b4fde..006ee3698cc 100644
--- a/source3/lib/g_lock.c
+++ b/source3/lib/g_lock.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "g_lock.h"
+#include "util_tdb.h"
 #include "ctdbd_conn.h"
 #include "../lib/util/select.h"
 #include "system/select.h"
@@ -107,7 +108,7 @@ static bool g_lock_parse(TALLOC_CTX *mem_ctx, TDB_DATA data,
 	DEBUG(10, ("locks:\n"));
 	for (i=0; i<num_locks; i++) {
 		DEBUGADD(10, ("%s: %s %s\n",
-			      procid_str(talloc_tos(), &locks[i].pid),
+			      server_id_str(talloc_tos(), &locks[i].pid),
 			      ((locks[i].lock_type & 1) == G_LOCK_READ) ?
 			      "read" : "write",
 			      (locks[i].lock_type & G_LOCK_PENDING) ?
@@ -117,7 +118,7 @@ static bool g_lock_parse(TALLOC_CTX *mem_ctx, TDB_DATA data,
 		    && !process_exists(locks[i].pid)) {
 
 			DEBUGADD(10, ("lock owner %s died -- discarding\n",
-				      procid_str(talloc_tos(),
+				      server_id_str(talloc_tos(),
 						 &locks[i].pid)));
 
 			if (i < (num_locks-1)) {
@@ -145,7 +146,7 @@ static void g_lock_cleanup(int *pnum_locks, struct g_lock_rec *locks)
 			continue;
 		}
 		DEBUGADD(10, ("%s does not exist -- discarding\n",
-			      procid_str(talloc_tos(), &locks[i].pid)));
+			      server_id_str(talloc_tos(), &locks[i].pid)));
 
 		if (i < (num_locks-1)) {
 			locks[i] = locks[num_locks-1];
@@ -392,7 +393,7 @@ NTSTATUS g_lock_lock(struct g_lock_ctx *ctx, const char *name,
 		 * sys_poll and in the clustering case we might have to add
 		 * the ctdb fd. This avoids the realloc then.
 		 */
-		pollfds = TALLOC_ARRAY(talloc_tos(), struct pollfd, 2);
+		pollfds = talloc_array(talloc_tos(), struct pollfd, 2);
 		if (pollfds == NULL) {
 			status = NT_STATUS_NO_MEMORY;
 			break;
@@ -495,7 +496,7 @@ static void g_lock_got_retry(struct messaging_context *msg,
 	bool *pretry = (bool *)private_data;
 
 	DEBUG(10, ("Got retry message from pid %s\n",
-		   procid_str(talloc_tos(), &server_id)));
+		   server_id_str(talloc_tos(), &server_id)));
 
 	*pretry = true;
 }
@@ -586,7 +587,7 @@ static NTSTATUS g_lock_force_unlock(struct g_lock_ctx *ctx, const char *name,
 						&data_blob_null);
 			if (!NT_STATUS_IS_OK(status)) {
 				DEBUG(1, ("sending retry to %s failed: %s\n",
-					  procid_str(talloc_tos(),
+					  server_id_str(talloc_tos(),
 						     &locks[i].pid),
 					  nt_errstr(status)));
 			} else {
diff --git a/source3/lib/gencache.c b/source3/lib/gencache.c
index 67c37f34e73..31ead7be7e3 100644
--- a/source3/lib/gencache.c
+++ b/source3/lib/gencache.c
@@ -24,6 +24,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "system/glob.h"
+#include "util_tdb.h"
 
 #undef  DBGC_CLASS
 #define DBGC_CLASS DBGC_TDB
@@ -80,14 +81,10 @@ again:
 				return false;
 			}
 			first_try = false;
-			DEBUG(0, ("gencache_init: tdb_check(%s) failed - retry after CLEAR_IF_FIRST\n",
+			DEBUG(0, ("gencache_init: tdb_check(%s) failed - retry after truncate\n",
 				  cache_fname));
-			cache = tdb_open_log(cache_fname, 0, TDB_CLEAR_IF_FIRST|TDB_INCOMPATIBLE_HASH, open_flags, 0644);
-			if (cache) {
-				tdb_close(cache);
-				cache = NULL;
-				goto again;
-			}
+			truncate(cache_fname, 0);
+			goto again;
 		}
 	}
 
@@ -125,7 +122,7 @@ again:
 static TDB_DATA last_stabilize_key(void)
 {
 	TDB_DATA result;
-	result.dptr = (uint8_t *)"@LAST_STABILIZED";
+	result.dptr = discard_const_p(uint8_t, "@LAST_STABILIZED");
 	result.dsize = 17;
 	return result;
 }
@@ -209,7 +206,7 @@ bool gencache_set_data_blob(const char *keystr, const DATA_BLOB *blob,
 	 */
 
 	last_stabilize = 0;
-	databuf = tdb_fetch(cache_notrans, last_stabilize_key());
+	databuf = tdb_fetch_compat(cache_notrans, last_stabilize_key());
 	if ((databuf.dptr != NULL)
 	    && (databuf.dptr[databuf.dsize-1] == '\0')) {
 		last_stabilize = atoi((char *)databuf.dptr);
@@ -347,11 +344,11 @@ bool gencache_parse(const char *keystr,
 	state.private_data = private_data;
 
 	ret = tdb_parse_record(cache_notrans, key, gencache_parse_fn, &state);
-	if (ret != -1) {
+	if (ret == 0) {
 		return true;
 	}
 	ret = tdb_parse_record(cache, key, gencache_parse_fn, &state);
-	return (ret != -1);
+	return (ret == 0);
 }
 
 struct gencache_get_data_blob_state {
@@ -464,9 +461,14 @@ bool gencache_stabilize(void)
 	}
 
 	res = tdb_transaction_start_nonblock(cache);
-	if (res == -1) {
-
-		if (tdb_error(cache) == TDB_ERR_NOLOCK) {
+	if (res != 0) {
+
+#if BUILD_TDB2
+		if (res == TDB_ERR_LOCK)
+#else
+		if (tdb_error(cache) == TDB_ERR_NOLOCK)
+#endif
+		{
 			/*
 			 * Someone else already does the stabilize,
 			 * this does not have to be done twice
@@ -475,15 +477,15 @@ bool gencache_stabilize(void)
 		}
 
 		DEBUG(10, ("Could not start transaction on gencache.tdb: "
-			   "%s\n", tdb_errorstr(cache)));
+			   "%s\n", tdb_errorstr_compat(cache)));
 		return false;
 	}
 	res = tdb_transaction_start(cache_notrans);
-	if (res == -1) {
+	if (res != 0) {
 		tdb_transaction_cancel(cache);
 		DEBUG(10, ("Could not start transaction on "
 			   "gencache_notrans.tdb: %s\n",
-			   tdb_errorstr(cache_notrans)));
+			   tdb_errorstr_compat(cache_notrans)));
 		return false;
 	}
 
@@ -491,36 +493,30 @@ bool gencache_stabilize(void)
 	state.written = false;
 
 	res = tdb_traverse(cache_notrans, stabilize_fn, &state);
-	if ((res == -1) || state.error) {
-		if ((tdb_transaction_cancel(cache_notrans) == -1)
-		    || (tdb_transaction_cancel(cache) == -1)) {
-			smb_panic("tdb_transaction_cancel failed\n");
-		}
+	if ((res < 0) || state.error) {
+		tdb_transaction_cancel(cache_notrans);
+		tdb_transaction_cancel(cache);
 		return false;
 	}
 
 	if (!state.written) {
-		if ((tdb_transaction_cancel(cache_notrans) == -1)
-		    || (tdb_transaction_cancel(cache) == -1)) {
-			smb_panic("tdb_transaction_cancel failed\n");
-		}
+		tdb_transaction_cancel(cache_notrans);
+		tdb_transaction_cancel(cache);
 		return true;
 	}
 
 	res = tdb_transaction_commit(cache);
-	if (res == -1) {
+	if (res != 0) {
 		DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
-			   "%s\n", tdb_errorstr(cache)));
-		if (tdb_transaction_cancel(cache_notrans) == -1) {
-			smb_panic("tdb_transaction_cancel failed\n");
-		}
+			   "%s\n", tdb_errorstr_compat(cache)));
+		tdb_transaction_cancel(cache_notrans);
 		return false;
 	}
 
 	res = tdb_transaction_commit(cache_notrans);
-	if (res == -1) {
+	if (res != 0) {
 		DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
-			   "%s\n", tdb_errorstr(cache)));
+			   "%s\n", tdb_errorstr_compat(cache)));
 		return false;
 	}
 
@@ -551,7 +547,7 @@ static int stabilize_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA val,
 	}
 	if ((timeout < time(NULL)) || (val.dsize == 0)) {
 		res = tdb_delete(cache, key);
-		if ((res == -1) && (tdb_error(cache) == TDB_ERR_NOEXIST)) {
+		if ((res != 0) && (tdb_error(cache) == TDB_ERR_NOEXIST)) {
 			res = 0;
 		} else {
 			state->written = true;
@@ -563,16 +559,16 @@ static int stabilize_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA val,
 		}
 	}
 
-	if (res == -1) {
+	if (res != 0) {
 		DEBUG(10, ("Transfer to gencache.tdb failed: %s\n",
-			   tdb_errorstr(cache)));
+			   tdb_errorstr_compat(cache)));
 		state->error = true;
 		return -1;
 	}
 
-	if (tdb_delete(cache_notrans, key) == -1) {
+	if (tdb_delete(cache_notrans, key) != 0) {
 		DEBUG(10, ("tdb_delete from gencache_notrans.tdb failed: "
-			   "%s\n", tdb_errorstr(cache_notrans)));
+			   "%s\n", tdb_errorstr_compat(cache_notrans)));
 		state->error = true;
 		return -1;
 	}
diff --git a/source3/lib/idmap_cache.h b/source3/lib/idmap_cache.h
index af2ba82bf8d..1a62dba64fc 100644
--- a/source3/lib/idmap_cache.h
+++ b/source3/lib/idmap_cache.h
@@ -1,3 +1,26 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * ID Mapping Cache
+ *
+ * Copyright (C) Volker Lendecke	2008
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LIB_IDMAP_CACHE_H_
+#define _LIB_IDMAP_CACHE_H_
+
 /* The following definitions come from lib/idmap_cache.c  */
 
 bool idmap_cache_find_sid2uid(const struct dom_sid *sid, uid_t *puid,
@@ -12,3 +35,5 @@ void idmap_cache_set_sid2gid(const struct dom_sid *sid, gid_t gid);
 bool idmap_cache_del_uid(uid_t uid);
 bool idmap_cache_del_gid(gid_t gid);
 bool idmap_cache_del_sid(const struct dom_sid *sid);
+
+#endif /* _LIB_IDMAP_CACHE_H_ */
diff --git a/source3/lib/interface.c b/source3/lib/interface.c
index 30e94069a52..39dc9cb04c2 100644
--- a/source3/lib/interface.c
+++ b/source3/lib/interface.c
@@ -19,7 +19,7 @@
 */
 
 #include "includes.h"
-#include "interfaces.h"
+#include "lib/socket/interfaces.h"
 
 static struct iface_struct *probed_ifaces;
 static int total_probed;
@@ -305,7 +305,7 @@ static void add_interface(const struct iface_struct *ifs)
 	char addr[INET6_ADDRSTRLEN];
 	struct interface *iface;
 
-	if (iface_find((struct sockaddr *)&ifs->ip, False)) {
+	if (iface_find((const struct sockaddr *)&ifs->ip, False)) {
 		DEBUG(3,("add_interface: not adding duplicate interface %s\n",
 			print_sockaddr(addr, sizeof(addr), &ifs->ip) ));
 		return;
diff --git a/source3/lib/memcache.c b/source3/lib/memcache.c
index 425861ed77a..88453f32dd8 100644
--- a/source3/lib/memcache.c
+++ b/source3/lib/memcache.c
@@ -72,7 +72,7 @@ struct memcache *memcache_init(TALLOC_CTX *mem_ctx, size_t max_size)
 {
 	struct memcache *result;
 
-	result = TALLOC_ZERO_P(mem_ctx, struct memcache);
+	result = talloc_zero(mem_ctx, struct memcache);
 	if (result == NULL) {
 		return NULL;
 	}
diff --git a/source3/lib/messages.c b/source3/lib/messages.c
index 4335554c2a0..76c1090b819 100644
--- a/source3/lib/messages.c
+++ b/source3/lib/messages.c
@@ -112,7 +112,7 @@ static int traverse_fn(struct db_record *rec, const struct server_id *id,
 	 * the msg has already been deleted from the messages.tdb.*/
 
 	status = messaging_send_buf(msg_all->msg_ctx, *id, msg_all->msg_type,
-				    (uint8 *)msg_all->buf, msg_all->len);
+				    (const uint8 *)msg_all->buf, msg_all->len);
 
 	if (NT_STATUS_EQUAL(status, NT_STATUS_INVALID_HANDLE)) {
 		
@@ -184,7 +184,7 @@ struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
 	struct messaging_context *ctx;
 	NTSTATUS status;
 
-	if (!(ctx = TALLOC_ZERO_P(mem_ctx, struct messaging_context))) {
+	if (!(ctx = talloc_zero(mem_ctx, struct messaging_context))) {
 		return NULL;
 	}
 
@@ -205,7 +205,7 @@ struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
 		status = messaging_ctdbd_init(ctx, ctx, &ctx->remote);
 
 		if (!NT_STATUS_IS_OK(status)) {
-			DEBUG(2, ("messaging_ctdb_init failed: %s\n",
+			DEBUG(2, ("messaging_ctdbd_init failed: %s\n",
 				  nt_errstr(status)));
 			TALLOC_FREE(ctx);
 			return NULL;
@@ -257,7 +257,7 @@ NTSTATUS messaging_reinit(struct messaging_context *msg_ctx,
 					      &msg_ctx->remote);
 
 		if (!NT_STATUS_IS_OK(status)) {
-			DEBUG(1, ("messaging_ctdb_init failed: %s\n",
+			DEBUG(1, ("messaging_ctdbd_init failed: %s\n",
 				  nt_errstr(status)));
 			return status;
 		}
diff --git a/source3/lib/messages_ctdbd.c b/source3/lib/messages_ctdbd.c
index 3e2fde0b8fd..b9ee049d01d 100644
--- a/source3/lib/messages_ctdbd.c
+++ b/source3/lib/messages_ctdbd.c
@@ -19,6 +19,7 @@
 
 #include "includes.h"
 #include "messages.h"
+#include "util_tdb.h"
 
 #ifdef CLUSTER_SUPPORT
 
@@ -104,12 +105,12 @@ NTSTATUS messaging_ctdbd_init(struct messaging_context *msg_ctx,
 	struct messaging_ctdbd_context *ctx;
 	NTSTATUS status;
 
-	if (!(result = TALLOC_P(mem_ctx, struct messaging_backend))) {
+	if (!(result = talloc(mem_ctx, struct messaging_backend))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	if (!(ctx = TALLOC_P(result, struct messaging_ctdbd_context))) {
+	if (!(ctx = talloc(result, struct messaging_ctdbd_context))) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(result);
 		return NT_STATUS_NO_MEMORY;
diff --git a/source3/lib/messages_local.c b/source3/lib/messages_local.c
index 18074cdd925..455f3d34766 100644
--- a/source3/lib/messages_local.c
+++ b/source3/lib/messages_local.c
@@ -45,6 +45,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "messages.h"
+#include "lib/util/tdb_wrap.h"
 
 struct messaging_tdb_context {
 	struct messaging_context *msg_ctx;
@@ -86,12 +87,12 @@ NTSTATUS messaging_tdb_init(struct messaging_context *msg_ctx,
 	struct messaging_backend *result;
 	struct messaging_tdb_context *ctx;
 
-	if (!(result = TALLOC_P(mem_ctx, struct messaging_backend))) {
+	if (!(result = talloc(mem_ctx, struct messaging_backend))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	ctx = TALLOC_ZERO_P(result, struct messaging_tdb_context);
+	ctx = talloc_zero(result, struct messaging_tdb_context);
 	if (!ctx) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(result);
@@ -186,11 +187,11 @@ static NTSTATUS messaging_tdb_fetch(TDB_CONTEXT *msg_tdb,
 	DATA_BLOB blob;
 	enum ndr_err_code ndr_err;
 
-	if (!(result = TALLOC_ZERO_P(mem_ctx, struct messaging_array))) {
+	if (!(result = talloc_zero(mem_ctx, struct messaging_array))) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	data = tdb_fetch(msg_tdb, key);
+	data = tdb_fetch_compat(msg_tdb, key);
 
 	if (data.dptr == NULL) {
 		*presult = result;
@@ -351,7 +352,7 @@ static NTSTATUS messaging_tdb_send(struct messaging_context *msg_ctx,
 
 	key = message_key_pid(frame, pid);
 
-	if (tdb_chainlock(tdb->tdb, key) == -1) {
+	if (tdb_chainlock(tdb->tdb, key) != 0) {
 		TALLOC_FREE(frame);
 		return NT_STATUS_LOCK_NOT_GRANTED;
 	}
@@ -370,7 +371,7 @@ static NTSTATUS messaging_tdb_send(struct messaging_context *msg_ctx,
 		goto done;
 	}
 
-	if (!(rec = TALLOC_REALLOC_ARRAY(talloc_tos(), msg_array->messages,
+	if (!(rec = talloc_realloc(talloc_tos(), msg_array->messages,
 					 struct messaging_rec,
 					 msg_array->num_messages+1))) {
 		status = NT_STATUS_NO_MEMORY;
@@ -419,7 +420,7 @@ static NTSTATUS retrieve_all_messages(TDB_CONTEXT *msg_tdb,
 	TDB_DATA key = message_key_pid(mem_ctx, id);
 	NTSTATUS status;
 
-	if (tdb_chainlock(msg_tdb, key) == -1) {
+	if (tdb_chainlock(msg_tdb, key) != 0) {
 		TALLOC_FREE(key.dptr);
 		return NT_STATUS_LOCK_NOT_GRANTED;
 	}
diff --git a/source3/lib/module.c b/source3/lib/module.c
index de136680092..bec4fddefdc 100644
--- a/source3/lib/module.c
+++ b/source3/lib/module.c
@@ -117,10 +117,10 @@ NTSTATUS smb_probe_module(const char *subsystem, const char *module)
 	}
 
 	full_path = talloc_asprintf(ctx,
-			"%s/%s.%s",
-			modules_path(subsystem),
-			module,
-			shlib_ext());
+				    "%s/%s.%s",
+				    modules_path(ctx, subsystem),
+				    module,
+				    shlib_ext());
 	if (!full_path) {
 		TALLOC_FREE(ctx);
 		return NT_STATUS_NO_MEMORY;
diff --git a/source3/lib/ms_fnmatch.c b/source3/lib/ms_fnmatch.c
index 31c66953a96..f02354bfd7d 100644
--- a/source3/lib/ms_fnmatch.c
+++ b/source3/lib/ms_fnmatch.c
@@ -166,7 +166,7 @@ int ms_fnmatch(const char *pattern, const char *string, bool translate_pattern,
 		if (is_case_sensitive) {
 			return strcmp(pattern, string);
 		} else {
-			return StrCaseCmp(pattern, string);
+			return strcasecmp_m(pattern, string);
 		}
 	}
 
@@ -229,10 +229,3 @@ int ms_fnmatch(const char *pattern, const char *string, bool translate_pattern,
 	TALLOC_FREE(s);
 	return ret;
 }
-
-
-/* a generic fnmatch function - uses for non-CIFS pattern matching */
-int gen_fnmatch(const char *pattern, const char *string)
-{
-	return ms_fnmatch(pattern, string, true, False);
-}
diff --git a/source3/lib/namearray.c b/source3/lib/namearray.c
new file mode 100644
index 00000000000..e5c3bd983b5
--- /dev/null
+++ b/source3/lib/namearray.c
@@ -0,0 +1,39 @@
+/*
+   Unix SMB/CIFS implementation.
+   Samba utility functions
+   Copyright (C) Andrew Tridgell 1992-1998
+   Copyright (C) Jeremy Allison 2001-2007
+   Copyright (C) Simo Sorce 2001
+   Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2003
+   Copyright (C) James Peach 2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+/****************************************************************************
+ Routine to free a namearray.
+****************************************************************************/
+
+void free_namearray(name_compare_entry *name_array)
+{
+	int i;
+
+	if(name_array == NULL)
+		return;
+
+	for(i=0; name_array[i].name!=NULL; i++)
+		SAFE_FREE(name_array[i].name);
+	SAFE_FREE(name_array);
+}
diff --git a/source3/lib/netapi/cm.c b/source3/lib/netapi/cm.c
index 290194ba2d9..47ccf8bb7a3 100644
--- a/source3/lib/netapi/cm.c
+++ b/source3/lib/netapi/cm.c
@@ -22,6 +22,7 @@
 
 #include "lib/netapi/netapi.h"
 #include "lib/netapi/netapi_private.h"
+#include "libsmb/libsmb.h"
 #include "rpc_client/cli_pipe.h"
 
 /********************************************************************
@@ -121,7 +122,7 @@ static WERROR libnetapi_open_ipc_connection(struct libnetapi_ctx *ctx,
 		return WERR_CAN_NOT_COMPLETE;
 	}
 
-	p = TALLOC_ZERO_P(ctx, struct client_ipc_connection);
+	p = talloc_zero(ctx, struct client_ipc_connection);
 	if (p == NULL) {
 		return WERR_NOMEM;
 	}
@@ -187,7 +188,7 @@ static NTSTATUS pipe_cm_connect(TALLOC_CTX *mem_ctx,
 	struct client_pipe_connection *p;
 	NTSTATUS status;
 
-	p = TALLOC_ZERO_ARRAY(mem_ctx, struct client_pipe_connection, 1);
+	p = talloc_zero_array(mem_ctx, struct client_pipe_connection, 1);
 	if (!p) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/lib/netapi/group.c b/source3/lib/netapi/group.c
index 4295d9f7bbf..710ec3790f2 100644
--- a/source3/lib/netapi/group.c
+++ b/source3/lib/netapi/group.c
@@ -1157,7 +1157,7 @@ static WERROR convert_samr_disp_groups_to_GROUP_INFO_0_buffer(TALLOC_CTX *mem_ct
 	struct GROUP_INFO_0 *g0;
 	int i;
 
-	g0 = TALLOC_ZERO_ARRAY(mem_ctx, struct GROUP_INFO_0, groups->count);
+	g0 = talloc_zero_array(mem_ctx, struct GROUP_INFO_0, groups->count);
 	W_ERROR_HAVE_NO_MEMORY(g0);
 
 	for (i=0; i<groups->count; i++) {
@@ -1183,7 +1183,7 @@ static WERROR convert_samr_disp_groups_to_GROUP_INFO_1_buffer(TALLOC_CTX *mem_ct
 	struct GROUP_INFO_1 *g1;
 	int i;
 
-	g1 = TALLOC_ZERO_ARRAY(mem_ctx, struct GROUP_INFO_1, groups->count);
+	g1 = talloc_zero_array(mem_ctx, struct GROUP_INFO_1, groups->count);
 	W_ERROR_HAVE_NO_MEMORY(g1);
 
 	for (i=0; i<groups->count; i++) {
@@ -1211,7 +1211,7 @@ static WERROR convert_samr_disp_groups_to_GROUP_INFO_2_buffer(TALLOC_CTX *mem_ct
 	struct GROUP_INFO_2 *g2;
 	int i;
 
-	g2 = TALLOC_ZERO_ARRAY(mem_ctx, struct GROUP_INFO_2, groups->count);
+	g2 = talloc_zero_array(mem_ctx, struct GROUP_INFO_2, groups->count);
 	W_ERROR_HAVE_NO_MEMORY(g2);
 
 	for (i=0; i<groups->count; i++) {
@@ -1242,7 +1242,7 @@ static WERROR convert_samr_disp_groups_to_GROUP_INFO_3_buffer(TALLOC_CTX *mem_ct
 	struct GROUP_INFO_3 *g3;
 	int i;
 
-	g3 = TALLOC_ZERO_ARRAY(mem_ctx, struct GROUP_INFO_3, groups->count);
+	g3 = talloc_zero_array(mem_ctx, struct GROUP_INFO_3, groups->count);
 	W_ERROR_HAVE_NO_MEMORY(g3);
 
 	for (i=0; i<groups->count; i++) {
diff --git a/source3/lib/netapi/libnetapi.h b/source3/lib/netapi/libnetapi.h
index 2711558dbc1..e832deaccc7 100644
--- a/source3/lib/netapi/libnetapi.h
+++ b/source3/lib/netapi/libnetapi.h
@@ -1,3 +1,22 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  NetApi Support
+ *  Copyright (C) Guenther Deschner 2007-2008
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
 #ifndef __LIBNETAPI_LIBNETAPI__
 #define __LIBNETAPI_LIBNETAPI__
 NET_API_STATUS NetJoinDomain(const char * server /* [in] [unique] */,
diff --git a/source3/lib/netapi/localgroup.c b/source3/lib/netapi/localgroup.c
index 51f4e1d7459..816afc230ff 100644
--- a/source3/lib/netapi/localgroup.c
+++ b/source3/lib/netapi/localgroup.c
@@ -610,7 +610,7 @@ static WERROR map_buffer_to_alias_info(TALLOC_CTX *mem_ctx,
 	struct LOCALGROUP_INFO_1002 *info1002;
 	union samr_AliasInfo *info = NULL;
 
-	info = TALLOC_ZERO_P(mem_ctx, union samr_AliasInfo);
+	info = talloc_zero(mem_ctx, union samr_AliasInfo);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	switch (level) {
@@ -1115,7 +1115,7 @@ static WERROR NetLocalGroupModifyMembers_r(struct libnetapi_ctx *ctx,
 	ZERO_STRUCT(domain_handle);
 	ZERO_STRUCT(alias_handle);
 
-	member_sids = TALLOC_ZERO_ARRAY(ctx, struct dom_sid,
+	member_sids = talloc_zero_array(ctx, struct dom_sid,
 					r->in.total_entries);
 	W_ERROR_HAVE_NO_MEMORY(member_sids);
 
@@ -1344,7 +1344,7 @@ static WERROR NetLocalGroupModifyMembers_r(struct libnetapi_ctx *ctx,
 	werr = WERR_OK;
 
  done:
-	if (is_valid_policy_hnd(&alias_handle)) {
+	if (b && is_valid_policy_hnd(&alias_handle)) {
 		dcerpc_samr_Close(b, talloc_tos(), &alias_handle, &result);
 	}
 
diff --git a/source3/lib/netapi/netapi.c b/source3/lib/netapi/netapi.c
index cd590273ce2..14259864ae2 100644
--- a/source3/lib/netapi/netapi.c
+++ b/source3/lib/netapi/netapi.c
@@ -24,7 +24,6 @@
 #include "krb5_env.h"
 
 struct libnetapi_ctx *stat_ctx = NULL;
-TALLOC_CTX *frame = NULL;
 static bool libnetapi_initialized = false;
 
 /****************************************************************
@@ -38,7 +37,7 @@ static NET_API_STATUS libnetapi_init_private_context(struct libnetapi_ctx *ctx)
 		return W_ERROR_V(WERR_INVALID_PARAM);
 	}
 
-	priv = TALLOC_ZERO_P(ctx, struct libnetapi_private_ctx);
+	priv = talloc_zero(ctx, struct libnetapi_private_ctx);
 	if (!priv) {
 		return W_ERROR_V(WERR_NOMEM);
 	}
@@ -57,6 +56,8 @@ were not expecting it.
 
 NET_API_STATUS libnetapi_init(struct libnetapi_ctx **context)
 {
+	NET_API_STATUS ret;
+	TALLOC_CTX *frame;
 	if (stat_ctx && libnetapi_initialized) {
 		*context = stat_ctx;
 		return NET_API_STATUS_SUCCESS;
@@ -76,7 +77,7 @@ NET_API_STATUS libnetapi_init(struct libnetapi_ctx **context)
 	lp_set_cmdline("log level", "0");
 	setup_logging("libnetapi", DEBUG_STDERR);
 
-	if (!lp_load(get_dyn_CONFIGFILE(), true, false, false, false)) {
+	if (!lp_load(get_dyn_CONFIGFILE(), true, false, false, true)) {
 		TALLOC_FREE(frame);
 		fprintf(stderr, "error loading %s\n", get_dyn_CONFIGFILE() );
 		return W_ERROR_V(WERR_GENERAL_FAILURE);
@@ -88,7 +89,9 @@ NET_API_STATUS libnetapi_init(struct libnetapi_ctx **context)
 
 	BlockSignals(True, SIGPIPE);
 
-	return libnetapi_net_init(context);
+	ret = libnetapi_net_init(context);
+	TALLOC_FREE(frame);
+	return ret;
 }
 
 /****************************************************************
@@ -105,7 +108,7 @@ NET_API_STATUS libnetapi_net_init(struct libnetapi_ctx **context)
 	struct libnetapi_ctx *ctx = NULL;
 	char *krb5_cc_env = NULL;
 
-	frame = talloc_stackframe();
+	TALLOC_CTX *frame = talloc_stackframe();
 
 	ctx = talloc_zero(frame, struct libnetapi_ctx);
 	if (!ctx) {
@@ -117,14 +120,14 @@ NET_API_STATUS libnetapi_net_init(struct libnetapi_ctx **context)
 
 	krb5_cc_env = getenv(KRB5_ENV_CCNAME);
 	if (!krb5_cc_env || (strlen(krb5_cc_env) == 0)) {
-		ctx->krb5_cc_env = talloc_strdup(frame, "MEMORY:libnetapi");
+		ctx->krb5_cc_env = talloc_strdup(ctx, "MEMORY:libnetapi");
 		setenv(KRB5_ENV_CCNAME, ctx->krb5_cc_env, 1);
 	}
 
 	if (getenv("USER")) {
-		ctx->username = talloc_strdup(frame, getenv("USER"));
+		ctx->username = talloc_strdup(ctx, getenv("USER"));
 	} else {
-		ctx->username = talloc_strdup(frame, "");
+		ctx->username = talloc_strdup(ctx, "");
 	}
 	if (!ctx->username) {
 		TALLOC_FREE(frame);
@@ -140,8 +143,10 @@ NET_API_STATUS libnetapi_net_init(struct libnetapi_ctx **context)
 
 	libnetapi_initialized = true;
 
+	talloc_steal(NULL, ctx);
 	*context = stat_ctx = ctx;
-
+	
+	TALLOC_FREE(frame);
 	return NET_API_STATUS_SUCCESS;
 }
 
@@ -187,8 +192,10 @@ NET_API_STATUS libnetapi_free(struct libnetapi_ctx *ctx)
 
 	secrets_shutdown();
 
+	if (ctx == stat_ctx) {
+		stat_ctx = NULL;
+	}
 	TALLOC_FREE(ctx);
-	TALLOC_FREE(frame);
 
 	gfree_debugsyms();
 
@@ -202,10 +209,14 @@ NET_API_STATUS libnetapi_free(struct libnetapi_ctx *ctx)
 NET_API_STATUS libnetapi_set_debuglevel(struct libnetapi_ctx *ctx,
 					const char *debuglevel)
 {
+	TALLOC_CTX *frame = talloc_stackframe();
 	ctx->debuglevel = talloc_strdup(ctx, debuglevel);
+	
 	if (!lp_set_cmdline("log level", debuglevel)) {
+		TALLOC_FREE(frame);
 		return W_ERROR_V(WERR_GENERAL_FAILURE);
 	}
+	TALLOC_FREE(frame);
 	return NET_API_STATUS_SUCCESS;
 }
 
@@ -272,15 +283,22 @@ NET_API_STATUS libnetapi_set_use_ccache(struct libnetapi_ctx *ctx)
 }
 
 /****************************************************************
+Return a libnetapi error as a string, caller must free with NetApiBufferFree
 ****************************************************************/
 
-const char *libnetapi_errstr(NET_API_STATUS status)
+char *libnetapi_errstr(NET_API_STATUS status)
 {
+	TALLOC_CTX *frame = talloc_stackframe();
+	char *ret;
 	if (status & 0xc0000000) {
-		return get_friendly_nt_error_msg(NT_STATUS(status));
+		ret = talloc_strdup(NULL, 
+				     get_friendly_nt_error_msg(NT_STATUS(status)));
+	} else {
+		ret = talloc_strdup(NULL,
+				    get_friendly_werror_msg(W_ERROR(status)));
 	}
-
-	return get_friendly_werror_msg(W_ERROR(status));
+	TALLOC_FREE(frame);
+	return ret;
 }
 
 /****************************************************************
@@ -304,9 +322,10 @@ NET_API_STATUS libnetapi_set_error_string(struct libnetapi_ctx *ctx,
 }
 
 /****************************************************************
+Return a libnetapi_errstr(), caller must free with NetApiBufferFree
 ****************************************************************/
 
-const char *libnetapi_get_error_string(struct libnetapi_ctx *ctx,
+char *libnetapi_get_error_string(struct libnetapi_ctx *ctx,
 				       NET_API_STATUS status_in)
 {
 	NET_API_STATUS status;
@@ -320,7 +339,7 @@ const char *libnetapi_get_error_string(struct libnetapi_ctx *ctx,
 	}
 
 	if (tmp_ctx->error_string) {
-		return tmp_ctx->error_string;
+		return talloc_strdup(NULL, tmp_ctx->error_string);
 	}
 
 	return libnetapi_errstr(status_in);
diff --git a/source3/lib/netapi/netapi.h b/source3/lib/netapi/netapi.h
index 9e1549df0eb..620d8cf5910 100644
--- a/source3/lib/netapi/netapi.h
+++ b/source3/lib/netapi/netapi.h
@@ -1411,15 +1411,18 @@ NET_API_STATUS libnetapi_set_use_kerberos(struct libnetapi_ctx *ctx);
 NET_API_STATUS libnetapi_set_use_ccache(struct libnetapi_ctx *ctx);
 
 /****************************************************************
+Return a specific libnetapi error as a string, caller must free with NetApiBufferFree
 ****************************************************************/
 
-const char *libnetapi_errstr(NET_API_STATUS status);
+char *libnetapi_errstr(NET_API_STATUS status);
 
 /****************************************************************
+Return the last libnetapi error as a string, caller must free with NetApiBufferFree
+ctx is optional
 ****************************************************************/
 
-const char *libnetapi_get_error_string(struct libnetapi_ctx *ctx,
-				       NET_API_STATUS status);
+char *libnetapi_get_error_string(struct libnetapi_ctx *ctx,
+				 NET_API_STATUS status);
 
 /****************************************************************
  NetApiBufferAllocate
diff --git a/source3/lib/netapi/netapi_private.h b/source3/lib/netapi/netapi_private.h
index 7559c615647..349287b9e96 100644
--- a/source3/lib/netapi/netapi_private.h
+++ b/source3/lib/netapi/netapi_private.h
@@ -29,6 +29,8 @@
 	} \
 	return fn ## _r(ctx, r);
 
+struct dcerpc_binding_handle;
+
 struct libnetapi_private_ctx {
 	struct {
 		const char *domain_name;
diff --git a/source3/lib/netapi/samr.c b/source3/lib/netapi/samr.c
index 544698004a0..1c3a94ee9cd 100644
--- a/source3/lib/netapi/samr.c
+++ b/source3/lib/netapi/samr.c
@@ -240,7 +240,7 @@ WERROR libnetapi_samr_open_builtin_domain(struct libnetapi_ctx *mem_ctx,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					connect_handle,
 					builtin_mask,
-					CONST_DISCARD(struct dom_sid *, &global_sid_Builtin),
+					discard_const_p(struct dom_sid, &global_sid_Builtin),
 					builtin_handle,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
diff --git a/source3/lib/netapi/serverinfo.c b/source3/lib/netapi/serverinfo.c
index 22c7df69d12..4b11339f945 100644
--- a/source3/lib/netapi/serverinfo.c
+++ b/source3/lib/netapi/serverinfo.c
@@ -36,9 +36,9 @@ static WERROR NetServerGetInfo_l_101(struct libnetapi_ctx *ctx,
 	struct SERVER_INFO_101 i;
 
 	i.sv101_platform_id	= PLATFORM_ID_NT;
-	i.sv101_name		= global_myname();
-	i.sv101_version_major	= lp_major_announce_version();
-	i.sv101_version_minor	= lp_minor_announce_version();
+	i.sv101_name		= lp_netbios_name();
+	i.sv101_version_major	= SAMBA_MAJOR_NBT_ANNOUNCE_VERSION;
+	i.sv101_version_minor	= SAMBA_MINOR_NBT_ANNOUNCE_VERSION;
 	i.sv101_type		= lp_default_server_announce();
 	i.sv101_comment		= lp_serverstring();
 
@@ -540,7 +540,8 @@ WERROR NetServerGetInfo_r(struct libnetapi_ctx *ctx,
 static WERROR NetServerSetInfo_l_1005(struct libnetapi_ctx *ctx,
 				      struct NetServerSetInfo *r)
 {
-	WERROR werr;
+	WERROR werr = WERR_OK;
+	sbcErr err;
 	struct smbconf_ctx *conf_ctx;
 	struct srvsvc_NetSrvInfo1005 *info1005;
 
@@ -563,13 +564,24 @@ static WERROR NetServerSetInfo_l_1005(struct libnetapi_ctx *ctx,
 		return WERR_NOT_SUPPORTED;
 	}
 
-	werr = smbconf_init_reg(ctx, &conf_ctx, NULL);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_init_reg(ctx, &conf_ctx, NULL);
+	if (!SBC_ERROR_IS_OK(err)) {
+		libnetapi_set_error_string(ctx,
+			"Could not initialize backend: %s",
+			sbcErrorString(err));
+		werr = WERR_NO_SUCH_SERVICE;
 		goto done;
 	}
 
-	werr = smbconf_set_global_parameter(conf_ctx, "server string",
+	err = smbconf_set_global_parameter(conf_ctx, "server string",
 					    info1005->comment);
+	if (!SBC_ERROR_IS_OK(err)) {
+		libnetapi_set_error_string(ctx,
+			"Could not set global parameter: %s",
+			sbcErrorString(err));
+		werr = WERR_NO_SUCH_SERVICE;
+		goto done;
+	}
 
  done:
 	smbconf_shutdown(conf_ctx);
diff --git a/source3/lib/netapi/share.c b/source3/lib/netapi/share.c
index c4ed1549ce7..d12fa1cf0c0 100644
--- a/source3/lib/netapi/share.c
+++ b/source3/lib/netapi/share.c
@@ -141,7 +141,7 @@ static NTSTATUS map_SHARE_INFO_buffer_to_srvsvc_share_info(TALLOC_CTX *mem_ctx,
 		case 2:
 			i2 = (struct SHARE_INFO_2 *)buffer;
 
-			s2 = TALLOC_P(mem_ctx, struct srvsvc_NetShareInfo2);
+			s2 = talloc(mem_ctx, struct srvsvc_NetShareInfo2);
 			NT_STATUS_HAVE_NO_MEMORY(s2);
 
 			s2->name		= i2->shi2_netname;
@@ -159,7 +159,7 @@ static NTSTATUS map_SHARE_INFO_buffer_to_srvsvc_share_info(TALLOC_CTX *mem_ctx,
 		case 1004:
 			i1004 = (struct SHARE_INFO_1004 *)buffer;
 
-			s1004 = TALLOC_P(mem_ctx, struct srvsvc_NetShareInfo1004);
+			s1004 = talloc(mem_ctx, struct srvsvc_NetShareInfo1004);
 			NT_STATUS_HAVE_NO_MEMORY(s1004);
 
 			s1004->comment		= i1004->shi1004_remark;
diff --git a/source3/lib/netapi/user.c b/source3/lib/netapi/user.c
index 63be7ac7512..3003a393971 100644
--- a/source3/lib/netapi/user.c
+++ b/source3/lib/netapi/user.c
@@ -322,7 +322,7 @@ static NTSTATUS set_user_info_USER_INFO_X(TALLOC_CTX *ctx,
 						  25,
 						  &user_info,
 						  &result);
-		if (NT_STATUS_EQUAL(status, NT_STATUS(DCERPC_FAULT_INVALID_TAG))) {
+		if (NT_STATUS_EQUAL(status, NT_STATUS_RPC_ENUM_VALUE_OUT_OF_RANGE)) {
 
 			user_info.info23.info = info21;
 
@@ -575,7 +575,7 @@ WERROR NetUserDel_r(struct libnetapi_ctx *ctx,
 	status = dcerpc_samr_OpenDomain(b, talloc_tos(),
 					&connect_handle,
 					SAMR_DOMAIN_ACCESS_OPEN_ACCOUNT,
-					CONST_DISCARD(struct dom_sid *, &global_sid_Builtin),
+					discard_const_p(struct dom_sid, &global_sid_Builtin),
 					&builtin_handle,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -1436,7 +1436,7 @@ static WERROR convert_samr_dispinfo_to_NET_DISPLAY_USER(TALLOC_CTX *mem_ctx,
 	struct NET_DISPLAY_USER *user = NULL;
 	int i;
 
-	user = TALLOC_ZERO_ARRAY(mem_ctx,
+	user = talloc_zero_array(mem_ctx,
 				 struct NET_DISPLAY_USER,
 				 info->count);
 	W_ERROR_HAVE_NO_MEMORY(user);
@@ -1480,7 +1480,7 @@ static WERROR convert_samr_dispinfo_to_NET_DISPLAY_MACHINE(TALLOC_CTX *mem_ctx,
 	struct NET_DISPLAY_MACHINE *machine = NULL;
 	int i;
 
-	machine = TALLOC_ZERO_ARRAY(mem_ctx,
+	machine = talloc_zero_array(mem_ctx,
 				    struct NET_DISPLAY_MACHINE,
 				    info->count);
 	W_ERROR_HAVE_NO_MEMORY(machine);
@@ -1522,7 +1522,7 @@ static WERROR convert_samr_dispinfo_to_NET_DISPLAY_GROUP(TALLOC_CTX *mem_ctx,
 	struct NET_DISPLAY_GROUP *group = NULL;
 	int i;
 
-	group = TALLOC_ZERO_ARRAY(mem_ctx,
+	group = talloc_zero_array(mem_ctx,
 				  struct NET_DISPLAY_GROUP,
 				  info->count);
 	W_ERROR_HAVE_NO_MEMORY(group);
@@ -3137,7 +3137,6 @@ WERROR NetUserSetGroups_r(struct libnetapi_ctx *ctx,
 	size_t num_del_rids = 0;
 
 	uint32_t *member_rids = NULL;
-	size_t num_member_rids = 0;
 
 	struct GROUP_USERS_INFO_0 *i0 = NULL;
 	struct GROUP_USERS_INFO_1 *i1 = NULL;
@@ -3263,7 +3262,6 @@ WERROR NetUserSetGroups_r(struct libnetapi_ctx *ctx,
 	}
 
 	member_rids = group_rids.ids;
-	num_member_rids = group_rids.count;
 
 	status = dcerpc_samr_GetGroupsForUser(b, talloc_tos(),
 					      &user_handle,
@@ -3574,7 +3572,7 @@ WERROR NetUserGetLocalGroups_r(struct libnetapi_ctx *ctx,
 	}
 
 	sid_array.num_sids = rid_array->count + 1;
-	sid_array.sids = TALLOC_ARRAY(ctx, struct lsa_SidPtr, sid_array.num_sids);
+	sid_array.sids = talloc_array(ctx, struct lsa_SidPtr, sid_array.num_sids);
 	if (!sid_array.sids) {
 		werr = WERR_NOMEM;
 		goto done;
diff --git a/source3/lib/popt_common.c b/source3/lib/popt_common.c
index 128b9c18706..0c8f691c255 100644
--- a/source3/lib/popt_common.c
+++ b/source3/lib/popt_common.c
@@ -129,7 +129,7 @@ static void popt_common_callback(poptContext con,
 
 	case 'n':
 		if (arg) {
-			set_global_myname(arg);
+			lp_set_cmdline("netbios name", arg);
 		}
 		break;
 
@@ -143,13 +143,13 @@ static void popt_common_callback(poptContext con,
 
 	case 'i':
 		if (arg) {
-			  set_global_scope(arg);
+			lp_set_cmdline("netbios scope", arg);
 		}
 		break;
 
 	case 'W':
 		if (arg) {
-			set_global_myworkgroup(arg);
+			lp_set_cmdline("workgroup", arg);
 		}
 		break;
 	}
@@ -473,7 +473,7 @@ static void get_credentials_file(struct user_auth_info *auth_info,
 		} else if (strwicmp("username", param) == 0) {
 			set_cmdline_auth_info_username(auth_info, val);
 		} else if (strwicmp("domain", param) == 0) {
-			set_global_myworkgroup(val);
+			set_cmdline_auth_info_domain(auth_info, val);
 		}
 		memset(buf, 0, sizeof(buf));
 	}
diff --git a/source3/lib/privileges.h b/source3/lib/privileges.h
index 13bc3eb0830..ca2a7c98331 100644
--- a/source3/lib/privileges.h
+++ b/source3/lib/privileges.h
@@ -1,3 +1,28 @@
+/*
+   Unix SMB/CIFS implementation.
+   Privileges handling functions
+   Copyright (C) Jean François Micouleau	1998-2001
+   Copyright (C) Simo Sorce			2002-2003
+   Copyright (C) Gerald (Jerry) Carter          2005
+   Copyright (C) Michael Adam			2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIB_PRIVILEGES_H_
+#define _LIB_PRIVILEGES_H_
+
 #include "../libcli/security/privileges.h"
 
 /* The following definitions come from lib/privileges.c  */
@@ -16,3 +41,5 @@ NTSTATUS privilege_create_account(const struct dom_sid *sid );
 NTSTATUS privilege_delete_account(const struct dom_sid *sid);
 bool is_privileged_sid( const struct dom_sid *sid );
 bool grant_all_privileges( const struct dom_sid *sid );
+
+#endif /* _LIB_PRIVILEGES_H_ */
diff --git a/source3/lib/pthreadpool/Makefile b/source3/lib/pthreadpool/Makefile
new file mode 100644
index 00000000000..48626bd2c0a
--- /dev/null
+++ b/source3/lib/pthreadpool/Makefile
@@ -0,0 +1,9 @@
+all: tests
+
+CFLAGS=-O3 -g -Wall
+
+pthreadpool.o: pthreadpool.c pthreadpool.h
+	gcc -c -O3 -o pthreadpool.o pthreadpool.c -I../../..
+
+tests: tests.o pthreadpool.o
+	gcc -o tests tests.o pthreadpool.o -lpthread
\ No newline at end of file
diff --git a/source3/lib/pthreadpool.c b/source3/lib/pthreadpool/pthreadpool.c
index b62bab0a2ee..9981ed25ae6 100644
--- a/source3/lib/pthreadpool.c
+++ b/source3/lib/pthreadpool/pthreadpool.c
@@ -26,8 +26,10 @@
 #include <signal.h>
 #include <assert.h>
 #include <fcntl.h>
+#include <sys/time.h>
 
 #include "pthreadpool.h"
+#include "lib/util/dlinklist.h"
 
 struct pthreadpool_job {
 	struct pthreadpool_job *next;
@@ -38,6 +40,11 @@ struct pthreadpool_job {
 
 struct pthreadpool {
 	/*
+	 * List pthreadpools for fork safety
+	 */
+	struct pthreadpool *prev, *next;
+
+	/*
 	 * Control access to this struct
 	 */
 	pthread_mutex_t mutex;
@@ -78,13 +85,18 @@ struct pthreadpool {
 	int num_idle;
 
 	/*
-	 * An array of threads that require joining, the array has
-	 * "max_threads" elements. It contains "num_exited" ids.
+	 * An array of threads that require joining.
 	 */
 	int			num_exited;
-	pthread_t		exited[1]; /* We alloc more */
+	pthread_t		*exited; /* We alloc more */
 };
 
+static pthread_mutex_t pthreadpools_mutex = PTHREAD_MUTEX_INITIALIZER;
+static struct pthreadpool *pthreadpools = NULL;
+static pthread_once_t pthreadpool_atfork_initialized = PTHREAD_ONCE_INIT;
+
+static void pthreadpool_prep_atfork(void);
+
 /*
  * Initialize a thread pool
  */
@@ -92,18 +104,24 @@ struct pthreadpool {
 int pthreadpool_init(unsigned max_threads, struct pthreadpool **presult)
 {
 	struct pthreadpool *pool;
-	size_t size;
 	int ret;
 
-	size = sizeof(struct pthreadpool) + max_threads * sizeof(pthread_t);
-
-	pool = (struct pthreadpool *)malloc(size);
+	pool = (struct pthreadpool *)malloc(sizeof(struct pthreadpool));
 	if (pool == NULL) {
 		return ENOMEM;
 	}
 
+	ret = pipe(pool->sig_pipe);
+	if (ret == -1) {
+		int err = errno;
+		free(pool);
+		return err;
+	}
+
 	ret = pthread_mutex_init(&pool->mutex, NULL);
 	if (ret != 0) {
+		close(pool->sig_pipe[0]);
+		close(pool->sig_pipe[1]);
 		free(pool);
 		return ret;
 	}
@@ -111,6 +129,8 @@ int pthreadpool_init(unsigned max_threads, struct pthreadpool **presult)
 	ret = pthread_cond_init(&pool->condvar, NULL);
 	if (ret != 0) {
 		pthread_mutex_destroy(&pool->mutex);
+		close(pool->sig_pipe[0]);
+		close(pool->sig_pipe[1]);
 		free(pool);
 		return ret;
 	}
@@ -119,46 +139,126 @@ int pthreadpool_init(unsigned max_threads, struct pthreadpool **presult)
 	pool->jobs = pool->last_job = NULL;
 	pool->num_threads = 0;
 	pool->num_exited = 0;
+	pool->exited = NULL;
 	pool->max_threads = max_threads;
 	pool->num_idle = 0;
-	pool->sig_pipe[0] = -1;
-	pool->sig_pipe[1] = -1;
+
+	ret = pthread_mutex_lock(&pthreadpools_mutex);
+	if (ret != 0) {
+		pthread_cond_destroy(&pool->condvar);
+		pthread_mutex_destroy(&pool->mutex);
+		close(pool->sig_pipe[0]);
+		close(pool->sig_pipe[1]);
+		free(pool);
+		return ret;
+	}
+	DLIST_ADD(pthreadpools, pool);
+
+	ret = pthread_mutex_unlock(&pthreadpools_mutex);
+	assert(ret == 0);
+
+	pthread_once(&pthreadpool_atfork_initialized, pthreadpool_prep_atfork);
 
 	*presult = pool;
+
 	return 0;
 }
 
-/*
- * Create and return a file descriptor which becomes readable when a job has
- * finished
- */
-
-int pthreadpool_sig_fd(struct pthreadpool *pool)
+static void pthreadpool_prepare(void)
 {
-	int result, ret;
+	int ret;
+	struct pthreadpool *pool;
 
-	ret = pthread_mutex_lock(&pool->mutex);
-	if (ret != 0) {
-		errno = ret;
-		return -1;
+	ret = pthread_mutex_lock(&pthreadpools_mutex);
+	assert(ret == 0);
+
+	pool = pthreadpools;
+
+	while (pool != NULL) {
+		ret = pthread_mutex_lock(&pool->mutex);
+		assert(ret == 0);
+		pool = pool->next;
 	}
+}
 
-	if (pool->sig_pipe[0] != -1) {
-		result = pool->sig_pipe[0];
-		goto done;
+static void pthreadpool_parent(void)
+{
+	int ret;
+	struct pthreadpool *pool;
+
+	pool = DLIST_TAIL(pthreadpools);
+
+	while (1) {
+		ret = pthread_mutex_unlock(&pool->mutex);
+		assert(ret == 0);
+
+		if (pool == pthreadpools) {
+			break;
+		}
+		pool = pool->prev;
 	}
 
-	ret = pipe(pool->sig_pipe);
-	if (ret == -1) {
-		result = -1;
-		goto done;
+	ret = pthread_mutex_unlock(&pthreadpools_mutex);
+	assert(ret == 0);
+}
+
+static void pthreadpool_child(void)
+{
+	int ret;
+	struct pthreadpool *pool;
+
+	pool = DLIST_TAIL(pthreadpools);
+
+	while (1) {
+		close(pool->sig_pipe[0]);
+		close(pool->sig_pipe[1]);
+
+		ret = pipe(pool->sig_pipe);
+		assert(ret == 0);
+
+		pool->num_threads = 0;
+
+		pool->num_exited = 0;
+		free(pool->exited);
+		pool->exited = NULL;
+
+		pool->num_idle = 0;
+
+		while (pool->jobs != NULL) {
+			struct pthreadpool_job *job;
+			job = pool->jobs;
+			pool->jobs = job->next;
+			free(job);
+		}
+		pool->last_job = NULL;
+
+		ret = pthread_mutex_unlock(&pool->mutex);
+		assert(ret == 0);
+
+		if (pool == pthreadpools) {
+			break;
+		}
+		pool = pool->prev;
 	}
 
-	result = pool->sig_pipe[0];
-done:
-	ret = pthread_mutex_unlock(&pool->mutex);
+	ret = pthread_mutex_unlock(&pthreadpools_mutex);
 	assert(ret == 0);
-	return result;
+}
+
+static void pthreadpool_prep_atfork(void)
+{
+	pthread_atfork(pthreadpool_prepare, pthreadpool_parent,
+		       pthreadpool_child);
+}
+
+/*
+ * Return the file descriptor which becomes readable when a job has
+ * finished
+ */
+
+int pthreadpool_signal_fd(struct pthreadpool *pool)
+{
+	return pool->sig_pipe[0];
 }
 
 /*
@@ -173,6 +273,11 @@ static void pthreadpool_join_children(struct pthreadpool *pool)
 		pthread_join(pool->exited[i], NULL);
 	}
 	pool->num_exited = 0;
+
+	/*
+	 * Deliberately not free and NULL pool->exited. That will be
+	 * re-used by realloc later.
+	 */
 }
 
 /*
@@ -181,59 +286,21 @@ static void pthreadpool_join_children(struct pthreadpool *pool)
 
 int pthreadpool_finished_job(struct pthreadpool *pool)
 {
-	int result, ret, fd;
+	int result;
 	ssize_t nread;
 
-	ret = pthread_mutex_lock(&pool->mutex);
-	if (ret != 0) {
-		errno = ret;
-		return -1;
-	}
-
-	/*
-	 * Just some cleanup under the mutex
-	 */
-	pthreadpool_join_children(pool);
-
-	fd = pool->sig_pipe[0];
-
-	ret = pthread_mutex_unlock(&pool->mutex);
-	assert(ret == 0);
-
-	if (fd == -1) {
-		errno = EINVAL;
-		return -1;
-	}
-
 	nread = -1;
 	errno = EINTR;
 
 	while ((nread == -1) && (errno == EINTR)) {
-		nread = read(fd, &result, sizeof(int));
+		nread = read(pool->sig_pipe[0], &result, sizeof(int));
 	}
-
-	/*
-	 * TODO: handle nread > 0 && nread < sizeof(int)
-	 */
-
-	/*
-	 * Lock the mutex to provide a memory barrier for data from the worker
-	 * thread to the main thread. The pipe access itself does not have to
-	 * be locked, for sizeof(int) the write to a pipe is atomic, and only
-	 * one thread reads from it. But we need to lock the mutex briefly
-	 * even if we don't do anything under the lock, to make sure we can
-	 * see all memory the helper thread has written.
-	 */
-
-	ret = pthread_mutex_lock(&pool->mutex);
-	if (ret == -1) {
-		errno = ret;
-		return -1;
+	if (nread == -1) {
+		return errno;
+	}
+	if (nread != sizeof(int)) {
+		return EINVAL;
 	}
-
-	ret = pthread_mutex_unlock(&pool->mutex);
-	assert(ret == 0);
-
 	return result;
 }
 
@@ -250,6 +317,12 @@ int pthreadpool_destroy(struct pthreadpool *pool)
 		return ret;
 	}
 
+	if ((pool->jobs != NULL) || pool->shutdown) {
+		ret = pthread_mutex_unlock(&pool->mutex);
+		assert(ret == 0);
+		return EBUSY;
+	}
+
 	if (pool->num_threads > 0) {
 		/*
 		 * We have active threads, tell them to finish, wait for that.
@@ -294,14 +367,31 @@ int pthreadpool_destroy(struct pthreadpool *pool)
 	ret = pthread_mutex_destroy(&pool->mutex);
 	ret1 = pthread_cond_destroy(&pool->condvar);
 
-	if ((ret == 0) && (ret1 == 0)) {
-		free(pool);
+	if (ret != 0) {
+		return ret;
+	}
+	if (ret1 != 0) {
+		return ret1;
 	}
 
+	ret = pthread_mutex_lock(&pthreadpools_mutex);
 	if (ret != 0) {
 		return ret;
 	}
-	return ret1;
+	DLIST_REMOVE(pthreadpools, pool);
+	ret = pthread_mutex_unlock(&pthreadpools_mutex);
+	assert(ret == 0);
+
+	close(pool->sig_pipe[0]);
+	pool->sig_pipe[0] = -1;
+
+	close(pool->sig_pipe[1]);
+	pool->sig_pipe[1] = -1;
+
+	free(pool->exited);
+	free(pool);
+
+	return 0;
 }
 
 /*
@@ -309,7 +399,19 @@ int pthreadpool_destroy(struct pthreadpool *pool)
  */
 static void pthreadpool_server_exit(struct pthreadpool *pool)
 {
+	pthread_t *exited;
+
 	pool->num_threads -= 1;
+
+	exited = (pthread_t *)realloc(
+		pool->exited, sizeof(pthread_t *) * (pool->num_exited + 1));
+
+	if (exited == NULL) {
+		/* lost a thread status */
+		return;
+	}
+	pool->exited = exited;
+
 	pool->exited[pool->num_exited] = pthread_self();
 	pool->num_exited += 1;
 }
@@ -325,7 +427,7 @@ static void *pthreadpool_server(void *arg)
 	}
 
 	while (1) {
-		struct timespec timeout;
+		struct timespec ts;
 		struct pthreadpool_job *job;
 
 		/*
@@ -333,14 +435,14 @@ static void *pthreadpool_server(void *arg)
 		 * time, exit this thread.
 		 */
 
-		timeout.tv_sec = time(NULL) + 1;
-		timeout.tv_nsec = 0;
+		clock_gettime(CLOCK_REALTIME, &ts);
+		ts.tv_sec += 1;
 
 		while ((pool->jobs == NULL) && (pool->shutdown == 0)) {
 
 			pool->num_idle += 1;
 			res = pthread_cond_timedwait(
-				&pool->condvar, &pool->mutex, &timeout);
+				&pool->condvar, &pool->mutex, &ts);
 			pool->num_idle -= 1;
 
 			if (res == ETIMEDOUT) {
@@ -363,7 +465,6 @@ static void *pthreadpool_server(void *arg)
 		job = pool->jobs;
 
 		if (job != NULL) {
-			int fd = pool->sig_pipe[1];
 			ssize_t written;
 
 			/*
@@ -376,7 +477,7 @@ static void *pthreadpool_server(void *arg)
 			}
 
 			/*
-			 * Do the work with the mutex unlocked :-)
+			 * Do the work with the mutex unlocked
 			 */
 
 			res = pthread_mutex_unlock(&pool->mutex);
@@ -384,17 +485,14 @@ static void *pthreadpool_server(void *arg)
 
 			job->fn(job->private_data);
 
-			written = sizeof(int);
+			written = write(pool->sig_pipe[1], &job->id,
+					sizeof(int));
+
+			free(job);
 
 			res = pthread_mutex_lock(&pool->mutex);
 			assert(res == 0);
 
-			if (fd != -1) {
-				written = write(fd, &job->id, sizeof(int));
-			}
-
-			free(job);
-
 			if (written != sizeof(int)) {
 				pthreadpool_server_exit(pool);
 				pthread_mutex_unlock(&pool->mutex);
@@ -447,6 +545,17 @@ int pthreadpool_add_job(struct pthreadpool *pool, int job_id,
 		return res;
 	}
 
+	if (pool->shutdown) {
+		/*
+		 * Protect against the pool being shut down while
+		 * trying to add a job
+		 */
+		res = pthread_mutex_unlock(&pool->mutex);
+		assert(res == 0);
+		free(job);
+		return EINVAL;
+	}
+
 	/*
 	 * Just some cleanup under the mutex
 	 */
@@ -472,7 +581,8 @@ int pthreadpool_add_job(struct pthreadpool *pool, int job_id,
 		return res;
 	}
 
-	if (pool->num_threads >= pool->max_threads) {
+	if ((pool->max_threads != 0) &&
+	    (pool->num_threads >= pool->max_threads)) {
 		/*
 		 * No more new threads, we just queue the request
 		 */
diff --git a/source3/lib/pthreadpool/pthreadpool.h b/source3/lib/pthreadpool/pthreadpool.h
new file mode 100644
index 00000000000..79704ea3853
--- /dev/null
+++ b/source3/lib/pthreadpool/pthreadpool.h
@@ -0,0 +1,97 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * threadpool implementation based on pthreads
+ * Copyright (C) Volker Lendecke 2009,2011
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __PTHREADPOOL_H__
+#define __PTHREADPOOL_H__
+
+struct pthreadpool;
+
+/**
+ * @defgroup pthreadpool The pthreadpool API
+ *
+ * This API provides a way to run threadsafe functions in a helper
+ * thread. It is initially intended to run getaddrinfo asynchronously.
+ */
+
+
+/**
+ * @brief Create a pthreadpool
+ *
+ * A struct pthreadpool is the basis for for running threads in the
+ * background.
+ *
+ * @param[in]	max_threads	Maximum parallelism in this pool
+ * @param[out]	presult		Pointer to the threadpool returned
+ * @return			success: 0, failure: errno
+ *
+ * max_threads=0 means unlimited parallelism. The caller has to take
+ * care to not overload the system.
+ */
+int pthreadpool_init(unsigned max_threads, struct pthreadpool **presult);
+
+/**
+ * @brief Destroy a pthreadpool
+ *
+ * Destroy a pthreadpool. If jobs are still active, this returns
+ * EBUSY.
+ *
+ * @param[in]	pool		The pool to destroy
+ * @return			success: 0, failure: errno
+ */
+int pthreadpool_destroy(struct pthreadpool *pool);
+
+/**
+ * @brief Add a job to a pthreadpool
+ *
+ * This adds a job to a pthreadpool. The job can be identified by
+ * job_id. This integer will be returned from
+ * pthreadpool_finished_job() then the job is completed.
+ *
+ * @param[in]	pool		The pool to run the job on
+ * @param[in]	job_id		A custom identifier
+ * @param[in]	fn		The function to run asynchronously
+ * @param[in]	private_data	Pointer passed to fn
+ * @return			success: 0, failure: errno
+ */
+int pthreadpool_add_job(struct pthreadpool *pool, int job_id,
+			void (*fn)(void *private_data), void *private_data);
+
+/**
+ * @brief Get the signalling fd from a pthreadpool
+ *
+ * Completion of a job is indicated by readability of the fd retuned
+ * by pthreadpool_signal_fd().
+ *
+ * @param[in]	pool		The pool in question
+ * @return			The fd to listen on for readability
+ */
+int pthreadpool_signal_fd(struct pthreadpool *pool);
+
+/**
+ * @brief Get the job_id of a finished job
+ *
+ * This blocks until a job has finished unless the fd returned by
+ * pthreadpool_signal_fd() is readable.
+ *
+ * @param[in]	pool		The pool to query for finished jobs
+ * @return			The job_id of the finished job
+ */
+int pthreadpool_finished_job(struct pthreadpool *pool);
+
+#endif
diff --git a/source3/lib/pthreadpool/tests.c b/source3/lib/pthreadpool/tests.c
new file mode 100644
index 00000000000..667ee01784e
--- /dev/null
+++ b/source3/lib/pthreadpool/tests.c
@@ -0,0 +1,362 @@
+#include <stdio.h>
+#include <string.h>
+#include <poll.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <unistd.h>
+#include "pthreadpool.h"
+
+static int test_init(void)
+{
+	struct pthreadpool *p;
+	int ret;
+
+	ret = pthreadpool_init(1, &p);
+	if (ret != 0) {
+		fprintf(stderr, "pthreadpool_init failed: %s\n",
+			strerror(ret));
+		return -1;
+	}
+	ret = pthreadpool_destroy(p);
+	if (ret != 0) {
+		fprintf(stderr, "pthreadpool_init failed: %s\n",
+			strerror(ret));
+		return -1;
+	}
+	return 0;
+}
+
+static void test_sleep(void *ptr)
+{
+	int *ptimeout = (int *)ptr;
+	int ret;
+	ret = poll(NULL, 0, *ptimeout);
+	if (ret != 0) {
+		fprintf(stderr, "poll returned %d (%s)\n",
+			ret, strerror(errno));
+	}
+}
+
+static int test_jobs(int num_threads, int num_jobs)
+{
+	char *finished;
+	struct pthreadpool *p;
+	int timeout = 1;
+	int i, ret;
+
+	finished = (char *)calloc(1, num_jobs);
+	if (finished == NULL) {
+		fprintf(stderr, "calloc failed\n");
+		return -1;
+	}
+
+	ret = pthreadpool_init(num_threads, &p);
+	if (ret != 0) {
+		fprintf(stderr, "pthreadpool_init failed: %s\n",
+			strerror(ret));
+		return -1;
+	}
+
+	for (i=0; i<num_jobs; i++) {
+		ret = pthreadpool_add_job(p, i, test_sleep, &timeout);
+		if (ret != 0) {
+			fprintf(stderr, "pthreadpool_add_job failed: %s\n",
+				strerror(ret));
+			return -1;
+		}
+	}
+
+	for (i=0; i<num_jobs; i++) {
+		ret = pthreadpool_finished_job(p);
+		if ((ret < 0) || (ret >= num_jobs)) {
+			fprintf(stderr, "invalid job number %d\n", ret);
+			return -1;
+		}
+		finished[ret] += 1;
+	}
+
+	for (i=0; i<num_jobs; i++) {
+		if (finished[i] != 1) {
+			fprintf(stderr, "finished[%d] = %d\n",
+				i, finished[i]);
+			return -1;
+		}
+	}
+
+	ret = pthreadpool_destroy(p);
+	if (ret != 0) {
+		fprintf(stderr, "pthreadpool_destroy failed: %s\n",
+			strerror(ret));
+		return -1;
+	}
+
+	free(finished);
+	return 0;
+}
+
+static int test_busydestroy(void)
+{
+	struct pthreadpool *p;
+	int timeout = 50;
+	struct pollfd pfd;
+	int ret;
+
+	ret = pthreadpool_init(1, &p);
+	if (ret != 0) {
+		fprintf(stderr, "pthreadpool_init failed: %s\n",
+			strerror(ret));
+		return -1;
+	}
+	ret = pthreadpool_add_job(p, 1, test_sleep, &timeout);
+	if (ret != 0) {
+		fprintf(stderr, "pthreadpool_add_job failed: %s\n",
+			strerror(ret));
+		return -1;
+	}
+	ret = pthreadpool_destroy(p);
+	if (ret != EBUSY) {
+		fprintf(stderr, "Could destroy a busy pool\n");
+		return -1;
+	}
+
+	pfd.fd = pthreadpool_signal_fd(p);
+	pfd.events = POLLIN|POLLERR;
+
+	poll(&pfd, 1, -1);
+
+	ret = pthreadpool_destroy(p);
+	if (ret != 0) {
+		fprintf(stderr, "pthreadpool_destroy failed: %s\n",
+			strerror(ret));
+		return -1;
+	}
+	return 0;
+}
+
+struct threaded_state {
+	pthread_t tid;
+	struct pthreadpool *p;
+	int start_job;
+	int num_jobs;
+	int timeout;
+};
+
+static void *test_threaded_worker(void *p)
+{
+	struct threaded_state *state = (struct threaded_state *)p;
+	int i;
+
+	for (i=0; i<state->num_jobs; i++) {
+		int ret = pthreadpool_add_job(state->p, state->start_job + i,
+					      test_sleep, &state->timeout);
+		if (ret != 0) {
+			fprintf(stderr, "pthreadpool_add_job failed: %s\n",
+				strerror(ret));
+			return NULL;
+		}
+	}
+	return NULL;
+}
+
+static int test_threaded_addjob(int num_pools, int num_threads, int poolsize,
+				int num_jobs)
+{
+	struct pthreadpool **pools;
+	struct threaded_state *states;
+	struct threaded_state *state;
+	struct pollfd *pfds;
+	char *finished;
+	pid_t child;
+	int i, ret, poolnum;
+	int received;
+
+	states = calloc(num_threads, sizeof(struct threaded_state));
+	if (states == NULL) {
+		fprintf(stderr, "calloc failed\n");
+		return -1;
+	}
+
+	finished = calloc(num_threads * num_jobs, 1);
+	if (finished == NULL) {
+		fprintf(stderr, "calloc failed\n");
+		return -1;
+	}
+
+	pools = calloc(num_pools, sizeof(struct pthreadpool *));
+	if (pools == NULL) {
+		fprintf(stderr, "calloc failed\n");
+		return -1;
+	}
+
+	pfds = calloc(num_pools, sizeof(struct pollfd));
+	if (pfds == NULL) {
+		fprintf(stderr, "calloc failed\n");
+		return -1;
+	}
+
+	for (i=0; i<num_pools; i++) {
+		ret = pthreadpool_init(poolsize, &pools[i]);
+		if (ret != 0) {
+			fprintf(stderr, "pthreadpool_init failed: %s\n",
+				strerror(ret));
+			return -1;
+		}
+		pfds[i].fd = pthreadpool_signal_fd(pools[i]);
+		pfds[i].events = POLLIN|POLLHUP;
+	}
+
+	poolnum = 0;
+
+	for (i=0; i<num_threads; i++) {
+		state = &states[i];
+
+		state->p = pools[poolnum];
+		poolnum = (poolnum + 1) % num_pools;
+
+		state->num_jobs = num_jobs;
+		state->timeout = 1;
+		state->start_job = i * num_jobs;
+
+		ret = pthread_create(&state->tid, NULL, test_threaded_worker,
+				     state);
+		if (ret != 0) {
+			fprintf(stderr, "pthread_create failed: %s\n",
+				strerror(ret));
+			return -1;
+		}
+	}
+
+	if (random() % 1) {
+		poll(NULL, 0, 1);
+	}
+
+	child = fork();
+	if (child < 0) {
+		fprintf(stderr, "fork failed: %s\n", strerror(errno));
+		return -1;
+	}
+	if (child == 0) {
+		for (i=0; i<num_pools; i++) {
+			ret = pthreadpool_destroy(pools[i]);
+			if (ret != 0) {
+				fprintf(stderr, "pthreadpool_destroy failed: "
+					"%s\n", strerror(ret));
+				exit(1);
+			}
+		}
+		/* child */
+		exit(0);
+	}
+
+	for (i=0; i<num_threads; i++) {
+		ret = pthread_join(states[i].tid, NULL);
+		if (ret != 0) {
+			fprintf(stderr, "pthread_join(%d) failed: %s\n",
+				i, strerror(ret));
+			return -1;
+		}
+	}
+
+	received = 0;
+
+	while (received < num_threads*num_jobs) {
+		int j;
+
+		ret = poll(pfds, num_pools, 1000);
+		if (ret == -1) {
+			fprintf(stderr, "poll failed: %s\n",
+				strerror(errno));
+			return -1;
+		}
+		if (ret == 0) {
+			fprintf(stderr, "\npoll timed out\n");
+			break;
+		}
+
+		for (j=0; j<num_pools; j++) {
+
+			if ((pfds[j].revents & (POLLIN|POLLHUP)) == 0) {
+				continue;
+			}
+
+			ret = pthreadpool_finished_job(pools[j]);
+			if ((ret < 0) || (ret >= num_jobs * num_threads)) {
+				fprintf(stderr, "invalid job number %d\n",
+					ret);
+				return -1;
+			}
+			finished[ret] += 1;
+			received += 1;
+		}
+	}
+
+	for (i=0; i<num_threads*num_jobs; i++) {
+		if (finished[i] != 1) {
+			fprintf(stderr, "finished[%d] = %d\n",
+				i, finished[i]);
+			return -1;
+		}
+	}
+
+	for (i=0; i<num_pools; i++) {
+		ret = pthreadpool_destroy(pools[i]);
+		if (ret != 0) {
+			fprintf(stderr, "pthreadpool_destroy failed: %s\n",
+				strerror(ret));
+			return -1;
+		}
+	}
+
+	free(pfds);
+	free(pools);
+	free(states);
+	free(finished);
+
+	return 0;
+}
+
+int main(void)
+{
+	int ret;
+
+	ret = test_init();
+	if (ret != 0) {
+		fprintf(stderr, "test_init failed\n");
+		return 1;
+	}
+
+	ret = test_jobs(10, 10000);
+	if (ret != 0) {
+		fprintf(stderr, "test_jobs failed\n");
+		return 1;
+	}
+
+	ret = test_busydestroy();
+	if (ret != 0) {
+		fprintf(stderr, "test_busydestroy failed\n");
+		return 1;
+	}
+
+	/*
+	 * Test 10 threads adding jobs on a single pool
+	 */
+	ret = test_threaded_addjob(1, 10, 5, 5000);
+	if (ret != 0) {
+		fprintf(stderr, "test_jobs failed\n");
+		return 1;
+	}
+
+	/*
+	 * Test 10 threads on 3 pools to verify our fork handling
+	 * works right.
+	 */
+	ret = test_threaded_addjob(3, 10, 5, 5000);
+	if (ret != 0) {
+		fprintf(stderr, "test_jobs failed\n");
+		return 1;
+	}
+
+	printf("success\n");
+	return 0;
+}
diff --git a/source3/lib/pthreadpool/wscript_build b/source3/lib/pthreadpool/wscript_build
new file mode 100644
index 00000000000..611ff5a4331
--- /dev/null
+++ b/source3/lib/pthreadpool/wscript_build
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+bld.SAMBA3_SUBSYSTEM('PTHREADPOOL',
+                     source='pthreadpool.c',
+                     deps='pthread rt',
+                     enabled=bld.env.WITH_PTHREADPOOL)
+
+bld.SAMBA3_BINARY('pthreadpooltest',
+                  source='tests.c',
+                  deps='PTHREADPOOL',
+                  enabled=bld.env.WITH_PTHREADPOOL)
diff --git a/source3/lib/server_mutex.c b/source3/lib/server_mutex.c
index 1ba9d6a65be..dc658191970 100644
--- a/source3/lib/server_mutex.c
+++ b/source3/lib/server_mutex.c
@@ -20,6 +20,8 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "lib/util/tdb_wrap.h"
+#include "util_tdb.h"
 
 /* For reasons known only to MS, many of their NT/Win2k versions
    need serialised access only.  Two connections at the same time
@@ -68,7 +70,7 @@ struct named_mutex *grab_named_mutex(TALLOC_CTX *mem_ctx, const char *name,
 	}
 
 	if (tdb_lock_bystring_with_timeout(result->tdb->tdb, name,
-					   timeout) == -1) {
+					   timeout) != 0) {
 		DEBUG(1, ("Could not get the lock for %s\n", name));
 		TALLOC_FREE(result);
 		return NULL;
diff --git a/source3/lib/serverid.c b/source3/lib/serverid.c
index f13c66e2c91..1a1141265a1 100644
--- a/source3/lib/serverid.c
+++ b/source3/lib/serverid.c
@@ -20,10 +20,13 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "serverid.h"
+#include "util_tdb.h"
 #include "dbwrap.h"
+#include "lib/util/tdb_wrap.h"
 
 struct serverid_key {
 	pid_t pid;
+	uint32_t task_id;
 	uint32_t vnn;
 };
 
@@ -70,6 +73,7 @@ static void serverid_fill_key(const struct server_id *id,
 {
 	ZERO_STRUCTP(key);
 	key->pid = id->pid;
+	key->task_id = id->task_id;
 	key->vnn = id->vnn;
 }
 
@@ -246,7 +250,7 @@ bool serverid_exists(const struct server_id *id)
 	state.id = id;
 	state.exists = false;
 
-	if (db->parse_record(db, tdbkey, server_exists_parse, &state) == -1) {
+	if (db->parse_record(db, tdbkey, server_exists_parse, &state) != 0) {
 		return false;
 	}
 	return state.exists;
@@ -273,6 +277,7 @@ static bool serverid_rec_parse(const struct db_record *rec,
 	memcpy(&data, rec->value.dptr, sizeof(data));
 
 	id->pid = key.pid;
+	id->task_id = key.task_id;
 	id->vnn = key.vnn;
 	id->unique_id = data.unique_id;
 	*msg_flags = data.msg_flags;
diff --git a/source3/lib/sessionid_tdb.c b/source3/lib/sessionid_tdb.c
index ab54ec9ca5f..de3ccab26ae 100644
--- a/source3/lib/sessionid_tdb.c
+++ b/source3/lib/sessionid_tdb.c
@@ -21,6 +21,7 @@
 #include "system/filesys.h"
 #include "dbwrap.h"
 #include "session.h"
+#include "util_tdb.h"
 
 static struct db_context *session_db_ctx(void)
 {
diff --git a/source3/lib/sharesec.c b/source3/lib/sharesec.c
index c84e8fa00b2..ed971a97a6c 100644
--- a/source3/lib/sharesec.c
+++ b/source3/lib/sharesec.c
@@ -22,6 +22,7 @@
 #include "../libcli/security/security.h"
 #include "../librpc/gen_ndr/ndr_security.h"
 #include "dbwrap.h"
+#include "util_tdb.h"
 
 /*******************************************************************
  Create the share security tdb.
@@ -462,7 +463,7 @@ bool parse_usershare_acl(TALLOC_CTX *ctx, const char *acl_str, struct security_d
 	/* Add the number of ',' characters to get the number of aces. */
 	num_aces += count_chars(pacl,',');
 
-	ace_list = TALLOC_ARRAY(ctx, struct security_ace, num_aces);
+	ace_list = talloc_array(ctx, struct security_ace, num_aces);
 	if (!ace_list) {
 		return False;
 	}
diff --git a/source3/lib/smbconf/smbconf_init.c b/source3/lib/smbconf/smbconf_init.c
index 36c51de5f03..2587a4fc53d 100644
--- a/source3/lib/smbconf/smbconf_init.c
+++ b/source3/lib/smbconf/smbconf_init.c
@@ -34,28 +34,28 @@
  * -  "registry" or "reg"
  * -  "txt" or "file"
  */
-WERROR smbconf_init(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
+sbcErr smbconf_init(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 		    const char *source)
 {
-	WERROR werr;
+	sbcErr err;
 	char *backend = NULL;
 	char *path = NULL;
 	char *sep;
 	TALLOC_CTX *tmp_ctx = talloc_stackframe();
 
 	if (conf_ctx == NULL) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	if ((source == NULL) || (*source == '\0')) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	backend = talloc_strdup(tmp_ctx, source);
 	if (backend == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
@@ -69,16 +69,16 @@ WERROR smbconf_init(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 	}
 
 	if (strequal(backend, "registry") || strequal(backend, "reg")) {
-		werr = smbconf_init_reg(mem_ctx, conf_ctx, path);
+		err = smbconf_init_reg(mem_ctx, conf_ctx, path);
 	} else if (strequal(backend, "file") || strequal(backend, "txt")) {
-		werr = smbconf_init_txt(mem_ctx, conf_ctx, path);
+		err = smbconf_init_txt(mem_ctx, conf_ctx, path);
 	} else if (sep == NULL) {
 		/*
 		 * If no separator was given in the source, and the string is
 		 * not a known backend, assume file backend and use the source
 		 * string as a path argument.
 		 */
-		werr = smbconf_init_txt(mem_ctx, conf_ctx, backend);
+		err = smbconf_init_txt(mem_ctx, conf_ctx, backend);
 	} else {
 		/*
 		 * Separator was specified but this is not a known backend.
@@ -87,10 +87,10 @@ WERROR smbconf_init(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 		 * This may occur with an include directive like this:
 		 * 'include = /path/to/file.%T'
 		 */
-		werr = smbconf_init_txt(mem_ctx, conf_ctx, source);
+		err = smbconf_init_txt(mem_ctx, conf_ctx, source);
 	}
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
diff --git a/source3/lib/smbconf/smbconf_init.h b/source3/lib/smbconf/smbconf_init.h
index abd62df2045..45ea8096240 100644
--- a/source3/lib/smbconf/smbconf_init.h
+++ b/source3/lib/smbconf/smbconf_init.h
@@ -26,7 +26,7 @@ struct smbconf_ctx;
  * intialization dispatcher function.
  * takes source string in the form of "backend:path"
  */
-WERROR smbconf_init(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
+sbcErr smbconf_init(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 		    const char *source);
 
 #endif /*  _LIBSMBCONF_INIT_H_  */
diff --git a/source3/lib/smbconf/smbconf_reg.c b/source3/lib/smbconf/smbconf_reg.c
index 67df03cbd2d..92dbcbae1c1 100644
--- a/source3/lib/smbconf/smbconf_reg.c
+++ b/source3/lib/smbconf/smbconf_reg.c
@@ -83,7 +83,7 @@ static bool smbconf_reg_valname_valid(const char *valname)
 /**
  * Open a subkey of the base key (i.e a service)
  */
-static WERROR smbconf_reg_open_service_key(TALLOC_CTX *mem_ctx,
+static sbcErr smbconf_reg_open_service_key(TALLOC_CTX *mem_ctx,
 					   struct smbconf_ctx *ctx,
 					   const char *servicename,
 					   uint32 desired_access,
@@ -93,16 +93,18 @@ static WERROR smbconf_reg_open_service_key(TALLOC_CTX *mem_ctx,
 
 	if (servicename == NULL) {
 		*key = rpd(ctx)->base_key;
-		return WERR_OK;
+		return SBC_ERR_OK;
 	}
 	werr = reg_openkey(mem_ctx, rpd(ctx)->base_key, servicename,
 			   desired_access, key);
-
 	if (W_ERROR_EQUAL(werr, WERR_BADFILE)) {
-		werr = WERR_NO_SUCH_SERVICE;
+		return SBC_ERR_NO_SUCH_SERVICE;
+	}
+	if (!W_ERROR_IS_OK(werr)) {
+		return SBC_ERR_NOMEM;
 	}
 
-	return werr;
+	return SBC_ERR_OK;
 }
 
 /**
@@ -127,12 +129,13 @@ static bool smbconf_value_exists(struct registry_key *key, const char *param)
 /**
  * create a subkey of the base key (i.e. a service...)
  */
-static WERROR smbconf_reg_create_service_key(TALLOC_CTX *mem_ctx,
+static sbcErr smbconf_reg_create_service_key(TALLOC_CTX *mem_ctx,
 					     struct smbconf_ctx *ctx,
 					     const char * subkeyname,
 					     struct registry_key **newkey)
 {
-	WERROR werr = WERR_OK;
+	WERROR werr;
+	sbcErr err = SBC_ERR_OK;
 	TALLOC_CTX *create_ctx;
 	enum winreg_CreateAction action = REG_ACTION_NONE;
 
@@ -145,26 +148,28 @@ static WERROR smbconf_reg_create_service_key(TALLOC_CTX *mem_ctx,
 			     REG_KEY_WRITE, newkey, &action);
 	if (W_ERROR_IS_OK(werr) && (action != REG_CREATED_NEW_KEY)) {
 		DEBUG(10, ("Key '%s' already exists.\n", subkeyname));
-		werr = WERR_FILE_EXISTS;
+		err = SBC_ERR_FILE_EXISTS;
 	}
 	if (!W_ERROR_IS_OK(werr)) {
 		DEBUG(5, ("Error creating key %s: %s\n",
 			 subkeyname, win_errstr(werr)));
+		err = SBC_ERR_UNKNOWN_FAILURE;
 	}
 
 	talloc_free(create_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * add a value to a key.
  */
-static WERROR smbconf_reg_set_value(struct registry_key *key,
+static sbcErr smbconf_reg_set_value(struct registry_key *key,
 				    const char *valname,
 				    const char *valstr)
 {
 	struct registry_value val;
 	WERROR werr = WERR_OK;
+	sbcErr err;
 	char *subkeyname;
 	const char *canon_valname;
 	const char *canon_valstr;
@@ -180,14 +185,14 @@ static WERROR smbconf_reg_set_value(struct registry_key *key,
 			DEBUG(5, ("invalid value '%s' given for "
 				  "parameter '%s'\n", valstr, valname));
 		}
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	if (smbconf_reg_valname_forbidden(canon_valname)) {
 		DEBUG(5, ("Parameter '%s' not allowed in registry.\n",
 			  canon_valname));
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
@@ -195,7 +200,7 @@ static WERROR smbconf_reg_set_value(struct registry_key *key,
 	if ((subkeyname == NULL) || (*(subkeyname +1) == '\0')) {
 		DEBUG(5, ("Invalid registry key '%s' given as "
 			  "smbconf section.\n", key->key->name));
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 	subkeyname++;
@@ -205,7 +210,7 @@ static WERROR smbconf_reg_set_value(struct registry_key *key,
 		DEBUG(5, ("Global parameter '%s' not allowed in "
 			  "service definition ('%s').\n", canon_valname,
 			  subkeyname));
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
@@ -213,7 +218,7 @@ static WERROR smbconf_reg_set_value(struct registry_key *key,
 
 	val.type = REG_SZ;
 	if (!push_reg_sz(talloc_tos(), &val.data, canon_valstr)) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
@@ -222,37 +227,41 @@ static WERROR smbconf_reg_set_value(struct registry_key *key,
 		DEBUG(5, ("Error adding value '%s' to "
 			  "key '%s': %s\n",
 			  canon_valname, key->key->name, win_errstr(werr)));
+		err = SBC_ERR_NOMEM;
+		goto done;
 	}
 
+	err = SBC_ERR_OK;
 done:
-	return werr;
+	return err;
 }
 
-static WERROR smbconf_reg_set_multi_sz_value(struct registry_key *key,
+static sbcErr smbconf_reg_set_multi_sz_value(struct registry_key *key,
 					     const char *valname,
 					     const uint32_t num_strings,
 					     const char **strings)
 {
 	WERROR werr;
+	sbcErr err = SBC_ERR_OK;
 	struct registry_value *value;
 	uint32_t count;
 	TALLOC_CTX *tmp_ctx = talloc_stackframe();
 	const char **array;
 
 	if (strings == NULL) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	array = talloc_zero_array(tmp_ctx, const char *, num_strings + 1);
 	if (array == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
-	value = TALLOC_ZERO_P(tmp_ctx, struct registry_value);
+	value = talloc_zero(tmp_ctx, struct registry_value);
 	if (value == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
@@ -261,13 +270,13 @@ static WERROR smbconf_reg_set_multi_sz_value(struct registry_key *key,
 	for (count = 0; count < num_strings; count++) {
 		array[count] = talloc_strdup(value, strings[count]);
 		if (array[count] == NULL) {
-			werr = WERR_NOMEM;
+			err = SBC_ERR_NOMEM;
 			goto done;
 		}
 	}
 
 	if (!push_reg_multi_sz(value, &value->data, array)) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
@@ -275,11 +284,12 @@ static WERROR smbconf_reg_set_multi_sz_value(struct registry_key *key,
 	if (!W_ERROR_IS_OK(werr)) {
 		DEBUG(5, ("Error adding value '%s' to key '%s': %s\n",
 			  valname, key->key->name, win_errstr(werr)));
+		err = SBC_ERR_ACCESS_DENIED;
 	}
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 /**
@@ -342,12 +352,13 @@ static char *smbconf_format_registry_value(TALLOC_CTX *mem_ctx,
 	return result;
 }
 
-static WERROR smbconf_reg_get_includes_internal(TALLOC_CTX *mem_ctx,
+static sbcErr smbconf_reg_get_includes_internal(TALLOC_CTX *mem_ctx,
 						struct registry_key *key,
 						uint32_t *num_includes,
 						char ***includes)
 {
 	WERROR werr;
+	sbcErr err;
 	uint32_t count;
 	struct registry_value *value = NULL;
 	char **tmp_includes = NULL;
@@ -358,31 +369,33 @@ static WERROR smbconf_reg_get_includes_internal(TALLOC_CTX *mem_ctx,
 		/* no includes */
 		*num_includes = 0;
 		*includes = NULL;
-		werr = WERR_OK;
+		err = SBC_ERR_OK;
 		goto done;
 	}
 
 	werr = reg_queryvalue(tmp_ctx, key, INCLUDES_VALNAME, &value);
 	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_ACCESS_DENIED;
 		goto done;
 	}
 
 	if (value->type != REG_MULTI_SZ) {
 		/* wrong type -- ignore */
+		err = SBC_ERR_OK;
 		goto done;
 	}
 
 	if (!pull_reg_multi_sz(tmp_ctx, &value->data, &array)) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
 	for (count = 0; array[count] != NULL; count++) {
-		werr = smbconf_add_string_to_array(tmp_ctx,
+		err = smbconf_add_string_to_array(tmp_ctx,
 					&tmp_includes,
 					count,
 					array[count]);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 	}
@@ -390,7 +403,7 @@ static WERROR smbconf_reg_get_includes_internal(TALLOC_CTX *mem_ctx,
 	if (count > 0) {
 		*includes = talloc_move(mem_ctx, &tmp_includes);
 		if (*includes == NULL) {
-			werr = WERR_NOMEM;
+			err = SBC_ERR_NOMEM;
 			goto done;
 		}
 		*num_includes = count;
@@ -399,16 +412,17 @@ static WERROR smbconf_reg_get_includes_internal(TALLOC_CTX *mem_ctx,
 		*includes = NULL;
 	}
 
+	err = SBC_ERR_OK;
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * Get the values of a key as a list of value names
  * and a list of value strings (ordered)
  */
-static WERROR smbconf_reg_get_values(TALLOC_CTX *mem_ctx,
+static sbcErr smbconf_reg_get_values(TALLOC_CTX *mem_ctx,
 				     struct registry_key *key,
 				     uint32_t *num_values,
 				     char ***value_names,
@@ -416,6 +430,7 @@ static WERROR smbconf_reg_get_values(TALLOC_CTX *mem_ctx,
 {
 	TALLOC_CTX *tmp_ctx = NULL;
 	WERROR werr = WERR_OK;
+	sbcErr err;
 	uint32_t count;
 	struct registry_value *valvalue = NULL;
 	char *valname = NULL;
@@ -428,7 +443,7 @@ static WERROR smbconf_reg_get_values(TALLOC_CTX *mem_ctx,
 	if ((num_values == NULL) || (value_names == NULL) ||
 	    (value_strings == NULL))
 	{
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
@@ -445,42 +460,44 @@ static WERROR smbconf_reg_get_values(TALLOC_CTX *mem_ctx,
 			continue;
 		}
 
-		werr = smbconf_add_string_to_array(tmp_ctx,
-						   &tmp_valnames,
-						   tmp_num_values, valname);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_add_string_to_array(tmp_ctx,
+						  &tmp_valnames,
+						  tmp_num_values, valname);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 
 		valstring = smbconf_format_registry_value(tmp_ctx, valvalue);
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_valstrings,
-						   tmp_num_values, valstring);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_valstrings,
+						  tmp_num_values, valstring);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 		tmp_num_values++;
 	}
 	if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
 	/* now add the includes at the end */
-	werr = smbconf_reg_get_includes_internal(tmp_ctx, key, &num_includes,
+	err = smbconf_reg_get_includes_internal(tmp_ctx, key, &num_includes,
 						 &includes);
-	if (!W_ERROR_IS_OK(werr)) {
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
+
 	for (count = 0; count < num_includes; count++) {
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_valnames,
-						   tmp_num_values, "include");
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_valnames,
+						  tmp_num_values, "include");
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_valstrings,
-						   tmp_num_values,
-						   includes[count]);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_valstrings,
+						  tmp_num_values,
+						  includes[count]);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 
@@ -498,7 +515,7 @@ static WERROR smbconf_reg_get_values(TALLOC_CTX *mem_ctx,
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 static bool smbconf_reg_key_has_values(struct registry_key *key)
@@ -527,9 +544,10 @@ static bool smbconf_reg_key_has_values(struct registry_key *key)
 /**
  * delete all values from a key
  */
-static WERROR smbconf_reg_delete_values(struct registry_key *key)
+static sbcErr smbconf_reg_delete_values(struct registry_key *key)
 {
 	WERROR werr;
+	sbcErr err;
 	char *valname;
 	struct registry_value *valvalue;
 	uint32_t count;
@@ -542,6 +560,7 @@ static WERROR smbconf_reg_delete_values(struct registry_key *key)
 	{
 		werr = reg_deletevalue(key, valname);
 		if (!W_ERROR_IS_OK(werr)) {
+			err = SBC_ERR_ACCESS_DENIED;
 			goto done;
 		}
 	}
@@ -550,14 +569,15 @@ static WERROR smbconf_reg_delete_values(struct registry_key *key)
 			  "Error enumerating values of %s: %s\n",
 			  key->key->name,
 			  win_errstr(werr)));
+		err = SBC_ERR_ACCESS_DENIED;
 		goto done;
 	}
 
-	werr = WERR_OK;
+	err = SBC_ERR_OK;
 
 done:
 	talloc_free(mem_ctx);
-	return werr;
+	return err;
 }
 
 /**********************************************************************
@@ -569,9 +589,10 @@ done:
 /**
  * initialize the registry smbconf backend
  */
-static WERROR smbconf_reg_init(struct smbconf_ctx *ctx, const char *path)
+static sbcErr smbconf_reg_init(struct smbconf_ctx *ctx, const char *path)
 {
 	WERROR werr = WERR_OK;
+	sbcErr err;
 	struct security_token *token;
 
 	if (path == NULL) {
@@ -579,26 +600,28 @@ static WERROR smbconf_reg_init(struct smbconf_ctx *ctx, const char *path)
 	}
 	ctx->path = talloc_strdup(ctx, path);
 	if (ctx->path == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
-	ctx->data = TALLOC_ZERO_P(ctx, struct reg_private_data);
+	ctx->data = talloc_zero(ctx, struct reg_private_data);
 
 	werr = ntstatus_to_werror(registry_create_admin_token(ctx, &token));
 	if (!W_ERROR_IS_OK(werr)) {
 		DEBUG(1, ("Error creating admin token\n"));
+		err = SBC_ERR_UNKNOWN_FAILURE;
 		goto done;
 	}
 	rpd(ctx)->open = false;
 
 	werr = registry_init_smbconf(path);
 	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_BADFILE;
 		goto done;
 	}
 
-	werr = ctx->ops->open_conf(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = ctx->ops->open_conf(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		DEBUG(1, ("Error opening the registry.\n"));
 		goto done;
 	}
@@ -607,11 +630,12 @@ static WERROR smbconf_reg_init(struct smbconf_ctx *ctx, const char *path)
 			     KEY_ENUMERATE_SUB_KEYS | REG_KEY_WRITE,
 			     token, &rpd(ctx)->base_key);
 	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_UNKNOWN_FAILURE;
 		goto done;
 	}
 
 done:
-	return werr;
+	return err;
 }
 
 static int smbconf_reg_shutdown(struct smbconf_ctx *ctx)
@@ -640,19 +664,21 @@ static bool smbconf_reg_is_writeable(struct smbconf_ctx *ctx)
 	return true;
 }
 
-static WERROR smbconf_reg_open(struct smbconf_ctx *ctx)
+static sbcErr smbconf_reg_open(struct smbconf_ctx *ctx)
 {
 	WERROR werr;
 
 	if (rpd(ctx)->open) {
-		return WERR_OK;
+		return SBC_ERR_OK;
 	}
 
 	werr = regdb_open();
-	if (W_ERROR_IS_OK(werr)) {
-		rpd(ctx)->open = true;
+	if (!W_ERROR_IS_OK(werr)) {
+		return SBC_ERR_BADFILE;
 	}
-	return werr;
+
+	rpd(ctx)->open = true;
+	return SBC_ERR_OK;
 }
 
 static int smbconf_reg_close(struct smbconf_ctx *ctx)
@@ -682,7 +708,7 @@ static void smbconf_reg_get_csn(struct smbconf_ctx *ctx,
 		return;
 	}
 
-	if (!W_ERROR_IS_OK(ctx->ops->open_conf(ctx))) {
+	if (!SBC_ERROR_IS_OK(ctx->ops->open_conf(ctx))) {
 		return;
 	}
 
@@ -692,10 +718,11 @@ static void smbconf_reg_get_csn(struct smbconf_ctx *ctx,
 /**
  * Drop the whole configuration (restarting empty) - registry version
  */
-static WERROR smbconf_reg_drop(struct smbconf_ctx *ctx)
+static sbcErr smbconf_reg_drop(struct smbconf_ctx *ctx)
 {
 	char *path, *p;
 	WERROR werr = WERR_OK;
+	sbcErr err = SBC_ERR_OK;
 	struct registry_key *parent_key = NULL;
 	struct registry_key *new_key = NULL;
 	TALLOC_CTX* mem_ctx = talloc_stackframe();
@@ -705,46 +732,51 @@ static WERROR smbconf_reg_drop(struct smbconf_ctx *ctx)
 	werr = ntstatus_to_werror(registry_create_admin_token(ctx, &token));
 	if (!W_ERROR_IS_OK(werr)) {
 		DEBUG(1, ("Error creating admin token\n"));
+		err = SBC_ERR_UNKNOWN_FAILURE;
 		goto done;
 	}
 
 	path = talloc_strdup(mem_ctx, ctx->path);
 	if (path == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 	p = strrchr(path, '\\');
 	if (p == NULL) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 	*p = '\0';
 	werr = reg_open_path(mem_ctx, path, REG_KEY_WRITE, token,
 			     &parent_key);
-
 	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_IO_FAILURE;
 		goto done;
 	}
 
 	werr = reg_deletekey_recursive(parent_key, p+1);
-
 	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_IO_FAILURE;
 		goto done;
 	}
 
 	werr = reg_createkey(mem_ctx, parent_key, p+1, REG_KEY_WRITE,
 			     &new_key, &action);
+	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_IO_FAILURE;
+		goto done;
+	}
 
 done:
 	talloc_free(mem_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * get the list of share names defined in the configuration.
  * registry version.
  */
-static WERROR smbconf_reg_get_share_names(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_get_share_names(struct smbconf_ctx *ctx,
 					  TALLOC_CTX *mem_ctx,
 					  uint32_t *num_shares,
 					  char ***share_names)
@@ -752,13 +784,13 @@ static WERROR smbconf_reg_get_share_names(struct smbconf_ctx *ctx,
 	uint32_t count;
 	uint32_t added_count = 0;
 	TALLOC_CTX *tmp_ctx = NULL;
-	WERROR werr = WERR_OK;
+	WERROR werr;
+	sbcErr err = SBC_ERR_OK;
 	char *subkey_name = NULL;
 	char **tmp_share_names = NULL;
 
 	if ((num_shares == NULL) || (share_names == NULL)) {
-		werr = WERR_INVALID_PARAM;
-		goto done;
+		return SBC_ERR_INVALID_PARAM;
 	}
 
 	tmp_ctx = talloc_stackframe();
@@ -766,9 +798,9 @@ static WERROR smbconf_reg_get_share_names(struct smbconf_ctx *ctx,
 	/* if there are values in the base key, return NULL as share name */
 
 	if (smbconf_reg_key_has_values(rpd(ctx)->base_key)) {
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
 						   0, NULL);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 		added_count++;
@@ -776,9 +808,9 @@ static WERROR smbconf_reg_get_share_names(struct smbconf_ctx *ctx,
 
 	/* make sure "global" is always listed first */
 	if (smbconf_share_exists(ctx, GLOBAL_NAME)) {
-		werr = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
-						   added_count, GLOBAL_NAME);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_add_string_to_array(tmp_ctx, &tmp_share_names,
+						  added_count, GLOBAL_NAME);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 		added_count++;
@@ -794,19 +826,20 @@ static WERROR smbconf_reg_get_share_names(struct smbconf_ctx *ctx,
 			continue;
 		}
 
-		werr = smbconf_add_string_to_array(tmp_ctx,
+		err = smbconf_add_string_to_array(tmp_ctx,
 						   &tmp_share_names,
 						   added_count,
 						   subkey_name);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 		added_count++;
 	}
 	if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
+		err = SBC_ERR_NO_MORE_ITEMS;
 		goto done;
 	}
-	werr = WERR_OK;
+	err = SBC_ERR_OK;
 
 	*num_shares = added_count;
 	if (added_count > 0) {
@@ -817,7 +850,7 @@ static WERROR smbconf_reg_get_share_names(struct smbconf_ctx *ctx,
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 /**
@@ -827,13 +860,13 @@ static bool smbconf_reg_share_exists(struct smbconf_ctx *ctx,
 				     const char *servicename)
 {
 	bool ret = false;
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 	struct registry_key *key = NULL;
 
-	werr = smbconf_reg_open_service_key(mem_ctx, ctx, servicename,
-					    REG_KEY_READ, &key);
-	if (W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(mem_ctx, ctx, servicename,
+					   REG_KEY_READ, &key);
+	if (SBC_ERROR_IS_OK(err)) {
 		ret = true;
 	}
 
@@ -844,286 +877,330 @@ static bool smbconf_reg_share_exists(struct smbconf_ctx *ctx,
 /**
  * Add a service if it does not already exist - registry version
  */
-static WERROR smbconf_reg_create_share(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_create_share(struct smbconf_ctx *ctx,
 				       const char *servicename)
 {
-	WERROR werr;
+	sbcErr err;
 	struct registry_key *key = NULL;
 
 	if (servicename == NULL) {
-		return WERR_OK;
+		return SBC_ERR_OK;
 	}
 
-	werr = smbconf_reg_create_service_key(talloc_tos(), ctx,
-					      servicename, &key);
+	err = smbconf_reg_create_service_key(talloc_tos(), ctx,
+					     servicename, &key);
 
 	talloc_free(key);
-	return werr;
+	return err;
 }
 
 /**
  * get a definition of a share (service) from configuration.
  */
-static WERROR smbconf_reg_get_share(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_get_share(struct smbconf_ctx *ctx,
 				    TALLOC_CTX *mem_ctx,
 				    const char *servicename,
 				    struct smbconf_service **service)
 {
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	struct registry_key *key = NULL;
 	struct smbconf_service *tmp_service = NULL;
 	TALLOC_CTX *tmp_ctx = talloc_stackframe();
 
-	werr = smbconf_reg_open_service_key(tmp_ctx, ctx, servicename,
-					    REG_KEY_READ, &key);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(tmp_ctx, ctx, servicename,
+					   REG_KEY_READ, &key);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
-	tmp_service = TALLOC_ZERO_P(tmp_ctx, struct smbconf_service);
+	tmp_service = talloc_zero(tmp_ctx, struct smbconf_service);
 	if (tmp_service == NULL) {
-		werr =  WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
 	if (servicename != NULL) {
 		tmp_service->name = talloc_strdup(tmp_service, servicename);
 		if (tmp_service->name == NULL) {
-			werr = WERR_NOMEM;
+			err = SBC_ERR_NOMEM;
 			goto done;
 		}
 	}
 
-	werr = smbconf_reg_get_values(tmp_service, key,
-				      &(tmp_service->num_params),
-				      &(tmp_service->param_names),
-				      &(tmp_service->param_values));
-
-	if (W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_get_values(tmp_service, key,
+				     &(tmp_service->num_params),
+				     &(tmp_service->param_names),
+				     &(tmp_service->param_values));
+	if (SBC_ERROR_IS_OK(err)) {
 		*service = talloc_move(mem_ctx, &tmp_service);
 	}
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * delete a service from configuration
  */
-static WERROR smbconf_reg_delete_share(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_delete_share(struct smbconf_ctx *ctx,
 				       const char *servicename)
 {
-	WERROR werr = WERR_OK;
+	WERROR werr;
+	sbcErr err = SBC_ERR_OK;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 
 	if (servicename != NULL) {
 		werr = reg_deletekey_recursive(rpd(ctx)->base_key, servicename);
+		if (!W_ERROR_IS_OK(werr)) {
+			err = SBC_ERR_ACCESS_DENIED;
+		}
 	} else {
-		werr = smbconf_reg_delete_values(rpd(ctx)->base_key);
+		err = smbconf_reg_delete_values(rpd(ctx)->base_key);
 	}
 
 	talloc_free(mem_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * set a configuration parameter to the value provided.
  */
-static WERROR smbconf_reg_set_parameter(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_set_parameter(struct smbconf_ctx *ctx,
 					const char *service,
 					const char *param,
 					const char *valstr)
 {
-	WERROR werr;
+	sbcErr err;
 	struct registry_key *key = NULL;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 
-	werr = smbconf_reg_open_service_key(mem_ctx, ctx, service,
-					    REG_KEY_WRITE, &key);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(mem_ctx, ctx, service,
+					   REG_KEY_WRITE, &key);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
-	werr = smbconf_reg_set_value(key, param, valstr);
+	err = smbconf_reg_set_value(key, param, valstr);
 
 done:
 	talloc_free(mem_ctx);
-	return werr;
+	return err;
 }
 
 /**
  * get the value of a configuration parameter as a string
  */
-static WERROR smbconf_reg_get_parameter(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_get_parameter(struct smbconf_ctx *ctx,
 					TALLOC_CTX *mem_ctx,
 					const char *service,
 					const char *param,
 					char **valstr)
 {
 	WERROR werr = WERR_OK;
+	sbcErr err;
 	struct registry_key *key = NULL;
 	struct registry_value *value = NULL;
 
-	werr = smbconf_reg_open_service_key(mem_ctx, ctx, service,
-					    REG_KEY_READ, &key);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(mem_ctx, ctx, service,
+					   REG_KEY_READ, &key);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
 	if (!smbconf_reg_valname_valid(param)) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	if (!smbconf_value_exists(key, param)) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	werr = reg_queryvalue(mem_ctx, key, param, &value);
 	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_NOMEM;
 		goto done;
 	}
 
 	*valstr = smbconf_format_registry_value(mem_ctx, value);
-
 	if (*valstr == NULL) {
-		werr = WERR_NOMEM;
+		err = SBC_ERR_NOMEM;
 	}
 
 done:
 	talloc_free(key);
 	talloc_free(value);
-	return werr;
+	return err;
 }
 
 /**
  * delete a parameter from configuration
  */
-static WERROR smbconf_reg_delete_parameter(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_delete_parameter(struct smbconf_ctx *ctx,
 					   const char *service,
 					   const char *param)
 {
 	struct registry_key *key = NULL;
-	WERROR werr = WERR_OK;
+	WERROR werr;
+	sbcErr err;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 
-	werr = smbconf_reg_open_service_key(mem_ctx, ctx, service,
-					    REG_KEY_ALL, &key);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(mem_ctx, ctx, service,
+					   REG_KEY_ALL, &key);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
 	if (!smbconf_reg_valname_valid(param)) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_INVALID_PARAM;
 		goto done;
 	}
 
 	if (!smbconf_value_exists(key, param)) {
-		werr = WERR_INVALID_PARAM;
+		err = SBC_ERR_OK;
 		goto done;
 	}
 
 	werr = reg_deletevalue(key, param);
+	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_ACCESS_DENIED;
+	}
 
 done:
 	talloc_free(mem_ctx);
-	return werr;
+	return err;
 }
 
-static WERROR smbconf_reg_get_includes(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_get_includes(struct smbconf_ctx *ctx,
 				       TALLOC_CTX *mem_ctx,
 				       const char *service,
 				       uint32_t *num_includes,
 				       char ***includes)
 {
-	WERROR werr;
+	sbcErr err;
 	struct registry_key *key = NULL;
 	TALLOC_CTX *tmp_ctx = talloc_stackframe();
 
-	werr = smbconf_reg_open_service_key(tmp_ctx, ctx, service,
-					    REG_KEY_READ, &key);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(tmp_ctx, ctx, service,
+					   REG_KEY_READ, &key);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
-	werr = smbconf_reg_get_includes_internal(mem_ctx, key, num_includes,
+	err = smbconf_reg_get_includes_internal(mem_ctx, key, num_includes,
 						 includes);
+	if (!SBC_ERROR_IS_OK(err)) {
+		goto done;
+	}
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
-static WERROR smbconf_reg_set_includes(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_set_includes(struct smbconf_ctx *ctx,
 				       const char *service,
 				       uint32_t num_includes,
 				       const char **includes)
 {
 	WERROR werr = WERR_OK;
+	sbcErr err;
 	struct registry_key *key = NULL;
 	TALLOC_CTX *tmp_ctx = talloc_stackframe();
 
-	werr = smbconf_reg_open_service_key(tmp_ctx, ctx, service,
-					    REG_KEY_ALL, &key);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(tmp_ctx, ctx, service,
+					   REG_KEY_ALL, &key);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
 	if (num_includes == 0) {
 		if (!smbconf_value_exists(key, INCLUDES_VALNAME)) {
+			err = SBC_ERR_OK;
 			goto done;
 		}
 		werr = reg_deletevalue(key, INCLUDES_VALNAME);
+		if (!W_ERROR_IS_OK(werr)) {
+			err = SBC_ERR_ACCESS_DENIED;
+			goto done;
+		}
 	} else {
-		werr = smbconf_reg_set_multi_sz_value(key, INCLUDES_VALNAME,
+		err = smbconf_reg_set_multi_sz_value(key, INCLUDES_VALNAME,
 						      num_includes, includes);
 	}
 
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
-static WERROR smbconf_reg_delete_includes(struct smbconf_ctx *ctx,
+static sbcErr smbconf_reg_delete_includes(struct smbconf_ctx *ctx,
 					  const char *service)
 {
-	WERROR werr = WERR_OK;
+	WERROR werr;
+	sbcErr err;
 	struct registry_key *key = NULL;
 	TALLOC_CTX *tmp_ctx = talloc_stackframe();
 
-	werr = smbconf_reg_open_service_key(tmp_ctx, ctx, service,
-					    REG_KEY_ALL, &key);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_reg_open_service_key(tmp_ctx, ctx, service,
+					   REG_KEY_ALL, &key);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
 	if (!smbconf_value_exists(key, INCLUDES_VALNAME)) {
+		err = SBC_ERR_OK;
 		goto done;
 	}
 
 	werr = reg_deletevalue(key, INCLUDES_VALNAME);
+	if (!W_ERROR_IS_OK(werr)) {
+		err = SBC_ERR_ACCESS_DENIED;
+		goto done;
+	}
 
-
+	err = SBC_ERR_OK;
 done:
 	talloc_free(tmp_ctx);
-	return werr;
+	return err;
 }
 
-static WERROR smbconf_reg_transaction_start(struct smbconf_ctx *ctx)
+static sbcErr smbconf_reg_transaction_start(struct smbconf_ctx *ctx)
 {
-	return regdb_transaction_start();
+	WERROR werr;
+
+	werr = regdb_transaction_start();
+	if (!W_ERROR_IS_OK(werr)) {
+		return SBC_ERR_IO_FAILURE;
+	}
+
+	return SBC_ERR_OK;
 }
 
-static WERROR smbconf_reg_transaction_commit(struct smbconf_ctx *ctx)
+static sbcErr smbconf_reg_transaction_commit(struct smbconf_ctx *ctx)
 {
-	return regdb_transaction_commit();
+	WERROR werr;
+
+	werr = regdb_transaction_commit();
+	if (!W_ERROR_IS_OK(werr)) {
+		return SBC_ERR_IO_FAILURE;
+	}
+
+	return SBC_ERR_OK;
 }
 
-static WERROR smbconf_reg_transaction_cancel(struct smbconf_ctx *ctx)
+static sbcErr smbconf_reg_transaction_cancel(struct smbconf_ctx *ctx)
 {
-	return regdb_transaction_cancel();
+	WERROR werr;
+
+	werr = regdb_transaction_cancel();
+	if (!W_ERROR_IS_OK(werr)) {
+		return SBC_ERR_IO_FAILURE;
+	}
+
+	return SBC_ERR_OK;
 }
 
 struct smbconf_ops smbconf_ops_reg = {
@@ -1156,7 +1233,7 @@ struct smbconf_ops smbconf_ops_reg = {
  * initialize the smbconf registry backend
  * the only function that is exported from this module
  */
-WERROR smbconf_init_reg(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
+sbcErr smbconf_init_reg(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 			const char *path)
 {
 	return smbconf_init_internal(mem_ctx, conf_ctx, path, &smbconf_ops_reg);
diff --git a/source3/lib/smbconf/smbconf_reg.h b/source3/lib/smbconf/smbconf_reg.h
index 7f54b6e32db..2c49057a946 100644
--- a/source3/lib/smbconf/smbconf_reg.h
+++ b/source3/lib/smbconf/smbconf_reg.h
@@ -26,7 +26,7 @@ struct smbconf_ctx;
  * initialization functions for the registry backend modules
  */
 
-WERROR smbconf_init_reg(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
+sbcErr smbconf_init_reg(TALLOC_CTX *mem_ctx, struct smbconf_ctx **conf_ctx,
 			const char *path);
 
 
diff --git a/source3/lib/smbconf/testsuite.c b/source3/lib/smbconf/testsuite.c
index 80754dd20c3..c2a9a59cbef 100644
--- a/source3/lib/smbconf/testsuite.c
+++ b/source3/lib/smbconf/testsuite.c
@@ -40,17 +40,17 @@ static void print_strings(const char *prefix,
 
 static bool test_get_includes(struct smbconf_ctx *ctx)
 {
-	WERROR werr;
+	sbcErr err;
 	bool ret = false;
 	uint32_t num_includes = 0;
 	char **includes = NULL;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 
 	printf("TEST: get_includes\n");
-	werr = smbconf_get_global_includes(ctx, mem_ctx,
-					   &num_includes, &includes);
-	if (!W_ERROR_IS_OK(werr)) {
-		printf("FAIL: get_includes - %s\n", win_errstr(werr));
+	err = smbconf_get_global_includes(ctx, mem_ctx,
+					  &num_includes, &includes);
+	if (!SBC_ERROR_IS_OK(err)) {
+		printf("FAIL: get_includes - %s\n", sbcErrorString(err));
 		goto done;
 	}
 
@@ -68,7 +68,7 @@ done:
 
 static bool test_set_get_includes(struct smbconf_ctx *ctx)
 {
-	WERROR werr;
+	sbcErr err;
 	uint32_t count;
 	bool ret = false;
 	const char *set_includes[] = {
@@ -82,18 +82,18 @@ static bool test_set_get_includes(struct smbconf_ctx *ctx)
 
 	printf("TEST: set_get_includes\n");
 
-	werr = smbconf_set_global_includes(ctx, set_num_includes, set_includes);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_set_global_includes(ctx, set_num_includes, set_includes);
+	if (!SBC_ERROR_IS_OK(err)) {
 		printf("FAIL: get_set_includes (setting includes) - %s\n",
-		       win_errstr(werr));
+		       sbcErrorString(err));
 		goto done;
 	}
 
-	werr = smbconf_get_global_includes(ctx, mem_ctx, &get_num_includes,
-					   &get_includes);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_global_includes(ctx, mem_ctx, &get_num_includes,
+					  &get_includes);
+	if (!SBC_ERROR_IS_OK(err)) {
 		printf("FAIL: get_set_includes (getting includes) - %s\n",
-		       win_errstr(werr));
+		       sbcErrorString(err));
 		goto done;
 	}
 
@@ -125,7 +125,7 @@ done:
 
 static bool test_delete_includes(struct smbconf_ctx *ctx)
 {
-	WERROR werr;
+	sbcErr err;
 	bool ret = false;
 	const char *set_includes[] = {
 		"/path/to/include",
@@ -137,25 +137,25 @@ static bool test_delete_includes(struct smbconf_ctx *ctx)
 
 	printf("TEST: delete_includes\n");
 
-	werr = smbconf_set_global_includes(ctx, set_num_includes, set_includes);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_set_global_includes(ctx, set_num_includes, set_includes);
+	if (!SBC_ERROR_IS_OK(err)) {
 		printf("FAIL: delete_includes (setting includes) - %s\n",
-		       win_errstr(werr));
+		       sbcErrorString(err));
 		goto done;
 	}
 
-	werr = smbconf_delete_global_includes(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_delete_global_includes(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		printf("FAIL: delete_includes (deleting includes) - %s\n",
-		       win_errstr(werr));
+		       sbcErrorString(err));
 		goto done;
 	}
 
-	werr = smbconf_get_global_includes(ctx, mem_ctx, &get_num_includes,
-					   &get_includes);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_global_includes(ctx, mem_ctx, &get_num_includes,
+					  &get_includes);
+	if (!SBC_ERROR_IS_OK(err)) {
 		printf("FAIL: delete_includes (getting includes) - %s\n",
-		       win_errstr(werr));
+		       sbcErrorString(err));
 		goto done;
 	}
 
@@ -164,10 +164,10 @@ static bool test_delete_includes(struct smbconf_ctx *ctx)
 		goto done;
 	}
 
-	werr = smbconf_delete_global_includes(ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_delete_global_includes(ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		printf("FAIL: delete_includes (delete empty includes) - "
-		       "%s\n", win_errstr(werr));
+		       "%s\n", sbcErrorString(err));
 		goto done;
 	}
 
@@ -203,7 +203,7 @@ static bool create_conf_file(const char *filename)
 
 static bool torture_smbconf_txt(void)
 {
-	WERROR werr;
+	sbcErr err;
 	bool ret = true;
 	const char *filename = "/tmp/smb.conf.smbconf_testsuite";
 	struct smbconf_ctx *conf_ctx = NULL;
@@ -217,9 +217,9 @@ static bool torture_smbconf_txt(void)
 	}
 
 	printf("TEST: init\n");
-	werr = smbconf_init_txt(mem_ctx, &conf_ctx, filename);
-	if (!W_ERROR_IS_OK(werr)) {
-		printf("FAIL: text backend failed: %s\n", win_errstr(werr));
+	err = smbconf_init_txt(mem_ctx, &conf_ctx, filename);
+	if (!SBC_ERROR_IS_OK(err)) {
+		printf("FAIL: text backend failed: %s\n", sbcErrorString(err));
 		ret = false;
 		goto done;
 	}
@@ -245,7 +245,7 @@ done:
 
 static bool torture_smbconf_reg(void)
 {
-	WERROR werr;
+	sbcErr err;
 	bool ret = true;
 	struct smbconf_ctx *conf_ctx = NULL;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
@@ -253,9 +253,9 @@ static bool torture_smbconf_reg(void)
 	printf("test: registry backend\n");
 
 	printf("TEST: init\n");
-	werr = smbconf_init_reg(mem_ctx, &conf_ctx, NULL);
-	if (!W_ERROR_IS_OK(werr)) {
-		printf("FAIL: init failed: %s\n", win_errstr(werr));
+	err = smbconf_init_reg(mem_ctx, &conf_ctx, NULL);
+	if (!SBC_ERROR_IS_OK(err)) {
+		printf("FAIL: init failed: %s\n", sbcErrorString(err));
 		ret = false;
 		goto done;
 	}
diff --git a/source3/lib/dummysmbd.c b/source3/lib/smbd_shim.c
index 2465e6552bc..72ae366ec36 100644
--- a/source3/lib/dummysmbd.c
+++ b/source3/lib/smbd_shim.c
@@ -3,6 +3,7 @@
    RPC pipe client
 
    Copyright (C) Gerald (Jerry) Carter          2004.
+   Copyright (C) Andrew Bartlett                2011.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -18,48 +19,55 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-/* Stupid dummy functions required due to the horrible dependency mess
+/* Shim functions required due to the horrible dependency mess
    in Samba. */
 
 #include "includes.h"
+#include "smbd_shim.h"
+#include "smbd/proto.h"
 
-int find_service(TALLOC_CTX *ctx, const char *service_in, char **p_service_out)
-{
-	return -1;
-}
+static struct smbd_shim shim;
 
-bool conn_snum_used(int snum)
+void set_smbd_shim(const struct smbd_shim *shim_functions)
 {
-	return False;
+	shim = *shim_functions;
 }
 
 void cancel_pending_lock_requests_by_fid(files_struct *fsp,
 			struct byte_range_lock *br_lck,
 			enum file_close_type close_type)
 {
+	if (shim.cancel_pending_lock_requests_by_fid) {
+
+		shim.cancel_pending_lock_requests_by_fid(fsp, br_lck, close_type);
+	}
 }
 
 void send_stat_cache_delete_message(struct messaging_context *msg_ctx,
 				    const char *name)
 {
+	if (shim.send_stat_cache_delete_message) {
+		shim.send_stat_cache_delete_message(msg_ctx, name);
+	}
 }
 
 NTSTATUS can_delete_directory(struct connection_struct *conn,
 				const char *dirname)
 {
+	if (shim.can_delete_directory) {
+		return shim.can_delete_directory(conn, dirname);
+	}
 	return NT_STATUS_OK;
 }
 
 bool change_to_root_user(void)
 {
+	if (shim.change_to_root_user) {
+		return shim.change_to_root_user();
+	}
 	return false;
 }
 
-struct event_context *smbd_event_context(void)
-{
-	return NULL;
-}
-
 /**
  * The following two functions need to be called from inside the low-level BRL
  * code for oplocks correctness in smbd.  Since other utility binaries also
@@ -70,11 +78,33 @@ struct event_context *smbd_event_context(void)
 void contend_level2_oplocks_begin(files_struct *fsp,
 				  enum level2_contention_type type)
 {
+	if (shim.contend_level2_oplocks_begin) {
+		shim.contend_level2_oplocks_begin(fsp, type);
+	}
 	return;
 }
 
 void contend_level2_oplocks_end(files_struct *fsp,
 				enum level2_contention_type type)
 {
+	if (shim.contend_level2_oplocks_end) {
+		shim.contend_level2_oplocks_end(fsp, type);
+	}
+	return;
+}
+
+void become_root(void)
+{
+	if (shim.become_root) {
+		shim.become_root();
+	}
+        return;
+}
+
+void unbecome_root(void)
+{
+	if (shim.unbecome_root) {
+		shim.unbecome_root();
+	}
 	return;
 }
diff --git a/source3/lib/smbd_shim.h b/source3/lib/smbd_shim.h
new file mode 100644
index 00000000000..89cae2b694d
--- /dev/null
+++ b/source3/lib/smbd_shim.h
@@ -0,0 +1,56 @@
+/*
+   Unix SMB/CIFS implementation.
+   RPC pipe client
+
+   Copyright (C) Gerald (Jerry) Carter          2004.
+   Copyright (C) Andrew Bartlett                2011.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/* 
+   shim functions are used required to allow library code to have
+   references to smbd specific code. The smbd daemon sets up the set
+   of function calls that it wants used by calling
+   set_smbd_shim(). Other executables don't make this call, and get
+   default (dummy) versions of these functions.
+*/
+
+struct smbd_shim
+{
+	void (*cancel_pending_lock_requests_by_fid)(files_struct *fsp,
+						    struct byte_range_lock *br_lck,
+						    enum file_close_type close_type);
+	void (*send_stat_cache_delete_message)(struct messaging_context *msg_ctx,
+					       const char *name);
+
+	NTSTATUS (*can_delete_directory)(struct connection_struct *conn,
+					 const char *dirname);
+
+	bool (*change_to_root_user)(void);
+
+	void (*contend_level2_oplocks_begin)(files_struct *fsp,
+					     enum level2_contention_type type);
+	
+	void (*contend_level2_oplocks_end)(files_struct *fsp,
+					   enum level2_contention_type type);
+
+	void (*become_root)(void);
+
+	void (*unbecome_root)(void);
+};
+
+void set_smbd_shim(const struct smbd_shim *shim_functions);
+
+
diff --git a/source3/lib/smbldap.c b/source3/lib/smbldap.c
index fe432374641..38c7b207f56 100644
--- a/source3/lib/smbldap.c
+++ b/source3/lib/smbldap.c
@@ -239,7 +239,7 @@ ATTRIB_MAP_ENTRY sidmap_attr_list[] = {
 		i++;
 	i++;
 
-	names = TALLOC_ARRAY( mem_ctx, const char*, i );
+	names = talloc_array( mem_ctx, const char*, i );
 	if ( !names ) {
 		DEBUG(0,("get_attr_list: out of memory\n"));
 		return NULL;
@@ -405,7 +405,7 @@ ATTRIB_MAP_ENTRY sidmap_attr_list[] = {
 			return NULL;
 		}
 
-		if (StrCaseCmp(tmp, result) < 0) {
+		if (strcasecmp_m(tmp, result) < 0) {
 			TALLOC_FREE(result);
 			result = tmp;
 		} else {
@@ -474,7 +474,7 @@ ATTRIB_MAP_ENTRY sidmap_attr_list[] = {
 		return;
 	}
 
-	handle = TALLOC_P(mem_ctx, LDAPMessage *);
+	handle = talloc(mem_ctx, LDAPMessage *);
 	SMB_ASSERT(handle != NULL);
 
 	*handle = result;
@@ -494,7 +494,7 @@ ATTRIB_MAP_ENTRY sidmap_attr_list[] = {
 		return;
 	}
 
-	handle = TALLOC_P(mem_ctx, LDAPMod **);
+	handle = talloc(mem_ctx, LDAPMod **);
 	SMB_ASSERT(handle != NULL);
 
 	*handle = mod;
@@ -654,7 +654,7 @@ static void smbldap_make_mod_internal(LDAP *ldap_struct, LDAPMessage *existing,
 			equal = (newblob && (data_blob_cmp(&oldblob, newblob) == 0));
 		} else {
 			/* all of our string attributes are case insensitive */
-			equal = (newval && (StrCaseCmp(oldval, newval) == 0));
+			equal = (newval && (strcasecmp_m(oldval, newval) == 0));
 		}
 
 		if (equal) {
@@ -1480,7 +1480,7 @@ static int smbldap_search_ext(struct smbldap_state *ldap_state,
 	while (another_ldap_try(ldap_state, &rc, &attempts, endtime)) {
 		rc = ldap_search_ext_s(ldap_state->ldap_struct, base, scope, 
 				       utf8_filter,
-				       CONST_DISCARD(char **, attrs),
+				       discard_const_p(char *, attrs),
 				       attrsonly, sctrls, cctrls, &timeout,
 				       sizelimit, res);
 		if (rc != LDAP_SUCCESS) {
@@ -1562,7 +1562,7 @@ int smbldap_search_paged(struct smbldap_state *ldap_state,
 	}
 	ber_flatten(cookie_be, &cookie_bv);
 
-	pr.ldctl_oid = CONST_DISCARD(char *, ADS_PAGE_CTL_OID);
+	pr.ldctl_oid = discard_const_p(char, ADS_PAGE_CTL_OID);
 	pr.ldctl_iscritical = (char) critical;
 	pr.ldctl_value.bv_len = cookie_bv->bv_len;
 	pr.ldctl_value.bv_val = cookie_bv->bv_val;
@@ -1862,7 +1862,7 @@ NTSTATUS smbldap_init(TALLOC_CTX *mem_ctx, struct event_context *event_ctx,
 		      const char *location,
 		      struct smbldap_state **smbldap_state)
 {
-	*smbldap_state = TALLOC_ZERO_P(mem_ctx, struct smbldap_state);
+	*smbldap_state = talloc_zero(mem_ctx, struct smbldap_state);
 	if (!*smbldap_state) {
 		DEBUG(0, ("talloc() failed for ldapsam private_data!\n"));
 		return NT_STATUS_NO_MEMORY;
@@ -1924,7 +1924,7 @@ static bool smbldap_check_root_dse(LDAP *ld, const char **attrs, const char *val
 	}
 
 	rc = ldap_search_s(ld, "", LDAP_SCOPE_BASE, 
-			   "(objectclass=*)", CONST_DISCARD(char **, attrs), 0 , &msg);
+			   "(objectclass=*)", discard_const_p(char *, attrs), 0 , &msg);
 
 	if (rc != LDAP_SUCCESS) {
 		DEBUG(3,("smbldap_check_root_dse: Could not search rootDSE\n"));
diff --git a/source3/lib/string_init.c b/source3/lib/string_init.c
new file mode 100644
index 00000000000..40a6ef059ba
--- /dev/null
+++ b/source3/lib/string_init.c
@@ -0,0 +1,77 @@
+/*
+   Unix SMB/CIFS implementation.
+   Samba utility functions
+
+   Copyright (C) Andrew Tridgell 1992-2001
+   Copyright (C) Simo Sorce      2001-2002
+   Copyright (C) Martin Pool     2003
+   Copyright (C) James Peach	 2006
+   Copyright (C) Jeremy Allison  1992-2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+
+/* this is used to prevent lots of mallocs of size 1 */
+static const char null_string[] = "";
+
+/**
+ Set a string value, allocing the space for the string
+**/
+
+static bool string_init(char **dest,const char *src)
+{
+	size_t l;
+
+	if (!src)
+		src = "";
+
+	l = strlen(src);
+
+	if (l == 0) {
+		*dest = discard_const_p(char, null_string);
+	} else {
+		(*dest) = SMB_STRDUP(src);
+		if ((*dest) == NULL) {
+			DEBUG(0,("Out of memory in string_init\n"));
+			return false;
+		}
+	}
+	return(true);
+}
+
+/**
+ Free a string value.
+**/
+
+void string_free(char **s)
+{
+	if (!s || !(*s))
+		return;
+	if (*s == null_string)
+		*s = NULL;
+	SAFE_FREE(*s);
+}
+
+/**
+ Set a string value, deallocating any existing space, and allocing the space
+ for the string
+**/
+
+bool string_set(char **dest,const char *src)
+{
+	string_free(dest);
+	return(string_init(dest,src));
+}
diff --git a/source3/lib/substitute.c b/source3/lib/substitute.c
index e72a8c3b616..df217bc03f7 100644
--- a/source3/lib/substitute.c
+++ b/source3/lib/substitute.c
@@ -80,7 +80,7 @@ bool set_local_machine_name(const char *local_name, bool perm)
 const char *get_local_machine_name(void)
 {
 	if (!local_machine || !*local_machine) {
-		return global_myname();
+		return lp_netbios_name();
 	}
 
 	return local_machine;
@@ -324,105 +324,6 @@ static char * realloc_expand_env_var(char *str, char *p)
 }
 
 /*******************************************************************
-*******************************************************************/
-
-static char *longvar_domainsid( void )
-{
-	struct dom_sid sid;
-	fstring tmp;
-	char *sid_string;
-
-	if ( !secrets_fetch_domain_sid( lp_workgroup(), &sid ) ) {
-		return NULL;
-	}
-
-	sid_string = SMB_STRDUP( sid_to_fstring( tmp, &sid ) );
-
-	if ( !sid_string ) {
-		DEBUG(0,("longvar_domainsid: failed to dup SID string!\n"));
-	}
-
-	return sid_string;
-}
-
-/*******************************************************************
-*******************************************************************/
-
-struct api_longvar {
-	const char *name;
-	char* (*fn)( void );
-};
-
-static struct api_longvar longvar_table[] = {
-	{ "DomainSID",		longvar_domainsid },
-	{ NULL, 		NULL }
-};
-
-static char *get_longvar_val( const char *varname )
-{
-	int i;
-
-	DEBUG(7,("get_longvar_val: expanding variable [%s]\n", varname));
-
-	for ( i=0; longvar_table[i].name; i++ ) {
-		if ( strequal( longvar_table[i].name, varname ) ) {
-			return longvar_table[i].fn();
-		}
-	}
-
-	return NULL;
-}
-
-/*******************************************************************
- Expand the long smb.conf variable names given a pointer to a %(NAME).
- Return the number of characters by which the pointer should be advanced.
- When this is called p points at the '%' character.
-********************************************************************/
-
-static char *realloc_expand_longvar(char *str, char *p)
-{
-	fstring varname;
-	char *value;
-	char *q, *r;
-	int copylen;
-
-	if ( p[0] != '%' || p[1] != '(' ) {
-		return str;
-	}
-
-	/* Look for the terminating ')'.*/
-
-	if ((q = strchr_m(p,')')) == NULL) {
-		DEBUG(0,("realloc_expand_longvar: Unterminated environment variable [%s]\n", p));
-		return str;
-	}
-
-	/* Extract the name from within the %(NAME) string.*/
-
-	r = p+2;
-	copylen = MIN( (q-r), (sizeof(varname)-1) );
-	strncpy(varname, r, copylen);
-	varname[copylen] = '\0';
-
-	if ((value = get_longvar_val(varname)) == NULL) {
-		DEBUG(0,("realloc_expand_longvar: Variable [%s] not set.  Skipping\n", varname));
-		return str;
-	}
-
-	/* Copy the full %(NAME) into envname so it can be replaced.*/
-
-	copylen = MIN( (q+1-p),(sizeof(varname)-1) );
-	strncpy( varname, p, copylen );
-	varname[copylen] = '\0';
-	r = realloc_string_sub(str, varname, value);
-	SAFE_FREE( value );
-
-	/* skip over the %(varname) */
-
-	return r;
-}
-
-/*******************************************************************
  Patch from jkf@soton.ac.uk
  Added this to implement %p (NIS auto-map version of %H)
 *******************************************************************/
@@ -489,7 +390,7 @@ static const char *automount_server(const char *user_name)
 	if (local_machine_name && *local_machine_name) {
 		server_name = talloc_strdup(ctx, local_machine_name);
 	} else {
-		server_name = talloc_strdup(ctx, global_myname());
+		server_name = talloc_strdup(ctx, lp_netbios_name());
 	}
 
 	if (!server_name) {
@@ -632,13 +533,13 @@ static char *alloc_sub_basic(const char *smb_name, const char *domain_name,
 				sub_sockaddr[0] ? sub_sockaddr : "0.0.0.0");
 			break;
 		case 'L' : 
-			if ( StrnCaseCmp(p, "%LOGONSERVER%", strlen("%LOGONSERVER%")) == 0 ) {
+			if ( strncasecmp_m(p, "%LOGONSERVER%", strlen("%LOGONSERVER%")) == 0 ) {
 				break;
 			}
 			if (local_machine_name && *local_machine_name) {
 				a_string = realloc_string_sub(a_string, "%L", local_machine_name); 
 			} else {
-				a_string = realloc_string_sub(a_string, "%L", global_myname()); 
+				a_string = realloc_string_sub(a_string, "%L", lp_netbios_name());
 			}
 			break;
 		case 'N':
@@ -680,9 +581,6 @@ static char *alloc_sub_basic(const char *smb_name, const char *domain_name,
 		case '$' :
 			a_string = realloc_expand_env_var(a_string, p); /* Expand environment variables */
 			break;
-		case '(':
-			a_string = realloc_expand_longvar( a_string, p );
-			break;
 		case 'V' :
 			slprintf(vnnstr,sizeof(vnnstr)-1, "%u", get_my_vnn());
 			a_string = realloc_string_sub(a_string, "%V", vnnstr);
@@ -926,3 +824,32 @@ char *standard_sub_conn(TALLOC_CTX *ctx, connection_struct *conn, const char *st
 				"",
 				str);
 }
+
+/******************************************************************************
+ version of standard_sub_basic() for string lists; uses talloc_sub_basic()
+ for the work
+ *****************************************************************************/
+
+bool str_list_sub_basic( char **list, const char *smb_name,
+			 const char *domain_name )
+{
+	TALLOC_CTX *ctx = list;
+	char *s, *tmpstr;
+
+	while ( *list ) {
+		s = *list;
+		tmpstr = talloc_sub_basic(ctx, smb_name, domain_name, s);
+		if ( !tmpstr ) {
+			DEBUG(0,("str_list_sub_basic: "
+				"alloc_sub_basic() return NULL!\n"));
+			return false;
+		}
+
+		TALLOC_FREE(*list);
+		*list = tmpstr;
+
+		list++;
+	}
+
+	return true;
+}
diff --git a/source3/lib/substitute_generic.c b/source3/lib/substitute_generic.c
new file mode 100644
index 00000000000..fa347fb8157
--- /dev/null
+++ b/source3/lib/substitute_generic.c
@@ -0,0 +1,116 @@
+/*
+   Unix SMB/CIFS implementation.
+   Samba utility functions
+
+   Copyright (C) Andrew Tridgell 1992-2001
+   Copyright (C) Simo Sorce      2001-2002
+   Copyright (C) Martin Pool     2003
+   Copyright (C) James Peach	 2006
+   Copyright (C) Jeremy Allison  1992-2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+
+void fstring_sub(char *s,const char *pattern,const char *insert)
+{
+	string_sub(s, pattern, insert, sizeof(fstring));
+}
+
+/**
+ Similar to string_sub2, but it will accept only allocated strings
+ and may realloc them so pay attention at what you pass on no
+ pointers inside strings, no const may be passed
+ as string.
+**/
+
+char *realloc_string_sub2(char *string,
+			const char *pattern,
+			const char *insert,
+			bool remove_unsafe_characters,
+			bool allow_trailing_dollar)
+{
+	char *p, *in;
+	char *s;
+	ssize_t ls,lp,li,ld, i;
+
+	if (!insert || !pattern || !*pattern || !string || !*string)
+		return NULL;
+
+	s = string;
+
+	in = SMB_STRDUP(insert);
+	if (!in) {
+		DEBUG(0, ("realloc_string_sub: out of memory!\n"));
+		return NULL;
+	}
+	ls = (ssize_t)strlen(s);
+	lp = (ssize_t)strlen(pattern);
+	li = (ssize_t)strlen(insert);
+	ld = li - lp;
+	for (i=0;i<li;i++) {
+		switch (in[i]) {
+			case '$':
+				/* allow a trailing $
+				 * (as in machine accounts) */
+				if (allow_trailing_dollar && (i == li - 1 )) {
+					break;
+				}
+			case '`':
+			case '"':
+			case '\'':
+			case ';':
+			case '%':
+			case '\r':
+			case '\n':
+				if ( remove_unsafe_characters ) {
+					in[i] = '_';
+					break;
+				}
+			default:
+				/* ok */
+				break;
+		}
+	}
+
+	while ((p = strstr_m(s,pattern))) {
+		if (ld > 0) {
+			int offset = PTR_DIFF(s,string);
+			string = (char *)SMB_REALLOC(string, ls + ld + 1);
+			if (!string) {
+				DEBUG(0, ("realloc_string_sub: "
+					"out of memory!\n"));
+				SAFE_FREE(in);
+				return NULL;
+			}
+			p = string + offset + (p - s);
+		}
+		if (li != lp) {
+			memmove(p+li,p+lp,strlen(p+lp)+1);
+		}
+		memcpy(p, in, li);
+		s = p + li;
+		ls += ld;
+	}
+	SAFE_FREE(in);
+	return string;
+}
+
+char *realloc_string_sub(char *string,
+			const char *pattern,
+			const char *insert)
+{
+	return realloc_string_sub2(string, pattern, insert, true, false);
+}
diff --git a/source3/lib/system.c b/source3/lib/system.c
index 52b64772666..2f7a55f48c1 100644
--- a/source3/lib/system.c
+++ b/source3/lib/system.c
@@ -48,42 +48,6 @@
 
 
 /*******************************************************************
- A wrapper for memalign
-********************************************************************/
-
-void *sys_memalign( size_t align, size_t size )
-{
-#if defined(HAVE_POSIX_MEMALIGN)
-	void *p = NULL;
-	int ret = posix_memalign( &p, align, size );
-	if ( ret == 0 )
-		return p;
-
-	return NULL;
-#elif defined(HAVE_MEMALIGN)
-	return memalign( align, size );
-#else
-	/* On *BSD systems memaligns doesn't exist, but memory will
-	 * be aligned on allocations of > pagesize. */
-#if defined(SYSCONF_SC_PAGESIZE)
-	size_t pagesize = (size_t)sysconf(_SC_PAGESIZE);
-#elif defined(HAVE_GETPAGESIZE)
-	size_t pagesize = (size_t)getpagesize();
-#else
-	size_t pagesize = (size_t)-1;
-#endif
-	if (pagesize == (size_t)-1) {
-		DEBUG(0,("memalign functionalaity not available on this platform!\n"));
-		return NULL;
-	}
-	if (size < pagesize) {
-		size = pagesize;
-	}
-	return SMB_MALLOC(size);
-#endif
-}
-
-/*******************************************************************
  A wrapper for usleep in case we don't have one.
 ********************************************************************/
 
@@ -545,13 +509,13 @@ void update_stat_ex_create_time(struct stat_ex *dst,
 }
 
 #if defined(HAVE_EXPLICIT_LARGEFILE_SUPPORT) && defined(HAVE_OFF64_T) && defined(HAVE_STAT64)
-static void init_stat_ex_from_stat (struct stat_ex *dst,
-				    const struct stat64 *src,
-				    bool fake_dir_create_times)
+void init_stat_ex_from_stat (struct stat_ex *dst,
+			    const struct stat64 *src,
+			    bool fake_dir_create_times)
 #else
-static void init_stat_ex_from_stat (struct stat_ex *dst,
-				    const struct stat *src,
-				    bool fake_dir_create_times)
+void init_stat_ex_from_stat (struct stat_ex *dst,
+			    const struct stat *src,
+			    bool fake_dir_create_times)
 #endif
 {
 	dst->st_ex_dev = src->st_dev;
@@ -833,6 +797,15 @@ FILE *sys_fopen(const char *path, const char *type)
 }
 
 
+#if HAVE_KERNEL_SHARE_MODES
+#ifndef LOCK_MAND
+#define LOCK_MAND	32	/* This is a mandatory flock */
+#define LOCK_READ	64	/* ... Which allows concurrent read operations */
+#define LOCK_WRITE	128	/* ... Which allows concurrent write operations */
+#define LOCK_RW		192	/* ... Which allows concurrent read & write ops */
+#endif
+#endif
+
 /*******************************************************************
  A flock() wrapper that will perform the kernel flock.
 ********************************************************************/
@@ -986,18 +959,45 @@ int sys_waitpid(pid_t pid,int *status,int options)
 }
 
 /*******************************************************************
- System wrapper for getwd
+ System wrapper for getwd. Always returns MALLOC'ed memory, or NULL
+ on error (malloc fail usually).
 ********************************************************************/
 
-char *sys_getwd(char *s)
+char *sys_getwd(void)
 {
-	char *wd;
-#ifdef HAVE_GETCWD
-	wd = (char *)getcwd(s, PATH_MAX);
+#ifdef GETCWD_TAKES_NULL
+	return getcwd(NULL, 0);
+#elif HAVE_GETCWD
+	char *wd = NULL, *s = NULL;
+	size_t allocated = PATH_MAX;
+
+	while (1) {
+		s = SMB_REALLOC_ARRAY(s, char, allocated);
+		if (s == NULL) {
+			return NULL;
+		}
+		wd = getcwd(s, allocated);
+		if (wd) {
+			break;
+		}
+		if (errno != ERANGE) {
+			SAFE_FREE(s);
+			break;
+		}
+		allocated *= 2;
+		if (allocated < PATH_MAX) {
+			SAFE_FREE(s);
+			break;
+		}
+	}
+	return wd;
 #else
-	wd = (char *)getwd(s);
+	char *s = SMB_MALLOC_ARRAY(char, PATH_MAX);
+	if (s == NULL) {
+		return NULL;
+	}
+	return getwd(s);
 #endif
-	return wd;
 }
 
 #if defined(HAVE_POSIX_CAPABILITIES)
@@ -1382,7 +1382,7 @@ static char **extract_args(TALLOC_CTX *mem_ctx, const char *command)
 
 	TALLOC_FREE(trunc_cmd);
 
-	if (!(argl = TALLOC_ARRAY(mem_ctx, char *, argcl + 1))) {
+	if (!(argl = talloc_array(mem_ctx, char *, argcl + 1))) {
 		goto nomem;
 	}
 
@@ -2367,7 +2367,7 @@ static ssize_t solaris_list_xattr(int attrdirfd, char *list, size_t size)
 	dirp = fdopendir(newfd);
 
 	while ((de = readdir(dirp))) {
-		size_t listlen = strlen(de->d_name);
+		size_t listlen = strlen(de->d_name) + 1;
 		if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
 			/* we don't want "." and ".." here: */
 			DEBUG(10,("skipped EA %s\n",de->d_name));
@@ -2376,18 +2376,16 @@ static ssize_t solaris_list_xattr(int attrdirfd, char *list, size_t size)
 
 		if (size == 0) {
 			/* return the current size of the list of extended attribute names*/
-			len += listlen + 1;
+			len += listlen;
 		} else {
 			/* check size and copy entrieѕ + nul into list. */
-			if ((len + listlen + 1) > size) {
+			if ((len + listlen) > size) {
 				errno = ERANGE;
 				len = -1;
 				break;
 			} else {
-				safe_strcpy(list + len, de->d_name, listlen);
+				strlcpy(list + len, de->d_name, listlen);
 				len += listlen;
-				list[len] = '\0';
-				++len;
 			}
 		}
 	}
@@ -2633,74 +2631,3 @@ int sys_aio_suspend(const SMB_STRUCT_AIOCB * const cblist[], int n, const struct
 	return -1;
 }
 #endif /* WITH_AIO */
-
-int sys_getpeereid( int s, uid_t *uid)
-{
-#if defined(HAVE_PEERCRED)
-	struct ucred cred;
-	socklen_t cred_len = sizeof(struct ucred);
-	int ret;
-
-	ret = getsockopt(s, SOL_SOCKET, SO_PEERCRED, (void *)&cred, &cred_len);
-	if (ret != 0) {
-		return -1;
-	}
-
-	if (cred_len != sizeof(struct ucred)) {
-		errno = EINVAL;
-		return -1;
-	}
-
-	*uid = cred.uid;
-	return 0;
-#else
-#if defined(HAVE_GETPEEREID)
-	gid_t gid;
-	return getpeereid(s, uid, &gid);
-#endif
-	errno = ENOSYS;
-	return -1;
-#endif
-}
-
-int sys_getnameinfo(const struct sockaddr *psa,
-			socklen_t salen,
-			char *host,
-			size_t hostlen,
-			char *service,
-			size_t servlen,
-			int flags)
-{
-	/*
-	 * For Solaris we must make sure salen is the
-	 * correct length for the incoming sa_family.
-	 */
-
-	if (salen == sizeof(struct sockaddr_storage)) {
-		salen = sizeof(struct sockaddr_in);
-#if defined(HAVE_IPV6)
-		if (psa->sa_family == AF_INET6) {
-			salen = sizeof(struct sockaddr_in6);
-		}
-#endif
-	}
-	return getnameinfo(psa, salen, host, hostlen, service, servlen, flags);
-}
-
-int sys_connect(int fd, const struct sockaddr * addr)
-{
-	socklen_t salen = (socklen_t)-1;
-
-	if (addr->sa_family == AF_INET) {
-	    salen = sizeof(struct sockaddr_in);
-	} else if (addr->sa_family == AF_UNIX) {
-	    salen = sizeof(struct sockaddr_un);
-	}
-#if defined(HAVE_IPV6)
-	else if (addr->sa_family == AF_INET6) {
-	    salen = sizeof(struct sockaddr_in6);
-	}
-#endif
-
-	return connect(fd, addr, salen);
-}
diff --git a/source3/lib/talloc_dict.c b/source3/lib/talloc_dict.c
index cbe5c7b5c2a..95ae0a3260d 100644
--- a/source3/lib/talloc_dict.c
+++ b/source3/lib/talloc_dict.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "dbwrap.h"
 #include "talloc_dict.h"
+#include "util_tdb.h"
 
 struct talloc_dict {
 	struct db_context *db;
diff --git a/source3/lib/tallocmsg.c b/source3/lib/tallocmsg.c
index da380c93355..9a0ce8ada18 100644
--- a/source3/lib/tallocmsg.c
+++ b/source3/lib/tallocmsg.c
@@ -65,7 +65,7 @@ static void msg_pool_usage_helper(const void *ptr, int depth, int max_depth, int
 			       (unsigned long)talloc_total_blocks(ptr),
 			       talloc_reference_count(ptr),
 			       MIN(50, talloc_get_size(ptr)),
-			       (char *)ptr);
+			       (const char *)ptr);
 		return;
 	}
 
diff --git a/source3/lib/tdb_validate.c b/source3/lib/tdb_validate.c
index b91ea7af834..385f4d0ef81 100644
--- a/source3/lib/tdb_validate.c
+++ b/source3/lib/tdb_validate.c
@@ -21,6 +21,7 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "util_tdb.h"
 #include "tdb_validate.h"
 
 /*
@@ -50,12 +51,13 @@ static int tdb_validate_child(struct tdb_context *tdb,
 	 * but I don't want to change all the callers...
 	 */
 	ret = tdb_check(tdb, NULL, NULL);
-	if (ret == -1) {
+	if (ret != 0) {
 		v_status.tdb_error = True;
 		v_status.success = False;
 		goto out;
 	}
 
+#ifndef BUILD_TDB2
 	/* Check if the tdb's freelist is good. */
 	if (tdb_validate_freelist(tdb, &num_entries) == -1) {
 		v_status.bad_freelist = True;
@@ -65,12 +67,13 @@ static int tdb_validate_child(struct tdb_context *tdb,
 
 	DEBUG(10,("tdb_validate_child: tdb %s freelist has %d entries\n",
 		  tdb_name(tdb), num_entries));
+#endif
 
 	/* Now traverse the tdb to validate it. */
 	num_entries = tdb_traverse(tdb, validate_fn, (void *)&v_status);
 	if (!v_status.success) {
 		goto out;
-	} else if (num_entries == -1) {
+	} else if (num_entries < 0) {
 		v_status.tdb_error = True;
 		v_status.success = False;
 		goto out;
@@ -288,8 +291,14 @@ static int tdb_backup(TALLOC_CTX *ctx, const char *src_path,
 	}
 
 	unlink(tmp_path);
-	dst_tdb = tdb_open_log(tmp_path,
-			       hash_size ? hash_size : tdb_hash_size(src_tdb),
+
+#ifndef BUILD_TDB2
+	if (!hash_size) {
+		hash_size = tdb_hash_size(src_tdb);
+	}
+#endif
+
+	dst_tdb = tdb_open_log(tmp_path, hash_size,
 			       TDB_DEFAULT, O_RDWR | O_CREAT | O_EXCL,
 			       st.st_mode & 0777);
 	if (dst_tdb == NULL) {
diff --git a/source3/lib/tdb_validate.h b/source3/lib/tdb_validate.h
index 3e7c20d04cf..5bb043d2924 100644
--- a/source3/lib/tdb_validate.h
+++ b/source3/lib/tdb_validate.h
@@ -23,7 +23,7 @@
 #define __TDB_VALIDATE_H__
 
 #include "lib/replace/replace.h"
-#include <tdb.h>
+#include "tdb_compat.h"
 
 /**
  * Flag field for keeping track of the status of a validation.
diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c
index cd1dea52ba5..8b04d006367 100644
--- a/source3/lib/tldap.c
+++ b/source3/lib/tldap.c
@@ -20,6 +20,8 @@
 #include "includes.h"
 #include "tldap.h"
 #include "../lib/util/asn1.h"
+#include "../lib/tsocket/tsocket.h"
+#include "../lib/util/tevent_unix.h"
 
 static int tldap_simple_recv(struct tevent_req *req);
 
@@ -187,7 +189,7 @@ bool tldap_context_setattr(struct tldap_context *ld,
 	struct tldap_ctx_attribute *tmp, *attr;
 	char *tmpname;
 	int num_attrs;
-	void **pptr = (void **)_pptr;
+	void **pptr = (void **)discard_const_p(void,_pptr);
 
 	attr = tldap_context_findattr(ld, name);
 	if (attr != NULL) {
@@ -539,7 +541,7 @@ static void tldap_msg_sent(struct tevent_req *subreq)
 	}
 
 	if (!tldap_msg_set_pending(req)) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return;
 	}
 }
@@ -933,10 +935,10 @@ struct tevent_req *tldap_simple_bind_send(TALLOC_CTX *mem_ctx,
 	DATA_BLOB cred;
 
 	if (passwd != NULL) {
-		cred.data = (uint8_t *)passwd;
+		cred.data = discard_const_p(uint8_t, passwd);
 		cred.length = strlen(passwd);
 	} else {
-		cred.data = (uint8_t *)"";
+		cred.data = discard_const_p(uint8_t, "");
 		cred.length = 0;
 	}
 	return tldap_sasl_bind_send(mem_ctx, ev, ld, dn, NULL, &cred, NULL, 0,
@@ -954,10 +956,10 @@ int tldap_simple_bind(struct tldap_context *ld, const char *dn,
 	DATA_BLOB cred;
 
 	if (passwd != NULL) {
-		cred.data = (uint8_t *)passwd;
+		cred.data = discard_const_p(uint8_t, passwd);
 		cred.length = strlen(passwd);
 	} else {
-		cred.data = (uint8_t *)"";
+		cred.data = discard_const_p(uint8_t, "");
 		cred.length = 0;
 	}
 	return tldap_sasl_bind(ld, dn, NULL, &cred, NULL, 0, NULL, 0);
@@ -1373,7 +1375,7 @@ static bool tldap_push_filter_basic(struct tldap_context *ld,
 				return false;
 			}
 
-			if (StrnCaseCmp(dn, "dn:", 3) != 0) {
+			if (strncasecmp_m(dn, "dn:", 3) != 0) {
 				if (rule == e) {
 					rule = dn;
 					dn = NULL;
@@ -1701,7 +1703,7 @@ static void tldap_search_done(struct tevent_req *subreq)
 	case TLDAP_RES_SEARCH_ENTRY:
 	case TLDAP_RES_SEARCH_REFERENCE:
 		if (!tldap_msg_set_pending(subreq)) {
-			tevent_req_nomem(NULL, req);
+			tevent_req_oom(req);
 			return;
 		}
 		tevent_req_notify_callback(req);
diff --git a/source3/lib/tldap_util.c b/source3/lib/tldap_util.c
index 0c22d6564e5..6b9f912e277 100644
--- a/source3/lib/tldap_util.c
+++ b/source3/lib/tldap_util.c
@@ -300,7 +300,7 @@ static int compare_utf8_blobs(const DATA_BLOB *d1, const DATA_BLOB *d2)
 		TALLOC_FREE(s1);
 		return 0;
 	}
-	ret = StrCaseCmp(s1, s2);
+	ret = strcasecmp_m(s1, s2);
 	TALLOC_FREE(s2);
 	TALLOC_FREE(s1);
 	return ret;
@@ -325,7 +325,7 @@ bool tldap_make_mod_fmt(struct tldap_message *existing, TALLOC_CTX *mem_ctx,
 
 	blob.length = strlen(newval);
 	if (blob.length != 0) {
-		blob.data = CONST_DISCARD(uint8_t *, newval);
+		blob.data = discard_const_p(uint8_t, newval);
 	}
 	ret = tldap_make_mod_blob_int(existing, mem_ctx, pmods, pnum_mods,
 				      attrib, blob, compare_utf8_blobs);
diff --git a/source3/lib/util.c b/source3/lib/util.c
index 0bb46db05f6..b8fc319a6ff 100644
--- a/source3/lib/util.c
+++ b/source3/lib/util.c
@@ -24,9 +24,11 @@
 #include "includes.h"
 #include "system/passwd.h"
 #include "system/filesys.h"
+#include "util_tdb.h"
 #include "ctdbd_conn.h"
 #include "../lib/util/util_pw.h"
 #include "messages.h"
+#include <ccan/hash/hash.h>
 
 /* Max allowable allococation - 256mb - 0x10000000 */
 #define MAX_ALLOC_SIZE (1024*1024*256)
@@ -71,87 +73,6 @@ void set_Protocol(enum protocol_types  p)
 
 static enum remote_arch_types ra_type = RA_UNKNOWN;
 
-/***********************************************************************
- Definitions for all names.
-***********************************************************************/
-
-static char *smb_scope;
-static int smb_num_netbios_names;
-static char **smb_my_netbios_names;
-
-/***********************************************************************
- Allocate and set scope. Ensure upper case.
-***********************************************************************/
-
-bool set_global_scope(const char *scope)
-{
-	SAFE_FREE(smb_scope);
-	smb_scope = SMB_STRDUP(scope);
-	if (!smb_scope)
-		return False;
-	strupper_m(smb_scope);
-	return True;
-}
-
-/*********************************************************************
- Ensure scope is never null string.
-*********************************************************************/
-
-const char *global_scope(void)
-{
-	if (!smb_scope)
-		set_global_scope("");
-	return smb_scope;
-}
-
-static void free_netbios_names_array(void)
-{
-	int i;
-
-	for (i = 0; i < smb_num_netbios_names; i++)
-		SAFE_FREE(smb_my_netbios_names[i]);
-
-	SAFE_FREE(smb_my_netbios_names);
-	smb_num_netbios_names = 0;
-}
-
-static bool allocate_my_netbios_names_array(size_t number)
-{
-	free_netbios_names_array();
-
-	smb_num_netbios_names = number + 1;
-	smb_my_netbios_names = SMB_MALLOC_ARRAY( char *, smb_num_netbios_names );
-
-	if (!smb_my_netbios_names)
-		return False;
-
-	memset(smb_my_netbios_names, '\0', sizeof(char *) * smb_num_netbios_names);
-	return True;
-}
-
-static bool set_my_netbios_names(const char *name, int i)
-{
-	SAFE_FREE(smb_my_netbios_names[i]);
-
-	smb_my_netbios_names[i] = SMB_STRDUP(name);
-	if (!smb_my_netbios_names[i])
-		return False;
-	strupper_m(smb_my_netbios_names[i]);
-	return True;
-}
-
-/***********************************************************************
- Free memory allocated to global objects
-***********************************************************************/
-
-void gfree_names(void)
-{
-	gfree_netbios_names();
-	SAFE_FREE( smb_scope );
-	free_netbios_names_array();
-	free_local_machine_name();
-}
-
 void gfree_all( void )
 {
 	gfree_names();
@@ -161,87 +82,6 @@ void gfree_all( void )
 	gfree_debugsyms();
 }
 
-const char *my_netbios_names(int i)
-{
-	return smb_my_netbios_names[i];
-}
-
-bool set_netbios_aliases(const char **str_array)
-{
-	size_t namecount;
-
-	/* Work out the max number of netbios aliases that we have */
-	for( namecount=0; str_array && (str_array[namecount] != NULL); namecount++ )
-		;
-
-	if ( global_myname() && *global_myname())
-		namecount++;
-
-	/* Allocate space for the netbios aliases */
-	if (!allocate_my_netbios_names_array(namecount))
-		return False;
-
-	/* Use the global_myname string first */
-	namecount=0;
-	if ( global_myname() && *global_myname()) {
-		set_my_netbios_names( global_myname(), namecount );
-		namecount++;
-	}
-
-	if (str_array) {
-		size_t i;
-		for ( i = 0; str_array[i] != NULL; i++) {
-			size_t n;
-			bool duplicate = False;
-
-			/* Look for duplicates */
-			for( n=0; n<namecount; n++ ) {
-				if( strequal( str_array[i], my_netbios_names(n) ) ) {
-					duplicate = True;
-					break;
-				}
-			}
-			if (!duplicate) {
-				if (!set_my_netbios_names(str_array[i], namecount))
-					return False;
-				namecount++;
-			}
-		}
-	}
-	return True;
-}
-
-/****************************************************************************
-  Common name initialization code.
-****************************************************************************/
-
-bool init_names(void)
-{
-	int n;
-
-	if (global_myname() == NULL || *global_myname() == '\0') {
-		if (!set_global_myname(myhostname())) {
-			DEBUG( 0, ( "init_names: malloc fail.\n" ) );
-			return False;
-		}
-	}
-
-	if (!set_netbios_aliases(lp_netbios_aliases())) {
-		DEBUG( 0, ( "init_names: malloc fail.\n" ) );
-		return False;
-	}
-
-	set_local_machine_name(global_myname(),false);
-
-	DEBUG( 5, ("Netbios name list:-\n") );
-	for( n=0; my_netbios_names(n); n++ ) {
-		DEBUGADD( 5, ("my_netbios_names[%d]=\"%s\"\n",
-					n, my_netbios_names(n) ) );
-	}
-
-	return( True );
-}
-
 /*******************************************************************
  Check if a file exists - call vfs_file_exist for samba files.
 ********************************************************************/
@@ -295,30 +135,10 @@ SMB_OFF_T get_file_size(char *file_name)
 }
 
 /*******************************************************************
- Return a string representing an attribute for a file.
-********************************************************************/
-
-char *attrib_string(uint16 mode)
-{
-	fstring attrstr;
-
-	attrstr[0] = 0;
-
-	if (mode & aVOLID) fstrcat(attrstr,"V");
-	if (mode & aDIR) fstrcat(attrstr,"D");
-	if (mode & aARCH) fstrcat(attrstr,"A");
-	if (mode & aHIDDEN) fstrcat(attrstr,"H");
-	if (mode & aSYSTEM) fstrcat(attrstr,"S");
-	if (mode & aRONLY) fstrcat(attrstr,"R");	  
-
-	return talloc_strdup(talloc_tos(), attrstr);
-}
-
-/*******************************************************************
  Show a smb message structure.
 ********************************************************************/
 
-void show_msg(char *buf)
+void show_msg(const char *buf)
 {
 	int i;
 	int bcc=0;
@@ -355,7 +175,7 @@ void show_msg(char *buf)
 	if (DEBUGLEVEL < 50)
 		bcc = MIN(bcc, 512);
 
-	dump_data(10, (uint8 *)smb_buf(buf), bcc);	
+	dump_data(10, (const uint8 *)smb_buf_const(buf), bcc);
 }
 
 /*******************************************************************
@@ -407,7 +227,7 @@ ssize_t message_push_blob(uint8 **outbuf, DATA_BLOB blob)
 	size_t newlen = smb_len(*outbuf) + 4 + blob.length;
 	uint8 *tmp;
 
-	if (!(tmp = TALLOC_REALLOC_ARRAY(NULL, *outbuf, uint8, newlen))) {
+	if (!(tmp = talloc_realloc(NULL, *outbuf, uint8, newlen))) {
 		DEBUG(0, ("talloc failed\n"));
 		return -1;
 	}
@@ -586,7 +406,7 @@ NTSTATUS reinit_after_fork(struct messaging_context *msg_ctx,
 	set_need_random_reseed();
 
 	/* tdb needs special fork handling */
-	if (tdb_reopen_all(parent_longlived ? 1 : 0) == -1) {
+	if (tdb_reopen_all(parent_longlived ? 1 : 0) != 0) {
 		DEBUG(0,("tdb_reopen_all failed.\n"));
 		status = NT_STATUS_OPEN_FAILED;
 		goto done;
@@ -611,149 +431,6 @@ NTSTATUS reinit_after_fork(struct messaging_context *msg_ctx,
 	return status;
 }
 
-#if defined(PARANOID_MALLOC_CHECKER)
-
-/****************************************************************************
- Internal malloc wrapper. Externally visible.
-****************************************************************************/
-
-void *malloc_(size_t size)
-{
-	if (size == 0) {
-		return NULL;
-	}
-#undef malloc
-	return malloc(size);
-#define malloc(s) __ERROR_DONT_USE_MALLOC_DIRECTLY
-}
-
-/****************************************************************************
- Internal calloc wrapper. Not externally visible.
-****************************************************************************/
-
-static void *calloc_(size_t count, size_t size)
-{
-	if (size == 0 || count == 0) {
-		return NULL;
-	}
-#undef calloc
-	return calloc(count, size);
-#define calloc(n,s) __ERROR_DONT_USE_CALLOC_DIRECTLY
-}
-
-/****************************************************************************
- Internal realloc wrapper. Not externally visible.
-****************************************************************************/
-
-static void *realloc_(void *ptr, size_t size)
-{
-#undef realloc
-	return realloc(ptr, size);
-#define realloc(p,s) __ERROR_DONT_USE_RELLOC_DIRECTLY
-}
-
-#endif /* PARANOID_MALLOC_CHECKER */
-
-/****************************************************************************
- Type-safe memalign
-****************************************************************************/
-
-void *memalign_array(size_t el_size, size_t align, unsigned int count)
-{
-	if (count >= MAX_ALLOC_SIZE/el_size) {
-		return NULL;
-	}
-
-	return sys_memalign(align, el_size*count);
-}
-
-/****************************************************************************
- Type-safe calloc.
-****************************************************************************/
-
-void *calloc_array(size_t size, size_t nmemb)
-{
-	if (nmemb >= MAX_ALLOC_SIZE/size) {
-		return NULL;
-	}
-	if (size == 0 || nmemb == 0) {
-		return NULL;
-	}
-#if defined(PARANOID_MALLOC_CHECKER)
-	return calloc_(nmemb, size);
-#else
-	return calloc(nmemb, size);
-#endif
-}
-
-/****************************************************************************
- Expand a pointer to be a particular size.
- Note that this version of Realloc has an extra parameter that decides
- whether to free the passed in storage on allocation failure or if the
- new size is zero.
-
- This is designed for use in the typical idiom of :
-
- p = SMB_REALLOC(p, size)
- if (!p) {
-    return error;
- }
-
- and not to have to keep track of the old 'p' contents to free later, nor
- to worry if the size parameter was zero. In the case where NULL is returned
- we guarentee that p has been freed.
-
- If free later semantics are desired, then pass 'free_old_on_error' as False which
- guarentees that the old contents are not freed on error, even if size == 0. To use
- this idiom use :
-
- tmp = SMB_REALLOC_KEEP_OLD_ON_ERROR(p, size);
- if (!tmp) {
-    SAFE_FREE(p);
-    return error;
- } else {
-    p = tmp;
- }
-
- Changes were instigated by Coverity error checking. JRA.
-****************************************************************************/
-
-void *Realloc(void *p, size_t size, bool free_old_on_error)
-{
-	void *ret=NULL;
-
-	if (size == 0) {
-		if (free_old_on_error) {
-			SAFE_FREE(p);
-		}
-		DEBUG(2,("Realloc asked for 0 bytes\n"));
-		return NULL;
-	}
-
-#if defined(PARANOID_MALLOC_CHECKER)
-	if (!p) {
-		ret = (void *)malloc_(size);
-	} else {
-		ret = (void *)realloc_(p,size);
-	}
-#else
-	if (!p) {
-		ret = (void *)malloc(size);
-	} else {
-		ret = (void *)realloc(p,size);
-	}
-#endif
-
-	if (!ret) {
-		if (free_old_on_error && p) {
-			SAFE_FREE(p);
-		}
-		DEBUG(0,("Memory allocation error: failed to expand to %d bytes\n",(int)size));
-	}
-
-	return(ret);
-}
-
 /****************************************************************************
  (Hopefully) efficient array append.
 ****************************************************************************/
@@ -1318,7 +995,7 @@ bool is_in_path(const char *name, name_compare_entry *namelist, bool case_sensit
 			}
 		} else {
 			if((case_sensitive && (strcmp(last_component, namelist->name) == 0))||
-						(!case_sensitive && (StrCaseCmp(last_component, namelist->name) == 0))) {
+						(!case_sensitive && (strcasecmp_m(last_component, namelist->name) == 0))) {
 				DEBUG(8,("is_in_path: match succeeded\n"));
 				return True;
 			}
@@ -1334,25 +1011,31 @@ bool is_in_path(const char *name, name_compare_entry *namelist, bool case_sensit
  passing to is_in_path(). We do this for
  speed so we can pre-parse all the names in the list 
  and don't do it for each call to is_in_path().
- namelist is modified here and is assumed to be 
- a copy owned by the caller.
  We also check if the entry contains a wildcard to
  remove a potentially expensive call to mask_match
  if possible.
 ********************************************************************/
 
-void set_namearray(name_compare_entry **ppname_array, const char *namelist)
+void set_namearray(name_compare_entry **ppname_array, const char *namelist_in)
 {
 	char *name_end;
-	char *nameptr = (char *)namelist;
+	char *namelist;
+	char *nameptr;
 	int num_entries = 0;
 	int i;
 
 	(*ppname_array) = NULL;
 
-	if((nameptr == NULL ) || ((nameptr != NULL) && (*nameptr == '\0'))) 
+	if((namelist_in == NULL ) || ((namelist_in != NULL) && (*namelist_in == '\0'))) 
 		return;
 
+	namelist = talloc_strdup(talloc_tos(), namelist_in);
+	if (namelist == NULL) {
+		DEBUG(0,("set_namearray: talloc fail\n"));
+		return;
+	}
+	nameptr = namelist;
+
 	/* We need to make two passes over the string. The
 		first to count the number of elements, the second
 		to split it.
@@ -1378,16 +1061,19 @@ void set_namearray(name_compare_entry **ppname_array, const char *namelist)
 		num_entries++;
 	}
 
-	if(num_entries == 0)
+	if(num_entries == 0) {
+		talloc_free(namelist);
 		return;
+	}
 
 	if(( (*ppname_array) = SMB_MALLOC_ARRAY(name_compare_entry, num_entries + 1)) == NULL) {
 		DEBUG(0,("set_namearray: malloc fail\n"));
+		talloc_free(namelist);
 		return;
 	}
 
 	/* Now copy out the names */
-	nameptr = (char *)namelist;
+	nameptr = namelist;
 	i = 0;
 	while(*nameptr) {
 		if ( *nameptr == '/' ) {
@@ -1409,6 +1095,7 @@ void set_namearray(name_compare_entry **ppname_array, const char *namelist)
 		(*ppname_array)[i].is_wild = ms_has_wild(nameptr);
 		if(((*ppname_array)[i].name = SMB_STRDUP(nameptr)) == NULL) {
 			DEBUG(0,("set_namearray: malloc fail (1)\n"));
+			talloc_free(namelist);
 			return;
 		}
 
@@ -1419,25 +1106,10 @@ void set_namearray(name_compare_entry **ppname_array, const char *namelist)
 
 	(*ppname_array)[i].name = NULL;
 
+	talloc_free(namelist);
 	return;
 }
 
-/****************************************************************************
- Routine to free a namearray.
-****************************************************************************/
-
-void free_namearray(name_compare_entry *name_array)
-{
-	int i;
-
-	if(name_array == NULL)
-		return;
-
-	for(i=0; name_array[i].name!=NULL; i++)
-		SAFE_FREE(name_array[i].name);
-	SAFE_FREE(name_array);
-}
-
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_LOCKING
 
@@ -1635,8 +1307,9 @@ const char *tab_depth(int level, int depth)
 
 int str_checksum(const char *s)
 {
-	TDB_DATA key = string_tdb_data(s);
-	return tdb_jenkins_hash(&key);
+	if (s == NULL)
+		return 0;
+	return hash(s, strlen(s), 0);
 }
 
 /*****************************************************************
@@ -1785,6 +1458,22 @@ char *myhostname(void)
 	return ret;
 }
 
+/*****************************************************************
+ Get local hostname and cache result.
+*****************************************************************/
+
+char *myhostname_upper(void)
+{
+	char *name;
+	static char *ret;
+	if (ret == NULL) {
+		name = get_myname(talloc_tos());
+		ret = strupper_talloc(NULL, name);
+		talloc_free(name);
+	}
+	return ret;
+}
+
 /**
  * @brief Returns an absolute path to a file concatenating the provided
  * @a rootpath and @a basename
@@ -1843,45 +1532,6 @@ char *pid_path(const char *name)
 }
 
 /**
- * @brief Returns an absolute path to a file in the Samba lib directory.
- *
- * @param name File to find, relative to LIBDIR.
- *
- * @retval Pointer to a string containing the full path.
- **/
-
-char *lib_path(const char *name)
-{
-	return talloc_asprintf(talloc_tos(), "%s/%s", get_dyn_LIBDIR(), name);
-}
-
-/**
- * @brief Returns an absolute path to a file in the Samba modules directory.
- *
- * @param name File to find, relative to MODULESDIR.
- *
- * @retval Pointer to a string containing the full path.
- **/
-
-char *modules_path(const char *name)
-{
-	return talloc_asprintf(talloc_tos(), "%s/%s", get_dyn_MODULESDIR(), name);
-}
-
-/**
- * @brief Returns an absolute path to a file in the Samba data directory.
- *
- * @param name File to find, relative to CODEPAGEDIR.
- *
- * @retval Pointer to a talloc'ed string containing the full path.
- **/
-
-char *data_path(const char *name)
-{
-	return talloc_asprintf(talloc_tos(), "%s/%s", get_dyn_CODEPAGEDIR(), name);
-}
-
-/**
  * @brief Returns an absolute path to a file in the Samba state directory.
  *
  * @param name File to find, relative to STATEDIR.
@@ -1907,17 +1557,6 @@ char *cache_path(const char *name)
 	return xx_path(name, lp_cachedir());
 }
 
-/**
- * @brief Returns the platform specific shared library extension.
- *
- * @retval Pointer to a const char * containing the extension.
- **/
-
-const char *shlib_ext(void)
-{
-	return get_dyn_SHLIBEXT();
-}
-
 /*******************************************************************
  Given a filename - get its directory name
 ********************************************************************/
@@ -1942,7 +1581,7 @@ bool parent_dirname(TALLOC_CTX *mem_ctx, const char *dir, char **parent,
 
 	len = p-dir;
 
-	if (!(*parent = (char *)TALLOC_MEMDUP(mem_ctx, dir, len+1))) {
+	if (!(*parent = (char *)talloc_memdup(mem_ctx, dir, len+1))) {
 		return False;
 	}
 	(*parent)[len] = '\0';
@@ -2211,7 +1850,7 @@ bool name_to_fqdn(fstring fqdn, const char *name)
 			}
 		}
 	}
-	if (full && (StrCaseCmp(full, "localhost.localdomain") == 0)) {
+	if (full && (strcasecmp_m(full, "localhost.localdomain") == 0)) {
 		DEBUG(1, ("WARNING: your /etc/hosts file may be broken!\n"));
 		DEBUGADD(1, ("    Specifing the machine hostname for address 127.0.0.1 may lead\n"));
 		DEBUGADD(1, ("    to Kerberos authentication problems as localhost.localdomain\n"));
@@ -2303,6 +1942,7 @@ struct server_id pid_to_procid(pid_t pid)
 {
 	struct server_id result;
 	result.pid = pid;
+	result.task_id = 0;
 	result.unique_id = my_unique_id;
 	result.vnn = my_vnn;
 	return result;
@@ -2317,6 +1957,8 @@ bool procid_equal(const struct server_id *p1, const struct server_id *p2)
 {
 	if (p1->pid != p2->pid)
 		return False;
+	if (p1->task_id != p2->task_id)
+		return False;
 	if (p1->vnn != p2->vnn)
 		return False;
 	return True;
@@ -2332,6 +1974,8 @@ bool procid_is_me(const struct server_id *pid)
 {
 	if (pid->pid != sys_getpid())
 		return False;
+	if (pid->task_id != 0)
+		return False;
 	if (pid->vnn != my_vnn)
 		return False;
 	return True;
@@ -2340,52 +1984,45 @@ bool procid_is_me(const struct server_id *pid)
 struct server_id interpret_pid(const char *pid_string)
 {
 	struct server_id result;
-	int pid;
-	unsigned int vnn;
-	if (sscanf(pid_string, "%u:%d", &vnn, &pid) == 2) {
+	unsigned long long pid;
+	unsigned int vnn, task_id = 0;
+
+	ZERO_STRUCT(result);
+
+	/* We accept various forms with 1, 2 or 3 component forms
+	 * because the server_id_str() can print different forms, and
+	 * we want backwards compatibility for scripts that may call
+	 * smbclient. */
+	if (sscanf(pid_string, "%u:%llu.%u", &vnn, &pid, &task_id) == 3) {
 		result.vnn = vnn;
 		result.pid = pid;
-	}
-	else if (sscanf(pid_string, "%d", &pid) == 1) {
+		result.task_id = task_id;
+	} else if (sscanf(pid_string, "%u:%llu", &vnn, &pid) == 2) {
+		result.vnn = vnn;
+		result.pid = pid;
+		result.task_id = 0;
+	} else if (sscanf(pid_string, "%llu.%u", &pid, &task_id) == 2) {
 		result.vnn = get_my_vnn();
 		result.pid = pid;
-	}
-	else {
+		result.task_id = task_id;
+	} else if (sscanf(pid_string, "%llu", &pid) == 1) {
+		result.vnn = get_my_vnn();
+		result.pid = pid;
+	} else {
 		result.vnn = NONCLUSTER_VNN;
-		result.pid = -1;
+		result.pid = (uint64_t)-1;
 	}
-	/* Assigning to result.pid may have overflowed
-	   Map negative pid to -1: i.e. error */
-	if (result.pid < 0) {
-		result.pid = -1;
-	}
-	result.unique_id = 0;
 	return result;
 }
 
-char *procid_str(TALLOC_CTX *mem_ctx, const struct server_id *pid)
-{
-	if (pid->vnn == NONCLUSTER_VNN) {
-		return talloc_asprintf(mem_ctx,
-				"%d",
-				(int)pid->pid);
-	}
-	else {
-		return talloc_asprintf(mem_ctx,
-					"%u:%d",
-					(unsigned)pid->vnn,
-					(int)pid->pid);
-	}
-}
-
 char *procid_str_static(const struct server_id *pid)
 {
-	return procid_str(talloc_tos(), pid);
+	return server_id_str(talloc_tos(), pid);
 }
 
 bool procid_valid(const struct server_id *pid)
 {
-	return (pid->pid != -1);
+	return (pid->pid != (uint64_t)-1);
 }
 
 bool procid_is_local(const struct server_id *pid)
@@ -2519,111 +2156,6 @@ void split_domain_user(TALLOC_CTX *mem_ctx,
 	}
 }
 
-#if 0
-
-Disable these now we have checked all code paths and ensured
-NULL returns on zero request. JRA.
-
-/****************************************************************
- talloc wrapper functions that guarentee a null pointer return
- if size == 0.
-****************************************************************/
-
-#ifndef MAX_TALLOC_SIZE
-#define MAX_TALLOC_SIZE 0x10000000
-#endif
-
-/*
- *    talloc and zero memory.
- *    - returns NULL if size is zero.
- */
-
-void *_talloc_zero_zeronull(const void *ctx, size_t size, const char *name)
-{
-	void *p;
-
-	if (size == 0) {
-		return NULL;
-	}
-
-	p = talloc_named_const(ctx, size, name);
-
-	if (p) {
-		memset(p, '\0', size);
-	}
-
-	return p;
-}
-
-/*
- *   memdup with a talloc.
- *   - returns NULL if size is zero.
- */
-
-void *_talloc_memdup_zeronull(const void *t, const void *p, size_t size, const char *name)
-{
-	void *newp;
-
-	if (size == 0) {
-		return NULL;
-	}
-
-	newp = talloc_named_const(t, size, name);
-	if (newp) {
-		memcpy(newp, p, size);
-	}
-
-	return newp;
-}
-
-/*
- *   alloc an array, checking for integer overflow in the array size.
- *   - returns NULL if count or el_size are zero.
- */
-
-void *_talloc_array_zeronull(const void *ctx, size_t el_size, unsigned count, const char *name)
-{
-	if (count >= MAX_TALLOC_SIZE/el_size) {
-		return NULL;
-	}
-
-	if (el_size == 0 || count == 0) {
-		return NULL;
-	}
-
-	return talloc_named_const(ctx, el_size * count, name);
-}
-
-/*
- *   alloc an zero array, checking for integer overflow in the array size
- *   - returns NULL if count or el_size are zero.
- */
-
-void *_talloc_zero_array_zeronull(const void *ctx, size_t el_size, unsigned count, const char *name)
-{
-	if (count >= MAX_TALLOC_SIZE/el_size) {
-		return NULL;
-	}
-
-	if (el_size == 0 || count == 0) {
-		return NULL;
-	}
-
-	return _talloc_zero(ctx, el_size * count, name);
-}
-
-/*
- *   Talloc wrapper that returns NULL if size == 0.
- */
-void *talloc_zeronull(const void *context, size_t size, const char *name)
-{
-	if (size == 0) {
-		return NULL;
-	}
-	return talloc_named_const(context, size, name);
-}
-#endif
-
 /****************************************************************
  strip off leading '\\' from a hostname
 ****************************************************************/
@@ -2672,3 +2204,37 @@ int timeval_to_msec(struct timeval t)
 {
 	return t.tv_sec * 1000 + (t.tv_usec+999) / 1000;
 }
+
+/*******************************************************************
+ Check a given DOS pathname is valid for a share.
+********************************************************************/
+
+char *valid_share_pathname(TALLOC_CTX *ctx, const char *dos_pathname)
+{
+	char *ptr = NULL;
+
+	if (!dos_pathname) {
+		return NULL;
+	}
+
+	ptr = talloc_strdup(ctx, dos_pathname);
+	if (!ptr) {
+		return NULL;
+	}
+	/* Convert any '\' paths to '/' */
+	unix_format(ptr);
+	ptr = unix_clean_name(ctx, ptr);
+	if (!ptr) {
+		return NULL;
+	}
+
+	/* NT is braindead - it wants a C: prefix to a pathname ! So strip it. */
+	if (strlen(ptr) > 2 && ptr[1] == ':' && ptr[0] != '/')
+		ptr += 2;
+
+	/* Only absolute paths allowed. */
+	if (*ptr != '/')
+		return NULL;
+
+	return ptr;
+}
diff --git a/source3/lib/util_builtin.c b/source3/lib/util_builtin.c
index e1b990619b4..b370a76c691 100644
--- a/source3/lib/util_builtin.c
+++ b/source3/lib/util_builtin.c
@@ -18,7 +18,6 @@
 */
 
 #include "includes.h"
-#include "passdb.h"
 #include "../libcli/security/security.h"
 
 struct rid_name_map {
diff --git a/source3/lib/util_cmdline.c b/source3/lib/util_cmdline.c
index cb0b79a5d30..39f136821ce 100644
--- a/source3/lib/util_cmdline.c
+++ b/source3/lib/util_cmdline.c
@@ -34,7 +34,7 @@ struct user_auth_info *user_auth_info_init(TALLOC_CTX *mem_ctx)
 {
 	struct user_auth_info *result;
 
-	result = TALLOC_ZERO_P(mem_ctx, struct user_auth_info);
+	result = talloc_zero(mem_ctx, struct user_auth_info);
 	if (result == NULL) {
 		return NULL;
 	}
@@ -228,7 +228,7 @@ bool set_cmdline_auth_info_machine_account_creds(struct user_auth_info *auth_inf
 		return false;
 	}
 
-	if (asprintf(&account, "%s$@%s", global_myname(), lp_realm()) < 0) {
+	if (asprintf(&account, "%s$@%s", lp_netbios_name(), lp_realm()) < 0) {
 		return false;
 	}
 
diff --git a/source3/lib/util_malloc.c b/source3/lib/util_malloc.c
new file mode 100644
index 00000000000..c052adc41bb
--- /dev/null
+++ b/source3/lib/util_malloc.c
@@ -0,0 +1,171 @@
+/* 
+   Unix SMB/CIFS implementation.
+   Samba utility functions
+   Copyright (C) Andrew Tridgell 1992-1998
+   Copyright (C) Jeremy Allison 2001-2007
+   Copyright (C) Simo Sorce 2001
+   Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2003
+   Copyright (C) James Peach 2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+
+/* Max allowable allococation - 256mb - 0x10000000 */
+#define MAX_ALLOC_SIZE (1024*1024*256)
+
+#if defined(PARANOID_MALLOC_CHECKER)
+
+/****************************************************************************
+ Internal malloc wrapper. Externally visible.
+****************************************************************************/
+
+void *malloc_(size_t size)
+{
+	if (size == 0) {
+		return NULL;
+	}
+#undef malloc
+	return malloc(size);
+#define malloc(s) __ERROR_DONT_USE_MALLOC_DIRECTLY
+}
+
+/****************************************************************************
+ Internal calloc wrapper. Not externally visible.
+****************************************************************************/
+
+static void *calloc_(size_t count, size_t size)
+{
+	if (size == 0 || count == 0) {
+		return NULL;
+	}
+#undef calloc
+	return calloc(count, size);
+#define calloc(n,s) __ERROR_DONT_USE_CALLOC_DIRECTLY
+}
+
+/****************************************************************************
+ Internal realloc wrapper. Not externally visible.
+****************************************************************************/
+
+static void *realloc_(void *ptr, size_t size)
+{
+#undef realloc
+	return realloc(ptr, size);
+#define realloc(p,s) __ERROR_DONT_USE_RELLOC_DIRECTLY
+}
+
+#endif /* PARANOID_MALLOC_CHECKER */
+
+/****************************************************************************
+ Type-safe memalign
+****************************************************************************/
+
+void *memalign_array(size_t el_size, size_t align, unsigned int count)
+{
+	if (count >= MAX_ALLOC_SIZE/el_size) {
+		return NULL;
+	}
+
+	return sys_memalign(align, el_size*count);
+}
+
+/****************************************************************************
+ Type-safe calloc.
+****************************************************************************/
+
+void *calloc_array(size_t size, size_t nmemb)
+{
+	if (nmemb >= MAX_ALLOC_SIZE/size) {
+		return NULL;
+	}
+	if (size == 0 || nmemb == 0) {
+		return NULL;
+	}
+#if defined(PARANOID_MALLOC_CHECKER)
+	return calloc_(nmemb, size);
+#else
+	return calloc(nmemb, size);
+#endif
+}
+
+/****************************************************************************
+ Expand a pointer to be a particular size.
+ Note that this version of Realloc has an extra parameter that decides
+ whether to free the passed in storage on allocation failure or if the
+ new size is zero.
+
+ This is designed for use in the typical idiom of :
+
+ p = SMB_REALLOC(p, size)
+ if (!p) {
+    return error;
+ }
+
+ and not to have to keep track of the old 'p' contents to free later, nor
+ to worry if the size parameter was zero. In the case where NULL is returned
+ we guarentee that p has been freed.
+
+ If free later semantics are desired, then pass 'free_old_on_error' as False which
+ guarentees that the old contents are not freed on error, even if size == 0. To use
+ this idiom use :
+
+ tmp = SMB_REALLOC_KEEP_OLD_ON_ERROR(p, size);
+ if (!tmp) {
+    SAFE_FREE(p);
+    return error;
+ } else {
+    p = tmp;
+ }
+
+ Changes were instigated by Coverity error checking. JRA.
+****************************************************************************/
+
+void *Realloc(void *p, size_t size, bool free_old_on_error)
+{
+	void *ret=NULL;
+
+	if (size == 0) {
+		if (free_old_on_error) {
+			SAFE_FREE(p);
+		}
+		DEBUG(2,("Realloc asked for 0 bytes\n"));
+		return NULL;
+	}
+
+#if defined(PARANOID_MALLOC_CHECKER)
+	if (!p) {
+		ret = (void *)malloc_(size);
+	} else {
+		ret = (void *)realloc_(p,size);
+	}
+#else
+	if (!p) {
+		ret = (void *)malloc(size);
+	} else {
+		ret = (void *)realloc(p,size);
+	}
+#endif
+
+	if (!ret) {
+		if (free_old_on_error && p) {
+			SAFE_FREE(p);
+		}
+		DEBUG(0,("Memory allocation error: failed to expand to %d bytes\n",(int)size));
+	}
+
+	return(ret);
+}
+
diff --git a/source3/lib/util_names.c b/source3/lib/util_names.c
index bd6e5c12026..0e128eab1cd 100644
--- a/source3/lib/util_names.c
+++ b/source3/lib/util_names.c
@@ -6,7 +6,7 @@
    Copyright (C) Simo Sorce 2001
    Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2003
    Copyright (C) James Peach 2006
-   Copyright (C) Andrew Bartlett 2010
+   Copyright (C) Andrew Bartlett 2010-2011
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -24,45 +24,131 @@
 
 #include "includes.h"
 
-static char *smb_myname;
-static char *smb_myworkgroup;
-
 /***********************************************************************
- Allocate and set myname. Ensure upper case.
+ Definitions for all names.
 ***********************************************************************/
 
-bool set_global_myname(const char *myname)
+static int smb_num_netbios_names;
+static char **smb_my_netbios_names;
+
+static void free_netbios_names_array(void)
+{
+	int i;
+
+	for (i = 0; i < smb_num_netbios_names; i++)
+		SAFE_FREE(smb_my_netbios_names[i]);
+
+	SAFE_FREE(smb_my_netbios_names);
+	smb_num_netbios_names = 0;
+}
+
+static bool allocate_my_netbios_names_array(size_t number)
 {
-	SAFE_FREE(smb_myname);
-	smb_myname = SMB_STRDUP(myname);
-	if (!smb_myname)
+	free_netbios_names_array();
+
+	smb_num_netbios_names = number + 1;
+	smb_my_netbios_names = SMB_MALLOC_ARRAY( char *, smb_num_netbios_names );
+
+	if (!smb_my_netbios_names)
 		return False;
-	strupper_m(smb_myname);
+
+	memset(smb_my_netbios_names, '\0', sizeof(char *) * smb_num_netbios_names);
 	return True;
 }
 
-const char *global_myname(void)
+static bool set_my_netbios_names(const char *name, int i)
 {
-	return smb_myname;
+	SAFE_FREE(smb_my_netbios_names[i]);
+
+	smb_my_netbios_names[i] = SMB_STRDUP(name);
+	if (!smb_my_netbios_names[i])
+		return False;
+	strupper_m(smb_my_netbios_names[i]);
+	return True;
 }
 
 /***********************************************************************
- Allocate and set myworkgroup. Ensure upper case.
+ Free memory allocated to global objects
 ***********************************************************************/
 
-bool set_global_myworkgroup(const char *myworkgroup)
+void gfree_names(void)
+{
+	free_netbios_names_array();
+	free_local_machine_name();
+}
+
+const char *my_netbios_names(int i)
+{
+	return smb_my_netbios_names[i];
+}
+
+bool set_netbios_aliases(const char **str_array)
 {
-	SAFE_FREE(smb_myworkgroup);
-	smb_myworkgroup = SMB_STRDUP(myworkgroup);
-	if (!smb_myworkgroup)
+	size_t namecount;
+
+	/* Work out the max number of netbios aliases that we have */
+	for( namecount=0; str_array && (str_array[namecount] != NULL); namecount++ )
+		;
+
+	if ( lp_netbios_name() && *lp_netbios_name())
+		namecount++;
+
+	/* Allocate space for the netbios aliases */
+	if (!allocate_my_netbios_names_array(namecount))
 		return False;
-	strupper_m(smb_myworkgroup);
+
+	/* Use the global_myname string first */
+	namecount=0;
+	if ( lp_netbios_name() && *lp_netbios_name()) {
+		set_my_netbios_names( lp_netbios_name(), namecount );
+		namecount++;
+	}
+
+	if (str_array) {
+		size_t i;
+		for ( i = 0; str_array[i] != NULL; i++) {
+			size_t n;
+			bool duplicate = False;
+
+			/* Look for duplicates */
+			for( n=0; n<namecount; n++ ) {
+				if( strequal( str_array[i], my_netbios_names(n) ) ) {
+					duplicate = True;
+					break;
+				}
+			}
+			if (!duplicate) {
+				if (!set_my_netbios_names(str_array[i], namecount))
+					return False;
+				namecount++;
+			}
+		}
+	}
 	return True;
 }
 
-const char *lp_workgroup(void)
+/****************************************************************************
+  Common name initialization code.
+****************************************************************************/
+
+bool init_names(void)
 {
-	return smb_myworkgroup;
+	int n;
+
+	if (!set_netbios_aliases(lp_netbios_aliases())) {
+		DEBUG( 0, ( "init_names: malloc fail.\n" ) );
+		return False;
+	}
+
+	set_local_machine_name(lp_netbios_name(),false);
+
+	DEBUG( 5, ("Netbios name list:-\n") );
+	for( n=0; my_netbios_names(n); n++ ) {
+		DEBUGADD( 5, ("my_netbios_names[%d]=\"%s\"\n",
+					n, my_netbios_names(n) ) );
+	}
+
+	return( True );
 }
 
 /******************************************************************
@@ -75,11 +161,6 @@ const char *get_global_sam_name(void)
 	if (IS_DC) {
 		return lp_workgroup();
 	}
-	return global_myname();
+	return lp_netbios_name();
 }
 
-void gfree_netbios_names(void)
-{
-	SAFE_FREE( smb_myname );
-	SAFE_FREE( smb_myworkgroup );
-}
diff --git a/source3/lib/util_nttoken.c b/source3/lib/util_nttoken.c
index 2fd0f088ab7..ffa858d7794 100644
--- a/source3/lib/util_nttoken.c
+++ b/source3/lib/util_nttoken.c
@@ -39,7 +39,7 @@ struct security_token *dup_nt_token(TALLOC_CTX *mem_ctx, const struct security_t
 	if (!ptoken)
 		return NULL;
 
-	token = TALLOC_ZERO_P(mem_ctx, struct security_token);
+	token = talloc_zero(mem_ctx, struct security_token);
 	if (token == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
@@ -80,7 +80,7 @@ NTSTATUS merge_nt_token(TALLOC_CTX *mem_ctx,
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	token = TALLOC_ZERO_P(mem_ctx, struct security_token);
+	token = talloc_zero(mem_ctx, struct security_token);
 	NT_STATUS_HAVE_NO_MEMORY(token);
 
 	for (i=0; i < token_1->num_sids; i++) {
diff --git a/source3/lib/util_sid.c b/source3/lib/util_sid.c
index ef46a38f8aa..f080d3dfb0c 100644
--- a/source3/lib/util_sid.c
+++ b/source3/lib/util_sid.c
@@ -121,7 +121,7 @@ char *sid_binstring_hex(const struct dom_sid *sid)
 	if (!buf)
 		return NULL;
 	sid_linearize(buf, len, sid);
-	hex_encode(buf, len, &s);
+	hex_encode((const unsigned char *)buf, len, &s);
 	free(buf);
 	return s;
 }
diff --git a/source3/lib/util_sock.c b/source3/lib/util_sock.c
index 71f6a8f29c9..9b8632b181a 100644
--- a/source3/lib/util_sock.c
+++ b/source3/lib/util_sock.c
@@ -24,158 +24,9 @@
 #include "memcache.h"
 #include "../lib/async_req/async_sock.h"
 #include "../lib/util/select.h"
-#include "interfaces.h"
-
-/****************************************************************************
- Get a port number in host byte order from a sockaddr_storage.
-****************************************************************************/
-
-uint16_t get_sockaddr_port(const struct sockaddr_storage *pss)
-{
-	uint16_t port = 0;
-
-	if (pss->ss_family != AF_INET) {
-#if defined(HAVE_IPV6)
-		/* IPv6 */
-		const struct sockaddr_in6 *sa6 =
-			(const struct sockaddr_in6 *)pss;
-		port = ntohs(sa6->sin6_port);
-#endif
-	} else {
-		const struct sockaddr_in *sa =
-			(const struct sockaddr_in *)pss;
-		port = ntohs(sa->sin_port);
-	}
-	return port;
-}
-
-/****************************************************************************
- Print out an IPv4 or IPv6 address from a struct sockaddr_storage.
-****************************************************************************/
-
-static char *print_sockaddr_len(char *dest,
-			size_t destlen,
-			const struct sockaddr *psa,
-			socklen_t psalen)
-{
-	if (destlen > 0) {
-		dest[0] = '\0';
-	}
-	(void)sys_getnameinfo(psa,
-			psalen,
-			dest, destlen,
-			NULL, 0,
-			NI_NUMERICHOST);
-	return dest;
-}
-
-/****************************************************************************
- Print out an IPv4 or IPv6 address from a struct sockaddr_storage.
-****************************************************************************/
-
-char *print_sockaddr(char *dest,
-			size_t destlen,
-			const struct sockaddr_storage *psa)
-{
-	return print_sockaddr_len(dest, destlen, (struct sockaddr *)psa,
-			sizeof(struct sockaddr_storage));
-}
-
-/****************************************************************************
- Print out a canonical IPv4 or IPv6 address from a struct sockaddr_storage.
-****************************************************************************/
-
-char *print_canonical_sockaddr(TALLOC_CTX *ctx,
-			const struct sockaddr_storage *pss)
-{
-	char addr[INET6_ADDRSTRLEN];
-	char *dest = NULL;
-	int ret;
-
-	/* Linux getnameinfo() man pages says port is unitialized if
-	   service name is NULL. */
-
-	ret = sys_getnameinfo((const struct sockaddr *)pss,
-			sizeof(struct sockaddr_storage),
-			addr, sizeof(addr),
-			NULL, 0,
-			NI_NUMERICHOST);
-	if (ret != 0) {
-		return NULL;
-	}
-
-	if (pss->ss_family != AF_INET) {
-#if defined(HAVE_IPV6)
-		dest = talloc_asprintf(ctx, "[%s]", addr);
-#else
-		return NULL;
-#endif
-	} else {
-		dest = talloc_asprintf(ctx, "%s", addr);
-	}
-
-	return dest;
-}
-
-/****************************************************************************
- Return the string of an IP address (IPv4 or IPv6).
-****************************************************************************/
-
-static const char *get_socket_addr(int fd, char *addr_buf, size_t addr_len)
-{
-	struct sockaddr_storage sa;
-	socklen_t length = sizeof(sa);
-
-	/* Ok, returning a hard coded IPv4 address
- 	 * is bogus, but it's just as bogus as a
- 	 * zero IPv6 address. No good choice here.
- 	 */
-
-	strlcpy(addr_buf, "0.0.0.0", addr_len);
-
-	if (fd == -1) {
-		return addr_buf;
-	}
-
-	if (getsockname(fd, (struct sockaddr *)&sa, &length) < 0) {
-		DEBUG(0,("getsockname failed. Error was %s\n",
-			strerror(errno) ));
-		return addr_buf;
-	}
-
-	return print_sockaddr_len(addr_buf, addr_len, (struct sockaddr *)&sa, length);
-}
-
-/****************************************************************************
- Return the port number we've bound to on a socket.
-****************************************************************************/
-
-int get_socket_port(int fd)
-{
-	struct sockaddr_storage sa;
-	socklen_t length = sizeof(sa);
-
-	if (fd == -1) {
-		return -1;
-	}
-
-	if (getsockname(fd, (struct sockaddr *)&sa, &length) < 0) {
-		int level = (errno == ENOTCONN) ? 2 : 0;
-		DEBUG(level, ("getsockname failed. Error was %s\n",
-			       strerror(errno)));
-		return -1;
-	}
-
-#if defined(HAVE_IPV6)
-	if (sa.ss_family == AF_INET6) {
-		return ntohs(((struct sockaddr_in6 *)&sa)->sin6_port);
-	}
-#endif
-	if (sa.ss_family == AF_INET) {
-		return ntohs(((struct sockaddr_in *)&sa)->sin_port);
-	}
-	return -1;
-}
+#include "lib/socket/interfaces.h"
+#include "../lib/util/tevent_unix.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 const char *client_name(int fd)
 {
@@ -187,11 +38,6 @@ const char *client_addr(int fd, char *addr, size_t addrlen)
 	return get_peer_addr(fd,addr,addrlen);
 }
 
-const char *client_socket_addr(int fd, char *addr, size_t addr_len)
-{
-	return get_socket_addr(fd, addr, addr_len);
-}
-
 #if 0
 /* Not currently used. JRA. */
 int client_socket_port(int fd)
@@ -232,166 +78,6 @@ bool is_a_socket(int fd)
 	return(getsockopt(fd, SOL_SOCKET, SO_TYPE, (char *)&v, &l) == 0);
 }
 
-enum SOCK_OPT_TYPES {OPT_BOOL,OPT_INT,OPT_ON};
-
-typedef struct smb_socket_option {
-	const char *name;
-	int level;
-	int option;
-	int value;
-	int opttype;
-} smb_socket_option;
-
-static const smb_socket_option socket_options[] = {
-  {"SO_KEEPALIVE", SOL_SOCKET, SO_KEEPALIVE, 0, OPT_BOOL},
-  {"SO_REUSEADDR", SOL_SOCKET, SO_REUSEADDR, 0, OPT_BOOL},
-  {"SO_BROADCAST", SOL_SOCKET, SO_BROADCAST, 0, OPT_BOOL},
-#ifdef TCP_NODELAY
-  {"TCP_NODELAY", IPPROTO_TCP, TCP_NODELAY, 0, OPT_BOOL},
-#endif
-#ifdef TCP_KEEPCNT
-  {"TCP_KEEPCNT", IPPROTO_TCP, TCP_KEEPCNT, 0, OPT_INT},
-#endif
-#ifdef TCP_KEEPIDLE
-  {"TCP_KEEPIDLE", IPPROTO_TCP, TCP_KEEPIDLE, 0, OPT_INT},
-#endif
-#ifdef TCP_KEEPINTVL
-  {"TCP_KEEPINTVL", IPPROTO_TCP, TCP_KEEPINTVL, 0, OPT_INT},
-#endif
-#ifdef IPTOS_LOWDELAY
-  {"IPTOS_LOWDELAY", IPPROTO_IP, IP_TOS, IPTOS_LOWDELAY, OPT_ON},
-#endif
-#ifdef IPTOS_THROUGHPUT
-  {"IPTOS_THROUGHPUT", IPPROTO_IP, IP_TOS, IPTOS_THROUGHPUT, OPT_ON},
-#endif
-#ifdef SO_REUSEPORT
-  {"SO_REUSEPORT", SOL_SOCKET, SO_REUSEPORT, 0, OPT_BOOL},
-#endif
-#ifdef SO_SNDBUF
-  {"SO_SNDBUF", SOL_SOCKET, SO_SNDBUF, 0, OPT_INT},
-#endif
-#ifdef SO_RCVBUF
-  {"SO_RCVBUF", SOL_SOCKET, SO_RCVBUF, 0, OPT_INT},
-#endif
-#ifdef SO_SNDLOWAT
-  {"SO_SNDLOWAT", SOL_SOCKET, SO_SNDLOWAT, 0, OPT_INT},
-#endif
-#ifdef SO_RCVLOWAT
-  {"SO_RCVLOWAT", SOL_SOCKET, SO_RCVLOWAT, 0, OPT_INT},
-#endif
-#ifdef SO_SNDTIMEO
-  {"SO_SNDTIMEO", SOL_SOCKET, SO_SNDTIMEO, 0, OPT_INT},
-#endif
-#ifdef SO_RCVTIMEO
-  {"SO_RCVTIMEO", SOL_SOCKET, SO_RCVTIMEO, 0, OPT_INT},
-#endif
-#ifdef TCP_FASTACK
-  {"TCP_FASTACK", IPPROTO_TCP, TCP_FASTACK, 0, OPT_INT},
-#endif
-#ifdef TCP_QUICKACK
-  {"TCP_QUICKACK", IPPROTO_TCP, TCP_QUICKACK, 0, OPT_BOOL},
-#endif
-#ifdef TCP_KEEPALIVE_THRESHOLD
-  {"TCP_KEEPALIVE_THRESHOLD", IPPROTO_TCP, TCP_KEEPALIVE_THRESHOLD, 0, OPT_INT},
-#endif
-#ifdef TCP_KEEPALIVE_ABORT_THRESHOLD
-  {"TCP_KEEPALIVE_ABORT_THRESHOLD", IPPROTO_TCP, TCP_KEEPALIVE_ABORT_THRESHOLD, 0, OPT_INT},
-#endif
-  {NULL,0,0,0,0}};
-
-/****************************************************************************
- Print socket options.
-****************************************************************************/
-
-static void print_socket_options(int s)
-{
-	int value;
-	socklen_t vlen = 4;
-	const smb_socket_option *p = &socket_options[0];
-
-	/* wrapped in if statement to prevent streams
-	 * leak in SCO Openserver 5.0 */
-	/* reported on samba-technical  --jerry */
-	if ( DEBUGLEVEL >= 5 ) {
-		DEBUG(5,("Socket options:\n"));
-		for (; p->name != NULL; p++) {
-			if (getsockopt(s, p->level, p->option,
-						(void *)&value, &vlen) == -1) {
-				DEBUGADD(5,("\tCould not test socket option %s.\n",
-							p->name));
-			} else {
-				DEBUGADD(5,("\t%s = %d\n",
-							p->name,value));
-			}
-		}
-	}
- }
-
-/****************************************************************************
- Set user socket options.
-****************************************************************************/
-
-void set_socket_options(int fd, const char *options)
-{
-	TALLOC_CTX *ctx = talloc_stackframe();
-	char *tok;
-
-	while (next_token_talloc(ctx, &options, &tok," \t,")) {
-		int ret=0,i;
-		int value = 1;
-		char *p;
-		bool got_value = false;
-
-		if ((p = strchr_m(tok,'='))) {
-			*p = 0;
-			value = atoi(p+1);
-			got_value = true;
-		}
-
-		for (i=0;socket_options[i].name;i++)
-			if (strequal(socket_options[i].name,tok))
-				break;
-
-		if (!socket_options[i].name) {
-			DEBUG(0,("Unknown socket option %s\n",tok));
-			continue;
-		}
-
-		switch (socket_options[i].opttype) {
-		case OPT_BOOL:
-		case OPT_INT:
-			ret = setsockopt(fd,socket_options[i].level,
-					socket_options[i].option,
-					(char *)&value,sizeof(int));
-			break;
-
-		case OPT_ON:
-			if (got_value)
-				DEBUG(0,("syntax error - %s "
-					"does not take a value\n",tok));
-
-			{
-				int on = socket_options[i].value;
-				ret = setsockopt(fd,socket_options[i].level,
-					socket_options[i].option,
-					(char *)&on,sizeof(int));
-			}
-			break;
-		}
-
-		if (ret != 0) {
-			/* be aware that some systems like Solaris return
-			 * EINVAL to a setsockopt() call when the client
-			 * sent a RST previously - no need to worry */
-			DEBUG(2,("Failed to set socket option %s (Error %s)\n",
-				tok, strerror(errno) ));
-		}
-	}
-
-	TALLOC_FREE(ctx);
-	print_socket_options(fd);
-}
-
 /****************************************************************************
  Read from a socket.
 ****************************************************************************/
@@ -571,7 +257,7 @@ ssize_t write_data_iov(int fd, const struct iovec *orig_iov, int iovcnt)
 	 * discarding elements.
 	 */
 
-	iov_copy = (struct iovec *)TALLOC_MEMDUP(
+	iov_copy = (struct iovec *)talloc_memdup(
 		talloc_tos(), orig_iov, sizeof(struct iovec) * iovcnt);
 
 	if (iov_copy == NULL) {
@@ -620,7 +306,7 @@ ssize_t write_data(int fd, const char *buffer, size_t N)
 {
 	struct iovec iov;
 
-	iov.iov_base = CONST_DISCARD(void *, buffer);
+	iov.iov_base = discard_const_p(void, buffer);
 	iov.iov_len = N;
 	return write_data_iov(fd, &iov, 1);
 }
@@ -787,6 +473,32 @@ int open_socket_in(int type,
 #endif /* SO_REUSEPORT */
 	}
 
+#ifdef HAVE_IPV6
+	/*
+	 * As IPV6_V6ONLY is the default on some systems,
+	 * we better try to be consistent and always use it.
+	 *
+	 * This also avoids using IPv4 via AF_INET6 sockets
+	 * and makes sure %I never resolves to a '::ffff:192.168.0.1'
+	 * string.
+	 */
+	if (sock.ss_family == AF_INET6) {
+		int val = 1;
+		int ret;
+
+		ret = setsockopt(res, IPPROTO_IPV6, IPV6_V6ONLY,
+				 (const void *)&val, sizeof(val));
+		if (ret == -1) {
+			if(DEBUGLVL(0)) {
+				dbgtext("open_socket_in(): IPV6_ONLY failed: ");
+				dbgtext("%s\n", strerror(errno));
+			}
+			close(res);
+			return -1;
+		}
+	}
+#endif
+
 	/* now we've got a socket - we need to bind it */
 	if (bind(res, (struct sockaddr *)&sock, slen) == -1 ) {
 		if( DEBUGLVL(dlevel) && (port == SMB_PORT1 ||
@@ -812,7 +524,7 @@ struct open_socket_out_state {
 	struct sockaddr_storage ss;
 	socklen_t salen;
 	uint16_t port;
-	int wait_nsec;
+	int wait_usec;
 };
 
 static void open_socket_out_connected(struct tevent_req *subreq);
@@ -848,7 +560,7 @@ struct tevent_req *open_socket_out_send(TALLOC_CTX *mem_ctx,
 	state->ev = ev;
 	state->ss = *pss;
 	state->port = port;
-	state->wait_nsec = 10000;
+	state->wait_usec = 10000;
 	state->salen = -1;
 
 	state->fd = socket(state->ss.ss_family, SOCK_STREAM, 0);
@@ -859,7 +571,7 @@ struct tevent_req *open_socket_out_send(TALLOC_CTX *mem_ctx,
 	talloc_set_destructor(state, open_socket_out_state_destructor);
 
 	if (!tevent_req_set_endtime(
-		    result, ev, timeval_current_ofs(0, timeout*1000))) {
+		    result, ev, timeval_current_ofs_msec(timeout))) {
 		goto fail;
 	}
 
@@ -896,7 +608,7 @@ struct tevent_req *open_socket_out_send(TALLOC_CTX *mem_ctx,
 	if ((subreq == NULL)
 	    || !tevent_req_set_endtime(
 		    subreq, state->ev,
-		    timeval_current_ofs(0, state->wait_nsec))) {
+		    timeval_current_ofs(0, state->wait_usec))) {
 		goto fail;
 	}
 	tevent_req_set_callback(subreq, open_socket_out_connected, result);
@@ -938,8 +650,8 @@ static void open_socket_out_connected(struct tevent_req *subreq)
 		 * retry
 		 */
 
-		if (state->wait_nsec < 250000) {
-			state->wait_nsec *= 1.5;
+		if (state->wait_usec < 250000) {
+			state->wait_usec *= 1.5;
 		}
 
 		subreq = async_connect_send(state, state->ev, state->fd,
@@ -950,7 +662,7 @@ static void open_socket_out_connected(struct tevent_req *subreq)
 		}
 		if (!tevent_req_set_endtime(
 			    subreq, state->ev,
-			    timeval_current_ofs(0, state->wait_nsec))) {
+			    timeval_current_ofs_usec(state->wait_usec))) {
 			tevent_req_nterror(req, NT_STATUS_NO_MEMORY);
 			return;
 		}
@@ -1248,7 +960,7 @@ static bool matchname(const char *remotehost,
 			continue;
 		}
 		if (sockaddr_equal((const struct sockaddr *)res->ai_addr,
-					(struct sockaddr *)pss)) {
+					(const struct sockaddr *)pss)) {
 			freeaddrinfo(ailist);
 			return true;
 		}
@@ -1454,14 +1166,21 @@ int create_pipe_sock(const char *socket_dir,
 	} else {
 		/* Check ownership and permission on existing directory */
 		if (!S_ISDIR(st.st_mode)) {
-			DEBUG(0, ("socket directory %s isn't a directory\n",
+			DEBUG(0, ("socket directory '%s' isn't a directory\n",
 				socket_dir));
 			goto out_umask;
 		}
-		if ((st.st_uid != sec_initial_uid()) ||
-				((st.st_mode & 0777) != dir_perms)) {
-			DEBUG(0, ("invalid permissions on socket directory "
-				"%s\n", socket_dir));
+		if (st.st_uid != sec_initial_uid()) {
+			DEBUG(0, ("invalid ownership on directory "
+				  "'%s'\n", socket_dir));
+			umask(old_umask);
+			goto out_umask;
+		}
+		if ((st.st_mode & 0777) != dir_perms) {
+			DEBUG(0, ("invalid permissions on directory "
+				  "'%s': has 0%o should be 0%o\n", socket_dir,
+				  (st.st_mode & 0777), dir_perms));
+			umask(old_umask);
 			goto out_umask;
 		}
 	}
@@ -1602,13 +1321,13 @@ static bool is_my_ipaddr(const char *ipaddr_str)
 		return false;
 	}
 
-	if (ismyaddr((struct sockaddr *)&ss)) {
-		return true;
+	if (is_zero_addr(&ss)) {
+		return false;
 	}
 
-	if (is_zero_addr(&ss) ||
-		is_loopback_addr((struct sockaddr *)&ss)) {
-		return false;
+	if (ismyaddr((struct sockaddr *)&ss) ||
+			is_loopback_addr((struct sockaddr *)&ss)) {
+		return true;
 	}
 
 	n = get_interfaces(talloc_tos(), &nics);
@@ -1651,7 +1370,7 @@ bool is_myname_or_ipaddr(const char *s)
 	}
 
 	/* Optimize for the common case */
-	if (strequal(servername, global_myname())) {
+	if (strequal(servername, lp_netbios_name())) {
 		return true;
 	}
 
@@ -1802,7 +1521,7 @@ int poll_one_fd(int fd, int events, int timeout, int *revents)
 	int ret;
 	int saved_errno;
 
-	fds = TALLOC_ZERO_ARRAY(talloc_tos(), struct pollfd, 2);
+	fds = talloc_zero_array(talloc_tos(), struct pollfd, 2);
 	if (fds == NULL) {
 		errno = ENOMEM;
 		return -1;
diff --git a/source3/lib/util_str.c b/source3/lib/util_str.c
index 88a3d703f4d..a348b389e86 100644
--- a/source3/lib/util_str.c
+++ b/source3/lib/util_str.c
@@ -36,164 +36,6 @@ const char toupper_ascii_fast_table[128] = {
 };
 
 /**
- * Case insensitive string compararison.
- *
- * iconv does not directly give us a way to compare strings in
- * arbitrary unix character sets -- all we can is convert and then
- * compare.  This is expensive.
- *
- * As an optimization, we do a first pass that considers only the
- * prefix of the strings that is entirely 7-bit.  Within this, we
- * check whether they have the same value.
- *
- * Hopefully this will often give the answer without needing to copy.
- * In particular it should speed comparisons to literal ascii strings
- * or comparisons of strings that are "obviously" different.
- *
- * If we find a non-ascii character we fall back to converting via
- * iconv.
- *
- * This should never be slower than convering the whole thing, and
- * often faster.
- *
- * A different optimization would be to compare for bitwise equality
- * in the binary encoding.  (It would be possible thought hairy to do
- * both simultaneously.)  But in that case if they turn out to be
- * different, we'd need to restart the whole thing.
- *
- * Even better is to implement strcasecmp for each encoding and use a
- * function pointer.
- **/
-int StrCaseCmp(const char *s, const char *t)
-{
-
-	const char *ps, *pt;
-	size_t size;
-	smb_ucs2_t *buffer_s, *buffer_t;
-	int ret;
-
-	for (ps = s, pt = t; ; ps++, pt++) {
-		char us, ut;
-
-		if (!*ps && !*pt)
-			return 0; /* both ended */
- 		else if (!*ps)
-			return -1; /* s is a prefix */
-		else if (!*pt)
-			return +1; /* t is a prefix */
-		else if ((*ps & 0x80) || (*pt & 0x80))
-			/* not ascii anymore, do it the hard way
-			 * from here on in */
-			break;
-
-		us = toupper_ascii_fast(*ps);
-		ut = toupper_ascii_fast(*pt);
-		if (us == ut)
-			continue;
-		else if (us < ut)
-			return -1;
-		else if (us > ut)
-			return +1;
-	}
-
-	if (!push_ucs2_talloc(talloc_tos(), &buffer_s, ps, &size)) {
-		return strcmp(ps, pt);
-		/* Not quite the right answer, but finding the right one
-		   under this failure case is expensive, and it's pretty
-		   close */
-	}
-
-	if (!push_ucs2_talloc(talloc_tos(), &buffer_t, pt, &size)) {
-		TALLOC_FREE(buffer_s);
-		return strcmp(ps, pt);
-		/* Not quite the right answer, but finding the right one
-		   under this failure case is expensive, and it's pretty
-		   close */
-	}
-
-	ret = strcasecmp_w(buffer_s, buffer_t);
-	TALLOC_FREE(buffer_s);
-	TALLOC_FREE(buffer_t);
-	return ret;
-}
-
-
-/**
- Case insensitive string compararison, length limited.
-**/
-int StrnCaseCmp(const char *s, const char *t, size_t len)
-{
-	size_t n = 0;
-	const char *ps, *pt;
-	size_t size;
-	smb_ucs2_t *buffer_s, *buffer_t;
-	int ret;
-
-	for (ps = s, pt = t; n < len ; ps++, pt++, n++) {
-		char us, ut;
-
-		if (!*ps && !*pt)
-			return 0; /* both ended */
- 		else if (!*ps)
-			return -1; /* s is a prefix */
-		else if (!*pt)
-			return +1; /* t is a prefix */
-		else if ((*ps & 0x80) || (*pt & 0x80))
-			/* not ascii anymore, do it the
-			 * hard way from here on in */
-			break;
-
-		us = toupper_ascii_fast(*ps);
-		ut = toupper_ascii_fast(*pt);
-		if (us == ut)
-			continue;
-		else if (us < ut)
-			return -1;
-		else if (us > ut)
-			return +1;
-	}
-
-	if (n == len) {
-		return 0;
-	}
-
-	if (!push_ucs2_talloc(talloc_tos(), &buffer_s, ps, &size)) {
-		return strncmp(ps, pt, len-n);
-		/* Not quite the right answer, but finding the right one
-		   under this failure case is expensive,
-		   and it's pretty close */
-	}
-
-	if (!push_ucs2_talloc(talloc_tos(), &buffer_t, pt, &size)) {
-		TALLOC_FREE(buffer_s);
-		return strncmp(ps, pt, len-n);
-		/* Not quite the right answer, but finding the right one
-		   under this failure case is expensive,
-		   and it's pretty close */
-	}
-
-	ret = strncasecmp_w(buffer_s, buffer_t, len-n);
-	TALLOC_FREE(buffer_s);
-	TALLOC_FREE(buffer_t);
-	return ret;
-}
-
-/**
- * Compare 2 strings.
- *
- * @note The comparison is case-insensitive.
- **/
-bool strequal(const char *s1, const char *s2)
-{
-	if (s1 == s2)
-		return(true);
-	if (!s1 || !s2)
-		return(false);
-
-	return(StrCaseCmp(s1,s2)==0);
-}
-
-/**
  * Compare 2 strings up to and including the nth char.
  *
  * @note The comparison is case-insensitive.
@@ -205,7 +47,7 @@ bool strnequal(const char *s1,const char *s2,size_t n)
 	if (!s1 || !s2 || !n)
 		return(false);
 
-	return(StrnCaseCmp(s1,s2,n)==0);
+	return(strncasecmp_m(s1,s2,n)==0);
 }
 
 /**
@@ -328,77 +170,6 @@ bool trim_char(char *s,char cfront,char cback)
 }
 
 /**
- Safe string copy into a known length string. maxlength does not
- include the terminating zero.
-**/
-
-char *safe_strcpy_fn(char *dest,
-		     const char *src,
-		     size_t maxlength)
-{
-	size_t len;
-
-	if (!dest) {
-		smb_panic("ERROR: NULL dest in safe_strcpy");
-	}
-
-	if (!src) {
-		*dest = 0;
-		return dest;
-	}
-
-	len = strnlen(src, maxlength+1);
-
-	if (len > maxlength) {
-		DEBUG(0,("ERROR: string overflow by "
-			"%lu (%lu - %lu) in safe_strcpy [%.50s]\n",
-			 (unsigned long)(len-maxlength), (unsigned long)len,
-			 (unsigned long)maxlength, src));
-		len = maxlength;
-	}
-
-	memmove(dest, src, len);
-	dest[len] = 0;
-	return dest;
-}
-
-/**
- Safe string cat into a string. maxlength does not
- include the terminating zero.
-**/
-char *safe_strcat_fn(char *dest,
-		     const char *src,
-		     size_t maxlength)
-{
-	size_t src_len, dest_len;
-
-	if (!dest) {
-		smb_panic("ERROR: NULL dest in safe_strcat");
-	}
-
-	if (!src)
-		return dest;
-
-	src_len = strnlen(src, maxlength + 1);
-	dest_len = strnlen(dest, maxlength + 1);
-
-	if (src_len + dest_len > maxlength) {
-		DEBUG(0,("ERROR: string overflow by %d "
-			"in safe_strcat [%.50s]\n",
-			 (int)(src_len + dest_len - maxlength), src));
-		if (maxlength > dest_len) {
-			memcpy(&dest[dest_len], src, maxlength - dest_len);
-		}
-		dest[maxlength] = 0;
-		return NULL;
-	}
-
-	memcpy(&dest[dest_len], src, src_len);
-	dest[dest_len + src_len] = 0;
-	return dest;
-}
-
-/**
  Like strncpy but always null terminates. Make sure there is room!
  The variable n should always be one less than the available size.
 **/
@@ -446,7 +217,7 @@ bool in_list(const char *s, const char *list, bool casesensitive)
 				break;
 			}
 		} else {
-			if (StrCaseCmp(tok,s) == 0) {
+			if (strcasecmp_m(tok,s) == 0) {
 				ret = true;
 				break;
 			}
@@ -456,235 +227,6 @@ bool in_list(const char *s, const char *list, bool casesensitive)
 	return ret;
 }
 
-/* this is used to prevent lots of mallocs of size 1 */
-static const char null_string[] = "";
-
-/**
- Set a string value, allocing the space for the string
-**/
-
-static bool string_init(char **dest,const char *src)
-{
-	size_t l;
-
-	if (!src)
-		src = "";
-
-	l = strlen(src);
-
-	if (l == 0) {
-		*dest = CONST_DISCARD(char*, null_string);
-	} else {
-		(*dest) = SMB_STRDUP(src);
-		if ((*dest) == NULL) {
-			DEBUG(0,("Out of memory in string_init\n"));
-			return false;
-		}
-	}
-	return(true);
-}
-
-/**
- Free a string value.
-**/
-
-void string_free(char **s)
-{
-	if (!s || !(*s))
-		return;
-	if (*s == null_string)
-		*s = NULL;
-	SAFE_FREE(*s);
-}
-
-/**
- Set a string value, deallocating any existing space, and allocing the space
- for the string
-**/
-
-bool string_set(char **dest,const char *src)
-{
-	string_free(dest);
-	return(string_init(dest,src));
-}
-
-/**
- Substitute a string for a pattern in another string. Make sure there is
- enough room!
-
- This routine looks for pattern in s and replaces it with
- insert. It may do multiple replacements or just one.
-
- Any of " ; ' $ or ` in the insert string are replaced with _
- if len==0 then the string cannot be extended. This is different from the old
- use of len==0 which was for no length checks to be done.
-**/
-
-void string_sub2(char *s,const char *pattern, const char *insert, size_t len,
-		 bool remove_unsafe_characters, bool replace_once,
-		 bool allow_trailing_dollar)
-{
-	char *p;
-	ssize_t ls,lp,li, i;
-
-	if (!insert || !pattern || !*pattern || !s)
-		return;
-
-	ls = (ssize_t)strlen(s);
-	lp = (ssize_t)strlen(pattern);
-	li = (ssize_t)strlen(insert);
-
-	if (len == 0)
-		len = ls + 1; /* len is number of *bytes* */
-
-	while (lp <= ls && (p = strstr_m(s,pattern))) {
-		if (ls + (li-lp) >= len) {
-			DEBUG(0,("ERROR: string overflow by "
-				"%d in string_sub(%.50s, %d)\n",
-				 (int)(ls + (li-lp) - len),
-				 pattern, (int)len));
-			break;
-		}
-		if (li != lp) {
-			memmove(p+li,p+lp,strlen(p+lp)+1);
-		}
-		for (i=0;i<li;i++) {
-			switch (insert[i]) {
-			case '$':
-				/* allow a trailing $
-				 * (as in machine accounts) */
-				if (allow_trailing_dollar && (i == li - 1 )) {
-					p[i] = insert[i];
-					break;
-				}
-			case '`':
-			case '"':
-			case '\'':
-			case ';':
-			case '%':
-			case '\r':
-			case '\n':
-				if ( remove_unsafe_characters ) {
-					p[i] = '_';
-					/* yes this break should be here
-					 * since we want to fall throw if
-					 * not replacing unsafe chars */
-					break;
-				}
-			default:
-				p[i] = insert[i];
-			}
-		}
-		s = p + li;
-		ls += (li-lp);
-
-		if (replace_once)
-			break;
-	}
-}
-
-void string_sub_once(char *s, const char *pattern,
-		const char *insert, size_t len)
-{
-	string_sub2( s, pattern, insert, len, true, true, false );
-}
-
-void string_sub(char *s,const char *pattern, const char *insert, size_t len)
-{
-	string_sub2( s, pattern, insert, len, true, false, false );
-}
-
-void fstring_sub(char *s,const char *pattern,const char *insert)
-{
-	string_sub(s, pattern, insert, sizeof(fstring));
-}
-
-/**
- Similar to string_sub2, but it will accept only allocated strings
- and may realloc them so pay attention at what you pass on no
- pointers inside strings, no const may be passed
- as string.
-**/
-
-char *realloc_string_sub2(char *string,
-			const char *pattern,
-			const char *insert,
-			bool remove_unsafe_characters,
-			bool allow_trailing_dollar)
-{
-	char *p, *in;
-	char *s;
-	ssize_t ls,lp,li,ld, i;
-
-	if (!insert || !pattern || !*pattern || !string || !*string)
-		return NULL;
-
-	s = string;
-
-	in = SMB_STRDUP(insert);
-	if (!in) {
-		DEBUG(0, ("realloc_string_sub: out of memory!\n"));
-		return NULL;
-	}
-	ls = (ssize_t)strlen(s);
-	lp = (ssize_t)strlen(pattern);
-	li = (ssize_t)strlen(insert);
-	ld = li - lp;
-	for (i=0;i<li;i++) {
-		switch (in[i]) {
-			case '$':
-				/* allow a trailing $
-				 * (as in machine accounts) */
-				if (allow_trailing_dollar && (i == li - 1 )) {
-					break;
-				}
-			case '`':
-			case '"':
-			case '\'':
-			case ';':
-			case '%':
-			case '\r':
-			case '\n':
-				if ( remove_unsafe_characters ) {
-					in[i] = '_';
-					break;
-				}
-			default:
-				/* ok */
-				break;
-		}
-	}
-
-	while ((p = strstr_m(s,pattern))) {
-		if (ld > 0) {
-			int offset = PTR_DIFF(s,string);
-			string = (char *)SMB_REALLOC(string, ls + ld + 1);
-			if (!string) {
-				DEBUG(0, ("realloc_string_sub: "
-					"out of memory!\n"));
-				SAFE_FREE(in);
-				return NULL;
-			}
-			p = string + offset + (p - s);
-		}
-		if (li != lp) {
-			memmove(p+li,p+lp,strlen(p+lp)+1);
-		}
-		memcpy(p, in, li);
-		s = p + li;
-		ls += ld;
-	}
-	SAFE_FREE(in);
-	return string;
-}
-
-char *realloc_string_sub(char *string,
-			const char *pattern,
-			const char *insert)
-{
-	return realloc_string_sub2(string, pattern, insert, true, false);
-}
-
 /*
  * Internal guts of talloc_string_sub and talloc_all_string_sub.
  * talloc version of string_sub2.
@@ -715,7 +257,7 @@ char *talloc_string_sub2(TALLOC_CTX *mem_ctx, const char *src,
 
 	s = string;
 
-	in = SMB_STRDUP(insert);
+	in = talloc_strdup(mem_ctx, insert);
 	if (!in) {
 		DEBUG(0, ("talloc_string_sub2: ENOMEM\n"));
 		return NULL;
@@ -758,7 +300,7 @@ char *talloc_string_sub2(TALLOC_CTX *mem_ctx, const char *src,
 			if (!string) {
 				DEBUG(0, ("talloc_string_sub: out of "
 					  "memory!\n"));
-				SAFE_FREE(in);
+				TALLOC_FREE(in);
 				return NULL;
 			}
 			p = string + offset + (p - s);
@@ -774,7 +316,7 @@ char *talloc_string_sub2(TALLOC_CTX *mem_ctx, const char *src,
 			break;
 		}
 	}
-	SAFE_FREE(in);
+	TALLOC_FREE(in);
 	return string;
 }
 
@@ -789,48 +331,6 @@ char *talloc_string_sub(TALLOC_CTX *mem_ctx,
 			true, false, false);
 }
 
-/**
- Similar to string_sub() but allows for any character to be substituted.
- Use with caution!
- if len==0 then the string cannot be extended. This is different from the old
- use of len==0 which was for no length checks to be done.
-**/
-
-void all_string_sub(char *s,const char *pattern,const char *insert, size_t len)
-{
-	char *p;
-	ssize_t ls,lp,li;
-
-	if (!insert || !pattern || !s)
-		return;
-
-	ls = (ssize_t)strlen(s);
-	lp = (ssize_t)strlen(pattern);
-	li = (ssize_t)strlen(insert);
-
-	if (!*pattern)
-		return;
-
-	if (len == 0)
-		len = ls + 1; /* len is number of *bytes* */
-
-	while (lp <= ls && (p = strstr_m(s,pattern))) {
-		if (ls + (li-lp) >= len) {
-			DEBUG(0,("ERROR: string overflow by "
-				"%d in all_string_sub(%.50s, %d)\n",
-				 (int)(ls + (li-lp) - len),
-				 pattern, (int)len));
-			break;
-		}
-		if (li != lp) {
-			memmove(p+li,p+lp,strlen(p+lp)+1);
-		}
-		memcpy(p, insert, li);
-		s = p + li;
-		ls += (li-lp);
-	}
-}
-
 char *talloc_all_string_sub(TALLOC_CTX *ctx,
 				const char *src,
 				const char *pattern,
@@ -898,93 +398,12 @@ char *strnrchr_m(const char *s, char c, unsigned int n)
 		/* Too hard to try and get right. */
 		return NULL;
 	}
-	ret = (char *)(s+strlen(s2));
+	ret = discard_const_p(char, (s+strlen(s2)));
 	TALLOC_FREE(ws);
 	TALLOC_FREE(s2);
 	return ret;
 }
 
-/***********************************************************************
- strstr_m - We convert via ucs2 for now.
-***********************************************************************/
-
-char *strstr_m(const char *src, const char *findstr)
-{
-	smb_ucs2_t *p;
-	smb_ucs2_t *src_w, *find_w;
-	const char *s;
-	char *s2;
-	char *retp;
-
-	size_t converted_size, findstr_len = 0;
-
-	/* for correctness */
-	if (!findstr[0]) {
-		return (char*)src;
-	}
-
-	/* Samba does single character findstr calls a *lot*. */
-	if (findstr[1] == '\0')
-		return strchr_m(src, *findstr);
-
-	/* We optimise for the ascii case, knowing that all our
-	   supported multi-byte character sets are ascii-compatible
-	   (ie. they match for the first 128 chars) */
-
-	for (s = src; *s && !(((unsigned char)s[0]) & 0x80); s++) {
-		if (*s == *findstr) {
-			if (!findstr_len)
-				findstr_len = strlen(findstr);
-
-			if (strncmp(s, findstr, findstr_len) == 0) {
-				return (char *)s;
-			}
-		}
-	}
-
-	if (!*s)
-		return NULL;
-
-#if 1 /* def BROKEN_UNICODE_COMPOSE_CHARACTERS */
-	/* 'make check' fails unless we do this */
-
-	/* With compose characters we must restart from the beginning. JRA. */
-	s = src;
-#endif
-
-	if (!push_ucs2_talloc(talloc_tos(), &src_w, src, &converted_size)) {
-		DEBUG(0,("strstr_m: src malloc fail\n"));
-		return NULL;
-	}
-
-	if (!push_ucs2_talloc(talloc_tos(), &find_w, findstr, &converted_size)) {
-		TALLOC_FREE(src_w);
-		DEBUG(0,("strstr_m: find malloc fail\n"));
-		return NULL;
-	}
-
-	p = strstr_w(src_w, find_w);
-
-	if (!p) {
-		TALLOC_FREE(src_w);
-		TALLOC_FREE(find_w);
-		return NULL;
-	}
-
-	*p = 0;
-	if (!pull_ucs2_talloc(talloc_tos(), &s2, src_w, &converted_size)) {
-		TALLOC_FREE(src_w);
-		TALLOC_FREE(find_w);
-		DEBUG(0,("strstr_m: dest malloc fail\n"));
-		return NULL;
-	}
-	retp = (char *)(s+strlen(s2));
-	TALLOC_FREE(src_w);
-	TALLOC_FREE(find_w);
-	TALLOC_FREE(s2);
-	return retp;
-}
-
 static bool unix_strlower(const char *src, size_t srclen, char *dest, size_t destlen)
 {
 	size_t size;
@@ -1005,6 +424,36 @@ static bool unix_strlower(const char *src, size_t srclen, char *dest, size_t des
 	return ret;
 }
 
+#if 0 /* Alternate function that avoid talloc calls for ASCII and non ASCII */
+
+/**
+ Convert a string to lower case.
+**/
+_PUBLIC_ void strlower_m(char *s)
+{
+	char *d;
+	struct smb_iconv_handle *iconv_handle;
+
+	iconv_handle = get_iconv_handle();
+
+	d = s;
+
+	while (*s) {
+		size_t c_size, c_size2;
+		codepoint_t c = next_codepoint_handle(iconv_handle, s, &c_size);
+		c_size2 = push_codepoint_handle(iconv_handle, d, tolower_m(c));
+		if (c_size2 > c_size) {
+			DEBUG(0,("FATAL: codepoint 0x%x (0x%x) expanded from %d to %d bytes in strlower_m\n",
+				 c, tolower_m(c), (int)c_size, (int)c_size2));
+			smb_panic("codepoint expansion in strlower_m\n");
+		}
+		s += c_size;
+		d += c_size2;
+	}
+	*d = 0;
+}
+
+#endif
 
 /**
  Convert a string to lower case.
@@ -1060,6 +509,37 @@ static bool unix_strupper(const char *src, size_t srclen, char *dest, size_t des
 	return ret;
 }
 
+#if 0 /* Alternate function that avoid talloc calls for ASCII and non ASCII */
+
+/**
+ Convert a string to UPPER case.
+**/
+_PUBLIC_ void strupper_m(char *s)
+{
+	char *d;
+	struct smb_iconv_handle *iconv_handle;
+
+	iconv_handle = get_iconv_handle();
+
+	d = s;
+
+	while (*s) {
+		size_t c_size, c_size2;
+		codepoint_t c = next_codepoint_handle(iconv_handle, s, &c_size);
+		c_size2 = push_codepoint_handle(iconv_handle, d, toupper_m(c));
+		if (c_size2 > c_size) {
+			DEBUG(0,("FATAL: codepoint 0x%x (0x%x) expanded from %d to %d bytes in strupper_m\n",
+				 c, toupper_m(c), (int)c_size, (int)c_size2));
+			smb_panic("codepoint expansion in strupper_m\n");
+		}
+		s += c_size;
+		d += c_size2;
+	}
+	*d = 0;
+}
+
+#endif
+
 /**
  Convert a string to upper case.
 **/
@@ -1116,35 +596,6 @@ int fstr_sprintf(fstring s, const char *fmt, ...)
 #define S_LIST_ABS 16 /* List Allocation Block Size */
 
 /******************************************************************************
- version of standard_sub_basic() for string lists; uses talloc_sub_basic()
- for the work
- *****************************************************************************/
-
-bool str_list_sub_basic( char **list, const char *smb_name,
-			 const char *domain_name )
-{
-	TALLOC_CTX *ctx = list;
-	char *s, *tmpstr;
-
-	while ( *list ) {
-		s = *list;
-		tmpstr = talloc_sub_basic(ctx, smb_name, domain_name, s);
-		if ( !tmpstr ) {
-			DEBUG(0,("str_list_sub_basic: "
-				"alloc_sub_basic() return NULL!\n"));
-			return false;
-		}
-
-		TALLOC_FREE(*list);
-		*list = tmpstr;
-
-		list++;
-	}
-
-	return true;
-}
-
-/******************************************************************************
  substitute a specific pattern in a string list
  *****************************************************************************/
 
@@ -1173,7 +624,7 @@ bool str_list_substitute(char **list, const char *pattern, const char *insert)
 			t = *list;
 			d = p -t;
 			if (ld) {
-				t = TALLOC_ARRAY(ctx, char, ls +ld +1);
+				t = talloc_array(ctx, char, ls +ld +1);
 				if (!t) {
 					DEBUG(0,("str_list_substitute: "
 						"Unable to allocate memory"));
@@ -1415,25 +866,16 @@ uint64_t STR_TO_SMB_BIG_UINT(const char *nptr, const char **entptr)
  *
  *  Returns 0 if the string can't be converted.
  */
-SMB_OFF_T conv_str_size(const char * str)
+uint64_t conv_str_size(const char * str)
 {
-	SMB_OFF_T lval_orig;
-        SMB_OFF_T lval;
+        uint64_t lval;
 	char * end;
 
         if (str == NULL || *str == '\0') {
                 return 0;
         }
 
-#ifdef HAVE_STRTOULL
-	if (sizeof(SMB_OFF_T) == 8) {
-		lval = strtoull(str, &end, 10 /* base */);
-	} else {
-		lval = strtoul(str, &end, 10 /* base */);
-	}
-#else
-	lval = strtoul(str, &end, 10 /* base */);
-#endif
+	lval = strtoull(str, &end, 10 /* base */);
 
         if (end == NULL || end == str) {
                 return 0;
@@ -1443,35 +885,24 @@ SMB_OFF_T conv_str_size(const char * str)
 		return lval;
 	}
 
-	lval_orig = lval;
-
 	if (strwicmp(end, "K") == 0) {
-		lval *= (SMB_OFF_T)1024;
+		lval *= 1024ULL;
 	} else if (strwicmp(end, "M") == 0) {
-		lval *= ((SMB_OFF_T)1024 * (SMB_OFF_T)1024);
+		lval *= (1024ULL * 1024ULL);
 	} else if (strwicmp(end, "G") == 0) {
-		lval *= ((SMB_OFF_T)1024 * (SMB_OFF_T)1024 *
-			 (SMB_OFF_T)1024);
+		lval *= (1024ULL * 1024ULL *
+			 1024ULL);
 	} else if (strwicmp(end, "T") == 0) {
-		lval *= ((SMB_OFF_T)1024 * (SMB_OFF_T)1024 *
-			 (SMB_OFF_T)1024 * (SMB_OFF_T)1024);
+		lval *= (1024ULL * 1024ULL *
+			 1024ULL * 1024ULL);
 	} else if (strwicmp(end, "P") == 0) {
-		lval *= ((SMB_OFF_T)1024 * (SMB_OFF_T)1024 *
-			 (SMB_OFF_T)1024 * (SMB_OFF_T)1024 *
-			 (SMB_OFF_T)1024);
+		lval *= (1024ULL * 1024ULL *
+			 1024ULL * 1024ULL *
+			 1024ULL);
 	} else {
 		return 0;
 	}
 
-	/*
-	 * Primitive attempt to detect wrapping on platforms with
-	 * 4-byte SMB_OFF_T. It's better to let the caller handle a
-	 * failure than some random number.
-	 */
-	if (lval_orig <= lval) {
-		return 0;
-	}
-
 	return lval;
 }
 
@@ -1495,7 +926,7 @@ void sprintf_append(TALLOC_CTX *mem_ctx, char **string, ssize_t *len,
 		if (*bufsize == 0)
 			*bufsize = 128;
 
-		*string = TALLOC_ARRAY(mem_ctx, char, *bufsize);
+		*string = talloc_array(mem_ctx, char, *bufsize);
 		if (*string == NULL)
 			goto error;
 	}
@@ -1517,7 +948,7 @@ void sprintf_append(TALLOC_CTX *mem_ctx, char **string, ssize_t *len,
 	}
 
 	if (increased) {
-		*string = TALLOC_REALLOC_ARRAY(mem_ctx, *string, char,
+		*string = talloc_realloc(mem_ctx, *string, char,
 					       *bufsize);
 		if (*string == NULL) {
 			goto error;
@@ -1589,33 +1020,6 @@ char *talloc_asprintf_strlower_m(TALLOC_CTX *t, const char *fmt, ...)
 }
 
 
-/*
-   Returns the substring from src between the first occurrence of
-   the char "front" and the first occurence of the char "back".
-   Mallocs the return string which must be freed.  Not for use
-   with wide character strings.
-*/
-char *sstring_sub(const char *src, char front, char back)
-{
-	char *temp1, *temp2, *temp3;
-	ptrdiff_t len;
-
-	temp1 = strchr(src, front);
-	if (temp1 == NULL) return NULL;
-	temp2 = strchr(src, back);
-	if (temp2 == NULL) return NULL;
-	len = temp2 - temp1;
-	if (len <= 0) return NULL;
-	temp3 = (char*)SMB_MALLOC(len);
-	if (temp3 == NULL) {
-		DEBUG(1,("Malloc failure in sstring_sub\n"));
-		return NULL;
-	}
-	memcpy(temp3, temp1+1, len-1);
-	temp3[len-1] = '\0';
-	return temp3;
-}
-
 /********************************************************************
  Check a string for any occurrences of a specified list of invalid
  characters.
@@ -1825,7 +1229,7 @@ char **str_list_make_v3(TALLOC_CTX *mem_ctx, const char *string,
 	if (!string || !*string)
 		return NULL;
 
-	list = TALLOC_ARRAY(mem_ctx, char *, S_LIST_ABS+1);
+	list = talloc_array(mem_ctx, char *, S_LIST_ABS+1);
 	if (list == NULL) {
 		return NULL;
 	}
@@ -1849,7 +1253,7 @@ char **str_list_make_v3(TALLOC_CTX *mem_ctx, const char *string,
 
 			lsize += S_LIST_ABS;
 
-			tmp = TALLOC_REALLOC_ARRAY(mem_ctx, list, char *,
+			tmp = talloc_realloc(mem_ctx, list, char *,
 						   lsize + 1);
 			if (tmp == NULL) {
 				DEBUG(0,("str_list_make: "
diff --git a/source3/lib/util_tdb.c b/source3/lib/util_tdb.c
index a9290ab94ae..ade46bf18ec 100644
--- a/source3/lib/util_tdb.c
+++ b/source3/lib/util_tdb.c
@@ -21,11 +21,108 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "util_tdb.h"
+
 #undef malloc
 #undef realloc
 #undef calloc
 #undef strdup
 
+#ifdef BUILD_TDB2
+static struct flock flock_struct;
+
+/* Return a value which is none of v1, v2 or v3. */
+static inline short int invalid_value(short int v1, short int v2, short int v3)
+{
+	short int try = (v1+v2+v3)^((v1+v2+v3) << 16);
+	while (try == v1 || try == v2 || try == v3)
+		try++;
+	return try;
+}
+
+/* We invalidate in as many ways as we can, so the OS rejects it */
+static void invalidate_flock_struct(int signum)
+{
+	flock_struct.l_type = invalid_value(F_RDLCK, F_WRLCK, F_UNLCK);
+	flock_struct.l_whence = invalid_value(SEEK_SET, SEEK_CUR, SEEK_END);
+	flock_struct.l_start = -1;
+	/* A large negative. */
+	flock_struct.l_len = (((off_t)1 << (sizeof(off_t)*CHAR_BIT - 1)) + 1);
+}
+
+static int timeout_lock(int fd, int rw, off_t off, off_t len, bool waitflag,
+			void *_timeout)
+{
+	int ret, saved_errno;
+	unsigned int timeout = *(unsigned int *)_timeout;
+
+	flock_struct.l_type = rw;
+	flock_struct.l_whence = SEEK_SET;
+	flock_struct.l_start = off;
+	flock_struct.l_len = len;
+
+	CatchSignal(SIGALRM, invalidate_flock_struct);
+	alarm(timeout);
+
+	for (;;) {
+		if (waitflag)
+			ret = fcntl(fd, F_SETLKW, &flock_struct);
+		else
+			ret = fcntl(fd, F_SETLK, &flock_struct);
+
+		if (ret == 0)
+			break;
+
+		/* Not signalled?  Something else went wrong. */
+		if (flock_struct.l_len == len) {
+			if (errno == EAGAIN || errno == EINTR)
+				continue;
+			saved_errno = errno;
+			break;
+		} else {
+			saved_errno = EINTR;
+			break;
+		}
+	}
+
+	alarm(0);
+	errno = saved_errno;
+	return ret;
+}
+
+static int tdb_chainlock_with_timeout_internal(struct tdb_context *tdb,
+					       TDB_DATA key,
+					       unsigned int timeout,
+					       int rw_type)
+{
+	union tdb_attribute locking;
+	enum TDB_ERROR ecode;
+
+	if (timeout) {
+		locking.base.attr = TDB_ATTRIBUTE_FLOCK;
+		ecode = tdb_get_attribute(tdb, &locking);
+		if (ecode != TDB_SUCCESS)
+			return ecode;
+
+		/* Replace locking function with our own. */
+		locking.flock.data = &timeout;
+		locking.flock.lock = timeout_lock;
+
+		ecode = tdb_set_attribute(tdb, &locking);
+		if (ecode != TDB_SUCCESS)
+			return ecode;
+	}
+	if (rw_type == F_RDLCK)
+		ecode = tdb_chainlock_read(tdb, key);
+	else
+		ecode = tdb_chainlock(tdb, key);
+
+	if (timeout) {
+		tdb_unset_attribute(tdb, TDB_ATTRIBUTE_FLOCK);
+	}
+	return ecode == TDB_SUCCESS ? 0 : -1;
+}
+#else
 /* these are little tdb utility functions that are meant to make
    dealing with a tdb database a little less cumbersome in Samba */
 
@@ -65,7 +162,7 @@ static int tdb_chainlock_with_timeout_internal( TDB_CONTEXT *tdb, TDB_DATA key,
 		alarm(0);
 		tdb_setalarm_sigptr(tdb, NULL);
 		CatchSignal(SIGALRM, SIG_IGN);
-		if (gotalarm && (ret == -1)) {
+		if (gotalarm && (ret != 0)) {
 			DEBUG(0,("tdb_chainlock_with_timeout_internal: alarm (%u) timed out for key %s in tdb %s\n",
 				timeout, key.dptr, tdb_name(tdb)));
 			/* TODO: If we time out waiting for a lock, it might
@@ -76,11 +173,12 @@ static int tdb_chainlock_with_timeout_internal( TDB_CONTEXT *tdb, TDB_DATA key,
 		}
 	}
 
-	return ret;
+	return ret == 0 ? 0 : -1;
 }
+#endif /* TDB1 */
 
 /****************************************************************************
- Write lock a chain. Return -1 if timeout or lock failed.
+ Write lock a chain. Return non-zero if timeout or lock failed.
 ****************************************************************************/
 
 int tdb_chainlock_with_timeout( TDB_CONTEXT *tdb, TDB_DATA key, unsigned int timeout)
@@ -97,7 +195,7 @@ int tdb_lock_bystring_with_timeout(TDB_CONTEXT *tdb, const char *keyval,
 }
 
 /****************************************************************************
- Read lock a chain by string. Return -1 if timeout or lock failed.
+ Read lock a chain by string. Return non-zero if timeout or lock failed.
 ****************************************************************************/
 
 int tdb_read_lock_bystring_with_timeout(TDB_CONTEXT *tdb, const char *keyval, unsigned int timeout)
@@ -229,7 +327,7 @@ bool tdb_pack_append(TALLOC_CTX *mem_ctx, uint8 **buf, size_t *len,
 	va_end(ap);
 
 	if (mem_ctx != NULL) {
-		*buf = TALLOC_REALLOC_ARRAY(mem_ctx, *buf, uint8,
+		*buf = talloc_realloc(mem_ctx, *buf, uint8,
 					    (*len) + len1);
 	} else {
 		*buf = SMB_REALLOC_ARRAY(*buf, uint8, (*len) + len1);
@@ -370,6 +468,14 @@ int tdb_unpack(const uint8 *buf, int bufsize, const char *fmt, ...)
  Log tdb messages via DEBUG().
 ****************************************************************************/
 
+#ifdef BUILD_TDB2
+static void tdb_log(TDB_CONTEXT *tdb, enum tdb_log_level level,
+		    const char *message, void *unused)
+{
+	DEBUG((int)level, ("tdb(%s): %s",
+			   tdb_name(tdb) ? tdb_name(tdb) : "unnamed", message));
+}
+#else
 static void tdb_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, const char *format, ...)
 {
 	va_list ap;
@@ -386,6 +492,7 @@ static void tdb_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, const char *fo
 	DEBUG((int)level, ("tdb(%s): %s", tdb_name(tdb) ? tdb_name(tdb) : "unnamed", ptr));
 	SAFE_FREE(ptr);
 }
+#endif /* TDB1 */
 
 /****************************************************************************
  Like tdb_open() but also setup a logging function that redirects to
@@ -396,14 +503,10 @@ TDB_CONTEXT *tdb_open_log(const char *name, int hash_size, int tdb_flags,
 			  int open_flags, mode_t mode)
 {
 	TDB_CONTEXT *tdb;
-	struct tdb_logging_context log_ctx;
 
 	if (!lp_use_mmap())
 		tdb_flags |= TDB_NOMMAP;
 
-	log_ctx.log_fn = tdb_log;
-	log_ctx.log_private = NULL;
-
 	if ((hash_size == 0) && (name != NULL)) {
 		const char *base = strrchr_m(name, '/');
 		if (base != NULL) {
@@ -415,8 +518,8 @@ TDB_CONTEXT *tdb_open_log(const char *name, int hash_size, int tdb_flags,
 		hash_size = lp_parm_int(-1, "tdb_hashsize", base, 0);
 	}
 
-	tdb = tdb_open_ex(name, hash_size, tdb_flags, 
-			  open_flags, mode, &log_ctx, NULL);
+	tdb = tdb_open_compat(name, hash_size, tdb_flags,
+			      open_flags, mode, tdb_log, NULL);
 	if (!tdb)
 		return NULL;
 
@@ -440,9 +543,7 @@ int tdb_trans_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf,
 
 	if ((res = tdb_store(tdb, key, dbuf, flag)) != 0) {
 		DEBUG(10, ("tdb_store failed\n"));
-		if (tdb_transaction_cancel(tdb) != 0) {
-			smb_panic("Cancelling transaction failed");
-		}
+		tdb_transaction_cancel(tdb);
 		return res;
 	}
 
@@ -469,9 +570,7 @@ int tdb_trans_delete(struct tdb_context *tdb, TDB_DATA key)
 
 	if ((res = tdb_delete(tdb, key)) != 0) {
 		DEBUG(10, ("tdb_delete failed\n"));
-		if (tdb_transaction_cancel(tdb) != 0) {
-			smb_panic("Cancelling transaction failed");
-		}
+		tdb_transaction_cancel(tdb);
 		return res;
 	}
 
@@ -482,165 +581,6 @@ int tdb_trans_delete(struct tdb_context *tdb, TDB_DATA key)
 	return res;
 }
 
-/*
- Log tdb messages via DEBUG().
-*/
-static void tdb_wrap_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, 
-			 const char *format, ...) PRINTF_ATTRIBUTE(3,4);
-
-static void tdb_wrap_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, 
-			 const char *format, ...)
-{
-	va_list ap;
-	char *ptr = NULL;
-	int debuglevel = 0;
-	int ret;
-
-	switch (level) {
-	case TDB_DEBUG_FATAL:
-		debuglevel = 0;
-		break;
-	case TDB_DEBUG_ERROR:
-		debuglevel = 1;
-		break;
-	case TDB_DEBUG_WARNING:
-		debuglevel = 2;
-		break;
-	case TDB_DEBUG_TRACE:
-		debuglevel = 5;
-		break;
-	default:
-		debuglevel = 0;
-	}		
-
-	va_start(ap, format);
-	ret = vasprintf(&ptr, format, ap);
-	va_end(ap);
-
-	if (ret != -1) {
-		const char *name = tdb_name(tdb);
-		DEBUG(debuglevel, ("tdb(%s): %s", name ? name : "unnamed", ptr));
-		free(ptr);
-	}
-}
-
-struct tdb_wrap_private {
-	struct tdb_context *tdb;
-	const char *name;
-	struct tdb_wrap_private *next, *prev;
-};
-
-static struct tdb_wrap_private *tdb_list;
-
-/* destroy the last connection to a tdb */
-static int tdb_wrap_private_destructor(struct tdb_wrap_private *w)
-{
-	tdb_close(w->tdb);
-	DLIST_REMOVE(tdb_list, w);
-	return 0;
-}				 
-
-static struct tdb_wrap_private *tdb_wrap_private_open(TALLOC_CTX *mem_ctx,
-						      const char *name,
-						      int hash_size,
-						      int tdb_flags,
-						      int open_flags,
-						      mode_t mode)
-{
-	struct tdb_wrap_private *result;
-	struct tdb_logging_context log_ctx;
-
-	result = talloc(mem_ctx, struct tdb_wrap_private);
-	if (result == NULL) {
-		return NULL;
-	}
-	result->name = talloc_strdup(result, name);
-	if (result->name == NULL) {
-		goto fail;
-	}
-
-	log_ctx.log_fn = tdb_wrap_log;
-
-	if (!lp_use_mmap()) {
-		tdb_flags |= TDB_NOMMAP;
-	}
-
-	if ((hash_size == 0) && (name != NULL)) {
-		const char *base;
-		base = strrchr_m(name, '/');
-
-		if (base != NULL) {
-			base += 1;
-		} else {
-			base = name;
-		}
-		hash_size = lp_parm_int(-1, "tdb_hashsize", base, 0);
-	}
-
-	result->tdb = tdb_open_ex(name, hash_size, tdb_flags,
-				  open_flags, mode, &log_ctx, NULL);
-	if (result->tdb == NULL) {
-		goto fail;
-	}
-	talloc_set_destructor(result, tdb_wrap_private_destructor);
-	DLIST_ADD(tdb_list, result);
-	return result;
-
-fail:
-	TALLOC_FREE(result);
-	return NULL;
-}
-
-/*
-  wrapped connection to a tdb database
-  to close just talloc_free() the tdb_wrap pointer
- */
-struct tdb_wrap *tdb_wrap_open(TALLOC_CTX *mem_ctx,
-			       const char *name, int hash_size, int tdb_flags,
-			       int open_flags, mode_t mode)
-{
-	struct tdb_wrap *result;
-	struct tdb_wrap_private *w;
-
-	result = talloc(mem_ctx, struct tdb_wrap);
-	if (result == NULL) {
-		return NULL;
-	}
-
-	for (w=tdb_list;w;w=w->next) {
-		if (strcmp(name, w->name) == 0) {
-			break;
-		}
-	}
-
-	if (w == NULL) {
-		w = tdb_wrap_private_open(result, name, hash_size, tdb_flags,
-					  open_flags, mode);
-	} else {
-		/*
-		 * Correctly use talloc_reference: The tdb will be
-		 * closed when "w" is being freed. The caller never
-		 * sees "w", so an incorrect use of talloc_free(w)
-		 * instead of calling talloc_unlink is not possible.
-		 * To avoid having to refcount ourselves, "w" will
-		 * have multiple parents that hang off all the
-		 * tdb_wrap's being returned from here. Those parents
-		 * can be freed without problem.
-		 */
-		if (talloc_reference(result, w) == NULL) {
-			goto fail;
-		}
-	}
-	if (w == NULL) {
-		goto fail;
-	}
-	result->tdb = w->tdb;
-	return result;
-fail:
-	TALLOC_FREE(result);
-	return NULL;
-}
-
 NTSTATUS map_nt_error_from_tdb(enum TDB_ERROR err)
 {
 	NTSTATUS result = NT_STATUS_INTERNAL_ERROR;
@@ -672,6 +612,7 @@ NTSTATUS map_nt_error_from_tdb(enum TDB_ERROR err)
 		result = NT_STATUS_FILE_LOCK_CONFLICT;
 		break;
 
+#ifndef BUILD_TDB2
 	case TDB_ERR_NOLOCK:
 	case TDB_ERR_LOCK_TIMEOUT:
 		/*
@@ -679,6 +620,7 @@ NTSTATUS map_nt_error_from_tdb(enum TDB_ERROR err)
 		 */
 		result = NT_STATUS_FILE_LOCK_CONFLICT;
 		break;
+#endif
 	case TDB_ERR_NOEXIST:
 		result = NT_STATUS_NOT_FOUND;
 		break;
@@ -688,9 +630,11 @@ NTSTATUS map_nt_error_from_tdb(enum TDB_ERROR err)
 	case TDB_ERR_RDONLY:
 		result = NT_STATUS_ACCESS_DENIED;
 		break;
+#ifndef BUILD_TDB2
 	case TDB_ERR_NESTING:
 		result = NT_STATUS_INTERNAL_ERROR;
 		break;
+#endif
 	};
 	return result;
 }
diff --git a/source3/lib/util_tsock.c b/source3/lib/util_tsock.c
index 1cb88c67bd0..35a97f5e692 100644
--- a/source3/lib/util_tsock.c
+++ b/source3/lib/util_tsock.c
@@ -18,6 +18,8 @@
 */
 
 #include "includes.h"
+#include "../lib/tsocket/tsocket.h"
+#include "../lib/util/tevent_unix.h"
 
 struct tstream_read_packet_state {
 	struct tevent_context *ev;
diff --git a/source3/lib/winbind_util.c b/source3/lib/winbind_util.c
index f30bcfc6126..b458ebe8472 100644
--- a/source3/lib/winbind_util.c
+++ b/source3/lib/winbind_util.c
@@ -237,15 +237,15 @@ bool winbind_lookup_rids(TALLOC_CTX *mem_ctx,
 	}	
 
 	*domain_name = talloc_strdup(mem_ctx, dom_name);
-	*names       = TALLOC_ARRAY(mem_ctx, const char*, num_rids);
-	*types       = TALLOC_ARRAY(mem_ctx, enum lsa_SidType, num_rids);
+	*names       = talloc_array(mem_ctx, const char*, num_rids);
+	*types       = talloc_array(mem_ctx, enum lsa_SidType, num_rids);
 
 	for(i=0; i<num_rids; i++) {
 		(*names)[i] = talloc_strdup(*names, namelist[i]);
 		(*types)[i] = (enum lsa_SidType)name_types[i];
 	}
 
-	wbcFreeMemory(CONST_DISCARD(char*, dom_name));
+	wbcFreeMemory(discard_const_p(char, dom_name));
 	wbcFreeMemory(namelist);
 	wbcFreeMemory(name_types);
 
@@ -284,7 +284,7 @@ bool winbind_get_groups(TALLOC_CTX * mem_ctx, const char *account, uint32_t *num
 	if (ret != WBC_ERR_SUCCESS)
 		return false;
 
-	*_groups = TALLOC_ARRAY(mem_ctx, gid_t, ngroups);
+	*_groups = talloc_array(mem_ctx, gid_t, ngroups);
 	if (*_groups == NULL) {
 	    wbcFreeMemory(group_list);
 	    return false;
@@ -313,7 +313,7 @@ bool winbind_get_sid_aliases(TALLOC_CTX *mem_ctx,
 
 	memcpy(&domain_sid, dom_sid, sizeof(*dom_sid));
 
-	sid_list = TALLOC_ARRAY(mem_ctx, struct wbcDomainSid, num_members);
+	sid_list = talloc_array(mem_ctx, struct wbcDomainSid, num_members);
 
 	for (i=0; i < num_members; i++) {
 	    memcpy(&sid_list[i], &members[i], sizeof(sid_list[i]));
@@ -328,7 +328,7 @@ bool winbind_get_sid_aliases(TALLOC_CTX *mem_ctx,
 		return false;
 	}
 
-	*pp_alias_rids = TALLOC_ARRAY(mem_ctx, uint32_t, num_rids);
+	*pp_alias_rids = talloc_array(mem_ctx, uint32_t, num_rids);
 	if (*pp_alias_rids == NULL) {
 		wbcFreeMemory(rids);
 		return false;
diff --git a/source3/lib/wins_srv.c b/source3/lib/wins_srv.c
index 6676f02e947..f9e8f3b0e1a 100644
--- a/source3/lib/wins_srv.c
+++ b/source3/lib/wins_srv.c
@@ -328,6 +328,48 @@ struct in_addr wins_srv_ip_tag(const char *tag, struct in_addr src_ip)
 	return t_ip.ip;
 }
 
+bool wins_server_tag_ips(const char *tag, TALLOC_CTX *mem_ctx,
+			 struct in_addr **pservers, int *pnum_servers)
+{
+	const char **list;
+	int i, num_servers;
+	struct in_addr *servers;
+
+	list = lp_wins_server_list();
+	if ((list == NULL) || (list[0] == NULL)) {
+		return false;
+	}
+
+	num_servers = 0;
+
+	for (i=0; list[i] != NULL; i++) {
+		struct tagged_ip t_ip;
+		parse_ip(&t_ip, list[i]);
+		if (strcmp(tag, t_ip.tag) == 0) {
+			num_servers += 1;
+		}
+	}
+
+	servers = talloc_array(mem_ctx, struct in_addr, num_servers);
+	if (servers == NULL) {
+		return false;
+	}
+
+	num_servers = 0;
+
+	for (i=0; list[i] != NULL; i++) {
+		struct tagged_ip t_ip;
+		parse_ip(&t_ip, list[i]);
+		if (strcmp(tag, t_ip.tag) == 0) {
+			servers[num_servers] = t_ip.ip;
+			num_servers += 1;
+		}
+	}
+	*pnum_servers = num_servers;
+	*pservers = servers;
+	return true;
+}
+
 
 /*
   return a count of the number of IPs for a particular tag, including
diff --git a/source3/libads/ads_ldap_protos.h b/source3/libads/ads_ldap_protos.h
index 03869fe5c99..0fb71349948 100644
--- a/source3/libads/ads_ldap_protos.h
+++ b/source3/libads/ads_ldap_protos.h
@@ -1,4 +1,30 @@
 /*
+   Unix SMB/CIFS implementation.
+   ads (active directory) utility library
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Remus Koos 2001
+   Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2002
+   Copyright (C) Guenther Deschner 2005
+   Copyright (C) Gerald Carter 2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBADS_ADS_LDAP_PROTOS_H_
+#define _LIBADS_ADS_LDAP_PROTOS_H_
+
+/*
  * Prototypes for ads
  */
 
@@ -112,3 +138,5 @@ ADS_STATUS ads_get_joinable_ous(ADS_STRUCT *ads,
 				TALLOC_CTX *mem_ctx,
 				char ***ous,
 				size_t *num_ous);
+
+#endif /* _LIBADS_ADS_LDAP_PROTOS_H_ */
diff --git a/source3/libads/ads_proto.h b/source3/libads/ads_proto.h
index ed702de8a6c..324fa2a9cc1 100644
--- a/source3/libads/ads_proto.h
+++ b/source3/libads/ads_proto.h
@@ -1,3 +1,37 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  ads (active directory) utility library
+ *
+ *  Copyright (C) Andrew Bartlett			2001
+ *  Copyright (C) Andrew Tridgell			2001
+ *  Copyright (C) Remus Koos (remuskoos@yahoo.com)	2001
+ *  Copyright (C) Alexey Kotovich			2002
+ *  Copyright (C) Jim McDonough <jmcd@us.ibm.com>	2002-2003
+ *  Copyright (C) Luke Howard				2003
+ *  Copyright (C) Guenther Deschner			2003-2008
+ *  Copyright (C) Rakesh Patel				2004
+ *  Copyright (C) Dan Perry				2004
+ *  Copyright (C) Jeremy Allison			2004
+ *  Copyright (C) Gerald Carter				2006
+ *  Copyright (C) Stefan Metzmacher			2007
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LIBADS_ADS_PROTO_H_
+#define _LIBADS_ADS_PROTO_H_
+
 /* The following definitions come from libads/ads_struct.c  */
 
 char *ads_build_path(const char *realm, const char *sep, const char *field, int reverse);
@@ -152,3 +186,5 @@ ADS_STATUS ads_setup_sasl_wrapping(ADS_STRUCT *ads,
 /* The following definitions come from libads/util.c  */
 
 ADS_STATUS ads_change_trust_account_password(ADS_STRUCT *ads, char *host_principal);
+
+#endif /* _LIBADS_ADS_PROTO_H_ */
diff --git a/source3/libads/ads_status.c b/source3/libads/ads_status.c
index b99464118de..a9e59edaddb 100644
--- a/source3/libads/ads_status.c
+++ b/source3/libads/ads_status.c
@@ -23,6 +23,7 @@
 #include "includes.h"
 #include "smb_krb5.h"
 #include "smb_ldap.h"
+#include "libads/ads_status.h"
 
 /*
   build a ADS_STATUS structure
diff --git a/source3/libads/ads_status.h b/source3/libads/ads_status.h
index 4c27e16f50e..ff7c1036380 100644
--- a/source3/libads/ads_status.h
+++ b/source3/libads/ads_status.h
@@ -1,3 +1,27 @@
+/*
+   Unix SMB/CIFS implementation.
+   ads (active directory) utility library
+   Copyright (C) Andrew Tridgell 2001
+   Copyright (C) Remus Koos 2001
+   Copyright (C) Andrew Bartlett 2001
+
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBADS_ADS_STATUS_H_
+#define _LIBADS_ADS_STATUS_H_
 
 /* there are 5 possible types of errors the ads subsystem can produce */
 enum ads_error_type {ENUM_ADS_ERROR_KRB5, ENUM_ADS_ERROR_GSS,
@@ -40,3 +64,5 @@ ADS_STATUS ads_build_nt_error(enum ads_error_type etype,
 NTSTATUS ads_ntstatus(ADS_STATUS status);
 const char *ads_errstr(ADS_STATUS status);
 NTSTATUS gss_err_to_ntstatus(uint32 maj, uint32 min);
+
+#endif /* _LIBADS_ADS_STATUS_H_ */
diff --git a/source3/libads/authdata.c b/source3/libads/authdata.c
index 6f49b857da7..44279a24d77 100644
--- a/source3/libads/authdata.c
+++ b/source3/libads/authdata.c
@@ -78,7 +78,7 @@ NTSTATUS kerberos_return_pac(TALLOC_CTX *mem_ctx,
 	NT_STATUS_HAVE_NO_MEMORY(auth_princ);
 
 	local_service = talloc_asprintf(mem_ctx, "%s$@%s",
-					global_myname(), lp_realm());
+					lp_netbios_name(), lp_realm());
 	NT_STATUS_HAVE_NO_MEMORY(local_service);
 
 	ret = kerberos_kinit_password_ext(auth_princ,
diff --git a/source3/libads/cldap.c b/source3/libads/cldap.c
index 5d2e900c05a..03fa17c26fd 100644
--- a/source3/libads/cldap.c
+++ b/source3/libads/cldap.c
@@ -30,7 +30,7 @@
 *******************************************************************/
 
 bool ads_cldap_netlogon(TALLOC_CTX *mem_ctx,
-			const char *server,
+			struct sockaddr_storage *ss,
 			const char *realm,
 			uint32_t nt_version,
 			struct netlogon_samlogon_response **_reply)
@@ -39,18 +39,12 @@ bool ads_cldap_netlogon(TALLOC_CTX *mem_ctx,
 	struct cldap_netlogon io;
 	struct netlogon_samlogon_response *reply;
 	NTSTATUS status;
-	struct sockaddr_storage ss;
 	char addrstr[INET6_ADDRSTRLEN];
 	const char *dest_str;
 	int ret;
 	struct tsocket_address *dest_addr;
 
-	if (!interpret_string_addr_prefer_ipv4(&ss, server, 0)) {
-		DEBUG(2,("Failed to resolve[%s] into an address for cldap\n",
-			server));
-		return false;
-	}
-	dest_str = print_sockaddr(addrstr, sizeof(addrstr), &ss);
+	dest_str = print_sockaddr(addrstr, sizeof(addrstr), ss);
 
 	ret = tsocket_address_inet_from_strings(mem_ctx, "ip",
 						dest_str, LDAP_PORT,
@@ -113,7 +107,7 @@ failed:
 *******************************************************************/
 
 bool ads_cldap_netlogon_5(TALLOC_CTX *mem_ctx,
-			  const char *server,
+			  struct sockaddr_storage *ss,
 			  const char *realm,
 			  struct NETLOGON_SAM_LOGON_RESPONSE_EX *reply5)
 {
@@ -121,7 +115,7 @@ bool ads_cldap_netlogon_5(TALLOC_CTX *mem_ctx,
 	struct netlogon_samlogon_response *reply = NULL;
 	bool ret;
 
-	ret = ads_cldap_netlogon(mem_ctx, server, realm, nt_version, &reply);
+	ret = ads_cldap_netlogon(mem_ctx, ss, realm, nt_version, &reply);
 	if (!ret) {
 		return false;
 	}
diff --git a/source3/libads/cldap.h b/source3/libads/cldap.h
index e2d05ce7313..60e1c56038d 100644
--- a/source3/libads/cldap.h
+++ b/source3/libads/cldap.h
@@ -1,12 +1,39 @@
+/*
+   Samba Unix/Linux SMB client library
+   net ads cldap functions
+   Copyright (C) 2001 Andrew Tridgell (tridge@samba.org)
+   Copyright (C) 2003 Jim McDonough (jmcd@us.ibm.com)
+   Copyright (C) 2008 Guenther Deschner (gd@samba.org)
+   Copyright (C) 2009 Stefan Metzmacher (metze@samba.org)
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBADS_CLDAP_H_
+#define _LIBADS_CLDAP_H_
+
 #include "../libcli/netlogon/netlogon.h"
 
 /* The following definitions come from libads/cldap.c  */
 bool ads_cldap_netlogon(TALLOC_CTX *mem_ctx,
-			const char *server,
+			struct sockaddr_storage *ss,
 			const char *realm,
 			uint32_t nt_version,
 			struct netlogon_samlogon_response **reply);
 bool ads_cldap_netlogon_5(TALLOC_CTX *mem_ctx,
-			  const char *server,
+			  struct sockaddr_storage *ss,
 			  const char *realm,
 			  struct NETLOGON_SAM_LOGON_RESPONSE_EX *reply5);
+
+#endif /* _LIBADS_CLDAP_H_ */
diff --git a/source3/libads/dns.c b/source3/libads/dns.c
index 113e63710bd..5eae10ec285 100644
--- a/source3/libads/dns.c
+++ b/source3/libads/dns.c
@@ -328,7 +328,7 @@ static NTSTATUS dns_send_req( TALLOC_CTX *ctx, const char *name, int q_type,
 		buf_len = resp_len * sizeof(uint8);
 
 		if (buf_len) {
-			if ((buffer = TALLOC_ARRAY(ctx, uint8, buf_len))
+			if ((buffer = talloc_array(ctx, uint8, buf_len))
 					== NULL ) {
 				DEBUG(0,("ads_dns_lookup_srv: "
 					"talloc() failed!\n"));
@@ -401,11 +401,19 @@ static NTSTATUS ads_dns_lookup_srv( TALLOC_CTX *ctx,
 	int rrnum;
 	int idx = 0;
 	NTSTATUS status;
+	const char *dns_hosts_file;
 
 	if ( !ctx || !name || !dclist ) {
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
+	dns_hosts_file = lp_parm_const_string(-1, "resolv", "host file", NULL);
+	if (dns_hosts_file) {
+		return resolve_dns_hosts_file_as_dns_rr(dns_hosts_file,
+							name, true, ctx,
+							dclist, numdcs);
+	}
+
 	/* Send the request.  May have to loop several times in case
 	   of large replies */
 
@@ -434,7 +442,7 @@ static NTSTATUS ads_dns_lookup_srv( TALLOC_CTX *ctx,
 		answer_count));
 
 	if (answer_count) {
-		if ((dcs = TALLOC_ZERO_ARRAY(ctx, struct dns_rr_srv,
+		if ((dcs = talloc_zero_array(ctx, struct dns_rr_srv,
 						answer_count)) == NULL ) {
 			DEBUG(0,("ads_dns_lookup_srv: "
 				"talloc() failure for %d char*'s\n",
@@ -526,13 +534,13 @@ static NTSTATUS ads_dns_lookup_srv( TALLOC_CTX *ctx,
 				/* allocate new memory */
 
 				if (dcs[i].num_ips == 0) {
-					if ((dcs[i].ss_s = TALLOC_ARRAY(dcs,
+					if ((dcs[i].ss_s = talloc_array(dcs,
 						struct sockaddr_storage, 1 ))
 							== NULL ) {
 						return NT_STATUS_NO_MEMORY;
 					}
 				} else {
-					if ((tmp_ss_s = TALLOC_REALLOC_ARRAY(dcs,
+					if ((tmp_ss_s = talloc_realloc(dcs,
 							dcs[i].ss_s,
 							struct sockaddr_storage,
 							dcs[i].num_ips+1))
@@ -590,11 +598,18 @@ NTSTATUS ads_dns_lookup_ns(TALLOC_CTX *ctx,
 	int rrnum;
 	int idx = 0;
 	NTSTATUS status;
+	const char *dns_hosts_file;
 
 	if ( !ctx || !dnsdomain || !nslist ) {
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
+	dns_hosts_file = lp_parm_const_string(-1, "resolv", "host file", NULL);
+	if (dns_hosts_file) {
+		DEBUG(1, ("NO 'NS' lookup available when using resolv:host file"));
+		return NT_STATUS_OBJECT_NAME_NOT_FOUND;
+	}
+
 	/* Send the request.  May have to loop several times in case
 	   of large replies */
 
@@ -623,7 +638,7 @@ NTSTATUS ads_dns_lookup_ns(TALLOC_CTX *ctx,
 		answer_count));
 
 	if (answer_count) {
-		if ((nsarray = TALLOC_ARRAY(ctx, struct dns_rr_ns,
+		if ((nsarray = talloc_array(ctx, struct dns_rr_ns,
 						answer_count)) == NULL ) {
 			DEBUG(0,("ads_dns_lookup_ns: "
 				"talloc() failure for %d char*'s\n",
diff --git a/source3/libads/dns.h b/source3/libads/dns.h
index b747e97bb77..f53153ba263 100644
--- a/source3/libads/dns.h
+++ b/source3/libads/dns.h
@@ -20,42 +20,7 @@
 #ifndef _ADS_DNS_H
 #define _ADS_DNS_H
 
-/* DNS query section in replies */
-
-struct dns_query {
-	const char *hostname;
-	uint16 type;
-	uint16 in_class;
-};
-
-/* DNS RR record in reply */
-
-struct dns_rr {
-	const char *hostname;
-	uint16 type;
-	uint16 in_class;
-	uint32 ttl;
-	uint16 rdatalen;
-	uint8 *rdata;
-};
-
-/* SRV records */
-
-struct dns_rr_srv {
-	const char *hostname;
-	uint16 priority;
-	uint16 weight;
-	uint16 port;
-	size_t num_ips;
-	struct sockaddr_storage *ss_s;	/* support multi-homed hosts */
-};
-
-/* NS records */
-
-struct dns_rr_ns {
-	const char *hostname;
-	struct sockaddr_storage ss;
-};
+#include "libcli/dns/dns.h"
 
 /* The following definitions come from libads/dns.c  */
 
diff --git a/source3/libads/kerberos.c b/source3/libads/kerberos.c
index d496ade71b0..913d6900493 100644
--- a/source3/libads/kerberos.c
+++ b/source3/libads/kerberos.c
@@ -223,8 +223,8 @@ int kerberos_kinit_password_ext(const char *principal,
 		krb5_get_init_creds_opt_set_address_list(opt, addr->addrs);
 	}
 
-	if ((code = krb5_get_init_creds_password(ctx, &my_creds, me, CONST_DISCARD(char *,password), 
-						 kerb_prompter, CONST_DISCARD(char *,password),
+	if ((code = krb5_get_init_creds_password(ctx, &my_creds, me, discard_const_p(char,password), 
+						 kerb_prompter, discard_const_p(char, password),
 						 0, NULL, opt))) {
 		goto out;
 	}
@@ -352,7 +352,7 @@ char* kerberos_standard_des_salt( void )
 {
 	fstring salt;
 
-	fstr_sprintf( salt, "host/%s.%s@", global_myname(), lp_realm() );
+	fstr_sprintf( salt, "host/%s.%s@", lp_netbios_name(), lp_realm() );
 	strlower_m( salt );
 	fstrcat( salt, lp_realm() );
 
@@ -958,22 +958,37 @@ bool create_local_private_krb5_conf_for_domain(const char *realm,
 	/* Insanity, sheer insanity..... */
 
 	if (strequal(realm, lp_realm())) {
-		char linkpath[PATH_MAX+1];
-		int lret;
-
-		lret = readlink(SYSTEM_KRB5_CONF_PATH, linkpath, sizeof(linkpath)-1);
-		if (lret != -1) {
-			linkpath[lret] = '\0';
-		}
-
-		if (lret != -1 || strcmp(linkpath, fname) == 0) {
-			/* Symlink already exists. */
-			goto done;
+		SMB_STRUCT_STAT sbuf;
+
+		if (sys_lstat(SYSTEM_KRB5_CONF_PATH, &sbuf, false) == 0) {
+			if (S_ISLNK(sbuf.st_ex_mode) && sbuf.st_ex_size) {
+				int lret;
+				size_t alloc_size = sbuf.st_ex_size + 1;
+				char *linkpath = talloc_array(talloc_tos(), char,
+						alloc_size);
+				if (!linkpath) {
+					goto done;
+				}
+				lret = readlink(SYSTEM_KRB5_CONF_PATH, linkpath,
+						alloc_size - 1);
+				if (lret == -1) {
+					TALLOC_FREE(linkpath);
+					goto done;
+				}
+				linkpath[lret] = '\0';
+
+				if (strcmp(linkpath, fname) == 0) {
+					/* Symlink already exists. */
+					TALLOC_FREE(linkpath);
+					goto done;
+				}
+				TALLOC_FREE(linkpath);
+			}
 		}
 
 		/* Try and replace with a symlink. */
 		if (symlink(fname, SYSTEM_KRB5_CONF_PATH) == -1) {
-			const char *newpath = SYSTEM_KRB5_CONF_PATH ## ".saved";
+			const char *newpath = SYSTEM_KRB5_CONF_PATH ".saved";
 			if (errno != EEXIST) {
 				DEBUG(0,("create_local_private_krb5_conf_for_domain: symlink "
 					"of %s to %s failed. Errno %s\n",
diff --git a/source3/libads/kerberos_keytab.c b/source3/libads/kerberos_keytab.c
index 6e1b7c08fe7..7654c666f4f 100644
--- a/source3/libads/kerberos_keytab.c
+++ b/source3/libads/kerberos_keytab.c
@@ -316,7 +316,7 @@ int ads_keytab_add_entry(ADS_STRUCT *ads, const char *srvPrinc)
 		goto out;
 	}
 
-	my_fqdn = ads_get_dnshostname(ads, tmpctx, global_myname());
+	my_fqdn = ads_get_dnshostname(ads, tmpctx, lp_netbios_name());
 	if (!my_fqdn) {
 		DEBUG(0, (__location__ ": unable to determine machine "
 			  "account's dns name in AD!\n"));
@@ -324,7 +324,7 @@ int ads_keytab_add_entry(ADS_STRUCT *ads, const char *srvPrinc)
 		goto out;
 	}
 
-	machine_name = ads_get_samaccountname(ads, tmpctx, global_myname());
+	machine_name = ads_get_samaccountname(ads, tmpctx, lp_netbios_name());
 	if (!machine_name) {
 		DEBUG(0, (__location__ ": unable to determine machine "
 			  "account's short name in AD!\n"));
@@ -380,7 +380,7 @@ int ads_keytab_add_entry(ADS_STRUCT *ads, const char *srvPrinc)
 				  "'%s'\n", princ_s));
 
 			aderr = ads_add_service_principal_name(ads,
-					global_myname(), my_fqdn, srvPrinc);
+					lp_netbios_name(), my_fqdn, srvPrinc);
 			if (!ADS_ERR_OK(aderr)) {
 				DEBUG(1, (__location__ ": failed to "
 					 "ads_add_service_principal_name.\n"));
@@ -389,7 +389,7 @@ int ads_keytab_add_entry(ADS_STRUCT *ads, const char *srvPrinc)
 		}
 	}
 
-	kvno = (krb5_kvno)ads_get_machine_kvno(ads, global_myname());
+	kvno = (krb5_kvno)ads_get_machine_kvno(ads, lp_netbios_name());
 	if (kvno == -1) {
 		/* -1 indicates failure, everything else is OK */
 		DEBUG(1, (__location__ ": ads_get_machine_kvno failed to "
@@ -458,7 +458,7 @@ int ads_keytab_flush(ADS_STRUCT *ads)
 		goto out;
 	}
 
-	kvno = (krb5_kvno)ads_get_machine_kvno(ads, global_myname());
+	kvno = (krb5_kvno)ads_get_machine_kvno(ads, lp_netbios_name());
 	if (kvno == -1) {
 		/* -1 indicates a failure */
 		DEBUG(1, (__location__ ": Error determining the kvno.\n"));
@@ -472,7 +472,7 @@ int ads_keytab_flush(ADS_STRUCT *ads)
 		goto out;
 	}
 
-	aderr = ads_clear_service_principal_names(ads, global_myname());
+	aderr = ads_clear_service_principal_names(ads, lp_netbios_name());
 	if (!ADS_ERR_OK(aderr)) {
 		DEBUG(1, (__location__ ": Error while clearing service "
 			  "principal listings in LDAP.\n"));
@@ -547,7 +547,7 @@ int ads_keytab_create_default(ADS_STRUCT *ads)
 		goto done;
 	}
 
-	machine_name = talloc_strdup(tmpctx, global_myname());
+	machine_name = talloc_strdup(tmpctx, lp_netbios_name());
 	if (!machine_name) {
 		ret = -1;
 		goto done;
diff --git a/source3/libads/kerberos_proto.h b/source3/libads/kerberos_proto.h
index afd57fe220c..406669cc49f 100644
--- a/source3/libads/kerberos_proto.h
+++ b/source3/libads/kerberos_proto.h
@@ -1,5 +1,39 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  kerberos utility library
+ *
+ *  Copyright (C) Andrew Tridgell			2001
+ *  Copyright (C) Remus Koos (remuskoos@yahoo.com)	2001
+ *  Copyright (C) Luke Howard				2002-2003
+ *  Copyright (C) Jim McDonough <jmcd@us.ibm.com>	2003
+ *  Copyright (C) Guenther Deschner			2003-2008
+ *  Copyright (C) Andrew Bartlett <abartlet@samba.org>	2004-2005
+ *  Copyright (C) Jeremy Allison			2004,2007
+ *  Copyright (C) Stefan Metzmacher			2004-2005
+ *  Copyright (C) Nalin Dahyabhai <nalin@redhat.com>	2004
+ *  Copyright (C) Gerald Carter				2006
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LIBADS_KERBEROS_PROTO_H_
+#define _LIBADS_KERBEROS_PROTO_H_
+
 struct PAC_LOGON_INFO;
 
+#include "libads/ads_status.h"
+
 /* The following definitions come from libads/kerberos_verify.c  */
 
 NTSTATUS ads_verify_ticket(TALLOC_CTX *mem_ctx,
@@ -67,3 +101,5 @@ ADS_STATUS kerberos_set_password(const char *kpasswd_server,
 				 const char *auth_principal, const char *auth_password,
 				 const char *target_principal, const char *new_password,
 				 int time_offset);
+
+#endif /* _LIBADS_KERBEROS_PROTO_H_ */
diff --git a/source3/libads/kerberos_util.c b/source3/libads/kerberos_util.c
index 4762e1b9143..4935da8e79c 100644
--- a/source3/libads/kerberos_util.c
+++ b/source3/libads/kerberos_util.c
@@ -72,13 +72,13 @@ int ads_kinit_password(ADS_STRUCT *ads)
 		account_name = lp_workgroup();
 	} else {
 		/* always use the sAMAccountName for security = domain */
-		/* global_myname()$@REA.LM */
+		/* lp_netbios_name()$@REA.LM */
 		if ( lp_security() == SEC_DOMAIN ) {
-			fstr_sprintf( acct_name, "%s$", global_myname() );
+			fstr_sprintf( acct_name, "%s$", lp_netbios_name() );
 			account_name = acct_name;
 		}
 		else
-			/* This looks like host/global_myname()@REA.LM */
+			/* This looks like host/lp_netbios_name()@REA.LM */
 			account_name = ads->auth.user_name;
 	}
 
diff --git a/source3/libads/kerberos_verify.c b/source3/libads/kerberos_verify.c
index fab38dc4deb..f11ea884778 100644
--- a/source3/libads/kerberos_verify.c
+++ b/source3/libads/kerberos_verify.c
@@ -151,10 +151,10 @@ static bool ads_keytab_verify_ticket(krb5_context context,
 	 * clients might want to use for authenticating to the file
 	 * service.  We allow name$,{host,cifs}/{name,fqdn,name.REALM}. */
 
-	fstrcpy(my_name, global_myname());
+	fstrcpy(my_name, lp_netbios_name());
 
 	my_fqdn[0] = '\0';
-	name_to_fqdn(my_fqdn, global_myname());
+	name_to_fqdn(my_fqdn, lp_netbios_name());
 
 	err = asprintf(&valid_princ_formats[0],
 			"%s$@%s", my_name, lp_realm());
@@ -534,7 +534,7 @@ NTSTATUS ads_verify_ticket(TALLOC_CTX *mem_ctx,
 		krb5_auth_con_setflags( context, auth_context, flags );
 	}
 
-	if (asprintf(&host_princ_s, "%s$", global_myname()) == -1) {
+	if (asprintf(&host_princ_s, "%s$", lp_netbios_name()) == -1) {
 		goto out;
 	}
 
diff --git a/source3/libads/ldap.c b/source3/libads/ldap.c
index 90c645c44ad..128c1337600 100644
--- a/source3/libads/ldap.c
+++ b/source3/libads/ldap.c
@@ -59,14 +59,47 @@ static void gotalarm_sig(int signum)
 	gotalarm = 1;
 }
 
- LDAP *ldap_open_with_timeout(const char *server, int port, unsigned int to)
+ LDAP *ldap_open_with_timeout(const char *server,
+			      struct sockaddr_storage *ss,
+			      int port, unsigned int to)
 {
 	LDAP *ldp = NULL;
 
-
 	DEBUG(10, ("Opening connection to LDAP server '%s:%d', timeout "
 		   "%u seconds\n", server, port, to));
 
+#if defined(HAVE_LDAP_INIT_FD) && defined(SOCKET_WRAPPER)
+	/* Only use this private LDAP function if we are in make test,
+	 * as this is the best way to get the emulated TCP socket into
+	 * OpenLDAP */
+	if (socket_wrapper_dir() != NULL) {
+		int fd, ldap_err;
+		NTSTATUS status;
+		char *uri;
+
+		status = open_socket_out(ss, port, to, &fd);
+
+		if (!NT_STATUS_IS_OK(status)) {
+			return NULL;
+		}
+
+#ifndef LDAP_PROTO_TCP
+#define LDAP_PROTO_TCP 1
+#endif
+		uri = talloc_asprintf(talloc_tos(), "ldap://%s:%u", server, port);
+		if (uri == NULL) {
+			return NULL;
+		}
+		ldap_err = ldap_init_fd(fd, LDAP_PROTO_TCP, uri, &ldp);
+		talloc_free(uri);
+
+		if (ldap_err != LDAP_SUCCESS) {
+			return NULL;
+		}
+		return ldp;
+	}
+#endif
+
 	/* Setup timeout */
 	gotalarm = 0;
 	CatchSignal(SIGALRM, gotalarm_sig);
@@ -196,45 +229,32 @@ bool ads_closest_dc(ADS_STRUCT *ads)
  */
 static bool ads_try_connect(ADS_STRUCT *ads, const char *server, bool gc)
 {
-	char *srv;
 	struct NETLOGON_SAM_LOGON_RESPONSE_EX cldap_reply;
 	TALLOC_CTX *frame = talloc_stackframe();
 	bool ret = false;
+	struct sockaddr_storage ss;
+	char addr[INET6_ADDRSTRLEN];
 
 	if (!server || !*server) {
 		TALLOC_FREE(frame);
 		return False;
 	}
 
-	if (!is_ipaddress(server)) {
-		struct sockaddr_storage ss;
-		char addr[INET6_ADDRSTRLEN];
-
-		if (!resolve_name(server, &ss, 0x20, true)) {
-			DEBUG(5,("ads_try_connect: unable to resolve name %s\n",
-				server ));
-			TALLOC_FREE(frame);
-			return false;
-		}
-		print_sockaddr(addr, sizeof(addr), &ss);
-		srv = talloc_strdup(frame, addr);
-	} else {
-		/* this copes with inet_ntoa brokenness */
-		srv = talloc_strdup(frame, server);
-	}
-
-	if (!srv) {
+	if (!resolve_name(server, &ss, 0x20, true)) {
+		DEBUG(5,("ads_try_connect: unable to resolve name %s\n",
+			 server ));
 		TALLOC_FREE(frame);
 		return false;
 	}
+	print_sockaddr(addr, sizeof(addr), &ss);
 
 	DEBUG(5,("ads_try_connect: sending CLDAP request to %s (realm: %s)\n", 
-		srv, ads->server.realm));
+		addr, ads->server.realm));
 
 	ZERO_STRUCT( cldap_reply );
 
-	if ( !ads_cldap_netlogon_5(frame, srv, ads->server.realm, &cldap_reply ) ) {
-		DEBUG(3,("ads_try_connect: CLDAP request %s failed.\n", srv));
+	if ( !ads_cldap_netlogon_5(frame, &ss, ads->server.realm, &cldap_reply ) ) {
+		DEBUG(3,("ads_try_connect: CLDAP request %s failed.\n", addr));
 		ret = false;
 		goto out;
 	}
@@ -243,7 +263,7 @@ static bool ads_try_connect(ADS_STRUCT *ads, const char *server, bool gc)
 
 	if ( !(cldap_reply.server_type & NBT_SERVER_LDAP) ) {
 		DEBUG(1,("ads_try_connect: %s's CLDAP reply says it is not an LDAP server!\n",
-			srv));
+			addr));
 		ret = false;
 		goto out;
 	}
@@ -273,13 +293,7 @@ static bool ads_try_connect(ADS_STRUCT *ads, const char *server, bool gc)
 	ads->server.workgroup          = SMB_STRDUP(cldap_reply.domain_name);
 
 	ads->ldap.port = gc ? LDAP_GC_PORT : LDAP_PORT;
-	if (!interpret_string_addr(&ads->ldap.ss, srv, 0)) {
-		DEBUG(1,("ads_try_connect: unable to convert %s "
-			"to an address\n",
-			srv));
-		ret = false;
-		goto out;
-	}
+	ads->ldap.ss = ss;
 
 	/* Store our site name. */
 	sitename_store( cldap_reply.domain_name, cldap_reply.client_site);
@@ -515,7 +529,7 @@ ADS_STATUS ads_connect_gc(ADS_STRUCT *ads)
 	TALLOC_CTX *frame = talloc_stackframe();
 	struct dns_rr_srv *gcs_list;
 	int num_gcs;
-	char *realm = ads->server.realm;
+	const char *realm = ads->server.realm;
 	NTSTATUS nt_status = NT_STATUS_UNSUCCESSFUL;
 	ADS_STATUS ads_status = ADS_ERROR_NT(NT_STATUS_UNSUCCESSFUL);
 	int i;
@@ -643,7 +657,7 @@ got_connection:
 		/* Must use the userPrincipalName value here or sAMAccountName
 		   and not servicePrincipalName; found by Guenther Deschner */
 
-		if (asprintf(&ads->auth.user_name, "%s$", global_myname() ) == -1) {
+		if (asprintf(&ads->auth.user_name, "%s$", lp_netbios_name() ) == -1) {
 			DEBUG(0,("ads_connect: asprintf fail.\n"));
 			ads->auth.user_name = NULL;
 		}
@@ -658,18 +672,6 @@ got_connection:
 		ads->auth.kdc_server = SMB_STRDUP(addr);
 	}
 
-#if KRB5_DNS_HACK
-	/* this is a really nasty hack to avoid ADS DNS problems. It needs a patch
-	   to MIT kerberos to work (tridge) */
-	{
-		char *env = NULL;
-		if (asprintf(&env, "KRB5_KDC_ADDRESS_%s", ads->config.realm) > 0) {
-			setenv(env, ads->auth.kdc_server, 1);
-			free(env);
-		}
-	}
-#endif
-
 	/* If the caller() requested no LDAP bind, then we are done */
 
 	if (ads->auth.flags & ADS_AUTH_NO_BIND) {
@@ -686,6 +688,7 @@ got_connection:
 	/* Otherwise setup the TCP LDAP session */
 
 	ads->ldap.ld = ldap_open_with_timeout(ads->config.ldap_server_name,
+					      &ads->ldap.ss,
 					      ads->ldap.port, lp_ldap_timeout());
 	if (ads->ldap.ld == NULL) {
 		status = ADS_ERROR(LDAP_OPERATIONS_ERROR);
@@ -781,13 +784,13 @@ static struct berval *dup_berval(TALLOC_CTX *ctx, const struct berval *in_val)
 
 	if (!in_val) return NULL;
 
-	value = TALLOC_ZERO_P(ctx, struct berval);
+	value = talloc_zero(ctx, struct berval);
 	if (value == NULL)
 		return NULL;
 	if (in_val->bv_len == 0) return value;
 
 	value->bv_len = in_val->bv_len;
-	value->bv_val = (char *)TALLOC_MEMDUP(ctx, in_val->bv_val,
+	value->bv_val = (char *)talloc_memdup(ctx, in_val->bv_val,
 					      in_val->bv_len);
 	return value;
 }
@@ -804,7 +807,7 @@ static struct berval **ads_dup_values(TALLOC_CTX *ctx,
 	if (!in_vals) return NULL;
 	for (i=0; in_vals[i]; i++)
 		; /* count values */
-	values = TALLOC_ZERO_ARRAY(ctx, struct berval *, i+1);
+	values = talloc_zero_array(ctx, struct berval *, i+1);
 	if (!values) return NULL;
 
 	for (i=0; in_vals[i]; i++) {
@@ -825,7 +828,7 @@ static char **ads_push_strvals(TALLOC_CTX *ctx, const char **in_vals)
 	if (!in_vals) return NULL;
 	for (i=0; in_vals[i]; i++)
 		; /* count values */
-	values = TALLOC_ZERO_ARRAY(ctx, char *, i+1);
+	values = talloc_zero_array(ctx, char *, i+1);
 	if (!values) return NULL;
 
 	for (i=0; in_vals[i]; i++) {
@@ -849,7 +852,7 @@ static char **ads_pull_strvals(TALLOC_CTX *ctx, const char **in_vals)
 	if (!in_vals) return NULL;
 	for (i=0; in_vals[i]; i++)
 		; /* count values */
-	values = TALLOC_ZERO_ARRAY(ctx, char *, i+1);
+	values = talloc_zero_array(ctx, char *, i+1);
 	if (!values) return NULL;
 
 	for (i=0; in_vals[i]; i++) {
@@ -937,21 +940,21 @@ static ADS_STATUS ads_do_paged_search_args(ADS_STRUCT *ads,
 		ber_printf(cookie_be, "{io}", (ber_int_t) 1000, "", 0);
 	}
 	ber_flatten(cookie_be, &cookie_bv);
-	PagedResults.ldctl_oid = CONST_DISCARD(char *, ADS_PAGE_CTL_OID);
+	PagedResults.ldctl_oid = discard_const_p(char, ADS_PAGE_CTL_OID);
 	PagedResults.ldctl_iscritical = (char) 1;
 	PagedResults.ldctl_value.bv_len = cookie_bv->bv_len;
 	PagedResults.ldctl_value.bv_val = cookie_bv->bv_val;
 
-	NoReferrals.ldctl_oid = CONST_DISCARD(char *, ADS_NO_REFERRALS_OID);
+	NoReferrals.ldctl_oid = discard_const_p(char, ADS_NO_REFERRALS_OID);
 	NoReferrals.ldctl_iscritical = (char) 0;
 	NoReferrals.ldctl_value.bv_len = 0;
-	NoReferrals.ldctl_value.bv_val = CONST_DISCARD(char *, "");
+	NoReferrals.ldctl_value.bv_val = discard_const_p(char, "");
 
 	if (external_control && 
 	    (strequal(external_control->control, ADS_EXTENDED_DN_OID) || 
 	     strequal(external_control->control, ADS_SD_FLAGS_OID))) {
 
-		ExternalCtrl.ldctl_oid = CONST_DISCARD(char *, external_control->control);
+		ExternalCtrl.ldctl_oid = discard_const_p(char, external_control->control);
 		ExternalCtrl.ldctl_iscritical = (char) external_control->critical;
 
 		/* win2k does not accept a ldctl_value beeing passed in */
@@ -1382,7 +1385,7 @@ ADS_MODLIST ads_init_mods(TALLOC_CTX *ctx)
 #define ADS_MODLIST_ALLOC_SIZE 10
 	LDAPMod **mods;
 
-	if ((mods = TALLOC_ZERO_ARRAY(ctx, LDAPMod *, ADS_MODLIST_ALLOC_SIZE + 1)))
+	if ((mods = talloc_zero_array(ctx, LDAPMod *, ADS_MODLIST_ALLOC_SIZE + 1)))
 		/* -1 is safety to make sure we don't go over the end.
 		   need to reset it to NULL before doing ldap modify */
 		mods[ADS_MODLIST_ALLOC_SIZE] = (LDAPMod *) -1;
@@ -1419,7 +1422,7 @@ static ADS_STATUS ads_modlist_add(TALLOC_CTX *ctx, ADS_MODLIST *mods,
 	for (curmod=0; modlist[curmod] && modlist[curmod] != (LDAPMod *) -1;
 	     curmod++);
 	if (modlist[curmod] == (LDAPMod *) -1) {
-		if (!(modlist = TALLOC_REALLOC_ARRAY(ctx, modlist, LDAPMod *,
+		if (!(modlist = talloc_realloc(ctx, modlist, LDAPMod *,
 				curmod+ADS_MODLIST_ALLOC_SIZE+1)))
 			return ADS_ERROR(LDAP_NO_MEMORY);
 		memset(&modlist[curmod], 0, 
@@ -1428,7 +1431,7 @@ static ADS_STATUS ads_modlist_add(TALLOC_CTX *ctx, ADS_MODLIST *mods,
 		*mods = (ADS_MODLIST)modlist;
 	}
 
-	if (!(modlist[curmod] = TALLOC_ZERO_P(ctx, LDAPMod)))
+	if (!(modlist[curmod] = talloc_zero(ctx, LDAPMod)))
 		return ADS_ERROR(LDAP_NO_MEMORY);
 	modlist[curmod]->mod_type = talloc_strdup(ctx, name);
 	if (mod_op & LDAP_MOD_BVALUES) {
@@ -1521,7 +1524,7 @@ ADS_STATUS ads_gen_mod(ADS_STRUCT *ads, const char *mod_dn, ADS_MODLIST mods)
 	   non-existent attribute (but allowable for the object) to run
 	*/
 	LDAPControl PermitModify = {
-                CONST_DISCARD(char *, ADS_PERMIT_MODIFY_OID),
+                discard_const_p(char, ADS_PERMIT_MODIFY_OID),
 		{0, NULL},
 		(char) 1};
 	LDAPControl *controls[2];
@@ -2225,7 +2228,7 @@ static bool ads_dump_field(ADS_STRUCT *ads, char *field, void **values, void *da
 	}
 
 	for (i=0; handlers[i].name; i++) {
-		if (StrCaseCmp(handlers[i].name, field) == 0) {
+		if (strcasecmp_m(handlers[i].name, field) == 0) {
 			if (!values) /* first time, indicate string or not */
 				return handlers[i].string;
 			handlers[i].handler(ads, field, (struct berval **) values);
@@ -2430,7 +2433,7 @@ int ads_count_replies(ADS_STRUCT *ads, void *res)
 
 	*num_values = ldap_count_values(values);
 
-	ret = TALLOC_ARRAY(mem_ctx, char *, *num_values + 1);
+	ret = talloc_array(mem_ctx, char *, *num_values + 1);
 	if (!ret) {
 		ldap_value_free(values);
 		return NULL;
@@ -2543,7 +2546,7 @@ int ads_count_replies(ADS_STRUCT *ads, void *res)
 		return NULL;
 	}
 
-	strings = TALLOC_REALLOC_ARRAY(mem_ctx, current_strings, char *,
+	strings = talloc_realloc(mem_ctx, current_strings, char *,
 				 *num_strings + num_new_strings);
 
 	if (strings == NULL) {
@@ -2666,7 +2669,7 @@ int ads_count_replies(ADS_STRUCT *ads, void *res)
 		/* nop */ ;
 
 	if (i) {
-		(*sids) = TALLOC_ARRAY(mem_ctx, struct dom_sid, i);
+		(*sids) = talloc_array(mem_ctx, struct dom_sid, i);
 		if (!(*sids)) {
 			ldap_value_free_len(values);
 			return 0;
@@ -2854,7 +2857,7 @@ ADS_STATUS ads_current_time(ADS_STRUCT *ads)
 
 	if (ads->config.current_time != 0) {
 		ads->auth.time_offset = ads->config.current_time - time(NULL);
-		DEBUG(4,("time offset is %d seconds\n", ads->auth.time_offset));
+		DEBUG(4,("KDC time offset is %d seconds\n", ads->auth.time_offset));
 	}
 
 	ads_msgfree(ads, res);
@@ -3284,7 +3287,7 @@ ADS_STATUS ads_get_sid_from_extended_dn(TALLOC_CTX *mem_ctx,
 		return 0;
 	}
 
-	(*sids) = TALLOC_ZERO_ARRAY(mem_ctx, struct dom_sid, dn_count + 1);
+	(*sids) = talloc_zero_array(mem_ctx, struct dom_sid, dn_count + 1);
 	if (!(*sids)) {
 		TALLOC_FREE(dn_strings);
 		return 0;
@@ -3322,10 +3325,10 @@ char* ads_get_dnshostname( ADS_STRUCT *ads, TALLOC_CTX *ctx, const char *machine
 	int count = 0;
 	char *name = NULL;
 
-	status = ads_find_machine_acct(ads, &res, global_myname());
+	status = ads_find_machine_acct(ads, &res, lp_netbios_name());
 	if (!ADS_ERR_OK(status)) {
 		DEBUG(0,("ads_get_dnshostname: Failed to find account for %s\n",
-			global_myname()));
+			lp_netbios_name()));
 		goto out;
 	}
 
@@ -3357,7 +3360,7 @@ char* ads_get_upn( ADS_STRUCT *ads, TALLOC_CTX *ctx, const char *machine_name )
 	status = ads_find_machine_acct(ads, &res, machine_name);
 	if (!ADS_ERR_OK(status)) {
 		DEBUG(0,("ads_get_upn: Failed to find account for %s\n",
-			global_myname()));
+			lp_netbios_name()));
 		goto out;
 	}
 
@@ -3386,10 +3389,10 @@ char* ads_get_samaccountname( ADS_STRUCT *ads, TALLOC_CTX *ctx, const char *mach
 	int count = 0;
 	char *name = NULL;
 
-	status = ads_find_machine_acct(ads, &res, global_myname());
+	status = ads_find_machine_acct(ads, &res, lp_netbios_name());
 	if (!ADS_ERR_OK(status)) {
 		DEBUG(0,("ads_get_dnshostname: Failed to find account for %s\n",
-			global_myname()));
+			lp_netbios_name()));
 		goto out;
 	}
 
@@ -3482,7 +3485,7 @@ ADS_STATUS ads_leave_realm(ADS_STRUCT *ads, const char *hostname)
 
 	pldap_control[0] = &ldap_control;
 	memset(&ldap_control, 0, sizeof(LDAPControl));
-	ldap_control.ldctl_oid = (char *)LDAP_SERVER_TREE_DELETE_OID;
+	ldap_control.ldctl_oid = discard_const_p(char, LDAP_SERVER_TREE_DELETE_OID);
 
 	/* hostname must be lowercase */
 	host = SMB_STRDUP(hostname);
diff --git a/source3/libads/ldap_printer.c b/source3/libads/ldap_printer.c
index 235ef7f9547..8ff9f9bdc9d 100644
--- a/source3/libads/ldap_printer.c
+++ b/source3/libads/ldap_printer.c
@@ -22,9 +22,7 @@
 #include "rpc_client/rpc_client.h"
 #include "../librpc/gen_ndr/ndr_spoolss_c.h"
 #include "rpc_client/cli_spoolss.h"
-#include "registry.h"
 #include "registry/reg_objects.h"
-#include "nt_printing.h"
 
 #ifdef HAVE_ADS
 
@@ -201,7 +199,7 @@ static bool map_multi_sz(TALLOC_CTX *ctx, ADS_MODLIST *mods,
 	};
 
 	if (num_vals) {
-		str_values = TALLOC_ARRAY(ctx, char *, num_vals + 1);
+		str_values = talloc_array(ctx, char *, num_vals + 1);
 		if (!str_values) {
 			return False;
 		}
@@ -291,7 +289,7 @@ static void map_regval_to_ads(TALLOC_CTX *ctx, ADS_MODLIST *mods,
 	int i;
 
 	for (i=0; map[i].valname; i++) {
-		if (StrCaseCmp(map[i].valname, regval_name(value)) == 0) {
+		if (strcasecmp_m(map[i].valname, regval_name(value)) == 0) {
 			if (!map[i].fn(ctx, mods, value)) {
 				DEBUG(5, ("Add of value %s to modlist failed\n", regval_name(value)));
 			} else {
diff --git a/source3/libads/ldap_schema.c b/source3/libads/ldap_schema.c
index 51aac14906c..7368be830fb 100644
--- a/source3/libads/ldap_schema.c
+++ b/source3/libads/ldap_schema.c
@@ -76,11 +76,11 @@ static ADS_STATUS ads_get_attrnames_by_oids(ADS_STRUCT *ads,
 		goto out;
 	}
 
-	if (((*names) = TALLOC_ARRAY(mem_ctx, char *, *count)) == NULL) {
+	if (((*names) = talloc_array(mem_ctx, char *, *count)) == NULL) {
 		status = ADS_ERROR(LDAP_NO_MEMORY);
 		goto out;
 	}
-	if (((*OIDs_out) = TALLOC_ARRAY(mem_ctx, char *, *count)) == NULL) {
+	if (((*OIDs_out) = talloc_array(mem_ctx, char *, *count)) == NULL) {
 		status = ADS_ERROR(LDAP_NO_MEMORY);
 		goto out;
 	}
@@ -252,7 +252,7 @@ ADS_STATUS ads_check_posix_schema_mapping(TALLOC_CTX *mem_ctx,
 		return ADS_ERROR(LDAP_NO_MEMORY);
 	}
 
-	if ( (schema = TALLOC_P(mem_ctx, struct posix_schema)) == NULL ) {
+	if ( (schema = talloc(mem_ctx, struct posix_schema)) == NULL ) {
 		TALLOC_FREE( ctx );
 		return ADS_ERROR(LDAP_NO_MEMORY);
 	}
diff --git a/source3/libads/ldap_schema.h b/source3/libads/ldap_schema.h
index 44964dd1c15..fc4ed078085 100644
--- a/source3/libads/ldap_schema.h
+++ b/source3/libads/ldap_schema.h
@@ -1,3 +1,26 @@
+/*
+   Unix SMB/CIFS implementation.
+   ads (active directory) utility library
+   Copyright (C) Guenther Deschner 2005-2007
+   Copyright (C) Gerald (Jerry) Carter 2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBADS_LDAP_SCHEMA_H_
+#define _LIBADS_LDAP_SCHEMA_H_
+
 /* used to remember the names of the posix attributes in AD */
 /* see the rfc2307 & sfu nss backends */
 
@@ -55,3 +78,5 @@ ADS_STATUS ads_check_posix_schema_mapping(TALLOC_CTX *mem_ctx,
 					  ADS_STRUCT *ads,
 					  enum wb_posix_mapping map_type,
 					  struct posix_schema **s ) ;
+
+#endif /* _LIBADS_LDAP_SCHEMA_H_ */
diff --git a/source3/libads/ldap_utils.c b/source3/libads/ldap_utils.c
index ab77abdd16b..2f1c1b8e0a5 100644
--- a/source3/libads/ldap_utils.c
+++ b/source3/libads/ldap_utils.c
@@ -242,7 +242,7 @@ ADS_STATUS ads_ranged_search(ADS_STRUCT *ads,
 	*num_strings = 0;
 	*strings = NULL;
 
-	attrs = TALLOC_ARRAY(mem_ctx, const char *, 3);
+	attrs = talloc_array(mem_ctx, const char *, 3);
 	ADS_ERROR_HAVE_NO_MEMORY(attrs);
 
 	attrs[0] = talloc_strdup(mem_ctx, range_attr);
diff --git a/source3/libads/ndr.c b/source3/libads/ndr.c
index c6b11213961..c667aa36376 100644
--- a/source3/libads/ndr.c
+++ b/source3/libads/ndr.c
@@ -22,7 +22,7 @@
 #include "includes.h"
 #include "ads.h"
 #include "../librpc/gen_ndr/ndr_netlogon.h"
-#include "librpc/ndr/util.h"
+#include "../librpc/ndr/libndr.h"
 
 static void ndr_print_ads_auth_flags(struct ndr_print *ndr, const char *name, uint32_t r)
 {
diff --git a/source3/libads/sasl.c b/source3/libads/sasl.c
index e7daa8aec63..45cf569c791 100644
--- a/source3/libads/sasl.c
+++ b/source3/libads/sasl.c
@@ -141,7 +141,7 @@ static ADS_STATUS ads_sasl_spnego_ntlmssp_bind(ADS_STRUCT *ads)
 	struct ntlmssp_state *ntlmssp_state;
 
 	nt_status = ntlmssp_client_start(NULL,
-					 global_myname(),
+					 lp_netbios_name(),
 					 lp_workgroup(),
 					 lp_client_ntlmv2_auth(),
 					 &ntlmssp_state);
@@ -388,7 +388,7 @@ static ADS_STATUS ads_sasl_spnego_gsskrb5_bind(ADS_STRUCT *ads, const gss_name_t
 	uint32 minor_status;
 	int gss_rc, rc;
 	gss_OID_desc krb5_mech_type =
-	{9, CONST_DISCARD(char *, "\x2a\x86\x48\x86\xf7\x12\x01\x02\x02") };
+	{9, discard_const_p(char, "\x2a\x86\x48\x86\xf7\x12\x01\x02\x02") };
 	gss_OID mech_type = &krb5_mech_type;
 	gss_OID actual_mech_type = GSS_C_NULL_OID;
 	const char *spnego_mechs[] = {OID_KERBEROS5_OLD, OID_KERBEROS5, OID_NTLMSSP, NULL};
@@ -722,7 +722,7 @@ static ADS_STATUS ads_generate_service_principal(ADS_STRUCT *ads,
 	gss_buffer_desc input_name;
 	/* GSS_KRB5_NT_PRINCIPAL_NAME */
 	gss_OID_desc nt_principal =
-	{10, CONST_DISCARD(char *, "\x2a\x86\x48\x86\xf7\x12\x01\x02\x02\x01")};
+	{10, discard_const_p(char, "\x2a\x86\x48\x86\xf7\x12\x01\x02\x02\x01")};
 	uint32 minor_status;
 	int gss_rc;
 #endif
diff --git a/source3/libgpo/gpext/registry.c b/source3/libgpo/gpext/registry.c
index eb49b19b203..b0ec7b88d90 100644
--- a/source3/libgpo/gpext/registry.c
+++ b/source3/libgpo/gpext/registry.c
@@ -111,14 +111,14 @@ static bool gp_reg_entry_from_file_entry(TALLOC_CTX *mem_ctx,
 
 	ZERO_STRUCTP(*reg_entry);
 
-	data = TALLOC_ZERO_P(mem_ctx, struct registry_value);
+	data = talloc_zero(mem_ctx, struct registry_value);
 	if (!data)
 		return false;
 
 	data->type = r->type;
 	data->data = data_blob_talloc(data, r->data, r->size);
 
-	entry = TALLOC_ZERO_P(mem_ctx, struct gp_registry_entry);
+	entry = talloc_zero(mem_ctx, struct gp_registry_entry);
 	if (!entry)
 		return false;
 
@@ -327,7 +327,7 @@ static NTSTATUS registry_get_reg_config(TALLOC_CTX *mem_ctx,
 		{ NULL, REG_NONE, NULL }
 	};
 
-	info = TALLOC_ZERO_P(mem_ctx, struct gp_extension_reg_info);
+	info = talloc_zero(mem_ctx, struct gp_extension_reg_info);
 	NT_STATUS_HAVE_NO_MEMORY(info);
 
 	status = gp_ext_info_add_entry(mem_ctx, GP_EXT_NAME,
diff --git a/source3/libgpo/gpext/scripts.c b/source3/libgpo/gpext/scripts.c
index 3e977378ee4..0b6748507a7 100644
--- a/source3/libgpo/gpext/scripts.c
+++ b/source3/libgpo/gpext/scripts.c
@@ -62,7 +62,7 @@ static NTSTATUS scripts_get_reg_config(TALLOC_CTX *mem_ctx,
 		{ NULL, REG_NONE, NULL },
 	};
 
-	info = TALLOC_ZERO_P(mem_ctx, struct gp_extension_reg_info);
+	info = talloc_zero(mem_ctx, struct gp_extension_reg_info);
 	NT_STATUS_HAVE_NO_MEMORY(info);
 
 	status = gp_ext_info_add_entry(mem_ctx, GP_EXT_NAME,
@@ -89,10 +89,10 @@ static NTSTATUS generate_gp_registry_entry(TALLOC_CTX *mem_ctx,
 	struct gp_registry_entry *entry = NULL;
 	struct registry_value *data = NULL;
 
-	entry = TALLOC_ZERO_P(mem_ctx, struct gp_registry_entry);
+	entry = talloc_zero(mem_ctx, struct gp_registry_entry);
 	NT_STATUS_HAVE_NO_MEMORY(entry);
 
-	data = TALLOC_ZERO_P(mem_ctx, struct registry_value);
+	data = talloc_zero(mem_ctx, struct registry_value);
 	NT_STATUS_HAVE_NO_MEMORY(data);
 
 	data->type = data_type;
diff --git a/source3/libgpo/gpext/security.c b/source3/libgpo/gpext/security.c
index dea0de33f93..b68840d2f6a 100644
--- a/source3/libgpo/gpext/security.c
+++ b/source3/libgpo/gpext/security.c
@@ -202,7 +202,7 @@ static NTSTATUS security_get_reg_config(TALLOC_CTX *mem_ctx,
 		{ NULL, REG_NONE, NULL }
 	};
 
-	info = TALLOC_ZERO_P(mem_ctx, struct gp_extension_reg_info);
+	info = talloc_zero(mem_ctx, struct gp_extension_reg_info);
 	NT_STATUS_HAVE_NO_MEMORY(info);
 
 	status = gp_ext_info_add_entry(mem_ctx, GP_EXT_NAME,
diff --git a/source3/libgpo/gpext/wscript_build b/source3/libgpo/gpext/wscript_build
index bfdb4d0bf6b..3c835dd8cd0 100644
--- a/source3/libgpo/gpext/wscript_build
+++ b/source3/libgpo/gpext/wscript_build
@@ -8,6 +8,7 @@ GPEXT_SRC = '''../../../libgpo/gpext/gpext.c'''
 
 bld.SAMBA3_SUBSYSTEM('gpext',
                     source=GPEXT_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_MODULE('gpext_registry',
diff --git a/source3/libgpo/gpo_filesync.c b/source3/libgpo/gpo_filesync.c
index be964a201e4..91e1c2725ac 100644
--- a/source3/libgpo/gpo_filesync.c
+++ b/source3/libgpo/gpo_filesync.c
@@ -19,6 +19,7 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "libsmb/libsmb.h"
 #include "../libgpo/gpo.h"
 #include "libgpo/gpo_proto.h"
 
@@ -150,7 +151,7 @@ static NTSTATUS gpo_sync_func(const char *mnt,
 	DEBUG(5,("gpo_sync_func: got mask: [%s], name: [%s]\n",
 		mask, info->name));
 
-	if (info->mode & aDIR) {
+	if (info->mode & FILE_ATTRIBUTE_DIRECTORY) {
 
 		DEBUG(3,("got dir: [%s]\n", info->name));
 
@@ -226,9 +227,9 @@ NTSTATUS gpo_sync_directories(TALLOC_CTX *mem_ctx,
 
 	ctx.mem_ctx 	= mem_ctx;
 	ctx.cli 	= cli;
-	ctx.remote_path	= CONST_DISCARD(char *, nt_path);
-	ctx.local_path	= CONST_DISCARD(char *, local_path);
-	ctx.attribute 	= (aSYSTEM | aHIDDEN | aDIR);
+	ctx.remote_path	= discard_const_p(char, nt_path);
+	ctx.local_path	= discard_const_p(char, local_path);
+	ctx.attribute 	= (FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY);
 
 	ctx.mask = talloc_asprintf(mem_ctx,
 				"%s\\*",
diff --git a/source3/libgpo/gpo_proto.h b/source3/libgpo/gpo_proto.h
index 8bc5874c9dd..fa2f87338d2 100644
--- a/source3/libgpo/gpo_proto.h
+++ b/source3/libgpo/gpo_proto.h
@@ -1,3 +1,25 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Group Policy Object Support
+ *
+ *  Copyright (C) Guenther Deschner 2006-2008
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LIBGPO_GPO_PROTO_H_
+#define _LIBGPO_GPO_PROTO_H_
 
 /* The following definitions come from libgpo/gpo_filesync.c  */
 
@@ -75,3 +97,5 @@ WERROR reg_apply_registry_entry(TALLOC_CTX *mem_ctx,
 				struct gp_registry_entry *entry,
 				const struct security_token *token,
 				uint32_t flags);
+
+#endif /* _LIBGPO_GPO_PROTO_H_ */
diff --git a/source3/libgpo/gpo_reg.c b/source3/libgpo/gpo_reg.c
index d9fcd43cafc..5142e919cc9 100644
--- a/source3/libgpo/gpo_reg.c
+++ b/source3/libgpo/gpo_reg.c
@@ -36,7 +36,7 @@ struct security_token *registry_create_system_token(TALLOC_CTX *mem_ctx)
 {
 	struct security_token *token = NULL;
 
-	token = TALLOC_ZERO_P(mem_ctx, struct security_token);
+	token = talloc_zero(mem_ctx, struct security_token);
 	if (!token) {
 		DEBUG(1,("talloc failed\n"));
 		return NULL;
@@ -74,7 +74,7 @@ WERROR gp_init_reg_ctx(TALLOC_CTX *mem_ctx,
 		return werr;
 	}
 
-	tmp_ctx = TALLOC_ZERO_P(mem_ctx, struct gp_registry_context);
+	tmp_ctx = talloc_zero(mem_ctx, struct gp_registry_context);
 	W_ERROR_HAVE_NO_MEMORY(tmp_ctx);
 
 	if (token) {
@@ -395,7 +395,7 @@ static WERROR gp_reg_read_groupmembership(TALLOC_CTX *mem_ctx,
 	int num_token_sids = 0;
 	struct security_token *tmp_token = NULL;
 
-	tmp_token = TALLOC_ZERO_P(mem_ctx, struct security_token);
+	tmp_token = talloc_zero(mem_ctx, struct security_token);
 	W_ERROR_HAVE_NO_MEMORY(tmp_token);
 
 	path = gp_reg_groupmembership_path(mem_ctx, object_sid, flags);
@@ -602,7 +602,7 @@ static WERROR gp_read_reg_gpo(TALLOC_CTX *mem_ctx,
 		return WERR_INVALID_PARAM;
 	}
 
-	gpo = TALLOC_ZERO_P(mem_ctx, struct GROUP_POLICY_OBJECT);
+	gpo = talloc_zero(mem_ctx, struct GROUP_POLICY_OBJECT);
 	W_ERROR_HAVE_NO_MEMORY(gpo);
 
 	werr = gp_read_reg_gpovals(mem_ctx, key, gpo);
@@ -901,7 +901,7 @@ bool add_gp_registry_entry_to_array(TALLOC_CTX *mem_ctx,
 				    struct gp_registry_entry **entries,
 				    size_t *num)
 {
-	*entries = TALLOC_REALLOC_ARRAY(mem_ctx, *entries,
+	*entries = talloc_realloc(mem_ctx, *entries,
 					struct gp_registry_entry,
 					(*num)+1);
 
diff --git a/source3/libnet/libnet_dssync.c b/source3/libnet/libnet_dssync.c
index 47ceff15092..c768226e8ba 100644
--- a/source3/libnet/libnet_dssync.c
+++ b/source3/libnet/libnet_dssync.c
@@ -54,7 +54,7 @@ NTSTATUS libnet_dssync_init_context(TALLOC_CTX *mem_ctx,
 {
 	struct dssync_context *ctx;
 
-	ctx = TALLOC_ZERO_P(mem_ctx, struct dssync_context);
+	ctx = talloc_zero(mem_ctx, struct dssync_context);
 	NT_STATUS_HAVE_NO_MEMORY(ctx);
 
 	talloc_set_destructor(ctx, libnet_dssync_free_context);
@@ -339,7 +339,7 @@ static NTSTATUS libnet_dssync_build_request(TALLOC_CTX *mem_ctx,
 		level = 5;
 	}
 
-	nc = TALLOC_ZERO_P(mem_ctx, struct drsuapi_DsReplicaObjectIdentifier);
+	nc = talloc_zero(mem_ctx, struct drsuapi_DsReplicaObjectIdentifier);
 	if (!nc) {
 		status = NT_STATUS_NO_MEMORY;
 		goto fail;
@@ -351,7 +351,7 @@ static NTSTATUS libnet_dssync_build_request(TALLOC_CTX *mem_ctx,
 	if (!ctx->single_object_replication &&
 	    !ctx->force_full_replication && utdv)
 	{
-		cursors = TALLOC_ZERO_P(mem_ctx,
+		cursors = talloc_zero(mem_ctx,
 					 struct drsuapi_DsReplicaCursorCtrEx);
 		if (!cursors) {
 			status = NT_STATUS_NO_MEMORY;
@@ -442,7 +442,7 @@ static NTSTATUS libnet_dssync_getncchanges(TALLOC_CTX *mem_ctx,
 	struct dcerpc_binding_handle *b = ctx->cli->binding_handle;
 
 	if (!ctx->single_object_replication) {
-		new_utdv = TALLOC_ZERO_P(mem_ctx, struct replUpToDateVectorBlob);
+		new_utdv = talloc_zero(mem_ctx, struct replUpToDateVectorBlob);
 		if (!new_utdv) {
 			status = NT_STATUS_NO_MEMORY;
 			goto out;
diff --git a/source3/libnet/libnet_dssync_keytab.c b/source3/libnet/libnet_dssync_keytab.c
index 65085cd96eb..763f8baf4d6 100644
--- a/source3/libnet/libnet_dssync_keytab.c
+++ b/source3/libnet/libnet_dssync_keytab.c
@@ -200,7 +200,7 @@ static  NTSTATUS parse_supplemental_credentials(TALLOC_CTX *mem_ctx,
 		goto done;
 	}
 
-	pkb = TALLOC_ZERO_P(mem_ctx, struct package_PrimaryKerberosBlob);
+	pkb = talloc_zero(mem_ctx, struct package_PrimaryKerberosBlob);
 	if (!pkb) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -276,7 +276,7 @@ static NTSTATUS parse_object(TALLOC_CTX *mem_ctx,
 		if (attr->attid == DRSUAPI_ATTID_servicePrincipalName) {
 			uint32_t count;
 			num_spns = attr->value_ctr.num_values;
-			spn = TALLOC_ARRAY(mem_ctx, char *, num_spns);
+			spn = talloc_array(mem_ctx, char *, num_spns);
 			for (count = 0; count < num_spns; count++) {
 				blob = attr->value_ctr.values[count].blob;
 				pull_string_talloc(spn, NULL, 0,
@@ -387,7 +387,7 @@ static NTSTATUS parse_object(TALLOC_CTX *mem_ctx,
 		entry = libnet_keytab_search(ctx, principal, 0, ENCTYPE_NULL,
 					     mem_ctx);
 		if (entry) {
-			name = (char *)TALLOC_MEMDUP(mem_ctx,
+			name = (char *)talloc_memdup(mem_ctx,
 						     entry->password.data,
 						     entry->password.length);
 			if (!name) {
diff --git a/source3/libnet/libnet_dssync_passdb.c b/source3/libnet/libnet_dssync_passdb.c
index ab347567f14..b8675d3d988 100644
--- a/source3/libnet/libnet_dssync_passdb.c
+++ b/source3/libnet/libnet_dssync_passdb.c
@@ -20,10 +20,10 @@
 #include "includes.h"
 #include "system/passwd.h"
 #include "libnet/libnet_dssync.h"
-#include "libnet/libnet_samsync.h"
 #include "../libcli/security/security.h"
 #include "../libds/common/flags.h"
 #include "../librpc/gen_ndr/ndr_drsuapi.h"
+#include "util_tdb.h"
 #include "dbwrap.h"
 #include "../libds/common/flag_mapping.h"
 #include "passdb.h"
@@ -106,7 +106,7 @@ static struct dssync_passdb_obj *dssync_search_obj_by_guid(struct dssync_passdb
 	TDB_DATA key;
 	TDB_DATA data;
 
-	key = make_tdb_data((const uint8_t *)(void *)guid,
+	key = make_tdb_data((const uint8_t *)(const void *)guid,
 			     sizeof(*guid));
 
 	ret = db->fetch(db, talloc_tos(), key, &data);
@@ -1531,11 +1531,11 @@ static NTSTATUS handle_alias_object(struct dssync_passdb *pctx,
 		map.sid_name_use = SID_NAME_ALIAS;
 	}
 
-	fstrcpy(map.nt_name, name);
+	strlcpy(map.nt_name, name, sizeof(map.nt_name));
 	if (description) {
-		fstrcpy(map.comment, comment);
+		strlcpy(map.comment, comment, sizeof(map.comment));
 	} else {
-		fstrcpy(map.comment, "");
+		strlcpy(map.comment, "", sizeof(map.comment));
 	}
 
 	if (insert)
@@ -1636,11 +1636,11 @@ static NTSTATUS handle_group_object(struct dssync_passdb *pctx,
 	map.gid = grp->gr_gid;
 	map.sid = group_sid;
 	map.sid_name_use = SID_NAME_DOM_GRP;
-	fstrcpy(map.nt_name, name);
+	strlcpy(map.nt_name, name, sizeof(map.nt_name));
 	if (description) {
-		fstrcpy(map.comment, comment);
+		strlcpy(map.comment, comment, sizeof(map.comment));
 	} else {
-		fstrcpy(map.comment, "");
+		strlcpy(map.comment, "", sizeof(map.comment));
 	}
 
 	if (insert)
diff --git a/source3/libnet/libnet_join.c b/source3/libnet/libnet_join.c
index 1e866c3dd19..16c068c66ee 100644
--- a/source3/libnet/libnet_join.c
+++ b/source3/libnet/libnet_join.c
@@ -38,6 +38,7 @@
 #include "krb5_env.h"
 #include "../libcli/security/security.h"
 #include "passdb.h"
+#include "libsmb/libsmb.h"
 
 /****************************************************************
 ****************************************************************/
@@ -1074,7 +1075,7 @@ static NTSTATUS libnet_join_joindomain_rpc(TALLOC_CTX *mem_ctx,
 					  &user_info,
 					  &result);
 
-	if (NT_STATUS_EQUAL(status, NT_STATUS(DCERPC_FAULT_INVALID_TAG))) {
+	if (NT_STATUS_EQUAL(status, NT_STATUS_RPC_ENUM_VALUE_OUT_OF_RANGE)) {
 
 		/* retry with level 24 */
 
@@ -1291,7 +1292,7 @@ static NTSTATUS libnet_join_unjoindomain_rpc(TALLOC_CTX *mem_ctx,
 	struct samr_Ids user_rids;
 	struct samr_Ids name_types;
 	union samr_UserInfo *info = NULL;
-	struct dcerpc_binding_handle *b;
+	struct dcerpc_binding_handle *b = NULL;
 
 	ZERO_STRUCT(sam_pol);
 	ZERO_STRUCT(domain_pol);
@@ -1432,7 +1433,7 @@ static NTSTATUS libnet_join_unjoindomain_rpc(TALLOC_CTX *mem_ctx,
 	dcerpc_samr_Close(b, mem_ctx, &user_pol, &result);
 
 done:
-	if (pipe_hnd) {
+	if (pipe_hnd && b) {
 		if (is_valid_policy_hnd(&domain_pol)) {
 			dcerpc_samr_Close(b, mem_ctx, &domain_pol, &result);
 		}
@@ -1454,40 +1455,61 @@ done:
 
 static WERROR do_join_modify_vals_config(struct libnet_JoinCtx *r)
 {
-	WERROR werr;
+	WERROR werr = WERR_OK;
+	sbcErr err;
 	struct smbconf_ctx *ctx;
 
-	werr = smbconf_init_reg(r, &ctx, NULL);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_init_reg(r, &ctx, NULL);
+	if (!SBC_ERROR_IS_OK(err)) {
+		werr = WERR_NO_SUCH_SERVICE;
 		goto done;
 	}
 
 	if (!(r->in.join_flags & WKSSVC_JOIN_FLAGS_JOIN_TYPE)) {
 
-		werr = smbconf_set_global_parameter(ctx, "security", "user");
-		W_ERROR_NOT_OK_GOTO_DONE(werr);
+		err = smbconf_set_global_parameter(ctx, "security", "user");
+		if (!SBC_ERROR_IS_OK(err)) {
+			werr = WERR_NO_SUCH_SERVICE;
+			goto done;
+		}
 
-		werr = smbconf_set_global_parameter(ctx, "workgroup",
-						    r->in.domain_name);
+		err = smbconf_set_global_parameter(ctx, "workgroup",
+						   r->in.domain_name);
+		if (!SBC_ERROR_IS_OK(err)) {
+			werr = WERR_NO_SUCH_SERVICE;
+			goto done;
+		}
 
 		smbconf_delete_global_parameter(ctx, "realm");
 		goto done;
 	}
 
-	werr = smbconf_set_global_parameter(ctx, "security", "domain");
-	W_ERROR_NOT_OK_GOTO_DONE(werr);
+	err = smbconf_set_global_parameter(ctx, "security", "domain");
+	if (!SBC_ERROR_IS_OK(err)) {
+		werr = WERR_NO_SUCH_SERVICE;
+		goto done;
+	}
 
-	werr = smbconf_set_global_parameter(ctx, "workgroup",
-					    r->out.netbios_domain_name);
-	W_ERROR_NOT_OK_GOTO_DONE(werr);
+	err = smbconf_set_global_parameter(ctx, "workgroup",
+					   r->out.netbios_domain_name);
+	if (!SBC_ERROR_IS_OK(err)) {
+		werr = WERR_NO_SUCH_SERVICE;
+		goto done;
+	}
 
 	if (r->out.domain_is_ad) {
-		werr = smbconf_set_global_parameter(ctx, "security", "ads");
-		W_ERROR_NOT_OK_GOTO_DONE(werr);
+		err = smbconf_set_global_parameter(ctx, "security", "ads");
+		if (!SBC_ERROR_IS_OK(err)) {
+			werr = WERR_NO_SUCH_SERVICE;
+			goto done;
+		}
 
-		werr = smbconf_set_global_parameter(ctx, "realm",
-						    r->out.dns_domain_name);
-		W_ERROR_NOT_OK_GOTO_DONE(werr);
+		err = smbconf_set_global_parameter(ctx, "realm",
+						   r->out.dns_domain_name);
+		if (!SBC_ERROR_IS_OK(err)) {
+			werr = WERR_NO_SUCH_SERVICE;
+			goto done;
+		}
 	}
 
  done:
@@ -1501,20 +1523,28 @@ static WERROR do_join_modify_vals_config(struct libnet_JoinCtx *r)
 static WERROR do_unjoin_modify_vals_config(struct libnet_UnjoinCtx *r)
 {
 	WERROR werr = WERR_OK;
+	sbcErr err;
 	struct smbconf_ctx *ctx;
 
-	werr = smbconf_init_reg(r, &ctx, NULL);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_init_reg(r, &ctx, NULL);
+	if (!SBC_ERROR_IS_OK(err)) {
+		werr = WERR_NO_SUCH_SERVICE;
 		goto done;
 	}
 
 	if (r->in.unjoin_flags & WKSSVC_JOIN_FLAGS_JOIN_TYPE) {
 
-		werr = smbconf_set_global_parameter(ctx, "security", "user");
-		W_ERROR_NOT_OK_GOTO_DONE(werr);
+		err = smbconf_set_global_parameter(ctx, "security", "user");
+		if (!SBC_ERROR_IS_OK(err)) {
+			werr = WERR_NO_SUCH_SERVICE;
+			goto done;
+		}
 
-		werr = smbconf_delete_global_parameter(ctx, "workgroup");
-		W_ERROR_NOT_OK_GOTO_DONE(werr);
+		err = smbconf_delete_global_parameter(ctx, "workgroup");
+		if (!SBC_ERROR_IS_OK(err)) {
+			werr = WERR_NO_SUCH_SERVICE;
+			goto done;
+		}
 
 		smbconf_delete_global_parameter(ctx, "realm");
 	}
@@ -1741,7 +1771,7 @@ static int libnet_destroy_JoinCtx(struct libnet_JoinCtx *r)
 	}
 
 	krb5_cc_env = getenv(KRB5_ENV_CCNAME);
-	if (krb5_cc_env && StrCaseCmp(krb5_cc_env, "MEMORY:libnetjoin")) {
+	if (krb5_cc_env && strcasecmp_m(krb5_cc_env, "MEMORY:libnetjoin")) {
 		unsetenv(KRB5_ENV_CCNAME);
 	}
 
@@ -1760,7 +1790,7 @@ static int libnet_destroy_UnjoinCtx(struct libnet_UnjoinCtx *r)
 	}
 
 	krb5_cc_env = getenv(KRB5_ENV_CCNAME);
-	if (krb5_cc_env && StrCaseCmp(krb5_cc_env, "MEMORY:libnetjoin")) {
+	if (krb5_cc_env && strcasecmp_m(krb5_cc_env, "MEMORY:libnetjoin")) {
 		unsetenv(KRB5_ENV_CCNAME);
 	}
 
@@ -1783,7 +1813,7 @@ WERROR libnet_init_JoinCtx(TALLOC_CTX *mem_ctx,
 
 	talloc_set_destructor(ctx, libnet_destroy_JoinCtx);
 
-	ctx->in.machine_name = talloc_strdup(mem_ctx, global_myname());
+	ctx->in.machine_name = talloc_strdup(mem_ctx, lp_netbios_name());
 	W_ERROR_HAVE_NO_MEMORY(ctx->in.machine_name);
 
 	krb5_cc_env = getenv(KRB5_ENV_CCNAME);
@@ -1816,7 +1846,7 @@ WERROR libnet_init_UnjoinCtx(TALLOC_CTX *mem_ctx,
 
 	talloc_set_destructor(ctx, libnet_destroy_UnjoinCtx);
 
-	ctx->in.machine_name = talloc_strdup(mem_ctx, global_myname());
+	ctx->in.machine_name = talloc_strdup(mem_ctx, lp_netbios_name());
 	W_ERROR_HAVE_NO_MEMORY(ctx->in.machine_name);
 
 	krb5_cc_env = getenv(KRB5_ENV_CCNAME);
diff --git a/source3/libnet/libnet_join.h b/source3/libnet/libnet_join.h
index 5659828a5ca..dccf03b715b 100644
--- a/source3/libnet/libnet_join.h
+++ b/source3/libnet/libnet_join.h
@@ -1,3 +1,26 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  libnet Join Support
+ *  Copyright (C) Gerald (Jerry) Carter 2006
+ *  Copyright (C) Guenther Deschner 2007-2008
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LIBNET_LIBNET_JOIN_H_
+#define _LIBNET_LIBNET_JOIN_H_
+
 /* The following definitions come from libnet/libnet_join.c  */
 
 NTSTATUS libnet_join_ok(const char *netbios_domain_name,
@@ -11,3 +34,5 @@ WERROR libnet_Join(TALLOC_CTX *mem_ctx,
 		   struct libnet_JoinCtx *r);
 WERROR libnet_Unjoin(TALLOC_CTX *mem_ctx,
 		     struct libnet_UnjoinCtx *r);
+
+#endif /* _LIBNET_LIBNET_JOIN_H_ */
diff --git a/source3/libnet/libnet_keytab.c b/source3/libnet/libnet_keytab.c
index f395c975e5c..6349c2291f7 100644
--- a/source3/libnet/libnet_keytab.c
+++ b/source3/libnet/libnet_keytab.c
@@ -66,7 +66,7 @@ krb5_error_code libnet_keytab_init(TALLOC_CTX *mem_ctx,
 
 	struct libnet_keytab_context *r;
 
-	r = TALLOC_ZERO_P(mem_ctx, struct libnet_keytab_context);
+	r = talloc_zero(mem_ctx, struct libnet_keytab_context);
 	if (!r) {
 		return ENOMEM;
 	}
diff --git a/source3/libnet/libnet_samsync.c b/source3/libnet/libnet_samsync.c
index 1141bed730b..75fc329e0af 100644
--- a/source3/libnet/libnet_samsync.c
+++ b/source3/libnet/libnet_samsync.c
@@ -23,7 +23,6 @@
 
 #include "includes.h"
 #include "libnet/libnet_samsync.h"
-#include "../lib/crypto/crypto.h"
 #include "../libcli/samsync/samsync.h"
 #include "../libcli/auth/libcli_auth.h"
 #include "rpc_client/rpc_client.h"
@@ -71,7 +70,7 @@ NTSTATUS libnet_samsync_init_context(TALLOC_CTX *mem_ctx,
 
 	*ctx_p = NULL;
 
-	ctx = TALLOC_ZERO_P(mem_ctx, struct samsync_context);
+	ctx = talloc_zero(mem_ctx, struct samsync_context);
 	NT_STATUS_HAVE_NO_MEMORY(ctx);
 
 	if (domain_sid) {
@@ -204,7 +203,7 @@ static NTSTATUS libnet_samsync_delta(TALLOC_CTX *mem_ctx,
 	NTSTATUS result, status;
 	NTSTATUS callback_status;
 	const char *logon_server = ctx->cli->desthost;
-	const char *computername = global_myname();
+	const char *computername = lp_netbios_name();
 	struct netr_Authenticator credential;
 	struct netr_Authenticator return_authenticator;
 	uint16_t restart_state = 0;
@@ -405,7 +404,7 @@ NTSTATUS pull_netr_AcctLockStr(TALLOC_CTX *mem_ctx,
 
 	*str_p = NULL;
 
-	str = TALLOC_ZERO_P(mem_ctx, struct netr_AcctLockStr);
+	str = talloc_zero(mem_ctx, struct netr_AcctLockStr);
 	if (!str) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/libnet/libnet_samsync_ldif.c b/source3/libnet/libnet_samsync_ldif.c
index 4154a07fdab..f535e28e338 100644
--- a/source3/libnet/libnet_samsync_ldif.c
+++ b/source3/libnet/libnet_samsync_ldif.c
@@ -68,6 +68,33 @@ struct samsync_ldif_context {
 	int num_alloced;
 };
 
+/*
+   Returns the substring from src between the first occurrence of
+   the char "front" and the first occurence of the char "back".
+   Mallocs the return string which must be freed.  Not for use
+   with wide character strings.
+*/
+static char *sstring_sub(const char *src, char front, char back)
+{
+	char *temp1, *temp2, *temp3;
+	ptrdiff_t len;
+
+	temp1 = strchr(src, front);
+	if (temp1 == NULL) return NULL;
+	temp2 = strchr(src, back);
+	if (temp2 == NULL) return NULL;
+	len = temp2 - temp1;
+	if (len <= 0) return NULL;
+	temp3 = (char*)SMB_MALLOC(len);
+	if (temp3 == NULL) {
+		DEBUG(1,("Malloc failure in sstring_sub\n"));
+		return NULL;
+	}
+	memcpy(temp3, temp1+1, len-1);
+	temp3[len-1] = '\0';
+	return temp3;
+}
+
 /****************************************************************
 ****************************************************************/
 
@@ -919,7 +946,7 @@ static NTSTATUS ldif_init_context(TALLOC_CTX *mem_ctx,
 		return NT_STATUS_OK;
 	}
 
-	r = TALLOC_ZERO_P(mem_ctx, struct samsync_ldif_context);
+	r = talloc_zero(mem_ctx, struct samsync_ldif_context);
 	NT_STATUS_HAVE_NO_MEMORY(r);
 
 	/* Get the ldap suffix */
@@ -966,8 +993,8 @@ static NTSTATUS ldif_init_context(TALLOC_CTX *mem_ctx,
 	}
 
 	/* Allocate initial memory for groupmap and accountmap arrays */
-	r->groupmap = TALLOC_ZERO_ARRAY(mem_ctx, GROUPMAP, 8);
-	r->accountmap = TALLOC_ZERO_ARRAY(mem_ctx, ACCOUNTMAP, 8);
+	r->groupmap = talloc_zero_array(mem_ctx, GROUPMAP, 8);
+	r->accountmap = talloc_zero_array(mem_ctx, ACCOUNTMAP, 8);
 	if (r->groupmap == NULL || r->accountmap == NULL) {
 		DEBUG(1,("GROUPMAP talloc failed\n"));
 		status = NT_STATUS_NO_MEMORY;
@@ -1173,12 +1200,12 @@ static NTSTATUS ldif_realloc_maps(TALLOC_CTX *mem_ctx,
 				  uint32_t num_entries)
 {
 	/* Re-allocate memory for groupmap and accountmap arrays */
-	l->groupmap = TALLOC_REALLOC_ARRAY(mem_ctx,
+	l->groupmap = talloc_realloc(mem_ctx,
 					   l->groupmap,
 					   GROUPMAP,
 					   num_entries + l->num_alloced);
 
-	l->accountmap = TALLOC_REALLOC_ARRAY(mem_ctx,
+	l->accountmap = talloc_realloc(mem_ctx,
 					     l->accountmap,
 					     ACCOUNTMAP,
 					     num_entries + l->num_alloced);
diff --git a/source3/libnet/libnet_samsync_passdb.c b/source3/libnet/libnet_samsync_passdb.c
index b579723f6fd..0cf2ed3323d 100644
--- a/source3/libnet/libnet_samsync_passdb.c
+++ b/source3/libnet/libnet_samsync_passdb.c
@@ -428,11 +428,11 @@ static NTSTATUS fetch_group_info(TALLOC_CTX *mem_ctx,
 	map.gid = grp->gr_gid;
 	map.sid = group_sid;
 	map.sid_name_use = SID_NAME_DOM_GRP;
-	fstrcpy(map.nt_name, name);
+	strlcpy(map.nt_name, name, sizeof(map.nt_name));
 	if (r->description.string) {
-		fstrcpy(map.comment, comment);
+		strlcpy(map.comment, comment, sizeof(map.comment));
 	} else {
-		fstrcpy(map.comment, "");
+		strlcpy(map.comment, "", sizeof(map.comment));
 	}
 
 	if (insert)
@@ -476,7 +476,7 @@ static NTSTATUS fetch_group_mem_info(TALLOC_CTX *mem_ctx,
 	d_printf("Group members of %s: ", grp->gr_name);
 
 	if (r->num_rids) {
-		if ((nt_members = TALLOC_ZERO_ARRAY(mem_ctx, char *, r->num_rids)) == NULL) {
+		if ((nt_members = talloc_zero_array(mem_ctx, char *, r->num_rids)) == NULL) {
 			DEBUG(0, ("talloc failed\n"));
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -617,8 +617,8 @@ static NTSTATUS fetch_alias_info(TALLOC_CTX *mem_ctx,
 	else
 		map.sid_name_use = SID_NAME_ALIAS;
 
-	fstrcpy(map.nt_name, name);
-	fstrcpy(map.comment, comment);
+	strlcpy(map.nt_name, name, sizeof(map.nt_name));
+	strlcpy(map.comment, comment, sizeof(map.comment));
 
 	if (insert)
 		pdb_add_group_mapping_entry(&map);
diff --git a/source3/librpc/crypto/cli_spnego.c b/source3/librpc/crypto/cli_spnego.c
index bf58e25d9a8..e4fdf315aaf 100644
--- a/source3/librpc/crypto/cli_spnego.c
+++ b/source3/librpc/crypto/cli_spnego.c
@@ -99,7 +99,7 @@ NTSTATUS spnego_ntlmssp_init_client(TALLOC_CTX *mem_ctx,
 	sp_ctx->mech = SPNEGO_NTLMSSP;
 
 	status = auth_ntlmssp_client_start(sp_ctx,
-					global_myname(),
+					lp_netbios_name(),
 					lp_workgroup(),
 					lp_client_ntlmv2_auth(),
 					&sp_ctx->mech_ctx.ntlmssp_state);
diff --git a/source3/librpc/crypto/gse.c b/source3/librpc/crypto/gse.c
index 0d9eead082e..c311c774d42 100644
--- a/source3/librpc/crypto/gse.c
+++ b/source3/librpc/crypto/gse.c
@@ -27,12 +27,6 @@
 #include "smb_krb5.h"
 #include "gse_krb5.h"
 
-#include <gssapi/gssapi.h>
-#include <gssapi/gssapi_krb5.h>
-#ifdef HAVE_GSSAPI_GSSAPI_EXT_H
-#include <gssapi/gssapi_ext.h>
-#endif
-
 #ifndef GSS_KRB5_INQ_SSPI_SESSION_KEY_OID
 #define GSS_KRB5_INQ_SSPI_SESSION_KEY_OID_LENGTH 11
 #define GSS_KRB5_INQ_SSPI_SESSION_KEY_OID "\x2a\x86\x48\x86\xf7\x12\x01\x02\x02\x05\x05"
@@ -62,16 +56,6 @@ gss_OID_desc gse_authz_data_oid = {
 	(void *)GSE_EXTRACT_RELEVANT_AUTHZ_DATA_OID
 };
 
-#ifndef GSS_KRB5_EXTRACT_AUTHTIME_FROM_SEC_CONTEXT_OID
-#define GSS_KRB5_EXTRACT_AUTHTIME_FROM_SEC_CONTEXT_OID_LENGTH 11
-#define GSS_KRB5_EXTRACT_AUTHTIME_FROM_SEC_CONTEXT_OID "\x2a\x86\x48\x86\xf7\x12\x01\x02\x02\x05\x0c"
-#endif
-
-gss_OID_desc gse_authtime_oid = {
-	GSS_KRB5_EXTRACT_AUTHTIME_FROM_SEC_CONTEXT_OID_LENGTH,
-	(void *)GSS_KRB5_EXTRACT_AUTHTIME_FROM_SEC_CONTEXT_OID
-};
-
 static char *gse_errstr(TALLOC_CTX *mem_ctx, OM_uint32 maj, OM_uint32 min);
 
 struct gse_context {
@@ -95,6 +79,24 @@ struct gse_context {
 	bool authenticated;
 };
 
+#ifndef HAVE_GSS_OID_EQUAL
+
+static bool gss_oid_equal(const gss_OID o1, const gss_OID o2)
+{
+	if (o1 == o2) {
+		return true;
+	}
+	if ((o1 == NULL && o2 != NULL) || (o1 != NULL && o2 == NULL)) {
+		return false;
+	}
+	if (o1->length != o2->length) {
+		return false;
+	}
+	return memcmp(o1->elements, o2->elements, o1->length) == false;
+}
+
+#endif
+
 /* free non talloc dependent contexts */
 static int gse_context_destructor(void *ptr)
 {
@@ -135,10 +137,19 @@ static int gse_context_destructor(void *ptr)
 		gss_maj = gss_release_cred(&gss_min,
 					   &gse_ctx->delegated_creds);
 	}
-	if (gse_ctx->ret_mech) {
-		gss_maj = gss_release_oid(&gss_min,
-					  &gse_ctx->ret_mech);
-	}
+
+	/* MIT and Heimdal differ as to if you can call
+	 * gss_release_oid() on this OID, generated by
+	 * gss_{accept,init}_sec_context().  However, as long as the
+	 * oid is gss_mech_krb5 (which it always is at the moment),
+	 * then this is a moot point, as both declare this particular
+	 * OID static, and so no memory is lost.  This assert is in
+	 * place to ensure that the programmer who wishes to extend
+	 * this code to EAP or other GSS mechanisms determines an
+	 * implementation-dependent way of releasing any dynamically
+	 * allocated OID */
+	SMB_ASSERT(gss_oid_equal(&gse_ctx->gss_mech, GSS_C_NO_OID) || gss_oid_equal(&gse_ctx->gss_mech, gss_mech_krb5));
+
 	return 0;
 }
 
@@ -348,8 +359,6 @@ NTSTATUS gse_init_server(TALLOC_CTX *mem_ctx,
 	OM_uint32 gss_maj, gss_min;
 	krb5_error_code ret;
 	NTSTATUS status;
-	const char *ktname;
-	gss_OID_set_desc mech_set;
 
 	status = gse_context_init(mem_ctx, do_sign, do_seal,
 				  NULL, add_gss_c_flags, &gse_ctx);
@@ -379,24 +388,27 @@ NTSTATUS gse_init_server(TALLOC_CTX *mem_ctx,
 	 * This call sets the default keytab for the whole server, not
 	 * just for this context. Need to find a way that does not alter
 	 * the state of the whole server ... */
+	{
+		const char *ktname;
+		gss_OID_set_desc mech_set;
 
-	ret = smb_krb5_keytab_name(gse_ctx, gse_ctx->k5ctx,
+		ret = smb_krb5_keytab_name(gse_ctx, gse_ctx->k5ctx,
 				   gse_ctx->keytab, &ktname);
-	if (ret) {
-		status = NT_STATUS_INTERNAL_ERROR;
-		goto done;
-	}
+		if (ret) {
+			status = NT_STATUS_INTERNAL_ERROR;
+			goto done;
+		}
 
-	ret = gsskrb5_register_acceptor_identity(ktname);
-	if (ret) {
-		status = NT_STATUS_INTERNAL_ERROR;
-		goto done;
-	}
+		ret = gsskrb5_register_acceptor_identity(ktname);
+		if (ret) {
+			status = NT_STATUS_INTERNAL_ERROR;
+			goto done;
+		}
 
-	mech_set.count = 1;
-	mech_set.elements = &gse_ctx->gss_mech;
-	
-	gss_maj = gss_acquire_cred(&gss_min,
+		mech_set.count = 1;
+		mech_set.elements = &gse_ctx->gss_mech;
+
+		gss_maj = gss_acquire_cred(&gss_min,
 				   GSS_C_NO_NAME,
 				   GSS_C_INDEFINITE,
 				   &mech_set,
@@ -404,11 +416,12 @@ NTSTATUS gse_init_server(TALLOC_CTX *mem_ctx,
 				   &gse_ctx->creds,
 				   NULL, NULL);
 
-	if (gss_maj) {
-		DEBUG(0, ("gss_acquire_creds failed with [%s]\n",
-			  gse_errstr(gse_ctx, gss_maj, gss_min)));
-		status = NT_STATUS_INTERNAL_ERROR;
-		goto done;
+		if (gss_maj) {
+			DEBUG(0, ("gss_acquire_creds failed with [%s]\n",
+				  gse_errstr(gse_ctx, gss_maj, gss_min)));
+			status = NT_STATUS_INTERNAL_ERROR;
+			goto done;
+		}
 	}
 #endif
 	status = NT_STATUS_OK;
@@ -692,42 +705,15 @@ NTSTATUS gse_get_authz_data(struct gse_context *gse_ctx,
 	return NT_STATUS_OK;
 }
 
-NTSTATUS gse_get_authtime(struct gse_context *gse_ctx, time_t *authtime)
+NTSTATUS gse_get_pac_blob(struct gse_context *gse_ctx,
+			  TALLOC_CTX *mem_ctx, DATA_BLOB *pac_blob)
 {
-	OM_uint32 gss_min, gss_maj;
-	gss_buffer_set_t set = GSS_C_NO_BUFFER_SET;
-	int32_t	tkttime;
-
 	if (!gse_ctx->authenticated) {
 		return NT_STATUS_ACCESS_DENIED;
 	}
 
-	gss_maj = gss_inquire_sec_context_by_oid(
-				&gss_min, gse_ctx->gss_ctx,
-				&gse_authtime_oid, &set);
-	if (gss_maj) {
-		DEBUG(0, ("gss_inquire_sec_context_by_oid failed [%s]\n",
-			  gse_errstr(talloc_tos(), gss_maj, gss_min)));
-		return NT_STATUS_NOT_FOUND;
-	}
-
-	if ((set == GSS_C_NO_BUFFER_SET) || (set->count != 1) != 0) {
-		DEBUG(0, ("gss_inquire_sec_context_by_oid returned unknown "
-			  "data in results.\n"));
-		return NT_STATUS_INTERNAL_ERROR;
-	}
-
-	if (set->elements[0].length != sizeof(int32_t)) {
-		DEBUG(0, ("Invalid authtime size!\n"));
-		return NT_STATUS_INTERNAL_ERROR;
-	}
-
-	tkttime = *((int32_t *)set->elements[0].value);
-
-	gss_maj = gss_release_buffer_set(&gss_min, &set);
-
-	*authtime = (time_t)tkttime;
-	return NT_STATUS_OK;
+	return gssapi_obtain_pac_blob(mem_ctx, gse_ctx->gss_ctx,
+					gse_ctx->client_name, pac_blob);
 }
 
 size_t gse_get_signature_length(struct gse_context *gse_ctx,
@@ -982,7 +968,8 @@ NTSTATUS gse_get_authz_data(struct gse_context *gse_ctx,
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
 
-NTSTATUS gse_get_authtime(struct gse_context *gse_ctx, time_t *authtime)
+NTSTATUS gse_get_pac_blob(struct gse_context *gse_ctx,
+			  TALLOC_CTX *mem_ctx, DATA_BLOB *pac_blob)
 {
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
@@ -1017,4 +1004,4 @@ NTSTATUS gse_sigcheck(TALLOC_CTX *mem_ctx, struct gse_context *gse_ctx,
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
 
-#endif /* HAVE_KRB5 && HAVE_GSSAPI_EXT_H && HAVE_GSS_WRAP_IOV */
+#endif /* HAVE_KRB5 && HAVE_GSS_WRAP_IOV */
diff --git a/source3/librpc/crypto/gse.h b/source3/librpc/crypto/gse.h
index fbcf5b6e10d..27cc2e92559 100644
--- a/source3/librpc/crypto/gse.h
+++ b/source3/librpc/crypto/gse.h
@@ -56,7 +56,8 @@ NTSTATUS gse_get_client_name(struct gse_context *gse_ctx,
 			     TALLOC_CTX *mem_ctx, char **client_name);
 NTSTATUS gse_get_authz_data(struct gse_context *gse_ctx,
 			    TALLOC_CTX *mem_ctx, DATA_BLOB *pac);
-NTSTATUS gse_get_authtime(struct gse_context *gse_ctx, time_t *authtime);
+NTSTATUS gse_get_pac_blob(struct gse_context *gse_ctx,
+			  TALLOC_CTX *mem_ctx, DATA_BLOB *pac_blob);
 
 size_t gse_get_signature_length(struct gse_context *gse_ctx,
 				int seal, size_t payload_size);
diff --git a/source3/librpc/crypto/gse_krb5.c b/source3/librpc/crypto/gse_krb5.c
index 830d517bc7c..81a9a075967 100644
--- a/source3/librpc/crypto/gse_krb5.c
+++ b/source3/librpc/crypto/gse_krb5.c
@@ -90,7 +90,7 @@ static krb5_error_code get_host_principal(krb5_context krbctx,
 	char *host_princ_s = NULL;
 	int err;
 
-	err = asprintf(&host_princ_s, "%s$@%s", global_myname(), lp_realm());
+	err = asprintf(&host_princ_s, "%s$@%s", lp_netbios_name(), lp_realm());
 	if (err == -1) {
 		return -1;
 	}
diff --git a/source3/librpc/idl/messaging.idl b/source3/librpc/idl/messaging.idl
index 8618d53c9a2..0c0672c7a6d 100644
--- a/source3/librpc/idl/messaging.idl
+++ b/source3/librpc/idl/messaging.idl
@@ -75,7 +75,6 @@ interface messaging
 
 		/* cluster reconfigure events */
 		MSG_SMB_BRL_VALIDATE		= 0x0311,
-		MSG_SMB_RELEASE_IP		= 0x0312,
 
 		/*Close a specific file given a share entry. */
 		MSG_SMB_CLOSE_FILE		= 0x0313,
diff --git a/source3/librpc/idl/wscript_build b/source3/librpc/idl/wscript_build
index 5a8dc31980b..3e07542320d 100644
--- a/source3/librpc/idl/wscript_build
+++ b/source3/librpc/idl/wscript_build
@@ -5,8 +5,8 @@ import os
 topinclude=os.path.join(bld.srcnode.abspath(), 'librpc/idl')
 
 bld.SAMBA_PIDL_LIST('PIDL',
-                    '''messaging.idl libnetapi.idl notify.idl
-                       perfcount.idl secrets.idl libnet_join.idl server_id.idl''',
+                    '''messaging.idl libnetapi.idl
+                       perfcount.idl secrets.idl libnet_join.idl''',
                     options='--includedir=%s --header --ndr-parser' % topinclude,
                     output_dir='../gen_ndr')
 
diff --git a/source3/librpc/ndr/util.h b/source3/librpc/ndr/util.h
deleted file mode 100644
index 3bf9c0eb438..00000000000
--- a/source3/librpc/ndr/util.h
+++ /dev/null
@@ -1,4 +0,0 @@
-
-/* The following definitions come from librpc/ndr/util.c  */
-
-_PUBLIC_ void ndr_print_sockaddr_storage(struct ndr_print *ndr, const char *name, const struct sockaddr_storage *ss);
diff --git a/source3/librpc/rpc/dcerpc.h b/source3/librpc/rpc/dcerpc.h
index 0a6ddaca173..d7e8e0c193d 100644
--- a/source3/librpc/rpc/dcerpc.h
+++ b/source3/librpc/rpc/dcerpc.h
@@ -33,7 +33,20 @@
 #define SMB_RPC_INTERFACE_VERSION 1
 
 struct NL_AUTH_MESSAGE;
-struct pipe_auth_data;
+
+/* auth state for all bind types. */
+
+struct pipe_auth_data {
+	enum dcerpc_AuthType auth_type;
+	enum dcerpc_AuthLevel auth_level;
+
+	void *auth_ctx;
+
+	/* Only the client code uses these 3 for now */
+	char *domain;
+	char *user_name;
+	DATA_BLOB user_session_key;
+};
 
 /* The following definitions come from librpc/rpc/dcerpc_helpers.c  */
 NTSTATUS dcerpc_push_ncacn_packet(TALLOC_CTX *mem_ctx,
@@ -75,4 +88,12 @@ NTSTATUS dcerpc_check_auth(struct pipe_auth_data *auth,
 			   DATA_BLOB *raw_pkt,
 			   size_t *pad_len);
 
+/* The following definitions come from librpc/rpc/rpc_common.c  */
+
+bool smb_register_ndr_interface(const struct ndr_interface_table *interface);
+const struct ndr_interface_table *get_iface_from_syntax(
+	const struct ndr_syntax_id *syntax);
+const char *get_pipe_name_from_syntax(TALLOC_CTX *mem_ctx,
+                                     const struct ndr_syntax_id *syntax);
+
 #endif /* __S3_DCERPC_H__ */
diff --git a/source3/librpc/rpc/dcerpc_ep.c b/source3/librpc/rpc/dcerpc_ep.c
index 76f7e91fb05..3d1acbbee22 100644
--- a/source3/librpc/rpc/dcerpc_ep.c
+++ b/source3/librpc/rpc/dcerpc_ep.c
@@ -23,6 +23,7 @@
 #include "../librpc/gen_ndr/ndr_epmapper_c.h"
 #include "rpc_client/cli_pipe.h"
 #include "auth.h"
+#include "rpc_server/rpc_ncacn_np.h"
 
 #define EPM_MAX_ANNOTATION_SIZE 64
 
@@ -77,7 +78,7 @@ NTSTATUS dcerpc_binding_vector_create(TALLOC_CTX *mem_ctx,
 
 		switch (b->transport) {
 			case NCACN_NP:
-				b->host = talloc_asprintf(b, "\\\\%s", global_myname());
+				b->host = talloc_asprintf(b, "\\\\%s", lp_netbios_name());
 				if (b->host == NULL) {
 					status = NT_STATUS_NO_MEMORY;
 					goto done;
@@ -168,7 +169,7 @@ static NTSTATUS ep_register(TALLOC_CTX *mem_ctx,
 					   "rpc_server", "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0) {
 		static struct client_address client_id;
 
 		strlcpy(client_id.addr, "localhost", sizeof(client_id.addr));
@@ -185,7 +186,7 @@ static NTSTATUS ep_register(TALLOC_CTX *mem_ctx,
 				  "epmapper (%s)", nt_errstr(status)));
 			goto done;
 		}
-	} else if (StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	} else if (strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		/* Connect to the endpoint mapper locally */
 		ncalrpc_sock = talloc_asprintf(tmp_ctx,
 					      "%s/%s",
diff --git a/source3/librpc/rpc/dcerpc_helpers.c b/source3/librpc/rpc/dcerpc_helpers.c
index 7e6990c0add..7520d767ba1 100644
--- a/source3/librpc/rpc/dcerpc_helpers.c
+++ b/source3/librpc/rpc/dcerpc_helpers.c
@@ -28,7 +28,6 @@
 #include "ntlmssp_wrap.h"
 #include "librpc/crypto/gse.h"
 #include "librpc/crypto/spnego.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_PARSE
diff --git a/source3/librpc/rpc/rpc_common.c b/source3/librpc/rpc/rpc_common.c
index ed0320adf09..65e3205f62a 100644
--- a/source3/librpc/rpc/rpc_common.c
+++ b/source3/librpc/rpc/rpc_common.c
@@ -18,7 +18,7 @@
  */
 
 #include "includes.h"
-#include "../librpc/gen_ndr/ndr_schannel.h"
+#include "librpc/rpc/dcerpc.h"
 #include "../librpc/gen_ndr/ndr_lsa.h"
 #include "../librpc/gen_ndr/ndr_dssetup.h"
 #include "../librpc/gen_ndr/ndr_samr.h"
diff --git a/source3/librpc/wscript_build b/source3/librpc/wscript_build
index d99936cc166..a4af551e7f2 100644
--- a/source3/librpc/wscript_build
+++ b/source3/librpc/wscript_build
@@ -10,21 +10,11 @@ bld.SAMBA3_SUBSYSTEM('NDR_LIBNET_JOIN',
 	public_deps='ndr'
 	)
 
-bld.SAMBA3_SUBSYSTEM('NDR_SERVER_ID',
-	source='gen_ndr/ndr_server_id.c',
-	public_deps='ndr'
-	)
-
 bld.SAMBA3_SUBSYSTEM('NDR_MESSAGING',
 	source='gen_ndr/ndr_messaging.c',
 	public_deps='ndr NDR_SERVER_ID'
 	)
 
-bld.SAMBA3_SUBSYSTEM('NDR_NOTIFY3',
-	source='gen_ndr/ndr_notify.c',
-	public_deps='ndr NDR_FILE_ID NDR_SERVER_ID'
-	)
-
 bld.SAMBA3_SUBSYSTEM('NDR_SECRETS',
 	source='gen_ndr/ndr_secrets.c',
 	public_deps='ndr'
diff --git a/source3/libsmb/async_smb.c b/source3/libsmb/async_smb.c
index 8af5aa10461..ee15caee7a5 100644
--- a/source3/libsmb/async_smb.c
+++ b/source3/libsmb/async_smb.c
@@ -18,97 +18,14 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "../lib/async_req/async_sock.h"
+#include "../lib/util/tevent_ntstatus.h"
+#include "../lib/util/tevent_unix.h"
 #include "async_smb.h"
 #include "smb_crypt.h"
 #include "libsmb/nmblib.h"
-
-/*
- * Read an smb packet asynchronously, discard keepalives
- */
-
-struct read_smb_state {
-	struct tevent_context *ev;
-	int fd;
-	uint8_t *buf;
-};
-
-static ssize_t read_smb_more(uint8_t *buf, size_t buflen, void *private_data);
-static void read_smb_done(struct tevent_req *subreq);
-
-static struct tevent_req *read_smb_send(TALLOC_CTX *mem_ctx,
-					struct tevent_context *ev,
-					int fd)
-{
-	struct tevent_req *result, *subreq;
-	struct read_smb_state *state;
-
-	result = tevent_req_create(mem_ctx, &state, struct read_smb_state);
-	if (result == NULL) {
-		return NULL;
-	}
-	state->ev = ev;
-	state->fd = fd;
-
-	subreq = read_packet_send(state, ev, fd, 4, read_smb_more, NULL);
-	if (subreq == NULL) {
-		goto fail;
-	}
-	tevent_req_set_callback(subreq, read_smb_done, result);
-	return result;
- fail:
-	TALLOC_FREE(result);
-	return NULL;
-}
-
-static ssize_t read_smb_more(uint8_t *buf, size_t buflen, void *private_data)
-{
-	if (buflen > 4) {
-		return 0;	/* We've been here, we're done */
-	}
-	return smb_len_large(buf);
-}
-
-static void read_smb_done(struct tevent_req *subreq)
-{
-	struct tevent_req *req = tevent_req_callback_data(
-		subreq, struct tevent_req);
-	struct read_smb_state *state = tevent_req_data(
-		req, struct read_smb_state);
-	ssize_t len;
-	int err;
-
-	len = read_packet_recv(subreq, state, &state->buf, &err);
-	TALLOC_FREE(subreq);
-	if (len == -1) {
-		tevent_req_error(req, err);
-		return;
-	}
-
-	if (CVAL(state->buf, 0) == SMBkeepalive) {
-		subreq = read_packet_send(state, state->ev, state->fd, 4,
-					  read_smb_more, NULL);
-		if (tevent_req_nomem(subreq, req)) {
-			return;
-		}
-		tevent_req_set_callback(subreq, read_smb_done, req);
-		return;
-	}
-	tevent_req_done(req);
-}
-
-static ssize_t read_smb_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
-			     uint8_t **pbuf, int *perrno)
-{
-	struct read_smb_state *state = tevent_req_data(
-		req, struct read_smb_state);
-
-	if (tevent_req_is_unix_error(req, perrno)) {
-		return -1;
-	}
-	*pbuf = talloc_move(mem_ctx, &state->buf);
-	return talloc_get_size(*pbuf);
-}
+#include "read_smb.h"
 
 /**
  * Fetch an error out of a NBT packet
@@ -236,6 +153,14 @@ void cli_smb_req_unset_pending(struct tevent_req *req)
 	int num_pending = talloc_array_length(cli->pending);
 	int i;
 
+	if (state->mid != 0) {
+		/*
+		 * This is a [nt]trans[2] request which waits
+		 * for more than one reply.
+		 */
+		return;
+	}
+
 	if (num_pending == 1) {
 		/*
 		 * The pending read_smb tevent_req is a child of
@@ -263,9 +188,7 @@ void cli_smb_req_unset_pending(struct tevent_req *req)
 	/*
 	 * Remove ourselves from the cli->pending array
 	 */
-	if (num_pending > 1) {
-		cli->pending[i] = cli->pending[num_pending-1];
-	}
+	cli->pending[i] = cli->pending[num_pending-1];
 
 	/*
 	 * No NULL check here, we're shrinking by sizeof(void *), and
@@ -278,6 +201,13 @@ void cli_smb_req_unset_pending(struct tevent_req *req)
 
 static int cli_smb_req_destructor(struct tevent_req *req)
 {
+	struct cli_smb_state *state = tevent_req_data(
+		req, struct cli_smb_state);
+	/*
+	 * Make sure we really remove it from
+	 * the pending array on destruction.
+	 */
+	state->mid = 0;
 	cli_smb_req_unset_pending(req);
 	return 0;
 }
@@ -340,6 +270,20 @@ void cli_smb_req_set_mid(struct tevent_req *req, uint16_t mid)
 	state->mid = mid;
 }
 
+uint32_t cli_smb_req_seqnum(struct tevent_req *req)
+{
+	struct cli_smb_state *state = tevent_req_data(
+		req, struct cli_smb_state);
+	return state->seqnum;
+}
+
+void cli_smb_req_set_seqnum(struct tevent_req *req, uint32_t seqnum)
+{
+	struct cli_smb_state *state = tevent_req_data(
+		req, struct cli_smb_state);
+	state->seqnum = seqnum;
+}
+
 static size_t iov_len(const struct iovec *iov, int count)
 {
 	size_t result = 0;
@@ -423,9 +367,9 @@ struct tevent_req *cli_smb_req_create(TALLOC_CTX *mem_ctx,
 	state->iov_count = iov_count + 3;
 
 	if (cli->timeout) {
-		endtime = timeval_current_ofs(0, cli->timeout * 1000);
+		endtime = timeval_current_ofs_msec(cli->timeout);
 		if (!tevent_req_set_endtime(result, ev, endtime)) {
-			tevent_req_nomem(NULL, result);
+			tevent_req_oom(result);
 		}
 	}
 	return result;
@@ -496,8 +440,8 @@ static NTSTATUS cli_smb_req_iov_send(struct tevent_req *req,
 		if (buf == NULL) {
 			return NT_STATUS_NO_MEMORY;
 		}
-		status = cli_encrypt_message(state->cli, (char *)buf,
-					     &enc_buf);
+		status = common_encrypt_buffer(state->cli->trans_enc_state,
+					       (char *)buf, &enc_buf);
 		TALLOC_FREE(buf);
 		if (!NT_STATUS_IS_OK(status)) {
 			DEBUG(0, ("Error in encrypting client message: %s\n",
@@ -544,7 +488,7 @@ struct tevent_req *cli_smb_send(TALLOC_CTX *mem_ctx,
 	struct iovec iov;
 	NTSTATUS status;
 
-	iov.iov_base = CONST_DISCARD(void *, bytes);
+	iov.iov_base = discard_const_p(void, bytes);
 	iov.iov_len = num_bytes;
 
 	req = cli_smb_req_create(mem_ctx, ev, cli, smb_command,
@@ -712,7 +656,7 @@ static void cli_smb_received(struct tevent_req *subreq)
 	if (state->chained_requests == NULL) {
 		state->inbuf = talloc_move(state, &inbuf);
 		talloc_set_destructor(req, NULL);
-		cli_smb_req_destructor(req);
+		cli_smb_req_unset_pending(req);
 		state->chain_num = 0;
 		state->chain_length = 1;
 		tevent_req_done(req);
@@ -756,7 +700,7 @@ static void cli_smb_received(struct tevent_req *subreq)
 	while (talloc_array_length(cli->pending) > 0) {
 		req = cli->pending[0];
 		talloc_set_destructor(req, NULL);
-		cli_smb_req_destructor(req);
+		cli_smb_req_unset_pending(req);
 		tevent_req_nterror(req, status);
 	}
 }
@@ -779,6 +723,24 @@ NTSTATUS cli_smb_recv(struct tevent_req *req,
 	}
 
 	if (state->inbuf == NULL) {
+		if (min_wct != 0) {
+			return NT_STATUS_INVALID_NETWORK_RESPONSE;
+		}
+		if (pinbuf) {
+			*pinbuf = NULL;
+		}
+		if (pwct) {
+			*pwct = 0;
+		}
+		if (pvwv) {
+			*pvwv = NULL;
+		}
+		if (pnum_bytes) {
+			*pnum_bytes = 0;
+		}
+		if (pbytes) {
+			*pbytes = NULL;
+		}
 		/* This was a request without a reply */
 		return NT_STATUS_OK;
 	}
@@ -821,6 +783,8 @@ NTSTATUS cli_smb_recv(struct tevent_req *req,
 
 	status = cli_pull_error((char *)state->inbuf);
 
+	cli_set_error(state->cli, status);
+
 	if (!have_andx_command((char *)state->inbuf, wct_ofs)) {
 
 		if ((cmd == SMBsesssetupX)
diff --git a/source3/libsmb/cli_np_tstream.c b/source3/libsmb/cli_np_tstream.c
index 1f9e5ffd36e..7521181ec80 100644
--- a/source3/libsmb/cli_np_tstream.c
+++ b/source3/libsmb/cli_np_tstream.c
@@ -19,7 +19,8 @@
 
 #include "includes.h"
 #include "system/network.h"
-#include "../util/tevent_unix.h"
+#include "libsmb/libsmb.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "../lib/tsocket/tsocket.h"
 #include "../lib/tsocket/tsocket_internal.h"
 #include "cli_np_tstream.h"
@@ -27,9 +28,24 @@
 static const struct tstream_context_ops tstream_cli_np_ops;
 
 /*
- * Window uses 1024 hardcoded for read size and trans max data
+ * Windows uses 4280 (the max xmit/recv size negotiated on DCERPC).
+ * This is fits into the max_xmit negotiated at the SMB layer.
+ *
+ * On the sending side they may use SMBtranss if the request does not
+ * fit into a single SMBtrans call.
+ *
+ * Windows uses 1024 as max data size of a SMBtrans request and then
+ * possibly reads the rest of the DCERPC fragment (up to 3256 bytes)
+ * via a SMBreadX.
+ *
+ * For now we just ask for the full 4280 bytes (max data size) in the SMBtrans
+ * request to get the whole fragment at once (like samba 3.5.x and below did.
+ *
+ * It is important that we use do SMBwriteX with the size of a full fragment,
+ * otherwise we may get NT_STATUS_PIPE_BUSY on the SMBtrans request
+ * from NT4 servers. (See bug #8195)
  */
-#define TSTREAM_CLI_NP_BUF_SIZE 1024
+#define TSTREAM_CLI_NP_MAX_BUF_SIZE 4280
 
 struct tstream_cli_np {
 	struct cli_state *cli;
@@ -47,7 +63,7 @@ struct tstream_cli_np {
 	struct {
 		off_t ofs;
 		size_t left;
-		uint8_t buf[TSTREAM_CLI_NP_BUF_SIZE];
+		uint8_t *buf;
 	} read, write;
 };
 
@@ -347,9 +363,26 @@ static void tstream_cli_np_writev_write_next(struct tevent_req *req)
 		tstream_context_data(state->stream,
 		struct tstream_cli_np);
 	struct tevent_req *subreq;
+	size_t i;
+	size_t left = 0;
+
+	for (i=0; i < state->count; i++) {
+		left += state->vector[i].iov_len;
+	}
+
+	if (left == 0) {
+		TALLOC_FREE(cli_nps->write.buf);
+		tevent_req_done(req);
+		return;
+	}
 
 	cli_nps->write.ofs = 0;
-	cli_nps->write.left = TSTREAM_CLI_NP_BUF_SIZE;
+	cli_nps->write.left = MIN(left, TSTREAM_CLI_NP_MAX_BUF_SIZE);
+	cli_nps->write.buf = talloc_realloc(cli_nps, cli_nps->write.buf,
+					    uint8_t, cli_nps->write.left);
+	if (tevent_req_nomem(cli_nps->write.buf, req)) {
+		return;
+	}
 
 	/*
 	 * copy the pending buffer first
@@ -375,11 +408,6 @@ static void tstream_cli_np_writev_write_next(struct tevent_req *req)
 		state->ret += len;
 	}
 
-	if (cli_nps->write.ofs == 0) {
-		tevent_req_done(req);
-		return;
-	}
-
 	if (cli_nps->trans.active && state->count == 0) {
 		cli_nps->trans.active = false;
 		cli_nps->trans.write_req = req;
@@ -478,9 +506,8 @@ static void tstream_cli_np_writev_disconnect_done(struct tevent_req *subreq)
 		tevent_req_data(req, struct tstream_cli_np_writev_state);
 	struct tstream_cli_np *cli_nps =
 		tstream_context_data(state->stream, struct tstream_cli_np);
-	NTSTATUS status;
 
-	status = cli_close_recv(subreq);
+	cli_close_recv(subreq);
 	TALLOC_FREE(subreq);
 
 	cli_nps->cli = NULL;
@@ -619,6 +646,10 @@ static void tstream_cli_np_readv_read_next(struct tevent_req *req)
 		state->ret += len;
 	}
 
+	if (cli_nps->read.left == 0) {
+		TALLOC_FREE(cli_nps->read.buf);
+	}
+
 	if (state->count == 0) {
 		tevent_req_done(req);
 		return;
@@ -637,7 +668,7 @@ static void tstream_cli_np_readv_read_next(struct tevent_req *req)
 	}
 
 	subreq = cli_read_andx_send(state, state->ev, cli_nps->cli,
-				    cli_nps->fnum, 0, TSTREAM_CLI_NP_BUF_SIZE);
+				    cli_nps->fnum, 0, TSTREAM_CLI_NP_MAX_BUF_SIZE);
 	if (tevent_req_nomem(subreq, req)) {
 		return;
 	}
@@ -673,7 +704,7 @@ static void tstream_cli_np_readv_trans_start(struct tevent_req *req)
 				NULL, 0, 0,
 				cli_nps->write.buf,
 				cli_nps->write.ofs,
-				TSTREAM_CLI_NP_BUF_SIZE);
+				TSTREAM_CLI_NP_MAX_BUF_SIZE);
 	if (tevent_req_nomem(subreq, req)) {
 		return;
 	}
@@ -713,7 +744,7 @@ static void tstream_cli_np_readv_trans_done(struct tevent_req *subreq)
 		return;
 	}
 
-	if (received > TSTREAM_CLI_NP_BUF_SIZE) {
+	if (received > TSTREAM_CLI_NP_MAX_BUF_SIZE) {
 		tstream_cli_np_readv_disconnect_now(req, EIO, __location__);
 		return;
 	}
@@ -725,8 +756,7 @@ static void tstream_cli_np_readv_trans_done(struct tevent_req *subreq)
 
 	cli_nps->read.ofs = 0;
 	cli_nps->read.left = received;
-	memcpy(cli_nps->read.buf, rcvbuf, received);
-	TALLOC_FREE(rcvbuf);
+	cli_nps->read.buf = talloc_move(cli_nps, &rcvbuf);
 
 	if (cli_nps->trans.write_req == NULL) {
 		tstream_cli_np_readv_read_next(req);
@@ -788,7 +818,7 @@ static void tstream_cli_np_readv_read_done(struct tevent_req *subreq)
 		return;
 	}
 
-	if (received > TSTREAM_CLI_NP_BUF_SIZE) {
+	if (received > TSTREAM_CLI_NP_MAX_BUF_SIZE) {
 		TALLOC_FREE(subreq);
 		tstream_cli_np_readv_disconnect_now(req, EIO, __location__);
 		return;
@@ -802,6 +832,12 @@ static void tstream_cli_np_readv_read_done(struct tevent_req *subreq)
 
 	cli_nps->read.ofs = 0;
 	cli_nps->read.left = received;
+	cli_nps->read.buf = talloc_array(cli_nps, uint8_t, received);
+	if (cli_nps->read.buf == NULL) {
+		TALLOC_FREE(subreq);
+		tevent_req_nomem(cli_nps->read.buf, req);
+		return;
+	}
 	memcpy(cli_nps->read.buf, rcvbuf, received);
 	TALLOC_FREE(subreq);
 
@@ -852,9 +888,8 @@ static void tstream_cli_np_readv_disconnect_done(struct tevent_req *subreq)
 		tevent_req_data(req, struct tstream_cli_np_readv_state);
 	struct tstream_cli_np *cli_nps =
 		tstream_context_data(state->stream, struct tstream_cli_np);
-	NTSTATUS status;
 
-	status = cli_close_recv(subreq);
+	cli_close_recv(subreq);
 	TALLOC_FREE(subreq);
 
 	cli_nps->cli = NULL;
diff --git a/source3/libsmb/cliconnect.c b/source3/libsmb/cliconnect.c
index e3d97b60fa7..62e3a35a369 100644
--- a/source3/libsmb/cliconnect.c
+++ b/source3/libsmb/cliconnect.c
@@ -3,6 +3,8 @@
    client connect/disconnect routines
    Copyright (C) Andrew Tridgell 1994-1998
    Copyright (C) Andrew Bartlett 2001-2003
+   Copyright (C) Volker Lendecke 2011
+   Copyright (C) Jeremy Allison 2011
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -19,6 +21,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "popt_common.h"
 #include "../libcli/auth/libcli_auth.h"
 #include "../libcli/auth/spnego.h"
@@ -26,8 +29,10 @@
 #include "../libcli/auth/ntlmssp.h"
 #include "libads/kerberos_proto.h"
 #include "krb5_env.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
 #include "libsmb/nmblib.h"
+#include "read_smb.h"
 
 static const struct {
 	int prot;
@@ -47,6 +52,40 @@ static const struct {
 
 #define STAR_SMBSERVER "*SMBSERVER"
 
+/********************************************************
+ Utility function to ensure we always return at least
+ a valid char * pointer to an empty string for the
+ cli->server_os, cli->server_type and cli->server_domain
+ strings.
+*******************************************************/
+
+static NTSTATUS smb_bytes_talloc_string(struct cli_state *cli,
+					char *inbuf,
+					char **dest,
+					uint8_t *src,
+					size_t srclen,
+					ssize_t *destlen)
+{
+	*destlen = clistr_pull_talloc(cli,
+				inbuf,
+				SVAL(inbuf, smb_flg2),
+				dest,
+				(char *)src,
+				srclen,
+				STR_TERMINATE);
+	if (*destlen == -1) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	if (*dest == NULL) {
+		*dest = talloc_strdup(cli, "");
+		if (*dest == NULL) {
+			return NT_STATUS_NO_MEMORY;
+		}
+	}
+	return NT_STATUS_OK;
+}
+
 /**
  * Set the user session key for a connection
  * @param cli The cli structure to add it too
@@ -63,86 +102,267 @@ static void cli_set_session_key (struct cli_state *cli, const DATA_BLOB session_
  Do an old lanman2 style session setup.
 ****************************************************************************/
 
-static NTSTATUS cli_session_setup_lanman2(struct cli_state *cli,
-					  const char *user, 
-					  const char *pass, size_t passlen,
-					  const char *workgroup)
+struct cli_session_setup_lanman2_state {
+	struct cli_state *cli;
+	uint16_t vwv[10];
+	const char *user;
+};
+
+static void cli_session_setup_lanman2_done(struct tevent_req *subreq);
+
+static struct tevent_req *cli_session_setup_lanman2_send(
+	TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+	struct cli_state *cli, const char *user,
+	const char *pass, size_t passlen,
+	const char *workgroup)
 {
+	struct tevent_req *req, *subreq;
+	struct cli_session_setup_lanman2_state *state;
 	DATA_BLOB lm_response = data_blob_null;
-	NTSTATUS status;
-	fstring pword;
-	char *p;
+	uint16_t *vwv;
+	uint8_t *bytes;
+	char *tmp;
 
-	if (passlen > sizeof(pword)-1) {
-		return NT_STATUS_INVALID_PARAMETER;
+	req = tevent_req_create(mem_ctx, &state,
+				struct cli_session_setup_lanman2_state);
+	if (req == NULL) {
+		return NULL;
 	}
+	state->cli = cli;
+	state->user = user;
+	vwv = state->vwv;
 
-	/* LANMAN servers predate NT status codes and Unicode and ignore those 
-	   smb flags so we must disable the corresponding default capabilities  
-	   that would otherwise cause the Unicode and NT Status flags to be
-	   set (and even returned by the server) */
+	/*
+	 * LANMAN servers predate NT status codes and Unicode and
+	 * ignore those smb flags so we must disable the corresponding
+	 * default capabilities that would otherwise cause the Unicode
+	 * and NT Status flags to be set (and even returned by the
+	 * server)
+	 */
 
 	cli->capabilities &= ~(CAP_UNICODE | CAP_STATUS32);
 
-	/* if in share level security then don't send a password now */
-	if (!(cli->sec_mode & NEGOTIATE_SECURITY_USER_LEVEL))
+	/*
+	 * if in share level security then don't send a password now
+	 */
+	if (!(cli->sec_mode & NEGOTIATE_SECURITY_USER_LEVEL)) {
 		passlen = 0;
+	}
 
-	if (passlen > 0 && (cli->sec_mode & NEGOTIATE_SECURITY_CHALLENGE_RESPONSE) && passlen != 24) {
-		/* Encrypted mode needed, and non encrypted password supplied. */
+	if (passlen > 0
+	    && (cli->sec_mode & NEGOTIATE_SECURITY_CHALLENGE_RESPONSE)
+	    && passlen != 24) {
+		/*
+		 * Encrypted mode needed, and non encrypted password
+		 * supplied.
+		 */
 		lm_response = data_blob(NULL, 24);
-		if (!SMBencrypt(pass, cli->secblob.data,(uchar *)lm_response.data)) {
-			DEBUG(1, ("Password is > 14 chars in length, and is therefore incompatible with Lanman authentication\n"));
-			return NT_STATUS_ACCESS_DENIED;
+		if (tevent_req_nomem(lm_response.data, req)) {
+			return tevent_req_post(req, ev);
+		}
+
+		if (!SMBencrypt(pass, cli->secblob.data,
+				(uint8_t *)lm_response.data)) {
+			DEBUG(1, ("Password is > 14 chars in length, and is "
+				  "therefore incompatible with Lanman "
+				  "authentication\n"));
+			tevent_req_nterror(req, NT_STATUS_ACCESS_DENIED);
+			return tevent_req_post(req, ev);
 		}
-	} else if ((cli->sec_mode & NEGOTIATE_SECURITY_CHALLENGE_RESPONSE) && passlen == 24) {
-		/* Encrypted mode needed, and encrypted password supplied. */
+	} else if ((cli->sec_mode & NEGOTIATE_SECURITY_CHALLENGE_RESPONSE)
+		   && passlen == 24) {
+		/*
+		 * Encrypted mode needed, and encrypted password
+		 * supplied.
+		 */
 		lm_response = data_blob(pass, passlen);
+		if (tevent_req_nomem(lm_response.data, req)) {
+			return tevent_req_post(req, ev);
+		}
 	} else if (passlen > 0) {
-		/* Plaintext mode needed, assume plaintext supplied. */
-		passlen = clistr_push(cli, pword, pass, sizeof(pword), STR_TERMINATE);
+		uint8_t *buf;
+		size_t converted_size;
+		/*
+		 * Plaintext mode needed, assume plaintext supplied.
+		 */
+		buf = talloc_array(talloc_tos(), uint8_t, 0);
+		buf = smb_bytes_push_str(buf, cli_ucs2(cli), pass, passlen+1,
+					 &converted_size);
+		if (tevent_req_nomem(buf, req)) {
+			return tevent_req_post(req, ev);
+		}
 		lm_response = data_blob(pass, passlen);
+		TALLOC_FREE(buf);
+		if (tevent_req_nomem(lm_response.data, req)) {
+			return tevent_req_post(req, ev);
+		}
 	}
 
-	/* send a session setup command */
-	memset(cli->outbuf,'\0',smb_size);
-	cli_set_message(cli->outbuf,10, 0, True);
-	SCVAL(cli->outbuf,smb_com,SMBsesssetupX);
-	cli_setup_packet(cli);
+	SCVAL(vwv+0, 0, 0xff);
+	SCVAL(vwv+0, 1, 0);
+	SSVAL(vwv+1, 0, 0);
+	SSVAL(vwv+2, 0, CLI_BUFFER_SIZE);
+	SSVAL(vwv+3, 0, 2);
+	SSVAL(vwv+4, 0, 1);
+	SIVAL(vwv+5, 0, cli->sesskey);
+	SSVAL(vwv+7, 0, lm_response.length);
 
-	SCVAL(cli->outbuf,smb_vwv0,0xFF);
-	SSVAL(cli->outbuf,smb_vwv2,cli->max_xmit);
-	SSVAL(cli->outbuf,smb_vwv3,2);
-	SSVAL(cli->outbuf,smb_vwv4,1);
-	SIVAL(cli->outbuf,smb_vwv5,cli->sesskey);
-	SSVAL(cli->outbuf,smb_vwv7,lm_response.length);
+	bytes = talloc_array(state, uint8_t, lm_response.length);
+	if (tevent_req_nomem(bytes, req)) {
+		return tevent_req_post(req, ev);
+	}
+	if (lm_response.length != 0) {
+		memcpy(bytes, lm_response.data, lm_response.length);
+	}
+	data_blob_free(&lm_response);
 
-	p = smb_buf(cli->outbuf);
-	memcpy(p,lm_response.data,lm_response.length);
-	p += lm_response.length;
-	p += clistr_push(cli, p, user, -1, STR_TERMINATE|STR_UPPER);
-	p += clistr_push(cli, p, workgroup, -1, STR_TERMINATE|STR_UPPER);
-	p += clistr_push(cli, p, "Unix", -1, STR_TERMINATE);
-	p += clistr_push(cli, p, "Samba", -1, STR_TERMINATE);
-	cli_setup_bcc(cli, p);
+	tmp = talloc_strdup_upper(talloc_tos(), user);
+	if (tevent_req_nomem(tmp, req)) {
+		return tevent_req_post(req, ev);
+	}
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), tmp, strlen(tmp)+1,
+				   NULL);
+	TALLOC_FREE(tmp);
 
-	if (!cli_send_smb(cli) || !cli_receive_smb(cli)) {
-		return cli_nt_error(cli);
+	tmp = talloc_strdup_upper(talloc_tos(), workgroup);
+	if (tevent_req_nomem(tmp, req)) {
+		return tevent_req_post(req, ev);
 	}
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), tmp, strlen(tmp)+1,
+				   NULL);
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), "Unix", 5, NULL);
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), "Samba", 6, NULL);
 
-	show_msg(cli->inbuf);
+	if (tevent_req_nomem(bytes, req)) {
+		return tevent_req_post(req, ev);
+	}
 
-	if (cli_is_error(cli)) {
-		return cli_nt_error(cli);
+	subreq = cli_smb_send(state, ev, cli, SMBsesssetupX, 0, 10, vwv,
+			      talloc_get_size(bytes), bytes);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
 	}
+	tevent_req_set_callback(subreq, cli_session_setup_lanman2_done, req);
+	return req;
+}
 
-	/* use the returned vuid from now on */
-	cli->vuid = SVAL(cli->inbuf,smb_uid);	
-	status = cli_set_username(cli, user);
+static void cli_session_setup_lanman2_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct cli_session_setup_lanman2_state *state = tevent_req_data(
+		req, struct cli_session_setup_lanman2_state);
+	struct cli_state *cli = state->cli;
+	uint32_t num_bytes;
+	uint8_t *in;
+	char *inbuf;
+	uint8_t *bytes;
+	uint8_t *p;
+	NTSTATUS status;
+	ssize_t ret;
+	uint8_t wct;
+	uint16_t *vwv;
+
+	status = cli_smb_recv(subreq, state, &in, 3, &wct, &vwv,
+			      &num_bytes, &bytes);
+	TALLOC_FREE(subreq);
 	if (!NT_STATUS_IS_OK(status)) {
-		return status;
+		tevent_req_nterror(req, status);
+		return;
 	}
-	return NT_STATUS_OK;
+
+	inbuf = (char *)in;
+	p = bytes;
+
+	cli->vuid = SVAL(inbuf, smb_uid);
+	cli->is_guestlogin = ((SVAL(vwv+2, 0) & 1) != 0);
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_os,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_type,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_domain,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	if (strstr(cli->server_type, "Samba")) {
+		cli->is_samba = True;
+	}
+	status = cli_set_username(cli, state->user);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	tevent_req_done(req);
+}
+
+static NTSTATUS cli_session_setup_lanman2_recv(struct tevent_req *req)
+{
+	return tevent_req_simple_recv_ntstatus(req);
+}
+
+static NTSTATUS cli_session_setup_lanman2(struct cli_state *cli, const char *user,
+					  const char *pass, size_t passlen,
+					  const char *workgroup)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	if (cli_has_async_calls(cli)) {
+		/*
+		 * Can't use sync call while an async call is in flight
+		 */
+		status = NT_STATUS_INVALID_PARAMETER;
+		goto fail;
+	}
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = cli_session_setup_lanman2_send(frame, ev, cli, user, pass, passlen,
+					     workgroup);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
+		goto fail;
+	}
+	status = cli_session_setup_lanman2_recv(req);
+ fail:
+	TALLOC_FREE(frame);
+	return status;
 }
 
 /****************************************************************************
@@ -267,8 +487,11 @@ static void cli_session_setup_guest_done(struct tevent_req *subreq)
 	uint8_t *bytes;
 	uint8_t *p;
 	NTSTATUS status;
+	ssize_t ret;
+	uint8_t wct;
+	uint16_t *vwv;
 
-	status = cli_smb_recv(subreq, state, &in, 0, NULL, NULL,
+	status = cli_smb_recv(subreq, state, &in, 3, &wct, &vwv,
 			      &num_bytes, &bytes);
 	TALLOC_FREE(subreq);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -280,13 +503,46 @@ static void cli_session_setup_guest_done(struct tevent_req *subreq)
 	p = bytes;
 
 	cli->vuid = SVAL(inbuf, smb_uid);
+	cli->is_guestlogin = ((SVAL(vwv+2, 0) & 1) != 0);
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_os,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_type,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
 
-	p += clistr_pull(inbuf, cli->server_os, (char *)p, sizeof(fstring),
-			 bytes+num_bytes-p, STR_TERMINATE);
-	p += clistr_pull(inbuf, cli->server_type, (char *)p, sizeof(fstring),
-			 bytes+num_bytes-p, STR_TERMINATE);
-	p += clistr_pull(inbuf, cli->server_domain, (char *)p, sizeof(fstring),
-			 bytes+num_bytes-p, STR_TERMINATE);
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_domain,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
 
 	if (strstr(cli->server_type, "Samba")) {
 		cli->is_samba = True;
@@ -340,9 +596,6 @@ static NTSTATUS cli_session_setup_guest(struct cli_state *cli)
 	status = cli_session_setup_guest_recv(req);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -350,82 +603,201 @@ static NTSTATUS cli_session_setup_guest(struct cli_state *cli)
  Do a NT1 plaintext session setup.
 ****************************************************************************/
 
-static NTSTATUS cli_session_setup_plaintext(struct cli_state *cli,
-					    const char *user, const char *pass,
-					    const char *workgroup)
+struct cli_session_setup_plain_state {
+	struct cli_state *cli;
+	uint16_t vwv[13];
+	const char *user;
+};
+
+static void cli_session_setup_plain_done(struct tevent_req *subreq);
+
+static struct tevent_req *cli_session_setup_plain_send(
+	TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+	struct cli_state *cli,
+	const char *user, const char *pass, const char *workgroup)
 {
-	uint32 capabilities = cli_session_setup_capabilities(cli);
-	char *p;
+	struct tevent_req *req, *subreq;
+	struct cli_session_setup_plain_state *state;
+	uint16_t *vwv;
+	uint8_t *bytes;
+	size_t passlen;
+	char *version;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct cli_session_setup_plain_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->cli = cli;
+	state->user = user;
+	vwv = state->vwv;
+
+	SCVAL(vwv+0, 0, 0xff);
+	SCVAL(vwv+0, 1, 0);
+	SSVAL(vwv+1, 0, 0);
+	SSVAL(vwv+2, 0, CLI_BUFFER_SIZE);
+	SSVAL(vwv+3, 0, 2);
+	SSVAL(vwv+4, 0, cli->pid);
+	SIVAL(vwv+5, 0, cli->sesskey);
+	SSVAL(vwv+7, 0, 0);
+	SSVAL(vwv+8, 0, 0);
+	SSVAL(vwv+9, 0, 0);
+	SSVAL(vwv+10, 0, 0);
+	SIVAL(vwv+11, 0, cli_session_setup_capabilities(cli));
+
+	bytes = talloc_array(state, uint8_t, 0);
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), pass, strlen(pass)+1,
+				   &passlen);
+	if (tevent_req_nomem(bytes, req)) {
+		return tevent_req_post(req, ev);
+	}
+	SSVAL(vwv + (cli_ucs2(cli) ? 8 : 7), 0, passlen);
+
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli),
+				   user, strlen(user)+1, NULL);
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli),
+				   workgroup, strlen(workgroup)+1, NULL);
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli),
+				   "Unix", 5, NULL);
+
+	version = talloc_asprintf(talloc_tos(), "Samba %s",
+				  samba_version_string());
+	if (tevent_req_nomem(version, req)){
+		return tevent_req_post(req, ev);
+	}
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli),
+				   version, strlen(version)+1, NULL);
+	TALLOC_FREE(version);
+
+	if (tevent_req_nomem(bytes, req)) {
+		return tevent_req_post(req, ev);
+	}
+
+	subreq = cli_smb_send(state, ev, cli, SMBsesssetupX, 0, 13, vwv,
+			      talloc_get_size(bytes), bytes);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, cli_session_setup_plain_done, req);
+	return req;
+}
+
+static void cli_session_setup_plain_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct cli_session_setup_plain_state *state = tevent_req_data(
+		req, struct cli_session_setup_plain_state);
+	struct cli_state *cli = state->cli;
+	uint32_t num_bytes;
+	uint8_t *in;
+	char *inbuf;
+	uint8_t *bytes;
+	uint8_t *p;
 	NTSTATUS status;
-	fstring lanman;
-
-	fstr_sprintf( lanman, "Samba %s", samba_version_string());
-
-	memset(cli->outbuf, '\0', smb_size);
-	cli_set_message(cli->outbuf,13,0,True);
-	SCVAL(cli->outbuf,smb_com,SMBsesssetupX);
-	cli_setup_packet(cli);
-
-	SCVAL(cli->outbuf,smb_vwv0,0xFF);
-	SSVAL(cli->outbuf,smb_vwv2,CLI_BUFFER_SIZE);
-	SSVAL(cli->outbuf,smb_vwv3,2);
-	SSVAL(cli->outbuf,smb_vwv4,cli->pid);
-	SIVAL(cli->outbuf,smb_vwv5,cli->sesskey);
-	SSVAL(cli->outbuf,smb_vwv8,0);
-	SIVAL(cli->outbuf,smb_vwv11,capabilities); 
-	p = smb_buf(cli->outbuf);
-
-	/* check wether to send the ASCII or UNICODE version of the password */
-
-	if ( (capabilities & CAP_UNICODE) == 0 ) {
-		p += clistr_push(cli, p, pass, -1, STR_TERMINATE); /* password */
-		SSVAL(cli->outbuf,smb_vwv7,PTR_DIFF(p, smb_buf(cli->outbuf)));
-	}
-	else {
-		/* For ucs2 passwords clistr_push calls ucs2_align, which causes
-		 * the space taken by the unicode password to be one byte too
-		 * long (as we're on an odd byte boundary here). Reduce the
-		 * count by 1 to cope with this. Fixes smbclient against NetApp
-		 * servers which can't cope. Fix from
-		 * bryan.kolodziej@allenlund.com in bug #3840.
-		 */
-		p += clistr_push(cli, p, pass, -1, STR_UNICODE|STR_TERMINATE); /* unicode password */
-		SSVAL(cli->outbuf,smb_vwv8,PTR_DIFF(p, smb_buf(cli->outbuf))-1);	
+	ssize_t ret;
+	uint8_t wct;
+	uint16_t *vwv;
+
+	status = cli_smb_recv(subreq, state, &in, 3, &wct, &vwv,
+			      &num_bytes, &bytes);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
 	}
 
-	p += clistr_push(cli, p, user, -1, STR_TERMINATE); /* username */
-	p += clistr_push(cli, p, workgroup, -1, STR_TERMINATE); /* workgroup */
-	p += clistr_push(cli, p, "Unix", -1, STR_TERMINATE);
-	p += clistr_push(cli, p, lanman, -1, STR_TERMINATE);
-	cli_setup_bcc(cli, p);
+	inbuf = (char *)in;
+	p = bytes;
+
+	cli->vuid = SVAL(inbuf, smb_uid);
+	cli->is_guestlogin = ((SVAL(vwv+2, 0) & 1) != 0);
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_os,
+					p,
+					bytes+num_bytes-p,
+					&ret);
 
-	if (!cli_send_smb(cli) || !cli_receive_smb(cli)) {
-		return cli_nt_error(cli);
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
 	}
+	p += ret;
 
-	show_msg(cli->inbuf);
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_type,
+					p,
+					bytes+num_bytes-p,
+					&ret);
 
-	if (cli_is_error(cli)) {
-		return cli_nt_error(cli);
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
 	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_domain,
+					p,
+					bytes+num_bytes-p,
+					&ret);
 
-	cli->vuid = SVAL(cli->inbuf,smb_uid);
-	p = smb_buf(cli->inbuf);
-	p += clistr_pull(cli->inbuf, cli->server_os, p, sizeof(fstring),
-			 -1, STR_TERMINATE);
-	p += clistr_pull(cli->inbuf, cli->server_type, p, sizeof(fstring),
-			 -1, STR_TERMINATE);
-	p += clistr_pull(cli->inbuf, cli->server_domain, p, sizeof(fstring),
-			 -1, STR_TERMINATE);
-	status = cli_set_username(cli, user);
 	if (!NT_STATUS_IS_OK(status)) {
-		return status;
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = cli_set_username(cli, state->user);
+	if (tevent_req_nterror(req, status)) {
+		return;
 	}
 	if (strstr(cli->server_type, "Samba")) {
 		cli->is_samba = True;
 	}
+	tevent_req_done(req);
+}
 
-	return NT_STATUS_OK;
+static NTSTATUS cli_session_setup_plain_recv(struct tevent_req *req)
+{
+	return tevent_req_simple_recv_ntstatus(req);
+}
+
+static NTSTATUS cli_session_setup_plain(struct cli_state *cli,
+					const char *user, const char *pass,
+					const char *workgroup)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	if (cli_has_async_calls(cli)) {
+		/*
+		 * Can't use sync call while an async call is in flight
+		 */
+		status = NT_STATUS_INVALID_PARAMETER;
+		goto fail;
+	}
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = cli_session_setup_plain_send(frame, ev, cli, user, pass,
+					   workgroup);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
+		goto fail;
+	}
+	status = cli_session_setup_plain_recv(req);
+ fail:
+	TALLOC_FREE(frame);
+	return status;
 }
 
 /****************************************************************************
@@ -438,18 +810,40 @@ static NTSTATUS cli_session_setup_plaintext(struct cli_state *cli,
    @param workgroup The user's domain.
 ****************************************************************************/
 
-static NTSTATUS cli_session_setup_nt1(struct cli_state *cli, const char *user, 
-				      const char *pass, size_t passlen,
-				      const char *ntpass, size_t ntpasslen,
-				      const char *workgroup)
+struct cli_session_setup_nt1_state {
+	struct cli_state *cli;
+	uint16_t vwv[13];
+	DATA_BLOB response;
+	DATA_BLOB session_key;
+	const char *user;
+};
+
+static void cli_session_setup_nt1_done(struct tevent_req *subreq);
+
+static struct tevent_req *cli_session_setup_nt1_send(
+	TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+	struct cli_state *cli, const char *user,
+	const char *pass, size_t passlen,
+	const char *ntpass, size_t ntpasslen,
+	const char *workgroup)
 {
-	uint32 capabilities = cli_session_setup_capabilities(cli);
+	struct tevent_req *req, *subreq;
+	struct cli_session_setup_nt1_state *state;
 	DATA_BLOB lm_response = data_blob_null;
 	DATA_BLOB nt_response = data_blob_null;
 	DATA_BLOB session_key = data_blob_null;
-	NTSTATUS result;
-	char *p;
-	bool ok;
+	uint16_t *vwv;
+	uint8_t *bytes;
+	char *workgroup_upper;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct cli_session_setup_nt1_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->cli = cli;
+	state->user = user;
+	vwv = state->vwv;
 
 	if (passlen == 0) {
 		/* do nothing - guest login */
@@ -457,20 +851,35 @@ static NTSTATUS cli_session_setup_nt1(struct cli_state *cli, const char *user,
 		if (lp_client_ntlmv2_auth()) {
 			DATA_BLOB server_chal;
 			DATA_BLOB names_blob;
-			server_chal = data_blob(cli->secblob.data, MIN(cli->secblob.length, 8)); 
 
-			/* note that the 'workgroup' here is a best guess - we don't know
-			   the server's domain at this point.  The 'server name' is also
-			   dodgy... 
-			*/
-			names_blob = NTLMv2_generate_names_blob(NULL, cli->called.name, workgroup);
+			server_chal = data_blob(cli->secblob.data,
+						MIN(cli->secblob.length, 8));
+			if (tevent_req_nomem(server_chal.data, req)) {
+				return tevent_req_post(req, ev);
+			}
+
+			/*
+			 * note that the 'workgroup' here is a best
+			 * guess - we don't know the server's domain
+			 * at this point.  The 'server name' is also
+			 * dodgy...
+			 */
+			names_blob = NTLMv2_generate_names_blob(
+				NULL, cli->called.name, workgroup);
+
+			if (tevent_req_nomem(names_blob.data, req)) {
+				return tevent_req_post(req, ev);
+			}
 
-			if (!SMBNTLMv2encrypt(NULL, user, workgroup, pass, &server_chal, 
-					      &names_blob,
-					      &lm_response, &nt_response, NULL, &session_key)) {
+			if (!SMBNTLMv2encrypt(NULL, user, workgroup, pass,
+					      &server_chal, &names_blob,
+					      &lm_response, &nt_response,
+					      NULL, &session_key)) {
 				data_blob_free(&names_blob);
 				data_blob_free(&server_chal);
-				return NT_STATUS_ACCESS_DENIED;
+				tevent_req_nterror(
+					req, NT_STATUS_ACCESS_DENIED);
+				return tevent_req_post(req, ev);
 			}
 			data_blob_free(&names_blob);
 			data_blob_free(&server_chal);
@@ -483,23 +892,50 @@ static NTSTATUS cli_session_setup_nt1(struct cli_state *cli, const char *user,
 			nt_response = data_blob_null;
 #else
 			nt_response = data_blob(NULL, 24);
-			SMBNTencrypt(pass,cli->secblob.data,nt_response.data);
+			if (tevent_req_nomem(nt_response.data, req)) {
+				return tevent_req_post(req, ev);
+			}
+
+			SMBNTencrypt(pass, cli->secblob.data,
+				     nt_response.data);
 #endif
 			/* non encrypted password supplied. Ignore ntpass. */
 			if (lp_client_lanman_auth()) {
+
 				lm_response = data_blob(NULL, 24);
-				if (!SMBencrypt(pass,cli->secblob.data, lm_response.data)) {
-					/* Oops, the LM response is invalid, just put 
-					   the NT response there instead */
+				if (tevent_req_nomem(lm_response.data, req)) {
+					return tevent_req_post(req, ev);
+				}
+
+				if (!SMBencrypt(pass,cli->secblob.data,
+						lm_response.data)) {
+					/*
+					 * Oops, the LM response is
+					 * invalid, just put the NT
+					 * response there instead
+					 */
 					data_blob_free(&lm_response);
-					lm_response = data_blob(nt_response.data, nt_response.length);
+					lm_response = data_blob(
+						nt_response.data,
+						nt_response.length);
 				}
 			} else {
-				/* LM disabled, place NT# in LM field instead */
-				lm_response = data_blob(nt_response.data, nt_response.length);
+				/*
+				 * LM disabled, place NT# in LM field
+				 * instead
+				 */
+				lm_response = data_blob(
+					nt_response.data, nt_response.length);
+			}
+
+			if (tevent_req_nomem(lm_response.data, req)) {
+				return tevent_req_post(req, ev);
 			}
 
 			session_key = data_blob(NULL, 16);
+			if (tevent_req_nomem(session_key.data, req)) {
+				return tevent_req_post(req, ev);
+			}
 #ifdef LANMAN_ONLY
 			E_deshash(pass, session_key.data);
 			memset(&session_key.data[8], '\0', 8);
@@ -514,94 +950,219 @@ static NTSTATUS cli_session_setup_nt1(struct cli_state *cli, const char *user,
 		   signing because we don't have original key */
 
 		lm_response = data_blob(pass, passlen);
+		if (tevent_req_nomem(lm_response.data, req)) {
+			return tevent_req_post(req, ev);
+		}
+
 		nt_response = data_blob(ntpass, ntpasslen);
+		if (tevent_req_nomem(nt_response.data, req)) {
+			return tevent_req_post(req, ev);
+		}
 	}
 
-	/* send a session setup command */
-	memset(cli->outbuf,'\0',smb_size);
+#ifdef LANMAN_ONLY
+	state->response = data_blob_talloc(
+		state, lm_response.data, lm_response.length);
+#else
+	state->response = data_blob_talloc(
+		state, nt_response.data, nt_response.length);
+#endif
+	if (tevent_req_nomem(state->response.data, req)) {
+		return tevent_req_post(req, ev);
+	}
 
-	cli_set_message(cli->outbuf,13,0,True);
-	SCVAL(cli->outbuf,smb_com,SMBsesssetupX);
-	cli_setup_packet(cli);
+	if (session_key.data) {
+		state->session_key = data_blob_talloc(
+			state, session_key.data, session_key.length);
+		if (tevent_req_nomem(state->session_key.data, req)) {
+			return tevent_req_post(req, ev);
+		}
+	}
+	data_blob_free(&session_key);
+
+	SCVAL(vwv+0, 0, 0xff);
+	SCVAL(vwv+0, 1, 0);
+	SSVAL(vwv+1, 0, 0);
+	SSVAL(vwv+2, 0, CLI_BUFFER_SIZE);
+	SSVAL(vwv+3, 0, 2);
+	SSVAL(vwv+4, 0, cli->pid);
+	SIVAL(vwv+5, 0, cli->sesskey);
+	SSVAL(vwv+7, 0, lm_response.length);
+	SSVAL(vwv+8, 0, nt_response.length);
+	SSVAL(vwv+9, 0, 0);
+	SSVAL(vwv+10, 0, 0);
+	SIVAL(vwv+11, 0, cli_session_setup_capabilities(cli));
 
-	SCVAL(cli->outbuf,smb_vwv0,0xFF);
-	SSVAL(cli->outbuf,smb_vwv2,CLI_BUFFER_SIZE);
-	SSVAL(cli->outbuf,smb_vwv3,2);
-	SSVAL(cli->outbuf,smb_vwv4,cli->pid);
-	SIVAL(cli->outbuf,smb_vwv5,cli->sesskey);
-	SSVAL(cli->outbuf,smb_vwv7,lm_response.length);
-	SSVAL(cli->outbuf,smb_vwv8,nt_response.length);
-	SIVAL(cli->outbuf,smb_vwv11,capabilities); 
-	p = smb_buf(cli->outbuf);
-	if (lm_response.length) {
-		memcpy(p,lm_response.data, lm_response.length); p += lm_response.length;
+	bytes = talloc_array(state, uint8_t,
+			     lm_response.length + nt_response.length);
+	if (tevent_req_nomem(bytes, req)) {
+		return tevent_req_post(req, ev);
+	}
+	if (lm_response.length != 0) {
+		memcpy(bytes, lm_response.data, lm_response.length);
 	}
-	if (nt_response.length) {
-		memcpy(p,nt_response.data, nt_response.length); p += nt_response.length;
+	if (nt_response.length != 0) {
+		memcpy(bytes + lm_response.length,
+		       nt_response.data, nt_response.length);
 	}
-	p += clistr_push(cli, p, user, -1, STR_TERMINATE);
+	data_blob_free(&lm_response);
+	data_blob_free(&nt_response);
 
-	/* Upper case here might help some NTLMv2 implementations */
-	p += clistr_push(cli, p, workgroup, -1, STR_TERMINATE|STR_UPPER);
-	p += clistr_push(cli, p, "Unix", -1, STR_TERMINATE);
-	p += clistr_push(cli, p, "Samba", -1, STR_TERMINATE);
-	cli_setup_bcc(cli, p);
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli),
+				   user, strlen(user)+1, NULL);
 
-	if (!cli_send_smb(cli) || !cli_receive_smb(cli)) {
-		result = cli_nt_error(cli);
-		goto end;
+	/*
+	 * Upper case here might help some NTLMv2 implementations
+	 */
+	workgroup_upper = talloc_strdup_upper(talloc_tos(), workgroup);
+	if (tevent_req_nomem(workgroup_upper, req)) {
+		return tevent_req_post(req, ev);
 	}
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli),
+				   workgroup_upper, strlen(workgroup_upper)+1,
+				   NULL);
+	TALLOC_FREE(workgroup_upper);
 
-	/* show_msg(cli->inbuf); */
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), "Unix", 5, NULL);
+	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), "Samba", 6, NULL);
+	if (tevent_req_nomem(bytes, req)) {
+		return tevent_req_post(req, ev);
+	}
 
-	if (cli_is_error(cli)) {
-		result = cli_nt_error(cli);
-		goto end;
+	subreq = cli_smb_send(state, ev, cli, SMBsesssetupX, 0, 13, vwv,
+			      talloc_get_size(bytes), bytes);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
 	}
+	tevent_req_set_callback(subreq, cli_session_setup_nt1_done, req);
+	return req;
+}
 
-#ifdef LANMAN_ONLY
-	ok = cli_simple_set_signing(cli, session_key, lm_response);
-#else
-	ok = cli_simple_set_signing(cli, session_key, nt_response);
-#endif
-	if (ok) {
-		if (!cli_check_sign_mac(cli, cli->inbuf, 1)) {
-			result = NT_STATUS_ACCESS_DENIED;
-			goto end;
-		}
+static void cli_session_setup_nt1_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct cli_session_setup_nt1_state *state = tevent_req_data(
+		req, struct cli_session_setup_nt1_state);
+	struct cli_state *cli = state->cli;
+	uint32_t num_bytes;
+	uint8_t *in;
+	char *inbuf;
+	uint8_t *bytes;
+	uint8_t *p;
+	NTSTATUS status;
+	ssize_t ret;
+	uint8_t wct;
+	uint16_t *vwv;
+
+	status = cli_smb_recv(subreq, state, &in, 3, &wct, &vwv,
+			      &num_bytes, &bytes);
+	TALLOC_FREE(subreq);
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
 	}
 
-	/* use the returned vuid from now on */
-	cli->vuid = SVAL(cli->inbuf,smb_uid);
+	inbuf = (char *)in;
+	p = bytes;
 
-	p = smb_buf(cli->inbuf);
-	p += clistr_pull(cli->inbuf, cli->server_os, p, sizeof(fstring),
-			 -1, STR_TERMINATE);
-	p += clistr_pull(cli->inbuf, cli->server_type, p, sizeof(fstring),
-			 -1, STR_TERMINATE);
-	p += clistr_pull(cli->inbuf, cli->server_domain, p, sizeof(fstring),
-			 -1, STR_TERMINATE);
+	cli->vuid = SVAL(inbuf, smb_uid);
+	cli->is_guestlogin = ((SVAL(vwv+2, 0) & 1) != 0);
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_os,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_type,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					inbuf,
+					&cli->server_domain,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
 
 	if (strstr(cli->server_type, "Samba")) {
 		cli->is_samba = True;
 	}
 
-	result = cli_set_username(cli, user);
-	if (!NT_STATUS_IS_OK(result)) {
-		goto end;
+	status = cli_set_username(cli, state->user);
+	if (tevent_req_nterror(req, status)) {
+		return;
 	}
-
-	if (session_key.data) {
+	if (cli_simple_set_signing(cli, state->session_key, state->response)
+	    && !cli_check_sign_mac(cli, (char *)in, 1)) {
+		tevent_req_nterror(req, NT_STATUS_ACCESS_DENIED);
+		return;
+	}
+	if (state->session_key.data) {
 		/* Have plaintext orginal */
-		cli_set_session_key(cli, session_key);
+		cli_set_session_key(cli, state->session_key);
 	}
+	tevent_req_done(req);
+}
 
-	result = NT_STATUS_OK;
-end:	
-	data_blob_free(&lm_response);
-	data_blob_free(&nt_response);
-	data_blob_free(&session_key);
-	return result;
+static NTSTATUS cli_session_setup_nt1_recv(struct tevent_req *req)
+{
+	return tevent_req_simple_recv_ntstatus(req);
+}
+
+static NTSTATUS cli_session_setup_nt1(struct cli_state *cli, const char *user,
+				      const char *pass, size_t passlen,
+				      const char *ntpass, size_t ntpasslen,
+				      const char *workgroup)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	if (cli_has_async_calls(cli)) {
+		/*
+		 * Can't use sync call while an async call is in flight
+		 */
+		status = NT_STATUS_INVALID_PARAMETER;
+		goto fail;
+	}
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = cli_session_setup_nt1_send(frame, ev, cli, user, pass, passlen,
+					 ntpass, ntpasslen, workgroup);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
+		goto fail;
+	}
+	status = cli_session_setup_nt1_recv(req);
+ fail:
+	TALLOC_FREE(frame);
+	return status;
 }
 
 /* The following is calculated from :
@@ -729,8 +1290,9 @@ static void cli_sesssetup_blob_done(struct tevent_req *subreq)
 	uint8_t *p;
 	uint16_t blob_length;
 	uint8_t *inbuf;
+	ssize_t ret;
 
-	status = cli_smb_recv(subreq, state, &inbuf, 1, &wct, &vwv,
+	status = cli_smb_recv(subreq, state, &inbuf, 4, &wct, &vwv,
 			      &num_bytes, &bytes);
 	TALLOC_FREE(subreq);
 	if (!NT_STATUS_IS_OK(status)
@@ -744,6 +1306,7 @@ static void cli_sesssetup_blob_done(struct tevent_req *subreq)
 
 	state->inbuf = (char *)inbuf;
 	cli->vuid = SVAL(state->inbuf, smb_uid);
+	cli->is_guestlogin = ((SVAL(vwv+2, 0) & 1) != 0);
 
 	blob_length = SVAL(vwv+3, 0);
 	if (blob_length > num_bytes) {
@@ -754,15 +1317,44 @@ static void cli_sesssetup_blob_done(struct tevent_req *subreq)
 
 	p = bytes + blob_length;
 
-	p += clistr_pull(state->inbuf, cli->server_os,
-			 (char *)p, sizeof(fstring),
-			 bytes+num_bytes-p, STR_TERMINATE);
-	p += clistr_pull(state->inbuf, cli->server_type,
-			 (char *)p, sizeof(fstring),
-			 bytes+num_bytes-p, STR_TERMINATE);
-	p += clistr_pull(state->inbuf, cli->server_domain,
-			 (char *)p, sizeof(fstring),
-			 bytes+num_bytes-p, STR_TERMINATE);
+	status = smb_bytes_talloc_string(cli,
+					(char *)inbuf,
+					&cli->server_os,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					(char *)inbuf,
+					&cli->server_type,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
+
+	status = smb_bytes_talloc_string(cli,
+					(char *)inbuf,
+					&cli->server_domain,
+					p,
+					bytes+num_bytes-p,
+					&ret);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		tevent_req_nterror(req, status);
+		return;
+	}
+	p += ret;
 
 	if (strstr(cli->server_type, "Samba")) {
 		cli->is_samba = True;
@@ -773,7 +1365,7 @@ static void cli_sesssetup_blob_done(struct tevent_req *subreq)
 		 * More to send
 		 */
 		if (!cli_sesssetup_blob_next(state, &subreq)) {
-			tevent_req_nomem(NULL, req);
+			tevent_req_oom(req);
 			return;
 		}
 		tevent_req_set_callback(subreq, cli_sesssetup_blob_done, req);
@@ -833,7 +1425,7 @@ static void cli_session_setup_kerberos_done(struct tevent_req *subreq);
 
 static struct tevent_req *cli_session_setup_kerberos_send(
 	TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct cli_state *cli,
-	const char *principal, const char *workgroup)
+	const char *principal)
 {
 	struct tevent_req *req, *subreq;
 	struct cli_session_setup_kerberos_state *state;
@@ -921,8 +1513,7 @@ static ADS_STATUS cli_session_setup_kerberos_recv(struct tevent_req *req)
 }
 
 static ADS_STATUS cli_session_setup_kerberos(struct cli_state *cli,
-					     const char *principal,
-					     const char *workgroup)
+					     const char *principal)
 {
 	struct tevent_context *ev;
 	struct tevent_req *req;
@@ -935,8 +1526,7 @@ static ADS_STATUS cli_session_setup_kerberos(struct cli_state *cli,
 	if (ev == NULL) {
 		goto fail;
 	}
-	req = cli_session_setup_kerberos_send(ev, ev, cli, principal,
-					      workgroup);
+	req = cli_session_setup_kerberos_send(ev, ev, cli, principal);
 	if (req == NULL) {
 		goto fail;
 	}
@@ -1000,7 +1590,7 @@ static struct tevent_req *cli_session_setup_ntlmssp_send(
 	cli_temp_set_signing(cli);
 
 	status = ntlmssp_client_start(state,
-				      global_myname(),
+				      lp_netbios_name(),
 				      lp_workgroup(),
 				      lp_client_ntlmv2_auth(),
 				      &state->ntlmssp_state);
@@ -1063,8 +1653,14 @@ static void cli_session_setup_ntlmssp_done(struct tevent_req *subreq)
 
 	if (NT_STATUS_IS_OK(status)) {
 		if (state->cli->server_domain[0] == '\0') {
-			fstrcpy(state->cli->server_domain,
-				state->ntlmssp_state->server.netbios_domain);
+			TALLOC_FREE(state->cli->server_domain);
+			state->cli->server_domain = talloc_strdup(state->cli,
+						state->ntlmssp_state->server.netbios_domain);
+			if (state->cli->server_domain == NULL) {
+				TALLOC_FREE(subreq);
+				tevent_req_nterror(req, NT_STATUS_NO_MEMORY);
+				return;
+			}
 		}
 		cli_set_session_key(
 			state->cli, state->ntlmssp_state->session_key);
@@ -1179,9 +1775,6 @@ static NTSTATUS cli_session_setup_ntlmssp(struct cli_state *cli,
 	status = cli_session_setup_ntlmssp_recv(req);
 fail:
 	TALLOC_FREE(ev);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1291,6 +1884,9 @@ ADS_STATUS cli_session_setup_spnego(struct cli_state *cli, const char *user,
 			host = strchr_m(cli->desthost, '.');
 			if (dest_realm) {
 				realm = SMB_STRDUP(dest_realm);
+				if (!realm) {
+					return ADS_ERROR_NT(NT_STATUS_NO_MEMORY);
+				}
 				strupper_m(realm);
 			} else {
 				if (host) {
@@ -1302,25 +1898,38 @@ ADS_STATUS cli_session_setup_spnego(struct cli_state *cli, const char *user,
 				}
 			}
 
-			if (realm && *realm) {
-				principal = talloc_asprintf(talloc_tos(),
-							    "cifs/%s@%s",
-							    cli->desthost,
-							    realm);
-				if (!principal) {
-					SAFE_FREE(realm);
+			if (realm == NULL || *realm == '\0') {
+				realm = SMB_STRDUP(lp_realm());
+				if (!realm) {
 					return ADS_ERROR_NT(NT_STATUS_NO_MEMORY);
 				}
-				DEBUG(3,("cli_session_setup_spnego: guessed "
-					"server principal=%s\n",
-					principal ? principal : "<null>"));
+				strupper_m(realm);
+				DEBUG(3,("cli_session_setup_spnego: cannot "
+					"get realm from dest_realm %s, "
+					"desthost %s. Using default "
+					"smb.conf realm %s\n",
+					dest_realm ? dest_realm : "<null>",
+					cli->desthost,
+					realm));
 			}
+
+			principal = talloc_asprintf(talloc_tos(),
+						    "cifs/%s@%s",
+						    cli->desthost,
+						    realm);
+			if (!principal) {
+				SAFE_FREE(realm);
+				return ADS_ERROR_NT(NT_STATUS_NO_MEMORY);
+			}
+			DEBUG(3,("cli_session_setup_spnego: guessed "
+				"server principal=%s\n",
+				principal ? principal : "<null>"));
+
 			SAFE_FREE(realm);
 		}
 
 		if (principal) {
-			rc = cli_session_setup_kerberos(cli, principal,
-				dest_realm);
+			rc = cli_session_setup_kerberos(cli, principal);
 			if (ADS_ERR_OK(rc) || !cli->fallback_after_kerberos) {
 				TALLOC_FREE(principal);
 				return rc;
@@ -1361,12 +1970,15 @@ NTSTATUS cli_session_setup(struct cli_state *cli,
 			   const char *workgroup)
 {
 	char *p;
-	fstring user2;
+	char *user2;
 
 	if (user) {
-		fstrcpy(user2, user);
+		user2 = talloc_strdup(talloc_tos(), user);
 	} else {
-		user2[0] ='\0';
+		user2 = talloc_strdup(talloc_tos(), "");
+	}
+	if (user2 == NULL) {
+		return NT_STATUS_NO_MEMORY;
 	}
 
 	if (!workgroup) {
@@ -1378,6 +1990,7 @@ NTSTATUS cli_session_setup(struct cli_state *cli,
 	    (p=strchr_m(user2,*lp_winbind_separator()))) {
 		*p = 0;
 		user = p+1;
+		strupper_m(user2);
 		workgroup = user2;
 	}
 
@@ -1420,7 +2033,7 @@ NTSTATUS cli_session_setup(struct cli_state *cli,
            connect */
 
 	if ((cli->sec_mode & NEGOTIATE_SECURITY_USER_LEVEL) == 0) 
-		return cli_session_setup_plaintext(cli, user, "", workgroup);
+		return cli_session_setup_plain(cli, user, "", workgroup);
 
 	/* if the server doesn't support encryption then we have to use 
 	   plaintext. The second password is ignored */
@@ -1431,7 +2044,7 @@ NTSTATUS cli_session_setup(struct cli_state *cli,
 				  " or 'client ntlmv2 auth = yes'\n"));
 			return NT_STATUS_ACCESS_DENIED;
 		}
-		return cli_session_setup_plaintext(cli, user, pass, workgroup);
+		return cli_session_setup_plain(cli, user, pass, workgroup);
 	}
 
 	/* if the server supports extended security then use SPNEGO */
@@ -1545,9 +2158,6 @@ NTSTATUS cli_ulogoff(struct cli_state *cli)
 	status = cli_ulogoff_recv(req);
 fail:
 	TALLOC_FREE(ev);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1572,7 +2182,7 @@ struct tevent_req *cli_tcon_andx_create(TALLOC_CTX *mem_ctx,
 {
 	struct tevent_req *req, *subreq;
 	struct cli_tcon_andx_state *state;
-	fstring pword;
+	uint8_t p24[24];
 	uint16_t *vwv;
 	char *tmp = NULL;
 	uint8_t *bytes;
@@ -1586,7 +2196,10 @@ struct tevent_req *cli_tcon_andx_create(TALLOC_CTX *mem_ctx,
 	state->cli = cli;
 	vwv = state->vwv;
 
-	fstrcpy(cli->share, share);
+	cli->share = talloc_strdup(cli, share);
+	if (!cli->share) {
+		return NULL;
+	}
 
 	/* in user level security don't send a password now */
 	if (cli->sec_mode & NEGOTIATE_SECURITY_USER_LEVEL) {
@@ -1611,12 +2224,15 @@ struct tevent_req *cli_tcon_andx_create(TALLOC_CTX *mem_ctx,
 		 * Non-encrypted passwords - convert to DOS codepage before
 		 * encryption.
 		 */
+		SMBencrypt(pass, cli->secblob.data, p24);
 		passlen = 24;
-		SMBencrypt(pass, cli->secblob.data, (uchar *)pword);
+		pass = (const char *)p24;
 	} else {
 		if((cli->sec_mode & (NEGOTIATE_SECURITY_USER_LEVEL
 				     |NEGOTIATE_SECURITY_CHALLENGE_RESPONSE))
 		   == 0) {
+			char *tmp_pass;
+
 			if (!lp_client_plaintext_auth() && (*pass)) {
 				DEBUG(1, ("Server requested plaintext "
 					  "password but "
@@ -1628,16 +2244,21 @@ struct tevent_req *cli_tcon_andx_create(TALLOC_CTX *mem_ctx,
 			 * Non-encrypted passwords - convert to DOS codepage
 			 * before using.
 			 */
-			passlen = clistr_push(cli, pword, pass, sizeof(pword),
-					      STR_TERMINATE);
-			if (passlen == -1) {
-				DEBUG(1, ("clistr_push(pword) failed\n"));
-				goto access_denied;
+			tmp_pass = talloc_array(talloc_tos(), char, 128);
+			if (tmp_pass == NULL) {
+				tevent_req_nterror(req, NT_STATUS_NO_MEMORY);
+				return tevent_req_post(req, ev);
 			}
-		} else {
-			if (passlen) {
-				memcpy(pword, pass, passlen);
+			passlen = clistr_push(cli,
+					tmp_pass,
+					pass,
+					talloc_get_size(tmp_pass),
+					STR_TERMINATE);
+			if (passlen == -1) {
+				tevent_req_nterror(req, NT_STATUS_NO_MEMORY);
+				return tevent_req_post(req, ev);
 			}
+			pass = tmp_pass;
 		}
 	}
 
@@ -1647,8 +2268,8 @@ struct tevent_req *cli_tcon_andx_create(TALLOC_CTX *mem_ctx,
 	SSVAL(vwv+2, 0, TCONX_FLAG_EXTENDED_RESPONSE);
 	SSVAL(vwv+3, 0, passlen);
 
-	if (passlen) {
-		bytes = (uint8_t *)talloc_memdup(state, pword, passlen);
+	if (passlen && pass) {
+		bytes = (uint8_t *)talloc_memdup(state, pass, passlen);
 	} else {
 		bytes = talloc_array(state, uint8_t, 0);
 	}
@@ -1750,8 +2371,24 @@ static void cli_tcon_andx_done(struct tevent_req *subreq)
 
 	inbuf = (char *)in;
 
-	clistr_pull(inbuf, cli->dev, bytes, sizeof(fstring), num_bytes,
-		    STR_TERMINATE|STR_ASCII);
+	if (num_bytes) {
+		if (clistr_pull_talloc(cli,
+				inbuf,
+				SVAL(inbuf, smb_flg2),
+				&cli->dev,
+				bytes,
+				num_bytes,
+				STR_TERMINATE|STR_ASCII) == -1) {
+			tevent_req_nterror(req, NT_STATUS_NO_MEMORY);
+			return;
+		}
+	} else {
+		cli->dev = talloc_strdup(cli, "");
+		if (cli->dev == NULL) {
+			tevent_req_nterror(req, NT_STATUS_NO_MEMORY);
+			return;
+		}
+	}
 
 	if ((cli->protocol >= PROTOCOL_NT1) && (num_bytes == 3)) {
 		/* almost certainly win95 - enable bug fixes */
@@ -1814,9 +2451,6 @@ NTSTATUS cli_tcon_andx(struct cli_state *cli, const char *share,
 	status = cli_tcon_andx_recv(req);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1897,9 +2531,6 @@ NTSTATUS cli_tdis(struct cli_state *cli)
 	status = cli_tdis_recv(req);
 fail:
 	TALLOC_FREE(ev);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -2028,11 +2659,14 @@ static void cli_negprot_done(struct tevent_req *subreq)
 		}
 		/* work out if they sent us a workgroup */
 		if (!(cli->capabilities & CAP_EXTENDED_SECURITY) &&
-		    smb_buflen(cli->inbuf) > 8) {
-			clistr_pull(cli->inbuf, cli->server_domain,
-				    bytes+8, sizeof(cli->server_domain),
-				    num_bytes-8,
-				    STR_UNICODE|STR_NOALIGN);
+		    smb_buflen(inbuf) > 8) {
+			ssize_t ret;
+			status = smb_bytes_talloc_string(
+				cli, (char *)inbuf, &cli->server_domain,
+				bytes + 8, num_bytes - 8, &ret);
+			if (tevent_req_nterror(req, status)) {
+				return;
+			}
 		}
 
 		/*
@@ -2149,306 +2783,141 @@ NTSTATUS cli_negprot(struct cli_state *cli)
 	status = cli_negprot_recv(req);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
-/****************************************************************************
- Send a session request. See rfc1002.txt 4.3 and 4.3.2.
-****************************************************************************/
-
-bool cli_session_request(struct cli_state *cli,
-			 struct nmb_name *calling, struct nmb_name *called)
+static NTSTATUS cli_connect_sock(const char *host, int name_type,
+				 const struct sockaddr_storage *pss,
+				 const char *myname, uint16_t port,
+				 int sec_timeout, int *pfd, uint16_t *pport)
 {
-	char *p;
-	int len = 4;
-	int namelen = 0;
-	char *tmp;
-
-	/* 445 doesn't have session request */
-	if (cli->port == 445)
-		return True;
-
-	memcpy(&(cli->calling), calling, sizeof(*calling));
-	memcpy(&(cli->called ), called , sizeof(*called ));
-
-	/* put in the destination name */
-
-	tmp = name_mangle(talloc_tos(), cli->called.name,
-			  cli->called.name_type);
-	if (tmp == NULL) {
-		return false;
-	}
-
-	p = cli->outbuf+len;
-	namelen = name_len((unsigned char *)tmp, talloc_get_size(tmp));
-	if (namelen > 0) {
-		memcpy(p, tmp, namelen);
-		len += namelen;
-	}
-	TALLOC_FREE(tmp);
-
-	/* and my name */
-
-	tmp = name_mangle(talloc_tos(), cli->calling.name,
-			  cli->calling.name_type);
-	if (tmp == NULL) {
-		return false;
-	}
-
-	p = cli->outbuf+len;
-	namelen = name_len((unsigned char *)tmp, talloc_get_size(tmp));
-	if (namelen > 0) {
-		memcpy(p, tmp, namelen);
-		len += namelen;
-	}
-	TALLOC_FREE(tmp);
-
-	/* send a session request (RFC 1002) */
-	/* setup the packet length
-         * Remove four bytes from the length count, since the length
-         * field in the NBT Session Service header counts the number
-         * of bytes which follow.  The cli_send_smb() function knows
-         * about this and accounts for those four bytes.
-         * CRH.
-         */
-        len -= 4;
-	_smb_setlen(cli->outbuf,len);
-	SCVAL(cli->outbuf,0,0x81);
-
-	cli_send_smb(cli);
-	DEBUG(5,("Sent session request\n"));
-
-	if (!cli_receive_smb(cli))
-		return False;
-
-	if (CVAL(cli->inbuf,0) == 0x84) {
-		/* C. Hoch  9/14/95 Start */
-		/* For information, here is the response structure.
-		 * We do the byte-twiddling to for portability.
-		struct RetargetResponse{
-		unsigned char type;
-		unsigned char flags;
-		int16 length;
-		int32 ip_addr;
-		int16 port;
-		};
-		*/
-		uint16_t port = (CVAL(cli->inbuf,8)<<8)+CVAL(cli->inbuf,9);
-		struct in_addr dest_ip;
-		NTSTATUS status;
-
-		/* SESSION RETARGET */
-		putip((char *)&dest_ip,cli->inbuf+4);
-		in_addr_to_sockaddr_storage(&cli->dest_ss, dest_ip);
-
-		status = open_socket_out(&cli->dest_ss, port,
-					 LONG_CONNECT_TIMEOUT, &cli->fd);
-		if (!NT_STATUS_IS_OK(status)) {
-			return False;
-		}
-
-		DEBUG(3,("Retargeted\n"));
-
-		set_socket_options(cli->fd, lp_socket_options());
-
-		/* Try again */
-		{
-			static int depth;
-			bool ret;
-			if (depth > 4) {
-				DEBUG(0,("Retarget recursion - failing\n"));
-				return False;
-			}
-			depth++;
-			ret = cli_session_request(cli, calling, called);
-			depth--;
-			return ret;
-		}
-	} /* C. Hoch 9/14/95 End */
-
-	if (CVAL(cli->inbuf,0) != 0x82) {
-                /* This is the wrong place to put the error... JRA. */
-		cli->rap_error = CVAL(cli->inbuf,4);
-		return False;
-	}
-	return(True);
-}
-
-struct fd_struct {
-	int fd;
-};
-
-static void smb_sock_connected(struct tevent_req *req)
-{
-	struct fd_struct *pfd = tevent_req_callback_data(
-		req, struct fd_struct);
-	int fd;
+	TALLOC_CTX *frame = talloc_stackframe();
+	const char *prog;
+	unsigned int i, num_addrs;
+	const char **called_names;
+	const char **calling_names;
+	int *called_types;
 	NTSTATUS status;
+	int fd;
 
-	status = open_socket_out_defer_recv(req, &fd);
-	if (NT_STATUS_IS_OK(status)) {
-		pfd->fd = fd;
-	}
-}
-
-static NTSTATUS open_smb_socket(const struct sockaddr_storage *pss,
-				uint16_t *port, int timeout, int *pfd)
-{
-	struct event_context *ev;
-	struct tevent_req *r139, *r445;
-	struct fd_struct *fd139, *fd445;
-	NTSTATUS status = NT_STATUS_NO_MEMORY;
-
-	if (*port != 0) {
-		return open_socket_out(pss, *port, timeout, pfd);
-	}
-
-	ev = event_context_init(talloc_tos());
-	if (ev == NULL) {
-		return NT_STATUS_NO_MEMORY;
-	}
-
-	fd139 = talloc(ev, struct fd_struct);
-	if (fd139 == NULL) {
+	prog = getenv("LIBSMB_PROG");
+	if (prog != NULL) {
+		fd = sock_exec(prog);
+		if (fd == -1) {
+			return map_nt_error_from_unix(errno);
+		}
+		port = 0;
 		goto done;
 	}
-	fd139->fd = -1;
 
-	fd445 = talloc(ev, struct fd_struct);
-	if (fd445 == NULL) {
-		goto done;
+	if ((pss == NULL) || is_zero_addr(pss)) {
+		struct sockaddr_storage *addrs;
+		status = resolve_name_list(talloc_tos(), host, name_type,
+					   &addrs, &num_addrs);
+		if (!NT_STATUS_IS_OK(status)) {
+			goto fail;
+		}
+		pss = addrs;
+	} else {
+		num_addrs = 1;
 	}
-	fd445->fd = -1;
 
-	r445 = open_socket_out_defer_send(ev, ev, timeval_set(0, 0),
-					  pss, 445, timeout);
-	r139 = open_socket_out_defer_send(ev, ev, timeval_set(0, 3000),
-					  pss, 139, timeout);
-	if ((r445 == NULL) || (r139 == NULL)) {
-		goto done;
+	called_names = talloc_array(talloc_tos(), const char *, num_addrs);
+	if (called_names == NULL) {
+		status = NT_STATUS_NO_MEMORY;
+		goto fail;
 	}
-	tevent_req_set_callback(r445, smb_sock_connected, fd445);
-	tevent_req_set_callback(r139, smb_sock_connected, fd139);
-
-	while ((fd445->fd == -1) && (fd139->fd == -1)
-	       && (tevent_req_is_in_progress(r139)
-		   || tevent_req_is_in_progress(r445))) {
-		event_loop_once(ev);
+	called_types = talloc_array(talloc_tos(), int, num_addrs);
+	if (called_types == NULL) {
+		status = NT_STATUS_NO_MEMORY;
+		goto fail;
 	}
-
-	if ((fd139->fd != -1) && (fd445->fd != -1)) {
-		close(fd139->fd);
-		fd139->fd = -1;
+	calling_names = talloc_array(talloc_tos(), const char *, num_addrs);
+	if (calling_names == NULL) {
+		status = NT_STATUS_NO_MEMORY;
+		goto fail;
 	}
-
-	if (fd445->fd != -1) {
-		*port = 445;
-		*pfd = fd445->fd;
-		status = NT_STATUS_OK;
-		goto done;
+	for (i=0; i<num_addrs; i++) {
+		called_names[i] = host;
+		called_types[i] = name_type;
+		calling_names[i] = myname;
 	}
-	if (fd139->fd != -1) {
-		*port = 139;
-		*pfd = fd139->fd;
-		status = NT_STATUS_OK;
-		goto done;
+	status = smbsock_any_connect(pss, called_names, called_types,
+				     calling_names, NULL, num_addrs, port,
+				     sec_timeout, &fd, NULL, &port);
+	if (!NT_STATUS_IS_OK(status)) {
+		goto fail;
 	}
-
-	status = open_socket_out_defer_recv(r445, &fd445->fd);
- done:
-	TALLOC_FREE(ev);
+	set_socket_options(fd, lp_socket_options());
+done:
+	*pfd = fd;
+	*pport = port;
+	status = NT_STATUS_OK;
+fail:
+	TALLOC_FREE(frame);
 	return status;
 }
 
-/****************************************************************************
- Open the client sockets.
-****************************************************************************/
-
-NTSTATUS cli_connect(struct cli_state *cli,
-		const char *host,
-		struct sockaddr_storage *dest_ss)
-
+NTSTATUS cli_connect_nb(const char *host, struct sockaddr_storage *pss,
+			uint16_t port, int name_type, const char *myname,
+			int signing_state, struct cli_state **pcli)
 {
-	int name_type = 0x20;
 	TALLOC_CTX *frame = talloc_stackframe();
-	unsigned int num_addrs = 0;
-	unsigned int i = 0;
-	struct sockaddr_storage *ss_arr = NULL;
-	char *p = NULL;
+	struct cli_state *cli;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+	int fd = -1;
+	char *desthost;
+	char *p;
+	socklen_t length;
+	int ret;
 
-	/* reasonable default hostname */
-	if (!host) {
-		host = STAR_SMBSERVER;
+	desthost = talloc_strdup(talloc_tos(), host);
+	if (desthost == NULL) {
+		goto fail;
 	}
 
-	fstrcpy(cli->desthost, host);
-
-	/* allow hostnames of the form NAME#xx and do a netbios lookup */
-	if ((p = strchr(cli->desthost, '#'))) {
+	p = strchr(host, '#');
+	if (p != NULL) {
 		name_type = strtol(p+1, NULL, 16);
-		*p = 0;
-	}
-
-	if (!dest_ss || is_zero_addr(dest_ss)) {
-		NTSTATUS status =resolve_name_list(frame,
-					cli->desthost,
-					name_type,
-					&ss_arr,
-					&num_addrs);
-		if (!NT_STATUS_IS_OK(status)) {
-			TALLOC_FREE(frame);
-			return NT_STATUS_BAD_NETWORK_NAME;
-                }
-	} else {
-		num_addrs = 1;
-		ss_arr = TALLOC_P(frame, struct sockaddr_storage);
-		if (!ss_arr) {
-			TALLOC_FREE(frame);
-			return NT_STATUS_NO_MEMORY;
+		host = talloc_strndup(talloc_tos(), host, p - host);
+		if (host == NULL) {
+			goto fail;
 		}
-		*ss_arr = *dest_ss;
 	}
 
-	for (i = 0; i < num_addrs; i++) {
-		cli->dest_ss = ss_arr[i];
-		if (getenv("LIBSMB_PROG")) {
-			cli->fd = sock_exec(getenv("LIBSMB_PROG"));
-		} else {
-			uint16_t port = cli->port;
-			NTSTATUS status;
-			status = open_smb_socket(&cli->dest_ss, &port,
-						 cli->timeout, &cli->fd);
-			if (NT_STATUS_IS_OK(status)) {
-				cli->port = port;
-			}
-		}
-		if (cli->fd == -1) {
-			char addr[INET6_ADDRSTRLEN];
-			print_sockaddr(addr, sizeof(addr), &ss_arr[i]);
-			DEBUG(2,("Error connecting to %s (%s)\n",
-				 dest_ss?addr:host,strerror(errno)));
-		} else {
-			/* Exit from loop on first connection. */
-			break;
-		}
+	cli = cli_initialise_ex(signing_state);
+	if (cli == NULL) {
+		goto fail;
 	}
+	cli->desthost = talloc_move(cli, &desthost);
 
-	if (cli->fd == -1) {
-		TALLOC_FREE(frame);
-		return map_nt_error_from_unix(errno);
+	status = cli_connect_sock(host, name_type, pss, myname, port, 20, &fd,
+				  &port);
+	if (!NT_STATUS_IS_OK(status)) {
+		cli_shutdown(cli);
+		goto fail;
 	}
+	cli->fd = fd;
+	cli->port = port;
 
-	if (dest_ss) {
-		*dest_ss = cli->dest_ss;
+	length = sizeof(cli->dest_ss);
+	ret = getpeername(fd, (struct sockaddr *)(void *)&cli->dest_ss,
+			  &length);
+	if (ret == -1) {
+		status = map_nt_error_from_unix(errno);
+		cli_shutdown(cli);
+		goto fail;
 	}
 
-	set_socket_options(cli->fd, lp_socket_options());
+	if (pss != NULL) {
+		*pss = cli->dest_ss;
+	}
 
+	*pcli = cli;
+	status = NT_STATUS_OK;
+fail:
 	TALLOC_FREE(frame);
-	return NT_STATUS_OK;
+	return status;
 }
 
 /**
@@ -2465,59 +2934,16 @@ NTSTATUS cli_start_connection(struct cli_state **output_cli,
 			      int signing_state, int flags)
 {
 	NTSTATUS nt_status;
-	struct nmb_name calling;
-	struct nmb_name called;
 	struct cli_state *cli;
-	struct sockaddr_storage ss;
-
-	if (!my_name) 
-		my_name = global_myname();
-
-	if (!(cli = cli_initialise_ex(signing_state))) {
-		return NT_STATUS_NO_MEMORY;
-	}
-
-	make_nmb_name(&calling, my_name, 0x0);
-	make_nmb_name(&called , dest_host, 0x20);
-
-	cli_set_port(cli, port);
-	cli_set_timeout(cli, 10000); /* 10 seconds. */
 
-	if (dest_ss) {
-		ss = *dest_ss;
-	} else {
-		zero_sockaddr(&ss);
-	}
-
-again:
-
-	DEBUG(3,("Connecting to host=%s\n", dest_host));
-
-	nt_status = cli_connect(cli, dest_host, &ss);
+	nt_status = cli_connect_nb(dest_host, dest_ss, port, 0x20, my_name,
+				   signing_state, &cli);
 	if (!NT_STATUS_IS_OK(nt_status)) {
-		char addr[INET6_ADDRSTRLEN];
-		print_sockaddr(addr, sizeof(addr), &ss);
-		DEBUG(1,("cli_start_connection: failed to connect to %s (%s). Error %s\n",
-			 nmb_namestr(&called), addr, nt_errstr(nt_status) ));
-		cli_shutdown(cli);
+		DEBUG(10, ("cli_connect_nb failed: %s\n",
+			   nt_errstr(nt_status)));
 		return nt_status;
 	}
 
-	if (!cli_session_request(cli, &calling, &called)) {
-		char *p;
-		DEBUG(1,("session request to %s failed (%s)\n",
-			 called.name, cli_errstr(cli)));
-		if ((p=strchr(called.name, '.')) && !is_ipaddress(called.name)) {
-			*p = 0;
-			goto again;
-		}
-		if (strcmp(called.name, STAR_SMBSERVER)) {
-			make_nmb_name(&called , STAR_SMBSERVER, 0x20);
-			goto again;
-		}
-		return NT_STATUS_BAD_NETWORK_NAME;
-	}
-
 	if (flags & CLI_FULL_CONNECTION_DONT_SPNEGO)
 		cli->use_spnego = False;
 	else if (flags & CLI_FULL_CONNECTION_USE_KERBEROS)
@@ -2631,71 +3057,6 @@ NTSTATUS cli_full_connection(struct cli_state **output_cli,
 }
 
 /****************************************************************************
- Attempt a NetBIOS session request, falling back to *SMBSERVER if needed.
-****************************************************************************/
-
-bool attempt_netbios_session_request(struct cli_state **ppcli, const char *srchost, const char *desthost,
-                                     struct sockaddr_storage *pdest_ss)
-{
-	struct nmb_name calling, called;
-
-	make_nmb_name(&calling, srchost, 0x0);
-
-	/*
-	 * If the called name is an IP address
-	 * then use *SMBSERVER immediately.
-	 */
-
-	if(is_ipaddress(desthost)) {
-		make_nmb_name(&called, STAR_SMBSERVER, 0x20);
-	} else {
-		make_nmb_name(&called, desthost, 0x20);
-	}
-
-	if (!cli_session_request(*ppcli, &calling, &called)) {
-		NTSTATUS status;
-		struct nmb_name smbservername;
-
-		make_nmb_name(&smbservername, STAR_SMBSERVER, 0x20);
-
-		/*
-		 * If the name wasn't *SMBSERVER then
-		 * try with *SMBSERVER if the first name fails.
-		 */
-
-		if (nmb_name_equal(&called, &smbservername)) {
-
-			/*
-			 * The name used was *SMBSERVER, don't bother with another name.
-			 */
-
-			DEBUG(0,("attempt_netbios_session_request: %s rejected the session for name *SMBSERVER \
-with error %s.\n", desthost, cli_errstr(*ppcli) ));
-			return False;
-		}
-
-		/* Try again... */
-		cli_shutdown(*ppcli);
-
-		*ppcli = cli_initialise();
-		if (!*ppcli) {
-			/* Out of memory... */
-			return False;
-		}
-
-		status = cli_connect(*ppcli, desthost, pdest_ss);
-		if (!NT_STATUS_IS_OK(status) ||
-				!cli_session_request(*ppcli, &calling, &smbservername)) {
-			DEBUG(0,("attempt_netbios_session_request: %s rejected the session for \
-name *SMBSERVER with error %s\n", desthost, cli_errstr(*ppcli) ));
-			return False;
-		}
-	}
-
-	return True;
-}
-
-/****************************************************************************
  Send an old style tcon.
 ****************************************************************************/
 NTSTATUS cli_raw_tcon(struct cli_state *cli, 
@@ -2786,7 +3147,7 @@ struct cli_state *get_ipc_connect(char *server,
  */
 
 struct cli_state *get_ipc_connect_master_ip(TALLOC_CTX *ctx,
-				struct ip_service *mb_ip,
+				struct sockaddr_storage *mb_ip,
 				const struct user_auth_info *user_info,
 				char **pp_workgroup_out)
 {
@@ -2797,7 +3158,7 @@ struct cli_state *get_ipc_connect_master_ip(TALLOC_CTX *ctx,
 
 	*pp_workgroup_out = NULL;
 
-	print_sockaddr(addr, sizeof(addr), &mb_ip->ss);
+	print_sockaddr(addr, sizeof(addr), mb_ip);
         DEBUG(99, ("Looking up name of master browser %s\n",
                    addr));
 
@@ -2812,8 +3173,8 @@ struct cli_state *get_ipc_connect_master_ip(TALLOC_CTX *ctx,
          * the original wildcard query as the first choice and fall back to
          * MSBROWSE if the wildcard query fails.
          */
-        if (!name_status_find("*", 0, 0x1d, &mb_ip->ss, name) &&
-            !name_status_find(MSBROWSE, 1, 0x1d, &mb_ip->ss, name)) {
+        if (!name_status_find("*", 0, 0x1d, mb_ip, name) &&
+            !name_status_find(MSBROWSE, 1, 0x1d, mb_ip, name)) {
 
                 DEBUG(99, ("Could not retrieve name status for %s\n",
                            addr));
@@ -2844,9 +3205,10 @@ struct cli_state *get_ipc_connect_master_ip_bcast(TALLOC_CTX *ctx,
 					const struct user_auth_info *user_info,
 					char **pp_workgroup_out)
 {
-	struct ip_service *ip_list;
+	struct sockaddr_storage *ip_list;
 	struct cli_state *cli;
 	int i, count;
+	NTSTATUS status;
 
 	*pp_workgroup_out = NULL;
 
@@ -2854,15 +3216,17 @@ struct cli_state *get_ipc_connect_master_ip_bcast(TALLOC_CTX *ctx,
 
         /* Go looking for workgroups by broadcasting on the local network */
 
-        if (!NT_STATUS_IS_OK(name_resolve_bcast(MSBROWSE, 1, &ip_list,
-						&count))) {
-                DEBUG(99, ("No master browsers responded\n"));
+	status = name_resolve_bcast(MSBROWSE, 1, talloc_tos(),
+				    &ip_list, &count);
+        if (!NT_STATUS_IS_OK(status)) {
+                DEBUG(99, ("No master browsers responded: %s\n",
+			   nt_errstr(status)));
                 return False;
         }
 
 	for (i = 0; i < count; i++) {
 		char addr[INET6_ADDRSTRLEN];
-		print_sockaddr(addr, sizeof(addr), &ip_list[i].ss);
+		print_sockaddr(addr, sizeof(addr), &ip_list[i]);
 		DEBUG(99, ("Found master browser %s\n", addr));
 
 		cli = get_ipc_connect_master_ip(ctx, &ip_list[i],
diff --git a/source3/libsmb/clidfs.c b/source3/libsmb/clidfs.c
index a0d60482ee6..5c5257fb6f8 100644
--- a/source3/libsmb/clidfs.c
+++ b/source3/libsmb/clidfs.c
@@ -20,6 +20,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "msdfs.h"
 #include "trans2.h"
@@ -87,10 +88,6 @@ static struct cli_state *do_connect(TALLOC_CTX *ctx,
 					int name_type)
 {
 	struct cli_state *c = NULL;
-	struct nmb_name called, calling;
-	const char *called_str;
-	const char *server_n;
-	struct sockaddr_storage ss;
 	char *servicename;
 	char *sharename;
 	char *newserver, *newshare;
@@ -106,7 +103,6 @@ static struct cli_state *do_connect(TALLOC_CTX *ctx,
 	sharename = servicename;
 	if (*sharename == '\\') {
 		sharename += 2;
-		called_str = sharename;
 		if (server == NULL) {
 			server = sharename;
 		}
@@ -116,36 +112,19 @@ static struct cli_state *do_connect(TALLOC_CTX *ctx,
 		}
 		*sharename = 0;
 		sharename++;
-	} else {
-		called_str = server;
 	}
-
-	server_n = server;
-
-	zero_sockaddr(&ss);
-
-	make_nmb_name(&calling, global_myname(), 0x0);
-	make_nmb_name(&called , called_str, name_type);
-
- again:
-	zero_sockaddr(&ss);
-
-	/* have to open a new connection */
-	c = cli_initialise_ex(get_cmdline_auth_info_signing_state(auth_info));
-	if (c == NULL) {
-		d_printf("Connection to %s failed\n", server_n);
+	if (server == NULL) {
 		return NULL;
 	}
-	if (port) {
-		cli_set_port(c, port);
-	}
 
-	status = cli_connect(c, server_n, &ss);
+	status = cli_connect_nb(
+		server, NULL, port, name_type, NULL,
+		get_cmdline_auth_info_signing_state(auth_info), &c);
+
 	if (!NT_STATUS_IS_OK(status)) {
 		d_printf("Connection to %s failed (Error %s)\n",
-				server_n,
+				server,
 				nt_errstr(status));
-		cli_shutdown(c);
 		return NULL;
 	}
 
@@ -158,23 +137,6 @@ static struct cli_state *do_connect(TALLOC_CTX *ctx,
 		get_cmdline_auth_info_fallback_after_kerberos(auth_info);
 	c->use_ccache = get_cmdline_auth_info_use_ccache(auth_info);
 
-	if (!cli_session_request(c, &calling, &called)) {
-		char *p;
-		d_printf("session request to %s failed (%s)\n",
-			 called.name, cli_errstr(c));
-		cli_shutdown(c);
-		c = NULL;
-		if ((p=strchr_m(called.name, '.'))) {
-			*p = 0;
-			goto again;
-		}
-		if (strcmp(called.name, "*SMBSERVER")) {
-			make_nmb_name(&called , "*SMBSERVER", 0x20);
-			goto again;
-		}
-		return NULL;
-	}
-
 	DEBUG(4,(" session request ok\n"));
 
 	status = cli_negprot(c);
diff --git a/source3/libsmb/clidgram.c b/source3/libsmb/clidgram.c
index 92f381e87b4..90f06fd5359 100644
--- a/source3/libsmb/clidgram.c
+++ b/source3/libsmb/clidgram.c
@@ -20,6 +20,8 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "libsmb/clidgram.h"
 #include "libsmb/nmblib.h"
 #include "messages.h"
@@ -138,7 +140,7 @@ static bool prep_getdc_request(const struct sockaddr_storage *dc_ss,
 		my_sid = *sid;
 	}
 
-	my_acct_name = talloc_asprintf(talloc_tos(), "%s$", global_myname());
+	my_acct_name = talloc_asprintf(talloc_tos(), "%s$", lp_netbios_name());
 	if (my_acct_name == NULL) {
 		goto fail;
 	}
@@ -147,7 +149,7 @@ static bool prep_getdc_request(const struct sockaddr_storage *dc_ss,
 	s		= &packet.req.logon;
 
 	s->request_count	= 0;
-	s->computer_name	= global_myname();
+	s->computer_name	= lp_netbios_name();
 	s->user_name		= my_acct_name;
 	s->mailslot_name	= my_mailslot;
 	s->acct_control		= ACB_WSTRUST;
@@ -168,7 +170,7 @@ static bool prep_getdc_request(const struct sockaddr_storage *dc_ss,
 
 	ret = cli_prep_mailslot(false, NBT_MAILSLOT_NTLOGON, 0,
 				(char *)blob.data, blob.length,
-				global_myname(), 0, domain_name, 0x1c,
+				lp_netbios_name(), 0, domain_name, 0x1c,
 				dc_ss, dgm_id, p);
 fail:
 	TALLOC_FREE(frame);
@@ -225,7 +227,7 @@ static bool parse_getdc_response(
 
 	blob = p.smb.body.trans.data;
 
-	r = TALLOC_ZERO_P(mem_ctx, struct netlogon_samlogon_response);
+	r = talloc_zero(mem_ctx, struct netlogon_samlogon_response);
 	if (!r) {
 		return false;
 	}
@@ -320,7 +322,7 @@ struct tevent_req *nbt_getdc_send(TALLOC_CTX *mem_ctx,
 		return tevent_req_post(req, ev);
 	}
 	state->my_mailslot = mailslot_name(
-		state, ((struct sockaddr_in *)dc_addr)->sin_addr);
+		state, ((const struct sockaddr_in *)dc_addr)->sin_addr);
 	if (tevent_req_nomem(state->my_mailslot, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/libsmb/clidgram.h b/source3/libsmb/clidgram.h
index 01a8a6a4049..a449724a41a 100644
--- a/source3/libsmb/clidgram.h
+++ b/source3/libsmb/clidgram.h
@@ -1,3 +1,27 @@
+/*
+   Unix SMB/CIFS implementation.
+   client dgram calls
+   Copyright (C) Andrew Tridgell 1994-1998
+   Copyright (C) Richard Sharpe 2001
+   Copyright (C) John Terpstra 2001
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBSMB_CLIDGRAM_H_
+#define _LIBSMB_CLIDGRAM_H_
+
 #include "../libcli/netlogon/netlogon.h"
 
 /* The following definitions come from libsmb/clidgram.c  */
@@ -21,3 +45,5 @@ NTSTATUS nbt_getdc(struct messaging_context *msg_ctx,
 		   uint32_t *pnt_version,
 		   const char **dc_name,
 		   struct netlogon_samlogon_response **samlogon_response);
+
+#endif /* _LIBSMB_CLIDGRAM_H_ */
diff --git a/source3/libsmb/clientgen.c b/source3/libsmb/clientgen.c
index 7eb10c4f1d6..d3b66b67e18 100644
--- a/source3/libsmb/clientgen.c
+++ b/source3/libsmb/clientgen.c
@@ -19,6 +19,8 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "smb_signing.h"
 #include "async_smb.h"
 
@@ -66,364 +68,6 @@ bool cli_ucs2(struct cli_state *cli)
 	return ((cli->capabilities & CAP_UNICODE) != 0);
 }
 
-
-/****************************************************************************
- Read an smb from a fd ignoring all keepalive packets.
- The timeout is in milliseconds
-
- This is exactly the same as receive_smb except that it never returns
- a session keepalive packet (just as receive_smb used to do).
- receive_smb was changed to return keepalives as the oplock processing means this call
- should never go into a blocking read.
-****************************************************************************/
-
-static ssize_t client_receive_smb(struct cli_state *cli, size_t maxlen)
-{
-	size_t len;
-
-	for(;;) {
-		NTSTATUS status;
-
-		set_smb_read_error(&cli->smb_rw_error, SMB_READ_OK);
-
-		status = receive_smb_raw(cli->fd, cli->inbuf, cli->bufsize,
-					cli->timeout, maxlen, &len);
-		if (!NT_STATUS_IS_OK(status)) {
-			DEBUG(10,("client_receive_smb failed\n"));
-			show_msg(cli->inbuf);
-
-			if (NT_STATUS_EQUAL(status, NT_STATUS_END_OF_FILE)) {
-				set_smb_read_error(&cli->smb_rw_error,
-						   SMB_READ_EOF);
-				return -1;
-			}
-
-			if (NT_STATUS_EQUAL(status, NT_STATUS_IO_TIMEOUT)) {
-				set_smb_read_error(&cli->smb_rw_error,
-						   SMB_READ_TIMEOUT);
-				return -1;
-			}
-
-			set_smb_read_error(&cli->smb_rw_error, SMB_READ_ERROR);
-			return -1;
-		}
-
-		/*
-		 * I don't believe len can be < 0 with NT_STATUS_OK
-		 * returned above, but this check doesn't hurt. JRA.
-		 */
-
-		if ((ssize_t)len < 0) {
-			return len;
-		}
-
-		/* Ignore session keepalive packets. */
-		if(CVAL(cli->inbuf,0) != SMBkeepalive) {
-			break;
-		}
-	}
-
-	if (cli_encryption_on(cli)) {
-		NTSTATUS status = cli_decrypt_message(cli);
-		if (!NT_STATUS_IS_OK(status)) {
-			DEBUG(0, ("SMB decryption failed on incoming packet! Error %s\n",
-				nt_errstr(status)));
-			cli->smb_rw_error = SMB_READ_BAD_DECRYPT;
-			return -1;
-		}
-	}
-
-	show_msg(cli->inbuf);
-	return len;
-}
-
-static bool cli_state_set_seqnum(struct cli_state *cli, uint16_t mid, uint32_t seqnum)
-{
-	struct cli_state_seqnum *c;
-
-	for (c = cli->seqnum; c; c = c->next) {
-		if (c->mid == mid) {
-			c->seqnum = seqnum;
-			return true;
-		}
-	}
-
-	c = talloc_zero(cli, struct cli_state_seqnum);
-	if (!c) {
-		return false;
-	}
-
-	c->mid = mid;
-	c->seqnum = seqnum;
-	c->persistent = false;
-	DLIST_ADD_END(cli->seqnum, c, struct cli_state_seqnum *);
-
-	return true;
-}
-
-bool cli_state_seqnum_persistent(struct cli_state *cli,
-				 uint16_t mid)
-{
-	struct cli_state_seqnum *c;
-
-	for (c = cli->seqnum; c; c = c->next) {
-		if (c->mid == mid) {
-			c->persistent = true;
-			return true;
-		}
-	}
-
-	return false;
-}
-
-bool cli_state_seqnum_remove(struct cli_state *cli,
-			     uint16_t mid)
-{
-	struct cli_state_seqnum *c;
-
-	for (c = cli->seqnum; c; c = c->next) {
-		if (c->mid == mid) {
-			DLIST_REMOVE(cli->seqnum, c);
-			TALLOC_FREE(c);
-			return true;
-		}
-	}
-
-	return false;
-}
-
-static uint32_t cli_state_get_seqnum(struct cli_state *cli, uint16_t mid)
-{
-	struct cli_state_seqnum *c;
-
-	for (c = cli->seqnum; c; c = c->next) {
-		if (c->mid == mid) {
-			uint32_t seqnum = c->seqnum;
-			if (!c->persistent) {
-				DLIST_REMOVE(cli->seqnum, c);
-				TALLOC_FREE(c);
-			}
-			return seqnum;
-		}
-	}
-
-	return 0;
-}
-
-/****************************************************************************
- Recv an smb.
-****************************************************************************/
-
-bool cli_receive_smb(struct cli_state *cli)
-{
-	ssize_t len;
-	uint16_t mid;
-	uint32_t seqnum;
-
-	/* fd == -1 causes segfaults -- Tom (tom@ninja.nl) */
-	if (cli->fd == -1)
-		return false; 
-
- again:
-	len = client_receive_smb(cli, 0);
-	
-	if (len > 0) {
-		/* it might be an oplock break request */
-		if (!(CVAL(cli->inbuf, smb_flg) & FLAG_REPLY) &&
-		    CVAL(cli->inbuf,smb_com) == SMBlockingX &&
-		    SVAL(cli->inbuf,smb_vwv6) == 0 &&
-		    SVAL(cli->inbuf,smb_vwv7) == 0) {
-			if (cli->oplock_handler) {
-				int fnum = SVAL(cli->inbuf,smb_vwv2);
-				unsigned char level = CVAL(cli->inbuf,smb_vwv3+1);
-				if (!NT_STATUS_IS_OK(cli->oplock_handler(cli, fnum, level))) {
-					return false;
-				}
-			}
-			/* try to prevent loops */
-			SCVAL(cli->inbuf,smb_com,0xFF);
-			goto again;
-		}
-	}
-
-	/* If the server is not responding, note that now */
-	if (len < 0) {
-		/*
-		 * only log if the connection should still be open and not when
-		 * the connection was closed due to a dropped ip message
-		 */
-		if (cli->fd != -1) {
-			char addr[INET6_ADDRSTRLEN];
-			print_sockaddr(addr, sizeof(addr), &cli->dest_ss);
-			DEBUG(0, ("Receiving SMB: Server %s stopped responding\n",
-				addr));
-			close(cli->fd);
-			cli->fd = -1;
-		}
-		return false;
-	}
-
-	mid = SVAL(cli->inbuf,smb_mid);
-	seqnum = cli_state_get_seqnum(cli, mid);
-
-	if (!cli_check_sign_mac(cli, cli->inbuf, seqnum+1)) {
-		/*
-		 * If we get a signature failure in sessionsetup, then
-		 * the server sometimes just reflects the sent signature
-		 * back to us. Detect this and allow the upper layer to
-		 * retrieve the correct Windows error message.
-		 */
-		if (CVAL(cli->outbuf,smb_com) == SMBsesssetupX &&
-			(smb_len(cli->inbuf) > (smb_ss_field + 8 - 4)) &&
-			(SVAL(cli->inbuf,smb_flg2) & FLAGS2_SMB_SECURITY_SIGNATURES) &&
-			memcmp(&cli->outbuf[smb_ss_field],&cli->inbuf[smb_ss_field],8) == 0 &&
-			cli_is_error(cli)) {
-
-			/*
-			 * Reflected signature on login error. 
-			 * Set bad sig but don't close fd.
-			 */
-			cli->smb_rw_error = SMB_READ_BAD_SIG;
-			return true;
-		}
-
-		DEBUG(0, ("SMB Signature verification failed on incoming packet!\n"));
-		cli->smb_rw_error = SMB_READ_BAD_SIG;
-		close(cli->fd);
-		cli->fd = -1;
-		return false;
-	};
-	return true;
-}
-
-static ssize_t write_socket(int fd, const char *buf, size_t len)
-{
-        ssize_t ret=0;
-
-        DEBUG(6,("write_socket(%d,%d)\n",fd,(int)len));
-        ret = write_data(fd,buf,len);
-
-        DEBUG(6,("write_socket(%d,%d) wrote %d\n",fd,(int)len,(int)ret));
-        if(ret <= 0)
-                DEBUG(0,("write_socket: Error writing %d bytes to socket %d: ERRNO = %s\n",
-                        (int)len, fd, strerror(errno) ));
-
-        return(ret);
-}
-
-/****************************************************************************
- Send an smb to a fd.
-****************************************************************************/
-
-bool cli_send_smb(struct cli_state *cli)
-{
-	size_t len;
-	size_t nwritten=0;
-	ssize_t ret;
-	char *buf_out = cli->outbuf;
-	bool enc_on = cli_encryption_on(cli);
-	uint32_t seqnum;
-
-	/* fd == -1 causes segfaults -- Tom (tom@ninja.nl) */
-	if (cli->fd == -1)
-		return false;
-
-	cli_calculate_sign_mac(cli, cli->outbuf, &seqnum);
-
-	if (!cli_state_set_seqnum(cli, cli->mid, seqnum)) {
-		DEBUG(0,("Failed to store mid[%u]/seqnum[%u]\n",
-			(unsigned int)cli->mid,
-			(unsigned int)seqnum));
-		return false;
-	}
-
-	if (enc_on) {
-		NTSTATUS status = cli_encrypt_message(cli, cli->outbuf,
-						      &buf_out);
-		if (!NT_STATUS_IS_OK(status)) {
-			close(cli->fd);
-			cli->fd = -1;
-			cli->smb_rw_error = SMB_WRITE_ERROR;
-			DEBUG(0,("Error in encrypting client message. Error %s\n",
-				nt_errstr(status) ));
-			return false;
-		}
-	}
-
-	len = smb_len(buf_out) + 4;
-
-	while (nwritten < len) {
-		ret = write_socket(cli->fd,buf_out+nwritten,len - nwritten);
-		if (ret <= 0) {
-			if (enc_on) {
-				cli_free_enc_buffer(cli, buf_out);
-			}
-			close(cli->fd);
-			cli->fd = -1;
-			cli->smb_rw_error = SMB_WRITE_ERROR;
-			DEBUG(0,("Error writing %d bytes to client. %d (%s)\n",
-				(int)len,(int)ret, strerror(errno) ));
-			return false;
-		}
-		nwritten += ret;
-	}
-
-	if (enc_on) {
-		cli_free_enc_buffer(cli, buf_out);
-	}
-
-	/* Increment the mid so we can tell between responses. */
-	cli->mid++;
-	if (!cli->mid)
-		cli->mid++;
-	return true;
-}
-
-/****************************************************************************
- Send a "direct" writeX smb to a fd.
-****************************************************************************/
-
-bool cli_send_smb_direct_writeX(struct cli_state *cli,
-				const char *p,
-				size_t extradata)
-{
-	/* First length to send is the offset to the data. */
-	size_t len = SVAL(cli->outbuf,smb_vwv11) + 4;
-	size_t nwritten=0;
-	struct iovec iov[2];
-
-	/* fd == -1 causes segfaults -- Tom (tom@ninja.nl) */
-	if (cli->fd == -1) {
-		return false;
-	}
-
-	if (client_is_signing_on(cli)) {
-		DEBUG(0,("cli_send_smb_large: cannot send signed packet.\n"));
-		return false;
-	}
-
-	iov[0].iov_base = (void *)cli->outbuf;
-	iov[0].iov_len = len;
-	iov[1].iov_base = CONST_DISCARD(void *, p);
-	iov[1].iov_len = extradata;
-
-	nwritten = write_data_iov(cli->fd, iov, 2);
-	if (nwritten < (len + extradata)) {
-		close(cli->fd);
-		cli->fd = -1;
-		cli->smb_rw_error = SMB_WRITE_ERROR;
-		DEBUG(0,("Error writing %d bytes to client. (%s)\n",
-			 (int)(len+extradata), strerror(errno)));
-		return false;
-	}
-
-	/* Increment the mid so we can tell between responses. */
-	cli->mid++;
-	if (!cli->mid)
-		cli->mid++;
-	return true;
-}
-
 /****************************************************************************
  Setup basics in a outgoing packet.
 ****************************************************************************/
@@ -460,20 +104,6 @@ void cli_setup_packet_buf(struct cli_state *cli, char *buf)
 	SSVAL(buf,smb_flg2, flags2);
 }
 
-void cli_setup_packet(struct cli_state *cli)
-{
-	cli_setup_packet_buf(cli, cli->outbuf);
-}
-
-/****************************************************************************
- Setup the bcc length of the packet from a pointer to the end of the data.
-****************************************************************************/
-
-void cli_setup_bcc(struct cli_state *cli, void *p)
-{
-	set_message_bcc(cli->outbuf, PTR_DIFF(p, smb_buf(cli->outbuf)));
-}
-
 /****************************************************************************
  Initialize Domain, user or password.
 ****************************************************************************/
@@ -552,7 +182,7 @@ struct cli_state *cli_initialise_ex(int signing_state)
 		return NULL;
 	}
 
-	cli = TALLOC_ZERO_P(NULL, struct cli_state);
+	cli = talloc_zero(NULL, struct cli_state);
 	if (!cli) {
 		return NULL;
 	}
@@ -572,7 +202,6 @@ struct cli_state *cli_initialise_ex(int signing_state)
 	cli->bufsize = CLI_BUFFER_SIZE+4;
 	cli->max_xmit = cli->bufsize;
 	cli->outbuf = (char *)SMB_MALLOC(cli->bufsize+SAFETY_MARGIN);
-	cli->seqnum = 0;
 	cli->inbuf = (char *)SMB_MALLOC(cli->bufsize+SAFETY_MARGIN);
 	cli->oplock_handler = cli_oplock_ack;
 	cli->case_sensitive = false;
diff --git a/source3/libsmb/clierror.c b/source3/libsmb/clierror.c
index 015afb18aaa..d5ac8937ed6 100644
--- a/source3/libsmb/clierror.c
+++ b/source3/libsmb/clierror.c
@@ -20,6 +20,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 
 /*****************************************************
  RAP error codes - a small start but will be extended.
@@ -322,44 +323,6 @@ bool cli_is_dos_error(struct cli_state *cli)
         return cli_is_error(cli) && !(flgs2 & FLAGS2_32_BIT_ERROR_CODES);
 }
 
-/* Return the last error always as an NTSTATUS. */
-
-NTSTATUS cli_get_nt_error(struct cli_state *cli)
-{
-	if (cli_is_nt_error(cli)) {
-		return cli_nt_error(cli);
-	} else if (cli_is_dos_error(cli)) {
-		uint32 ecode;
-		uint8 eclass;
-		cli_dos_error(cli, &eclass, &ecode);
-		return dos_to_ntstatus(eclass, ecode);
-	} else {
-		/* Something went wrong, we don't know what. */
-		return NT_STATUS_UNSUCCESSFUL;
-	}
-}
-
-/* Push an error code into the inbuf to be returned on the next
- * query. */
-
-void cli_set_nt_error(struct cli_state *cli, NTSTATUS status)
-{
-	SSVAL(cli->inbuf,smb_flg2, SVAL(cli->inbuf,smb_flg2)|FLAGS2_32_BIT_ERROR_CODES);
-	SIVAL(cli->inbuf, smb_rcls, NT_STATUS_V(status));
-}
-
-/* Reset an error. */
-
-void cli_reset_error(struct cli_state *cli)
-{
-        if (SVAL(cli->inbuf,smb_flg2) & FLAGS2_32_BIT_ERROR_CODES) {
-		SIVAL(cli->inbuf, smb_rcls, NT_STATUS_V(NT_STATUS_OK));
-	} else {
-		SCVAL(cli->inbuf,smb_rcls,0);
-		SSVAL(cli->inbuf,smb_err,0);
-	}
-}
-
 bool cli_state_is_connected(struct cli_state *cli)
 {
 	if (cli == NULL) {
diff --git a/source3/libsmb/clifile.c b/source3/libsmb/clifile.c
index 28f59f0d88c..88cecd505ce 100644
--- a/source3/libsmb/clifile.c
+++ b/source3/libsmb/clifile.c
@@ -20,10 +20,13 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "libsmb/libsmb.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
 #include "libsmb/clirap.h"
 #include "trans2.h"
 #include "ntioctl.h"
+#include "libcli/security/secdesc.h"
 
 /***********************************************************
  Common function for pushing stings, used by smb_bytes_push_str()
@@ -46,11 +49,13 @@ static uint8_t *internal_bytes_push_str(uint8_t *buf, bool ucs2,
 
 	buflen = talloc_get_size(buf);
 
-	if (align_odd && ucs2 && (buflen % 2 == 0)) {
+	if (ucs2 &&
+	    ((align_odd && (buflen % 2 == 0)) ||
+	     (!align_odd && (buflen % 2 == 1)))) {
 		/*
 		 * We're pushing into an SMB buffer, align odd
 		 */
-		buf = TALLOC_REALLOC_ARRAY(NULL, buf, uint8_t, buflen + 1);
+		buf = talloc_realloc(NULL, buf, uint8_t, buflen + 1);
 		if (buf == NULL) {
 			return NULL;
 		}
@@ -65,7 +70,7 @@ static uint8_t *internal_bytes_push_str(uint8_t *buf, bool ucs2,
 		return NULL;
 	}
 
-	buf = TALLOC_REALLOC_ARRAY(NULL, buf, uint8_t,
+	buf = talloc_realloc(NULL, buf, uint8_t,
 				   buflen + converted_size);
 	if (buf == NULL) {
 		TALLOC_FREE(converted);
@@ -106,7 +111,7 @@ uint8_t *smb_bytes_push_bytes(uint8_t *buf, uint8_t prefix,
 	}
 	buflen = talloc_get_size(buf);
 
-	buf = TALLOC_REALLOC_ARRAY(NULL, buf, uint8_t,
+	buf = talloc_realloc(NULL, buf, uint8_t,
 				   buflen + 1 + num_bytes);
 	if (buf == NULL) {
 		return NULL;
@@ -159,7 +164,7 @@ struct tevent_req *cli_setpathinfo_send(TALLOC_CTX *mem_ctx,
 	SSVAL(&state->setup, 0, TRANSACT2_SETPATHINFO);
 
 	/* Setup param array. */
-	state->param = TALLOC_ZERO_ARRAY(state, uint8_t, 6);
+	state->param = talloc_zero_array(state, uint8_t, 6);
 	if (tevent_req_nomem(state->param, req)) {
 		return tevent_req_post(req, ev);
 	}
@@ -356,9 +361,6 @@ NTSTATUS cli_posix_symlink(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -416,8 +418,7 @@ static void cli_posix_readlink_done(struct tevent_req *subreq)
 	status = cli_qpathinfo_recv(subreq, state, &state->data,
 				    &state->num_data);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	/*
@@ -501,9 +502,6 @@ NTSTATUS cli_posix_readlink(struct cli_state *cli, const char *fname,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -568,107 +566,10 @@ NTSTATUS cli_posix_hardlink(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
 /****************************************************************************
- Map standard UNIX permissions onto wire representations.
-****************************************************************************/
-
-uint32_t unix_perms_to_wire(mode_t perms)
-{
-        unsigned int ret = 0;
-
-        ret |= ((perms & S_IXOTH) ?  UNIX_X_OTH : 0);
-        ret |= ((perms & S_IWOTH) ?  UNIX_W_OTH : 0);
-        ret |= ((perms & S_IROTH) ?  UNIX_R_OTH : 0);
-        ret |= ((perms & S_IXGRP) ?  UNIX_X_GRP : 0);
-        ret |= ((perms & S_IWGRP) ?  UNIX_W_GRP : 0);
-        ret |= ((perms & S_IRGRP) ?  UNIX_R_GRP : 0);
-        ret |= ((perms & S_IXUSR) ?  UNIX_X_USR : 0);
-        ret |= ((perms & S_IWUSR) ?  UNIX_W_USR : 0);
-        ret |= ((perms & S_IRUSR) ?  UNIX_R_USR : 0);
-#ifdef S_ISVTX
-        ret |= ((perms & S_ISVTX) ?  UNIX_STICKY : 0);
-#endif
-#ifdef S_ISGID
-        ret |= ((perms & S_ISGID) ?  UNIX_SET_GID : 0);
-#endif
-#ifdef S_ISUID
-        ret |= ((perms & S_ISUID) ?  UNIX_SET_UID : 0);
-#endif
-        return ret;
-}
-
-/****************************************************************************
- Map wire permissions to standard UNIX.
-****************************************************************************/
-
-mode_t wire_perms_to_unix(uint32_t perms)
-{
-        mode_t ret = (mode_t)0;
-
-        ret |= ((perms & UNIX_X_OTH) ? S_IXOTH : 0);
-        ret |= ((perms & UNIX_W_OTH) ? S_IWOTH : 0);
-        ret |= ((perms & UNIX_R_OTH) ? S_IROTH : 0);
-        ret |= ((perms & UNIX_X_GRP) ? S_IXGRP : 0);
-        ret |= ((perms & UNIX_W_GRP) ? S_IWGRP : 0);
-        ret |= ((perms & UNIX_R_GRP) ? S_IRGRP : 0);
-        ret |= ((perms & UNIX_X_USR) ? S_IXUSR : 0);
-        ret |= ((perms & UNIX_W_USR) ? S_IWUSR : 0);
-        ret |= ((perms & UNIX_R_USR) ? S_IRUSR : 0);
-#ifdef S_ISVTX
-        ret |= ((perms & UNIX_STICKY) ? S_ISVTX : 0);
-#endif
-#ifdef S_ISGID
-        ret |= ((perms & UNIX_SET_GID) ? S_ISGID : 0);
-#endif
-#ifdef S_ISUID
-        ret |= ((perms & UNIX_SET_UID) ? S_ISUID : 0);
-#endif
-        return ret;
-}
-
-/****************************************************************************
- Return the file type from the wire filetype for UNIX extensions.
-****************************************************************************/
-
-static mode_t unix_filetype_from_wire(uint32_t wire_type)
-{
-	switch (wire_type) {
-		case UNIX_TYPE_FILE:
-			return S_IFREG;
-		case UNIX_TYPE_DIR:
-			return S_IFDIR;
-#ifdef S_IFLNK
-		case UNIX_TYPE_SYMLINK:
-			return S_IFLNK;
-#endif
-#ifdef S_IFCHR
-		case UNIX_TYPE_CHARDEV:
-			return S_IFCHR;
-#endif
-#ifdef S_IFBLK
-		case UNIX_TYPE_BLKDEV:
-			return S_IFBLK;
-#endif
-#ifdef S_IFIFO
-		case UNIX_TYPE_FIFO:
-			return S_IFIFO;
-#endif
-#ifdef S_IFSOCK
-		case UNIX_TYPE_SOCKET:
-			return S_IFSOCK;
-#endif
-		default:
-			return (mode_t)0;
-	}
-}
-
-/****************************************************************************
  Do a POSIX getfacl (UNIX extensions).
 ****************************************************************************/
 
@@ -711,8 +612,7 @@ static void cli_posix_getfacl_done(struct tevent_req *subreq)
 	status = cli_qpathinfo_recv(subreq, state, &state->data,
 				    &state->num_data);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -777,9 +677,6 @@ NTSTATUS cli_posix_getfacl(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -825,8 +722,7 @@ static void cli_posix_stat_done(struct tevent_req *subreq)
 	status = cli_qpathinfo_recv(subreq, state, &state->data,
 				    &state->num_data);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -912,9 +808,6 @@ NTSTATUS cli_posix_stat(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1029,9 +922,6 @@ NTSTATUS cli_posix_chmod(struct cli_state *cli, const char *fname, mode_t mode)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1102,9 +992,6 @@ NTSTATUS cli_posix_chown(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1134,7 +1021,7 @@ struct tevent_req *cli_rename_send(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
-	SSVAL(state->vwv+0, 0, aSYSTEM | aHIDDEN | aDIR);
+	SSVAL(state->vwv+0, 0, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY);
 
 	bytes = talloc_array(state, uint8_t, 1);
 	if (tevent_req_nomem(bytes, req)) {
@@ -1147,7 +1034,7 @@ struct tevent_req *cli_rename_send(TALLOC_CTX *mem_ctx,
 		return tevent_req_post(req, ev);
 	}
 
-	bytes = TALLOC_REALLOC_ARRAY(state, bytes, uint8_t,
+	bytes = talloc_realloc(state, bytes, uint8_t,
 			talloc_get_size(bytes)+1);
 	if (tevent_req_nomem(bytes, req)) {
 		return tevent_req_post(req, ev);
@@ -1177,8 +1064,7 @@ static void cli_rename_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -1225,9 +1111,6 @@ NTSTATUS cli_rename(struct cli_state *cli, const char *fname_src, const char *fn
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1259,7 +1142,7 @@ static struct tevent_req *cli_ntrename_internal_send(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
-	SSVAL(state->vwv+0, 0 ,aSYSTEM | aHIDDEN | aDIR);
+	SSVAL(state->vwv+0, 0 ,FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY);
 	SSVAL(state->vwv+1, 0, rename_flag);
 
 	bytes = talloc_array(state, uint8_t, 1);
@@ -1273,7 +1156,7 @@ static struct tevent_req *cli_ntrename_internal_send(TALLOC_CTX *mem_ctx,
 		return tevent_req_post(req, ev);
 	}
 
-	bytes = TALLOC_REALLOC_ARRAY(state, bytes, uint8_t,
+	bytes = talloc_realloc(state, bytes, uint8_t,
 			talloc_get_size(bytes)+1);
 	if (tevent_req_nomem(bytes, req)) {
 		return tevent_req_post(req, ev);
@@ -1303,8 +1186,7 @@ static void cli_ntrename_internal_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -1370,9 +1252,6 @@ NTSTATUS cli_ntrename(struct cli_state *cli, const char *fname_src, const char *
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1435,9 +1314,6 @@ NTSTATUS cli_nt_hardlink(struct cli_state *cli, const char *fname_src, const cha
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1498,8 +1374,7 @@ static void cli_unlink_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -1546,9 +1421,6 @@ NTSTATUS cli_unlink(struct cli_state *cli, const char *fname, uint16_t mayhave_a
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1606,8 +1478,7 @@ static void cli_mkdir_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -1654,9 +1525,6 @@ NTSTATUS cli_mkdir(struct cli_state *cli, const char *dname)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1714,8 +1582,7 @@ static void cli_rmdir_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -1762,9 +1629,6 @@ NTSTATUS cli_rmdir(struct cli_state *cli, const char *dname)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1879,9 +1743,6 @@ NTSTATUS cli_nt_delete_on_close(struct cli_state *cli, uint16_t fnum, bool flag)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1976,8 +1837,7 @@ static void cli_ntcreate_done(struct tevent_req *subreq)
 	status = cli_smb_recv(subreq, state, &inbuf, 3, &wct, &vwv,
 			      &num_bytes, &bytes);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	state->fnum = SVAL(vwv+2, 1);
@@ -2044,9 +1904,184 @@ NTSTATUS cli_ntcreate(struct cli_state *cli,
 	status = cli_ntcreate_recv(req, pfid);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
+	return status;
+}
+
+struct cli_nttrans_create_state {
+	uint16_t fnum;
+};
+
+static void cli_nttrans_create_done(struct tevent_req *subreq);
+
+struct tevent_req *cli_nttrans_create_send(TALLOC_CTX *mem_ctx,
+					   struct event_context *ev,
+					   struct cli_state *cli,
+					   const char *fname,
+					   uint32_t CreatFlags,
+					   uint32_t DesiredAccess,
+					   uint32_t FileAttributes,
+					   uint32_t ShareAccess,
+					   uint32_t CreateDisposition,
+					   uint32_t CreateOptions,
+					   uint8_t SecurityFlags,
+					   struct security_descriptor *secdesc,
+					   struct ea_struct *eas,
+					   int num_eas)
+{
+	struct tevent_req *req, *subreq;
+	struct cli_nttrans_create_state *state;
+	uint8_t *param;
+	uint8_t *secdesc_buf;
+	size_t secdesc_len;
+	NTSTATUS status;
+	size_t converted_len;
+
+	req = tevent_req_create(mem_ctx,
+				&state, struct cli_nttrans_create_state);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	if (secdesc != NULL) {
+		status = marshall_sec_desc(talloc_tos(), secdesc,
+					   &secdesc_buf, &secdesc_len);
+		if (tevent_req_nterror(req, status)) {
+			DEBUG(10, ("marshall_sec_desc failed: %s\n",
+				   nt_errstr(status)));
+			return tevent_req_post(req, ev);
+		}
+	} else {
+		secdesc_buf = NULL;
+		secdesc_len = 0;
+	}
+
+	if (num_eas != 0) {
+		/*
+		 * TODO ;-)
+		 */
+		tevent_req_nterror(req, NT_STATUS_NOT_IMPLEMENTED);
+		return tevent_req_post(req, ev);
 	}
+
+	param = talloc_array(state, uint8_t, 53);
+	if (tevent_req_nomem(param, req)) {
+		return tevent_req_post(req, ev);
+	}
+
+	param = trans2_bytes_push_str(param, cli_ucs2(cli),
+				      fname, strlen(fname),
+				      &converted_len);
+	if (tevent_req_nomem(param, req)) {
+		return tevent_req_post(req, ev);
+	}
+
+	SIVAL(param, 0, CreatFlags);
+	SIVAL(param, 4, 0x0);	/* RootDirectoryFid */
+	SIVAL(param, 8, DesiredAccess);
+	SIVAL(param, 12, 0x0);	/* AllocationSize */
+	SIVAL(param, 16, 0x0);	/* AllocationSize */
+	SIVAL(param, 20, FileAttributes);
+	SIVAL(param, 24, ShareAccess);
+	SIVAL(param, 28, CreateDisposition);
+	SIVAL(param, 32, CreateOptions);
+	SIVAL(param, 36, secdesc_len);
+	SIVAL(param, 40, 0);	 /* EA length*/
+	SIVAL(param, 44, converted_len);
+	SIVAL(param, 48, 0x02); /* ImpersonationLevel */
+	SCVAL(param, 52, SecurityFlags);
+
+	subreq = cli_trans_send(state, ev, cli, SMBnttrans,
+				NULL, -1, /* name, fid */
+				NT_TRANSACT_CREATE, 0,
+				NULL, 0, 0, /* setup */
+				param, talloc_get_size(param), 128, /* param */
+				secdesc_buf, secdesc_len, 0); /* data */
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, cli_nttrans_create_done, req);
+	return req;
+}
+
+static void cli_nttrans_create_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct cli_nttrans_create_state *state = tevent_req_data(
+		req, struct cli_nttrans_create_state);
+	uint8_t *param;
+	uint32_t num_param;
+	NTSTATUS status;
+
+	status = cli_trans_recv(subreq, talloc_tos(), NULL,
+				NULL, 0, NULL, /* rsetup */
+				&param, 69, &num_param,
+				NULL, 0, NULL);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	state->fnum = SVAL(param, 2);
+	TALLOC_FREE(param);
+	tevent_req_done(req);
+}
+
+NTSTATUS cli_nttrans_create_recv(struct tevent_req *req, uint16_t *fnum)
+{
+	struct cli_nttrans_create_state *state = tevent_req_data(
+		req, struct cli_nttrans_create_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+	*fnum = state->fnum;
+	return NT_STATUS_OK;
+}
+
+NTSTATUS cli_nttrans_create(struct cli_state *cli,
+			    const char *fname,
+			    uint32_t CreatFlags,
+			    uint32_t DesiredAccess,
+			    uint32_t FileAttributes,
+			    uint32_t ShareAccess,
+			    uint32_t CreateDisposition,
+			    uint32_t CreateOptions,
+			    uint8_t SecurityFlags,
+			    struct security_descriptor *secdesc,
+			    struct ea_struct *eas,
+			    int num_eas,
+			    uint16_t *pfid)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	if (cli_has_async_calls(cli)) {
+		/*
+		 * Can't use sync call while an async call is in flight
+		 */
+		status = NT_STATUS_INVALID_PARAMETER;
+		goto fail;
+	}
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = cli_nttrans_create_send(frame, ev, cli, fname, CreatFlags,
+				      DesiredAccess, FileAttributes,
+				      ShareAccess, CreateDisposition,
+				      CreateOptions, SecurityFlags,
+				      secdesc, eas, num_eas);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
+		goto fail;
+	}
+	status = cli_nttrans_create_recv(req, pfid);
+ fail:
+	TALLOC_FREE(frame);
 	return status;
 }
 
@@ -2115,7 +2150,7 @@ struct tevent_req *cli_open_create(TALLOC_CTX *mem_ctx,
 	SSVAL(state->vwv + 1, 0, 0);
 	SSVAL(state->vwv + 2, 0, 0);  /* no additional info */
 	SSVAL(state->vwv + 3, 0, accessmode);
-	SSVAL(state->vwv + 4, 0, aSYSTEM | aHIDDEN);
+	SSVAL(state->vwv + 4, 0, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	SSVAL(state->vwv + 5, 0, 0);
 	SIVAL(state->vwv + 6, 0, 0);
 	SSVAL(state->vwv + 8, 0, openfn);
@@ -2169,8 +2204,7 @@ struct tevent_req *cli_open_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
 	}
 
 	status = cli_smb_req_send(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return tevent_req_post(req, ev);
 	}
 	return req;
@@ -2190,8 +2224,7 @@ static void cli_open_done(struct tevent_req *subreq)
 	status = cli_smb_recv(subreq, state, &inbuf, 3, &wct, &vwv, NULL,
 			      NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	state->fnum = SVAL(vwv+2, 0);
@@ -2247,9 +2280,6 @@ NTSTATUS cli_open(struct cli_state *cli, const char *fname, int flags,
 	status = cli_open_recv(req, pfnum);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -2305,8 +2335,7 @@ struct tevent_req *cli_close_send(TALLOC_CTX *mem_ctx,
 	}
 
 	status = cli_smb_req_send(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return tevent_req_post(req, ev);
 	}
 	return req;
@@ -2320,8 +2349,7 @@ static void cli_close_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -2367,9 +2395,6 @@ NTSTATUS cli_close(struct cli_state *cli, uint16_t fnum)
 	status = cli_close_recv(req);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -2485,9 +2510,6 @@ NTSTATUS cli_ftruncate(struct cli_state *cli, uint16_t fnum, uint64_t size)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -2547,7 +2569,6 @@ bool cli_lock(struct cli_state *cli, uint16_t fnum,
 
 	status = cli_locktype(cli, fnum, offset, len, timeout,
 			      (lock_type == READ_LOCK? 1 : 0));
-	cli_set_error(cli, status);
 	return NT_STATUS_IS_OK(status);
 }
 
@@ -2607,8 +2628,7 @@ static void cli_unlock_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -2659,9 +2679,6 @@ NTSTATUS cli_unlock(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -2712,7 +2729,6 @@ bool cli_lock64(struct cli_state *cli, uint16_t fnum,
 
 	cli->timeout = saved_timeout;
 
-	cli_set_error(cli, status);
 	return NT_STATUS_IS_OK(status);
 }
 
@@ -2772,8 +2788,7 @@ static void cli_unlock64_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -2828,9 +2843,6 @@ NTSTATUS cli_unlock64(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3002,9 +3014,6 @@ NTSTATUS cli_posix_lock(struct cli_state *cli, uint16_t fnum,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3069,9 +3078,6 @@ NTSTATUS cli_posix_unlock(struct cli_state *cli, uint16_t fnum, uint64_t offset,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3131,8 +3137,7 @@ static void cli_getattrE_done(struct tevent_req *subreq)
 	status = cli_smb_recv(subreq, state, &inbuf, 11, &wct, &vwv,
 			      NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -3224,9 +3229,6 @@ NTSTATUS cli_getattrE(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3295,8 +3297,7 @@ static void cli_getatr_done(struct tevent_req *subreq)
 	status = cli_smb_recv(subreq, state, &inbuf, 4, &wct, &vwv, NULL,
 			      NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -3374,9 +3375,6 @@ NTSTATUS cli_getatr(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3432,8 +3430,7 @@ static void cli_setattrE_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -3490,9 +3487,6 @@ NTSTATUS cli_setattrE(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3536,7 +3530,7 @@ struct tevent_req *cli_setatr_send(TALLOC_CTX *mem_ctx,
 	if (tevent_req_nomem(bytes, req)) {
 		return tevent_req_post(req, ev);
 	}
-	bytes = TALLOC_REALLOC_ARRAY(state, bytes, uint8_t,
+	bytes = talloc_realloc(state, bytes, uint8_t,
 			talloc_get_size(bytes)+1);
 	if (tevent_req_nomem(bytes, req)) {
 		return tevent_req_post(req, ev);
@@ -3566,8 +3560,7 @@ static void cli_setatr_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -3617,9 +3610,6 @@ NTSTATUS cli_setatr(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3677,8 +3667,7 @@ static void cli_chkpath_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -3740,9 +3729,6 @@ NTSTATUS cli_chkpath(struct cli_state *cli, const char *path)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3794,8 +3780,7 @@ static void cli_dskattr_done(struct tevent_req *subreq)
 	status = cli_smb_recv(subreq, state, &inbuf, 4, &wct, &vwv, NULL,
 			      NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	state->bsize = SVAL(vwv+1, 0)*SVAL(vwv+2,0);
@@ -3855,9 +3840,6 @@ NTSTATUS cli_dskattr(struct cli_state *cli, int *bsize, int *total, int *avail)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -3927,8 +3909,7 @@ static void cli_ctemp_done(struct tevent_req *subreq)
 	status = cli_smb_recv(subreq, state, &inbuf, 1, &wcnt, &vwv,
 			      &num_bytes, &bytes);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -4013,9 +3994,6 @@ NTSTATUS cli_ctemp(struct cli_state *cli,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -4201,7 +4179,7 @@ static bool parse_ea_blob(TALLOC_CTX *ctx, const uint8_t *rdata,
 		return true;
 	}
 
-	ea_list = TALLOC_ARRAY(ctx, struct ea_struct, num_eas);
+	ea_list = talloc_array(ctx, struct ea_struct, num_eas);
 	if (!ea_list) {
 		return false;
 	}
@@ -4288,8 +4266,7 @@ static void cli_get_ea_list_path_done(struct tevent_req *subreq)
 	status = cli_qpathinfo_recv(subreq, state, &state->data,
 				    &state->num_data);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -4343,9 +4320,6 @@ NTSTATUS cli_get_ea_list_path(struct cli_state *cli, const char *path,
 	status = cli_get_ea_list_path_recv(req, ctx, pnum_eas, pea_list);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -4395,7 +4369,6 @@ static uint32_t open_flags_to_wire(int flags)
 #endif
 #if defined(O_DIRECTORY)
 	if (flags & O_DIRECTORY) {
-		ret &= ~(SMB_O_RDONLY|SMB_O_RDWR|SMB_O_WRONLY);
 		ret |= SMB_O_DIRECTORY;
 	}
 #endif
@@ -4425,8 +4398,7 @@ static void cli_posix_open_internal_done(struct tevent_req *subreq)
 	status = cli_trans_recv(subreq, state, NULL, NULL, 0, NULL,
 				NULL, 0, NULL, &data, 12, &num_data);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	state->fnum = SVAL(data,2);
@@ -4470,7 +4442,6 @@ static struct tevent_req *cli_posix_open_internal_send(TALLOC_CTX *mem_ctx,
 
 	/* Setup data words. */
 	if (is_dir) {
-		wire_flags &= ~(SMB_O_RDONLY|SMB_O_RDWR|SMB_O_WRONLY);
 		wire_flags |= SMB_O_DIRECTORY;
 	}
 
@@ -4575,9 +4546,6 @@ NTSTATUS cli_posix_open(struct cli_state *cli, const char *fname,
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -4636,9 +4604,6 @@ NTSTATUS cli_posix_mkdir(struct cli_state *cli, const char *fname, mode_t mode)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -4744,9 +4709,6 @@ NTSTATUS cli_posix_unlink(struct cli_state *cli, const char *fname)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -4808,9 +4770,6 @@ NTSTATUS cli_posix_rmdir(struct cli_state *cli, const char *fname)
 
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -4884,9 +4843,8 @@ static void cli_notify_done(struct tevent_req *subreq)
 	status = cli_trans_recv(subreq, talloc_tos(), &flags2, NULL, 0, NULL,
 				&params, 0, &num_params, NULL, 0, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
+	if (tevent_req_nterror(req, status)) {
 		DEBUG(10, ("cli_trans_recv returned %s\n", nt_errstr(status)));
-		tevent_req_nterror(req, status);
 		return;
 	}
 
@@ -5035,8 +4993,7 @@ static void cli_qpathinfo_done(struct tevent_req *subreq)
 				NULL, 0, NULL,
 				&state->rdata, state->min_rdata,
 				&state->num_rdata);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -5095,9 +5052,6 @@ NTSTATUS cli_qpathinfo(TALLOC_CTX *mem_ctx, struct cli_state *cli,
 	status = cli_qpathinfo_recv(req, mem_ctx, rdata, num_rdata);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -5168,8 +5122,7 @@ static void cli_qfileinfo_done(struct tevent_req *subreq)
 				NULL, 0, NULL,
 				&state->rdata, state->min_rdata,
 				&state->num_rdata);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -5228,9 +5181,6 @@ NTSTATUS cli_qfileinfo(TALLOC_CTX *mem_ctx, struct cli_state *cli,
 	status = cli_qfileinfo_recv(req, mem_ctx, rdata, num_rdata);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -5271,8 +5221,7 @@ static void cli_flush_done(struct tevent_req *subreq)
 
 	status = cli_smb_recv(subreq, NULL, NULL, 0, NULL, NULL, NULL, NULL);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -5311,9 +5260,6 @@ NTSTATUS cli_flush(TALLOC_CTX *mem_ctx, struct cli_state *cli, uint16_t fnum)
 	status = cli_flush_recv(req);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -5374,8 +5320,7 @@ static void cli_shadow_copy_data_done(struct tevent_req *subreq)
 				NULL, 0, NULL, /* param */
 				&state->data, 12, &state->num_data);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -5460,8 +5405,5 @@ NTSTATUS cli_shadow_copy_data(TALLOC_CTX *mem_ctx, struct cli_state *cli,
 	status = cli_shadow_copy_data_recv(req, mem_ctx, pnames, pnum_names);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
diff --git a/source3/libsmb/clifsinfo.c b/source3/libsmb/clifsinfo.c
index ebd04e6d2b6..78024c8f987 100644
--- a/source3/libsmb/clifsinfo.c
+++ b/source3/libsmb/clifsinfo.c
@@ -19,8 +19,10 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "../libcli/auth/spnego.h"
 #include "../libcli/auth/ntlmssp.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
 #include "smb_crypt.h"
 #include "trans2.h"
@@ -151,9 +153,6 @@ NTSTATUS cli_unix_extensions_version(struct cli_state *cli, uint16 *pmajor,
 						  pcaphigh);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -256,9 +255,6 @@ NTSTATUS cli_set_unix_extensions_capabilities(struct cli_state *cli,
 	status = cli_set_unix_extensions_capabilities_recv(req);
 fail:
 	TALLOC_FREE(ev);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -355,9 +351,6 @@ NTSTATUS cli_get_fs_attr_info(struct cli_state *cli, uint32_t *fs_attr)
 	status = cli_get_fs_attr_info_recv(req, fs_attr);
 fail:
 	TALLOC_FREE(ev);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -599,7 +592,7 @@ NTSTATUS cli_raw_ntlm_smb_encryption_start(struct cli_state *cli,
 		return NT_STATUS_NO_MEMORY;
 	}
 	status = ntlmssp_client_start(NULL,
-				      global_myname(),
+				      lp_netbios_name(),
 				      lp_workgroup(),
 				      lp_client_ntlmv2_auth(),
 				      &es->s.ntlmssp_state);
@@ -692,7 +685,7 @@ static NTSTATUS make_cli_gss_blob(TALLOC_CTX *ctx,
 	NTSTATUS status = NT_STATUS_OK;
 
 	gss_OID_desc nt_hostbased_service =
-	{10, CONST_DISCARD(char *,"\x2a\x86\x48\x86\xf7\x12\x01\x02\x01\x04")};
+	{10, discard_const_p(char, "\x2a\x86\x48\x86\xf7\x12\x01\x02\x01\x04")};
 
 	memset(&tok_out, '\0', sizeof(tok_out));
 
diff --git a/source3/libsmb/clikrb5.c b/source3/libsmb/clikrb5.c
index a872d2df61e..b11aed8c7b2 100644
--- a/source3/libsmb/clikrb5.c
+++ b/source3/libsmb/clikrb5.c
@@ -538,6 +538,7 @@ static krb5_error_code setup_auth_context(krb5_context context,
 	return retval;
 }
 
+#if defined(TKT_FLG_OK_AS_DELEGATE ) && defined(HAVE_KRB5_FWD_TGT_CREDS) && defined(HAVE_KRB5_AUTH_CON_SETUSERUSERKEY) && defined(KRB5_AUTH_CONTEXT_USE_SUBKEY) && defined(HAVE_KRB5_AUTH_CON_SET_REQ_CKSUMTYPE)
 static krb5_error_code create_gss_checksum(krb5_data *in_data, /* [inout] */
 						uint32_t gss_flags)
 {
@@ -589,6 +590,7 @@ static krb5_error_code create_gss_checksum(krb5_data *in_data, /* [inout] */
 	in_data->length = base_cksum_size + orig_length;
 	return 0;
 }
+#endif
 
 /*
   we can't use krb5_mk_req because w2k wants the service to be in a particular format
@@ -610,7 +612,6 @@ static krb5_error_code ads_krb5_mk_req(krb5_context context,
 	krb5_data in_data;
 	bool creds_ready = False;
 	int i = 0, maxtries = 3;
-	uint32_t gss_flags = 0;
 
 	ZERO_STRUCT(in_data);
 
@@ -690,80 +691,84 @@ static krb5_error_code ads_krb5_mk_req(krb5_context context,
 	}
 
 #if defined(TKT_FLG_OK_AS_DELEGATE ) && defined(HAVE_KRB5_FWD_TGT_CREDS) && defined(HAVE_KRB5_AUTH_CON_SETUSERUSERKEY) && defined(KRB5_AUTH_CONTEXT_USE_SUBKEY) && defined(HAVE_KRB5_AUTH_CON_SET_REQ_CKSUMTYPE)
-	if( credsp->ticket_flags & TKT_FLG_OK_AS_DELEGATE ) {
-		/* Fetch a forwarded TGT from the KDC so that we can hand off a 2nd ticket
-		 as part of the kerberos exchange. */
+	{
+		uint32_t gss_flags = 0;
 
-		DEBUG( 3, ("ads_krb5_mk_req: server marked as OK to delegate to, building forwardable TGT\n")  );
+		if( credsp->ticket_flags & TKT_FLG_OK_AS_DELEGATE ) {
+			/* Fetch a forwarded TGT from the KDC so that we can hand off a 2nd ticket
+			 as part of the kerberos exchange. */
 
-		retval = krb5_auth_con_setuseruserkey(context,
+			DEBUG( 3, ("ads_krb5_mk_req: server marked as OK to delegate to, building forwardable TGT\n")  );
+
+			retval = krb5_auth_con_setuseruserkey(context,
 					*auth_context,
 					&credsp->keyblock );
-		if (retval) {
-			DEBUG(1,("krb5_auth_con_setuseruserkey failed (%s)\n",
-				error_message(retval)));
-			goto cleanup_creds;
-		}
+			if (retval) {
+				DEBUG(1,("krb5_auth_con_setuseruserkey failed (%s)\n",
+					error_message(retval)));
+				goto cleanup_creds;
+			}
 
-		/* Must use a subkey for forwarded tickets. */
-		retval = krb5_auth_con_setflags(context,
+			/* Must use a subkey for forwarded tickets. */
+			retval = krb5_auth_con_setflags(context,
 				*auth_context,
 				KRB5_AUTH_CONTEXT_USE_SUBKEY);
-		if (retval) {
-			DEBUG(1,("krb5_auth_con_setflags failed (%s)\n",
-				error_message(retval)));
-			goto cleanup_creds;
-		}
+			if (retval) {
+				DEBUG(1,("krb5_auth_con_setflags failed (%s)\n",
+					error_message(retval)));
+				goto cleanup_creds;
+			}
 
-		retval = krb5_fwd_tgt_creds(context,/* Krb5 context [in] */
+			retval = krb5_fwd_tgt_creds(context,/* Krb5 context [in] */
 				*auth_context,  /* Authentication context [in] */
-				CONST_DISCARD(char *, KRB5_TGS_NAME),  /* Ticket service name ("krbtgt") [in] */
+				discard_const_p(char, KRB5_TGS_NAME),  /* Ticket service name ("krbtgt") [in] */
 				credsp->client, /* Client principal for the tgt [in] */
 				credsp->server, /* Server principal for the tgt [in] */
 				ccache,         /* Credential cache to use for storage [in] */
 				1,              /* Turn on for "Forwardable ticket" [in] */
 				&in_data );     /* Resulting response [out] */
 
-		if (retval) {
-			DEBUG( 3, ("krb5_fwd_tgt_creds failed (%s)\n",
-				   error_message( retval ) ) );
-
-			/*
-			 * This is not fatal. Delete the *auth_context and continue
-			 * with krb5_mk_req_extended to get a non-forwardable ticket.
-			 */
-
-			if (in_data.data) {
-				free( in_data.data );
-				in_data.data = NULL;
-				in_data.length = 0;
-			}
-			krb5_auth_con_free(context, *auth_context);
-			*auth_context = NULL;
-			retval = setup_auth_context(context, auth_context);
 			if (retval) {
-				DEBUG(1,("setup_auth_context failed (%s)\n",
-					error_message(retval)));
-				goto cleanup_creds;
+				DEBUG( 3, ("krb5_fwd_tgt_creds failed (%s)\n",
+					   error_message( retval ) ) );
+
+				/*
+				 * This is not fatal. Delete the *auth_context and continue
+				 * with krb5_mk_req_extended to get a non-forwardable ticket.
+				 */
+
+				if (in_data.data) {
+					free( in_data.data );
+					in_data.data = NULL;
+					in_data.length = 0;
+				}
+				krb5_auth_con_free(context, *auth_context);
+				*auth_context = NULL;
+				retval = setup_auth_context(context, auth_context);
+				if (retval) {
+					DEBUG(1,("setup_auth_context failed (%s)\n",
+						error_message(retval)));
+					goto cleanup_creds;
+				}
+			} else {
+				/* We got a delegated ticket. */
+				gss_flags |= GSS_C_DELEG_FLAG;
 			}
-		} else {
-			/* We got a delegated ticket. */
-			gss_flags |= GSS_C_DELEG_FLAG;
 		}
-	}
 
-	/* Frees and reallocates in_data into a GSS checksum blob. */
-	retval = create_gss_checksum(&in_data, gss_flags);
-	if (retval) {
-		goto cleanup_data;
-	}
+		/* Frees and reallocates in_data into a GSS checksum blob. */
+		retval = create_gss_checksum(&in_data, gss_flags);
+		if (retval) {
+			goto cleanup_data;
+		}
 
-	/* We always want GSS-checksum types. */
-	retval = krb5_auth_con_set_req_cksumtype(context, *auth_context, GSSAPI_CHECKSUM );
-	if (retval) {
-		DEBUG(1,("krb5_auth_con_set_req_cksumtype failed (%s)\n",
-			error_message(retval)));
-		goto cleanup_data;
+		/* We always want GSS-checksum types. */
+		retval = krb5_auth_con_set_req_cksumtype(context, *auth_context, GSSAPI_CHECKSUM );
+		if (retval) {
+			DEBUG(1,("krb5_auth_con_set_req_cksumtype failed (%s)\n",
+				error_message(retval)));
+			goto cleanup_data;
+		}
 	}
 #endif
 
@@ -774,7 +779,10 @@ static krb5_error_code ads_krb5_mk_req(krb5_context context,
 			 error_message(retval)));
 	}
 
+#if defined(TKT_FLG_OK_AS_DELEGATE ) && defined(HAVE_KRB5_FWD_TGT_CREDS) && defined(HAVE_KRB5_AUTH_CON_SETUSERUSERKEY) && defined(KRB5_AUTH_CONTEXT_USE_SUBKEY) && defined(HAVE_KRB5_AUTH_CON_SET_REQ_CKSUMTYPE)
 cleanup_data:
+#endif
+
 	if (in_data.data) {
 		free( in_data.data );
 		in_data.length = 0;
@@ -919,7 +927,7 @@ done:
 {
 	static krb5_data kdata;
 
-	kdata.data = (char *)krb5_principal_get_comp_string(context, principal, i);
+	kdata.data = discard_const_p(char, krb5_principal_get_comp_string(context, principal, i));
 	kdata.length = strlen((const char *)kdata.data);
 	return &kdata;
 }
@@ -1131,7 +1139,7 @@ out:
 	DEBUG(10,("smb_krb5_renew_ticket: using %s as ccache\n", ccache_string));
 
 	/* FIXME: we should not fall back to defaults */
-	ret = krb5_cc_resolve(context, CONST_DISCARD(char *, ccache_string), &ccache);
+	ret = krb5_cc_resolve(context, discard_const_p(char, ccache_string), &ccache);
 	if (ret) {
 		goto done;
 	}
@@ -1150,7 +1158,7 @@ out:
 
 #ifdef HAVE_KRB5_GET_RENEWED_CREDS	/* MIT */
 	{
-		ret = krb5_get_renewed_creds(context, &creds, client, ccache, CONST_DISCARD(char *, service_string));
+		ret = krb5_get_renewed_creds(context, &creds, client, ccache, discard_const_p(char, service_string));
 		if (ret) {
 			DEBUG(10,("smb_krb5_renew_ticket: krb5_get_kdc_cred failed: %s\n", error_message(ret)));
 			goto done;
@@ -1265,7 +1273,7 @@ done:
 		return ENOMEM;
 	}
 
-	put_name(buf, global_myname(), ' ', 0x20);
+	put_name(buf, lp_netbios_name(), ' ', 0x20);
 
 #if defined(HAVE_MAGIC_IN_KRB5_ADDRESS) && defined(HAVE_ADDRTYPE_IN_KRB5_ADDRESS) /* MIT */
 	{
@@ -1956,11 +1964,11 @@ char *smb_krb5_principal_get_realm(krb5_context context,
 				   krb5_principal principal)
 {
 #ifdef HAVE_KRB5_PRINCIPAL_GET_REALM /* Heimdal */
-	return krb5_principal_get_realm(context, principal);
+	return discard_const_p(char, krb5_principal_get_realm(context, principal));
 #elif defined(krb5_princ_realm) /* MIT */
 	krb5_data *realm;
 	realm = krb5_princ_realm(context, principal);
-	return (char *)realm->data;
+	return discard_const_p(char, realm->data);
 #else
 	return NULL;
 #endif
diff --git a/source3/libsmb/clilist.c b/source3/libsmb/clilist.c
index 552de002428..9eec97fbeb1 100644
--- a/source3/libsmb/clilist.c
+++ b/source3/libsmb/clilist.c
@@ -18,6 +18,8 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
 #include "trans2.h"
 
@@ -319,7 +321,7 @@ static struct tevent_req *cli_list_old_send(TALLOC_CTX *mem_ctx,
 	bytes = smb_bytes_push_str(bytes, cli_ucs2(cli), mask,
 				   strlen(mask)+1, NULL);
 
-	bytes = smb_bytes_push_bytes(bytes, 5, (uint8_t *)&zero, 2);
+	bytes = smb_bytes_push_bytes(bytes, 5, (const uint8_t *)&zero, 2);
 	if (tevent_req_nomem(bytes, req)) {
 		return tevent_req_post(req, ev);
 	}
@@ -385,7 +387,7 @@ static void cli_list_old_done(struct tevent_req *subreq)
 
 		dirlist_len = talloc_get_size(state->dirlist);
 
-		tmp = TALLOC_REALLOC_ARRAY(
+		tmp = talloc_realloc(
 			state, state->dirlist, uint8_t,
 			dirlist_len + received * DIR_STRUCT_SIZE);
 		if (tevent_req_nomem(tmp, req)) {
@@ -450,7 +452,7 @@ static NTSTATUS cli_list_old_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
 
 	num_received = talloc_array_length(state->dirlist) / DIR_STRUCT_SIZE;
 
-	finfo = TALLOC_ARRAY(mem_ctx, struct file_info, num_received);
+	finfo = talloc_array(mem_ctx, struct file_info, num_received);
 	if (finfo == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -512,9 +514,6 @@ NTSTATUS cli_list_old(struct cli_state *cli, const char *mask,
 	}
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -574,7 +573,7 @@ static struct tevent_req *cli_list_trans_send(TALLOC_CTX *mem_ctx,
 	state->setup[0] = TRANSACT2_FINDFIRST;
 
 	nlen = 2*(strlen(mask)+1);
-	state->param = TALLOC_ARRAY(state, uint8_t, 12+nlen+2);
+	state->param = talloc_array(state, uint8_t, 12+nlen+2);
 	if (tevent_req_nomem(state->param, req)) {
 		return tevent_req_post(req, ev);
 	}
@@ -655,7 +654,7 @@ static void cli_list_trans_done(struct tevent_req *subreq)
 
 	old_num_finfo = talloc_array_length(state->finfo);
 
-	tmp = TALLOC_REALLOC_ARRAY(state, state->finfo, struct file_info,
+	tmp = talloc_realloc(state, state->finfo, struct file_info,
 				   old_num_finfo + ff_searchcount);
 	if (tevent_req_nomem(tmp, req)) {
 		return;
@@ -712,7 +711,7 @@ static void cli_list_trans_done(struct tevent_req *subreq)
 	/*
 	 * Shrink state->finfo to the real length we received
 	 */
-	tmp = TALLOC_REALLOC_ARRAY(state, state->finfo, struct file_info,
+	tmp = talloc_realloc(state, state->finfo, struct file_info,
 				   old_num_finfo + i);
 	if (tevent_req_nomem(tmp, req)) {
 		return;
@@ -737,7 +736,7 @@ static void cli_list_trans_done(struct tevent_req *subreq)
 
 	nlen = 2*(strlen(state->mask) + 1);
 
-	param = TALLOC_REALLOC_ARRAY(state, state->param, uint8_t,
+	param = talloc_realloc(state, state->param, uint8_t,
 				     12 + nlen + last_name_raw.length + 2);
 	if (tevent_req_nomem(param, req)) {
 		return;
@@ -841,9 +840,6 @@ NTSTATUS cli_list_trans(struct cli_state *cli, const char *mask,
 	}
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -966,8 +962,5 @@ NTSTATUS cli_list(struct cli_state *cli, const char *mask, uint16 attribute,
 	}
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
diff --git a/source3/libsmb/climessage.c b/source3/libsmb/climessage.c
index 2c2921aa0cc..55ffd8d9a59 100644
--- a/source3/libsmb/climessage.c
+++ b/source3/libsmb/climessage.c
@@ -18,7 +18,9 @@
 */
 
 #include "includes.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
+#include "libsmb/libsmb.h"
 
 struct cli_message_start_state {
 	uint16_t grp;
diff --git a/source3/libsmb/clioplock.c b/source3/libsmb/clioplock.c
index 78e93a4f316..da55f971a80 100644
--- a/source3/libsmb/clioplock.c
+++ b/source3/libsmb/clioplock.c
@@ -18,7 +18,9 @@
 */
 
 #include "includes.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
+#include "libsmb/libsmb.h"
 
 /****************************************************************************
 send an ack for an oplock break request
@@ -116,9 +118,6 @@ NTSTATUS cli_oplock_ack(struct cli_state *cli, uint16_t fnum, unsigned char leve
 	status = cli_oplock_ack_recv(req);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
diff --git a/source3/libsmb/cliprint.c b/source3/libsmb/cliprint.c
index 10d1759de82..06cb02521f6 100644
--- a/source3/libsmb/cliprint.c
+++ b/source3/libsmb/cliprint.c
@@ -18,6 +18,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 
 /*****************************************************************************
@@ -65,16 +66,16 @@ int cli_print_queue(struct cli_state *cli,
 	p = param;
 	SSVAL(p,0,76);         /* API function number 76 (DosPrintJobEnum) */
 	p += 2;
-	safe_strcpy_base(p,"zWrLeh", param, sizeof(param));   /* parameter description? */
+	strlcpy_base(p,"zWrLeh", param, sizeof(param));   /* parameter description? */
 	p = skip_string(param,sizeof(param),p);
-	safe_strcpy_base(p,"WWzWWDDzz", param, sizeof(param));  /* returned data format */
+	strlcpy_base(p,"WWzWWDDzz", param, sizeof(param));  /* returned data format */
 	p = skip_string(param,sizeof(param),p);
-	safe_strcpy_base(p,cli->share, param, sizeof(param));    /* name of queue */
+	strlcpy_base(p,cli->share, param, sizeof(param));    /* name of queue */
 	p = skip_string(param,sizeof(param),p);
 	SSVAL(p,0,2);   /* API function level 2, PRJINFO_2 data structure */
 	SSVAL(p,2,1000); /* size of bytes of returned data buffer */
 	p += 4;
-	safe_strcpy_base(p,"", param,sizeof(param));   /* subformat */
+	strlcpy_base(p,"", param,sizeof(param));   /* subformat */
 	p = skip_string(param,sizeof(param),p);
 
 	DEBUG(4,("doing cli_print_queue for %s\n", cli->share));
@@ -136,9 +137,9 @@ int cli_printjob_del(struct cli_state *cli, int job)
 	p = param;
 	SSVAL(p,0,81);		/* DosPrintJobDel() */
 	p += 2;
-	safe_strcpy_base(p,"W", param,sizeof(param));
+	strlcpy_base(p,"W", param,sizeof(param));
 	p = skip_string(param,sizeof(param),p);
-	safe_strcpy_base(p,"", param,sizeof(param));
+	strlcpy_base(p,"", param,sizeof(param));
 	p = skip_string(param,sizeof(param),p);
 	SSVAL(p,0,job);
 	p += 2;
diff --git a/source3/libsmb/cliquota.c b/source3/libsmb/cliquota.c
index e5c8c831b73..9136506e48c 100644
--- a/source3/libsmb/cliquota.c
+++ b/source3/libsmb/cliquota.c
@@ -18,6 +18,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "../librpc/gen_ndr/ndr_security.h"
 #include "fake_file.h"
 #include "../libcli/security/security.h"
@@ -88,7 +89,7 @@ static bool parse_user_quota_record(const uint8_t *rdata,
 	/* the hard quotas 8 bytes (uint64_t)*/
 	qt.hardlim = BVAL(rdata,32);
 
-	if (!sid_parse((char *)rdata+40,sid_len,&qt.sid)) {
+	if (!sid_parse((const char *)rdata+40,sid_len,&qt.sid)) {
 		return false;
 	}
 
@@ -262,19 +263,19 @@ NTSTATUS cli_list_user_quota(struct cli_state *cli, int quota_fnum,
 		((curdata)&&(curdata_count>=8)&&(offset>0));
 		curdata +=offset,curdata_count -= offset) {
 		ZERO_STRUCT(qt);
-		if (!parse_user_quota_record((uint8_t *)curdata, curdata_count,
+		if (!parse_user_quota_record((const uint8_t *)curdata, curdata_count,
 					     &offset, &qt)) {
 			DEBUG(1,("Failed to parse the quota record\n"));
 			goto cleanup;
 		}
 
-		if ((tmp_list_ent=TALLOC_ZERO_P(mem_ctx,SMB_NTQUOTA_LIST))==NULL) {
+		if ((tmp_list_ent=talloc_zero(mem_ctx,SMB_NTQUOTA_LIST))==NULL) {
 			DEBUG(0,("TALLOC_ZERO() failed\n"));
 			talloc_destroy(mem_ctx);
 			return NT_STATUS_NO_MEMORY;
 		}
 
-		if ((tmp_list_ent->quotas=TALLOC_ZERO_P(mem_ctx,SMB_NTQUOTA_STRUCT))==NULL) {
+		if ((tmp_list_ent->quotas=talloc_zero(mem_ctx,SMB_NTQUOTA_STRUCT))==NULL) {
 			DEBUG(0,("TALLOC_ZERO() failed\n"));
 			talloc_destroy(mem_ctx);
 			return NT_STATUS_NO_MEMORY;
@@ -318,20 +319,20 @@ NTSTATUS cli_list_user_quota(struct cli_state *cli, int quota_fnum,
 			((curdata)&&(curdata_count>=8)&&(offset>0));
 			curdata +=offset,curdata_count -= offset) {
 			ZERO_STRUCT(qt);
-			if (!parse_user_quota_record((uint8_t *)curdata,
+			if (!parse_user_quota_record((const uint8_t *)curdata,
 						     curdata_count, &offset,
 						     &qt)) {
 				DEBUG(1,("Failed to parse the quota record\n"));
 				goto cleanup;
 			}
 
-			if ((tmp_list_ent=TALLOC_ZERO_P(mem_ctx,SMB_NTQUOTA_LIST))==NULL) {
+			if ((tmp_list_ent=talloc_zero(mem_ctx,SMB_NTQUOTA_LIST))==NULL) {
 				DEBUG(0,("TALLOC_ZERO() failed\n"));
 				talloc_destroy(mem_ctx);
 				goto cleanup;
 			}
 
-			if ((tmp_list_ent->quotas=TALLOC_ZERO_P(mem_ctx,SMB_NTQUOTA_STRUCT))==NULL) {
+			if ((tmp_list_ent->quotas=talloc_zero(mem_ctx,SMB_NTQUOTA_STRUCT))==NULL) {
 				DEBUG(0,("TALLOC_ZERO() failed\n"));
 				talloc_destroy(mem_ctx);
 				goto cleanup;
diff --git a/source3/libsmb/clirap.c b/source3/libsmb/clirap.c
index 8d1be2d66e2..e5ff1abc18b 100644
--- a/source3/libsmb/clirap.c
+++ b/source3/libsmb/clirap.c
@@ -23,7 +23,9 @@
 #include "../libcli/auth/libcli_auth.h"
 #include "../librpc/gen_ndr/rap.h"
 #include "../lib/crypto/arcfour.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "trans2.h"
 
@@ -687,9 +689,6 @@ NTSTATUS cli_qpathinfo1(struct cli_state *cli,
 				     write_time, size, mode);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -865,9 +864,6 @@ NTSTATUS cli_qpathinfo2(struct cli_state *cli, const char *fname,
 				     write_time, change_time, size, mode, ino);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -979,9 +975,6 @@ NTSTATUS cli_qpathinfo_streams(struct cli_state *cli, const char *fname,
 					    pstreams);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -1005,7 +998,7 @@ static bool parse_streams_blob(TALLOC_CTX *mem_ctx, const uint8_t *rdata,
 		struct stream_struct *tmp;
 		uint8_t *tmp_buf;
 
-		tmp = TALLOC_REALLOC_ARRAY(mem_ctx, streams,
+		tmp = talloc_realloc(mem_ctx, streams,
 					   struct stream_struct,
 					   num_streams+1);
 
@@ -1030,7 +1023,7 @@ static bool parse_streams_blob(TALLOC_CTX *mem_ctx, const uint8_t *rdata,
 		 * convert_string_talloc??
 		 */
 
-		tmp_buf = TALLOC_ARRAY(streams, uint8_t, nlen+2);
+		tmp_buf = talloc_array(streams, uint8_t, nlen+2);
 		if (tmp_buf == NULL) {
 			goto fail;
 		}
@@ -1086,7 +1079,7 @@ NTSTATUS cli_qfilename(struct cli_state *cli, uint16_t fnum, char *name,
 		return status;
 	}
 
-	clistr_pull(cli->inbuf, name, rdata+4, namelen, IVAL(rdata, 0),
+	clistr_pull((const char *)rdata, name, rdata+4, namelen, IVAL(rdata, 0),
 		    STR_UNICODE);
 	TALLOC_FREE(rdata);
 	return NT_STATUS_OK;
@@ -1247,9 +1240,6 @@ NTSTATUS cli_qpathinfo_basic(struct cli_state *cli, const char *name,
 	status = cli_qpathinfo_basic_recv(req, sbuf, attributes);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
diff --git a/source3/libsmb/clirap.h b/source3/libsmb/clirap.h
index 6ea6978c045..26265dec50c 100644
--- a/source3/libsmb/clirap.h
+++ b/source3/libsmb/clirap.h
@@ -25,6 +25,8 @@
 #ifndef _LIBSMB_CLIRAP_H
 #define _LIBSMB_CLIRAP_H
 
+struct cli_state;
+
 /* The following definitions come from libsmb/clirap.c  */
 
 bool cli_api(struct cli_state *cli,
diff --git a/source3/libsmb/clirap2.c b/source3/libsmb/clirap2.c
index 44a9d7c8c49..03b4ca75412 100644
--- a/source3/libsmb/clirap2.c
+++ b/source3/libsmb/clirap2.c
@@ -76,6 +76,7 @@
 /*****************************************************/
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "../librpc/gen_ndr/rap.h"
 #include "../librpc/gen_ndr/svcctl.h"
 #include "libsmb/clirap.h"
diff --git a/source3/libsmb/clireadwrite.c b/source3/libsmb/clireadwrite.c
index e243f8a0fc4..14b6401051e 100644
--- a/source3/libsmb/clireadwrite.c
+++ b/source3/libsmb/clireadwrite.c
@@ -18,6 +18,8 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
 #include "trans2.h"
 
@@ -26,6 +28,11 @@
 ****************************************************************************/
 static size_t cli_read_max_bufsize(struct cli_state *cli)
 {
+	size_t data_offset = smb_size - 4;
+	size_t wct = 12;
+
+	size_t useable_space;
+
 	if (!client_is_signing_on(cli) && !cli_encryption_on(cli)
 	    && (cli->server_posix_capabilities & CIFS_UNIX_LARGE_READ_CAP)) {
 		return CLI_SAMBA_MAX_POSIX_LARGE_READX_SIZE;
@@ -35,13 +42,21 @@ static size_t cli_read_max_bufsize(struct cli_state *cli)
 			? CLI_SAMBA_MAX_LARGE_READX_SIZE
 			: CLI_WINDOWS_MAX_LARGE_READX_SIZE;
 	}
-	return (cli->max_xmit - (smb_size+32)) & ~1023;
+
+	data_offset += wct * sizeof(uint16_t);
+	data_offset += 1; /* pad */
+
+	useable_space = cli->max_xmit - data_offset;
+
+	return useable_space;
 }
 
 /****************************************************************************
   Calculate the recommended write buffer size
 ****************************************************************************/
-static size_t cli_write_max_bufsize(struct cli_state *cli, uint16_t write_mode)
+static size_t cli_write_max_bufsize(struct cli_state *cli,
+				    uint16_t write_mode,
+				    uint8_t wct)
 {
         if (write_mode == 0 &&
 	    !client_is_signing_on(cli) &&
@@ -60,13 +75,15 @@ static size_t cli_write_max_bufsize(struct cli_state *cli, uint16_t write_mode)
 	if (((cli->capabilities & CAP_LARGE_WRITEX) == 0)
 	    || client_is_signing_on(cli)
 	    || strequal(cli->dev, "LPT1:")) {
+		size_t data_offset = smb_size - 4;
+		size_t useable_space;
 
-		/*
-		 * Printer devices are restricted to max_xmit writesize in
-		 * Vista and XPSP3 as are signing connections.
-		 */
+		data_offset += wct * sizeof(uint16_t);
+		data_offset += 1; /* pad */
 
-		return (cli->max_xmit - (smb_size+32)) & ~1023;
+		useable_space = cli->max_xmit - data_offset;
+
+		return useable_space;
 	}
 
 	return CLI_WINDOWS_MAX_LARGE_WRITEX_SIZE;
@@ -196,7 +213,7 @@ static void cli_read_andx_done(struct tevent_req *subreq)
 		return;
 	}
 
-	state->buf = (uint8_t *)smb_base(inbuf) + SVAL(vwv+6, 0);
+	state->buf = discard_const_p(uint8_t, smb_base(inbuf)) + SVAL(vwv+6, 0);
 
 	if (trans_oob(smb_len(inbuf), SVAL(vwv+6, 0), state->received)
 	    || ((state->received != 0) && (state->buf < bytes))) {
@@ -465,7 +482,7 @@ struct tevent_req *cli_pull_send(TALLOC_CTX *mem_ctx,
 	state->num_reqs = MAX(window_size/state->chunk_size, 1);
 	state->num_reqs = MIN(state->num_reqs, cli->max_mux);
 
-	state->reqs = TALLOC_ZERO_ARRAY(state, struct cli_pull_subreq,
+	state->reqs = talloc_zero_array(state, struct cli_pull_subreq,
 					state->num_reqs);
 	if (state->reqs == NULL) {
 		goto failed;
@@ -663,9 +680,6 @@ NTSTATUS cli_pull(struct cli_state *cli, uint16_t fnum,
 	status = cli_pull_recv(req, received);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
 
@@ -686,171 +700,12 @@ ssize_t cli_read(struct cli_state *cli, uint16_t fnum, char *buf,
 	status = cli_pull(cli, fnum, offset, size, size,
 			  cli_read_sink, &buf, &ret);
 	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
 		return -1;
 	}
 	return ret;
 }
 
 /****************************************************************************
- Issue a single SMBwrite and don't wait for a reply.
-****************************************************************************/
-
-static bool cli_issue_write(struct cli_state *cli,
-				uint16_t fnum,
-				off_t offset,
-				uint16 mode,
-				const char *buf,
-				size_t size)
-{
-	char *p;
-	bool large_writex = false;
-	/* We can only do direct writes if not signing and not encrypting. */
-	bool direct_writes = !client_is_signing_on(cli) && !cli_encryption_on(cli);
-
-	if (!direct_writes && size + 1 > cli->bufsize) {
-		cli->outbuf = (char *)SMB_REALLOC(cli->outbuf, size + 1024);
-		if (!cli->outbuf) {
-			return False;
-		}
-		cli->inbuf = (char *)SMB_REALLOC(cli->inbuf, size + 1024);
-		if (cli->inbuf == NULL) {
-			SAFE_FREE(cli->outbuf);
-			return False;
-		}
-		cli->bufsize = size + 1024;
-	}
-
-	memset(cli->outbuf,'\0',smb_size);
-	memset(cli->inbuf,'\0',smb_size);
-
-	if (cli->capabilities & CAP_LARGE_FILES) {
-		large_writex = True;
-	}
-
-	if (large_writex) {
-		cli_set_message(cli->outbuf,14,0,True);
-	} else {
-		cli_set_message(cli->outbuf,12,0,True);
-	}
-
-	SCVAL(cli->outbuf,smb_com,SMBwriteX);
-	SSVAL(cli->outbuf,smb_tid,cli->cnum);
-	cli_setup_packet(cli);
-
-	SCVAL(cli->outbuf,smb_vwv0,0xFF);
-	SSVAL(cli->outbuf,smb_vwv2,fnum);
-
-	SIVAL(cli->outbuf,smb_vwv3,offset);
-	SIVAL(cli->outbuf,smb_vwv5,0);
-	SSVAL(cli->outbuf,smb_vwv7,mode);
-
-	SSVAL(cli->outbuf,smb_vwv8,(mode & 0x0008) ? size : 0);
-	/*
-	 * According to CIFS-TR-1p00, this following field should only
-	 * be set if CAP_LARGE_WRITEX is set. We should check this
-	 * locally. However, this check might already have been
-	 * done by our callers.
-	 */
-	SSVAL(cli->outbuf,smb_vwv9,(size>>16));
-	SSVAL(cli->outbuf,smb_vwv10,size);
-	/* +1 is pad byte. */
-	SSVAL(cli->outbuf,smb_vwv11,
-	      smb_buf(cli->outbuf) - smb_base(cli->outbuf) + 1);
-
-	if (large_writex) {
-		SIVAL(cli->outbuf,smb_vwv12,(((uint64_t)offset)>>32) & 0xffffffff);
-	}
-
-	p = smb_base(cli->outbuf) + SVAL(cli->outbuf,smb_vwv11) -1;
-	*p++ = '\0'; /* pad byte. */
-	if (!direct_writes) {
-		memcpy(p, buf, size);
-	}
-	if (size > 0x1FFFF) {
-		/* This is a POSIX 14 word large write. */
-		set_message_bcc(cli->outbuf, 0); /* Set bcc to zero. */
-		_smb_setlen_large(cli->outbuf,smb_size + 28 + 1 /* pad */ + size - 4);
-	} else {
-		cli_setup_bcc(cli, p+size);
-	}
-
-	show_msg(cli->outbuf);
-	if (direct_writes) {
-		/* For direct writes we now need to write the data
-		 * directly out of buf. */
-		return cli_send_smb_direct_writeX(cli, buf, size);
-	} else {
-		return cli_send_smb(cli);
-	}
-}
-
-/****************************************************************************
-  write to a file
-  write_mode: 0x0001 disallow write cacheing
-              0x0002 return bytes remaining
-              0x0004 use raw named pipe protocol
-              0x0008 start of message mode named pipe protocol
-****************************************************************************/
-
-ssize_t cli_write(struct cli_state *cli,
-    	         uint16_t fnum, uint16 write_mode,
-		 const char *buf, off_t offset, size_t size)
-{
-	ssize_t bwritten = 0;
-	unsigned int issued = 0;
-	unsigned int received = 0;
-	int mpx = 1;
-	size_t writesize;
-	int blocks;
-
-	if(cli->max_mux > 1) {
-		mpx = cli->max_mux-1;
-	} else {
-		mpx = 1;
-	}
-
-	writesize = cli_write_max_bufsize(cli, write_mode);
-
-	blocks = (size + (writesize-1)) / writesize;
-
-	while (received < blocks) {
-
-		while ((issued - received < mpx) && (issued < blocks)) {
-			ssize_t bsent = issued * writesize;
-			ssize_t size1 = MIN(writesize, size - bsent);
-
-			if (!cli_issue_write(cli, fnum, offset + bsent,
-			                write_mode,
-			                buf + bsent,
-					size1))
-				return -1;
-			issued++;
-		}
-
-		if (!cli_receive_smb(cli)) {
-			return bwritten;
-		}
-
-		received++;
-
-		if (cli_is_error(cli))
-			break;
-
-		bwritten += SVAL(cli->inbuf, smb_vwv2);
-		if (writesize > 0xFFFF) {
-			bwritten += (((int)(SVAL(cli->inbuf, smb_vwv4)))<<16);
-		}
-	}
-
-	while (received < issued && cli_receive_smb(cli)) {
-		received++;
-	}
-
-	return bwritten;
-}
-
-/****************************************************************************
   write to a file using a SMBwrite and not bypassing 0 byte writes
 ****************************************************************************/
 
@@ -864,7 +719,7 @@ NTSTATUS cli_smbwrite(struct cli_state *cli, uint16_t fnum, char *buf,
 	 * 3 bytes prefix
 	 */
 
-	bytes = TALLOC_ARRAY(talloc_tos(), uint8_t, 3);
+	bytes = talloc_array(talloc_tos(), uint8_t, 3);
 	if (bytes == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -882,7 +737,7 @@ NTSTATUS cli_smbwrite(struct cli_state *cli, uint16_t fnum, char *buf,
 		SIVAL(vwv+2, 0, offset);
 		SSVAL(vwv+4, 0, 0);
 
-		bytes = TALLOC_REALLOC_ARRAY(talloc_tos(), bytes, uint8_t,
+		bytes = talloc_realloc(talloc_tos(), bytes, uint8_t,
 					     size+3);
 		if (bytes == NULL) {
 			return NT_STATUS_NO_MEMORY;
@@ -944,7 +799,7 @@ struct tevent_req *cli_write_andx_create(TALLOC_CTX *mem_ctx,
 	struct cli_write_andx_state *state;
 	bool bigoffset = ((cli->capabilities & CAP_LARGE_FILES) != 0);
 	uint8_t wct = bigoffset ? 14 : 12;
-	size_t max_write = cli_write_max_bufsize(cli, mode);
+	size_t max_write = cli_write_max_bufsize(cli, mode, wct);
 	uint16_t *vwv;
 
 	req = tevent_req_create(mem_ctx, &state, struct cli_write_andx_state);
@@ -981,7 +836,7 @@ struct tevent_req *cli_write_andx_create(TALLOC_CTX *mem_ctx,
 	state->pad = 0;
 	state->iov[0].iov_base = (void *)&state->pad;
 	state->iov[0].iov_len = 1;
-	state->iov[1].iov_base = CONST_DISCARD(void *, buf);
+	state->iov[1].iov_base = discard_const_p(void, buf);
 	state->iov[1].iov_len = size;
 
 	subreq = cli_smb_req_create(state, ev, cli, SMBwriteX, 0, wct, vwv,
@@ -1138,9 +993,54 @@ static void cli_writeall_written(struct tevent_req *subreq)
 	tevent_req_set_callback(subreq, cli_writeall_written, req);
 }
 
-static NTSTATUS cli_writeall_recv(struct tevent_req *req)
+static NTSTATUS cli_writeall_recv(struct tevent_req *req,
+				  size_t *pwritten)
 {
-	return tevent_req_simple_recv_ntstatus(req);
+	struct cli_writeall_state *state = tevent_req_data(
+		req, struct cli_writeall_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+	if (pwritten != NULL) {
+		*pwritten = state->written;
+	}
+	return NT_STATUS_OK;
+}
+
+NTSTATUS cli_writeall(struct cli_state *cli, uint16_t fnum, uint16_t mode,
+		      const uint8_t *buf, off_t offset, size_t size,
+		      size_t *pwritten)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	if (cli_has_async_calls(cli)) {
+		/*
+		 * Can't use sync call while an async call is in flight
+		 */
+		status = NT_STATUS_INVALID_PARAMETER;
+		goto fail;
+	}
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = cli_writeall_send(frame, ev, cli, fnum, mode, buf, offset, size);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll(req, ev)) {
+		status = map_nt_error_from_unix(errno);
+		goto fail;
+	}
+	status = cli_writeall_recv(req, pwritten);
+ fail:
+	TALLOC_FREE(frame);
+	return status;
 }
 
 struct cli_push_write_state {
@@ -1252,7 +1152,7 @@ struct tevent_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
 	state->pending = 0;
 	state->next_offset = start_offset;
 
-	state->chunk_size = cli_write_max_bufsize(cli, mode);
+	state->chunk_size = cli_write_max_bufsize(cli, mode, 14);
 
 	if (window_size == 0) {
 		window_size = cli->max_mux * state->chunk_size;
@@ -1264,7 +1164,7 @@ struct tevent_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
 	state->num_reqs = MIN(state->num_reqs, cli->max_mux);
 	state->num_reqs = MAX(state->num_reqs, 1);
 
-	state->reqs = TALLOC_ZERO_ARRAY(state, struct cli_push_write_state *,
+	state->reqs = talloc_zero_array(state, struct cli_push_write_state *,
 					state->num_reqs);
 	if (state->reqs == NULL) {
 		goto failed;
@@ -1305,7 +1205,7 @@ static void cli_push_written(struct tevent_req *subreq)
 	state->reqs[idx] = NULL;
 	state->pending -= 1;
 
-	status = cli_writeall_recv(subreq);
+	status = cli_writeall_recv(subreq, NULL);
 	TALLOC_FREE(subreq);
 	TALLOC_FREE(substate);
 	if (tevent_req_nterror(req, status)) {
@@ -1369,8 +1269,5 @@ NTSTATUS cli_push(struct cli_state *cli, uint16_t fnum, uint16_t mode,
 	status = cli_push_recv(req);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
diff --git a/source3/libsmb/clisecdesc.c b/source3/libsmb/clisecdesc.c
index 5543ce50330..d703b1f7744 100644
--- a/source3/libsmb/clisecdesc.c
+++ b/source3/libsmb/clisecdesc.c
@@ -18,6 +18,8 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
+#include "../libcli/security/secdesc.h"
 
 /****************************************************************************
   query the security descriptor for a open file
diff --git a/source3/libsmb/clisigning.c b/source3/libsmb/clisigning.c
index 1752edb023c..ac4db7626f5 100644
--- a/source3/libsmb/clisigning.c
+++ b/source3/libsmb/clisigning.c
@@ -20,6 +20,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "smb_signing.h"
 
 bool cli_simple_set_signing(struct cli_state *cli,
diff --git a/source3/libsmb/clistr.c b/source3/libsmb/clistr.c
index 944479c2a8f..ff18ef6bcfe 100644
--- a/source3/libsmb/clistr.c
+++ b/source3/libsmb/clistr.c
@@ -19,6 +19,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 
 size_t clistr_push_fn(struct cli_state *cli,
 			void *dest,
@@ -74,8 +75,3 @@ size_t clistr_pull_talloc(TALLOC_CTX *ctx,
 				  src_len,
 				  flags);
 }
-
-size_t clistr_align_out(struct cli_state *cli, const void *p, int flags)
-{
-	return align_string(cli->outbuf, (const char *)p, flags);
-}
diff --git a/source3/libsmb/clitrans.c b/source3/libsmb/clitrans.c
index dab8c10c843..dd8063bf1e3 100644
--- a/source3/libsmb/clitrans.c
+++ b/source3/libsmb/clitrans.c
@@ -18,6 +18,8 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "async_smb.h"
 
 struct trans_recvblob {
@@ -30,7 +32,6 @@ struct cli_trans_state {
 	struct event_context *ev;
 	uint8_t cmd;
 	uint16_t mid;
-	uint32_t seqnum;
 	const char *pipe_name;
 	uint8_t *pipe_name_conv;
 	size_t pipe_name_conv_len;
@@ -50,13 +51,29 @@ struct cli_trans_state {
 	struct trans_recvblob rdata;
 	uint16_t recv_flags2;
 
-	TALLOC_CTX *secondary_request_ctx;
-
-	struct iovec iov[4];
+	struct iovec iov[6];
 	uint8_t pad[4];
+	uint8_t zero_pad[4];
 	uint16_t vwv[32];
+
+	struct tevent_req *primary_subreq;
 };
 
+static void cli_trans_cleanup_primary(struct cli_trans_state *state)
+{
+	if (state->primary_subreq) {
+		cli_smb_req_set_mid(state->primary_subreq, 0);
+		cli_smb_req_unset_pending(state->primary_subreq);
+		TALLOC_FREE(state->primary_subreq);
+	}
+}
+
+static int cli_trans_state_destructor(struct cli_trans_state *state)
+{
+	cli_trans_cleanup_primary(state);
+	return 0;
+}
+
 static NTSTATUS cli_pull_trans(uint8_t *inbuf,
 			       uint8_t wct, uint16_t *vwv,
 			       uint16_t num_bytes, uint8_t *bytes,
@@ -146,7 +163,7 @@ static NTSTATUS cli_trans_pull_blob(TALLOC_CTX *mem_ctx,
 			return NT_STATUS_INVALID_NETWORK_RESPONSE;
 		}
 		blob->total = total;
-		blob->data = TALLOC_ARRAY(mem_ctx, uint8_t, total);
+		blob->data = talloc_array(mem_ctx, uint8_t, total);
 		if (blob->data == NULL) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -171,9 +188,12 @@ static void cli_trans_format(struct cli_trans_state *state, uint8_t *pwct,
 	struct iovec *iov = state->iov;
 	uint8_t *pad = state->pad;
 	uint16_t *vwv = state->vwv;
-	uint16_t param_offset;
-	uint16_t this_param = 0;
-	uint16_t this_data = 0;
+	uint32_t param_offset;
+	uint32_t this_param = 0;
+	uint32_t param_pad;
+	uint32_t data_offset;
+	uint32_t this_data = 0;
+	uint32_t data_pad;
 	uint32_t useable_space;
 	uint8_t cmd;
 
@@ -221,7 +241,18 @@ static void cli_trans_format(struct cli_trans_state *state, uint8_t *pwct,
 		break;
 	}
 
-	useable_space = state->cli->max_xmit - smb_size - sizeof(uint16_t)*wct;
+	param_offset += wct * sizeof(uint16_t);
+	useable_space = state->cli->max_xmit - param_offset;
+
+	param_pad = param_offset % 4;
+	if (param_pad > 0) {
+		param_pad = MIN(param_pad, useable_space);
+		iov[0].iov_base = (void *)state->zero_pad;
+		iov[0].iov_len = param_pad;
+		iov += 1;
+		param_offset += param_pad;
+	}
+	useable_space = state->cli->max_xmit - param_offset;
 
 	if (state->param_sent < state->num_param) {
 		this_param = MIN(state->num_param - state->param_sent,
@@ -231,27 +262,41 @@ static void cli_trans_format(struct cli_trans_state *state, uint8_t *pwct,
 		iov += 1;
 	}
 
+	data_offset = param_offset + this_param;
+	useable_space = state->cli->max_xmit - data_offset;
+
+	data_pad = data_offset % 4;
+	if (data_pad > 0) {
+		data_pad = MIN(data_pad, useable_space);
+		iov[0].iov_base = (void *)state->zero_pad;
+		iov[0].iov_len = data_pad;
+		iov += 1;
+		data_offset += data_pad;
+	}
+	useable_space = state->cli->max_xmit - data_offset;
+
 	if (state->data_sent < state->num_data) {
 		this_data = MIN(state->num_data - state->data_sent,
-				useable_space - this_param);
+				useable_space);
 		iov[0].iov_base = (void *)(state->data + state->data_sent);
 		iov[0].iov_len = this_data;
 		iov += 1;
 	}
 
-	param_offset += wct * sizeof(uint16_t);
-
 	DEBUG(10, ("num_setup=%u, max_setup=%u, "
 		   "param_total=%u, this_param=%u, max_param=%u, "
 		   "data_total=%u, this_data=%u, max_data=%u, "
-		   "param_offset=%u, param_disp=%u, data_disp=%u\n",
+		   "param_offset=%u, param_pad=%u, param_disp=%u, "
+		   "data_offset=%u, data_pad=%u, data_disp=%u\n",
 		   (unsigned)state->num_setup, (unsigned)state->max_setup,
 		   (unsigned)state->num_param, (unsigned)this_param,
 		   (unsigned)state->rparam.max,
 		   (unsigned)state->num_data, (unsigned)this_data,
 		   (unsigned)state->rdata.max,
-		   (unsigned)param_offset,
-		   (unsigned)state->param_sent, (unsigned)state->data_sent));
+		   (unsigned)param_offset, (unsigned)param_pad,
+		   (unsigned)state->param_sent,
+		   (unsigned)data_offset, (unsigned)data_pad,
+		   (unsigned)state->data_sent));
 
 	switch (cmd) {
 	case SMBtrans:
@@ -268,7 +313,7 @@ static void cli_trans_format(struct cli_trans_state *state, uint8_t *pwct,
 		SSVAL(vwv + 9, 0, this_param);
 		SSVAL(vwv +10, 0, param_offset);
 		SSVAL(vwv +11, 0, this_data);
-		SSVAL(vwv +12, 0, param_offset + this_param);
+		SSVAL(vwv +12, 0, data_offset);
 		SCVAL(vwv +13, 0, state->num_setup);
 		SCVAL(vwv +13, 1, 0);	/* reserved */
 		memcpy(vwv + 14, state->setup,
@@ -282,40 +327,40 @@ static void cli_trans_format(struct cli_trans_state *state, uint8_t *pwct,
 		SSVAL(vwv + 3, 0, param_offset);
 		SSVAL(vwv + 4, 0, state->param_sent);
 		SSVAL(vwv + 5, 0, this_data);
-		SSVAL(vwv + 6, 0, param_offset + this_param);
+		SSVAL(vwv + 6, 0, data_offset);
 		SSVAL(vwv + 7, 0, state->data_sent);
 		if (cmd == SMBtranss2) {
 			SSVAL(vwv + 8, 0, state->fid);
 		}
 		break;
 	case SMBnttrans:
-		SCVAL(vwv,  0, state->max_setup);
-		SSVAL(vwv,  1, 0); /* reserved */
-		SIVAL(vwv,  3, state->num_param);
-		SIVAL(vwv,  7, state->num_data);
-		SIVAL(vwv, 11, state->rparam.max);
-		SIVAL(vwv, 15, state->rdata.max);
-		SIVAL(vwv, 19, this_param);
-		SIVAL(vwv, 23, param_offset);
-		SIVAL(vwv, 27, this_data);
-		SIVAL(vwv, 31, param_offset + this_param);
-		SCVAL(vwv, 35, state->num_setup);
-		SSVAL(vwv, 36, state->function);
+		SCVAL(vwv + 0, 0, state->max_setup);
+		SSVAL(vwv + 0, 1, 0); /* reserved */
+		SIVAL(vwv + 1, 1, state->num_param);
+		SIVAL(vwv + 3, 1, state->num_data);
+		SIVAL(vwv + 5, 1, state->rparam.max);
+		SIVAL(vwv + 7, 1, state->rdata.max);
+		SIVAL(vwv + 9, 1, this_param);
+		SIVAL(vwv +11, 1, param_offset);
+		SIVAL(vwv +13, 1, this_data);
+		SIVAL(vwv +15, 1, data_offset);
+		SCVAL(vwv +17, 1, state->num_setup);
+		SSVAL(vwv +18, 0, state->function);
 		memcpy(vwv + 19, state->setup,
 		       sizeof(uint16_t) * state->num_setup);
 		break;
 	case SMBnttranss:
-		SSVAL(vwv,  0, 0); /* reserved */
-		SCVAL(vwv,  2, 0); /* reserved */
-		SIVAL(vwv,  3, state->num_param);
-		SIVAL(vwv,  7, state->num_data);
-		SIVAL(vwv, 11, this_param);
-		SIVAL(vwv, 15, param_offset);
-		SIVAL(vwv, 19, state->param_sent);
-		SIVAL(vwv, 23, this_data);
-		SIVAL(vwv, 27, param_offset + this_param);
-		SIVAL(vwv, 31, state->data_sent);
-		SCVAL(vwv, 35, 0); /* reserved */
+		SSVAL(vwv + 0, 0, 0); /* reserved */
+		SCVAL(vwv + 1, 0, 0); /* reserved */
+		SIVAL(vwv + 1, 1, state->num_param);
+		SIVAL(vwv + 3, 1, state->num_data);
+		SIVAL(vwv + 5, 1, this_param);
+		SIVAL(vwv + 7, 1, param_offset);
+		SIVAL(vwv + 9, 1, state->param_sent);
+		SIVAL(vwv +11, 1, this_data);
+		SIVAL(vwv +13, 1, data_offset);
+		SIVAL(vwv +15, 1, state->data_sent);
+		SCVAL(vwv +17, 1, 0); /* reserved */
 		break;
 	}
 
@@ -412,17 +457,30 @@ struct tevent_req *cli_trans_send(
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
-	state->mid = cli_smb_req_mid(subreq);
 	status = cli_smb_req_send(subreq);
 	if (!NT_STATUS_IS_OK(status)) {
 		tevent_req_nterror(req, status);
 		return tevent_req_post(req, state->ev);
 	}
-	cli_state_seqnum_persistent(cli, state->mid);
 	tevent_req_set_callback(subreq, cli_trans_done, req);
+
+	/*
+	 * Now get the MID of the primary request
+	 * and mark it as persistent. This means
+	 * we will able to send and receive multiple
+	 * SMB pdus using this MID in both directions
+	 * (including correct SMB signing).
+	 */
+	state->mid = cli_smb_req_mid(subreq);
+	cli_smb_req_set_mid(subreq, state->mid);
+	state->primary_subreq = subreq;
+	talloc_set_destructor(state, cli_trans_state_destructor);
+
 	return req;
 }
 
+static void cli_trans_done2(struct tevent_req *subreq);
+
 static void cli_trans_done(struct tevent_req *subreq)
 {
 	struct tevent_req *req = tevent_req_callback_data(
@@ -478,25 +536,25 @@ static void cli_trans_done(struct tevent_req *subreq)
 
 	if (!sent_all) {
 		int iov_count;
-
-		TALLOC_FREE(subreq);
+		struct tevent_req *subreq2;
 
 		cli_trans_format(state, &wct, &iov_count);
 
-		subreq = cli_smb_req_create(state, state->ev, state->cli,
-					    state->cmd + 1, 0, wct, state->vwv,
-					    iov_count, state->iov);
-		if (tevent_req_nomem(subreq, req)) {
+		subreq2 = cli_smb_req_create(state, state->ev, state->cli,
+					     state->cmd + 1, 0, wct, state->vwv,
+					     iov_count, state->iov);
+		if (tevent_req_nomem(subreq2, req)) {
 			return;
 		}
-		cli_smb_req_set_mid(subreq, state->mid);
+		cli_smb_req_set_mid(subreq2, state->mid);
 
-		status = cli_smb_req_send(subreq);
+		status = cli_smb_req_send(subreq2);
 
 		if (!NT_STATUS_IS_OK(status)) {
 			goto fail;
 		}
-		tevent_req_set_callback(subreq, cli_trans_done, req);
+		tevent_req_set_callback(subreq2, cli_trans_done2, req);
+
 		return;
 	}
 
@@ -521,23 +579,80 @@ static void cli_trans_done(struct tevent_req *subreq)
 	if ((state->rparam.total == state->rparam.received)
 	    && (state->rdata.total == state->rdata.received)) {
 		state->recv_flags2 = SVAL(inbuf, smb_flg2);
-		TALLOC_FREE(subreq);
-		cli_state_seqnum_remove(state->cli, state->mid);
+		cli_trans_cleanup_primary(state);
 		tevent_req_done(req);
 		return;
 	}
 
 	TALLOC_FREE(inbuf);
 
-	if (!cli_smb_req_set_pending(subreq)) {
-		status = NT_STATUS_NO_MEMORY;
+	return;
+
+ fail:
+	cli_trans_cleanup_primary(state);
+	tevent_req_nterror(req, status);
+}
+
+static void cli_trans_done2(struct tevent_req *subreq2)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq2, struct tevent_req);
+	struct cli_trans_state *state = tevent_req_data(
+		req, struct cli_trans_state);
+	NTSTATUS status;
+	bool sent_all;
+	uint8_t wct;
+	uint32_t seqnum;
+
+	/*
+	 * First backup the seqnum of the secondary request
+	 * and attach it to the primary request.
+	 */
+	seqnum = cli_smb_req_seqnum(subreq2);
+	cli_smb_req_set_seqnum(state->primary_subreq, seqnum);
+
+	status = cli_smb_recv(subreq2, state, NULL, 0, &wct, NULL,
+			      NULL, NULL);
+	TALLOC_FREE(subreq2);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		goto fail;
+	}
+
+	if (wct != 0) {
+		status = NT_STATUS_INVALID_NETWORK_RESPONSE;
 		goto fail;
 	}
+
+	sent_all = ((state->param_sent == state->num_param)
+		    && (state->data_sent == state->num_data));
+
+	if (!sent_all) {
+		int iov_count;
+
+		cli_trans_format(state, &wct, &iov_count);
+
+		subreq2 = cli_smb_req_create(state, state->ev, state->cli,
+					     state->cmd + 1, 0, wct, state->vwv,
+					     iov_count, state->iov);
+		if (tevent_req_nomem(subreq2, req)) {
+			return;
+		}
+		cli_smb_req_set_mid(subreq2, state->mid);
+
+		status = cli_smb_req_send(subreq2);
+
+		if (!NT_STATUS_IS_OK(status)) {
+			goto fail;
+		}
+		tevent_req_set_callback(subreq2, cli_trans_done2, req);
+		return;
+	}
+
 	return;
 
  fail:
-	cli_state_seqnum_remove(state->cli, state->mid);
-	TALLOC_FREE(subreq);
+	cli_trans_cleanup_primary(state);
 	tevent_req_nterror(req, status);
 }
 
@@ -554,6 +669,8 @@ NTSTATUS cli_trans_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
 		req, struct cli_trans_state);
 	NTSTATUS status;
 
+	cli_trans_cleanup_primary(state);
+
 	if (tevent_req_is_nterror(req, &status)) {
 		return status;
 	}
@@ -644,8 +761,5 @@ NTSTATUS cli_trans(TALLOC_CTX *mem_ctx, struct cli_state *cli,
 				rdata, min_rdata, num_rdata);
  fail:
 	TALLOC_FREE(frame);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_set_error(cli, status);
-	}
 	return status;
 }
diff --git a/source3/libsmb/dsgetdcname.c b/source3/libsmb/dsgetdcname.c
index e062818ddae..5df833f40f8 100644
--- a/source3/libsmb/dsgetdcname.c
+++ b/source3/libsmb/dsgetdcname.c
@@ -32,7 +32,6 @@
 
 struct ip_service_name {
 	struct sockaddr_storage ss;
-	unsigned port;
 	const char *hostname;
 };
 
@@ -340,7 +339,7 @@ static NTSTATUS dsgetdcname_cache_fetch(TALLOC_CTX *mem_ctx,
 		return NT_STATUS_NOT_FOUND;
 	}
 
-	info = TALLOC_ZERO_P(mem_ctx, struct netr_DsRGetDCNameInfo);
+	info = talloc_zero(mem_ctx, struct netr_DsRGetDCNameInfo);
 	if (!info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -499,7 +498,7 @@ static NTSTATUS discover_dc_netbios(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	dclist = TALLOC_ZERO_ARRAY(mem_ctx, struct ip_service_name, count);
+	dclist = talloc_zero_array(mem_ctx, struct ip_service_name, count);
 	if (!dclist) {
 		SAFE_FREE(iplist);
 		return NT_STATUS_NO_MEMORY;
@@ -514,7 +513,6 @@ static NTSTATUS discover_dc_netbios(TALLOC_CTX *mem_ctx,
 			       &iplist[i].ss);
 
 		r->ss	= iplist[i].ss;
-		r->port = iplist[i].port;
 		r->hostname = talloc_strdup(mem_ctx, addr);
 		if (!r->hostname) {
 			SAFE_FREE(iplist);
@@ -581,7 +579,7 @@ static NTSTATUS discover_dc_dns(TALLOC_CTX *mem_ctx,
 		numaddrs += MAX(dcs[i].num_ips,1);
 	}
 
-	dclist = TALLOC_ZERO_ARRAY(mem_ctx,
+	dclist = talloc_zero_array(mem_ctx,
 				   struct ip_service_name,
 				   numaddrs);
 	if (!dclist) {
@@ -598,7 +596,6 @@ static NTSTATUS discover_dc_dns(TALLOC_CTX *mem_ctx,
 
 		struct ip_service_name *r = &dclist[count];
 
-		r->port = dcs[i].port;
 		r->hostname = dcs[i].hostname;
 
 		/* If we don't have an IP list for a name, lookup it up */
@@ -609,7 +606,7 @@ static NTSTATUS discover_dc_dns(TALLOC_CTX *mem_ctx,
 			i++;
 			j = 0;
 		} else {
-			/* use the IP addresses from the SRV sresponse */
+			/* use the IP addresses from the SRV response */
 
 			if (j >= dcs[i].num_ips) {
 				i++;
@@ -623,8 +620,8 @@ static NTSTATUS discover_dc_dns(TALLOC_CTX *mem_ctx,
 
 		/* make sure it is a valid IP.  I considered checking the
 		 * negative connection cache, but this is the wrong place for
-		 * it.  Maybe only as a hac.  After think about it, if all of
-		 * the IP addresses retuend from DNS are dead, what hope does a
+		 * it.  Maybe only as a hack. After think about it, if all of
+		 * the IP addresses returned from DNS are dead, what hope does a
 		 * netbios name lookup have?  The standard reason for falling
 		 * back to netbios lookups is that our DNS server doesn't know
 		 * anything about the DC's   -- jerry */
@@ -662,7 +659,7 @@ static NTSTATUS make_domain_controller_info(TALLOC_CTX *mem_ctx,
 {
 	struct netr_DsRGetDCNameInfo *info;
 
-	info = TALLOC_ZERO_P(mem_ctx, struct netr_DsRGetDCNameInfo);
+	info = talloc_zero(mem_ctx, struct netr_DsRGetDCNameInfo);
 	NT_STATUS_HAVE_NO_MEMORY(info);
 
 	if (dc_unc) {
@@ -864,9 +861,10 @@ static NTSTATUS process_dc_dns(TALLOC_CTX *mem_ctx,
 
 	for (i=0; i<num_dcs; i++) {
 
+
 		DEBUG(10,("LDAP ping to %s\n", dclist[i].hostname));
 
-		if (ads_cldap_netlogon(mem_ctx, dclist[i].hostname,
+		if (ads_cldap_netlogon(mem_ctx, &dclist[i].ss,
 					domain_name,
 					nt_version,
 					&r))
@@ -938,10 +936,8 @@ static NTSTATUS process_dc_netbios(TALLOC_CTX *mem_ctx,
 
 	for (i=0; i<num_dcs; i++) {
 		uint16_t val;
-		int dgm_id;
 
 		generate_random_buffer((uint8_t *)&val, 2);
-		dgm_id = val;
 
 		ip_list.ss = dclist[i].ss;
 		ip_list.port = 0;
@@ -967,7 +963,7 @@ static NTSTATUS process_dc_netbios(TALLOC_CTX *mem_ctx,
 		{
 			struct NETLOGON_SAM_LOGON_RESPONSE_NT40 logon1;
 
-			r = TALLOC_ZERO_P(mem_ctx, struct netlogon_samlogon_response);
+			r = talloc_zero(mem_ctx, struct netlogon_samlogon_response);
 			NT_STATUS_HAVE_NO_MEMORY(r);
 
 			ZERO_STRUCT(logon1);
diff --git a/source3/libsmb/errormap.c b/source3/libsmb/errormap.c
index bc9676bbff6..35b4137ec3a 100644
--- a/source3/libsmb/errormap.c
+++ b/source3/libsmb/errormap.c
@@ -29,600 +29,6 @@
 #include <gssapi.h>
 #endif
 
-/* This map was extracted by the ERRMAPEXTRACT smbtorture command. 
-   The setup was a Samba HEAD (2002-01-03) PDC and an Win2k member 
-   workstation.  The PDC was modified (by using the 'name_to_nt_status'
-   authentication module) to convert the username (in hex) into the
-   corresponding NTSTATUS error return. 
-
-   By opening two nbt sessions to the Win2k workstation, one negotiating
-   DOS and one negotiating NT errors it was possible to extract the
-   error mapping.  (Because the server only supplies NT errors, the 
-   NT4 workstation had to use its own error tables to convert these
-   to dos errors). 
-
-   Some errors show up as 'squashed' because the NT error connection
-   got back a different error to the one it sent, so a mapping could
-   not be determined (a guess has been made in this case, to map the
-   error as squashed).  This is done mainly to prevent users from getting
-   NT_STATUS_WRONG_PASSWORD and NT_STATUS_NO_SUCH_USER errors (they get
-   NT_STATUS_LOGON_FAILURE instead.
-
-   -- abartlet (2002-01-03)
-*/
-
-/* NT status -> dos error map */
-static const struct {
-	uint8 dos_class;
-	uint32 dos_code;
-	NTSTATUS ntstatus;
-} ntstatus_to_dos_map[] = {
-	{ERRDOS,	ERRgeneral,	NT_STATUS_UNSUCCESSFUL},
-	{ERRDOS,	ERRbadfunc,	NT_STATUS_NOT_IMPLEMENTED},
-	{ERRDOS,	87,	NT_STATUS_INVALID_INFO_CLASS},
-	{ERRDOS,	24,	NT_STATUS_INFO_LENGTH_MISMATCH},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ACCESS_VIOLATION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_IN_PAGE_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PAGEFILE_QUOTA},
-	{ERRDOS,	ERRbadfid,	NT_STATUS_INVALID_HANDLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_INITIAL_STACK},
-	{ERRDOS,	193,	NT_STATUS_BAD_INITIAL_PC},
-	{ERRDOS,	87,	NT_STATUS_INVALID_CID},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TIMER_NOT_CANCELED},
-	{ERRDOS,	ERRinvalidparam,	NT_STATUS_INVALID_PARAMETER},
-	{ERRDOS,	ERRbadfile,	NT_STATUS_NO_SUCH_DEVICE},
-	{ERRDOS,	ERRbadfile,	NT_STATUS_NO_SUCH_FILE},
-	{ERRDOS,	ERRbadfunc,	NT_STATUS_INVALID_DEVICE_REQUEST},
-	{ERRDOS,	38,	NT_STATUS_END_OF_FILE},
-	{ERRDOS,	34,	NT_STATUS_WRONG_VOLUME},
-	{ERRDOS,	21,	NT_STATUS_NO_MEDIA_IN_DEVICE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNRECOGNIZED_MEDIA},
-	{ERRDOS,	27,	NT_STATUS_NONEXISTENT_SECTOR},
-/** Session setup succeeded.  This shouldn't happen...*/
-/** Session setup succeeded.  This shouldn't happen...*/
-/** NT error on DOS connection! (NT_STATUS_OK) */
-/*	{ This NT error code was 'sqashed'
-	 from NT_STATUS_MORE_PROCESSING_REQUIRED to NT_STATUS_OK 
-	 during the session setup }
-*/
-#if 0
-	{SUCCESS,	0,	NT_STATUS_OK},
-#endif
-	{ERRDOS,	ERRnomem,	NT_STATUS_NO_MEMORY},
-	{ERRDOS,	487,	NT_STATUS_CONFLICTING_ADDRESSES},
-	{ERRDOS,	487,	NT_STATUS_NOT_MAPPED_VIEW},
-	{ERRDOS,	87,	NT_STATUS_UNABLE_TO_FREE_VM},
-	{ERRDOS,	87,	NT_STATUS_UNABLE_TO_DELETE_SECTION},
-	{ERRDOS,	2142,	NT_STATUS_INVALID_SYSTEM_SERVICE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ILLEGAL_INSTRUCTION},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_INVALID_LOCK_SEQUENCE},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_INVALID_VIEW_SIZE},
-	{ERRDOS,	193,	NT_STATUS_INVALID_FILE_FOR_SECTION},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_ALREADY_COMMITTED},
-/*	{ This NT error code was 'sqashed'
-	 from NT_STATUS_ACCESS_DENIED to NT_STATUS_TRUSTED_RELATIONSHIP_FAILURE 
-	 during the session setup }
-*/
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_ACCESS_DENIED},
-	{ERRDOS,	111,	NT_STATUS_BUFFER_TOO_SMALL},
-/*
- * Not an official error, as only bit 0x80000000, not bits 0xC0000000 are set.
- */
-	{ERRDOS,	ERRmoredata,	STATUS_BUFFER_OVERFLOW},
-	{ERRDOS,	ERRnofiles,	STATUS_NO_MORE_FILES},
-	{ERRDOS,	ERRbadfid,	NT_STATUS_OBJECT_TYPE_MISMATCH},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NONCONTINUABLE_EXCEPTION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_DISPOSITION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNWIND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_STACK},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_UNWIND_TARGET},
-	{ERRDOS,	158,	NT_STATUS_NOT_LOCKED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PARITY_ERROR},
-	{ERRDOS,	487,	NT_STATUS_UNABLE_TO_DECOMMIT_VM},
-	{ERRDOS,	487,	NT_STATUS_NOT_COMMITTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_PORT_ATTRIBUTES},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PORT_MESSAGE_TOO_LONG},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_MIX},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_QUOTA_LOWER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DISK_CORRUPT_ERROR},
-	{ERRDOS,	ERRinvalidname,	NT_STATUS_OBJECT_NAME_INVALID},
-	{ERRDOS,	ERRbadfile,	NT_STATUS_OBJECT_NAME_NOT_FOUND},
-	{ERRDOS,	183,	NT_STATUS_OBJECT_NAME_COLLISION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_HANDLE_NOT_WAITABLE},
-	{ERRDOS,	ERRbadfid,	NT_STATUS_PORT_DISCONNECTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DEVICE_ALREADY_ATTACHED},
-	{ERRDOS,	ERRinvalidpath,	NT_STATUS_OBJECT_PATH_INVALID},
-	{ERRDOS,	ERRbadpath,	NT_STATUS_OBJECT_PATH_NOT_FOUND},
-	{ERRDOS,	ERRinvalidpath,	NT_STATUS_OBJECT_PATH_SYNTAX_BAD},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DATA_OVERRUN},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DATA_LATE_ERROR},
-	{ERRDOS,	23,	NT_STATUS_DATA_ERROR},
-	{ERRDOS,	23,	NT_STATUS_CRC_ERROR},
-	{ERRDOS,	ERRnomem,	NT_STATUS_SECTION_TOO_BIG},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_PORT_CONNECTION_REFUSED},
-	{ERRDOS,	ERRbadfid,	NT_STATUS_INVALID_PORT_HANDLE},
-	{ERRDOS,	ERRbadshare,	NT_STATUS_SHARING_VIOLATION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_QUOTA_EXCEEDED},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PAGE_PROTECTION},
-	{ERRDOS,	288,	NT_STATUS_MUTANT_NOT_OWNED},
-	{ERRDOS,	298,	NT_STATUS_SEMAPHORE_LIMIT_EXCEEDED},
-	{ERRDOS,	87,	NT_STATUS_PORT_ALREADY_SET},
-	{ERRDOS,	87,	NT_STATUS_SECTION_NOT_IMAGE},
-	{ERRDOS,	156,	NT_STATUS_SUSPEND_COUNT_EXCEEDED},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_THREAD_IS_TERMINATING},
-	{ERRDOS,	87,	NT_STATUS_BAD_WORKING_SET_LIMIT},
-	{ERRDOS,	87,	NT_STATUS_INCOMPATIBLE_FILE_MAP},
-	{ERRDOS,	87,	NT_STATUS_SECTION_PROTECTION},
-	{ERRDOS,	ERReasnotsupported,	NT_STATUS_EAS_NOT_SUPPORTED},
-	{ERRDOS,	255,	NT_STATUS_EA_TOO_LARGE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NONEXISTENT_EA_ENTRY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_EAS_ON_FILE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_EA_CORRUPT_ERROR},
-	{ERRDOS,	ERRlock,	NT_STATUS_FILE_LOCK_CONFLICT},
-	{ERRDOS,	ERRlock,	NT_STATUS_LOCK_NOT_GRANTED},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_DELETE_PENDING},
-	{ERRDOS,	ERRunsup,	NT_STATUS_CTL_FILE_NOT_SUPPORTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNKNOWN_REVISION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REVISION_MISMATCH},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_OWNER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_PRIMARY_GROUP},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_IMPERSONATION_TOKEN},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANT_DISABLE_MANDATORY},
-	{ERRDOS,	2215,	NT_STATUS_NO_LOGON_SERVERS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SUCH_LOGON_SESSION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SUCH_PRIVILEGE},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_PRIVILEGE_NOT_HELD},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_ACCOUNT_NAME},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_USER_EXISTS},
-/*	{ This NT error code was 'sqashed'
-	 from NT_STATUS_NO_SUCH_USER to NT_STATUS_LOGON_FAILURE 
-	 during the session setup }
-*/
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_NO_SUCH_USER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_GROUP_EXISTS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SUCH_GROUP},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MEMBER_IN_GROUP},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MEMBER_NOT_IN_GROUP},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LAST_ADMIN},
-/*	{ This NT error code was 'sqashed'
-	 from NT_STATUS_WRONG_PASSWORD to NT_STATUS_LOGON_FAILURE 
-	 during the session setup }
-*/
-	{ERRSRV,	ERRbadpw,	NT_STATUS_WRONG_PASSWORD},
-	{ERRSRV,	ERRbaduid,	NT_STATUS_USER_SESSION_DELETED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ILL_FORMED_PASSWORD},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PASSWORD_RESTRICTION},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_LOGON_FAILURE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ACCOUNT_RESTRICTION},
-	{ERRSRV,	2241,	NT_STATUS_INVALID_LOGON_HOURS},
-	{ERRSRV,	2240,	NT_STATUS_INVALID_WORKSTATION},
-	{ERRSRV,	2242,	NT_STATUS_PASSWORD_EXPIRED},
-	{ERRSRV,	2239,	NT_STATUS_ACCOUNT_DISABLED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NONE_MAPPED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TOO_MANY_LUIDS_REQUESTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LUIDS_EXHAUSTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_SUB_AUTHORITY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_ACL},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_SID},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_SECURITY_DESCR},
-	{ERRDOS,	127,	NT_STATUS_PROCEDURE_NOT_FOUND},
-	{ERRDOS,	193,	NT_STATUS_INVALID_IMAGE_FORMAT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_TOKEN},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_INHERITANCE_ACL},
-	{ERRDOS,	158,	NT_STATUS_RANGE_NOT_LOCKED},
-	{ERRDOS,	112,	NT_STATUS_DISK_FULL},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SERVER_DISABLED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SERVER_NOT_DISABLED},
-	{ERRDOS,	68,	NT_STATUS_TOO_MANY_GUIDS_REQUESTED},
-	{ERRDOS,	259,	NT_STATUS_GUIDS_EXHAUSTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_ID_AUTHORITY},
-	{ERRDOS,	259,	NT_STATUS_AGENTS_EXHAUSTED},
-	{ERRDOS,	154,	NT_STATUS_INVALID_VOLUME_LABEL},
-	{ERRDOS,	ERRres,	NT_STATUS_SECTION_NOT_EXTENDED},
-	{ERRDOS,	487,	NT_STATUS_NOT_MAPPED_DATA},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RESOURCE_DATA_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RESOURCE_TYPE_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RESOURCE_NAME_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ARRAY_BOUNDS_EXCEEDED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOAT_DENORMAL_OPERAND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOAT_DIVIDE_BY_ZERO},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOAT_INEXACT_RESULT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOAT_INVALID_OPERATION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOAT_OVERFLOW},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOAT_STACK_CHECK},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOAT_UNDERFLOW},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INTEGER_DIVIDE_BY_ZERO},
-	{ERRDOS,	534,	NT_STATUS_INTEGER_OVERFLOW},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PRIVILEGED_INSTRUCTION},
-	{ERRDOS,	ERRnomem,	NT_STATUS_TOO_MANY_PAGING_FILES},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FILE_INVALID},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ALLOTTED_SPACE_EXCEEDED},
-/*	{ This NT error code was 'sqashed'
-	 from NT_STATUS_INSUFFICIENT_RESOURCES to NT_STATUS_INSUFF_SERVER_RESOURCES 
-	 during the session setup }
-*/
-	{ERRDOS,	ERRnomem,	NT_STATUS_INSUFFICIENT_RESOURCES},
-	{ERRDOS,	ERRbadpath,	NT_STATUS_DFS_EXIT_PATH_FOUND},
-	{ERRDOS,	23,	NT_STATUS_DEVICE_DATA_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DEVICE_NOT_CONNECTED},
-	{ERRDOS,	21,	NT_STATUS_DEVICE_POWER_FAILURE},
-	{ERRDOS,	487,	NT_STATUS_FREE_VM_NOT_AT_BASE},
-	{ERRDOS,	487,	NT_STATUS_MEMORY_NOT_ALLOCATED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_WORKING_SET_QUOTA},
-	{ERRDOS,	19,	NT_STATUS_MEDIA_WRITE_PROTECTED},
-	{ERRDOS,	21,	NT_STATUS_DEVICE_NOT_READY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_GROUP_ATTRIBUTES},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_IMPERSONATION_LEVEL},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANT_OPEN_ANONYMOUS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_VALIDATION_CLASS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_TOKEN_TYPE},
-	{ERRDOS,	87,	NT_STATUS_BAD_MASTER_BOOT_RECORD},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INSTRUCTION_MISALIGNMENT},
-	{ERRDOS,	ERRpipebusy,	NT_STATUS_INSTANCE_NOT_AVAILABLE},
-	{ERRDOS,	ERRpipebusy,	NT_STATUS_PIPE_NOT_AVAILABLE},
-	{ERRDOS,	ERRbadpipe,	NT_STATUS_INVALID_PIPE_STATE},
-	{ERRDOS,	ERRpipebusy,	NT_STATUS_PIPE_BUSY},
-	{ERRDOS,	ERRbadfunc,	NT_STATUS_ILLEGAL_FUNCTION},
-	{ERRDOS,	ERRnotconnected,	NT_STATUS_PIPE_DISCONNECTED},
-	{ERRDOS,	ERRpipeclosing,	NT_STATUS_PIPE_CLOSING},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PIPE_CONNECTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PIPE_LISTENING},
-	{ERRDOS,	ERRbadpipe,	NT_STATUS_INVALID_READ_MODE},
-	{ERRDOS,	121,	NT_STATUS_IO_TIMEOUT},
-	{ERRDOS,	38,	NT_STATUS_FILE_FORCED_CLOSED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PROFILING_NOT_STARTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PROFILING_NOT_STOPPED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_COULD_NOT_INTERPRET},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_FILE_IS_A_DIRECTORY},
-	{ERRDOS,	ERRunsup,	NT_STATUS_NOT_SUPPORTED},
-	{ERRDOS,	51,	NT_STATUS_REMOTE_NOT_LISTENING},
-	{ERRDOS,	52,	NT_STATUS_DUPLICATE_NAME},
-	{ERRDOS,	53,	NT_STATUS_BAD_NETWORK_PATH},
-	{ERRDOS,	54,	NT_STATUS_NETWORK_BUSY},
-	{ERRDOS,	55,	NT_STATUS_DEVICE_DOES_NOT_EXIST},
-	{ERRDOS,	56,	NT_STATUS_TOO_MANY_COMMANDS},
-	{ERRDOS,	57,	NT_STATUS_ADAPTER_HARDWARE_ERROR},
-	{ERRDOS,	58,	NT_STATUS_INVALID_NETWORK_RESPONSE},
-	{ERRDOS,	59,	NT_STATUS_UNEXPECTED_NETWORK_ERROR},
-	{ERRDOS,	60,	NT_STATUS_BAD_REMOTE_ADAPTER},
-	{ERRDOS,	61,	NT_STATUS_PRINT_QUEUE_FULL},
-	{ERRDOS,	62,	NT_STATUS_NO_SPOOL_SPACE},
-	{ERRDOS,	63,	NT_STATUS_PRINT_CANCELLED},
-	{ERRDOS,	64,	NT_STATUS_NETWORK_NAME_DELETED},
-	{ERRDOS,	65,	NT_STATUS_NETWORK_ACCESS_DENIED},
-	{ERRDOS,	66,	NT_STATUS_BAD_DEVICE_TYPE},
-	{ERRDOS,	ERRnosuchshare,	NT_STATUS_BAD_NETWORK_NAME},
-	{ERRDOS,	68,	NT_STATUS_TOO_MANY_NAMES},
-	{ERRDOS,	69,	NT_STATUS_TOO_MANY_SESSIONS},
-	{ERRDOS,	70,	NT_STATUS_SHARING_PAUSED},
-	{ERRDOS,	71,	NT_STATUS_REQUEST_NOT_ACCEPTED},
-	{ERRDOS,	72,	NT_STATUS_REDIRECTOR_PAUSED},
-	{ERRDOS,	88,	NT_STATUS_NET_WRITE_FAULT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PROFILING_AT_LIMIT},
-	{ERRDOS,	ERRdiffdevice,	NT_STATUS_NOT_SAME_DEVICE},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_FILE_RENAMED},
-	{ERRDOS,	240,	NT_STATUS_VIRTUAL_CIRCUIT_CLOSED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SECURITY_ON_OBJECT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANT_WAIT},
-	{ERRDOS,	ERRpipeclosing,	NT_STATUS_PIPE_EMPTY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANT_ACCESS_DOMAIN_INFO},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANT_TERMINATE_SELF},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_SERVER_STATE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_DOMAIN_STATE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_DOMAIN_ROLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SUCH_DOMAIN},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DOMAIN_EXISTS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DOMAIN_LIMIT_EXCEEDED},
-	{ERRDOS,	300,	NT_STATUS_OPLOCK_NOT_GRANTED},
-	{ERRDOS,	301,	NT_STATUS_INVALID_OPLOCK_PROTOCOL},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INTERNAL_DB_CORRUPTION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INTERNAL_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_GENERIC_NOT_MAPPED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_DESCRIPTOR_FORMAT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_USER_BUFFER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNEXPECTED_IO_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNEXPECTED_MM_CREATE_ERR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNEXPECTED_MM_MAP_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNEXPECTED_MM_EXTEND_ERR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NOT_LOGON_PROCESS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOGON_SESSION_EXISTS},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_1},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_2},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_3},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_4},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_5},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_6},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_7},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_8},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_9},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_10},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_11},
-	{ERRDOS,	87,	NT_STATUS_INVALID_PARAMETER_12},
-	{ERRDOS,	ERRbadpath,	NT_STATUS_REDIRECTOR_NOT_STARTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REDIRECTOR_STARTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_STACK_OVERFLOW},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SUCH_PACKAGE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_FUNCTION_TABLE},
-	{ERRDOS,	203,	NT_STATUS(0xc0000100)},
-	{ERRDOS,	145,	NT_STATUS_DIRECTORY_NOT_EMPTY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FILE_CORRUPT_ERROR},
-	{ERRDOS,	ERRbaddirectory,	NT_STATUS_NOT_A_DIRECTORY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_LOGON_SESSION_STATE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOGON_SESSION_COLLISION},
-	{ERRDOS,	206,	NT_STATUS_NAME_TOO_LONG},
-	{ERRDOS,	2401,	NT_STATUS_FILES_OPEN},
-	{ERRDOS,	2404,	NT_STATUS_CONNECTION_IN_USE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MESSAGE_NOT_FOUND},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_PROCESS_IS_TERMINATING},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_LOGON_TYPE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_GUID_TRANSLATION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANNOT_IMPERSONATE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_IMAGE_ALREADY_LOADED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_NOT_PRESENT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_LID_NOT_EXIST},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_LID_ALREADY_OWNED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_NOT_LID_OWNER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_INVALID_COMMAND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_INVALID_LID},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_SELECTOR_NOT_AVAILABLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ABIOS_INVALID_SELECTOR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_LDT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_LDT_SIZE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_LDT_OFFSET},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_LDT_DESCRIPTOR},
-	{ERRDOS,	193,	NT_STATUS_INVALID_IMAGE_NE_FORMAT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RXACT_INVALID_STATE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RXACT_COMMIT_FAILURE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MAPPED_FILE_SIZE_ZERO},
-	{ERRDOS,	ERRnofids,	NT_STATUS_TOO_MANY_OPENED_FILES},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANCELLED},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_CANNOT_DELETE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_COMPUTER_NAME},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_FILE_DELETED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SPECIAL_ACCOUNT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SPECIAL_GROUP},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SPECIAL_USER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MEMBERS_PRIMARY_GROUP},
-	{ERRDOS,	ERRbadfid,	NT_STATUS_FILE_CLOSED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TOO_MANY_THREADS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_THREAD_NOT_IN_PROCESS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TOKEN_ALREADY_IN_USE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PAGEFILE_QUOTA_EXCEEDED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_COMMITMENT_LIMIT},
-	{ERRDOS,	193,	NT_STATUS_INVALID_IMAGE_LE_FORMAT},
-	{ERRDOS,	193,	NT_STATUS_INVALID_IMAGE_NOT_MZ},
-	{ERRDOS,	193,	NT_STATUS_INVALID_IMAGE_PROTECT},
-	{ERRDOS,	193,	NT_STATUS_INVALID_IMAGE_WIN_16},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOGON_SERVER_CONFLICT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TIME_DIFFERENCE_AT_DC},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SYNCHRONIZATION_REQUIRED},
-	{ERRDOS,	126,	NT_STATUS_DLL_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_OPEN_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_IO_PRIVILEGE_FAILED},
-	{ERRDOS,	182,	NT_STATUS_ORDINAL_NOT_FOUND},
-	{ERRDOS,	127,	NT_STATUS_ENTRYPOINT_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CONTROL_C_EXIT},
-	{ERRDOS,	64,	NT_STATUS_LOCAL_DISCONNECT},
-	{ERRDOS,	64,	NT_STATUS_REMOTE_DISCONNECT},
-	{ERRDOS,	51,	NT_STATUS_REMOTE_RESOURCES},
-	{ERRDOS,	59,	NT_STATUS_LINK_FAILED},
-	{ERRDOS,	59,	NT_STATUS_LINK_TIMEOUT},
-	{ERRDOS,	59,	NT_STATUS_INVALID_CONNECTION},
-	{ERRDOS,	59,	NT_STATUS_INVALID_ADDRESS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DLL_INIT_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MISSING_SYSTEMFILE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNHANDLED_EXCEPTION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_APP_INIT_FAILURE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PAGEFILE_CREATE_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_PAGEFILE},
-	{ERRDOS,	ERRunknownlevel,	NT_STATUS_INVALID_LEVEL},
-	{ERRDOS,	86,	NT_STATUS_WRONG_PASSWORD_CORE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ILLEGAL_FLOAT_CONTEXT},
-	{ERRDOS,	109,	NT_STATUS_PIPE_BROKEN},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REGISTRY_CORRUPT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REGISTRY_IO_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_EVENT_PAIR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNRECOGNIZED_VOLUME},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SERIAL_NO_DEVICE_INITED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SUCH_ALIAS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MEMBER_NOT_IN_ALIAS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MEMBER_IN_ALIAS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ALIAS_EXISTS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOGON_NOT_GRANTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TOO_MANY_SECRETS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SECRET_TOO_LONG},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INTERNAL_DB_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FULLSCREEN_MODE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TOO_MANY_CONTEXT_IDS},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_LOGON_TYPE_NOT_GRANTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NOT_REGISTRY_FILE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NT_CROSS_ENCRYPTION_REQUIRED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DOMAIN_CTRLR_CONFIG_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FT_MISSING_MEMBER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ILL_FORMED_SERVICE_ENTRY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ILLEGAL_CHARACTER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNMAPPABLE_CHARACTER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNDEFINED_CHARACTER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOPPY_VOLUME},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOPPY_ID_MARK_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOPPY_WRONG_CYLINDER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOPPY_UNKNOWN_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FLOPPY_BAD_REGISTERS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DISK_RECALIBRATE_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DISK_OPERATION_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DISK_RESET_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SHARED_IRQ_BUSY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FT_ORPHANING},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000016e)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000016f)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc0000170)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc0000171)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PARTITION_FAILURE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_BLOCK_LENGTH},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DEVICE_NOT_PARTITIONED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNABLE_TO_LOCK_MEDIA},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNABLE_TO_UNLOAD_MEDIA},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_EOM_OVERFLOW},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_MEDIA},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc0000179)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_SUCH_MEMBER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_MEMBER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_KEY_DELETED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_LOG_SPACE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TOO_MANY_SIDS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LM_CROSS_ENCRYPTION_REQUIRED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_KEY_HAS_CHILDREN},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CHILD_MUST_BE_VOLATILE},
-	{ERRDOS,	87,	NT_STATUS_DEVICE_CONFIGURATION_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DRIVER_INTERNAL_ERROR},
-	{ERRDOS,	22,	NT_STATUS_INVALID_DEVICE_STATE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_IO_DEVICE_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DEVICE_PROTOCOL_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BACKUP_CONTROLLER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOG_FILE_FULL},
-	{ERRDOS,	19,	NT_STATUS_TOO_LATE},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_NO_TRUST_LSA_SECRET},
-/*	{ This NT error code was 'sqashed'
-	 from NT_STATUS_NO_TRUST_SAM_ACCOUNT to NT_STATUS_TRUSTED_RELATIONSHIP_FAILURE 
-	 during the session setup }
-*/
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_NO_TRUST_SAM_ACCOUNT},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_TRUSTED_DOMAIN_FAILURE},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_TRUSTED_RELATIONSHIP_FAILURE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_EVENTLOG_FILE_CORRUPT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_EVENTLOG_CANT_START},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_TRUST_FAILURE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MUTANT_LIMIT_EXCEEDED},
-	{ERRDOS,	ERRinvgroup,	NT_STATUS_NETLOGON_NOT_STARTED},
-	{ERRSRV,	2239,	NT_STATUS_ACCOUNT_EXPIRED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_POSSIBLE_DEADLOCK},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NETWORK_CREDENTIAL_CONFLICT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REMOTE_SESSION_LIMIT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_EVENTLOG_FILE_CHANGED},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_NOLOGON_SERVER_TRUST_ACCOUNT},
-/*	{ This NT error code was 'sqashed'
-	 from NT_STATUS_DOMAIN_TRUST_INCONSISTENT to NT_STATUS_LOGON_FAILURE 
-	 during the session setup }
-*/
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_DOMAIN_TRUST_INCONSISTENT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FS_DRIVER_REQUIRED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_USER_SESSION_KEY},
-	{ERRDOS,	59,	NT_STATUS_USER_SESSION_DELETED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RESOURCE_LANG_NOT_FOUND},
-	{ERRDOS,	ERRnomem,	NT_STATUS_INSUFF_SERVER_RESOURCES},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_BUFFER_SIZE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_ADDRESS_COMPONENT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_ADDRESS_WILDCARD},
-	{ERRDOS,	68,	NT_STATUS_TOO_MANY_ADDRESSES},
-	{ERRDOS,	52,	NT_STATUS_ADDRESS_ALREADY_EXISTS},
-	{ERRDOS,	64,	NT_STATUS_ADDRESS_CLOSED},
-	{ERRDOS,	64,	NT_STATUS_CONNECTION_DISCONNECTED},
-	{ERRDOS,	64,	NT_STATUS_CONNECTION_RESET},
-	{ERRDOS,	68,	NT_STATUS_TOO_MANY_NODES},
-	{ERRDOS,	59,	NT_STATUS_TRANSACTION_ABORTED},
-	{ERRDOS,	59,	NT_STATUS_TRANSACTION_TIMED_OUT},
-	{ERRDOS,	59,	NT_STATUS_TRANSACTION_NO_RELEASE},
-	{ERRDOS,	59,	NT_STATUS_TRANSACTION_NO_MATCH},
-	{ERRDOS,	59,	NT_STATUS_TRANSACTION_RESPONDED},
-	{ERRDOS,	59,	NT_STATUS_TRANSACTION_INVALID_ID},
-	{ERRDOS,	59,	NT_STATUS_TRANSACTION_INVALID_TYPE},
-	{ERRDOS,	ERRunsup,	NT_STATUS_NOT_SERVER_SESSION},
-	{ERRDOS,	ERRunsup,	NT_STATUS_NOT_CLIENT_SESSION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CANNOT_LOAD_REGISTRY_FILE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DEBUG_ATTACH_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_SYSTEM_PROCESS_TERMINATED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DATA_NOT_ACCEPTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_BROWSER_SERVERS_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_VDM_HARD_ERROR},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DRIVER_CANCEL_TIMEOUT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REPLY_MESSAGE_MISMATCH},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MAPPED_ALIGNMENT},
-	{ERRDOS,	193,	NT_STATUS_IMAGE_CHECKSUM_MISMATCH},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOST_WRITEBEHIND_DATA},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CLIENT_SERVER_PARAMETERS_INVALID},
-	{ERRSRV,	2242,	NT_STATUS_PASSWORD_MUST_CHANGE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NOT_TINY_STREAM},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RECOVERY_FAILURE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_STACK_OVERFLOW_READ},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FAIL_CHECK},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DUPLICATE_OBJECTID},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_OBJECTID_EXISTS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CONVERT_TO_LARGE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_RETRY},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FOUND_OUT_OF_SCOPE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ALLOCATE_BUCKET},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PROPSET_NOT_FOUND},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_MARSHALL_OVERFLOW},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_VARIANT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS_ACCOUNT_LOCKED_OUT},
-	{ERRDOS,	ERRbadfid,	NT_STATUS_HANDLE_NOT_CLOSABLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CONNECTION_REFUSED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_GRACEFUL_DISCONNECT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ADDRESS_ALREADY_ASSOCIATED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_ADDRESS_NOT_ASSOCIATED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CONNECTION_INVALID},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CONNECTION_ACTIVE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NETWORK_UNREACHABLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_HOST_UNREACHABLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PROTOCOL_UNREACHABLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PORT_UNREACHABLE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REQUEST_ABORTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CONNECTION_ABORTED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_COMPRESSION_BUFFER},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_USER_MAPPED_FILE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_AUDIT_FAILED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TIMER_RESOLUTION_NOT_SET},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_CONNECTION_COUNT_LIMIT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOGIN_TIME_RESTRICTION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LOGIN_WKSTA_RESTRICTION},
-	{ERRDOS,	193,	NT_STATUS_IMAGE_MP_UP_MISMATCH},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000024a)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000024b)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000024c)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000024d)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000024e)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000024f)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INSUFFICIENT_LOGON_INFO},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_DLL_ENTRYPOINT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_BAD_SERVICE_ENTRYPOINT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LPC_REPLY_LOST},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_IP_ADDRESS_CONFLICT1},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_IP_ADDRESS_CONFLICT2},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_REGISTRY_QUOTA_LIMIT},
-	{ERRSRV,	ERRbadtype,	NT_STATUS_PATH_NOT_COVERED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_NO_CALLBACK_ACTIVE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_LICENSE_QUOTA_EXCEEDED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PWD_TOO_SHORT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PWD_TOO_RECENT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PWD_HISTORY_CONFLICT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000025d)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_PLUGPLAY_NO_DEVICE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_UNSUPPORTED_COMPRESSION},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_HW_PROFILE},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_INVALID_PLUGPLAY_DEVICE_PATH},
-	{ERRDOS,	182,	NT_STATUS_DRIVER_ORDINAL_NOT_FOUND},
-	{ERRDOS,	127,	NT_STATUS_DRIVER_ENTRYPOINT_NOT_FOUND},
-	{ERRDOS,	288,	NT_STATUS_RESOURCE_NOT_OWNED},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_TOO_MANY_LINKS},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_QUOTA_LIST_INCONSISTENT},
-	{ERRHRD,	ERRgeneral,	NT_STATUS_FILE_IS_OFFLINE},
-	{ERRDOS,	21,	NT_STATUS(0xc000026e)},
-	{ERRDOS,	161,	NT_STATUS(0xc0000281)},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS(0xc000028a)},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS(0xc000028b)},
-	{ERRHRD,	ERRgeneral,	NT_STATUS(0xc000028c)},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS(0xc000028d)},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS(0xc000028e)},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS(0xc000028f)},
-	{ERRDOS,	ERRnoaccess,	NT_STATUS(0xc0000290)},
-	{ERRDOS,	ERRbadfunc,	NT_STATUS(0xc000029c)},
-};
-
-
 /* dos -> nt status error map */
 static const struct {
 	uint8 dos_class;
@@ -879,555 +285,6 @@ static const struct {
 	{ERRHRD,	1022,	NT_STATUS(0x0000010c)},
 };
 
-/* errmap NTSTATUS->Win32 */
-static const struct {
-	NTSTATUS ntstatus;
-	WERROR werror;
-} ntstatus_to_werror_map[] = {
-	{NT_STATUS(0x103), W_ERROR(0x3e5)},
-	{NT_STATUS(0x105), W_ERROR(0xea)},
-	{NT_STATUS(0x106), W_ERROR(0x514)},
-	{NT_STATUS(0x107), W_ERROR(0x515)},
-	{NT_STATUS(0x10c), W_ERROR(0x3fe)},
-	{NT_STATUS(0x10d), W_ERROR(0x516)},
-	{NT_STATUS(0x121), W_ERROR(0x2009)},
-	{NT_STATUS(0xc0000001), W_ERROR(0x1f)},
-	{NT_STATUS(0xc0000002), W_ERROR(0x1)},
-	{NT_STATUS(0xc0000003), W_ERROR(0x57)},
-	{NT_STATUS(0xc0000004), W_ERROR(0x18)},
-	{NT_STATUS(0xc0000005), W_ERROR(0x3e6)},
-	{NT_STATUS(0xc0000006), W_ERROR(0x3e7)},
-	{NT_STATUS(0xc0000007), W_ERROR(0x5ae)},
-	{NT_STATUS(0xc0000008), W_ERROR(0x6)},
-	{NT_STATUS(0xc0000009), W_ERROR(0x3e9)},
-	{NT_STATUS(0xc000000a), W_ERROR(0xc1)},
-	{NT_STATUS(0xc000000b), W_ERROR(0x57)},
-	{NT_STATUS(0xc000000d), W_ERROR(0x57)},
-	{NT_STATUS(0xc000000e), W_ERROR(0x2)},
-	{NT_STATUS(0xc000000f), W_ERROR(0x2)},
-	{NT_STATUS(0xc0000010), W_ERROR(0x1)},
-	{NT_STATUS(0xc0000011), W_ERROR(0x26)},
-	{NT_STATUS(0xc0000012), W_ERROR(0x22)},
-	{NT_STATUS(0xc0000013), W_ERROR(0x15)},
-	{NT_STATUS(0xc0000014), W_ERROR(0x6f9)},
-	{NT_STATUS(0xc0000015), W_ERROR(0x1b)},
-	{NT_STATUS(0xc0000016), W_ERROR(0xea)},
-	{NT_STATUS(0xc0000017), W_ERROR(0x8)},
-	{NT_STATUS(0xc0000018), W_ERROR(0x1e7)},
-	{NT_STATUS(0xc0000019), W_ERROR(0x1e7)},
-	{NT_STATUS(0xc000001a), W_ERROR(0x57)},
-	{NT_STATUS(0xc000001b), W_ERROR(0x57)},
-	{NT_STATUS(0xc000001c), W_ERROR(0x1)},
-	{NT_STATUS(0xc000001d), W_ERROR(0xc000001d)},
-	{NT_STATUS(0xc000001e), W_ERROR(0x5)},
-	{NT_STATUS(0xc000001f), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000020), W_ERROR(0xc1)},
-	{NT_STATUS(0xc0000021), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000022), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000023), W_ERROR(0x7a)},
-	{NT_STATUS(0xc0000024), W_ERROR(0x6)},
-	{NT_STATUS(0xc0000025), W_ERROR(0xc0000025)},
-	{NT_STATUS(0xc0000026), W_ERROR(0xc0000026)},
-	{NT_STATUS(0xc000002a), W_ERROR(0x9e)},
-	{NT_STATUS(0xc000002b), W_ERROR(0xc000002b)},
-	{NT_STATUS(0xc000002c), W_ERROR(0x1e7)},
-	{NT_STATUS(0xc000002d), W_ERROR(0x1e7)},
-	{NT_STATUS(0xc0000030), W_ERROR(0x57)},
-	{NT_STATUS(0xc0000032), W_ERROR(0x571)},
-	{NT_STATUS(0xc0000033), W_ERROR(0x7b)},
-	{NT_STATUS(0xc0000034), W_ERROR(0x2)},
-	{NT_STATUS(0xc0000035), W_ERROR(0xb7)},
-	{NT_STATUS(0xc0000037), W_ERROR(0x6)},
-	{NT_STATUS(0xc0000039), W_ERROR(0xa1)},
-	{NT_STATUS(0xc000003a), W_ERROR(0x3)},
-	{NT_STATUS(0xc000003b), W_ERROR(0xa1)},
-	{NT_STATUS(0xc000003c), W_ERROR(0x45d)},
-	{NT_STATUS(0xc000003d), W_ERROR(0x45d)},
-	{NT_STATUS(0xc000003e), W_ERROR(0x17)},
-	{NT_STATUS(0xc000003f), W_ERROR(0x17)},
-	{NT_STATUS(0xc0000040), W_ERROR(0x8)},
-	{NT_STATUS(0xc0000041), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000042), W_ERROR(0x6)},
-	{NT_STATUS(0xc0000043), W_ERROR(0x20)},
-	{NT_STATUS(0xc0000044), W_ERROR(0x718)},
-	{NT_STATUS(0xc0000045), W_ERROR(0x57)},
-	{NT_STATUS(0xc0000046), W_ERROR(0x120)},
-	{NT_STATUS(0xc0000047), W_ERROR(0x12a)},
-	{NT_STATUS(0xc0000048), W_ERROR(0x57)},
-	{NT_STATUS(0xc0000049), W_ERROR(0x57)},
-	{NT_STATUS(0xc000004a), W_ERROR(0x9c)},
-	{NT_STATUS(0xc000004b), W_ERROR(0x5)},
-	{NT_STATUS(0xc000004c), W_ERROR(0x57)},
-	{NT_STATUS(0xc000004d), W_ERROR(0x57)},
-	{NT_STATUS(0xc000004e), W_ERROR(0x57)},
-	{NT_STATUS(0xc000004f), W_ERROR(0x11a)},
-	{NT_STATUS(0xc0000050), W_ERROR(0xff)},
-	{NT_STATUS(0xc0000051), W_ERROR(0x570)},
-	{NT_STATUS(0xc0000052), W_ERROR(0x570)},
-	{NT_STATUS(0xc0000053), W_ERROR(0x570)},
-	{NT_STATUS(0xc0000054), W_ERROR(0x21)},
-	{NT_STATUS(0xc0000055), W_ERROR(0x21)},
-	{NT_STATUS(0xc0000056), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000057), W_ERROR(0x32)},
-	{NT_STATUS(0xc0000058), W_ERROR(0x519)},
-	{NT_STATUS(0xc0000059), W_ERROR(0x51a)},
-	{NT_STATUS(0xc000005a), W_ERROR(0x51b)},
-	{NT_STATUS(0xc000005b), W_ERROR(0x51c)},
-	{NT_STATUS(0xc000005c), W_ERROR(0x51d)},
-	{NT_STATUS(0xc000005d), W_ERROR(0x51e)},
-	{NT_STATUS(0xc000005e), W_ERROR(0x51f)},
-	{NT_STATUS(0xc000005f), W_ERROR(0x520)},
-	{NT_STATUS(0xc0000060), W_ERROR(0x521)},
-	{NT_STATUS(0xc0000061), W_ERROR(0x522)},
-	{NT_STATUS(0xc0000062), W_ERROR(0x523)},
-	{NT_STATUS(0xc0000063), W_ERROR(0x524)},
-	{NT_STATUS(0xc0000064), W_ERROR(0x525)},
-	{NT_STATUS(0xc0000065), W_ERROR(0x526)},
-	{NT_STATUS(0xc0000066), W_ERROR(0x527)},
-	{NT_STATUS(0xc0000067), W_ERROR(0x528)},
-	{NT_STATUS(0xc0000068), W_ERROR(0x529)},
-	{NT_STATUS(0xc0000069), W_ERROR(0x52a)},
-	{NT_STATUS(0xc000006a), W_ERROR(0x56)},
-	{NT_STATUS(0xc000006b), W_ERROR(0x52c)},
-	{NT_STATUS(0xc000006c), W_ERROR(0x52d)},
-	{NT_STATUS(0xc000006d), W_ERROR(0x52e)},
-	{NT_STATUS(0xc000006e), W_ERROR(0x52f)},
-	{NT_STATUS(0xc000006f), W_ERROR(0x530)},
-	{NT_STATUS(0xc0000070), W_ERROR(0x531)},
-	{NT_STATUS(0xc0000071), W_ERROR(0x532)},
-	{NT_STATUS(0xc0000072), W_ERROR(0x533)},
-	{NT_STATUS(0xc0000073), W_ERROR(0x534)},
-	{NT_STATUS(0xc0000074), W_ERROR(0x535)},
-	{NT_STATUS(0xc0000075), W_ERROR(0x536)},
-	{NT_STATUS(0xc0000076), W_ERROR(0x537)},
-	{NT_STATUS(0xc0000077), W_ERROR(0x538)},
-	{NT_STATUS(0xc0000078), W_ERROR(0x539)},
-	{NT_STATUS(0xc0000079), W_ERROR(0x53a)},
-	{NT_STATUS(0xc000007a), W_ERROR(0x7f)},
-	{NT_STATUS(0xc000007b), W_ERROR(0xc1)},
-	{NT_STATUS(0xc000007c), W_ERROR(0x3f0)},
-	{NT_STATUS(0xc000007d), W_ERROR(0x53c)},
-	{NT_STATUS(0xc000007e), W_ERROR(0x9e)},
-	{NT_STATUS(0xc000007f), W_ERROR(0x70)},
-	{NT_STATUS(0xc0000080), W_ERROR(0x53d)},
-	{NT_STATUS(0xc0000081), W_ERROR(0x53e)},
-	{NT_STATUS(0xc0000082), W_ERROR(0x44)},
-	{NT_STATUS(0xc0000083), W_ERROR(0x103)},
-	{NT_STATUS(0xc0000084), W_ERROR(0x53f)},
-	{NT_STATUS(0xc0000085), W_ERROR(0x103)},
-	{NT_STATUS(0xc0000086), W_ERROR(0x9a)},
-	{NT_STATUS(0xc0000087), W_ERROR(0xe)},
-	{NT_STATUS(0xc0000088), W_ERROR(0x1e7)},
-	{NT_STATUS(0xc0000089), W_ERROR(0x714)},
-	{NT_STATUS(0xc000008a), W_ERROR(0x715)},
-	{NT_STATUS(0xc000008b), W_ERROR(0x716)},
-	{NT_STATUS(0xc000008c), W_ERROR(0xc000008c)},
-	{NT_STATUS(0xc000008d), W_ERROR(0xc000008d)},
-	{NT_STATUS(0xc000008e), W_ERROR(0xc000008e)},
-	{NT_STATUS(0xc000008f), W_ERROR(0xc000008f)},
-	{NT_STATUS(0xc0000090), W_ERROR(0xc0000090)},
-	{NT_STATUS(0xc0000091), W_ERROR(0xc0000091)},
-	{NT_STATUS(0xc0000092), W_ERROR(0xc0000092)},
-	{NT_STATUS(0xc0000093), W_ERROR(0xc0000093)},
-	{NT_STATUS(0xc0000094), W_ERROR(0xc0000094)},
-	{NT_STATUS(0xc0000095), W_ERROR(0x216)},
-	{NT_STATUS(0xc0000096), W_ERROR(0xc0000096)},
-	{NT_STATUS(0xc0000097), W_ERROR(0x8)},
-	{NT_STATUS(0xc0000098), W_ERROR(0x3ee)},
-	{NT_STATUS(0xc0000099), W_ERROR(0x540)},
-	{NT_STATUS(0xc000009a), W_ERROR(0x5aa)},
-	{NT_STATUS(0xc000009b), W_ERROR(0x3)},
-	{NT_STATUS(0xc000009c), W_ERROR(0x17)},
-	{NT_STATUS(0xc000009d), W_ERROR(0x48f)},
-	{NT_STATUS(0xc000009e), W_ERROR(0x15)},
-	{NT_STATUS(0xc000009f), W_ERROR(0x1e7)},
-	{NT_STATUS(0xc00000a0), W_ERROR(0x1e7)},
-	{NT_STATUS(0xc00000a1), W_ERROR(0x5ad)},
-	{NT_STATUS(0xc00000a2), W_ERROR(0x13)},
-	{NT_STATUS(0xc00000a3), W_ERROR(0x15)},
-	{NT_STATUS(0xc00000a4), W_ERROR(0x541)},
-	{NT_STATUS(0xc00000a5), W_ERROR(0x542)},
-	{NT_STATUS(0xc00000a6), W_ERROR(0x543)},
-	{NT_STATUS(0xc00000a7), W_ERROR(0x544)},
-	{NT_STATUS(0xc00000a8), W_ERROR(0x545)},
-	{NT_STATUS(0xc00000a9), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000ab), W_ERROR(0xe7)},
-	{NT_STATUS(0xc00000ac), W_ERROR(0xe7)},
-	{NT_STATUS(0xc00000ad), W_ERROR(0xe6)},
-	{NT_STATUS(0xc00000ae), W_ERROR(0xe7)},
-	{NT_STATUS(0xc00000af), W_ERROR(0x1)},
-	{NT_STATUS(0xc00000b0), W_ERROR(0xe9)},
-	{NT_STATUS(0xc00000b1), W_ERROR(0xe8)},
-	{NT_STATUS(0xc00000b2), W_ERROR(0x217)},
-	{NT_STATUS(0xc00000b3), W_ERROR(0x218)},
-	{NT_STATUS(0xc00000b4), W_ERROR(0xe6)},
-	{NT_STATUS(0xc00000b5), W_ERROR(0x79)},
-	{NT_STATUS(0xc00000b6), W_ERROR(0x26)},
-	{NT_STATUS(0xc00000ba), W_ERROR(0x5)},
-	{NT_STATUS(0xc00000bb), W_ERROR(0x32)},
-	{NT_STATUS(0xc00000bc), W_ERROR(0x33)},
-	{NT_STATUS(0xc00000bd), W_ERROR(0x34)},
-	{NT_STATUS(0xc00000be), W_ERROR(0x35)},
-	{NT_STATUS(0xc00000bf), W_ERROR(0x36)},
-	{NT_STATUS(0xc00000c0), W_ERROR(0x37)},
-	{NT_STATUS(0xc00000c1), W_ERROR(0x38)},
-	{NT_STATUS(0xc00000c2), W_ERROR(0x39)},
-	{NT_STATUS(0xc00000c3), W_ERROR(0x3a)},
-	{NT_STATUS(0xc00000c4), W_ERROR(0x3b)},
-	{NT_STATUS(0xc00000c5), W_ERROR(0x3c)},
-	{NT_STATUS(0xc00000c6), W_ERROR(0x3d)},
-	{NT_STATUS(0xc00000c7), W_ERROR(0x3e)},
-	{NT_STATUS(0xc00000c8), W_ERROR(0x3f)},
-	{NT_STATUS(0xc00000c9), W_ERROR(0x40)},
-	{NT_STATUS(0xc00000ca), W_ERROR(0x41)},
-	{NT_STATUS(0xc00000cb), W_ERROR(0x42)},
-	{NT_STATUS(0xc00000cc), W_ERROR(0x43)},
-	{NT_STATUS(0xc00000cd), W_ERROR(0x44)},
-	{NT_STATUS(0xc00000ce), W_ERROR(0x45)},
-	{NT_STATUS(0xc00000cf), W_ERROR(0x46)},
-	{NT_STATUS(0xc00000d0), W_ERROR(0x47)},
-	{NT_STATUS(0xc00000d1), W_ERROR(0x48)},
-	{NT_STATUS(0xc00000d2), W_ERROR(0x58)},
-	{NT_STATUS(0xc00000d4), W_ERROR(0x11)},
-	{NT_STATUS(0xc00000d5), W_ERROR(0x5)},
-	{NT_STATUS(0xc00000d6), W_ERROR(0xf0)},
-	{NT_STATUS(0xc00000d7), W_ERROR(0x546)},
-	{NT_STATUS(0xc00000d9), W_ERROR(0xe8)},
-	{NT_STATUS(0xc00000da), W_ERROR(0x547)},
-	{NT_STATUS(0xc00000dc), W_ERROR(0x548)},
-	{NT_STATUS(0xc00000dd), W_ERROR(0x549)},
-	{NT_STATUS(0xc00000de), W_ERROR(0x54a)},
-	{NT_STATUS(0xc00000df), W_ERROR(0x54b)},
-	{NT_STATUS(0xc00000e0), W_ERROR(0x54c)},
-	{NT_STATUS(0xc00000e1), W_ERROR(0x54d)},
-	{NT_STATUS(0xc00000e2), W_ERROR(0x12c)},
-	{NT_STATUS(0xc00000e3), W_ERROR(0x12d)},
-	{NT_STATUS(0xc00000e4), W_ERROR(0x54e)},
-	{NT_STATUS(0xc00000e5), W_ERROR(0x54f)},
-	{NT_STATUS(0xc00000e6), W_ERROR(0x550)},
-	{NT_STATUS(0xc00000e7), W_ERROR(0x551)},
-	{NT_STATUS(0xc00000e8), W_ERROR(0x6f8)},
-	{NT_STATUS(0xc00000ed), W_ERROR(0x552)},
-	{NT_STATUS(0xc00000ee), W_ERROR(0x553)},
-	{NT_STATUS(0xc00000ef), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f0), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f1), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f2), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f3), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f4), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f5), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f6), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f7), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f8), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000f9), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000fa), W_ERROR(0x57)},
-	{NT_STATUS(0xc00000fb), W_ERROR(0x3)},
-	{NT_STATUS(0xc00000fd), W_ERROR(0x3e9)},
-	{NT_STATUS(0xc00000fe), W_ERROR(0x554)},
-	{NT_STATUS(0xc0000100), W_ERROR(0xcb)},
-	{NT_STATUS(0xc0000101), W_ERROR(0x91)},
-	{NT_STATUS(0xc0000102), W_ERROR(0x570)},
-	{NT_STATUS(0xc0000103), W_ERROR(0x10b)},
-	{NT_STATUS(0xc0000104), W_ERROR(0x555)},
-	{NT_STATUS(0xc0000105), W_ERROR(0x556)},
-	{NT_STATUS(0xc0000106), W_ERROR(0xce)},
-	{NT_STATUS(0xc0000107), W_ERROR(0x961)},
-	{NT_STATUS(0xc0000108), W_ERROR(0x964)},
-	{NT_STATUS(0xc000010a), W_ERROR(0x5)},
-	{NT_STATUS(0xc000010b), W_ERROR(0x557)},
-	{NT_STATUS(0xc000010d), W_ERROR(0x558)},
-	{NT_STATUS(0xc000010e), W_ERROR(0x420)},
-	{NT_STATUS(0xc0000117), W_ERROR(0x5a4)},
-	{NT_STATUS(0xc000011b), W_ERROR(0xc1)},
-	{NT_STATUS(0xc000011c), W_ERROR(0x559)},
-	{NT_STATUS(0xc000011d), W_ERROR(0x55a)},
-	{NT_STATUS(0xc000011e), W_ERROR(0x3ee)},
-	{NT_STATUS(0xc000011f), W_ERROR(0x4)},
-	{NT_STATUS(0xc0000120), W_ERROR(0x3e3)},
-	{NT_STATUS(0xc0000121), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000122), W_ERROR(0x4ba)},
-	{NT_STATUS(0xc0000123), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000124), W_ERROR(0x55b)},
-	{NT_STATUS(0xc0000125), W_ERROR(0x55c)},
-	{NT_STATUS(0xc0000126), W_ERROR(0x55d)},
-	{NT_STATUS(0xc0000127), W_ERROR(0x55e)},
-	{NT_STATUS(0xc0000128), W_ERROR(0x6)},
-	{NT_STATUS(0xc000012b), W_ERROR(0x55f)},
-	{NT_STATUS(0xc000012d), W_ERROR(0x5af)},
-	{NT_STATUS(0xc000012e), W_ERROR(0xc1)},
-	{NT_STATUS(0xc000012f), W_ERROR(0xc1)},
-	{NT_STATUS(0xc0000130), W_ERROR(0xc1)},
-	{NT_STATUS(0xc0000131), W_ERROR(0xc1)},
-	{NT_STATUS(0xc0000133), W_ERROR(0x576)},
-	{NT_STATUS(0xc0000135), W_ERROR(0x7e)},
-	{NT_STATUS(0xc0000138), W_ERROR(0xb6)},
-	{NT_STATUS(0xc0000139), W_ERROR(0x7f)},
-	{NT_STATUS(0xc000013b), W_ERROR(0x40)},
-	{NT_STATUS(0xc000013c), W_ERROR(0x40)},
-	{NT_STATUS(0xc000013d), W_ERROR(0x33)},
-	{NT_STATUS(0xc000013e), W_ERROR(0x3b)},
-	{NT_STATUS(0xc000013f), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000140), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000141), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000142), W_ERROR(0x45a)},
-	{NT_STATUS(0xc0000148), W_ERROR(0x7c)},
-	{NT_STATUS(0xc0000149), W_ERROR(0x56)},
-	{NT_STATUS(0xc000014b), W_ERROR(0x6d)},
-	{NT_STATUS(0xc000014c), W_ERROR(0x3f1)},
-	{NT_STATUS(0xc000014d), W_ERROR(0x3f8)},
-	{NT_STATUS(0xc000014f), W_ERROR(0x3ed)},
-	{NT_STATUS(0xc0000150), W_ERROR(0x45e)},
-	{NT_STATUS(0xc0000151), W_ERROR(0x560)},
-	{NT_STATUS(0xc0000152), W_ERROR(0x561)},
-	{NT_STATUS(0xc0000153), W_ERROR(0x562)},
-	{NT_STATUS(0xc0000154), W_ERROR(0x563)},
-	{NT_STATUS(0xc0000155), W_ERROR(0x564)},
-	{NT_STATUS(0xc0000156), W_ERROR(0x565)},
-	{NT_STATUS(0xc0000157), W_ERROR(0x566)},
-	{NT_STATUS(0xc0000158), W_ERROR(0x567)},
-	{NT_STATUS(0xc0000159), W_ERROR(0x3ef)},
-	{NT_STATUS(0xc000015a), W_ERROR(0x568)},
-	{NT_STATUS(0xc000015b), W_ERROR(0x569)},
-	{NT_STATUS(0xc000015c), W_ERROR(0x3f9)},
-	{NT_STATUS(0xc000015d), W_ERROR(0x56a)},
-	{NT_STATUS(0xc000015f), W_ERROR(0x45d)},
-	{NT_STATUS(0xc0000162), W_ERROR(0x459)},
-	{NT_STATUS(0xc0000165), W_ERROR(0x462)},
-	{NT_STATUS(0xc0000166), W_ERROR(0x463)},
-	{NT_STATUS(0xc0000167), W_ERROR(0x464)},
-	{NT_STATUS(0xc0000168), W_ERROR(0x465)},
-	{NT_STATUS(0xc0000169), W_ERROR(0x466)},
-	{NT_STATUS(0xc000016a), W_ERROR(0x467)},
-	{NT_STATUS(0xc000016b), W_ERROR(0x468)},
-	{NT_STATUS(0xc000016c), W_ERROR(0x45f)},
-	{NT_STATUS(0xc000016d), W_ERROR(0x45d)},
-	{NT_STATUS(0xc0000172), W_ERROR(0x451)},
-	{NT_STATUS(0xc0000173), W_ERROR(0x452)},
-	{NT_STATUS(0xc0000174), W_ERROR(0x453)},
-	{NT_STATUS(0xc0000175), W_ERROR(0x454)},
-	{NT_STATUS(0xc0000176), W_ERROR(0x455)},
-	{NT_STATUS(0xc0000177), W_ERROR(0x469)},
-	{NT_STATUS(0xc0000178), W_ERROR(0x458)},
-	{NT_STATUS(0xc000017a), W_ERROR(0x56b)},
-	{NT_STATUS(0xc000017b), W_ERROR(0x56c)},
-	{NT_STATUS(0xc000017c), W_ERROR(0x3fa)},
-	{NT_STATUS(0xc000017d), W_ERROR(0x3fb)},
-	{NT_STATUS(0xc000017e), W_ERROR(0x56d)},
-	{NT_STATUS(0xc000017f), W_ERROR(0x56e)},
-	{NT_STATUS(0xc0000180), W_ERROR(0x3fc)},
-	{NT_STATUS(0xc0000181), W_ERROR(0x3fd)},
-	{NT_STATUS(0xc0000182), W_ERROR(0x57)},
-	{NT_STATUS(0xc0000183), W_ERROR(0x45d)},
-	{NT_STATUS(0xc0000184), W_ERROR(0x16)},
-	{NT_STATUS(0xc0000185), W_ERROR(0x45d)},
-	{NT_STATUS(0xc0000186), W_ERROR(0x45d)},
-	{NT_STATUS(0xc0000188), W_ERROR(0x5de)},
-	{NT_STATUS(0xc0000189), W_ERROR(0x13)},
-	{NT_STATUS(0xc000018a), W_ERROR(0x6fa)},
-	{NT_STATUS(0xc000018b), W_ERROR(0x6fb)},
-	{NT_STATUS(0xc000018c), W_ERROR(0x6fc)},
-	{NT_STATUS(0xc000018d), W_ERROR(0x6fd)},
-	{NT_STATUS(0xc000018e), W_ERROR(0x5dc)},
-	{NT_STATUS(0xc000018f), W_ERROR(0x5dd)},
-	{NT_STATUS(0xc0000190), W_ERROR(0x6fe)},
-	{NT_STATUS(0xc0000192), W_ERROR(0x700)},
-	{NT_STATUS(0xc0000193), W_ERROR(0x701)},
-	{NT_STATUS(0xc0000194), W_ERROR(0x46b)},
-	{NT_STATUS(0xc0000195), W_ERROR(0x4c3)},
-	{NT_STATUS(0xc0000196), W_ERROR(0x4c4)},
-	{NT_STATUS(0xc0000197), W_ERROR(0x5df)},
-	{NT_STATUS(0xc0000198), W_ERROR(0x70f)},
-	{NT_STATUS(0xc0000199), W_ERROR(0x710)},
-	{NT_STATUS(0xc000019a), W_ERROR(0x711)},
-	{NT_STATUS(0xc000019b), W_ERROR(0x712)},
-	{NT_STATUS(0xc0000202), W_ERROR(0x572)},
-	{NT_STATUS(0xc0000203), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000204), W_ERROR(0x717)},
-	{NT_STATUS(0xc0000205), W_ERROR(0x46a)},
-	{NT_STATUS(0xc0000206), W_ERROR(0x6f8)},
-	{NT_STATUS(0xc0000207), W_ERROR(0x4be)},
-	{NT_STATUS(0xc0000208), W_ERROR(0x4be)},
-	{NT_STATUS(0xc0000209), W_ERROR(0x44)},
-	{NT_STATUS(0xc000020a), W_ERROR(0x34)},
-	{NT_STATUS(0xc000020b), W_ERROR(0x40)},
-	{NT_STATUS(0xc000020c), W_ERROR(0x40)},
-	{NT_STATUS(0xc000020d), W_ERROR(0x40)},
-	{NT_STATUS(0xc000020e), W_ERROR(0x44)},
-	{NT_STATUS(0xc000020f), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000210), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000211), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000212), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000213), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000214), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000215), W_ERROR(0x3b)},
-	{NT_STATUS(0xc0000216), W_ERROR(0x32)},
-	{NT_STATUS(0xc0000217), W_ERROR(0x32)},
-	{NT_STATUS(0xc000021c), W_ERROR(0x17e6)},
-	{NT_STATUS(0xc0000220), W_ERROR(0x46c)},
-	{NT_STATUS(0xc0000221), W_ERROR(0xc1)},
-	{NT_STATUS(0xc0000224), W_ERROR(0x773)},
-	{NT_STATUS(0xc0000225), W_ERROR(0x490)},
-	{NT_STATUS(0xc000022a), W_ERROR(0xc000022a)},
-	{NT_STATUS(0xc000022b), W_ERROR(0xc000022b)},
-	{NT_STATUS(0xc000022d), W_ERROR(0x4d5)},
-	{NT_STATUS(0xc0000230), W_ERROR(0x492)},
-	{NT_STATUS(0xc0000233), W_ERROR(0x774)},
-	{NT_STATUS(0xc0000234), W_ERROR(0x775)},
-	{NT_STATUS(0xc0000235), W_ERROR(0x6)},
-	{NT_STATUS(0xc0000236), W_ERROR(0x4c9)},
-	{NT_STATUS(0xc0000237), W_ERROR(0x4ca)},
-	{NT_STATUS(0xc0000238), W_ERROR(0x4cb)},
-	{NT_STATUS(0xc0000239), W_ERROR(0x4cc)},
-	{NT_STATUS(0xc000023a), W_ERROR(0x4cd)},
-	{NT_STATUS(0xc000023b), W_ERROR(0x4ce)},
-	{NT_STATUS(0xc000023c), W_ERROR(0x4cf)},
-	{NT_STATUS(0xc000023d), W_ERROR(0x4d0)},
-	{NT_STATUS(0xc000023e), W_ERROR(0x4d1)},
-	{NT_STATUS(0xc000023f), W_ERROR(0x4d2)},
-	{NT_STATUS(0xc0000240), W_ERROR(0x4d3)},
-	{NT_STATUS(0xc0000241), W_ERROR(0x4d4)},
-	{NT_STATUS(0xc0000243), W_ERROR(0x4c8)},
-	{NT_STATUS(0xc0000246), W_ERROR(0x4d6)},
-	{NT_STATUS(0xc0000247), W_ERROR(0x4d7)},
-	{NT_STATUS(0xc0000248), W_ERROR(0x4d8)},
-	{NT_STATUS(0xc0000249), W_ERROR(0xc1)},
-	{NT_STATUS(0xc0000253), W_ERROR(0x54f)},
-	{NT_STATUS(0xc0000257), W_ERROR(0x4d0)},
-	{NT_STATUS(0xc0000259), W_ERROR(0x573)},
-	{NT_STATUS(0xc000025e), W_ERROR(0x422)},
-	{NT_STATUS(0xc0000262), W_ERROR(0xb6)},
-	{NT_STATUS(0xc0000263), W_ERROR(0x7f)},
-	{NT_STATUS(0xc0000264), W_ERROR(0x120)},
-	{NT_STATUS(0xc0000265), W_ERROR(0x476)},
-	{NT_STATUS(0xc0000267), W_ERROR(0x10fe)},
-	{NT_STATUS(0xc000026c), W_ERROR(0x7d1)},
-	{NT_STATUS(0xc000026d), W_ERROR(0x4b1)},
-	{NT_STATUS(0xc000026e), W_ERROR(0x15)},
-	{NT_STATUS(0xc0000272), W_ERROR(0x491)},
-	{NT_STATUS(0xc0000275), W_ERROR(0x1126)},
-	{NT_STATUS(0xc0000276), W_ERROR(0x1129)},
-	{NT_STATUS(0xc0000277), W_ERROR(0x112a)},
-	{NT_STATUS(0xc0000278), W_ERROR(0x1128)},
-	{NT_STATUS(0xc0000279), W_ERROR(0x780)},
-	{NT_STATUS(0xc0000280), W_ERROR(0x781)},
-	{NT_STATUS(0xc0000281), W_ERROR(0xa1)},
-	{NT_STATUS(0xc0000283), W_ERROR(0x488)},
-	{NT_STATUS(0xc0000284), W_ERROR(0x489)},
-	{NT_STATUS(0xc0000285), W_ERROR(0x48a)},
-	{NT_STATUS(0xc0000286), W_ERROR(0x48b)},
-	{NT_STATUS(0xc0000287), W_ERROR(0x48c)},
-	{NT_STATUS(0xc000028a), W_ERROR(0x5)},
-	{NT_STATUS(0xc000028b), W_ERROR(0x5)},
-	{NT_STATUS(0xc000028d), W_ERROR(0x5)},
-	{NT_STATUS(0xc000028e), W_ERROR(0x5)},
-	{NT_STATUS(0xc000028f), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000290), W_ERROR(0x5)},
-	{NT_STATUS(0xc0000291), W_ERROR(0x1777)},
-	{NT_STATUS(0xc0000292), W_ERROR(0x1778)},
-	{NT_STATUS(0xc0000293), W_ERROR(0x1772)},
-	{NT_STATUS(0xc0000295), W_ERROR(0x1068)},
-	{NT_STATUS(0xc0000296), W_ERROR(0x1069)},
-	{NT_STATUS(0xc0000297), W_ERROR(0x106a)},
-	{NT_STATUS(0xc0000298), W_ERROR(0x106b)},
-	{NT_STATUS(0xc0000299), W_ERROR(0x201a)},
-	{NT_STATUS(0xc000029a), W_ERROR(0x201b)},
-	{NT_STATUS(0xc000029b), W_ERROR(0x201c)},
-	{NT_STATUS(0xc000029c), W_ERROR(0x1)},
-	{NT_STATUS(0xc000029d), W_ERROR(0x10ff)},
-	{NT_STATUS(0xc000029e), W_ERROR(0x1100)},
-	{NT_STATUS(0xc000029f), W_ERROR(0x494)},
-	{NT_STATUS(0xc00002a1), W_ERROR(0x200a)},
-	{NT_STATUS(0xc00002a2), W_ERROR(0x200b)},
-	{NT_STATUS(0xc00002a3), W_ERROR(0x200c)},
-	{NT_STATUS(0xc00002a4), W_ERROR(0x200d)},
-	{NT_STATUS(0xc00002a5), W_ERROR(0x200e)},
-	{NT_STATUS(0xc00002a6), W_ERROR(0x200f)},
-	{NT_STATUS(0xc00002a7), W_ERROR(0x2010)},
-	{NT_STATUS(0xc00002a8), W_ERROR(0x2011)},
-	{NT_STATUS(0xc00002a9), W_ERROR(0x2012)},
-	{NT_STATUS(0xc00002aa), W_ERROR(0x2013)},
-	{NT_STATUS(0xc00002ab), W_ERROR(0x2014)},
-	{NT_STATUS(0xc00002ac), W_ERROR(0x2015)},
-	{NT_STATUS(0xc00002ad), W_ERROR(0x2016)},
-	{NT_STATUS(0xc00002ae), W_ERROR(0x2017)},
-	{NT_STATUS(0xc00002af), W_ERROR(0x2018)},
-	{NT_STATUS(0xc00002b0), W_ERROR(0x2019)},
-	{NT_STATUS(0xc00002b1), W_ERROR(0x211e)},
-	{NT_STATUS(0xc00002b2), W_ERROR(0x1127)},
-	{NT_STATUS(0xc00002b6), W_ERROR(0x651)},
-	{NT_STATUS(0xc00002b7), W_ERROR(0x49a)},
-	{NT_STATUS(0xc00002b8), W_ERROR(0x49b)},
-	{NT_STATUS(0xc00002c1), W_ERROR(0x2024)},
-	{NT_STATUS(0xc00002c3), W_ERROR(0x575)},
-	{NT_STATUS(0xc00002c5), W_ERROR(0x3e6)},
-	{NT_STATUS(0xc00002c6), W_ERROR(0x1075)},
-	{NT_STATUS(0xc00002c7), W_ERROR(0x1076)},
-	{NT_STATUS(0xc00002ca), W_ERROR(0x10e8)},
-	{NT_STATUS(0xc00002cb), W_ERROR(0x2138)},
-	{NT_STATUS(0xc00002cc), W_ERROR(0x4e3)},
-	{NT_STATUS(0xc00002cd), W_ERROR(0x2139)},
-	{NT_STATUS(0xc00002cf), W_ERROR(0x49d)},
-	{NT_STATUS(0xc00002d0), W_ERROR(0x213a)},
-	{NT_STATUS(0xc00002d4), W_ERROR(0x2141)},
-	{NT_STATUS(0xc00002d5), W_ERROR(0x2142)},
-	{NT_STATUS(0xc00002d6), W_ERROR(0x2143)},
-	{NT_STATUS(0xc00002d7), W_ERROR(0x2144)},
-	{NT_STATUS(0xc00002d8), W_ERROR(0x2145)},
-	{NT_STATUS(0xc00002d9), W_ERROR(0x2146)},
-	{NT_STATUS(0xc00002da), W_ERROR(0x2147)},
-	{NT_STATUS(0xc00002db), W_ERROR(0x2148)},
-	{NT_STATUS(0xc00002dc), W_ERROR(0x2149)},
-	{NT_STATUS(0xc00002dd), W_ERROR(0x32)},
-	{NT_STATUS(0xc00002df), W_ERROR(0x2151)},
-	{NT_STATUS(0xc00002e0), W_ERROR(0x2152)},
-	{NT_STATUS(0xc00002e1), W_ERROR(0x2153)},
-	{NT_STATUS(0xc00002e2), W_ERROR(0x2154)},
-	{NT_STATUS(0xc00002e3), W_ERROR(0x215d)},
-	{NT_STATUS(0xc00002e4), W_ERROR(0x2163)},
-	{NT_STATUS(0xc00002e5), W_ERROR(0x2164)},
-	{NT_STATUS(0xc00002e6), W_ERROR(0x2165)},
-	{NT_STATUS(0xc00002e7), W_ERROR(0x216d)},
-	{NT_STATUS(0xc00002fe), W_ERROR(0x45b)},
-	{NT_STATUS(0xc00002ff), W_ERROR(0x4e7)},
-	{NT_STATUS(0xc0000300), W_ERROR(0x4e6)},
-	{NT_STATUS(0x80000001), W_ERROR(0x80000001)},
-	{NT_STATUS(0x80000002), W_ERROR(0x3e6)},
-	{NT_STATUS(0x80000003), W_ERROR(0x80000003)},
-	{NT_STATUS(0x80000004), W_ERROR(0x80000004)},
-	{NT_STATUS(0x80000005), W_ERROR(0xea)},
-	{NT_STATUS(0x80000006), W_ERROR(0x12)},
-	{NT_STATUS(0x8000000b), W_ERROR(0x56f)},
-	{NT_STATUS(0x8000000d), W_ERROR(0x12b)},
-	{NT_STATUS(0x8000000e), W_ERROR(0x1c)},
-	{NT_STATUS(0x8000000f), W_ERROR(0x15)},
-	{NT_STATUS(0x80000010), W_ERROR(0x15)},
-	{NT_STATUS(0x80000011), W_ERROR(0xaa)},
-	{NT_STATUS(0x80000012), W_ERROR(0x103)},
-	{NT_STATUS(0x80000013), W_ERROR(0xfe)},
-	{NT_STATUS(0x80000014), W_ERROR(0xff)},
-	{NT_STATUS(0x80000015), W_ERROR(0xff)},
-	{NT_STATUS(0x80000016), W_ERROR(0x456)},
-	{NT_STATUS(0x8000001a), W_ERROR(0x103)},
-	{NT_STATUS(0x8000001b), W_ERROR(0x44d)},
-	{NT_STATUS(0x8000001c), W_ERROR(0x456)},
-	{NT_STATUS(0x8000001d), W_ERROR(0x457)},
-	{NT_STATUS(0x8000001e), W_ERROR(0x44c)},
-	{NT_STATUS(0x8000001f), W_ERROR(0x44e)},
-	{NT_STATUS(0x80000021), W_ERROR(0x44f)},
-	{NT_STATUS(0x80000022), W_ERROR(0x450)},
-	{NT_STATUS(0x80000025), W_ERROR(0x962)},
-	{NT_STATUS(0x80000288), W_ERROR(0x48d)},
-	{NT_STATUS(0x80000289), W_ERROR(0x48e)},
-	{NT_STATUS_OK, WERR_OK}};
-
-static const struct {
-	WERROR werror;
-	NTSTATUS ntstatus;
-} werror_to_ntstatus_map[] = {
-	{ W_ERROR(0x5), NT_STATUS_ACCESS_DENIED },
-	{ WERR_OK, NT_STATUS_OK }
-};
-
 /*****************************************************************************
 convert a dos eclas/ecode to a NT status32 code
  *****************************************************************************/
@@ -1444,76 +301,6 @@ NTSTATUS dos_to_ntstatus(uint8 eclass, uint32 ecode)
 	return NT_STATUS_UNSUCCESSFUL;
 }
 
-
-/*****************************************************************************
-convert a NT status code to a dos class/code
- *****************************************************************************/
-void ntstatus_to_dos(NTSTATUS ntstatus, uint8 *eclass, uint32 *ecode)
-{
-	int i;
-	if (NT_STATUS_IS_OK(ntstatus)) {
-		*eclass = 0;
-		*ecode = 0;
-		return;
-	}
-	for (i=0; NT_STATUS_V(ntstatus_to_dos_map[i].ntstatus); i++) {
-		if (NT_STATUS_V(ntstatus) == 
-		    NT_STATUS_V(ntstatus_to_dos_map[i].ntstatus)) {
-			*eclass = ntstatus_to_dos_map[i].dos_class;
-			*ecode = ntstatus_to_dos_map[i].dos_code;
-			return;
-		}
-	}
-	*eclass = ERRHRD;
-	*ecode = ERRgeneral;
-}
-
-
-/*****************************************************************************
-convert a WERROR to a NT status32 code
- *****************************************************************************/
-NTSTATUS werror_to_ntstatus(WERROR error)
-{
-	int i;
-	if (W_ERROR_IS_OK(error)) return NT_STATUS_OK;
-
-	for (i=0; !W_ERROR_IS_OK(werror_to_ntstatus_map[i].werror); i++) {
-		if (W_ERROR_V(error) == 
-		    W_ERROR_V(werror_to_ntstatus_map[i].werror)) {
-			return werror_to_ntstatus_map[i].ntstatus;
-		}
-	}
-
-	for (i=0; NT_STATUS_V(ntstatus_to_werror_map[i].ntstatus); i++) {
-		if (W_ERROR_V(error) == 
-		    W_ERROR_V(ntstatus_to_werror_map[i].werror)) {
-			return ntstatus_to_werror_map[i].ntstatus;
-		}
-	}
-
-	/* just guess ... */
-	return NT_STATUS(W_ERROR_V(error) | 0xc0000000);
-}
-
-/*****************************************************************************
-convert a NTSTATUS to a WERROR
- *****************************************************************************/
-WERROR ntstatus_to_werror(NTSTATUS error)
-{
-	int i;
-	if (NT_STATUS_IS_OK(error)) return WERR_OK;
-	for (i=0; NT_STATUS_V(ntstatus_to_werror_map[i].ntstatus); i++) {
-		if (NT_STATUS_V(error) == 
-		    NT_STATUS_V(ntstatus_to_werror_map[i].ntstatus)) {
-			return ntstatus_to_werror_map[i].werror;
-		}
-	}
-
-	/* a lame guess */
-	return W_ERROR(NT_STATUS_V(error) & 0xffff);
-}
-
-
 #if defined(HAVE_GSSAPI)
 /*******************************************************************************
  Map between gssapi errors and NT status. I made these up :-(. JRA.
diff --git a/source3/libsmb/errormap_wbc.h b/source3/libsmb/errormap_wbc.h
index 2042da4efca..6323deec1dd 100644
--- a/source3/libsmb/errormap_wbc.h
+++ b/source3/libsmb/errormap_wbc.h
@@ -1,3 +1,29 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  error mapping functions
+ *  Copyright (C) Andrew Tridgell 2001
+ *  Copyright (C) Andrew Bartlett 2001
+ *  Copyright (C) Tim Potter 2000
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LIBSMB_ERRORMAP_WBC_H_
+#define _LIBSMB_ERRORMAP_WBC_H_
+
 /* The following definitions come from libsmb/errormap_wbc.c  */
 
 NTSTATUS map_nt_error_from_wbcErr(wbcErr wbc_err);
+
+#endif /* _LIBSMB_ERRORMAP_WBC_H_ */
diff --git a/source3/libsmb/libsmb.h b/source3/libsmb/libsmb.h
new file mode 100644
index 00000000000..061f317e1e5
--- /dev/null
+++ b/source3/libsmb/libsmb.h
@@ -0,0 +1,30 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   Copyright (C) Andrew Tridgell 1992-1998,2001
+   Copyright (C) Jeremy Allison 1998
+   Copyright (C) Remus Koos 2001
+   Copyright (C) Andrew Bartlett 2001
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBSMB_LIBSMB_H
+#define _LIBSMB_LIBSMB_H
+
+#include "client.h"
+#include "libads/ads_status.h"
+#include "libsmb/proto.h"
+
+#endif /* _LIBSMB_LIBSMB_H */
diff --git a/source3/libsmb/libsmb_cache.c b/source3/libsmb/libsmb_cache.c
index f9770d363ca..598fa81376f 100644
--- a/source3/libsmb/libsmb_cache.c
+++ b/source3/libsmb/libsmb_cache.c
@@ -21,6 +21,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 
diff --git a/source3/libsmb/libsmb_context.c b/source3/libsmb/libsmb_context.c
index 9c8429aab47..dbf70e02c8d 100644
--- a/source3/libsmb/libsmb_context.c
+++ b/source3/libsmb/libsmb_context.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 #include "secrets.h"
@@ -456,11 +457,11 @@ smbc_option_get(SMBCCTX *context,
                 switch(smbc_getOptionSmbEncryptionLevel(context))
                 {
                 case 0:
-                        return (void *) "none";
+                        return discard_const_p(void, "none");
                 case 1:
-                        return (void *) "request";
+                        return discard_const_p(void, "request");
                 case 2:
-                        return (void *) "require";
+                        return discard_const_p(void, "require");
                 }
 
         } else if (strcmp(option_name, "smb_encrypt_on") == 0) {
@@ -598,8 +599,8 @@ smbc_init_context(SMBCCTX *context)
                  * our hostname etc
                  */
                 char *netbios_name;
-                if (global_myname()) {
-                        netbios_name = SMB_STRDUP(global_myname());
+                if (lp_netbios_name()) {
+                        netbios_name = SMB_STRDUP(lp_netbios_name());
                 } else {
                         /*
                          * Hmmm, I want to get hostname as well, but I am too
@@ -749,6 +750,7 @@ void smbc_set_credentials_with_fallback(SMBCCTX *context,
 	}
 
         set_cmdline_auth_info_username(auth_info, user);
+        set_cmdline_auth_info_domain(auth_info, workgroup);
         set_cmdline_auth_info_password(auth_info, password);
         set_cmdline_auth_info_use_kerberos(auth_info, use_kerberos);
         set_cmdline_auth_info_signing_state(auth_info, signing_state);
@@ -756,7 +758,6 @@ void smbc_set_credentials_with_fallback(SMBCCTX *context,
 		smbc_getOptionFallbackAfterKerberos(context));
 	set_cmdline_auth_info_use_ccache(
 		auth_info, smbc_getOptionUseCCache(context));
-        set_global_myworkgroup(workgroup);
 
 	TALLOC_FREE(context->internal->auth_info);
 
diff --git a/source3/libsmb/libsmb_dir.c b/source3/libsmb/libsmb_dir.c
index c0c6e83bb63..ea75dbf2649 100644
--- a/source3/libsmb/libsmb_dir.c
+++ b/source3/libsmb/libsmb_dir.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "popt_common.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
@@ -245,7 +246,7 @@ dir_list_fn(const char *mnt,
 {
 
 	if (add_dirent((SMBCFILE *)state, finfo->name, "",
-		       (finfo->mode&aDIR?SMBC_DIR:SMBC_FILE)) < 0) {
+		       (finfo->mode&FILE_ATTRIBUTE_DIRECTORY?SMBC_DIR:SMBC_FILE)) < 0) {
 		SMBCFILE *dir = (SMBCFILE *)state;
 		return map_nt_error_from_unix(dir->dir_error);
 	}
@@ -456,9 +457,10 @@ SMBC_opendir_ctx(SMBCCTX *context,
                 int i;
                 int count;
                 int max_lmb_count;
-                struct ip_service *ip_list;
-                struct ip_service server_addr;
+                struct sockaddr_storage *ip_list;
+                struct sockaddr_storage server_addr;
                 struct user_auth_info u_info;
+		NTSTATUS status;
 
 		if (share[0] != (char)0 || path[0] != (char)0) {
 
@@ -498,13 +500,14 @@ SMBC_opendir_ctx(SMBCCTX *context,
                  */
 
                 ip_list = NULL;
-                if (!NT_STATUS_IS_OK(name_resolve_bcast(MSBROWSE, 1, &ip_list,
-                                                        &count)))
+		status = name_resolve_bcast(MSBROWSE, 1, talloc_tos(),
+					    &ip_list, &count);
+                if (!NT_STATUS_IS_OK(status))
 		{
 
-                        SAFE_FREE(ip_list);
+                        TALLOC_FREE(ip_list);
 
-                        if (!find_master_ip(workgroup, &server_addr.ss)) {
+                        if (!find_master_ip(workgroup, &server_addr)) {
 
 				if (dir) {
 					SAFE_FREE(dir->fname);
@@ -515,8 +518,9 @@ SMBC_opendir_ctx(SMBCCTX *context,
                                 return NULL;
                         }
 
-			ip_list = (struct ip_service *)memdup(
-				&server_addr, sizeof(server_addr));
+			ip_list = (struct sockaddr_storage *)talloc_memdup(
+				talloc_tos(), &server_addr,
+				sizeof(server_addr));
 			if (ip_list == NULL) {
 				if (dir) {
 					SAFE_FREE(dir->fname);
@@ -534,7 +538,7 @@ SMBC_opendir_ctx(SMBCCTX *context,
 			char *wg_ptr = NULL;
                 	struct cli_state *cli = NULL;
 
-			print_sockaddr(addr, sizeof(addr), &ip_list[i].ss);
+			print_sockaddr(addr, sizeof(addr), &ip_list[i]);
                         DEBUG(99, ("Found master browser %d of %d: %s\n",
                                    i+1, MAX(count, max_lmb_count),
                                    addr));
@@ -594,7 +598,7 @@ SMBC_opendir_ctx(SMBCCTX *context,
                         }
                 }
 
-                SAFE_FREE(ip_list);
+                TALLOC_FREE(ip_list);
         } else {
                 /*
                  * Server not an empty string ... Check the rest and see what
@@ -816,7 +820,7 @@ SMBC_opendir_ctx(SMBCCTX *context,
 			}
 
 			status = cli_list(targetcli, targetpath,
-					  aDIR | aSYSTEM | aHIDDEN,
+					  FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN,
 					  dir_list_fn, (void *)dir);
 			if (!NT_STATUS_IS_OK(status)) {
 				if (dir) {
@@ -1345,7 +1349,7 @@ SMBC_rmdir_ctx(SMBCCTX *context,
 			}
 
 			status = cli_list(targetcli, lpath,
-					  aDIR | aSYSTEM | aHIDDEN,
+					  FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN,
 					  rmdir_list_fn,
 					  &smbc_rmdir_dirempty);
 
@@ -1611,10 +1615,10 @@ SMBC_chmod_ctx(SMBCCTX *context,
 
 	mode = 0;
 
-	if (!(newmode & (S_IWUSR | S_IWGRP | S_IWOTH))) mode |= aRONLY;
-	if ((newmode & S_IXUSR) && lp_map_archive(-1)) mode |= aARCH;
-	if ((newmode & S_IXGRP) && lp_map_system(-1)) mode |= aSYSTEM;
-	if ((newmode & S_IXOTH) && lp_map_hidden(-1)) mode |= aHIDDEN;
+	if (!(newmode & (S_IWUSR | S_IWGRP | S_IWOTH))) mode |= FILE_ATTRIBUTE_READONLY;
+	if ((newmode & S_IXUSR) && lp_map_archive(-1)) mode |= FILE_ATTRIBUTE_ARCHIVE;
+	if ((newmode & S_IXGRP) && lp_map_system(-1)) mode |= FILE_ATTRIBUTE_SYSTEM;
+	if ((newmode & S_IXOTH) && lp_map_hidden(-1)) mode |= FILE_ATTRIBUTE_HIDDEN;
 
 	if (!NT_STATUS_IS_OK(cli_setatr(targetcli, targetpath, mode, 0))) {
 		errno = SMBC_errno(context, targetcli);
@@ -1803,7 +1807,7 @@ SMBC_unlink_ctx(SMBCCTX *context,
 	}
 	/*d_printf(">>>unlink: resolved path as %s\n", targetpath);*/
 
-	if (!NT_STATUS_IS_OK(cli_unlink(targetcli, targetpath, aSYSTEM | aHIDDEN))) {
+	if (!NT_STATUS_IS_OK(cli_unlink(targetcli, targetpath, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN))) {
 
 		errno = SMBC_errno(context, targetcli);
 
@@ -2009,7 +2013,7 @@ SMBC_rename_ctx(SMBCCTX *ocontext,
 		int eno = SMBC_errno(ocontext, targetcli1);
 
 		if (eno != EEXIST ||
-		    !NT_STATUS_IS_OK(cli_unlink(targetcli1, targetpath2, aSYSTEM | aHIDDEN)) ||
+		    !NT_STATUS_IS_OK(cli_unlink(targetcli1, targetpath2, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN)) ||
 		    !NT_STATUS_IS_OK(cli_rename(targetcli1, targetpath1, targetpath2))) {
 
 			errno = eno;
diff --git a/source3/libsmb/libsmb_file.c b/source3/libsmb/libsmb_file.c
index 58403dfbb36..c822eab7985 100644
--- a/source3/libsmb/libsmb_file.c
+++ b/source3/libsmb/libsmb_file.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 
@@ -318,13 +319,13 @@ SMBC_write_ctx(SMBCCTX *context,
                const void *buf,
                size_t count)
 {
-	int ret;
         off_t offset;
 	char *server = NULL, *share = NULL, *user = NULL, *password = NULL;
 	char *path = NULL;
 	char *targetpath = NULL;
 	struct cli_state *targetcli = NULL;
 	TALLOC_CTX *frame = talloc_stackframe();
+	NTSTATUS status;
 
 	/* First check all pointers before dereferencing them */
 
@@ -377,18 +378,18 @@ SMBC_write_ctx(SMBCCTX *context,
 	}
 	/*d_printf(">>>write: resolved path as %s\n", targetpath);*/
 
-	ret = cli_write(targetcli, file->cli_fd,
-                        0, (char *)buf, offset, count);
-	if (ret <= 0) {
-		errno = SMBC_errno(context, targetcli);
+	status = cli_writeall(targetcli, file->cli_fd,
+			      0, (const uint8_t *)buf, offset, count, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		errno = map_errno_from_nt_status(status);
 		TALLOC_FREE(frame);
 		return -1;
 	}
 
-	file->offset += ret;
+	file->offset += count;
 
 	TALLOC_FREE(frame);
-	return ret;  /* Success, 0 bytes of data ... */
+	return count;  /* Success, 0 bytes of data ... */
 }
 
 /*
@@ -480,7 +481,7 @@ SMBC_close_ctx(SMBCCTX *context,
 bool
 SMBC_getatr(SMBCCTX * context,
             SMBCSRV *srv,
-            char *path,
+            const char *path,
             uint16 *mode,
             SMB_OFF_T *size,
             struct timespec *create_time_ts,
diff --git a/source3/libsmb/libsmb_misc.c b/source3/libsmb/libsmb_misc.c
index a6e96350f54..7808d8e4f0e 100644
--- a/source3/libsmb/libsmb_misc.c
+++ b/source3/libsmb/libsmb_misc.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 
diff --git a/source3/libsmb/libsmb_path.c b/source3/libsmb/libsmb_path.c
index 64a956d9ee4..1dcf2dd5167 100644
--- a/source3/libsmb/libsmb_path.c
+++ b/source3/libsmb/libsmb_path.c
@@ -88,7 +88,7 @@ urldecode_talloc(TALLOC_CTX *ctx, char **pp_dest, const char *src)
 		newlen++;
 	}
 
-	dest = TALLOC_ARRAY(ctx, char, newlen);
+	dest = talloc_array(ctx, char, newlen);
 	if (!dest) {
 		return err_count;
 	}
diff --git a/source3/libsmb/libsmb_printjob.c b/source3/libsmb/libsmb_printjob.c
index ea0cb37a144..db46ceee9cb 100644
--- a/source3/libsmb/libsmb_printjob.c
+++ b/source3/libsmb/libsmb_printjob.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 
diff --git a/source3/libsmb/libsmb_server.c b/source3/libsmb/libsmb_server.c
index 961c9cde83e..755ef2d32b0 100644
--- a/source3/libsmb/libsmb_server.c
+++ b/source3/libsmb/libsmb_server.c
@@ -24,6 +24,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 #include "../librpc/gen_ndr/ndr_lsa.h"
@@ -245,20 +246,14 @@ SMBC_server_internal(TALLOC_CTX *ctx,
 {
 	SMBCSRV *srv=NULL;
 	char *workgroup = NULL;
-	struct cli_state *c;
-	struct nmb_name called, calling;
+	struct cli_state *c = NULL;
 	const char *server_n = server;
-	struct sockaddr_storage ss;
-	int tried_reverse = 0;
-        int port_try_first;
-        int port_try_next;
         int is_ipc = (share != NULL && strcmp(share, "IPC$") == 0);
 	uint32 fs_attrs = 0;
         const char *username_used;
  	NTSTATUS status;
 	char *newserver, *newshare;
 
-	zero_sockaddr(&ss);
 	ZERO_STRUCT(c);
 	*in_cache = false;
 
@@ -392,20 +387,32 @@ SMBC_server_internal(TALLOC_CTX *ctx,
 		return NULL;
 	}
 
-	make_nmb_name(&calling, smbc_getNetbiosName(context), 0x0);
-	make_nmb_name(&called , server, 0x20);
-
 	DEBUG(4,("SMBC_server: server_n=[%s] server=[%s]\n", server_n, server));
 
 	DEBUG(4,(" -> server_n=[%s] server=[%s]\n", server_n, server));
 
-again:
+	status = NT_STATUS_UNSUCCESSFUL;
 
-	zero_sockaddr(&ss);
+        if (share == NULL || *share == '\0' || is_ipc) {
+		/*
+		 * Try 139 first for IPC$
+		 */
+		status = cli_connect_nb(server_n, NULL, 139, 0x20,
+					smbc_getNetbiosName(context),
+					Undefined, &c);
+	}
 
-	/* have to open a new connection */
-	if ((c = cli_initialise()) == NULL) {
-		errno = ENOMEM;
+	if (!NT_STATUS_IS_OK(status)) {
+		/*
+		 * No IPC$ or 139 did not work
+		 */
+		status = cli_connect_nb(server_n, NULL, 0, 0x20,
+					smbc_getNetbiosName(context),
+					Undefined, &c);
+	}
+
+	if (!NT_STATUS_IS_OK(status)) {
+		errno = map_errno_from_nt_status(status);
 		return NULL;
 	}
 
@@ -423,73 +430,6 @@ again:
 
 	c->timeout = smbc_getTimeout(context);
 
-        /*
-         * Force use of port 139 for first try if share is $IPC, empty, or
-         * null, so browse lists can work
-         */
-        if (share == NULL || *share == '\0' || is_ipc) {
-                port_try_first = 139;
-                port_try_next = 445;
-        } else {
-                port_try_first = 445;
-                port_try_next = 139;
-        }
-
-        c->port = port_try_first;
-
-	status = cli_connect(c, server_n, &ss);
-	if (!NT_STATUS_IS_OK(status)) {
-
-                /* First connection attempt failed.  Try alternate port. */
-                c->port = port_try_next;
-
-                status = cli_connect(c, server_n, &ss);
-		if (!NT_STATUS_IS_OK(status)) {
-			cli_shutdown(c);
-			errno = ETIMEDOUT;
-			return NULL;
-		}
-	}
-
-	if (!cli_session_request(c, &calling, &called)) {
-		cli_shutdown(c);
-		if (strcmp(called.name, "*SMBSERVER")) {
-			make_nmb_name(&called , "*SMBSERVER", 0x20);
-			goto again;
-		} else {  /* Try one more time, but ensure we don't loop */
-
-			/* Only try this if server is an IP address ... */
-
-			if (is_ipaddress(server) && !tried_reverse) {
-				fstring remote_name;
-				struct sockaddr_storage rem_ss;
-
-				if (!interpret_string_addr(&rem_ss, server,
-                                                           NI_NUMERICHOST)) {
-					DEBUG(4, ("Could not convert IP address "
-                                                  "%s to struct sockaddr_storage\n",
-                                                  server));
-					errno = ETIMEDOUT;
-					return NULL;
-				}
-
-				tried_reverse++; /* Yuck */
-
-				if (name_status_find("*", 0, 0,
-                                                     &rem_ss, remote_name)) {
-					make_nmb_name(&called,
-                                                      remote_name,
-                                                      0x20);
-					goto again;
-				}
-			}
-		}
-		errno = ETIMEDOUT;
-		return NULL;
-	}
-
-	DEBUG(4,(" session request ok\n"));
-
 	status = cli_negprot(c);
 
 	if (!NT_STATUS_IS_OK(status)) {
@@ -646,6 +586,10 @@ done:
 		workgroup = *pp_workgroup;
 	}
 	if(!workgroup) {
+		if (c != NULL) {
+			cli_shutdown(c);
+		}
+		SAFE_FREE(srv);
 		return NULL;
 	}
 
@@ -773,7 +717,7 @@ SMBC_attr_server(TALLOC_CTX *ctx,
 
                 zero_sockaddr(&ss);
                 nt_status = cli_full_connection(&ipc_cli,
-						global_myname(), server,
+						lp_netbios_name(), server,
 						&ss, 0, "IPC$", "?????",
 						*pp_username,
 						*pp_workgroup,
diff --git a/source3/libsmb/libsmb_stat.c b/source3/libsmb/libsmb_stat.c
index 9c613508eb4..b579a5f5c1d 100644
--- a/source3/libsmb/libsmb_stat.c
+++ b/source3/libsmb/libsmb_stat.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 
@@ -52,7 +53,7 @@ generate_inode(SMBCCTX *context,
 static int
 setup_stat(SMBCCTX *context,
            struct stat *st,
-           char *fname,
+           const char *fname,
            SMB_OFF_T size,
            int mode)
 {
@@ -180,7 +181,7 @@ SMBC_stat_ctx(SMBCCTX *context,
 
 	st->st_ino = ino;
 
-	setup_stat(context, st, (char *) fname, size, mode);
+	setup_stat(context, st, fname, size, mode);
 
 	st->st_atime = convert_timespec_to_time_t(access_time_ts);
 	st->st_ctime = convert_timespec_to_time_t(change_time_ts);
diff --git a/source3/libsmb/libsmb_xattr.c b/source3/libsmb/libsmb_xattr.c
index af2c56ad403..eeff9a90306 100644
--- a/source3/libsmb/libsmb_xattr.c
+++ b/source3/libsmb/libsmb_xattr.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "libsmbclient.h"
 #include "libsmb_internal.h"
 #include "../librpc/gen_ndr/ndr_lsa.h"
@@ -320,9 +321,9 @@ parse_ace(struct cli_state *ipc_cli,
 		return false;
 	}
 
-	if (StrnCaseCmp(tok, "ALLOWED", strlen("ALLOWED")) == 0) {
+	if (strncasecmp_m(tok, "ALLOWED", strlen("ALLOWED")) == 0) {
 		atype = SEC_ACE_TYPE_ACCESS_ALLOWED;
-	} else if (StrnCaseCmp(tok, "DENIED", strlen("DENIED")) == 0) {
+	} else if (strncasecmp_m(tok, "DENIED", strlen("DENIED")) == 0) {
 		atype = SEC_ACE_TYPE_ACCESS_DENIED;
 	} else {
 		TALLOC_FREE(frame);
@@ -435,12 +436,12 @@ sec_desc_parse(TALLOC_CTX *ctx,
 
 	while (next_token_talloc(ctx, &p, &tok, "\t,\r\n")) {
 
-		if (StrnCaseCmp(tok,"REVISION:", 9) == 0) {
+		if (strncasecmp_m(tok,"REVISION:", 9) == 0) {
 			revision = strtol(tok+9, NULL, 16);
 			continue;
 		}
 
-		if (StrnCaseCmp(tok,"OWNER:", 6) == 0) {
+		if (strncasecmp_m(tok,"OWNER:", 6) == 0) {
 			if (owner_sid) {
 				DEBUG(5,("OWNER specified more than once!\n"));
 				goto done;
@@ -456,7 +457,7 @@ sec_desc_parse(TALLOC_CTX *ctx,
 			continue;
 		}
 
-		if (StrnCaseCmp(tok,"OWNER+:", 7) == 0) {
+		if (strncasecmp_m(tok,"OWNER+:", 7) == 0) {
 			if (owner_sid) {
 				DEBUG(5,("OWNER specified more than once!\n"));
 				goto done;
@@ -472,7 +473,7 @@ sec_desc_parse(TALLOC_CTX *ctx,
 			continue;
 		}
 
-		if (StrnCaseCmp(tok,"GROUP:", 6) == 0) {
+		if (strncasecmp_m(tok,"GROUP:", 6) == 0) {
 			if (group_sid) {
 				DEBUG(5,("GROUP specified more than once!\n"));
 				goto done;
@@ -488,7 +489,7 @@ sec_desc_parse(TALLOC_CTX *ctx,
 			continue;
 		}
 
-		if (StrnCaseCmp(tok,"GROUP+:", 7) == 0) {
+		if (strncasecmp_m(tok,"GROUP+:", 7) == 0) {
 			if (group_sid) {
 				DEBUG(5,("GROUP specified more than once!\n"));
 				goto done;
@@ -504,7 +505,7 @@ sec_desc_parse(TALLOC_CTX *ctx,
 			continue;
 		}
 
-		if (StrnCaseCmp(tok,"ACL:", 4) == 0) {
+		if (strncasecmp_m(tok,"ACL:", 4) == 0) {
 			struct security_ace ace;
 			if (!parse_ace(ipc_cli, pol, &ace, numeric, tok+4)) {
 				DEBUG(5, ("Failed to parse ACL %s\n", tok));
@@ -517,7 +518,7 @@ sec_desc_parse(TALLOC_CTX *ctx,
 			continue;
 		}
 
-		if (StrnCaseCmp(tok,"ACL+:", 5) == 0) {
+		if (strncasecmp_m(tok,"ACL+:", 5) == 0) {
 			struct security_ace ace;
 			if (!parse_ace(ipc_cli, pol, &ace, False, tok+5)) {
 				DEBUG(5, ("Failed to parse ACL %s\n", tok));
@@ -560,14 +561,14 @@ dos_attr_query(SMBCCTX *context,
 	SMB_INO_T inode = 0;
         DOS_ATTR_DESC *ret;
 
-        ret = TALLOC_P(ctx, DOS_ATTR_DESC);
+        ret = talloc(ctx, DOS_ATTR_DESC);
         if (!ret) {
                 errno = ENOMEM;
                 return NULL;
         }
 
         /* Obtain the DOS attributes */
-        if (!SMBC_getatr(context, srv, CONST_DISCARD(char *, filename),
+        if (!SMBC_getatr(context, srv, filename,
                          &mode, &size,
                          &create_time_ts,
                          &access_time_ts,
@@ -636,7 +637,7 @@ dos_attr_parse(SMBCCTX *context,
 
 	frame = talloc_stackframe();
 	while (next_token_talloc(frame, &p, &tok, "\t,\r\n")) {
-		if (StrnCaseCmp(tok, "MODE:", 5) == 0) {
+		if (strncasecmp_m(tok, "MODE:", 5) == 0) {
                         long request = strtol(tok+5, NULL, 16);
                         if (request == 0) {
                                 dad->mode = (request |
@@ -649,32 +650,32 @@ dos_attr_parse(SMBCCTX *context,
 			continue;
 		}
 
-		if (StrnCaseCmp(tok, "SIZE:", 5) == 0) {
+		if (strncasecmp_m(tok, "SIZE:", 5) == 0) {
                         dad->size = (SMB_OFF_T)atof(tok+5);
 			continue;
 		}
 
                 n = strlen(attr_strings.access_time_attr);
-                if (StrnCaseCmp(tok, attr_strings.access_time_attr, n) == 0) {
+                if (strncasecmp_m(tok, attr_strings.access_time_attr, n) == 0) {
                         dad->access_time = (time_t)strtol(tok+n+1, NULL, 10);
 			continue;
 		}
 
                 n = strlen(attr_strings.change_time_attr);
-                if (StrnCaseCmp(tok, attr_strings.change_time_attr, n) == 0) {
+                if (strncasecmp_m(tok, attr_strings.change_time_attr, n) == 0) {
                         dad->change_time = (time_t)strtol(tok+n+1, NULL, 10);
 			continue;
 		}
 
                 n = strlen(attr_strings.write_time_attr);
-                if (StrnCaseCmp(tok, attr_strings.write_time_attr, n) == 0) {
+                if (strncasecmp_m(tok, attr_strings.write_time_attr, n) == 0) {
                         dad->write_time = (time_t)strtol(tok+n+1, NULL, 10);
 			continue;
 		}
 
 		if (attr_strings.create_time_attr != NULL) {
 			n = strlen(attr_strings.create_time_attr);
-			if (StrnCaseCmp(tok, attr_strings.create_time_attr,
+			if (strncasecmp_m(tok, attr_strings.create_time_attr,
 					n) == 0) {
 				dad->create_time = (time_t)strtol(tok+n+1,
 								  NULL, 10);
@@ -682,7 +683,7 @@ dos_attr_parse(SMBCCTX *context,
 			}
 		}
 
-		if (StrnCaseCmp(tok, "INODE:", 6) == 0) {
+		if (strncasecmp_m(tok, "INODE:", 6) == 0) {
                         dad->inode = (SMB_INO_T)atof(tok+6);
 			continue;
 		}
@@ -700,8 +701,8 @@ cacl_get(SMBCCTX *context,
          SMBCSRV *srv,
          struct cli_state *ipc_cli,
          struct policy_handle *pol,
-         char *filename,
-         char *attr_name,
+         const char *filename,
+         const char *attr_name,
          char *buf,
          int bufsize)
 {
@@ -799,12 +800,12 @@ cacl_get(SMBCCTX *context,
                 *pExclude++ = '\0';
         }
 
-        all = (StrnCaseCmp(name, "system.*", 8) == 0);
-        all_nt = (StrnCaseCmp(name, "system.nt_sec_desc.*", 20) == 0);
-        all_nt_acls = (StrnCaseCmp(name, "system.nt_sec_desc.acl.*", 24) == 0);
-        all_dos = (StrnCaseCmp(name, "system.dos_attr.*", 17) == 0);
-        some_nt = (StrnCaseCmp(name, "system.nt_sec_desc.", 19) == 0);
-        some_dos = (StrnCaseCmp(name, "system.dos_attr.", 16) == 0);
+        all = (strncasecmp_m(name, "system.*", 8) == 0);
+        all_nt = (strncasecmp_m(name, "system.nt_sec_desc.*", 20) == 0);
+        all_nt_acls = (strncasecmp_m(name, "system.nt_sec_desc.acl.*", 24) == 0);
+        all_dos = (strncasecmp_m(name, "system.dos_attr.*", 17) == 0);
+        some_nt = (strncasecmp_m(name, "system.nt_sec_desc.", 19) == 0);
+        some_dos = (strncasecmp_m(name, "system.dos_attr.", 16) == 0);
         numeric = (* (name + strlen(name) - 1) != '+');
 
         /* Look for exclusions from "all" requests */
@@ -821,48 +822,48 @@ cacl_get(SMBCCTX *context,
                         }
 
                         /* Which exclusion name is this? */
-                        if (StrCaseCmp(pExclude,
+                        if (strcasecmp_m(pExclude,
                                        "nt_sec_desc.revision") == 0) {
                                 exclude_nt_revision = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             "nt_sec_desc.owner") == 0) {
                                 exclude_nt_owner = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             "nt_sec_desc.group") == 0) {
                                 exclude_nt_group = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             "nt_sec_desc.acl") == 0) {
                                 exclude_nt_acl = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             "dos_attr.mode") == 0) {
                                 exclude_dos_mode = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             "dos_attr.size") == 0) {
                                 exclude_dos_size = True;
                         }
                         else if (excl_attr_strings.create_time_attr != NULL &&
-                                 StrCaseCmp(pExclude,
+                                 strcasecmp_m(pExclude,
                                             excl_attr_strings.change_time_attr) == 0) {
                                 exclude_dos_create_time = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             excl_attr_strings.access_time_attr) == 0) {
                                 exclude_dos_access_time = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             excl_attr_strings.write_time_attr) == 0) {
                                 exclude_dos_write_time = True;
                         }
-                        else if (StrCaseCmp(pExclude,
+                        else if (strcasecmp_m(pExclude,
                                             excl_attr_strings.change_time_attr) == 0) {
                                 exclude_dos_change_time = True;
                         }
-                        else if (StrCaseCmp(pExclude, "dos_attr.inode") == 0) {
+                        else if (strcasecmp_m(pExclude, "dos_attr.inode") == 0) {
                                 exclude_dos_inode = True;
                         }
                         else {
@@ -932,7 +933,7 @@ cacl_get(SMBCCTX *context,
                                                      "REVISION:%d",
                                                      sd->revision);
                                 }
-                        } else if (StrCaseCmp(name, "revision") == 0) {
+                        } else if (strcasecmp_m(name, "revision") == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "%d",
                                                             sd->revision);
@@ -981,7 +982,7 @@ cacl_get(SMBCCTX *context,
                                         n = snprintf(buf, bufsize,
                                                      ",OWNER:%s", sidstr);
                                 }
-                        } else if (StrnCaseCmp(name, "owner", 5) == 0) {
+                        } else if (strncasecmp_m(name, "owner", 5) == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "%s", sidstr);
                                         if (!p) {
@@ -1027,7 +1028,7 @@ cacl_get(SMBCCTX *context,
                                         n = snprintf(buf, bufsize,
                                                      ",GROUP:%s", sidstr);
                                 }
-                        } else if (StrnCaseCmp(name, "group", 5) == 0) {
+                        } else if (strncasecmp_m(name, "group", 5) == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "%s", sidstr);
                                         if (!p) {
@@ -1084,10 +1085,10 @@ cacl_get(SMBCCTX *context,
                                                         ace->flags,
                                                         ace->access_mask);
                                         }
-                                } else if ((StrnCaseCmp(name, "acl", 3) == 0 &&
-                                            StrCaseCmp(name+3, sidstr) == 0) ||
-                                           (StrnCaseCmp(name, "acl+", 4) == 0 &&
-                                            StrCaseCmp(name+4, sidstr) == 0)) {
+                                } else if ((strncasecmp_m(name, "acl", 3) == 0 &&
+                                            strcasecmp_m(name+3, sidstr) == 0) ||
+                                           (strncasecmp_m(name, "acl+", 4) == 0 &&
+                                            strcasecmp_m(name+4, sidstr) == 0)) {
                                         if (determine_size) {
                                                 p = talloc_asprintf(
                                                         ctx, 
@@ -1192,7 +1193,7 @@ cacl_get(SMBCCTX *context,
                                                       : ""),
                                                      mode);
                                 }
-                        } else if (StrCaseCmp(name, "mode") == 0) {
+                        } else if (strcasecmp_m(name, "mode") == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "0x%x", mode);
                                         if (!p) {
@@ -1233,7 +1234,7 @@ cacl_get(SMBCCTX *context,
                                                      ",SIZE:%.0f",
                                                      (double)size);
                                 }
-                        } else if (StrCaseCmp(name, "size") == 0) {
+                        } else if (strcasecmp_m(name, "size") == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(
                                                 ctx,
@@ -1280,7 +1281,7 @@ cacl_get(SMBCCTX *context,
                                                      attr_strings.create_time_attr,
                                                      (unsigned long) create_time);
                                 }
-                        } else if (StrCaseCmp(name, attr_strings.create_time_attr) == 0) {
+                        } else if (strcasecmp_m(name, attr_strings.create_time_attr) == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "%lu", (unsigned long) create_time);
                                         if (!p) {
@@ -1322,7 +1323,7 @@ cacl_get(SMBCCTX *context,
                                                      attr_strings.access_time_attr,
                                                      (unsigned long) access_time);
                                 }
-                        } else if (StrCaseCmp(name, attr_strings.access_time_attr) == 0) {
+                        } else if (strcasecmp_m(name, attr_strings.access_time_attr) == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "%lu", (unsigned long) access_time);
                                         if (!p) {
@@ -1364,7 +1365,7 @@ cacl_get(SMBCCTX *context,
                                                      attr_strings.write_time_attr,
                                                      (unsigned long) write_time);
                                 }
-                        } else if (StrCaseCmp(name, attr_strings.write_time_attr) == 0) {
+                        } else if (strcasecmp_m(name, attr_strings.write_time_attr) == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "%lu", (unsigned long) write_time);
                                         if (!p) {
@@ -1406,7 +1407,7 @@ cacl_get(SMBCCTX *context,
                                                      attr_strings.change_time_attr,
                                                      (unsigned long) change_time);
                                 }
-                        } else if (StrCaseCmp(name, attr_strings.change_time_attr) == 0) {
+                        } else if (strcasecmp_m(name, attr_strings.change_time_attr) == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(ctx, "%lu", (unsigned long) change_time);
                                         if (!p) {
@@ -1447,7 +1448,7 @@ cacl_get(SMBCCTX *context,
                                                      ",INODE:%.0f",
                                                      (double) ino);
                                 }
-                        } else if (StrCaseCmp(name, "inode") == 0) {
+                        } else if (strcasecmp_m(name, "inode") == 0) {
                                 if (determine_size) {
                                         p = talloc_asprintf(
                                                 ctx,
@@ -1774,8 +1775,8 @@ SMBC_setxattr_ctx(SMBCCTX *context,
         /*
          * Are they asking to set the entire set of known attributes?
          */
-        if (StrCaseCmp(name, "system.*") == 0 ||
-            StrCaseCmp(name, "system.*+") == 0) {
+        if (strcasecmp_m(name, "system.*") == 0 ||
+            strcasecmp_m(name, "system.*+") == 0) {
                 /* Yup. */
                 char *namevalue =
                         talloc_asprintf(talloc_tos(), "%s:%s",
@@ -1834,11 +1835,11 @@ SMBC_setxattr_ctx(SMBCCTX *context,
          * Are they asking to set an access control element or to set
          * the entire access control list?
          */
-        if (StrCaseCmp(name, "system.nt_sec_desc.*") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.*+") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.revision") == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.acl", 22) == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.acl+", 23) == 0) {
+        if (strcasecmp_m(name, "system.nt_sec_desc.*") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.*+") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.revision") == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.acl", 22) == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.acl+", 23) == 0) {
 
                 /* Yup. */
                 char *namevalue =
@@ -1867,8 +1868,8 @@ SMBC_setxattr_ctx(SMBCCTX *context,
         /*
          * Are they asking to set the owner?
          */
-        if (StrCaseCmp(name, "system.nt_sec_desc.owner") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.owner+") == 0) {
+        if (strcasecmp_m(name, "system.nt_sec_desc.owner") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.owner+") == 0) {
 
                 /* Yup. */
                 char *namevalue =
@@ -1893,8 +1894,8 @@ SMBC_setxattr_ctx(SMBCCTX *context,
         /*
          * Are they asking to set the group?
          */
-        if (StrCaseCmp(name, "system.nt_sec_desc.group") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.group+") == 0) {
+        if (strcasecmp_m(name, "system.nt_sec_desc.group") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.group+") == 0) {
 
                 /* Yup. */
                 char *namevalue =
@@ -1935,13 +1936,13 @@ SMBC_setxattr_ctx(SMBCCTX *context,
         /*
          * Are they asking to set a DOS attribute?
          */
-        if (StrCaseCmp(name, "system.dos_attr.*") == 0 ||
-            StrCaseCmp(name, "system.dos_attr.mode") == 0 ||
+        if (strcasecmp_m(name, "system.dos_attr.*") == 0 ||
+            strcasecmp_m(name, "system.dos_attr.mode") == 0 ||
             (attr_strings.create_time_attr != NULL &&
-             StrCaseCmp(name, attr_strings.create_time_attr) == 0) ||
-            StrCaseCmp(name, attr_strings.access_time_attr) == 0 ||
-            StrCaseCmp(name, attr_strings.write_time_attr) == 0 ||
-            StrCaseCmp(name, attr_strings.change_time_attr) == 0) {
+             strcasecmp_m(name, attr_strings.create_time_attr) == 0) ||
+            strcasecmp_m(name, attr_strings.access_time_attr) == 0 ||
+            strcasecmp_m(name, attr_strings.write_time_attr) == 0 ||
+            strcasecmp_m(name, attr_strings.change_time_attr) == 0) {
 
                 /* get a DOS Attribute Descriptor with current attributes */
                 dad = dos_attr_query(context, talloc_tos(), path, srv);
@@ -2080,39 +2081,39 @@ SMBC_getxattr_ctx(SMBCCTX *context,
         }
 
         /* Are they requesting a supported attribute? */
-        if (StrCaseCmp(name, "system.*") == 0 ||
-            StrnCaseCmp(name, "system.*!", 9) == 0 ||
-            StrCaseCmp(name, "system.*+") == 0 ||
-            StrnCaseCmp(name, "system.*+!", 10) == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.*") == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.*!", 21) == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.*+") == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.*+!", 22) == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.revision") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.owner") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.owner+") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.group") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.group+") == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.acl", 22) == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.acl+", 23) == 0 ||
-            StrCaseCmp(name, "system.dos_attr.*") == 0 ||
-            StrnCaseCmp(name, "system.dos_attr.*!", 18) == 0 ||
-            StrCaseCmp(name, "system.dos_attr.mode") == 0 ||
-            StrCaseCmp(name, "system.dos_attr.size") == 0 ||
+        if (strcasecmp_m(name, "system.*") == 0 ||
+            strncasecmp_m(name, "system.*!", 9) == 0 ||
+            strcasecmp_m(name, "system.*+") == 0 ||
+            strncasecmp_m(name, "system.*+!", 10) == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.*") == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.*!", 21) == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.*+") == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.*+!", 22) == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.revision") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.owner") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.owner+") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.group") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.group+") == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.acl", 22) == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.acl+", 23) == 0 ||
+            strcasecmp_m(name, "system.dos_attr.*") == 0 ||
+            strncasecmp_m(name, "system.dos_attr.*!", 18) == 0 ||
+            strcasecmp_m(name, "system.dos_attr.mode") == 0 ||
+            strcasecmp_m(name, "system.dos_attr.size") == 0 ||
             (attr_strings.create_time_attr != NULL &&
-             StrCaseCmp(name, attr_strings.create_time_attr) == 0) ||
-            StrCaseCmp(name, attr_strings.access_time_attr) == 0 ||
-            StrCaseCmp(name, attr_strings.write_time_attr) == 0 ||
-            StrCaseCmp(name, attr_strings.change_time_attr) == 0 ||
-            StrCaseCmp(name, "system.dos_attr.inode") == 0) {
+             strcasecmp_m(name, attr_strings.create_time_attr) == 0) ||
+            strcasecmp_m(name, attr_strings.access_time_attr) == 0 ||
+            strcasecmp_m(name, attr_strings.write_time_attr) == 0 ||
+            strcasecmp_m(name, attr_strings.change_time_attr) == 0 ||
+            strcasecmp_m(name, "system.dos_attr.inode") == 0) {
 
                 /* Yup. */
-                char *filename = (char *) name;
+                const char *filename = name;
                 ret = cacl_get(context, talloc_tos(), srv,
                                ipc_srv == NULL ? NULL : ipc_srv->cli, 
                                &ipc_srv->pol, path,
                                filename,
-                               CONST_DISCARD(char *, value),
+                               discard_const_p(char, value),
                                size);
                 if (ret < 0 && errno == 0) {
                         errno = SMBC_errno(context, srv->cli);
@@ -2205,8 +2206,8 @@ SMBC_removexattr_ctx(SMBCCTX *context,
         }
 
         /* Are they asking to set the entire ACL? */
-        if (StrCaseCmp(name, "system.nt_sec_desc.*") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.*+") == 0) {
+        if (strcasecmp_m(name, "system.nt_sec_desc.*") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.*+") == 0) {
 
                 /* Yup. */
                 ret = cacl_set(context, talloc_tos(), srv->cli,
@@ -2220,18 +2221,18 @@ SMBC_removexattr_ctx(SMBCCTX *context,
          * Are they asking to remove one or more spceific security descriptor
          * attributes?
          */
-        if (StrCaseCmp(name, "system.nt_sec_desc.revision") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.owner") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.owner+") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.group") == 0 ||
-            StrCaseCmp(name, "system.nt_sec_desc.group+") == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.acl", 22) == 0 ||
-            StrnCaseCmp(name, "system.nt_sec_desc.acl+", 23) == 0) {
+        if (strcasecmp_m(name, "system.nt_sec_desc.revision") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.owner") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.owner+") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.group") == 0 ||
+            strcasecmp_m(name, "system.nt_sec_desc.group+") == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.acl", 22) == 0 ||
+            strncasecmp_m(name, "system.nt_sec_desc.acl+", 23) == 0) {
 
                 /* Yup. */
                 ret = cacl_set(context, talloc_tos(), srv->cli,
                                ipc_srv->cli, &ipc_srv->pol, path,
-                               CONST_DISCARD(char *, name) + 19,
+                               discard_const_p(char, name) + 19,
                                SMBC_XATTR_MODE_REMOVE, 0);
 		TALLOC_FREE(frame);
                 return ret;
diff --git a/source3/libsmb/namequery.c b/source3/libsmb/namequery.c
index ef7aab6a6aa..dca740d3e6b 100644
--- a/source3/libsmb/namequery.c
+++ b/source3/libsmb/namequery.c
@@ -19,6 +19,7 @@
 */
 
 #include "includes.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "libads/sitename_cache.h"
 #include "libads/dns.h"
 #include "../libcli/netlogon/netlogon.h"
@@ -196,6 +197,26 @@ char *saf_fetch( const char *domain )
 	return server;
 }
 
+static void set_socket_addr_v4(struct sockaddr_storage *addr)
+{
+	if (!interpret_string_addr(addr, lp_socket_address(),
+				   AI_NUMERICHOST|AI_PASSIVE)) {
+		zero_sockaddr(addr);
+	}
+	if (addr->ss_family != AF_INET) {
+		zero_sockaddr(addr);
+	}
+}
+
+static struct in_addr my_socket_addr_v4(void)
+{
+	struct sockaddr_storage my_addr;
+	struct sockaddr_in *in_addr = (struct sockaddr_in *)((char *)&my_addr);
+
+	set_socket_addr_v4(&my_addr);
+	return in_addr->sin_addr;
+}
+
 /****************************************************************************
  Generate a random trn_id.
 ****************************************************************************/
@@ -225,7 +246,7 @@ static struct node_status *parse_node_status(TALLOC_CTX *mem_ctx, char *p,
 	if (*num_names == 0)
 		return NULL;
 
-	ret = TALLOC_ARRAY(mem_ctx, struct node_status,*num_names);
+	ret = talloc_array(mem_ctx, struct node_status,*num_names);
 	if (!ret)
 		return NULL;
 
@@ -704,10 +725,7 @@ struct tevent_req *node_status_query_send(TALLOC_CTX *mem_ctx,
 	in_addr = (struct sockaddr_in *)(void *)&state->addr;
 	in_addr->sin_port = htons(NMB_PORT);
 
-	if (!interpret_string_addr(&state->my_addr, lp_socket_address(),
-				   AI_NUMERICHOST|AI_PASSIVE)) {
-		zero_sockaddr(&state->my_addr);
-	}
+	set_socket_addr_v4(&state->my_addr);
 
 	ZERO_STRUCT(p);
 	nmb->header.name_trn_id = generate_trn_id();
@@ -891,10 +909,7 @@ bool name_status_find(const char *q_name,
 		return false;
 	}
 
-	if (!interpret_string_addr(&ss, lp_socket_address(),
-				AI_NUMERICHOST|AI_PASSIVE)) {
-		zero_sockaddr(&ss);
-	}
+	set_socket_addr_v4(&ss);
 
 	/* W2K PDC's seem not to respond to '*'#0. JRA */
 	make_nmb_name(&nname, q_name, q_type);
@@ -962,9 +977,9 @@ static int addr_compare(const struct sockaddr_storage *ss1,
 
 	for (i=0;i<num_interfaces;i++) {
 		const struct sockaddr_storage *pss = iface_n_bcast(i);
-		unsigned char *p_ss1 = NULL;
-		unsigned char *p_ss2 = NULL;
-		unsigned char *p_if = NULL;
+		const unsigned char *p_ss1 = NULL;
+		const unsigned char *p_ss2 = NULL;
+		const unsigned char *p_if = NULL;
 		size_t len = 0;
 		int bits1, bits2;
 
@@ -973,21 +988,21 @@ static int addr_compare(const struct sockaddr_storage *ss1,
 			continue;
 		}
 		if (pss->ss_family == AF_INET) {
-			p_if = (unsigned char *)
+			p_if = (const unsigned char *)
 				&((const struct sockaddr_in *)pss)->sin_addr;
-			p_ss1 = (unsigned char *)
+			p_ss1 = (const unsigned char *)
 				&((const struct sockaddr_in *)ss1)->sin_addr;
-			p_ss2 = (unsigned char *)
+			p_ss2 = (const unsigned char *)
 				&((const struct sockaddr_in *)ss2)->sin_addr;
 			len = 4;
 		}
 #if defined(HAVE_IPV6)
 		if (pss->ss_family == AF_INET6) {
-			p_if = (unsigned char *)
+			p_if = (const unsigned char *)
 				&((const struct sockaddr_in6 *)pss)->sin6_addr;
-			p_ss1 = (unsigned char *)
+			p_ss1 = (const unsigned char *)
 				&((const struct sockaddr_in6 *)ss1)->sin6_addr;
-			p_ss2 = (unsigned char *)
+			p_ss2 = (const unsigned char *)
 				&((const struct sockaddr_in6 *)ss2)->sin6_addr;
 			len = 16;
 		}
@@ -1002,14 +1017,14 @@ static int addr_compare(const struct sockaddr_storage *ss1,
 	}
 
 	/* Bias towards directly reachable IPs */
-	if (iface_local((struct sockaddr *)ss1)) {
+	if (iface_local((const struct sockaddr *)ss1)) {
 		if (ss1->ss_family == AF_INET) {
 			max_bits1 += 32;
 		} else {
 			max_bits1 += 128;
 		}
 	}
-	if (iface_local((struct sockaddr *)ss2)) {
+	if (iface_local((const struct sockaddr *)ss2)) {
 		if (ss2->ss_family == AF_INET) {
 			max_bits2 += 32;
 		} else {
@@ -1084,7 +1099,8 @@ static int remove_duplicate_addrs2(struct ip_service *iplist, int count )
 		}
 
 		for ( j=i+1; j<count; j++ ) {
-			if (sockaddr_equal((struct sockaddr *)&iplist[i].ss, (struct sockaddr *)&iplist[j].ss) &&
+			if (sockaddr_equal((struct sockaddr *)(void *)&iplist[i].ss,
+					   (struct sockaddr *)(void *)&iplist[j].ss) &&
 					iplist[i].port == iplist[j].port) {
 				zero_sockaddr(&iplist[j].ss);
 			}
@@ -1111,7 +1127,7 @@ static int remove_duplicate_addrs2(struct ip_service *iplist, int count )
 static bool prioritize_ipv4_list(struct ip_service *iplist, int count)
 {
 	TALLOC_CTX *frame = talloc_stackframe();
-	struct ip_service *iplist_new = TALLOC_ARRAY(frame, struct ip_service, count);
+	struct ip_service *iplist_new = talloc_array(frame, struct ip_service, count);
 	int i, j;
 
 	if (iplist_new == NULL) {
@@ -1177,7 +1193,6 @@ struct tevent_req *name_query_send(TALLOC_CTX *mem_ctx,
 	struct packet_struct p;
 	struct nmb_packet *nmb = &p.packet.nmb;
 	struct sockaddr_in *in_addr;
-	struct timeval timeout;
 
 	req = tevent_req_create(mem_ctx, &state, struct name_query_state);
 	if (req == NULL) {
@@ -1202,10 +1217,7 @@ struct tevent_req *name_query_send(TALLOC_CTX *mem_ctx,
 	in_addr = (struct sockaddr_in *)(void *)&state->addr;
 	in_addr->sin_port = htons(NMB_PORT);
 
-	if (!interpret_string_addr(&state->my_addr, lp_socket_address(),
-				   AI_NUMERICHOST|AI_PASSIVE)) {
-		zero_sockaddr(&state->my_addr);
-	}
+	set_socket_addr_v4(&state->my_addr);
 
 	ZERO_STRUCT(p);
 	nmb->header.name_trn_id = generate_trn_id();
@@ -1243,14 +1255,6 @@ struct tevent_req *name_query_send(TALLOC_CTX *mem_ctx,
 		DEBUG(10, ("nb_trans_send failed\n"));
 		return tevent_req_post(req, ev);
 	}
-	if (bcast) {
-		timeout = timeval_current_ofs(0, 250000);
-	} else {
-		timeout = timeval_current_ofs(2, 0);
-	}
-	if (!tevent_req_set_endtime(req, ev, timeout)) {
-		return tevent_req_post(req, ev);
-	}
 	tevent_req_set_callback(subreq, name_query_done, req);
 	return req;
 }
@@ -1261,6 +1265,7 @@ static bool name_query_validator(struct packet_struct *p, void *private_data)
 		private_data, struct name_query_state);
 	struct nmb_packet *nmb = &p->packet.nmb;
 	struct sockaddr_storage *tmp_addrs;
+	bool got_unique_netbios_name = false;
 	int i;
 
 	debug_nmb_packet(p);
@@ -1322,7 +1327,7 @@ static bool name_query_validator(struct packet_struct *p, void *private_data)
 		return false;
 	}
 
-	tmp_addrs = TALLOC_REALLOC_ARRAY(
+	tmp_addrs = talloc_realloc(
 		state, state->addrs, struct sockaddr_storage,
 		state->num_addrs + nmb->answers->rdlength/6);
 	if (tmp_addrs == NULL) {
@@ -1335,11 +1340,32 @@ static bool name_query_validator(struct packet_struct *p, void *private_data)
 		 "from %s ( ", inet_ntoa(p->ip)));
 
 	for (i=0; i<nmb->answers->rdlength/6; i++) {
+		uint16_t flags;
 		struct in_addr ip;
+		struct sockaddr_storage addr;
+		int j;
+
+		flags = RSVAL(&nmb->answers->rdata[i*6], 0);
+		got_unique_netbios_name |= ((flags & 0x8000) == 0);
+
 		putip((char *)&ip,&nmb->answers->rdata[2+i*6]);
-		in_addr_to_sockaddr_storage(
-			&state->addrs[state->num_addrs], ip);
+		in_addr_to_sockaddr_storage(&addr, ip);
+
+		for (j=0; j<state->num_addrs; j++) {
+			if (sockaddr_equal(
+				    (struct sockaddr *)(void *)&addr,
+				    (struct sockaddr *)(void *)&state->addrs[j])) {
+				break;
+			}
+		}
+		if (j < state->num_addrs) {
+			/* Already got it */
+			continue;
+		}
+
 		DEBUGADD(2,("%s ",inet_ntoa(ip)));
+
+		state->addrs[state->num_addrs] = addr;
 		state->num_addrs += 1;
 	}
 	DEBUGADD(2,(")\n"));
@@ -1360,10 +1386,10 @@ static bool name_query_validator(struct packet_struct *p, void *private_data)
 
 	if (state->bcast) {
 		/*
-		 * We have to collect all entries coming in from
-		 * broadcast queries
+		 * We have to collect all entries coming in from broadcast
+		 * queries. If we got a unique name, we're done.
 		 */
-		return false;
+		return got_unique_netbios_name;
 	}
 	/*
 	 * WINS responses are accepted when they are received
@@ -1407,9 +1433,18 @@ NTSTATUS name_query_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
 		req, struct name_query_state);
 	NTSTATUS status;
 
-	if (tevent_req_is_nterror(req, &status)
-	    && !NT_STATUS_EQUAL(status, NT_STATUS_IO_TIMEOUT)) {
-		return status;
+	if (tevent_req_is_nterror(req, &status)) {
+		if (state->bcast &&
+		    NT_STATUS_EQUAL(status, NT_STATUS_IO_TIMEOUT)) {
+			/*
+			 * In the broadcast case we collect replies until the
+			 * timeout.
+			 */
+			status = NT_STATUS_OK;
+		}
+		if (!NT_STATUS_IS_OK(status)) {
+			return status;
+		}
 	}
 	if (state->num_addrs == 0) {
 		return NT_STATUS_NOT_FOUND;
@@ -1433,6 +1468,7 @@ NTSTATUS name_query(const char *name, int name_type,
 	TALLOC_CTX *frame = talloc_stackframe();
 	struct tevent_context *ev;
 	struct tevent_req *req;
+	struct timeval timeout;
 	NTSTATUS status = NT_STATUS_NO_MEMORY;
 
 	ev = tevent_context_init(frame);
@@ -1443,6 +1479,14 @@ NTSTATUS name_query(const char *name, int name_type,
 	if (req == NULL) {
 		goto fail;
 	}
+	if (bcast) {
+		timeout = timeval_current_ofs(0, 250000);
+	} else {
+		timeout = timeval_current_ofs(2, 0);
+	}
+	if (!tevent_req_set_endtime(req, ev, timeout)) {
+		goto fail;
+	}
 	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
 		goto fail;
 	}
@@ -1482,113 +1526,507 @@ static bool convert_ss2service(struct ip_service **return_iplist,
 	return true;
 }
 
+struct name_queries_state {
+	struct tevent_context *ev;
+	const char *name;
+	int name_type;
+	bool bcast;
+	bool recurse;
+	const struct sockaddr_storage *addrs;
+	int num_addrs;
+	int wait_msec;
+	int timeout_msec;
+
+	struct tevent_req **subreqs;
+	int num_received;
+	int num_sent;
+
+	int received_index;
+	struct sockaddr_storage *result_addrs;
+	int num_result_addrs;
+	uint8_t flags;
+};
+
+static void name_queries_done(struct tevent_req *subreq);
+static void name_queries_next(struct tevent_req *subreq);
+
+/*
+ * Send a name query to multiple destinations with a wait time in between
+ */
+
+static struct tevent_req *name_queries_send(
+	TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+	const char *name, int name_type,
+	bool bcast, bool recurse,
+	const struct sockaddr_storage *addrs,
+	int num_addrs, int wait_msec, int timeout_msec)
+{
+	struct tevent_req *req, *subreq;
+	struct name_queries_state *state;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct name_queries_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->ev = ev;
+	state->name = name;
+	state->name_type = name_type;
+	state->bcast = bcast;
+	state->recurse = recurse;
+	state->addrs = addrs;
+	state->num_addrs = num_addrs;
+	state->wait_msec = wait_msec;
+	state->timeout_msec = timeout_msec;
+
+	state->subreqs = talloc_zero_array(
+		state, struct tevent_req *, num_addrs);
+	if (tevent_req_nomem(state->subreqs, req)) {
+		return tevent_req_post(req, ev);
+	}
+	state->num_sent = 0;
+
+	subreq = name_query_send(
+		state->subreqs, state->ev, name, name_type, bcast, recurse,
+		&state->addrs[state->num_sent]);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	if (!tevent_req_set_endtime(
+		    subreq, state->ev,
+		    timeval_current_ofs(0, state->timeout_msec * 1000))) {
+		tevent_req_oom(req);
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, name_queries_done, req);
+
+	state->subreqs[state->num_sent] = subreq;
+	state->num_sent += 1;
+
+	if (state->num_sent < state->num_addrs) {
+		subreq = tevent_wakeup_send(
+			state, state->ev,
+			timeval_current_ofs(0, state->wait_msec * 1000));
+		if (tevent_req_nomem(subreq, req)) {
+			return tevent_req_post(req, ev);
+		}
+		tevent_req_set_callback(subreq, name_queries_next, req);
+	}
+	return req;
+}
+
+static void name_queries_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct name_queries_state *state = tevent_req_data(
+		req, struct name_queries_state);
+	int i;
+	NTSTATUS status;
+
+	status = name_query_recv(subreq, state, &state->result_addrs,
+				 &state->num_result_addrs, &state->flags);
+
+	for (i=0; i<state->num_sent; i++) {
+		if (state->subreqs[i] == subreq) {
+			break;
+		}
+	}
+	if (i == state->num_sent) {
+		tevent_req_nterror(req, NT_STATUS_INTERNAL_ERROR);
+		return;
+	}
+	TALLOC_FREE(state->subreqs[i]);
+
+	state->num_received += 1;
+
+	if (!NT_STATUS_IS_OK(status)) {
+
+		if (state->num_received >= state->num_addrs) {
+			tevent_req_nterror(req, status);
+			return;
+		}
+		/*
+		 * Still outstanding requests, just wait
+		 */
+		return;
+	}
+	state->received_index = i;
+	tevent_req_done(req);
+}
+
+static void name_queries_next(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct name_queries_state *state = tevent_req_data(
+		req, struct name_queries_state);
+
+	if (!tevent_wakeup_recv(subreq)) {
+		tevent_req_nterror(req, NT_STATUS_INTERNAL_ERROR);
+		return;
+	}
+
+	subreq = name_query_send(
+		state->subreqs, state->ev,
+		state->name, state->name_type, state->bcast, state->recurse,
+		&state->addrs[state->num_sent]);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, name_queries_done, req);
+	if (!tevent_req_set_endtime(
+		    subreq, state->ev,
+		    timeval_current_ofs(0, state->timeout_msec * 1000))) {
+		tevent_req_oom(req);
+		return;
+	}
+	state->subreqs[state->num_sent] = subreq;
+	state->num_sent += 1;
+
+	if (state->num_sent < state->num_addrs) {
+		subreq = tevent_wakeup_send(
+			state, state->ev,
+			timeval_current_ofs(0, state->wait_msec * 1000));
+		if (tevent_req_nomem(subreq, req)) {
+			return;
+		}
+		tevent_req_set_callback(subreq, name_queries_next, req);
+	}
+}
+
+static NTSTATUS name_queries_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+				  struct sockaddr_storage **result_addrs,
+				  int *num_result_addrs, uint8_t *flags,
+				  int *received_index)
+{
+	struct name_queries_state *state = tevent_req_data(
+		req, struct name_queries_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+
+	if (result_addrs != NULL) {
+		*result_addrs = talloc_move(mem_ctx, &state->result_addrs);
+	}
+	if (num_result_addrs != NULL) {
+		*num_result_addrs = state->num_result_addrs;
+	}
+	if (flags != NULL) {
+		*flags = state->flags;
+	}
+	if (received_index != NULL) {
+		*received_index = state->received_index;
+	}
+	return NT_STATUS_OK;
+}
+
 /********************************************************
  Resolve via "bcast" method.
 *********************************************************/
 
-NTSTATUS name_resolve_bcast(const char *name,
-			int name_type,
-			struct ip_service **return_iplist,
-			int *return_count)
+struct name_resolve_bcast_state {
+	struct sockaddr_storage *addrs;
+	int num_addrs;
+};
+
+static void name_resolve_bcast_done(struct tevent_req *subreq);
+
+struct tevent_req *name_resolve_bcast_send(TALLOC_CTX *mem_ctx,
+					   struct tevent_context *ev,
+					   const char *name,
+					   int name_type)
 {
-	int i;
-	int num_interfaces = iface_count();
-	struct sockaddr_storage *ss_list;
-	struct sockaddr_storage ss;
-	NTSTATUS status = NT_STATUS_NOT_FOUND;
+	struct tevent_req *req, *subreq;
+	struct name_resolve_bcast_state *state;
+	struct sockaddr_storage *bcast_addrs;
+	int i, num_addrs, num_bcast_addrs;
 
-	if (lp_disable_netbios()) {
-		DEBUG(5,("name_resolve_bcast(%s#%02x): netbios is disabled\n",
-					name, name_type));
-		return NT_STATUS_INVALID_PARAMETER;
+	req = tevent_req_create(mem_ctx, &state,
+				struct name_resolve_bcast_state);
+	if (req == NULL) {
+		return NULL;
 	}
 
-	*return_iplist = NULL;
-	*return_count = 0;
+	if (lp_disable_netbios()) {
+		DEBUG(5, ("name_resolve_bcast(%s#%02x): netbios is disabled\n",
+			  name, name_type));
+		tevent_req_nterror(req, NT_STATUS_INVALID_PARAMETER);
+		return tevent_req_post(req, ev);
+	}
 
 	/*
 	 * "bcast" means do a broadcast lookup on all the local interfaces.
 	 */
 
-	DEBUG(3,("name_resolve_bcast: Attempting broadcast lookup "
-		"for name %s<0x%x>\n", name, name_type));
+	DEBUG(3, ("name_resolve_bcast: Attempting broadcast lookup "
+		  "for name %s<0x%x>\n", name, name_type));
 
-	if (!interpret_string_addr(&ss, lp_socket_address(),
-				AI_NUMERICHOST|AI_PASSIVE)) {
-		zero_sockaddr(&ss);
+	num_addrs = iface_count();
+	bcast_addrs = talloc_array(state, struct sockaddr_storage, num_addrs);
+	if (tevent_req_nomem(bcast_addrs, req)) {
+		return tevent_req_post(req, ev);
 	}
 
 	/*
 	 * Lookup the name on all the interfaces, return on
 	 * the first successful match.
 	 */
-	for( i = num_interfaces-1; i >= 0; i--) {
+	num_bcast_addrs = 0;
+
+	for (i=0; i<num_addrs; i++) {
 		const struct sockaddr_storage *pss = iface_n_bcast(i);
 
-		/* Done this way to fix compiler error on IRIX 5.x */
-		if (!pss) {
+		if (pss->ss_family != AF_INET) {
 			continue;
 		}
-		status = name_query(name, name_type, true, true, pss,
-				    talloc_tos(), &ss_list, return_count,
-				    NULL);
-		if (NT_STATUS_IS_OK(status)) {
-			goto success;
-		}
+		bcast_addrs[num_bcast_addrs] = *pss;
+		num_bcast_addrs += 1;
 	}
 
-	/* failed - no response */
+	subreq = name_queries_send(state, ev, name, name_type, true, true,
+				   bcast_addrs, num_bcast_addrs, 0, 1000);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, name_resolve_bcast_done, req);
+	return req;
+}
 
-	return status;
+static void name_resolve_bcast_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct name_resolve_bcast_state *state = tevent_req_data(
+		req, struct name_resolve_bcast_state);
+	NTSTATUS status;
 
-success:
+	status = name_queries_recv(subreq, state,
+				   &state->addrs, &state->num_addrs,
+				   NULL, NULL);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	tevent_req_done(req);
+}
 
-	if (!convert_ss2service(return_iplist, ss_list, *return_count) )
-		status = NT_STATUS_NO_MEMORY;
+NTSTATUS name_resolve_bcast_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+				 struct sockaddr_storage **addrs,
+				 int *num_addrs)
+{
+	struct name_resolve_bcast_state *state = tevent_req_data(
+		req, struct name_resolve_bcast_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+	*addrs = talloc_move(mem_ctx, &state->addrs);
+	*num_addrs = state->num_addrs;
+	return NT_STATUS_OK;
+}
 
-	TALLOC_FREE(ss_list);
+NTSTATUS name_resolve_bcast(const char *name,
+			int name_type,
+			TALLOC_CTX *mem_ctx,
+			struct sockaddr_storage **return_iplist,
+			int *return_count)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = name_resolve_bcast_send(frame, ev, name, name_type);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
+		goto fail;
+	}
+	status = name_resolve_bcast_recv(req, mem_ctx, return_iplist,
+					 return_count);
+ fail:
+	TALLOC_FREE(frame);
 	return status;
 }
 
-/********************************************************
- Resolve via "wins" method.
-*********************************************************/
+struct query_wins_list_state {
+	struct tevent_context *ev;
+	const char *name;
+	uint8_t name_type;
+	struct in_addr *servers;
+	uint32_t num_servers;
+	struct sockaddr_storage server;
+	uint32_t num_sent;
 
-NTSTATUS resolve_wins(const char *name,
-		int name_type,
-		struct ip_service **return_iplist,
-		int *return_count)
+	struct sockaddr_storage *addrs;
+	int num_addrs;
+	uint8_t flags;
+};
+
+static void query_wins_list_done(struct tevent_req *subreq);
+
+/*
+ * Query a list of (replicating) wins servers in sequence, call them
+ * dead if they don't reply
+ */
+
+static struct tevent_req *query_wins_list_send(
+	TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+	struct in_addr src_ip, const char *name, uint8_t name_type,
+	struct in_addr *servers, int num_servers)
 {
-	int t, i;
-	char **wins_tags;
-	struct sockaddr_storage src_ss, *ss_list = NULL;
-	struct in_addr src_ip;
+	struct tevent_req *req, *subreq;
+	struct query_wins_list_state *state;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct query_wins_list_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->ev = ev;
+	state->name = name;
+	state->name_type = name_type;
+	state->servers = servers;
+	state->num_servers = num_servers;
+
+	if (state->num_servers == 0) {
+		tevent_req_nterror(req, NT_STATUS_NOT_FOUND);
+		return tevent_req_post(req, ev);
+	}
+
+	in_addr_to_sockaddr_storage(
+		&state->server, state->servers[state->num_sent]);
+
+	subreq = name_query_send(state, state->ev,
+				 state->name, state->name_type,
+				 false, true, &state->server);
+	state->num_sent += 1;
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	if (!tevent_req_set_endtime(subreq, state->ev,
+				    timeval_current_ofs(2, 0))) {
+		tevent_req_oom(req);
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, query_wins_list_done, req);
+	return req;
+}
+
+static void query_wins_list_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct query_wins_list_state *state = tevent_req_data(
+		req, struct query_wins_list_state);
 	NTSTATUS status;
 
-	if (lp_disable_netbios()) {
-		DEBUG(5,("resolve_wins(%s#%02x): netbios is disabled\n",
-					name, name_type));
-		return NT_STATUS_INVALID_PARAMETER;
+	status = name_query_recv(subreq, state,
+				 &state->addrs, &state->num_addrs,
+				 &state->flags);
+	TALLOC_FREE(subreq);
+	if (NT_STATUS_IS_OK(status)) {
+		tevent_req_done(req);
+		return;
+	}
+	if (!NT_STATUS_EQUAL(status, NT_STATUS_IO_TIMEOUT)) {
+		tevent_req_nterror(req, status);
+		return;
 	}
+	wins_srv_died(state->servers[state->num_sent-1],
+		      my_socket_addr_v4());
 
-	*return_iplist = NULL;
-	*return_count = 0;
+	if (state->num_sent == state->num_servers) {
+		tevent_req_nterror(req, NT_STATUS_NOT_FOUND);
+		return;
+	}
 
-	DEBUG(3,("resolve_wins: Attempting wins lookup for name %s<0x%x>\n",
-				name, name_type));
+	in_addr_to_sockaddr_storage(
+		&state->server, state->servers[state->num_sent]);
 
-	if (wins_srv_count() < 1) {
-		DEBUG(3,("resolve_wins: WINS server resolution selected "
-			"and no WINS servers listed.\n"));
-		return NT_STATUS_INVALID_PARAMETER;
+	subreq = name_query_send(state, state->ev,
+				 state->name, state->name_type,
+				 false, true, &state->server);
+	state->num_sent += 1;
+	if (tevent_req_nomem(subreq, req)) {
+		return;
 	}
+	if (!tevent_req_set_endtime(subreq, state->ev,
+				    timeval_current_ofs(2, 0))) {
+		tevent_req_oom(req);
+		return;
+	}
+	tevent_req_set_callback(subreq, query_wins_list_done, req);
+}
 
-	/* we try a lookup on each of the WINS tags in turn */
-	wins_tags = wins_srv_tags();
+static NTSTATUS query_wins_list_recv(struct tevent_req *req,
+				     TALLOC_CTX *mem_ctx,
+				     struct sockaddr_storage **addrs,
+				     int *num_addrs,
+				     uint8_t *flags)
+{
+	struct query_wins_list_state *state = tevent_req_data(
+		req, struct query_wins_list_state);
+	NTSTATUS status;
 
-	if (!wins_tags) {
-		/* huh? no tags?? give up in disgust */
-		return NT_STATUS_INVALID_PARAMETER;
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+	if (addrs != NULL) {
+		*addrs = talloc_move(mem_ctx, &state->addrs);
+	}
+	if (num_addrs != NULL) {
+		*num_addrs = state->num_addrs;
+	}
+	if (flags != NULL) {
+		*flags = state->flags;
+	}
+	return NT_STATUS_OK;
+}
+
+struct resolve_wins_state {
+	int num_sent;
+	int num_received;
+
+	struct sockaddr_storage *addrs;
+	int num_addrs;
+	uint8_t flags;
+};
+
+static void resolve_wins_done(struct tevent_req *subreq);
+
+struct tevent_req *resolve_wins_send(TALLOC_CTX *mem_ctx,
+				     struct tevent_context *ev,
+				     const char *name,
+				     int name_type)
+{
+	struct tevent_req *req, *subreq;
+	struct resolve_wins_state *state;
+	char **wins_tags = NULL;
+	struct sockaddr_storage src_ss;
+	struct in_addr src_ip;
+	int i, num_wins_tags;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct resolve_wins_state);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	if (wins_srv_count() < 1) {
+		DEBUG(3,("resolve_wins: WINS server resolution selected "
+			"and no WINS servers listed.\n"));
+		tevent_req_nterror(req, NT_STATUS_INVALID_PARAMETER);
+		goto fail;
 	}
 
 	/* the address we will be sending from */
@@ -1603,80 +2041,164 @@ NTSTATUS resolve_wins(const char *name,
 		DEBUG(3,("resolve_wins: cannot receive WINS replies "
 			"on IPv6 address %s\n",
 			addr));
-		wins_srv_tags_free(wins_tags);
-		return NT_STATUS_INVALID_PARAMETER;
+		tevent_req_nterror(req, NT_STATUS_INVALID_PARAMETER);
+		goto fail;
 	}
 
-	src_ip = ((struct sockaddr_in *)&src_ss)->sin_addr;
+	src_ip = ((const struct sockaddr_in *)(void *)&src_ss)->sin_addr;
 
-	/* in the worst case we will try every wins server with every
-	   tag! */
-	for (t=0; wins_tags && wins_tags[t]; t++) {
-		int srv_count = wins_srv_count_tag(wins_tags[t]);
-		for (i=0; i<srv_count; i++) {
-			struct sockaddr_storage wins_ss;
-			struct in_addr wins_ip;
+	wins_tags = wins_srv_tags();
+	if (wins_tags == NULL) {
+		tevent_req_nterror(req, NT_STATUS_INVALID_PARAMETER);
+		goto fail;
+	}
+
+	num_wins_tags = 0;
+	while (wins_tags[num_wins_tags] != NULL) {
+		num_wins_tags += 1;
+	}
+
+	for (i=0; i<num_wins_tags; i++) {
+		int num_servers, num_alive;
+		struct in_addr *servers, *alive;
+		int j;
+
+		if (!wins_server_tag_ips(wins_tags[i], talloc_tos(),
+					 &servers, &num_servers)) {
+			DEBUG(10, ("wins_server_tag_ips failed for tag %s\n",
+				   wins_tags[i]));
+			continue;
+		}
+
+		alive = talloc_array(state, struct in_addr, num_servers);
+		if (tevent_req_nomem(alive, req)) {
+			goto fail;
+		}
 
-			wins_ip = wins_srv_ip_tag(wins_tags[t], src_ip);
+		num_alive = 0;
+		for (j=0; j<num_servers; j++) {
+			struct in_addr wins_ip = servers[j];
 
 			if (global_in_nmbd && ismyip_v4(wins_ip)) {
 				/* yikes! we'll loop forever */
 				continue;
 			}
-
 			/* skip any that have been unresponsive lately */
 			if (wins_srv_is_dead(wins_ip, src_ip)) {
 				continue;
 			}
+			DEBUG(3, ("resolve_wins: using WINS server %s "
+				 "and tag '%s'\n",
+				  inet_ntoa(wins_ip), wins_tags[i]));
+			alive[num_alive] = wins_ip;
+			num_alive += 1;
+		}
+		TALLOC_FREE(servers);
 
-			DEBUG(3,("resolve_wins: using WINS server %s "
-				"and tag '%s'\n",
-				inet_ntoa(wins_ip), wins_tags[t]));
+		if (num_alive == 0) {
+			continue;
+		}
 
-			in_addr_to_sockaddr_storage(&wins_ss, wins_ip);
-			status = name_query(name,
-						name_type,
-						false,
-						true,
-						&wins_ss,
-						talloc_tos(),
-						&ss_list,
-						return_count,
-						NULL);
+		subreq = query_wins_list_send(
+			state, ev, src_ip, name, name_type,
+			alive, num_alive);
+		if (tevent_req_nomem(subreq, req)) {
+			goto fail;
+		}
+		tevent_req_set_callback(subreq, resolve_wins_done, req);
+		state->num_sent += 1;
+	}
 
-			/* exit loop if we got a list of addresses */
+	if (state->num_sent == 0) {
+		tevent_req_nterror(req, NT_STATUS_NOT_FOUND);
+		goto fail;
+	}
 
-			if (NT_STATUS_IS_OK(status)) {
-				goto success;
-			}
+	wins_srv_tags_free(wins_tags);
+	return req;
+fail:
+	wins_srv_tags_free(wins_tags);
+	return tevent_req_post(req, ev);
+}
 
-			if (NT_STATUS_EQUAL(status,
-					    NT_STATUS_IO_TIMEOUT)) {
-				/* Timed out waiting for WINS server to
-				 * respond.
-				 * Mark it dead. */
-				wins_srv_died(wins_ip, src_ip);
-			} else {
-				/* The name definitely isn't in this
-				   group of WINS servers.
-				   goto the next group  */
-				break;
-			}
-		}
+static void resolve_wins_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct resolve_wins_state *state = tevent_req_data(
+		req, struct resolve_wins_state);
+	NTSTATUS status;
+
+	status = query_wins_list_recv(subreq, state, &state->addrs,
+				      &state->num_addrs, &state->flags);
+	if (NT_STATUS_IS_OK(status)) {
+		tevent_req_done(req);
+		return;
 	}
 
-	wins_srv_tags_free(wins_tags);
-	return NT_STATUS_NO_LOGON_SERVERS;
+	state->num_received += 1;
 
-success:
+	if (state->num_received < state->num_sent) {
+		/*
+		 * Wait for the others
+		 */
+		return;
+	}
+	tevent_req_nterror(req, status);
+}
 
-	status = NT_STATUS_OK;
-	if (!convert_ss2service(return_iplist, ss_list, *return_count))
-		status = NT_STATUS_INVALID_PARAMETER;
+NTSTATUS resolve_wins_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+			   struct sockaddr_storage **addrs,
+			   int *num_addrs, uint8_t *flags)
+{
+	struct resolve_wins_state *state = tevent_req_data(
+		req, struct resolve_wins_state);
+	NTSTATUS status;
 
-	TALLOC_FREE(ss_list);
-	wins_srv_tags_free(wins_tags);
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+	if (addrs != NULL) {
+		*addrs = talloc_move(mem_ctx, &state->addrs);
+	}
+	if (num_addrs != NULL) {
+		*num_addrs = state->num_addrs;
+	}
+	if (flags != NULL) {
+		*flags = state->flags;
+	}
+	return NT_STATUS_OK;
+}
 
+/********************************************************
+ Resolve via "wins" method.
+*********************************************************/
+
+NTSTATUS resolve_wins(const char *name,
+		int name_type,
+		TALLOC_CTX *mem_ctx,
+		struct sockaddr_storage **return_iplist,
+		int *return_count)
+{
+	struct tevent_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	ev = tevent_context_init(talloc_tos());
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = resolve_wins_send(ev, ev, name, name_type);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
+		goto fail;
+	}
+	status = resolve_wins_recv(req, mem_ctx, return_iplist, return_count,
+				   NULL);
+fail:
+	TALLOC_FREE(ev);
 	return status;
 }
 
@@ -1744,6 +2266,7 @@ static NTSTATUS resolve_hosts(const char *name, int name_type,
 	struct addrinfo *res = NULL;
 	int ret = -1;
 	int i = 0;
+	const char *dns_hosts_file;
 
 	if ( name_type != 0x20 && name_type != 0x0) {
 		DEBUG(5, ("resolve_hosts: not appropriate "
@@ -1768,6 +2291,32 @@ static NTSTATUS resolve_hosts(const char *name, int name_type,
 	hints.ai_family = AF_INET;
 #endif
 
+	dns_hosts_file = lp_parm_const_string(-1, "resolv", "host file", NULL);
+	if (dns_hosts_file) {
+		struct sockaddr_storage *ss_list;
+		NTSTATUS status;
+		TALLOC_CTX *ctx = talloc_stackframe();
+		if (!ctx) {
+			return NT_STATUS_NO_MEMORY;
+		}
+
+		status = resolve_dns_hosts_file_as_sockaddr(dns_hosts_file, name, false,
+							    ctx, &ss_list, return_count);
+		if (NT_STATUS_IS_OK(status)) {
+			if (convert_ss2service(return_iplist,
+					       ss_list,
+					       *return_count)) {
+				talloc_free(ctx);
+				return NT_STATUS_OK;
+			} else {
+				talloc_free(ctx);
+				return NT_STATUS_NO_MEMORY;
+			}
+		}
+		talloc_free(ctx);
+		return NT_STATUS_UNSUCCESSFUL;
+	}
+
 	ret = getaddrinfo(name,
 			NULL,
 			&hints,
@@ -2044,19 +2593,32 @@ NTSTATUS internal_resolve_name(const char *name,
 			}
 		} else if(strequal( tok, "wins")) {
 			/* don't resolve 1D via WINS */
+			struct sockaddr_storage *ss_list;
 			if (name_type != 0x1D) {
 				status = resolve_wins(name, name_type,
-						      return_iplist,
+						      talloc_tos(),
+						      &ss_list,
 						      return_count);
 				if (NT_STATUS_IS_OK(status)) {
+					if (!convert_ss2service(return_iplist,
+								ss_list,
+								*return_count)) {
+						status = NT_STATUS_NO_MEMORY;
+					}
 					goto done;
 				}
 			}
 		} else if(strequal( tok, "bcast")) {
-			status = name_resolve_bcast(name, name_type,
-						    return_iplist,
-						    return_count);
+			struct sockaddr_storage *ss_list;
+			status = name_resolve_bcast(
+				name, name_type, talloc_tos(),
+				&ss_list, return_count);
 			if (NT_STATUS_IS_OK(status)) {
+				if (!convert_ss2service(return_iplist,
+							ss_list,
+							*return_count)) {
+					status = NT_STATUS_NO_MEMORY;
+				}
 				goto done;
 			}
 		} else {
@@ -2152,7 +2714,7 @@ bool resolve_name(const char *name,
 		if (prefer_ipv4) {
 			for (i=0; i<count; i++) {
 				if (!is_zero_addr(&ss_list[i].ss) &&
-						!is_broadcast_addr((struct sockaddr *)&ss_list[i].ss) &&
+				    !is_broadcast_addr((struct sockaddr *)(void *)&ss_list[i].ss) &&
 						(ss_list[i].ss.ss_family == AF_INET)) {
 					*return_ss = ss_list[i].ss;
 					SAFE_FREE(ss_list);
@@ -2165,7 +2727,7 @@ bool resolve_name(const char *name,
 		/* only return valid addresses for TCP connections */
 		for (i=0; i<count; i++) {
 			if (!is_zero_addr(&ss_list[i].ss) &&
-					!is_broadcast_addr((struct sockaddr *)&ss_list[i].ss)) {
+			    !is_broadcast_addr((struct sockaddr *)(void *)&ss_list[i].ss)) {
 				*return_ss = ss_list[i].ss;
 				SAFE_FREE(ss_list);
 				SAFE_FREE(sitename);
@@ -2203,7 +2765,7 @@ NTSTATUS resolve_name_list(TALLOC_CTX *ctx,
 	*return_ss_arr = NULL;
 
 	if (is_ipaddress(name)) {
-		*return_ss_arr = TALLOC_P(ctx, struct sockaddr_storage);
+		*return_ss_arr = talloc(ctx, struct sockaddr_storage);
 		if (!*return_ss_arr) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -2229,7 +2791,7 @@ NTSTATUS resolve_name_list(TALLOC_CTX *ctx,
 	/* only return valid addresses for TCP connections */
 	for (i=0, num_entries = 0; i<count; i++) {
 		if (!is_zero_addr(&ss_list[i].ss) &&
-				!is_broadcast_addr((struct sockaddr *)&ss_list[i].ss)) {
+		    !is_broadcast_addr((struct sockaddr *)(void *)&ss_list[i].ss)) {
 			num_entries++;
 		}
 	}
@@ -2238,7 +2800,7 @@ NTSTATUS resolve_name_list(TALLOC_CTX *ctx,
 		return NT_STATUS_BAD_NETWORK_NAME;
 	}
 
-	*return_ss_arr = TALLOC_ARRAY(ctx,
+	*return_ss_arr = talloc_array(ctx,
 				struct sockaddr_storage,
 				num_entries);
 	if (!(*return_ss_arr)) {
@@ -2248,7 +2810,7 @@ NTSTATUS resolve_name_list(TALLOC_CTX *ctx,
 
 	for (i=0, num_entries = 0; i<count; i++) {
 		if (!is_zero_addr(&ss_list[i].ss) &&
-				!is_broadcast_addr((struct sockaddr *)&ss_list[i].ss)) {
+		    !is_broadcast_addr((struct sockaddr *)(void *)&ss_list[i].ss)) {
 			(*return_ss_arr)[num_entries++] = ss_list[i].ss;
 		}
 	}
diff --git a/source3/libsmb/nmblib.c b/source3/libsmb/nmblib.c
index dbf996ee5e7..767ff81476e 100644
--- a/source3/libsmb/nmblib.c
+++ b/source3/libsmb/nmblib.c
@@ -288,7 +288,7 @@ void put_name(char *dest, const char *name, int pad, unsigned int name_type)
  If buf == NULL this is a length calculation.
 ******************************************************************/
 
-static int put_nmb_name(char *buf,int offset,struct nmb_name *name)
+static int put_nmb_name(char *buf, size_t buflen, int offset,struct nmb_name *name)
 {
 	int ret,m;
 	nstring buf1;
@@ -302,6 +302,9 @@ static int put_nmb_name(char *buf,int offset,struct nmb_name *name)
 	}
 
 	if (buf) {
+		if (offset >= buflen) {
+			return 0;
+		}
 		buf[offset] = 0x20;
 	}
 
@@ -309,6 +312,9 @@ static int put_nmb_name(char *buf,int offset,struct nmb_name *name)
 
 	for (m=0;m<MAX_NETBIOSNAME_LEN;m++) {
 		if (buf) {
+			if (offset+2+2*m >= buflen) {
+				return 0;
+			}
 			buf[offset+1+2*m] = 'A' + ((buf1[m]>>4)&0xF);
 			buf[offset+2+2*m] = 'A' + (buf1[m]&0xF);
 		}
@@ -316,20 +322,30 @@ static int put_nmb_name(char *buf,int offset,struct nmb_name *name)
 	offset += 33;
 
 	if (buf) {
+		if (offset >= buflen) {
+			return 0;
+		}
 		buf[offset] = 0;
 	}
 
 	if (name->scope[0]) {
 		/* XXXX this scope handling needs testing */
-		ret += strlen(name->scope) + 1;
+		size_t scopenamelen = strlen(name->scope) + 1;
+		ret += scopenamelen;
 		if (buf) {
-			safe_strcpy(&buf[offset+1],name->scope,
-					sizeof(name->scope));
+			if (offset+1+scopenamelen >= buflen) {
+				return 0;
+			}
+			strlcpy(&buf[offset+1],name->scope,
+					buflen - (offset+1));
 
 			p = &buf[offset+1];
 			while ((p = strchr_m(p,'.'))) {
 				buf[offset] = PTR_DIFF(p,&buf[offset+1]);
 				offset += (buf[offset] + 1);
+				if (offset+1 >= buflen) {
+					return 0;
+				}
 				p = &buf[offset+1];
 			}
 			buf[offset] = strlen(&buf[offset+1]);
@@ -404,13 +420,13 @@ static bool parse_alloc_res_rec(char *inbuf,int *offset,int length,
  If buf == NULL this is a length calculation.
 ******************************************************************/
 
-static int put_res_rec(char *buf,int offset,struct res_rec *recs,int count)
+static int put_res_rec(char *buf, size_t buflen, int offset,struct res_rec *recs,int count)
 {
 	int ret=0;
 	int i;
 
 	for (i=0;i<count;i++) {
-		int l = put_nmb_name(buf,offset,&recs[i].rr_name);
+		int l = put_nmb_name(buf,buflen,offset,&recs[i].rr_name);
 		offset += l;
 		ret += l;
 		if (buf) {
@@ -887,8 +903,8 @@ static int build_dgram(char *buf, size_t len, struct dgram_packet *dgram)
 	if (dgram->header.msg_type == 0x10 ||
 			dgram->header.msg_type == 0x11 ||
 			dgram->header.msg_type == 0x12) {
-		offset += put_nmb_name((char *)ubuf,offset,&dgram->source_name);
-		offset += put_nmb_name((char *)ubuf,offset,&dgram->dest_name);
+		offset += put_nmb_name((char *)ubuf,len,offset,&dgram->source_name);
+		offset += put_nmb_name((char *)ubuf,len,offset,&dgram->dest_name);
 	}
 
 	if (buf) {
@@ -920,7 +936,7 @@ void make_nmb_name( struct nmb_name *n, const char *name, int type)
 	strupper_m(unix_name);
 	push_ascii(n->name, unix_name, sizeof(n->name), STR_TERMINATE);
 	n->name_type = (unsigned int)type & 0xFF;
-	push_ascii(n->scope,  global_scope(), 64, STR_TERMINATE);
+	push_ascii(n->scope,  lp_netbios_scope(), 64, STR_TERMINATE);
 }
 
 /*******************************************************************
@@ -979,13 +995,13 @@ static int build_nmb(char *buf, size_t len, struct nmb_packet *nmb)
 		/* XXXX this doesn't handle a qdcount of > 1 */
 		if (len) {
 			/* Length check. */
-			int extra = put_nmb_name(NULL,offset,
+			int extra = put_nmb_name(NULL,0,offset,
 					&nmb->question.question_name);
 			if (offset + extra > len) {
 				return 0;
 			}
 		}
-		offset += put_nmb_name((char *)ubuf,offset,
+		offset += put_nmb_name((char *)ubuf,len,offset,
 				&nmb->question.question_name);
 		if (buf) {
 			RSSVAL(ubuf,offset,nmb->question.question_type);
@@ -997,26 +1013,26 @@ static int build_nmb(char *buf, size_t len, struct nmb_packet *nmb)
 	if (nmb->header.ancount) {
 		if (len) {
 			/* Length check. */
-			int extra = put_res_rec(NULL,offset,nmb->answers,
+			int extra = put_res_rec(NULL,0,offset,nmb->answers,
 					nmb->header.ancount);
 			if (offset + extra > len) {
 				return 0;
 			}
 		}
-		offset += put_res_rec((char *)ubuf,offset,nmb->answers,
+		offset += put_res_rec((char *)ubuf,len,offset,nmb->answers,
 				nmb->header.ancount);
 	}
 
 	if (nmb->header.nscount) {
 		if (len) {
 			/* Length check. */
-			int extra = put_res_rec(NULL,offset,nmb->nsrecs,
+			int extra = put_res_rec(NULL,0,offset,nmb->nsrecs,
 				nmb->header.nscount);
 			if (offset + extra > len) {
 				return 0;
 			}
 		}
-		offset += put_res_rec((char *)ubuf,offset,nmb->nsrecs,
+		offset += put_res_rec((char *)ubuf,len,offset,nmb->nsrecs,
 				nmb->header.nscount);
 	}
 
@@ -1048,13 +1064,13 @@ static int build_nmb(char *buf, size_t len, struct nmb_packet *nmb)
 	} else if (nmb->header.arcount) {
 		if (len) {
 			/* Length check. */
-			int extra = put_res_rec(NULL,offset,nmb->additional,
+			int extra = put_res_rec(NULL,0,offset,nmb->additional,
 				nmb->header.arcount);
 			if (offset + extra > len) {
 				return 0;
 			}
 		}
-		offset += put_res_rec((char *)ubuf,offset,nmb->additional,
+		offset += put_res_rec((char *)ubuf,len,offset,nmb->additional,
 			nmb->header.arcount);
 	}
 	return offset;
@@ -1131,7 +1147,7 @@ bool match_mailslot_name(struct packet_struct *p, const char *mailslot_name)
  Return the number of bits that match between two len character buffers
 ***************************************************************************/
 
-int matching_len_bits(unsigned char *p1, unsigned char *p2, size_t len)
+int matching_len_bits(const unsigned char *p1, const unsigned char *p2, size_t len)
 {
 	size_t i, j;
 	int ret = 0;
@@ -1245,7 +1261,7 @@ char *name_mangle(TALLOC_CTX *mem_ctx, const char *In, char name_type)
 	char *result;
 	char *p;
 
-	result = talloc_array(mem_ctx, char, 33 + strlen(global_scope()) + 2);
+	result = talloc_array(mem_ctx, char, 33 + strlen(lp_netbios_scope()) + 2);
 	if (result == NULL) {
 		return NULL;
 	}
@@ -1280,8 +1296,8 @@ char *name_mangle(TALLOC_CTX *mem_ctx, const char *In, char name_type)
 	p[0] = '\0';
 
 	/* Add the scope string. */
-	for( i = 0, len = 0; *(global_scope()) != '\0'; i++, len++ ) {
-		switch( (global_scope())[i] ) {
+	for( i = 0, len = 0; *(lp_netbios_scope()) != '\0'; i++, len++ ) {
+		switch( (lp_netbios_scope())[i] ) {
 			case '\0':
 				p[0] = len;
 				if( len > 0 )
@@ -1293,7 +1309,7 @@ char *name_mangle(TALLOC_CTX *mem_ctx, const char *In, char name_type)
 				len  = -1;
 				break;
 			default:
-				p[len+1] = (global_scope())[i];
+				p[len+1] = (lp_netbios_scope())[i];
 				break;
 		}
 	}
diff --git a/source3/libsmb/nmblib.h b/source3/libsmb/nmblib.h
index a6666a84c61..d8290dc0765 100644
--- a/source3/libsmb/nmblib.h
+++ b/source3/libsmb/nmblib.h
@@ -1,3 +1,28 @@
+/*
+   Unix SMB/CIFS implementation.
+   handle unexpected packets
+   NBT netbios library routines
+   Copyright (C) Andrew Tridgell 1994-1998, 2000
+   Copyright (C) Jeremy Allison 2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef _LIBSMB_NMBLIB_H_
+#define _LIBSMB_NMBLIB_H_
+
 /* The following definitions come from libsmb/unexpected.c  */
 
 #include "nameserv.h"
@@ -42,8 +67,10 @@ bool nmb_name_equal(struct nmb_name *n1, struct nmb_name *n2);
 int build_packet(char *buf, size_t buflen, struct packet_struct *p);
 bool send_packet(struct packet_struct *p);
 bool match_mailslot_name(struct packet_struct *p, const char *mailslot_name);
-int matching_len_bits(unsigned char *p1, unsigned char *p2, size_t len);
+int matching_len_bits(const unsigned char *p1, const unsigned char *p2, size_t len);
 void sort_query_replies(char *data, int n, struct in_addr ip);
 char *name_mangle(TALLOC_CTX *mem_ctx, const char *In, char name_type);
 int name_extract(unsigned char *buf,size_t buf_len, unsigned int ofs, fstring name);
 int name_len(unsigned char *s1, size_t buf_len);
+
+#endif /* _LIBSMB_NMBLIB_H_ */
diff --git a/source3/libsmb/nterr.c b/source3/libsmb/nterr.c
deleted file mode 100644
index 321965878d1..00000000000
--- a/source3/libsmb/nterr.c
+++ /dev/null
@@ -1,802 +0,0 @@
-/*
- *  Unix SMB/CIFS implementation.
- *  RPC Pipe client / server routines
- *  Copyright (C) Luke Kenneth Casson Leighton 1997-2001.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* NT error codes.  please read nterr.h */
-
-#include "includes.h"
-#include "smb_ldap.h"
-#undef strcasecmp
-
-#if !defined(N_)
-#define N_(string) string
-#endif
-
-typedef struct
-{
-	const char *nt_errstr;
-	NTSTATUS nt_errcode;
-} nt_err_code_struct;
-
-static const nt_err_code_struct nt_errs[] =
-{
-	{ "NT_STATUS_OK", NT_STATUS_OK },
-	{ "STATUS_NO_MORE_FILES", STATUS_NO_MORE_FILES },
-	{ "STATUS_NO_MORE_EAS", STATUS_NO_MORE_EAS },
-	{ "STATUS_INVALID_EA_NAME", STATUS_INVALID_EA_NAME },
-	{ "STATUS_EA_LIST_INCONSISTENT", STATUS_EA_LIST_INCONSISTENT },
-	{ "STATUS_INVALID_EA_FLAG", STATUS_INVALID_EA_FLAG },
-	{ "NT_STATUS_UNSUCCESSFUL", NT_STATUS_UNSUCCESSFUL },
-	{ "NT_STATUS_NOT_IMPLEMENTED", NT_STATUS_NOT_IMPLEMENTED },
-	{ "NT_STATUS_INVALID_INFO_CLASS", NT_STATUS_INVALID_INFO_CLASS },
-	{ "NT_STATUS_INFO_LENGTH_MISMATCH", NT_STATUS_INFO_LENGTH_MISMATCH },
-	{ "NT_STATUS_ACCESS_VIOLATION", NT_STATUS_ACCESS_VIOLATION },
-	{ "STATUS_BUFFER_OVERFLOW", STATUS_BUFFER_OVERFLOW },
-	{ "NT_STATUS_IN_PAGE_ERROR", NT_STATUS_IN_PAGE_ERROR },
-	{ "NT_STATUS_PAGEFILE_QUOTA", NT_STATUS_PAGEFILE_QUOTA },
-	{ "NT_STATUS_INVALID_HANDLE", NT_STATUS_INVALID_HANDLE },
-	{ "NT_STATUS_BAD_INITIAL_STACK", NT_STATUS_BAD_INITIAL_STACK },
-	{ "NT_STATUS_BAD_INITIAL_PC", NT_STATUS_BAD_INITIAL_PC },
-	{ "NT_STATUS_INVALID_CID", NT_STATUS_INVALID_CID },
-	{ "NT_STATUS_TIMER_NOT_CANCELED", NT_STATUS_TIMER_NOT_CANCELED },
-	{ "NT_STATUS_INVALID_PARAMETER", NT_STATUS_INVALID_PARAMETER },
-	{ "NT_STATUS_NO_SUCH_DEVICE", NT_STATUS_NO_SUCH_DEVICE },
-	{ "NT_STATUS_NO_SUCH_FILE", NT_STATUS_NO_SUCH_FILE },
-	{ "NT_STATUS_INVALID_DEVICE_REQUEST", NT_STATUS_INVALID_DEVICE_REQUEST },
-	{ "NT_STATUS_END_OF_FILE", NT_STATUS_END_OF_FILE },
-	{ "NT_STATUS_WRONG_VOLUME", NT_STATUS_WRONG_VOLUME },
-	{ "NT_STATUS_NO_MEDIA_IN_DEVICE", NT_STATUS_NO_MEDIA_IN_DEVICE },
-	{ "NT_STATUS_UNRECOGNIZED_MEDIA", NT_STATUS_UNRECOGNIZED_MEDIA },
-	{ "NT_STATUS_NONEXISTENT_SECTOR", NT_STATUS_NONEXISTENT_SECTOR },
-	{ "NT_STATUS_MORE_PROCESSING_REQUIRED", NT_STATUS_MORE_PROCESSING_REQUIRED },
-	{ "NT_STATUS_NO_MEMORY", NT_STATUS_NO_MEMORY },
-	{ "NT_STATUS_CONFLICTING_ADDRESSES", NT_STATUS_CONFLICTING_ADDRESSES },
-	{ "NT_STATUS_NOT_MAPPED_VIEW", NT_STATUS_NOT_MAPPED_VIEW },
-	{ "NT_STATUS_UNABLE_TO_FREE_VM", NT_STATUS_UNABLE_TO_FREE_VM },
-	{ "NT_STATUS_UNABLE_TO_DELETE_SECTION", NT_STATUS_UNABLE_TO_DELETE_SECTION },
-	{ "NT_STATUS_INVALID_SYSTEM_SERVICE", NT_STATUS_INVALID_SYSTEM_SERVICE },
-	{ "NT_STATUS_ILLEGAL_INSTRUCTION", NT_STATUS_ILLEGAL_INSTRUCTION },
-	{ "NT_STATUS_INVALID_LOCK_SEQUENCE", NT_STATUS_INVALID_LOCK_SEQUENCE },
-	{ "NT_STATUS_INVALID_VIEW_SIZE", NT_STATUS_INVALID_VIEW_SIZE },
-	{ "NT_STATUS_INVALID_FILE_FOR_SECTION", NT_STATUS_INVALID_FILE_FOR_SECTION },
-	{ "NT_STATUS_ALREADY_COMMITTED", NT_STATUS_ALREADY_COMMITTED },
-	{ "NT_STATUS_ACCESS_DENIED", NT_STATUS_ACCESS_DENIED },
-	{ "NT_STATUS_BUFFER_TOO_SMALL", NT_STATUS_BUFFER_TOO_SMALL },
-	{ "NT_STATUS_OBJECT_TYPE_MISMATCH", NT_STATUS_OBJECT_TYPE_MISMATCH },
-	{ "NT_STATUS_NONCONTINUABLE_EXCEPTION", NT_STATUS_NONCONTINUABLE_EXCEPTION },
-	{ "NT_STATUS_INVALID_DISPOSITION", NT_STATUS_INVALID_DISPOSITION },
-	{ "NT_STATUS_UNWIND", NT_STATUS_UNWIND },
-	{ "NT_STATUS_BAD_STACK", NT_STATUS_BAD_STACK },
-	{ "NT_STATUS_INVALID_UNWIND_TARGET", NT_STATUS_INVALID_UNWIND_TARGET },
-	{ "NT_STATUS_NOT_LOCKED", NT_STATUS_NOT_LOCKED },
-	{ "NT_STATUS_PARITY_ERROR", NT_STATUS_PARITY_ERROR },
-	{ "NT_STATUS_UNABLE_TO_DECOMMIT_VM", NT_STATUS_UNABLE_TO_DECOMMIT_VM },
-	{ "NT_STATUS_NOT_COMMITTED", NT_STATUS_NOT_COMMITTED },
-	{ "NT_STATUS_INVALID_PORT_ATTRIBUTES", NT_STATUS_INVALID_PORT_ATTRIBUTES },
-	{ "NT_STATUS_PORT_MESSAGE_TOO_LONG", NT_STATUS_PORT_MESSAGE_TOO_LONG },
-	{ "NT_STATUS_INVALID_PARAMETER_MIX", NT_STATUS_INVALID_PARAMETER_MIX },
-	{ "NT_STATUS_INVALID_QUOTA_LOWER", NT_STATUS_INVALID_QUOTA_LOWER },
-	{ "NT_STATUS_DISK_CORRUPT_ERROR", NT_STATUS_DISK_CORRUPT_ERROR },
-	{ "NT_STATUS_OBJECT_NAME_INVALID", NT_STATUS_OBJECT_NAME_INVALID },
-	{ "NT_STATUS_OBJECT_NAME_NOT_FOUND", NT_STATUS_OBJECT_NAME_NOT_FOUND },
-	{ "NT_STATUS_OBJECT_NAME_COLLISION", NT_STATUS_OBJECT_NAME_COLLISION },
-	{ "NT_STATUS_HANDLE_NOT_WAITABLE", NT_STATUS_HANDLE_NOT_WAITABLE },
-	{ "NT_STATUS_PORT_DISCONNECTED", NT_STATUS_PORT_DISCONNECTED },
-	{ "NT_STATUS_DEVICE_ALREADY_ATTACHED", NT_STATUS_DEVICE_ALREADY_ATTACHED },
-	{ "NT_STATUS_OBJECT_PATH_INVALID", NT_STATUS_OBJECT_PATH_INVALID },
-	{ "NT_STATUS_OBJECT_PATH_NOT_FOUND", NT_STATUS_OBJECT_PATH_NOT_FOUND },
-	{ "NT_STATUS_OBJECT_PATH_SYNTAX_BAD", NT_STATUS_OBJECT_PATH_SYNTAX_BAD },
-	{ "NT_STATUS_DATA_OVERRUN", NT_STATUS_DATA_OVERRUN },
-	{ "NT_STATUS_DATA_LATE_ERROR", NT_STATUS_DATA_LATE_ERROR },
-	{ "NT_STATUS_DATA_ERROR", NT_STATUS_DATA_ERROR },
-	{ "NT_STATUS_CRC_ERROR", NT_STATUS_CRC_ERROR },
-	{ "NT_STATUS_SECTION_TOO_BIG", NT_STATUS_SECTION_TOO_BIG },
-	{ "NT_STATUS_PORT_CONNECTION_REFUSED", NT_STATUS_PORT_CONNECTION_REFUSED },
-	{ "NT_STATUS_INVALID_PORT_HANDLE", NT_STATUS_INVALID_PORT_HANDLE },
-	{ "NT_STATUS_SHARING_VIOLATION", NT_STATUS_SHARING_VIOLATION },
-	{ "NT_STATUS_QUOTA_EXCEEDED", NT_STATUS_QUOTA_EXCEEDED },
-	{ "NT_STATUS_INVALID_PAGE_PROTECTION", NT_STATUS_INVALID_PAGE_PROTECTION },
-	{ "NT_STATUS_MUTANT_NOT_OWNED", NT_STATUS_MUTANT_NOT_OWNED },
-	{ "NT_STATUS_SEMAPHORE_LIMIT_EXCEEDED", NT_STATUS_SEMAPHORE_LIMIT_EXCEEDED },
-	{ "NT_STATUS_PORT_ALREADY_SET", NT_STATUS_PORT_ALREADY_SET },
-	{ "NT_STATUS_SECTION_NOT_IMAGE", NT_STATUS_SECTION_NOT_IMAGE },
-	{ "NT_STATUS_SUSPEND_COUNT_EXCEEDED", NT_STATUS_SUSPEND_COUNT_EXCEEDED },
-	{ "NT_STATUS_THREAD_IS_TERMINATING", NT_STATUS_THREAD_IS_TERMINATING },
-	{ "NT_STATUS_BAD_WORKING_SET_LIMIT", NT_STATUS_BAD_WORKING_SET_LIMIT },
-	{ "NT_STATUS_INCOMPATIBLE_FILE_MAP", NT_STATUS_INCOMPATIBLE_FILE_MAP },
-	{ "NT_STATUS_SECTION_PROTECTION", NT_STATUS_SECTION_PROTECTION },
-	{ "NT_STATUS_EAS_NOT_SUPPORTED", NT_STATUS_EAS_NOT_SUPPORTED },
-	{ "NT_STATUS_EA_TOO_LARGE", NT_STATUS_EA_TOO_LARGE },
-	{ "NT_STATUS_NONEXISTENT_EA_ENTRY", NT_STATUS_NONEXISTENT_EA_ENTRY },
-	{ "NT_STATUS_NO_EAS_ON_FILE", NT_STATUS_NO_EAS_ON_FILE },
-	{ "NT_STATUS_EA_CORRUPT_ERROR", NT_STATUS_EA_CORRUPT_ERROR },
-	{ "NT_STATUS_FILE_LOCK_CONFLICT", NT_STATUS_FILE_LOCK_CONFLICT },
-	{ "NT_STATUS_LOCK_NOT_GRANTED", NT_STATUS_LOCK_NOT_GRANTED },
-	{ "NT_STATUS_DELETE_PENDING", NT_STATUS_DELETE_PENDING },
-	{ "NT_STATUS_CTL_FILE_NOT_SUPPORTED", NT_STATUS_CTL_FILE_NOT_SUPPORTED },
-	{ "NT_STATUS_UNKNOWN_REVISION", NT_STATUS_UNKNOWN_REVISION },
-	{ "NT_STATUS_REVISION_MISMATCH", NT_STATUS_REVISION_MISMATCH },
-	{ "NT_STATUS_INVALID_OWNER", NT_STATUS_INVALID_OWNER },
-	{ "NT_STATUS_INVALID_PRIMARY_GROUP", NT_STATUS_INVALID_PRIMARY_GROUP },
-	{ "NT_STATUS_NO_IMPERSONATION_TOKEN", NT_STATUS_NO_IMPERSONATION_TOKEN },
-	{ "NT_STATUS_CANT_DISABLE_MANDATORY", NT_STATUS_CANT_DISABLE_MANDATORY },
-	{ "NT_STATUS_NO_LOGON_SERVERS", NT_STATUS_NO_LOGON_SERVERS },
-	{ "NT_STATUS_NO_SUCH_LOGON_SESSION", NT_STATUS_NO_SUCH_LOGON_SESSION },
-	{ "NT_STATUS_NO_SUCH_PRIVILEGE", NT_STATUS_NO_SUCH_PRIVILEGE },
-	{ "NT_STATUS_PRIVILEGE_NOT_HELD", NT_STATUS_PRIVILEGE_NOT_HELD },
-	{ "NT_STATUS_INVALID_ACCOUNT_NAME", NT_STATUS_INVALID_ACCOUNT_NAME },
-	{ "NT_STATUS_USER_EXISTS", NT_STATUS_USER_EXISTS },
-	{ "NT_STATUS_NO_SUCH_USER", NT_STATUS_NO_SUCH_USER },
-	{ "NT_STATUS_GROUP_EXISTS", NT_STATUS_GROUP_EXISTS },
-	{ "NT_STATUS_NO_SUCH_GROUP", NT_STATUS_NO_SUCH_GROUP },
-	{ "NT_STATUS_MEMBER_IN_GROUP", NT_STATUS_MEMBER_IN_GROUP },
-	{ "NT_STATUS_MEMBER_NOT_IN_GROUP", NT_STATUS_MEMBER_NOT_IN_GROUP },
-	{ "NT_STATUS_LAST_ADMIN", NT_STATUS_LAST_ADMIN },
-	{ "NT_STATUS_WRONG_PASSWORD", NT_STATUS_WRONG_PASSWORD },
-	{ "NT_STATUS_ILL_FORMED_PASSWORD", NT_STATUS_ILL_FORMED_PASSWORD },
-	{ "NT_STATUS_PASSWORD_RESTRICTION", NT_STATUS_PASSWORD_RESTRICTION },
-	{ "NT_STATUS_LOGON_FAILURE", NT_STATUS_LOGON_FAILURE },
-	{ "NT_STATUS_ACCOUNT_RESTRICTION", NT_STATUS_ACCOUNT_RESTRICTION },
-	{ "NT_STATUS_INVALID_LOGON_HOURS", NT_STATUS_INVALID_LOGON_HOURS },
-	{ "NT_STATUS_INVALID_WORKSTATION", NT_STATUS_INVALID_WORKSTATION },
-	{ "NT_STATUS_PASSWORD_EXPIRED", NT_STATUS_PASSWORD_EXPIRED },
-	{ "NT_STATUS_ACCOUNT_DISABLED", NT_STATUS_ACCOUNT_DISABLED },
-	{ "NT_STATUS_NONE_MAPPED", NT_STATUS_NONE_MAPPED },
-	{ "NT_STATUS_TOO_MANY_LUIDS_REQUESTED", NT_STATUS_TOO_MANY_LUIDS_REQUESTED },
-	{ "NT_STATUS_LUIDS_EXHAUSTED", NT_STATUS_LUIDS_EXHAUSTED },
-	{ "NT_STATUS_INVALID_SUB_AUTHORITY", NT_STATUS_INVALID_SUB_AUTHORITY },
-	{ "NT_STATUS_INVALID_ACL", NT_STATUS_INVALID_ACL },
-	{ "NT_STATUS_INVALID_SID", NT_STATUS_INVALID_SID },
-	{ "NT_STATUS_INVALID_SECURITY_DESCR", NT_STATUS_INVALID_SECURITY_DESCR },
-	{ "NT_STATUS_PROCEDURE_NOT_FOUND", NT_STATUS_PROCEDURE_NOT_FOUND },
-	{ "NT_STATUS_INVALID_IMAGE_FORMAT", NT_STATUS_INVALID_IMAGE_FORMAT },
-	{ "NT_STATUS_NO_TOKEN", NT_STATUS_NO_TOKEN },
-	{ "NT_STATUS_BAD_INHERITANCE_ACL", NT_STATUS_BAD_INHERITANCE_ACL },
-	{ "NT_STATUS_RANGE_NOT_LOCKED", NT_STATUS_RANGE_NOT_LOCKED },
-	{ "NT_STATUS_DISK_FULL", NT_STATUS_DISK_FULL },
-	{ "NT_STATUS_SERVER_DISABLED", NT_STATUS_SERVER_DISABLED },
-	{ "NT_STATUS_SERVER_NOT_DISABLED", NT_STATUS_SERVER_NOT_DISABLED },
-	{ "NT_STATUS_TOO_MANY_GUIDS_REQUESTED", NT_STATUS_TOO_MANY_GUIDS_REQUESTED },
-	{ "NT_STATUS_GUIDS_EXHAUSTED", NT_STATUS_GUIDS_EXHAUSTED },
-	{ "NT_STATUS_INVALID_ID_AUTHORITY", NT_STATUS_INVALID_ID_AUTHORITY },
-	{ "NT_STATUS_AGENTS_EXHAUSTED", NT_STATUS_AGENTS_EXHAUSTED },
-	{ "NT_STATUS_INVALID_VOLUME_LABEL", NT_STATUS_INVALID_VOLUME_LABEL },
-	{ "NT_STATUS_SECTION_NOT_EXTENDED", NT_STATUS_SECTION_NOT_EXTENDED },
-	{ "NT_STATUS_NOT_MAPPED_DATA", NT_STATUS_NOT_MAPPED_DATA },
-	{ "NT_STATUS_RESOURCE_DATA_NOT_FOUND", NT_STATUS_RESOURCE_DATA_NOT_FOUND },
-	{ "NT_STATUS_RESOURCE_TYPE_NOT_FOUND", NT_STATUS_RESOURCE_TYPE_NOT_FOUND },
-	{ "NT_STATUS_RESOURCE_NAME_NOT_FOUND", NT_STATUS_RESOURCE_NAME_NOT_FOUND },
-	{ "NT_STATUS_ARRAY_BOUNDS_EXCEEDED", NT_STATUS_ARRAY_BOUNDS_EXCEEDED },
-	{ "NT_STATUS_FLOAT_DENORMAL_OPERAND", NT_STATUS_FLOAT_DENORMAL_OPERAND },
-	{ "NT_STATUS_FLOAT_DIVIDE_BY_ZERO", NT_STATUS_FLOAT_DIVIDE_BY_ZERO },
-	{ "NT_STATUS_FLOAT_INEXACT_RESULT", NT_STATUS_FLOAT_INEXACT_RESULT },
-	{ "NT_STATUS_FLOAT_INVALID_OPERATION", NT_STATUS_FLOAT_INVALID_OPERATION },
-	{ "NT_STATUS_FLOAT_OVERFLOW", NT_STATUS_FLOAT_OVERFLOW },
-	{ "NT_STATUS_FLOAT_STACK_CHECK", NT_STATUS_FLOAT_STACK_CHECK },
-	{ "NT_STATUS_FLOAT_UNDERFLOW", NT_STATUS_FLOAT_UNDERFLOW },
-	{ "NT_STATUS_INTEGER_DIVIDE_BY_ZERO", NT_STATUS_INTEGER_DIVIDE_BY_ZERO },
-	{ "NT_STATUS_INTEGER_OVERFLOW", NT_STATUS_INTEGER_OVERFLOW },
-	{ "NT_STATUS_PRIVILEGED_INSTRUCTION", NT_STATUS_PRIVILEGED_INSTRUCTION },
-	{ "NT_STATUS_TOO_MANY_PAGING_FILES", NT_STATUS_TOO_MANY_PAGING_FILES },
-	{ "NT_STATUS_FILE_INVALID", NT_STATUS_FILE_INVALID },
-	{ "NT_STATUS_ALLOTTED_SPACE_EXCEEDED", NT_STATUS_ALLOTTED_SPACE_EXCEEDED },
-	{ "NT_STATUS_INSUFFICIENT_RESOURCES", NT_STATUS_INSUFFICIENT_RESOURCES },
-	{ "NT_STATUS_DFS_EXIT_PATH_FOUND", NT_STATUS_DFS_EXIT_PATH_FOUND },
-	{ "NT_STATUS_DEVICE_DATA_ERROR", NT_STATUS_DEVICE_DATA_ERROR },
-	{ "NT_STATUS_DEVICE_NOT_CONNECTED", NT_STATUS_DEVICE_NOT_CONNECTED },
-	{ "NT_STATUS_DEVICE_POWER_FAILURE", NT_STATUS_DEVICE_POWER_FAILURE },
-	{ "NT_STATUS_FREE_VM_NOT_AT_BASE", NT_STATUS_FREE_VM_NOT_AT_BASE },
-	{ "NT_STATUS_MEMORY_NOT_ALLOCATED", NT_STATUS_MEMORY_NOT_ALLOCATED },
-	{ "NT_STATUS_WORKING_SET_QUOTA", NT_STATUS_WORKING_SET_QUOTA },
-	{ "NT_STATUS_MEDIA_WRITE_PROTECTED", NT_STATUS_MEDIA_WRITE_PROTECTED },
-	{ "NT_STATUS_DEVICE_NOT_READY", NT_STATUS_DEVICE_NOT_READY },
-	{ "NT_STATUS_INVALID_GROUP_ATTRIBUTES", NT_STATUS_INVALID_GROUP_ATTRIBUTES },
-	{ "NT_STATUS_BAD_IMPERSONATION_LEVEL", NT_STATUS_BAD_IMPERSONATION_LEVEL },
-	{ "NT_STATUS_CANT_OPEN_ANONYMOUS", NT_STATUS_CANT_OPEN_ANONYMOUS },
-	{ "NT_STATUS_BAD_VALIDATION_CLASS", NT_STATUS_BAD_VALIDATION_CLASS },
-	{ "NT_STATUS_BAD_TOKEN_TYPE", NT_STATUS_BAD_TOKEN_TYPE },
-	{ "NT_STATUS_BAD_MASTER_BOOT_RECORD", NT_STATUS_BAD_MASTER_BOOT_RECORD },
-	{ "NT_STATUS_INSTRUCTION_MISALIGNMENT", NT_STATUS_INSTRUCTION_MISALIGNMENT },
-	{ "NT_STATUS_INSTANCE_NOT_AVAILABLE", NT_STATUS_INSTANCE_NOT_AVAILABLE },
-	{ "NT_STATUS_PIPE_NOT_AVAILABLE", NT_STATUS_PIPE_NOT_AVAILABLE },
-	{ "NT_STATUS_INVALID_PIPE_STATE", NT_STATUS_INVALID_PIPE_STATE },
-	{ "NT_STATUS_PIPE_BUSY", NT_STATUS_PIPE_BUSY },
-	{ "NT_STATUS_ILLEGAL_FUNCTION", NT_STATUS_ILLEGAL_FUNCTION },
-	{ "NT_STATUS_PIPE_DISCONNECTED", NT_STATUS_PIPE_DISCONNECTED },
-	{ "NT_STATUS_PIPE_CLOSING", NT_STATUS_PIPE_CLOSING },
-	{ "NT_STATUS_PIPE_CONNECTED", NT_STATUS_PIPE_CONNECTED },
-	{ "NT_STATUS_PIPE_LISTENING", NT_STATUS_PIPE_LISTENING },
-	{ "NT_STATUS_INVALID_READ_MODE", NT_STATUS_INVALID_READ_MODE },
-	{ "NT_STATUS_IO_TIMEOUT", NT_STATUS_IO_TIMEOUT },
-	{ "NT_STATUS_FILE_FORCED_CLOSED", NT_STATUS_FILE_FORCED_CLOSED },
-	{ "NT_STATUS_PROFILING_NOT_STARTED", NT_STATUS_PROFILING_NOT_STARTED },
-	{ "NT_STATUS_PROFILING_NOT_STOPPED", NT_STATUS_PROFILING_NOT_STOPPED },
-	{ "NT_STATUS_COULD_NOT_INTERPRET", NT_STATUS_COULD_NOT_INTERPRET },
-	{ "NT_STATUS_FILE_IS_A_DIRECTORY", NT_STATUS_FILE_IS_A_DIRECTORY },
-	{ "NT_STATUS_NOT_SUPPORTED", NT_STATUS_NOT_SUPPORTED },
-	{ "NT_STATUS_REMOTE_NOT_LISTENING", NT_STATUS_REMOTE_NOT_LISTENING },
-	{ "NT_STATUS_DUPLICATE_NAME", NT_STATUS_DUPLICATE_NAME },
-	{ "NT_STATUS_BAD_NETWORK_PATH", NT_STATUS_BAD_NETWORK_PATH },
-	{ "NT_STATUS_NETWORK_BUSY", NT_STATUS_NETWORK_BUSY },
-	{ "NT_STATUS_DEVICE_DOES_NOT_EXIST", NT_STATUS_DEVICE_DOES_NOT_EXIST },
-	{ "NT_STATUS_TOO_MANY_COMMANDS", NT_STATUS_TOO_MANY_COMMANDS },
-	{ "NT_STATUS_ADAPTER_HARDWARE_ERROR", NT_STATUS_ADAPTER_HARDWARE_ERROR },
-	{ "NT_STATUS_INVALID_NETWORK_RESPONSE", NT_STATUS_INVALID_NETWORK_RESPONSE },
-	{ "NT_STATUS_UNEXPECTED_NETWORK_ERROR", NT_STATUS_UNEXPECTED_NETWORK_ERROR },
-	{ "NT_STATUS_BAD_REMOTE_ADAPTER", NT_STATUS_BAD_REMOTE_ADAPTER },
-	{ "NT_STATUS_PRINT_QUEUE_FULL", NT_STATUS_PRINT_QUEUE_FULL },
-	{ "NT_STATUS_NO_SPOOL_SPACE", NT_STATUS_NO_SPOOL_SPACE },
-	{ "NT_STATUS_PRINT_CANCELLED", NT_STATUS_PRINT_CANCELLED },
-	{ "NT_STATUS_NETWORK_NAME_DELETED", NT_STATUS_NETWORK_NAME_DELETED },
-	{ "NT_STATUS_NETWORK_ACCESS_DENIED", NT_STATUS_NETWORK_ACCESS_DENIED },
-	{ "NT_STATUS_BAD_DEVICE_TYPE", NT_STATUS_BAD_DEVICE_TYPE },
-	{ "NT_STATUS_BAD_NETWORK_NAME", NT_STATUS_BAD_NETWORK_NAME },
-	{ "NT_STATUS_TOO_MANY_NAMES", NT_STATUS_TOO_MANY_NAMES },
-	{ "NT_STATUS_TOO_MANY_SESSIONS", NT_STATUS_TOO_MANY_SESSIONS },
-	{ "NT_STATUS_SHARING_PAUSED", NT_STATUS_SHARING_PAUSED },
-	{ "NT_STATUS_REQUEST_NOT_ACCEPTED", NT_STATUS_REQUEST_NOT_ACCEPTED },
-	{ "NT_STATUS_REDIRECTOR_PAUSED", NT_STATUS_REDIRECTOR_PAUSED },
-	{ "NT_STATUS_NET_WRITE_FAULT", NT_STATUS_NET_WRITE_FAULT },
-	{ "NT_STATUS_PROFILING_AT_LIMIT", NT_STATUS_PROFILING_AT_LIMIT },
-	{ "NT_STATUS_NOT_SAME_DEVICE", NT_STATUS_NOT_SAME_DEVICE },
-	{ "NT_STATUS_FILE_RENAMED", NT_STATUS_FILE_RENAMED },
-	{ "NT_STATUS_VIRTUAL_CIRCUIT_CLOSED", NT_STATUS_VIRTUAL_CIRCUIT_CLOSED },
-	{ "NT_STATUS_NO_SECURITY_ON_OBJECT", NT_STATUS_NO_SECURITY_ON_OBJECT },
-	{ "NT_STATUS_CANT_WAIT", NT_STATUS_CANT_WAIT },
-	{ "NT_STATUS_PIPE_EMPTY", NT_STATUS_PIPE_EMPTY },
-	{ "NT_STATUS_CANT_ACCESS_DOMAIN_INFO", NT_STATUS_CANT_ACCESS_DOMAIN_INFO },
-	{ "NT_STATUS_CANT_TERMINATE_SELF", NT_STATUS_CANT_TERMINATE_SELF },
-	{ "NT_STATUS_INVALID_SERVER_STATE", NT_STATUS_INVALID_SERVER_STATE },
-	{ "NT_STATUS_INVALID_DOMAIN_STATE", NT_STATUS_INVALID_DOMAIN_STATE },
-	{ "NT_STATUS_INVALID_DOMAIN_ROLE", NT_STATUS_INVALID_DOMAIN_ROLE },
-	{ "NT_STATUS_NO_SUCH_DOMAIN", NT_STATUS_NO_SUCH_DOMAIN },
-	{ "NT_STATUS_DOMAIN_EXISTS", NT_STATUS_DOMAIN_EXISTS },
-	{ "NT_STATUS_DOMAIN_LIMIT_EXCEEDED", NT_STATUS_DOMAIN_LIMIT_EXCEEDED },
-	{ "NT_STATUS_OPLOCK_NOT_GRANTED", NT_STATUS_OPLOCK_NOT_GRANTED },
-	{ "NT_STATUS_INVALID_OPLOCK_PROTOCOL", NT_STATUS_INVALID_OPLOCK_PROTOCOL },
-	{ "NT_STATUS_INTERNAL_DB_CORRUPTION", NT_STATUS_INTERNAL_DB_CORRUPTION },
-	{ "NT_STATUS_INTERNAL_ERROR", NT_STATUS_INTERNAL_ERROR },
-	{ "NT_STATUS_GENERIC_NOT_MAPPED", NT_STATUS_GENERIC_NOT_MAPPED },
-	{ "NT_STATUS_BAD_DESCRIPTOR_FORMAT", NT_STATUS_BAD_DESCRIPTOR_FORMAT },
-	{ "NT_STATUS_INVALID_USER_BUFFER", NT_STATUS_INVALID_USER_BUFFER },
-	{ "NT_STATUS_UNEXPECTED_IO_ERROR", NT_STATUS_UNEXPECTED_IO_ERROR },
-	{ "NT_STATUS_UNEXPECTED_MM_CREATE_ERR", NT_STATUS_UNEXPECTED_MM_CREATE_ERR },
-	{ "NT_STATUS_UNEXPECTED_MM_MAP_ERROR", NT_STATUS_UNEXPECTED_MM_MAP_ERROR },
-	{ "NT_STATUS_UNEXPECTED_MM_EXTEND_ERR", NT_STATUS_UNEXPECTED_MM_EXTEND_ERR },
-	{ "NT_STATUS_NOT_LOGON_PROCESS", NT_STATUS_NOT_LOGON_PROCESS },
-	{ "NT_STATUS_LOGON_SESSION_EXISTS", NT_STATUS_LOGON_SESSION_EXISTS },
-	{ "NT_STATUS_INVALID_PARAMETER_1", NT_STATUS_INVALID_PARAMETER_1 },
-	{ "NT_STATUS_INVALID_PARAMETER_2", NT_STATUS_INVALID_PARAMETER_2 },
-	{ "NT_STATUS_INVALID_PARAMETER_3", NT_STATUS_INVALID_PARAMETER_3 },
-	{ "NT_STATUS_INVALID_PARAMETER_4", NT_STATUS_INVALID_PARAMETER_4 },
-	{ "NT_STATUS_INVALID_PARAMETER_5", NT_STATUS_INVALID_PARAMETER_5 },
-	{ "NT_STATUS_INVALID_PARAMETER_6", NT_STATUS_INVALID_PARAMETER_6 },
-	{ "NT_STATUS_INVALID_PARAMETER_7", NT_STATUS_INVALID_PARAMETER_7 },
-	{ "NT_STATUS_INVALID_PARAMETER_8", NT_STATUS_INVALID_PARAMETER_8 },
-	{ "NT_STATUS_INVALID_PARAMETER_9", NT_STATUS_INVALID_PARAMETER_9 },
-	{ "NT_STATUS_INVALID_PARAMETER_10", NT_STATUS_INVALID_PARAMETER_10 },
-	{ "NT_STATUS_INVALID_PARAMETER_11", NT_STATUS_INVALID_PARAMETER_11 },
-	{ "NT_STATUS_INVALID_PARAMETER_12", NT_STATUS_INVALID_PARAMETER_12 },
-	{ "NT_STATUS_REDIRECTOR_NOT_STARTED", NT_STATUS_REDIRECTOR_NOT_STARTED },
-	{ "NT_STATUS_REDIRECTOR_STARTED", NT_STATUS_REDIRECTOR_STARTED },
-	{ "NT_STATUS_STACK_OVERFLOW", NT_STATUS_STACK_OVERFLOW },
-	{ "NT_STATUS_NO_SUCH_PACKAGE", NT_STATUS_NO_SUCH_PACKAGE },
-	{ "NT_STATUS_BAD_FUNCTION_TABLE", NT_STATUS_BAD_FUNCTION_TABLE },
-	{ "NT_STATUS_DIRECTORY_NOT_EMPTY", NT_STATUS_DIRECTORY_NOT_EMPTY },
-	{ "NT_STATUS_FILE_CORRUPT_ERROR", NT_STATUS_FILE_CORRUPT_ERROR },
-	{ "NT_STATUS_NOT_A_DIRECTORY", NT_STATUS_NOT_A_DIRECTORY },
-	{ "NT_STATUS_BAD_LOGON_SESSION_STATE", NT_STATUS_BAD_LOGON_SESSION_STATE },
-	{ "NT_STATUS_LOGON_SESSION_COLLISION", NT_STATUS_LOGON_SESSION_COLLISION },
-	{ "NT_STATUS_NAME_TOO_LONG", NT_STATUS_NAME_TOO_LONG },
-	{ "NT_STATUS_FILES_OPEN", NT_STATUS_FILES_OPEN },
-	{ "NT_STATUS_CONNECTION_IN_USE", NT_STATUS_CONNECTION_IN_USE },
-	{ "NT_STATUS_MESSAGE_NOT_FOUND", NT_STATUS_MESSAGE_NOT_FOUND },
-	{ "NT_STATUS_PROCESS_IS_TERMINATING", NT_STATUS_PROCESS_IS_TERMINATING },
-	{ "NT_STATUS_INVALID_LOGON_TYPE", NT_STATUS_INVALID_LOGON_TYPE },
-	{ "NT_STATUS_NO_GUID_TRANSLATION", NT_STATUS_NO_GUID_TRANSLATION },
-	{ "NT_STATUS_CANNOT_IMPERSONATE", NT_STATUS_CANNOT_IMPERSONATE },
-	{ "NT_STATUS_IMAGE_ALREADY_LOADED", NT_STATUS_IMAGE_ALREADY_LOADED },
-	{ "NT_STATUS_ABIOS_NOT_PRESENT", NT_STATUS_ABIOS_NOT_PRESENT },
-	{ "NT_STATUS_ABIOS_LID_NOT_EXIST", NT_STATUS_ABIOS_LID_NOT_EXIST },
-	{ "NT_STATUS_ABIOS_LID_ALREADY_OWNED", NT_STATUS_ABIOS_LID_ALREADY_OWNED },
-	{ "NT_STATUS_ABIOS_NOT_LID_OWNER", NT_STATUS_ABIOS_NOT_LID_OWNER },
-	{ "NT_STATUS_ABIOS_INVALID_COMMAND", NT_STATUS_ABIOS_INVALID_COMMAND },
-	{ "NT_STATUS_ABIOS_INVALID_LID", NT_STATUS_ABIOS_INVALID_LID },
-	{ "NT_STATUS_ABIOS_SELECTOR_NOT_AVAILABLE", NT_STATUS_ABIOS_SELECTOR_NOT_AVAILABLE },
-	{ "NT_STATUS_ABIOS_INVALID_SELECTOR", NT_STATUS_ABIOS_INVALID_SELECTOR },
-	{ "NT_STATUS_NO_LDT", NT_STATUS_NO_LDT },
-	{ "NT_STATUS_INVALID_LDT_SIZE", NT_STATUS_INVALID_LDT_SIZE },
-	{ "NT_STATUS_INVALID_LDT_OFFSET", NT_STATUS_INVALID_LDT_OFFSET },
-	{ "NT_STATUS_INVALID_LDT_DESCRIPTOR", NT_STATUS_INVALID_LDT_DESCRIPTOR },
-	{ "NT_STATUS_INVALID_IMAGE_NE_FORMAT", NT_STATUS_INVALID_IMAGE_NE_FORMAT },
-	{ "NT_STATUS_RXACT_INVALID_STATE", NT_STATUS_RXACT_INVALID_STATE },
-	{ "NT_STATUS_RXACT_COMMIT_FAILURE", NT_STATUS_RXACT_COMMIT_FAILURE },
-	{ "NT_STATUS_MAPPED_FILE_SIZE_ZERO", NT_STATUS_MAPPED_FILE_SIZE_ZERO },
-	{ "NT_STATUS_TOO_MANY_OPENED_FILES", NT_STATUS_TOO_MANY_OPENED_FILES },
-	{ "NT_STATUS_CANCELLED", NT_STATUS_CANCELLED },
-	{ "NT_STATUS_CANNOT_DELETE", NT_STATUS_CANNOT_DELETE },
-	{ "NT_STATUS_INVALID_COMPUTER_NAME", NT_STATUS_INVALID_COMPUTER_NAME },
-	{ "NT_STATUS_FILE_DELETED", NT_STATUS_FILE_DELETED },
-	{ "NT_STATUS_SPECIAL_ACCOUNT", NT_STATUS_SPECIAL_ACCOUNT },
-	{ "NT_STATUS_SPECIAL_GROUP", NT_STATUS_SPECIAL_GROUP },
-	{ "NT_STATUS_SPECIAL_USER", NT_STATUS_SPECIAL_USER },
-	{ "NT_STATUS_MEMBERS_PRIMARY_GROUP", NT_STATUS_MEMBERS_PRIMARY_GROUP },
-	{ "NT_STATUS_FILE_CLOSED", NT_STATUS_FILE_CLOSED },
-	{ "NT_STATUS_TOO_MANY_THREADS", NT_STATUS_TOO_MANY_THREADS },
-	{ "NT_STATUS_THREAD_NOT_IN_PROCESS", NT_STATUS_THREAD_NOT_IN_PROCESS },
-	{ "NT_STATUS_TOKEN_ALREADY_IN_USE", NT_STATUS_TOKEN_ALREADY_IN_USE },
-	{ "NT_STATUS_PAGEFILE_QUOTA_EXCEEDED", NT_STATUS_PAGEFILE_QUOTA_EXCEEDED },
-	{ "NT_STATUS_COMMITMENT_LIMIT", NT_STATUS_COMMITMENT_LIMIT },
-	{ "NT_STATUS_INVALID_IMAGE_LE_FORMAT", NT_STATUS_INVALID_IMAGE_LE_FORMAT },
-	{ "NT_STATUS_INVALID_IMAGE_NOT_MZ", NT_STATUS_INVALID_IMAGE_NOT_MZ },
-	{ "NT_STATUS_INVALID_IMAGE_PROTECT", NT_STATUS_INVALID_IMAGE_PROTECT },
-	{ "NT_STATUS_INVALID_IMAGE_WIN_16", NT_STATUS_INVALID_IMAGE_WIN_16 },
-	{ "NT_STATUS_LOGON_SERVER_CONFLICT", NT_STATUS_LOGON_SERVER_CONFLICT },
-	{ "NT_STATUS_TIME_DIFFERENCE_AT_DC", NT_STATUS_TIME_DIFFERENCE_AT_DC },
-	{ "NT_STATUS_SYNCHRONIZATION_REQUIRED", NT_STATUS_SYNCHRONIZATION_REQUIRED },
-	{ "NT_STATUS_DLL_NOT_FOUND", NT_STATUS_DLL_NOT_FOUND },
-	{ "NT_STATUS_OPEN_FAILED", NT_STATUS_OPEN_FAILED },
-	{ "NT_STATUS_IO_PRIVILEGE_FAILED", NT_STATUS_IO_PRIVILEGE_FAILED },
-	{ "NT_STATUS_ORDINAL_NOT_FOUND", NT_STATUS_ORDINAL_NOT_FOUND },
-	{ "NT_STATUS_ENTRYPOINT_NOT_FOUND", NT_STATUS_ENTRYPOINT_NOT_FOUND },
-	{ "NT_STATUS_CONTROL_C_EXIT", NT_STATUS_CONTROL_C_EXIT },
-	{ "NT_STATUS_LOCAL_DISCONNECT", NT_STATUS_LOCAL_DISCONNECT },
-	{ "NT_STATUS_REMOTE_DISCONNECT", NT_STATUS_REMOTE_DISCONNECT },
-	{ "NT_STATUS_REMOTE_RESOURCES", NT_STATUS_REMOTE_RESOURCES },
-	{ "NT_STATUS_LINK_FAILED", NT_STATUS_LINK_FAILED },
-	{ "NT_STATUS_LINK_TIMEOUT", NT_STATUS_LINK_TIMEOUT },
-	{ "NT_STATUS_INVALID_CONNECTION", NT_STATUS_INVALID_CONNECTION },
-	{ "NT_STATUS_INVALID_ADDRESS", NT_STATUS_INVALID_ADDRESS },
-	{ "NT_STATUS_DLL_INIT_FAILED", NT_STATUS_DLL_INIT_FAILED },
-	{ "NT_STATUS_MISSING_SYSTEMFILE", NT_STATUS_MISSING_SYSTEMFILE },
-	{ "NT_STATUS_UNHANDLED_EXCEPTION", NT_STATUS_UNHANDLED_EXCEPTION },
-	{ "NT_STATUS_APP_INIT_FAILURE", NT_STATUS_APP_INIT_FAILURE },
-	{ "NT_STATUS_PAGEFILE_CREATE_FAILED", NT_STATUS_PAGEFILE_CREATE_FAILED },
-	{ "NT_STATUS_NO_PAGEFILE", NT_STATUS_NO_PAGEFILE },
-	{ "NT_STATUS_INVALID_LEVEL", NT_STATUS_INVALID_LEVEL },
-	{ "NT_STATUS_WRONG_PASSWORD_CORE", NT_STATUS_WRONG_PASSWORD_CORE },
-	{ "NT_STATUS_ILLEGAL_FLOAT_CONTEXT", NT_STATUS_ILLEGAL_FLOAT_CONTEXT },
-	{ "NT_STATUS_PIPE_BROKEN", NT_STATUS_PIPE_BROKEN },
-	{ "NT_STATUS_REGISTRY_CORRUPT", NT_STATUS_REGISTRY_CORRUPT },
-	{ "NT_STATUS_REGISTRY_IO_FAILED", NT_STATUS_REGISTRY_IO_FAILED },
-	{ "NT_STATUS_NO_EVENT_PAIR", NT_STATUS_NO_EVENT_PAIR },
-	{ "NT_STATUS_UNRECOGNIZED_VOLUME", NT_STATUS_UNRECOGNIZED_VOLUME },
-	{ "NT_STATUS_SERIAL_NO_DEVICE_INITED", NT_STATUS_SERIAL_NO_DEVICE_INITED },
-	{ "NT_STATUS_NO_SUCH_ALIAS", NT_STATUS_NO_SUCH_ALIAS },
-	{ "NT_STATUS_MEMBER_NOT_IN_ALIAS", NT_STATUS_MEMBER_NOT_IN_ALIAS },
-	{ "NT_STATUS_MEMBER_IN_ALIAS", NT_STATUS_MEMBER_IN_ALIAS },
-	{ "NT_STATUS_ALIAS_EXISTS", NT_STATUS_ALIAS_EXISTS },
-	{ "NT_STATUS_LOGON_NOT_GRANTED", NT_STATUS_LOGON_NOT_GRANTED },
-	{ "NT_STATUS_TOO_MANY_SECRETS", NT_STATUS_TOO_MANY_SECRETS },
-	{ "NT_STATUS_SECRET_TOO_LONG", NT_STATUS_SECRET_TOO_LONG },
-	{ "NT_STATUS_INTERNAL_DB_ERROR", NT_STATUS_INTERNAL_DB_ERROR },
-	{ "NT_STATUS_FULLSCREEN_MODE", NT_STATUS_FULLSCREEN_MODE },
-	{ "NT_STATUS_TOO_MANY_CONTEXT_IDS", NT_STATUS_TOO_MANY_CONTEXT_IDS },
-	{ "NT_STATUS_LOGON_TYPE_NOT_GRANTED", NT_STATUS_LOGON_TYPE_NOT_GRANTED },
-	{ "NT_STATUS_NOT_REGISTRY_FILE", NT_STATUS_NOT_REGISTRY_FILE },
-	{ "NT_STATUS_NT_CROSS_ENCRYPTION_REQUIRED", NT_STATUS_NT_CROSS_ENCRYPTION_REQUIRED },
-	{ "NT_STATUS_DOMAIN_CTRLR_CONFIG_ERROR", NT_STATUS_DOMAIN_CTRLR_CONFIG_ERROR },
-	{ "NT_STATUS_FT_MISSING_MEMBER", NT_STATUS_FT_MISSING_MEMBER },
-	{ "NT_STATUS_ILL_FORMED_SERVICE_ENTRY", NT_STATUS_ILL_FORMED_SERVICE_ENTRY },
-	{ "NT_STATUS_ILLEGAL_CHARACTER", NT_STATUS_ILLEGAL_CHARACTER },
-	{ "NT_STATUS_UNMAPPABLE_CHARACTER", NT_STATUS_UNMAPPABLE_CHARACTER },
-	{ "NT_STATUS_UNDEFINED_CHARACTER", NT_STATUS_UNDEFINED_CHARACTER },
-	{ "NT_STATUS_FLOPPY_VOLUME", NT_STATUS_FLOPPY_VOLUME },
-	{ "NT_STATUS_FLOPPY_ID_MARK_NOT_FOUND", NT_STATUS_FLOPPY_ID_MARK_NOT_FOUND },
-	{ "NT_STATUS_FLOPPY_WRONG_CYLINDER", NT_STATUS_FLOPPY_WRONG_CYLINDER },
-	{ "NT_STATUS_FLOPPY_UNKNOWN_ERROR", NT_STATUS_FLOPPY_UNKNOWN_ERROR },
-	{ "NT_STATUS_FLOPPY_BAD_REGISTERS", NT_STATUS_FLOPPY_BAD_REGISTERS },
-	{ "NT_STATUS_DISK_RECALIBRATE_FAILED", NT_STATUS_DISK_RECALIBRATE_FAILED },
-	{ "NT_STATUS_DISK_OPERATION_FAILED", NT_STATUS_DISK_OPERATION_FAILED },
-	{ "NT_STATUS_DISK_RESET_FAILED", NT_STATUS_DISK_RESET_FAILED },
-	{ "NT_STATUS_SHARED_IRQ_BUSY", NT_STATUS_SHARED_IRQ_BUSY },
-	{ "NT_STATUS_FT_ORPHANING", NT_STATUS_FT_ORPHANING },
-	{ "NT_STATUS_PARTITION_FAILURE", NT_STATUS_PARTITION_FAILURE },
-	{ "NT_STATUS_INVALID_BLOCK_LENGTH", NT_STATUS_INVALID_BLOCK_LENGTH },
-	{ "NT_STATUS_DEVICE_NOT_PARTITIONED", NT_STATUS_DEVICE_NOT_PARTITIONED },
-	{ "NT_STATUS_UNABLE_TO_LOCK_MEDIA", NT_STATUS_UNABLE_TO_LOCK_MEDIA },
-	{ "NT_STATUS_UNABLE_TO_UNLOAD_MEDIA", NT_STATUS_UNABLE_TO_UNLOAD_MEDIA },
-	{ "NT_STATUS_EOM_OVERFLOW", NT_STATUS_EOM_OVERFLOW },
-	{ "NT_STATUS_NO_MEDIA", NT_STATUS_NO_MEDIA },
-	{ "NT_STATUS_NO_SUCH_MEMBER", NT_STATUS_NO_SUCH_MEMBER },
-	{ "NT_STATUS_INVALID_MEMBER", NT_STATUS_INVALID_MEMBER },
-	{ "NT_STATUS_KEY_DELETED", NT_STATUS_KEY_DELETED },
-	{ "NT_STATUS_NO_LOG_SPACE", NT_STATUS_NO_LOG_SPACE },
-	{ "NT_STATUS_TOO_MANY_SIDS", NT_STATUS_TOO_MANY_SIDS },
-	{ "NT_STATUS_LM_CROSS_ENCRYPTION_REQUIRED", NT_STATUS_LM_CROSS_ENCRYPTION_REQUIRED },
-	{ "NT_STATUS_KEY_HAS_CHILDREN", NT_STATUS_KEY_HAS_CHILDREN },
-	{ "NT_STATUS_CHILD_MUST_BE_VOLATILE", NT_STATUS_CHILD_MUST_BE_VOLATILE },
-	{ "NT_STATUS_DEVICE_CONFIGURATION_ERROR", NT_STATUS_DEVICE_CONFIGURATION_ERROR },
-	{ "NT_STATUS_DRIVER_INTERNAL_ERROR", NT_STATUS_DRIVER_INTERNAL_ERROR },
-	{ "NT_STATUS_INVALID_DEVICE_STATE", NT_STATUS_INVALID_DEVICE_STATE },
-	{ "NT_STATUS_IO_DEVICE_ERROR", NT_STATUS_IO_DEVICE_ERROR },
-	{ "NT_STATUS_DEVICE_PROTOCOL_ERROR", NT_STATUS_DEVICE_PROTOCOL_ERROR },
-	{ "NT_STATUS_BACKUP_CONTROLLER", NT_STATUS_BACKUP_CONTROLLER },
-	{ "NT_STATUS_LOG_FILE_FULL", NT_STATUS_LOG_FILE_FULL },
-	{ "NT_STATUS_TOO_LATE", NT_STATUS_TOO_LATE },
-	{ "NT_STATUS_NO_TRUST_LSA_SECRET", NT_STATUS_NO_TRUST_LSA_SECRET },
-	{ "NT_STATUS_NO_TRUST_SAM_ACCOUNT", NT_STATUS_NO_TRUST_SAM_ACCOUNT },
-	{ "NT_STATUS_TRUSTED_DOMAIN_FAILURE", NT_STATUS_TRUSTED_DOMAIN_FAILURE },
-	{ "NT_STATUS_TRUSTED_RELATIONSHIP_FAILURE", NT_STATUS_TRUSTED_RELATIONSHIP_FAILURE },
-	{ "NT_STATUS_EVENTLOG_FILE_CORRUPT", NT_STATUS_EVENTLOG_FILE_CORRUPT },
-	{ "NT_STATUS_EVENTLOG_CANT_START", NT_STATUS_EVENTLOG_CANT_START },
-	{ "NT_STATUS_TRUST_FAILURE", NT_STATUS_TRUST_FAILURE },
-	{ "NT_STATUS_MUTANT_LIMIT_EXCEEDED", NT_STATUS_MUTANT_LIMIT_EXCEEDED },
-	{ "NT_STATUS_NETLOGON_NOT_STARTED", NT_STATUS_NETLOGON_NOT_STARTED },
-	{ "NT_STATUS_ACCOUNT_EXPIRED", NT_STATUS_ACCOUNT_EXPIRED },
-	{ "NT_STATUS_POSSIBLE_DEADLOCK", NT_STATUS_POSSIBLE_DEADLOCK },
-	{ "NT_STATUS_NETWORK_CREDENTIAL_CONFLICT", NT_STATUS_NETWORK_CREDENTIAL_CONFLICT },
-	{ "NT_STATUS_REMOTE_SESSION_LIMIT", NT_STATUS_REMOTE_SESSION_LIMIT },
-	{ "NT_STATUS_EVENTLOG_FILE_CHANGED", NT_STATUS_EVENTLOG_FILE_CHANGED },
-	{ "NT_STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT", NT_STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT },
-	{ "NT_STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT", NT_STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT },
-	{ "NT_STATUS_NOLOGON_SERVER_TRUST_ACCOUNT", NT_STATUS_NOLOGON_SERVER_TRUST_ACCOUNT },
-	{ "NT_STATUS_DOMAIN_TRUST_INCONSISTENT", NT_STATUS_DOMAIN_TRUST_INCONSISTENT },
-	{ "NT_STATUS_FS_DRIVER_REQUIRED", NT_STATUS_FS_DRIVER_REQUIRED },
-	{ "NT_STATUS_NO_USER_SESSION_KEY", NT_STATUS_NO_USER_SESSION_KEY },
-	{ "NT_STATUS_USER_SESSION_DELETED", NT_STATUS_USER_SESSION_DELETED },
-	{ "NT_STATUS_RESOURCE_LANG_NOT_FOUND", NT_STATUS_RESOURCE_LANG_NOT_FOUND },
-	{ "NT_STATUS_INSUFF_SERVER_RESOURCES", NT_STATUS_INSUFF_SERVER_RESOURCES },
-	{ "NT_STATUS_INVALID_BUFFER_SIZE", NT_STATUS_INVALID_BUFFER_SIZE },
-	{ "NT_STATUS_INVALID_ADDRESS_COMPONENT", NT_STATUS_INVALID_ADDRESS_COMPONENT },
-	{ "NT_STATUS_INVALID_ADDRESS_WILDCARD", NT_STATUS_INVALID_ADDRESS_WILDCARD },
-	{ "NT_STATUS_TOO_MANY_ADDRESSES", NT_STATUS_TOO_MANY_ADDRESSES },
-	{ "NT_STATUS_ADDRESS_ALREADY_EXISTS", NT_STATUS_ADDRESS_ALREADY_EXISTS },
-	{ "NT_STATUS_ADDRESS_CLOSED", NT_STATUS_ADDRESS_CLOSED },
-	{ "NT_STATUS_CONNECTION_DISCONNECTED", NT_STATUS_CONNECTION_DISCONNECTED },
-	{ "NT_STATUS_CONNECTION_RESET", NT_STATUS_CONNECTION_RESET },
-	{ "NT_STATUS_TOO_MANY_NODES", NT_STATUS_TOO_MANY_NODES },
-	{ "NT_STATUS_TRANSACTION_ABORTED", NT_STATUS_TRANSACTION_ABORTED },
-	{ "NT_STATUS_TRANSACTION_TIMED_OUT", NT_STATUS_TRANSACTION_TIMED_OUT },
-	{ "NT_STATUS_TRANSACTION_NO_RELEASE", NT_STATUS_TRANSACTION_NO_RELEASE },
-	{ "NT_STATUS_TRANSACTION_NO_MATCH", NT_STATUS_TRANSACTION_NO_MATCH },
-	{ "NT_STATUS_TRANSACTION_RESPONDED", NT_STATUS_TRANSACTION_RESPONDED },
-	{ "NT_STATUS_TRANSACTION_INVALID_ID", NT_STATUS_TRANSACTION_INVALID_ID },
-	{ "NT_STATUS_TRANSACTION_INVALID_TYPE", NT_STATUS_TRANSACTION_INVALID_TYPE },
-	{ "NT_STATUS_NOT_SERVER_SESSION", NT_STATUS_NOT_SERVER_SESSION },
-	{ "NT_STATUS_NOT_CLIENT_SESSION", NT_STATUS_NOT_CLIENT_SESSION },
-	{ "NT_STATUS_CANNOT_LOAD_REGISTRY_FILE", NT_STATUS_CANNOT_LOAD_REGISTRY_FILE },
-	{ "NT_STATUS_DEBUG_ATTACH_FAILED", NT_STATUS_DEBUG_ATTACH_FAILED },
-	{ "NT_STATUS_SYSTEM_PROCESS_TERMINATED", NT_STATUS_SYSTEM_PROCESS_TERMINATED },
-	{ "NT_STATUS_DATA_NOT_ACCEPTED", NT_STATUS_DATA_NOT_ACCEPTED },
-	{ "NT_STATUS_NO_BROWSER_SERVERS_FOUND", NT_STATUS_NO_BROWSER_SERVERS_FOUND },
-	{ "NT_STATUS_VDM_HARD_ERROR", NT_STATUS_VDM_HARD_ERROR },
-	{ "NT_STATUS_DRIVER_CANCEL_TIMEOUT", NT_STATUS_DRIVER_CANCEL_TIMEOUT },
-	{ "NT_STATUS_REPLY_MESSAGE_MISMATCH", NT_STATUS_REPLY_MESSAGE_MISMATCH },
-	{ "NT_STATUS_MAPPED_ALIGNMENT", NT_STATUS_MAPPED_ALIGNMENT },
-	{ "NT_STATUS_IMAGE_CHECKSUM_MISMATCH", NT_STATUS_IMAGE_CHECKSUM_MISMATCH },
-	{ "NT_STATUS_LOST_WRITEBEHIND_DATA", NT_STATUS_LOST_WRITEBEHIND_DATA },
-	{ "NT_STATUS_CLIENT_SERVER_PARAMETERS_INVALID", NT_STATUS_CLIENT_SERVER_PARAMETERS_INVALID },
-	{ "NT_STATUS_PASSWORD_MUST_CHANGE", NT_STATUS_PASSWORD_MUST_CHANGE },
-	{ "NT_STATUS_NOT_FOUND", NT_STATUS_NOT_FOUND },
-	{ "NT_STATUS_NOT_TINY_STREAM", NT_STATUS_NOT_TINY_STREAM },
-	{ "NT_STATUS_RECOVERY_FAILURE", NT_STATUS_RECOVERY_FAILURE },
-	{ "NT_STATUS_STACK_OVERFLOW_READ", NT_STATUS_STACK_OVERFLOW_READ },
-	{ "NT_STATUS_FAIL_CHECK", NT_STATUS_FAIL_CHECK },
-	{ "NT_STATUS_DUPLICATE_OBJECTID", NT_STATUS_DUPLICATE_OBJECTID },
-	{ "NT_STATUS_OBJECTID_EXISTS", NT_STATUS_OBJECTID_EXISTS },
-	{ "NT_STATUS_CONVERT_TO_LARGE", NT_STATUS_CONVERT_TO_LARGE },
-	{ "NT_STATUS_RETRY", NT_STATUS_RETRY },
-	{ "NT_STATUS_FOUND_OUT_OF_SCOPE", NT_STATUS_FOUND_OUT_OF_SCOPE },
-	{ "NT_STATUS_ALLOCATE_BUCKET", NT_STATUS_ALLOCATE_BUCKET },
-	{ "NT_STATUS_PROPSET_NOT_FOUND", NT_STATUS_PROPSET_NOT_FOUND },
-	{ "NT_STATUS_MARSHALL_OVERFLOW", NT_STATUS_MARSHALL_OVERFLOW },
-	{ "NT_STATUS_INVALID_VARIANT", NT_STATUS_INVALID_VARIANT },
-	{ "NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND", NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND },
-	{ "NT_STATUS_ACCOUNT_LOCKED_OUT", NT_STATUS_ACCOUNT_LOCKED_OUT },
-	{ "NT_STATUS_HANDLE_NOT_CLOSABLE", NT_STATUS_HANDLE_NOT_CLOSABLE },
-	{ "NT_STATUS_CONNECTION_REFUSED", NT_STATUS_CONNECTION_REFUSED },
-	{ "NT_STATUS_GRACEFUL_DISCONNECT", NT_STATUS_GRACEFUL_DISCONNECT },
-	{ "NT_STATUS_ADDRESS_ALREADY_ASSOCIATED", NT_STATUS_ADDRESS_ALREADY_ASSOCIATED },
-	{ "NT_STATUS_ADDRESS_NOT_ASSOCIATED", NT_STATUS_ADDRESS_NOT_ASSOCIATED },
-	{ "NT_STATUS_CONNECTION_INVALID", NT_STATUS_CONNECTION_INVALID },
-	{ "NT_STATUS_CONNECTION_ACTIVE", NT_STATUS_CONNECTION_ACTIVE },
-	{ "NT_STATUS_NETWORK_UNREACHABLE", NT_STATUS_NETWORK_UNREACHABLE },
-	{ "NT_STATUS_HOST_UNREACHABLE", NT_STATUS_HOST_UNREACHABLE },
-	{ "NT_STATUS_PROTOCOL_UNREACHABLE", NT_STATUS_PROTOCOL_UNREACHABLE },
-	{ "NT_STATUS_PORT_UNREACHABLE", NT_STATUS_PORT_UNREACHABLE },
-	{ "NT_STATUS_REQUEST_ABORTED", NT_STATUS_REQUEST_ABORTED },
-	{ "NT_STATUS_CONNECTION_ABORTED", NT_STATUS_CONNECTION_ABORTED },
-	{ "NT_STATUS_BAD_COMPRESSION_BUFFER", NT_STATUS_BAD_COMPRESSION_BUFFER },
-	{ "NT_STATUS_USER_MAPPED_FILE", NT_STATUS_USER_MAPPED_FILE },
-	{ "NT_STATUS_AUDIT_FAILED", NT_STATUS_AUDIT_FAILED },
-	{ "NT_STATUS_TIMER_RESOLUTION_NOT_SET", NT_STATUS_TIMER_RESOLUTION_NOT_SET },
-	{ "NT_STATUS_CONNECTION_COUNT_LIMIT", NT_STATUS_CONNECTION_COUNT_LIMIT },
-	{ "NT_STATUS_LOGIN_TIME_RESTRICTION", NT_STATUS_LOGIN_TIME_RESTRICTION },
-	{ "NT_STATUS_LOGIN_WKSTA_RESTRICTION", NT_STATUS_LOGIN_WKSTA_RESTRICTION },
-	{ "NT_STATUS_IMAGE_MP_UP_MISMATCH", NT_STATUS_IMAGE_MP_UP_MISMATCH },
-	{ "NT_STATUS_INSUFFICIENT_LOGON_INFO", NT_STATUS_INSUFFICIENT_LOGON_INFO },
-	{ "NT_STATUS_BAD_DLL_ENTRYPOINT", NT_STATUS_BAD_DLL_ENTRYPOINT },
-	{ "NT_STATUS_BAD_SERVICE_ENTRYPOINT", NT_STATUS_BAD_SERVICE_ENTRYPOINT },
-	{ "NT_STATUS_LPC_REPLY_LOST", NT_STATUS_LPC_REPLY_LOST },
-	{ "NT_STATUS_IP_ADDRESS_CONFLICT1", NT_STATUS_IP_ADDRESS_CONFLICT1 },
-	{ "NT_STATUS_IP_ADDRESS_CONFLICT2", NT_STATUS_IP_ADDRESS_CONFLICT2 },
-	{ "NT_STATUS_REGISTRY_QUOTA_LIMIT", NT_STATUS_REGISTRY_QUOTA_LIMIT },
-	{ "NT_STATUS_PATH_NOT_COVERED", NT_STATUS_PATH_NOT_COVERED },
-	{ "NT_STATUS_NO_CALLBACK_ACTIVE", NT_STATUS_NO_CALLBACK_ACTIVE },
-	{ "NT_STATUS_LICENSE_QUOTA_EXCEEDED", NT_STATUS_LICENSE_QUOTA_EXCEEDED },
-	{ "NT_STATUS_PWD_TOO_SHORT", NT_STATUS_PWD_TOO_SHORT },
-	{ "NT_STATUS_PWD_TOO_RECENT", NT_STATUS_PWD_TOO_RECENT },
-	{ "NT_STATUS_PWD_HISTORY_CONFLICT", NT_STATUS_PWD_HISTORY_CONFLICT },
-	{ "NT_STATUS_PLUGPLAY_NO_DEVICE", NT_STATUS_PLUGPLAY_NO_DEVICE },
-	{ "NT_STATUS_UNSUPPORTED_COMPRESSION", NT_STATUS_UNSUPPORTED_COMPRESSION },
-	{ "NT_STATUS_INVALID_HW_PROFILE", NT_STATUS_INVALID_HW_PROFILE },
-	{ "NT_STATUS_INVALID_PLUGPLAY_DEVICE_PATH", NT_STATUS_INVALID_PLUGPLAY_DEVICE_PATH },
-	{ "NT_STATUS_DRIVER_ORDINAL_NOT_FOUND", NT_STATUS_DRIVER_ORDINAL_NOT_FOUND },
-	{ "NT_STATUS_DRIVER_ENTRYPOINT_NOT_FOUND", NT_STATUS_DRIVER_ENTRYPOINT_NOT_FOUND },
-	{ "NT_STATUS_RESOURCE_NOT_OWNED", NT_STATUS_RESOURCE_NOT_OWNED },
-	{ "NT_STATUS_TOO_MANY_LINKS", NT_STATUS_TOO_MANY_LINKS },
-	{ "NT_STATUS_QUOTA_LIST_INCONSISTENT", NT_STATUS_QUOTA_LIST_INCONSISTENT },
-	{ "NT_STATUS_FILE_IS_OFFLINE", NT_STATUS_FILE_IS_OFFLINE },
-	{ "NT_STATUS_DS_NO_MORE_RIDS", NT_STATUS_DS_NO_MORE_RIDS },
-	{ "NT_STATUS_NOT_A_REPARSE_POINT", NT_STATUS_NOT_A_REPARSE_POINT },
-	{ "NT_STATUS_NO_MORE_ENTRIES", NT_STATUS_NO_MORE_ENTRIES },
-	{ "NT_STATUS_RPC_PROTSEQ_NOT_SUPPORTED", NT_STATUS_RPC_PROTSEQ_NOT_SUPPORTED },
-	{ "NT_STATUS_RPC_UNSUPPORTED_NAME_SYNTAX", NT_STATUS_RPC_UNSUPPORTED_NAME_SYNTAX },
-	{ "NT_STATUS_RPC_UNKNOWN_IF", NT_STATUS_RPC_UNKNOWN_IF },
-	{ "NT_STATUS_RPC_CALL_FAILED", NT_STATUS_RPC_CALL_FAILED },
-	{ "NT_STATUS_RPC_PROTOCOL_ERROR", NT_STATUS_RPC_PROTOCOL_ERROR },
-	{ "NT_STATUS_RPC_PROCNUM_OUT_OF_RANGE", NT_STATUS_RPC_PROCNUM_OUT_OF_RANGE },
-	{ "NT_STATUS_RPC_CANNOT_SUPPORT", NT_STATUS_RPC_CANNOT_SUPPORT },
-	{ "NT_STATUS_RPC_SEC_PKG_ERROR", NT_STATUS_RPC_SEC_PKG_ERROR },
-	{ "NT_STATUS_RPC_SS_CONTEXT_MISMATCH", NT_STATUS_RPC_SS_CONTEXT_MISMATCH },
-	{ "NT_STATUS_RPC_ENUM_VALUE_OUT_OF_RANGE", NT_STATUS_RPC_ENUM_VALUE_OUT_OF_RANGE },
-	{ "NT_STATUS_RPC_BAD_STUB_DATA", NT_STATUS_RPC_BAD_STUB_DATA },
-	{ "NT_STATUS_RPC_INVALID_PIPE_OBJECT", NT_STATUS_RPC_INVALID_PIPE_OBJECT },
-	{ "NT_STATUS_RPC_INVALID_PIPE_OPERATION", NT_STATUS_RPC_INVALID_PIPE_OPERATION },
-	{ "NT_STATUS_RPC_WRONG_PIPE_VERSION", NT_STATUS_RPC_WRONG_PIPE_VERSION },
-	{ "NT_STATUS_RPC_PIPE_CLOSED", NT_STATUS_RPC_PIPE_CLOSED },
-	{ "NT_STATUS_RPC_PIPE_DISCIPLINE_ERROR", NT_STATUS_RPC_PIPE_DISCIPLINE_ERROR },
-	{ "NT_STATUS_RPC_PIPE_EMPTY", NT_STATUS_RPC_PIPE_EMPTY },
-	{ "NT_STATUS_CURRENT_DOMAIN_NOT_ALLOWED", NT_STATUS_CURRENT_DOMAIN_NOT_ALLOWED },
-	{ "NT_STATUS_OBJECTID_NOT_FOUND", NT_STATUS_OBJECTID_NOT_FOUND },
-	{ "NT_STATUS_DOWNGRADE_DETECTED", NT_STATUS_DOWNGRADE_DETECTED },
-	{ "NT_STATUS_INVALID_LOCK_RANGE", NT_STATUS_INVALID_LOCK_RANGE },
-	{ "NT_STATUS_ERROR_DS_OBJ_STRING_NAME_EXISTS", NT_STATUS_ERROR_DS_OBJ_STRING_NAME_EXISTS },
-	{ "NT_STATUS_ERROR_DS_INCOMPATIBLE_VERSION", NT_STATUS_ERROR_DS_INCOMPATIBLE_VERSION },
-	{ "STATUS_MORE_ENTRIES", STATUS_MORE_ENTRIES },
-	{ "STATUS_SOME_UNMAPPED", STATUS_SOME_UNMAPPED },
-	{ "STATUS_NOTIFY_CLEANUP", STATUS_NOTIFY_CLEANUP },
-	{ "STATUS_NOTIFY_ENUM_DIR", STATUS_NOTIFY_ENUM_DIR },
-
-	{ NULL, NT_STATUS(0) }
-};
-
-/* These need sorting..... */
-
-static const nt_err_code_struct nt_err_desc[] =
-{
-	{ N_("Success"), 				NT_STATUS_OK },
-	{ N_("Undetermined error"), 		NT_STATUS_UNSUCCESSFUL },
-	{ N_("Access denied"), 			NT_STATUS_ACCESS_DENIED },
-	{ N_("Account locked out"), 		NT_STATUS_ACCOUNT_LOCKED_OUT },
-	{ N_("Must change password"), 		NT_STATUS_PASSWORD_MUST_CHANGE },
-	{ N_("Password is too short"), 		NT_STATUS_PWD_TOO_SHORT },
-	{ N_("Password is too recent"), 		NT_STATUS_PWD_TOO_RECENT },
-	{ N_("Password history conflict"), 		NT_STATUS_PWD_HISTORY_CONFLICT },
-	{ N_("No logon servers"), 			NT_STATUS_NO_LOGON_SERVERS },
-	{ N_("Improperly formed account name"), 	NT_STATUS_INVALID_ACCOUNT_NAME },
-	{ N_("User exists"), 			NT_STATUS_USER_EXISTS },
-	{ N_("No such user"), 			NT_STATUS_NO_SUCH_USER },
-	{ N_("Group exists"), 			NT_STATUS_GROUP_EXISTS },
-	{ N_("No such group"), 			NT_STATUS_NO_SUCH_GROUP },
-	{ N_("Member not in group"), 		NT_STATUS_MEMBER_NOT_IN_GROUP },
-	{ N_("Wrong Password"), 			NT_STATUS_WRONG_PASSWORD },
-	{ N_("Ill formed password"), 		NT_STATUS_ILL_FORMED_PASSWORD },
-	{ N_("Password restriction"), 		NT_STATUS_PASSWORD_RESTRICTION },
-	{ N_("Logon failure"), 			NT_STATUS_LOGON_FAILURE },
-	{ N_("Account restriction"), 		NT_STATUS_ACCOUNT_RESTRICTION },
-	{ N_("Invalid logon hours"), 		NT_STATUS_INVALID_LOGON_HOURS },
-	{ N_("Invalid workstation"), 		NT_STATUS_INVALID_WORKSTATION },
-	{ N_("Password expired"), 			NT_STATUS_PASSWORD_EXPIRED },
-	{ N_("Account disabled"), 			NT_STATUS_ACCOUNT_DISABLED },
-	{ N_("Unexpected information received"), 	NT_STATUS_INVALID_PARAMETER },
-	{ N_("Memory allocation error"), 		NT_STATUS_NO_MEMORY },
-	{ N_("No domain controllers located"), 	NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND },
-	{ N_("Account locked out"), 		NT_STATUS_ACCOUNT_LOCKED_OUT },
-	{ N_("Named pipe not available"), 		NT_STATUS_PIPE_NOT_AVAILABLE },
-	{ N_("Not implemented"), 			NT_STATUS_NOT_IMPLEMENTED },
-	{ N_("Invalid information class"), 		NT_STATUS_INVALID_INFO_CLASS },
-	{ N_("Information length mismatch"), 	NT_STATUS_INFO_LENGTH_MISMATCH },
-	{ N_("Access violation"), 			NT_STATUS_ACCESS_VIOLATION },
-	{ N_("Invalid handle"), 			NT_STATUS_INVALID_HANDLE },
-	{ N_("Invalid parameter"), 			NT_STATUS_INVALID_PARAMETER },
-	{ N_("No memory"), 				NT_STATUS_NO_MEMORY },
-	{ N_("Buffer too small"), 			NT_STATUS_BUFFER_TOO_SMALL },
-	{ N_("Revision mismatch"), 			NT_STATUS_REVISION_MISMATCH },
-	{ N_("No logon servers"), 			NT_STATUS_NO_LOGON_SERVERS },
-	{ N_("No such logon session"), 		NT_STATUS_NO_SUCH_LOGON_SESSION },
-	{ N_("No such privilege"), 			NT_STATUS_NO_SUCH_PRIVILEGE },
-	{ N_("Procedure not found"), 		NT_STATUS_PROCEDURE_NOT_FOUND },
-	{ N_("Server disabled"), 			NT_STATUS_SERVER_DISABLED },
-	{ N_("Invalid pipe state"), 		NT_STATUS_INVALID_PIPE_STATE },
-	{ N_("Named pipe busy"), 			NT_STATUS_PIPE_BUSY },
-	{ N_("Illegal function"),		 	NT_STATUS_ILLEGAL_FUNCTION },
-	{ N_("Named pipe disconnected"), 		NT_STATUS_PIPE_DISCONNECTED },
-	{ N_("Named pipe closing"), 		NT_STATUS_PIPE_CLOSING },
-	{ N_("Remote host not listening"), 		NT_STATUS_REMOTE_NOT_LISTENING },
-	{ N_("Duplicate name on network"), 		NT_STATUS_DUPLICATE_NAME },
-	{ N_("Print queue is full"), 		NT_STATUS_PRINT_QUEUE_FULL },
-	{ N_("No print spool space available"), 	NT_STATUS_NO_SPOOL_SPACE },
-	{ N_("The network name cannot be found"),   NT_STATUS_BAD_NETWORK_NAME },
-	{ N_("The connection was refused"),         NT_STATUS_CONNECTION_REFUSED },
-	{ N_("Too many names"), 			NT_STATUS_TOO_MANY_NAMES },
-	{ N_("Too many sessions"), 			NT_STATUS_TOO_MANY_SESSIONS },
-	{ N_("Invalid server state"), 		NT_STATUS_INVALID_SERVER_STATE },
-	{ N_("Invalid domain state"), 		NT_STATUS_INVALID_DOMAIN_STATE },
-	{ N_("Invalid domain role"), 		NT_STATUS_INVALID_DOMAIN_ROLE },
-	{ N_("No such domain"), 			NT_STATUS_NO_SUCH_DOMAIN },
-	{ N_("Domain exists"), 			NT_STATUS_DOMAIN_EXISTS },
-	{ N_("Domain limit exceeded"), 		NT_STATUS_DOMAIN_LIMIT_EXCEEDED },
-	{ N_("Bad logon session state"), 		NT_STATUS_BAD_LOGON_SESSION_STATE },
-	{ N_("Logon session collision"), 		NT_STATUS_LOGON_SESSION_COLLISION },
-	{ N_("Invalid logon type"), 		NT_STATUS_INVALID_LOGON_TYPE },
-	{ N_("Cancelled"), 				NT_STATUS_CANCELLED },
-	{ N_("Invalid computer name"), 		NT_STATUS_INVALID_COMPUTER_NAME },
-	{ N_("Logon server conflict"), 		NT_STATUS_LOGON_SERVER_CONFLICT },
-	{ N_("Time difference at domain controller"), NT_STATUS_TIME_DIFFERENCE_AT_DC },
-	{ N_("Pipe broken"), 			NT_STATUS_PIPE_BROKEN },
-	{ N_("Registry corrupt"), 			NT_STATUS_REGISTRY_CORRUPT },
-	{ N_("Too many secrets"), 			NT_STATUS_TOO_MANY_SECRETS },
-	{ N_("Too many SIDs"), 			NT_STATUS_TOO_MANY_SIDS },
-	{ N_("Lanmanager cross encryption required"), NT_STATUS_LM_CROSS_ENCRYPTION_REQUIRED },
-	{ N_("Log file full"), 			NT_STATUS_LOG_FILE_FULL },
-	{ N_("No trusted LSA secret"), 		NT_STATUS_NO_TRUST_LSA_SECRET },
-	{ N_("No trusted SAM account"), 		NT_STATUS_NO_TRUST_SAM_ACCOUNT },
-	{ N_("Trusted domain failure"), 		NT_STATUS_TRUSTED_DOMAIN_FAILURE },
-	{ N_("Trust relationship failure"), 	NT_STATUS_TRUSTED_RELATIONSHIP_FAILURE },
-	{ N_("Trust failure"), 			NT_STATUS_TRUST_FAILURE },
-	{ N_("Netlogon service not started"), 	NT_STATUS_NETLOGON_NOT_STARTED },
-	{ N_("Account expired"), 			NT_STATUS_ACCOUNT_EXPIRED },
-	{ N_("Network credential conflict"), 	NT_STATUS_NETWORK_CREDENTIAL_CONFLICT },
-	{ N_("Remote session limit"), 		NT_STATUS_REMOTE_SESSION_LIMIT },
-	{ N_("No logon interdomain trust account"), NT_STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT },
-	{ N_("No logon workstation trust account"), NT_STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT },
-	{ N_("No logon server trust account"), 	NT_STATUS_NOLOGON_SERVER_TRUST_ACCOUNT },
-	{ N_("Domain trust inconsistent"), 		NT_STATUS_DOMAIN_TRUST_INCONSISTENT },
-	{ N_("No user session key available"), 	NT_STATUS_NO_USER_SESSION_KEY },
-	{ N_("User session deleted"), 		NT_STATUS_USER_SESSION_DELETED },
-	{ N_("Insufficient server resources"), 	NT_STATUS_INSUFF_SERVER_RESOURCES },
-	{ N_("Insufficient logon information"), 	NT_STATUS_INSUFFICIENT_LOGON_INFO },
-
-	{ N_("License quota exceeded"), 		NT_STATUS_LICENSE_QUOTA_EXCEEDED },
-	{ N_("No more files"),			STATUS_NO_MORE_FILES },
-
-	{ NULL, NT_STATUS(0) }
-};
-
-/*****************************************************************************
- Returns an NT error message.  not amazingly helpful, but better than a number.
- *****************************************************************************/
-
-const char *nt_errstr(NTSTATUS nt_code)
-{
-	int idx = 0;
-	char *result;
-
-#ifdef HAVE_LDAP
-	if (NT_STATUS_IS_LDAP(nt_code)) {
-		return ldap_err2string(NT_STATUS_LDAP_CODE(nt_code));
-	}
-#endif
-
-	if (NT_STATUS_IS_DOS(nt_code)) {
-		return smb_dos_err_name(NT_STATUS_DOS_CLASS(nt_code),
-					NT_STATUS_DOS_CODE(nt_code));
-	}
-
-	while (nt_errs[idx].nt_errstr != NULL) {
-		if (NT_STATUS_V(nt_errs[idx].nt_errcode) ==
-		    NT_STATUS_V(nt_code)) {
-			return nt_errs[idx].nt_errstr;
-		}
-		idx++;
-	}
-
-	result = talloc_asprintf(talloc_tos(), "NT code 0x%08x",
-				 NT_STATUS_V(nt_code));
-	SMB_ASSERT(result != NULL);
-	return result;
-}
-
-/************************************************************************
- Print friendler version fo NT error code
- ***********************************************************************/
-
-const char *get_friendly_nt_error_msg(NTSTATUS nt_code)
-{
-	int idx = 0;
-
-	while (nt_err_desc[idx].nt_errstr != NULL) {
-		if (NT_STATUS_V(nt_err_desc[idx].nt_errcode) == NT_STATUS_V(nt_code)) {
-			return nt_err_desc[idx].nt_errstr;
-		}
-		idx++;
-	}
-
-	/* fall back to NT_STATUS_XXX string */
-
-	return nt_errstr(nt_code);
-}
-
-/*****************************************************************************
- Returns an NT_STATUS constant as a string for inclusion in autogen C code.
- *****************************************************************************/
-
-const char *get_nt_error_c_code(NTSTATUS nt_code)
-{
-	char *result;
-	int idx = 0;
-
-	while (nt_errs[idx].nt_errstr != NULL) {
-		if (NT_STATUS_V(nt_errs[idx].nt_errcode) ==
-		    NT_STATUS_V(nt_code)) {
-			return nt_errs[idx].nt_errstr;
-		}
-		idx++;
-	}
-
-	result = talloc_asprintf(talloc_tos(), "NT_STATUS(0x%08x)",
-				 NT_STATUS_V(nt_code));
-	SMB_ASSERT(result);
-	return result;
-}
-
-/*****************************************************************************
- Returns the NT_STATUS constant matching the string supplied (as an NTSTATUS)
- *****************************************************************************/
-
-NTSTATUS nt_status_string_to_code(const char *nt_status_str)
-{
-	int idx = 0;
-
-	while (nt_errs[idx].nt_errstr != NULL) {
-		if (strcasecmp(nt_errs[idx].nt_errstr, nt_status_str) == 0) {
-			return nt_errs[idx].nt_errcode;
-		}
-		idx++;
-	}
-	return NT_STATUS_UNSUCCESSFUL;
-}
-
-/**
- * Squash an NT_STATUS in line with security requirements.
- * In an attempt to avoid giving the whole game away when users
- * are authenticating, NT replaces both NT_STATUS_NO_SUCH_USER and
- * NT_STATUS_WRONG_PASSWORD with NT_STATUS_LOGON_FAILURE in certain situations
- * (session setups in particular).
- *
- * @param nt_status NTSTATUS input for squashing.
- * @return the 'squashed' nt_status
- **/
-
-NTSTATUS nt_status_squash(NTSTATUS nt_status)
-{
-	if NT_STATUS_IS_OK(nt_status) {
-		return nt_status;
-	} else if NT_STATUS_EQUAL(nt_status, NT_STATUS_NO_SUCH_USER) {
-		/* Match WinXP and don't give the game away */
-		return NT_STATUS_LOGON_FAILURE;
-
-	} else if NT_STATUS_EQUAL(nt_status, NT_STATUS_WRONG_PASSWORD) {
-		/* Match WinXP and don't give the game away */
-		return NT_STATUS_LOGON_FAILURE;
-	} else {
-		return nt_status;
-	}
-}
diff --git a/source3/libsmb/ntlmssp.c b/source3/libsmb/ntlmssp.c
index d000ed59c03..fe2fb663bac 100644
--- a/source3/libsmb/ntlmssp.c
+++ b/source3/libsmb/ntlmssp.c
@@ -117,9 +117,9 @@ NTSTATUS ntlmssp_set_hashes(struct ntlmssp_state *ntlmssp_state,
 			    const uint8_t nt_hash[16])
 {
 	ntlmssp_state->lm_hash = (uint8_t *)
-		TALLOC_MEMDUP(ntlmssp_state, lm_hash, 16);
+		talloc_memdup(ntlmssp_state, lm_hash, 16);
 	ntlmssp_state->nt_hash = (uint8_t *)
-		TALLOC_MEMDUP(ntlmssp_state, nt_hash, 16);
+		talloc_memdup(ntlmssp_state, nt_hash, 16);
 	if (!ntlmssp_state->lm_hash || !ntlmssp_state->nt_hash) {
 		TALLOC_FREE(ntlmssp_state->lm_hash);
 		TALLOC_FREE(ntlmssp_state->nt_hash);
diff --git a/source3/libsmb/passchange.c b/source3/libsmb/passchange.c
index d37389a4858..bf2103db684 100644
--- a/source3/libsmb/passchange.c
+++ b/source3/libsmb/passchange.c
@@ -2,17 +2,17 @@
    Unix SMB/CIFS implementation.
    SMB client password change routine
    Copyright (C) Andrew Tridgell 1994-1998
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
@@ -21,6 +21,7 @@
 #include "../librpc/gen_ndr/ndr_samr.h"
 #include "rpc_client/cli_pipe.h"
 #include "rpc_client/cli_samr.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "libsmb/nmblib.h"
 
@@ -32,10 +33,8 @@ NTSTATUS remote_password_change(const char *remote_machine, const char *user_nam
 				const char *old_passwd, const char *new_passwd,
 				char **err_str)
 {
-	struct nmb_name calling, called;
 	struct cli_state *cli = NULL;
 	struct rpc_pipe_client *pipe_hnd = NULL;
-	struct sockaddr_storage ss;
 	char *user, *domain, *p;
 
 	NTSTATUS result;
@@ -56,41 +55,14 @@ NTSTATUS remote_password_change(const char *remote_machine, const char *user_nam
 
 	*err_str = NULL;
 
-	if(!resolve_name( remote_machine, &ss, 0x20, false)) {
-		if (asprintf(err_str, "Unable to find an IP address for machine "
-			 "%s.\n", remote_machine) == -1) {
-			*err_str = NULL;
-		}
-		return NT_STATUS_UNSUCCESSFUL;
-	}
-
-	cli = cli_initialise();
-	if (!cli) {
-		return NT_STATUS_NO_MEMORY;
-	}
-
-	result = cli_connect(cli, remote_machine, &ss);
+	result = cli_connect_nb(remote_machine, NULL, 0, 0x20, NULL,
+				Undefined, &cli);
 	if (!NT_STATUS_IS_OK(result)) {
 		if (asprintf(err_str, "Unable to connect to SMB server on "
 			 "machine %s. Error was : %s.\n",
 			 remote_machine, nt_errstr(result))==-1) {
 			*err_str = NULL;
 		}
-		cli_shutdown(cli);
-		return result;
-	}
-
-	make_nmb_name(&calling, global_myname() , 0x0);
-	make_nmb_name(&called , remote_machine, 0x20);
-
-	if (!cli_session_request(cli, &calling, &called)) {
-		if (asprintf(err_str, "machine %s rejected the session setup. "
-			 "Error was : %s.\n",
-			 remote_machine, cli_errstr(cli)) == -1) {
-			*err_str = NULL;
-		}
-		result = cli_nt_error(cli);
-		cli_shutdown(cli);
 		return result;
 	}
 
@@ -125,7 +97,7 @@ NTSTATUS remote_password_change(const char *remote_machine, const char *user_nam
 		if (!NT_STATUS_EQUAL(result, NT_STATUS_PASSWORD_MUST_CHANGE) &&
 		    !NT_STATUS_EQUAL(result, NT_STATUS_PASSWORD_EXPIRED)) {
 			if (asprintf(err_str, "Could not connect to machine %s: "
-				 "%s\n", remote_machine, cli_errstr(cli)) == -1) {
+				 "%s\n", remote_machine, nt_errstr(result)) == -1) {
 				*err_str = NULL;
 			}
 			cli_shutdown(cli);
@@ -145,7 +117,7 @@ NTSTATUS remote_password_change(const char *remote_machine, const char *user_nam
 		if (!NT_STATUS_IS_OK(result)) {
 			if (asprintf(err_str, "machine %s rejected the session "
 				 "setup. Error was : %s.\n",        
-				 remote_machine, cli_errstr(cli)) == -1) {
+				 remote_machine, nt_errstr(result)) == -1) {
 				*err_str = NULL;
 			}
 			cli_shutdown(cli);
@@ -203,12 +175,12 @@ NTSTATUS remote_password_change(const char *remote_machine, const char *user_nam
 		if (lp_client_lanman_auth()) {
 			/* Use the old RAP method. */
 			if (!cli_oem_change_password(cli, user_name, new_passwd, old_passwd)) {
+				result = cli_nt_error(cli);
 				if (asprintf(err_str, "machine %s rejected the "
 					 "password change: Error was : %s.\n",
-					 remote_machine, cli_errstr(cli)) == -1) {
+					 remote_machine, nt_errstr(result)) == -1) {
 					*err_str = NULL;
 				}
-				result = cli_nt_error(cli);
 				cli_shutdown(cli);
 				return result;
 			}
@@ -294,12 +266,13 @@ NTSTATUS remote_password_change(const char *remote_machine, const char *user_nam
 				cli_shutdown(cli);
 				return NT_STATUS_OK;
 			}
+
+			result = cli_nt_error(cli);
 			if (asprintf(err_str, "machine %s rejected the password "
 				 "change: Error was : %s.\n",
-				 remote_machine, cli_errstr(cli)) == -1) {
+				 remote_machine, nt_errstr(result)) == -1) {
 				*err_str = NULL;
 			}
-			result = cli_nt_error(cli);
 			cli_shutdown(cli);
 			return result;
 		} else {
diff --git a/source3/libsmb/proto.h b/source3/libsmb/proto.h
new file mode 100644
index 00000000000..f88a891d585
--- /dev/null
+++ b/source3/libsmb/proto.h
@@ -0,0 +1,854 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   Copyright (C) Andrew Bartlett 2001-2003
+   Copyright (C) Andrew Tridgell 1994-1998,2000-2001
+   Copyright (C) Gerald (Jerry) Carter 2004
+   Copyright (C) Jelmer Vernooij 2003
+   Copyright (C) Jeremy Allison 2001-2009,2011
+   Copyright (C) Stefan Metzmacher 2003,2009
+   Copyright (C) Volker Lendecke 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LIBSMB_PROTO_H_
+#define _LIBSMB_PROTO_H_
+
+#include "ads.h"
+
+/* The following definitions come from libsmb/cliconnect.c  */
+
+ADS_STATUS cli_session_setup_spnego(struct cli_state *cli, const char *user,
+			      const char *pass, const char *user_domain,
+				    const char * dest_realm);
+
+NTSTATUS cli_session_setup(struct cli_state *cli,
+			   const char *user,
+			   const char *pass, int passlen,
+			   const char *ntpass, int ntpasslen,
+			   const char *workgroup);
+struct tevent_req *cli_session_setup_guest_create(TALLOC_CTX *mem_ctx,
+						  struct event_context *ev,
+						  struct cli_state *cli,
+						  struct tevent_req **psmbreq);
+struct tevent_req *cli_session_setup_guest_send(TALLOC_CTX *mem_ctx,
+						struct event_context *ev,
+						struct cli_state *cli);
+NTSTATUS cli_session_setup_guest_recv(struct tevent_req *req);
+struct tevent_req *cli_ulogoff_send(TALLOC_CTX *mem_ctx,
+				    struct tevent_context *ev,
+				    struct cli_state *cli);
+NTSTATUS cli_ulogoff_recv(struct tevent_req *req);
+NTSTATUS cli_ulogoff(struct cli_state *cli);
+struct tevent_req *cli_tcon_andx_create(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *share, const char *dev,
+					const char *pass, int passlen,
+					struct tevent_req **psmbreq);
+struct tevent_req *cli_tcon_andx_send(TALLOC_CTX *mem_ctx,
+				      struct event_context *ev,
+				      struct cli_state *cli,
+				      const char *share, const char *dev,
+				      const char *pass, int passlen);
+NTSTATUS cli_tcon_andx_recv(struct tevent_req *req);
+NTSTATUS cli_tcon_andx(struct cli_state *cli, const char *share,
+		       const char *dev, const char *pass, int passlen);
+struct tevent_req *cli_tdis_send(TALLOC_CTX *mem_ctx,
+                                 struct tevent_context *ev,
+                                 struct cli_state *cli);
+NTSTATUS cli_tdis_recv(struct tevent_req *req);
+NTSTATUS cli_tdis(struct cli_state *cli);
+NTSTATUS cli_negprot(struct cli_state *cli);
+struct tevent_req *cli_negprot_send(TALLOC_CTX *mem_ctx,
+				    struct event_context *ev,
+				    struct cli_state *cli);
+NTSTATUS cli_negprot_recv(struct tevent_req *req);
+NTSTATUS cli_connect_nb(const char *host, struct sockaddr_storage *pss,
+			uint16_t port, int name_type, const char *myname,
+			int signing_state, struct cli_state **pcli);
+NTSTATUS cli_start_connection(struct cli_state **output_cli,
+			      const char *my_name,
+			      const char *dest_host,
+			      struct sockaddr_storage *dest_ss, int port,
+			      int signing_state, int flags);
+NTSTATUS cli_full_connection(struct cli_state **output_cli,
+			     const char *my_name,
+			     const char *dest_host,
+			     struct sockaddr_storage *dest_ss, int port,
+			     const char *service, const char *service_type,
+			     const char *user, const char *domain,
+			     const char *password, int flags,
+			     int signing_state);
+NTSTATUS cli_raw_tcon(struct cli_state *cli,
+		      const char *service, const char *pass, const char *dev,
+		      uint16 *max_xmit, uint16 *tid);
+struct cli_state *get_ipc_connect(char *server,
+				struct sockaddr_storage *server_ss,
+				const struct user_auth_info *user_info);
+struct cli_state *get_ipc_connect_master_ip(TALLOC_CTX *ctx,
+				struct sockaddr_storage *mb_ip,
+				const struct user_auth_info *user_info,
+				char **pp_workgroup_out);
+struct cli_state *get_ipc_connect_master_ip_bcast(TALLOC_CTX *ctx,
+					const struct user_auth_info *user_info,
+					char **pp_workgroup_out);
+
+/* The following definitions come from libsmb/clidfs.c  */
+
+NTSTATUS cli_cm_force_encryption(struct cli_state *c,
+			const char *username,
+			const char *password,
+			const char *domain,
+			const char *sharename);
+struct cli_state *cli_cm_open(TALLOC_CTX *ctx,
+				struct cli_state *referring_cli,
+				const char *server,
+				const char *share,
+				const struct user_auth_info *auth_info,
+				bool show_hdr,
+				bool force_encrypt,
+				int max_protocol,
+				int port,
+				int name_type);
+void cli_cm_display(const struct cli_state *c);
+struct client_dfs_referral;
+NTSTATUS cli_dfs_get_referral(TALLOC_CTX *ctx,
+			struct cli_state *cli,
+			const char *path,
+			struct client_dfs_referral **refs,
+			size_t *num_refs,
+			size_t *consumed);
+bool cli_resolve_path(TALLOC_CTX *ctx,
+			const char *mountpt,
+			const struct user_auth_info *dfs_auth_info,
+			struct cli_state *rootcli,
+			const char *path,
+			struct cli_state **targetcli,
+			char **pp_targetpath);
+
+bool cli_check_msdfs_proxy(TALLOC_CTX *ctx,
+			struct cli_state *cli,
+			const char *sharename,
+			char **pp_newserver,
+			char **pp_newshare,
+			bool force_encrypt,
+			const char *username,
+			const char *password,
+			const char *domain);
+
+/* The following definitions come from libsmb/clientgen.c  */
+
+int cli_set_message(char *buf,int num_words,int num_bytes,bool zero);
+unsigned int cli_set_timeout(struct cli_state *cli, unsigned int timeout);
+void cli_set_port(struct cli_state *cli, int port);
+void cli_setup_packet_buf(struct cli_state *cli, char *buf);
+NTSTATUS cli_set_domain(struct cli_state *cli, const char *domain);
+NTSTATUS cli_set_username(struct cli_state *cli, const char *username);
+NTSTATUS cli_set_password(struct cli_state *cli, const char *password);
+NTSTATUS cli_init_creds(struct cli_state *cli, const char *username, const char *domain, const char *password);
+struct cli_state *cli_initialise(void);
+struct cli_state *cli_initialise_ex(int signing_state);
+void cli_nt_pipes_close(struct cli_state *cli);
+void cli_shutdown(struct cli_state *cli);
+void cli_sockopt(struct cli_state *cli, const char *options);
+uint16 cli_setpid(struct cli_state *cli, uint16 pid);
+bool cli_set_case_sensitive(struct cli_state *cli, bool case_sensitive);
+struct tevent_req *cli_echo_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
+				 struct cli_state *cli, uint16_t num_echos,
+				 DATA_BLOB data);
+NTSTATUS cli_echo_recv(struct tevent_req *req);
+NTSTATUS cli_echo(struct cli_state *cli, uint16_t num_echos, DATA_BLOB data);
+bool cli_ucs2(struct cli_state *cli);
+bool is_andx_req(uint8_t cmd);
+NTSTATUS cli_smb(TALLOC_CTX *mem_ctx, struct cli_state *cli,
+		 uint8_t smb_command, uint8_t additional_flags,
+		 uint8_t wct, uint16_t *vwv,
+		 uint32_t num_bytes, const uint8_t *bytes,
+		 struct tevent_req **result_parent,
+		 uint8_t min_wct, uint8_t *pwct, uint16_t **pvwv,
+		 uint32_t *pnum_bytes, uint8_t **pbytes);
+
+/* The following definitions come from libsmb/clierror.c  */
+
+const char *cli_errstr(struct cli_state *cli);
+NTSTATUS cli_nt_error(struct cli_state *cli);
+void cli_dos_error(struct cli_state *cli, uint8 *eclass, uint32 *ecode);
+int cli_errno(struct cli_state *cli);
+bool cli_is_error(struct cli_state *cli);
+bool cli_is_nt_error(struct cli_state *cli);
+bool cli_is_dos_error(struct cli_state *cli);
+bool cli_state_is_connected(struct cli_state *cli);
+
+/* The following definitions come from libsmb/clifile.c  */
+
+struct tevent_req *cli_setpathinfo_send(TALLOC_CTX *mem_ctx,
+					struct tevent_context *ev,
+					struct cli_state *cli,
+					uint16_t level,
+					const char *path,
+					uint8_t *data,
+					size_t data_len);
+NTSTATUS cli_setpathinfo_recv(struct tevent_req *req);
+NTSTATUS cli_setpathinfo(struct cli_state *cli,
+			 uint16_t level,
+			 const char *path,
+			 uint8_t *data,
+			 size_t data_len);
+
+struct tevent_req *cli_posix_symlink_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *oldname,
+					const char *newname);
+NTSTATUS cli_posix_symlink_recv(struct tevent_req *req);
+NTSTATUS cli_posix_symlink(struct cli_state *cli,
+			const char *oldname,
+			const char *newname);
+struct tevent_req *cli_posix_readlink_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname,
+					size_t len);
+NTSTATUS cli_posix_readlink_recv(struct tevent_req *req, struct cli_state *cli,
+				char *retpath, size_t len);
+NTSTATUS cli_posix_readlink(struct cli_state *cli, const char *fname,
+			char *linkpath, size_t len);
+struct tevent_req *cli_posix_hardlink_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *oldname,
+					const char *newname);
+NTSTATUS cli_posix_hardlink_recv(struct tevent_req *req);
+NTSTATUS cli_posix_hardlink(struct cli_state *cli,
+			const char *oldname,
+			const char *newname);
+uint32_t unix_perms_to_wire(mode_t perms);
+mode_t wire_perms_to_unix(uint32_t perms);
+struct tevent_req *cli_posix_getfacl_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname);
+NTSTATUS cli_posix_getfacl_recv(struct tevent_req *req,
+				TALLOC_CTX *mem_ctx,
+				size_t *prb_size,
+				char **retbuf);
+NTSTATUS cli_posix_getfacl(struct cli_state *cli,
+			const char *fname,
+			TALLOC_CTX *mem_ctx,
+			size_t *prb_size,
+			char **retbuf);
+struct tevent_req *cli_posix_stat_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname);
+NTSTATUS cli_posix_stat_recv(struct tevent_req *req,
+				SMB_STRUCT_STAT *sbuf);
+NTSTATUS cli_posix_stat(struct cli_state *cli,
+			const char *fname,
+			SMB_STRUCT_STAT *sbuf);
+struct tevent_req *cli_posix_chmod_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname,
+					mode_t mode);
+NTSTATUS cli_posix_chmod_recv(struct tevent_req *req);
+NTSTATUS cli_posix_chmod(struct cli_state *cli, const char *fname, mode_t mode);
+struct tevent_req *cli_posix_chown_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname,
+					uid_t uid,
+					gid_t gid);
+NTSTATUS cli_posix_chown_recv(struct tevent_req *req);
+NTSTATUS cli_posix_chown(struct cli_state *cli,
+			const char *fname,
+			uid_t uid,
+			gid_t gid);
+struct tevent_req *cli_rename_send(TALLOC_CTX *mem_ctx,
+                                struct event_context *ev,
+                                struct cli_state *cli,
+                                const char *fname_src,
+                                const char *fname_dst);
+NTSTATUS cli_rename_recv(struct tevent_req *req);
+NTSTATUS cli_rename(struct cli_state *cli, const char *fname_src, const char *fname_dst);
+struct tevent_req *cli_ntrename_send(TALLOC_CTX *mem_ctx,
+                                struct event_context *ev,
+                                struct cli_state *cli,
+                                const char *fname_src,
+                                const char *fname_dst);
+NTSTATUS cli_ntrename_recv(struct tevent_req *req);
+NTSTATUS cli_ntrename(struct cli_state *cli, const char *fname_src, const char *fname_dst);
+
+struct tevent_req *cli_nt_hardlink_send(TALLOC_CTX *mem_ctx,
+                                struct event_context *ev,
+                                struct cli_state *cli,
+                                const char *fname_src,
+                                const char *fname_dst);
+NTSTATUS cli_nt_hardlink_recv(struct tevent_req *req);
+NTSTATUS cli_nt_hardlink(struct cli_state *cli, const char *fname_src, const char *fname_dst);
+
+struct tevent_req *cli_unlink_send(TALLOC_CTX *mem_ctx,
+                                struct event_context *ev,
+                                struct cli_state *cli,
+                                const char *fname,
+                                uint16_t mayhave_attrs);
+NTSTATUS cli_unlink_recv(struct tevent_req *req);
+NTSTATUS cli_unlink(struct cli_state *cli, const char *fname, uint16_t mayhave_attrs);
+
+struct tevent_req *cli_mkdir_send(TALLOC_CTX *mem_ctx,
+				  struct event_context *ev,
+				  struct cli_state *cli,
+				  const char *dname);
+NTSTATUS cli_mkdir_recv(struct tevent_req *req);
+NTSTATUS cli_mkdir(struct cli_state *cli, const char *dname);
+struct tevent_req *cli_rmdir_send(TALLOC_CTX *mem_ctx,
+				  struct event_context *ev,
+				  struct cli_state *cli,
+				  const char *dname);
+NTSTATUS cli_rmdir_recv(struct tevent_req *req);
+NTSTATUS cli_rmdir(struct cli_state *cli, const char *dname);
+struct tevent_req *cli_nt_delete_on_close_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					uint16_t fnum,
+					bool flag);
+NTSTATUS cli_nt_delete_on_close_recv(struct tevent_req *req);
+NTSTATUS cli_nt_delete_on_close(struct cli_state *cli, uint16_t fnum, bool flag);
+struct tevent_req *cli_ntcreate_send(TALLOC_CTX *mem_ctx,
+				     struct event_context *ev,
+				     struct cli_state *cli,
+				     const char *fname,
+				     uint32_t CreatFlags,
+				     uint32_t DesiredAccess,
+				     uint32_t FileAttributes,
+				     uint32_t ShareAccess,
+				     uint32_t CreateDisposition,
+				     uint32_t CreateOptions,
+				     uint8_t SecurityFlags);
+NTSTATUS cli_ntcreate_recv(struct tevent_req *req, uint16_t *pfnum);
+NTSTATUS cli_ntcreate(struct cli_state *cli,
+		      const char *fname,
+		      uint32_t CreatFlags,
+		      uint32_t DesiredAccess,
+		      uint32_t FileAttributes,
+		      uint32_t ShareAccess,
+		      uint32_t CreateDisposition,
+		      uint32_t CreateOptions,
+		      uint8_t SecurityFlags,
+		      uint16_t *pfid);
+uint8_t *smb_bytes_push_str(uint8_t *buf, bool ucs2, const char *str,
+			    size_t str_len, size_t *pconverted_size);
+uint8_t *smb_bytes_push_bytes(uint8_t *buf, uint8_t prefix,
+			      const uint8_t *bytes, size_t num_bytes);
+struct tevent_req *cli_open_create(TALLOC_CTX *mem_ctx,
+				   struct event_context *ev,
+				   struct cli_state *cli, const char *fname,
+				   int flags, int share_mode,
+				   struct tevent_req **psmbreq);
+struct tevent_req *cli_open_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
+				 struct cli_state *cli, const char *fname,
+				 int flags, int share_mode);
+NTSTATUS cli_open_recv(struct tevent_req *req, uint16_t *fnum);
+NTSTATUS cli_open(struct cli_state *cli, const char *fname, int flags, int share_mode, uint16_t *pfnum);
+struct tevent_req *cli_close_create(TALLOC_CTX *mem_ctx,
+				    struct event_context *ev,
+				    struct cli_state *cli, uint16_t fnum,
+				    struct tevent_req **psubreq);
+struct tevent_req *cli_close_send(TALLOC_CTX *mem_ctx,
+				  struct event_context *ev,
+				  struct cli_state *cli, uint16_t fnum);
+NTSTATUS cli_close_recv(struct tevent_req *req);
+NTSTATUS cli_close(struct cli_state *cli, uint16_t fnum);
+struct tevent_req *cli_ftruncate_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					uint16_t fnum,
+					uint64_t size);
+NTSTATUS cli_ftruncate_recv(struct tevent_req *req);
+NTSTATUS cli_ftruncate(struct cli_state *cli, uint16_t fnum, uint64_t size);
+NTSTATUS cli_locktype(struct cli_state *cli, uint16_t fnum,
+		      uint32_t offset, uint32_t len,
+		      int timeout, unsigned char locktype);
+bool cli_lock(struct cli_state *cli, uint16_t fnum,
+	      uint32_t offset, uint32_t len, int timeout, enum brl_type lock_type);
+struct tevent_req *cli_unlock_send(TALLOC_CTX *mem_ctx,
+                                struct event_context *ev,
+                                struct cli_state *cli,
+                                uint16_t fnum,
+                                uint64_t offset,
+                                uint64_t len);
+NTSTATUS cli_unlock_recv(struct tevent_req *req);
+NTSTATUS cli_unlock(struct cli_state *cli, uint16_t fnum, uint32_t offset, uint32_t len);
+bool cli_lock64(struct cli_state *cli, uint16_t fnum,
+		uint64_t offset, uint64_t len, int timeout, enum brl_type lock_type);
+struct tevent_req *cli_unlock64_send(TALLOC_CTX *mem_ctx,
+                                struct event_context *ev,
+                                struct cli_state *cli,
+                                uint16_t fnum,
+                                uint64_t offset,
+                                uint64_t len);
+NTSTATUS cli_unlock64_recv(struct tevent_req *req);
+NTSTATUS cli_unlock64(struct cli_state *cli, uint16_t fnum, uint64_t offset, uint64_t len);
+struct tevent_req *cli_posix_lock_send(TALLOC_CTX *mem_ctx,
+                                        struct event_context *ev,
+                                        struct cli_state *cli,
+                                        uint16_t fnum,
+                                        uint64_t offset,
+                                        uint64_t len,
+                                        bool wait_lock,
+                                        enum brl_type lock_type);
+NTSTATUS cli_posix_lock_recv(struct tevent_req *req);
+NTSTATUS cli_posix_lock(struct cli_state *cli, uint16_t fnum,
+			uint64_t offset, uint64_t len,
+			bool wait_lock, enum brl_type lock_type);
+struct tevent_req *cli_posix_unlock_send(TALLOC_CTX *mem_ctx,
+                                        struct event_context *ev,
+                                        struct cli_state *cli,
+                                        uint16_t fnum,
+                                        uint64_t offset,
+                                        uint64_t len);
+NTSTATUS cli_posix_unlock_recv(struct tevent_req *req);
+NTSTATUS cli_posix_unlock(struct cli_state *cli, uint16_t fnum, uint64_t offset, uint64_t len);
+struct tevent_req *cli_getattrE_send(TALLOC_CTX *mem_ctx,
+				struct event_context *ev,
+				struct cli_state *cli,
+                                uint16_t fnum);
+NTSTATUS cli_getattrE_recv(struct tevent_req *req,
+                        uint16_t *attr,
+                        SMB_OFF_T *size,
+                        time_t *change_time,
+                        time_t *access_time,
+                        time_t *write_time);
+NTSTATUS cli_getattrE(struct cli_state *cli,
+			uint16_t fnum,
+			uint16_t *attr,
+			SMB_OFF_T *size,
+			time_t *change_time,
+			time_t *access_time,
+			time_t *write_time);
+struct tevent_req *cli_setattrE_send(TALLOC_CTX *mem_ctx,
+				struct event_context *ev,
+				struct cli_state *cli,
+				uint16_t fnum,
+				time_t change_time,
+				time_t access_time,
+				time_t write_time);
+NTSTATUS cli_setattrE_recv(struct tevent_req *req);
+NTSTATUS cli_setattrE(struct cli_state *cli,
+			uint16_t fnum,
+			time_t change_time,
+			time_t access_time,
+			time_t write_time);
+struct tevent_req *cli_getatr_send(TALLOC_CTX *mem_ctx,
+				struct event_context *ev,
+				struct cli_state *cli,
+				const char *fname);
+NTSTATUS cli_getatr_recv(struct tevent_req *req,
+				uint16_t *attr,
+				SMB_OFF_T *size,
+				time_t *write_time);
+NTSTATUS cli_getatr(struct cli_state *cli,
+			const char *fname,
+			uint16_t *attr,
+			SMB_OFF_T *size,
+			time_t *write_time);
+struct tevent_req *cli_setatr_send(TALLOC_CTX *mem_ctx,
+				struct event_context *ev,
+				struct cli_state *cli,
+				const char *fname,
+				uint16_t attr,
+				time_t mtime);
+NTSTATUS cli_setatr_recv(struct tevent_req *req);
+NTSTATUS cli_setatr(struct cli_state *cli,
+                const char *fname,
+                uint16_t attr,
+                time_t mtime);
+struct tevent_req *cli_chkpath_send(TALLOC_CTX *mem_ctx,
+				  struct event_context *ev,
+				  struct cli_state *cli,
+				  const char *fname);
+NTSTATUS cli_chkpath_recv(struct tevent_req *req);
+NTSTATUS cli_chkpath(struct cli_state *cli, const char *path);
+struct tevent_req *cli_dskattr_send(TALLOC_CTX *mem_ctx,
+				  struct event_context *ev,
+				  struct cli_state *cli);
+NTSTATUS cli_dskattr_recv(struct tevent_req *req, int *bsize, int *total,
+			  int *avail);
+NTSTATUS cli_dskattr(struct cli_state *cli, int *bsize, int *total, int *avail);
+struct tevent_req *cli_ctemp_send(TALLOC_CTX *mem_ctx,
+				struct event_context *ev,
+				struct cli_state *cli,
+				const char *path);
+NTSTATUS cli_ctemp_recv(struct tevent_req *req,
+			TALLOC_CTX *ctx,
+			uint16_t *pfnum,
+			char **outfile);
+NTSTATUS cli_ctemp(struct cli_state *cli,
+			TALLOC_CTX *ctx,
+			const char *path,
+			uint16_t *pfnum,
+			char **out_path);
+NTSTATUS cli_raw_ioctl(struct cli_state *cli, uint16_t fnum, uint32_t code, DATA_BLOB *blob);
+NTSTATUS cli_set_ea_path(struct cli_state *cli, const char *path,
+			 const char *ea_name, const char *ea_val,
+			 size_t ea_len);
+NTSTATUS cli_set_ea_fnum(struct cli_state *cli, uint16_t fnum,
+			 const char *ea_name, const char *ea_val,
+			 size_t ea_len);
+struct tevent_req *cli_get_ea_list_path_send(TALLOC_CTX *mem_ctx,
+					     struct tevent_context *ev,
+					     struct cli_state *cli,
+					     const char *fname);
+NTSTATUS cli_get_ea_list_path_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+				   size_t *pnum_eas, struct ea_struct **peas);
+NTSTATUS cli_get_ea_list_path(struct cli_state *cli, const char *path,
+		TALLOC_CTX *ctx,
+		size_t *pnum_eas,
+		struct ea_struct **pea_list);
+struct tevent_req *cli_posix_open_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname,
+					int flags,
+					mode_t mode);
+NTSTATUS cli_posix_open_recv(struct tevent_req *req, uint16_t *pfnum);
+NTSTATUS cli_posix_open(struct cli_state *cli, const char *fname,
+			int flags, mode_t mode, uint16_t *fnum);
+struct tevent_req *cli_posix_mkdir_send(TALLOC_CTX *mem_ctx,
+                                        struct event_context *ev,
+                                        struct cli_state *cli,
+                                        const char *fname,
+                                        mode_t mode);
+NTSTATUS cli_posix_mkdir_recv(struct tevent_req *req);
+NTSTATUS cli_posix_mkdir(struct cli_state *cli, const char *fname, mode_t mode);
+
+struct tevent_req *cli_posix_unlink_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname);
+NTSTATUS cli_posix_unlink_recv(struct tevent_req *req);
+NTSTATUS cli_posix_unlink(struct cli_state *cli, const char *fname);
+
+struct tevent_req *cli_posix_rmdir_send(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli,
+					const char *fname);
+NTSTATUS cli_posix_rmdir_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx);
+NTSTATUS cli_posix_rmdir(struct cli_state *cli, const char *fname);
+struct tevent_req *cli_notify_send(TALLOC_CTX *mem_ctx,
+				   struct tevent_context *ev,
+				   struct cli_state *cli, uint16_t fnum,
+				   uint32_t buffer_size,
+				   uint32_t completion_filter, bool recursive);
+NTSTATUS cli_notify_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+			 uint32_t *pnum_changes,
+			 struct notify_change **pchanges);
+
+struct tevent_req *cli_nttrans_create_send(TALLOC_CTX *mem_ctx,
+					   struct event_context *ev,
+					   struct cli_state *cli,
+					   const char *fname,
+					   uint32_t CreatFlags,
+					   uint32_t DesiredAccess,
+					   uint32_t FileAttributes,
+					   uint32_t ShareAccess,
+					   uint32_t CreateDisposition,
+					   uint32_t CreateOptions,
+					   uint8_t SecurityFlags,
+					   struct security_descriptor *secdesc,
+					   struct ea_struct *eas,
+					   int num_eas);
+NTSTATUS cli_nttrans_create_recv(struct tevent_req *req, uint16_t *fnum);
+NTSTATUS cli_nttrans_create(struct cli_state *cli,
+			    const char *fname,
+			    uint32_t CreatFlags,
+			    uint32_t DesiredAccess,
+			    uint32_t FileAttributes,
+			    uint32_t ShareAccess,
+			    uint32_t CreateDisposition,
+			    uint32_t CreateOptions,
+			    uint8_t SecurityFlags,
+			    struct security_descriptor *secdesc,
+			    struct ea_struct *eas,
+			    int num_eas,
+			    uint16_t *pfid);
+
+/* The following definitions come from libsmb/clifsinfo.c  */
+
+struct tevent_req *cli_unix_extensions_version_send(TALLOC_CTX *mem_ctx,
+						    struct tevent_context *ev,
+						    struct cli_state *cli);
+NTSTATUS cli_unix_extensions_version_recv(struct tevent_req *req,
+					  uint16_t *pmajor, uint16_t *pminor,
+					  uint32_t *pcaplow,
+					  uint32_t *pcaphigh);
+NTSTATUS cli_unix_extensions_version(struct cli_state *cli, uint16 *pmajor,
+				     uint16 *pminor, uint32 *pcaplow,
+				     uint32 *pcaphigh);
+struct tevent_req *cli_set_unix_extensions_capabilities_send(
+	TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct cli_state *cli,
+	uint16_t major, uint16_t minor, uint32_t caplow, uint32_t caphigh);
+NTSTATUS cli_set_unix_extensions_capabilities_recv(struct tevent_req *req);
+NTSTATUS cli_set_unix_extensions_capabilities(struct cli_state *cli,
+					      uint16 major, uint16 minor,
+					      uint32 caplow, uint32 caphigh);
+struct tevent_req *cli_get_fs_attr_info_send(TALLOC_CTX *mem_ctx,
+					     struct tevent_context *ev,
+					     struct cli_state *cli);
+NTSTATUS cli_get_fs_attr_info_recv(struct tevent_req *req, uint32_t *fs_attr);
+NTSTATUS cli_get_fs_attr_info(struct cli_state *cli, uint32_t *fs_attr);
+NTSTATUS cli_get_fs_volume_info(struct cli_state *cli, fstring volume_name,
+				uint32 *pserial_number, time_t *pdate);
+NTSTATUS cli_get_fs_full_size_info(struct cli_state *cli,
+				   uint64_t *total_allocation_units,
+				   uint64_t *caller_allocation_units,
+				   uint64_t *actual_allocation_units,
+				   uint64_t *sectors_per_allocation_unit,
+				   uint64_t *bytes_per_sector);
+NTSTATUS cli_get_posix_fs_info(struct cli_state *cli,
+			       uint32 *optimal_transfer_size,
+			       uint32 *block_size,
+			       uint64_t *total_blocks,
+			       uint64_t *blocks_available,
+			       uint64_t *user_blocks_available,
+			       uint64_t *total_file_nodes,
+			       uint64_t *free_file_nodes,
+			       uint64_t *fs_identifier);
+NTSTATUS cli_raw_ntlm_smb_encryption_start(struct cli_state *cli,
+				const char *user,
+				const char *pass,
+				const char *domain);
+NTSTATUS cli_gss_smb_encryption_start(struct cli_state *cli);
+NTSTATUS cli_force_encryption(struct cli_state *c,
+			const char *username,
+			const char *password,
+			const char *domain);
+
+/* The following definitions come from libsmb/clilist.c  */
+
+NTSTATUS cli_list_old(struct cli_state *cli,const char *Mask,uint16 attribute,
+		      NTSTATUS (*fn)(const char *, struct file_info *,
+				 const char *, void *), void *state);
+NTSTATUS cli_list_trans(struct cli_state *cli, const char *mask,
+			uint16_t attribute, int info_level,
+			NTSTATUS (*fn)(const char *mnt, struct file_info *finfo,
+				   const char *mask, void *private_data),
+			void *private_data);
+struct tevent_req *cli_list_send(TALLOC_CTX *mem_ctx,
+				 struct tevent_context *ev,
+				 struct cli_state *cli,
+				 const char *mask,
+				 uint16_t attribute,
+				 uint16_t info_level);
+NTSTATUS cli_list_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+		       struct file_info **finfo, size_t *num_finfo);
+NTSTATUS cli_list(struct cli_state *cli,const char *Mask,uint16 attribute,
+		  NTSTATUS (*fn)(const char *, struct file_info *, const char *,
+			     void *), void *state);
+
+/* The following definitions come from libsmb/climessage.c  */
+
+struct tevent_req *cli_message_send(TALLOC_CTX *mem_ctx,
+				    struct tevent_context *ev,
+				    struct cli_state *cli,
+				    const char *host, const char *username,
+				    const char *message);
+NTSTATUS cli_message_recv(struct tevent_req *req);
+NTSTATUS cli_message(struct cli_state *cli, const char *host,
+		     const char *username, const char *message);
+
+/* The following definitions come from libsmb/clioplock.c  */
+
+struct tevent_req *cli_oplock_ack_send(TALLOC_CTX *mem_ctx,
+				       struct tevent_context *ev,
+				       struct cli_state *cli,
+				       uint16_t fnum, uint8_t level);
+NTSTATUS cli_oplock_ack_recv(struct tevent_req *req);
+NTSTATUS cli_oplock_ack(struct cli_state *cli, uint16_t fnum, unsigned char level);
+void cli_oplock_handler(struct cli_state *cli,
+			NTSTATUS (*handler)(struct cli_state *, uint16_t, unsigned char));
+
+/* The following definitions come from libsmb/cliprint.c  */
+
+int cli_print_queue(struct cli_state *cli,
+		    void (*fn)(struct print_job_info *));
+int cli_printjob_del(struct cli_state *cli, int job);
+
+/* The following definitions come from libsmb/cliquota.c  */
+
+NTSTATUS cli_get_quota_handle(struct cli_state *cli, uint16_t *quota_fnum);
+void free_ntquota_list(SMB_NTQUOTA_LIST **qt_list);
+NTSTATUS cli_get_user_quota(struct cli_state *cli, int quota_fnum,
+			    SMB_NTQUOTA_STRUCT *pqt);
+NTSTATUS cli_set_user_quota(struct cli_state *cli, int quota_fnum,
+			    SMB_NTQUOTA_STRUCT *pqt);
+NTSTATUS cli_list_user_quota(struct cli_state *cli, int quota_fnum,
+			     SMB_NTQUOTA_LIST **pqt_list);
+NTSTATUS cli_get_fs_quota_info(struct cli_state *cli, int quota_fnum,
+			       SMB_NTQUOTA_STRUCT *pqt);
+NTSTATUS cli_set_fs_quota_info(struct cli_state *cli, int quota_fnum,
+			       SMB_NTQUOTA_STRUCT *pqt);
+
+/* The following definitions come from libsmb/clireadwrite.c  */
+
+struct tevent_req *cli_read_andx_create(TALLOC_CTX *mem_ctx,
+					struct event_context *ev,
+					struct cli_state *cli, uint16_t fnum,
+					off_t offset, size_t size,
+					struct tevent_req **psmbreq);
+struct tevent_req *cli_read_andx_send(TALLOC_CTX *mem_ctx,
+				      struct event_context *ev,
+				      struct cli_state *cli, uint16_t fnum,
+				      off_t offset, size_t size);
+NTSTATUS cli_read_andx_recv(struct tevent_req *req, ssize_t *received,
+			    uint8_t **rcvbuf);
+struct tevent_req *cli_pull_send(TALLOC_CTX *mem_ctx,
+				 struct event_context *ev,
+				 struct cli_state *cli,
+				 uint16_t fnum, off_t start_offset,
+				 SMB_OFF_T size, size_t window_size,
+				 NTSTATUS (*sink)(char *buf, size_t n,
+						  void *priv),
+				 void *priv);
+NTSTATUS cli_pull_recv(struct tevent_req *req, SMB_OFF_T *received);
+NTSTATUS cli_pull(struct cli_state *cli, uint16_t fnum,
+		  off_t start_offset, SMB_OFF_T size, size_t window_size,
+		  NTSTATUS (*sink)(char *buf, size_t n, void *priv),
+		  void *priv, SMB_OFF_T *received);
+ssize_t cli_read(struct cli_state *cli, uint16_t fnum, char *buf,
+		 off_t offset, size_t size);
+NTSTATUS cli_smbwrite(struct cli_state *cli, uint16_t fnum, char *buf,
+		      off_t offset, size_t size1, size_t *ptotal);
+struct tevent_req *cli_write_andx_create(TALLOC_CTX *mem_ctx,
+					 struct event_context *ev,
+					 struct cli_state *cli, uint16_t fnum,
+					 uint16_t mode, const uint8_t *buf,
+					 off_t offset, size_t size,
+					 struct tevent_req **reqs_before,
+					 int num_reqs_before,
+					 struct tevent_req **psmbreq);
+struct tevent_req *cli_write_andx_send(TALLOC_CTX *mem_ctx,
+				       struct event_context *ev,
+				       struct cli_state *cli, uint16_t fnum,
+				       uint16_t mode, const uint8_t *buf,
+				       off_t offset, size_t size);
+NTSTATUS cli_write_andx_recv(struct tevent_req *req, size_t *pwritten);
+
+NTSTATUS cli_writeall(struct cli_state *cli, uint16_t fnum, uint16_t mode,
+		      const uint8_t *buf, off_t offset, size_t size,
+		      size_t *pwritten);
+
+struct tevent_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
+				 struct cli_state *cli,
+				 uint16_t fnum, uint16_t mode,
+				 off_t start_offset, size_t window_size,
+				 size_t (*source)(uint8_t *buf, size_t n,
+						  void *priv),
+				 void *priv);
+NTSTATUS cli_push_recv(struct tevent_req *req);
+NTSTATUS cli_push(struct cli_state *cli, uint16_t fnum, uint16_t mode,
+		  off_t start_offset, size_t window_size,
+		  size_t (*source)(uint8_t *buf, size_t n, void *priv),
+		  void *priv);
+
+/* The following definitions come from libsmb/clisecdesc.c  */
+
+struct security_descriptor *cli_query_secdesc(struct cli_state *cli, uint16_t fnum,
+			    TALLOC_CTX *mem_ctx);
+NTSTATUS cli_set_secdesc(struct cli_state *cli, uint16_t fnum,
+			 struct security_descriptor *sd);
+
+/* The following definitions come from libsmb/clistr.c  */
+
+size_t clistr_push_fn(struct cli_state *cli,
+			void *dest,
+			const char *src,
+			int dest_len,
+			int flags);
+size_t clistr_pull_fn(const char *inbuf,
+			char *dest,
+			const void *src,
+			int dest_len,
+			int src_len,
+			int flags);
+size_t clistr_pull_talloc(TALLOC_CTX *ctx,
+			  const char *base,
+			  uint16_t flags2,
+			  char **pp_dest,
+			  const void *src,
+			  int src_len,
+			  int flags);
+
+/* The following definitions come from libsmb/clitrans.c  */
+
+struct tevent_req *cli_trans_send(
+	TALLOC_CTX *mem_ctx, struct event_context *ev,
+	struct cli_state *cli, uint8_t cmd,
+	const char *pipe_name, uint16_t fid, uint16_t function, int flags,
+	uint16_t *setup, uint8_t num_setup, uint8_t max_setup,
+	uint8_t *param, uint32_t num_param, uint32_t max_param,
+	uint8_t *data, uint32_t num_data, uint32_t max_data);
+NTSTATUS cli_trans_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+			uint16_t *recv_flags2,
+			uint16_t **setup, uint8_t min_setup,
+			uint8_t *num_setup,
+			uint8_t **param, uint32_t min_param,
+			uint32_t *num_param,
+			uint8_t **data, uint32_t min_data,
+			uint32_t *num_data);
+NTSTATUS cli_trans(TALLOC_CTX *mem_ctx, struct cli_state *cli,
+		   uint8_t trans_cmd,
+		   const char *pipe_name, uint16_t fid, uint16_t function,
+		   int flags,
+		   uint16_t *setup, uint8_t num_setup, uint8_t max_setup,
+		   uint8_t *param, uint32_t num_param, uint32_t max_param,
+		   uint8_t *data, uint32_t num_data, uint32_t max_data,
+		   uint16_t *recv_flags2,
+		   uint16_t **rsetup, uint8_t min_rsetup, uint8_t *num_rsetup,
+		   uint8_t **rparam, uint32_t min_rparam, uint32_t *num_rparam,
+		   uint8_t **rdata, uint32_t min_rdata, uint32_t *num_rdata);
+
+/* The following definitions come from libsmb/smb_seal.c  */
+
+NTSTATUS get_enc_ctx_num(const uint8_t *buf, uint16 *p_enc_ctx_num);
+bool common_encryption_on(struct smb_trans_enc_state *es);
+NTSTATUS common_ntlm_decrypt_buffer(struct ntlmssp_state *ntlmssp_state, char *buf);
+NTSTATUS common_ntlm_encrypt_buffer(struct ntlmssp_state *ntlmssp_state,
+				uint16 enc_ctx_num,
+				char *buf,
+				char **ppbuf_out);
+NTSTATUS common_encrypt_buffer(struct smb_trans_enc_state *es, char *buffer, char **buf_out);
+NTSTATUS common_decrypt_buffer(struct smb_trans_enc_state *es, char *buf);
+void common_free_encryption_state(struct smb_trans_enc_state **pp_es);
+void common_free_enc_buffer(struct smb_trans_enc_state *es, char *buf);
+bool cli_encryption_on(struct cli_state *cli);
+void cli_free_encryption_context(struct cli_state *cli);
+void cli_free_enc_buffer(struct cli_state *cli, char *buf);
+
+/* The following definitions come from libsmb/clisigning.c  */
+
+bool cli_simple_set_signing(struct cli_state *cli,
+			    const DATA_BLOB user_session_key,
+			    const DATA_BLOB response);
+bool cli_temp_set_signing(struct cli_state *cli);
+void cli_calculate_sign_mac(struct cli_state *cli, char *buf, uint32_t *seqnum);
+bool cli_check_sign_mac(struct cli_state *cli, const char *buf, uint32_t seqnum);
+bool client_is_signing_on(struct cli_state *cli);
+bool client_is_signing_allowed(struct cli_state *cli);
+bool client_is_signing_mandatory(struct cli_state *cli);
+void cli_set_signing_negotiated(struct cli_state *cli);
+
+#endif /* _LIBSMB_PROTO_H_ */
diff --git a/source3/libsmb/read_smb.c b/source3/libsmb/read_smb.c
new file mode 100644
index 00000000000..f530633c9f8
--- /dev/null
+++ b/source3/libsmb/read_smb.c
@@ -0,0 +1,134 @@
+/*
+   Unix SMB/CIFS implementation.
+   Infrastructure for async SMB client requests
+   Copyright (C) Volker Lendecke 2008
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/async_req/async_sock.h"
+#include "read_smb.h"
+#include "lib/util/tevent_unix.h"
+
+/*
+ * Read an smb packet asynchronously, discard keepalives
+ */
+
+struct read_smb_state {
+	struct tevent_context *ev;
+	int fd;
+	uint8_t *buf;
+};
+
+static ssize_t read_smb_more(uint8_t *buf, size_t buflen, void *private_data);
+static void read_smb_done(struct tevent_req *subreq);
+
+struct tevent_req *read_smb_send(TALLOC_CTX *mem_ctx,
+				 struct tevent_context *ev,
+				 int fd)
+{
+	struct tevent_req *result, *subreq;
+	struct read_smb_state *state;
+
+	result = tevent_req_create(mem_ctx, &state, struct read_smb_state);
+	if (result == NULL) {
+		return NULL;
+	}
+	state->ev = ev;
+	state->fd = fd;
+
+	subreq = read_packet_send(state, ev, fd, 4, read_smb_more, NULL);
+	if (subreq == NULL) {
+		goto fail;
+	}
+	tevent_req_set_callback(subreq, read_smb_done, result);
+	return result;
+ fail:
+	TALLOC_FREE(result);
+	return NULL;
+}
+
+static ssize_t read_smb_more(uint8_t *buf, size_t buflen, void *private_data)
+{
+	if (buflen > 4) {
+		return 0;	/* We've been here, we're done */
+	}
+	return smb_len_large(buf);
+}
+
+static void read_smb_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct read_smb_state *state = tevent_req_data(
+		req, struct read_smb_state);
+	ssize_t len;
+	int err;
+
+	len = read_packet_recv(subreq, state, &state->buf, &err);
+	TALLOC_FREE(subreq);
+	if (len == -1) {
+		tevent_req_error(req, err);
+		return;
+	}
+
+	if (CVAL(state->buf, 0) == SMBkeepalive) {
+		subreq = read_packet_send(state, state->ev, state->fd, 4,
+					  read_smb_more, NULL);
+		if (tevent_req_nomem(subreq, req)) {
+			return;
+		}
+		tevent_req_set_callback(subreq, read_smb_done, req);
+		return;
+	}
+	tevent_req_done(req);
+}
+
+ssize_t read_smb_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+		      uint8_t **pbuf, int *perrno)
+{
+	struct read_smb_state *state = tevent_req_data(
+		req, struct read_smb_state);
+
+	if (tevent_req_is_unix_error(req, perrno)) {
+		return -1;
+	}
+	*pbuf = talloc_move(mem_ctx, &state->buf);
+	return talloc_get_size(*pbuf);
+}
+
+ssize_t read_smb(int fd, TALLOC_CTX *mem_ctx, uint8_t **pbuf, int *perrno)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	ssize_t ret = -1;
+
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = read_smb_send(frame, ev, fd);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll(req, ev)) {
+		goto fail;
+	}
+	ret = read_smb_recv(req, mem_ctx, pbuf, perrno);
+ fail:
+	TALLOC_FREE(frame);
+	return ret;
+}
diff --git a/source3/libsmb/read_smb.h b/source3/libsmb/read_smb.h
new file mode 100644
index 00000000000..b0846c1ec3c
--- /dev/null
+++ b/source3/libsmb/read_smb.h
@@ -0,0 +1,34 @@
+/*
+   Unix SMB/CIFS implementation.
+   Infrastructure for async SMB client requests
+   Copyright (C) Volker Lendecke 2008
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __LIBSMB_READ_SMB_H
+#define __LIBSMB_READ_SMB_H
+
+#include "lib/talloc/talloc.h"
+#include "lib/tevent/tevent.h"
+
+struct tevent_req *read_smb_send(TALLOC_CTX *mem_ctx,
+				 struct tevent_context *ev,
+				 int fd);
+
+ssize_t read_smb_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+		      uint8_t **pbuf, int *perrno);
+ssize_t read_smb(int fd, TALLOC_CTX *mem_ctx, uint8_t **pbuf, int *perrno);
+
+#endif
diff --git a/source3/libsmb/samlogon_cache.c b/source3/libsmb/samlogon_cache.c
index facdbc7dc19..618a570c4e0 100644
--- a/source3/libsmb/samlogon_cache.c
+++ b/source3/libsmb/samlogon_cache.c
@@ -25,6 +25,7 @@
 #include "system/filesys.h"
 #include "librpc/gen_ndr/ndr_krb5pac.h"
 #include "../libcli/security/security.h"
+#include "util_tdb.h"
 
 #define NETSAMLOGON_TDB	"netsamlogon_cache.tdb"
 
@@ -70,16 +71,9 @@ clear:
 	}
 	first_try = false;
 
-	DEBUG(0,("retry after CLEAR_IF_FIRST for '%s'\n", path));
-	tdb = tdb_open_log(path, 0, TDB_CLEAR_IF_FIRST|TDB_INCOMPATIBLE_HASH,
-			   O_RDWR | O_CREAT, 0600);
-	if (tdb) {
-		tdb_close(tdb);
-		goto again;
-	}
-	DEBUG(0,("tdb_open_log(%s) with CLEAR_IF_FIRST - failed\n", path));
-
-	return false;
+	DEBUG(0,("retry after truncate for '%s'\n", path));
+	truncate(path, 0);
+	goto again;
 }
 
 
@@ -155,7 +149,7 @@ bool netsamlogon_cache_store(const char *username, struct netr_SamInfo3 *info3)
 
 	/* Prepare data */
 
-	if (!(mem_ctx = TALLOC_P( NULL, int))) {
+	if (!(mem_ctx = talloc( NULL, int))) {
 		DEBUG(0,("netsamlogon_cache_store: talloc() failed!\n"));
 		return false;
 	}
@@ -185,7 +179,7 @@ bool netsamlogon_cache_store(const char *username, struct netr_SamInfo3 *info3)
 	data.dsize = blob.length;
 	data.dptr = blob.data;
 
-	if (tdb_store_bystring(netsamlogon_tdb, keystr, data, TDB_REPLACE) != -1) {
+	if (tdb_store_bystring(netsamlogon_tdb, keystr, data, TDB_REPLACE) == 0) {
 		result = true;
 	}
 
@@ -223,7 +217,7 @@ struct netr_SamInfo3 *netsamlogon_cache_get(TALLOC_CTX *mem_ctx, const struct do
 		return NULL;
 	}
 
-	info3 = TALLOC_ZERO_P(mem_ctx, struct netr_SamInfo3);
+	info3 = talloc_zero(mem_ctx, struct netr_SamInfo3);
 	if (!info3) {
 		goto done;
 	}
diff --git a/source3/libsmb/smb_seal.c b/source3/libsmb/smb_seal.c
index 0eed15d4a3c..73786ac54a8 100644
--- a/source3/libsmb/smb_seal.c
+++ b/source3/libsmb/smb_seal.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "../libcli/auth/ntlmssp.h"
 #include "smb_crypt.h"
+#include "libsmb/libsmb.h"
 
 /******************************************************************************
  Pull out the encryption context for this packet. 0 means global context.
@@ -464,46 +465,3 @@ void cli_free_enc_buffer(struct cli_state *cli, char *buf)
 	 */
 	common_free_enc_buffer(cli->trans_enc_state, buf);
 }
-
-/******************************************************************************
- Decrypt an incoming buffer.
-******************************************************************************/
-
-NTSTATUS cli_decrypt_message(struct cli_state *cli)
-{
-	NTSTATUS status;
-	uint16 enc_ctx_num;
-
-	/* Ignore non-session messages. */
-	if(CVAL(cli->inbuf,0)) {
-		return NT_STATUS_OK;
-	}
-
-	status = get_enc_ctx_num((const uint8_t *)cli->inbuf, &enc_ctx_num);
-	if (!NT_STATUS_IS_OK(status)) {
-		return status;
-	}
-
-	if (enc_ctx_num != cli->trans_enc_state->enc_ctx_num) {
-		return NT_STATUS_INVALID_HANDLE;
-	}
-
-	return common_decrypt_buffer(cli->trans_enc_state, cli->inbuf);
-}
-
-/******************************************************************************
- Encrypt an outgoing buffer. Return the encrypted pointer in buf_out.
-******************************************************************************/
-
-NTSTATUS cli_encrypt_message(struct cli_state *cli, char *buf, char **buf_out)
-{
-	/* Ignore non-session messages. */
-	if (CVAL(buf,0)) {
-		return NT_STATUS_OK;
-	}
-
-	/* If we supported multiple encrytion contexts
-	 * here we'd look up based on tid.
-	 */
-	return common_encrypt_buffer(cli->trans_enc_state, buf, buf_out);
-}
diff --git a/source3/libsmb/smb_share_modes.c b/source3/libsmb/smb_share_modes.c
index 1a6c2123ed6..7c0a6d2a4e6 100644
--- a/source3/libsmb/smb_share_modes.c
+++ b/source3/libsmb/smb_share_modes.c
@@ -27,6 +27,8 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "smb_share_modes.h"
+#include "tdb_compat.h"
+#include <ccan/hash/hash.h>
 
 /* Database context handle. */
 struct smbdb_ctx {
@@ -67,10 +69,12 @@ struct smbdb_ctx *smb_share_mode_db_open(const char *db_path)
 
 	memset(smb_db, '\0', sizeof(struct smbdb_ctx));
 
-	smb_db->smb_tdb = tdb_open(db_path,
-				0, TDB_DEFAULT|TDB_CLEAR_IF_FIRST|TDB_INCOMPATIBLE_HASH,
-				O_RDWR|O_CREAT,
-				0644);
+	/* FIXME: We should *never* open a tdb without logging! */
+	smb_db->smb_tdb = tdb_open_compat(db_path,
+					  0, TDB_DEFAULT|TDB_CLEAR_IF_FIRST|TDB_INCOMPATIBLE_HASH,
+					  O_RDWR|O_CREAT,
+					  0644,
+					  NULL, NULL);
 
 	if (!smb_db->smb_tdb) {
 		free(smb_db);
@@ -120,7 +124,7 @@ int smb_lock_share_mode_entry(struct smbdb_ctx *db_ctx,
 {
 	struct locking_key lk;
 	return tdb_chainlock(db_ctx->smb_tdb, get_locking_key(&lk, dev, ino,
-							      extid));
+							      extid)) == 0 ? 0 : -1;
 }
 
 int smb_unlock_share_mode_entry(struct smbdb_ctx *db_ctx,
@@ -129,8 +133,9 @@ int smb_unlock_share_mode_entry(struct smbdb_ctx *db_ctx,
                                 uint64_t extid)
 {
 	struct locking_key lk;
-	return tdb_chainunlock(db_ctx->smb_tdb,
-			       get_locking_key(&lk, dev, ino, extid));
+	tdb_chainunlock(db_ctx->smb_tdb,
+			get_locking_key(&lk, dev, ino, extid));
+	return 0;
 }
 
 /*
@@ -199,8 +204,8 @@ int smb_get_share_mode_entries(struct smbdb_ctx *db_ctx,
 	*pp_list = NULL;
 	*p_delete_on_close = 0;
 
-	db_data = tdb_fetch(db_ctx->smb_tdb, get_locking_key(&lk, dev, ino,
-							     extid));
+	db_data = tdb_fetch_compat(db_ctx->smb_tdb,
+				   get_locking_key(&lk, dev, ino, extid));
 	if (!db_data.dptr) {
 		return 0;
 	}
@@ -266,7 +271,6 @@ int smb_get_share_mode_entries(struct smbdb_ctx *db_ctx,
 
 static uint32_t smb_name_hash(const char *sharepath, const char *filename, int *err)
 {
-	TDB_DATA key;
 	char *fullpath = NULL;
 	size_t sharepath_size = strlen(sharepath);
 	size_t filename_size = strlen(filename);
@@ -282,9 +286,7 @@ static uint32_t smb_name_hash(const char *sharepath, const char *filename, int *
 	fullpath[sharepath_size] = '/';
 	memcpy(&fullpath[sharepath_size + 1], filename, filename_size + 1);
 
-	key.dptr = (uint8_t *)fullpath;
-	key.dsize = strlen(fullpath) + 1;
-	name_hash = tdb_jenkins_hash(&key);
+	name_hash = hash(fullpath, strlen(fullpath) + 1, 0);
 	free(fullpath);
 	return name_hash;
 }
@@ -316,7 +318,7 @@ int smb_create_share_mode_entry_ex(struct smbdb_ctx *db_ctx,
 		return -1;
 	}
 
-	db_data = tdb_fetch(db_ctx->smb_tdb, locking_key);
+	db_data = tdb_fetch_compat(db_ctx->smb_tdb, locking_key);
 	if (!db_data.dptr) {
 		/* We must create the entry. */
 		db_data.dptr = (uint8 *)malloc(
@@ -345,7 +347,7 @@ int smb_create_share_mode_entry_ex(struct smbdb_ctx *db_ctx,
 		db_data.dsize = sizeof(struct locking_data) + sizeof(struct share_mode_entry) +
 					strlen(sharepath) + 1 +
 					strlen(filename) + 1;
-		if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_INSERT) == -1) {
+		if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_INSERT) != 0) {
 			free(db_data.dptr);
 			return -1;
 		}
@@ -388,7 +390,7 @@ int smb_create_share_mode_entry_ex(struct smbdb_ctx *db_ctx,
 	db_data.dptr = new_data_p;
 	db_data.dsize = new_data_size;
 
-	if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_REPLACE) == -1) {
+	if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_REPLACE) != 0) {
 		free(db_data.dptr);
 		return -1;
 	}
@@ -433,7 +435,7 @@ int smb_delete_share_mode_entry(struct smbdb_ctx *db_ctx,
 	size_t i, num_share_modes;
 	const uint8 *remaining_ptr = NULL;
 
-	db_data = tdb_fetch(db_ctx->smb_tdb, locking_key);
+	db_data = tdb_fetch_compat(db_ctx->smb_tdb, locking_key);
 	if (!db_data.dptr) {
 		return -1; /* Error - missing entry ! */
 	}
@@ -451,7 +453,7 @@ int smb_delete_share_mode_entry(struct smbdb_ctx *db_ctx,
 		}
 		/* It's ours - just remove the entire record. */
 		free(db_data.dptr);
-		return tdb_delete(db_ctx->smb_tdb, locking_key);
+		return tdb_delete(db_ctx->smb_tdb, locking_key) ? -1 : 0;
 	}
 
 	/* More than one - allocate a new record minus the one we'll delete. */
@@ -490,7 +492,7 @@ int smb_delete_share_mode_entry(struct smbdb_ctx *db_ctx,
 		/* None left after pruning. Delete record. */
 		free(db_data.dptr);
 		free(new_data_p);
-		return tdb_delete(db_ctx->smb_tdb, locking_key);
+		return tdb_delete(db_ctx->smb_tdb, locking_key) ? -1 : 0;
 	}
 
 	/* Copy any delete tokens plus the terminating filenames. */
@@ -511,7 +513,7 @@ int smb_delete_share_mode_entry(struct smbdb_ctx *db_ctx,
 
 	db_data.dsize = sizeof(struct locking_data) + (num_share_modes * sizeof(struct share_mode_entry)) + remaining_size;
 
-	if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_REPLACE) == -1) {
+	if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_REPLACE) != 0) {
 		free(db_data.dptr);
 		return -1;
 	}
@@ -535,7 +537,7 @@ int smb_change_share_mode_entry(struct smbdb_ctx *db_ctx,
 	size_t i;
 	int found_entry = 0;
 
-	db_data = tdb_fetch(db_ctx->smb_tdb, locking_key);
+	db_data = tdb_fetch_compat(db_ctx->smb_tdb, locking_key);
 	if (!db_data.dptr) {
 		return -1; /* Error - missing entry ! */
 	}
@@ -566,7 +568,7 @@ int smb_change_share_mode_entry(struct smbdb_ctx *db_ctx,
 	}
 
 	/* Save modified data. */
-	if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_REPLACE) == -1) {
+	if (tdb_store(db_ctx->smb_tdb, locking_key, db_data, TDB_REPLACE) != 0) {
 		free(db_data.dptr);
 		return -1;
 	}
diff --git a/source3/libsmb/smbsock_connect.c b/source3/libsmb/smbsock_connect.c
index fd8626f5ad2..3a228b1b9ed 100644
--- a/source3/libsmb/smbsock_connect.c
+++ b/source3/libsmb/smbsock_connect.c
@@ -18,7 +18,8 @@
 */
 
 #include "includes.h"
-#include "../lib/async_req/async_sock.h"
+#include "../lib/util/tevent_ntstatus.h"
+#include "client.h"
 #include "async_smb.h"
 #include "libsmb/nmblib.h"
 
@@ -215,7 +216,7 @@ struct tevent_req *smbsock_connect_send(TALLOC_CTX *mem_ctx,
 	state->called_type =
 		(called_type != -1) ? called_type : 0x20;
 	state->calling_name =
-		(calling_name != NULL) ? calling_name : global_myname();
+		(calling_name != NULL) ? calling_name : lp_netbios_name();
 	state->calling_type =
 		(calling_type != -1) ? calling_type : 0x00;
 
@@ -270,6 +271,7 @@ static int smbsock_connect_state_destructor(
 {
 	if (state->sock != -1) {
 		close(state->sock);
+		state->sock = -1;
 	}
 	return 0;
 }
@@ -368,7 +370,7 @@ NTSTATUS smbsock_connect_recv(struct tevent_req *req, int *sock,
 NTSTATUS smbsock_connect(const struct sockaddr_storage *addr, uint16_t port,
 			 const char *called_name, int called_type,
 			 const char *calling_name, int calling_type,
-			 int *pfd, uint16_t *ret_port)
+			 int *pfd, uint16_t *ret_port, int sec_timeout)
 {
 	TALLOC_CTX *frame = talloc_stackframe();
 	struct event_context *ev;
@@ -385,6 +387,11 @@ NTSTATUS smbsock_connect(const struct sockaddr_storage *addr, uint16_t port,
 	if (req == NULL) {
 		goto fail;
 	}
+	if ((sec_timeout != 0) &&
+	    !tevent_req_set_endtime(
+		    req, ev, timeval_current_ofs(sec_timeout, 0))) {
+		goto fail;
+	}
 	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
 		goto fail;
 	}
@@ -568,7 +575,7 @@ static void smbsock_any_connect_connected(struct tevent_req *subreq)
 	}
 
 	state->num_received += 1;
-	if (state->num_received <= state->num_addrs) {
+	if (state->num_received < state->num_addrs) {
 		/*
 		 * More addrs pending, wait for the others
 		 */
@@ -610,6 +617,7 @@ NTSTATUS smbsock_any_connect(const struct sockaddr_storage *addrs,
 			     int *calling_types,
 			     size_t num_addrs,
 			     uint16_t port,
+			     int sec_timeout,
 			     int *pfd, size_t *chosen_index,
 			     uint16_t *chosen_port)
 {
@@ -629,6 +637,11 @@ NTSTATUS smbsock_any_connect(const struct sockaddr_storage *addrs,
 	if (req == NULL) {
 		goto fail;
 	}
+	if ((sec_timeout != 0) &&
+	    !tevent_req_set_endtime(
+		    req, ev, timeval_current_ofs(sec_timeout, 0))) {
+		goto fail;
+	}
 	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
 		goto fail;
 	}
diff --git a/source3/libsmb/trusts_util.c b/source3/libsmb/trusts_util.c
index 08377fbfbc6..dc2cf03a044 100644
--- a/source3/libsmb/trusts_util.c
+++ b/source3/libsmb/trusts_util.c
@@ -27,6 +27,7 @@
 #include "../librpc/gen_ndr/ndr_netlogon.h"
 #include "secrets.h"
 #include "passdb.h"
+#include "libsmb/libsmb.h"
 
 /*********************************************************
  Change the domain password on the PDC.
@@ -168,7 +169,7 @@ bool enumerate_domain_trusts( TALLOC_CTX *mem_ctx, const char *domain,
 
 	/* setup the anonymous connection */
 
-	status = cli_full_connection( &cli, global_myname(), dc_name, &dc_ss, 0, "IPC$", "IPC",
+	status = cli_full_connection( &cli, lp_netbios_name(), dc_name, &dc_ss, 0, "IPC$", "IPC",
 		"", "", "", 0, Undefined);
 	if ( !NT_STATUS_IS_OK(status) )
 		goto done;
@@ -207,20 +208,20 @@ bool enumerate_domain_trusts( TALLOC_CTX *mem_ctx, const char *domain,
 
 	*num_domains = dom_list.count;
 
-	*domain_names = TALLOC_ZERO_ARRAY(mem_ctx, char *, *num_domains);
+	*domain_names = talloc_zero_array(mem_ctx, char *, *num_domains);
 	if (!*domain_names) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
 	}
 
-	*sids = TALLOC_ZERO_ARRAY(mem_ctx, struct dom_sid, *num_domains);
+	*sids = talloc_zero_array(mem_ctx, struct dom_sid, *num_domains);
 	if (!*sids) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
 	}
 
 	for (i=0; i< *num_domains; i++) {
-		(*domain_names)[i] = CONST_DISCARD(char *, dom_list.domains[i].name.string);
+		(*domain_names)[i] = discard_const_p(char, dom_list.domains[i].name.string);
 		(*sids)[i] = *dom_list.domains[i].sid;
 	}
 
@@ -263,7 +264,7 @@ NTSTATUS change_trust_account_password( const char *domain, const char *remote_m
 	/* if this next call fails, then give up.  We can't do
 	   password changes on BDC's  --jerry */
 
-	if (!NT_STATUS_IS_OK(cli_full_connection(&cli, global_myname(), dc_name,
+	if (!NT_STATUS_IS_OK(cli_full_connection(&cli, lp_netbios_name(), dc_name,
 					   NULL, 0,
 					   "IPC$", "IPC",
 					   "", "",
diff --git a/source3/libsmb/unexpected.c b/source3/libsmb/unexpected.c
index c5ad9ecb595..cf9ed7dbc6d 100644
--- a/source3/libsmb/unexpected.c
+++ b/source3/libsmb/unexpected.c
@@ -19,6 +19,7 @@
 */
 
 #include "includes.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "lib/async_req/async_sock.h"
 #include "libsmb/nmblib.h"
 
@@ -72,7 +73,7 @@ NTSTATUS nb_packet_server_create(TALLOC_CTX *mem_ctx,
 	struct tevent_fd *fde;
 	NTSTATUS status;
 
-	result = TALLOC_ZERO_P(mem_ctx, struct nb_packet_server);
+	result = talloc_zero(mem_ctx, struct nb_packet_server);
 	if (result == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto fail;
@@ -139,7 +140,7 @@ static void nb_packet_server_listener(struct tevent_context *ev,
 	}
 	DEBUG(6,("accepted socket %d\n", sock));
 
-	client = TALLOC_ZERO_P(server, struct nb_packet_client);
+	client = talloc_zero(server, struct nb_packet_client);
 	if (client == NULL) {
 		DEBUG(10, ("talloc failed\n"));
 		close(sock);
@@ -377,7 +378,7 @@ static void nb_packet_client_send(struct nb_packet_client *client,
 		return;
 	}
 
-	state = TALLOC_ZERO_P(client, struct nb_packet_client_state);
+	state = talloc_zero(client, struct nb_packet_client_state);
 	if (state == NULL) {
 		DEBUG(10, ("talloc failed\n"));
 		return;
@@ -484,7 +485,7 @@ struct tevent_req *nb_packet_reader_send(TALLOC_CTX *mem_ctx,
 		state->query.mailslot_namelen = strlen(mailslot_name);
 	}
 
-	state->reader = TALLOC_ZERO_P(state, struct nb_packet_reader);
+	state->reader = talloc_zero(state, struct nb_packet_reader);
 	if (tevent_req_nomem(state->reader, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/locale/net/de.po b/source3/locale/net/de.po
index a12fabd9001..2244cd6983b 100644
--- a/source3/locale/net/de.po
+++ b/source3/locale/net/de.po
@@ -415,7 +415,7 @@ msgid "  net getdomainsid"
 msgstr ""
 
 #: ../../utils/net.c:663
-msgid "Display the maximul RID currently used"
+msgid "Display the maximum RID currently used"
 msgstr "Die höchste verwendete RID anzeigen"
 
 #: ../../utils/net.c:664
@@ -924,7 +924,7 @@ msgstr ""
 #: ../../utils/net_ads.c:1070
 #, c-format
 msgid "Join is OK\n"
-msgstr "Beitritt ist OK\n"
+msgstr "Mitgliedschaft ist OK\n"
 
 #: ../../utils/net_ads.c:1081
 msgid "Host is not configured as a member server.\n"
diff --git a/source3/locking/brlock.c b/source3/locking/brlock.c
index c325338062d..6eef695f691 100644
--- a/source3/locking/brlock.c
+++ b/source3/locking/brlock.c
@@ -5,17 +5,17 @@
 
    Copyright (C) Andrew Tridgell 1992-2000
    Copyright (C) Jeremy Allison 1992-2000
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
@@ -51,8 +51,8 @@ static void print_lock_struct(unsigned int i, struct lock_struct *pls)
 			i,
 			(unsigned long long)pls->context.smblctx,
 			(unsigned int)pls->context.tid,
-			procid_str(talloc_tos(), &pls->context.pid) ));
-	
+			server_id_str(talloc_tos(), &pls->context.pid) ));
+
 	DEBUG(10,("start = %.0f, size = %.0f, fnum = %d, %s %s\n",
 		(double)pls->start,
 		(double)pls->size,
@@ -180,7 +180,7 @@ static bool brl_conflict1(const struct lock_struct *lck1,
 	    lck2->start >= (lck1->start + lck1->size)) {
 		return False;
 	}
-	    
+
 	return True;
 } 
 #endif
@@ -1521,7 +1521,7 @@ void brl_close_fnum(struct messaging_context *msg_ctx,
 
 			/* Copy the current lock array. */
 			if (br_lck->num_locks) {
-				locks_copy = (struct lock_struct *)TALLOC_MEMDUP(br_lck, locks, br_lck->num_locks * sizeof(struct lock_struct));
+				locks_copy = (struct lock_struct *)talloc_memdup(br_lck, locks, br_lck->num_locks * sizeof(struct lock_struct));
 				if (!locks_copy) {
 					smb_panic("brl_close_fnum: talloc failed");
 	 			}
@@ -1823,7 +1823,7 @@ static struct byte_range_lock *brl_get_locks_internal(TALLOC_CTX *mem_ctx,
 					files_struct *fsp, bool read_only)
 {
 	TDB_DATA key, data;
-	struct byte_range_lock *br_lck = TALLOC_P(mem_ctx, struct byte_range_lock);
+	struct byte_range_lock *br_lck = talloc(mem_ctx, struct byte_range_lock);
 	bool do_read_only = read_only;
 
 	if (br_lck == NULL) {
@@ -1845,7 +1845,7 @@ static struct byte_range_lock *brl_get_locks_internal(TALLOC_CTX *mem_ctx,
 	}
 
 	if (do_read_only) {
-		if (brlock_db->fetch(brlock_db, br_lck, key, &data) == -1) {
+		if (brlock_db->fetch(brlock_db, br_lck, key, &data) != 0) {
 			DEBUG(3, ("Could not fetch byte range lock record\n"));
 			TALLOC_FREE(br_lck);
 			return NULL;
@@ -1881,7 +1881,7 @@ static struct byte_range_lock *brl_get_locks_internal(TALLOC_CTX *mem_ctx,
 
 		memcpy(br_lck->lock_data, data.dptr, data.dsize);
 	}
-	
+
 	if (!fsp->lockdb_clean) {
 		int orig_num_locks = br_lck->num_locks;
 
@@ -1993,8 +1993,8 @@ static void brl_revalidate_collect(struct file_id id, struct server_id pid,
 
 static int compare_procids(const void *p1, const void *p2)
 {
-	const struct server_id *i1 = (struct server_id *)p1;
-	const struct server_id *i2 = (struct server_id *)p2;
+	const struct server_id *i1 = (const struct server_id *)p1;
+	const struct server_id *i2 = (const struct server_id *)p2;
 
 	if (i1->pid < i2->pid) return -1;
 	if (i2->pid > i2->pid) return 1;
@@ -2020,7 +2020,7 @@ static void brl_revalidate(struct messaging_context *msg_ctx,
 	uint32 i;
 	struct server_id last_pid;
 
-	if (!(state = TALLOC_ZERO_P(NULL, struct brl_revalidate_state))) {
+	if (!(state = talloc_zero(NULL, struct brl_revalidate_state))) {
 		DEBUG(0, ("talloc failed\n"));
 		return;
 	}
diff --git a/source3/locking/locking.c b/source3/locking/locking.c
index f5892ddf0d7..81e93a555a1 100644
--- a/source3/locking/locking.c
+++ b/source3/locking/locking.c
@@ -4,17 +4,17 @@
    Copyright (C) Andrew Tridgell 1992-2000
    Copyright (C) Jeremy Allison 1992-2006
    Copyright (C) Volker Lendecke 2005
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
@@ -43,6 +43,7 @@
 #include "../libcli/security/security.h"
 #include "serverid.h"
 #include "messages.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_LOCKING
@@ -309,15 +310,15 @@ NTSTATUS do_unlock(struct messaging_context *msg_ctx,
 {
 	bool ok = False;
 	struct byte_range_lock *br_lck = NULL;
-	
+
 	if (!fsp->can_lock) {
 		return fsp->is_directory ? NT_STATUS_INVALID_DEVICE_REQUEST : NT_STATUS_INVALID_HANDLE;
 	}
-	
+
 	if (!lp_locking(fsp->conn->params)) {
 		return NT_STATUS_OK;
 	}
-	
+
 	DEBUG(10,("do_unlock: unlock start=%.0f len=%.0f requested for fnum %d file %s\n",
 		  (double)offset, (double)count, fsp->fnum,
 		  fsp_str_dbg(fsp)));
@@ -334,7 +335,7 @@ NTSTATUS do_unlock(struct messaging_context *msg_ctx,
 			offset,
 			count,
 			lock_flav);
-   
+
 	TALLOC_FREE(br_lck);
 
 	if (!ok) {
@@ -364,7 +365,7 @@ NTSTATUS do_lock_cancel(files_struct *fsp,
 		return fsp->is_directory ?
 			NT_STATUS_INVALID_DEVICE_REQUEST : NT_STATUS_INVALID_HANDLE;
 	}
-	
+
 	if (!lp_locking(fsp->conn->params)) {
 		return NT_STATUS_DOS(ERRDOS, ERRcancelviolation);
 	}
@@ -578,7 +579,7 @@ static int parse_delete_tokens_list(struct share_mode_lock *lck,
 
 		p += sizeof(token_len);
 
-		pdtl = TALLOC_ZERO_P(lck, struct delete_token_list);
+		pdtl = talloc_zero(lck, struct delete_token_list);
 		if (pdtl == NULL) {
 			DEBUG(0,("parse_delete_tokens_list: talloc failed"));
 			return -1;
@@ -587,7 +588,7 @@ static int parse_delete_tokens_list(struct share_mode_lock *lck,
 		memcpy(&pdtl->name_hash, p, sizeof(pdtl->name_hash));
 		p += sizeof(pdtl->name_hash);
 
-		pdtl->delete_token = TALLOC_ZERO_P(pdtl, struct security_unix_token);
+		pdtl->delete_token = talloc_zero(pdtl, struct security_unix_token);
 		if (pdtl->delete_token == NULL) {
 			DEBUG(0,("parse_delete_tokens_list: talloc failed"));
 			return -1;
@@ -615,7 +616,7 @@ static int parse_delete_tokens_list(struct share_mode_lock *lck,
 			}
 
 			pdtl->delete_token->ngroups = token_len / sizeof(gid_t);
-			pdtl->delete_token->groups = TALLOC_ARRAY(pdtl->delete_token, gid_t,
+			pdtl->delete_token->groups = talloc_array(pdtl->delete_token, gid_t,
 						pdtl->delete_token->ngroups);
 			if (pdtl->delete_token->groups == NULL) {
 				DEBUG(0,("parse_delete_tokens_list: talloc failed"));
@@ -671,7 +672,7 @@ static bool parse_share_modes(const TDB_DATA dbuf, struct share_mode_lock *lck)
 	}
 
 	lck->share_modes = NULL;
-	
+
 	if (lck->num_share_modes != 0) {
 
 		if (dbuf.dsize < (sizeof(struct locking_data) +
@@ -679,9 +680,9 @@ static bool parse_share_modes(const TDB_DATA dbuf, struct share_mode_lock *lck)
 				   sizeof(struct share_mode_entry)))) {
 			smb_panic("parse_share_modes: buffer too short");
 		}
-				  
+
 		lck->share_modes = (struct share_mode_entry *)
-			TALLOC_MEMDUP(lck,
+			talloc_memdup(lck,
 				      dbuf.dptr+sizeof(struct locking_data),
 				      lck->num_share_modes *
 				      sizeof(struct share_mode_entry));
@@ -781,7 +782,7 @@ static TDB_DATA unparse_share_modes(const struct share_mode_lock *lck)
 		sp_len + 1 +
 		bn_len + 1 +
 		sn_len + 1;
-	result.dptr = TALLOC_ARRAY(lck, uint8, result.dsize);
+	result.dptr = talloc_array(lck, uint8, result.dsize);
 
 	if (result.dptr == NULL) {
 		smb_panic("talloc failed");
@@ -838,14 +839,17 @@ static TDB_DATA unparse_share_modes(const struct share_mode_lock *lck)
 		offset += token_size;
 	}
 
-	safe_strcpy((char *)result.dptr + offset, lck->servicepath,
-		    result.dsize - offset - 1);
+	strlcpy((char *)result.dptr + offset,
+		lck->servicepath ? lck->servicepath : "",
+		result.dsize - offset);
 	offset += sp_len + 1;
-	safe_strcpy((char *)result.dptr + offset, lck->base_name,
-		    result.dsize - offset - 1);
+	strlcpy((char *)result.dptr + offset,
+		lck->base_name ? lck->base_name : "",
+		result.dsize - offset);
 	offset += bn_len + 1;
-	safe_strcpy((char *)result.dptr + offset, lck->stream_name,
-		    result.dsize - offset - 1);
+	strlcpy((char *)result.dptr + offset,
+		lck->stream_name ? lck->stream_name : "",
+		result.dsize - offset);
 
 	if (DEBUGLEVEL >= 10) {
 		print_share_mode_table(data);
@@ -969,7 +973,7 @@ struct share_mode_lock *get_share_mode_lock(TALLOC_CTX *mem_ctx,
 	struct file_id tmp;
 	TDB_DATA key = locking_key(&id, &tmp);
 
-	if (!(lck = TALLOC_P(mem_ctx, struct share_mode_lock))) {
+	if (!(lck = talloc(mem_ctx, struct share_mode_lock))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
@@ -1000,12 +1004,12 @@ struct share_mode_lock *fetch_share_mode_unlocked(TALLOC_CTX *mem_ctx,
 	TDB_DATA key = locking_key(&id, &tmp);
 	TDB_DATA data;
 
-	if (!(lck = TALLOC_P(mem_ctx, struct share_mode_lock))) {
+	if (!(lck = talloc(mem_ctx, struct share_mode_lock))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
 
-	if (lock_db->fetch(lock_db, lck, key, &data) == -1) {
+	if (lock_db->fetch(lock_db, lck, key, &data) != 0) {
 		DEBUG(3, ("Could not fetch share entry\n"));
 		TALLOC_FREE(lck);
 		return NULL;
@@ -1078,7 +1082,7 @@ bool rename_share_filename(struct messaging_context *msg_ctx,
 	    sn_len + 1;
 
 	/* Set up the name changed message. */
-	frm = TALLOC_ARRAY(lck, char, msg_len);
+	frm = talloc_array(lck, char, msg_len);
 	if (!frm) {
 		return False;
 	}
@@ -1087,10 +1091,15 @@ bool rename_share_filename(struct messaging_context *msg_ctx,
 
 	DEBUG(10,("rename_share_filename: msg_len = %u\n", (unsigned int)msg_len ));
 
-	safe_strcpy(&frm[24], lck->servicepath, sp_len);
-	safe_strcpy(&frm[24 + sp_len + 1], lck->base_name, bn_len);
-	safe_strcpy(&frm[24 + sp_len + 1 + bn_len + 1], lck->stream_name,
-		    sn_len);
+	strlcpy(&frm[24],
+		lck->servicepath ? lck->servicepath : "",
+		sp_len+1);
+	strlcpy(&frm[24 + sp_len + 1],
+		lck->base_name ? lck->base_name : "",
+		bn_len+1);
+	strlcpy(&frm[24 + sp_len + 1 + bn_len + 1],
+		lck->stream_name ? lck->stream_name : "",
+		sn_len+1);
 
 	/* Send the messages. */
 	for (i=0; i<lck->num_share_modes; i++) {
@@ -1424,7 +1433,7 @@ NTSTATUS can_set_delete_on_close(files_struct *fsp, uint32 dosmode)
 	 * Only allow delete on close for writable files.
 	 */
 
-	if ((dosmode & aRONLY) &&
+	if ((dosmode & FILE_ATTRIBUTE_READONLY) &&
 	    !lp_delete_readonly(SNUM(fsp->conn))) {
 		DEBUG(10,("can_set_delete_on_close: file %s delete on close "
 			  "flag set but file attribute is readonly.\n",
@@ -1474,7 +1483,7 @@ static struct security_unix_token *copy_unix_token(TALLOC_CTX *ctx, const struct
 {
 	struct security_unix_token *cpy;
 
-	cpy = TALLOC_P(ctx, struct security_unix_token);
+	cpy = talloc(ctx, struct security_unix_token);
 	if (!cpy) {
 		return NULL;
 	}
@@ -1484,11 +1493,12 @@ static struct security_unix_token *copy_unix_token(TALLOC_CTX *ctx, const struct
 	cpy->ngroups = tok->ngroups;
 	if (tok->ngroups) {
 		/* Make this a talloc child of cpy. */
-		cpy->groups = TALLOC_ARRAY(cpy, gid_t, tok->ngroups);
+		cpy->groups = (gid_t *)talloc_memdup(
+			cpy, tok->groups, tok->ngroups * sizeof(gid_t));
 		if (!cpy->groups) {
+			TALLOC_FREE(cpy);
 			return NULL;
 		}
-		memcpy(cpy->groups, tok->groups, tok->ngroups * sizeof(gid_t));
 	}
 	return cpy;
 }
@@ -1503,7 +1513,7 @@ static bool add_delete_on_close_token(struct share_mode_lock *lck,
 {
 	struct delete_token_list *dtl;
 
-	dtl = TALLOC_ZERO_P(lck, struct delete_token_list);
+	dtl = talloc_zero(lck, struct delete_token_list);
 	if (dtl == NULL) {
 		return false;
 	}
@@ -1573,7 +1583,7 @@ void set_delete_on_close_lck(files_struct *fsp,
 bool set_delete_on_close(files_struct *fsp, bool delete_on_close, const struct security_unix_token *tok)
 {
 	struct share_mode_lock *lck;
-	
+
 	DEBUG(10,("set_delete_on_close: %s delete on close flag for "
 		  "fnum = %d, file %s\n",
 		  delete_on_close ? "Adding" : "Removing", fsp->fnum,
diff --git a/source3/locking/posix.c b/source3/locking/posix.c
index cea8d2f8a2e..51151df9a43 100644
--- a/source3/locking/posix.c
+++ b/source3/locking/posix.c
@@ -25,6 +25,7 @@
 #include "system/filesys.h"
 #include "locking/proto.h"
 #include "dbwrap.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_LOCKING
@@ -557,7 +558,7 @@ static void add_fd_to_close_entry(files_struct *fsp)
 
 	SMB_ASSERT(rec != NULL);
 
-	new_data = TALLOC_ARRAY(
+	new_data = talloc_array(
 		rec, uint8_t, rec->value.dsize + sizeof(fsp->fh->fd));
 
 	SMB_ASSERT(new_data != NULL);
@@ -891,7 +892,7 @@ BECOMES.....
         | l_curr|         | l_new   |
         +-------+         +---------+
 **********************************************/
-				struct lock_list *l_new = TALLOC_P(ctx, struct lock_list);
+				struct lock_list *l_new = talloc(ctx, struct lock_list);
 
 				if(l_new == NULL) {
 					DEBUG(0,("posix_lock_list: talloc fail.\n"));
@@ -999,7 +1000,7 @@ bool set_posix_lock_windows_flavour(files_struct *fsp,
 		return False;
 	}
 
-	if ((ll = TALLOC_P(l_ctx, struct lock_list)) == NULL) {
+	if ((ll = talloc(l_ctx, struct lock_list)) == NULL) {
 		DEBUG(0,("set_posix_lock_windows_flavour: unable to talloc unlock list.\n"));
 		talloc_destroy(l_ctx);
 		return False;
@@ -1118,7 +1119,7 @@ bool release_posix_lock_windows_flavour(files_struct *fsp,
 		return False;
 	}
 
-	if ((ul = TALLOC_P(ul_ctx, struct lock_list)) == NULL) {
+	if ((ul = talloc(ul_ctx, struct lock_list)) == NULL) {
 		DEBUG(0,("release_posix_lock_windows_flavour: unable to talloc unlock list.\n"));
 		talloc_destroy(ul_ctx);
 		return False;
@@ -1280,7 +1281,7 @@ bool release_posix_lock_posix_flavour(files_struct *fsp,
 		return False;
 	}
 
-	if ((ul = TALLOC_P(ul_ctx, struct lock_list)) == NULL) {
+	if ((ul = talloc(ul_ctx, struct lock_list)) == NULL) {
 		DEBUG(0,("release_posix_lock_windows_flavour: unable to talloc unlock list.\n"));
 		talloc_destroy(ul_ctx);
 		return False;
diff --git a/source3/locking/proto.h b/source3/locking/proto.h
index 6f130961a86..b7c8990b1af 100644
--- a/source3/locking/proto.h
+++ b/source3/locking/proto.h
@@ -1,3 +1,27 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Locking functions
+ *
+ *  Copyright (C) Andrew Tridgell	1992-2000
+ *  Copyright (C) Jeremy Allison	1992-2006
+ *  Copyright (C) Volker Lendecke	2005
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _LOCKING_PROTO_H_
+#define _LOCKING_PROTO_H_
 
 /* The following definitions come from locking/brlock.c  */
 
@@ -199,3 +223,5 @@ bool release_posix_lock_posix_flavour(files_struct *fsp,
 				const struct lock_context *lock_ctx,
 				const struct lock_struct *plocks,
 				int num_locks);
+
+#endif /* _LOCKING_PROTO_H_ */
diff --git a/source3/m4/aclocal.m4 b/source3/m4/aclocal.m4
index 301445f562d..afd7c7491a1 100644
--- a/source3/m4/aclocal.m4
+++ b/source3/m4/aclocal.m4
@@ -54,7 +54,7 @@ AC_DEFUN(SMB_SUBSYSTEM,
 ])
 
 
-dnl SMB_LIBRARY(name, version, default, reason)
+dnl SMB_LIBRARY(name, soversion, fullversion, default, reason)
 dnl
 dnl configure build and use of an (internal) shared library
 dnl
@@ -74,6 +74,7 @@ LIBUC[_TARGET]=
 [UNINSTALL_]LIBUC=
 
 m4_if([$2], [], [LIBUC[_SOVER]=0], [LIBUC[_SOVER]=$2])
+m4_if([$3], [], [LIBUC[_FULLVER]=$LIBUC[_SOVER]], [LIBUC[_FULLVER]=$3])
 
 AC_SUBST(LIBUC[_SHARED_TARGET])
 AC_SUBST(LIBUC[_STATIC_TARGET])
@@ -84,15 +85,16 @@ AC_SUBST(LIBUC[_TARGET])
 AC_SUBST([INSTALL_]LIBUC)
 AC_SUBST([UNINSTALL_]LIBUC)
 AC_SUBST(LIBUC[_SOVER])
+AC_SUBST(LIBUC[_FULLVER])
 
 AC_MSG_CHECKING([whether to build the LIBNAME shared library])
-m4_if([$3], [no], [
+m4_if([$4], [no], [
 dnl set the default to not build the shared lib
 AC_ARG_WITH(LIBNAME,
 AS_HELP_STRING([--with-]LIBNAME,
-	m4_if([$4], [],
+	m4_if([$5], [],
 		[Build the LIBNAME shared library (default=no)],
-		[Build the LIBNAME shared library (default=no ($4))])),
+		[Build the LIBNAME shared library (default=no ($5))])),
 [
 case "$withval" in
 	yes)
diff --git a/source3/m4/samba_version.m4 b/source3/m4/samba_version.m4
index dbea1f3b3a1..8ee0232df81 100644
--- a/source3/m4/samba_version.m4
+++ b/source3/m4/samba_version.m4
@@ -8,7 +8,7 @@ dnl http://www.gnu.org/licenses/
 dnl
 dnl
 
-SMB_VERSION_STRING=`cat $srcdir/include/version.h | grep 'SAMBA_VERSION_OFFICIAL_STRING' | cut -d '"' -f2`
+SMB_VERSION_STRING=`cat $srcdir/include/version.h | grep '#define SAMBA_VERSION_OFFICIAL_STRING' | cut -d '"' -f2`
 echo "SAMBA VERSION: ${SMB_VERSION_STRING}"
 
 SAMBA_VERSION_GIT_COMMIT_FULLREV=`cat $srcdir/include/version.h | grep 'SAMBA_VERSION_GIT_COMMIT_FULLREV' | cut -d ' ' -f3- | cut -d '"' -f2`
diff --git a/source3/modules/gpfs.c b/source3/modules/gpfs.c
index ef770995483..d2832ba9dc3 100644
--- a/source3/modules/gpfs.c
+++ b/source3/modules/gpfs.c
@@ -18,6 +18,7 @@
  */
 
 #include "includes.h"
+#include "system/filesys.h"
 #include "smbd/smbd.h"
 
 #ifdef HAVE_GPFS
@@ -40,6 +41,7 @@ static int (*gpfs_set_winattrs_path_fn)(char *pathname, int flags, struct gpfs_w
 static int (*gpfs_get_winattrs_path_fn)(char *pathname, struct gpfs_winattr *attrs);
 static int (*gpfs_get_winattrs_fn)(int fd, struct gpfs_winattr *attrs);
 static int (*gpfs_ftruncate_fn)(int fd, gpfs_off64_t length);
+static int (*gpfs_lib_init_fn)(int flags);
 
 bool set_gpfs_sharemode(files_struct *fsp, uint32 access_mask,
 			uint32 share_access)
@@ -189,6 +191,17 @@ int set_gpfs_winattrs(char *pathname,int flags,struct gpfs_winattr *attrs)
         return gpfs_set_winattrs_path_fn(pathname,flags, attrs);
 }
 
+void smbd_gpfs_lib_init()
+{
+	if (gpfs_lib_init_fn) {
+		int rc = gpfs_lib_init_fn(0);
+		DEBUG(10, ("gpfs_lib_init() finished with rc %d "
+			   "and errno %d\n", rc, errno));
+	} else {
+		DEBUG(10, ("libgpfs lacks gpfs_lib_init\n"));
+	}
+}
+
 static bool init_gpfs_function_lib(void *plibhandle_pointer,
 				   const char *libname,
 				   void *pfn_pointer, const char *fn_name)
@@ -251,6 +264,7 @@ void init_gpfs(void)
         init_gpfs_function(&gpfs_set_winattrs_path_fn,"gpfs_set_winattrs_path");
         init_gpfs_function(&gpfs_get_winattrs_fn,"gpfs_get_winattrs");
 	init_gpfs_function(&gpfs_ftruncate_fn, "gpfs_ftruncate");
+        init_gpfs_function(&gpfs_lib_init_fn,"gpfs_lib_init");
 
 	gpfs_getrealfilename = lp_parm_bool(-1, "gpfs", "getrealfilename",
 					    True);
@@ -311,6 +325,11 @@ int get_gpfs_winattrs(char *pathname,struct gpfs_winattr *attrs)
         return -1;
 }
 
+void smbd_gpfs_lib_init()
+{
+	return;
+}
+
 void init_gpfs(void)
 {
 	return;
diff --git a/source3/modules/nfs4_acls.c b/source3/modules/nfs4_acls.c
index 14d464451fa..dbfa36e6c2a 100644
--- a/source3/modules/nfs4_acls.c
+++ b/source3/modules/nfs4_acls.c
@@ -26,6 +26,7 @@
 #include "include/dbwrap.h"
 #include "system/filesys.h"
 #include "passdb/lookup_sid.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_ACLS
@@ -51,6 +52,57 @@ typedef struct _SMB_ACL4_INT_T
 	SMB_ACE4_INT_T	*last;
 } SMB_ACL4_INT_T;
 
+/************************************************
+ Split the ACE flag mapping between nfs4 and Windows
+ into two separate functions rather than trying to do
+ it inline. Allows us to carefully control what flags
+ are mapped to what in one place.
+************************************************/
+
+static uint32_t map_nfs4_ace_flags_to_windows_ace_flags(uint32_t nfs4_ace_flags)
+{
+	uint32_t win_ace_flags = 0;
+
+	/* The nfs4 flags <= 0xf map perfectly. */
+	win_ace_flags = nfs4_ace_flags & (SEC_ACE_FLAG_OBJECT_INHERIT|
+				      SEC_ACE_FLAG_CONTAINER_INHERIT|
+				      SEC_ACE_FLAG_NO_PROPAGATE_INHERIT|
+				      SEC_ACE_FLAG_INHERIT_ONLY);
+
+	/* flags greater than 0xf have diverged :-(. */
+	/* See the nfs4 ace flag definitions here:
+	   http://www.ietf.org/rfc/rfc3530.txt.
+	   And the Windows ace flag definitions here:
+	   librpc/idl/security.idl. */
+	if (nfs4_ace_flags & SMB_ACE4_INHERITED_ACE) {
+		win_ace_flags |= SEC_ACE_FLAG_INHERITED_ACE;
+	}
+
+	return win_ace_flags;
+}
+
+static uint32_t map_windows_ace_flags_to_nfs4_ace_flags(uint32_t win_ace_flags)
+{
+	uint32_t nfs4_ace_flags = 0;
+
+	/* The windows flags <= 0xf map perfectly. */
+	nfs4_ace_flags = win_ace_flags & (SMB_ACE4_FILE_INHERIT_ACE|
+				      SMB_ACE4_DIRECTORY_INHERIT_ACE|
+				      SMB_ACE4_NO_PROPAGATE_INHERIT_ACE|
+				      SMB_ACE4_INHERIT_ONLY_ACE);
+
+	/* flags greater than 0xf have diverged :-(. */
+	/* See the nfs4 ace flag definitions here:
+	   http://www.ietf.org/rfc/rfc3530.txt.
+	   And the Windows ace flag definitions here:
+	   librpc/idl/security.idl. */
+	if (win_ace_flags & SEC_ACE_FLAG_INHERITED_ACE) {
+		nfs4_ace_flags |= SMB_ACE4_INHERITED_ACE;
+	}
+
+	return nfs4_ace_flags;
+}
+
 static SMB_ACL4_INT_T *get_validated_aclint(SMB4ACL_T *theacl)
 {
 	SMB_ACL4_INT_T *aclint = (SMB_ACL4_INT_T *)theacl;
@@ -233,7 +285,7 @@ static bool smbacl4_nfs42win(TALLOC_CTX *mem_ctx, SMB4ACL_T *theacl, /* in */
 		uint32_t mask;
 		struct dom_sid sid;
 		SMB_ACE4PROP_T	*ace = &aceint->prop;
-		uint32_t mapped_ace_flags;
+		uint32_t win_ace_flags;
 
 		DEBUG(10, ("magic: 0x%x, type: %d, iflags: %x, flags: %x, mask: %x, "
 			"who: %d\n", aceint->magic, ace->aceType, ace->flags,
@@ -270,25 +322,25 @@ static bool smbacl4_nfs42win(TALLOC_CTX *mem_ctx, SMB4ACL_T *theacl, /* in */
 			ace->aceMask |= SMB_ACE4_DELETE_CHILD;
 		}
 
-		mapped_ace_flags = ace->aceFlags & 0xf;
-		if (!is_directory && (mapped_ace_flags & (SMB_ACE4_FILE_INHERIT_ACE|SMB_ACE4_DIRECTORY_INHERIT_ACE))) {
+		win_ace_flags = map_nfs4_ace_flags_to_windows_ace_flags(ace->aceFlags);
+		if (!is_directory && (win_ace_flags & (SEC_ACE_FLAG_OBJECT_INHERIT|SEC_ACE_FLAG_CONTAINER_INHERIT))) {
 			/*
 			 * GPFS sets inherits dir_inhert and file_inherit flags
 			 * to files, too, which confuses windows, and seems to
 			 * be wrong anyways. ==> Map these bits away for files.
 			 */
 			DEBUG(10, ("removing inherit flags from nfs4 ace\n"));
-			mapped_ace_flags &= ~(SMB_ACE4_FILE_INHERIT_ACE|SMB_ACE4_DIRECTORY_INHERIT_ACE);
+			win_ace_flags &= ~(SEC_ACE_FLAG_OBJECT_INHERIT|SEC_ACE_FLAG_CONTAINER_INHERIT);
 		}
-		DEBUG(10, ("mapped ace flags: 0x%x => 0x%x\n",
-		      ace->aceFlags, mapped_ace_flags));
+		DEBUG(10, ("Windows mapped ace flags: 0x%x => 0x%x\n",
+		      ace->aceFlags, win_ace_flags));
 
 		/* Windows clients expect SYNC on acls to
 		   correctly allow rename. See bug #7909. */
 		mask = ace->aceMask | SMB_ACE4_SYNCHRONIZE;
 		init_sec_ace(&nt_ace_list[good_aces++], &sid,
 			ace->aceType, mask,
-			mapped_ace_flags);
+			win_ace_flags);
 	}
 
 	*ppnt_ace_list = nt_ace_list;
@@ -397,13 +449,15 @@ static int smbacl4_get_vfs_params(
 {
 	static const struct enum_list enum_smbacl4_modes[] = {
 		{ e_simple, "simple" },
-		{ e_special, "special" }
+		{ e_special, "special" },
+		{ -1 , NULL }
 	};
 	static const struct enum_list enum_smbacl4_acedups[] = {
 		{ e_dontcare, "dontcare" },
 		{ e_reject, "reject" },
 		{ e_ignore, "ignore" },
 		{ e_merge, "merge" },
+		{ -1 , NULL }
 	};
 
 	memset(params, 0, sizeof(smbacl4_vfs_params));
@@ -520,7 +574,7 @@ static bool nfs4_map_sid(smbacl4_vfs_params *params, const struct dom_sid *src,
 	
 	if (mapping_db->fetch(mapping_db, NULL,
 			      string_term_tdb_data(sid_string_tos(src)),
-			      &data) == -1) {
+			      &data) != 0) {
 		DEBUG(10, ("could not find mapping for SID %s\n",
 			   sid_string_dbg(src)));
 		return False;
@@ -560,7 +614,7 @@ static bool smbacl4_fill_ace4(
 
 	memset(ace_v4, 0, sizeof(SMB_ACE4PROP_T));
 	ace_v4->aceType = ace_nt->type; /* only ACCESS|DENY supported right now */
-	ace_v4->aceFlags = ace_nt->flags & SEC_ACE_FLAG_VALID_INHERIT;
+	ace_v4->aceFlags = map_windows_ace_flags_to_nfs4_ace_flags(ace_nt->flags);
 	ace_v4->aceMask = ace_nt->access_mask &
 		(SEC_STD_ALL | SEC_FILE_ALL);
 
diff --git a/source3/modules/nfs4_acls.h b/source3/modules/nfs4_acls.h
index f4576b0c137..fcab6359154 100644
--- a/source3/modules/nfs4_acls.h
+++ b/source3/modules/nfs4_acls.h
@@ -76,9 +76,10 @@ typedef struct _SMB_ACE4PROP_T {
 #define SMB_ACE4_SUCCESSFUL_ACCESS_ACE_FLAG   0x00000010
 #define SMB_ACE4_FAILED_ACCESS_ACE_FLAG       0x00000020
 #define SMB_ACE4_IDENTIFIER_GROUP             0x00000040
+#define SMB_ACE4_INHERITED_ACE                0x00000080
 #define SMB_ACE4_ALL_FLAGS	( SMB_ACE4_FILE_INHERIT_ACE | SMB_ACE4_DIRECTORY_INHERIT_ACE \
 | SMB_ACE4_NO_PROPAGATE_INHERIT_ACE | SMB_ACE4_INHERIT_ONLY_ACE | SMB_ACE4_SUCCESSFUL_ACCESS_ACE_FLAG \
-| SMB_ACE4_FAILED_ACCESS_ACE_FLAG | SMB_ACE4_IDENTIFIER_GROUP )
+| SMB_ACE4_FAILED_ACCESS_ACE_FLAG | SMB_ACE4_IDENTIFIER_GROUP | SMB_ACE4_INHERITED_ACE)
 
 	uint32	aceMask;	/* Access rights */
 /*The bitmask constants used for the access mask field are as follows: */
diff --git a/source3/modules/onefs_acl.c b/source3/modules/onefs_acl.c
index e986b2a9789..f78d1f0e2e6 100644
--- a/source3/modules/onefs_acl.c
+++ b/source3/modules/onefs_acl.c
@@ -554,7 +554,7 @@ static bool add_sfs_aces(files_struct *fsp, struct ifs_security_descriptor *sd)
 
 		/* Use existing samba logic to derive the mode bits. */
 		file_mode = unix_mode(fsp->conn, 0, fsp->fsp_name, NULL);
-		dir_mode = unix_mode(fsp->conn, aDIR, fsp->fsp_name, NULL);
+		dir_mode = unix_mode(fsp->conn, FILE_ATTRIBUTE_DIRECTORY, fsp->fsp_name, NULL);
 
 		/* Initialize ACEs. */
 		new_aces[0] = onefs_init_ace(fsp->conn, file_mode, false, USR);
diff --git a/source3/modules/onefs_cbrl.c b/source3/modules/onefs_cbrl.c
index 5c69bec566c..2f20d04ddb6 100644
--- a/source3/modules/onefs_cbrl.c
+++ b/source3/modules/onefs_cbrl.c
@@ -251,7 +251,7 @@ static void onefs_init_cbrl(void)
 	DEBUG(10, ("cbrl_event_fd = %d\n", cbrl_event_fd));
 
 	/* Register the CBRL event_fd with samba's event system */
-	cbrl_fde = event_add_fd(smbd_event_context(),
+	cbrl_fde = event_add_fd(server_event_context(),
 				     NULL,
 				     cbrl_event_fd,
 				     EVENT_FD_READ,
diff --git a/source3/modules/onefs_open.c b/source3/modules/onefs_open.c
index 80f799ca6f6..101dc5bc6e2 100644
--- a/source3/modules/onefs_open.c
+++ b/source3/modules/onefs_open.c
@@ -498,9 +498,9 @@ NTSTATUS onefs_open_file_ntcreate(connection_struct *conn,
 		unx_mode = (mode_t)(new_dos_attributes & ~FILE_FLAG_POSIX_SEMANTICS);
 		new_dos_attributes = 0;
 	} else {
-		/* We add aARCH to this as this mode is only used if the file is
+		/* We add FILE_ATTRIBUTE_ARCHIVE to this as this mode is only used if the file is
 		 * created new. */
-		unx_mode = unix_mode(conn, new_dos_attributes | aARCH,
+		unx_mode = unix_mode(conn, new_dos_attributes | FILE_ATTRIBUTE_ARCHIVE,
 				     smb_fname, parent_dir);
 	}
 
@@ -587,7 +587,7 @@ NTSTATUS onefs_open_file_ntcreate(connection_struct *conn,
 
 	/* Setup dos_attributes to be set by ifs_createfile */
 	if (lp_store_dos_attributes(SNUM(conn))) {
-		createfile_attributes = (new_dos_attributes | aARCH) &
+		createfile_attributes = (new_dos_attributes | FILE_ATTRIBUTE_ARCHIVE) &
 		    ~(FILE_ATTRIBUTE_NONINDEXED | FILE_ATTRIBUTE_COMPRESSED);
 	}
 
@@ -1310,7 +1310,8 @@ NTSTATUS onefs_open_file_ntcreate(connection_struct *conn,
 		new_file_created = True;
 	}
 
-	set_share_mode(lck, fsp, get_current_uid(conn), 0,
+	set_share_mode(lck, fsp, get_current_uid(conn),
+			req ? req->mid : 0,
 		       fsp->oplock_type);
 
 	/* Handle strange delete on close create semantics. */
@@ -1482,7 +1483,7 @@ static NTSTATUS onefs_open_directory(connection_struct *conn,
 		mode = (mode_t)(file_attributes & ~FILE_FLAG_POSIX_SEMANTICS);
 		file_attributes = 0;
 	} else {
-		mode = unix_mode(conn, aDIR, smb_dname, parent_dir);
+		mode = unix_mode(conn, FILE_ATTRIBUTE_DIRECTORY, smb_dname, parent_dir);
 	}
 
 	/*
@@ -1666,7 +1667,8 @@ static NTSTATUS onefs_open_directory(connection_struct *conn,
 		return NT_STATUS_DELETE_PENDING;
 	}
 
-	set_share_mode(lck, fsp, get_current_uid(conn), 0, NO_OPLOCK);
+	set_share_mode(lck, fsp, get_current_uid(conn),
+		req ? req->mid : 0, NO_OPLOCK);
 
 	/*
 	 * For directories the delete on close bit at open time seems
diff --git a/source3/modules/onefs_streams.c b/source3/modules/onefs_streams.c
index 85ab25683f3..78230bcad67 100644
--- a/source3/modules/onefs_streams.c
+++ b/source3/modules/onefs_streams.c
@@ -56,7 +56,7 @@ NTSTATUS onefs_stream_prep_smb_fname(TALLOC_CTX *ctx,
 		/* Strip off the :$DATA if one exists. */
 		str_tmp = strrchr_m(stream_name, ':');
 		if (str_tmp) {
-			if (StrCaseCmp(str_tmp, ":$DATA") != 0) {
+			if (strcasecmp_m(str_tmp, ":$DATA") != 0) {
 				return NT_STATUS_INVALID_PARAMETER;
 			}
 			str_tmp[0] = '\0';
@@ -552,7 +552,7 @@ static bool add_one_stream(TALLOC_CTX *mem_ctx, unsigned int *num_streams,
 {
 	struct stream_struct *tmp;
 
-	tmp = TALLOC_REALLOC_ARRAY(mem_ctx, *streams, struct stream_struct,
+	tmp = talloc_realloc(mem_ctx, *streams, struct stream_struct,
 				   (*num_streams)+1);
 	if (tmp == NULL) {
 		return false;
diff --git a/source3/modules/vfs_acl_common.c b/source3/modules/vfs_acl_common.c
index 6c57acb13dd..bee7966dfc9 100644
--- a/source3/modules/vfs_acl_common.c
+++ b/source3/modules/vfs_acl_common.c
@@ -144,7 +144,7 @@ static NTSTATUS create_acl_blob(const struct security_descriptor *psd,
 
 	xacl.version = 3;
 	xacl.info.sd_hs3 = &sd_hs3;
-	xacl.info.sd_hs3->sd = CONST_DISCARD(struct security_descriptor *, psd);
+	xacl.info.sd_hs3->sd = discard_const_p(struct security_descriptor, psd);
 	xacl.info.sd_hs3->hash_type = hash_type;
 	memcpy(&xacl.info.sd_hs3->hash[0], hash, XATTR_SD_HASH_SIZE);
 
@@ -179,7 +179,7 @@ static void add_directory_inheritable_components(vfs_handle_struct *handle,
 	mode_t dir_mode;
 	mode_t file_mode;
 	mode_t mode;
-	struct security_ace *new_ace_list = TALLOC_ZERO_ARRAY(talloc_tos(),
+	struct security_ace *new_ace_list = talloc_zero_array(talloc_tos(),
 						struct security_ace,
 						num_aces + 3);
 
@@ -190,7 +190,7 @@ static void add_directory_inheritable_components(vfs_handle_struct *handle,
 	/* Fake a quick smb_filename. */
 	ZERO_STRUCT(smb_fname);
 	smb_fname.st = *psbuf;
-	smb_fname.base_name = CONST_DISCARD(char *, name);
+	smb_fname.base_name = discard_const_p(char, name);
 
 	dir_mode = unix_mode(conn,
 			FILE_ATTRIBUTE_DIRECTORY, &smb_fname, NULL);
@@ -448,10 +448,14 @@ static NTSTATUS inherit_new_acl(vfs_handle_struct *handle,
 	struct security_descriptor *psd = NULL;
 	struct dom_sid *owner_sid = NULL;
 	struct dom_sid *group_sid = NULL;
+	uint32_t security_info_sent = (SECINFO_OWNER | SECINFO_GROUP | SECINFO_DACL);
 	bool inherit_owner = lp_inherit_owner(SNUM(handle->conn));
+	bool inheritable_components = sd_has_inheritable_components(parent_desc,
+					is_directory);
 	size_t size;
 
-	if (!sd_has_inheritable_components(parent_desc, is_directory)) {
+	if (!inheritable_components && !inherit_owner) {
+		/* Nothing to inherit and not setting owner. */
 		return NT_STATUS_OK;
 	}
 
@@ -487,6 +491,17 @@ static NTSTATUS inherit_new_acl(vfs_handle_struct *handle,
 		return status;
 	}
 
+	/* If inheritable_components == false,
+	   se_create_child_secdesc()
+	   creates a security desriptor with a NULL dacl
+	   entry, but with SEC_DESC_DACL_PRESENT. We need
+	   to remove that flag. */
+
+	if (!inheritable_components) {
+		security_info_sent &= ~SECINFO_DACL;
+		psd->type &= ~SEC_DESC_DACL_PRESENT;
+	}
+
 	if (DEBUGLEVEL >= 10) {
 		DEBUG(10,("inherit_new_acl: child acl for %s is:\n",
 			fsp_str_dbg(fsp) ));
@@ -498,9 +513,7 @@ static NTSTATUS inherit_new_acl(vfs_handle_struct *handle,
 		become_root();
 	}
 	status = SMB_VFS_FSET_NT_ACL(fsp,
-				(SECINFO_OWNER |
-				 SECINFO_GROUP |
-				 SECINFO_DACL),
+				security_info_sent,
 				psd);
 	if (inherit_owner) {
 		unbecome_root();
@@ -729,7 +742,7 @@ static NTSTATUS fset_nt_acl_common(vfs_handle_struct *handle, files_struct *fsp,
 		DEBUG(10,("fset_nt_acl_xattr: incoming sd for file %s\n",
 			  fsp_str_dbg(fsp)));
 		NDR_PRINT_DEBUG(security_descriptor,
-			CONST_DISCARD(struct security_descriptor *,orig_psd));
+			discard_const_p(struct security_descriptor, orig_psd));
 	}
 
 	status = get_nt_acl_internal(handle, fsp,
@@ -784,7 +797,7 @@ static NTSTATUS fset_nt_acl_common(vfs_handle_struct *handle, files_struct *fsp,
 		DEBUG(10,("fset_nt_acl_xattr: storing xattr sd for file %s\n",
 			  fsp_str_dbg(fsp)));
 		NDR_PRINT_DEBUG(security_descriptor,
-			CONST_DISCARD(struct security_descriptor *,psd));
+			discard_const_p(struct security_descriptor, psd));
 	}
 	create_acl_blob(psd, &blob, XATTR_SD_HASH_TYPE_SHA256, hash);
 	store_acl_blob_fsp(handle, fsp, &blob);
@@ -836,7 +849,7 @@ static int acl_common_remove_object(vfs_handle_struct *handle,
 	}
 
 	ZERO_STRUCT(local_fname);
-	local_fname.base_name = CONST_DISCARD(char *,final_component);
+	local_fname.base_name = discard_const_p(char, final_component);
 
 	/* Must use lstat here. */
 	ret = SMB_VFS_LSTAT(conn, &local_fname);
diff --git a/source3/modules/vfs_acl_tdb.c b/source3/modules/vfs_acl_tdb.c
index f2e04152936..0b03896dd90 100644
--- a/source3/modules/vfs_acl_tdb.c
+++ b/source3/modules/vfs_acl_tdb.c
@@ -28,6 +28,7 @@
 #include "../lib/crypto/crypto.h"
 #include "dbwrap.h"
 #include "auth.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_VFS
@@ -176,7 +177,7 @@ static NTSTATUS get_acl_blob(TALLOC_CTX *ctx,
 	if (db->fetch(db,
 			ctx,
 			make_tdb_data(id_buf, sizeof(id_buf)),
-			&data) == -1) {
+			&data) != 0) {
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
diff --git a/source3/modules/vfs_acl_xattr.c b/source3/modules/vfs_acl_xattr.c
index 2a83430fa94..70e979b71b0 100644
--- a/source3/modules/vfs_acl_xattr.c
+++ b/source3/modules/vfs_acl_xattr.c
@@ -55,7 +55,7 @@ static NTSTATUS get_acl_blob(TALLOC_CTX *ctx,
 
   again:
 
-	tmp = TALLOC_REALLOC_ARRAY(ctx, val, uint8_t, size);
+	tmp = talloc_realloc(ctx, val, uint8_t, size);
 	if (tmp == NULL) {
 		TALLOC_FREE(val);
 		return NT_STATUS_NO_MEMORY;
diff --git a/source3/modules/vfs_afsacl.c b/source3/modules/vfs_afsacl.c
index 06ce6f5f8d8..2ef6adf953d 100644
--- a/source3/modules/vfs_afsacl.c
+++ b/source3/modules/vfs_afsacl.c
@@ -30,7 +30,7 @@
 #include <afs/venus.h>
 #include <afs/prs_fs.h>
 
-#define MAXSIZE 2048
+#define MAXSIZE 2049
 
 extern const struct dom_sid global_sid_World;
 extern const struct dom_sid global_sid_Builtin_Administrators;
@@ -89,7 +89,7 @@ static void free_afs_acl(struct afs_acl *acl)
 
 static struct afs_ace *clone_afs_ace(TALLOC_CTX *mem_ctx, struct afs_ace *ace)
 {
-	struct afs_ace *result = TALLOC_P(mem_ctx, struct afs_ace);
+	struct afs_ace *result = talloc(mem_ctx, struct afs_ace);
 
 	if (result == NULL)
 		return NULL;
@@ -167,7 +167,7 @@ static struct afs_ace *new_afs_ace(TALLOC_CTX *mem_ctx,
 		}
 	}
 
-	result = TALLOC_P(mem_ctx, struct afs_ace);
+	result = talloc(mem_ctx, struct afs_ace);
 
 	if (result == NULL) {
 		DEBUG(0, ("Could not talloc AFS ace\n"));
@@ -234,10 +234,10 @@ static bool parse_afs_acl(struct afs_acl *acl, const char *acl_str)
 	int nplus, nminus;
 	int aces;
 
-	char str[MAXSIZE+1];
+	char str[MAXSIZE];
 	char *p = str;
 
-	strncpy(str, acl_str, MAXSIZE);
+	strlcpy(str, acl_str, MAXSIZE);
 
 	if (sscanf(p, "%d", &nplus) != 1)
 		return False;
@@ -313,16 +313,16 @@ static bool unparse_afs_acl(struct afs_acl *acl, char *acl_str)
 	}
 
 	fstr_sprintf(line, "%d\n", positives);
-	safe_strcat(acl_str, line, MAXSIZE);
+	strlcat(acl_str, line, MAXSIZE);
 
 	fstr_sprintf(line, "%d\n", negatives);
-	safe_strcat(acl_str, line, MAXSIZE);
+	strlcat(acl_str, line, MAXSIZE);
 
 	ace = acl->acelist;
 
 	while (ace != NULL) {
 		fstr_sprintf(line, "%s\t%d\n", ace->name, ace->rights);
-		safe_strcat(acl_str, line, MAXSIZE);
+		strlcat(acl_str, line, MAXSIZE);
 		ace = ace->next;
 	}
 	return True;
@@ -605,7 +605,7 @@ static size_t afs_to_nt_acl_common(struct afs_acl *afs_acl,
 	gid_to_sid(&group_sid, psbuf->st_ex_gid);
 
 	if (afs_acl->num_aces) {
-		nt_ace_list = TALLOC_ARRAY(mem_ctx, struct security_ace, afs_acl->num_aces);
+		nt_ace_list = talloc_array(mem_ctx, struct security_ace, afs_acl->num_aces);
 
 		if (nt_ace_list == NULL)
 			return 0;
@@ -897,7 +897,7 @@ static NTSTATUS afs_set_nt_acl(vfs_handle_struct *handle, files_struct *fsp,
 {
 	struct afs_acl old_afs_acl, new_afs_acl;
 	struct afs_acl dir_acl, file_acl;
-	char acl_string[2049];
+	char acl_string[MAXSIZE];
 	struct afs_iob iob;
 	int ret = -1;
 	char *name = NULL;
diff --git a/source3/modules/vfs_aio_fork.c b/source3/modules/vfs_aio_fork.c
index 41b5a89dd29..961efd85de2 100644
--- a/source3/modules/vfs_aio_fork.c
+++ b/source3/modules/vfs_aio_fork.c
@@ -260,7 +260,7 @@ static void aio_child_cleanup(struct event_context *event_ctx,
 		/*
 		 * Re-schedule the next cleanup round
 		 */
-		list->cleanup_event = event_add_timed(smbd_event_context(), list,
+		list->cleanup_event = event_add_timed(server_event_context(), list,
 						      timeval_add(&now, 30, 0),
 						      aio_child_cleanup, list);
 
@@ -277,7 +277,7 @@ static struct aio_child_list *init_aio_children(struct vfs_handle_struct *handle
 	}
 
 	if (data == NULL) {
-		data = TALLOC_ZERO_P(NULL, struct aio_child_list);
+		data = talloc_zero(NULL, struct aio_child_list);
 		if (data == NULL) {
 			return NULL;
 		}
@@ -290,7 +290,7 @@ static struct aio_child_list *init_aio_children(struct vfs_handle_struct *handle
 	 */
 
 	if (data->cleanup_event == NULL) {
-		data->cleanup_event = event_add_timed(smbd_event_context(), data,
+		data->cleanup_event = event_add_timed(server_event_context(), data,
 						      timeval_current_ofs(30, 0),
 						      aio_child_cleanup, data);
 		if (data->cleanup_event == NULL) {
@@ -481,7 +481,7 @@ static NTSTATUS create_aio_child(struct smbd_server_connection *sconn,
 
 	fdpair[0] = fdpair[1] = -1;
 
-	result = TALLOC_ZERO_P(children, struct aio_child);
+	result = talloc_zero(children, struct aio_child);
 	NT_STATUS_HAVE_NO_MEMORY(result);
 
 	if (socketpair(AF_UNIX, SOCK_STREAM, 0, fdpair) == -1) {
@@ -519,7 +519,7 @@ static NTSTATUS create_aio_child(struct smbd_server_connection *sconn,
 	result->sockfd = fdpair[0];
 	close(fdpair[1]);
 
-	result->sock_event = event_add_fd(smbd_event_context(), result,
+	result->sock_event = event_add_fd(server_event_context(), result,
 					  result->sockfd, EVENT_FD_READ,
 					  handle_aio_completion,
 					  result);
diff --git a/source3/modules/vfs_audit.c b/source3/modules/vfs_audit.c
index 349600fc43f..8213bafa1cb 100644
--- a/source3/modules/vfs_audit.c
+++ b/source3/modules/vfs_audit.c
@@ -40,7 +40,8 @@ static int audit_syslog_facility(vfs_handle_struct *handle)
 		{ LOG_LOCAL4, "LOCAL4" },
 		{ LOG_LOCAL5, "LOCAL5" },
 		{ LOG_LOCAL6, "LOCAL6" },
-		{ LOG_LOCAL7, "LOCAL7" }
+		{ LOG_LOCAL7, "LOCAL7" },
+		{ -1, NULL}
 	};
 
 	int facility;
@@ -61,7 +62,8 @@ static int audit_syslog_priority(vfs_handle_struct *handle)
 		{ LOG_WARNING, "WARNING" },
 		{ LOG_NOTICE, "NOTICE" },
 		{ LOG_INFO, "INFO" },
-		{ LOG_DEBUG, "DEBUG" }
+		{ LOG_DEBUG, "DEBUG" },
+		{ -1, NULL}
 	};
 
 	int priority;
diff --git a/source3/modules/vfs_cap.c b/source3/modules/vfs_cap.c
index 7aab6c3973b..15bebf6fea0 100644
--- a/source3/modules/vfs_cap.c
+++ b/source3/modules/vfs_cap.c
@@ -75,7 +75,7 @@ static SMB_STRUCT_DIRENT *cap_readdir(vfs_handle_struct *handle,
 	}
 	DEBUG(3,("cap: cap_readdir: %s\n", newname));
 	newnamelen = strlen(newname)+1;
-	newdirent = (SMB_STRUCT_DIRENT *)TALLOC_ARRAY(talloc_tos(),
+	newdirent = (SMB_STRUCT_DIRENT *)talloc_array(talloc_tos(),
 			char,
 			sizeof(SMB_STRUCT_DIRENT)+
 				newnamelen);
@@ -663,7 +663,7 @@ static char *capencode(TALLOC_CTX *ctx, const char *from)
 	}
 	len++;
 
-	to = TALLOC_ARRAY(ctx, char, len);
+	to = talloc_array(ctx, char, len);
 	if (!to) {
 		return NULL;
 	}
@@ -704,7 +704,7 @@ static char *capdecode(TALLOC_CTX *ctx, const char *from)
 	}
 	len++;
 
-	to = TALLOC_ARRAY(ctx, char, len);
+	to = talloc_array(ctx, char, len);
 	if (!to) {
 		return NULL;
 	}
diff --git a/source3/modules/vfs_default.c b/source3/modules/vfs_default.c
index cf6e24fca04..2dc7ec75b82 100644
--- a/source3/modules/vfs_default.c
+++ b/source3/modules/vfs_default.c
@@ -84,7 +84,10 @@ static int vfswrap_set_quota(struct vfs_handle_struct *handle,  enum SMB_QUOTA_T
 #endif
 }
 
-static int vfswrap_get_shadow_copy_data(struct vfs_handle_struct *handle, struct files_struct *fsp, SHADOW_COPY_DATA *shadow_copy_data, bool labels)
+static int vfswrap_get_shadow_copy_data(struct vfs_handle_struct *handle,
+					struct files_struct *fsp,
+					struct shadow_copy_data *shadow_copy_data,
+					bool labels)
 {
 	errno = ENOSYS;
 	return -1;  /* Not implemented. */
@@ -745,12 +748,12 @@ static int vfswrap_chdir(vfs_handle_struct *handle,  const char *path)
 	return result;
 }
 
-static char *vfswrap_getwd(vfs_handle_struct *handle,  char *path)
+static char *vfswrap_getwd(vfs_handle_struct *handle)
 {
 	char *result;
 
 	START_PROFILE(syscall_getwd);
-	result = sys_getwd(path);
+	result = sys_getwd();
 	END_PROFILE(syscall_getwd);
 	return result;
 }
@@ -1512,12 +1515,12 @@ static ssize_t vfswrap_listxattr(struct vfs_handle_struct *handle, const char *p
 	return sys_listxattr(path, list, size);
 }
 
-ssize_t vfswrap_llistxattr(struct vfs_handle_struct *handle, const char *path, char *list, size_t size)
+static ssize_t vfswrap_llistxattr(struct vfs_handle_struct *handle, const char *path, char *list, size_t size)
 {
 	return sys_llistxattr(path, list, size);
 }
 
-ssize_t vfswrap_flistxattr(struct vfs_handle_struct *handle, struct files_struct *fsp, char *list, size_t size)
+static ssize_t vfswrap_flistxattr(struct vfs_handle_struct *handle, struct files_struct *fsp, char *list, size_t size)
 {
 	return sys_flistxattr(fsp->fh->fd, list, size);
 }
diff --git a/source3/modules/vfs_dirsort.c b/source3/modules/vfs_dirsort.c
index adeab0425d5..ba8441c2dff 100644
--- a/source3/modules/vfs_dirsort.c
+++ b/source3/modules/vfs_dirsort.c
@@ -24,7 +24,7 @@
 
 static int compare_dirent (const SMB_STRUCT_DIRENT *da, const SMB_STRUCT_DIRENT *db)
 {
-	return StrCaseCmp(da->d_name, db->d_name);
+	return strcasecmp_m(da->d_name, db->d_name);
 }
 
 struct dirsort_privates {
diff --git a/source3/modules/vfs_expand_msdfs.c b/source3/modules/vfs_expand_msdfs.c
index f1f82d243ee..eb7c6fed6b5 100644
--- a/source3/modules/vfs_expand_msdfs.c
+++ b/source3/modules/vfs_expand_msdfs.c
@@ -178,7 +178,7 @@ static int expand_msdfs_readlink(struct vfs_handle_struct *handle,
 {
 	TALLOC_CTX *ctx = talloc_tos();
 	int result;
-	char *target = TALLOC_ARRAY(ctx, char, PATH_MAX+1);
+	char *target = talloc_array(ctx, char, PATH_MAX+1);
 	size_t len;
 
 	if (!target) {
diff --git a/source3/modules/vfs_extd_audit.c b/source3/modules/vfs_extd_audit.c
index 34a43c712f8..192b0750ab1 100644
--- a/source3/modules/vfs_extd_audit.c
+++ b/source3/modules/vfs_extd_audit.c
@@ -43,7 +43,8 @@ static int audit_syslog_facility(vfs_handle_struct *handle)
 		{ LOG_LOCAL4, "LOCAL4" },
 		{ LOG_LOCAL5, "LOCAL5" },
 		{ LOG_LOCAL6, "LOCAL6" },
-		{ LOG_LOCAL7, "LOCAL7" }
+		{ LOG_LOCAL7, "LOCAL7" },
+		{ -1, NULL}
 	};
 
 	int facility;
@@ -64,7 +65,8 @@ static int audit_syslog_priority(vfs_handle_struct *handle)
 		{ LOG_WARNING, "WARNING" },
 		{ LOG_NOTICE, "NOTICE" },
 		{ LOG_INFO, "INFO" },
-		{ LOG_DEBUG, "DEBUG" }
+		{ LOG_DEBUG, "DEBUG" },
+		{ -1, NULL}
 	};
 
 	int priority;
diff --git a/source3/modules/vfs_fileid.c b/source3/modules/vfs_fileid.c
index 4c230a0566d..1168d50cb2e 100644
--- a/source3/modules/vfs_fileid.c
+++ b/source3/modules/vfs_fileid.c
@@ -66,7 +66,7 @@ static void fileid_load_mount_entries(struct fileid_handle_data *data)
 			m->mnt_fsname += 5;
 		}
 
-		data->mount_entries = TALLOC_REALLOC_ARRAY(data,
+		data->mount_entries = talloc_realloc(data,
 							   data->mount_entries,
 							   struct fileid_mount_entry,
 							   data->num_mount_entries+1);
@@ -144,7 +144,7 @@ static uint64_t fileid_device_mapping_fsname(struct fileid_handle_data *data,
 	if (!m) return dev;
 
 	if (m->devid == (uint64_t)-1) {
-		m->devid = fileid_uint64_hash((uint8_t *)m->mnt_fsname,
+		m->devid = fileid_uint64_hash((const uint8_t *)m->mnt_fsname,
 					      strlen(m->mnt_fsname));
 	}
 
diff --git a/source3/modules/vfs_full_audit.c b/source3/modules/vfs_full_audit.c
index a723a0ce2d6..e8129f41436 100644
--- a/source3/modules/vfs_full_audit.c
+++ b/source3/modules/vfs_full_audit.c
@@ -355,7 +355,8 @@ static int audit_syslog_facility(vfs_handle_struct *handle)
 		{ LOG_LOCAL4, "LOCAL4" },
 		{ LOG_LOCAL5, "LOCAL5" },
 		{ LOG_LOCAL6, "LOCAL6" },
-		{ LOG_LOCAL7, "LOCAL7" }
+		{ LOG_LOCAL7, "LOCAL7" },
+		{ -1, NULL}
 	};
 
 	int facility;
@@ -375,7 +376,8 @@ static int audit_syslog_priority(vfs_handle_struct *handle)
 		{ LOG_WARNING, "WARNING" },
 		{ LOG_NOTICE, "NOTICE" },
 		{ LOG_INFO, "INFO" },
-		{ LOG_DEBUG, "DEBUG" }
+		{ LOG_DEBUG, "DEBUG" },
+		{ -1, NULL}
 	};
 
 	int priority;
@@ -609,13 +611,13 @@ static int smb_full_audit_connect(vfs_handle_struct *handle,
 		return result;
 	}
 
-	pd = TALLOC_ZERO_P(handle, struct vfs_full_audit_private_data);
+	pd = talloc_zero(handle, struct vfs_full_audit_private_data);
 	if (!pd) {
 		SMB_VFS_NEXT_DISCONNECT(handle);
 		return -1;
 	}
 
-#ifndef WITH_SYSLOG
+#ifdef WITH_SYSLOG
 	openlog("smbd_audit", 0, audit_syslog_facility(handle));
 #endif
 
@@ -695,7 +697,8 @@ static int smb_full_audit_set_quota(struct vfs_handle_struct *handle,
 
 static int smb_full_audit_get_shadow_copy_data(struct vfs_handle_struct *handle,
 				struct files_struct *fsp,
-				SHADOW_COPY_DATA *shadow_copy_data, bool labels)
+				struct shadow_copy_data *shadow_copy_data,
+				bool labels)
 {
 	int result;
 
@@ -1205,14 +1208,14 @@ static int smb_full_audit_chdir(vfs_handle_struct *handle,
 	return result;
 }
 
-static char *smb_full_audit_getwd(vfs_handle_struct *handle,
-			 char *path)
+static char *smb_full_audit_getwd(vfs_handle_struct *handle)
 {
 	char *result;
 
-	result = SMB_VFS_NEXT_GETWD(handle, path);
+	result = SMB_VFS_NEXT_GETWD(handle);
 	
-	do_log(SMB_VFS_OP_GETWD, (result != NULL), handle, "%s", path);
+	do_log(SMB_VFS_OP_GETWD, (result != NULL), handle, "%s",
+		result == NULL? "" : result);
 
 	return result;
 }
diff --git a/source3/modules/vfs_gpfs.c b/source3/modules/vfs_gpfs.c
index c38d7c2337f..0a684e0d8e5 100644
--- a/source3/modules/vfs_gpfs.c
+++ b/source3/modules/vfs_gpfs.c
@@ -32,6 +32,7 @@
 #include <gpfs_gpl.h>
 #include "nfs4_acls.h"
 #include "vfs_gpfs.h"
+#include "system/filesys.h"
 
 struct gpfs_config_data {
 	bool sharemodes;
@@ -998,7 +999,7 @@ static int gpfs_set_xattr(struct vfs_handle_struct *handle,  const char *path,
 	}
 
 
-        ret = set_gpfs_winattrs(CONST_DISCARD(char *, path),
+        ret = set_gpfs_winattrs(discard_const_p(char, path),
 				GPFS_WINATTR_SET_ATTRS, &attrs);
         if ( ret == -1){
 		if (errno == ENOSYS) {
@@ -1029,7 +1030,7 @@ static ssize_t gpfs_get_xattr(struct vfs_handle_struct *handle,  const char *pat
                 return SMB_VFS_NEXT_GETXATTR(handle,path,name,value,size);
         }
 
-        ret = get_gpfs_winattrs(CONST_DISCARD(char *, path), &attrs);
+        ret = get_gpfs_winattrs(discard_const_p(char, path), &attrs);
         if ( ret == -1){
 		if (errno == ENOSYS) {
 			return SMB_VFS_NEXT_GETXATTR(handle, path, name, value,
@@ -1083,11 +1084,12 @@ static int vfs_gpfs_stat(struct vfs_handle_struct *handle,
 		errno = map_errno_from_nt_status(status);
 		return -1;
 	}
-	ret = get_gpfs_winattrs(CONST_DISCARD(char *, fname), &attrs);
+	ret = get_gpfs_winattrs(discard_const_p(char, fname), &attrs);
 	TALLOC_FREE(fname);
 	if (ret == 0) {
 		smb_fname->st.st_ex_btime.tv_sec = attrs.creationTime.tv_sec;
 		smb_fname->st.st_ex_btime.tv_nsec = attrs.creationTime.tv_nsec;
+		smb_fname->st.vfs_private = attrs.winAttrs;
 	}
 	return 0;
 }
@@ -1130,11 +1132,12 @@ static int vfs_gpfs_lstat(struct vfs_handle_struct *handle,
 		errno = map_errno_from_nt_status(status);
 		return -1;
 	}
-	ret = get_gpfs_winattrs(CONST_DISCARD(char *, path), &attrs);
+	ret = get_gpfs_winattrs(discard_const_p(char, path), &attrs);
 	TALLOC_FREE(path);
 	if (ret == 0) {
 		smb_fname->st.st_ex_btime.tv_sec = attrs.creationTime.tv_sec;
 		smb_fname->st.st_ex_btime.tv_nsec = attrs.creationTime.tv_nsec;
+		smb_fname->st.vfs_private = attrs.winAttrs;
 	}
 	return 0;
 }
@@ -1170,7 +1173,7 @@ static int vfs_gpfs_ntimes(struct vfs_handle_struct *handle,
         attrs.creationTime.tv_sec = ft->create_time.tv_sec;
         attrs.creationTime.tv_nsec = ft->create_time.tv_nsec;
 
-        ret = set_gpfs_winattrs(CONST_DISCARD(char *, path),
+        ret = set_gpfs_winattrs(discard_const_p(char, path),
                                 GPFS_WINATTR_SET_CREATION_TIME, &attrs);
         if(ret == -1 && errno != ENOSYS){
                 DEBUG(1,("vfs_gpfs_ntimes: set GPFS ntimes failed %d\n",ret));
@@ -1199,18 +1202,23 @@ static bool vfs_gpfs_is_offline(struct vfs_handle_struct *handle,
 	struct gpfs_winattr attrs;
 	char *path = NULL;
 	NTSTATUS status;
-	int ret;
 
 	status = get_full_smb_filename(talloc_tos(), fname, &path);
 	if (!NT_STATUS_IS_OK(status)) {
 		errno = map_errno_from_nt_status(status);
 		return -1;
 	}
-	ret = get_gpfs_winattrs(path, &attrs);
 
-	if (ret == -1) {
-		TALLOC_FREE(path);
-		return false;
+	if (VALID_STAT(*sbuf)) {
+		attrs.winAttrs = sbuf->vfs_private;
+	} else {
+		int ret;
+		ret = get_gpfs_winattrs(path, &attrs);
+
+		if (ret == -1) {
+			TALLOC_FREE(path);
+			return false;
+		}
 	}
 	if ((attrs.winAttrs & GPFS_WINATTR_OFFLINE) != 0) {
 		DEBUG(10, ("%s is offline\n", path));
@@ -1228,10 +1236,24 @@ static bool vfs_gpfs_aio_force(struct vfs_handle_struct *handle,
 	return vfs_gpfs_is_offline(handle, fsp->fsp_name, &fsp->fsp_name->st);
 }
 
+static ssize_t vfs_gpfs_sendfile(vfs_handle_struct *handle, int tofd,
+				 files_struct *fsp, const DATA_BLOB *hdr,
+				 SMB_OFF_T offset, size_t n)
+{
+	if ((fsp->fsp_name->st.vfs_private & GPFS_WINATTR_OFFLINE) != 0) {
+		errno = ENOSYS;
+		return -1;
+	}
+	return SMB_VFS_NEXT_SENDFILE(handle, tofd, fsp, hdr, offset, n);
+}
+
 int vfs_gpfs_connect(struct vfs_handle_struct *handle, const char *service,
 			const char *user)
 {
 	struct gpfs_config_data *config;
+
+	smbd_gpfs_lib_init();
+
 	int ret = SMB_VFS_NEXT_CONNECT(handle, service, user);
 
 	if (ret < 0) {
@@ -1278,6 +1300,17 @@ static uint32_t vfs_gpfs_capabilities(struct vfs_handle_struct *handle,
 	return next;
 }
 
+static int vfs_gpfs_open(struct vfs_handle_struct *handle,
+			 struct smb_filename *smb_fname, files_struct *fsp,
+			 int flags, mode_t mode)
+{
+	if (lp_parm_bool(fsp->conn->params->service, "gpfs", "syncio",
+			 false)) {
+		flags |= O_SYNC;
+	}
+	return SMB_VFS_NEXT_OPEN(handle, smb_fname, fsp, flags, mode);
+}
+
 
 static struct vfs_fn_pointers vfs_gpfs_fns = {
 	.connect_fn = vfs_gpfs_connect,
@@ -1304,6 +1337,8 @@ static struct vfs_fn_pointers vfs_gpfs_fns = {
 	.ntimes = vfs_gpfs_ntimes,
 	.is_offline = vfs_gpfs_is_offline,
 	.aio_force = vfs_gpfs_aio_force,
+	.sendfile = vfs_gpfs_sendfile,
+	.open_fn = vfs_gpfs_open,
 	.ftruncate = vfs_gpfs_ftruncate
 };
 
diff --git a/source3/modules/vfs_gpfs.h b/source3/modules/vfs_gpfs.h
index 46cfdfea0ac..4a05841feba 100644
--- a/source3/modules/vfs_gpfs.h
+++ b/source3/modules/vfs_gpfs.h
@@ -1,26 +1,25 @@
 /*
    Unix SMB/CIFS implementation.
    Wrap gpfs calls in vfs functions.
- 
+
    Copyright (C) Christian Ambach <cambach1@de.ibm.com> 2006
-   
+
    Major code contributions by Chetan Shringarpure <chetan.sh@in.ibm.com>
                             and Gomati Mohanan <gomati.mohanan@in.ibm.com>
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-  
 
 */
 
@@ -36,3 +35,4 @@ int get_gpfs_winattrs(char * pathname,struct gpfs_winattr *attrs);
 int set_gpfs_winattrs(char * pathname,int flags,struct gpfs_winattr *attrs);
 int smbd_gpfs_ftruncate(int fd, gpfs_off64_t length);
 void init_gpfs(void);
+void smbd_gpfs_lib_init();
diff --git a/source3/modules/vfs_gpfs_hsm_notify.c b/source3/modules/vfs_gpfs_hsm_notify.c
new file mode 100644
index 00000000000..fa24db99833
--- /dev/null
+++ b/source3/modules/vfs_gpfs_hsm_notify.c
@@ -0,0 +1,110 @@
+/*
+   Unix SMB/CIFS implementation.
+   Make sure offline->online changes are propagated by notifies
+
+   This module must come before aio_fork in the chain, because
+   aio_fork (correcly!) does not propagate the aio calls further
+
+   Copyright (C) Volker Lendecke 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "smbd/smbd.h"
+#include "librpc/gen_ndr/ndr_xattr.h"
+#include "include/smbprofile.h"
+
+#undef DBGC_CLASS
+#define DBGC_CLASS DBGC_VFS
+
+#include <gpfs_gpl.h>
+#include "nfs4_acls.h"
+#include "vfs_gpfs.h"
+
+static ssize_t vfs_gpfs_hsm_notify_pread(vfs_handle_struct *handle, files_struct *fsp,
+			      void *data, size_t n, SMB_OFF_T offset)
+{
+	ssize_t ret;
+
+	ret = SMB_VFS_NEXT_PREAD(handle, fsp, data, n, offset);
+
+	DEBUG(10, ("vfs_private = %x\n",
+		   (unsigned int)fsp->fsp_name->st.vfs_private));
+
+	if ((ret != -1) &&
+	    ((fsp->fsp_name->st.vfs_private & GPFS_WINATTR_OFFLINE) != 0)) {
+		fsp->fsp_name->st.vfs_private &= ~GPFS_WINATTR_OFFLINE;
+		notify_fname(handle->conn, NOTIFY_ACTION_MODIFIED,
+			     FILE_NOTIFY_CHANGE_ATTRIBUTES,
+			     fsp->fsp_name->base_name);
+	}
+
+	return ret;
+}
+
+static ssize_t vfs_gpfs_hsm_notify_pwrite(struct vfs_handle_struct *handle,
+			       struct files_struct *fsp,
+			       const void *data, size_t n, SMB_OFF_T offset)
+{
+	ssize_t ret;
+
+	ret = SMB_VFS_NEXT_PWRITE(handle, fsp, data, n, offset);
+
+	if ((ret != -1) &&
+	    ((fsp->fsp_name->st.vfs_private & GPFS_WINATTR_OFFLINE) != 0)) {
+		fsp->fsp_name->st.vfs_private &= ~GPFS_WINATTR_OFFLINE;
+		notify_fname(handle->conn, NOTIFY_ACTION_MODIFIED,
+			     FILE_NOTIFY_CHANGE_ATTRIBUTES,
+			     fsp->fsp_name->base_name);
+	}
+
+	return ret;
+}
+
+static ssize_t vfs_gpfs_hsm_notify_aio_return(struct vfs_handle_struct *handle,
+				   struct files_struct *fsp,
+				   SMB_STRUCT_AIOCB *aiocb)
+{
+	ssize_t ret;
+
+	ret = SMB_VFS_NEXT_AIO_RETURN(handle, fsp, aiocb);
+
+	DEBUG(10, ("vfs_gpfs_hsm_notify_aio_return: vfs_private = %x\n",
+		   (unsigned int)fsp->fsp_name->st.vfs_private));
+
+	if ((ret != -1) &&
+	    ((fsp->fsp_name->st.vfs_private & GPFS_WINATTR_OFFLINE) != 0)) {
+		fsp->fsp_name->st.vfs_private &= ~GPFS_WINATTR_OFFLINE;
+		DEBUG(10, ("sending notify\n"));
+		notify_fname(handle->conn, NOTIFY_ACTION_MODIFIED,
+			     FILE_NOTIFY_CHANGE_ATTRIBUTES,
+			     fsp->fsp_name->base_name);
+	}
+
+	return ret;
+}
+
+static struct vfs_fn_pointers vfs_gpfs_hsm_notify_fns = {
+	.pread = vfs_gpfs_hsm_notify_pread,
+	.pwrite = vfs_gpfs_hsm_notify_pwrite,
+	.aio_return_fn = vfs_gpfs_hsm_notify_aio_return
+};
+
+NTSTATUS vfs_gpfs_hsm_notify_init(void);
+NTSTATUS vfs_gpfs_hsm_notify_init(void)
+{
+	return smb_register_vfs(SMB_VFS_INTERFACE_VERSION, "gpfs_hsm_notify",
+				&vfs_gpfs_hsm_notify_fns);
+}
diff --git a/source3/modules/vfs_hpuxacl.c b/source3/modules/vfs_hpuxacl.c
index 2155945b95a..8a574443526 100644
--- a/source3/modules/vfs_hpuxacl.c
+++ b/source3/modules/vfs_hpuxacl.c
@@ -307,7 +307,7 @@ int hpuxacl_sys_acl_set_file(vfs_handle_struct *handle,
 	}
 	DEBUG(10, ("resulting acl is valid.\n"));
 
-	ret = acl(CONST_DISCARD(char *, smb_fname->base_name), ACL_SET, count,
+	ret = acl(discard_const_p(char, smb_fname->base_name), ACL_SET, count,
 		  hpux_acl);
 	if (ret != 0) {
 		DEBUG(0, ("ERROR calling acl: %s\n", strerror(errno)));
@@ -394,7 +394,7 @@ int hpuxacl_sys_acl_delete_def_file(vfs_handle_struct *handle,
 		DEBUG(10, ("resulting acl is not valid!\n"));
 		goto done;
 	}
-	ret = acl(CONST_DISCARD(char *, path), ACL_SET, count, hpux_acl);
+	ret = acl(discard_const_p(char, path), ACL_SET, count, hpux_acl);
 	if (ret != 0) {
 		DEBUG(10, ("settinge file acl failed!\n"));
 	}
@@ -682,7 +682,7 @@ static bool hpux_acl_get_file(const char *name, HPUX_ACL_T *hpux_acl,
 	 * instantiated or malloced each time this function is
 	 * called). Btw: the count parameter does not seem to matter...
 	 */
-	*count = acl(CONST_DISCARD(char *, name), ACL_CNT, 0, &dummy_ace);
+	*count = acl(discard_const_p(char, name), ACL_CNT, 0, &dummy_ace);
 	if (*count < 0) {
 		DEBUG(10, ("acl ACL_CNT failed: %s\n", strerror(errno)));
 		goto done;
@@ -692,7 +692,7 @@ static bool hpux_acl_get_file(const char *name, HPUX_ACL_T *hpux_acl,
 		DEBUG(10, ("error allocating memory for hpux acl...\n"));
 		goto done;
 	}
-	*count = acl(CONST_DISCARD(char *, name), ACL_GET, *count, *hpux_acl);
+	*count = acl(discard_const_p(char, name), ACL_GET, *count, *hpux_acl);
 	if (*count < 0) {
 		DEBUG(10, ("acl ACL_GET failed: %s\n", strerror(errno)));
 		goto done;
diff --git a/source3/modules/vfs_notify_fam.c b/source3/modules/vfs_notify_fam.c
index a4e7fd1e2ba..1f76a05fc08 100644
--- a/source3/modules/vfs_notify_fam.c
+++ b/source3/modules/vfs_notify_fam.c
@@ -249,7 +249,7 @@ static NTSTATUS fam_watch(vfs_handle_struct *vfs_handle,
 		fam_connection_initialized = True;
 	}
 
-	if (!(watch = TALLOC_P(ctx, struct fam_watch_context))) {
+	if (!(watch = talloc(ctx, struct fam_watch_context))) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
diff --git a/source3/modules/vfs_preopen.c b/source3/modules/vfs_preopen.c
index 82969e48d6a..3d7f6c1b038 100644
--- a/source3/modules/vfs_preopen.c
+++ b/source3/modules/vfs_preopen.c
@@ -166,7 +166,7 @@ static bool preopen_helper_open_one(int sock_fd, char **pnamebuf,
 		nread += thistime;
 
 		if (nread == talloc_get_size(namebuf)) {
-			namebuf = TALLOC_REALLOC_ARRAY(
+			namebuf = talloc_realloc(
 				NULL, namebuf, char,
 				talloc_get_size(namebuf) * 2);
 			if (namebuf == NULL) {
@@ -193,7 +193,7 @@ static bool preopen_helper(int fd, size_t to_read)
 	char *namebuf;
 	void *readbuf;
 
-	namebuf = TALLOC_ARRAY(NULL, char, 1024);
+	namebuf = talloc_array(NULL, char, 1024);
 	if (namebuf == NULL) {
 		return false;
 	}
@@ -237,7 +237,7 @@ static NTSTATUS preopen_init_helper(struct preopen_helper *h)
 	}
 	close(fdpair[1]);
 	h->fd = fdpair[0];
-	h->fde = event_add_fd(smbd_event_context(), h->state, h->fd,
+	h->fde = event_add_fd(server_event_context(), h->state, h->fd,
 			      EVENT_FD_READ, preopen_helper_readable, h);
 	if (h->fde == NULL) {
 		close(h->fd);
@@ -261,7 +261,7 @@ static NTSTATUS preopen_init_helpers(TALLOC_CTX *mem_ctx, size_t to_read,
 	}
 
 	result->num_helpers = num_helpers;
-	result->helpers = TALLOC_ARRAY(result, struct preopen_helper,
+	result->helpers = talloc_array(result, struct preopen_helper,
 				       num_helpers);
 	if (result->helpers == NULL) {
 		TALLOC_FREE(result);
@@ -322,7 +322,7 @@ static struct preopen_state *preopen_state_get(vfs_handle_struct *handle)
 		return NULL;
 	}
 
-	set_namearray(&state->preopen_names, (char *)namelist);
+	set_namearray(&state->preopen_names, namelist);
 
 	if (state->preopen_names == NULL) {
 		TALLOC_FREE(state);
diff --git a/source3/modules/vfs_recycle.c b/source3/modules/vfs_recycle.c
index 22f61b4b33c..5c4f8706cbf 100644
--- a/source3/modules/vfs_recycle.c
+++ b/source3/modules/vfs_recycle.c
@@ -35,33 +35,9 @@ static int vfs_recycle_debug_level = DBGC_VFS;
 #undef DBGC_CLASS
 #define DBGC_CLASS vfs_recycle_debug_level
  
-static int recycle_connect(vfs_handle_struct *handle, const char *service, const char *user);
-static void recycle_disconnect(vfs_handle_struct *handle);
 static int recycle_unlink(vfs_handle_struct *handle,
 			  const struct smb_filename *smb_fname);
 
-static int recycle_connect(vfs_handle_struct *handle, const char *service, const char *user)
-{
-	int ret = SMB_VFS_NEXT_CONNECT(handle, service, user);
-
-	if (ret < 0) {
-		return ret;
-	}
-
-	DEBUG(10,("recycle_connect() connect to service[%s] as user[%s].\n",
-		service,user));
-
-	return 0;
-}
-
-static void recycle_disconnect(vfs_handle_struct *handle)
-{
-	DEBUG(10,("recycle_disconnect() connect to service[%s].\n",
-		lp_servicename(SNUM(handle->conn))));
-
-	SMB_VFS_NEXT_DISCONNECT(handle);
-}
-
 static const char *recycle_repository(vfs_handle_struct *handle)
 {
 	const char *tmp_str = NULL;
@@ -304,13 +280,13 @@ static bool recycle_create_dir(vfs_handle_struct *handle, const char *dname)
 	*new_dir = '\0';
 	if (dname[0] == '/') {
 		/* Absolute path. */
-		safe_strcat(new_dir,"/",len);
+		strlcat(new_dir,"/",len+1);
 	}
 
 	/* Create directory tree if neccessary */
 	for(token = strtok_r(tok_str, "/", &saveptr); token;
 	    token = strtok_r(NULL, "/", &saveptr)) {
-		safe_strcat(new_dir, token, len);
+		strlcat(new_dir, token, len+1);
 		if (recycle_directory_exist(handle, new_dir))
 			DEBUG(10, ("recycle: dir %s already exists\n", new_dir));
 		else {
@@ -321,7 +297,7 @@ static bool recycle_create_dir(vfs_handle_struct *handle, const char *dname)
 				goto done;
 			}
 		}
-		safe_strcat(new_dir, "/", len);
+		strlcat(new_dir, "/", len+1);
 		mode = recycle_subdir_mode(handle);
 	}
 
@@ -663,8 +639,6 @@ done:
 }
 
 static struct vfs_fn_pointers vfs_recycle_fns = {
-	.connect_fn = recycle_connect,
-	.disconnect = recycle_disconnect,
 	.unlink = recycle_unlink
 };
 
diff --git a/source3/modules/vfs_scannedonly.c b/source3/modules/vfs_scannedonly.c
index d0eba212d24..6d748c4c238 100644
--- a/source3/modules/vfs_scannedonly.c
+++ b/source3/modules/vfs_scannedonly.c
@@ -75,12 +75,12 @@ struct Tscannedonly {
 	bool rm_hidden_files_on_rmdir;
 	bool hide_nonscanned_files;
 	bool allow_nonscanned_files;
-	char *socketname;
-	char *scanhost;
-	char *scanning_message;
-	char *p_scanned; /* prefix for scanned files */
-	char *p_virus; /* prefix for virus containing files */
-	char *p_failed; /* prefix for failed to scan files */
+	const char *socketname;
+	const char *scanhost;
+	const char *scanning_message;
+	const char *p_scanned; /* prefix for scanned files */
+	const char *p_virus; /* prefix for virus containing files */
+	const char *p_failed; /* prefix for failed to scan files */
 	char gsendbuffer[SENDBUFFERSIZE + 1];
 };
 
@@ -146,14 +146,15 @@ static char *cachefile_name_f_fullpath(TALLOC_CTX *ctx,
 				       const char *p_scanned)
 {
 	const char *base;
-	char *tmp, *cachefile, *shortname;
+	char *tmp, *cachefile;
+	const char *shortname;
 	tmp = strrchr(fullpath, '/');
 	if (tmp) {
 		base = talloc_strndup(ctx, fullpath, (tmp - fullpath) + 1);
 		shortname = tmp + 1;
 	} else {
 		base = "";
-		shortname = (char *)fullpath;
+		shortname = (const char *)fullpath;
 	}
 	cachefile = cachefile_name(ctx, shortname, base, p_scanned);
 	DEBUG(SCANNEDONLY_DEBUG,
@@ -164,7 +165,7 @@ static char *cachefile_name_f_fullpath(TALLOC_CTX *ctx,
 static char *construct_full_path(TALLOC_CTX *ctx, vfs_handle_struct * handle,
 				 const char *somepath, bool ending_slash)
 {
-	char *tmp;
+	const char *tmp;
 
 	if (!somepath) {
 		return NULL;
@@ -175,7 +176,7 @@ static char *construct_full_path(TALLOC_CTX *ctx, vfs_handle_struct * handle,
 		}
 		return talloc_strdup(ctx,somepath);
 	}
-	tmp=(char *)somepath;
+	tmp = somepath;
 	if (tmp[0]=='.'&&tmp[1]=='/') {
 		tmp+=2;
 	}
@@ -308,14 +309,14 @@ static void flush_sendbuffer(vfs_handle_struct * handle)
 
 static void notify_scanner(vfs_handle_struct * handle, const char *scanfile)
 {
-	char *tmp;
+	const char *tmp;
 	int tmplen, gsendlen;
 	struct Tscannedonly *so = (struct Tscannedonly *)handle->data;
 	TALLOC_CTX *ctx=talloc_tos();
 	if (scanfile[0] != '/') {
 		tmp = construct_full_path(ctx,handle, scanfile, false);
 	} else {
-		tmp = (char *)scanfile;
+		tmp = (const char *)scanfile;
 	}
 	tmplen = strlen(tmp);
 	gsendlen = strlen(so->gsendbuffer);
@@ -515,7 +516,7 @@ static SMB_STRUCT_DIR *scannedonly_opendir(vfs_handle_struct * handle,
 		return NULL;
 	}
 
-	sDIR = TALLOC_P(NULL, struct scannedonly_DIR);
+	sDIR = talloc(NULL, struct scannedonly_DIR);
 	if (fname[0] != '/') {
 		sDIR->base = construct_full_path(sDIR,handle, fname, true);
 	} else {
@@ -543,7 +544,7 @@ static SMB_STRUCT_DIR *scannedonly_fdopendir(vfs_handle_struct * handle,
 
 	fname = (const char *)fsp->fsp_name->base_name;
 
-	sDIR = TALLOC_P(NULL, struct scannedonly_DIR);
+	sDIR = talloc(NULL, struct scannedonly_DIR);
 	if (fname[0] != '/') {
 		sDIR->base = construct_full_path(sDIR,handle, fname, true);
 	} else {
@@ -631,7 +632,7 @@ static SMB_STRUCT_DIRENT *scannedonly_readdir(vfs_handle_struct *handle,
 		ctx,"%s %s",result->d_name,
 		STRUCTSCANO(handle->data)->scanning_message);
 	namelen = strlen(notify_name);
-	newdirent = (SMB_STRUCT_DIRENT *)TALLOC_ARRAY(
+	newdirent = (SMB_STRUCT_DIRENT *)talloc_array(
 		ctx, char, sizeof(SMB_STRUCT_DIRENT) + namelen + 1);
 	if (!newdirent) {
 		return NULL;
@@ -948,15 +949,14 @@ static int scannedonly_connect(struct vfs_handle_struct *handle,
 	so->domain_socket =
 		lp_parm_bool(SNUM(handle->conn), "scannedonly",
 			     "domain_socket", True);
-	so->socketname =
-		(char *)lp_parm_const_string(SNUM(handle->conn),
+	so->socketname = lp_parm_const_string(SNUM(handle->conn),
 					     "scannedonly", "socketname",
 					     "/var/lib/scannedonly/scan");
+
 	so->portnum =
 		lp_parm_int(SNUM(handle->conn), "scannedonly", "portnum",
 			    2020);
-	so->scanhost =
-		(char *)lp_parm_const_string(SNUM(handle->conn),
+	so->scanhost = lp_parm_const_string(SNUM(handle->conn),
 					     "scannedonly", "scanhost",
 					     "localhost");
 
@@ -972,8 +972,7 @@ static int scannedonly_connect(struct vfs_handle_struct *handle,
 	so->allow_nonscanned_files =
 		lp_parm_bool(SNUM(handle->conn), "scannedonly",
 			     "allow_nonscanned_files", False);
-	so->scanning_message =
-		(char *)lp_parm_const_string(SNUM(handle->conn),
+	so->scanning_message = lp_parm_const_string(SNUM(handle->conn),
 					     "scannedonly",
 					     "scanning_message",
 					     "is being scanned for viruses");
@@ -995,17 +994,17 @@ static int scannedonly_connect(struct vfs_handle_struct *handle,
 			    "recheck_tries_readdir", 20);
 
 	so->p_scanned =
-		(char *)lp_parm_const_string(SNUM(handle->conn),
+		lp_parm_const_string(SNUM(handle->conn),
 					     "scannedonly",
 					     "pref_scanned",
 					     ".scanned:");
 	so->p_virus =
-		(char *)lp_parm_const_string(SNUM(handle->conn),
+		lp_parm_const_string(SNUM(handle->conn),
 					     "scannedonly",
 					     "pref_virus",
 					     ".virus:");
 	so->p_failed =
-		(char *)lp_parm_const_string(SNUM(handle->conn),
+		lp_parm_const_string(SNUM(handle->conn),
 					     "scannedonly",
 					     "pref_failed",
 					     ".failed:");
diff --git a/source3/modules/vfs_shadow_copy.c b/source3/modules/vfs_shadow_copy.c
index b93f98dedee..1db47d216be 100644
--- a/source3/modules/vfs_shadow_copy.c
+++ b/source3/modules/vfs_shadow_copy.c
@@ -216,7 +216,10 @@ static int shadow_copy_closedir(vfs_handle_struct *handle, SMB_STRUCT_DIR *_dirp
 	return 0;	
 }
 
-static int shadow_copy_get_shadow_copy_data(vfs_handle_struct *handle, files_struct *fsp, SHADOW_COPY_DATA *shadow_copy_data, bool labels)
+static int shadow_copy_get_shadow_copy_data(vfs_handle_struct *handle,
+					    files_struct *fsp,
+					    struct shadow_copy_data *shadow_copy_data,
+					    bool labels)
 {
 	SMB_STRUCT_DIR *p = SMB_VFS_NEXT_OPENDIR(handle,fsp->conn->connectpath,NULL,0);
 
@@ -250,7 +253,7 @@ static int shadow_copy_get_shadow_copy_data(vfs_handle_struct *handle, files_str
 			continue;
 		}
 
-		tlabels = (SHADOW_COPY_LABEL *)TALLOC_REALLOC(shadow_copy_data->mem_ctx,
+		tlabels = (SHADOW_COPY_LABEL *)TALLOC_REALLOC(shadow_copy_data,
 									shadow_copy_data->labels,
 									(shadow_copy_data->num_volumes+1)*sizeof(SHADOW_COPY_LABEL));
 		if (tlabels == NULL) {
diff --git a/source3/modules/vfs_shadow_copy2.c b/source3/modules/vfs_shadow_copy2.c
index df3bde72c94..ca33b6d3444 100644
--- a/source3/modules/vfs_shadow_copy2.c
+++ b/source3/modules/vfs_shadow_copy2.c
@@ -1,32 +1,29 @@
-/* 
- * implementation of an Shadow Copy module - version 2
+/*
+ * Third attempt at a shadow copy module
  *
- * Copyright (C) Andrew Tridgell     2007
- * Copyright (C) Ed Plese            2009
+ * Copyright (C) Andrew Tridgell   2007 (portions taken from shadow_copy2)
+ * Copyright (C) Ed Plese          2009
+ * Copyright (C) Volker Lendecke   2011
+ * Copyright (C) Christian Ambach  2011
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
- *  
+ *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- *  
+ *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#include "includes.h"
-#include "smbd/smbd.h"
-#include "system/filesys.h"
-#include "ntioctl.h"
-
 /*
 
-  This is a 2nd implemetation of a shadow copy module for exposing
+  This is a 3rd implemetation of a shadow copy module for exposing
   snapshots to windows clients as shadow copies. This version has the
   following features:
 
@@ -96,243 +93,169 @@
   The following command would generate a correctly formatted directory name
   for use with the default parameters:
      date -u +@GMT-%Y.%m.%d-%H.%M.%S
-  
  */
 
-static int vfs_shadow_copy2_debug_level = DBGC_VFS;
-
-#undef DBGC_CLASS
-#define DBGC_CLASS vfs_shadow_copy2_debug_level
+#include "includes.h"
+#include "system/filesys.h"
+#include "include/ntioctl.h"
+#include "smbd/proto.h"
+#include <tdb.h>
+#include "util_tdb.h"
 
 #define GMT_NAME_LEN 24 /* length of a @GMT- name */
-#define SHADOW_COPY2_GMT_FORMAT "@GMT-%Y.%m.%d-%H.%M.%S"
+#define GMT_FORMAT "@GMT-%Y.%m.%d-%H.%M.%S"
 
-#define SHADOW_COPY2_DEFAULT_SORT NULL
-#define SHADOW_COPY2_DEFAULT_FORMAT "@GMT-%Y.%m.%d-%H.%M.%S"
-#define SHADOW_COPY2_DEFAULT_LOCALTIME false
-
-/*
-  make very sure it is one of our special names 
- */
-static inline bool shadow_copy2_match_name(const char *name, const char **gmt_start)
+static bool shadow_copy2_find_slashes(TALLOC_CTX *mem_ctx, const char *str,
+				      size_t **poffsets,
+				      unsigned *pnum_offsets)
 {
-	unsigned year, month, day, hr, min, sec;
+	unsigned num_offsets;
+	size_t *offsets;
 	const char *p;
-	if (gmt_start) {
-		(*gmt_start) = NULL;
-	}
-	p = strstr_m(name, "@GMT-");
-	if (p == NULL) return false;
-	if (p > name && p[-1] != '/') return False;
-	if (sscanf(p, "@GMT-%04u.%02u.%02u-%02u.%02u.%02u", &year, &month,
-		   &day, &hr, &min, &sec) != 6) {
-		return False;
-	}
-	if (p[24] != 0 && p[24] != '/') {
-		return False;
-	}
-	if (gmt_start) {
-		(*gmt_start) = p;
-	}
-	return True;
-}
 
-static char *shadow_copy2_snapshot_to_gmt(TALLOC_CTX *mem_ctx,
-				vfs_handle_struct *handle, const char *name)
-{
-	struct tm timestamp;
-	time_t timestamp_t;
-	char gmt[GMT_NAME_LEN + 1];
-	const char *fmt;
+	num_offsets = 0;
 
-	fmt = lp_parm_const_string(SNUM(handle->conn), "shadow",
-				   "format", SHADOW_COPY2_DEFAULT_FORMAT);
+	p = str;
+	while ((p = strchr(p, '/')) != NULL) {
+		num_offsets += 1;
+		p += 1;
+	}
 
-	ZERO_STRUCT(timestamp);
-	if (strptime(name, fmt, &timestamp) == NULL) {
-		DEBUG(10, ("shadow_copy2_snapshot_to_gmt: no match %s: %s\n",
-			   fmt, name));
-		return NULL;
+	offsets = talloc_array(mem_ctx, size_t, num_offsets);
+	if (offsets == NULL) {
+		return false;
 	}
 
-	DEBUG(10, ("shadow_copy2_snapshot_to_gmt: match %s: %s\n", fmt, name));
-	if (lp_parm_bool(SNUM(handle->conn), "shadow", "localtime",
-		         SHADOW_COPY2_DEFAULT_LOCALTIME))
-	{
-		timestamp.tm_isdst = -1;
-		timestamp_t = mktime(&timestamp);
-		gmtime_r(&timestamp_t, &timestamp);
+	p = str;
+	num_offsets = 0;
+	while ((p = strchr(p, '/')) != NULL) {
+		offsets[num_offsets] = p-str;
+		num_offsets += 1;
+		p += 1;
 	}
-	strftime(gmt, sizeof(gmt), SHADOW_COPY2_GMT_FORMAT, &timestamp);
 
-	return talloc_strdup(mem_ctx, gmt);
+	*poffsets = offsets;
+	*pnum_offsets = num_offsets;
+	return true;
 }
 
-/*
-  shadow copy paths can also come into the server in this form:
-
-    /foo/bar/@GMT-XXXXX/some/file
-
-  This function normalises the filename to be of the form:
-
-    @GMT-XXXX/foo/bar/some/file
- */
-static const char *shadow_copy2_normalise_path(TALLOC_CTX *mem_ctx, const char *path, const char *gmt_start)
+static char *shadow_copy2_insert_string(TALLOC_CTX *mem_ctx,
+					struct vfs_handle_struct *handle,
+					time_t snapshot)
 {
-	char *pcopy;
-	char buf[GMT_NAME_LEN];
-	size_t prefix_len;
+	struct tm snap_tm;
+	fstring gmt;
+	size_t gmt_len;
 
-	if (path == gmt_start) {
-		return path;
+	if (localtime_r(&snapshot, &snap_tm) == 0) {
+		DEBUG(10, ("gmtime_r failed\n"));
+		return NULL;
 	}
-
-	prefix_len = gmt_start - path - 1;
-
-	DEBUG(10, ("path=%s, gmt_start=%s, prefix_len=%d\n", path, gmt_start,
-		   (int)prefix_len));
-
-	/*
-	 * We've got a/b/c/@GMT-YYYY.MM.DD-HH.MM.SS/d/e. convert to
-	 * @GMT-YYYY.MM.DD-HH.MM.SS/a/b/c/d/e before further
-	 * processing. As many VFS calls provide a const char *,
-	 * unfortunately we have to make a copy.
-	 */
-
-	pcopy = talloc_strdup(talloc_tos(), path);
-	if (pcopy == NULL) {
+	gmt_len = strftime(gmt, sizeof(gmt),
+			   lp_parm_const_string(SNUM(handle->conn), "shadow",
+						"format", GMT_FORMAT),
+			   &snap_tm);
+	if (gmt_len == 0) {
+		DEBUG(10, ("strftime failed\n"));
 		return NULL;
 	}
-
-	gmt_start = pcopy + prefix_len;
-
-	/*
-	 * Copy away "@GMT-YYYY.MM.DD-HH.MM.SS"
-	 */
-	memcpy(buf, gmt_start+1, GMT_NAME_LEN);
-
-	/*
-	 * Make space for it including a trailing /
-	 */
-	memmove(pcopy + GMT_NAME_LEN + 1, pcopy, prefix_len);
-
-	/*
-	 * Move in "@GMT-YYYY.MM.DD-HH.MM.SS/" at the beginning again
-	 */
-	memcpy(pcopy, buf, GMT_NAME_LEN);
-	pcopy[GMT_NAME_LEN] = '/';
-
-	DEBUG(10, ("shadow_copy2_normalise_path: %s -> %s\n", path, pcopy));
-
-	return pcopy;
+	return talloc_asprintf(talloc_tos(), "/%s/%s",
+			       lp_parm_const_string(
+				       SNUM(handle->conn), "shadow", "snapdir",
+				       ".snapshots"),
+			       gmt);
 }
 
-/*
-  convert a name to the shadow directory
- */
-
-#define _SHADOW2_NEXT(op, args, rtype, eret, extra) do { \
-	const char *name = fname; \
-	const char *gmt_start; \
-	if (shadow_copy2_match_name(fname, &gmt_start)) {	\
-		char *name2; \
-		rtype ret; \
-		name2 = convert_shadow2_name(handle, fname, gmt_start);	\
-		if (name2 == NULL) { \
-			errno = EINVAL; \
-			return eret; \
-		} \
-		name = name2; \
-		ret = SMB_VFS_NEXT_ ## op args; \
-		talloc_free(name2); \
-		if (ret != eret) extra; \
-		return ret; \
-	} else { \
-		return SMB_VFS_NEXT_ ## op args; \
-	} \
-} while (0)
-
-#define _SHADOW2_NEXT_SMB_FNAME(op, args, rtype, eret, extra) do { \
-	const char *gmt_start; \
-	if (shadow_copy2_match_name(smb_fname->base_name, &gmt_start)) { \
-		char *name2; \
-		char *smb_base_name_tmp = NULL; \
-		rtype ret; \
-		name2 = convert_shadow2_name(handle, smb_fname->base_name, gmt_start); \
-		if (name2 == NULL) { \
-			errno = EINVAL; \
-			return eret; \
-		} \
-		smb_base_name_tmp = smb_fname->base_name; \
-		smb_fname->base_name = name2; \
-		ret = SMB_VFS_NEXT_ ## op args; \
-		smb_fname->base_name = smb_base_name_tmp; \
-		talloc_free(name2); \
-		if (ret != eret) extra; \
-		return ret; \
-	} else { \
-		return SMB_VFS_NEXT_ ## op args; \
-	} \
-} while (0)
+static bool shadow_copy2_strip_snapshot(TALLOC_CTX *mem_ctx,
+					struct vfs_handle_struct *handle,
+					const char *name,
+					time_t *ptimestamp,
+					char **pstripped)
+{
+	struct tm tm;
+	time_t timestamp;
+	const char *p;
+	char *q;
+	char *stripped;
+	size_t rest_len, dst_len;
 
-/*
-  convert a name to the shadow directory: NTSTATUS-specific handling
- */
+	p = strstr_m(name, "@GMT-");
+	if (p == NULL) {
+		goto no_snapshot;
+	}
+	if ((p > name) && (p[-1] != '/')) {
+		goto no_snapshot;
+	}
+	q = strptime(p, GMT_FORMAT, &tm);
+	if (q == NULL) {
+		goto no_snapshot;
+	}
+	tm.tm_isdst = -1;
+	timestamp = mktime(&tm);
+	if (timestamp == (time_t)-1) {
+		goto no_snapshot;
+	}
+	if ((p == name) && (q[0] == '\0')) {
+		if (pstripped != NULL) {
+			stripped = talloc_strdup(mem_ctx, "");
+			if (stripped == NULL) {
+				return false;
+			}
+			*pstripped = stripped;
+		}
+		*ptimestamp = timestamp;
+		return true;
+	}
+	if (q[0] != '/') {
+		goto no_snapshot;
+	}
+	q += 1;
+
+	rest_len = strlen(q);
+	dst_len = (p-name) + rest_len;
+
+	if (lp_parm_bool(SNUM(handle->conn), "shadow", "snapdirseverywhere",
+			 false)) {
+		char *insert;
+		bool have_insert;
+		insert = shadow_copy2_insert_string(talloc_tos(), handle,
+						    timestamp);
+		if (insert == NULL) {
+			errno = ENOMEM;
+			return false;
+		}
 
-#define _SHADOW2_NTSTATUS_NEXT(op, args, eret, extra) do { \
-        const char *name = fname; \
-        const char *gmt_start; \
-        if (shadow_copy2_match_name(fname, &gmt_start)) {	\
-                char *name2; \
-                NTSTATUS ret; \
-                name2 = convert_shadow2_name(handle, fname, gmt_start);	\
-                if (name2 == NULL) { \
-                        errno = EINVAL; \
-                        return eret; \
-                } \
-                name = name2; \
-                ret = SMB_VFS_NEXT_ ## op args; \
-                talloc_free(name2); \
-                if (!NT_STATUS_EQUAL(ret, eret)) extra; \
-                return ret; \
-        } else { \
-                return SMB_VFS_NEXT_ ## op args; \
-        } \
-} while (0)
-
-#define SHADOW2_NTSTATUS_NEXT(op, args, eret) _SHADOW2_NTSTATUS_NEXT(op, args, eret, )
-
-#define SHADOW2_NEXT(op, args, rtype, eret) _SHADOW2_NEXT(op, args, rtype, eret, )
-
-#define SHADOW2_NEXT_SMB_FNAME(op, args, rtype, eret) _SHADOW2_NEXT_SMB_FNAME(op, args, rtype, eret, )
-
-#define SHADOW2_NEXT2(op, args) do { \
-	const char *gmt_start1, *gmt_start2; \
-	if (shadow_copy2_match_name(oldname, &gmt_start1) || \
-	    shadow_copy2_match_name(newname, &gmt_start2)) {	\
-		errno = EROFS; \
-		return -1; \
-	} else { \
-		return SMB_VFS_NEXT_ ## op args; \
-	} \
-} while (0)
-
-#define SHADOW2_NEXT2_SMB_FNAME(op, args) do { \
-	const char *gmt_start1, *gmt_start2; \
-	if (shadow_copy2_match_name(smb_fname_src->base_name, &gmt_start1) || \
-	    shadow_copy2_match_name(smb_fname_dst->base_name, &gmt_start2)) { \
-		errno = EROFS; \
-		return -1; \
-	} else { \
-		return SMB_VFS_NEXT_ ## op args; \
-	} \
-} while (0)
+		have_insert = (strstr(name, insert+1) != NULL);
+		TALLOC_FREE(insert);
+		if (have_insert) {
+			goto no_snapshot;
+		}
+	}
 
+	if (pstripped != NULL) {
+		stripped = talloc_array(mem_ctx, char, dst_len+1);
+		if (stripped == NULL) {
+			errno = ENOMEM;
+			return false;
+		}
+		if (p > name) {
+			memcpy(stripped, name, p-name);
+		}
+		if (rest_len > 0) {
+			memcpy(stripped + (p-name), q, rest_len);
+		}
+		stripped[dst_len] = '\0';
+		*pstripped = stripped;
+	}
+	*ptimestamp = timestamp;
+	return true;
+no_snapshot:
+	*ptimestamp = 0;
+	return true;
+}
 
-/*
-  find the mount point of a filesystem
- */
-static char *find_mount_point(TALLOC_CTX *mem_ctx, vfs_handle_struct *handle)
+static char *shadow_copy2_find_mount_point(TALLOC_CTX *mem_ctx,
+					   vfs_handle_struct *handle)
 {
 	char *path = talloc_strdup(mem_ctx, handle->conn->connectpath);
 	dev_t dev;
@@ -358,164 +281,152 @@ static char *find_mount_point(TALLOC_CTX *mem_ctx, vfs_handle_struct *handle)
 		}
 	}
 
-	return path;	
+	return path;
 }
 
-/*
-  work out the location of the snapshot for this share
- */
-static const char *shadow_copy2_find_snapdir(TALLOC_CTX *mem_ctx, vfs_handle_struct *handle)
+static char *shadow_copy2_convert(TALLOC_CTX *mem_ctx,
+				  struct vfs_handle_struct *handle,
+				  const char *name, time_t timestamp)
 {
-	const char *snapdir;
-	char *mount_point;
-	const char *ret;
-
-	snapdir = lp_parm_const_string(SNUM(handle->conn), "shadow", "snapdir", NULL);
-	if (snapdir == NULL) {
-		return NULL;
-	}
-	/* if its an absolute path, we're done */
-	if (*snapdir == '/') {
-		return snapdir;
+	struct smb_filename converted_fname;
+	char *result = NULL;
+	size_t *slashes = NULL;
+	unsigned num_slashes;
+	char *path = NULL;
+	size_t pathlen;
+	char *insert = NULL;
+	char *converted = NULL;
+	size_t insertlen;
+	int i, saved_errno;
+	size_t min_offset;
+
+	path = talloc_asprintf(mem_ctx, "%s/%s", handle->conn->connectpath,
+			       name);
+	if (path == NULL) {
+		errno = ENOMEM;
+		goto fail;
 	}
+	pathlen = talloc_get_size(path)-1;
 
-	/* other its relative to the filesystem mount point */
-	mount_point = find_mount_point(mem_ctx, handle);
-	if (mount_point == NULL) {
-		return NULL;
-	}
-
-	ret = talloc_asprintf(mem_ctx, "%s/%s", mount_point, snapdir);
-	talloc_free(mount_point);
-	return ret;
-}
-
-/*
-  work out the location of the base directory for snapshots of this share
- */
-static const char *shadow_copy2_find_basedir(TALLOC_CTX *mem_ctx, vfs_handle_struct *handle)
-{
-	const char *basedir = lp_parm_const_string(SNUM(handle->conn), "shadow", "basedir", NULL);
+	DEBUG(10, ("converting %s\n", path));
 
-	/* other its the filesystem mount point */
-	if (basedir == NULL) {
-		basedir = find_mount_point(mem_ctx, handle);
+	if (!shadow_copy2_find_slashes(talloc_tos(), path,
+				       &slashes, &num_slashes)) {
+		goto fail;
 	}
-
-	return basedir;
-}
-
-/*
-  convert a filename from a share relative path, to a path in the
-  snapshot directory
- */
-static char *convert_shadow2_name(vfs_handle_struct *handle, const char *fname, const char *gmt_path)
-{
-	TALLOC_CTX *tmp_ctx = talloc_new(handle->data);
-	const char *snapdir, *relpath, *baseoffset, *basedir;
-	size_t baselen;
-	char *ret, *prefix;
-
-	struct tm timestamp;
-	time_t timestamp_t;
-	char snapshot[MAXPATHLEN];
-	const char *fmt;
-
-	fmt = lp_parm_const_string(SNUM(handle->conn), "shadow",
-				   "format", SHADOW_COPY2_DEFAULT_FORMAT);
-
-	snapdir = shadow_copy2_find_snapdir(tmp_ctx, handle);
-	if (snapdir == NULL) {
-		DEBUG(2,("no snapdir found for share at %s\n", handle->conn->connectpath));
-		talloc_free(tmp_ctx);
-		return NULL;
+	insert = shadow_copy2_insert_string(talloc_tos(), handle, timestamp);
+	if (insert == NULL) {
+		goto fail;
 	}
-
-	basedir = shadow_copy2_find_basedir(tmp_ctx, handle);
-	if (basedir == NULL) {
-		DEBUG(2,("no basedir found for share at %s\n", handle->conn->connectpath));
-		talloc_free(tmp_ctx);
-		return NULL;
+	insertlen = talloc_get_size(insert)-1;
+	converted = talloc_array(mem_ctx, char, pathlen + insertlen + 1);
+	if (converted == NULL) {
+		goto fail;
 	}
 
-	prefix = talloc_asprintf(tmp_ctx, "%s/@GMT-", snapdir);
-	if (strncmp(fname, prefix, (talloc_get_size(prefix)-1)) == 0) {
-		/* this looks like as we have already normalized it, leave it untouched*/
-		talloc_free(tmp_ctx);
-		return talloc_strdup(handle->data, fname);
+	if (path[pathlen-1] != '/') {
+		/*
+		 * Append a fake slash to find the snapshot root
+		 */
+		size_t *tmp;
+		tmp = talloc_realloc(talloc_tos(), slashes,
+				     size_t, num_slashes+1);
+		if (tmp == NULL) {
+			goto fail;
+		}
+		slashes = tmp;
+		slashes[num_slashes] = pathlen;
+		num_slashes += 1;
 	}
 
-	if (strncmp(fname, "@GMT-", 5) != 0) {
-		fname = shadow_copy2_normalise_path(tmp_ctx, fname, gmt_path);
-		if (fname == NULL) {
-			talloc_free(tmp_ctx);
-			return NULL;
+	min_offset = 0;
+
+	if (!lp_parm_bool(SNUM(handle->conn), "shadow", "crossmountpoints",
+			  false)) {
+		char *mount_point;
+
+		mount_point = shadow_copy2_find_mount_point(talloc_tos(),
+							    handle);
+		if (mount_point == NULL) {
+			goto fail;
 		}
+		min_offset = strlen(mount_point);
+		TALLOC_FREE(mount_point);
 	}
 
-	ZERO_STRUCT(timestamp);
-	relpath = strptime(fname, SHADOW_COPY2_GMT_FORMAT, &timestamp);
-	if (relpath == NULL) {
-		talloc_free(tmp_ctx);
-		return NULL;
-	}
+	memcpy(converted, path, pathlen+1);
+	converted[pathlen+insertlen] = '\0';
 
-	/* relpath is the remaining portion of the path after the @GMT-xxx */
+	ZERO_STRUCT(converted_fname);
+	converted_fname.base_name = converted;
 
-	if (lp_parm_bool(SNUM(handle->conn), "shadow", "localtime",
-			 SHADOW_COPY2_DEFAULT_LOCALTIME))
-	{
-		timestamp_t = timegm(&timestamp);
-		localtime_r(&timestamp_t, &timestamp);
-	}
+	for (i = num_slashes-1; i>=0; i--) {
+		int ret;
+		size_t offset;
 
-	strftime(snapshot, MAXPATHLEN, fmt, &timestamp);
+		offset = slashes[i];
 
-	baselen = strlen(basedir);
-	baseoffset = handle->conn->connectpath + baselen;
+		if (offset < min_offset) {
+			errno = ENOENT;
+			goto fail;
+		}
 
-	/* some sanity checks */
-	if (strncmp(basedir, handle->conn->connectpath, baselen) != 0 ||
-	    (handle->conn->connectpath[baselen] != 0 && handle->conn->connectpath[baselen] != '/')) {
-		DEBUG(0,("convert_shadow2_name: basedir %s is not a parent of %s\n",
-			 basedir, handle->conn->connectpath));
-		talloc_free(tmp_ctx);
-		return NULL;
-	}
+		memcpy(converted+offset, insert, insertlen);
 
-	if (*relpath == '/') relpath++;
-	if (*baseoffset == '/') baseoffset++;
+		offset += insertlen;
+		memcpy(converted+offset, path + slashes[i],
+		       pathlen - slashes[i]);
 
-	ret = talloc_asprintf(handle->data, "%s/%s/%s/%s",
-			      snapdir, 
-			      snapshot,
-			      baseoffset, 
-			      relpath);
-	DEBUG(6,("convert_shadow2_name: '%s' -> '%s'\n", fname, ret));
-	talloc_free(tmp_ctx);
-	return ret;
-}
+		ret = SMB_VFS_NEXT_LSTAT(handle, &converted_fname);
 
+		DEBUG(10, ("Trying %s: %d (%s)\n", converted,
+			   ret, ret == 0 ? "ok" : strerror(errno)));
+		if (ret == 0) {
+			/* success */
+			break;
+		}
+		if (errno == ENOTDIR) {
+			/*
+			 * This is a valid condition: We appended the
+			 * .snaphots/@GMT.. to a file name. Just try
+			 * with the upper levels.
+			 */
+			continue;
+		}
+		if (errno != ENOENT) {
+			/* Other problem than "not found" */
+			goto fail;
+		}
+	}
 
-/*
-  simple string hash
- */
-static uint32 string_hash(const char *s)
-{
-        uint32 n = 0;
-	while (*s) {
-                n = ((n << 5) + n) ^ (uint32)(*s++);
-        }
-        return n;
+	if (i >= 0) {
+		/*
+		 * Found something
+		 */
+		DEBUG(10, ("Found %s\n", converted));
+		result = converted;
+		converted = NULL;
+	} else {
+		errno = ENOENT;
+	}
+fail:
+	saved_errno = errno;
+	TALLOC_FREE(converted);
+	TALLOC_FREE(insert);
+	TALLOC_FREE(slashes);
+	TALLOC_FREE(path);
+	errno = saved_errno;
+	return result;
 }
 
 /*
   modify a sbuf return to ensure that inodes in the shadow directory
   are different from those in the main directory
  */
-static void convert_sbuf(vfs_handle_struct *handle, const char *fname, SMB_STRUCT_STAT *sbuf)
+static void convert_sbuf(vfs_handle_struct *handle, const char *fname,
+			 SMB_STRUCT_STAT *sbuf)
 {
-	if (lp_parm_bool(SNUM(handle->conn), "shadow", "fixinodes", False)) {		
+	if (lp_parm_bool(SNUM(handle->conn), "shadow", "fixinodes", False)) {
 		/* some snapshot systems, like GPFS, return the name
 		   device:inode for the snapshot files as the current
 		   files. That breaks the 'restore' button in the shadow copy
@@ -526,7 +437,10 @@ static void convert_sbuf(vfs_handle_struct *handle, const char *fname, SMB_STRUC
 		   number collision, but I can't see a better approach
 		   without significant VFS changes
 		*/
-		uint32_t shash = string_hash(fname) & 0xFF000000;
+		uint32_t shash;
+		TDB_DATA data = string_tdb_data(fname);
+
+		shash = tdb_jenkins_hash(&data) & 0xFF000000;
 		if (shash == 0) {
 			shash = 1;
 		}
@@ -534,316 +448,607 @@ static void convert_sbuf(vfs_handle_struct *handle, const char *fname, SMB_STRUC
 	}
 }
 
+static SMB_STRUCT_DIR *shadow_copy2_opendir(vfs_handle_struct *handle,
+					    const char *fname,
+					    const char *mask,
+					    uint32 attr)
+{
+	time_t timestamp;
+	char *stripped;
+	SMB_STRUCT_DIR *ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return NULL;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_OPENDIR(handle, fname, mask, attr);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return NULL;
+	}
+	ret = SMB_VFS_NEXT_OPENDIR(handle, conv, mask, attr);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
 static int shadow_copy2_rename(vfs_handle_struct *handle,
 			       const struct smb_filename *smb_fname_src,
 			       const struct smb_filename *smb_fname_dst)
 {
-	if (shadow_copy2_match_name(smb_fname_src->base_name, NULL)) {
+	time_t timestamp_src, timestamp_dst;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 smb_fname_src->base_name,
+					 &timestamp_src, NULL)) {
+		return -1;
+	}
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 smb_fname_dst->base_name,
+					 &timestamp_dst, NULL)) {
+		return -1;
+	}
+	if (timestamp_src != 0) {
 		errno = EXDEV;
 		return -1;
 	}
-	SHADOW2_NEXT2_SMB_FNAME(RENAME,
-				(handle, smb_fname_src, smb_fname_dst));
+	if (timestamp_dst != 0) {
+		errno = EROFS;
+		return -1;
+	}
+	return SMB_VFS_NEXT_RENAME(handle, smb_fname_src, smb_fname_dst);
 }
 
 static int shadow_copy2_symlink(vfs_handle_struct *handle,
 				const char *oldname, const char *newname)
 {
-	SHADOW2_NEXT2(SYMLINK, (handle, oldname, newname));
-}
+	time_t timestamp_old, timestamp_new;
 
-static int shadow_copy2_link(vfs_handle_struct *handle,
-			  const char *oldname, const char *newname)
-{
-	SHADOW2_NEXT2(LINK, (handle, oldname, newname));
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, oldname,
+					 &timestamp_old, NULL)) {
+		return -1;
+	}
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, newname,
+					 &timestamp_new, NULL)) {
+		return -1;
+	}
+	if ((timestamp_old != 0) || (timestamp_new != 0)) {
+		errno = EROFS;
+		return -1;
+	}
+	return SMB_VFS_NEXT_SYMLINK(handle, oldname, newname);
 }
 
-static int shadow_copy2_open(vfs_handle_struct *handle,
-			     struct smb_filename *smb_fname, files_struct *fsp,
-			     int flags, mode_t mode)
+static int shadow_copy2_link(vfs_handle_struct *handle,
+			     const char *oldname, const char *newname)
 {
-	SHADOW2_NEXT_SMB_FNAME(OPEN,
-			       (handle, smb_fname, fsp, flags, mode),
-			       int, -1);
-}
+	time_t timestamp_old, timestamp_new;
 
-static SMB_STRUCT_DIR *shadow_copy2_opendir(vfs_handle_struct *handle,
-			  const char *fname, const char *mask, uint32 attr)
-{
-        SHADOW2_NEXT(OPENDIR, (handle, name, mask, attr), SMB_STRUCT_DIR *, NULL);
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, oldname,
+					 &timestamp_old, NULL)) {
+		return -1;
+	}
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, newname,
+					 &timestamp_new, NULL)) {
+		return -1;
+	}
+	if ((timestamp_old != 0) || (timestamp_new != 0)) {
+		errno = EROFS;
+		return -1;
+	}
+	return SMB_VFS_NEXT_LINK(handle, oldname, newname);
 }
 
 static int shadow_copy2_stat(vfs_handle_struct *handle,
 			     struct smb_filename *smb_fname)
 {
-        _SHADOW2_NEXT_SMB_FNAME(STAT, (handle, smb_fname), int, -1,
-				convert_sbuf(handle, smb_fname->base_name,
-					     &smb_fname->st));
+	time_t timestamp;
+	char *stripped, *tmp;
+	int ret, saved_errno;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 smb_fname->base_name,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_STAT(handle, smb_fname);
+	}
+
+	tmp = smb_fname->base_name;
+	smb_fname->base_name = shadow_copy2_convert(
+		talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+
+	if (smb_fname->base_name == NULL) {
+		smb_fname->base_name = tmp;
+		return -1;
+	}
+
+	ret = SMB_VFS_NEXT_STAT(handle, smb_fname);
+	saved_errno = errno;
+
+	TALLOC_FREE(smb_fname->base_name);
+	smb_fname->base_name = tmp;
+
+	if (ret == 0) {
+		convert_sbuf(handle, smb_fname->base_name, &smb_fname->st);
+	}
+	errno = saved_errno;
+	return ret;
 }
 
 static int shadow_copy2_lstat(vfs_handle_struct *handle,
 			      struct smb_filename *smb_fname)
 {
-        _SHADOW2_NEXT_SMB_FNAME(LSTAT, (handle, smb_fname), int, -1,
-				convert_sbuf(handle, smb_fname->base_name,
-					     &smb_fname->st));
+	time_t timestamp;
+	char *stripped, *tmp;
+	int ret, saved_errno;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 smb_fname->base_name,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_LSTAT(handle, smb_fname);
+	}
+
+	tmp = smb_fname->base_name;
+	smb_fname->base_name = shadow_copy2_convert(
+		talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+
+	if (smb_fname->base_name == NULL) {
+		smb_fname->base_name = tmp;
+		return -1;
+	}
+
+	ret = SMB_VFS_NEXT_LSTAT(handle, smb_fname);
+	saved_errno = errno;
+
+	TALLOC_FREE(smb_fname->base_name);
+	smb_fname->base_name = tmp;
+
+	if (ret == 0) {
+		convert_sbuf(handle, smb_fname->base_name, &smb_fname->st);
+	}
+	errno = saved_errno;
+	return ret;
 }
 
-static int shadow_copy2_fstat(vfs_handle_struct *handle, files_struct *fsp, SMB_STRUCT_STAT *sbuf)
+static int shadow_copy2_fstat(vfs_handle_struct *handle, files_struct *fsp,
+			      SMB_STRUCT_STAT *sbuf)
 {
-	int ret = SMB_VFS_NEXT_FSTAT(handle, fsp, sbuf);
-	if (ret == 0 && shadow_copy2_match_name(fsp->fsp_name->base_name, NULL)) {
+	time_t timestamp;
+	int ret;
+
+	ret = SMB_VFS_NEXT_FSTAT(handle, fsp, sbuf);
+	if (ret == -1) {
+		return ret;
+	}
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 fsp->fsp_name->base_name,
+					 &timestamp, NULL)) {
+		return 0;
+	}
+	if (timestamp != 0) {
 		convert_sbuf(handle, fsp->fsp_name->base_name, sbuf);
 	}
+	return 0;
+}
+
+static int shadow_copy2_open(vfs_handle_struct *handle,
+			     struct smb_filename *smb_fname, files_struct *fsp,
+			     int flags, mode_t mode)
+{
+	time_t timestamp;
+	char *stripped, *tmp;
+	int ret, saved_errno;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 smb_fname->base_name,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_OPEN(handle, smb_fname, fsp, flags, mode);
+	}
+
+	tmp = smb_fname->base_name;
+	smb_fname->base_name = shadow_copy2_convert(
+		talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+
+	if (smb_fname->base_name == NULL) {
+		smb_fname->base_name = tmp;
+		return -1;
+	}
+
+	ret = SMB_VFS_NEXT_OPEN(handle, smb_fname, fsp, flags, mode);
+	saved_errno = errno;
+
+	TALLOC_FREE(smb_fname->base_name);
+	smb_fname->base_name = tmp;
+
+	errno = saved_errno;
 	return ret;
 }
 
 static int shadow_copy2_unlink(vfs_handle_struct *handle,
-			       const struct smb_filename *smb_fname_in)
+			       const struct smb_filename *smb_fname)
 {
-	struct smb_filename *smb_fname = NULL;
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	struct smb_filename *conv;
 	NTSTATUS status;
 
-	status = copy_smb_filename(talloc_tos(), smb_fname_in, &smb_fname);
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 smb_fname->base_name,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_UNLINK(handle, smb_fname);
+	}
+	status = copy_smb_filename(talloc_tos(), smb_fname, &conv);
 	if (!NT_STATUS_IS_OK(status)) {
-		errno = map_errno_from_nt_status(status);
+		errno = ENOMEM;
 		return -1;
 	}
-
-        SHADOW2_NEXT_SMB_FNAME(UNLINK, (handle, smb_fname), int, -1);
+	conv->base_name = shadow_copy2_convert(
+		conv, handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv->base_name == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_UNLINK(handle, conv);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
 }
 
-static int shadow_copy2_chmod(vfs_handle_struct *handle,
-		       const char *fname, mode_t mode)
+static int shadow_copy2_chmod(vfs_handle_struct *handle, const char *fname,
+			      mode_t mode)
 {
-        SHADOW2_NEXT(CHMOD, (handle, name, mode), int, -1);
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_CHMOD(handle, fname, mode);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_CHMOD(handle, conv, mode);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
 }
 
-static int shadow_copy2_chown(vfs_handle_struct *handle,
-		       const char *fname, uid_t uid, gid_t gid)
+static int shadow_copy2_chown(vfs_handle_struct *handle, const char *fname,
+			      uid_t uid, gid_t gid)
 {
-        SHADOW2_NEXT(CHOWN, (handle, name, uid, gid), int, -1);
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_CHOWN(handle, fname, uid, gid);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_CHOWN(handle, conv, uid, gid);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
 }
 
 static int shadow_copy2_chdir(vfs_handle_struct *handle,
-		       const char *fname)
+			      const char *fname)
 {
-	SHADOW2_NEXT(CHDIR, (handle, name), int, -1);
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_CHDIR(handle, fname);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_CHDIR(handle, conv);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
 }
 
 static int shadow_copy2_ntimes(vfs_handle_struct *handle,
-			       const struct smb_filename *smb_fname_in,
+			       const struct smb_filename *smb_fname,
 			       struct smb_file_time *ft)
 {
-	struct smb_filename *smb_fname = NULL;
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	struct smb_filename *conv;
 	NTSTATUS status;
 
-	status = copy_smb_filename(talloc_tos(), smb_fname_in, &smb_fname);
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 smb_fname->base_name,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_NTIMES(handle, smb_fname, ft);
+	}
+	status = copy_smb_filename(talloc_tos(), smb_fname, &conv);
 	if (!NT_STATUS_IS_OK(status)) {
-		errno = map_errno_from_nt_status(status);
+		errno = ENOMEM;
 		return -1;
 	}
-
-        SHADOW2_NEXT_SMB_FNAME(NTIMES, (handle, smb_fname, ft), int, -1);
+	conv->base_name = shadow_copy2_convert(
+		conv, handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv->base_name == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_NTIMES(handle, conv, ft);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
 }
 
 static int shadow_copy2_readlink(vfs_handle_struct *handle,
 				 const char *fname, char *buf, size_t bufsiz)
 {
-        SHADOW2_NEXT(READLINK, (handle, name, buf, bufsiz), int, -1);
-}
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
 
-static int shadow_copy2_mknod(vfs_handle_struct *handle,
-		       const char *fname, mode_t mode, SMB_DEV_T dev)
-{
-        SHADOW2_NEXT(MKNOD, (handle, name, mode, dev), int, -1);
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_READLINK(handle, fname, buf, bufsiz);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_READLINK(handle, conv, buf, bufsiz);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
 }
 
-static char *shadow_copy2_realpath(vfs_handle_struct *handle,
-			    const char *fname)
+static int shadow_copy2_mknod(vfs_handle_struct *handle,
+			      const char *fname, mode_t mode, SMB_DEV_T dev)
 {
-	const char *gmt;
-
-	if (shadow_copy2_match_name(fname, &gmt)
-	    && (gmt[GMT_NAME_LEN] == '\0')) {
-		char *copy;
-
-		copy = talloc_strdup(talloc_tos(), fname);
-		if (copy == NULL) {
-			errno = ENOMEM;
-			return NULL;
-		}
-
-		copy[gmt - fname] = '.';
-		copy[gmt - fname + 1] = '\0';
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
 
-		DEBUG(10, ("calling NEXT_REALPATH with %s\n", copy));
-		SHADOW2_NEXT(REALPATH, (handle, name), char *,
-			     NULL);
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_MKNOD(handle, fname, mode, dev);
 	}
-        SHADOW2_NEXT(REALPATH, (handle, name), char *, NULL);
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_MKNOD(handle, conv, mode, dev);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
 }
 
-static const char *shadow_copy2_connectpath(struct vfs_handle_struct *handle,
-					    const char *fname)
+static char *shadow_copy2_realpath(vfs_handle_struct *handle,
+				   const char *fname)
 {
-	TALLOC_CTX *tmp_ctx;
-	const char *snapdir, *baseoffset, *basedir, *gmt_start;
-	size_t baselen;
-	char *ret;
-
-	DEBUG(10, ("shadow_copy2_connectpath called with %s\n", fname));
-
-	if (!shadow_copy2_match_name(fname, &gmt_start)) {
-		return handle->conn->connectpath;
+	time_t timestamp;
+	char *stripped = NULL;
+	char *tmp = NULL;
+	char *result = NULL;
+	char *inserted = NULL;
+	char *inserted_to, *inserted_end;
+	int saved_errno;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		goto done;
 	}
-
-        /*
-         * We have to create a real temporary context because we have
-         * to put our result on talloc_tos(). Thus we can't use a
-         * talloc_stackframe() here.
-         */
-	tmp_ctx = talloc_new(talloc_tos());
-
-	fname = shadow_copy2_normalise_path(tmp_ctx, fname, gmt_start);
-	if (fname == NULL) {
-		TALLOC_FREE(tmp_ctx);
-		return NULL;
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_REALPATH(handle, fname);
 	}
 
-	snapdir = shadow_copy2_find_snapdir(tmp_ctx, handle);
-	if (snapdir == NULL) {
-		DEBUG(2,("no snapdir found for share at %s\n",
-			 handle->conn->connectpath));
-		TALLOC_FREE(tmp_ctx);
-		return NULL;
+	tmp = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	if (tmp == NULL) {
+		goto done;
 	}
 
-	basedir = shadow_copy2_find_basedir(tmp_ctx, handle);
-	if (basedir == NULL) {
-		DEBUG(2,("no basedir found for share at %s\n",
-			 handle->conn->connectpath));
-		TALLOC_FREE(tmp_ctx);
-		return NULL;
+	result = SMB_VFS_NEXT_REALPATH(handle, tmp);
+	if (result == NULL) {
+		goto done;
 	}
 
-	baselen = strlen(basedir);
-	baseoffset = handle->conn->connectpath + baselen;
-
-	/* some sanity checks */
-	if (strncmp(basedir, handle->conn->connectpath, baselen) != 0 ||
-	    (handle->conn->connectpath[baselen] != 0
-	     && handle->conn->connectpath[baselen] != '/')) {
-		DEBUG(0,("shadow_copy2_connectpath: basedir %s is not a "
-			 "parent of %s\n", basedir,
-			 handle->conn->connectpath));
-		TALLOC_FREE(tmp_ctx);
-		return NULL;
+	/*
+	 * Take away what we've inserted. This removes the @GMT-thingy
+	 * completely, but will give a path under the share root.
+	 */
+	inserted = shadow_copy2_insert_string(talloc_tos(), handle, timestamp);
+	if (inserted == NULL) {
+		goto done;
 	}
-
-	if (*baseoffset == '/') baseoffset++;
-
-	ret = talloc_asprintf(talloc_tos(), "%s/%.*s/%s",
-			      snapdir,
-			      GMT_NAME_LEN, fname,
-			      baseoffset);
-	DEBUG(6,("shadow_copy2_connectpath: '%s' -> '%s'\n", fname, ret));
-	TALLOC_FREE(tmp_ctx);
-	return ret;
+	inserted_to = strstr_m(result, inserted);
+	if (inserted_to == NULL) {
+		DEBUG(2, ("SMB_VFS_NEXT_REALPATH removed %s\n", inserted));
+		goto done;
+	}
+	inserted_end = inserted_to + talloc_get_size(inserted) - 1;
+	memmove(inserted_to, inserted_end, strlen(inserted_end)+1);
+
+done:
+	saved_errno = errno;
+	TALLOC_FREE(inserted);
+	TALLOC_FREE(tmp);
+	TALLOC_FREE(stripped);
+	errno = saved_errno;
+	return result;
 }
 
-static NTSTATUS shadow_copy2_get_nt_acl(vfs_handle_struct *handle,
-			       const char *fname, uint32 security_info,
-			       struct security_descriptor **ppdesc)
+static char *have_snapdir(struct vfs_handle_struct *handle,
+			  const char *path)
 {
-        SHADOW2_NTSTATUS_NEXT(GET_NT_ACL, (handle, name, security_info, ppdesc), NT_STATUS_ACCESS_DENIED);
-}
+	struct smb_filename smb_fname;
+	int ret;
+
+	ZERO_STRUCT(smb_fname);
+	smb_fname.base_name = talloc_asprintf(
+		talloc_tos(), "%s/%s", path,
+		lp_parm_const_string(SNUM(handle->conn), "shadow", "snapdir",
+				     ".snapshots"));
+	if (smb_fname.base_name == NULL) {
+		return NULL;
+	}
 
-static int shadow_copy2_mkdir(vfs_handle_struct *handle,  const char *fname, mode_t mode)
-{
-        SHADOW2_NEXT(MKDIR, (handle, name, mode), int, -1);
+	ret = SMB_VFS_NEXT_STAT(handle, &smb_fname);
+	if ((ret == 0) && (S_ISDIR(smb_fname.st.st_ex_mode))) {
+		return smb_fname.base_name;
+	}
+	TALLOC_FREE(smb_fname.base_name);
+	return NULL;
 }
 
-static int shadow_copy2_rmdir(vfs_handle_struct *handle,  const char *fname)
+static char *shadow_copy2_find_snapdir(TALLOC_CTX *mem_ctx,
+				       struct vfs_handle_struct *handle,
+				       struct smb_filename *smb_fname)
 {
-        SHADOW2_NEXT(RMDIR, (handle, name), int, -1);
-}
+	char *path, *p;
+	char *snapdir;
 
-static int shadow_copy2_chflags(vfs_handle_struct *handle, const char *fname,
-				unsigned int flags)
-{
-        SHADOW2_NEXT(CHFLAGS, (handle, name, flags), int, -1);
-}
+	path = talloc_asprintf(mem_ctx, "%s/%s",
+			       handle->conn->connectpath,
+			       smb_fname->base_name);
+	if (path == NULL) {
+		return NULL;
+	}
 
-static ssize_t shadow_copy2_getxattr(vfs_handle_struct *handle,
-				  const char *fname, const char *aname, void *value, size_t size)
-{
-        SHADOW2_NEXT(GETXATTR, (handle, name, aname, value, size), ssize_t, -1);
-}
+	snapdir = have_snapdir(handle, path);
+	if (snapdir != NULL) {
+		TALLOC_FREE(path);
+		return snapdir;
+	}
 
-static ssize_t shadow_copy2_lgetxattr(vfs_handle_struct *handle,
-				      const char *fname, const char *aname, void *value, size_t size)
-{
-        SHADOW2_NEXT(LGETXATTR, (handle, name, aname, value, size), ssize_t, -1);
-}
+	while ((p = strrchr(path, '/')) && (p > path)) {
 
-static ssize_t shadow_copy2_listxattr(struct vfs_handle_struct *handle, const char *fname, 
-				      char *list, size_t size)
-{
-	SHADOW2_NEXT(LISTXATTR, (handle, name, list, size), ssize_t, -1);
-}
+		p[0] = '\0';
 
-static int shadow_copy2_removexattr(struct vfs_handle_struct *handle, const char *fname, 
-				    const char *aname)
-{
-	SHADOW2_NEXT(REMOVEXATTR, (handle, name, aname), int, -1);
+		snapdir = have_snapdir(handle, path);
+		if (snapdir != NULL) {
+			TALLOC_FREE(path);
+			return snapdir;
+		}
+	}
+	TALLOC_FREE(path);
+	return NULL;
 }
 
-static int shadow_copy2_lremovexattr(struct vfs_handle_struct *handle, const char *fname, 
-				     const char *aname)
+static bool shadow_copy2_snapshot_to_gmt(TALLOC_CTX *mem_ctx,
+					 vfs_handle_struct *handle,
+					 const char *name,
+					 char *gmt, size_t gmt_len)
 {
-	SHADOW2_NEXT(LREMOVEXATTR, (handle, name, aname), int, -1);
-}
+	struct tm timestamp;
+	time_t timestamp_t;
+	const char *fmt;
 
-static int shadow_copy2_setxattr(struct vfs_handle_struct *handle, const char *fname, 
-				 const char *aname, const void *value, size_t size, int flags)
-{
-	SHADOW2_NEXT(SETXATTR, (handle, name, aname, value, size, flags), int, -1);
-}
+	fmt = lp_parm_const_string(SNUM(handle->conn), "shadow",
+				   "format", GMT_FORMAT);
 
-static int shadow_copy2_lsetxattr(struct vfs_handle_struct *handle, const char *fname, 
-				  const char *aname, const void *value, size_t size, int flags)
-{
-	SHADOW2_NEXT(LSETXATTR, (handle, name, aname, value, size, flags), int, -1);
-}
+	ZERO_STRUCT(timestamp);
+	if (strptime(name, fmt, &timestamp) == NULL) {
+		DEBUG(10, ("shadow_copy2_snapshot_to_gmt: no match %s: %s\n",
+			   fmt, name));
+		return false;
+	}
 
-static int shadow_copy2_chmod_acl(vfs_handle_struct *handle,
-			   const char *fname, mode_t mode)
-{
-        SHADOW2_NEXT(CHMOD_ACL, (handle, name, mode), int, -1);
+	DEBUG(10, ("shadow_copy2_snapshot_to_gmt: match %s: %s\n", fmt, name));
+
+	if (lp_parm_bool(SNUM(handle->conn), "shadow", "localtime", false)) {
+		timestamp.tm_isdst = -1;
+		timestamp_t = mktime(&timestamp);
+		gmtime_r(&timestamp_t, &timestamp);
+	}
+	strftime(gmt, gmt_len, GMT_FORMAT, &timestamp);
+	return true;
 }
 
 static int shadow_copy2_label_cmp_asc(const void *x, const void *y)
 {
-	return strncmp((char *)x, (char *)y, sizeof(SHADOW_COPY_LABEL));
+	return strncmp((const char *)x, (const char *)y, sizeof(SHADOW_COPY_LABEL));
 }
 
 static int shadow_copy2_label_cmp_desc(const void *x, const void *y)
 {
-	return -strncmp((char *)x, (char *)y, sizeof(SHADOW_COPY_LABEL));
+	return -strncmp((const char *)x, (const char *)y, sizeof(SHADOW_COPY_LABEL));
 }
 
 /*
   sort the shadow copy data in ascending or descending order
  */
 static void shadow_copy2_sort_data(vfs_handle_struct *handle,
-				   SHADOW_COPY_DATA *shadow_copy2_data)
+				   struct shadow_copy_data *shadow_copy2_data)
 {
 	int (*cmpfunc)(const void *, const void *);
 	const char *sort;
 
 	sort = lp_parm_const_string(SNUM(handle->conn), "shadow",
-				    "sort", SHADOW_COPY2_DEFAULT_SORT);
+				    "sort", NULL);
 	if (sort == NULL) {
 		return;
 	}
@@ -867,18 +1072,17 @@ static void shadow_copy2_sort_data(vfs_handle_struct *handle,
 	return;
 }
 
-static int shadow_copy2_get_shadow_copy2_data(vfs_handle_struct *handle, 
-					      files_struct *fsp, 
-					      SHADOW_COPY_DATA *shadow_copy2_data, 
-					      bool labels)
+static int shadow_copy2_get_shadow_copy_data(
+	vfs_handle_struct *handle, files_struct *fsp,
+	struct shadow_copy_data *shadow_copy2_data,
+	bool labels)
 {
 	SMB_STRUCT_DIR *p;
 	const char *snapdir;
 	SMB_STRUCT_DIRENT *d;
-	TALLOC_CTX *tmp_ctx = talloc_new(handle->data);
-	char *snapshot;
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
 
-	snapdir = shadow_copy2_find_snapdir(tmp_ctx, handle);
+	snapdir = shadow_copy2_find_snapdir(tmp_ctx, handle, fsp->fsp_name);
 	if (snapdir == NULL) {
 		DEBUG(0,("shadow:snapdir not found for %s in get_shadow_copy_data\n",
 			 handle->conn->connectpath));
@@ -901,16 +1105,23 @@ static int shadow_copy2_get_shadow_copy2_data(vfs_handle_struct *handle,
 	shadow_copy2_data->labels      = NULL;
 
 	while ((d = SMB_VFS_NEXT_READDIR(handle, p, NULL))) {
+		char snapshot[GMT_NAME_LEN+1];
 		SHADOW_COPY_LABEL *tlabels;
 
-		/* ignore names not of the right form in the snapshot directory */
-		snapshot = shadow_copy2_snapshot_to_gmt(tmp_ctx, handle,
-							d->d_name);
-		DEBUG(6,("shadow_copy2_get_shadow_copy2_data: %s -> %s\n",
-			 d->d_name, snapshot));
-		if (!snapshot) {
+		/*
+		 * ignore names not of the right form in the snapshot
+		 * directory
+		 */
+		if (!shadow_copy2_snapshot_to_gmt(
+			    tmp_ctx, handle, d->d_name,
+			    snapshot, sizeof(snapshot))) {
+
+			DEBUG(6, ("shadow_copy2_get_shadow_copy_data: "
+				  "ignoring %s\n", d->d_name));
 			continue;
 		}
+		DEBUG(6,("shadow_copy2_get_shadow_copy_data: %s -> %s\n",
+			 d->d_name, snapshot));
 
 		if (!labels) {
 			/* the caller doesn't want the labels */
@@ -918,9 +1129,10 @@ static int shadow_copy2_get_shadow_copy2_data(vfs_handle_struct *handle,
 			continue;
 		}
 
-		tlabels = talloc_realloc(shadow_copy2_data->mem_ctx,
+		tlabels = talloc_realloc(shadow_copy2_data,
 					 shadow_copy2_data->labels,
-					 SHADOW_COPY_LABEL, shadow_copy2_data->num_volumes+1);
+					 SHADOW_COPY_LABEL,
+					 shadow_copy2_data->num_volumes+1);
 		if (tlabels == NULL) {
 			DEBUG(0,("shadow_copy2: out of memory\n"));
 			SMB_VFS_NEXT_CLOSEDIR(handle, p);
@@ -930,7 +1142,6 @@ static int shadow_copy2_get_shadow_copy2_data(vfs_handle_struct *handle,
 
 		strlcpy(tlabels[shadow_copy2_data->num_volumes], snapshot,
 			sizeof(*tlabels));
-		talloc_free(snapshot);
 
 		shadow_copy2_data->num_volumes++;
 		shadow_copy2_data->labels = tlabels;
@@ -944,59 +1155,455 @@ static int shadow_copy2_get_shadow_copy2_data(vfs_handle_struct *handle,
 	return 0;
 }
 
-static struct vfs_fn_pointers vfs_shadow_copy2_fns = {
-        .opendir = shadow_copy2_opendir,
-        .mkdir = shadow_copy2_mkdir,
-        .rmdir = shadow_copy2_rmdir,
-        .chflags = shadow_copy2_chflags,
-        .getxattr = shadow_copy2_getxattr,
-        .lgetxattr = shadow_copy2_lgetxattr,
-        .listxattr = shadow_copy2_listxattr,
-        .removexattr = shadow_copy2_removexattr,
-        .lremovexattr = shadow_copy2_lremovexattr,
-        .setxattr = shadow_copy2_setxattr,
-        .lsetxattr = shadow_copy2_lsetxattr,
-        .open_fn = shadow_copy2_open,
-        .rename = shadow_copy2_rename,
-        .stat = shadow_copy2_stat,
-        .lstat = shadow_copy2_lstat,
-        .fstat = shadow_copy2_fstat,
-        .unlink = shadow_copy2_unlink,
-        .chmod = shadow_copy2_chmod,
-        .chown = shadow_copy2_chown,
-        .chdir = shadow_copy2_chdir,
-        .ntimes = shadow_copy2_ntimes,
-        .symlink = shadow_copy2_symlink,
-        .vfs_readlink = shadow_copy2_readlink,
-        .link = shadow_copy2_link,
-        .mknod = shadow_copy2_mknod,
-        .realpath = shadow_copy2_realpath,
-        .connectpath = shadow_copy2_connectpath,
-        .get_nt_acl = shadow_copy2_get_nt_acl,
-        .chmod_acl = shadow_copy2_chmod_acl,
-	.get_shadow_copy_data = shadow_copy2_get_shadow_copy2_data,
-};
+static NTSTATUS shadow_copy2_fget_nt_acl(vfs_handle_struct *handle,
+					struct files_struct *fsp,
+					uint32 security_info,
+					struct security_descriptor **ppdesc)
+{
+	time_t timestamp;
+	char *stripped;
+	NTSTATUS status;
+	char *conv;
 
-NTSTATUS vfs_shadow_copy2_init(void);
-NTSTATUS vfs_shadow_copy2_init(void)
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle,
+					 fsp->fsp_name->base_name,
+					 &timestamp, &stripped)) {
+		return map_nt_error_from_unix(errno);
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_FGET_NT_ACL(handle, fsp, security_info,
+						ppdesc);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return map_nt_error_from_unix(errno);
+	}
+	status = SMB_VFS_NEXT_GET_NT_ACL(handle, conv, security_info, ppdesc);
+	TALLOC_FREE(conv);
+	return status;
+}
+
+static NTSTATUS shadow_copy2_get_nt_acl(vfs_handle_struct *handle,
+					const char *fname,
+					uint32 security_info,
+					struct security_descriptor **ppdesc)
+{
+	time_t timestamp;
+	char *stripped;
+	NTSTATUS status;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return map_nt_error_from_unix(errno);
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_GET_NT_ACL(handle, fname, security_info,
+					       ppdesc);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return map_nt_error_from_unix(errno);
+	}
+	status = SMB_VFS_NEXT_GET_NT_ACL(handle, conv, security_info, ppdesc);
+	TALLOC_FREE(conv);
+	return status;
+}
+
+static int shadow_copy2_mkdir(vfs_handle_struct *handle,
+			      const char *fname, mode_t mode)
 {
-	NTSTATUS ret;
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
 
-	ret = smb_register_vfs(SMB_VFS_INTERFACE_VERSION, "shadow_copy2",
-			       &vfs_shadow_copy2_fns);
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_MKDIR(handle, fname, mode);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_MKDIR(handle, conv, mode);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
 
-	if (!NT_STATUS_IS_OK(ret))
-		return ret;
+static int shadow_copy2_rmdir(vfs_handle_struct *handle, const char *fname)
+{
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
 
-	vfs_shadow_copy2_debug_level = debug_add_class("shadow_copy2");
-	if (vfs_shadow_copy2_debug_level == -1) {
-		vfs_shadow_copy2_debug_level = DBGC_VFS;
-		DEBUG(0, ("%s: Couldn't register custom debugging class!\n",
-			"vfs_shadow_copy2_init"));
-	} else {
-		DEBUG(10, ("%s: Debug class number of '%s': %d\n", 
-			"vfs_shadow_copy2_init","shadow_copy2",vfs_shadow_copy2_debug_level));
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_RMDIR(handle, fname);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_RMDIR(handle, conv);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static int shadow_copy2_chflags(vfs_handle_struct *handle, const char *fname,
+				unsigned int flags)
+{
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_CHFLAGS(handle, fname, flags);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_CHFLAGS(handle, conv, flags);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static ssize_t shadow_copy2_getxattr(vfs_handle_struct *handle,
+				     const char *fname, const char *aname,
+				     void *value, size_t size)
+{
+	time_t timestamp;
+	char *stripped;
+	ssize_t ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_GETXATTR(handle, fname, aname, value,
+					     size);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_GETXATTR(handle, conv, aname, value, size);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static ssize_t shadow_copy2_lgetxattr(vfs_handle_struct *handle,
+				      const char *fname, const char *aname,
+				      void *value, size_t size)
+{
+	time_t timestamp;
+	char *stripped;
+	ssize_t ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_LGETXATTR(handle, fname, aname, value,
+					      size);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_LGETXATTR(handle, conv, aname, value, size);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static ssize_t shadow_copy2_listxattr(struct vfs_handle_struct *handle,
+				      const char *fname,
+				      char *list, size_t size)
+{
+	time_t timestamp;
+	char *stripped;
+	ssize_t ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_LISTXATTR(handle, fname, list, size);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_LISTXATTR(handle, conv, list, size);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static int shadow_copy2_removexattr(vfs_handle_struct *handle,
+				    const char *fname, const char *aname)
+{
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_REMOVEXATTR(handle, fname, aname);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_REMOVEXATTR(handle, conv, aname);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static int shadow_copy2_lremovexattr(vfs_handle_struct *handle,
+				     const char *fname, const char *aname)
+{
+	time_t timestamp;
+	char *stripped;
+	int ret, saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_LREMOVEXATTR(handle, fname, aname);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_LREMOVEXATTR(handle, conv, aname);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static int shadow_copy2_setxattr(struct vfs_handle_struct *handle,
+				 const char *fname,
+				 const char *aname, const void *value,
+				 size_t size, int flags)
+{
+	time_t timestamp;
+	char *stripped;
+	ssize_t ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_SETXATTR(handle, fname, aname, value, size,
+					     flags);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_SETXATTR(handle, conv, aname, value, size, flags);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static int shadow_copy2_lsetxattr(struct vfs_handle_struct *handle,
+				  const char *fname,
+				  const char *aname, const void *value,
+				  size_t size, int flags)
+{
+	time_t timestamp;
+	char *stripped;
+	ssize_t ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_LSETXATTR(handle, fname, aname, value,
+					      size, flags);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_LSETXATTR(handle, conv, aname, value, size, flags);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
+
+static int shadow_copy2_chmod_acl(vfs_handle_struct *handle,
+				  const char *fname, mode_t mode)
+{
+	time_t timestamp;
+	char *stripped;
+	ssize_t ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, fname,
+					 &timestamp, &stripped)) {
+		return -1;
 	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_CHMOD_ACL(handle, fname, mode);
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_CHMOD_ACL(handle, conv, mode);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
+	return ret;
+}
 
+static int shadow_copy2_get_real_filename(struct vfs_handle_struct *handle,
+					  const char *path,
+					  const char *name,
+					  TALLOC_CTX *mem_ctx,
+					  char **found_name)
+{
+	time_t timestamp;
+	char *stripped;
+	ssize_t ret;
+	int saved_errno;
+	char *conv;
+
+	if (!shadow_copy2_strip_snapshot(talloc_tos(), handle, path,
+					 &timestamp, &stripped)) {
+		return -1;
+	}
+	if (timestamp == 0) {
+		return SMB_VFS_NEXT_GET_REAL_FILENAME(handle, path, name,
+						      mem_ctx, found_name);
+	}
+	if (stripped[0] == '\0') {
+		*found_name = talloc_strdup(mem_ctx, name);
+		if (*found_name == NULL) {
+			errno = ENOMEM;
+			return -1;
+		}
+		return 0;
+	}
+	conv = shadow_copy2_convert(talloc_tos(), handle, stripped, timestamp);
+	TALLOC_FREE(stripped);
+	if (conv == NULL) {
+		return -1;
+	}
+	ret = SMB_VFS_NEXT_GET_REAL_FILENAME(handle, conv, name,
+					     mem_ctx, found_name);
+	saved_errno = errno;
+	TALLOC_FREE(conv);
+	errno = saved_errno;
 	return ret;
 }
+
+
+static struct vfs_fn_pointers vfs_shadow_copy2_fns = {
+	.opendir = shadow_copy2_opendir,
+	.rename = shadow_copy2_rename,
+	.link = shadow_copy2_link,
+	.symlink = shadow_copy2_symlink,
+	.stat = shadow_copy2_stat,
+	.lstat = shadow_copy2_lstat,
+	.fstat = shadow_copy2_fstat,
+	.open_fn = shadow_copy2_open,
+	.unlink = shadow_copy2_unlink,
+	.chmod = shadow_copy2_chmod,
+	.chown = shadow_copy2_chown,
+	.chdir = shadow_copy2_chdir,
+	.ntimes = shadow_copy2_ntimes,
+	.vfs_readlink = shadow_copy2_readlink,
+	.mknod = shadow_copy2_mknod,
+	.realpath = shadow_copy2_realpath,
+	.get_nt_acl = shadow_copy2_get_nt_acl,
+	.fget_nt_acl = shadow_copy2_fget_nt_acl,
+	.get_shadow_copy_data = shadow_copy2_get_shadow_copy_data,
+	.mkdir = shadow_copy2_mkdir,
+	.rmdir = shadow_copy2_rmdir,
+	.getxattr = shadow_copy2_getxattr,
+	.lgetxattr = shadow_copy2_lgetxattr,
+	.listxattr = shadow_copy2_listxattr,
+	.removexattr = shadow_copy2_removexattr,
+	.lremovexattr = shadow_copy2_lremovexattr,
+	.setxattr = shadow_copy2_setxattr,
+	.lsetxattr = shadow_copy2_lsetxattr,
+	.chmod_acl = shadow_copy2_chmod_acl,
+	.chflags = shadow_copy2_chflags,
+	.get_real_filename = shadow_copy2_get_real_filename,
+};
+
+NTSTATUS vfs_shadow_copy2_init(void);
+NTSTATUS vfs_shadow_copy2_init(void)
+{
+	return smb_register_vfs(SMB_VFS_INTERFACE_VERSION,
+				"shadow_copy2", &vfs_shadow_copy2_fns);
+}
diff --git a/source3/modules/vfs_smb_traffic_analyzer.c b/source3/modules/vfs_smb_traffic_analyzer.c
index e8aa385d69c..ddce7e7dc5e 100644
--- a/source3/modules/vfs_smb_traffic_analyzer.c
+++ b/source3/modules/vfs_smb_traffic_analyzer.c
@@ -174,7 +174,7 @@ static char *smb_traffic_analyzer_encrypt( TALLOC_CTX *ctx,
 	char *output;
 	unsigned char crypted[18];
 	if (akey == NULL) return NULL;
-	samba_AES_set_encrypt_key((unsigned char *) akey, 128, &key);
+	samba_AES_set_encrypt_key((const unsigned char *) akey, 128, &key);
 	s1 = strlen(str) / 16;
 	s2 = strlen(str) % 16;
 	for (h = 0; h < s2; h++) *(filler+h)=*(str+(s1*16)+h);
@@ -183,10 +183,10 @@ static char *smb_traffic_analyzer_encrypt( TALLOC_CTX *ctx,
 	output = talloc_array(ctx, char, (s1*16)+17 );
 	d=0;
 	for (h = 0; h < s1; h++) {
-		samba_AES_encrypt((unsigned char *) str+(16*h), crypted, &key);
+		samba_AES_encrypt((const unsigned char *) str+(16*h), crypted, &key);
 		for (d = 0; d<16; d++) output[d+(16*h)]=crypted[d];
 	}
-	samba_AES_encrypt( (unsigned char *) str+(16*h), filler, &key );
+	samba_AES_encrypt( (const unsigned char *) str+(16*h), filler, &key );
 	for (d = 0;d < 16; d++) output[d+(16*h)]=*(filler+d);
 	*len = (s1*16)+16;
 	return output;	
@@ -417,6 +417,17 @@ static void smb_traffic_analyzer_send_data(vfs_handle_struct *handle,
 	 */
 	char state_flags[9] = "000000\0";
 
+	/**
+	 * The first byte of the state flag string represents
+	 * the modules protocol subversion number, defined
+	 * in smb_traffic_analyzer.h. smbtatools/smbtad are designed
+	 * to handle not yet implemented protocol enhancements
+	 * by ignoring them. By recognizing the SMBTA_SUBRELEASE
+	 * smbtatools can tell the user to update the client
+	 * software.
+	 */
+	state_flags[0] = SMBTA_SUBRELEASE;
+
 	SMB_VFS_HANDLE_GET_DATA(handle, rf_sock, struct refcounted_sock, return);
 
 	if (rf_sock == NULL || rf_sock->sock == -1) {
@@ -452,7 +463,7 @@ static void smb_traffic_analyzer_send_data(vfs_handle_struct *handle,
 					"protocol_version", NULL );
 
 
-	if ( protocol_version == NULL || strcmp( protocol_version,"V1") == 0) {
+	if (protocol_version != NULL && strcmp(protocol_version,"V1") == 0) {
 
 		struct rw_data *s_data = (struct rw_data *) data;
 
@@ -489,7 +500,10 @@ static void smb_traffic_analyzer_send_data(vfs_handle_struct *handle,
 		return;
 		}
 
-	} else if ( strcmp( protocol_version, "V2") == 0) {
+	} else {
+		/**
+		 * Protocol 2 is used by default.
+		 */
 
 		switch( vfs_operation ) {
 		case vfs_id_open: ;
@@ -557,10 +571,6 @@ static void smb_traffic_analyzer_send_data(vfs_handle_struct *handle,
 			return;
 		}
 
-	} else {
-		DEBUG(1, ("smb_traffic_analyzer_send_data_socket: "
-			"error, unknown protocol given!\n"));
-		return;
 	}
 
 	if (!str) {
@@ -656,7 +666,7 @@ static int smb_traffic_analyzer_connect(struct vfs_handle_struct *handle,
 		rf_sock->ref_count++;
 	} else {
 		/* New connection. */
-		rf_sock = TALLOC_ZERO_P(NULL, struct refcounted_sock);
+		rf_sock = talloc_zero(NULL, struct refcounted_sock);
 		if (rf_sock == NULL) {
 			SMB_VFS_NEXT_DISCONNECT(handle);
 			errno = ENOMEM;
diff --git a/source3/modules/vfs_smb_traffic_analyzer.h b/source3/modules/vfs_smb_traffic_analyzer.h
index bfc0614132d..817ffd83d12 100644
--- a/source3/modules/vfs_smb_traffic_analyzer.h
+++ b/source3/modules/vfs_smb_traffic_analyzer.h
@@ -77,7 +77,7 @@
  */
 
 /* Protocol subrelease number */
-#define SMBTA_SUBRELEASE 0
+#define SMBTA_SUBRELEASE '0'
 
 /*
  * Every data block sends a number of blocks sending common data
diff --git a/source3/modules/vfs_streams_depot.c b/source3/modules/vfs_streams_depot.c
index 01851cd2f83..3569405da30 100644
--- a/source3/modules/vfs_streams_depot.c
+++ b/source3/modules/vfs_streams_depot.c
@@ -324,7 +324,7 @@ static NTSTATUS stream_smb_fname(vfs_handle_struct *handle,
 	stype = strchr_m(smb_fname->stream_name + 1, ':');
 
 	if (stype) {
-		if (StrCaseCmp(stype, ":$DATA") != 0) {
+		if (strcasecmp_m(stype, ":$DATA") != 0) {
 			return NT_STATUS_INVALID_PARAMETER;
 		}
 	}
@@ -716,7 +716,7 @@ static bool add_one_stream(TALLOC_CTX *mem_ctx, unsigned int *num_streams,
 {
 	struct stream_struct *tmp;
 
-	tmp = TALLOC_REALLOC_ARRAY(mem_ctx, *streams, struct stream_struct,
+	tmp = talloc_realloc(mem_ctx, *streams, struct stream_struct,
 				   (*num_streams)+1);
 	if (tmp == NULL) {
 		return false;
diff --git a/source3/modules/vfs_streams_xattr.c b/source3/modules/vfs_streams_xattr.c
index 34e01b0b5f2..2772c96c4d8 100644
--- a/source3/modules/vfs_streams_xattr.c
+++ b/source3/modules/vfs_streams_xattr.c
@@ -52,9 +52,9 @@ static SMB_INO_T stream_inode(const SMB_STRUCT_STAT *sbuf, const char *sname)
 	SMB_ASSERT(upper_sname != NULL);
 
         MD5Init(&ctx);
-        MD5Update(&ctx, (unsigned char *)&(sbuf->st_ex_dev),
+        MD5Update(&ctx, (const unsigned char *)&(sbuf->st_ex_dev),
 		  sizeof(sbuf->st_ex_dev));
-        MD5Update(&ctx, (unsigned char *)&(sbuf->st_ex_ino),
+        MD5Update(&ctx, (const unsigned char *)&(sbuf->st_ex_ino),
 		  sizeof(sbuf->st_ex_ino));
         MD5Update(&ctx, (unsigned char *)upper_sname,
 		  talloc_get_size(upper_sname)-1);
@@ -614,7 +614,7 @@ static int streams_xattr_rename(vfs_handle_struct *handle,
 	}
 
 	/* Don't rename if the streams are identical. */
-	if (StrCaseCmp(smb_fname_src->stream_name,
+	if (strcasecmp_m(smb_fname_src->stream_name,
 		       smb_fname_dst->stream_name) == 0) {
 		goto done;
 	}
@@ -731,7 +731,7 @@ static bool add_one_stream(TALLOC_CTX *mem_ctx, unsigned int *num_streams,
 {
 	struct stream_struct *tmp;
 
-	tmp = TALLOC_REALLOC_ARRAY(mem_ctx, *streams, struct stream_struct,
+	tmp = talloc_realloc(mem_ctx, *streams, struct stream_struct,
 				   (*num_streams)+1);
 	if (tmp == NULL) {
 		return false;
@@ -880,7 +880,7 @@ static ssize_t streams_xattr_pwrite(vfs_handle_struct *handle,
         if ((offset + n) > ea.value.length-1) {
 		uint8 *tmp;
 
-		tmp = TALLOC_REALLOC_ARRAY(talloc_tos(), ea.value.data, uint8,
+		tmp = talloc_realloc(talloc_tos(), ea.value.data, uint8,
 					   offset + n + 1);
 
 		if (tmp == NULL) {
@@ -986,7 +986,7 @@ static int streams_xattr_ftruncate(struct vfs_handle_struct *handle,
 		return -1;
 	}
 
-	tmp = TALLOC_REALLOC_ARRAY(talloc_tos(), ea.value.data, uint8,
+	tmp = talloc_realloc(talloc_tos(), ea.value.data, uint8,
 				   offset + 1);
 
 	if (tmp == NULL) {
diff --git a/source3/modules/vfs_time_audit.c b/source3/modules/vfs_time_audit.c
index 37fe1c6e35b..25332e4e038 100644
--- a/source3/modules/vfs_time_audit.c
+++ b/source3/modules/vfs_time_audit.c
@@ -144,7 +144,7 @@ static int smb_time_audit_set_quota(struct vfs_handle_struct *handle,
 
 static int smb_time_audit_get_shadow_copy_data(struct vfs_handle_struct *handle,
 					       struct files_struct *fsp,
-					       SHADOW_COPY_DATA *shadow_copy_data,
+					       struct shadow_copy_data *shadow_copy_data,
 					       bool labels)
 {
 	int result;
@@ -564,7 +564,7 @@ static SMB_OFF_T smb_time_audit_lseek(vfs_handle_struct *handle,
 				      files_struct *fsp,
 				      SMB_OFF_T offset, int whence)
 {
-	ssize_t result;
+	SMB_OFF_T result;
 	struct timespec ts1,ts2;
 	double timediff;
 
@@ -721,7 +721,7 @@ static uint64_t smb_time_audit_get_alloc_size(vfs_handle_struct *handle,
 					      files_struct *fsp,
 					      const SMB_STRUCT_STAT *sbuf)
 {
-	int result;
+	uint64_t result;
 	struct timespec ts1,ts2;
 	double timediff;
 
@@ -869,14 +869,14 @@ static int smb_time_audit_chdir(vfs_handle_struct *handle, const char *path)
 	return result;
 }
 
-static char *smb_time_audit_getwd(vfs_handle_struct *handle, char *path)
+static char *smb_time_audit_getwd(vfs_handle_struct *handle)
 {
 	char *result;
 	struct timespec ts1,ts2;
 	double timediff;
 
 	clock_gettime_mono(&ts1);
-	result = SMB_VFS_NEXT_GETWD(handle, path);
+	result = SMB_VFS_NEXT_GETWD(handle);
 	clock_gettime_mono(&ts2);
 	timediff = nsec_time_diff(&ts2,&ts1)*1.0e-9;
 
@@ -2216,7 +2216,7 @@ static ssize_t smb_time_audit_aio_return(struct vfs_handle_struct *handle,
 					 struct files_struct *fsp,
 					 SMB_STRUCT_AIOCB *aiocb)
 {
-	int result;
+	ssize_t result;
 	struct timespec ts1,ts2;
 	double timediff;
 
diff --git a/source3/modules/vfs_tsmsm.c b/source3/modules/vfs_tsmsm.c
index d53d2918322..e9fd83bfea7 100644
--- a/source3/modules/vfs_tsmsm.c
+++ b/source3/modules/vfs_tsmsm.c
@@ -97,7 +97,7 @@ static int tsmsm_connect(struct vfs_handle_struct *handle,
 		return ret;
 	}
 
-	tsmd = TALLOC_ZERO_P(handle, struct tsmsm_struct);
+	tsmd = talloc_zero(handle, struct tsmsm_struct);
 	if (!tsmd) {
 		SMB_VFS_NEXT_DISCONNECT(handle);
 		DEBUG(0,("tsmsm_connect: out of memory!\n"));
diff --git a/source3/modules/vfs_xattr_tdb.c b/source3/modules/vfs_xattr_tdb.c
index 5b62565d62f..40ccf06dd95 100644
--- a/source3/modules/vfs_xattr_tdb.c
+++ b/source3/modules/vfs_xattr_tdb.c
@@ -24,6 +24,7 @@
 #include "librpc/gen_ndr/ndr_xattr.h"
 #include "../librpc/gen_ndr/ndr_netlogon.h"
 #include "dbwrap.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_VFS
@@ -40,7 +41,7 @@ static NTSTATUS xattr_tdb_pull_attrs(TALLOC_CTX *mem_ctx,
 	enum ndr_err_code ndr_err;
 	struct tdb_xattrs *result;
 
-	if (!(result = TALLOC_ZERO_P(mem_ctx, struct tdb_xattrs))) {
+	if (!(result = talloc_zero(mem_ctx, struct tdb_xattrs))) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
@@ -107,7 +108,7 @@ static NTSTATUS xattr_tdb_load_attrs(TALLOC_CTX *mem_ctx,
 
 	if (db_ctx->fetch(db_ctx, mem_ctx,
 			  make_tdb_data(id_buf, sizeof(id_buf)),
-			  &data) == -1) {
+			  &data) != 0) {
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
@@ -298,12 +299,12 @@ static int xattr_tdb_setattr(struct db_context *db_ctx,
 			return -1;
 		}
 
-		tmp = TALLOC_REALLOC_ARRAY(
+		tmp = talloc_realloc(
 			attribs, attribs->eas, struct xattr_EA,
 			attribs->num_eas+ 1);
 
 		if (tmp == NULL) {
-			DEBUG(0, ("TALLOC_REALLOC_ARRAY failed\n"));
+			DEBUG(0, ("talloc_realloc failed\n"));
 			TALLOC_FREE(rec);
 			errno = ENOMEM;
 			return -1;
@@ -314,7 +315,7 @@ static int xattr_tdb_setattr(struct db_context *db_ctx,
 	}
 
 	attribs->eas[i].name = name;
-	attribs->eas[i].value.data = CONST_DISCARD(uint8 *, value);
+	attribs->eas[i].value.data = discard_const_p(uint8, value);
 	attribs->eas[i].value.length = size;
 
 	status = xattr_tdb_save_attrs(rec, attribs);
diff --git a/source3/modules/wscript_build b/source3/modules/wscript_build
index 201610e8a29..8ccc4cac875 100644
--- a/source3/modules/wscript_build
+++ b/source3/modules/wscript_build
@@ -30,6 +30,7 @@ VFS_CACHEPRIME_SRC = 'vfs_cacheprime.c'
 VFS_PREALLOC_SRC = 'vfs_prealloc.c'
 VFS_COMMIT_SRC = 'vfs_commit.c'
 VFS_GPFS_SRC = 'vfs_gpfs.c gpfs.c'
+VFS_GPFS_HSM_NOTIFY_SRC = 'vfs_gpfs_hsm_notify.c'
 VFS_NOTIFY_FAM_SRC = 'vfs_notify_fam.c'
 VFS_READAHEAD_SRC = 'vfs_readahead.c'
 VFS_TSMSM_SRC = 'vfs_tsmsm.c'
@@ -52,7 +53,8 @@ VFS_TIME_AUDIT_SRC = 'vfs_time_audit.c'
 
 
 bld.SAMBA3_SUBSYSTEM('NFS4_ACLS',
-                    source='nfs4_acls.c')
+                    source='nfs4_acls.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('VFS_AIXACL_UTIL',
                     source='vfs_aixacl_util.c',
@@ -66,6 +68,7 @@ bld.SAMBA3_SUBSYSTEM('vfs',
 bld.SAMBA3_MODULE('vfs_default',
                  subsystem='vfs',
                  source=VFS_DEFAULT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_default'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_default'))
@@ -73,6 +76,7 @@ bld.SAMBA3_MODULE('vfs_default',
 bld.SAMBA3_MODULE('vfs_audit',
                  subsystem='vfs',
                  source=VFS_AUDIT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_audit'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_audit'))
@@ -80,6 +84,7 @@ bld.SAMBA3_MODULE('vfs_audit',
 bld.SAMBA3_MODULE('vfs_extd_audit',
                  subsystem='vfs',
                  source=VFS_EXTD_AUDIT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_extd_audit'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_extd_audit'))
@@ -87,6 +92,7 @@ bld.SAMBA3_MODULE('vfs_extd_audit',
 bld.SAMBA3_MODULE('vfs_full_audit',
                  subsystem='vfs',
                  source=VFS_FULL_AUDIT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_full_audit'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_full_audit'))
@@ -94,7 +100,7 @@ bld.SAMBA3_MODULE('vfs_full_audit',
 bld.SAMBA3_MODULE('vfs_fake_perms',
                  subsystem='vfs',
                  source=VFS_FAKE_PERMS_SRC,
-                 deps='acl attr',
+                 deps='acl attr samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_fake_perms'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_fake_perms'))
@@ -102,6 +108,7 @@ bld.SAMBA3_MODULE('vfs_fake_perms',
 bld.SAMBA3_MODULE('vfs_recycle',
                  subsystem='vfs',
                  source=VFS_RECYCLE_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_recycle'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_recycle'))
@@ -109,6 +116,7 @@ bld.SAMBA3_MODULE('vfs_recycle',
 bld.SAMBA3_MODULE('vfs_netatalk',
                  subsystem='vfs',
                  source=VFS_NETATALK_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_netatalk'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_netatalk'))
@@ -116,6 +124,7 @@ bld.SAMBA3_MODULE('vfs_netatalk',
 bld.SAMBA3_MODULE('vfs_default_quota',
                  subsystem='vfs',
                  source=VFS_DEFAULT_QUOTA_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_default_quota'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_default_quota'))
@@ -123,6 +132,7 @@ bld.SAMBA3_MODULE('vfs_default_quota',
 bld.SAMBA3_MODULE('vfs_readonly',
                  subsystem='vfs',
                  source=VFS_READONLY_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_readonly'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_readonly'))
@@ -130,6 +140,7 @@ bld.SAMBA3_MODULE('vfs_readonly',
 bld.SAMBA3_MODULE('vfs_cap',
                  subsystem='vfs',
                  source=VFS_CAP_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_cap'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_cap'))
@@ -137,6 +148,7 @@ bld.SAMBA3_MODULE('vfs_cap',
 bld.SAMBA3_MODULE('vfs_expand_msdfs',
                  subsystem='vfs',
                  source=VFS_EXPAND_MSDFS_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_expand_msdfs'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_expand_msdfs'))
@@ -144,6 +156,7 @@ bld.SAMBA3_MODULE('vfs_expand_msdfs',
 bld.SAMBA3_MODULE('vfs_shadow_copy',
                  subsystem='vfs',
                  source=VFS_SHADOW_COPY_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_shadow_copy'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_shadow_copy'))
@@ -151,6 +164,7 @@ bld.SAMBA3_MODULE('vfs_shadow_copy',
 bld.SAMBA3_MODULE('vfs_shadow_copy2',
                  subsystem='vfs',
                  source=VFS_SHADOW_COPY2_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_shadow_copy2'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_shadow_copy2'))
@@ -158,6 +172,7 @@ bld.SAMBA3_MODULE('vfs_shadow_copy2',
 bld.SAMBA3_MODULE('vfs_afsacl',
                  subsystem='vfs',
                  source=VFS_AFSACL_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_afsacl'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_afsacl'))
@@ -233,6 +248,7 @@ bld.SAMBA3_MODULE('vfs_tru64acl',
 bld.SAMBA3_MODULE('vfs_catia',
                  subsystem='vfs',
                  source=VFS_CATIA_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_catia'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_catia'))
@@ -240,6 +256,7 @@ bld.SAMBA3_MODULE('vfs_catia',
 bld.SAMBA3_MODULE('vfs_streams_xattr',
                  subsystem='vfs',
                  source=VFS_STREAMS_XATTR_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_streams_xattr'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_streams_xattr'))
@@ -247,6 +264,7 @@ bld.SAMBA3_MODULE('vfs_streams_xattr',
 bld.SAMBA3_MODULE('vfs_streams_depot',
                  subsystem='vfs',
                  source=VFS_STREAMS_DEPOT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_streams_depot'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_streams_depot'))
@@ -254,6 +272,7 @@ bld.SAMBA3_MODULE('vfs_streams_depot',
 bld.SAMBA3_MODULE('vfs_cacheprime',
                  subsystem='vfs',
                  source=VFS_CACHEPRIME_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_cacheprime'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_cacheprime'))
@@ -261,6 +280,7 @@ bld.SAMBA3_MODULE('vfs_cacheprime',
 bld.SAMBA3_MODULE('vfs_prealloc',
                  subsystem='vfs',
                  source=VFS_PREALLOC_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_prealloc'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_prealloc'))
@@ -268,6 +288,7 @@ bld.SAMBA3_MODULE('vfs_prealloc',
 bld.SAMBA3_MODULE('vfs_commit',
                  subsystem='vfs',
                  source=VFS_COMMIT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_commit'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_commit'))
@@ -280,6 +301,13 @@ bld.SAMBA3_MODULE('vfs_gpfs',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_gpfs'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_gpfs'))
 
+bld.SAMBA3_MODULE('vfs_gpfs_hsm_notify',
+                 subsystem='vfs',
+                 source=VFS_GPFS_HSM_NOTIFY_SRC,
+                 init_function='',
+                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_gpfs_hsm_notify'),
+                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_gpfs_hsm_notify'))
+
 bld.SAMBA3_MODULE('vfs_notify_fam',
                  subsystem='vfs',
                  source=VFS_NOTIFY_FAM_SRC,
@@ -290,6 +318,7 @@ bld.SAMBA3_MODULE('vfs_notify_fam',
 bld.SAMBA3_MODULE('vfs_readahead',
                  subsystem='vfs',
                  source=VFS_READAHEAD_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_readahead'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_readahead'))
@@ -297,6 +326,7 @@ bld.SAMBA3_MODULE('vfs_readahead',
 bld.SAMBA3_MODULE('vfs_tsmsm',
                  subsystem='vfs',
                  source=VFS_TSMSM_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_tsmsm'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_tsmsm'))
@@ -304,6 +334,7 @@ bld.SAMBA3_MODULE('vfs_tsmsm',
 bld.SAMBA3_MODULE('vfs_fileid',
                  subsystem='vfs',
                  source=VFS_FILEID_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_fileid'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_fileid'))
@@ -311,6 +342,7 @@ bld.SAMBA3_MODULE('vfs_fileid',
 bld.SAMBA3_MODULE('vfs_aio_fork',
                  subsystem='vfs',
                  source=VFS_AIO_FORK_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_aio_fork'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_aio_fork'))
@@ -318,6 +350,7 @@ bld.SAMBA3_MODULE('vfs_aio_fork',
 bld.SAMBA3_MODULE('vfs_preopen',
                  subsystem='vfs',
                  source=VFS_PREOPEN_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_preopen'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_preopen'))
@@ -325,6 +358,7 @@ bld.SAMBA3_MODULE('vfs_preopen',
 bld.SAMBA3_MODULE('vfs_syncops',
                  subsystem='vfs',
                  source=VFS_SYNCOPS_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_syncops'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_syncops'))
@@ -332,6 +366,7 @@ bld.SAMBA3_MODULE('vfs_syncops',
 bld.SAMBA3_MODULE('vfs_acl_xattr',
                  subsystem='vfs',
                  source=VFS_ACL_XATTR_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_acl_xattr'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_acl_xattr'))
@@ -347,6 +382,7 @@ bld.SAMBA3_MODULE('vfs_acl_tdb',
 bld.SAMBA3_MODULE('vfs_smb_traffic_analyzer',
                  subsystem='vfs',
                  source=VFS_SMB_TRAFFIC_ANALYZER_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_smb_traffic_analyzer'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_smb_traffic_analyzer'))
@@ -354,6 +390,7 @@ bld.SAMBA3_MODULE('vfs_smb_traffic_analyzer',
 bld.SAMBA3_MODULE('vfs_onefs',
                  subsystem='vfs',
                  source=VFS_ONEFS_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_onefs'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_onefs'))
@@ -361,6 +398,7 @@ bld.SAMBA3_MODULE('vfs_onefs',
 bld.SAMBA3_MODULE('vfs_onefs_shadow_copy',
                  subsystem='vfs',
                  source=VFS_ONEFS_SHADOW_COPY_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_onefs_shadow_copy'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_onefs_shadow_copy'))
@@ -368,6 +406,7 @@ bld.SAMBA3_MODULE('vfs_onefs_shadow_copy',
 bld.SAMBA3_MODULE('vfs_dirsort',
                  subsystem='vfs',
                  source=VFS_DIRSORT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_dirsort'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_dirsort'))
@@ -375,6 +414,7 @@ bld.SAMBA3_MODULE('vfs_dirsort',
 bld.SAMBA3_MODULE('vfs_scannedonly',
                  subsystem='vfs',
                  source=VFS_SCANNEDONLY_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_scannedonly'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_scannedonly'))
@@ -382,6 +422,7 @@ bld.SAMBA3_MODULE('vfs_scannedonly',
 bld.SAMBA3_MODULE('vfs_crossrename',
                  subsystem='vfs',
                  source=VFS_CROSSRENAME_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_crossrename'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_crossrename'))
@@ -389,6 +430,7 @@ bld.SAMBA3_MODULE('vfs_crossrename',
 bld.SAMBA3_MODULE('vfs_linux_xfs_sgid',
                  subsystem='vfs',
                  source=VFS_LINUX_XFS_SGID_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_linux_xfs_sgid'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_linux_xfs_sgid'))
@@ -396,51 +438,13 @@ bld.SAMBA3_MODULE('vfs_linux_xfs_sgid',
 bld.SAMBA3_MODULE('vfs_time_audit',
                  subsystem='vfs',
                  source=VFS_TIME_AUDIT_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_time_audit'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_time_audit'))
 
 
 
-CHARSET_WEIRD_SRC = 'weird.c'
-CHARSET_CP850_SRC = 'CP850.c'
-CHARSET_CP437_SRC = 'CP437.c'
-CHARSET_MACOSXFS_SRC = 'charset_macosxfs.c'
-
-
-bld.SAMBA3_SUBSYSTEM('charset',
-                    source='',
-                    vars=locals())
-
-bld.SAMBA3_MODULE('charset_weird',
-                 subsystem='charset',
-                 source=CHARSET_WEIRD_SRC,
-                 init_function='',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_weird'),
-                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_weird'))
-
-bld.SAMBA3_MODULE('charset_CP850',
-                 subsystem='charset',
-                 source=CHARSET_CP850_SRC,
-                 init_function='',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_CP850'),
-                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_CP850'))
-
-bld.SAMBA3_MODULE('charset_CP437',
-                 subsystem='charset',
-                 source=CHARSET_CP437_SRC,
-                 init_function='',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_CP437'),
-                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_CP437'))
-
-bld.SAMBA3_MODULE('charset_macosxfs',
-                 subsystem='charset',
-                 source=CHARSET_MACOSXFS_SRC,
-                 init_function='',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_macosxfs'),
-                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_macosxfs'))
-
-
 PERFCOUNT_ONEFS_SRC = 'perfcount_onefs.c'
 PERFCOUNT_TEST_SRC = 'perfcount_test.c'
 
diff --git a/source3/nmbd/nmbd.c b/source3/nmbd/nmbd.c
index f8108d9d52b..d8a4b17a071 100644
--- a/source3/nmbd/nmbd.c
+++ b/source3/nmbd/nmbd.c
@@ -240,8 +240,8 @@ static void reload_interfaces(time_t t)
 			continue;
 		}
 
-		ip = ((struct sockaddr_in *)(void *)&iface->ip)->sin_addr;
-		nmask = ((struct sockaddr_in *)(void *)
+		ip = ((const struct sockaddr_in *)(const void *)&iface->ip)->sin_addr;
+		nmask = ((const struct sockaddr_in *)(const void *)
 			 &iface->netmask)->sin_addr;
 
 		/*
@@ -250,7 +250,7 @@ static void reload_interfaces(time_t t)
 		 * ignore it here. JRA.
 		 */
 
-		if (is_loopback_addr((struct sockaddr *)(void *)&iface->ip)) {
+		if (is_loopback_addr((const struct sockaddr *)(const void *)&iface->ip)) {
 			DEBUG(2,("reload_interfaces: Ignoring loopback "
 				"interface %s\n",
 				print_sockaddr(str, sizeof(str), &iface->ip) ));
diff --git a/source3/nmbd/nmbd.h b/source3/nmbd/nmbd.h
index e100e7d6932..47940e19401 100644
--- a/source3/nmbd/nmbd.h
+++ b/source3/nmbd/nmbd.h
@@ -1,6 +1,31 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  NBT netbios routines and daemon - version 2
+ *
+ *  Copyright (C) Guenther Deschner 2011
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _NMBD_NMBD_H_
+#define _NMBD_NMBD_H_
+
 #ifndef HAVE_PIPE
 #define SYNC_DNS 1
 #endif
 
 #include "libsmb/nmblib.h"
 #include "nmbd/nmbd_proto.h"
+
+#endif /* _NMBD_NMBD_H_ */
diff --git a/source3/nmbd/nmbd_become_dmb.c b/source3/nmbd/nmbd_become_dmb.c
index 160bcb3c339..276c381f6c8 100644
--- a/source3/nmbd/nmbd_become_dmb.c
+++ b/source3/nmbd/nmbd_become_dmb.c
@@ -51,10 +51,10 @@ workgroup %s on subnet %s\n", failname, subrec->subnet_name));
 	/* Set the state back to DOMAIN_NONE. */
 	work->dom_state = DOMAIN_NONE;
 
-	if((servrec = find_server_in_workgroup( work, global_myname())) == NULL) {
+	if((servrec = find_server_in_workgroup( work, lp_netbios_name())) == NULL) {
 		DEBUG(0,("become_domain_master_fail: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), work->work_group, subrec->subnet_name));
+			lp_netbios_name(), work->work_group, subrec->subnet_name));
 		return;
 	}
 
@@ -92,10 +92,10 @@ workgroup %s on subnet %s\n", regname, subrec->subnet_name));
 		return;
 	}
 
-	if((servrec = find_server_in_workgroup( work, global_myname())) == NULL) {
+	if((servrec = find_server_in_workgroup( work, lp_netbios_name())) == NULL) {
 		DEBUG(0,("become_domain_master_stage2: Error - cannot find server %s \
 in workgroup %s on subnet %s\n", 
-		global_myname(), regname, subrec->subnet_name));
+		lp_netbios_name(), regname, subrec->subnet_name));
 		work->dom_state = DOMAIN_NONE;
 		return;
 	}
@@ -110,7 +110,7 @@ in workgroup %s on subnet %s\n",
 	subrec->work_changed = True;
 
 	if( DEBUGLVL( 0 ) ) {
-		dbgtext( "*****\n\nSamba server %s ", global_myname() );
+		dbgtext( "*****\n\nSamba server %s ", lp_netbios_name() );
 		dbgtext( "is now a domain master browser for " );
 		dbgtext( "workgroup %s ", work->work_group );
 		dbgtext( "on subnet %s\n\n*****\n", subrec->subnet_name );
@@ -126,7 +126,7 @@ in workgroup %s on subnet %s\n",
 		   will stop us syncing with ourself if we are also
 		   a local master browser. */
 
-		make_nmb_name(&nmbname, global_myname(), 0x20);
+		make_nmb_name(&nmbname, lp_netbios_name(), 0x20);
 
 		work->dmb_name = nmbname;
 
diff --git a/source3/nmbd/nmbd_become_lmb.c b/source3/nmbd/nmbd_become_lmb.c
index ffd92cf0d8f..f94efd03711 100644
--- a/source3/nmbd/nmbd_become_lmb.c
+++ b/source3/nmbd/nmbd_become_lmb.c
@@ -85,10 +85,10 @@ subnet %s.\n", workgroup_name, subrec->subnet_name ));
 		return;
 	}
 
-	if((servrec = find_server_in_workgroup( work, global_myname())) == NULL) {
+	if((servrec = find_server_in_workgroup( work, lp_netbios_name())) == NULL) {
 		DEBUG(0,("reset_workgroup_state: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), work->work_group, subrec->subnet_name));
+			lp_netbios_name(), work->work_group, subrec->subnet_name));
 		work->mst_state = lp_local_master() ? MST_POTENTIAL : MST_NONE;
 		return;
 	}
@@ -149,7 +149,7 @@ static void unbecome_local_master_success(struct subnet_record *subrec,
 
 	if( DEBUGLVL( 0 ) ) {
 		dbgtext( "*****\n\n" );
-		dbgtext( "Samba name server %s ", global_myname() );
+		dbgtext( "Samba name server %s ", lp_netbios_name() );
 		dbgtext( "has stopped being a local master browser " );
 		dbgtext( "for workgroup %s ", relname );
 		dbgtext( "on subnet %s\n\n*****\n", subrec->subnet_name );
@@ -185,7 +185,7 @@ Removing from namelist anyway.\n", nmb_namestr(fail_name)));
 
 	if( DEBUGLVL( 0 ) ) {
 		dbgtext( "*****\n\n" );
-		dbgtext( "Samba name server %s ", global_myname() );
+		dbgtext( "Samba name server %s ", lp_netbios_name() );
 		dbgtext( "has stopped being a local master browser " );
 		dbgtext( "for workgroup %s ", failname );
 		dbgtext( "on subnet %s\n\n*****\n", subrec->subnet_name );
@@ -280,10 +280,10 @@ void unbecome_local_master_browser(struct subnet_record *subrec, struct work_rec
 	DEBUG(2,("unbecome_local_master_browser: unbecoming local master for workgroup %s \
 on subnet %s\n",work->work_group, subrec->subnet_name));
   
-	if(find_server_in_workgroup( work, global_myname()) == NULL) {
+	if(find_server_in_workgroup( work, lp_netbios_name()) == NULL) {
 		DEBUG(0,("unbecome_local_master_browser: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), work->work_group, subrec->subnet_name));
+			lp_netbios_name(), work->work_group, subrec->subnet_name));
 			work->mst_state = lp_local_master() ? MST_POTENTIAL : MST_NONE;
 		return;
 	}
@@ -342,10 +342,10 @@ workgroup %s on subnet %s\n", regname, subrec->subnet_name));
 		return;
 	}
 
-	if((servrec = find_server_in_workgroup( work, global_myname())) == NULL) {
+	if((servrec = find_server_in_workgroup( work, lp_netbios_name())) == NULL) {
 		DEBUG(0,("become_local_master_stage2: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), regname, subrec->subnet_name));
+			lp_netbios_name(), regname, subrec->subnet_name));
 			work->mst_state = lp_local_master() ? MST_POTENTIAL : MST_NONE;
 		return;
 	}
@@ -363,7 +363,7 @@ on subnet %s\n", work->work_group, subrec->subnet_name));
 	subrec->work_changed = True;
 
 	/* Add this name to the workgroup as local master browser. */
-	set_workgroup_local_master_browser_name( work, global_myname());
+	set_workgroup_local_master_browser_name( work, lp_netbios_name());
 
 	/* Count the number of servers we have on our list. If it's
 		less than 10 (just a heuristic) request the servers
@@ -396,7 +396,7 @@ on subnet %s\n", work->work_group, subrec->subnet_name));
 
 	if( DEBUGLVL( 0 ) ) {
 		dbgtext( "*****\n\n" );
-		dbgtext( "Samba name server %s ", global_myname() );
+		dbgtext( "Samba name server %s ", lp_netbios_name() );
 		dbgtext( "is now a local master browser " );
 		dbgtext( "for workgroup %s ", work->work_group );
 		dbgtext( "on subnet %s\n\n*****\n", subrec->subnet_name );
@@ -487,10 +487,10 @@ workgroup %s on subnet %s\n", work_name, subrec->subnet_name));
 		return;
 	}
 
-	if(find_server_in_workgroup(work, global_myname()) == NULL) {
+	if(find_server_in_workgroup(work, lp_netbios_name()) == NULL) {
 		DEBUG(0,("become_local_master_fail1: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), work->work_group, subrec->subnet_name));
+			lp_netbios_name(), work->work_group, subrec->subnet_name));
 		return;
 	}
 
@@ -527,10 +527,10 @@ void become_local_master_browser(struct subnet_record *subrec, struct work_recor
 		return;
 	}
 
-	if(find_server_in_workgroup( work, global_myname()) == NULL) {
+	if(find_server_in_workgroup( work, lp_netbios_name()) == NULL) {
 		DEBUG(0,("become_local_master_browser: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), work->work_group, subrec->subnet_name));
+			lp_netbios_name(), work->work_group, subrec->subnet_name));
 		return;
 	}
 
@@ -554,7 +554,7 @@ in workgroup %s on subnet %s\n",
 	userdata->copy_fn = NULL;
 	userdata->free_fn = NULL;
 	userdata->userdata_len = strlen(work->work_group)+1;
-	overmalloc_safe_strcpy(userdata->data, work->work_group, size - sizeof(*userdata) - 1);
+	strlcpy(userdata->data, work->work_group, size - sizeof(*userdata));
 
 	/* Register the special browser group name. */
 	register_name(subrec, MSBROWSE, 0x01, samba_nb_type|NB_GROUP,
diff --git a/source3/nmbd/nmbd_browsesync.c b/source3/nmbd/nmbd_browsesync.c
index 3dc89b50261..c6ded584e54 100644
--- a/source3/nmbd/nmbd_browsesync.c
+++ b/source3/nmbd/nmbd_browsesync.c
@@ -119,7 +119,7 @@ static void announce_local_master_browser_to_domain_master_browser( struct work_
 	SCVAL(p,0,ANN_MasterAnnouncement);
 	p++;
 
-	unstrcpy(myname, global_myname());
+	unstrcpy(myname, lp_netbios_name());
 	strupper_m(myname);
 	myname[15]='\0';
 	/* The call below does CH_UNIX -> CH_DOS conversion. JRA */
@@ -137,7 +137,7 @@ static void announce_local_master_browser_to_domain_master_browser( struct work_
 	/* Target name for send_mailslot must be in UNIX charset. */
 	pull_ascii_nstring(dmb_name, sizeof(dmb_name), work->dmb_name.name);
 	send_mailslot(True, BROWSE_MAILSLOT, outbuf,PTR_DIFF(p,outbuf),
-		global_myname(), 0x0, dmb_name, 0x0,
+		lp_netbios_name(), 0x0, dmb_name, 0x0,
 		work->dmb_addr, FIRST_SUBNET->myip, DGRAM_PORT);
 }
 
@@ -330,7 +330,7 @@ static void find_domain_master_name_query_success(struct subnet_record *subrec,
 	userdata->copy_fn = NULL;
 	userdata->free_fn = NULL;
 	userdata->userdata_len = strlen(work->work_group)+1;
-	overmalloc_safe_strcpy(userdata->data, work->work_group, size - sizeof(*userdata) - 1);
+	strlcpy(userdata->data, work->work_group, size - sizeof(*userdata));
 
 	node_status( subrec, &nmbname, answer_ip, 
 		domain_master_node_status_success,
@@ -435,7 +435,7 @@ static void get_domain_master_name_node_status_success(struct subnet_record *sub
 			if(!(nb_flags & NB_GROUP) && (name_type == 0x00) && 
 					server_name[0] == 0) {
 				/* this is almost certainly the server netbios name */
-				unstrcpy(server_name, qname);
+				strlcpy(server_name, qname, sizeof(server_name));
 				continue;
 			}
 
@@ -461,7 +461,9 @@ static void get_domain_master_name_node_status_success(struct subnet_record *sub
 						return;
 
 					/* remember who the master is */
-					unstrcpy(work->local_master_browser_name, server_name);
+					strlcpy(work->local_master_browser_name,
+						server_name,
+						sizeof(work->local_master_browser_name));
 					make_nmb_name(&nmbname, server_name, 0x20);
 					work->dmb_name = nmbname;
 					work->dmb_addr = from_ip;
diff --git a/source3/nmbd/nmbd_elections.c b/source3/nmbd/nmbd_elections.c
index 899c273fe08..ac0d8735120 100644
--- a/source3/nmbd/nmbd_elections.c
+++ b/source3/nmbd/nmbd_elections.c
@@ -57,7 +57,7 @@ static void send_election_dgram(struct subnet_record *subrec, const char *workgr
 	p = skip_string(outbuf,sizeof(outbuf),p);
 
 	send_mailslot(False, BROWSE_MAILSLOT, outbuf, PTR_DIFF(p,outbuf),
-		global_myname(), 0,
+		lp_netbios_name(), 0,
 		workgroup_name, 0x1e,
 		subrec->bcast_ip, subrec->myip, DGRAM_PORT);
 }
@@ -194,7 +194,7 @@ yet registered on subnet %s\n", nmb_namestr(&nmbname), subrec->subnet_name ));
 				}
 
 				send_election_dgram(subrec, work->work_group, work->ElectionCriterion,
-						t - StartupTime, global_myname());
+						t - StartupTime, lp_netbios_name());
 	      
 				if (work->ElectionCount++ >= 4) {
 					/* Won election (4 packets were sent out uncontested. */
@@ -231,7 +231,7 @@ static bool win_election(struct work_record *work, int version,
 			version, ELECTION_VERSION,
 			criterion, mycriterion,
 			timeup, mytimeup,
-			server_name, global_myname()));
+			server_name, lp_netbios_name()));
 
 	if (version > ELECTION_VERSION)
 		return(False);
@@ -248,7 +248,7 @@ static bool win_election(struct work_record *work, int version,
 	if (timeup < mytimeup)
 		return(True);
 
-	if (StrCaseCmp(global_myname(), server_name) > 0)
+	if (strcasecmp_m(lp_netbios_name(), server_name) > 0)
 		return(False);
   
 	return(True);
@@ -258,7 +258,7 @@ static bool win_election(struct work_record *work, int version,
   Process an incoming election datagram packet.
 ******************************************************************/
 
-void process_election(struct subnet_record *subrec, struct packet_struct *p, char *buf)
+void process_election(struct subnet_record *subrec, struct packet_struct *p, const char *buf)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	int version = CVAL(buf,0);
diff --git a/source3/nmbd/nmbd_incomingdgrams.c b/source3/nmbd/nmbd_incomingdgrams.c
index 6993e183583..582848c1fcf 100644
--- a/source3/nmbd/nmbd_incomingdgrams.c
+++ b/source3/nmbd/nmbd_incomingdgrams.c
@@ -95,7 +95,7 @@ void tell_become_backup(void)
   Process an incoming host announcement packet.
 *******************************************************************/
 
-void process_host_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf)
+void process_host_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	int ttl = IVAL(buf,1)/1000;
@@ -144,7 +144,7 @@ void process_host_announce(struct subnet_record *subrec, struct packet_struct *p
 	 * to be our primary workgroup name.
 	 */
 
-	if(strequal(work_name, global_myname()))
+	if(strequal(work_name, lp_netbios_name()))
 		unstrcpy(work_name,lp_workgroup());
 
 	/*
@@ -172,7 +172,7 @@ void process_host_announce(struct subnet_record *subrec, struct packet_struct *p
 			/* Update the record. */
 			servrec->serv.type = servertype|SV_TYPE_LOCAL_LIST_ONLY;
 			update_server_ttl( servrec, ttl);
-			fstrcpy(servrec->serv.comment,comment);
+			strlcpy(servrec->serv.comment,comment,sizeof(servrec->serv.comment));
 		}
 	} else {
 		/*
@@ -195,7 +195,7 @@ done:
   Process an incoming WORKGROUP announcement packet.
 *******************************************************************/
 
-void process_workgroup_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf)
+void process_workgroup_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	int ttl = IVAL(buf,1)/1000;
@@ -252,7 +252,7 @@ done:
   Process an incoming local master browser announcement packet.
 *******************************************************************/
 
-void process_local_master_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf)
+void process_local_master_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	int ttl = IVAL(buf,1)/1000;
@@ -338,7 +338,7 @@ a local master browser for workgroup %s and we think we are master. Forcing elec
 			/* Update the record. */
 			servrec->serv.type = servertype|SV_TYPE_LOCAL_LIST_ONLY;
 			update_server_ttl(servrec, ttl);
-			fstrcpy(servrec->serv.comment,comment);
+			strlcpy(servrec->serv.comment,comment,sizeof(servrec->serv.comment));
 		}
 	
 		set_workgroup_local_master_browser_name( work, server_name );
@@ -367,7 +367,7 @@ done:
 ******************************************************************/
 
 void process_master_browser_announce(struct subnet_record *subrec, 
-                                     struct packet_struct *p,char *buf)
+                                     struct packet_struct *p,const char *buf)
 {
 	unstring local_master_name;
 	struct work_record *work;
@@ -418,7 +418,7 @@ done:
   Process an incoming LanMan host announcement packet.
 *******************************************************************/
 
-void process_lm_host_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf, int len)
+void process_lm_host_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf, int len)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	uint32 servertype = IVAL(buf,1);
@@ -431,7 +431,7 @@ void process_lm_host_announce(struct subnet_record *subrec, struct packet_struct
 	unstring work_name;
 	unstring source_name;
 	fstring comment;
-	char *s = get_safe_str_ptr(buf,len,buf,9);
+	char *s = get_safe_str_ptr(buf,len,discard_const_p(char, buf),9);
 
 	START_PROFILE(lm_host_announce);
 	if (!s) {
@@ -484,7 +484,7 @@ originate from OS/2 Warp client. Ignoring packet.\n"));
 	 * not needed in the LanMan announce code, but it won't hurt.
 	 */
 
-	if(strequal(work_name, global_myname()))
+	if(strequal(work_name, lp_netbios_name()))
 		unstrcpy(work_name,lp_workgroup());
 
 	/*
@@ -512,7 +512,7 @@ originate from OS/2 Warp client. Ignoring packet.\n"));
 			/* Update the record. */
 			servrec->serv.type = servertype|SV_TYPE_LOCAL_LIST_ONLY;
 			update_server_ttl( servrec, ttl);
-			fstrcpy(servrec->serv.comment,comment);
+			strlcpy(servrec->serv.comment,comment,sizeof(servrec->serv.comment));
 		}
 	} else {
 		/*
@@ -571,7 +571,7 @@ static void send_backup_list_response(struct subnet_record *subrec,
 
 	/* We always return at least one name - our own. */
 	count = 1;
-	unstrcpy(myname, global_myname());
+	unstrcpy(myname, lp_netbios_name());
 	strupper_m(myname);
 	myname[15]='\0';
 	push_ascii(p, myname, sizeof(outbuf)-PTR_DIFF(p,outbuf)-1, STR_TERMINATE);
@@ -599,7 +599,7 @@ static void send_backup_list_response(struct subnet_record *subrec,
     if(count >= (unsigned int)max_number_requested)
       break;
 
-    if(strnequal(servrec->serv.name, global_myname(),15))
+    if(strnequal(servrec->serv.name, lp_netbios_name(),15))
       continue;
 
     if(!(servrec->serv.type & SV_TYPE_BACKUP_BROWSER))
@@ -625,7 +625,7 @@ static void send_backup_list_response(struct subnet_record *subrec,
 
 	send_mailslot(True, BROWSE_MAILSLOT,
 		outbuf,PTR_DIFF(p,outbuf),
-		global_myname(), 0, 
+		lp_netbios_name(), 0,
 		send_to_namestr,0,
 		sendto_ip, subrec->myip, port);
 }
@@ -641,7 +641,7 @@ static void send_backup_list_response(struct subnet_record *subrec,
 ********************************************************************/
 
 void process_get_backup_list_request(struct subnet_record *subrec,
-                                     struct packet_struct *p,char *buf)
+                                     struct packet_struct *p,const char *buf)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	struct work_record *work;
@@ -724,7 +724,7 @@ done:
 ******************************************************************/
 
 void process_reset_browser(struct subnet_record *subrec,
-                                  struct packet_struct *p,char *buf)
+                                  struct packet_struct *p,const char *buf)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	int state = CVAL(buf,0);
@@ -776,7 +776,7 @@ request from %s IP %s state=0x%X\n",
   announcement is needed soon.
 ******************************************************************/
 
-void process_announce_request(struct subnet_record *subrec, struct packet_struct *p, char *buf)
+void process_announce_request(struct subnet_record *subrec, struct packet_struct *p, const char *buf)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	struct work_record *work;
@@ -817,7 +817,7 @@ done:
   through the "lm announce" parameter in smb.conf)
 ******************************************************************/
 
-void process_lm_announce_request(struct subnet_record *subrec, struct packet_struct *p, char *buf, int len)
+void process_lm_announce_request(struct subnet_record *subrec, struct packet_struct *p, const char *buf, int len)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	unstring workgroup_name;
diff --git a/source3/nmbd/nmbd_incomingrequests.c b/source3/nmbd/nmbd_incomingrequests.c
index d0138ddd4a9..b2545f102fc 100644
--- a/source3/nmbd/nmbd_incomingrequests.c
+++ b/source3/nmbd/nmbd_incomingrequests.c
@@ -290,14 +290,14 @@ static int status_compare(char *n1,char *n2)
 		;
 	for (l2=0;l2<15 && n2[l2] && n2[l2] != ' ';l2++)
 		;
-	l3 = strlen(global_myname());
+	l3 = strlen(lp_netbios_name());
 
-	if ((l1==l3) && strncmp(n1,global_myname(),l3) == 0 && 
-			(l2!=l3 || strncmp(n2,global_myname(),l3) != 0))
+	if ((l1==l3) && strncmp(n1,lp_netbios_name(),l3) == 0 &&
+			(l2!=l3 || strncmp(n2,lp_netbios_name(),l3) != 0))
 		return -1;
 
-	if ((l2==l3) && strncmp(n2,global_myname(),l3) == 0 && 
-			(l1!=l3 || strncmp(n1,global_myname(),l3) != 0))
+	if ((l2==l3) && strncmp(n2,lp_netbios_name(),l3) == 0 &&
+			(l1!=l3 || strncmp(n1,lp_netbios_name(),l3) != 0))
 		return 1;
 
 	return memcmp(n1,n2,sizeof(name1));
diff --git a/source3/nmbd/nmbd_logonnames.c b/source3/nmbd/nmbd_logonnames.c
index f99092055bb..9a05095bfd0 100644
--- a/source3/nmbd/nmbd_logonnames.c
+++ b/source3/nmbd/nmbd_logonnames.c
@@ -46,10 +46,10 @@ workgroup %s on subnet %s\n", failname, subrec->subnet_name));
 		return;
 	}
 
-	if((servrec = find_server_in_workgroup( work, global_myname())) == NULL) {
+	if((servrec = find_server_in_workgroup( work, lp_netbios_name())) == NULL) {
 		DEBUG(0,("become_logon_server_fail: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), failname, subrec->subnet_name));
+			lp_netbios_name(), failname, subrec->subnet_name));
 		work->log_state = LOGON_NONE;
 		return;
 	}
@@ -87,10 +87,10 @@ workgroup %s on subnet %s\n", reg_name, subrec->subnet_name));
 		return;
 	}
 
-	if((servrec = find_server_in_workgroup( work, global_myname())) == NULL) {
+	if((servrec = find_server_in_workgroup( work, lp_netbios_name())) == NULL) {
 		DEBUG(0,("become_logon_server_success: Error - cannot find server %s \
 in workgroup %s on subnet %s\n",
-			global_myname(), reg_name, subrec->subnet_name));
+			lp_netbios_name(), reg_name, subrec->subnet_name));
 		work->log_state = LOGON_NONE;
 		return;
 	}
diff --git a/source3/nmbd/nmbd_packets.c b/source3/nmbd/nmbd_packets.c
index 36dacfb6f31..a89f49c8fc9 100644
--- a/source3/nmbd/nmbd_packets.c
+++ b/source3/nmbd/nmbd_packets.c
@@ -23,6 +23,7 @@
 #include "nmbd/nmbd.h"
 #include "../lib/util/select.h"
 #include "system/select.h"
+#include "libsmb/libsmb.h"
 
 extern int ClientNMB;
 extern int ClientDGRAM;
@@ -108,7 +109,7 @@ void set_nb_flags(char *buf, uint16 nb_flags)
 Dumps out the browse packet data.
 **************************************************************************/
 
-static void debug_browse_data(char *outbuf, int len)
+static void debug_browse_data(const char *outbuf, int len)
 {
 	int i,j;
 
@@ -1062,7 +1063,7 @@ static struct subnet_record *find_subnet_for_dgram_browse_packet(struct packet_s
 Dispatch a browse frame from port 138 to the correct processing function.
 ****************************************************************************/
 
-static void process_browse_packet(struct packet_struct *p, char *buf,int len)
+static void process_browse_packet(struct packet_struct *p, const char *buf,int len)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	int command = CVAL(buf,0);
@@ -1072,9 +1073,9 @@ static void process_browse_packet(struct packet_struct *p, char *buf,int len)
 
 	/* Drop the packet if it's a different NetBIOS scope, or the source is from one of our names. */
 	pull_ascii(scope, dgram->dest_name.scope, 64, 64, STR_TERMINATE);
-	if (!strequal(scope, global_scope())) {
+	if (!strequal(scope, lp_netbios_scope())) {
 		DEBUG(7,("process_browse_packet: Discarding datagram from IP %s. Scope (%s) \
-mismatch with our scope (%s).\n", inet_ntoa(p->ip), scope, global_scope()));
+mismatch with our scope (%s).\n", inet_ntoa(p->ip), scope, lp_netbios_scope()));
 		return;
 	}
 
@@ -1149,7 +1150,7 @@ command code %d from %s IP %s to %s\n", subrec->subnet_name, command, nmb_namest
  Dispatch a LanMan browse frame from port 138 to the correct processing function.
 ****************************************************************************/
 
-static void process_lanman_packet(struct packet_struct *p, char *buf,int len)
+static void process_lanman_packet(struct packet_struct *p, const char *buf,int len)
 {
 	struct dgram_packet *dgram = &p->packet.dgram;
 	int command = SVAL(buf,0);
@@ -1160,9 +1161,9 @@ static void process_lanman_packet(struct packet_struct *p, char *buf,int len)
 	/* Drop the packet if it's a different NetBIOS scope, or the source is from one of our names. */
 
 	pull_ascii(scope, dgram->dest_name.scope, 64, 64, STR_TERMINATE);
-	if (!strequal(scope, global_scope())) {
+	if (!strequal(scope, lp_netbios_scope())) {
 		DEBUG(7,("process_lanman_packet: Discarding datagram from IP %s. Scope (%s) \
-mismatch with our scope (%s).\n", inet_ntoa(p->ip), scope, global_scope()));
+mismatch with our scope (%s).\n", inet_ntoa(p->ip), scope, lp_netbios_scope()));
 		return;
 	}
 
@@ -1216,8 +1217,8 @@ static bool listening(struct packet_struct *p,struct nmb_name *nbname)
 
 static void process_dgram(struct packet_struct *p)
 {
-	char *buf;
-	char *buf2;
+	const char *buf;
+	const char *buf2;
 	int len;
 	struct dgram_packet *dgram = &p->packet.dgram;
 
@@ -1290,28 +1291,28 @@ packet sent to name %s from IP %s\n",
 
 	DEBUG(4,("process_dgram: datagram from %s to %s IP %s for %s of type %d len=%d\n",
 		nmb_namestr(&dgram->source_name),nmb_namestr(&dgram->dest_name),
-		inet_ntoa(p->ip), smb_buf(buf),CVAL(buf2,0),len));
+		inet_ntoa(p->ip), smb_buf_const(buf),CVAL(buf2,0),len));
 
 	/* Datagram packet received for the browser mailslot */
-	if (strequal(smb_buf(buf),BROWSE_MAILSLOT)) {
+	if (strequal(smb_buf_const(buf),BROWSE_MAILSLOT)) {
 		process_browse_packet(p,buf2,len);
 		return;
 	}
 
 	/* Datagram packet received for the LAN Manager mailslot */
-	if (strequal(smb_buf(buf),LANMAN_MAILSLOT)) {
+	if (strequal(smb_buf_const(buf),LANMAN_MAILSLOT)) {
 		process_lanman_packet(p,buf2,len);
 		return;
 	}
 
 	/* Datagram packet received for the domain logon mailslot */
-	if (strequal(smb_buf(buf),NET_LOGON_MAILSLOT)) {
+	if (strequal(smb_buf_const(buf),NET_LOGON_MAILSLOT)) {
 		process_logon_packet(p,buf2,len,NET_LOGON_MAILSLOT);
 		return;
 	}
 
 	/* Datagram packet received for the NT domain logon mailslot */
-	if (strequal(smb_buf(buf),NT_LOGON_MAILSLOT)) {
+	if (strequal(smb_buf_const(buf),NT_LOGON_MAILSLOT)) {
 		process_logon_packet(p,buf2,len,NT_LOGON_MAILSLOT);
 		return;
 	}
@@ -1706,14 +1707,14 @@ static bool create_listen_pollfds(struct pollfd **pfds,
 		}
 	}
 
-	fds = TALLOC_ZERO_ARRAY(NULL, struct pollfd, count);
+	fds = talloc_zero_array(NULL, struct pollfd, count);
 	if (fds == NULL) {
 		DEBUG(1, ("create_listen_pollfds: malloc fail for fds. "
 			  "size %d\n", count));
 		return true;
 	}
 
-	attrs = TALLOC_ARRAY(NULL, struct socket_attributes, count);
+	attrs = talloc_array(NULL, struct socket_attributes, count);
 	if (fds == NULL) {
 		DEBUG(1, ("create_listen_pollfds: malloc fail for attrs. "
 			  "size %d\n", count));
@@ -1884,7 +1885,7 @@ bool listen_for_packets(bool run_election)
 	 * create_listen_pollfds.
 	 */
 
-	fds = TALLOC_REALLOC_ARRAY(NULL, fds, struct pollfd, listen_number);
+	fds = talloc_realloc(NULL, fds, struct pollfd, listen_number);
 	if (fds == NULL) {
 		return true;
 	}
@@ -1893,7 +1894,7 @@ bool listen_for_packets(bool run_election)
 #ifndef SYNC_DNS
 	dns_fd = asyncdns_fd();
 	if (dns_fd != -1) {
-		fds = TALLOC_REALLOC_ARRAY(NULL, fds, struct pollfd, num_sockets+1);
+		fds = talloc_realloc(NULL, fds, struct pollfd, num_sockets+1);
 		if (fds == NULL) {
 			return true;
 		}
@@ -2087,7 +2088,7 @@ bool send_mailslot(bool unique, const char *mailslot,char *buf, size_t len,
 	SSVAL(ptr,smb_vwv15,1);
 	SSVAL(ptr,smb_vwv16,2);
 	p2 = smb_buf(ptr);
-	safe_strcpy_base(p2, mailslot, dgram->data, sizeof(dgram->data));
+	strlcpy_base(p2, mailslot, dgram->data, sizeof(dgram->data));
 	p2 = skip_string(ptr,MAX_DGRAM_SIZE,p2);
 
 	if (((p2+len) > dgram->data+sizeof(dgram->data)) || ((p2+len) < p2)) {
diff --git a/source3/nmbd/nmbd_processlogon.c b/source3/nmbd/nmbd_processlogon.c
index 79768176647..2adc81d76e1 100644
--- a/source3/nmbd/nmbd_processlogon.c
+++ b/source3/nmbd/nmbd_processlogon.c
@@ -157,7 +157,7 @@ static void nmbd_proxy_logon_done(struct tevent_req *subreq);
 static void nmbd_proxy_logon(struct nmbd_proxy_logon_context *ctx,
 			     struct in_addr local_ip,
 			     struct packet_struct *p,
-			     uint8_t *buf,
+			     const uint8_t *buf,
 			     uint32_t len)
 {
 	struct nmbd_proxy_logon_state *state;
@@ -173,7 +173,7 @@ static void nmbd_proxy_logon(struct nmbd_proxy_logon_context *ctx,
 	fstring source_name;
 	struct dgram_packet *dgram = &p->packet.dgram;
 
-	state = TALLOC_ZERO_P(ctx, struct nmbd_proxy_logon_state);
+	state = talloc_zero(ctx, struct nmbd_proxy_logon_state);
 	if (!state) {
 		DEBUG(0,("failed to allocate nmbd_proxy_logon_state\n"));
 		return;
@@ -291,7 +291,7 @@ static void nmbd_proxy_logon_done(struct tevent_req *subreq)
 
 	send_mailslot(true, state->remote_mailslot,
 		      (char *)response.data, response.length,
-		      global_myname(), 0x0,
+		      lp_netbios_name(), 0x0,
 		      state->remote_name,
 		      state->remote_name_type,
 		      state->p->ip,
@@ -304,7 +304,7 @@ static void nmbd_proxy_logon_done(struct tevent_req *subreq)
 Process a domain logon packet
 **************************************************************************/
 
-void process_logon_packet(struct packet_struct *p, char *buf,int len,
+void process_logon_packet(struct packet_struct *p, const char *buf,int len,
                           const char *mailslot)
 {
 	fstring source_name;
@@ -328,7 +328,7 @@ void process_logon_packet(struct packet_struct *p, char *buf,int len,
 			inet_ntoa(p->ip) ));
 		return;
 	}
-	ip = ((struct sockaddr_in *)pss)->sin_addr;
+	ip = ((const struct sockaddr_in *)pss)->sin_addr;
 
 	if (!lp_domain_logons()) {
 		DEBUG(5,("process_logon_packet: Logon packet received from IP %s and domain \
@@ -338,7 +338,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 
 	pull_ascii_nstring(source_name, sizeof(source_name), dgram->source_name.name);
 
-	pdc_name = talloc_asprintf(talloc_tos(), "\\\\%s", global_myname());
+	pdc_name = talloc_asprintf(talloc_tos(), "\\\\%s", lp_netbios_name());
 	if (!pdc_name) {
 		return;
 	}
@@ -391,7 +391,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 		send_mailslot(True, request.req.logon0.mailslot_name,
 				(char *)blob_out.data,
 				blob_out.length,
-				global_myname(), 0x0,
+				lp_netbios_name(), 0x0,
 				source_name,
 				dgram->source_name.name_type,
 				p->ip, ip, p->port);
@@ -411,7 +411,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 			"reporting %s domain %s 0x%x ntversion=%x lm_nt token=%x lm_20 token=%x\n",
 			request.req.pdc.computer_name,
 			inet_ntoa(p->ip),
-			global_myname(),
+			lp_netbios_name(),
 			lp_workgroup(),
 			NETLOGON_RESPONSE_FROM_PDC,
 			request.req.pdc.nt_version,
@@ -419,9 +419,9 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 			request.req.pdc.lm20_token));
 
 		get_pdc.command			= NETLOGON_RESPONSE_FROM_PDC;
-		get_pdc.pdc_name		= global_myname();
+		get_pdc.pdc_name		= lp_netbios_name();
 		get_pdc._pad			= data_blob_null;
-		get_pdc.unicode_pdc_name	= global_myname();
+		get_pdc.unicode_pdc_name	= lp_netbios_name();
 		get_pdc.domain_name		= lp_workgroup();
 		get_pdc.nt_version		= NETLOGON_NT_VERSION_1;
 		get_pdc.lmnt_token		= 0xffff;
@@ -443,7 +443,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 		send_mailslot(True, request.req.pdc.mailslot_name,
 			(char *)blob_out.data,
 			blob_out.length,
-			global_myname(), 0x0,
+			lp_netbios_name(), 0x0,
 			source_name,
 			dgram->source_name.name_type,
 			p->ip, ip, p->port);
@@ -459,7 +459,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 
 		if (global_nmbd_proxy_logon) {
 			nmbd_proxy_logon(global_nmbd_proxy_logon,
-					 ip, p, (uint8_t *)buf, len);
+					 ip, p, (const uint8_t *)buf, len);
 			return;
 		}
 
@@ -553,7 +553,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 			nt5_ex.dns_domain	= domain;
 			nt5_ex.pdc_dns_name	= hostname;
 			nt5_ex.domain_name	= lp_workgroup();
-			nt5_ex.pdc_name		= global_myname();
+			nt5_ex.pdc_name		= lp_netbios_name();
 			nt5_ex.user_name	= request.req.logon.user_name;
 			nt5_ex.server_site	= "Default-First-Site-Name";
 			nt5_ex.client_site	= "Default-First-Site-Name";
@@ -604,8 +604,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 				  source_name, source_addr,
 				  lp_init_logon_delay()));
 
-			when = timeval_current_ofs(0,
-				lp_init_logon_delay() * 1000);
+			when = timeval_current_ofs_msec(lp_init_logon_delay());
 			p->locked = true;
 			event_add_timed(nmbd_event_context(),
 					NULL,
@@ -622,7 +621,7 @@ logons are not enabled.\n", inet_ntoa(p->ip) ));
 			send_mailslot(true, request.req.logon.mailslot_name,
 				(char *)blob_out.data,
 				blob_out.length,
-				global_myname(), 0x0,
+				lp_netbios_name(), 0x0,
 				source_name,
 				dgram->source_name.name_type,
 				p->ip, ip, p->port);
diff --git a/source3/nmbd/nmbd_proto.h b/source3/nmbd/nmbd_proto.h
index b58ca41e5a0..0258b4a478b 100644
--- a/source3/nmbd/nmbd_proto.h
+++ b/source3/nmbd/nmbd_proto.h
@@ -1,3 +1,28 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  NBT netbios routines and daemon - version 2
+ *
+ *  Copyright (C) Andrew Tridgell			1994-1998
+ *  Copyright (C) Jeremy Allison			1994-2005
+ *  Copyright (C) Luke Kenneth Casson Leighton		1994-1998
+ *  Copyright (C) John H Terpstra			1995-1998
+ *  Copyright (C) Christopher R. Hertel			1998
+ *  Copyright (C) Jim McDonough <jmcd@us.ibm.com>	2002
+ *  Copyright (C) Jelmer Vernooij			2002,2003
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
 /* The following definitions come from nmbd/asyncdns.c  */
 
@@ -48,7 +73,7 @@ void sync_all_dmbs(time_t t);
 
 void check_master_browser_exists(time_t t);
 void run_elections(time_t t);
-void process_election(struct subnet_record *subrec, struct packet_struct *p, char *buf);
+void process_election(struct subnet_record *subrec, struct packet_struct *p, const char *buf);
 bool check_elections(void);
 void nmbd_message_election(struct messaging_context *msg,
 			   void *private_data,
@@ -59,18 +84,18 @@ void nmbd_message_election(struct messaging_context *msg,
 /* The following definitions come from nmbd/nmbd_incomingdgrams.c  */
 
 void tell_become_backup(void);
-void process_host_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf);
-void process_workgroup_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf);
-void process_local_master_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf);
+void process_host_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf);
+void process_workgroup_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf);
+void process_local_master_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf);
 void process_master_browser_announce(struct subnet_record *subrec,
-                                     struct packet_struct *p,char *buf);
-void process_lm_host_announce(struct subnet_record *subrec, struct packet_struct *p, char *buf, int len);
+                                     struct packet_struct *p,const char *buf);
+void process_lm_host_announce(struct subnet_record *subrec, struct packet_struct *p, const char *buf, int len);
 void process_get_backup_list_request(struct subnet_record *subrec,
-                                     struct packet_struct *p,char *buf);
+                                     struct packet_struct *p,const char *buf);
 void process_reset_browser(struct subnet_record *subrec,
-                                  struct packet_struct *p,char *buf);
-void process_announce_request(struct subnet_record *subrec, struct packet_struct *p, char *buf);
-void process_lm_announce_request(struct subnet_record *subrec, struct packet_struct *p, char *buf, int len);
+                                  struct packet_struct *p,const char *buf);
+void process_announce_request(struct subnet_record *subrec, struct packet_struct *p, const char *buf);
+void process_lm_announce_request(struct subnet_record *subrec, struct packet_struct *p, const char *buf, int len);
 
 /* The following definitions come from nmbd/nmbd_incomingrequests.c  */
 
@@ -250,7 +275,7 @@ bool send_mailslot(bool unique, const char *mailslot,char *buf, size_t len,
 
 bool initialize_nmbd_proxy_logon(void);
 
-void process_logon_packet(struct packet_struct *p, char *buf,int len,
+void process_logon_packet(struct packet_struct *p, const char *buf,int len,
                           const char *mailslot);
 
 /* The following definitions come from nmbd/nmbd_responserecordsdb.c  */
diff --git a/source3/nmbd/nmbd_sendannounce.c b/source3/nmbd/nmbd_sendannounce.c
index e04e1f6fe3c..39e63d29abb 100644
--- a/source3/nmbd/nmbd_sendannounce.c
+++ b/source3/nmbd/nmbd_sendannounce.c
@@ -50,7 +50,7 @@ void send_browser_reset(int reset_type, const char *to_name, int to_type, struct
 	p++;
 
 	send_mailslot(True, BROWSE_MAILSLOT, outbuf,PTR_DIFF(p,outbuf),
-		global_myname(), 0x0, to_name, to_type, to_ip, 
+		lp_netbios_name(), 0x0, to_name, to_type, to_ip,
 		FIRST_SUBNET->myip, DGRAM_PORT);
 }
 
@@ -76,10 +76,10 @@ to subnet %s\n", work->work_group, subrec->subnet_name));
 
 	SCVAL(p,0,work->token); /* (local) Unique workgroup token id. */
 	p++;
-	p +=  push_string_check(p+1, global_myname(), 15, STR_ASCII|STR_UPPER|STR_TERMINATE);
+	p +=  push_string_check(p+1, lp_netbios_name(), 15, STR_ASCII|STR_UPPER|STR_TERMINATE);
   
 	send_mailslot(False, BROWSE_MAILSLOT, outbuf,PTR_DIFF(p,outbuf),
-		global_myname(), 0x0, work->work_group,0x1e, subrec->bcast_ip, 
+		lp_netbios_name(), 0x0, work->work_group,0x1e, subrec->bcast_ip,
 		subrec->myip, DGRAM_PORT);
 }
 
@@ -105,12 +105,12 @@ static void send_announcement(struct subnet_record *subrec, int announce_type,
 	SCVAL(p,0,updatecount);
 	SIVAL(p,1,announce_interval*1000); /* Milliseconds - despite the spec. */
 
-	safe_strcpy(upper_server_name, server_name, sizeof(upper_server_name)-1);
+	strlcpy(upper_server_name, server_name ? server_name : "", sizeof(upper_server_name));
 	strupper_m(upper_server_name);
 	push_string_check(p+5, upper_server_name, 16, STR_ASCII|STR_TERMINATE);
 
-	SCVAL(p,21,lp_major_announce_version()); /* Major version. */
-	SCVAL(p,22,lp_minor_announce_version()); /* Minor version. */
+	SCVAL(p,21,SAMBA_MAJOR_NBT_ANNOUNCE_VERSION); /* Major version. */
+	SCVAL(p,22,SAMBA_MINOR_NBT_ANNOUNCE_VERSION); /* Minor version. */
 
 	SIVAL(p,23,server_type & ~SV_TYPE_LOCAL_LIST_ONLY);
 	/* Browse version: got from NT/AS 4.00  - Value defined in smb.h (JHT). */
@@ -140,8 +140,8 @@ static void send_lm_announcement(struct subnet_record *subrec, int announce_type
 
 	SSVAL(p,0,announce_type);
 	SIVAL(p,2,server_type & ~SV_TYPE_LOCAL_LIST_ONLY);
-	SCVAL(p,6,lp_major_announce_version()); /* Major version. */
-	SCVAL(p,7,lp_minor_announce_version()); /* Minor version. */
+	SCVAL(p,6,SAMBA_MAJOR_NBT_ANNOUNCE_VERSION); /* Major version. */
+	SCVAL(p,7,SAMBA_MINOR_NBT_ANNOUNCE_VERSION); /* Minor version. */
 	SSVAL(p,8,announce_interval);            /* In seconds - according to spec. */
 
 	p += 10;
@@ -164,14 +164,14 @@ static void send_local_master_announcement(struct subnet_record *subrec, struct
 	uint32 type = servrec->serv.type & ~SV_TYPE_LOCAL_LIST_ONLY;
 
 	DEBUG(3,("send_local_master_announcement: type %x for name %s on subnet %s for workgroup %s\n",
-		type, global_myname(), subrec->subnet_name, work->work_group));
+		type, lp_netbios_name(), subrec->subnet_name, work->work_group));
 
 	send_announcement(subrec, ANN_LocalMasterAnnouncement,
-			global_myname(),                 /* From nbt name. */
+			lp_netbios_name(),                 /* From nbt name. */
 			work->work_group, 0x1e,          /* To nbt name. */
 			subrec->bcast_ip,                /* To ip. */
 			work->announce_interval,         /* Time until next announce. */
-			global_myname(),                 /* Name to announce. */
+			lp_netbios_name(),                 /* Name to announce. */
 			type,                            /* Type field. */
 			servrec->serv.comment);
 }
@@ -186,13 +186,13 @@ static void send_workgroup_announcement(struct subnet_record *subrec, struct wor
 		subrec->subnet_name, work->work_group));
 
 	send_announcement(subrec, ANN_DomainAnnouncement,
-			global_myname(),                 /* From nbt name. */
+			lp_netbios_name(),                 /* From nbt name. */
 			MSBROWSE, 0x1,                   /* To nbt name. */
 			subrec->bcast_ip,                /* To ip. */
 			work->announce_interval,         /* Time until next announce. */
 			work->work_group,                /* Name to announce. */
 			SV_TYPE_DOMAIN_ENUM|SV_TYPE_NT,  /* workgroup announce flags. */
-			global_myname());                /* From name as comment. */
+			lp_netbios_name());                /* From name as comment. */
 }
 
 /****************************************************************************
@@ -251,7 +251,7 @@ static void announce_server(struct subnet_record *subrec, struct work_record *wo
 	/* Only do domain announcements if we are a master and it's
 		our primary name we're being asked to announce. */
 
-	if (AM_LOCAL_MASTER_BROWSER(work) && strequal(global_myname(),servrec->serv.name)) {
+	if (AM_LOCAL_MASTER_BROWSER(work) && strequal(lp_netbios_name(),servrec->serv.name)) {
 		send_local_master_announcement(subrec, work, servrec);
 		send_workgroup_announcement(subrec, work);
 	} else {
@@ -566,7 +566,7 @@ for workgroup %s on subnet %s.\n", lp_workgroup(), FIRST_SUBNET->subnet_name ));
 	SCVAL(p,0,ANN_MasterAnnouncement);
 	p++;
 
-	unstrcpy(myname, global_myname());
+	unstrcpy(myname, lp_netbios_name());
 	strupper_m(myname);
 	myname[15]='\0';
 	push_ascii(p, myname, sizeof(outbuf)-PTR_DIFF(p,outbuf)-1, STR_TERMINATE);
@@ -579,10 +579,10 @@ for workgroup %s on subnet %s.\n", lp_workgroup(), FIRST_SUBNET->subnet_name ));
 		addr = interpret_addr2(s2);
 
 		DEBUG(5,("announce_remote: Doing remote browse sync announce for server %s to IP %s.\n",
-			global_myname(), inet_ntoa(addr) ));
+			lp_netbios_name(), inet_ntoa(addr) ));
 
 		send_mailslot(True, BROWSE_MAILSLOT, outbuf,PTR_DIFF(p,outbuf),
-			global_myname(), 0x0, "*", 0x0, addr, FIRST_SUBNET->myip, DGRAM_PORT);
+			lp_netbios_name(), 0x0, "*", 0x0, addr, FIRST_SUBNET->myip, DGRAM_PORT);
 	}
 	TALLOC_FREE(frame);
 }
diff --git a/source3/nmbd/nmbd_subnetdb.c b/source3/nmbd/nmbd_subnetdb.c
index 3a60038dcdc..666679a90b4 100644
--- a/source3/nmbd/nmbd_subnetdb.c
+++ b/source3/nmbd/nmbd_subnetdb.c
@@ -296,7 +296,7 @@ bool create_subnets(void)
 		 * ignore it here. JRA.
 		 */
 
-		if (is_loopback_addr((struct sockaddr *)&iface->ip)) {
+		if (is_loopback_addr((const struct sockaddr *)&iface->ip)) {
 			DEBUG(2,("create_subnets: Ignoring loopback interface.\n" ));
 			continue;
 		}
diff --git a/source3/nmbd/nmbd_synclists.c b/source3/nmbd/nmbd_synclists.c
index fc2b25edcae..964975e1a52 100644
--- a/source3/nmbd/nmbd_synclists.c
+++ b/source3/nmbd/nmbd_synclists.c
@@ -4,20 +4,19 @@
    Copyright (C) Andrew Tridgell 1994-1998
    Copyright (C) Luke Kenneth Casson Leighton 1994-1998
    Copyright (C) Jeremy Allison 1994-1998
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-   
 */
 
 /* this file handles asynchronous browse synchronisation requests. The
@@ -30,6 +29,7 @@
 #include "system/filesys.h"
 #include "../librpc/gen_ndr/svcctl.h"
 #include "nmbd/nmbd.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "smbprofile.h"
 
@@ -72,7 +72,6 @@ static void sync_child(char *name, int nm_type,
 	fstring unix_workgroup;
 	struct cli_state *cli;
 	uint32 local_type = local ? SV_TYPE_LOCAL_LIST_ONLY : 0;
-	struct nmb_name called, calling;
 	struct sockaddr_storage ss;
 	NTSTATUS status;
 
@@ -80,25 +79,12 @@ static void sync_child(char *name, int nm_type,
 	 * Patch from Andy Levine andyl@epicrealm.com.
 	 */
 
-	cli = cli_initialise();
-	if (!cli) {
-		return;
-	}
-
-	cli_set_port(cli, 139);
-
 	in_addr_to_sockaddr_storage(&ss, ip);
-	status = cli_connect(cli, name, &ss);
-	if (!NT_STATUS_IS_OK(status)) {
-		cli_shutdown(cli);
-		return;
-	}
-
-	make_nmb_name(&calling, get_local_machine_name(), 0x0);
-	make_nmb_name(&called , name, nm_type);
 
-	if (!cli_session_request(cli, &calling, &called)) {
-		cli_shutdown(cli);
+	status = cli_connect_nb(name, &ss, 139, nm_type,
+				get_local_machine_name(), Undefined,
+				&cli);
+	if (!NT_STATUS_IS_OK(status)) {
 		return;
 	}
 
@@ -126,7 +112,7 @@ static void sync_child(char *name, int nm_type,
 	cli_NetServerEnum(cli, unix_workgroup,
 			  local_type|SV_TYPE_DOMAIN_ENUM, 
 			  callback, NULL);
-	
+
 	/* Now fetch a server list. */
 	if (servers) {
 		fstrcpy(unix_workgroup, workgroup);
@@ -134,7 +120,7 @@ static void sync_child(char *name, int nm_type,
 				  local?SV_TYPE_LOCAL_LIST_ONLY:SV_TYPE_ALL,
 				  callback, NULL);
 	}
-	
+
 	cli_shutdown(cli);
 }
 
diff --git a/source3/nmbd/nmbd_winsserver.c b/source3/nmbd/nmbd_winsserver.c
index 976b99023eb..50cdc0acf93 100644
--- a/source3/nmbd/nmbd_winsserver.c
+++ b/source3/nmbd/nmbd_winsserver.c
@@ -23,6 +23,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "nmbd/nmbd.h"
+#include "util_tdb.h"
 
 #define WINS_LIST "wins.dat"
 #define WINS_VERSION 1
@@ -74,7 +75,7 @@ static void wins_delete_all_1b_in_memory_records(void)
 }
 
 /****************************************************************************
- Convert a wins.tdb record to a struct name_record. Add in our global_scope().
+ Convert a wins.tdb record to a struct name_record. Add in our lp_netbios_scope().
 *****************************************************************************/
 
 static struct name_record *wins_record_to_name_record(TDB_DATA key, TDB_DATA data)
@@ -127,7 +128,7 @@ static struct name_record *wins_record_to_name_record(TDB_DATA key, TDB_DATA dat
 	push_ascii_nstring(namerec->name.name, (const char *)key.dptr);
 	namerec->name.name_type = key.dptr[sizeof(unstring)];
 	/* Add the scope. */
-	push_ascii(namerec->name.scope, global_scope(), 64, STR_TERMINATE);
+	push_ascii(namerec->name.scope, lp_netbios_scope(), 64, STR_TERMINATE);
 
         /* We're using a byte-by-byte compare, so we must be sure that
          * unused space doesn't have garbage in it.
@@ -240,7 +241,7 @@ struct name_record *find_name_on_wins_subnet(const struct nmb_name *nmbname, boo
 	}
 
 	key = name_to_key(nmbname);
-	data = tdb_fetch(wins_tdb, key);
+	data = tdb_fetch_compat(wins_tdb, key);
 
 	if (data.dsize == 0) {
 		return NULL;
diff --git a/source3/nmbd/nmbd_workgroupdb.c b/source3/nmbd/nmbd_workgroupdb.c
index 5ce10e64d46..43a28aa6699 100644
--- a/source3/nmbd/nmbd_workgroupdb.c
+++ b/source3/nmbd/nmbd_workgroupdb.c
@@ -52,7 +52,7 @@ static void name_to_unstring(unstring unname, const char *name)
 	if (errno == E2BIG) {
 		unstring tname;
 		pull_ascii_nstring(tname, sizeof(tname), nname);
-		unstrcpy(unname, tname);
+		strlcpy(unname, tname, sizeof(nname));
 		DEBUG(0,("name_to_nstring: workgroup name %s is too long. Truncating to %s\n",
 			name, tname));
 	} else {
@@ -250,7 +250,7 @@ workgroup %s on subnet %s\n", work->work_group, subrec->subnet_name));
 		const char *name = my_netbios_names(i);
 		int stype = lp_default_server_announce() | (lp_local_master() ?  SV_TYPE_POTENTIAL_BROWSER : 0 );
    
-		if(!strequal(global_myname(), name))
+		if(!strequal(lp_netbios_name(), name))
 			stype &= ~(SV_TYPE_MASTER_BROWSER|SV_TYPE_POTENTIAL_BROWSER|SV_TYPE_DOMAIN_MASTER|SV_TYPE_DOMAIN_MEMBER);
    
 		create_server_on_workgroup(work,name,stype|SV_TYPE_LOCAL_LIST_ONLY, PERMANENT_TTL, 
diff --git a/source3/pam_smbpass/support.c b/source3/pam_smbpass/support.c
index 79f92a8758d..3dfdff3c028 100644
--- a/source3/pam_smbpass/support.c
+++ b/source3/pam_smbpass/support.c
@@ -150,7 +150,7 @@ int make_remark( pam_handle_t * pamh, unsigned int ctrl
 		struct pam_response *resp;
 
 		pmsg[0] = &msg[0];
-		msg[0].msg = CONST_DISCARD(char *, text);
+		msg[0].msg = discard_const_p(char, text);
 		msg[0].msg_style = type;
 		resp = NULL;
 
@@ -550,7 +550,7 @@ int _smb_read_password( pam_handle_t * pamh, unsigned int ctrl,
     if (comment != NULL && off(SMB__QUIET, ctrl)) {
         pmsg[0] = &msg[0];
         msg[0].msg_style = PAM_TEXT_INFO;
-        msg[0].msg = CONST_DISCARD(char *, comment);
+        msg[0].msg = discard_const_p(char, comment);
         i = 1;
     } else {
         i = 0;
@@ -558,12 +558,12 @@ int _smb_read_password( pam_handle_t * pamh, unsigned int ctrl,
 
     pmsg[i] = &msg[i];
     msg[i].msg_style = PAM_PROMPT_ECHO_OFF;
-    msg[i++].msg = CONST_DISCARD(char *, prompt1);
+    msg[i++].msg = discard_const_p(char, prompt1);
 
     if (prompt2 != NULL) {
         pmsg[i] = &msg[i];
         msg[i].msg_style = PAM_PROMPT_ECHO_OFF;
-        msg[i++].msg = CONST_DISCARD(char *, prompt2);
+        msg[i++].msg = discard_const_p(char, prompt2);
         expect = 2;
     } else
         expect = 1;
diff --git a/source3/pam_smbpass/wscript_build b/source3/pam_smbpass/wscript_build
index 0d200c6fd22..200482c7cf4 100644
--- a/source3/pam_smbpass/wscript_build
+++ b/source3/pam_smbpass/wscript_build
@@ -6,8 +6,9 @@ if bld.CONFIG_SET('WITH_PAM_MODULES'):
                   pam_smb_passwd.c
                   pam_smb_acct.c
                   support.c''',
-        deps='''tdb talloc pam PAM_ERRORS wbclient cap asn1util param LIB_NONSMBD passdb SMBLDAP
+        deps='''tdb talloc pam PAM_ERRORS wbclient cap asn1util param passdb SMBLDAP
                 LIBNTLMSSP LIBTSOCKET''',
         cflags='-DLOCALEDIR=\"%s/locale\"' % bld.env.DATADIR,
         realname='pam_smbpass.so',
+                       allow_undefined_symbols=False
         )
diff --git a/source3/param/loadparm.c b/source3/param/loadparm.c
index 0f76f0925a5..3f622ac1972 100644
--- a/source3/param/loadparm.c
+++ b/source3/param/loadparm.c
@@ -53,10 +53,10 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "util_tdb.h"
 #include "printing.h"
 #include "lib/smbconf/smbconf.h"
 #include "lib/smbconf/smbconf_init.h"
-#include "lib/smbconf/smbconf_reg.h"
 
 #include "ads.h"
 #include "../librpc/gen_ndr/svcctl.h"
@@ -130,7 +130,6 @@ struct global {
 	char *smb_ports;
 	char *dos_charset;
 	char *unix_charset;
-	char *display_charset;
 	char *szPrintcapname;
 	char *szAddPortCommand;
 	char *szEnumPortsCommand;
@@ -159,6 +158,8 @@ struct global {
 	char *szPasswordServer;
 	char *szSocketOptions;
 	char *szRealm;
+	char *szRealmUpper;
+	char *szDnsDomain;
 	char *szAfsUsernameMap;
 	int iAfsTokenLifetime;
 	char *szLogNtTokenCommand;
@@ -174,7 +175,6 @@ struct global {
 	char *szSocketAddress;
 	bool bNmbdBindExplicitBroadcast;
 	char *szNISHomeMapName;
-	char *szAnnounceVersion;	/* This is initialised in init_globals */
 	char *szWorkgroup;
 	char *szNetbiosName;
 	char **szNetbiosAliases;
@@ -259,7 +259,6 @@ struct global {
 	int min_wins_ttl;
 	int lm_announce;
 	int lm_interval;
-	int announce_as;	/* This is initialised in init_globals */
 	int machine_password_timeout;
 	int map_to_guest;
 	int oplock_break_wait_time;
@@ -699,15 +698,14 @@ static int default_server_announce;
 /* prototypes for the special type handlers */
 static bool handle_include( int snum, const char *pszParmValue, char **ptr);
 static bool handle_copy( int snum, const char *pszParmValue, char **ptr);
-static bool handle_netbios_name( int snum, const char *pszParmValue, char **ptr);
 static bool handle_idmap_backend(int snum, const char *pszParmValue, char **ptr);
 static bool handle_idmap_uid( int snum, const char *pszParmValue, char **ptr);
 static bool handle_idmap_gid( int snum, const char *pszParmValue, char **ptr);
 static bool handle_debug_list( int snum, const char *pszParmValue, char **ptr );
-static bool handle_workgroup( int snum, const char *pszParmValue, char **ptr );
+static bool handle_realm( int snum, const char *pszParmValue, char **ptr );
 static bool handle_netbios_aliases( int snum, const char *pszParmValue, char **ptr );
-static bool handle_netbios_scope( int snum, const char *pszParmValue, char **ptr );
 static bool handle_charset( int snum, const char *pszParmValue, char **ptr );
+static bool handle_dos_charset( int snum, const char *pszParmValue, char **ptr );
 static bool handle_printing( int snum, const char *pszParmValue, char **ptr);
 static bool handle_ldap_debug_level( int snum, const char *pszParmValue, char **ptr);
 
@@ -798,21 +796,6 @@ static const struct enum_list enum_ldap_passwd_sync[] = {
 	{-1, NULL}
 };
 
-/* Types of machine we can announce as. */
-#define ANNOUNCE_AS_NT_SERVER 1
-#define ANNOUNCE_AS_WIN95 2
-#define ANNOUNCE_AS_WFW 3
-#define ANNOUNCE_AS_NT_WORKSTATION 4
-
-static const struct enum_list enum_announce_as[] = {
-	{ANNOUNCE_AS_NT_SERVER, "NT"},
-	{ANNOUNCE_AS_NT_SERVER, "NT Server"},
-	{ANNOUNCE_AS_NT_WORKSTATION, "NT Workstation"},
-	{ANNOUNCE_AS_WIN95, "win95"},
-	{ANNOUNCE_AS_WFW, "WfW"},
-	{-1, NULL}
-};
-
 static const struct enum_list enum_map_readonly[] = {
 	{MAP_READONLY_NO, "no"},
 	{MAP_READONLY_NO, "false"},
@@ -955,7 +938,7 @@ static struct parm_struct parm_table[] = {
 		.type		= P_STRING,
 		.p_class	= P_GLOBAL,
 		.ptr		= &Globals.dos_charset,
-		.special	= handle_charset,
+		.special	= handle_dos_charset,
 		.enum_list	= NULL,
 		.flags		= FLAG_ADVANCED
 	},
@@ -969,15 +952,6 @@ static struct parm_struct parm_table[] = {
 		.flags		= FLAG_ADVANCED
 	},
 	{
-		.label		= "display charset",
-		.type		= P_STRING,
-		.p_class	= P_GLOBAL,
-		.ptr		= &Globals.display_charset,
-		.special	= handle_charset,
-		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED
-	},
-	{
 		.label		= "comment",
 		.type		= P_STRING,
 		.p_class	= P_LOCAL,
@@ -1009,27 +983,25 @@ static struct parm_struct parm_table[] = {
 		.type		= P_USTRING,
 		.p_class	= P_GLOBAL,
 		.ptr		= &Globals.szWorkgroup,
-		.special	= handle_workgroup,
+		.special	= NULL,
 		.enum_list	= NULL,
 		.flags		= FLAG_BASIC | FLAG_ADVANCED | FLAG_WIZARD,
 	},
-#ifdef WITH_ADS
 	{
 		.label		= "realm",
 		.type		= P_USTRING,
 		.p_class	= P_GLOBAL,
 		.ptr		= &Globals.szRealm,
-		.special	= NULL,
+		.special	= handle_realm,
 		.enum_list	= NULL,
 		.flags		= FLAG_BASIC | FLAG_ADVANCED | FLAG_WIZARD,
 	},
-#endif
 	{
 		.label		= "netbios name",
 		.type		= P_USTRING,
 		.p_class	= P_GLOBAL,
 		.ptr		= &Globals.szNetbiosName,
-		.special	= handle_netbios_name,
+		.special	= NULL,
 		.enum_list	= NULL,
 		.flags		= FLAG_BASIC | FLAG_ADVANCED | FLAG_WIZARD,
 	},
@@ -1047,7 +1019,7 @@ static struct parm_struct parm_table[] = {
 		.type		= P_USTRING,
 		.p_class	= P_GLOBAL,
 		.ptr		= &Globals.szNetbiosScope,
-		.special	= handle_netbios_scope,
+		.special	= NULL,
 		.enum_list	= NULL,
 		.flags		= FLAG_ADVANCED,
 	},
@@ -1160,7 +1132,7 @@ static struct parm_struct parm_table[] = {
 		.ptr		= &Globals.bNullPasswords,
 		.special	= NULL,
 		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED,
+		.flags		= FLAG_ADVANCED | FLAG_DEPRECATED,
 	},
 	{
 		.label		= "obey pam restrictions",
@@ -1259,7 +1231,7 @@ static struct parm_struct parm_table[] = {
 		.ptr		= &Globals.bEnablePrivileges,
 		.special	= NULL,
 		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED,
+		.flags		= FLAG_ADVANCED | FLAG_DEPRECATED,
 	},
 
 	{
@@ -1332,7 +1304,7 @@ static struct parm_struct parm_table[] = {
 		.ptr		= &Globals.pwordlevel,
 		.special	= NULL,
 		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED,
+		.flags		= FLAG_ADVANCED | FLAG_DEPRECATED,
 	},
 	{
 		.label		= "username level",
@@ -1431,7 +1403,7 @@ static struct parm_struct parm_table[] = {
 		.ptr		= &sDefault.szUsername,
 		.special	= NULL,
 		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED | FLAG_GLOBAL | FLAG_SHARE,
+		.flags		= FLAG_ADVANCED | FLAG_GLOBAL | FLAG_SHARE | FLAG_DEPRECATED,
 	},
 	{
 		.label		= "user",
@@ -2180,24 +2152,6 @@ static struct parm_struct parm_table[] = {
 		.flags		= FLAG_ADVANCED | FLAG_GLOBAL | FLAG_SHARE,
 	},
 	{
-		.label		= "announce version",
-		.type		= P_STRING,
-		.p_class	= P_GLOBAL,
-		.ptr		= &Globals.szAnnounceVersion,
-		.special	= NULL,
-		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED,
-	},
-	{
-		.label		= "announce as",
-		.type		= P_ENUM,
-		.p_class	= P_GLOBAL,
-		.ptr		= &Globals.announce_as,
-		.special	= NULL,
-		.enum_list	= enum_announce_as,
-		.flags		= FLAG_ADVANCED,
-	},
-	{
 		.label		= "map acl inherit",
 		.type		= P_BOOL,
 		.p_class	= P_LOCAL,
@@ -2294,7 +2248,7 @@ static struct parm_struct parm_table[] = {
 		.ptr		= &Globals.bUseSpnego,
 		.special	= NULL,
 		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED,
+		.flags		= FLAG_ADVANCED | FLAG_DEPRECATED,
 	},
 	{
 		.label		= "client signing",
@@ -4139,15 +4093,6 @@ static struct parm_struct parm_table[] = {
 		.flags		= FLAG_ADVANCED,
 	},
 	{
-		.label		= "time offset",
-		.type		= P_INTEGER,
-		.p_class	= P_GLOBAL,
-		.ptr		= &extra_time_offset,
-		.special	= NULL,
-		.enum_list	= NULL,
-		.flags		= FLAG_ADVANCED,
-	},
-	{
 		.label		= "NIS homedir",
 		.type		= P_BOOL,
 		.p_class	= P_GLOBAL,
@@ -5172,6 +5117,7 @@ static void init_globals(bool reinit_globals)
 		}
 	}
 
+
 	string_set(&sDefault.fstype, FSTYPE_STRING);
 	string_set(&sDefault.szPrintjobUsername, "%U");
 
@@ -5180,6 +5126,9 @@ static void init_globals(bool reinit_globals)
 
 	DEBUG(3, ("Initialising global parameters\n"));
 
+	/* Must manually force to upper case here, as this does not go via the handler */
+	string_set(&Globals.szNetbiosName, myhostname_upper());
+
 	string_set(&Globals.szSMBPasswdFile, get_dyn_SMB_PASSWD_FILE());
 	string_set(&Globals.szPrivateDir, get_dyn_PRIVATE_DIR());
 
@@ -5192,14 +5141,6 @@ static void init_globals(bool reinit_globals)
 	/* using UTF8 by default allows us to support all chars */
 	string_set(&Globals.unix_charset, DEFAULT_UNIX_CHARSET);
 
-#if defined(HAVE_NL_LANGINFO) && defined(CODESET)
-	/* If the system supports nl_langinfo(), try to grab the value
-	   from the user's locale */
-	string_set(&Globals.display_charset, "LOCALE");
-#else
-	string_set(&Globals.display_charset, DEFAULT_DISPLAY_CHARSET);
-#endif
-
 	/* Use codepage 850 as a default for the dos character set */
 	string_set(&Globals.dos_charset, DEFAULT_DOS_CHARSET);
 
@@ -5208,11 +5149,7 @@ static void init_globals(bool reinit_globals)
 	 */
 	string_set(&Globals.szPasswdChat, DEFAULT_PASSWD_CHAT);
 
-	set_global_myname(myhostname());
-	string_set(&Globals.szNetbiosName,global_myname());
-
-	set_global_myworkgroup(WORKGROUP);
-	string_set(&Globals.szWorkgroup, lp_workgroup());
+	string_set(&Globals.szWorkgroup, WORKGROUP);
 
 	string_set(&Globals.szPasswdProgram, "");
 	string_set(&Globals.szLockDir, get_dyn_LOCKDIR());
@@ -5231,12 +5168,6 @@ static void init_globals(bool reinit_globals)
 	}
 	string_set(&Globals.szServerString, s);
 	SAFE_FREE(s);
-	if (asprintf(&s, "%d.%d", DEFAULT_MAJOR_VERSION,
-			DEFAULT_MINOR_VERSION) < 0) {
-		smb_panic("init_globals: ENOMEM");
-	}
-	string_set(&Globals.szAnnounceVersion, s);
-	SAFE_FREE(s);
 #ifdef DEVELOPER
 	string_set(&Globals.szPanicAction, "/bin/sleep 999999999");
 #endif
@@ -5301,7 +5232,6 @@ static void init_globals(bool reinit_globals)
 	Globals.machine_password_timeout = 60 * 60 * 24 * 7;	/* 7 days default. */
 	Globals.lm_announce = 2;	/* = Auto: send only if LM clients found */
 	Globals.lm_interval = 60;
-	Globals.announce_as = ANNOUNCE_AS_NT_SERVER;
 #if (defined(HAVE_NETGROUP) && defined(WITH_AUTOMOUNT))
 	Globals.bNISHomeMap = False;
 #ifdef WITH_NISPLUS_HOME
@@ -5533,17 +5463,17 @@ static char *lp_string(const char *s)
 */
 
 #define FN_GLOBAL_STRING(fn_name,ptr) \
- char *fn_name(void) {return(lp_string(*(char **)(ptr) ? *(char **)(ptr) : ""));}
+ char *fn_name(void) {return(lp_string(*(char **)(&Globals.ptr) ? *(char **)(&Globals.ptr) : ""));}
 #define FN_GLOBAL_CONST_STRING(fn_name,ptr) \
- const char *fn_name(void) {return(*(const char **)(ptr) ? *(const char **)(ptr) : "");}
+ const char *fn_name(void) {return(*(const char **)(&Globals.ptr) ? *(const char **)(&Globals.ptr) : "");}
 #define FN_GLOBAL_LIST(fn_name,ptr) \
- const char **fn_name(void) {return(*(const char ***)(ptr));}
+ const char **fn_name(void) {return(*(const char ***)(&Globals.ptr));}
 #define FN_GLOBAL_BOOL(fn_name,ptr) \
- bool fn_name(void) {return(*(bool *)(ptr));}
+ bool fn_name(void) {return(*(bool *)(&Globals.ptr));}
 #define FN_GLOBAL_CHAR(fn_name,ptr) \
- char fn_name(void) {return(*(char *)(ptr));}
+ char fn_name(void) {return(*(char *)(&Globals.ptr));}
 #define FN_GLOBAL_INTEGER(fn_name,ptr) \
- int fn_name(void) {return(*(int *)(ptr));}
+ int fn_name(void) {return(*(int *)(&Globals.ptr));}
 
 #define FN_LOCAL_STRING(fn_name,val) \
  char *fn_name(int i) {return(lp_string((LP_SNUM_OK(i) && ServicePtrs[(i)]->val) ? ServicePtrs[(i)]->val : sDefault.val));}
@@ -5563,78 +5493,80 @@ static char *lp_string(const char *s)
 #define FN_LOCAL_CHAR(fn_name,val) \
  char fn_name(const struct share_params *p) {return(LP_SNUM_OK(p->service)? ServicePtrs[(p->service)]->val : sDefault.val);}
 
-FN_GLOBAL_STRING(lp_smb_ports, &Globals.smb_ports)
-FN_GLOBAL_STRING(lp_dos_charset, &Globals.dos_charset)
-FN_GLOBAL_STRING(lp_unix_charset, &Globals.unix_charset)
-FN_GLOBAL_STRING(lp_display_charset, &Globals.display_charset)
-FN_GLOBAL_STRING(lp_logfile, &Globals.szLogFile)
-FN_GLOBAL_STRING(lp_configfile, &Globals.szConfigFile)
-FN_GLOBAL_STRING(lp_smb_passwd_file, &Globals.szSMBPasswdFile)
-FN_GLOBAL_STRING(lp_private_dir, &Globals.szPrivateDir)
-FN_GLOBAL_STRING(lp_serverstring, &Globals.szServerString)
-FN_GLOBAL_INTEGER(lp_printcap_cache_time, &Globals.PrintcapCacheTime)
-FN_GLOBAL_STRING(lp_addport_cmd, &Globals.szAddPortCommand)
-FN_GLOBAL_STRING(lp_enumports_cmd, &Globals.szEnumPortsCommand)
-FN_GLOBAL_STRING(lp_addprinter_cmd, &Globals.szAddPrinterCommand)
-FN_GLOBAL_STRING(lp_deleteprinter_cmd, &Globals.szDeletePrinterCommand)
-FN_GLOBAL_STRING(lp_os2_driver_map, &Globals.szOs2DriverMap)
-FN_GLOBAL_STRING(lp_lockdir, &Globals.szLockDir)
+FN_GLOBAL_CONST_STRING(lp_smb_ports, smb_ports)
+FN_GLOBAL_CONST_STRING(lp_dos_charset, dos_charset)
+FN_GLOBAL_CONST_STRING(lp_unix_charset, unix_charset)
+FN_GLOBAL_STRING(lp_logfile, szLogFile)
+FN_GLOBAL_STRING(lp_configfile, szConfigFile)
+FN_GLOBAL_CONST_STRING(lp_smb_passwd_file, szSMBPasswdFile)
+FN_GLOBAL_CONST_STRING(lp_private_dir, szPrivateDir)
+FN_GLOBAL_STRING(lp_serverstring, szServerString)
+FN_GLOBAL_INTEGER(lp_printcap_cache_time, PrintcapCacheTime)
+FN_GLOBAL_STRING(lp_addport_cmd, szAddPortCommand)
+FN_GLOBAL_STRING(lp_enumports_cmd, szEnumPortsCommand)
+FN_GLOBAL_STRING(lp_addprinter_cmd, szAddPrinterCommand)
+FN_GLOBAL_STRING(lp_deleteprinter_cmd, szDeletePrinterCommand)
+FN_GLOBAL_STRING(lp_os2_driver_map, szOs2DriverMap)
+FN_GLOBAL_CONST_STRING(lp_lockdir, szLockDir)
 /* If lp_statedir() and lp_cachedir() are explicitely set during the
  * build process or in smb.conf, we use that value.  Otherwise they
  * default to the value of lp_lockdir(). */
-char *lp_statedir(void) {
+const char *lp_statedir(void) {
 	if ((strcmp(get_dyn_STATEDIR(), get_dyn_LOCKDIR()) != 0) ||
 	    (strcmp(get_dyn_STATEDIR(), Globals.szStateDir) != 0))
-		return(lp_string(*(char **)(&Globals.szStateDir) ?
-		    *(char **)(&Globals.szStateDir) : ""));
+		return(*(char **)(&Globals.szStateDir) ?
+		       *(char **)(&Globals.szStateDir) : "");
 	else
-		return(lp_string(*(char **)(&Globals.szLockDir) ?
-		    *(char **)(&Globals.szLockDir) : ""));
+		return(*(char **)(&Globals.szLockDir) ?
+		       *(char **)(&Globals.szLockDir) : "");
 }
-char *lp_cachedir(void) {
+const char *lp_cachedir(void) {
 	if ((strcmp(get_dyn_CACHEDIR(), get_dyn_LOCKDIR()) != 0) ||
 	    (strcmp(get_dyn_CACHEDIR(), Globals.szCacheDir) != 0))
-		return(lp_string(*(char **)(&Globals.szCacheDir) ?
-		    *(char **)(&Globals.szCacheDir) : ""));
+		return(*(char **)(&Globals.szCacheDir) ?
+		       *(char **)(&Globals.szCacheDir) : "");
 	else
-		return(lp_string(*(char **)(&Globals.szLockDir) ?
-		    *(char **)(&Globals.szLockDir) : ""));
-}
-FN_GLOBAL_STRING(lp_piddir, &Globals.szPidDir)
-FN_GLOBAL_STRING(lp_mangling_method, &Globals.szManglingMethod)
-FN_GLOBAL_INTEGER(lp_mangle_prefix, &Globals.mangle_prefix)
-FN_GLOBAL_STRING(lp_utmpdir, &Globals.szUtmpDir)
-FN_GLOBAL_STRING(lp_wtmpdir, &Globals.szWtmpDir)
-FN_GLOBAL_BOOL(lp_utmp, &Globals.bUtmp)
-FN_GLOBAL_STRING(lp_rootdir, &Globals.szRootdir)
-FN_GLOBAL_STRING(lp_perfcount_module, &Globals.szSMBPerfcountModule)
-FN_GLOBAL_STRING(lp_defaultservice, &Globals.szDefaultService)
-FN_GLOBAL_STRING(lp_msg_command, &Globals.szMsgCommand)
-FN_GLOBAL_STRING(lp_get_quota_command, &Globals.szGetQuota)
-FN_GLOBAL_STRING(lp_set_quota_command, &Globals.szSetQuota)
-FN_GLOBAL_STRING(lp_auto_services, &Globals.szAutoServices)
-FN_GLOBAL_STRING(lp_passwd_program, &Globals.szPasswdProgram)
-FN_GLOBAL_STRING(lp_passwd_chat, &Globals.szPasswdChat)
-FN_GLOBAL_STRING(lp_passwordserver, &Globals.szPasswordServer)
-FN_GLOBAL_STRING(lp_name_resolve_order, &Globals.szNameResolveOrder)
-FN_GLOBAL_STRING(lp_realm, &Globals.szRealm)
-FN_GLOBAL_CONST_STRING(lp_afs_username_map, &Globals.szAfsUsernameMap)
-FN_GLOBAL_INTEGER(lp_afs_token_lifetime, &Globals.iAfsTokenLifetime)
-FN_GLOBAL_STRING(lp_log_nt_token_command, &Globals.szLogNtTokenCommand)
-FN_GLOBAL_STRING(lp_username_map, &Globals.szUsernameMap)
-FN_GLOBAL_CONST_STRING(lp_logon_script, &Globals.szLogonScript)
-FN_GLOBAL_CONST_STRING(lp_logon_path, &Globals.szLogonPath)
-FN_GLOBAL_CONST_STRING(lp_logon_drive, &Globals.szLogonDrive)
-FN_GLOBAL_CONST_STRING(lp_logon_home, &Globals.szLogonHome)
-FN_GLOBAL_STRING(lp_remote_announce, &Globals.szRemoteAnnounce)
-FN_GLOBAL_STRING(lp_remote_browse_sync, &Globals.szRemoteBrowseSync)
-FN_GLOBAL_BOOL(lp_nmbd_bind_explicit_broadcast, &Globals.bNmbdBindExplicitBroadcast)
-FN_GLOBAL_LIST(lp_wins_server_list, &Globals.szWINSservers)
-FN_GLOBAL_LIST(lp_interfaces, &Globals.szInterfaces)
-FN_GLOBAL_STRING(lp_nis_home_map_name, &Globals.szNISHomeMapName)
-static FN_GLOBAL_STRING(lp_announce_version, &Globals.szAnnounceVersion)
-FN_GLOBAL_LIST(lp_netbios_aliases, &Globals.szNetbiosAliases)
-/* FN_GLOBAL_STRING(lp_passdb_backend, &Globals.szPassdbBackend)
+		return(*(char **)(&Globals.szLockDir) ?
+		       *(char **)(&Globals.szLockDir) : "");
+}
+FN_GLOBAL_CONST_STRING(lp_piddir, szPidDir)
+FN_GLOBAL_STRING(lp_mangling_method, szManglingMethod)
+FN_GLOBAL_INTEGER(lp_mangle_prefix, mangle_prefix)
+FN_GLOBAL_CONST_STRING(lp_utmpdir, szUtmpDir)
+FN_GLOBAL_CONST_STRING(lp_wtmpdir, szWtmpDir)
+FN_GLOBAL_BOOL(lp_utmp, bUtmp)
+FN_GLOBAL_STRING(lp_rootdir, szRootdir)
+FN_GLOBAL_STRING(lp_perfcount_module, szSMBPerfcountModule)
+FN_GLOBAL_STRING(lp_defaultservice, szDefaultService)
+FN_GLOBAL_STRING(lp_msg_command, szMsgCommand)
+FN_GLOBAL_STRING(lp_get_quota_command, szGetQuota)
+FN_GLOBAL_STRING(lp_set_quota_command, szSetQuota)
+FN_GLOBAL_STRING(lp_auto_services, szAutoServices)
+FN_GLOBAL_STRING(lp_passwd_program, szPasswdProgram)
+FN_GLOBAL_STRING(lp_passwd_chat, szPasswdChat)
+FN_GLOBAL_CONST_STRING(lp_passwordserver, szPasswordServer)
+FN_GLOBAL_CONST_STRING(lp_name_resolve_order, szNameResolveOrder)
+FN_GLOBAL_CONST_STRING(lp_workgroup, szWorkgroup)
+FN_GLOBAL_CONST_STRING(lp_netbios_name, szNetbiosName)
+FN_GLOBAL_CONST_STRING(lp_netbios_scope, szNetbiosScope)
+FN_GLOBAL_CONST_STRING(lp_realm, szRealmUpper)
+FN_GLOBAL_CONST_STRING(lp_dnsdomain, szDnsDomain)
+FN_GLOBAL_CONST_STRING(lp_afs_username_map, szAfsUsernameMap)
+FN_GLOBAL_INTEGER(lp_afs_token_lifetime, iAfsTokenLifetime)
+FN_GLOBAL_STRING(lp_log_nt_token_command, szLogNtTokenCommand)
+FN_GLOBAL_STRING(lp_username_map, szUsernameMap)
+FN_GLOBAL_CONST_STRING(lp_logon_script, szLogonScript)
+FN_GLOBAL_CONST_STRING(lp_logon_path, szLogonPath)
+FN_GLOBAL_CONST_STRING(lp_logon_drive, szLogonDrive)
+FN_GLOBAL_CONST_STRING(lp_logon_home, szLogonHome)
+FN_GLOBAL_STRING(lp_remote_announce, szRemoteAnnounce)
+FN_GLOBAL_STRING(lp_remote_browse_sync, szRemoteBrowseSync)
+FN_GLOBAL_BOOL(lp_nmbd_bind_explicit_broadcast, bNmbdBindExplicitBroadcast)
+FN_GLOBAL_LIST(lp_wins_server_list, szWINSservers)
+FN_GLOBAL_LIST(lp_interfaces, szInterfaces)
+FN_GLOBAL_STRING(lp_nis_home_map_name, szNISHomeMapName)
+FN_GLOBAL_LIST(lp_netbios_aliases, szNetbiosAliases)
+/* FN_GLOBAL_STRING(lp_passdb_backend, szPassdbBackend)
  * lp_passdb_backend() should be replace by the this macro again after
  * some releases.
  * */
@@ -5676,46 +5608,46 @@ warn:
 out:
 	return Globals.szPassdbBackend;
 }
-FN_GLOBAL_LIST(lp_preload_modules, &Globals.szPreloadModules)
-FN_GLOBAL_STRING(lp_panic_action, &Globals.szPanicAction)
-FN_GLOBAL_STRING(lp_adduser_script, &Globals.szAddUserScript)
-FN_GLOBAL_STRING(lp_renameuser_script, &Globals.szRenameUserScript)
-FN_GLOBAL_STRING(lp_deluser_script, &Globals.szDelUserScript)
-
-FN_GLOBAL_CONST_STRING(lp_guestaccount, &Globals.szGuestaccount)
-FN_GLOBAL_STRING(lp_addgroup_script, &Globals.szAddGroupScript)
-FN_GLOBAL_STRING(lp_delgroup_script, &Globals.szDelGroupScript)
-FN_GLOBAL_STRING(lp_addusertogroup_script, &Globals.szAddUserToGroupScript)
-FN_GLOBAL_STRING(lp_deluserfromgroup_script, &Globals.szDelUserFromGroupScript)
-FN_GLOBAL_STRING(lp_setprimarygroup_script, &Globals.szSetPrimaryGroupScript)
-
-FN_GLOBAL_STRING(lp_addmachine_script, &Globals.szAddMachineScript)
-
-FN_GLOBAL_STRING(lp_shutdown_script, &Globals.szShutdownScript)
-FN_GLOBAL_STRING(lp_abort_shutdown_script, &Globals.szAbortShutdownScript)
-FN_GLOBAL_STRING(lp_username_map_script, &Globals.szUsernameMapScript)
-FN_GLOBAL_INTEGER(lp_username_map_cache_time, &Globals.iUsernameMapCacheTime)
-
-FN_GLOBAL_STRING(lp_check_password_script, &Globals.szCheckPasswordScript)
-
-FN_GLOBAL_STRING(lp_wins_hook, &Globals.szWINSHook)
-FN_GLOBAL_CONST_STRING(lp_template_homedir, &Globals.szTemplateHomedir)
-FN_GLOBAL_CONST_STRING(lp_template_shell, &Globals.szTemplateShell)
-FN_GLOBAL_CONST_STRING(lp_winbind_separator, &Globals.szWinbindSeparator)
-FN_GLOBAL_INTEGER(lp_acl_compatibility, &Globals.iAclCompat)
-FN_GLOBAL_BOOL(lp_winbind_enum_users, &Globals.bWinbindEnumUsers)
-FN_GLOBAL_BOOL(lp_winbind_enum_groups, &Globals.bWinbindEnumGroups)
-FN_GLOBAL_BOOL(lp_winbind_use_default_domain, &Globals.bWinbindUseDefaultDomain)
-FN_GLOBAL_BOOL(lp_winbind_trusted_domains_only, &Globals.bWinbindTrustedDomainsOnly)
-FN_GLOBAL_BOOL(lp_winbind_nested_groups, &Globals.bWinbindNestedGroups)
-FN_GLOBAL_INTEGER(lp_winbind_expand_groups, &Globals.winbind_expand_groups)
-FN_GLOBAL_BOOL(lp_winbind_refresh_tickets, &Globals.bWinbindRefreshTickets)
-FN_GLOBAL_BOOL(lp_winbind_offline_logon, &Globals.bWinbindOfflineLogon)
-FN_GLOBAL_BOOL(lp_winbind_normalize_names, &Globals.bWinbindNormalizeNames)
-FN_GLOBAL_BOOL(lp_winbind_rpc_only, &Globals.bWinbindRpcOnly)
-FN_GLOBAL_BOOL(lp_create_krb5_conf, &Globals.bCreateKrb5Conf)
+FN_GLOBAL_LIST(lp_preload_modules, szPreloadModules)
+FN_GLOBAL_STRING(lp_panic_action, szPanicAction)
+FN_GLOBAL_STRING(lp_adduser_script, szAddUserScript)
+FN_GLOBAL_STRING(lp_renameuser_script, szRenameUserScript)
+FN_GLOBAL_STRING(lp_deluser_script, szDelUserScript)
+
+FN_GLOBAL_CONST_STRING(lp_guestaccount, szGuestaccount)
+FN_GLOBAL_STRING(lp_addgroup_script, szAddGroupScript)
+FN_GLOBAL_STRING(lp_delgroup_script, szDelGroupScript)
+FN_GLOBAL_STRING(lp_addusertogroup_script, szAddUserToGroupScript)
+FN_GLOBAL_STRING(lp_deluserfromgroup_script, szDelUserFromGroupScript)
+FN_GLOBAL_STRING(lp_setprimarygroup_script, szSetPrimaryGroupScript)
+
+FN_GLOBAL_STRING(lp_addmachine_script, szAddMachineScript)
+
+FN_GLOBAL_STRING(lp_shutdown_script, szShutdownScript)
+FN_GLOBAL_STRING(lp_abort_shutdown_script, szAbortShutdownScript)
+FN_GLOBAL_STRING(lp_username_map_script, szUsernameMapScript)
+FN_GLOBAL_INTEGER(lp_username_map_cache_time, iUsernameMapCacheTime)
+
+FN_GLOBAL_STRING(lp_check_password_script, szCheckPasswordScript)
+
+FN_GLOBAL_STRING(lp_wins_hook, szWINSHook)
+FN_GLOBAL_CONST_STRING(lp_template_homedir, szTemplateHomedir)
+FN_GLOBAL_CONST_STRING(lp_template_shell, szTemplateShell)
+FN_GLOBAL_CONST_STRING(lp_winbind_separator, szWinbindSeparator)
+FN_GLOBAL_INTEGER(lp_acl_compatibility, iAclCompat)
+FN_GLOBAL_BOOL(lp_winbind_enum_users, bWinbindEnumUsers)
+FN_GLOBAL_BOOL(lp_winbind_enum_groups, bWinbindEnumGroups)
+FN_GLOBAL_BOOL(lp_winbind_use_default_domain, bWinbindUseDefaultDomain)
+FN_GLOBAL_BOOL(lp_winbind_trusted_domains_only, bWinbindTrustedDomainsOnly)
+FN_GLOBAL_BOOL(lp_winbind_nested_groups, bWinbindNestedGroups)
+FN_GLOBAL_INTEGER(lp_winbind_expand_groups, winbind_expand_groups)
+FN_GLOBAL_BOOL(lp_winbind_refresh_tickets, bWinbindRefreshTickets)
+FN_GLOBAL_BOOL(lp_winbind_offline_logon, bWinbindOfflineLogon)
+FN_GLOBAL_BOOL(lp_winbind_normalize_names, bWinbindNormalizeNames)
+FN_GLOBAL_BOOL(lp_winbind_rpc_only, bWinbindRpcOnly)
+FN_GLOBAL_BOOL(lp_create_krb5_conf, bCreateKrb5Conf)
 static FN_GLOBAL_INTEGER(lp_winbind_max_domain_connections_int,
-		  &Globals.winbindMaxDomainConnections)
+		  winbindMaxDomainConnections)
 
 int lp_winbind_max_domain_connections(void)
 {
@@ -5728,119 +5660,119 @@ int lp_winbind_max_domain_connections(void)
 	return MAX(1, lp_winbind_max_domain_connections_int());
 }
 
-FN_GLOBAL_CONST_STRING(lp_idmap_backend, &Globals.szIdmapBackend)
-FN_GLOBAL_INTEGER(lp_idmap_cache_time, &Globals.iIdmapCacheTime)
-FN_GLOBAL_INTEGER(lp_idmap_negative_cache_time, &Globals.iIdmapNegativeCacheTime)
-FN_GLOBAL_INTEGER(lp_keepalive, &Globals.iKeepalive)
-FN_GLOBAL_BOOL(lp_passdb_expand_explicit, &Globals.bPassdbExpandExplicit)
-
-FN_GLOBAL_STRING(lp_ldap_suffix, &Globals.szLdapSuffix)
-FN_GLOBAL_STRING(lp_ldap_admin_dn, &Globals.szLdapAdminDn)
-FN_GLOBAL_INTEGER(lp_ldap_ssl, &Globals.ldap_ssl)
-FN_GLOBAL_BOOL(lp_ldap_ssl_ads, &Globals.ldap_ssl_ads)
-FN_GLOBAL_INTEGER(lp_ldap_deref, &Globals.ldap_deref)
-FN_GLOBAL_INTEGER(lp_ldap_follow_referral, &Globals.ldap_follow_referral)
-FN_GLOBAL_INTEGER(lp_ldap_passwd_sync, &Globals.ldap_passwd_sync)
-FN_GLOBAL_BOOL(lp_ldap_delete_dn, &Globals.ldap_delete_dn)
-FN_GLOBAL_INTEGER(lp_ldap_replication_sleep, &Globals.ldap_replication_sleep)
-FN_GLOBAL_INTEGER(lp_ldap_timeout, &Globals.ldap_timeout)
-FN_GLOBAL_INTEGER(lp_ldap_connection_timeout, &Globals.ldap_connection_timeout)
-FN_GLOBAL_INTEGER(lp_ldap_page_size, &Globals.ldap_page_size)
-FN_GLOBAL_INTEGER(lp_ldap_debug_level, &Globals.ldap_debug_level)
-FN_GLOBAL_INTEGER(lp_ldap_debug_threshold, &Globals.ldap_debug_threshold)
-FN_GLOBAL_STRING(lp_add_share_cmd, &Globals.szAddShareCommand)
-FN_GLOBAL_STRING(lp_change_share_cmd, &Globals.szChangeShareCommand)
-FN_GLOBAL_STRING(lp_delete_share_cmd, &Globals.szDeleteShareCommand)
-FN_GLOBAL_STRING(lp_usershare_path, &Globals.szUsersharePath)
-FN_GLOBAL_LIST(lp_usershare_prefix_allow_list, &Globals.szUsersharePrefixAllowList)
-FN_GLOBAL_LIST(lp_usershare_prefix_deny_list, &Globals.szUsersharePrefixDenyList)
-
-FN_GLOBAL_LIST(lp_eventlog_list, &Globals.szEventLogs)
-
-FN_GLOBAL_BOOL(lp_registry_shares, &Globals.bRegistryShares)
-FN_GLOBAL_BOOL(lp_usershare_allow_guests, &Globals.bUsershareAllowGuests)
-FN_GLOBAL_BOOL(lp_usershare_owner_only, &Globals.bUsershareOwnerOnly)
-FN_GLOBAL_BOOL(lp_disable_netbios, &Globals.bDisableNetbios)
-FN_GLOBAL_BOOL(lp_reset_on_zero_vc, &Globals.bResetOnZeroVC)
+FN_GLOBAL_CONST_STRING(lp_idmap_backend, szIdmapBackend)
+FN_GLOBAL_INTEGER(lp_idmap_cache_time, iIdmapCacheTime)
+FN_GLOBAL_INTEGER(lp_idmap_negative_cache_time, iIdmapNegativeCacheTime)
+FN_GLOBAL_INTEGER(lp_keepalive, iKeepalive)
+FN_GLOBAL_BOOL(lp_passdb_expand_explicit, bPassdbExpandExplicit)
+
+FN_GLOBAL_STRING(lp_ldap_suffix, szLdapSuffix)
+FN_GLOBAL_STRING(lp_ldap_admin_dn, szLdapAdminDn)
+FN_GLOBAL_INTEGER(lp_ldap_ssl, ldap_ssl)
+FN_GLOBAL_BOOL(lp_ldap_ssl_ads, ldap_ssl_ads)
+FN_GLOBAL_INTEGER(lp_ldap_deref, ldap_deref)
+FN_GLOBAL_INTEGER(lp_ldap_follow_referral, ldap_follow_referral)
+FN_GLOBAL_INTEGER(lp_ldap_passwd_sync, ldap_passwd_sync)
+FN_GLOBAL_BOOL(lp_ldap_delete_dn, ldap_delete_dn)
+FN_GLOBAL_INTEGER(lp_ldap_replication_sleep, ldap_replication_sleep)
+FN_GLOBAL_INTEGER(lp_ldap_timeout, ldap_timeout)
+FN_GLOBAL_INTEGER(lp_ldap_connection_timeout, ldap_connection_timeout)
+FN_GLOBAL_INTEGER(lp_ldap_page_size, ldap_page_size)
+FN_GLOBAL_INTEGER(lp_ldap_debug_level, ldap_debug_level)
+FN_GLOBAL_INTEGER(lp_ldap_debug_threshold, ldap_debug_threshold)
+FN_GLOBAL_STRING(lp_add_share_cmd, szAddShareCommand)
+FN_GLOBAL_STRING(lp_change_share_cmd, szChangeShareCommand)
+FN_GLOBAL_STRING(lp_delete_share_cmd, szDeleteShareCommand)
+FN_GLOBAL_STRING(lp_usershare_path, szUsersharePath)
+FN_GLOBAL_LIST(lp_usershare_prefix_allow_list, szUsersharePrefixAllowList)
+FN_GLOBAL_LIST(lp_usershare_prefix_deny_list, szUsersharePrefixDenyList)
+
+FN_GLOBAL_LIST(lp_eventlog_list, szEventLogs)
+
+FN_GLOBAL_BOOL(lp_registry_shares, bRegistryShares)
+FN_GLOBAL_BOOL(lp_usershare_allow_guests, bUsershareAllowGuests)
+FN_GLOBAL_BOOL(lp_usershare_owner_only, bUsershareOwnerOnly)
+FN_GLOBAL_BOOL(lp_disable_netbios, bDisableNetbios)
+FN_GLOBAL_BOOL(lp_reset_on_zero_vc, bResetOnZeroVC)
 FN_GLOBAL_BOOL(lp_log_writeable_files_on_exit,
-	       &Globals.bLogWriteableFilesOnExit)
-FN_GLOBAL_BOOL(lp_ms_add_printer_wizard, &Globals.bMsAddPrinterWizard)
-FN_GLOBAL_BOOL(lp_dns_proxy, &Globals.bDNSproxy)
-FN_GLOBAL_BOOL(lp_wins_support, &Globals.bWINSsupport)
-FN_GLOBAL_BOOL(lp_we_are_a_wins_server, &Globals.bWINSsupport)
-FN_GLOBAL_BOOL(lp_wins_proxy, &Globals.bWINSproxy)
-FN_GLOBAL_BOOL(lp_local_master, &Globals.bLocalMaster)
-FN_GLOBAL_BOOL(lp_domain_logons, &Globals.bDomainLogons)
-FN_GLOBAL_LIST(lp_init_logon_delayed_hosts, &Globals.szInitLogonDelayedHosts)
-FN_GLOBAL_INTEGER(lp_init_logon_delay, &Globals.InitLogonDelay)
-FN_GLOBAL_BOOL(lp_load_printers, &Globals.bLoadPrinters)
-FN_GLOBAL_BOOL(_lp_readraw, &Globals.bReadRaw)
-FN_GLOBAL_BOOL(lp_large_readwrite, &Globals.bLargeReadwrite)
-FN_GLOBAL_BOOL(_lp_writeraw, &Globals.bWriteRaw)
-FN_GLOBAL_BOOL(lp_null_passwords, &Globals.bNullPasswords)
-FN_GLOBAL_BOOL(lp_obey_pam_restrictions, &Globals.bObeyPamRestrictions)
-FN_GLOBAL_BOOL(lp_encrypted_passwords, &Globals.bEncryptPasswords)
-FN_GLOBAL_INTEGER(lp_client_schannel, &Globals.clientSchannel)
-FN_GLOBAL_INTEGER(lp_server_schannel, &Globals.serverSchannel)
-FN_GLOBAL_BOOL(lp_syslog_only, &Globals.bSyslogOnly)
-FN_GLOBAL_BOOL(lp_timestamp_logs, &Globals.bTimestampLogs)
-FN_GLOBAL_BOOL(lp_debug_prefix_timestamp, &Globals.bDebugPrefixTimestamp)
-FN_GLOBAL_BOOL(lp_debug_hires_timestamp, &Globals.bDebugHiresTimestamp)
-FN_GLOBAL_BOOL(lp_debug_pid, &Globals.bDebugPid)
-FN_GLOBAL_BOOL(lp_debug_uid, &Globals.bDebugUid)
-FN_GLOBAL_BOOL(lp_debug_class, &Globals.bDebugClass)
-FN_GLOBAL_BOOL(lp_enable_core_files, &Globals.bEnableCoreFiles)
-FN_GLOBAL_BOOL(lp_browse_list, &Globals.bBrowseList)
-FN_GLOBAL_BOOL(lp_nis_home_map, &Globals.bNISHomeMap)
-static FN_GLOBAL_BOOL(lp_time_server, &Globals.bTimeServer)
-FN_GLOBAL_BOOL(lp_bind_interfaces_only, &Globals.bBindInterfacesOnly)
-FN_GLOBAL_BOOL(lp_pam_password_change, &Globals.bPamPasswordChange)
-FN_GLOBAL_BOOL(lp_unix_password_sync, &Globals.bUnixPasswdSync)
-FN_GLOBAL_BOOL(lp_passwd_chat_debug, &Globals.bPasswdChatDebug)
-FN_GLOBAL_INTEGER(lp_passwd_chat_timeout, &Globals.iPasswdChatTimeout)
-FN_GLOBAL_BOOL(lp_nt_pipe_support, &Globals.bNTPipeSupport)
-FN_GLOBAL_BOOL(lp_nt_status_support, &Globals.bNTStatusSupport)
-FN_GLOBAL_BOOL(lp_stat_cache, &Globals.bStatCache)
-FN_GLOBAL_INTEGER(lp_max_stat_cache_size, &Globals.iMaxStatCacheSize)
-FN_GLOBAL_BOOL(lp_allow_trusted_domains, &Globals.bAllowTrustedDomains)
-FN_GLOBAL_BOOL(lp_map_untrusted_to_domain, &Globals.bMapUntrustedToDomain)
-FN_GLOBAL_INTEGER(lp_restrict_anonymous, &Globals.restrict_anonymous)
-FN_GLOBAL_BOOL(lp_lanman_auth, &Globals.bLanmanAuth)
-FN_GLOBAL_BOOL(lp_ntlm_auth, &Globals.bNTLMAuth)
-FN_GLOBAL_BOOL(lp_client_plaintext_auth, &Globals.bClientPlaintextAuth)
-FN_GLOBAL_BOOL(lp_client_lanman_auth, &Globals.bClientLanManAuth)
-FN_GLOBAL_BOOL(lp_client_ntlmv2_auth, &Globals.bClientNTLMv2Auth)
-FN_GLOBAL_BOOL(lp_host_msdfs, &Globals.bHostMSDfs)
-FN_GLOBAL_BOOL(lp_kernel_oplocks, &Globals.bKernelOplocks)
-FN_GLOBAL_BOOL(lp_enhanced_browsing, &Globals.enhanced_browsing)
-FN_GLOBAL_BOOL(lp_use_mmap, &Globals.bUseMmap)
-FN_GLOBAL_BOOL(lp_unix_extensions, &Globals.bUnixExtensions)
-FN_GLOBAL_BOOL(lp_use_spnego, &Globals.bUseSpnego)
-FN_GLOBAL_BOOL(lp_client_use_spnego, &Globals.bClientUseSpnego)
-FN_GLOBAL_BOOL(lp_client_use_spnego_principal, &Globals.client_use_spnego_principal)
-FN_GLOBAL_BOOL(lp_send_spnego_principal, &Globals.send_spnego_principal)
-FN_GLOBAL_BOOL(lp_hostname_lookups, &Globals.bHostnameLookups)
+	       bLogWriteableFilesOnExit)
+FN_GLOBAL_BOOL(lp_ms_add_printer_wizard, bMsAddPrinterWizard)
+FN_GLOBAL_BOOL(lp_dns_proxy, bDNSproxy)
+FN_GLOBAL_BOOL(lp_wins_support, bWINSsupport)
+FN_GLOBAL_BOOL(lp_we_are_a_wins_server, bWINSsupport)
+FN_GLOBAL_BOOL(lp_wins_proxy, bWINSproxy)
+FN_GLOBAL_BOOL(lp_local_master, bLocalMaster)
+FN_GLOBAL_BOOL(lp_domain_logons, bDomainLogons)
+FN_GLOBAL_LIST(lp_init_logon_delayed_hosts, szInitLogonDelayedHosts)
+FN_GLOBAL_INTEGER(lp_init_logon_delay, InitLogonDelay)
+FN_GLOBAL_BOOL(lp_load_printers, bLoadPrinters)
+FN_GLOBAL_BOOL(_lp_readraw, bReadRaw)
+FN_GLOBAL_BOOL(lp_large_readwrite, bLargeReadwrite)
+FN_GLOBAL_BOOL(_lp_writeraw, bWriteRaw)
+FN_GLOBAL_BOOL(lp_null_passwords, bNullPasswords)
+FN_GLOBAL_BOOL(lp_obey_pam_restrictions, bObeyPamRestrictions)
+FN_GLOBAL_BOOL(lp_encrypted_passwords, bEncryptPasswords)
+FN_GLOBAL_INTEGER(lp_client_schannel, clientSchannel)
+FN_GLOBAL_INTEGER(lp_server_schannel, serverSchannel)
+FN_GLOBAL_BOOL(lp_syslog_only, bSyslogOnly)
+FN_GLOBAL_BOOL(lp_timestamp_logs, bTimestampLogs)
+FN_GLOBAL_BOOL(lp_debug_prefix_timestamp, bDebugPrefixTimestamp)
+FN_GLOBAL_BOOL(lp_debug_hires_timestamp, bDebugHiresTimestamp)
+FN_GLOBAL_BOOL(lp_debug_pid, bDebugPid)
+FN_GLOBAL_BOOL(lp_debug_uid, bDebugUid)
+FN_GLOBAL_BOOL(lp_debug_class, bDebugClass)
+FN_GLOBAL_BOOL(lp_enable_core_files, bEnableCoreFiles)
+FN_GLOBAL_BOOL(lp_browse_list, bBrowseList)
+FN_GLOBAL_BOOL(lp_nis_home_map, bNISHomeMap)
+static FN_GLOBAL_BOOL(lp_time_server, bTimeServer)
+FN_GLOBAL_BOOL(lp_bind_interfaces_only, bBindInterfacesOnly)
+FN_GLOBAL_BOOL(lp_pam_password_change, bPamPasswordChange)
+FN_GLOBAL_BOOL(lp_unix_password_sync, bUnixPasswdSync)
+FN_GLOBAL_BOOL(lp_passwd_chat_debug, bPasswdChatDebug)
+FN_GLOBAL_INTEGER(lp_passwd_chat_timeout, iPasswdChatTimeout)
+FN_GLOBAL_BOOL(lp_nt_pipe_support, bNTPipeSupport)
+FN_GLOBAL_BOOL(lp_nt_status_support, bNTStatusSupport)
+FN_GLOBAL_BOOL(lp_stat_cache, bStatCache)
+FN_GLOBAL_INTEGER(lp_max_stat_cache_size, iMaxStatCacheSize)
+FN_GLOBAL_BOOL(lp_allow_trusted_domains, bAllowTrustedDomains)
+FN_GLOBAL_BOOL(lp_map_untrusted_to_domain, bMapUntrustedToDomain)
+FN_GLOBAL_INTEGER(lp_restrict_anonymous, restrict_anonymous)
+FN_GLOBAL_BOOL(lp_lanman_auth, bLanmanAuth)
+FN_GLOBAL_BOOL(lp_ntlm_auth, bNTLMAuth)
+FN_GLOBAL_BOOL(lp_client_plaintext_auth, bClientPlaintextAuth)
+FN_GLOBAL_BOOL(lp_client_lanman_auth, bClientLanManAuth)
+FN_GLOBAL_BOOL(lp_client_ntlmv2_auth, bClientNTLMv2Auth)
+FN_GLOBAL_BOOL(lp_host_msdfs, bHostMSDfs)
+FN_GLOBAL_BOOL(lp_kernel_oplocks, bKernelOplocks)
+FN_GLOBAL_BOOL(lp_enhanced_browsing, enhanced_browsing)
+FN_GLOBAL_BOOL(lp_use_mmap, bUseMmap)
+FN_GLOBAL_BOOL(lp_unix_extensions, bUnixExtensions)
+FN_GLOBAL_BOOL(lp_use_spnego, bUseSpnego)
+FN_GLOBAL_BOOL(lp_client_use_spnego, bClientUseSpnego)
+FN_GLOBAL_BOOL(lp_client_use_spnego_principal, client_use_spnego_principal)
+FN_GLOBAL_BOOL(lp_send_spnego_principal, send_spnego_principal)
+FN_GLOBAL_BOOL(lp_hostname_lookups, bHostnameLookups)
 FN_LOCAL_PARM_BOOL(lp_change_notify, bChangeNotify)
 FN_LOCAL_PARM_BOOL(lp_kernel_change_notify, bKernelChangeNotify)
-FN_GLOBAL_STRING(lp_dedicated_keytab_file, &Globals.szDedicatedKeytabFile)
-FN_GLOBAL_INTEGER(lp_kerberos_method, &Globals.iKerberosMethod)
-FN_GLOBAL_BOOL(lp_defer_sharing_violations, &Globals.bDeferSharingViolations)
-FN_GLOBAL_BOOL(lp_enable_privileges, &Globals.bEnablePrivileges)
-FN_GLOBAL_BOOL(lp_enable_asu_support, &Globals.bASUSupport)
-FN_GLOBAL_INTEGER(lp_os_level, &Globals.os_level)
-FN_GLOBAL_INTEGER(lp_max_ttl, &Globals.max_ttl)
-FN_GLOBAL_INTEGER(lp_max_wins_ttl, &Globals.max_wins_ttl)
-FN_GLOBAL_INTEGER(lp_min_wins_ttl, &Globals.min_wins_ttl)
-FN_GLOBAL_INTEGER(lp_max_log_size, &Globals.max_log_size)
-FN_GLOBAL_INTEGER(lp_max_open_files, &Globals.max_open_files)
-FN_GLOBAL_INTEGER(lp_open_files_db_hash_size, &Globals.open_files_db_hash_size)
-FN_GLOBAL_INTEGER(lp_maxxmit, &Globals.max_xmit)
-FN_GLOBAL_INTEGER(lp_maxmux, &Globals.max_mux)
-FN_GLOBAL_INTEGER(lp_passwordlevel, &Globals.pwordlevel)
-FN_GLOBAL_INTEGER(lp_usernamelevel, &Globals.unamelevel)
-FN_GLOBAL_INTEGER(lp_deadtime, &Globals.deadtime)
-FN_GLOBAL_BOOL(lp_getwd_cache, &Globals.getwd_cache)
-static FN_GLOBAL_INTEGER(_lp_maxprotocol, &Globals.maxprotocol)
+FN_GLOBAL_CONST_STRING(lp_dedicated_keytab_file, szDedicatedKeytabFile)
+FN_GLOBAL_INTEGER(lp_kerberos_method, iKerberosMethod)
+FN_GLOBAL_BOOL(lp_defer_sharing_violations, bDeferSharingViolations)
+FN_GLOBAL_BOOL(lp_enable_privileges, bEnablePrivileges)
+FN_GLOBAL_BOOL(lp_enable_asu_support, bASUSupport)
+FN_GLOBAL_INTEGER(lp_os_level, os_level)
+FN_GLOBAL_INTEGER(lp_max_ttl, max_ttl)
+FN_GLOBAL_INTEGER(lp_max_wins_ttl, max_wins_ttl)
+FN_GLOBAL_INTEGER(lp_min_wins_ttl, min_wins_ttl)
+FN_GLOBAL_INTEGER(lp_max_log_size, max_log_size)
+FN_GLOBAL_INTEGER(lp_max_open_files, max_open_files)
+FN_GLOBAL_INTEGER(lp_open_files_db_hash_size, open_files_db_hash_size)
+FN_GLOBAL_INTEGER(lp_maxxmit, max_xmit)
+FN_GLOBAL_INTEGER(lp_maxmux, max_mux)
+FN_GLOBAL_INTEGER(lp_passwordlevel, pwordlevel)
+FN_GLOBAL_INTEGER(lp_usernamelevel, unamelevel)
+FN_GLOBAL_INTEGER(lp_deadtime, deadtime)
+FN_GLOBAL_BOOL(lp_getwd_cache, getwd_cache)
+static FN_GLOBAL_INTEGER(_lp_maxprotocol, maxprotocol)
 int lp_maxprotocol(void)
 {
 	int ret = _lp_maxprotocol();
@@ -5852,28 +5784,27 @@ int lp_maxprotocol(void)
 	}
 	return ret;
 }
-FN_GLOBAL_INTEGER(lp_minprotocol, &Globals.minprotocol)
-FN_GLOBAL_INTEGER(lp_security, &Globals.security)
-FN_GLOBAL_LIST(lp_auth_methods, &Globals.AuthMethods)
-FN_GLOBAL_BOOL(lp_paranoid_server_security, &Globals.paranoid_server_security)
-FN_GLOBAL_INTEGER(lp_maxdisksize, &Globals.maxdisksize)
-FN_GLOBAL_INTEGER(lp_lpqcachetime, &Globals.lpqcachetime)
-FN_GLOBAL_INTEGER(lp_max_smbd_processes, &Globals.iMaxSmbdProcesses)
-FN_GLOBAL_BOOL(_lp_disable_spoolss, &Globals.bDisableSpoolss)
-FN_GLOBAL_INTEGER(lp_syslog, &Globals.syslog)
-static FN_GLOBAL_INTEGER(lp_announce_as, &Globals.announce_as)
-FN_GLOBAL_INTEGER(lp_lm_announce, &Globals.lm_announce)
-FN_GLOBAL_INTEGER(lp_lm_interval, &Globals.lm_interval)
-FN_GLOBAL_INTEGER(lp_machine_password_timeout, &Globals.machine_password_timeout)
-FN_GLOBAL_INTEGER(lp_map_to_guest, &Globals.map_to_guest)
-FN_GLOBAL_INTEGER(lp_oplock_break_wait_time, &Globals.oplock_break_wait_time)
-FN_GLOBAL_INTEGER(lp_lock_spin_time, &Globals.iLockSpinTime)
-FN_GLOBAL_INTEGER(lp_usershare_max_shares, &Globals.iUsershareMaxShares)
-FN_GLOBAL_CONST_STRING(lp_socket_options, &Globals.szSocketOptions)
-FN_GLOBAL_INTEGER(lp_config_backend, &Globals.ConfigBackend)
-FN_GLOBAL_INTEGER(lp_smb2_max_read, &Globals.ismb2_max_read)
-FN_GLOBAL_INTEGER(lp_smb2_max_write, &Globals.ismb2_max_write)
-FN_GLOBAL_INTEGER(lp_smb2_max_trans, &Globals.ismb2_max_trans)
+FN_GLOBAL_INTEGER(lp_minprotocol, minprotocol)
+FN_GLOBAL_INTEGER(lp_security, security)
+FN_GLOBAL_LIST(lp_auth_methods, AuthMethods)
+FN_GLOBAL_BOOL(lp_paranoid_server_security, paranoid_server_security)
+FN_GLOBAL_INTEGER(lp_maxdisksize, maxdisksize)
+FN_GLOBAL_INTEGER(lp_lpqcachetime, lpqcachetime)
+FN_GLOBAL_INTEGER(lp_max_smbd_processes, iMaxSmbdProcesses)
+FN_GLOBAL_BOOL(_lp_disable_spoolss, bDisableSpoolss)
+FN_GLOBAL_INTEGER(lp_syslog, syslog)
+FN_GLOBAL_INTEGER(lp_lm_announce, lm_announce)
+FN_GLOBAL_INTEGER(lp_lm_interval, lm_interval)
+FN_GLOBAL_INTEGER(lp_machine_password_timeout, machine_password_timeout)
+FN_GLOBAL_INTEGER(lp_map_to_guest, map_to_guest)
+FN_GLOBAL_INTEGER(lp_oplock_break_wait_time, oplock_break_wait_time)
+FN_GLOBAL_INTEGER(lp_lock_spin_time, iLockSpinTime)
+FN_GLOBAL_INTEGER(lp_usershare_max_shares, iUsershareMaxShares)
+FN_GLOBAL_CONST_STRING(lp_socket_options, szSocketOptions)
+FN_GLOBAL_INTEGER(lp_config_backend, ConfigBackend)
+FN_GLOBAL_INTEGER(lp_smb2_max_read, ismb2_max_read)
+FN_GLOBAL_INTEGER(lp_smb2_max_write, ismb2_max_write)
+FN_GLOBAL_INTEGER(lp_smb2_max_trans, ismb2_max_trans)
 int lp_smb2_max_credits(void)
 {
 	if (Globals.ismb2_max_credits == 0) {
@@ -5893,9 +5824,9 @@ FN_LOCAL_STRING(lp_username, szUsername)
 FN_LOCAL_LIST(lp_invalid_users, szInvalidUsers)
 FN_LOCAL_LIST(lp_valid_users, szValidUsers)
 FN_LOCAL_LIST(lp_admin_users, szAdminUsers)
-FN_GLOBAL_LIST(lp_svcctl_list, &Globals.szServicesList)
+FN_GLOBAL_LIST(lp_svcctl_list, szServicesList)
 FN_LOCAL_STRING(lp_cups_options, szCupsOptions)
-FN_GLOBAL_STRING(lp_cups_server, &Globals.szCupsServer)
+FN_GLOBAL_STRING(lp_cups_server, szCupsServer)
 int lp_cups_encrypt(void)
 {
 	int result = 0;
@@ -5914,13 +5845,13 @@ int lp_cups_encrypt(void)
 #endif
 	return result;
 }
-FN_GLOBAL_STRING(lp_iprint_server, &Globals.szIPrintServer)
-FN_GLOBAL_INTEGER(lp_cups_connection_timeout, &Globals.cups_connection_timeout)
-FN_GLOBAL_CONST_STRING(lp_ctdbd_socket, &Globals.ctdbdSocket)
-FN_GLOBAL_LIST(lp_cluster_addresses, &Globals.szClusterAddresses)
-FN_GLOBAL_BOOL(lp_clustering, &Globals.clustering)
-FN_GLOBAL_INTEGER(lp_ctdb_timeout, &Globals.ctdb_timeout)
-FN_GLOBAL_INTEGER(lp_ctdb_locktime_warn_threshold, &Globals.ctdb_locktime_warn_threshold)
+FN_GLOBAL_STRING(lp_iprint_server, szIPrintServer)
+FN_GLOBAL_INTEGER(lp_cups_connection_timeout, cups_connection_timeout)
+FN_GLOBAL_CONST_STRING(lp_ctdbd_socket, ctdbdSocket)
+FN_GLOBAL_LIST(lp_cluster_addresses, szClusterAddresses)
+FN_GLOBAL_BOOL(lp_clustering, clustering)
+FN_GLOBAL_INTEGER(lp_ctdb_timeout, ctdb_timeout)
+FN_GLOBAL_INTEGER(lp_ctdb_locktime_warn_threshold, ctdb_locktime_warn_threshold)
 FN_LOCAL_STRING(lp_printcommand, szPrintcommand)
 FN_LOCAL_STRING(lp_lpqcommand, szLpqcommand)
 FN_LOCAL_STRING(lp_lprmcommand, szLprmcommand)
@@ -5993,8 +5924,8 @@ FN_LOCAL_BOOL(lp_dos_filemode, bDosFilemode)
 FN_LOCAL_BOOL(lp_dos_filetimes, bDosFiletimes)
 FN_LOCAL_BOOL(lp_dos_filetime_resolution, bDosFiletimeResolution)
 FN_LOCAL_BOOL(lp_fake_dir_create_times, bFakeDirCreateTimes)
-FN_GLOBAL_BOOL(lp_async_smb_echo_handler, &Globals.bAsyncSMBEchoHandler)
-FN_GLOBAL_BOOL(lp_multicast_dns_register, &Globals.bMulticastDnsRegister)
+FN_GLOBAL_BOOL(lp_async_smb_echo_handler, bAsyncSMBEchoHandler)
+FN_GLOBAL_BOOL(lp_multicast_dns_register, bMulticastDnsRegister)
 FN_LOCAL_BOOL(lp_blocking_locks, bBlockingLocks)
 FN_LOCAL_BOOL(lp_inherit_perms, bInheritPerms)
 FN_LOCAL_BOOL(lp_inherit_acls, bInheritACLS)
@@ -6037,17 +5968,17 @@ FN_LOCAL_INTEGER(lp_map_readonly, iMap_readonly)
 FN_LOCAL_INTEGER(lp_directory_name_cache_size, iDirectoryNameCacheSize)
 FN_LOCAL_INTEGER(lp_smb_encrypt, ismb_encrypt)
 FN_LOCAL_CHAR(lp_magicchar, magic_char)
-FN_GLOBAL_INTEGER(lp_winbind_cache_time, &Globals.winbind_cache_time)
-FN_GLOBAL_INTEGER(lp_winbind_reconnect_delay, &Globals.winbind_reconnect_delay)
-FN_GLOBAL_INTEGER(lp_winbind_max_clients, &Globals.winbind_max_clients)
-FN_GLOBAL_LIST(lp_winbind_nss_info, &Globals.szWinbindNssInfo)
-FN_GLOBAL_INTEGER(lp_algorithmic_rid_base, &Globals.AlgorithmicRidBase)
-FN_GLOBAL_INTEGER(lp_name_cache_timeout, &Globals.name_cache_timeout)
-FN_GLOBAL_INTEGER(lp_client_signing, &Globals.client_signing)
-FN_GLOBAL_INTEGER(lp_server_signing, &Globals.server_signing)
-FN_GLOBAL_INTEGER(lp_client_ldap_sasl_wrapping, &Globals.client_ldap_sasl_wrapping)
-
-FN_GLOBAL_STRING(lp_ncalrpc_dir, &Globals.ncalrpc_dir)
+FN_GLOBAL_INTEGER(lp_winbind_cache_time, winbind_cache_time)
+FN_GLOBAL_INTEGER(lp_winbind_reconnect_delay, winbind_reconnect_delay)
+FN_GLOBAL_INTEGER(lp_winbind_max_clients, winbind_max_clients)
+FN_GLOBAL_LIST(lp_winbind_nss_info, szWinbindNssInfo)
+FN_GLOBAL_INTEGER(lp_algorithmic_rid_base, AlgorithmicRidBase)
+FN_GLOBAL_INTEGER(lp_name_cache_timeout, name_cache_timeout)
+FN_GLOBAL_INTEGER(lp_client_signing, client_signing)
+FN_GLOBAL_INTEGER(lp_server_signing, server_signing)
+FN_GLOBAL_INTEGER(lp_client_ldap_sasl_wrapping, client_ldap_sasl_wrapping)
+
+FN_GLOBAL_CONST_STRING(lp_ncalrpc_dir, ncalrpc_dir)
 
 /* local prototypes */
 
@@ -7132,10 +7063,14 @@ static void copy_service(struct service *pserviceDest, struct service *pserviceS
 					break;
 
 				case P_USTRING:
+				{
+					char *upper_string = strupper_talloc(talloc_tos(), 
+									     *(char **)src_ptr);
 					string_set((char **)dest_ptr,
-						   *(char **)src_ptr);
-					strupper_m(*(char **)dest_ptr);
+						   upper_string);
+					TALLOC_FREE(upper_string);
 					break;
+				}
 				case P_LIST:
 					TALLOC_FREE(*((char ***)dest_ptr));
 					*((char ***)dest_ptr) = str_list_copy(NULL, 
@@ -7208,14 +7143,14 @@ bool service_ok(int iService)
 
 static struct smbconf_ctx *lp_smbconf_ctx(void)
 {
-	WERROR werr;
+	sbcErr err;
 	static struct smbconf_ctx *conf_ctx = NULL;
 
 	if (conf_ctx == NULL) {
-		werr = smbconf_init(NULL, &conf_ctx, "registry:");
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_init(NULL, &conf_ctx, "registry:");
+		if (!SBC_ERROR_IS_OK(err)) {
 			DEBUG(1, ("error initializing registry configuration: "
-				  "%s\n", win_errstr(werr)));
+				  "%s\n", sbcErrorString(err)));
 			conf_ctx = NULL;
 		}
 	}
@@ -7255,7 +7190,7 @@ static bool process_smbconf_service(struct smbconf_service *service)
  */
 bool process_registry_service(const char *service_name)
 {
-	WERROR werr;
+	sbcErr err;
 	struct smbconf_service *service = NULL;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 	struct smbconf_ctx *conf_ctx = lp_smbconf_ctx();
@@ -7276,8 +7211,8 @@ bool process_registry_service(const char *service_name)
 		goto done;
 	}
 
-	werr = smbconf_get_share(conf_ctx, mem_ctx, service_name, &service);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_share(conf_ctx, mem_ctx, service_name, &service);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
@@ -7313,7 +7248,7 @@ static bool process_registry_globals(void)
 
 bool process_registry_shares(void)
 {
-	WERROR werr;
+	sbcErr err;
 	uint32_t count;
 	struct smbconf_service **service = NULL;
 	uint32_t num_shares = 0;
@@ -7325,8 +7260,8 @@ bool process_registry_shares(void)
 		goto done;
 	}
 
-	werr = smbconf_get_config(conf_ctx, mem_ctx, &num_shares, &service);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_config(conf_ctx, mem_ctx, &num_shares, &service);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
@@ -7499,27 +7434,18 @@ bool lp_file_list_changed(void)
 }
 
 
-/***************************************************************************
- Run standard_sub_basic on netbios name... needed because global_myname
- is not accessed through any lp_ macro.
- Note: We must *NOT* use string_set() here as ptr points to global_myname.
-***************************************************************************/
-
-static bool handle_netbios_name(int snum, const char *pszParmValue, char **ptr)
+/**
+ * Initialize iconv conversion descriptors.
+ *
+ * This is called the first time it is needed, and also called again
+ * every time the configuration is reloaded, because the charset or
+ * codepage might have changed.
+ **/
+static void init_iconv(void)
 {
-	bool ret;
-	char *netbios_name = talloc_sub_basic(
-		talloc_tos(), get_current_username(), current_user_info.domain,
-		pszParmValue);
-
-	ret = set_global_myname(netbios_name);
-	TALLOC_FREE(netbios_name);
-	string_set(&Globals.szNetbiosName,global_myname());
-
-	DEBUG(4, ("handle_netbios_name: set global_myname to: %s\n",
-	       global_myname()));
-
-	return ret;
+	global_iconv_handle = smb_iconv_handle_reinit(NULL, lp_dos_charset(),
+						      lp_unix_charset(),
+						      true, global_iconv_handle);
 }
 
 static bool handle_charset(int snum, const char *pszParmValue, char **ptr)
@@ -7531,24 +7457,54 @@ static bool handle_charset(int snum, const char *pszParmValue, char **ptr)
 	return True;
 }
 
-
-
-static bool handle_workgroup(int snum, const char *pszParmValue, char **ptr)
+static bool handle_dos_charset(int snum, const char *pszParmValue, char **ptr)
 {
-	bool ret;
+	bool is_utf8 = false;
+	size_t len = strlen(pszParmValue);
 
-	ret = set_global_myworkgroup(pszParmValue);
-	string_set(&Globals.szWorkgroup,lp_workgroup());
+	if (len == 4 || len == 5) {
+		/* Don't use StrCaseCmp here as we don't want to
+		   initialize iconv. */
+		if ((toupper_ascii(pszParmValue[0]) == 'U') &&
+		    (toupper_ascii(pszParmValue[1]) == 'T') &&
+		    (toupper_ascii(pszParmValue[2]) == 'F')) {
+			if (len == 4) {
+				if (pszParmValue[3] == '8') {
+					is_utf8 = true;
+				}
+			} else {
+				if (pszParmValue[3] == '-' &&
+				    pszParmValue[4] == '8') {
+					is_utf8 = true;
+				}
+			}
+		}
+	}
 
-	return ret;
+	if (strcmp(*ptr, pszParmValue) != 0) {
+		if (is_utf8) {
+			DEBUG(0,("ERROR: invalid DOS charset: 'dos charset' must not "
+				"be UTF8, using (default value) %s instead.\n",
+				DEFAULT_DOS_CHARSET));
+			pszParmValue = DEFAULT_DOS_CHARSET;
+		}
+		string_set(ptr, pszParmValue);
+		init_iconv();
+	}
+	return true;
 }
 
-static bool handle_netbios_scope(int snum, const char *pszParmValue, char **ptr)
+static bool handle_realm(int snum, const char *pszParmValue, char **ptr)
 {
-	bool ret;
+	bool ret = true;
+	char *realm = strupper_talloc(talloc_tos(), pszParmValue);
+	char *dnsdomain = strlower_talloc(talloc_tos(), pszParmValue);
 
-	ret = set_global_scope(pszParmValue);
-	string_set(&Globals.szNetbiosScope,global_scope());
+	ret &= string_set(&Globals.szRealm, pszParmValue);
+	ret &= string_set(&Globals.szRealmUpper, realm);
+	ret &= string_set(&Globals.szDnsDomain, dnsdomain);
+	TALLOC_FREE(realm);
+	TALLOC_FREE(dnsdomain);
 
 	return ret;
 }
@@ -7983,10 +7939,13 @@ bool lp_do_parameter(int snum, const char *pszParmName, const char *pszParmValue
 			break;
 
 		case P_USTRING:
-			string_set((char **)parm_ptr, pszParmValue);
-			strupper_m(*(char **)parm_ptr);
+		{
+			char *upper_string = strupper_talloc(talloc_tos(), 
+							     pszParmValue);
+			string_set((char **)parm_ptr, upper_string);
+			TALLOC_FREE(upper_string);
 			break;
-
+		}
 		case P_ENUM:
 			lp_set_enum_parm( &parm_table[parmnum], pszParmValue, (int*)parm_ptr );
 			break;
@@ -8610,7 +8569,8 @@ static void lp_add_auto_services(char *str)
  Auto-load one printer.
 ***************************************************************************/
 
-void lp_add_one_printer(const char *name, const char *comment, void *pdata)
+void lp_add_one_printer(const char *name, const char *comment,
+			const char *location, void *pdata)
 {
 	int printers = lp_servicenumber(PRINTERS_NAME);
 	int i;
@@ -8637,7 +8597,8 @@ bool lp_loaded(void)
  Unload unused services.
 ***************************************************************************/
 
-void lp_killunused(bool (*snumused) (int))
+void lp_killunused(struct smbd_server_connection *sconn,
+		   bool (*snumused) (struct smbd_server_connection *, int))
 {
 	int i;
 	for (i = 0; i < iNumServices; i++) {
@@ -8650,7 +8611,7 @@ void lp_killunused(bool (*snumused) (int))
 			continue;
 		}
 
-		if (!snumused || !snumused(i)) {
+		if (!snumused || !snumused(sconn, i)) {
 			free_service_byindex(i);
 		}
 	}
@@ -8661,7 +8622,7 @@ void lp_killunused(bool (*snumused) (int))
  */
 void lp_kill_all_services(void)
 {
-	lp_killunused(NULL);
+	lp_killunused(NULL, NULL);
 }
 
 /***************************************************************************
@@ -9275,7 +9236,7 @@ int load_usershare_service(const char *servicename)
  been removed.
 ***************************************************************************/
 
-int load_usershare_shares(void)
+int load_usershare_shares(struct smbd_server_connection *sconn)
 {
 	SMB_STRUCT_DIR *dp;
 	SMB_STRUCT_STAT sbuf;
@@ -9413,7 +9374,7 @@ int load_usershare_shares(void)
 	   not currently in use. */
 	for (iService = iNumServices - 1; iService >= 0; iService--) {
 		if (VALID(iService) && (ServicePtrs[iService]->usershare == USERSHARE_PENDING_DELETE)) {
-			if (conn_snum_used(iService)) {
+			if (conn_snum_used(sconn, iService)) {
 				continue;
 			}
 			/* Remove from the share ACL db. */
@@ -9579,6 +9540,17 @@ static bool lp_load_ex(const char *pszFname,
 	set_default_server_announce_type();
 	set_allowed_client_auth();
 
+	if (lp_security() == SEC_SHARE) {
+		DEBUG(1, ("WARNING: The security=share option is deprecated\n"));
+	} else if (lp_security() == SEC_SERVER) {
+		DEBUG(1, ("WARNING: The security=server option is deprecated\n"));
+	}
+
+	if (lp_security() == SEC_ADS && strchr(lp_passwordserver(), ':')) {
+		DEBUG(1, ("WARNING: The optional ':port' in password server = %s is deprecated\n",
+			  lp_passwordserver()));
+	}
+
 	bLoaded = True;
 
 	/* Now we check bWINSsupport and set szWINSserver to 127.0.0.1 */
@@ -9747,87 +9719,6 @@ bool share_defined(const char *service_name)
 	return (lp_servicenumber(service_name) != -1);
 }
 
-struct share_params *get_share_params(TALLOC_CTX *mem_ctx,
-				      const char *sharename)
-{
-	struct share_params *result;
-	char *sname = NULL;
-	int snum;
-
-	snum = find_service(mem_ctx, sharename, &sname);
-	if (snum < 0 || sname == NULL) {
-		return NULL;
-	}
-
-	if (!(result = TALLOC_P(mem_ctx, struct share_params))) {
-		DEBUG(0, ("talloc failed\n"));
-		return NULL;
-	}
-
-	result->service = snum;
-	return result;
-}
-
-struct share_iterator *share_list_all(TALLOC_CTX *mem_ctx)
-{
-	struct share_iterator *result;
-
-	if (!(result = TALLOC_P(mem_ctx, struct share_iterator))) {
-		DEBUG(0, ("talloc failed\n"));
-		return NULL;
-	}
-
-	result->next_id = 0;
-	return result;
-}
-
-struct share_params *next_share(struct share_iterator *list)
-{
-	struct share_params *result;
-
-	while (!lp_snum_ok(list->next_id) &&
-	       (list->next_id < lp_numservices())) {
-		list->next_id += 1;
-	}
-
-	if (list->next_id >= lp_numservices()) {
-		return NULL;
-	}
-
-	if (!(result = TALLOC_P(list, struct share_params))) {
-		DEBUG(0, ("talloc failed\n"));
-		return NULL;
-	}
-
-	result->service = list->next_id;
-	list->next_id += 1;
-	return result;
-}
-
-struct share_params *next_printer(struct share_iterator *list)
-{
-	struct share_params *result;
-
-	while ((result = next_share(list)) != NULL) {
-		if (lp_print_ok(result->service)) {
-			break;
-		}
-	}
-	return result;
-}
-
-/*
- * This is a hack for a transition period until we transformed all code from
- * service numbers to struct share_params.
- */
-
-struct share_params *snum2params_static(int snum)
-{
-	static struct share_params result;
-	result.service = snum;
-	return &result;
-}
-
 /*******************************************************************
  A useful volume label function. 
 ********************************************************************/
@@ -9865,22 +9756,8 @@ static void set_default_server_announce_type(void)
 
 	default_server_announce |= SV_TYPE_PRINTQ_SERVER;
 
-	switch (lp_announce_as()) {
-		case ANNOUNCE_AS_NT_SERVER:
-			default_server_announce |= SV_TYPE_SERVER_NT;
-			/* fall through... */
-		case ANNOUNCE_AS_NT_WORKSTATION:
-			default_server_announce |= SV_TYPE_NT;
-			break;
-		case ANNOUNCE_AS_WIN95:
-			default_server_announce |= SV_TYPE_WIN95_PLUS;
-			break;
-		case ANNOUNCE_AS_WFW:
-			default_server_announce |= SV_TYPE_WFW;
-			break;
-		default:
-			break;
-	}
+	default_server_announce |= SV_TYPE_SERVER_NT;
+	default_server_announce |= SV_TYPE_NT;
 
 	switch (lp_server_role()) {
 		case ROLE_DOMAIN_MEMBER:
@@ -9973,54 +9850,6 @@ int lp_default_server_announce(void)
 	return default_server_announce;
 }
 
-/*******************************************************************
- Split the announce version into major and minor numbers.
-********************************************************************/
-
-int lp_major_announce_version(void)
-{
-	static bool got_major = False;
-	static int major_version = DEFAULT_MAJOR_VERSION;
-	char *vers;
-	char *p;
-
-	if (got_major)
-		return major_version;
-
-	got_major = True;
-	if ((vers = lp_announce_version()) == NULL)
-		return major_version;
-
-	if ((p = strchr_m(vers, '.')) == 0)
-		return major_version;
-
-	*p = '\0';
-	major_version = atoi(vers);
-	return major_version;
-}
-
-int lp_minor_announce_version(void)
-{
-	static bool got_minor = False;
-	static int minor_version = DEFAULT_MINOR_VERSION;
-	char *vers;
-	char *p;
-
-	if (got_minor)
-		return minor_version;
-
-	got_minor = True;
-	if ((vers = lp_announce_version()) == NULL)
-		return minor_version;
-
-	if ((p = strchr_m(vers, '.')) == 0)
-		return minor_version;
-
-	p++;
-	minor_version = atoi(p);
-	return minor_version;
-}
-
 /***********************************************************
  Set the global name resolution order (used in smbclient).
 ************************************************************/
diff --git a/source3/param/loadparm_ctx.c b/source3/param/loadparm_ctx.c
new file mode 100644
index 00000000000..e80f6f18444
--- /dev/null
+++ b/source3/param/loadparm_ctx.c
@@ -0,0 +1,68 @@
+/* 
+   Unix SMB/CIFS implementation.
+   Parameter loading functions
+   Copyright (C) Andrew Bartlett 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "../source4/param/s3_param.h"
+
+static const char *get_parametric(const char *type, const char *option)
+{
+	return lp_parm_const_string(-1, type, option, NULL);
+}
+
+/* These are in the order that they appear in the s4 loadparm file.
+ * All of the s4 loadparm functions should be here eventually, once
+ * they are implemented in the s3 loadparm, have the same format (enum
+ * values in particular) and defaults. */
+static const struct loadparm_s3_context s3_fns = 
+{
+	.get_parametric = get_parametric,
+
+	.server_role = lp_server_role,
+
+	.winbind_separator = lp_winbind_separator,
+	.template_homedir = lp_template_homedir,
+	.template_shell = lp_template_shell,
+
+	.dos_charset = lp_dos_charset,
+	.unix_charset = lp_unix_charset,
+
+	.realm = lp_realm,
+	.dnsdomain = lp_dnsdomain,
+	.socket_options = lp_socket_options,
+	.workgroup = lp_workgroup,
+
+	.netbios_name = lp_netbios_name,
+	.netbios_scope = lp_netbios_scope,
+
+	.lanman_auth = lp_lanman_auth,
+	.ntlm_auth = lp_ntlm_auth,
+
+	.client_plaintext_auth = lp_client_plaintext_auth,
+	.client_lanman_auth = lp_client_lanman_auth,
+	.client_ntlmv2_auth = lp_client_ntlmv2_auth,
+
+	.private_dir = lp_private_dir,
+	.ncalrpc_dir = lp_ncalrpc_dir,
+	.lockdir = lp_lockdir
+};
+
+const struct loadparm_s3_context *loadparm_s3_context(void)
+{
+	return &s3_fns;
+}
diff --git a/source3/param/service.c b/source3/param/service.c
new file mode 100644
index 00000000000..358b7af2dee
--- /dev/null
+++ b/source3/param/service.c
@@ -0,0 +1,276 @@
+/*
+   Unix SMB/CIFS implementation.
+   service (connection) opening and closing
+   Copyright (C) Andrew Tridgell 1992-1998
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "system/filesys.h"
+#include "../lib/tsocket/tsocket.h"
+#include "smbd/smbd.h"
+#include "smbd/globals.h"
+#include "../librpc/gen_ndr/netlogon.h"
+#include "../libcli/security/security.h"
+#include "printing/pcap.h"
+#include "passdb/lookup_sid.h"
+#include "auth.h"
+
+static int load_registry_service(const char *servicename)
+{
+	if (!lp_registry_shares()) {
+		return -1;
+	}
+
+	if ((servicename == NULL) || (*servicename == '\0')) {
+		return -1;
+	}
+
+	if (strequal(servicename, GLOBAL_NAME)) {
+		return -2;
+	}
+
+	if (!process_registry_service(servicename)) {
+		return -1;
+	}
+
+	return lp_servicenumber(servicename);
+}
+
+void load_registry_shares(void)
+{
+	DEBUG(8, ("load_registry_shares()\n"));
+	if (!lp_registry_shares()) {
+		return;
+	}
+
+	process_registry_shares();
+
+	return;
+}
+
+/****************************************************************************
+ Add a home service. Returns the new service number or -1 if fail.
+****************************************************************************/
+
+int add_home_service(const char *service, const char *username, const char *homedir)
+{
+	int iHomeService;
+
+	if (!service || !homedir || homedir[0] == '\0')
+		return -1;
+
+	if ((iHomeService = lp_servicenumber(HOMES_NAME)) < 0) {
+		if ((iHomeService = load_registry_service(HOMES_NAME)) < 0) {
+			return -1;
+		}
+	}
+
+	/*
+	 * If this is a winbindd provided username, remove
+	 * the domain component before adding the service.
+	 * Log a warning if the "path=" parameter does not
+	 * include any macros.
+	 */
+
+	{
+		const char *p = strchr(service,*lp_winbind_separator());
+
+		/* We only want the 'user' part of the string */
+		if (p) {
+			service = p + 1;
+		}
+	}
+
+	if (!lp_add_home(service, iHomeService, username, homedir)) {
+		return -1;
+	}
+
+	return lp_servicenumber(service);
+
+}
+
+/**
+ * Find a service entry.
+ *
+ * @param service is modified (to canonical form??)
+ **/
+
+int find_service(TALLOC_CTX *ctx, const char *service_in, char **p_service_out)
+{
+	int iService;
+
+	if (!service_in) {
+		return -1;
+	}
+
+	/* First make a copy. */
+	*p_service_out = talloc_strdup(ctx, service_in);
+	if (!*p_service_out) {
+		return -1;
+	}
+
+	all_string_sub(*p_service_out,"\\","/",0);
+
+	iService = lp_servicenumber(*p_service_out);
+
+	/* now handle the special case of a home directory */
+	if (iService < 0) {
+		char *phome_dir = get_user_home_dir(ctx, *p_service_out);
+
+		if(!phome_dir) {
+			/*
+			 * Try mapping the servicename, it may
+			 * be a Windows to unix mapped user name.
+			 */
+			if(map_username(ctx, *p_service_out, p_service_out)) {
+				if (*p_service_out == NULL) {
+					/* Out of memory. */
+					return -1;
+				}
+				phome_dir = get_user_home_dir(
+						ctx, *p_service_out);
+			}
+		}
+
+		DEBUG(3,("checking for home directory %s gave %s\n",*p_service_out,
+			phome_dir?phome_dir:"(NULL)"));
+
+		iService = add_home_service(*p_service_out,*p_service_out /* 'username' */, phome_dir);
+	}
+
+	/* If we still don't have a service, attempt to add it as a printer. */
+	if (iService < 0) {
+		int iPrinterService;
+
+		if ((iPrinterService = lp_servicenumber(PRINTERS_NAME)) < 0) {
+			iPrinterService = load_registry_service(PRINTERS_NAME);
+		}
+		if (iPrinterService >= 0) {
+			DEBUG(3,("checking whether %s is a valid printer name...\n",
+				*p_service_out));
+			if (pcap_printername_ok(*p_service_out)) {
+				DEBUG(3,("%s is a valid printer name\n",
+					*p_service_out));
+				DEBUG(3,("adding %s as a printer service\n",
+					*p_service_out));
+				lp_add_printer(*p_service_out, iPrinterService);
+				iService = lp_servicenumber(*p_service_out);
+				if (iService < 0) {
+					DEBUG(0,("failed to add %s as a printer service!\n",
+						*p_service_out));
+				}
+			} else {
+				DEBUG(3,("%s is not a valid printer name\n",
+					*p_service_out));
+			}
+		}
+	}
+
+	/* Check for default vfs service?  Unsure whether to implement this */
+	if (iService < 0) {
+	}
+
+	if (iService < 0) {
+		iService = load_registry_service(*p_service_out);
+	}
+
+	/* Is it a usershare service ? */
+	if (iService < 0 && *lp_usershare_path()) {
+		/* Ensure the name is canonicalized. */
+		strlower_m(*p_service_out);
+		iService = load_usershare_service(*p_service_out);
+	}
+
+	/* just possibly it's a default service? */
+	if (iService < 0) {
+		char *pdefservice = lp_defaultservice();
+		if (pdefservice &&
+				*pdefservice &&
+				!strequal(pdefservice, *p_service_out)
+				&& !strstr_m(*p_service_out,"..")) {
+			/*
+			 * We need to do a local copy here as lp_defaultservice()
+			 * returns one of the rotating lp_string buffers that
+			 * could get overwritten by the recursive find_service() call
+			 * below. Fix from Josef Hinteregger <joehtg@joehtg.co.at>.
+			 */
+			char *defservice = talloc_strdup(ctx, pdefservice);
+
+			if (!defservice) {
+				goto fail;
+			}
+
+			/* Disallow anything except explicit share names. */
+			if (strequal(defservice,HOMES_NAME) ||
+					strequal(defservice, PRINTERS_NAME) ||
+					strequal(defservice, "IPC$")) {
+				TALLOC_FREE(defservice);
+				goto fail;
+			}
+
+			iService = find_service(ctx, defservice, p_service_out);
+			if (!*p_service_out) {
+				TALLOC_FREE(defservice);
+				iService = -1;
+				goto fail;
+			}
+			if (iService >= 0) {
+				all_string_sub(*p_service_out, "_","/",0);
+				iService = lp_add_service(*p_service_out, iService);
+			}
+			TALLOC_FREE(defservice);
+		}
+	}
+
+	if (iService >= 0) {
+		if (!VALID_SNUM(iService)) {
+			DEBUG(0,("Invalid snum %d for %s\n",iService,
+				*p_service_out));
+			iService = -1;
+		}
+	}
+
+  fail:
+
+	if (iService < 0) {
+		DEBUG(3,("find_service() failed to find service %s\n",
+			*p_service_out));
+	}
+
+	return (iService);
+}
+
+
+struct share_params *get_share_params(TALLOC_CTX *mem_ctx,
+				      const char *sharename)
+{
+	struct share_params *result;
+	char *sname = NULL;
+	int snum;
+
+	snum = find_service(mem_ctx, sharename, &sname);
+	if (snum < 0 || sname == NULL) {
+		return NULL;
+	}
+
+	if (!(result = talloc(mem_ctx, struct share_params))) {
+		DEBUG(0, ("talloc failed\n"));
+		return NULL;
+	}
+
+	result->service = snum;
+	return result;
+}
diff --git a/source3/passdb/login_cache.c b/source3/passdb/login_cache.c
index 1785a4dacac..a53fe5b24f7 100644
--- a/source3/passdb/login_cache.c
+++ b/source3/passdb/login_cache.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "passdb.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_PASSDB
@@ -58,10 +59,10 @@ bool login_cache_init(void)
 
 bool login_cache_shutdown(void)
 {
-	/* tdb_close routine returns -1 on error */
+	/* tdb_close routine returns non-zero on error */
 	if (!cache) return False;
 	DEBUG(5, ("Closing cache file\n"));
-	return tdb_close(cache) != -1;
+	return tdb_close(cache) == 0;
 }
 
 /* if we can't read the cache, oh well, no need to return anything */
diff --git a/source3/passdb/lookup_sid.c b/source3/passdb/lookup_sid.c
index 5cf391f11c7..8b5330c0974 100644
--- a/source3/passdb/lookup_sid.c
+++ b/source3/passdb/lookup_sid.c
@@ -482,8 +482,8 @@ static bool lookup_rids(TALLOC_CTX *mem_ctx, const struct dom_sid *domain_sid,
 		   sid_string_dbg(domain_sid)));
 
 	if (num_rids) {
-		*names = TALLOC_ZERO_ARRAY(mem_ctx, const char *, num_rids);
-		*types = TALLOC_ARRAY(mem_ctx, enum lsa_SidType, num_rids);
+		*names = talloc_zero_array(mem_ctx, const char *, num_rids);
+		*types = talloc_array(mem_ctx, enum lsa_SidType, num_rids);
 
 		if ((*names == NULL) || (*types == NULL)) {
 			return false;
@@ -750,7 +750,7 @@ NTSTATUS lookup_sids(TALLOC_CTX *mem_ctx, int num_sids,
 	}
 
 	if (num_sids) {
-		name_infos = TALLOC_ARRAY(mem_ctx, struct lsa_name_info, num_sids);
+		name_infos = talloc_array(mem_ctx, struct lsa_name_info, num_sids);
 		if (name_infos == NULL) {
 			result = NT_STATUS_NO_MEMORY;
 			goto fail;
@@ -759,7 +759,7 @@ NTSTATUS lookup_sids(TALLOC_CTX *mem_ctx, int num_sids,
 		name_infos = NULL;
 	}
 
-	dom_infos = TALLOC_ZERO_ARRAY(mem_ctx, struct lsa_dom_info,
+	dom_infos = talloc_zero_array(mem_ctx, struct lsa_dom_info,
 				      LSA_REF_DOMAIN_LIST_MULTIPLIER);
 	if (dom_infos == NULL) {
 		result = NT_STATUS_NO_MEMORY;
@@ -896,7 +896,7 @@ NTSTATUS lookup_sids(TALLOC_CTX *mem_ctx, int num_sids,
 		}
 
 		if (dom->num_idxs) {
-			if (!(rids = TALLOC_ARRAY(tmp_ctx, uint32, dom->num_idxs))) {
+			if (!(rids = talloc_array(tmp_ctx, uint32, dom->num_idxs))) {
 				result = NT_STATUS_NO_MEMORY;
 				goto fail;
 			}
@@ -1404,7 +1404,7 @@ bool sids_to_unix_ids(const struct dom_sid *sids, uint32_t num_sids,
 	wbcErr err;
 	bool ret = false;
 
-	wbc_sids = TALLOC_ARRAY(talloc_tos(), struct wbcDomainSid, num_sids);
+	wbc_sids = talloc_array(talloc_tos(), struct wbcDomainSid, num_sids);
 	if (wbc_sids == NULL) {
 		return false;
 	}
@@ -1455,7 +1455,7 @@ bool sids_to_unix_ids(const struct dom_sid *sids, uint32_t num_sids,
 	if (num_not_cached == 0) {
 		goto done;
 	}
-	wbc_ids = TALLOC_ARRAY(talloc_tos(), struct wbcUnixId, num_not_cached);
+	wbc_ids = talloc_array(talloc_tos(), struct wbcUnixId, num_not_cached);
 	if (wbc_ids == NULL) {
 		goto fail;
 	}
diff --git a/source3/passdb/machine_account_secrets.c b/source3/passdb/machine_account_secrets.c
index a51a99c9a2f..665e2f89a45 100644
--- a/source3/passdb/machine_account_secrets.c
+++ b/source3/passdb/machine_account_secrets.c
@@ -28,6 +28,7 @@
 #include "secrets.h"
 #include "dbwrap.h"
 #include "../librpc/ndr/libndr.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_PASSDB
@@ -56,6 +57,17 @@ bool secrets_store_domain_sid(const char *domain, const struct dom_sid  *sid)
 {
 	bool ret;
 
+#ifdef _SAMBA_WAF_BUILD_
+	if (strequal(domain, get_global_sam_name()) &&
+	    (pdb_capabilities() & PDB_CAP_ADS)) {
+		/* If we have a ADS-capable passdb backend, we
+		 * must never make up our own SID, it will
+		 * already be in the directory */
+		DEBUG(0, ("Refusing to store a Domain SID, this should be read from the directory not stored here\n"));
+		return false;
+	}
+#endif
+
 	ret = secrets_store(domain_sid_keystr(domain), sid, sizeof(struct dom_sid ));
 
 	/* Force a re-query, in case we modified our domain */
@@ -69,6 +81,24 @@ bool secrets_fetch_domain_sid(const char *domain, struct dom_sid  *sid)
 	struct dom_sid  *dyn_sid;
 	size_t size = 0;
 
+#ifdef _SAMBA_WAF_BUILD_
+	if (strequal(domain, get_global_sam_name()) &&
+	    (pdb_capabilities() & PDB_CAP_ADS)) {
+		struct pdb_domain_info *domain_info;
+		domain_info = pdb_get_domain_info(talloc_tos());
+		if (!domain_info) {
+			/* If we have a ADS-capable passdb backend, we
+			 * must never make up our own SID, it will
+			 * already be in the directory */
+			DEBUG(0, ("Unable to fetch a Domain SID from the directory!\n"));
+			return false;
+		}
+
+		*sid = domain_info->sid;
+		return true;
+	}
+#endif
+
 	dyn_sid = (struct dom_sid  *)secrets_fetch(domain_sid_keystr(domain), &size);
 
 	if (dyn_sid == NULL)
@@ -88,6 +118,17 @@ bool secrets_store_domain_guid(const char *domain, struct GUID *guid)
 {
 	fstring key;
 
+#ifdef _SAMBA_WAF_BUILD_
+	if (strequal(domain, get_global_sam_name()) &&
+	    (pdb_capabilities() & PDB_CAP_ADS)) {
+		/* If we have a ADS-capable passdb backend, we
+		 * must never make up our own GUID, it will
+		 * already be in the directory */
+		DEBUG(0, ("Refusing to store a Domain GUID, this should be read from the directory not stored here\n"));
+		return false;
+	}
+#endif
+
 	slprintf(key, sizeof(key)-1, "%s/%s", SECRETS_DOMAIN_GUID, domain);
 	strupper_m(key);
 	return secrets_store(key, guid, sizeof(struct GUID));
@@ -100,6 +141,24 @@ bool secrets_fetch_domain_guid(const char *domain, struct GUID *guid)
 	size_t size = 0;
 	struct GUID new_guid;
 
+#ifdef _SAMBA_WAF_BUILD_
+	if (strequal(domain, get_global_sam_name()) &&
+	    (pdb_capabilities() & PDB_CAP_ADS)) {
+		struct pdb_domain_info *domain_info;
+		domain_info = pdb_get_domain_info(talloc_tos());
+		if (!domain_info) {
+			/* If we have a ADS-capable passdb backend, we
+			 * must never make up our own SID, it will
+			 * already be in the directory */
+			DEBUG(0, ("Unable to fetch a Domain GUID from the directory!\n"));
+			return false;
+		}
+
+		*guid = domain_info->guid;
+		return true;
+	}
+#endif
+
 	slprintf(key, sizeof(key)-1, "%s/%s", SECRETS_DOMAIN_GUID, domain);
 	strupper_m(key);
 	dyn_guid = (struct GUID *)secrets_fetch(key, &size);
diff --git a/source3/passdb/machine_sid.c b/source3/passdb/machine_sid.c
index 8fd51031648..6d4cf237c77 100644
--- a/source3/passdb/machine_sid.c
+++ b/source3/passdb/machine_sid.c
@@ -97,7 +97,7 @@ static struct dom_sid *pdb_generate_sam_sid(void)
 		}
 	}
 
-	if (secrets_fetch_domain_sid(global_myname(), sam_sid)) {
+	if (secrets_fetch_domain_sid(lp_netbios_name(), sam_sid)) {
 
 		/* We got our sid. If not a pdc/bdc, we're done. */
 		if ( !IS_DC )
@@ -120,7 +120,7 @@ static struct dom_sid *pdb_generate_sam_sid(void)
 			/* Domain name sid doesn't match global sam sid. Re-store domain sid as 'local' sid. */
 
 			DEBUG(0,("pdb_generate_sam_sid: Mismatched SIDs as a pdc/bdc.\n"));
-			if (!secrets_store_domain_sid(global_myname(), &domain_sid)) {
+			if (!secrets_store_domain_sid(lp_netbios_name(), &domain_sid)) {
 				DEBUG(0,("pdb_generate_sam_sid: Can't re-store domain SID for local sid as PDC/BDC.\n"));
 				SAFE_FREE(sam_sid);
 				return NULL;
@@ -139,7 +139,7 @@ static struct dom_sid *pdb_generate_sam_sid(void)
 
 	if (read_sid_from_file(fname, sam_sid)) {
 		/* remember it for future reference and unlink the old MACHINE.SID */
-		if (!secrets_store_domain_sid(global_myname(), sam_sid)) {
+		if (!secrets_store_domain_sid(lp_netbios_name(), sam_sid)) {
 			DEBUG(0,("pdb_generate_sam_sid: Failed to store SID from file.\n"));
 			SAFE_FREE(fname);
 			SAFE_FREE(sam_sid);
@@ -166,7 +166,7 @@ static struct dom_sid *pdb_generate_sam_sid(void)
            generate one and save it */
 	generate_random_sid(sam_sid);
 
-	if (!secrets_store_domain_sid(global_myname(), sam_sid)) {
+	if (!secrets_store_domain_sid(lp_netbios_name(), sam_sid)) {
 		DEBUG(0,("pdb_generate_sam_sid: Failed to store generated machine SID.\n"));
 		SAFE_FREE(sam_sid);
 		return NULL;
diff --git a/source3/passdb/machine_sid.h b/source3/passdb/machine_sid.h
index 2da281ac34c..12fde2f3ac6 100644
--- a/source3/passdb/machine_sid.h
+++ b/source3/passdb/machine_sid.h
@@ -1,3 +1,24 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Password and authentication handling
+ *  Copyright (C) Jeremy Allison 		1996-2002
+ *  Copyright (C) Andrew Tridgell		2002
+ *  Copyright (C) Gerald (Jerry) Carter		2000
+ *  Copyright (C) Stefan (metze) Metzmacher	2002
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
 /* The following definitions come from passdb/machine_sid.c  */
 
diff --git a/source3/passdb/passdb.c b/source3/passdb/passdb.c
index 62af5be7eb4..5116396cd1b 100644
--- a/source3/passdb/passdb.c
+++ b/source3/passdb/passdb.c
@@ -29,6 +29,7 @@
 #include "secrets.h"
 #include "../libcli/security/security.h"
 #include "../lib/util/util_pw.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_PASSDB
@@ -45,7 +46,7 @@ const char *my_sam_name(void)
 {
        /* Standalone servers can only use the local netbios name */
        if ( lp_server_role() == ROLE_STANDALONE )
-               return global_myname();
+               return lp_netbios_name();
 
        /* Default to the DOMAIN name when not specified */
        return lp_workgroup();
@@ -73,7 +74,7 @@ struct samu *samu_new( TALLOC_CTX *ctx )
 {
 	struct samu *user;
 
-	if ( !(user = TALLOC_ZERO_P( ctx, struct samu )) ) {
+	if ( !(user = talloc_zero( ctx, struct samu )) ) {
 		DEBUG(0,("samuser_new: Talloc failed!\n"));
 		return NULL;
 	}
@@ -148,7 +149,7 @@ static int count_commas(const char *str)
 static NTSTATUS samu_set_unix_internal(struct samu *user, const struct passwd *pwd, bool create)
 {
 	const char *guest_account = lp_guestaccount();
-	const char *domain = global_myname();
+	const char *domain = lp_netbios_name();
 	char *fullname;
 	uint32_t urid;
 
@@ -384,9 +385,9 @@ void pdb_sethexpwd(char p[33], const unsigned char *pwd, uint32_t acct_ctrl)
 			slprintf(&p[i*2], 3, "%02X", pwd[i]);
 	} else {
 		if (acct_ctrl & ACB_PWNOTREQ)
-			safe_strcpy(p, "NO PASSWORDXXXXXXXXXXXXXXXXXXXXX", 32);
+			strlcpy(p, "NO PASSWORDXXXXXXXXXXXXXXXXXXXXX", 33);
 		else
-			safe_strcpy(p, "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", 32);
+			strlcpy(p, "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", 33);
 	}
 }
 
@@ -435,7 +436,7 @@ void pdb_sethexhours(char *p, const unsigned char *hours)
 			slprintf(&p[i*2], 3, "%02X", hours[i]);
 		}
 	} else {
-		safe_strcpy(p, "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 43);
+		strlcpy(p, "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 44);
 	}
 }
 
@@ -2354,7 +2355,7 @@ bool get_trust_pw_clear(const char *domain, char **ret_pwd,
 	if (pwd != NULL) {
 		*ret_pwd = pwd;
 		if (account_name != NULL) {
-			*account_name = global_myname();
+			*account_name = lp_netbios_name();
 		}
 
 		return true;
@@ -2392,7 +2393,7 @@ bool get_trust_pw_hash(const char *domain, uint8_t ret_pwd[16],
 							channel))
 	{
 		if (account_name != NULL) {
-			*account_name = global_myname();
+			*account_name = lp_netbios_name();
 		}
 
 		return true;
diff --git a/source3/passdb/pdb_get_set.c b/source3/passdb/pdb_get_set.c
index 6c1a2ab23d7..eec82f9c984 100644
--- a/source3/passdb/pdb_get_set.c
+++ b/source3/passdb/pdb_get_set.c
@@ -123,8 +123,7 @@ time_t pdb_get_pass_must_change_time(const struct samu *sampass)
 
 bool pdb_get_pass_can_change(const struct samu *sampass)
 {
-	if (sampass->pass_can_change_time == get_time_t_max() &&
-	    sampass->pass_last_set_time != 0)
+	if (sampass->pass_can_change_time == get_time_t_max())
 		return False;
 	return True;
 }
@@ -512,7 +511,7 @@ bool pdb_set_group_sid(struct samu *sampass, const struct dom_sid *g_sid, enum p
 	if (!g_sid)
 		return False;
 
-	if ( !(sampass->group_sid = TALLOC_P( sampass, struct dom_sid )) ) {
+	if ( !(sampass->group_sid = talloc( sampass, struct dom_sid )) ) {
 		return False;
 	}
 
diff --git a/source3/passdb/pdb_interface.c b/source3/passdb/pdb_interface.c
index 36d5aaa01a7..94ed355e2c5 100644
--- a/source3/passdb/pdb_interface.c
+++ b/source3/passdb/pdb_interface.c
@@ -25,6 +25,8 @@
 #include "passdb.h"
 #include "secrets.h"
 #include "../librpc/gen_ndr/samr.h"
+#include "../librpc/gen_ndr/drsblobs.h"
+#include "../librpc/gen_ndr/ndr_drsblobs.h"
 #include "memcache.h"
 #include "nsswitch/winbind_client.h"
 #include "../libcli/security/security.h"
@@ -1518,7 +1520,7 @@ static NTSTATUS pdb_default_enum_group_members(struct pdb_methods *methods,
 	if (num_uids == 0)
 		return NT_STATUS_OK;
 
-	*pp_member_rids = TALLOC_ZERO_ARRAY(mem_ctx, uint32_t, num_uids);
+	*pp_member_rids = talloc_zero_array(mem_ctx, uint32_t, num_uids);
 
 	for (i=0; i<num_uids; i++) {
 		struct dom_sid sid;
@@ -1570,7 +1572,7 @@ static NTSTATUS pdb_default_enum_group_memberships(struct pdb_methods *methods,
 		smb_panic("primary group missing");
 	}
 
-	*pp_sids = TALLOC_ARRAY(mem_ctx, struct dom_sid, *p_num_groups);
+	*pp_sids = talloc_array(mem_ctx, struct dom_sid, *p_num_groups);
 
 	if (*pp_sids == NULL) {
 		TALLOC_FREE(*pp_gids);
@@ -2144,7 +2146,62 @@ static NTSTATUS pdb_default_get_trusted_domain(struct pdb_methods *methods,
 					       const char *domain,
 					       struct pdb_trusted_domain **td)
 {
-	return NT_STATUS_NOT_IMPLEMENTED;
+	struct trustAuthInOutBlob taiob;
+	struct AuthenticationInformation aia;
+	struct pdb_trusted_domain *tdom;
+	enum ndr_err_code ndr_err;
+	time_t last_set_time;
+	char *pwd;
+	bool ok;
+
+	tdom = talloc(mem_ctx, struct pdb_trusted_domain);
+	if (!tdom) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	tdom->domain_name = talloc_strdup(tdom, domain);
+	tdom->netbios_name = talloc_strdup(tdom, domain);
+	if (!tdom->domain_name || !tdom->netbios_name) {
+		talloc_free(tdom);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	tdom->trust_auth_incoming = data_blob_null;
+
+	ok = pdb_get_trusteddom_pw(domain, &pwd, &tdom->security_identifier,
+				   &last_set_time);
+	if (!ok) {
+		talloc_free(tdom);
+		return NT_STATUS_UNSUCCESSFUL;
+	}
+
+	ZERO_STRUCT(taiob);
+	ZERO_STRUCT(aia);
+	taiob.count = 1;
+	taiob.current.count = 1;
+	taiob.current.array = &aia;
+	unix_to_nt_time(&aia.LastUpdateTime, last_set_time);
+	aia.AuthType = TRUST_AUTH_TYPE_CLEAR;
+	aia.AuthInfo.clear.password = (uint8_t *) pwd;
+	aia.AuthInfo.clear.size = strlen(pwd);
+	taiob.previous.count = 0;
+	taiob.previous.array = NULL;
+
+	ndr_err = ndr_push_struct_blob(&tdom->trust_auth_outgoing,
+					tdom, &taiob,
+			(ndr_push_flags_fn_t)ndr_push_trustAuthInOutBlob);
+	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+		talloc_free(tdom);
+		return NT_STATUS_UNSUCCESSFUL;
+	}
+
+	tdom->trust_direction = LSA_TRUST_DIRECTION_OUTBOUND;
+	tdom->trust_type = LSA_TRUST_TYPE_DOWNLEVEL;
+	tdom->trust_attributes = 0;
+	tdom->trust_forest_trust_info = data_blob_null;
+
+	*td = tdom;
+	return NT_STATUS_OK;
 }
 
 static NTSTATUS pdb_default_get_trusted_domain_by_sid(struct pdb_methods *methods,
@@ -2155,11 +2212,54 @@ static NTSTATUS pdb_default_get_trusted_domain_by_sid(struct pdb_methods *method
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
 
+#define IS_NULL_DATA_BLOB(d) ((d).data == NULL && (d).length == 0)
+
 static NTSTATUS pdb_default_set_trusted_domain(struct pdb_methods *methods,
 					       const char* domain,
 					       const struct pdb_trusted_domain *td)
 {
-	return NT_STATUS_NOT_IMPLEMENTED;
+	struct trustAuthInOutBlob taiob;
+	struct AuthenticationInformation *aia;
+	enum ndr_err_code ndr_err;
+	char *pwd;
+	bool ok;
+
+	if (td->trust_attributes != 0 ||
+	    td->trust_type != LSA_TRUST_TYPE_DOWNLEVEL ||
+	    td->trust_direction != LSA_TRUST_DIRECTION_OUTBOUND ||
+	    !IS_NULL_DATA_BLOB(td->trust_auth_incoming) ||
+	    !IS_NULL_DATA_BLOB(td->trust_forest_trust_info)) {
+	    return NT_STATUS_NOT_IMPLEMENTED;
+	}
+
+	ZERO_STRUCT(taiob);
+	ndr_err = ndr_pull_struct_blob(&td->trust_auth_outgoing, talloc_tos(),
+			      &taiob,
+			      (ndr_pull_flags_fn_t)ndr_pull_trustAuthInOutBlob);
+	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+		return NT_STATUS_UNSUCCESSFUL;
+	}
+
+	aia = (struct AuthenticationInformation *) taiob.current.array;
+
+	if (taiob.count != 1 || taiob.current.count != 1 ||
+	    taiob.previous.count != 0 ||
+	    aia->AuthType != TRUST_AUTH_TYPE_CLEAR) {
+	    return NT_STATUS_NOT_IMPLEMENTED;
+	}
+
+	pwd = talloc_strndup(talloc_tos(), (char *) aia->AuthInfo.clear.password,
+			     aia->AuthInfo.clear.size);
+	if (!pwd) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	ok = pdb_set_trusteddom_pw(domain, pwd, &td->security_identifier);
+	if (!ok) {
+		return NT_STATUS_UNSUCCESSFUL;
+	}
+
+	return NT_STATUS_OK;
 }
 
 static NTSTATUS pdb_default_del_trusted_domain(struct pdb_methods *methods,
diff --git a/source3/passdb/pdb_ipa.c b/source3/passdb/pdb_ipa.c
index 5f89be7a29f..02f7bb68888 100644
--- a/source3/passdb/pdb_ipa.c
+++ b/source3/passdb/pdb_ipa.c
@@ -636,7 +636,7 @@ static NTSTATUS ipasam_enum_trusted_domains(struct pdb_methods *methods,
 	}
 
 	*num_domains = 0;
-	if (!(*domains = TALLOC_ARRAY(mem_ctx, struct pdb_trusted_domain *, 1))) {
+	if (!(*domains = talloc_array(mem_ctx, struct pdb_trusted_domain *, 1))) {
 		DEBUG(1, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -684,7 +684,7 @@ static NTSTATUS ipasam_enum_trusteddoms(struct pdb_methods *methods,
 		return NT_STATUS_OK;
 	}
 
-	if (!(*domains = TALLOC_ARRAY(mem_ctx, struct trustdom_info *,
+	if (!(*domains = talloc_array(mem_ctx, struct trustdom_info *,
 				      *num_domains))) {
 		DEBUG(1, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
@@ -693,7 +693,7 @@ static NTSTATUS ipasam_enum_trusteddoms(struct pdb_methods *methods,
 	for (i = 0; i < *num_domains; i++) {
 		struct trustdom_info *dom_info;
 
-		dom_info = TALLOC_P(*domains, struct trustdom_info);
+		dom_info = talloc(*domains, struct trustdom_info);
 		if (dom_info == NULL) {
 			DEBUG(1, ("talloc failed\n"));
 			return NT_STATUS_NO_MEMORY;
diff --git a/source3/passdb/pdb_ldap.c b/source3/passdb/pdb_ldap.c
index 7ce5edcefa9..f8da0ff2553 100644
--- a/source3/passdb/pdb_ldap.c
+++ b/source3/passdb/pdb_ldap.c
@@ -208,7 +208,7 @@ static NTSTATUS ldapsam_get_seq_num(struct pdb_methods *my_methods, time_t *seq_
 	if (mem_ctx == NULL)
 		return NT_STATUS_NO_MEMORY;
 
-	if ((attrs = TALLOC_ARRAY(mem_ctx, const char *, 2)) == NULL) {
+	if ((attrs = talloc_array(mem_ctx, const char *, 2)) == NULL) {
 		ntstatus = NT_STATUS_NO_MEMORY;
 		goto done;
 	}
@@ -887,7 +887,7 @@ static bool init_sam_from_ldap(struct ldapsam_privates *ldap_state,
 	if (pwHistLen > 0){
 		uint8 *pwhist = NULL;
 		int i;
-		char *history_string = TALLOC_ARRAY(ctx, char,
+		char *history_string = talloc_array(ctx, char,
 						MAX_PW_HISTORY_LEN*64);
 
 		if (!history_string) {
@@ -896,7 +896,7 @@ static bool init_sam_from_ldap(struct ldapsam_privates *ldap_state,
 
 		pwHistLen = MIN(pwHistLen, MAX_PW_HISTORY_LEN);
 
-		pwhist = TALLOC_ARRAY(ctx, uint8,
+		pwhist = talloc_array(ctx, uint8,
 				      pwHistLen * PW_HISTORY_ENTRY_LEN);
 		if (pwhist == NULL) {
 			DEBUG(0, ("init_sam_from_ldap: talloc failed!\n"));
@@ -1532,7 +1532,7 @@ static void append_attr(TALLOC_CTX *mem_ctx, const char ***attr_list,
 		;
 	}
 
-	(*attr_list) = TALLOC_REALLOC_ARRAY(mem_ctx, (*attr_list),
+	(*attr_list) = talloc_realloc(mem_ctx, (*attr_list),
 					    const char *,  i+2);
 	SMB_ASSERT((*attr_list) != NULL);
 	(*attr_list)[i] = talloc_strdup((*attr_list), new_attr);
@@ -4340,7 +4340,7 @@ static const char **talloc_attrs(TALLOC_CTX *mem_ctx, ...)
 		num += 1;
 	va_end(ap);
 
-	if ((result = TALLOC_ARRAY(mem_ctx, const char *, num+1)) == NULL) {
+	if ((result = talloc_array(mem_ctx, const char *, num+1)) == NULL) {
 		return NULL;
 	}
 
@@ -4564,7 +4564,7 @@ static bool ldapuser2displayentry(struct ldap_search_state *state,
 		return False;
 	}
 	if (!pull_utf8_talloc(mem_ctx,
-			      CONST_DISCARD(char **, &result->account_name),
+			      discard_const_p(char *, &result->account_name),
 			      vals[0], &converted_size))
 	{
 		DEBUG(0,("ldapuser2displayentry: pull_utf8_talloc failed: %s",
@@ -4577,7 +4577,7 @@ static bool ldapuser2displayentry(struct ldap_search_state *state,
 	if ((vals == NULL) || (vals[0] == NULL))
 		DEBUG(8, ("\"displayName\" not found\n"));
 	else if (!pull_utf8_talloc(mem_ctx,
-				   CONST_DISCARD(char **, &result->fullname),
+				   discard_const_p(char *, &result->fullname),
 				   vals[0], &converted_size))
 	{
 		DEBUG(0,("ldapuser2displayentry: pull_utf8_talloc failed: %s",
@@ -4590,7 +4590,7 @@ static bool ldapuser2displayentry(struct ldap_search_state *state,
 	if ((vals == NULL) || (vals[0] == NULL))
 		DEBUG(8, ("\"description\" not found\n"));
 	else if (!pull_utf8_talloc(mem_ctx,
-				   CONST_DISCARD(char **, &result->description),
+				   discard_const_p(char *, &result->description),
 				   vals[0], &converted_size))
 	{
 		DEBUG(0,("ldapuser2displayentry: pull_utf8_talloc failed: %s",
@@ -4724,7 +4724,7 @@ static bool ldapgroup2displayentry(struct ldap_search_state *state,
 			return False;
 		}
 		if (!pull_utf8_talloc(mem_ctx,
-				      CONST_DISCARD(char **,
+				      discard_const_p(char *,
 						    &result->account_name),
 				      vals[0], &converted_size))
 		{
@@ -4733,7 +4733,7 @@ static bool ldapgroup2displayentry(struct ldap_search_state *state,
 		}
 	}
 	else if (!pull_utf8_talloc(mem_ctx,
-				   CONST_DISCARD(char **,
+				   discard_const_p(char *,
 						 &result->account_name),
 				   vals[0], &converted_size))
 	{
@@ -4747,7 +4747,7 @@ static bool ldapgroup2displayentry(struct ldap_search_state *state,
 	if ((vals == NULL) || (vals[0] == NULL))
 		DEBUG(8, ("\"description\" not found\n"));
 	else if (!pull_utf8_talloc(mem_ctx,
-				   CONST_DISCARD(char **, &result->description),
+				   discard_const_p(char *, &result->description),
 				   vals[0], &converted_size))
 	{
 		DEBUG(0,("ldapgroup2displayentry: pull_utf8_talloc failed: %s",
@@ -6356,7 +6356,7 @@ static NTSTATUS ldapsam_enum_trusteddoms(struct pdb_methods *methods,
 	}
 
 	*num_domains = 0;
-	if (!(*domains = TALLOC_ARRAY(mem_ctx, struct trustdom_info *, 1))) {
+	if (!(*domains = talloc_array(mem_ctx, struct trustdom_info *, 1))) {
 		DEBUG(1, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -6368,7 +6368,7 @@ static NTSTATUS ldapsam_enum_trusteddoms(struct pdb_methods *methods,
 		char *dom_name, *dom_sid_str;
 		struct trustdom_info *dom_info;
 
-		dom_info = TALLOC_P(*domains, struct trustdom_info);
+		dom_info = talloc(*domains, struct trustdom_info);
 		if (dom_info == NULL) {
 			DEBUG(1, ("talloc failed\n"));
 			return NT_STATUS_NO_MEMORY;
@@ -6480,7 +6480,7 @@ static NTSTATUS pdb_init_ldapsam_common(struct pdb_methods **pdb_method, const c
 
 	/* TODO: Setup private data and free */
 
-	if ( !(ldap_state = TALLOC_ZERO_P(*pdb_method, struct ldapsam_privates)) ) {
+	if ( !(ldap_state = talloc_zero(*pdb_method, struct ldapsam_privates)) ) {
 		DEBUG(0, ("pdb_init_ldapsam_common: talloc() failed for ldapsam private_data!\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/passdb/pdb_samba4.c b/source3/passdb/pdb_samba4.c
new file mode 100644
index 00000000000..5848c23de8e
--- /dev/null
+++ b/source3/passdb/pdb_samba4.c
@@ -0,0 +1,2100 @@
+/*
+   Unix SMB/CIFS implementation.
+   pdb glue module for samba4
+   Copyright (C) Volker Lendecke 2009-2011
+   Copyright (C) Andrew Bartlett 2010
+   Copyright (C) Matthias Dieter Wallnöfer                 2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/* This module, is a port of Volker's pdb_ads to ldb and DSDB APIs */
+
+#include "includes.h"
+#include "source3/include/passdb.h"
+#include "source4/dsdb/samdb/samdb.h"
+#include "ldb_errors.h"
+#include "libcli/security/dom_sid.h"
+#include "source4/winbind/idmap.h"
+#include "librpc/gen_ndr/ndr_security.h"
+#include "libds/common/flag_mapping.h"
+#include "source4/lib/events/events.h"
+#include "source4/auth/session.h"
+#include "source4/auth/system_session_proto.h"
+#include "source4/param/param.h"
+
+struct pdb_samba4_state {
+	struct tevent_context *ev;
+	struct ldb_context *ldb;
+	struct idmap_context *idmap_ctx;
+	struct loadparm_context *lp_ctx;
+};
+
+static NTSTATUS pdb_samba4_getsampwsid(struct pdb_methods *m,
+				    struct samu *sam_acct,
+				    const struct dom_sid *sid);
+static NTSTATUS pdb_samba4_getsamupriv(struct pdb_samba4_state *state,
+				    const char *filter,
+				    TALLOC_CTX *mem_ctx,
+				    struct ldb_message **pmsg);
+static bool pdb_samba4_sid_to_id(struct pdb_methods *m, const struct dom_sid *sid,
+				 union unid_t *id, enum lsa_SidType *type);
+
+static bool pdb_samba4_pull_time(struct ldb_message *msg, const char *attr,
+			      time_t *ptime)
+{
+	uint64_t tmp;
+	if (! ldb_msg_find_element(msg, attr)) {
+		return false;
+	}
+	tmp = ldb_msg_find_attr_as_uint64(msg, attr, 0);
+	*ptime = uint64s_nt_time_to_unix_abs(&tmp);
+	return true;
+}
+
+static struct pdb_domain_info *pdb_samba4_get_domain_info(
+	struct pdb_methods *m, TALLOC_CTX *mem_ctx)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct pdb_domain_info *info;
+	struct dom_sid *domain_sid;
+	struct ldb_dn *forest_dn, *domain_dn;
+	struct ldb_result *dom_res = NULL;
+	const char *dom_attrs[] = {
+		"objectSid", 
+		"objectGUID", 
+		"nTMixedDomain",
+		"fSMORoleOwner",
+		NULL
+	};
+	char *p;
+	int ret;
+
+	info = talloc(mem_ctx, struct pdb_domain_info);
+	if (info == NULL) {
+		return NULL;
+	}
+
+	domain_dn = ldb_get_default_basedn(state->ldb); 
+
+	ret = ldb_search(state->ldb, info, &dom_res,
+			 domain_dn, LDB_SCOPE_BASE, dom_attrs, NULL);
+	if (ret != LDB_SUCCESS) {
+		goto fail;
+	}
+	if (dom_res->count != 1) {
+		goto fail;
+	}
+
+	info->guid = samdb_result_guid(dom_res->msgs[0], "objectGUID");
+
+	domain_sid = samdb_result_dom_sid(state, dom_res->msgs[0], "objectSid");
+	if (!domain_sid) {
+		goto fail;
+	}
+	info->sid = *domain_sid;
+
+	TALLOC_FREE(dom_res);
+
+	info->name = talloc_strdup(info, lpcfg_sam_name(state->lp_ctx));
+	info->dns_domain = ldb_dn_canonical_string(info, domain_dn);
+
+	if (!info->dns_domain) {
+		goto fail;
+	}
+	p = strchr(info->dns_domain, '/');
+	if (p) {
+		*p = '\0';
+	}
+
+	forest_dn = ldb_get_root_basedn(state->ldb);
+	if (!forest_dn) {
+		goto fail;
+	}
+
+	info->dns_forest = ldb_dn_canonical_string(info, forest_dn);
+	if (!info->dns_forest) {
+		goto fail;
+	}
+	p = strchr(info->dns_forest, '/');
+	if (p) {
+		*p = '\0';
+	}
+
+	return info;
+
+fail:
+	TALLOC_FREE(dom_res);
+	TALLOC_FREE(info);
+	return NULL;
+}
+
+static struct ldb_message *pdb_samba4_get_samu_private(
+	struct pdb_methods *m, struct samu *sam)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct ldb_message *msg;
+	char *sidstr, *filter;
+	NTSTATUS status;
+
+	msg = (struct ldb_message *)
+		pdb_get_backend_private_data(sam, m);
+
+	if (msg != NULL) {
+		return talloc_get_type_abort(msg, struct ldb_message);
+	}
+
+	sidstr = dom_sid_string(talloc_tos(), pdb_get_user_sid(sam));
+	if (sidstr == NULL) {
+		return NULL;
+	}
+
+	filter = talloc_asprintf(
+		talloc_tos(), "(&(objectsid=%s)(objectclass=user))", sidstr);
+	TALLOC_FREE(sidstr);
+	if (filter == NULL) {
+		return NULL;
+	}
+
+	status = pdb_samba4_getsamupriv(state, filter, sam, &msg);
+	TALLOC_FREE(filter);
+	if (!NT_STATUS_IS_OK(status)) {
+		return NULL;
+	}
+
+	return msg;
+}
+
+static NTSTATUS pdb_samba4_init_sam_from_priv(struct pdb_methods *m,
+					   struct samu *sam,
+					   struct ldb_message *msg)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	TALLOC_CTX *frame = talloc_stackframe();
+	NTSTATUS status = NT_STATUS_INTERNAL_DB_CORRUPTION;
+	const char *str;
+	time_t tmp_time;
+	struct dom_sid *sid, group_sid;
+	uint64_t n;
+	const DATA_BLOB *blob;
+
+	str = ldb_msg_find_attr_as_string(msg, "samAccountName", NULL);
+	if (str == NULL) {
+		DEBUG(10, ("no samAccountName\n"));
+		goto fail;
+	}
+	pdb_set_username(sam, str, PDB_SET);
+
+	if (pdb_samba4_pull_time(msg, "lastLogon", &tmp_time)) {
+		pdb_set_logon_time(sam, tmp_time, PDB_SET);
+	}
+	if (pdb_samba4_pull_time(msg, "lastLogoff", &tmp_time)) {
+		pdb_set_logoff_time(sam, tmp_time, PDB_SET);
+	}
+	if (pdb_samba4_pull_time(msg, "pwdLastSet", &tmp_time)) {
+		pdb_set_pass_last_set_time(sam, tmp_time, PDB_SET);
+	}
+	if (pdb_samba4_pull_time(msg, "accountExpires", &tmp_time)) {
+		pdb_set_kickoff_time(sam, tmp_time, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "displayName",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_fullname(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "homeDirectory",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_homedir(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "homeDrive", NULL);
+	if (str != NULL) {
+		pdb_set_dir_drive(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "scriptPath", NULL);
+	if (str != NULL) {
+		pdb_set_logon_script(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "profilePath",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_profile_path(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "profilePath",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_profile_path(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "comment",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_comment(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "description",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_acct_desc(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "userWorkstations",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_workstations(sam, str, PDB_SET);
+	}
+
+	str = ldb_msg_find_attr_as_string(msg, "userParameters",
+					    NULL);
+	if (str != NULL) {
+		pdb_set_munged_dial(sam, str, PDB_SET);
+	}
+
+	sid = samdb_result_dom_sid(talloc_tos(), msg, "objectSid");
+	if (!sid) {
+		DEBUG(10, ("Could not pull SID\n"));
+		goto fail;
+	}
+	pdb_set_user_sid(sam, sid, PDB_SET);
+
+	n = ldb_msg_find_attr_as_uint(msg, "userAccountControl", 0);
+	if (n == 0) {
+		DEBUG(10, ("Could not pull userAccountControl\n"));
+		goto fail;
+	}
+	pdb_set_acct_ctrl(sam, ds_uf2acb(n), PDB_SET);
+
+	blob = ldb_msg_find_ldb_val(msg, "unicodePwd");
+	if (blob) {
+		if (blob->length != NT_HASH_LEN) {
+			DEBUG(0, ("Got NT hash of length %d, expected %d\n",
+				  (int)blob->length, NT_HASH_LEN));
+			goto fail;
+		}
+		pdb_set_nt_passwd(sam, blob->data, PDB_SET);
+	}
+	
+	blob = ldb_msg_find_ldb_val(msg, "dBCSPwd");
+	if (blob) {
+		if (blob->length != LM_HASH_LEN) {
+			DEBUG(0, ("Got LM hash of length %d, expected %d\n",
+				  (int)blob->length, LM_HASH_LEN));
+			goto fail;
+		}
+		pdb_set_lanman_passwd(sam, blob->data, PDB_SET);
+	}
+	
+	n = ldb_msg_find_attr_as_uint(msg, "primaryGroupID", 0);
+	if (n == 0) {
+		DEBUG(10, ("Could not pull primaryGroupID\n"));
+		goto fail;
+	}
+	sid_compose(&group_sid, samdb_domain_sid(state->ldb), n);
+	pdb_set_group_sid(sam, &group_sid, PDB_SET);
+
+	status = NT_STATUS_OK;
+fail:
+	TALLOC_FREE(frame);
+	return status;
+}
+
+static bool pdb_samba4_add_time(struct ldb_message *msg, 
+				const char *attrib, time_t t)
+{
+	uint64_t nt_time;
+
+	unix_to_nt_time(&nt_time, t);
+
+	return ldb_msg_add_fmt(msg, attrib, "%llu", (unsigned long long) nt_time);
+}
+
+/* Like in pdb_ldap(), this will need to be a function pointer when we
+ * start to support 'adds' for migrations from samba3 passdb backends
+ * to samba4 */
+static bool update_required(struct samu *sam, enum pdb_elements element)
+{
+	return (IS_SAM_CHANGED(sam, element));
+}
+
+static bool pdb_samba4_init_samba4_from_sam(struct pdb_samba4_state *state,
+					    struct ldb_message *existing,
+					    TALLOC_CTX *mem_ctx,
+					    struct ldb_message **pmods, 
+					    struct samu *sam)
+{
+	int ret = LDB_SUCCESS;
+	const char *pw;
+	struct ldb_message *msg;
+
+	/* TODO: All fields :-) */
+
+	msg = ldb_msg_new(mem_ctx);
+	if (!msg) {
+		return false;
+	}
+
+	msg->dn = existing->dn;
+
+	pw = pdb_get_plaintext_passwd(sam);
+	if (update_required(sam, PDB_PLAINTEXT_PW)) {
+		if (pw == NULL) {
+			ret = LDB_ERR_OPERATIONS_ERROR;
+			goto fail;
+		}
+		
+		ret |= ldb_msg_add_string(msg, "clearTextPassword", pw);
+	}
+
+	if (update_required(sam, PDB_FULLNAME)) {
+		ret |= ldb_msg_add_string(msg, "displayName", pdb_get_fullname(sam));
+	}
+
+	if (update_required(sam, PDB_SMBHOME)) {
+		ret |= ldb_msg_add_string(msg, "homeDirectory",
+					  pdb_get_homedir(sam));
+	}
+
+	if (update_required(sam, PDB_PROFILE)) {
+		ret |= ldb_msg_add_string(msg, "profilePath",
+					  pdb_get_profile_path(sam));
+	}
+
+	if (update_required(sam, PDB_DRIVE)) {
+		ret |= ldb_msg_add_string(msg, "homeDrive",
+					  pdb_get_dir_drive(sam));
+	}
+
+	if (update_required(sam, PDB_LOGONSCRIPT)) {
+		ret |= ldb_msg_add_string(msg, "scriptPath",
+					  pdb_get_logon_script(sam));
+	}
+
+	if (update_required(sam, PDB_KICKOFFTIME)) {
+		ret |= pdb_samba4_add_time(msg, "accountExpires",
+					pdb_get_kickoff_time(sam));
+	}
+
+	if (update_required(sam, PDB_USERNAME)) {
+		ret |= ldb_msg_add_string(msg, "samAccountName",
+					  pdb_get_username(sam));
+	}
+
+	if (update_required(sam, PDB_HOURSLEN) || update_required(sam, PDB_HOURS)) {
+		struct ldb_val hours = data_blob_const(pdb_get_hours(sam), pdb_get_hours_len(sam));
+		ret |= ldb_msg_add_value(msg, "logonHours",
+					 &hours, NULL);
+	}
+
+	if (update_required(sam, PDB_ACCTCTRL)) {
+		ret |= ldb_msg_add_fmt(msg, "userAccountControl",
+				       "%d", ds_acb2uf(pdb_get_acct_ctrl(sam)));
+	}
+
+	if (update_required(sam, PDB_COMMENT)) {
+		ret |= ldb_msg_add_string(msg, "comment",
+					  pdb_get_comment(sam));
+	}
+
+	if (update_required(sam, PDB_ACCTDESC)) {
+		ret |= ldb_msg_add_string(msg, "description",
+					  pdb_get_acct_desc(sam));
+	}
+
+	if (update_required(sam, PDB_WORKSTATIONS)) {
+		ret |= ldb_msg_add_string(msg, "userWorkstations",
+					  pdb_get_workstations(sam));
+	}
+
+	/* This will need work, it is actually a UTF8 'string' with internal NULLs, to handle TS parameters */
+	if (update_required(sam, PDB_MUNGEDDIAL)) {
+		ret |= ldb_msg_add_string(msg, "userParameters",
+					  pdb_get_munged_dial(sam));
+	}
+
+	if (update_required(sam, PDB_COUNTRY_CODE)) {
+		ret |= ldb_msg_add_fmt(msg, "countryCode",
+				       "%i", (int)pdb_get_country_code(sam));
+	}
+
+	if (update_required(sam, PDB_CODE_PAGE)) {
+		ret |= ldb_msg_add_fmt(msg, "codePage",
+				       "%i", (int)pdb_get_code_page(sam));
+	}
+
+	/* Not yet handled here or not meaningful for modifies on a Samba4 backend:
+	PDB_LOGONTIME,
+	PDB_LOGOFFTIME,
+	PDB_BAD_PASSWORD_TIME,
+	PDB_CANCHANGETIME,
+	PDB_MUSTCHANGETIME,
+	PDB_DOMAIN,
+	PDB_NTUSERNAME,
+	PDB_LOGONDIVS,
+	PDB_USERSID,
+	PDB_GROUPSID,
+	PDB_PASSLASTSET,
+	PDB_FIELDS_PRESENT,
+	PDB_BAD_PASSWORD_COUNT,
+	PDB_LOGON_COUNT,
+	PDB_UNKNOWN6,
+	PDB_LMPASSWD,
+	PDB_NTPASSWD,
+	PDB_PWHISTORY,
+	PDB_BACKEND_PRIVATE_DATA,
+
+ */
+
+	*pmods = msg;
+fail:
+	return ret == LDB_SUCCESS;
+}
+
+static NTSTATUS pdb_samba4_getsamupriv(struct pdb_samba4_state *state,
+				    const char *filter,
+				    TALLOC_CTX *mem_ctx,
+				    struct ldb_message **msg)
+{
+	const char * attrs[] = {
+		"lastLogon", "lastLogoff", "pwdLastSet", "accountExpires",
+		"sAMAccountName", "displayName", "homeDirectory",
+		"homeDrive", "scriptPath", "profilePath", "description",
+		"userWorkstations", "comment", "userParameters", "objectSid",
+		"primaryGroupID", "userAccountControl", "logonHours",
+		"badPwdCount", "logonCount", "countryCode", "codePage",
+		"unicodePwd", "dBCSPwd", NULL };
+
+	int rc = dsdb_search_one(state->ldb, mem_ctx, msg, NULL, LDB_SCOPE_SUBTREE, attrs, 0, "%s", filter); 
+	if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("ldap_search failed %s\n",
+			   ldb_errstring(state->ldb)));
+		return NT_STATUS_LDAP(rc);
+	}
+
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_getsampwfilter(struct pdb_methods *m,
+					  struct pdb_samba4_state *state,
+					  struct samu *sam_acct,
+					  const char *exp_fmt, ...) _PRINTF_ATTRIBUTE(4, 5)
+{
+	struct ldb_message *priv;
+	NTSTATUS status;
+	va_list ap;
+	char *expression = NULL;
+	TALLOC_CTX *tmp_ctx = talloc_new(state);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	va_start(ap, exp_fmt);
+	expression = talloc_vasprintf(tmp_ctx, exp_fmt, ap);
+	va_end(ap);
+	
+	if (!expression) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	status = pdb_samba4_getsamupriv(state, expression, sam_acct, &priv);
+	talloc_free(tmp_ctx);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(10, ("pdb_samba4_getsamupriv failed: %s\n",
+			   nt_errstr(status)));
+		return status;
+	}
+
+	status = pdb_samba4_init_sam_from_priv(m, sam_acct, priv);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(10, ("pdb_samba4_init_sam_from_priv failed: %s\n",
+			   nt_errstr(status)));
+		TALLOC_FREE(priv);
+		return status;
+	}
+
+	pdb_set_backend_private_data(sam_acct, priv, NULL, m, PDB_SET);
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_getsampwnam(struct pdb_methods *m,
+				    struct samu *sam_acct,
+				    const char *username)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+
+	return pdb_samba4_getsampwfilter(m, state, sam_acct, 
+					 "(&(samaccountname=%s)(objectclass=user))",
+					 username);
+}
+
+static NTSTATUS pdb_samba4_getsampwsid(struct pdb_methods *m,
+				    struct samu *sam_acct,
+				    const struct dom_sid *sid)
+{
+	NTSTATUS status;
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	char *sidstr;
+
+	sidstr = dom_sid_string(talloc_tos(), sid);
+	NT_STATUS_HAVE_NO_MEMORY(sidstr);
+
+	status = pdb_samba4_getsampwfilter(m, state, sam_acct,  
+					   "(&(objectsid=%s)(objectclass=user))", 
+					   sidstr);
+	talloc_free(sidstr);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_create_user(struct pdb_methods *m,
+				    TALLOC_CTX *mem_ctx,
+				    const char *name, uint32 acct_flags,
+				    uint32 *rid)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct dom_sid *sid;
+	struct ldb_dn *dn;
+	NTSTATUS status;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	/* Internally this uses transactions to ensure all the steps
+	 * happen or fail as one */
+	status = dsdb_add_user(state->ldb, tmp_ctx, name, acct_flags, &sid, &dn);
+	if (!NT_STATUS_IS_OK(status)) {
+		talloc_free(tmp_ctx);
+		return status;
+	}
+	sid_peek_rid(sid, rid);
+	talloc_free(tmp_ctx);
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_delete_user(struct pdb_methods *m,
+				       TALLOC_CTX *mem_ctx,
+				       struct samu *sam)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct ldb_dn *dn;
+	int rc;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	dn = ldb_dn_new_fmt(tmp_ctx, state->ldb, "<SID=%s>", dom_sid_string(tmp_ctx, pdb_get_user_sid(sam)));
+	if (!dn || !ldb_dn_validate(dn)) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+	rc = ldb_delete(state->ldb, dn);
+
+	if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("ldb_delete for %s failed: %s\n", ldb_dn_get_linearized(dn),
+			   ldb_errstring(state->ldb)));
+		talloc_free(tmp_ctx);
+		return NT_STATUS_LDAP(rc);
+	}
+	talloc_free(tmp_ctx);
+	return NT_STATUS_OK;
+}
+
+/* This interface takes a fully populated struct samu and places it in
+ * the database.  This is not implemented at this time as we need to
+ * be careful around the creation of arbitary SIDs (ie, we must ensrue
+ * they are not left in a RID pool */
+static NTSTATUS pdb_samba4_add_sam_account(struct pdb_methods *m,
+					struct samu *sampass)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+/*
+ * Update the Samba4 LDB with the changes from a struct samu.
+ *
+ * This takes care not to update elements that have not been changed
+ * by the caller
+ */
+static NTSTATUS pdb_samba4_update_sam_account(struct pdb_methods *m,
+					   struct samu *sam)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct ldb_message *msg = pdb_samba4_get_samu_private(
+		m, sam);
+	struct ldb_message *replace_msg;
+	int rc;
+
+	if (!pdb_samba4_init_samba4_from_sam(state, msg, talloc_tos(),
+					     &replace_msg, sam)) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	if (replace_msg->num_elements == 0) {
+		/* Nothing to do, just return success */
+		return NT_STATUS_OK;
+	}
+
+	rc = dsdb_replace(state->ldb, replace_msg, 0);
+	TALLOC_FREE(replace_msg);
+	if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("dsdb_replace for %s failed: %s\n", ldb_dn_get_linearized(replace_msg->dn),
+			   ldb_errstring(state->ldb)));
+		return NT_STATUS_LDAP(rc);
+	}
+
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_delete_sam_account(struct pdb_methods *m,
+					   struct samu *username)
+{
+	NTSTATUS status;
+	TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+	status = pdb_samba4_delete_user(m, tmp_ctx, username);
+	talloc_free(tmp_ctx);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_rename_sam_account(struct pdb_methods *m,
+					   struct samu *oldname,
+					   const char *newname)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+/* This is not implemented, as this module is exptected to be used
+ * with auth_samba4, and this is responible for login counters etc
+ *
+ */
+static NTSTATUS pdb_samba4_update_login_attempts(struct pdb_methods *m,
+					      struct samu *sam_acct,
+					      bool success)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+static NTSTATUS pdb_samba4_getgrfilter(struct pdb_methods *m, GROUP_MAP *map,
+				    const char *exp_fmt, ...) _PRINTF_ATTRIBUTE(4, 5)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	const char *attrs[] = { "objectSid", "description", "samAccountName",
+				NULL };
+	struct ldb_message *msg;
+	va_list ap;
+	char *expression = NULL;
+	struct dom_sid *sid;
+	const char *str;
+	int rc;
+	union unid_t id;
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+	
+	va_start(ap, exp_fmt);
+	expression = talloc_vasprintf(tmp_ctx, exp_fmt, ap);
+	va_end(ap);
+	
+	if (!expression) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	rc = dsdb_search_one(state->ldb, tmp_ctx, &msg, NULL, LDB_SCOPE_SUBTREE, attrs, 0, "%s", expression);
+	if (rc == LDB_ERR_NO_SUCH_OBJECT) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_SUCH_GROUP;
+	} else if (rc != LDB_SUCCESS) {
+		talloc_free(tmp_ctx);
+		DEBUG(10, ("dsdb_search_one failed %s\n",
+			   ldb_errstring(state->ldb)));
+		return NT_STATUS_LDAP(rc);
+	}
+
+	sid = samdb_result_dom_sid(tmp_ctx, msg, "objectSid");
+	if (!sid) {
+		talloc_free(tmp_ctx);
+		DEBUG(10, ("Could not pull SID\n"));
+		return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	}
+	
+	map->sid = *sid;
+	
+	if (!pdb_samba4_sid_to_id(m, sid, &id, &map->sid_name_use)) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_SUCH_GROUP;
+	}
+	if (map->sid_name_use == SID_NAME_USER) {
+		DEBUG(1, (__location__ "Got SID_NAME_USER when searching for a group with %s", expression));
+		return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	}
+	map->gid = id.gid;
+
+	str = ldb_msg_find_attr_as_string(msg, "samAccountName",
+					  NULL);
+	if (str == NULL) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	}
+	fstrcpy(map->nt_name, str);
+
+	str = ldb_msg_find_attr_as_string(msg, "description",
+					    NULL);
+	if (str != NULL) {
+		fstrcpy(map->comment, str);
+	} else {
+		map->comment[0] = '\0';
+	}
+
+	talloc_free(tmp_ctx);
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_getgrsid(struct pdb_methods *m, GROUP_MAP *map,
+				 struct dom_sid sid)
+{
+	char *filter;
+	NTSTATUS status;
+
+	filter = talloc_asprintf(talloc_tos(),
+				 "(&(objectsid=%s)(objectclass=group))",
+				 sid_string_talloc(talloc_tos(), &sid));
+	if (filter == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	status = pdb_samba4_getgrfilter(m, map, filter);
+	TALLOC_FREE(filter);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_getgrgid(struct pdb_methods *m, GROUP_MAP *map,
+				 gid_t gid)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	NTSTATUS status;
+	struct id_map id_map;
+	struct id_map *id_maps[2];
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	id_map.xid.id = gid;
+	id_map.xid.type = ID_TYPE_GID;
+	id_maps[0] = &id_map;
+	id_maps[1] = NULL;
+
+	status = idmap_xids_to_sids(state->idmap_ctx, tmp_ctx, id_maps);
+	if (!NT_STATUS_IS_OK(status)) {
+		return status;
+	}
+	status = pdb_samba4_getgrsid(m, map, *id_map.sid);
+	talloc_free(tmp_ctx);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_getgrnam(struct pdb_methods *m, GROUP_MAP *map,
+				 const char *name)
+{
+	char *filter;
+	NTSTATUS status;
+
+	filter = talloc_asprintf(talloc_tos(),
+				 "(&(samaccountname=%s)(objectclass=group))",
+				 name);
+	if (filter == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	status = pdb_samba4_getgrfilter(m, map, filter);
+	TALLOC_FREE(filter);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_create_dom_group(struct pdb_methods *m,
+					 TALLOC_CTX *mem_ctx, const char *name,
+					 uint32 *rid)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	NTSTATUS status;
+	struct dom_sid *sid;
+	struct ldb_dn *dn;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	status = dsdb_add_domain_group(state->ldb, tmp_ctx, name, &sid, &dn);
+	if (!NT_STATUS_IS_OK(status)) {
+		talloc_free(tmp_ctx);
+		return status;
+	}
+
+	sid_peek_rid(sid, rid);
+	talloc_free(tmp_ctx);
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_delete_dom_group(struct pdb_methods *m,
+					 TALLOC_CTX *mem_ctx, uint32 rid)
+{
+	const char *attrs[] = { NULL };
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct dom_sid sid;
+	struct ldb_message *msg;
+	struct ldb_dn *dn;
+	int rc;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	sid_compose(&sid, samdb_domain_sid(state->ldb), rid);
+	
+	if (ldb_transaction_start(state->ldb) != LDB_SUCCESS) {
+		DEBUG(0, ("Unable to start transaction in pdb_samba4_delete_dom_group()\n"));
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
+	dn = ldb_dn_new_fmt(tmp_ctx, state->ldb, "<SID=%s>", dom_sid_string(tmp_ctx, &sid));
+	if (!dn || !ldb_dn_validate(dn)) {
+		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(state->ldb);
+		return NT_STATUS_NO_MEMORY;
+	}
+	rc = dsdb_search_one(state->ldb, tmp_ctx, &msg, dn, LDB_SCOPE_BASE, attrs, 0, "objectclass=group");
+	if (rc == LDB_ERR_NO_SUCH_OBJECT) {
+		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(state->ldb);
+		return NT_STATUS_NO_SUCH_GROUP;
+	}
+	rc = ldb_delete(state->ldb, dn);
+	if (rc == LDB_ERR_NO_SUCH_OBJECT) {
+		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(state->ldb);
+		return NT_STATUS_NO_SUCH_GROUP;
+	} else if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("ldb_delete failed %s\n",
+			   ldb_errstring(state->ldb)));
+		ldb_transaction_cancel(state->ldb);
+		return NT_STATUS_LDAP(rc);
+	}
+
+	if (ldb_transaction_commit(state->ldb) != LDB_SUCCESS) {
+		DEBUG(0, ("Unable to commit transaction in pdb_samba4_delete_dom_group()\n"));
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_add_group_mapping_entry(struct pdb_methods *m,
+						GROUP_MAP *map)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+static NTSTATUS pdb_samba4_update_group_mapping_entry(struct pdb_methods *m,
+						   GROUP_MAP *map)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+static NTSTATUS pdb_samba4_delete_group_mapping_entry(struct pdb_methods *m,
+						   struct dom_sid sid)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+static NTSTATUS pdb_samba4_enum_group_mapping(struct pdb_methods *m,
+					   const struct dom_sid *sid,
+					   enum lsa_SidType sid_name_use,
+					   GROUP_MAP **pp_rmap,
+					   size_t *p_num_entries,
+					   bool unix_only)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+static NTSTATUS pdb_samba4_enum_group_members(struct pdb_methods *m,
+					   TALLOC_CTX *mem_ctx,
+					   const struct dom_sid *group,
+					   uint32_t **pmembers,
+					   size_t *pnum_members)
+{
+	unsigned int i, num_sids, num_members;
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct dom_sid *members_as_sids;
+	struct dom_sid *dom_sid;
+	uint32_t *members;
+	struct ldb_dn *dn;
+	NTSTATUS status;
+
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	dn = ldb_dn_new_fmt(tmp_ctx, state->ldb, "<SID=%s>", dom_sid_string(tmp_ctx, group));
+	if (!dn || !ldb_dn_validate(dn)) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	status = dsdb_enum_group_mem(state->ldb, tmp_ctx, dn, &members_as_sids, &num_sids);
+	if (!NT_STATUS_IS_OK(status)) {
+		talloc_free(tmp_ctx);
+		return status;
+	}
+	status = dom_sid_split_rid(tmp_ctx, group, &dom_sid, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		talloc_free(tmp_ctx);
+		return status;
+	}
+
+	*pmembers = members = talloc_array(mem_ctx, uint32_t, num_sids);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(*pmembers, tmp_ctx);
+	num_members = 0;
+
+	for (i = 0; i < num_sids; i++) {
+		if (!dom_sid_in_domain(dom_sid, &members_as_sids[i])) {
+			continue;
+		}
+		status = dom_sid_split_rid(NULL, &members_as_sids[i], 
+					   NULL, &members[num_members]);
+		if (!NT_STATUS_IS_OK(status)) {
+			talloc_free(tmp_ctx);
+			return status;
+		}
+		num_members++;
+	}
+	*pnum_members = num_members;
+	return NT_STATUS_OK;
+}
+
+/* Just convert the primary group SID into a group */
+static NTSTATUS fake_enum_group_memberships(struct pdb_samba4_state *state,
+					    TALLOC_CTX *mem_ctx,
+					    struct samu *user,
+					    struct dom_sid **pp_sids,
+					    gid_t **pp_gids,
+					    uint32_t *p_num_groups)
+{
+	NTSTATUS status;
+	size_t num_groups = 0;
+	struct dom_sid *group_sids;
+	gid_t *gids;
+	TALLOC_CTX *tmp_ctx;
+	
+	tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	if (user->group_sid) {
+		struct id_map *id_maps[2];
+		struct id_map id_map;
+
+		num_groups = 1;
+
+		group_sids = talloc_array(tmp_ctx, struct dom_sid, num_groups);
+		if (group_sids == NULL) {
+			talloc_free(tmp_ctx);
+			return NT_STATUS_NO_MEMORY;
+		}
+		gids = talloc_array(tmp_ctx, gid_t, num_groups);
+		if (gids == NULL) {
+			talloc_free(tmp_ctx);
+			return NT_STATUS_NO_MEMORY;
+		}
+
+		group_sids[0] = *user->group_sid;
+
+		ZERO_STRUCT(id_map);
+		id_map.sid = &group_sids[0];
+		id_maps[0] = &id_map;
+		id_maps[1] = NULL;
+		
+		status = idmap_sids_to_xids(state->idmap_ctx, tmp_ctx, id_maps);
+		if (!NT_STATUS_IS_OK(status)) {
+			talloc_free(tmp_ctx);
+			return status;
+		}
+		if (id_map.xid.type == ID_TYPE_GID || id_map.xid.type == ID_TYPE_BOTH) {
+			gids[0] = id_map.xid.id;
+		} else {
+			DEBUG(1, (__location__ 
+				  "Group %s, of which %s is a member, could not be converted to a GID\n",
+				  dom_sid_string(tmp_ctx, &group_sids[0]),
+				  dom_sid_string(tmp_ctx, &user->user_sid)));
+			talloc_free(tmp_ctx);
+			/* We must error out, otherwise a user might
+			 * avoid a DENY acl based on a group they
+			 * missed out on */
+			return NT_STATUS_NO_SUCH_GROUP;
+		}
+	}
+
+	*pp_sids = talloc_steal(mem_ctx, group_sids);
+	*pp_gids = talloc_steal(mem_ctx, gids);
+	*p_num_groups = num_groups;
+	talloc_free(tmp_ctx);
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_enum_group_memberships(struct pdb_methods *m,
+					       TALLOC_CTX *mem_ctx,
+					       struct samu *user,
+					       struct dom_sid **pp_sids,
+					       gid_t **pp_gids,
+					       uint32_t *p_num_groups)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct ldb_message *msg = pdb_samba4_get_samu_private(
+		m, user);
+	const char *attrs[] = { "tokenGroups", NULL};
+	struct ldb_message *tokengroups_msg;
+	struct ldb_message_element *tokengroups;
+	int i, rc;
+	NTSTATUS status;
+	unsigned int count = 0;
+	size_t num_groups;
+	struct dom_sid *group_sids;
+	gid_t *gids;
+	TALLOC_CTX *tmp_ctx;
+		
+	if (msg == NULL) {
+		/* Fake up some things here */
+		return fake_enum_group_memberships(state, 
+						   mem_ctx, 
+						   user, pp_sids,
+						   pp_gids, p_num_groups);
+	}
+
+	tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	rc = dsdb_search_one(state->ldb, tmp_ctx, &tokengroups_msg, msg->dn, LDB_SCOPE_BASE, attrs, 0, NULL);
+
+	if (rc == LDB_ERR_NO_SUCH_OBJECT) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_SUCH_USER;
+	} else if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("dsdb_search_one failed %s\n",
+			   ldb_errstring(state->ldb)));
+		talloc_free(tmp_ctx);
+		return NT_STATUS_LDAP(rc);
+	}
+
+	tokengroups = ldb_msg_find_element(tokengroups_msg, "tokenGroups");
+
+	if (tokengroups) {
+		count = tokengroups->num_values;
+	}
+
+	group_sids = talloc_array(tmp_ctx, struct dom_sid, count);
+	if (group_sids == NULL) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+	gids = talloc_array(tmp_ctx, gid_t, count);
+	if (gids == NULL) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+	num_groups = 0;
+
+	for (i=0; i<count; i++) {
+		struct id_map *id_maps[2];
+		struct id_map id_map;
+		struct ldb_val *v = &tokengroups->values[i];
+		enum ndr_err_code ndr_err
+			= ndr_pull_struct_blob(v, group_sids, &group_sids[num_groups],
+					       (ndr_pull_flags_fn_t)ndr_pull_dom_sid);
+		if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+			talloc_free(tmp_ctx);
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+		
+		ZERO_STRUCT(id_map);
+		id_map.sid = &group_sids[num_groups];
+		id_maps[0] = &id_map;
+		id_maps[1] = NULL;
+
+		status = idmap_sids_to_xids(state->idmap_ctx, tmp_ctx, id_maps);
+		if (!NT_STATUS_IS_OK(status)) {
+			talloc_free(tmp_ctx);
+			return status;
+		}
+		if (id_map.xid.type == ID_TYPE_GID || id_map.xid.type == ID_TYPE_BOTH) {
+			gids[num_groups] = id_map.xid.id;
+		} else {
+			DEBUG(1, (__location__ 
+				  "Group %s, of which %s is a member, could not be converted to a GID\n",
+				  dom_sid_string(tmp_ctx, &group_sids[num_groups]),
+				  ldb_dn_get_linearized(msg->dn)));
+			talloc_free(tmp_ctx);
+			/* We must error out, otherwise a user might
+			 * avoid a DENY acl based on a group they
+			 * missed out on */
+			return NT_STATUS_NO_SUCH_GROUP;
+		}
+
+		num_groups += 1;
+		if (num_groups == count) {
+			break;
+		}
+	}
+
+	*pp_sids = talloc_steal(mem_ctx, group_sids);
+	*pp_gids = talloc_steal(mem_ctx, gids);
+	*p_num_groups = num_groups;
+	talloc_free(tmp_ctx);
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_set_unix_primary_group(struct pdb_methods *m,
+					       TALLOC_CTX *mem_ctx,
+					       struct samu *user)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+static NTSTATUS pdb_samba4_mod_groupmem_by_sid(struct pdb_methods *m,
+					       TALLOC_CTX *mem_ctx,
+					       const struct dom_sid *groupsid, 
+					       const struct dom_sid *membersid,
+					       int mod_op)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct ldb_message *msg;
+	int ret;
+	struct ldb_message_element *el;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+	msg = ldb_msg_new(tmp_ctx);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(msg, tmp_ctx);
+
+	msg->dn = ldb_dn_new_fmt(msg, state->ldb, "<SID=%s>", dom_sid_string(tmp_ctx, groupsid));
+	if (!msg->dn || !ldb_dn_validate(msg->dn)) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+	ret = ldb_msg_add_fmt(msg, "member", "<SID=%s>", dom_sid_string(tmp_ctx, membersid));
+	if (ret != LDB_SUCCESS) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+	el = ldb_msg_find_element(msg, "member");
+	el->flags = mod_op;
+
+	/* No need for transactions here, the ldb auto-transaction
+	 * code will handle things for the single operation */
+	ret = ldb_modify(state->ldb, msg);
+	talloc_free(tmp_ctx);
+	if (ret != LDB_SUCCESS) {
+		DEBUG(10, ("ldb_modify failed: %s\n",
+			   ldb_errstring(state->ldb)));
+		if (ret == LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS) {
+			return NT_STATUS_MEMBER_IN_GROUP;
+		}
+		if (ret == LDB_ERR_NO_SUCH_ATTRIBUTE) {
+			return NT_STATUS_MEMBER_NOT_IN_GROUP;
+		}
+		return NT_STATUS_LDAP(ret);
+	}
+
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_mod_groupmem(struct pdb_methods *m,
+				     TALLOC_CTX *mem_ctx,
+				     uint32 grouprid, uint32 memberrid,
+				     int mod_op)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	const struct dom_sid *dom_sid, *groupsid, *membersid;
+	NTSTATUS status;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	dom_sid = samdb_domain_sid(state->ldb);
+
+	groupsid = dom_sid_add_rid(tmp_ctx, dom_sid, grouprid);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(groupsid, tmp_ctx);
+	membersid = dom_sid_add_rid(tmp_ctx, dom_sid, memberrid);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(membersid, tmp_ctx);
+	status = pdb_samba4_mod_groupmem_by_sid(m, tmp_ctx, groupsid, membersid, mod_op);
+	talloc_free(tmp_ctx);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_add_groupmem(struct pdb_methods *m,
+				     TALLOC_CTX *mem_ctx,
+				     uint32 group_rid, uint32 member_rid)
+{
+	return pdb_samba4_mod_groupmem(m, mem_ctx, group_rid, member_rid,
+				    LDB_FLAG_MOD_ADD);
+}
+
+static NTSTATUS pdb_samba4_del_groupmem(struct pdb_methods *m,
+				     TALLOC_CTX *mem_ctx,
+				     uint32 group_rid, uint32 member_rid)
+{
+	return pdb_samba4_mod_groupmem(m, mem_ctx, group_rid, member_rid,
+				       LDB_FLAG_MOD_DELETE);
+}
+
+static NTSTATUS pdb_samba4_create_alias(struct pdb_methods *m,
+				     const char *name, uint32 *rid)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct dom_sid *sid;
+
+	struct ldb_dn *dn;
+	NTSTATUS status;
+	
+	/* Internally this uses transactions to ensure all the steps
+	 * happen or fail as one */
+	status = dsdb_add_domain_alias(state->ldb, frame, name, &sid, &dn);
+	if (!NT_STATUS_IS_OK(status)) {
+		TALLOC_FREE(frame);
+	}
+
+	sid_peek_rid(sid, rid);
+	TALLOC_FREE(frame);
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_delete_alias(struct pdb_methods *m,
+				     const struct dom_sid *sid)
+{
+	const char *attrs[] = { NULL };
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct ldb_message *msg;
+	struct ldb_dn *dn;
+	int rc;
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	dn = ldb_dn_new_fmt(tmp_ctx, state->ldb, "<SID=%s>", dom_sid_string(tmp_ctx, sid));
+	if (!dn || !ldb_dn_validate(dn)) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	if (ldb_transaction_start(state->ldb) != LDB_SUCCESS) {
+		DEBUG(0, ("Failed to start transaction in dsdb_add_domain_alias(): %s\n", ldb_errstring(state->ldb)));
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
+	rc = dsdb_search_one(state->ldb, tmp_ctx, &msg, dn, LDB_SCOPE_BASE, attrs, 0, "(objectclass=group)"
+			     "(|(grouptype=%d)(grouptype=%d)))",
+			     GTYPE_SECURITY_BUILTIN_LOCAL_GROUP,
+			     GTYPE_SECURITY_DOMAIN_LOCAL_GROUP);
+	if (rc == LDB_ERR_NO_SUCH_OBJECT) {
+		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(state->ldb);
+		return NT_STATUS_NO_SUCH_ALIAS;
+	}
+	rc = ldb_delete(state->ldb, dn);
+	if (rc == LDB_ERR_NO_SUCH_OBJECT) {
+		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(state->ldb);
+		return NT_STATUS_NO_SUCH_ALIAS;
+	} else if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("ldb_delete failed %s\n",
+			   ldb_errstring(state->ldb)));
+		ldb_transaction_cancel(state->ldb);
+		return NT_STATUS_LDAP(rc);
+	}
+
+	if (ldb_transaction_commit(state->ldb) != LDB_SUCCESS) {
+		DEBUG(0, ("Failed to commit transaction in pdb_samba4_delete_alias(): %s\n",
+			  ldb_errstring(state->ldb)));
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
+	return NT_STATUS_OK;
+}
+
+#if 0
+static NTSTATUS pdb_samba4_set_aliasinfo(struct pdb_methods *m,
+				      const struct dom_sid *sid,
+				      struct acct_info *info)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct tldap_context *ld;
+	const char *attrs[3] = { "objectSid", "description",
+				 "samAccountName" };
+	struct ldb_message **msg;
+	char *sidstr, *dn;
+	int rc;
+	struct tldap_mod *mods;
+	int num_mods;
+	bool ok;
+
+	ld = pdb_samba4_ld(state);
+	if (ld == NULL) {
+		return NT_STATUS_LDAP(TLDAP_SERVER_DOWN);
+	}
+
+	sidstr = sid_binstring(talloc_tos(), sid);
+	NT_STATUS_HAVE_NO_MEMORY(sidstr);
+
+	rc = pdb_samba4_search_fmt(state, state->domaindn, TLDAP_SCOPE_SUB,
+				attrs, ARRAY_SIZE(attrs), 0, talloc_tos(),
+				&msg, "(&(objectSid=%s)(objectclass=group)"
+				"(|(grouptype=%d)(grouptype=%d)))",
+				sidstr, GTYPE_SECURITY_BUILTIN_LOCAL_GROUP,
+				GTYPE_SECURITY_DOMAIN_LOCAL_GROUP);
+	TALLOC_FREE(sidstr)
+	if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("ldap_search failed %s\n",
+			   ldb_errstring(state->ldb)));
+		return NT_STATUS_LDAP(rc);
+	}
+	switch talloc_array_length(msg) {
+	case 0:
+		return NT_STATUS_NO_SUCH_ALIAS;
+	case 1:
+		break;
+	default:
+		return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	}
+
+	if (!tldap_entry_dn(msg[0], &dn)) {
+		TALLOC_FREE(msg);
+		return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	}
+
+	mods = NULL;
+	num_mods = 0;
+	ok = true;
+
+	ok &= tldap_make_mod_fmt(
+		msg[0], msg, &num_mods, &mods, "description",
+		"%s", info->acct_desc);
+	ok &= tldap_make_mod_fmt(
+		msg[0], msg, &num_mods, &mods, "samAccountName",
+		"%s", info->acct_name);
+	if (!ok) {
+		TALLOC_FREE(msg);
+		return NT_STATUS_NO_MEMORY;
+	}
+	if (num_mods == 0) {
+		/* no change */
+		TALLOC_FREE(msg);
+		return NT_STATUS_OK;
+	}
+
+	rc = tldap_modify(ld, dn, num_mods, mods, NULL, 0, NULL, 0);
+	TALLOC_FREE(msg);
+	if (rc != LDB_SUCCESS) {
+		DEBUG(10, ("ldap_modify failed: %s\n",
+			   ldb_errstring(state->ldb)));
+		return NT_STATUS_LDAP(rc);
+	}
+	return NT_STATUS_OK;
+}
+#endif
+static NTSTATUS pdb_samba4_add_aliasmem(struct pdb_methods *m,
+				     const struct dom_sid *alias,
+				     const struct dom_sid *member)
+{
+	NTSTATUS status;
+	TALLOC_CTX *frame = talloc_stackframe();
+	status = pdb_samba4_mod_groupmem_by_sid(m, frame, alias, member, LDB_FLAG_MOD_ADD);
+	talloc_free(frame);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_del_aliasmem(struct pdb_methods *m,
+				     const struct dom_sid *alias,
+				     const struct dom_sid *member)
+{
+	NTSTATUS status;
+	TALLOC_CTX *frame = talloc_stackframe();
+	status = pdb_samba4_mod_groupmem_by_sid(m, frame, alias, member, LDB_FLAG_MOD_DELETE);
+	talloc_free(frame);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_enum_aliasmem(struct pdb_methods *m,
+				      const struct dom_sid *alias,
+				      TALLOC_CTX *mem_ctx,
+				      struct dom_sid **pmembers,
+				      size_t *pnum_members)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct ldb_dn *dn;
+	unsigned int num_members;
+	NTSTATUS status;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+
+	dn = ldb_dn_new_fmt(tmp_ctx, state->ldb, "<SID=%s>", dom_sid_string(tmp_ctx, alias));
+	if (!dn || !ldb_dn_validate(dn)) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	status = dsdb_enum_group_mem(state->ldb, mem_ctx, dn, pmembers, &num_members);
+	*pnum_members = num_members;
+	if (NT_STATUS_IS_OK(status)) {
+		talloc_steal(mem_ctx, pmembers);
+	}
+	talloc_free(tmp_ctx);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_enum_alias_memberships(struct pdb_methods *m,
+					       TALLOC_CTX *mem_ctx,
+					       const struct dom_sid *domain_sid,
+					       const struct dom_sid *members,
+					       size_t num_members,
+					       uint32_t **palias_rids,
+					       size_t *pnum_alias_rids)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	uint32_t *alias_rids = NULL;
+	size_t num_alias_rids = 0;
+	int i;
+	struct dom_sid *groupSIDs = NULL;
+	unsigned int num_groupSIDs = 0;
+	char *filter;
+	NTSTATUS status;
+	const char *sid_string;
+	const char *sid_dn;
+	DATA_BLOB sid_blob;
+
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+	/*
+	 * TODO: Get the filter right so that we only get the aliases from
+	 * either the SAM or BUILTIN
+	 */
+
+	filter = talloc_asprintf(tmp_ctx, "(&(objectClass=group)(groupType:1.2.840.113556.1.4.803:=%u))",
+				 GROUP_TYPE_BUILTIN_LOCAL_GROUP);
+	if (filter == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	for (i = 0; i < num_members; i++) {
+		sid_string = dom_sid_string(tmp_ctx, &members[i]);
+		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(sid_string, tmp_ctx);
+
+		sid_dn = talloc_asprintf(tmp_ctx, "<SID=%s>", sid_string);
+		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(sid_dn, tmp_ctx);
+		
+		sid_blob = data_blob_string_const(sid_dn);
+		
+		status = dsdb_expand_nested_groups(state->ldb, &sid_blob, true, filter,
+						   tmp_ctx, &groupSIDs, &num_groupSIDs);
+		if (!NT_STATUS_IS_OK(status)) {
+			talloc_free(tmp_ctx);
+			return status;
+		}
+	}
+
+	alias_rids = talloc_array(mem_ctx, uint32_t, num_groupSIDs);
+	if (alias_rids == NULL) {
+		talloc_free(tmp_ctx);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	for (i=0; i<num_groupSIDs; i++) {
+		if (sid_peek_check_rid(domain_sid, &groupSIDs[i],
+				       &alias_rids[num_alias_rids])) {
+			num_alias_rids++;;
+		}
+	}
+
+	*palias_rids = alias_rids;
+	*pnum_alias_rids = num_alias_rids;
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS pdb_samba4_lookup_rids(struct pdb_methods *m,
+				    const struct dom_sid *domain_sid,
+				    int num_rids,
+				    uint32 *rids,
+				    const char **names,
+				    enum lsa_SidType *lsa_attrs)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	NTSTATUS status;
+
+	TALLOC_CTX *tmp_ctx;
+
+	if (num_rids == 0) {
+		return NT_STATUS_NONE_MAPPED;
+	}
+
+	tmp_ctx = talloc_stackframe();
+	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
+	
+	status = dsdb_lookup_rids(state->ldb, tmp_ctx, domain_sid, num_rids, rids, names, lsa_attrs);
+	talloc_free(tmp_ctx);
+	return status;
+}
+
+static NTSTATUS pdb_samba4_lookup_names(struct pdb_methods *m,
+				     const struct dom_sid *domain_sid,
+				     int num_names,
+				     const char **pp_names,
+				     uint32 *rids,
+				     enum lsa_SidType *attrs)
+{
+	return NT_STATUS_NOT_IMPLEMENTED;
+}
+
+static NTSTATUS pdb_samba4_get_account_policy(struct pdb_methods *m,
+					   enum pdb_policy_type type,
+					   uint32_t *value)
+{
+	return account_policy_get(type, value)
+		? NT_STATUS_OK : NT_STATUS_UNSUCCESSFUL;
+}
+
+static NTSTATUS pdb_samba4_set_account_policy(struct pdb_methods *m,
+					   enum pdb_policy_type type,
+					   uint32_t value)
+{
+	return account_policy_set(type, value)
+		? NT_STATUS_OK : NT_STATUS_UNSUCCESSFUL;
+}
+
+static NTSTATUS pdb_samba4_get_seq_num(struct pdb_methods *m,
+				    time_t *seq_num_out)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	uint64_t seq_num;
+	int ret = ldb_sequence_number(state->ldb, LDB_SEQ_HIGHEST_SEQ, &seq_num);
+	if (ret == LDB_SUCCESS) {
+		*seq_num_out = seq_num;
+		return NT_STATUS_OK;
+	} else {
+		return NT_STATUS_UNSUCCESSFUL;
+	}
+}
+
+struct pdb_samba4_search_state {
+	uint32_t acct_flags;
+	struct samr_displayentry *entries;
+	uint32_t num_entries;
+	ssize_t array_size;
+	uint32_t current;
+};
+
+static bool pdb_samba4_next_entry(struct pdb_search *search,
+			       struct samr_displayentry *entry)
+{
+	struct pdb_samba4_search_state *state = talloc_get_type_abort(
+		search->private_data, struct pdb_samba4_search_state);
+
+	if (state->current == state->num_entries) {
+		return false;
+	}
+
+	entry->idx = state->entries[state->current].idx;
+	entry->rid = state->entries[state->current].rid;
+	entry->acct_flags = state->entries[state->current].acct_flags;
+
+	entry->account_name = talloc_strdup(
+		search, state->entries[state->current].account_name);
+	entry->fullname = talloc_strdup(
+		search, state->entries[state->current].fullname);
+	entry->description = talloc_strdup(
+		search, state->entries[state->current].description);
+
+	state->current += 1;
+	return true;
+}
+
+static void pdb_samba4_search_end(struct pdb_search *search)
+{
+	struct pdb_samba4_search_state *state = talloc_get_type_abort(
+		search->private_data, struct pdb_samba4_search_state);
+	talloc_free(state);
+}
+
+static bool pdb_samba4_search_filter(struct pdb_methods *m,
+				     struct pdb_search *search,
+				     struct pdb_samba4_search_state **pstate,
+				     const char *exp_fmt, ...) _PRINTF_ATTRIBUTE(4, 5)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct pdb_samba4_search_state *sstate;
+	const char * attrs[] = { "objectSid", "sAMAccountName", "displayName",
+				 "userAccountControl", "description", NULL };
+	struct ldb_result *res;
+	int i, rc, num_users;
+
+	va_list ap;
+	char *expression = NULL;
+
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
+	if (!tmp_ctx) {
+		return false;
+	}
+
+	va_start(ap, exp_fmt);
+	expression = talloc_vasprintf(tmp_ctx, exp_fmt, ap);
+	va_end(ap);
+	
+	if (!expression) {
+		talloc_free(tmp_ctx);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	sstate = talloc_zero(tmp_ctx, struct pdb_samba4_search_state);
+	if (sstate == NULL) {
+		talloc_free(tmp_ctx);
+		return false;
+	}
+
+	rc = dsdb_search(state->ldb, tmp_ctx, &res, NULL, LDB_SCOPE_SUBTREE, attrs, 0, "%s", expression);
+	if (rc != LDB_SUCCESS) {
+		talloc_free(tmp_ctx);
+		DEBUG(10, ("dsdb_search failed: %s\n",
+			   ldb_errstring(state->ldb)));
+		return false;
+	}
+
+	num_users = res->count;
+
+	sstate->entries = talloc_array(sstate, struct samr_displayentry,
+				       num_users);
+	if (sstate->entries == NULL) {
+		talloc_free(tmp_ctx);
+		DEBUG(10, ("talloc failed\n"));
+		return false;
+	}
+
+	sstate->num_entries = 0;
+
+	for (i=0; i<num_users; i++) {
+		struct samr_displayentry *e;
+		struct dom_sid *sid;
+
+		e = &sstate->entries[sstate->num_entries];
+
+		e->idx = sstate->num_entries;
+		sid = samdb_result_dom_sid(tmp_ctx, res->msgs[i], "objectSid");
+		if (!sid) {
+			talloc_free(tmp_ctx);
+			DEBUG(10, ("Could not pull SID\n"));
+			return false;
+		}
+		sid_peek_rid(sid, &e->rid);
+	
+		e->acct_flags = samdb_result_acct_flags(state->ldb, tmp_ctx,
+							res->msgs[i], 
+							ldb_get_default_basedn(state->ldb));
+		e->account_name = ldb_msg_find_attr_as_string(
+			res->msgs[i], "samAccountName", NULL);
+		if (e->account_name == NULL) {
+			talloc_free(tmp_ctx);
+			return false;
+		}
+		e->fullname = ldb_msg_find_attr_as_string(
+                        res->msgs[i], "displayName", "");
+		e->description = ldb_msg_find_attr_as_string(
+                        res->msgs[i], "description", "");
+
+		sstate->num_entries += 1;
+		if (sstate->num_entries >= num_users) {
+			break;
+		}
+	}
+	talloc_steal(sstate->entries, res->msgs);
+	search->private_data = talloc_steal(search, sstate);
+	search->next_entry = pdb_samba4_next_entry;
+	search->search_end = pdb_samba4_search_end;
+	*pstate = sstate;
+	talloc_free(tmp_ctx);
+	return true;
+}
+
+static bool pdb_samba4_search_users(struct pdb_methods *m,
+				 struct pdb_search *search,
+				 uint32 acct_flags)
+{
+	struct pdb_samba4_search_state *sstate;
+	bool ret;
+
+	ret = pdb_samba4_search_filter(m, search, &sstate, "(objectclass=user)");
+	if (!ret) {
+		return false;
+	}
+	sstate->acct_flags = acct_flags;
+	return true;
+}
+
+static bool pdb_samba4_search_groups(struct pdb_methods *m,
+				  struct pdb_search *search)
+{
+	struct pdb_samba4_search_state *sstate;
+	bool ret;
+
+	ret = pdb_samba4_search_filter(m, search, &sstate, 
+				       "(&(grouptype=%d)(objectclass=group))",
+				       GTYPE_SECURITY_GLOBAL_GROUP);
+	if (!ret) {
+		return false;
+	}
+	sstate->acct_flags = 0;
+	return true;
+}
+
+static bool pdb_samba4_search_aliases(struct pdb_methods *m,
+				   struct pdb_search *search,
+				   const struct dom_sid *sid)
+{
+	struct pdb_samba4_search_state *sstate;
+	bool ret;
+
+	ret = pdb_samba4_search_filter(m, search, &sstate, 
+				       "(&(grouptype=%d)(objectclass=group))",
+				       sid_check_is_builtin(sid)
+				       ? GTYPE_SECURITY_BUILTIN_LOCAL_GROUP
+				       : GTYPE_SECURITY_DOMAIN_LOCAL_GROUP);
+	if (!ret) {
+		return false;
+	}
+	sstate->acct_flags = 0;
+	return true;
+}
+
+static bool pdb_samba4_uid_to_sid(struct pdb_methods *m, uid_t uid,
+			       struct dom_sid *sid)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	NTSTATUS status;
+	struct id_map id_map;
+	struct id_map *id_maps[2];
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
+	if (!tmp_ctx) {
+		return false;
+	}
+
+	id_map.xid.id = uid;
+	id_map.xid.type = ID_TYPE_UID;
+	id_maps[0] = &id_map;
+	id_maps[1] = NULL;
+
+	status = idmap_xids_to_sids(state->idmap_ctx, tmp_ctx, id_maps);
+	if (!NT_STATUS_IS_OK(status)) {
+		talloc_free(tmp_ctx);
+		return false;
+	}
+	*sid = *id_map.sid;
+	talloc_free(tmp_ctx);
+	return true;
+}
+
+static bool pdb_samba4_gid_to_sid(struct pdb_methods *m, gid_t gid,
+			       struct dom_sid *sid)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	NTSTATUS status;
+	struct id_map id_map;
+	struct id_map *id_maps[2];
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
+	if (!tmp_ctx) {
+		return false;
+	}
+
+	id_map.xid.id = gid;
+	id_map.xid.type = ID_TYPE_GID;
+	id_maps[0] = &id_map;
+	id_maps[1] = NULL;
+
+	status = idmap_xids_to_sids(state->idmap_ctx, tmp_ctx, id_maps);
+	if (!NT_STATUS_IS_OK(status)) {
+		return false;
+	}
+	*sid = *id_map.sid;
+	talloc_free(tmp_ctx);
+	return true;
+}
+
+static bool pdb_samba4_sid_to_id(struct pdb_methods *m, const struct dom_sid *sid,
+				 union unid_t *id, enum lsa_SidType *type)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		m->private_data, struct pdb_samba4_state);
+	struct id_map id_map;
+	struct id_map *id_maps[2];
+	const char *attrs[] = { "objectClass", "groupType", NULL };
+	struct ldb_message *msg;
+	struct ldb_dn *dn;
+	NTSTATUS status;
+	int rc;
+	TALLOC_CTX *tmp_ctx = talloc_stackframe();
+	if (!tmp_ctx) {
+		return false;
+	}
+
+	ZERO_STRUCT(id_map);
+
+	dn = ldb_dn_new_fmt(tmp_ctx, state->ldb, "<SID=%s>", dom_sid_string(tmp_ctx, sid));
+	if (!dn || !ldb_dn_validate(dn)) {
+		talloc_free(tmp_ctx);
+		return false;
+	}
+	rc = dsdb_search_one(state->ldb, tmp_ctx, &msg, dn, LDB_SCOPE_BASE, attrs, 0, NULL);
+	if (rc == LDB_ERR_NO_SUCH_OBJECT) {
+		DEBUG(5, (__location__ "SID to Unix ID lookup failed because SID %s could not be found in the samdb\n", dom_sid_string(tmp_ctx, sid)));
+		talloc_free(tmp_ctx);
+		return false;
+	}
+	if (samdb_find_attribute(state->ldb, msg, "objectClass", "group")) {
+		uint32_t grouptype = ldb_msg_find_attr_as_uint(msg, "groupType", 0);
+		switch (grouptype) {
+		case GTYPE_SECURITY_BUILTIN_LOCAL_GROUP:
+		case GTYPE_SECURITY_DOMAIN_LOCAL_GROUP:
+			*type = SID_NAME_ALIAS;
+			break;
+		case GTYPE_SECURITY_GLOBAL_GROUP:
+			*type = SID_NAME_DOM_GRP;
+			break;
+		default:
+			talloc_free(tmp_ctx);
+			DEBUG(10, ("Could not pull groupType\n"));
+			return false;
+		}
+
+		*type = SID_NAME_DOM_GRP;
+
+		ZERO_STRUCT(id_map);
+		id_map.sid = sid;
+		id_maps[0] = &id_map;
+		id_maps[1] = NULL;
+		
+		status = idmap_sids_to_xids(state->idmap_ctx, tmp_ctx, id_maps);
+		talloc_free(tmp_ctx);
+		if (!NT_STATUS_IS_OK(status)) {
+			return false;
+		}
+		if (id_map.xid.type == ID_TYPE_GID || id_map.xid.type == ID_TYPE_BOTH) {
+			id->gid = id_map.xid.id;
+			return true;
+		}
+		return false;
+	} else if (samdb_find_attribute(state->ldb, msg, "objectClass", "user")) {
+		*type = SID_NAME_USER;
+		ZERO_STRUCT(id_map);
+		id_map.sid = sid;
+		id_maps[0] = &id_map;
+		id_maps[1] = NULL;
+		
+		status = idmap_sids_to_xids(state->idmap_ctx, tmp_ctx, id_maps);
+		talloc_free(tmp_ctx);
+		if (!NT_STATUS_IS_OK(status)) {
+			return false;
+		}
+		if (id_map.xid.type == ID_TYPE_UID || id_map.xid.type == ID_TYPE_BOTH) {
+			id->uid = id_map.xid.id;
+			return true;
+		}
+		return false;
+	}
+	DEBUG(5, (__location__ "SID to Unix ID lookup failed because SID %s was found, but was not a user or group\n", dom_sid_string(tmp_ctx, sid)));
+	talloc_free(tmp_ctx);
+	return false;
+}
+
+static uint32_t pdb_samba4_capabilities(struct pdb_methods *m)
+{
+	return PDB_CAP_STORE_RIDS | PDB_CAP_ADS;
+}
+
+static bool pdb_samba4_new_rid(struct pdb_methods *m, uint32 *rid)
+{
+	return false;
+}
+
+static bool pdb_samba4_get_trusteddom_pw(struct pdb_methods *m,
+				      const char *domain, char** pwd,
+				      struct dom_sid *sid,
+				      time_t *pass_last_set_time)
+{
+	return false;
+}
+
+static bool pdb_samba4_set_trusteddom_pw(struct pdb_methods *m,
+				      const char* domain, const char* pwd,
+				      const struct dom_sid *sid)
+{
+	return false;
+}
+
+static bool pdb_samba4_del_trusteddom_pw(struct pdb_methods *m,
+				      const char *domain)
+{
+	return false;
+}
+
+static NTSTATUS pdb_samba4_enum_trusteddoms(struct pdb_methods *m,
+					 TALLOC_CTX *mem_ctx,
+					 uint32 *num_domains,
+					 struct trustdom_info ***domains)
+{
+	*num_domains = 0;
+	*domains = NULL;
+	return NT_STATUS_OK;
+}
+
+static void pdb_samba4_init_methods(struct pdb_methods *m)
+{
+	m->name = "samba4";
+	m->get_domain_info = pdb_samba4_get_domain_info;
+	m->getsampwnam = pdb_samba4_getsampwnam;
+	m->getsampwsid = pdb_samba4_getsampwsid;
+	m->create_user = pdb_samba4_create_user;
+	m->delete_user = pdb_samba4_delete_user;
+	m->add_sam_account = pdb_samba4_add_sam_account;
+	m->update_sam_account = pdb_samba4_update_sam_account;
+	m->delete_sam_account = pdb_samba4_delete_sam_account;
+	m->rename_sam_account = pdb_samba4_rename_sam_account;
+	m->update_login_attempts = pdb_samba4_update_login_attempts;
+	m->getgrsid = pdb_samba4_getgrsid;
+	m->getgrgid = pdb_samba4_getgrgid;
+	m->getgrnam = pdb_samba4_getgrnam;
+	m->create_dom_group = pdb_samba4_create_dom_group;
+	m->delete_dom_group = pdb_samba4_delete_dom_group;
+	m->add_group_mapping_entry = pdb_samba4_add_group_mapping_entry;
+	m->update_group_mapping_entry = pdb_samba4_update_group_mapping_entry;
+	m->delete_group_mapping_entry =	pdb_samba4_delete_group_mapping_entry;
+	m->enum_group_mapping = pdb_samba4_enum_group_mapping;
+	m->enum_group_members = pdb_samba4_enum_group_members;
+	m->enum_group_memberships = pdb_samba4_enum_group_memberships;
+	m->set_unix_primary_group = pdb_samba4_set_unix_primary_group;
+	m->add_groupmem = pdb_samba4_add_groupmem;
+	m->del_groupmem = pdb_samba4_del_groupmem;
+	m->create_alias = pdb_samba4_create_alias;
+	m->delete_alias = pdb_samba4_delete_alias;
+	m->get_aliasinfo = pdb_default_get_aliasinfo;
+	m->add_aliasmem = pdb_samba4_add_aliasmem;
+	m->del_aliasmem = pdb_samba4_del_aliasmem;
+	m->enum_aliasmem = pdb_samba4_enum_aliasmem;
+	m->enum_alias_memberships = pdb_samba4_enum_alias_memberships;
+	m->lookup_rids = pdb_samba4_lookup_rids;
+	m->lookup_names = pdb_samba4_lookup_names;
+	m->get_account_policy = pdb_samba4_get_account_policy;
+	m->set_account_policy = pdb_samba4_set_account_policy;
+	m->get_seq_num = pdb_samba4_get_seq_num;
+	m->search_users = pdb_samba4_search_users;
+	m->search_groups = pdb_samba4_search_groups;
+	m->search_aliases = pdb_samba4_search_aliases;
+	m->uid_to_sid = pdb_samba4_uid_to_sid;
+	m->gid_to_sid = pdb_samba4_gid_to_sid;
+	m->sid_to_id = pdb_samba4_sid_to_id;
+	m->capabilities = pdb_samba4_capabilities;
+	m->new_rid = pdb_samba4_new_rid;
+	m->get_trusteddom_pw = pdb_samba4_get_trusteddom_pw;
+	m->set_trusteddom_pw = pdb_samba4_set_trusteddom_pw;
+	m->del_trusteddom_pw = pdb_samba4_del_trusteddom_pw;
+	m->enum_trusteddoms = pdb_samba4_enum_trusteddoms;
+}
+
+static void free_private_data(void **vp)
+{
+	struct pdb_samba4_state *state = talloc_get_type_abort(
+		*vp, struct pdb_samba4_state);
+	talloc_unlink(state, state->ldb);
+	return;
+}
+
+static NTSTATUS pdb_init_samba4(struct pdb_methods **pdb_method,
+			     const char *location)
+{
+	struct pdb_methods *m;
+	struct pdb_samba4_state *state;
+	NTSTATUS status;
+
+	if ( !NT_STATUS_IS_OK(status = make_pdb_method( &m )) ) {
+		return status;
+	}
+
+	state = talloc_zero(m, struct pdb_samba4_state);
+	if (state == NULL) {
+		goto nomem;
+	}
+	m->private_data = state;
+	m->free_private_data = free_private_data;
+	pdb_samba4_init_methods(m);
+
+	state->ev = s4_event_context_init(state);
+	if (!state->ev) {
+		DEBUG(10, ("s4_event_context_init failed\n"));
+		goto fail;
+	}
+
+	state->lp_ctx = loadparm_init_s3(state, loadparm_s3_context());
+	if (state->lp_ctx == NULL) {
+		DEBUG(10, ("loadparm_init_s3 failed\n"));
+		goto fail;
+	}
+
+	state->ldb = samdb_connect(state,
+				   state->ev,
+				   state->lp_ctx,
+				   system_session(state->lp_ctx), 0);
+
+	if (!state->ldb) {
+		DEBUG(10, ("samdb_connect failed\n"));
+		goto fail;
+	}
+
+	state->idmap_ctx = idmap_init(state, state->ev,
+				      state->lp_ctx);
+	if (!state->idmap_ctx) {
+		DEBUG(10, ("samdb_connect failed\n"));
+		goto fail;
+	}
+
+	*pdb_method = m;
+	return NT_STATUS_OK;
+nomem:
+	status = NT_STATUS_NO_MEMORY;
+fail:
+	TALLOC_FREE(m);
+	return status;
+}
+
+NTSTATUS pdb_samba4_init(void);
+NTSTATUS pdb_samba4_init(void)
+{
+	return smb_register_passdb(PASSDB_INTERFACE_VERSION, "samba4",
+				   pdb_init_samba4);
+}
diff --git a/source3/passdb/pdb_smbpasswd.c b/source3/passdb/pdb_smbpasswd.c
index 43d6eaf6645..2cfacd3a136 100644
--- a/source3/passdb/pdb_smbpasswd.c
+++ b/source3/passdb/pdb_smbpasswd.c
@@ -549,7 +549,7 @@ static struct smb_passwd *getsmbfilepwent(struct smbpasswd_privates *smbpasswd_s
 			}
 			if(*p == ':') {
 				p++;
-				if(*p && (StrnCaseCmp((char *)p, "LCT-", 4)==0)) {
+				if(*p && (strncasecmp_m((char *)p, "LCT-", 4)==0)) {
 					int i;
 					p += 4;
 					for(i = 0; i < 8; i++) {
@@ -975,7 +975,7 @@ This is no longer supported.!\n", pwd->smb_name));
 			p++;
 
 			/* We should be pointing at the LCT entry. */
-			if((linebuf_len > (PTR_DIFF(p, linebuf) + 13)) && (StrnCaseCmp((char *)p, "LCT-", 4) == 0)) {
+			if((linebuf_len > (PTR_DIFF(p, linebuf) + 13)) && (strncasecmp_m((char *)p, "LCT-", 4) == 0)) {
 				p += 4;
 				for(i = 0; i < 8; i++) {
 					if(p[i] == '\0' || !isxdigit(p[i])) {
@@ -990,7 +990,7 @@ This is no longer supported.!\n", pwd->smb_name));
 					 */
 					got_pass_last_set_time = True;
 				} /* i == 8 */
-			} /* *p && StrnCaseCmp() */
+			} /* *p && strncasecmp_m() */
 		} /* p == ':' */
 	} /* p == '[' */
 
@@ -1694,7 +1694,7 @@ static NTSTATUS pdb_init_smbpasswd( struct pdb_methods **pdb_method, const char
 
 	/* Setup private data and free function */
 
-	if ( !(privates = TALLOC_ZERO_P( *pdb_method, struct smbpasswd_privates )) ) {
+	if ( !(privates = talloc_zero( *pdb_method, struct smbpasswd_privates )) ) {
 		DEBUG(0, ("talloc() failed for smbpasswd private_data!\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/passdb/pdb_tdb.c b/source3/passdb/pdb_tdb.c
index 79c0ed196af..28461d0ac3d 100644
--- a/source3/passdb/pdb_tdb.c
+++ b/source3/passdb/pdb_tdb.c
@@ -27,6 +27,7 @@
 #include "passdb.h"
 #include "dbwrap.h"
 #include "../libcli/security/security.h"
+#include "util_tdb.h"
 
 #if 0 /* when made a module use this */
 
diff --git a/source3/passdb/pdb_wbc_sam.c b/source3/passdb/pdb_wbc_sam.c
index 64529c78961..91adc8108b3 100644
--- a/source3/passdb/pdb_wbc_sam.c
+++ b/source3/passdb/pdb_wbc_sam.c
@@ -112,7 +112,7 @@ static NTSTATUS pdb_wbc_sam_enum_group_memberships(struct pdb_methods *methods,
 		smb_panic("primary group missing");
 	}
 
-	*pp_sids = TALLOC_ARRAY(mem_ctx, struct dom_sid, *p_num_groups);
+	*pp_sids = talloc_array(mem_ctx, struct dom_sid, *p_num_groups);
 
 	if (*pp_sids == NULL) {
 		TALLOC_FREE(*pp_gids);
diff --git a/source3/passdb/proto.h b/source3/passdb/proto.h
index a64985c7091..8b95b729597 100644
--- a/source3/passdb/proto.h
+++ b/source3/passdb/proto.h
@@ -1,3 +1,41 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  passdb - password and authentication handling
+ *
+ *  Copyright (C) Andrew Tridgell		1992-1998
+ *  Copyright (C) Jeremy Allison 		1995-2009
+ *  Copyright (C) Luke Kenneth Casson Leighton	1996-1998
+ *  Copyright (C) Jean François Micouleau	1998-2001
+ *  Copyright (C) Gerald (Jerry) Carter		2000-2006
+ *  Copyright (C) Simo Sorce			2000-2003,2006
+ *  Copyright (C) Andrew Bartlett		2001-2002
+ *  Copyright (C) Shahms King			2001
+ *  Copyright (C) Jelmer Vernooij		2002
+ *  Copyright (C) Rafal Szczesniak		2002
+ *  Copyright (C) Stefan (metze) Metzmacher	2002-2003
+ *  Copyright (C) Guenther Deschner		2004-2005
+ *  Copyright (C) Jim McDonough (jmcd@us.ibm.com) 2004-2005
+ *  Copyright (C) Vince Brimhall		2004-2005
+ *  Copyright (C) Volker Lendecke 		2006
+ *  Copyright (C) Michael Adam			2007
+ *  Copyright (C) Dan Sledz			2009
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _PASSDB_PROTO_H_
+#define _PASSDB_PROTO_H_
 
 /* The following definitions come from passdb/account_pol.c  */
 
@@ -300,3 +338,5 @@ NTSTATUS pdb_tdbsam_init(void);
 
 NTSTATUS create_builtin_users(const struct dom_sid *sid);
 NTSTATUS create_builtin_administrators(const struct dom_sid *sid);
+
+#endif /* _PASSDB_PROTO_H_ */
diff --git a/source3/passdb/secrets.c b/source3/passdb/secrets.c
index c16d15807bd..c9101ca0506 100644
--- a/source3/passdb/secrets.c
+++ b/source3/passdb/secrets.c
@@ -30,6 +30,7 @@
 #include "secrets.h"
 #include "dbwrap.h"
 #include "../libcli/security/security.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_PASSDB
@@ -419,7 +420,7 @@ static int list_trusted_domain(struct db_record *rec, void *private_data)
 		return 0;
 	}
 
-	if (!(dom_info = TALLOC_P(state->domains, struct trustdom_info))) {
+	if (!(dom_info = talloc(state->domains, struct trustdom_info))) {
 		DEBUG(0, ("talloc failed\n"));
 		return 0;
 	}
@@ -458,7 +459,7 @@ NTSTATUS secrets_trusted_domains(TALLOC_CTX *mem_ctx, uint32 *num_domains,
 	 * exists
 	 */
 
-	if (!(state.domains = TALLOC_ARRAY(
+	if (!(state.domains = talloc_array(
 		      mem_ctx, struct trustdom_info *, 1))) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/passdb/wscript_build b/source3/passdb/wscript_build
index 79b143507f1..90ce2c4c4d2 100644
--- a/source3/passdb/wscript_build
+++ b/source3/passdb/wscript_build
@@ -14,6 +14,7 @@ bld.SAMBA3_SUBSYSTEM('pdb',
 bld.SAMBA3_MODULE('pdb_tdbsam',
                  subsystem='pdb',
                  source=PDB_TDBSAM_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('pdb_tdbsam'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('pdb_tdbsam'))
@@ -36,6 +37,7 @@ bld.SAMBA3_MODULE('pdb_ads',
 bld.SAMBA3_MODULE('pdb_smbpasswd',
                  subsystem='pdb',
                  source=PDB_SMBPASSWD_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('pdb_smbpasswd'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('pdb_smbpasswd'))
@@ -43,6 +45,17 @@ bld.SAMBA3_MODULE('pdb_smbpasswd',
 bld.SAMBA3_MODULE('pdb_wbc_sam',
                  subsystem='pdb',
                  source=PDB_WBC_SAM_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('pdb_wbc_sam'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('pdb_wbc_sam'))
+
+bld.SAMBA3_MODULE('pdb_samba4',
+                  subsystem='pdb',
+                  source='pdb_samba4.c',
+                  init_function='',
+                  deps='IDMAP samdb',
+                  allow_undefined_symbols=False,
+                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('pdb_samba4'),
+                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('pdb_samba4'))
+
diff --git a/source3/printing/load.h b/source3/printing/load.h
index df401a43772..4611e9748c2 100644
--- a/source3/printing/load.h
+++ b/source3/printing/load.h
@@ -1,4 +1,28 @@
+/*
+   Unix SMB/CIFS implementation.
+   load printer lists
+   Copyright (C) Andrew Tridgell 1992-2000
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _PRINTING_LOAD_H_
+#define _PRINTING_LOAD_H_
+
 /* The following definitions come from printing/load.c  */
 
 void load_printers(struct tevent_context *ev,
 		   struct messaging_context *msg_ctx);
+
+#endif /* _PRINTING_LOAD_H_ */
diff --git a/source3/printing/lpq_parse.c b/source3/printing/lpq_parse.c
index 16b9b09fef3..9104497f9f6 100644
--- a/source3/printing/lpq_parse.c
+++ b/source3/printing/lpq_parse.c
@@ -801,8 +801,9 @@ static bool parse_lpq_plp(char *line,print_queue_struct *buf,bool first)
 		fstring tmp;
 		char *p = strrchr_m(tok[6],'/');
 		if (p) {
+			size_t len = strlen(tok[6])+1;
 			fstrcpy(tmp,p+1);
-			fstrcpy(tok[6],tmp);
+			strlcpy(tok[6],tmp, len);
 		}
 	}
 
diff --git a/source3/printing/notify.c b/source3/printing/notify.c
index 836f7df3d11..b6b7ea045e9 100644
--- a/source3/printing/notify.c
+++ b/source3/printing/notify.c
@@ -25,6 +25,7 @@
 #include "nt_printing.h"
 #include "printing/notify.h"
 #include "messages.h"
+#include "util_tdb.h"
 
 static TALLOC_CTX *send_ctx;
 
@@ -253,9 +254,9 @@ static bool copy_notify2_msg( SPOOLSS_NOTIFY_MSG *to, SPOOLSS_NOTIFY_MSG *from )
 	memcpy( to, from, sizeof(SPOOLSS_NOTIFY_MSG) );
 	
 	if ( from->len ) {
-		to->notify.data = (char *)TALLOC_MEMDUP(send_ctx, from->notify.data, from->len );
+		to->notify.data = (char *)talloc_memdup(send_ctx, from->notify.data, from->len );
 		if ( !to->notify.data ) {
-			DEBUG(0,("copy_notify2_msg: TALLOC_MEMDUP() of size [%d] failed!\n", from->len ));
+			DEBUG(0,("copy_notify2_msg: talloc_memdup() of size [%d] failed!\n", from->len ));
 			return False;
 		}
 	}
@@ -304,7 +305,7 @@ static void send_spoolss_notify2_msg(struct tevent_context *ev,
 
 	/* Store the message on the pending queue. */
 
-	pnqueue = TALLOC_P(send_ctx, struct notify_queue);
+	pnqueue = talloc(send_ctx, struct notify_queue);
 	if (!pnqueue) {
 		DEBUG(0,("send_spoolss_notify2_msg: Out of memory.\n"));
 		return;
@@ -312,7 +313,7 @@ static void send_spoolss_notify2_msg(struct tevent_context *ev,
 
 	/* allocate a new msg structure and copy the fields */
 	
-	if ( !(pnqueue->msg = TALLOC_P(send_ctx, SPOOLSS_NOTIFY_MSG)) ) {
+	if ( !(pnqueue->msg = talloc(send_ctx, SPOOLSS_NOTIFY_MSG)) ) {
 		DEBUG(0,("send_spoolss_notify2_msg: talloc() of size [%lu] failed!\n", 
 			(unsigned long)sizeof(SPOOLSS_NOTIFY_MSG)));
 		return;
@@ -356,7 +357,7 @@ static void send_notify_field_values(struct tevent_context *ev,
 	if (!create_send_ctx())
 		return;
 
-	msg = TALLOC_P(send_ctx, struct spoolss_notify_msg);
+	msg = talloc(send_ctx, struct spoolss_notify_msg);
 	if (!msg)
 		return;
 
@@ -387,7 +388,7 @@ static void send_notify_field_buffer(struct tevent_context *ev,
 	if (!create_send_ctx())
 		return;
 
-	msg = TALLOC_P(send_ctx, struct spoolss_notify_msg);
+	msg = talloc(send_ctx, struct spoolss_notify_msg);
 	if (!msg)
 		return;
 
@@ -398,7 +399,7 @@ static void send_notify_field_buffer(struct tevent_context *ev,
 	msg->field = field;
 	msg->id = id;
 	msg->len = len;
-	msg->notify.data = CONST_DISCARD(char *,buffer);
+	msg->notify.data = discard_const_p(char, buffer);
 
 	send_spoolss_notify2_msg(ev, msg_ctx, msg);
 }
@@ -631,7 +632,7 @@ static bool print_notify_pid_list(const char *printername, TALLOC_CTX *mem_ctx,
 		return False;
 	tdb = pdb->tdb;
 
-	if (tdb_read_lock_bystring_with_timeout(tdb, NOTIFY_PID_LIST_KEY, 10) == -1) {
+	if (tdb_read_lock_bystring_with_timeout(tdb, NOTIFY_PID_LIST_KEY, 10) != 0) {
 		DEBUG(0,("print_notify_pid_list: Failed to lock printer %s database\n",
 					printername));
 		if (pdb)
@@ -649,7 +650,7 @@ static bool print_notify_pid_list(const char *printername, TALLOC_CTX *mem_ctx,
 	num_pids = data.dsize / 8;
 
 	if (num_pids) {
-		if ((pid_list = TALLOC_ARRAY(mem_ctx, pid_t, num_pids)) == NULL) {
+		if ((pid_list = talloc_array(mem_ctx, pid_t, num_pids)) == NULL) {
 			ret = False;
 			goto done;
 		}
diff --git a/source3/printing/nt_printing.c b/source3/printing/nt_printing.c
index a7539f6478f..45d4191ae76 100644
--- a/source3/printing/nt_printing.c
+++ b/source3/printing/nt_printing.c
@@ -20,12 +20,7 @@
  */
 
 #include "includes.h"
-#include "printing/pcap.h"
 #include "printing/nt_printing_tdb.h"
-#include "printing/nt_printing_migrate.h"
-#include "registry.h"
-#include "registry/reg_objects.h"
-#include "../librpc/gen_ndr/ndr_security.h"
 #include "../librpc/gen_ndr/ndr_spoolss.h"
 #include "rpc_server/spoolss/srv_spoolss_util.h"
 #include "nt_printing.h"
@@ -36,7 +31,7 @@
 #include "smbd/smbd.h"
 #include "auth.h"
 #include "messages.h"
-#include "ntdomain.h"
+#include "rpc_server/spoolss/srv_spoolss_nt.h"
 
 /* Map generic permissions to printer object specific permissions */
 
@@ -152,7 +147,7 @@ const char *get_short_archi(const char *long_archi)
         do {
                 i++;
         } while ( (archi_table[i].long_archi!=NULL ) &&
-                  StrCaseCmp(long_archi, archi_table[i].long_archi) );
+                  strcasecmp_m(long_archi, archi_table[i].long_archi) );
 
         if (archi_table[i].long_archi==NULL) {
                 DEBUGADD(10,("Unknown architecture [%s] !\n", long_archi));
@@ -578,7 +573,7 @@ static int file_version_is_newer(connection_struct *conn, fstring new_file, fstr
 /****************************************************************************
 Determine the correct cVersion associated with an architecture and driver
 ****************************************************************************/
-static uint32 get_correct_cversion(struct pipes_struct *p,
+static uint32 get_correct_cversion(struct auth_serversupplied_info *session_info,
 				   const char *architecture,
 				   const char *driverpath_in,
 				   WERROR *perr)
@@ -621,7 +616,7 @@ static uint32 get_correct_cversion(struct pipes_struct *p,
 
 	nt_status = create_conn_struct(talloc_tos(), &conn, printdollar_snum,
 				       lp_pathname(printdollar_snum),
-				       p->session_info, &oldcwd);
+				       session_info, &oldcwd);
 	if (!NT_STATUS_IS_OK(nt_status)) {
 		DEBUG(0,("get_correct_cversion: create_conn_struct "
 			 "returned %s\n", nt_errstr(nt_status)));
@@ -636,7 +631,7 @@ static uint32 get_correct_cversion(struct pipes_struct *p,
 		goto error_free_conn;
 	}
 
-	if (!become_user_by_session(conn, p->session_info)) {
+	if (!become_user_by_session(conn, session_info)) {
 		DEBUG(0, ("failed to become user\n"));
 		*perr = WERR_ACCESS_DENIED;
 		goto error_free_conn;
@@ -766,7 +761,7 @@ static uint32 get_correct_cversion(struct pipes_struct *p,
 } while (0);
 
 static WERROR clean_up_driver_struct_level(TALLOC_CTX *mem_ctx,
-					   struct pipes_struct *rpc_pipe,
+					   struct auth_serversupplied_info *session_info,
 					   const char *architecture,
 					   const char **driver_path,
 					   const char **data_file,
@@ -826,7 +821,7 @@ static WERROR clean_up_driver_struct_level(TALLOC_CTX *mem_ctx,
 	 *	NT2K: cversion=3
 	 */
 
-	*version = get_correct_cversion(rpc_pipe, short_architecture,
+	*version = get_correct_cversion(session_info, short_architecture,
 					*driver_path, &err);
 	if (*version == -1) {
 		return err;
@@ -839,12 +834,12 @@ static WERROR clean_up_driver_struct_level(TALLOC_CTX *mem_ctx,
 ****************************************************************************/
 
 WERROR clean_up_driver_struct(TALLOC_CTX *mem_ctx,
-			      struct pipes_struct *rpc_pipe,
+			      struct auth_serversupplied_info *session_info,
 			      struct spoolss_AddDriverInfoCtr *r)
 {
 	switch (r->level) {
 	case 3:
-		return clean_up_driver_struct_level(mem_ctx, rpc_pipe,
+		return clean_up_driver_struct_level(mem_ctx, session_info,
 						    r->info.info3->architecture,
 						    &r->info.info3->driver_path,
 						    &r->info.info3->data_file,
@@ -853,7 +848,7 @@ WERROR clean_up_driver_struct(TALLOC_CTX *mem_ctx,
 						    r->info.info3->dependent_files,
 						    &r->info.info3->version);
 	case 6:
-		return clean_up_driver_struct_level(mem_ctx, rpc_pipe,
+		return clean_up_driver_struct_level(mem_ctx, session_info,
 						    r->info.info6->architecture,
 						    &r->info.info6->driver_path,
 						    &r->info.info6->data_file,
@@ -924,7 +919,7 @@ static WERROR move_driver_file_to_download_area(TALLOC_CTX *mem_ctx,
 		}
 
 		/* Setup a synthetic smb_filename struct */
-		smb_fname_new = TALLOC_ZERO_P(mem_ctx, struct smb_filename);
+		smb_fname_new = talloc_zero(mem_ctx, struct smb_filename);
 		if (!smb_fname_new) {
 			ret = WERR_NOMEM;
 			goto out;
@@ -958,7 +953,7 @@ static WERROR move_driver_file_to_download_area(TALLOC_CTX *mem_ctx,
 	return ret;
 }
 
-WERROR move_driver_to_download_area(struct pipes_struct *p,
+WERROR move_driver_to_download_area(struct auth_serversupplied_info *session_info,
 				    struct spoolss_AddDriverInfoCtr *r)
 {
 	struct spoolss_AddDriverInfo3 *driver;
@@ -1004,7 +999,7 @@ WERROR move_driver_to_download_area(struct pipes_struct *p,
 
 	nt_status = create_conn_struct(talloc_tos(), &conn, printdollar_snum,
 				       lp_pathname(printdollar_snum),
-				       p->session_info, &oldcwd);
+				       session_info, &oldcwd);
 	if (!NT_STATUS_IS_OK(nt_status)) {
 		DEBUG(0,("move_driver_to_download_area: create_conn_struct "
 			 "returned %s\n", nt_errstr(nt_status)));
@@ -1019,7 +1014,7 @@ WERROR move_driver_to_download_area(struct pipes_struct *p,
 		goto err_free_conn;
 	}
 
-	if (!become_user_by_session(conn, p->session_info)) {
+	if (!become_user_by_session(conn, session_info)) {
 		DEBUG(0, ("failed to become user\n"));
 		err = WERR_ACCESS_DENIED;
 		goto err_free_conn;
diff --git a/source3/printing/nt_printing_ads.c b/source3/printing/nt_printing_ads.c
index ea414716136..6290417260f 100644
--- a/source3/printing/nt_printing_ads.c
+++ b/source3/printing/nt_printing_ads.c
@@ -28,6 +28,7 @@
 #include "krb5_env.h"
 #include "../libcli/registry/util_reg.h"
 #include "auth.h"
+#include "../librpc/ndr/libndr.h"
 
 #ifdef HAVE_ADS
 /*****************************************************************
@@ -110,7 +111,7 @@ static WERROR nt_printer_publish_ads(struct messaging_context *msg_ctx,
 	DEBUG(5, ("publishing printer %s\n", printer));
 
 	/* figure out where to publish */
-	ads_find_machine_acct(ads, &res, global_myname());
+	ads_find_machine_acct(ads, &res, lp_netbios_name());
 
 	/* We use ldap_get_dn here as we need the answer
 	 * in utf8 to call ldap_explode_dn(). JRA. */
@@ -213,7 +214,7 @@ static WERROR nt_printer_unpublish_ads(ADS_STRUCT *ads,
 
 	/* remove the printer from the directory */
 	ads_rc = ads_find_printer_on_server(ads, &res,
-					    printer, global_myname());
+					    printer, lp_netbios_name());
 
 	if (ADS_ERR_OK(ads_rc) && res && ads_count_replies(ads, res)) {
 		prt_dn = ads_get_dn(ads, talloc_tos(), res);
diff --git a/source3/printing/nt_printing_migrate.c b/source3/printing/nt_printing_migrate.c
index 55e49c973b4..ec66dfe8162 100644
--- a/source3/printing/nt_printing_migrate.c
+++ b/source3/printing/nt_printing_migrate.c
@@ -25,10 +25,10 @@
 #include "rpc_client/rpc_client.h"
 #include "librpc/gen_ndr/ndr_ntprinting.h"
 #include "librpc/gen_ndr/ndr_spoolss_c.h"
-#include "rpc_client/cli_spoolss.h"
 #include "librpc/gen_ndr/ndr_security.h"
 #include "rpc_server/rpc_ncacn_np.h"
 #include "auth.h"
+#include "util_tdb.h"
 
 #define FORMS_PREFIX "FORMS/"
 #define DRIVERS_PREFIX "DRIVERS/"
@@ -73,7 +73,7 @@ static NTSTATUS migrate_form(TALLOC_CTX *mem_ctx,
 
 	DEBUG(2, ("Migrating Form: %s\n", key_name));
 
-	srv_name_slash = talloc_asprintf(mem_ctx, "\\\\%s", global_myname());
+	srv_name_slash = talloc_asprintf(mem_ctx, "\\\\%s", lp_netbios_name());
 	if (srv_name_slash == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -164,7 +164,7 @@ static NTSTATUS migrate_driver(TALLOC_CTX *mem_ctx,
 
 	DEBUG(2, ("Migrating Printer Driver: %s\n", key_name));
 
-	srv_name_slash = talloc_asprintf(mem_ctx, "\\\\%s", global_myname());
+	srv_name_slash = talloc_asprintf(mem_ctx, "\\\\%s", lp_netbios_name());
 	if (srv_name_slash == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -360,13 +360,13 @@ static NTSTATUS migrate_printer(TALLOC_CTX *mem_ctx,
 	/* migrate printerdata */
 	for (j = 0; j < r.count; j++) {
 		char *valuename;
-		char *keyname;
+		const char *keyname;
 
 		if (r.printer_data[j].type == REG_NONE) {
 			continue;
 		}
 
-		keyname = CONST_DISCARD(char *, r.printer_data[j].name);
+		keyname = r.printer_data[j].name;
 		valuename = strchr(keyname, '\\');
 		if (valuename == NULL) {
 			continue;
@@ -532,7 +532,7 @@ static NTSTATUS migrate_internal(TALLOC_CTX *mem_ctx,
 				 struct rpc_pipe_client *pipe_hnd)
 {
 	const char *backup_suffix = ".bak";
-	TDB_DATA kbuf, newkey, dbuf;
+	TDB_DATA kbuf, dbuf;
 	TDB_CONTEXT *tdb;
 	NTSTATUS status;
 	int rc;
@@ -549,11 +549,11 @@ static NTSTATUS migrate_internal(TALLOC_CTX *mem_ctx,
 		return NT_STATUS_NO_SUCH_FILE;
 	}
 
-	for (kbuf = tdb_firstkey(tdb);
+	for (kbuf = tdb_firstkey_compat(tdb);
 	     kbuf.dptr;
-	     newkey = tdb_nextkey(tdb, kbuf), free(kbuf.dptr), kbuf = newkey)
+	     kbuf = tdb_nextkey_compat(tdb, kbuf))
 	{
-		dbuf = tdb_fetch(tdb, kbuf);
+		dbuf = tdb_fetch_compat(tdb, kbuf);
 		if (!dbuf.dptr) {
 			continue;
 		}
@@ -651,7 +651,7 @@ bool nt_printing_tdb_migrate(struct messaging_context *msg_ctx)
 		return false;
 	}
 
-	status = rpc_pipe_open_internal(tmp_ctx,
+	status = rpc_pipe_open_interface(tmp_ctx,
 					&ndr_table_spoolss.syntax_id,
 					session_info,
 					NULL,
diff --git a/source3/printing/nt_printing_tdb.c b/source3/printing/nt_printing_tdb.c
index 98ac46797fc..028dc0b1da9 100644
--- a/source3/printing/nt_printing_tdb.c
+++ b/source3/printing/nt_printing_tdb.c
@@ -26,6 +26,7 @@
 #include "librpc/gen_ndr/spoolss.h"
 #include "librpc/gen_ndr/ndr_security.h"
 #include "libcli/security/security.h"
+#include "util_tdb.h"
 
 #define FORMS_PREFIX "FORMS/"
 #define DRIVERS_PREFIX "DRIVERS/"
@@ -87,25 +88,25 @@ static TDB_DATA make_printers_secdesc_tdbkey(TALLOC_CTX *ctx,
 
 static bool upgrade_to_version_3(void)
 {
-	TDB_DATA kbuf, newkey, dbuf;
+	TDB_DATA kbuf, dbuf;
 
 	DEBUG(0,("upgrade_to_version_3: upgrading print tdb's to version 3\n"));
 
-	for (kbuf = tdb_firstkey(tdb_drivers); kbuf.dptr;
-			newkey = tdb_nextkey(tdb_drivers, kbuf), free(kbuf.dptr), kbuf=newkey) {
+	for (kbuf = tdb_firstkey_compat(tdb_drivers); kbuf.dptr;
+			kbuf = tdb_nextkey_compat(tdb_drivers, kbuf)) {
 
-		dbuf = tdb_fetch(tdb_drivers, kbuf);
+		dbuf = tdb_fetch_compat(tdb_drivers, kbuf);
 
 		if (strncmp((const char *)kbuf.dptr, FORMS_PREFIX, strlen(FORMS_PREFIX)) == 0) {
 			DEBUG(0,("upgrade_to_version_3:moving form\n"));
 			if (tdb_store(tdb_forms, kbuf, dbuf, TDB_REPLACE) != 0) {
 				SAFE_FREE(dbuf.dptr);
-				DEBUG(0,("upgrade_to_version_3: failed to move form. Error (%s).\n", tdb_errorstr(tdb_forms)));
+				DEBUG(0,("upgrade_to_version_3: failed to move form. Error (%s).\n", tdb_errorstr_compat(tdb_forms)));
 				return False;
 			}
 			if (tdb_delete(tdb_drivers, kbuf) != 0) {
 				SAFE_FREE(dbuf.dptr);
-				DEBUG(0,("upgrade_to_version_3: failed to delete form. Error (%s)\n", tdb_errorstr(tdb_drivers)));
+				DEBUG(0,("upgrade_to_version_3: failed to delete form. Error (%s)\n", tdb_errorstr_compat(tdb_drivers)));
 				return False;
 			}
 		}
@@ -114,12 +115,12 @@ static bool upgrade_to_version_3(void)
 			DEBUG(0,("upgrade_to_version_3:moving printer\n"));
 			if (tdb_store(tdb_printers, kbuf, dbuf, TDB_REPLACE) != 0) {
 				SAFE_FREE(dbuf.dptr);
-				DEBUG(0,("upgrade_to_version_3: failed to move printer. Error (%s)\n", tdb_errorstr(tdb_printers)));
+				DEBUG(0,("upgrade_to_version_3: failed to move printer. Error (%s)\n", tdb_errorstr_compat(tdb_printers)));
 				return False;
 			}
 			if (tdb_delete(tdb_drivers, kbuf) != 0) {
 				SAFE_FREE(dbuf.dptr);
-				DEBUG(0,("upgrade_to_version_3: failed to delete printer. Error (%s)\n", tdb_errorstr(tdb_drivers)));
+				DEBUG(0,("upgrade_to_version_3: failed to delete printer. Error (%s)\n", tdb_errorstr_compat(tdb_drivers)));
 				return False;
 			}
 		}
@@ -128,12 +129,12 @@ static bool upgrade_to_version_3(void)
 			DEBUG(0,("upgrade_to_version_3:moving secdesc\n"));
 			if (tdb_store(tdb_printers, kbuf, dbuf, TDB_REPLACE) != 0) {
 				SAFE_FREE(dbuf.dptr);
-				DEBUG(0,("upgrade_to_version_3: failed to move secdesc. Error (%s)\n", tdb_errorstr(tdb_printers)));
+				DEBUG(0,("upgrade_to_version_3: failed to move secdesc. Error (%s)\n", tdb_errorstr_compat(tdb_printers)));
 				return False;
 			}
 			if (tdb_delete(tdb_drivers, kbuf) != 0) {
 				SAFE_FREE(dbuf.dptr);
-				DEBUG(0,("upgrade_to_version_3: failed to delete secdesc. Error (%s)\n", tdb_errorstr(tdb_drivers)));
+				DEBUG(0,("upgrade_to_version_3: failed to delete secdesc. Error (%s)\n", tdb_errorstr_compat(tdb_drivers)));
 				return False;
 			}
 		}
@@ -161,7 +162,6 @@ static int sec_desc_upg_fn( TDB_CONTEXT *the_tdb, TDB_DATA key,
 	struct security_descriptor *sec, *new_sec;
 	TALLOC_CTX *ctx = state;
 	int result, i;
-	uint32 sd_size;
 	size_t size_new_sec;
 
 	if (!data.dptr || data.dsize == 0) {
@@ -235,9 +235,6 @@ static int sec_desc_upg_fn( TDB_CONTEXT *the_tdb, TDB_DATA key,
 
 	/* store it back */
 
-	sd_size = ndr_size_security_descriptor(sd_store->sd, 0)
-		+ sizeof(struct sec_desc_buf);
-
 	status = marshall_sec_desc_buf(ctx, sd_store, &data.dptr, &data.dsize);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0,("sec_desc_upg_fn: Failed to parse new sec_desc for %s\n", key.dptr ));
@@ -248,7 +245,7 @@ static int sec_desc_upg_fn( TDB_CONTEXT *the_tdb, TDB_DATA key,
 
 	/* 0 to continue and non-zero to stop traversal */
 
-	return (result == -1);
+	return (result != 0);
 }
 
 /*******************************************************************
@@ -269,7 +266,7 @@ static bool upgrade_to_version_4(void)
 
 	talloc_destroy( ctx );
 
-	return ( result != -1 );
+	return ( result >= 0 );
 }
 
 /*******************************************************************
@@ -337,7 +334,7 @@ static bool upgrade_to_version_5(void)
 
 	talloc_destroy( ctx );
 
-	return ( result != -1 );
+	return ( result >= 0 );
 }
 
 bool nt_printing_tdb_upgrade(void)
diff --git a/source3/printing/pcap.c b/source3/printing/pcap.c
index 7208f4b7f8c..62db4f5efc1 100644
--- a/source3/printing/pcap.c
+++ b/source3/printing/pcap.c
@@ -43,10 +43,11 @@
 struct pcap_cache {
 	char *name;
 	char *comment;
+	char *location;
 	struct pcap_cache *next;
 };
 
-bool pcap_cache_add_specific(struct pcap_cache **ppcache, const char *name, const char *comment)
+bool pcap_cache_add_specific(struct pcap_cache **ppcache, const char *name, const char *comment, const char *location)
 {
 	struct pcap_cache *p;
 
@@ -55,9 +56,11 @@ bool pcap_cache_add_specific(struct pcap_cache **ppcache, const char *name, cons
 
 	p->name = SMB_STRDUP(name);
 	p->comment = (comment && *comment) ? SMB_STRDUP(comment) : NULL;
+	p->location = (location && *location) ? SMB_STRDUP(location) : NULL;
 
-	DEBUG(11,("pcap_cache_add_specific: Adding name %s info %s\n",
-		p->name, p->comment ? p->comment : ""));
+	DEBUG(11,("pcap_cache_add_specific: Adding name %s info %s, location: %s\n",
+		p->name, p->comment ? p->comment : "",
+		p->location ? p->location : ""));
 
 	p->next = *ppcache;
 	*ppcache = p;
@@ -74,17 +77,18 @@ void pcap_cache_destroy_specific(struct pcap_cache **pp_cache)
 
 		SAFE_FREE(p->name);
 		SAFE_FREE(p->comment);
+		SAFE_FREE(p->location);
 		SAFE_FREE(p);
 	}
 	*pp_cache = NULL;
 }
 
-bool pcap_cache_add(const char *name, const char *comment)
+bool pcap_cache_add(const char *name, const char *comment, const char *location)
 {
 	NTSTATUS status;
 	time_t t = time_mono(NULL);
 
-	status = printer_list_set_printer(talloc_tos(), name, comment, t);
+	status = printer_list_set_printer(talloc_tos(), name, comment, location, t);
 	return NT_STATUS_IS_OK(status);
 }
 
@@ -109,7 +113,7 @@ bool pcap_cache_replace(const struct pcap_cache *pcache)
 	}
 
 	for (p = pcache; p; p = p->next) {
-		pcap_cache_add(p->name, p->comment);
+		pcap_cache_add(p->name, p->comment, p->location);
 	}
 
 	status = printer_list_clean_old();
@@ -205,7 +209,7 @@ bool pcap_printername_ok(const char *printername)
 {
 	NTSTATUS status;
 
-	status = printer_list_get_printer(talloc_tos(), printername, NULL, 0);
+	status = printer_list_get_printer(talloc_tos(), printername, NULL, NULL, 0);
 	return NT_STATUS_IS_OK(status);
 }
 
@@ -214,18 +218,18 @@ run a function on each printer name in the printcap file.
 ***************************************************************************/
 
 void pcap_printer_fn_specific(const struct pcap_cache *pc,
-			void (*fn)(const char *, const char *, void *),
+			void (*fn)(const char *, const char *, const char *, void *),
 			void *pdata)
 {
 	const struct pcap_cache *p;
 
 	for (p = pc; p != NULL; p = p->next)
-		fn(p->name, p->comment, pdata);
+		fn(p->name, p->comment, p->location, pdata);
 
 	return;
 }
 
-void pcap_printer_fn(void (*fn)(const char *, const char *, void *), void *pdata)
+void pcap_printer_fn(void (*fn)(const char *, const char *, const char *, void *), void *pdata)
 {
 	NTSTATUS status;
 
diff --git a/source3/printing/pcap.h b/source3/printing/pcap.h
index e24142e8b2a..70562137ac8 100644
--- a/source3/printing/pcap.h
+++ b/source3/printing/pcap.h
@@ -2,6 +2,16 @@
    Unix SMB/CIFS implementation.
    printcap headers
 
+   Copyright (C) Karl Auer 1993-1998
+
+   Re-working by Martin Kiff, 1994
+
+   Re-written again by Andrew Tridgell
+
+   Modified for SVID support by Norm Jacobs, 1997
+
+   Modified for CUPS support by Michael Sweet, 1999
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
@@ -16,17 +26,20 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#ifndef _PRINTING_PCAP_H_
+#define _PRINTING_PCAP_H_
+
 struct pcap_cache;
 
 /* The following definitions come from printing/pcap.c  */
 
-bool pcap_cache_add_specific(struct pcap_cache **ppcache, const char *name, const char *comment);
+bool pcap_cache_add_specific(struct pcap_cache **ppcache, const char *name, const char *comment, const char *location);
 void pcap_cache_destroy_specific(struct pcap_cache **ppcache);
-bool pcap_cache_add(const char *name, const char *comment);
+bool pcap_cache_add(const char *name, const char *comment, const char *location);
 bool pcap_cache_loaded(void);
 bool pcap_cache_replace(const struct pcap_cache *cache);
-void pcap_printer_fn_specific(const struct pcap_cache *, void (*fn)(const char *, const char *, void *), void *);
-void pcap_printer_fn(void (*fn)(const char *, const char *, void *), void *);
+void pcap_printer_fn_specific(const struct pcap_cache *, void (*fn)(const char *, const char *, const char *, void *), void *);
+void pcap_printer_fn(void (*fn)(const char *, const char *, const char *, void *), void *);
 
 void pcap_cache_reload(struct tevent_context *ev,
 		       struct messaging_context *msg_ctx,
@@ -44,10 +57,6 @@ bool cups_cache_reload(struct tevent_context *ev,
 		       struct messaging_context *msg_ctx,
 		       void (*post_cache_fill_fn)(struct tevent_context *,
 						  struct messaging_context *));
-bool cups_pull_comment_location(TALLOC_CTX *mem_ctx,
-				const char *printername,
-				char **comment,
-				char **location);
 
 /* The following definitions come from printing/print_iprint.c  */
 
@@ -59,3 +68,5 @@ bool sysv_cache_reload(void);
 
 /* The following definitions come from printing/print_standard.c  */
 bool std_pcap_cache_reload(const char *pcap_name);
+
+#endif /* _PRINTING_PCAP_H_ */
diff --git a/source3/printing/print_aix.c b/source3/printing/print_aix.c
index b7198d36322..23d9a86fe37 100644
--- a/source3/printing/print_aix.c
+++ b/source3/printing/print_aix.c
@@ -86,7 +86,7 @@ bool aix_cache_reload(void)
 				/* name is found without stanza device  */
 				/* probably a good printer ???		*/
 				iEtat = 0;
-				if (!pcap_cache_add(name, NULL)) {
+				if (!pcap_cache_add(name, NULL, NULL)) {
 					SAFE_FREE(line);
 					x_fclose(pfile);
 					TALLOC_FREE(ctx);
@@ -101,7 +101,7 @@ bool aix_cache_reload(void)
 			} else if (strstr_m(line, "device")) {
 				/* it's a good virtual printer */
 				iEtat = 0;
-				if (!pcap_cache_add(name, NULL)) {
+				if (!pcap_cache_add(name, NULL, NULL)) {
 					SAFE_FREE(line);
 					x_fclose(pfile);
 					TALLOC_FREE(ctx);
diff --git a/source3/printing/print_cups.c b/source3/printing/print_cups.c
index da42cb6f85d..b8bbddfebd0 100644
--- a/source3/printing/print_cups.c
+++ b/source3/printing/print_cups.c
@@ -163,6 +163,7 @@ static bool process_cups_printers_response(TALLOC_CTX *mem_ctx,
 	ipp_attribute_t	*attr;
 	char *name;
 	char *info;
+	char *location = NULL;
 	struct pcap_printer *printer;
 	bool ret_ok = false;
 
@@ -206,6 +207,16 @@ static bool process_cups_printers_response(TALLOC_CTX *mem_ctx,
 				}
 			}
 
+			if (strcmp(attr->name, "printer-location") == 0 &&
+			    attr->value_tag == IPP_TAG_TEXT) {
+				if (!pull_utf8_talloc(mem_ctx,
+						&location,
+						attr->values[0].string.text,
+						&size)) {
+					goto err_out;
+				}
+			}
+
 			attr = attr->next;
 		}
 
@@ -229,6 +240,7 @@ static bool process_cups_printers_response(TALLOC_CTX *mem_ctx,
 		pcap_data->printers = printer;
 		pcap_data->printers[pcap_data->count].name = name;
 		pcap_data->printers[pcap_data->count].info = info;
+		pcap_data->printers[pcap_data->count].location = location;
 		pcap_data->count++;
 	}
 
@@ -252,7 +264,8 @@ static bool cups_cache_reload_async(int fd)
 	static const char *requested[] =/* Requested attributes */
 			{
 			  "printer-name",
-			  "printer-info"
+			  "printer-info",
+			  "printer-location"
 			};
 	bool ret = False;
 	enum ndr_err_code ndr_ret;
@@ -478,7 +491,8 @@ static void cups_async_callback(struct event_context *event_ctx,
 	for (i = 0; i < pcap_data.count; i++) {
 		ret_ok = pcap_cache_add_specific(&tmp_pcap_cache,
 						 pcap_data.printers[i].name,
-						 pcap_data.printers[i].info);
+						 pcap_data.printers[i].info,
+						 pcap_data.printers[i].location);
 		if (!ret_ok) {
 			DEBUG(0, ("failed to add to tmp pcap cache\n"));
 			goto err_out;
@@ -510,7 +524,7 @@ bool cups_cache_reload(struct tevent_context *ev,
 	struct cups_async_cb_args *cb_args;
 	int *p_pipe_fd;
 
-	cb_args = TALLOC_P(NULL, struct cups_async_cb_args);
+	cb_args = talloc(NULL, struct cups_async_cb_args);
 	if (cb_args == NULL) {
 		return false;
 	}
@@ -1573,178 +1587,6 @@ struct printif	cups_printif =
 	cups_job_submit,
 };
 
-bool cups_pull_comment_location(TALLOC_CTX *mem_ctx,
-				const char *printername,
-				char **comment,
-				char **location)
-{
-	TALLOC_CTX *frame = talloc_stackframe();
-	http_t		*http = NULL;		/* HTTP connection to server */
-	ipp_t		*request = NULL,	/* IPP Request */
-			*response = NULL;	/* IPP Response */
-	ipp_attribute_t	*attr;		/* Current attribute */
-	cups_lang_t	*language = NULL;	/* Default language */
-	char		uri[HTTP_MAX_URI];
-	char *server = NULL;
-	char *sharename = NULL;
-	char *name = NULL;
-	static const char *requested[] =/* Requested attributes */
-			{
-			  "printer-name",
-			  "printer-info",
-			  "printer-location"
-			};
-	bool ret = False;
-	size_t size;
-
-	DEBUG(5, ("pulling %s location\n", printername));
-
-	/*
-	 * Make sure we don't ask for passwords...
-	 */
-
-        cupsSetPasswordCB(cups_passwd_cb);
-
-	/*
-	 * Try to connect to the server...
-	 */
-
-	if ((http = cups_connect(frame)) == NULL) {
-		goto out;
-	}
-
-	request = ippNew();
-
-	request->request.op.operation_id = IPP_GET_PRINTER_ATTRIBUTES;
-	request->request.op.request_id   = 1;
-
-	language = cupsLangDefault();
-
-	ippAddString(request, IPP_TAG_OPERATION, IPP_TAG_CHARSET,
-                     "attributes-charset", NULL, "utf-8");
-
-	ippAddString(request, IPP_TAG_OPERATION, IPP_TAG_LANGUAGE,
-                     "attributes-natural-language", NULL, language->language);
-
-	if (lp_cups_server() != NULL && strlen(lp_cups_server()) > 0) {
-		if (!push_utf8_talloc(frame, &server, lp_cups_server(), &size)) {
-			goto out;
-		}
-	} else {
-		server = talloc_strdup(frame,cupsServer());
-	}
-	if (server) {
-		goto out;
-	}
-	if (!push_utf8_talloc(frame, &sharename, printername, &size)) {
-		goto out;
-	}
-	slprintf(uri, sizeof(uri) - 1, "ipp://%s/printers/%s",
-		 server, sharename);
-
-	ippAddString(request, IPP_TAG_OPERATION, IPP_TAG_URI,
-                     "printer-uri", NULL, uri);
-
-        ippAddStrings(request, IPP_TAG_OPERATION, IPP_TAG_NAME,
-	              "requested-attributes",
-		      (sizeof(requested) / sizeof(requested[0])),
-		      NULL, requested);
-
-	/*
-	 * Do the request and get back a response...
-	 */
-
-	if ((response = cupsDoRequest(http, request, "/")) == NULL) {
-		DEBUG(0,("Unable to get printer attributes - %s\n",
-			 ippErrorString(cupsLastError())));
-		goto out;
-	}
-
-	for (attr = response->attrs; attr != NULL;) {
-		/*
-		 * Skip leading attributes until we hit a printer...
-		 */
-
-		while (attr != NULL && attr->group_tag != IPP_TAG_PRINTER)
-			attr = attr->next;
-
-		if (attr == NULL)
-        		break;
-
-		/*
-		 * Pull the needed attributes from this printer...
-		 */
-
-		while ( attr && (attr->group_tag == IPP_TAG_PRINTER) ) {
-        		if (strcmp(attr->name, "printer-name") == 0 &&
-			    attr->value_tag == IPP_TAG_NAME) {
-				if (!pull_utf8_talloc(frame,
-						&name,
-						attr->values[0].string.text,
-						&size)) {
-					goto out;
-				}
-			}
-
-			/* Grab the comment if we don't have one */
-        		if ( (strcmp(attr->name, "printer-info") == 0)
-			     && (attr->value_tag == IPP_TAG_TEXT))
-			{
-				if (!pull_utf8_talloc(mem_ctx,
-						comment,
-						attr->values[0].string.text,
-						&size)) {
-					goto out;
-				}
-				DEBUG(5,("cups_pull_comment_location: Using cups comment: %s\n",
-					 *comment));
-			}
-
-			/* Grab the location if we don't have one */
-			if ( (strcmp(attr->name, "printer-location") == 0)
-			     && (attr->value_tag == IPP_TAG_TEXT))
-			{
-				if (!pull_utf8_talloc(mem_ctx,
-						location,
-						attr->values[0].string.text,
-						&size)) {
-					goto out;
-				}
-				DEBUG(5,("cups_pull_comment_location: Using cups location: %s\n",
-					 *location));
-			}
-
-        		attr = attr->next;
-		}
-
-		/*
-		 * We have everything needed...
-		 */
-
-		if (name != NULL)
-			break;
-	}
-
-	ret = True;
-
- out:
-	if (response)
-		ippDelete(response);
-
-	if (request) {
-		ippDelete(request);
-	}
-
-	if (language)
-		cupsLangFree(language);
-
-	if (http)
-		httpClose(http);
-
-	TALLOC_FREE(frame);
-	return ret;
-}
-
 #else
  /* this keeps fussy compilers happy */
  void print_cups_dummy(void);
diff --git a/source3/printing/print_generic.c b/source3/printing/print_generic.c
index b925bedaceb..14f4c6dbe79 100644
--- a/source3/printing/print_generic.c
+++ b/source3/printing/print_generic.c
@@ -155,17 +155,17 @@ static int generic_job_submit(int snum, struct printjob *pjob)
 
 	/* we print from the directory path to give the best chance of
            parsing the lpq output */
-	current_directory = TALLOC_ARRAY(ctx,
-					char,
-					PATH_MAX+1);
-	if (!current_directory) {
-		return -1;
-	}
-	wd = sys_getwd(current_directory);
+	wd = sys_getwd();
 	if (!wd) {
 		return -1;
 	}
 
+	current_directory = talloc_strdup(ctx, wd);
+	SAFE_FREE(wd);
+
+	if (!current_directory) {
+		return -1;
+	}
 	print_directory = talloc_strdup(ctx, pjob->filename);
 	if (!print_directory) {
 		return -1;
@@ -205,7 +205,7 @@ static int generic_job_submit(int snum, struct printjob *pjob)
 
  out:
 
-	if (chdir(wd) == -1) {
+	if (chdir(current_directory) == -1) {
 		smb_panic("chdir failed in generic_job_submit");
 	}
 	TALLOC_FREE(current_directory);
diff --git a/source3/printing/print_iprint.c b/source3/printing/print_iprint.c
index ea375da3b70..1392cba7b8f 100644
--- a/source3/printing/print_iprint.c
+++ b/source3/printing/print_iprint.c
@@ -297,7 +297,7 @@ static int iprint_cache_add_printer(http_t *http,
 		*/
 
 		if (name != NULL && !secure && smb_enabled) 
-			pcap_cache_add(name, info);
+			pcap_cache_add(name, info, NULL);
 	}
 
  out:
diff --git a/source3/printing/print_standard.c b/source3/printing/print_standard.c
index a8460935c99..c4f9c5b7ae3 100644
--- a/source3/printing/print_standard.c
+++ b/source3/printing/print_standard.c
@@ -93,6 +93,10 @@ bool std_pcap_cache_reload(const char *pcap_name)
 
 			has_punctuation = (strchr_m(p, ' ') ||
 			                   strchr_m(p, '\t') ||
+					   strchr_m(p, '"') ||
+					   strchr_m(p, '\'') ||
+					   strchr_m(p, ';') ||
+					   strchr_m(p, ',') ||
 			                   strchr_m(p, '(') ||
 			                   strchr_m(p, ')'));
 
@@ -101,11 +105,7 @@ bool std_pcap_cache_reload(const char *pcap_name)
 				continue;
 			}
 
-			if (strlen(p) <= MAXPRINTERLEN &&
-			    strlen(p) > strlen(name) && !has_punctuation) {
-				if (!*comment) {
-					strlcpy(comment, name, sizeof(comment));
-				}
+			if (strlen(p) <= MAXPRINTERLEN && *name == '\0' && !has_punctuation) {
 				strlcpy(name, p, sizeof(name));
 				continue;
 			}
@@ -117,10 +117,7 @@ bool std_pcap_cache_reload(const char *pcap_name)
 			}
 		}
 
-		comment[60] = 0;
-		name[MAXPRINTERLEN] = 0;
-
-		if (*name && !pcap_cache_add(name, comment)) {
+		if (*name && !pcap_cache_add(name, comment, NULL)) {
 			x_fclose(pcap_file);
 			return false;
 		}
diff --git a/source3/printing/print_svid.c b/source3/printing/print_svid.c
index 4929be14bc7..222649308ca 100644
--- a/source3/printing/print_svid.c
+++ b/source3/printing/print_svid.c
@@ -111,7 +111,7 @@ bool sysv_cache_reload(void)
 			*tmp = '\0';
 		
 		/* add it to the cache */
-		if (!pcap_cache_add(name, NULL)) {
+		if (!pcap_cache_add(name, NULL, NULL)) {
 			TALLOC_FREE(lines);
 			return False;
 		}
diff --git a/source3/printing/printer_list.c b/source3/printing/printer_list.c
index 417eea777bb..d28278d069b 100644
--- a/source3/printing/printer_list.c
+++ b/source3/printing/printer_list.c
@@ -20,13 +20,14 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "dbwrap.h"
+#include "util_tdb.h"
 #include "printer_list.h"
 
 #define PL_DB_NAME() lock_path("printer_list.tdb")
 #define PL_KEY_PREFIX "PRINTERLIST/PRN/"
 #define PL_KEY_FORMAT PL_KEY_PREFIX"%s"
 #define PL_TIMESTAMP_KEY "PRINTERLIST/GLOBAL/LAST_REFRESH"
-#define PL_DATA_FORMAT "ddPP"
+#define PL_DATA_FORMAT "ddPPP"
 #define PL_TSTAMP_FORMAT "dd"
 
 static struct db_context *get_printer_list_db(void)
@@ -64,6 +65,7 @@ bool printer_list_parent_init(void)
 NTSTATUS printer_list_get_printer(TALLOC_CTX *mem_ctx,
 				  const char *name,
 				  const char **comment,
+				  const char **location,
 				  time_t *last_refresh)
 {
 	struct db_context *db;
@@ -72,6 +74,7 @@ NTSTATUS printer_list_get_printer(TALLOC_CTX *mem_ctx,
 	uint32_t time_h, time_l;
 	char *nstr = NULL;
 	char *cstr = NULL;
+	char *lstr = NULL;
 	NTSTATUS status;
 	int ret;
 
@@ -95,7 +98,7 @@ NTSTATUS printer_list_get_printer(TALLOC_CTX *mem_ctx,
 
 	ret = tdb_unpack(data.dptr, data.dsize,
 			 PL_DATA_FORMAT,
-			 &time_h, &time_l, &nstr, &cstr);
+			 &time_h, &time_l, &nstr, &cstr, &lstr);
 	if (ret == -1) {
 		DEBUG(1, ("Failed to un pack printer data"));
 		status = NT_STATUS_INTERNAL_DB_CORRUPTION;
@@ -115,6 +118,15 @@ NTSTATUS printer_list_get_printer(TALLOC_CTX *mem_ctx,
 		}
 	}
 
+	if (location) {
+		*location = talloc_strdup(mem_ctx, lstr);
+		if (*location == NULL) {
+			DEBUG(1, ("Failed to strdup location!\n"));
+			status = NT_STATUS_NO_MEMORY;
+			goto done;
+		}
+	}
+
 	status = NT_STATUS_OK;
 
 done:
@@ -127,6 +139,7 @@ done:
 NTSTATUS printer_list_set_printer(TALLOC_CTX *mem_ctx,
 				  const char *name,
 				  const char *comment,
+				  const char *location,
 				  time_t last_refresh)
 {
 	struct db_context *db;
@@ -135,6 +148,7 @@ NTSTATUS printer_list_set_printer(TALLOC_CTX *mem_ctx,
 	uint64_t time_64;
 	uint32_t time_h, time_l;
 	const char *str = NULL;
+	const char *str2 = NULL;
 	NTSTATUS status;
 	int len;
 
@@ -155,11 +169,18 @@ NTSTATUS printer_list_set_printer(TALLOC_CTX *mem_ctx,
 		str = "";
 	}
 
+	if (location) {
+		str2 = location;
+	} else {
+		str2 = "";
+	}
+
+
 	time_64 = last_refresh;
 	time_l = time_64 & 0xFFFFFFFFL;
 	time_h = time_64 >> 32;
 
-	len = tdb_pack(NULL, 0, PL_DATA_FORMAT, time_h, time_l, name, str);
+	len = tdb_pack(NULL, 0, PL_DATA_FORMAT, time_h, time_l, name, str, str2);
 
 	data.dptr = talloc_array(key, uint8_t, len);
 	if (!data.dptr) {
@@ -288,6 +309,7 @@ static int printer_list_clean_fn(struct db_record *rec, void *private_data)
 	time_t refresh;
 	char *name;
 	char *comment;
+	char *location;
 	int ret;
 
 	/* skip anything that does not contain PL_DATA_FORMAT data */
@@ -297,7 +319,8 @@ static int printer_list_clean_fn(struct db_record *rec, void *private_data)
 	}
 
 	ret = tdb_unpack(rec->value.dptr, rec->value.dsize,
-			 PL_DATA_FORMAT, &time_h, &time_l, &name, &comment);
+			 PL_DATA_FORMAT, &time_h, &time_l, &name, &comment,
+			 &location);
 	if (ret == -1) {
 		DEBUG(1, ("Failed to un pack printer data"));
 		state->status = NT_STATUS_INTERNAL_DB_CORRUPTION;
@@ -306,6 +329,7 @@ static int printer_list_clean_fn(struct db_record *rec, void *private_data)
 
 	SAFE_FREE(name);
 	SAFE_FREE(comment);
+	SAFE_FREE(location);
 
 	refresh = (time_t)(((uint64_t)time_h << 32) + time_l);
 
@@ -341,7 +365,7 @@ NTSTATUS printer_list_clean_old(void)
 }
 
 struct printer_list_exec_state {
-	void (*fn)(const char *, const char *, void *);
+	void (*fn)(const char *, const char *, const char *, void *);
 	void *private_data;
 	NTSTATUS status;
 };
@@ -353,6 +377,7 @@ static int printer_list_exec_fn(struct db_record *rec, void *private_data)
 	uint32_t time_h, time_l;
 	char *name;
 	char *comment;
+	char *location;
 	int ret;
 
 	/* always skip PL_TIMESTAMP_KEY key */
@@ -361,21 +386,23 @@ static int printer_list_exec_fn(struct db_record *rec, void *private_data)
 	}
 
 	ret = tdb_unpack(rec->value.dptr, rec->value.dsize,
-			 PL_DATA_FORMAT, &time_h, &time_l, &name, &comment);
+			 PL_DATA_FORMAT, &time_h, &time_l, &name, &comment,
+			 &location);
 	if (ret == -1) {
 		DEBUG(1, ("Failed to un pack printer data"));
 		state->status = NT_STATUS_INTERNAL_DB_CORRUPTION;
 		return -1;
 	}
 
-	state->fn(name, comment, state->private_data);
+	state->fn(name, comment, location, state->private_data);
 
 	SAFE_FREE(name);
 	SAFE_FREE(comment);
+	SAFE_FREE(location);
 	return 0;
 }
 
-NTSTATUS printer_list_run_fn(void (*fn)(const char *, const char *, void *),
+NTSTATUS printer_list_run_fn(void (*fn)(const char *, const char *, const char *, void *),
 			     void *private_data)
 {
 	struct printer_list_exec_state state;
diff --git a/source3/printing/printer_list.h b/source3/printing/printer_list.h
index fce3e34248c..fb2e007ae6c 100644
--- a/source3/printing/printer_list.h
+++ b/source3/printing/printer_list.h
@@ -32,6 +32,8 @@ bool printer_list_parent_init(void);
  *
  * @param[out] comment  A pointer to store the comment of the printer.
  *
+ * @param[out] location A pointer to store the location of the printer.
+ *
  * @param[out] last_refresh A pointer to store the last refresh time of the
  *                          printer.
  *
@@ -41,6 +43,7 @@ bool printer_list_parent_init(void);
 NTSTATUS printer_list_get_printer(TALLOC_CTX *mem_ctx,
 				  const char *name,
 				  const char **comment,
+				  const char **location,
 				  time_t *last_refresh);
 
 /**
@@ -52,6 +55,8 @@ NTSTATUS printer_list_get_printer(TALLOC_CTX *mem_ctx,
  *
  * @param[in]  comment  The comment to store in the db.
  *
+ * @param[in]  location  The location to store in the db.
+ *
  * @param[in]  last_refresh The last refresh time of the printer to store in
  *                          the db.
  *
@@ -61,6 +66,7 @@ NTSTATUS printer_list_get_printer(TALLOC_CTX *mem_ctx,
 NTSTATUS printer_list_set_printer(TALLOC_CTX *mem_ctx,
 				  const char *name,
 				  const char *comment,
+				  const char *location,
 				  time_t last_refresh);
 
 /**
@@ -94,6 +100,6 @@ NTSTATUS printer_list_mark_reload(void);
  */
 NTSTATUS printer_list_clean_old(void);
 
-NTSTATUS printer_list_run_fn(void (*fn)(const char *, const char *, void *),
+NTSTATUS printer_list_run_fn(void (*fn)(const char *, const char *, const char *, void *),
 			     void *private_data);
 #endif /* _PRINTER_LIST_H_ */
diff --git a/source3/printing/printing.c b/source3/printing/printing.c
index 785e893d3a5..0c070c76682 100644
--- a/source3/printing/printing.c
+++ b/source3/printing/printing.c
@@ -32,6 +32,7 @@
 #include "smbd/smbd.h"
 #include "auth.h"
 #include "messages.h"
+#include "util_tdb.h"
 
 extern struct current_user current_user;
 extern userdom_struct current_user_info;
@@ -87,7 +88,7 @@ uint16 pjobid_to_rap(const char* sharename, uint32 jobid)
 	key.dptr = (uint8 *)&jinfo;
 	key.dsize = sizeof(jinfo);
 
-	data = tdb_fetch(rap_tdb, key);
+	data = tdb_fetch_compat(rap_tdb, key);
 	if (data.dptr && data.dsize == sizeof(uint16)) {
 		rap_jobid = SVAL(data.dptr, 0);
 		SAFE_FREE(data.dptr);
@@ -124,7 +125,7 @@ bool rap_to_pjobid(uint16 rap_jobid, fstring sharename, uint32 *pjobid)
 	SSVAL(buf,0,rap_jobid);
 	key.dptr = buf;
 	key.dsize = sizeof(rap_jobid);
-	data = tdb_fetch(rap_tdb, key);
+	data = tdb_fetch_compat(rap_tdb, key);
 	if ( data.dptr && data.dsize == sizeof(struct rap_jobid_key) )
 	{
 		struct rap_jobid_key *jinfo = (struct rap_jobid_key*)data.dptr;
@@ -162,7 +163,7 @@ void rap_jobid_delete(const char* sharename, uint32 jobid)
 	key.dptr = (uint8 *)&jinfo;
 	key.dsize = sizeof(jinfo);
 
-	data = tdb_fetch(rap_tdb, key);
+	data = tdb_fetch_compat(rap_tdb, key);
 	if (!data.dptr || (data.dsize != sizeof(uint16))) {
 		DEBUG(10,("rap_jobid_delete: cannot find jobid %u\n",
 			(unsigned int)jobid ));
@@ -207,7 +208,7 @@ bool print_backend_init(struct messaging_context *msg_ctx)
 		pdb = get_print_db_byname(lp_const_servicename(snum));
 		if (!pdb)
 			continue;
-		if (tdb_lock_bystring(pdb->tdb, sversion) == -1) {
+		if (tdb_lock_bystring(pdb->tdb, sversion) != 0) {
 			DEBUG(0,("print_backend_init: Failed to open printer %s database\n", lp_const_servicename(snum) ));
 			release_print_db(pdb);
 			return False;
@@ -440,7 +441,7 @@ static struct printjob *print_job_find(const char *sharename, uint32 jobid)
 		return NULL;
 	}
 
-	ret = tdb_fetch(pdb->tdb, print_key(jobid, &tmp));
+	ret = tdb_fetch_compat(pdb->tdb, print_key(jobid, &tmp));
 	release_print_db(pdb);
 
 	if (!ret.dptr) {
@@ -604,12 +605,12 @@ static bool remove_from_jobs_changed(const char* sharename, uint32_t jobid)
 
 	key = string_tdb_data("INFO/jobs_changed");
 
-	if (tdb_chainlock_with_timeout(pdb->tdb, key, 5) == -1)
+	if (tdb_chainlock_with_timeout(pdb->tdb, key, 5) != 0)
 		goto out;
 
 	gotlock = True;
 
-	data = tdb_fetch(pdb->tdb, key);
+	data = tdb_fetch_compat(pdb->tdb, key);
 
 	if (data.dptr == NULL || data.dsize == 0 || (data.dsize % 4 != 0))
 		goto out;
@@ -623,7 +624,7 @@ static bool remove_from_jobs_changed(const char* sharename, uint32_t jobid)
 			if (i < job_count -1 )
 				memmove(data.dptr + (i*4), data.dptr + (i*4) + 4, (job_count - i - 1)*4 );
 			data.dsize -= 4;
-			if (tdb_store(pdb->tdb, key, data, TDB_REPLACE) == -1)
+			if (tdb_store(pdb->tdb, key, data, TDB_REPLACE) != 0)
 				goto out;
 			break;
 		}
@@ -727,7 +728,7 @@ static bool pjob_store(struct tevent_context *ev,
 
 	/* Get old data */
 
-	old_data = tdb_fetch(pdb->tdb, print_key(jobid, &tmp));
+	old_data = tdb_fetch_compat(pdb->tdb, print_key(jobid, &tmp));
 
 	/* Doh!  Now we have to pack/unpack data since the NT_DEVICEMODE was added */
 
@@ -1077,7 +1078,7 @@ static pid_t get_updating_pid(const char *sharename)
 	slprintf(keystr, sizeof(keystr)-1, "UPDATING/%s", sharename);
     	key = string_tdb_data(keystr);
 
-	data = tdb_fetch(pdb->tdb, key);
+	data = tdb_fetch_compat(pdb->tdb, key);
 	release_print_db(pdb);
 	if (!data.dptr || data.dsize != sizeof(pid_t)) {
 		SAFE_FREE(data.dptr);
@@ -1224,7 +1225,7 @@ static TDB_DATA get_jobs_added_data(struct tdb_print_db *pdb)
 
 	ZERO_STRUCT(data);
 
-	data = tdb_fetch(pdb->tdb, string_tdb_data("INFO/jobs_added"));
+	data = tdb_fetch_compat(pdb->tdb, string_tdb_data("INFO/jobs_added"));
 	if (data.dptr == NULL || data.dsize == 0 || (data.dsize % 4 != 0)) {
 		SAFE_FREE(data.dptr);
 		ZERO_STRUCT(data);
@@ -1515,7 +1516,7 @@ static void print_queue_update_with_lock( struct tevent_context *ev,
 
 	slprintf(keystr, sizeof(keystr) - 1, "LOCK/%s", sharename);
 	/* Only wait 10 seconds for this. */
-	if (tdb_lock_bystring_with_timeout(pdb->tdb, keystr, 10) == -1) {
+	if (tdb_lock_bystring_with_timeout(pdb->tdb, keystr, 10) != 0) {
 		DEBUG(0,("print_queue_update_with_lock: Failed to lock printer %s database\n", sharename));
 		release_print_db(pdb);
 		return;
@@ -1884,7 +1885,7 @@ bool print_notify_register_pid(int snum)
 		tdb = pdb->tdb;
 	}
 
-	if (tdb_lock_bystring_with_timeout(tdb, NOTIFY_PID_LIST_KEY, 10) == -1) {
+	if (tdb_lock_bystring_with_timeout(tdb, NOTIFY_PID_LIST_KEY, 10) != 0) {
 		DEBUG(0,("print_notify_register_pid: Failed to lock printer %s\n",
 					printername));
 		if (pdb)
@@ -1918,7 +1919,7 @@ bool print_notify_register_pid(int snum)
 	}
 
 	/* Store back the record. */
-	if (tdb_store_bystring(tdb, NOTIFY_PID_LIST_KEY, data, TDB_REPLACE) == -1) {
+	if (tdb_store_bystring(tdb, NOTIFY_PID_LIST_KEY, data, TDB_REPLACE) != 0) {
 		DEBUG(0,("print_notify_register_pid: Failed to update pid \
 list for printer %s\n", printername));
 		goto done;
@@ -1974,7 +1975,7 @@ bool print_notify_deregister_pid(int snum)
 		tdb = pdb->tdb;
 	}
 
-	if (tdb_lock_bystring_with_timeout(tdb, NOTIFY_PID_LIST_KEY, 10) == -1) {
+	if (tdb_lock_bystring_with_timeout(tdb, NOTIFY_PID_LIST_KEY, 10) != 0) {
 		DEBUG(0,("print_notify_register_pid: Failed to lock \
 printer %s database\n", printername));
 		if (pdb)
@@ -2008,7 +2009,7 @@ printer %s database\n", printername));
 		SAFE_FREE(data.dptr);
 
 	/* Store back the record. */
-	if (tdb_store_bystring(tdb, NOTIFY_PID_LIST_KEY, data, TDB_REPLACE) == -1) {
+	if (tdb_store_bystring(tdb, NOTIFY_PID_LIST_KEY, data, TDB_REPLACE) != 0) {
 		DEBUG(0,("print_notify_register_pid: Failed to update pid \
 list for printer %s\n", printername));
 		goto done;
@@ -2133,12 +2134,12 @@ static bool remove_from_jobs_added(const char* sharename, uint32 jobid)
 
 	key = string_tdb_data("INFO/jobs_added");
 
-	if (tdb_chainlock_with_timeout(pdb->tdb, key, 5) == -1)
+	if (tdb_chainlock_with_timeout(pdb->tdb, key, 5) != 0)
 		goto out;
 
 	gotlock = True;
 
-	data = tdb_fetch(pdb->tdb, key);
+	data = tdb_fetch_compat(pdb->tdb, key);
 
 	if (data.dptr == NULL || data.dsize == 0 || (data.dsize % 4 != 0))
 		goto out;
@@ -2152,7 +2153,7 @@ static bool remove_from_jobs_added(const char* sharename, uint32 jobid)
 			if (i < job_count -1 )
 				memmove(data.dptr + (i*4), data.dptr + (i*4) + 4, (job_count - i - 1)*4 );
 			data.dsize -= 4;
-			if (tdb_store(pdb->tdb, key, data, TDB_REPLACE) == -1)
+			if (tdb_store(pdb->tdb, key, data, TDB_REPLACE) != 0)
 				goto out;
 			break;
 		}
@@ -2501,7 +2502,7 @@ static int get_queue_status(const char* sharename, print_status_struct *status)
 
 	if (status) {
 		fstr_sprintf(keystr, "STATUS/%s", sharename);
-		data = tdb_fetch(pdb->tdb, string_tdb_data(keystr));
+		data = tdb_fetch_compat(pdb->tdb, string_tdb_data(keystr));
 		if (data.dptr) {
 			if (data.dsize == sizeof(print_status_struct))
 				/* this memcpy is ok since the status struct was
@@ -2560,7 +2561,7 @@ static WERROR allocate_print_jobid(struct tdb_print_db *pdb, int snum,
 		/* Lock the database - only wait 20 seconds. */
 		ret = tdb_lock_bystring_with_timeout(pdb->tdb,
 						     "INFO/nextjob", 20);
-		if (ret == -1) {
+		if (ret != 0) {
 			DEBUG(0, ("allocate_print_jobid: "
 				  "Failed to lock printing database %s\n",
 				  sharename));
@@ -2588,7 +2589,7 @@ static WERROR allocate_print_jobid(struct tdb_print_db *pdb, int snum,
 		jobid = NEXT_JOBID(jobid);
 
 		ret = tdb_store_int32(pdb->tdb, "INFO/nextjob", jobid);
-		if (ret == -1) {
+		if (ret != 0) {
 			terr = tdb_error(pdb->tdb);
 			DEBUG(3, ("allocate_print_jobid: "
 				  "Failed to store INFO/nextjob.\n"));
@@ -2621,7 +2622,7 @@ static WERROR allocate_print_jobid(struct tdb_print_db *pdb, int snum,
 		dum.dptr = NULL;
 		dum.dsize = 0;
 		if (tdb_store(pdb->tdb, print_key(jobid, &tmp), dum,
-			      TDB_INSERT) == -1) {
+			      TDB_INSERT) != 0) {
 			DEBUG(3, ("allocate_print_jobid: "
 				  "jobid (%d) failed to store placeholder.\n",
 				  jobid ));
@@ -3037,18 +3038,18 @@ static bool get_stored_queue_info(struct messaging_context *msg_ctx,
 	ZERO_STRUCT(cgdata);
 
 	/* Get the stored queue data. */
-	data = tdb_fetch(pdb->tdb, string_tdb_data("INFO/linear_queue_array"));
+	data = tdb_fetch_compat(pdb->tdb, string_tdb_data("INFO/linear_queue_array"));
 
 	if (data.dptr && data.dsize >= sizeof(qcount))
 		len += tdb_unpack(data.dptr + len, data.dsize - len, "d", &qcount);
 
 	/* Get the added jobs list. */
-	cgdata = tdb_fetch(pdb->tdb, string_tdb_data("INFO/jobs_added"));
+	cgdata = tdb_fetch_compat(pdb->tdb, string_tdb_data("INFO/jobs_added"));
 	if (cgdata.dptr != NULL && (cgdata.dsize % 4 == 0))
 		extra_count = cgdata.dsize/4;
 
 	/* Get the changed jobs list. */
-	jcdata = tdb_fetch(pdb->tdb, string_tdb_data("INFO/jobs_changed"));
+	jcdata = tdb_fetch_compat(pdb->tdb, string_tdb_data("INFO/jobs_changed"));
 	if (jcdata.dptr != NULL && (jcdata.dsize % 4 == 0))
 		changed_count = jcdata.dsize / 4;
 
@@ -3215,7 +3216,7 @@ int print_queue_status(struct messaging_context *msg_ctx, int snum,
 	slprintf(keystr, sizeof(keystr)-1, "STATUS/%s", sharename);
 	key = string_tdb_data(keystr);
 
-	data = tdb_fetch(pdb->tdb, key);
+	data = tdb_fetch_compat(pdb->tdb, key);
 	if (data.dptr) {
 		if (data.dsize == sizeof(*status)) {
 			/* this memcpy is ok since the status struct was
diff --git a/source3/printing/printing_db.c b/source3/printing/printing_db.c
index 70accab9350..e442551b833 100644
--- a/source3/printing/printing_db.c
+++ b/source3/printing/printing_db.c
@@ -22,6 +22,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "printing.h"
+#include "util_tdb.h"
 
 static struct tdb_print_db *print_db_head;
 
@@ -159,7 +160,7 @@ void close_all_print_db(void)
  messages. data needs freeing on exit.
 ****************************************************************************/
 
-TDB_DATA get_printer_notify_pid_list(TDB_CONTEXT *tdb, const char *printer_name, bool cleanlist)
+TDB_DATA get_printer_notify_pid_list(struct tdb_context *tdb, const char *printer_name, bool cleanlist)
 {
 	TDB_DATA data;
 	size_t i;
diff --git a/source3/printing/printspoolss.c b/source3/printing/printspoolss.c
index 2f6f19c3da7..3d8b0d8b1bb 100644
--- a/source3/printing/printspoolss.c
+++ b/source3/printing/printspoolss.c
@@ -22,7 +22,6 @@
 #include "rpc_client/rpc_client.h"
 #include "../librpc/gen_ndr/ndr_spoolss_c.h"
 #include "rpc_server/rpc_ncacn_np.h"
-#include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/security/security.h"
 
diff --git a/source3/printing/spoolssd.c b/source3/printing/spoolssd.c
index 340bf0a5f14..896aee0a539 100644
--- a/source3/printing/spoolssd.c
+++ b/source3/printing/spoolssd.c
@@ -19,15 +19,16 @@
 #include "includes.h"
 #include "serverid.h"
 #include "smbd/smbd.h"
-#include "locking/proto.h"
 
 #include "messages.h"
 #include "include/printing.h"
 #include "printing/nt_printing_migrate.h"
+#include "ntdomain.h"
 #include "librpc/gen_ndr/srv_winreg.h"
 #include "librpc/gen_ndr/srv_spoolss.h"
 #include "rpc_server/rpc_server.h"
 #include "rpc_server/rpc_ep_setup.h"
+#include "rpc_server/spoolss/srv_spoolss_nt.h"
 
 #define SPOOLSS_PIPE_NAME "spoolss"
 #define DAEMON_NAME "spoolssd"
diff --git a/source3/printing/tests/vlp.c b/source3/printing/tests/vlp.c
index 4bff290e28a..66860e5ada3 100644
--- a/source3/printing/tests/vlp.c
+++ b/source3/printing/tests/vlp.c
@@ -23,6 +23,7 @@
 #include "system/passwd.h"
 #include "system/filesys.h"
 #include "printing.h"
+#include "util_tdb.h"
 
 #ifdef malloc
 #undef malloc
@@ -110,10 +111,9 @@ static int next_jobnum(char *printer)
 static void set_printer_status(char *printer, int status)
 {
 	fstring keystr;
-	int result;
 
 	slprintf(keystr, sizeof(keystr) - 1, "STATUS/%s", printer);
-	result = tdb_store_int32(tdb, keystr, status);
+	tdb_store_int32(tdb, keystr, status);
 }
 
 static int get_printer_status(char *printer)
@@ -392,8 +392,9 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	if (!(tdb = tdb_open(printdb_path, 0, 0, O_RDWR | O_CREAT,
-			     0666))) {
+	/* FIXME: We should *never* open a tdb without logging! */
+	if (!(tdb = tdb_open_compat(printdb_path, 0, 0, O_RDWR | O_CREAT,
+				    0666, NULL, NULL))) {
 		printf("%s: unable to open %s\n", argv[0], printdb_path);
 		return 1;
 	}
diff --git a/source3/registry/reg_api.c b/source3/registry/reg_api.c
index 935d2441afa..51bd98e9395 100644
--- a/source3/registry/reg_api.c
+++ b/source3/registry/reg_api.c
@@ -142,9 +142,9 @@ static WERROR regkey_open_onelevel(TALLOC_CTX *mem_ctx,
 
 	SMB_ASSERT(strchr(name, '\\') == NULL);
 
-	if (!(regkey = TALLOC_ZERO_P(mem_ctx, struct registry_key)) ||
+	if (!(regkey = talloc_zero(mem_ctx, struct registry_key)) ||
 	    !(regkey->token = dup_nt_token(regkey, token)) ||
-	    !(regkey->key = TALLOC_ZERO_P(regkey, struct registry_key_handle)))
+	    !(regkey->key = talloc_zero(regkey, struct registry_key_handle)))
 	{
 		result = WERR_NOMEM;
 		goto done;
@@ -188,7 +188,7 @@ static WERROR regkey_open_onelevel(TALLOC_CTX *mem_ctx,
 
 	/* Tag this as a Performance Counter Key */
 
-	if( StrnCaseCmp(key->name, KEY_HKPD, strlen(KEY_HKPD)) == 0 )
+	if( strncasecmp_m(key->name, KEY_HKPD, strlen(KEY_HKPD)) == 0 )
 		key->type = REG_KEY_HKPD;
 
 	/* Look up the table of registry I/O operations */
diff --git a/source3/registry/reg_backend_current_version.c b/source3/registry/reg_backend_current_version.c
index ab7bd517b0c..adf304e8c95 100644
--- a/source3/registry/reg_backend_current_version.c
+++ b/source3/registry/reg_backend_current_version.c
@@ -58,8 +58,8 @@ static int current_version_fetch_values(const char *key, struct regval_ctr *valu
 
 	regval_ctr_addvalue_sz(values, "SystemRoot", sysroot_string);
 
-	fstr_sprintf(sysversion, "%d.%d", lp_major_announce_version(),
-		     lp_minor_announce_version());
+	fstr_sprintf(sysversion, "%d.%d", SAMBA_MAJOR_NBT_ANNOUNCE_VERSION,
+		     SAMBA_MINOR_NBT_ANNOUNCE_VERSION);
 
 	regval_ctr_addvalue_sz(values, "CurrentVersion", sysversion);
 
diff --git a/source3/registry/reg_backend_db.c b/source3/registry/reg_backend_db.c
index 6024a354c47..812dd0d43cf 100644
--- a/source3/registry/reg_backend_db.c
+++ b/source3/registry/reg_backend_db.c
@@ -28,7 +28,9 @@
 #include "reg_backend_db.h"
 #include "reg_objects.h"
 #include "nt_printing.h"
+#include "util_tdb.h"
 #include "dbwrap.h"
+#include "../libcli/security/secdesc.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_REGISTRY
@@ -47,6 +49,8 @@ static int regdb_fetch_values_internal(struct db_context *db, const char* key,
 static bool regdb_store_values_internal(struct db_context *db, const char *key,
 					struct regval_ctr *values);
 
+static NTSTATUS create_sorted_subkeys(const char *key);
+
 /* List the deepest path into the registry.  All part components will be created.*/
 
 /* If you want to have a part of the path controlled by the tdb and part by
@@ -478,7 +482,7 @@ static WERROR regdb_upgrade_v1_to_v2(void)
 
 	talloc_destroy(mem_ctx);
 
-	if (rc == -1) {
+	if (rc < 0) {
 		return WERR_REG_IO_FAILURE;
 	}
 
@@ -831,22 +835,9 @@ static WERROR regdb_store_keys_internal2(struct db_context *db,
 	W_ERROR_NOT_OK_GOTO_DONE(werr);
 
 	/*
-	 * Delete a sorted subkey cache for regdb_key_exists, will be
-	 * recreated automatically
+	 * recreate the sorted subkey cache for regdb_key_exists()
 	 */
-	keyname = talloc_asprintf(ctx, "%s\\%s", REG_SORTED_SUBKEYS_PREFIX,
-				  keyname);
-	if (keyname == NULL) {
-		werr = WERR_NOMEM;
-		goto done;
-	}
-
-	werr = ntstatus_to_werror(dbwrap_delete_bystring(db, keyname));
-
-	/* don't treat WERR_NOT_FOUND as an error here */
-	if (W_ERROR_EQUAL(werr, WERR_NOT_FOUND)) {
-		werr = WERR_OK;
-	}
+	werr = ntstatus_to_werror(create_sorted_subkeys(keyname));
 
 done:
 	TALLOC_FREE(ctx);
@@ -1310,7 +1301,7 @@ done:
 
 static int cmp_keynames(char **p1, char **p2)
 {
-	return StrCaseCmp(*p1, *p2);
+	return strcasecmp_m(*p1, *p2);
 }
 
 struct create_sorted_subkeys_context {
@@ -1407,7 +1398,8 @@ done:
 	return status;
 }
 
-static bool create_sorted_subkeys(const char *key, const char *sorted_keyname)
+static NTSTATUS create_sorted_subkeys_internal(const char *key,
+					       const char *sorted_keyname)
 {
 	NTSTATUS status;
 	struct create_sorted_subkeys_context sorted_ctx;
@@ -1419,7 +1411,26 @@ static bool create_sorted_subkeys(const char *key, const char *sorted_keyname)
 				 create_sorted_subkeys_action,
 				 &sorted_ctx);
 
-	return NT_STATUS_IS_OK(status);
+	return status;
+}
+
+static NTSTATUS create_sorted_subkeys(const char *key)
+{
+	char *sorted_subkeys_keyname;
+	NTSTATUS status;
+
+	sorted_subkeys_keyname = talloc_asprintf(talloc_tos(), "%s\\%s",
+						 REG_SORTED_SUBKEYS_PREFIX,
+						 key);
+	if (sorted_subkeys_keyname == NULL) {
+		status = NT_STATUS_NO_MEMORY;
+		goto done;
+	}
+
+	status = create_sorted_subkeys_internal(key, sorted_subkeys_keyname);
+
+done:
+	return status;
 }
 
 struct scan_subkey_state {
@@ -1499,13 +1510,21 @@ static bool scan_parent_subkeys(struct db_context *db, const char *parent,
 	if (state.scanned) {
 		result = state.found;
 	} else {
+		NTSTATUS status;
+
 		res = db->transaction_start(db);
 		if (res != 0) {
-			DEBUG(0, ("error starting transacion\n"));
+			DEBUG(0, ("error starting transaction\n"));
 			goto fail;
 		}
 
-		if (!create_sorted_subkeys(path, key)) {
+		DEBUG(2, (__location__ " WARNING: recreating the sorted "
+			  "subkeys cache for key '%s' from scan_parent_subkeys "
+			  "this should not happen (too frequently)...\n",
+			  path));
+
+		status = create_sorted_subkeys_internal(path, key);
+		if (!NT_STATUS_IS_OK(status)) {
 			res = db->transaction_cancel(db);
 			if (res != 0) {
 				smb_panic("Failed to cancel transaction.");
@@ -1801,7 +1820,7 @@ static bool regdb_store_values_internal(struct db_context *db, const char *key,
 		goto done;
 	}
 
-	data.dptr = TALLOC_ARRAY(ctx, uint8, len);
+	data.dptr = talloc_array(ctx, uint8, len);
 	data.dsize = len;
 
 	len = regdb_pack_values(values, data.dptr, data.dsize);
diff --git a/source3/registry/reg_objects.c b/source3/registry/reg_objects.c
index 0fab3f7d8be..62487e1e4f4 100644
--- a/source3/registry/reg_objects.c
+++ b/source3/registry/reg_objects.c
@@ -23,6 +23,7 @@
 #include "includes.h"
 #include "registry.h"
 #include "reg_objects.h"
+#include "util_tdb.h"
 #include "dbwrap.h"
 #include "../libcli/registry/util_reg.h"
 
@@ -61,7 +62,7 @@ struct regsubkey_ctr {
  context for internal private data.
 
  There is no longer a regval_ctr_intit() and regval_ctr_destroy()
- pair of functions.  Simply TALLOC_ZERO_P() and TALLOC_FREE() the
+ pair of functions.  Simply talloc_zero() and TALLOC_FREE() the
  object.
 
  **********************************************************************/
@@ -210,7 +211,7 @@ WERROR regsubkey_ctr_addkey( struct regsubkey_ctr *ctr, const char *keyname )
 		return WERR_OK;
 	}
 
-	if (!(newkeys = TALLOC_REALLOC_ARRAY(ctr, ctr->subkeys, char *,
+	if (!(newkeys = talloc_realloc(ctr, ctr->subkeys, char *,
 					     ctr->num_subkeys+1))) {
 		return WERR_NOMEM;
 	}
@@ -465,7 +466,7 @@ struct regval_blob *regval_compose(TALLOC_CTX *ctx, const char *name,
 				   uint32_t type,
 				   const uint8_t *data_p, size_t size)
 {
-	struct regval_blob *regval = TALLOC_P(ctx, struct regval_blob);
+	struct regval_blob *regval = talloc(ctx, struct regval_blob);
 
 	if (regval == NULL) {
 		return NULL;
@@ -474,7 +475,7 @@ struct regval_blob *regval_compose(TALLOC_CTX *ctx, const char *name,
 	fstrcpy(regval->valuename, name);
 	regval->type = type;
 	if (size) {
-		regval->data_p = (uint8_t *)TALLOC_MEMDUP(regval, data_p, size);
+		regval->data_p = (uint8_t *)talloc_memdup(regval, data_p, size);
 		if (!regval->data_p) {
 			TALLOC_FREE(regval);
 			return NULL;
@@ -504,9 +505,9 @@ int regval_ctr_addvalue(struct regval_ctr *ctr, const char *name, uint32_t type,
 	/* allocate a slot in the array of pointers */
 
 	if (  ctr->num_values == 0 ) {
-		ctr->values = TALLOC_P( ctr, struct regval_blob *);
+		ctr->values = talloc( ctr, struct regval_blob *);
 	} else {
-		ctr->values = TALLOC_REALLOC_ARRAY(ctr, ctr->values,
+		ctr->values = talloc_realloc(ctr, ctr->values,
 						   struct regval_blob *,
 						   ctr->num_values+1);
 	}
diff --git a/source3/registry/reg_parse.c b/source3/registry/reg_parse.c
index 0c596a8f5e0..f1258fca6d1 100644
--- a/source3/registry/reg_parse.c
+++ b/source3/registry/reg_parse.c
@@ -622,7 +622,7 @@ static bool lookslike_utf16(const char* line, size_t len, bool* little_endian)
 	bool le;
 
 	size_t l = MIN(len/2, 64);
-	uint16_t* u = (uint16_t*)line;
+	const uint16_t* u = (const uint16_t*)line;
 	int i;
 
 	assert(len >= 2);
diff --git a/source3/registry/reg_parse_internal.c b/source3/registry/reg_parse_internal.c
index dedbe123d86..208844856f0 100644
--- a/source3/registry/reg_parse_internal.c
+++ b/source3/registry/reg_parse_internal.c
@@ -1,4 +1,9 @@
-/* * Samba Unix/Linux SMB client library
+/*
+ * Unix SMB/CIFS implementation.
+ *
+ * Registry helper routines
+ *
+ * Copyright (C) Gregor Beck 2010
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -308,7 +313,7 @@ int write_bom(FILE* file, const char* charset, charset_t ctype)
 		DEBUG(0, ("No Byte Order Mark for charset_t: %u\n", (unsigned)ctype));
 	} else {
 		for (i=0; BOM[i].name; i++) {
-			if (StrCaseCmp(BOM[i].name, charset) == 0) {
+			if (strcasecmp_m(BOM[i].name, charset) == 0) {
 				return fwrite(BOM[i].seq, 1, BOM[i].len, file);
 			}
 		}
diff --git a/source3/registry/reg_parse_internal.h b/source3/registry/reg_parse_internal.h
index ef591617569..14fed89aa95 100644
--- a/source3/registry/reg_parse_internal.h
+++ b/source3/registry/reg_parse_internal.h
@@ -1,4 +1,9 @@
-/* Samba Unix/Linux SMB client library
+/*
+ * Unix SMB/CIFS implementation.
+ *
+ * Registry helper routines
+ *
+ * Copyright (C) Gregor Beck 2010
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
diff --git a/source3/registry/reg_perfcount.c b/source3/registry/reg_perfcount.c
index 4ed3305c158..64e3cbee0f6 100644
--- a/source3/registry/reg_perfcount.c
+++ b/source3/registry/reg_perfcount.c
@@ -25,6 +25,7 @@
 #include "registry.h"
 #include "reg_perfcount.h"
 #include "../libcli/registry/util_reg.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_REGISTRY
@@ -94,7 +95,7 @@ uint32 reg_perfcount_get_base_index(void)
 	   and so on.
 	   So last_counter becomes num_counters*2, and last_help will be last_counter+1 */
 	kbuf = string_tdb_data(key);
-	dbuf = tdb_fetch(names, kbuf);
+	dbuf = tdb_fetch_compat(names, kbuf);
 	if(dbuf.dptr == NULL)
 	{
 		DEBUG(1, ("reg_perfcount_get_base_index: failed to find key \'1\' in [%s].\n", fname));
@@ -161,7 +162,7 @@ static uint32 _reg_perfcount_multi_sz_from_tdb(TDB_CONTEXT *tdb,
 	memset(temp, 0, sizeof(temp));
 	snprintf(temp, sizeof(temp), "%d", keyval);
 	kbuf = string_tdb_data(temp);
-	dbuf = tdb_fetch(tdb, kbuf);
+	dbuf = tdb_fetch_compat(tdb, kbuf);
 	if(dbuf.dptr == NULL)
 	{
 		/* If a key isn't there, just bypass it -- this really shouldn't 
@@ -346,7 +347,7 @@ static uint32 _reg_perfcount_get_numinst(int objInd, TDB_CONTEXT *names)
 	char buf[PERFCOUNT_MAX_LEN];
 
 	_reg_perfcount_make_key(&key, buf, PERFCOUNT_MAX_LEN, objInd, "inst");
-	data = tdb_fetch(names, key);
+	data = tdb_fetch_compat(names, key);
 
 	if(data.dptr == NULL)
 		return (uint32)PERF_NO_INSTANCES;
@@ -375,7 +376,7 @@ static bool _reg_perfcount_add_object(struct PERF_DATA_BLOCK *block,
 	bool success = True;
 	struct PERF_OBJECT_TYPE *obj;
 
-	block->objects = (struct PERF_OBJECT_TYPE *)TALLOC_REALLOC_ARRAY(mem_ctx,
+	block->objects = (struct PERF_OBJECT_TYPE *)talloc_realloc(mem_ctx,
 								  block->objects,
 								  struct PERF_OBJECT_TYPE,
 								  block->NumObjectTypes+1);
@@ -420,7 +421,7 @@ static bool _reg_perfcount_get_counter_data(TDB_DATA key, TDB_DATA *data)
 		return False;
 	}    
 
-	*data = tdb_fetch(counters, key);
+	*data = tdb_fetch_compat(counters, key);
 
 	tdb_close(counters);
 
@@ -487,7 +488,7 @@ static bool _reg_perfcount_get_counter_info(struct PERF_DATA_BLOCK *block,
 	padding = 0;
 
 	_reg_perfcount_make_key(&key, buf, PERFCOUNT_MAX_LEN, CounterIndex, "type");
-	data = tdb_fetch(names, key);
+	data = tdb_fetch_compat(names, key);
 	if(data.dptr == NULL)
 	{
 		DEBUG(3, ("_reg_perfcount_get_counter_info: No type data for counter [%d].\n", CounterIndex));
@@ -546,7 +547,7 @@ static bool _reg_perfcount_get_counter_info(struct PERF_DATA_BLOCK *block,
 	SAFE_FREE(data.dptr);
 
 	obj->counter_data.ByteLength += dsize + padding;
-	obj->counter_data.data = TALLOC_REALLOC_ARRAY(mem_ctx,
+	obj->counter_data.data = talloc_realloc(mem_ctx,
 						      obj->counter_data.data,
 						      uint8,
 						      obj->counter_data.ByteLength - sizeof(uint32));
@@ -634,7 +635,7 @@ static bool _reg_perfcount_add_counter(struct PERF_DATA_BLOCK *block,
 				  parent, num));
 			return False;
 		}
-		obj->counters = (struct PERF_COUNTER_DEFINITION *)TALLOC_REALLOC_ARRAY(mem_ctx,
+		obj->counters = (struct PERF_COUNTER_DEFINITION *)talloc_realloc(mem_ctx,
 										obj->counters,
 										struct PERF_COUNTER_DEFINITION,
 										obj->NumCounters+1);
@@ -686,7 +687,7 @@ static bool _reg_perfcount_get_instance_info(struct PERF_INSTANCE_DEFINITION *in
 		return False;
 	}
 	inst->counter_data.ByteLength = data.dsize + sizeof(inst->counter_data.ByteLength);
-	inst->counter_data.data = TALLOC_REALLOC_ARRAY(mem_ctx,
+	inst->counter_data.data = talloc_realloc(mem_ctx,
 						       inst->counter_data.data,
 						       uint8,
 						       data.dsize);
@@ -700,7 +701,7 @@ static bool _reg_perfcount_get_instance_info(struct PERF_INSTANCE_DEFINITION *in
 	memset(temp, 0, PERFCOUNT_MAX_LEN);
 	snprintf(temp, PERFCOUNT_MAX_LEN, "i%dname", instId);
 	_reg_perfcount_make_key(&key, buf, PERFCOUNT_MAX_LEN, obj->ObjectNameTitleIndex, temp);
-	data = tdb_fetch(names, key);
+	data = tdb_fetch_compat(names, key);
 	if(data.dptr == NULL)
 	{
 		/* Not actually an error, but possibly unintended? -- just logging FYI */
@@ -718,7 +719,7 @@ static bool _reg_perfcount_get_instance_info(struct PERF_INSTANCE_DEFINITION *in
 			SAFE_FREE(data.dptr);
 			return False;
 		}
-		inst->data = TALLOC_REALLOC_ARRAY(mem_ctx,
+		inst->data = talloc_realloc(mem_ctx,
 						  inst->data,
 						  uint8,
 						  inst->NameLength);
@@ -740,7 +741,7 @@ static bool _reg_perfcount_get_instance_info(struct PERF_INSTANCE_DEFINITION *in
 	if((pad = (inst->ByteLength % 8)))
 	{
 		pad = 8 - pad;
-		inst->data = TALLOC_REALLOC_ARRAY(mem_ctx,
+		inst->data = talloc_realloc(mem_ctx,
 						  inst->data,
 						  uint8,
 						  inst->NameLength + pad);
@@ -762,7 +763,7 @@ static bool _reg_perfcount_add_instance(struct PERF_OBJECT_TYPE *obj,
 	struct PERF_INSTANCE_DEFINITION *inst;
 
 	if(obj->instances == NULL) {
-		obj->instances = TALLOC_REALLOC_ARRAY(mem_ctx,
+		obj->instances = talloc_realloc(mem_ctx,
 						      obj->instances,
 						      struct PERF_INSTANCE_DEFINITION,
 						      obj->NumInstances);
@@ -792,7 +793,7 @@ static int _reg_perfcount_assemble_global(struct PERF_DATA_BLOCK *block,
 	{
 		j = i*2;
 		_reg_perfcount_make_key(&key, keybuf, PERFCOUNT_MAX_LEN, j, "rel");
-		data = tdb_fetch(names, key);
+		data = tdb_fetch_compat(names, key);
 		if(data.dptr != NULL)
 		{
 			if(_reg_perfcount_isparent(data))
@@ -830,7 +831,7 @@ static bool _reg_perfcount_get_64(uint64_t *retval,
 
 	_reg_perfcount_make_key(&key, buf, PERFCOUNT_MAX_LEN, key_part1, key_part2);
 
-	data = tdb_fetch(tdb, key);
+	data = tdb_fetch_compat(tdb, key);
 	if(data.dptr == NULL)
 	{
 		DEBUG(3,("_reg_perfcount_get_64: No data found for key [%s].\n", key.dptr));
@@ -942,9 +943,9 @@ static bool _reg_perfcount_init_data_block(struct PERF_DATA_BLOCK *block,
 	make_systemtime(&(block->SystemTime), gmtime(&tm));
 	_reg_perfcount_init_data_block_perf(block, names);
 	memset(temp, 0, sizeof(temp));
-	rpcstr_push((void *)temp, global_myname(), sizeof(temp), STR_TERMINATE);
+	rpcstr_push((void *)temp, lp_netbios_name(), sizeof(temp), STR_TERMINATE);
 	block->SystemNameLength = (strlen_w(temp) * 2) + 2;
-	block->data = TALLOC_ZERO_ARRAY(mem_ctx, uint8, block->SystemNameLength + (8 - (block->SystemNameLength % 8)));
+	block->data = talloc_zero_array(mem_ctx, uint8, block->SystemNameLength + (8 - (block->SystemNameLength % 8)));
 	if (block->data == NULL) {
 		return False;
 	}
@@ -992,7 +993,7 @@ static uint32 _reg_perfcount_perf_data_block_fixup(struct PERF_DATA_BLOCK *block
 				counter_data = &(instance->counter_data);
 				counter = &(object[obj].counters[object[obj].NumCounters - 1]);
 				counter_data->ByteLength = counter->CounterOffset + counter->CounterSize + sizeof(counter_data->ByteLength);
-				temp = TALLOC_REALLOC_ARRAY(mem_ctx,
+				temp = talloc_realloc(mem_ctx,
 							    temp, 
 							    char, 
 							    counter_data->ByteLength- sizeof(counter_data->ByteLength));
@@ -1013,7 +1014,7 @@ static uint32 _reg_perfcount_perf_data_block_fixup(struct PERF_DATA_BLOCK *block
 				{
 					pad = 8 - pad;
 				}
-				counter_data->data = TALLOC_REALLOC_ARRAY(mem_ctx,
+				counter_data->data = talloc_realloc(mem_ctx,
 									 counter_data->data,
 									 uint8,
 									 counter_data->ByteLength - sizeof(counter_data->ByteLength) + pad);
@@ -1033,7 +1034,7 @@ static uint32 _reg_perfcount_perf_data_block_fixup(struct PERF_DATA_BLOCK *block
 			if((pad = (object[obj].counter_data.ByteLength % 8)))
 			{
 				pad = 8 - pad;
-				object[obj].counter_data.data = TALLOC_REALLOC_ARRAY(mem_ctx,
+				object[obj].counter_data.data = talloc_realloc(mem_ctx,
 										     object[obj].counter_data.data,
 										     uint8, 
 										     object[obj].counter_data.ByteLength + pad);
diff --git a/source3/registry/reg_util_internal.c b/source3/registry/reg_util_internal.c
index a1aeaa405cd..b15015b988d 100644
--- a/source3/registry/reg_util_internal.c
+++ b/source3/registry/reg_util_internal.c
@@ -97,12 +97,11 @@ char *normalize_reg_path(TALLOC_CTX *ctx, const char *keyname )
 	char *nkeyname;
 
 	/* skip leading '\' chars */
-	p = (char *)keyname;
-	while (*p == '\\') {
-		p++;
+	while (*keyname == '\\') {
+		keyname++;
 	}
 
-	nkeyname = talloc_strdup(ctx, p);
+	nkeyname = talloc_strdup(ctx, keyname);
 	if (nkeyname == NULL) {
 		return NULL;
 	}
diff --git a/source3/registry/reg_util_token.c b/source3/registry/reg_util_token.c
index ca0159a6491..d599b3a33dd 100644
--- a/source3/registry/reg_util_token.c
+++ b/source3/registry/reg_util_token.c
@@ -38,7 +38,7 @@ NTSTATUS registry_create_admin_token(TALLOC_CTX *mem_ctx,
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	token = TALLOC_ZERO_P(mem_ctx, struct security_token);
+	token = talloc_zero(mem_ctx, struct security_token);
 	if (token == NULL) {
 		DEBUG(1, ("talloc failed\n"));
 		status = NT_STATUS_NO_MEMORY;
diff --git a/source3/registry/regfio.c b/source3/registry/regfio.c
index bd5380787c6..6d2beccfef1 100644
--- a/source3/registry/regfio.c
+++ b/source3/registry/regfio.c
@@ -22,6 +22,7 @@
 #include "regfio.h"
 #include "../librpc/gen_ndr/ndr_security.h"
 #include "../libcli/security/security_descriptor.h"
+#include "../libcli/security/secdesc.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_REGISTRY
@@ -497,7 +498,7 @@ static REGF_HBIN* read_hbin_block( REGF_FILE *file, off_t offset )
 	REGF_HBIN *hbin;
 	uint32 record_size, curr_off, block_size, header;
 	
-	if ( !(hbin = TALLOC_ZERO_P(file->mem_ctx, REGF_HBIN)) ) 
+	if ( !(hbin = talloc_zero(file->mem_ctx, REGF_HBIN)) ) 
 		return NULL;
 	hbin->file_off = offset;
 	hbin->free_off = -1;
@@ -1072,7 +1073,7 @@ static bool hbin_prs_key( REGF_FILE *file, REGF_HBIN *hbin, REGF_NK_REC *nk )
 			}
 		}
 		
-		if ( !(nk->sec_desc = TALLOC_ZERO_P( file->mem_ctx, REGF_SK_REC )) )
+		if ( !(nk->sec_desc = talloc_zero( file->mem_ctx, REGF_SK_REC )) )
 			return False;
 		nk->sec_desc->sk_off = nk->sk_off;
 		if ( !hbin_prs_sk_rec( "sk_rec", sub_hbin, depth, nk->sec_desc ))
@@ -1378,7 +1379,7 @@ REGF_NK_REC* regfio_rootkey( REGF_FILE *file )
 	if ( !file )
 		return NULL;
 		
-	if ( !(nk = TALLOC_ZERO_P( file->mem_ctx, REGF_NK_REC )) ) {
+	if ( !(nk = talloc_zero( file->mem_ctx, REGF_NK_REC )) ) {
 		DEBUG(0,("regfio_rootkey: talloc() failed!\n"));
 		return NULL;
 	}
@@ -1446,7 +1447,7 @@ REGF_NK_REC* regfio_rootkey( REGF_FILE *file )
 		return NULL;
 		
 	nk->subkey_index++;
-	if ( !(subkey = TALLOC_ZERO_P( file->mem_ctx, REGF_NK_REC )) )
+	if ( !(subkey = talloc_zero( file->mem_ctx, REGF_NK_REC )) )
 		return NULL;
 		
 	if ( !hbin_prs_key( file, hbin, subkey ) )
@@ -1464,7 +1465,7 @@ static REGF_HBIN* regf_hbin_allocate( REGF_FILE *file, uint32 block_size )
 	REGF_HBIN *hbin;
 	SMB_STRUCT_STAT sbuf;
 
-	if ( !(hbin = TALLOC_ZERO_P( file->mem_ctx, REGF_HBIN )) )
+	if ( !(hbin = talloc_zero( file->mem_ctx, REGF_HBIN )) )
 		return NULL;
 
 	memcpy( hbin->header, "hbin", sizeof(HBIN_HDR_SIZE) );
@@ -1719,7 +1720,7 @@ static bool create_vk_record(REGF_FILE *file, REGF_VK_REC *vk,
 	if ( vk->data_size > sizeof(uint32) ) {
 		uint32 data_size = ( (vk->data_size+sizeof(uint32)) & 0xfffffff8 ) + 8;
 
-		vk->data = (uint8 *)TALLOC_MEMDUP( file->mem_ctx,
+		vk->data = (uint8 *)talloc_memdup( file->mem_ctx,
 						   regval_data_p(value),
 						   vk->data_size );
 		if (vk->data == NULL) {
@@ -1750,7 +1751,7 @@ static bool create_vk_record(REGF_FILE *file, REGF_VK_REC *vk,
 
 static int hashrec_cmp( REGF_HASH_REC *h1, REGF_HASH_REC *h2 )
 {
-	return StrCaseCmp( h1->fullname, h2->fullname );
+	return strcasecmp_m( h1->fullname, h2->fullname );
 }
 
 /*******************************************************************
@@ -1764,7 +1765,7 @@ static int hashrec_cmp( REGF_HASH_REC *h1, REGF_HASH_REC *h2 )
 	REGF_HBIN *vlist_hbin = NULL;
 	uint32 size;
 
-	if ( !(nk = TALLOC_ZERO_P( file->mem_ctx, REGF_NK_REC )) )
+	if ( !(nk = talloc_zero( file->mem_ctx, REGF_NK_REC )) )
 		return NULL;
 
 	memcpy( nk->header, "nk", REC_HDR_SIZE );
@@ -1830,7 +1831,7 @@ static int hashrec_cmp( REGF_HASH_REC *h1, REGF_HASH_REC *h2 )
 				return NULL;
 			}
 
-			if ( !(nk->sec_desc = TALLOC_ZERO_P( file->mem_ctx, REGF_SK_REC )) )
+			if ( !(nk->sec_desc = talloc_zero( file->mem_ctx, REGF_SK_REC )) )
 				return NULL;
 	
 			/* now we have to store the security descriptor in the list and 
@@ -1897,7 +1898,7 @@ static int hashrec_cmp( REGF_HASH_REC *h1, REGF_HASH_REC *h2 )
 		
 		nk->subkeys.num_keys = nk->num_subkeys;
 		if (nk->subkeys.num_keys) {
-			if ( !(nk->subkeys.hashes = TALLOC_ZERO_ARRAY( file->mem_ctx, REGF_HASH_REC, nk->subkeys.num_keys )) )
+			if ( !(nk->subkeys.hashes = talloc_zero_array( file->mem_ctx, REGF_HASH_REC, nk->subkeys.num_keys )) )
 				return NULL;
 		} else {
 			nk->subkeys.hashes = NULL;
@@ -1925,7 +1926,7 @@ static int hashrec_cmp( REGF_HASH_REC *h1, REGF_HASH_REC *h2 )
 		nk->values_off = prs_offset( &vlist_hbin->ps ) + vlist_hbin->first_hbin_off - HBIN_HDR_SIZE;
 	
 		if (nk->num_values) {
-			if ( !(nk->values = TALLOC_ARRAY( file->mem_ctx, REGF_VK_REC, nk->num_values )) )
+			if ( !(nk->values = talloc_array( file->mem_ctx, REGF_VK_REC, nk->num_values )) )
 				return NULL;
 		} else {
 			nk->values = NULL;
diff --git a/source3/rpc_client/cli_lsarpc.c b/source3/rpc_client/cli_lsarpc.c
index e5995711819..66192bdf54f 100644
--- a/source3/rpc_client/cli_lsarpc.c
+++ b/source3/rpc_client/cli_lsarpc.c
@@ -187,7 +187,7 @@ static NTSTATUS dcerpc_lsa_lookup_sids_noalloc(struct dcerpc_binding_handle *h,
 	ZERO_STRUCT(lsa_names);
 
 	sid_array.num_sids = num_sids;
-	sid_array.sids = TALLOC_ARRAY(mem_ctx, struct lsa_SidPtr, num_sids);
+	sid_array.sids = talloc_array(mem_ctx, struct lsa_SidPtr, num_sids);
 	if (sid_array.sids == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -354,19 +354,19 @@ static NTSTATUS dcerpc_lsa_lookup_sids_generic(struct dcerpc_binding_handle *h,
 	bool have_unmapped = false;
 
 	if (num_sids) {
-		if (!(domains = TALLOC_ARRAY(mem_ctx, char *, num_sids))) {
+		if (!(domains = talloc_array(mem_ctx, char *, num_sids))) {
 			DEBUG(0, ("rpccli_lsa_lookup_sids(): out of memory\n"));
 			status = NT_STATUS_NO_MEMORY;
 			goto fail;
 		}
 
-		if (!(names = TALLOC_ARRAY(mem_ctx, char *, num_sids))) {
+		if (!(names = talloc_array(mem_ctx, char *, num_sids))) {
 			DEBUG(0, ("rpccli_lsa_lookup_sids(): out of memory\n"));
 			status = NT_STATUS_NO_MEMORY;
 			goto fail;
 		}
 
-		if (!(types = TALLOC_ARRAY(mem_ctx, enum lsa_SidType, num_sids))) {
+		if (!(types = talloc_array(mem_ctx, enum lsa_SidType, num_sids))) {
 			DEBUG(0, ("rpccli_lsa_lookup_sids(): out of memory\n"));
 			status = NT_STATUS_NO_MEMORY;
 			goto fail;
@@ -589,7 +589,7 @@ static NTSTATUS dcerpc_lsa_lookup_names_generic(struct dcerpc_binding_handle *h,
 	ZERO_STRUCT(sid_array);
 	ZERO_STRUCT(sid_array3);
 
-	lsa_names = TALLOC_ARRAY(mem_ctx, struct lsa_String, num_names);
+	lsa_names = talloc_array(mem_ctx, struct lsa_String, num_names);
 	if (lsa_names == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -639,20 +639,20 @@ static NTSTATUS dcerpc_lsa_lookup_names_generic(struct dcerpc_binding_handle *h,
 	}
 
 	if (num_names) {
-		if (!((*sids = TALLOC_ARRAY(mem_ctx, struct dom_sid, num_names)))) {
+		if (!((*sids = talloc_array(mem_ctx, struct dom_sid, num_names)))) {
 			DEBUG(0, ("cli_lsa_lookup_sids(): out of memory\n"));
 			*presult = NT_STATUS_NO_MEMORY;
 			goto done;
 		}
 
-		if (!((*types = TALLOC_ARRAY(mem_ctx, enum lsa_SidType, num_names)))) {
+		if (!((*types = talloc_array(mem_ctx, enum lsa_SidType, num_names)))) {
 			DEBUG(0, ("cli_lsa_lookup_sids(): out of memory\n"));
 			*presult = NT_STATUS_NO_MEMORY;
 			goto done;
 		}
 
 		if (dom_names != NULL) {
-			*dom_names = TALLOC_ARRAY(mem_ctx, const char *, num_names);
+			*dom_names = talloc_array(mem_ctx, const char *, num_names);
 			if (*dom_names == NULL) {
 				DEBUG(0, ("cli_lsa_lookup_sids(): out of memory\n"));
 				*presult = NT_STATUS_NO_MEMORY;
diff --git a/source3/rpc_client/cli_netlogon.c b/source3/rpc_client/cli_netlogon.c
index bd3232d2cde..3c685fd9a7f 100644
--- a/source3/rpc_client/cli_netlogon.c
+++ b/source3/rpc_client/cli_netlogon.c
@@ -177,7 +177,7 @@ NTSTATUS rpccli_netlogon_sam_logon(struct rpc_pipe_client *cli,
 
 	ZERO_STRUCT(ret_creds);
 
-	logon = TALLOC_ZERO_P(mem_ctx, union netr_LogonLevel);
+	logon = talloc_zero(mem_ctx, union netr_LogonLevel);
 	if (!logon) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -185,7 +185,7 @@ NTSTATUS rpccli_netlogon_sam_logon(struct rpc_pipe_client *cli,
 	if (workstation) {
 		fstr_sprintf( clnt_name_slash, "\\\\%s", workstation );
 	} else {
-		fstr_sprintf( clnt_name_slash, "\\\\%s", global_myname() );
+		fstr_sprintf( clnt_name_slash, "\\\\%s", lp_netbios_name() );
 	}
 
 	/* Initialise input parameters */
@@ -200,7 +200,7 @@ NTSTATUS rpccli_netlogon_sam_logon(struct rpc_pipe_client *cli,
 		struct samr_Password lmpassword;
 		struct samr_Password ntpassword;
 
-		password_info = TALLOC_ZERO_P(mem_ctx, struct netr_PasswordInfo);
+		password_info = talloc_zero(mem_ctx, struct netr_PasswordInfo);
 		if (!password_info) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -240,7 +240,7 @@ NTSTATUS rpccli_netlogon_sam_logon(struct rpc_pipe_client *cli,
 		ZERO_STRUCT(lm);
 		ZERO_STRUCT(nt);
 
-		network_info = TALLOC_ZERO_P(mem_ctx, struct netr_NetworkInfo);
+		network_info = talloc_zero(mem_ctx, struct netr_NetworkInfo);
 		if (!network_info) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -279,7 +279,7 @@ NTSTATUS rpccli_netlogon_sam_logon(struct rpc_pipe_client *cli,
 
 	status = dcerpc_netr_LogonSamLogon(b, mem_ctx,
 					   cli->srv_name_slash,
-					   global_myname(),
+					   lp_netbios_name(),
 					   &clnt_creds,
 					   &ret_creds,
 					   logon_type,
@@ -388,12 +388,12 @@ NTSTATUS rpccli_netlogon_sam_network_logon(struct rpc_pipe_client *cli,
 	ZERO_STRUCT(lm);
 	ZERO_STRUCT(nt);
 
-	logon = TALLOC_ZERO_P(mem_ctx, union netr_LogonLevel);
+	logon = talloc_zero(mem_ctx, union netr_LogonLevel);
 	if (!logon) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	network_info = TALLOC_ZERO_P(mem_ctx, struct netr_NetworkInfo);
+	network_info = talloc_zero(mem_ctx, struct netr_NetworkInfo);
 	if (!network_info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -441,7 +441,7 @@ NTSTATUS rpccli_netlogon_sam_network_logon(struct rpc_pipe_client *cli,
 
 	status = dcerpc_netr_LogonSamLogon(b, mem_ctx,
 					   server_name_slash,
-					   global_myname(),
+					   lp_netbios_name(),
 					   &clnt_creds,
 					   &ret_creds,
 					   NetlogonNetworkInformation,
@@ -505,12 +505,12 @@ NTSTATUS rpccli_netlogon_sam_network_logon_ex(struct rpc_pipe_client *cli,
 	ZERO_STRUCT(lm);
 	ZERO_STRUCT(nt);
 
-	logon = TALLOC_ZERO_P(mem_ctx, union netr_LogonLevel);
+	logon = talloc_zero(mem_ctx, union netr_LogonLevel);
 	if (!logon) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	network_info = TALLOC_ZERO_P(mem_ctx, struct netr_NetworkInfo);
+	network_info = talloc_zero(mem_ctx, struct netr_NetworkInfo);
 	if (!network_info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -556,7 +556,7 @@ NTSTATUS rpccli_netlogon_sam_network_logon_ex(struct rpc_pipe_client *cli,
 
 	status = dcerpc_netr_LogonSamLogonEx(b, mem_ctx,
 					     server_name_slash,
-					     global_myname(),
+					     lp_netbios_name(),
 					     NetlogonNetworkInformation,
 					     logon,
 					     validation_level,
@@ -608,7 +608,7 @@ NTSTATUS rpccli_netlogon_set_trust_password(struct rpc_pipe_client *cli,
 		result = rpccli_netlogon_setup_creds(cli,
 						     cli->desthost, /* server name */
 						     lp_workgroup(), /* domain */
-						     global_myname(), /* client name */
+						     lp_netbios_name(), /* client name */
 						     account_name, /* machine account name */
 						     orig_trust_passwd_hash,
 						     sec_channel_type,
diff --git a/source3/rpc_client/cli_netlogon.h b/source3/rpc_client/cli_netlogon.h
index 808a4283ce7..ad59d5b2039 100644
--- a/source3/rpc_client/cli_netlogon.h
+++ b/source3/rpc_client/cli_netlogon.h
@@ -1,3 +1,28 @@
+/*
+   Unix SMB/CIFS implementation.
+   NT Domain Authentication SMB / MSRPC client
+   Copyright (C) Andrew Tridgell 1992-2000
+   Copyright (C) Jeremy Allison                    1998.
+   Largely re-written by Jeremy Allison (C)	   2005.
+   Copyright (C) Guenther Deschner                 2008.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _RPC_CLIENT_CLI_NETLOGON_H_
+#define _RPC_CLIENT_CLI_NETLOGON_H_
+
 /* The following definitions come from rpc_client/cli_netlogon.c  */
 
 NTSTATUS rpccli_netlogon_setup_creds(struct rpc_pipe_client *cli,
@@ -49,3 +74,4 @@ NTSTATUS rpccli_netlogon_set_trust_password(struct rpc_pipe_client *cli,
 					    const unsigned char new_trust_passwd_hash[16],
 					    enum netr_SchannelType sec_channel_type);
 
+#endif /* _RPC_CLIENT_CLI_NETLOGON_H_ */
diff --git a/source3/rpc_client/cli_pipe.c b/source3/rpc_client/cli_pipe.c
index d7f990af2c9..0b9470620d9 100644
--- a/source3/rpc_client/cli_pipe.c
+++ b/source3/rpc_client/cli_pipe.c
@@ -19,12 +19,12 @@
  */
 
 #include "includes.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "librpc/gen_ndr/ndr_epmapper_c.h"
 #include "../librpc/gen_ndr/ndr_schannel.h"
 #include "../librpc/gen_ndr/ndr_dssetup.h"
 #include "../libcli/auth/schannel.h"
 #include "../libcli/auth/spnego.h"
-#include "smb_krb5.h"
 #include "../libcli/auth/ntlmssp.h"
 #include "ntlmssp_wrap.h"
 #include "librpc/gen_ndr/ndr_dcerpc.h"
@@ -33,7 +33,7 @@
 #include "librpc/crypto/spnego.h"
 #include "rpc_dce.h"
 #include "cli_pipe.h"
-#include "ntdomain.h"
+#include "client.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_CLI
@@ -469,11 +469,7 @@ static NTSTATUS cli_pipe_validate_current_pdu(TALLOC_CTX *mem_ctx,
 			  pkt->u.fault.status),
 			  rpccli_pipe_txt(talloc_tos(), cli)));
 
-		if (NT_STATUS_IS_OK(NT_STATUS(pkt->u.fault.status))) {
-			return NT_STATUS_UNSUCCESSFUL;
-		} else {
-			return NT_STATUS(pkt->u.fault.status);
-		}
+		return dcerpc_fault_to_nt_status(pkt->u.fault.status);
 
 	default:
 		DEBUG(0, (__location__ "Unknown packet type %u received "
@@ -610,7 +606,7 @@ static void cli_api_pipe_write_done(struct tevent_req *subreq)
 		return;
 	}
 
-	state->rdata = TALLOC_ARRAY(state, uint8_t, RPC_HEADER_LEN);
+	state->rdata = talloc_array(state, uint8_t, RPC_HEADER_LEN);
 	if (tevent_req_nomem(state->rdata, req)) {
 		return;
 	}
@@ -1091,7 +1087,7 @@ static NTSTATUS create_schannel_auth_rpc_bind_req(struct rpc_pipe_client *cli,
 	r.Flags				= NL_FLAG_OEM_NETBIOS_DOMAIN_NAME |
 					  NL_FLAG_OEM_NETBIOS_COMPUTER_NAME;
 	r.oem_netbios_domain.a		= cli->auth->domain;
-	r.oem_netbios_computer.a	= global_myname();
+	r.oem_netbios_computer.a	= lp_netbios_name();
 
 	status = dcerpc_push_schannel_bind(cli, &r, auth_token);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -2334,7 +2330,7 @@ static NTSTATUS rpccli_ntlmssp_bind_data(TALLOC_CTX *mem_ctx,
 	}
 
 	status = auth_ntlmssp_client_start(NULL,
-				      global_myname(),
+				      lp_netbios_name(),
 				      lp_workgroup(),
 				      lp_client_ntlmv2_auth(),
 				      &ntlmssp_ctx);
@@ -2435,7 +2431,7 @@ static NTSTATUS rpc_pipe_open_tcp_port(TALLOC_CTX *mem_ctx, const char *host,
 	NTSTATUS status;
 	int fd;
 
-	result = TALLOC_ZERO_P(mem_ctx, struct rpc_pipe_client);
+	result = talloc_zero(mem_ctx, struct rpc_pipe_client);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -2544,7 +2540,7 @@ static NTSTATUS rpc_pipe_get_tcp_port(const char *host,
 
 	/* create tower for asking the epmapper */
 
-	map_binding = TALLOC_ZERO_P(tmp_ctx, struct dcerpc_binding);
+	map_binding = talloc_zero(tmp_ctx, struct dcerpc_binding);
 	if (map_binding == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -2555,7 +2551,7 @@ static NTSTATUS rpc_pipe_get_tcp_port(const char *host,
 	map_binding->host = host; /* needed? */
 	map_binding->endpoint = "0"; /* correct? needed? */
 
-	map_tower = TALLOC_ZERO_P(tmp_ctx, struct epm_twr_t);
+	map_tower = talloc_zero(tmp_ctx, struct epm_twr_t);
 	if (map_tower == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -2569,14 +2565,14 @@ static NTSTATUS rpc_pipe_get_tcp_port(const char *host,
 
 	/* allocate further parameters for the epm_Map call */
 
-	res_towers = TALLOC_ARRAY(tmp_ctx, struct epm_twr_t, max_towers);
+	res_towers = talloc_array(tmp_ctx, struct epm_twr_t, max_towers);
 	if (res_towers == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
 	}
 	towers.twr = res_towers;
 
-	entry_handle = TALLOC_ZERO_P(tmp_ctx, struct policy_handle);
+	entry_handle = talloc_zero(tmp_ctx, struct policy_handle);
 	if (entry_handle == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -2586,7 +2582,7 @@ static NTSTATUS rpc_pipe_get_tcp_port(const char *host,
 
 	status = dcerpc_epm_Map(epm_handle,
 				tmp_ctx,
-				CONST_DISCARD(struct GUID *,
+				discard_const_p(struct GUID,
 					      &(abstract_syntax->uuid)),
 				map_tower,
 				entry_handle,
@@ -2760,7 +2756,7 @@ static NTSTATUS rpc_pipe_open_np(struct cli_state *cli,
 		return NT_STATUS_INVALID_HANDLE;
 	}
 
-	result = TALLOC_ZERO_P(NULL, struct rpc_pipe_client);
+	result = talloc_zero(NULL, struct rpc_pipe_client);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/rpc_client/cli_pipe_schannel.c b/source3/rpc_client/cli_pipe_schannel.c
index 0535e930be7..f48c4c7318f 100644
--- a/source3/rpc_client/cli_pipe_schannel.c
+++ b/source3/rpc_client/cli_pipe_schannel.c
@@ -26,6 +26,7 @@
 #include "librpc/gen_ndr/ndr_dcerpc.h"
 #include "librpc/rpc/dcerpc.h"
 #include "passdb.h"
+#include "client.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_CLI
@@ -57,7 +58,7 @@ static NTSTATUS get_schannel_session_key_common(struct rpc_pipe_client *netlogon
 	status = rpccli_netlogon_setup_creds(netlogon_pipe,
 					cli->desthost, /* server name */
 					domain,	       /* domain */
-					global_myname(), /* client name */
+					lp_netbios_name(), /* client name */
 					machine_account, /* machine account name */
 					machine_pwd,
 					sec_chan_type,
diff --git a/source3/rpc_client/cli_spoolss.c b/source3/rpc_client/cli_spoolss.c
index 831da85205f..5c8448b29b6 100644
--- a/source3/rpc_client/cli_spoolss.c
+++ b/source3/rpc_client/cli_spoolss.c
@@ -26,7 +26,6 @@
 #include "rpc_client/rpc_client.h"
 #include "../librpc/gen_ndr/ndr_spoolss_c.h"
 #include "rpc_client/cli_spoolss.h"
-#include "ntdomain.h"
 
 /**********************************************************************
  convencience wrapper around rpccli_spoolss_OpenPrinterEx
@@ -48,7 +47,7 @@ WERROR rpccli_spoolss_openprinter_ex(struct rpc_pipe_client *cli,
 	ZERO_STRUCT(devmode_ctr);
 
 	level1.size	= 28;
-	level1.client	= talloc_asprintf(mem_ctx, "\\\\%s", global_myname());
+	level1.client	= talloc_asprintf(mem_ctx, "\\\\%s", lp_netbios_name());
 	W_ERROR_HAVE_NO_MEMORY(level1.client);
 	level1.user	= cli->auth->user_name;
 	level1.build	= 1381;
@@ -231,7 +230,7 @@ WERROR rpccli_spoolss_addprinterex(struct rpc_pipe_client *cli,
 	level1.major		= 2;
 	level1.minor		= 0;
 	level1.processor	= 0;
-	level1.client		= talloc_asprintf(mem_ctx, "\\\\%s", global_myname());
+	level1.client		= talloc_asprintf(mem_ctx, "\\\\%s", lp_netbios_name());
 	W_ERROR_HAVE_NO_MEMORY(level1.client);
 	level1.user		= cli->auth->user_name;
 
diff --git a/source3/rpc_client/cli_spoolss.h b/source3/rpc_client/cli_spoolss.h
index 285a34fab40..9883874cd68 100644
--- a/source3/rpc_client/cli_spoolss.h
+++ b/source3/rpc_client/cli_spoolss.h
@@ -1,3 +1,30 @@
+/*
+   Unix SMB/CIFS implementation.
+   RPC pipe client
+
+   Copyright (C) Gerald Carter                2001-2005,
+   Copyright (C) Tim Potter                   2000-2002,
+   Copyright (C) Andrew Tridgell              1994-2000,
+   Copyright (C) Jean-Francois Micouleau      1999-2000.
+   Copyright (C) Jeremy Allison                         2005.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _RPC_CLIENT_CLI_SPOOLSS_H_
+#define _RPC_CLIENT_CLI_SPOOLSS_H_
+
 /* The following definitions come from rpc_client/cli_spoolss.c  */
 
 WERROR rpccli_spoolss_openprinter_ex(struct rpc_pipe_client *cli,
@@ -122,3 +149,5 @@ WERROR rpccli_spoolss_enumprinterdataex(struct rpc_pipe_client *cli,
 					uint32_t offered,
 					uint32_t *count,
 					struct spoolss_PrinterEnumValues **info);
+
+#endif /* _RPC_CLIENT_CLI_SPOOLSS_H_ */
diff --git a/source3/rpc_client/cli_winreg.c b/source3/rpc_client/cli_winreg.c
index 2517dbc50ab..e5ebc25ed80 100644
--- a/source3/rpc_client/cli_winreg.c
+++ b/source3/rpc_client/cli_winreg.c
@@ -33,7 +33,7 @@ NTSTATUS dcerpc_winreg_query_dword(TALLOC_CTX *mem_ctx,
 				   WERROR *pwerr)
 {
 	struct winreg_String wvalue;
-	enum winreg_Type type;
+	enum winreg_Type type = REG_NONE;
 	uint32_t value_len = 0;
 	uint32_t data_size = 0;
 	WERROR result = WERR_OK;
@@ -69,7 +69,7 @@ NTSTATUS dcerpc_winreg_query_dword(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	blob = data_blob_talloc(mem_ctx, NULL, data_size);
+	blob = data_blob_talloc_zero(mem_ctx, data_size);
 	if (blob.data == NULL) {
 		*pwerr = WERR_NOMEM;
 		return status;
@@ -108,13 +108,14 @@ NTSTATUS dcerpc_winreg_query_binary(TALLOC_CTX *mem_ctx,
 				    WERROR *pwerr)
 {
 	struct winreg_String wvalue;
-	enum winreg_Type type;
+	enum winreg_Type type = REG_NONE;
 	WERROR result = WERR_OK;
 	uint32_t value_len = 0;
 	uint32_t data_size = 0;
 	NTSTATUS status;
 	DATA_BLOB blob;
 
+	ZERO_STRUCT(wvalue);
 	wvalue.name = value;
 
 	status = dcerpc_winreg_QueryValue(h,
@@ -139,7 +140,7 @@ NTSTATUS dcerpc_winreg_query_binary(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	blob = data_blob_talloc(mem_ctx, NULL, data_size);
+	blob = data_blob_talloc_zero(mem_ctx, data_size);
 	if (blob.data == NULL) {
 		*pwerr = WERR_NOMEM;
 		return status;
@@ -179,7 +180,7 @@ NTSTATUS dcerpc_winreg_query_multi_sz(TALLOC_CTX *mem_ctx,
 				      WERROR *pwerr)
 {
 	struct winreg_String wvalue;
-	enum winreg_Type type;
+	enum winreg_Type type = REG_NONE;
 	WERROR result = WERR_OK;
 	uint32_t value_len = 0;
 	uint32_t data_size = 0;
@@ -210,7 +211,7 @@ NTSTATUS dcerpc_winreg_query_multi_sz(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	blob = data_blob_talloc(mem_ctx, NULL, data_size);
+	blob = data_blob_talloc_zero(mem_ctx, data_size);
 	if (blob.data == NULL) {
 		*pwerr = WERR_NOMEM;
 		return status;
@@ -254,7 +255,7 @@ NTSTATUS dcerpc_winreg_query_sz(TALLOC_CTX *mem_ctx,
 				      WERROR *pwerr)
 {
 	struct winreg_String wvalue;
-	enum winreg_Type type;
+	enum winreg_Type type = REG_NONE;
 	WERROR result = WERR_OK;
 	uint32_t value_len = 0;
 	uint32_t data_size = 0;
@@ -285,7 +286,7 @@ NTSTATUS dcerpc_winreg_query_sz(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	blob = data_blob_talloc(mem_ctx, NULL, data_size);
+	blob = data_blob_talloc_zero(mem_ctx, data_size);
 	if (blob.data == NULL) {
 		*pwerr = WERR_NOMEM;
 		return status;
@@ -380,13 +381,14 @@ NTSTATUS dcerpc_winreg_set_dword(TALLOC_CTX *mem_ctx,
 				 uint32_t data,
 				 WERROR *pwerr)
 {
-	struct winreg_String wvalue = { 0, };
+	struct winreg_String wvalue;
 	DATA_BLOB blob;
 	WERROR result = WERR_OK;
 	NTSTATUS status;
 
+	ZERO_STRUCT(wvalue);
 	wvalue.name = value;
-	blob = data_blob_talloc(mem_ctx, NULL, 4);
+	blob = data_blob_talloc_zero(mem_ctx, 4);
 	SIVAL(blob.data, 0, data);
 
 	status = dcerpc_winreg_SetValue(h,
@@ -615,7 +617,7 @@ NTSTATUS dcerpc_winreg_add_multi_sz(TALLOC_CTX *mem_ctx,
 	/* count the elements */
 	for (p = a, i = 0; p && *p; p++, i++);
 
-	p = TALLOC_REALLOC_ARRAY(mem_ctx, a, const char *, i + 2);
+	p = talloc_realloc(mem_ctx, a, const char *, i + 2);
 	if (p == NULL) {
 		*pwerr = WERR_NOMEM;
 		return NT_STATUS_OK;
diff --git a/source3/rpc_client/cli_winreg_int.c b/source3/rpc_client/cli_winreg_int.c
index 8e11ae832d6..1828b419a95 100644
--- a/source3/rpc_client/cli_winreg_int.c
+++ b/source3/rpc_client/cli_winreg_int.c
@@ -21,7 +21,6 @@
 
 #include "includes.h"
 #include "include/registry.h"
-#include "utils/net_registry_util.h"
 #include "librpc/gen_ndr/ndr_winreg_c.h"
 #include "rpc_client/cli_winreg_int.h"
 #include "rpc_server/rpc_ncacn_np.h"
diff --git a/source3/rpc_client/cli_winreg_int.h b/source3/rpc_client/cli_winreg_int.h
index f175c17f5c9..6e9d07f95cc 100644
--- a/source3/rpc_client/cli_winreg_int.h
+++ b/source3/rpc_client/cli_winreg_int.h
@@ -23,6 +23,7 @@
 #define CLI_WINREG_INT_H
 
 struct auth_serversupplied_info;
+struct dcerpc_binding_handle;
 
 /**
  * @brief Connect to the interal winreg server and open the given key.
diff --git a/source3/rpc_client/init_lsa.h b/source3/rpc_client/init_lsa.h
index 670caed2128..e4e5c119636 100644
--- a/source3/rpc_client/init_lsa.h
+++ b/source3/rpc_client/init_lsa.h
@@ -1,3 +1,25 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Copyright (C) Guenther Deschner                  2008.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_CLIENT_INIT_LSA_H_
+#define _RPC_CLIENT_INIT_LSA_H_
+
 struct lsa_String;
 struct lsa_StringLarge;
 struct lsa_AsciiString;
@@ -9,3 +31,5 @@ void init_lsa_String(struct lsa_String *name, const char *s);
 void init_lsa_StringLarge(struct lsa_StringLarge *name, const char *s);
 void init_lsa_AsciiString(struct lsa_AsciiString *name, const char *s);
 void init_lsa_AsciiStringLarge(struct lsa_AsciiStringLarge *name, const char *s);
+
+#endif /* _RPC_CLIENT_INIT_LSA_H_ */
diff --git a/source3/rpc_client/init_netlogon.h b/source3/rpc_client/init_netlogon.h
index d87a9ea493a..826d0c56bb4 100644
--- a/source3/rpc_client/init_netlogon.h
+++ b/source3/rpc_client/init_netlogon.h
@@ -1,5 +1,29 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Copyright (C) Guenther Deschner                  2008.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_CLIENT_INIT_NETLOGON_H_
+#define _RPC_CLIENT_INIT_NETLOGON_H_
+
 /* The following definitions come from rpc_client/init_netlogon.c  */
 
 void init_netr_CryptPassword(const char *pwd,
 			     unsigned char session_key[16],
 			     struct netr_CryptPassword *pwd_buf);
+
+#endif /* _RPC_CLIENT_INIT_NETLOGON_H_ */
diff --git a/source3/rpc_client/init_samr.h b/source3/rpc_client/init_samr.h
index 1ddaef66153..223fa91e3d9 100644
--- a/source3/rpc_client/init_samr.h
+++ b/source3/rpc_client/init_samr.h
@@ -1,3 +1,25 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Copyright (C) Guenther Deschner                  2008.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_CLIENT_INIT_SAMR_H_
+#define _RPC_CLIENT_INIT_SAMR_H_
+
 /* The following definitions come from rpc_client/init_samr.c  */
 
 void init_samr_CryptPasswordEx(const char *pwd,
@@ -7,3 +29,4 @@ void init_samr_CryptPassword(const char *pwd,
 			     DATA_BLOB *session_key,
 			     struct samr_CryptPassword *pwd_buf);
 
+#endif /* _RPC_CLIENT_INIT_SAMR_H_ */
diff --git a/source3/rpc_client/init_spoolss.h b/source3/rpc_client/init_spoolss.h
index 60ee041ae59..a9374453e11 100644
--- a/source3/rpc_client/init_spoolss.h
+++ b/source3/rpc_client/init_spoolss.h
@@ -1,3 +1,24 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Copyright (C) Guenther Deschner                  2009.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_CLIENT_INIT_SPOOLSS_H_
+#define _RPC_CLIENT_INIT_SPOOLSS_H_
 
 /* The following definitions come from rpc_client/init_spoolss.c  */
 
@@ -14,3 +35,4 @@ WERROR push_spoolss_PrinterData(TALLOC_CTX *mem_ctx, DATA_BLOB *blob,
 void spoolss_printerinfo2_to_setprinterinfo2(const struct spoolss_PrinterInfo2 *i,
 					     struct spoolss_SetPrinterInfo2 *s);
 
+#endif /* _RPC_CLIENT_INIT_SPOOLSS_H_ */
diff --git a/source3/rpc_client/rpc_transport_np.c b/source3/rpc_client/rpc_transport_np.c
index 2be01621448..c66d4b80227 100644
--- a/source3/rpc_client/rpc_transport_np.c
+++ b/source3/rpc_client/rpc_transport_np.c
@@ -18,6 +18,7 @@
  */
 
 #include "includes.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "rpc_client/rpc_transport.h"
 #include "libsmb/cli_np_tstream.h"
 
diff --git a/source3/rpc_client/rpc_transport_sock.c b/source3/rpc_client/rpc_transport_sock.c
index 71f979c2af9..a833e4469a3 100644
--- a/source3/rpc_client/rpc_transport_sock.c
+++ b/source3/rpc_client/rpc_transport_sock.c
@@ -18,8 +18,8 @@
  */
 
 #include "includes.h"
+#include "../lib/tsocket/tsocket.h"
 #include "rpc_client/rpc_transport.h"
-#include "../lib/async_req/async_sock.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_CLI
diff --git a/source3/rpc_client/rpc_transport_tstream.c b/source3/rpc_client/rpc_transport_tstream.c
index 248b09f0491..488c093d3b7 100644
--- a/source3/rpc_client/rpc_transport_tstream.c
+++ b/source3/rpc_client/rpc_transport_tstream.c
@@ -18,6 +18,7 @@
  */
 
 #include "includes.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "rpc_client/rpc_transport.h"
 #include "lib/tsocket/tsocket.h"
 #include "libsmb/cli_np_tstream.h"
@@ -201,7 +202,7 @@ static struct tevent_req *rpc_tstream_read_send(TALLOC_CTX *mem_ctx,
 		return tevent_req_post(req, ev);
 	}
 
-	endtime = timeval_current_ofs(0, transp->timeout * 1000);
+	endtime = timeval_current_ofs_msec(transp->timeout);
 	if (!tevent_req_set_endtime(subreq, ev, endtime)) {
 		goto fail;
 	}
@@ -285,7 +286,7 @@ static struct tevent_req *rpc_tstream_write_send(TALLOC_CTX *mem_ctx,
 		goto fail;
 	}
 
-	endtime = timeval_current_ofs(0, transp->timeout * 1000);
+	endtime = timeval_current_ofs_msec(transp->timeout);
 	if (!tevent_req_set_endtime(subreq, ev, endtime)) {
 		goto fail;
 	}
@@ -373,7 +374,7 @@ static struct tevent_req *rpc_tstream_trans_send(TALLOC_CTX *mem_ctx,
 	state->req.iov_base = discard_const_p(void *, data);
 	state->max_rdata_len = max_rdata_len;
 
-	endtime = timeval_current_ofs(0, transp->timeout * 1000);
+	endtime = timeval_current_ofs_msec(transp->timeout);
 
 	subreq = tstream_writev_queue_send(state, ev,
 					   transp->stream,
diff --git a/source3/rpc_client/util_netlogon.h b/source3/rpc_client/util_netlogon.h
index 42e4326f7dc..cea9787acb6 100644
--- a/source3/rpc_client/util_netlogon.h
+++ b/source3/rpc_client/util_netlogon.h
@@ -1,5 +1,29 @@
+/*
+   Unix SMB/CIFS implementation.
+   Authentication utility functions
+   Copyright (C) Volker Lendecke 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _RPC_CLIENT_UTIL_NETLOGON_H_
+#define _RPC_CLIENT_UTIL_NETLOGON_H_
+
 /* The following definitions come from rpc_client/util_netlogon.c  */
 
 NTSTATUS copy_netr_SamBaseInfo(TALLOC_CTX *mem_ctx,
 			       const struct netr_SamBaseInfo *in,
 			       struct netr_SamBaseInfo *out);
+
+#endif /* _RPC_CLIENT_UTIL_NETLOGON_H_ */
diff --git a/source3/rpc_server/dcesrv_gssapi.c b/source3/rpc_server/dcesrv_gssapi.c
index ec024596332..534e8a41896 100644
--- a/source3/rpc_server/dcesrv_gssapi.c
+++ b/source3/rpc_server/dcesrv_gssapi.c
@@ -23,7 +23,9 @@
 #include "../librpc/gen_ndr/ndr_krb5pac.h"
 #include "librpc/crypto/gse.h"
 #include "auth.h"
-
+#ifdef HAVE_KRB5
+#include "libcli/auth/krb5_wrap.h"
+#endif
 NTSTATUS gssapi_server_auth_start(TALLOC_CTX *mem_ctx,
 				  bool do_sign,
 				  bool do_seal,
@@ -105,14 +107,9 @@ NTSTATUS gssapi_server_get_user_info(struct gse_context *gse_ctx,
 				     struct auth_serversupplied_info **server_info)
 {
 	TALLOC_CTX *tmp_ctx;
-	DATA_BLOB auth_data;
-	time_t tgs_authtime;
-	NTTIME tgs_authtime_nttime;
-	DATA_BLOB pac;
-	struct PAC_DATA *pac_data;
-	struct PAC_LOGON_NAME *logon_name = NULL;
+	DATA_BLOB pac_blob;
+	struct PAC_DATA *pac_data = NULL;
 	struct PAC_LOGON_INFO *logon_info = NULL;
-	enum ndr_err_code ndr_err;
 	unsigned int i;
 	bool is_mapped;
 	bool is_guest;
@@ -122,14 +119,13 @@ NTSTATUS gssapi_server_get_user_info(struct gse_context *gse_ctx,
 	char *username;
 	struct passwd *pw;
 	NTSTATUS status;
-	bool bret;
 
 	tmp_ctx = talloc_new(mem_ctx);
 	if (!tmp_ctx) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	status = gse_get_authz_data(gse_ctx, tmp_ctx, &auth_data);
+	status = gse_get_pac_blob(gse_ctx, tmp_ctx, &pac_blob);
 	if (NT_STATUS_EQUAL(status, NT_STATUS_NOT_FOUND)) {
 		/* TODO: Fetch user by principal name ? */
 		status = NT_STATUS_ACCESS_DENIED;
@@ -139,37 +135,22 @@ NTSTATUS gssapi_server_get_user_info(struct gse_context *gse_ctx,
 		goto done;
 	}
 
-	bret = unwrap_pac(tmp_ctx, &auth_data, &pac);
-	if (!bret) {
-		DEBUG(1, ("Failed to unwrap PAC\n"));
-		status = NT_STATUS_ACCESS_DENIED;
-		goto done;
-	}
-
-	status = gse_get_client_name(gse_ctx, tmp_ctx, &princ_name);
+#ifdef HAVE_KRB5
+	status = kerberos_decode_pac(tmp_ctx,
+				     pac_blob,
+				     NULL, NULL, NULL, NULL, 0, &pac_data);
+#else
+	status = NT_STATUS_ACCESS_DENIED;
+#endif
+	data_blob_free(&pac_blob);
 	if (!NT_STATUS_IS_OK(status)) {
 		goto done;
 	}
 
-	status = gse_get_authtime(gse_ctx, &tgs_authtime);
+	status = gse_get_client_name(gse_ctx, tmp_ctx, &princ_name);
 	if (!NT_STATUS_IS_OK(status)) {
 		goto done;
 	}
-	unix_to_nt_time(&tgs_authtime_nttime, tgs_authtime);
-
-	pac_data = talloc_zero(tmp_ctx, struct PAC_DATA);
-	if (!pac_data) {
-		status = NT_STATUS_NO_MEMORY;
-		goto done;
-	}
-
-	ndr_err = ndr_pull_struct_blob(&pac, pac_data, pac_data,
-				(ndr_pull_flags_fn_t)ndr_pull_PAC_DATA);
-	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
-		DEBUG(1, ("Failed to parse the PAC for %s\n", princ_name));
-		status = ndr_map_error2ntstatus(ndr_err);
-		goto done;
-	}
 
 	/* get logon name and logon info */
 	for (i = 0; i < pac_data->num_buffers; i++) {
@@ -182,9 +163,6 @@ NTSTATUS gssapi_server_get_user_info(struct gse_context *gse_ctx,
 			}
 			logon_info = data_buf->info->logon_info.info;
 			break;
-		case PAC_TYPE_LOGON_NAME:
-			logon_name = &data_buf->info->logon_name;
-			break;
 		default:
 			break;
 		}
@@ -194,25 +172,6 @@ NTSTATUS gssapi_server_get_user_info(struct gse_context *gse_ctx,
 		status = NT_STATUS_NOT_FOUND;
 		goto done;
 	}
-	if (!logon_name) {
-		DEBUG(1, ("Invalid PAC data, missing logon info!\n"));
-		status = NT_STATUS_NOT_FOUND;
-		goto done;
-	}
-
-	/* check time */
-	if (tgs_authtime_nttime != logon_name->logon_time) {
-		DEBUG(1, ("Logon time mismatch between ticket and PAC!\n"
-			  "PAC Time = %s | Ticket Time = %s\n",
-			  nt_time_string(tmp_ctx, logon_name->logon_time),
-			  nt_time_string(tmp_ctx, tgs_authtime_nttime)));
-		status = NT_STATUS_ACCESS_DENIED;
-		goto done;
-	}
-
-	/* TODO: Should we check princ_name against account_name in
-	 * logon_name ? Are they supposed to be identical, or can an
-	 * account_name be different from the UPN ? */
 
 	status = get_user_from_kerberos_info(tmp_ctx, client_id->name,
 					     princ_name, logon_info,
diff --git a/source3/rpc_server/dfs/srv_dfs_nt.c b/source3/rpc_server/dfs/srv_dfs_nt.c
index 45e9d9a6ff3..5b4e423393c 100644
--- a/source3/rpc_server/dfs/srv_dfs_nt.c
+++ b/source3/rpc_server/dfs/srv_dfs_nt.c
@@ -22,11 +22,12 @@
 /* This is the implementation of the dfs pipe. */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_dfs.h"
 #include "msdfs.h"
 #include "smbd/smbd.h"
+#include "smbd/globals.h"
 #include "auth.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_MSDFS
@@ -58,7 +59,7 @@ WERROR _dfs_Add(struct pipes_struct *p, struct dfs_Add *r)
 		return WERR_ACCESS_DENIED;
 	}
 
-	jn = TALLOC_ZERO_P(ctx, struct junction_map);
+	jn = talloc_zero(ctx, struct junction_map);
 	if (!jn) {
 		return WERR_NOMEM;
 	}
@@ -87,7 +88,7 @@ WERROR _dfs_Add(struct pipes_struct *p, struct dfs_Add *r)
 		return WERR_NOMEM;
 	}
 
-	jn->referral_list = TALLOC_ARRAY(ctx, struct referral, jn->referral_count);
+	jn->referral_list = talloc_array(ctx, struct referral, jn->referral_count);
 	if(jn->referral_list == NULL) {
 		DEBUG(0,("init_reply_dfs_add: talloc failed for referral list!\n"));
 		return WERR_DFS_INTERNAL_ERROR;
@@ -123,7 +124,7 @@ WERROR _dfs_Remove(struct pipes_struct *p, struct dfs_Remove *r)
 		return WERR_ACCESS_DENIED;
 	}
 
-	jn = TALLOC_ZERO_P(ctx, struct junction_map);
+	jn = talloc_zero(ctx, struct junction_map);
 	if (!jn) {
 		return WERR_NOMEM;
 	}
@@ -192,7 +193,7 @@ WERROR _dfs_Remove(struct pipes_struct *p, struct dfs_Remove *r)
 static bool init_reply_dfs_info_1(TALLOC_CTX *mem_ctx, struct junction_map* j,struct dfs_Info1* dfs1)
 {
 	dfs1->path = talloc_asprintf(mem_ctx,
-				"\\\\%s\\%s\\%s", global_myname(),
+				"\\\\%s\\%s\\%s", lp_netbios_name(),
 				j->service_name, j->volume_name);
 	if (dfs1->path == NULL)
 		return False;
@@ -204,7 +205,7 @@ static bool init_reply_dfs_info_1(TALLOC_CTX *mem_ctx, struct junction_map* j,st
 static bool init_reply_dfs_info_2(TALLOC_CTX *mem_ctx, struct junction_map* j, struct dfs_Info2* dfs2)
 {
 	dfs2->path = talloc_asprintf(mem_ctx,
-			"\\\\%s\\%s\\%s", global_myname(), j->service_name, j->volume_name);
+			"\\\\%s\\%s\\%s", lp_netbios_name(), j->service_name, j->volume_name);
 	if (dfs2->path == NULL)
 		return False;
 	dfs2->comment = talloc_strdup(mem_ctx, j->comment);
@@ -218,9 +219,9 @@ static bool init_reply_dfs_info_3(TALLOC_CTX *mem_ctx, struct junction_map* j, s
 	int ii;
 	if (j->volume_name[0] == '\0')
 		dfs3->path = talloc_asprintf(mem_ctx, "\\\\%s\\%s",
-			global_myname(), j->service_name);
+			lp_netbios_name(), j->service_name);
 	else
-		dfs3->path = talloc_asprintf(mem_ctx, "\\\\%s\\%s\\%s", global_myname(),
+		dfs3->path = talloc_asprintf(mem_ctx, "\\\\%s\\%s\\%s", lp_netbios_name(),
 			j->service_name, j->volume_name);
 
 	if (dfs3->path == NULL)
@@ -232,7 +233,7 @@ static bool init_reply_dfs_info_3(TALLOC_CTX *mem_ctx, struct junction_map* j, s
 
 	/* also enumerate the stores */
 	if (j->referral_count) {
-		dfs3->stores = TALLOC_ARRAY(mem_ctx, struct dfs_StorageInfo, j->referral_count);
+		dfs3->stores = talloc_array(mem_ctx, struct dfs_StorageInfo, j->referral_count);
 		if (!dfs3->stores)
 			return False;
 		memset(dfs3->stores, '\0', j->referral_count * sizeof(struct dfs_StorageInfo));
@@ -278,7 +279,8 @@ WERROR _dfs_Enum(struct pipes_struct *p, struct dfs_Enum *r)
 	size_t i;
 	TALLOC_CTX *ctx = talloc_tos();
 
-	jn = enum_msdfs_links(ctx, &num_jn);
+	jn = enum_msdfs_links(msg_ctx_to_sconn(p->msg_ctx),
+			      ctx, &num_jn);
 	if (!jn || num_jn == 0) {
 		num_jn = 0;
 		jn = NULL;
@@ -293,7 +295,7 @@ WERROR _dfs_Enum(struct pipes_struct *p, struct dfs_Enum *r)
 	switch (r->in.level) {
 	case 1:
 		if (num_jn) {
-			if ((r->out.info->e.info1->s = TALLOC_ARRAY(ctx, struct dfs_Info1, num_jn)) == NULL) {
+			if ((r->out.info->e.info1->s = talloc_array(ctx, struct dfs_Info1, num_jn)) == NULL) {
 				return WERR_NOMEM;
 			}
 		} else {
@@ -303,7 +305,7 @@ WERROR _dfs_Enum(struct pipes_struct *p, struct dfs_Enum *r)
 		break;
 	case 2:
 		if (num_jn) {
-			if ((r->out.info->e.info2->s = TALLOC_ARRAY(ctx, struct dfs_Info2, num_jn)) == NULL) {
+			if ((r->out.info->e.info2->s = talloc_array(ctx, struct dfs_Info2, num_jn)) == NULL) {
 				return WERR_NOMEM;
 			}
 		} else {
@@ -313,7 +315,7 @@ WERROR _dfs_Enum(struct pipes_struct *p, struct dfs_Enum *r)
 		break;
 	case 3:
 		if (num_jn) {
-			if ((r->out.info->e.info3->s = TALLOC_ARRAY(ctx, struct dfs_Info3, num_jn)) == NULL) {
+			if ((r->out.info->e.info3->s = talloc_array(ctx, struct dfs_Info3, num_jn)) == NULL) {
 				return WERR_NOMEM;
 			}
 		} else {
@@ -352,7 +354,7 @@ WERROR _dfs_GetInfo(struct pipes_struct *p, struct dfs_GetInfo *r)
 	TALLOC_CTX *ctx = talloc_tos();
 	bool ret;
 
-	jn = TALLOC_ZERO_P(ctx, struct junction_map);
+	jn = talloc_zero(ctx, struct junction_map);
 	if (!jn) {
 		return WERR_NOMEM;
 	}
@@ -370,28 +372,28 @@ WERROR _dfs_GetInfo(struct pipes_struct *p, struct dfs_GetInfo *r)
 
 	switch (r->in.level) {
 		case 1:
-			r->out.info->info1 = TALLOC_ZERO_P(ctx,struct dfs_Info1);
+			r->out.info->info1 = talloc_zero(ctx,struct dfs_Info1);
 			if (!r->out.info->info1) {
 				return WERR_NOMEM;
 			}
 			ret = init_reply_dfs_info_1(ctx, jn, r->out.info->info1);
 			break;
 		case 2:
-			r->out.info->info2 = TALLOC_ZERO_P(ctx,struct dfs_Info2);
+			r->out.info->info2 = talloc_zero(ctx,struct dfs_Info2);
 			if (!r->out.info->info2) {
 				return WERR_NOMEM;
 			}
 			ret = init_reply_dfs_info_2(ctx, jn, r->out.info->info2);
 			break;
 		case 3:
-			r->out.info->info3 = TALLOC_ZERO_P(ctx,struct dfs_Info3);
+			r->out.info->info3 = talloc_zero(ctx,struct dfs_Info3);
 			if (!r->out.info->info3) {
 				return WERR_NOMEM;
 			}
 			ret = init_reply_dfs_info_3(ctx, jn, r->out.info->info3);
 			break;
 		case 100:
-			r->out.info->info100 = TALLOC_ZERO_P(ctx,struct dfs_Info100);
+			r->out.info->info100 = talloc_zero(ctx,struct dfs_Info100);
 			if (!r->out.info->info100) {
 				return WERR_NOMEM;
 			}
diff --git a/source3/rpc_server/dssetup/srv_dssetup_nt.c b/source3/rpc_server/dssetup/srv_dssetup_nt.c
index 73617df99f1..1cf4ab80746 100644
--- a/source3/rpc_server/dssetup/srv_dssetup_nt.c
+++ b/source3/rpc_server/dssetup/srv_dssetup_nt.c
@@ -23,9 +23,9 @@
  */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_dssetup.h"
 #include "secrets.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -42,7 +42,7 @@ static WERROR fill_dsrole_dominfo_basic(TALLOC_CTX *ctx,
 
 	DEBUG(10,("fill_dsrole_dominfo_basic: enter\n"));
 
-	basic = TALLOC_ZERO_P(ctx, struct dssetup_DsRolePrimaryDomInfoBasic);
+	basic = talloc_zero(ctx, struct dssetup_DsRolePrimaryDomInfoBasic);
 	if (!basic) {
 		DEBUG(0,("fill_dsrole_dominfo_basic: out of memory\n"));
 		return WERR_NOMEM;
diff --git a/source3/rpc_server/echo/srv_echo_nt.c b/source3/rpc_server/echo/srv_echo_nt.c
index 7f27984b8bd..7c8ae19b82b 100644
--- a/source3/rpc_server/echo/srv_echo_nt.c
+++ b/source3/rpc_server/echo/srv_echo_nt.c
@@ -22,8 +22,8 @@
 /* This is the interface to the rpcecho pipe. */
 
 #include "includes.h"
-#include "../librpc/gen_ndr/srv_echo.h"
 #include "ntdomain.h"
+#include "../librpc/gen_ndr/srv_echo.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -48,7 +48,7 @@ void _echo_EchoData(struct pipes_struct *p, struct echo_EchoData *r)
 		return;
 	}
 
-	r->out.out_data = TALLOC_ARRAY(p->mem_ctx, uint8, r->in.len);
+	r->out.out_data = talloc_array(p->mem_ctx, uint8, r->in.len);
 	memcpy( r->out.out_data, r->in.in_data, r->in.len );
 	return;
 }
@@ -76,7 +76,7 @@ void _echo_SourceData(struct pipes_struct *p, struct echo_SourceData *r)
 		return;
 	}
 
-	r->out.data = TALLOC_ARRAY(p->mem_ctx, uint8, r->in.len );
+	r->out.data = talloc_array(p->mem_ctx, uint8, r->in.len );
 
 	for (i = 0; i < r->in.len; i++ ) {
 		r->out.data[i] = i & 0xff;
diff --git a/source3/rpc_server/epmapper/srv_epmapper.c b/source3/rpc_server/epmapper/srv_epmapper.c
index d96fd1e929f..70f665dadf6 100644
--- a/source3/rpc_server/epmapper/srv_epmapper.c
+++ b/source3/rpc_server/epmapper/srv_epmapper.c
@@ -20,12 +20,11 @@
 */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../libcli/security/security.h"
-#include "librpc/gen_ndr/ndr_epmapper.h"
 #include "librpc/gen_ndr/srv_epmapper.h"
 #include "srv_epmapper.h"
 #include "auth.h"
-#include "ntdomain.h"
 
 typedef uint32_t error_status_t;
 
diff --git a/source3/rpc_server/epmd.c b/source3/rpc_server/epmd.c
index 5e82b276fd2..bb241ff2c1f 100644
--- a/source3/rpc_server/epmd.c
+++ b/source3/rpc_server/epmd.c
@@ -22,6 +22,7 @@
 #include "includes.h"
 
 #include "serverid.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_epmapper.h"
 #include "rpc_server/rpc_server.h"
 #include "rpc_server/epmapper/srv_epmapper.h"
@@ -53,7 +54,6 @@ static bool epmd_open_sockets(struct tevent_context *ev_ctx,
 
 			port = setup_dcerpc_ncacn_tcpip_socket(ev_ctx,
 							       msg_ctx,
-							       ndr_table_epmapper.syntax_id,
 							       ifss,
 							       135);
 			if (port == 0) {
@@ -68,7 +68,7 @@ static bool epmd_open_sockets(struct tevent_context *ev_ctx,
 		if (strequal(sock_addr, "0.0.0.0") ||
 		    strequal(sock_addr, "::")) {
 #if HAVE_IPV6
-			sock_addr = "::";
+			sock_addr = "::,0.0.0.0";
 #else
 			sock_addr = "0.0.0.0";
 #endif
@@ -88,7 +88,6 @@ static bool epmd_open_sockets(struct tevent_context *ev_ctx,
 
 			port = setup_dcerpc_ncacn_tcpip_socket(ev_ctx,
 							       msg_ctx,
-							       ndr_table_epmapper.syntax_id,
 							       &ss,
 							       135);
 			if (port == 0) {
@@ -262,7 +261,6 @@ void start_epmd(struct tevent_context *ev_ctx,
 
 	ok = setup_dcerpc_ncalrpc_socket(ev_ctx,
 					 msg_ctx,
-					 ndr_table_epmapper.syntax_id,
 					 "EPMAPPER",
 					 srv_epmapper_delete_endpoints);
 	if (!ok) {
diff --git a/source3/rpc_server/eventlog/srv_eventlog_nt.c b/source3/rpc_server/eventlog/srv_eventlog_nt.c
index 2e1c1faa8c5..16a0c974881 100644
--- a/source3/rpc_server/eventlog/srv_eventlog_nt.c
+++ b/source3/rpc_server/eventlog/srv_eventlog_nt.c
@@ -21,16 +21,16 @@
  */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_eventlog.h"
 #include "lib/eventlog/eventlog.h"
-#include "registry.h"
 #include "../libcli/security/security.h"
 #include "../librpc/gen_ndr/ndr_winreg_c.h"
 #include "rpc_client/cli_winreg_int.h"
 #include "rpc_client/cli_winreg.h"
 #include "smbd/smbd.h"
 #include "auth.h"
-#include "ntdomain.h"
+#include "util_tdb.h"
 
 #undef  DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -212,7 +212,7 @@ static NTSTATUS elog_open( struct pipes_struct * p, const char *logname, struct
 	if ( !elog_validate_logname( logname ) )
 		return NT_STATUS_OBJECT_PATH_INVALID;
 
-	if ( !(elog = TALLOC_ZERO_P( NULL, EVENTLOG_INFO )) )
+	if ( !(elog = talloc_zero( NULL, EVENTLOG_INFO )) )
 		return NT_STATUS_NO_MEMORY;
 	talloc_set_destructor(elog, eventlog_info_destructor);
 
diff --git a/source3/rpc_server/initshutdown/srv_initshutdown_nt.c b/source3/rpc_server/initshutdown/srv_initshutdown_nt.c
index 8644d95e7d0..9b16d806096 100644
--- a/source3/rpc_server/initshutdown/srv_initshutdown_nt.c
+++ b/source3/rpc_server/initshutdown/srv_initshutdown_nt.c
@@ -21,9 +21,9 @@
 /* Implementation of registry functions. */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_initshutdown.h"
 #include "../librpc/gen_ndr/srv_winreg.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
diff --git a/source3/rpc_server/lsa/srv_lsa_nt.c b/source3/rpc_server/lsa/srv_lsa_nt.c
index 9ea872b0276..c6f45eaad0f 100644
--- a/source3/rpc_server/lsa/srv_lsa_nt.c
+++ b/source3/rpc_server/lsa/srv_lsa_nt.c
@@ -30,6 +30,7 @@
 /* This is the implementation of the lsa server code. */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_lsa.h"
 #include "secrets.h"
 #include "../librpc/gen_ndr/netlogon.h"
@@ -43,7 +44,6 @@
 #include "../librpc/gen_ndr/ndr_security.h"
 #include "passdb.h"
 #include "auth.h"
-#include "ntdomain.h"
 #include "lib/privileges.h"
 #include "rpc_server/srv_access_check.h"
 
@@ -122,7 +122,7 @@ static int init_lsa_ref_domain_list(TALLOC_CTX *mem_ctx,
 	ref->count = num + 1;
 	ref->max_size = LSA_REF_DOMAIN_LIST_MULTIPLIER;
 
-	ref->domains = TALLOC_REALLOC_ARRAY(mem_ctx, ref->domains,
+	ref->domains = talloc_realloc(mem_ctx, ref->domains,
 					    struct lsa_DomainInfo, ref->count);
 	if (!ref->domains) {
 		return -1;
@@ -520,7 +520,7 @@ NTSTATUS _lsa_EnumTrustDom(struct pipes_struct *p,
 		return nt_status;
 	}
 
-	entries = TALLOC_ZERO_ARRAY(p->mem_ctx, struct lsa_DomainInfo, count);
+	entries = talloc_zero_array(p->mem_ctx, struct lsa_DomainInfo, count);
 	if (!entries) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -628,7 +628,7 @@ NTSTATUS _lsa_QueryInfoPolicy(struct pipes_struct *p,
 		/* return NT_STATUS_ACCESS_DENIED; */
 	}
 
-	info = TALLOC_ZERO_P(p->mem_ctx, union lsa_PolicyInformation);
+	info = talloc_zero(p->mem_ctx, union lsa_PolicyInformation);
 	if (!info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -681,7 +681,7 @@ NTSTATUS _lsa_QueryInfoPolicy(struct pipes_struct *p,
 
 		info->audit_events.auditing_mode = true;
 		info->audit_events.count = LSA_AUDIT_NUM_CATEGORIES;
-		info->audit_events.settings = TALLOC_ZERO_ARRAY(p->mem_ctx,
+		info->audit_events.settings = talloc_zero_array(p->mem_ctx,
 								enum lsa_PolicyAuditPolicy,
 								info->audit_events.count);
 		if (!info->audit_events.settings) {
@@ -858,8 +858,8 @@ static NTSTATUS _lsa_lookup_sids_internal(struct pipes_struct *p,
 		return NT_STATUS_OK;
 	}
 
-	sids = TALLOC_ARRAY(p->mem_ctx, const struct dom_sid *, num_sids);
-	ref = TALLOC_ZERO_P(p->mem_ctx, struct lsa_RefDomainList);
+	sids = talloc_array(p->mem_ctx, const struct dom_sid *, num_sids);
+	ref = talloc_zero(p->mem_ctx, struct lsa_RefDomainList);
 
 	if (sids == NULL || ref == NULL) {
 		return NT_STATUS_NO_MEMORY;
@@ -876,7 +876,7 @@ static NTSTATUS _lsa_lookup_sids_internal(struct pipes_struct *p,
 		return status;
 	}
 
-	names = TALLOC_ARRAY(p->mem_ctx, struct lsa_TranslatedName2, num_sids);
+	names = talloc_array(p->mem_ctx, struct lsa_TranslatedName2, num_sids);
 	if (names == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -998,7 +998,7 @@ NTSTATUS _lsa_LookupSids(struct pipes_struct *p,
 	}
 
 	/* Convert from lsa_TranslatedName2 to lsa_TranslatedName */
-	names_out = TALLOC_ARRAY(p->mem_ctx, struct lsa_TranslatedName,
+	names_out = talloc_array(p->mem_ctx, struct lsa_TranslatedName,
 				 num_sids);
 	if (!names_out) {
 		return NT_STATUS_NO_MEMORY;
@@ -1168,13 +1168,13 @@ NTSTATUS _lsa_LookupNames(struct pipes_struct *p,
 
 	flags = lsa_lookup_level_to_flags(r->in.level);
 
-	domains = TALLOC_ZERO_P(p->mem_ctx, struct lsa_RefDomainList);
+	domains = talloc_zero(p->mem_ctx, struct lsa_RefDomainList);
 	if (!domains) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
 	if (num_entries) {
-		rids = TALLOC_ZERO_ARRAY(p->mem_ctx, struct lsa_TranslatedSid,
+		rids = talloc_zero_array(p->mem_ctx, struct lsa_TranslatedSid,
 					 num_entries);
 		if (!rids) {
 			return NT_STATUS_NO_MEMORY;
@@ -1235,7 +1235,7 @@ NTSTATUS _lsa_LookupNames2(struct pipes_struct *p,
 	struct lsa_TransSidArray *sid_array = NULL;
 	uint32_t i;
 
-	sid_array = TALLOC_ZERO_P(p->mem_ctx, struct lsa_TransSidArray);
+	sid_array = talloc_zero(p->mem_ctx, struct lsa_TransSidArray);
 	if (!sid_array) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1257,7 +1257,7 @@ NTSTATUS _lsa_LookupNames2(struct pipes_struct *p,
 	status = _lsa_LookupNames(p, &q);
 
 	sid_array2->count = sid_array->count;
-	sid_array2->sids = TALLOC_ARRAY(p->mem_ctx, struct lsa_TranslatedSid2, sid_array->count);
+	sid_array2->sids = talloc_array(p->mem_ctx, struct lsa_TranslatedSid2, sid_array->count);
 	if (!sid_array2->sids) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1310,13 +1310,13 @@ NTSTATUS _lsa_LookupNames3(struct pipes_struct *p,
 		flags = LOOKUP_NAME_ALL;
 	}
 
-	domains = TALLOC_ZERO_P(p->mem_ctx, struct lsa_RefDomainList);
+	domains = talloc_zero(p->mem_ctx, struct lsa_RefDomainList);
 	if (!domains) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
 	if (num_entries) {
-		trans_sids = TALLOC_ZERO_ARRAY(p->mem_ctx, struct lsa_TranslatedSid3,
+		trans_sids = talloc_zero_array(p->mem_ctx, struct lsa_TranslatedSid3,
 					       num_entries);
 		if (!trans_sids) {
 			return NT_STATUS_NO_MEMORY;
@@ -2012,7 +2012,7 @@ NTSTATUS _lsa_QueryTrustedDomainInfo(struct pipes_struct *p,
 		return status;
 	}
 
-	info = TALLOC_ZERO_P(p->mem_ctx, union lsa_TrustedDomainInfo);
+	info = talloc_zero(p->mem_ctx, union lsa_TrustedDomainInfo);
 	if (!info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -2230,7 +2230,7 @@ NTSTATUS _lsa_EnumPrivs(struct pipes_struct *p,
 		return NT_STATUS_ACCESS_DENIED;
 
 	if (num_privs) {
-		entries = TALLOC_ZERO_ARRAY(p->mem_ctx, struct lsa_PrivEntry, num_privs);
+		entries = talloc_zero_array(p->mem_ctx, struct lsa_PrivEntry, num_privs);
 		if (!entries) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -2299,7 +2299,7 @@ NTSTATUS _lsa_LookupPrivDisplayName(struct pipes_struct *p,
 
 	DEBUG(10,("_lsa_LookupPrivDisplayName: display name = %s\n", description));
 
-	lsa_name = TALLOC_ZERO_P(p->mem_ctx, struct lsa_StringLarge);
+	lsa_name = talloc_zero(p->mem_ctx, struct lsa_StringLarge);
 	if (!lsa_name) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -2351,7 +2351,7 @@ NTSTATUS _lsa_EnumAccounts(struct pipes_struct *p,
 	}
 
 	if (num_entries - *r->in.resume_handle) {
-		sids = TALLOC_ZERO_ARRAY(p->mem_ctx, struct lsa_SidPtr,
+		sids = talloc_zero_array(p->mem_ctx, struct lsa_SidPtr,
 					 num_entries - *r->in.resume_handle);
 		if (!sids) {
 			talloc_free(sid_list);
@@ -2412,14 +2412,14 @@ NTSTATUS _lsa_GetUserName(struct pipes_struct *p,
 		domname = p->session_info->info3->base.domain.string;
 	}
 
-	account_name = TALLOC_P(p->mem_ctx, struct lsa_String);
+	account_name = talloc(p->mem_ctx, struct lsa_String);
 	if (!account_name) {
 		return NT_STATUS_NO_MEMORY;
 	}
 	init_lsa_String(account_name, username);
 
 	if (r->out.authority_name) {
-		authority_name = TALLOC_P(p->mem_ctx, struct lsa_String);
+		authority_name = talloc(p->mem_ctx, struct lsa_String);
 		if (!authority_name) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -2597,7 +2597,7 @@ NTSTATUS _lsa_EnumPrivsAccount(struct pipes_struct *p,
 		return status;
 	}
 
-	*r->out.privs = priv_set = TALLOC_ZERO_P(p->mem_ctx, struct lsa_PrivilegeSet);
+	*r->out.privs = priv_set = talloc_zero(p->mem_ctx, struct lsa_PrivilegeSet);
 	if (!priv_set) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -2801,7 +2801,7 @@ NTSTATUS _lsa_LookupPrivName(struct pipes_struct *p,
 		return NT_STATUS_NO_SUCH_PRIVILEGE;
 	}
 
-	lsa_name = TALLOC_ZERO_P(p->mem_ctx, struct lsa_StringLarge);
+	lsa_name = talloc_zero(p->mem_ctx, struct lsa_StringLarge);
 	if (!lsa_name) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -3030,7 +3030,7 @@ static NTSTATUS init_lsa_right_set(TALLOC_CTX *mem_ctx,
 
 	if (num_priv) {
 
-		r->names = TALLOC_ZERO_ARRAY(mem_ctx, struct lsa_StringLarge,
+		r->names = talloc_zero_array(mem_ctx, struct lsa_StringLarge,
 					     num_priv);
 		if (!r->names) {
 			return NT_STATUS_NO_MEMORY;
@@ -3323,7 +3323,7 @@ NTSTATUS _lsa_EnumTrustedDomainsEx(struct pipes_struct *p,
 		return nt_status;
 	}
 
-	entries = TALLOC_ZERO_ARRAY(p->mem_ctx, struct lsa_TrustDomainInfoInfoEx,
+	entries = talloc_zero_array(p->mem_ctx, struct lsa_TrustDomainInfoInfoEx,
 				    count);
 	if (!entries) {
 		return NT_STATUS_NO_MEMORY;
@@ -3488,7 +3488,7 @@ static int dns_cmp(const char *s1, size_t l1,
 	int cret;
 
 	if (l1 == l2) {
-		if (StrCaseCmp(s1, s2) == 0) {
+		if (strcasecmp_m(s1, s2) == 0) {
 			return DNS_CMP_MATCH;
 		}
 		return DNS_CMP_NO_MATCH;
@@ -3512,7 +3512,7 @@ static int dns_cmp(const char *s1, size_t l1,
 		return DNS_CMP_NO_MATCH;
 	}
 
-	if (StrCaseCmp(&p1[t1 - t2], p2) == 0) {
+	if (strcasecmp_m(&p1[t1 - t2], p2) == 0) {
 		return cret;
 	}
 
@@ -3701,7 +3701,7 @@ static NTSTATUS check_ft_info(TALLOC_CTX *mem_ctx,
 				sid_conflict = true;
 			}
 			if (!(trec->flags & LSA_NB_DISABLED_ADMIN) &&
-			    StrCaseCmp(trec->data.info.netbios_name.string,
+			    strcasecmp_m(trec->data.info.netbios_name.string,
 				       nb_name) == 0) {
 				nb_conflict = true;
 			}
@@ -3876,7 +3876,7 @@ NTSTATUS _lsa_lsaRSetForestTrustInformation(struct pipes_struct *p,
 		if (domains[i]->domain_name == NULL) {
 			return NT_STATUS_INVALID_DOMAIN_STATE;
 		}
-		if (StrCaseCmp(domains[i]->domain_name,
+		if (strcasecmp_m(domains[i]->domain_name,
 			       r->in.trusted_domain_name->string) == 0) {
 			break;
 		}
diff --git a/source3/rpc_server/netlogon/srv_netlog_nt.c b/source3/rpc_server/netlogon/srv_netlog_nt.c
index f0535c72953..03897d77a47 100644
--- a/source3/rpc_server/netlogon/srv_netlog_nt.c
+++ b/source3/rpc_server/netlogon/srv_netlog_nt.c
@@ -25,10 +25,9 @@
 /* This is the implementation of the netlogon pipe. */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../libcli/auth/schannel.h"
 #include "../librpc/gen_ndr/srv_netlogon.h"
-#include "../librpc/gen_ndr/srv_samr.h"
-#include "../librpc/gen_ndr/srv_lsa.h"
 #include "../librpc/gen_ndr/ndr_samr_c.h"
 #include "../librpc/gen_ndr/ndr_lsa_c.h"
 #include "rpc_client/cli_lsarpc.h"
@@ -45,7 +44,6 @@
 #include "passdb.h"
 #include "auth.h"
 #include "messages.h"
-#include "ntdomain.h"
 
 extern userdom_struct current_user_info;
 
@@ -348,7 +346,7 @@ WERROR _netr_LogonControl2Ex(struct pipes_struct *p,
 
 	switch (r->in.level) {
 	case 1:
-		info1 = TALLOC_ZERO_P(p->mem_ctx, struct netr_NETLOGON_INFO_1);
+		info1 = talloc_zero(p->mem_ctx, struct netr_NETLOGON_INFO_1);
 		W_ERROR_HAVE_NO_MEMORY(info1);
 
 		info1->flags			= flags;
@@ -357,7 +355,7 @@ WERROR _netr_LogonControl2Ex(struct pipes_struct *p,
 		r->out.query->info1 = info1;
 		break;
 	case 2:
-		info2 = TALLOC_ZERO_P(p->mem_ctx, struct netr_NETLOGON_INFO_2);
+		info2 = talloc_zero(p->mem_ctx, struct netr_NETLOGON_INFO_2);
 		W_ERROR_HAVE_NO_MEMORY(info2);
 
 		info2->flags			= flags;
@@ -368,7 +366,7 @@ WERROR _netr_LogonControl2Ex(struct pipes_struct *p,
 		r->out.query->info2 = info2;
 		break;
 	case 3:
-		info3 = TALLOC_ZERO_P(p->mem_ctx, struct netr_NETLOGON_INFO_3);
+		info3 = talloc_zero(p->mem_ctx, struct netr_NETLOGON_INFO_3);
 		W_ERROR_HAVE_NO_MEMORY(info3);
 
 		info3->flags			= flags;
@@ -377,7 +375,7 @@ WERROR _netr_LogonControl2Ex(struct pipes_struct *p,
 		r->out.query->info3 = info3;
 		break;
 	case 4:
-		info4 = TALLOC_ZERO_P(p->mem_ctx, struct netr_NETLOGON_INFO_4);
+		info4 = talloc_zero(p->mem_ctx, struct netr_NETLOGON_INFO_4);
 		W_ERROR_HAVE_NO_MEMORY(info4);
 
 		info4->trusted_dc_name		= dc_name;
@@ -524,7 +522,7 @@ static NTSTATUS samr_find_machine_account(TALLOC_CTX *mem_ctx,
 	uint32_t rid;
 
 	status = dcerpc_samr_Connect2(b, mem_ctx,
-				      global_myname(),
+				      lp_netbios_name(),
 				      SAMR_ACCESS_CONNECT_TO_SERVER |
 				      SAMR_ACCESS_ENUM_DOMAINS |
 				      SAMR_ACCESS_LOOKUP_DOMAIN,
@@ -1423,19 +1421,19 @@ static NTSTATUS _netr_LogonSamLogon_base(struct pipes_struct *p,
 
 	switch (r->in.validation_level) {
 	case 2:
-		r->out.validation->sam2 = TALLOC_ZERO_P(p->mem_ctx, struct netr_SamInfo2);
+		r->out.validation->sam2 = talloc_zero(p->mem_ctx, struct netr_SamInfo2);
 		if (!r->out.validation->sam2) {
 			return NT_STATUS_NO_MEMORY;
 		}
 		break;
 	case 3:
-		r->out.validation->sam3 = TALLOC_ZERO_P(p->mem_ctx, struct netr_SamInfo3);
+		r->out.validation->sam3 = talloc_zero(p->mem_ctx, struct netr_SamInfo3);
 		if (!r->out.validation->sam3) {
 			return NT_STATUS_NO_MEMORY;
 		}
 		break;
 	case 6:
-		r->out.validation->sam6 = TALLOC_ZERO_P(p->mem_ctx, struct netr_SamInfo6);
+		r->out.validation->sam6 = talloc_zero(p->mem_ctx, struct netr_SamInfo6);
 		if (!r->out.validation->sam6) {
 			return NT_STATUS_NO_MEMORY;
 		}
diff --git a/source3/rpc_server/ntsvcs/srv_ntsvcs_nt.c b/source3/rpc_server/ntsvcs/srv_ntsvcs_nt.c
index af80254a0b2..0a00e000255 100644
--- a/source3/rpc_server/ntsvcs/srv_ntsvcs_nt.c
+++ b/source3/rpc_server/ntsvcs/srv_ntsvcs_nt.c
@@ -20,10 +20,10 @@
  */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_ntsvcs.h"
 #include "services/svc_winreg_glue.h"
 #include "../libcli/registry/util_reg.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
diff --git a/source3/rpc_server/rpc_contexts.c b/source3/rpc_server/rpc_contexts.c
new file mode 100644
index 00000000000..bb5c0eaf6cd
--- /dev/null
+++ b/source3/rpc_server/rpc_contexts.c
@@ -0,0 +1,40 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Almost completely rewritten by (C) Jeremy Allison 2005 - 2010
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "includes.h"
+#include "ntdomain.h"
+
+#include "rpc_contexts.h"
+
+struct pipe_rpc_fns *find_pipe_fns_by_context(struct pipe_rpc_fns *list,
+					      uint32_t context_id)
+{
+	struct pipe_rpc_fns *fns = NULL;
+
+	if ( !list ) {
+		DEBUG(0,("find_pipe_fns_by_context: ERROR!  No context list for pipe!\n"));
+		return NULL;
+	}
+
+	for (fns=list; fns; fns=fns->next ) {
+		if ( fns->context_id == context_id )
+			return fns;
+	}
+	return NULL;
+}
diff --git a/source3/rpc_server/rpc_contexts.h b/source3/rpc_server/rpc_contexts.h
new file mode 100644
index 00000000000..8463414bbb3
--- /dev/null
+++ b/source3/rpc_server/rpc_contexts.h
@@ -0,0 +1,28 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Almost completely rewritten by (C) Jeremy Allison 2005 - 2010
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_SERVER_RPC_CONTEXTS_H_
+#define _RPC_SERVER_RPC_CONTEXTS_H_
+
+struct pipe_rpc_fns;
+
+struct pipe_rpc_fns *find_pipe_fns_by_context(struct pipe_rpc_fns *list,
+					      uint32_t context_id);
+
+#endif /* _RPC_SERVER_RPC_CONTEXTS_H_*/
diff --git a/source3/rpc_server/rpc_ep_setup.c b/source3/rpc_server/rpc_ep_setup.c
index d517bb88516..e5059dae807 100644
--- a/source3/rpc_server/rpc_ep_setup.c
+++ b/source3/rpc_server/rpc_ep_setup.c
@@ -20,6 +20,7 @@
  */
 
 #include "includes.h"
+#include "ntdomain.h"
 
 #include "../librpc/gen_ndr/ndr_epmapper_c.h"
 #include "../librpc/gen_ndr/srv_epmapper.h"
@@ -41,6 +42,8 @@
 #include "printing/nt_printing_migrate.h"
 #include "rpc_server/eventlog/srv_eventlog_reg.h"
 #include "rpc_server/svcctl/srv_svcctl_reg.h"
+#include "rpc_server/spoolss/srv_spoolss_nt.h"
+#include "rpc_server/svcctl/srv_svcctl_nt.h"
 
 #include "librpc/rpc/dcerpc_ep.h"
 
@@ -76,7 +79,6 @@ static uint16_t _open_sockets(struct tevent_context *ev_ctx,
 
 			p = setup_dcerpc_ncacn_tcpip_socket(ev_ctx,
 							    msg_ctx,
-							    syntax_id,
 							    ifss,
 							    port);
 			if (p == 0) {
@@ -92,7 +94,7 @@ static uint16_t _open_sockets(struct tevent_context *ev_ctx,
 		if (strequal(sock_addr, "0.0.0.0") ||
 		    strequal(sock_addr, "::")) {
 #if HAVE_IPV6
-			sock_addr = "::";
+			sock_addr = "::,0.0.0.0";
 #else
 			sock_addr = "0.0.0.0";
 #endif
@@ -112,7 +114,6 @@ static uint16_t _open_sockets(struct tevent_context *ev_ctx,
 
 			p = setup_dcerpc_ncacn_tcpip_socket(ev_ctx,
 							    msg_ctx,
-							    syntax_id,
 							    &ss,
 							    port);
 			if (p == 0) {
@@ -130,7 +131,7 @@ static NTSTATUS rpc_ep_setup_try_register(TALLOC_CTX *mem_ctx,
 					  struct tevent_context *ev_ctx,
 					  struct messaging_context *msg_ctx,
 					  const struct ndr_interface_table *iface,
-					  const char *name,
+					  const char *ncalrpc,
 					  uint16_t port,
 					  struct dcerpc_binding_handle **pbh);
 
@@ -253,7 +254,7 @@ static NTSTATUS rpc_ep_setup_try_register(TALLOC_CTX *mem_ctx,
 					  struct tevent_context *ev_ctx,
 					  struct messaging_context *msg_ctx,
 					  const struct ndr_interface_table *iface,
-					  const char *name,
+					  const char *ncalrpc,
 					  uint16_t port,
 					  struct dcerpc_binding_handle **pbh)
 {
@@ -263,7 +264,7 @@ static NTSTATUS rpc_ep_setup_try_register(TALLOC_CTX *mem_ctx,
 	status = dcerpc_binding_vector_create(mem_ctx,
 					      iface,
 					      port,
-					      name,
+					      ncalrpc,
 					      &v);
 	if (!NT_STATUS_IS_OK(status)) {
 		return status;
@@ -273,7 +274,7 @@ static NTSTATUS rpc_ep_setup_try_register(TALLOC_CTX *mem_ctx,
 				    iface,
 				    v,
 				    &iface->syntax_id.uuid,
-				    name,
+				    iface->name,
 				    pbh);
 	talloc_free(v);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -420,14 +421,13 @@ static bool winreg_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -468,14 +468,13 @@ static bool srvsvc_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -517,14 +516,13 @@ static bool lsarpc_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -566,14 +564,13 @@ static bool samr_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -615,14 +612,13 @@ static bool netlogon_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -670,8 +666,8 @@ static bool spoolss_init_cb(void *ptr)
 		return false;
 	}
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 
 		status =rpc_ep_setup_register(ep_ctx->ev_ctx,
@@ -714,8 +710,8 @@ static bool svcctl_init_cb(void *ptr)
 	/* initialize the control hooks */
 	init_service_op_table();
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 
 		status = rpc_ep_setup_register(ep_ctx->ev_ctx,
@@ -749,8 +745,8 @@ static bool ntsvcs_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 
 		status = rpc_ep_setup_register(ep_ctx->ev_ctx,
@@ -783,8 +779,8 @@ static bool eventlog_init_cb(void *ptr)
 		return false;
 	}
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 
 		status =rpc_ep_setup_register(ep_ctx->ev_ctx,
@@ -811,8 +807,8 @@ static bool initshutdown_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 
 		status = rpc_ep_setup_register(ep_ctx->ev_ctx,
@@ -840,8 +836,8 @@ static bool rpcecho_init_cb(void *ptr) {
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 
 		port = _open_sockets(ep_ctx->ev_ctx,
@@ -880,14 +876,13 @@ static bool netdfs_init_cb(void *ptr)
 					   "rpc_server",
 					   "epmapper",
 					   "none");
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -929,14 +924,13 @@ static bool dssetup_init_cb(void *ptr)
 					   "epmapper",
 					   "none");
 
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -977,14 +971,13 @@ static bool wkssvc_init_cb(void *ptr)
 					   "rpc_server",
 					   "epmapper",
 					   "none");
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0 ||
-	    StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0 ||
+	    strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		NTSTATUS status;
 		bool ok;
 
 		ok = setup_dcerpc_ncalrpc_socket(ep_ctx->ev_ctx,
 						 ep_ctx->msg_ctx,
-						 abstract_syntax,
 						 pipe_name,
 						 NULL);
 		if (!ok) {
@@ -1053,7 +1046,7 @@ bool dcesrv_ep_setup(struct tevent_context *ev_ctx,
 					   "rpc_server",
 					   "epmapper",
 					   "none");
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0) {
 		epmapper_cb.init         = epmapper_init_cb;
 		epmapper_cb.shutdown     = epmapper_shutdown_cb;
 		epmapper_cb.private_data = ep_ctx;
@@ -1061,7 +1054,7 @@ bool dcesrv_ep_setup(struct tevent_context *ev_ctx,
 		if (!NT_STATUS_IS_OK(rpc_epmapper_init(&epmapper_cb))) {
 			return false;
 		}
-	} else if (StrCaseCmp(rpcsrv_type, "daemon") == 0) {
+	} else if (strcasecmp_m(rpcsrv_type, "daemon") == 0) {
 		if (!NT_STATUS_IS_OK(rpc_epmapper_init(NULL))) {
 			return false;
 		}
@@ -1107,15 +1100,15 @@ bool dcesrv_ep_setup(struct tevent_context *ev_ctx,
 					   "rpc_server",
 					   "spoolss",
 					   "embedded");
-	if (StrCaseCmp(rpcsrv_type, "embedded") == 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") == 0) {
 		spoolss_cb.init         = spoolss_init_cb;
 		spoolss_cb.shutdown     = spoolss_shutdown_cb;
 		spoolss_cb.private_data = ep_ctx;
 		if (!NT_STATUS_IS_OK(rpc_spoolss_init(&spoolss_cb))) {
 			return false;
 		}
-	} else if (StrCaseCmp(rpcsrv_type, "daemon") == 0 ||
-		   StrCaseCmp(rpcsrv_type, "external") == 0) {
+	} else if (strcasecmp_m(rpcsrv_type, "daemon") == 0 ||
+		   strcasecmp_m(rpcsrv_type, "external") == 0) {
 		if (!NT_STATUS_IS_OK(rpc_spoolss_init(NULL))) {
 			return false;
 		}
diff --git a/source3/rpc_server/rpc_ep_setup.h b/source3/rpc_server/rpc_ep_setup.h
index b96de36b16e..57b06abb39f 100644
--- a/source3/rpc_server/rpc_ep_setup.h
+++ b/source3/rpc_server/rpc_ep_setup.h
@@ -22,6 +22,8 @@
 #ifndef _SMBD_RPC_CALLBACKS_H
 #define _SMBD_RPC_CALLBACKS_H
 
+struct ndr_interface_table;
+
 /**
  * @brief Register an endpoint at the endpoint mapper.
  *
diff --git a/source3/rpc_server/rpc_handles.c b/source3/rpc_server/rpc_handles.c
index 889e87e9b5a..7fa59b6b117 100644
--- a/source3/rpc_server/rpc_handles.c
+++ b/source3/rpc_server/rpc_handles.c
@@ -24,6 +24,7 @@
 #include "../librpc/gen_ndr/ndr_samr.h"
 #include "auth.h"
 #include "ntdomain.h"
+#include "rpc_server/rpc_ncacn_np.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -83,15 +84,28 @@ bool init_pipe_handles(struct pipes_struct *p, const struct ndr_syntax_id *synta
 	for (plist = get_first_internal_pipe();
 	     plist;
 	     plist = get_next_internal_pipe(plist)) {
-		if (ndr_syntax_id_equal(syntax, &plist->syntax)) {
-			break;
+		struct pipe_rpc_fns *p_ctx;
+		bool stop = false;
+
+		for (p_ctx = plist->contexts;
+		     p_ctx != NULL;
+		     p_ctx = p_ctx->next) {
+			if (ndr_syntax_id_equal(syntax, &p_ctx->syntax)) {
+				stop = true;
+				break;
+			}
+			if (is_samr_lsa_pipe(&p_ctx->syntax)
+			    && is_samr_lsa_pipe(syntax)) {
+				/*
+				 * samr and lsa share a handle space (same process
+				 * under Windows?)
+				 */
+				stop = true;
+				break;
+			}
 		}
-		if (is_samr_lsa_pipe(&plist->syntax)
-		    && is_samr_lsa_pipe(syntax)) {
-			/*
-			 * samr and lsa share a handle space (same process
-			 * under Windows?)
-			 */
+
+		if (stop) {
 			break;
 		}
 	}
@@ -229,7 +243,7 @@ static struct dcesrv_handle *find_policy_by_hnd_internal(struct pipes_struct *p,
 	for (h = p->pipe_handles->handles; h != NULL; h = h->next) {
 		if (memcmp(&h->wire_handle, hnd, sizeof(*hnd)) == 0) {
 			DEBUG(4,("Found policy hnd[%u] ", count));
-			dump_data(4, (uint8 *)hnd, sizeof(*hnd));
+			dump_data(4, (const uint8 *)hnd, sizeof(*hnd));
 			if (data_p) {
 				*data_p = h->data;
 			}
@@ -239,7 +253,7 @@ static struct dcesrv_handle *find_policy_by_hnd_internal(struct pipes_struct *p,
 	}
 
 	DEBUG(4,("Policy not found: "));
-	dump_data(4, (uint8_t *)hnd, sizeof(*hnd));
+	dump_data(4, (const uint8_t *)hnd, sizeof(*hnd));
 
 	p->bad_handle_fault_state = true;
 
@@ -293,6 +307,10 @@ bool close_policy_hnd(struct pipes_struct *p, struct policy_handle *hnd)
 
 void close_policy_by_pipe(struct pipes_struct *p)
 {
+	if (p->pipe_handles == NULL) {
+		return;
+	}
+
 	p->pipe_handles->pipe_ref_count--;
 
 	if (p->pipe_handles->pipe_ref_count == 0) {
@@ -301,9 +319,9 @@ void close_policy_by_pipe(struct pipes_struct *p)
 		 */
 		TALLOC_FREE(p->pipe_handles);
 
-		DEBUG(10,("close_policy_by_pipe: deleted handle list for "
-			  "pipe %s\n",
-			  get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+		DEBUG(10,("Deleted handle list for RPC connection %s\n",
+			  get_pipe_name_from_syntax(talloc_tos(),
+						    &p->contexts->syntax)));
 	}
 }
 
@@ -344,9 +362,10 @@ void *_policy_handle_create(struct pipes_struct *p, struct policy_handle *hnd,
 	void *data;
 
 	if (p->pipe_handles->count > MAX_OPEN_POLS) {
-		DEBUG(0, ("policy_handle_create: ERROR: too many handles (%d) "
-			  "on pipe %s.\n", (int)p->pipe_handles->count,
-			  get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+		DEBUG(0, ("ERROR: Too many handles (%d) for RPC connection %s\n",
+			  (int) p->pipe_handles->count,
+			  get_pipe_name_from_syntax(talloc_tos(),
+						    &p->contexts->syntax)));
 		*pstatus = NT_STATUS_INSUFFICIENT_RESOURCES;
 		return NULL;
 	}
diff --git a/source3/rpc_server/rpc_ncacn_np.c b/source3/rpc_server/rpc_ncacn_np.c
index e89a366f9e9..f14aae540d3 100644
--- a/source3/rpc_server/rpc_ncacn_np.c
+++ b/source3/rpc_server/rpc_ncacn_np.c
@@ -31,12 +31,13 @@
 #include "../auth/auth_sam_reply.h"
 #include "auth.h"
 #include "ntdomain.h"
+#include "../lib/tsocket/tsocket.h"
+#include "../lib/util/tevent_ntstatus.h"
+#include "rpc_contexts.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
 
-static int pipes_open;
-
 static struct pipes_struct *InternalPipes;
 
 /* TODO
@@ -102,11 +103,11 @@ int close_internal_rpc_pipe_hnd(struct pipes_struct *p)
 
 	TALLOC_FREE(p->auth.auth_ctx);
 
-	free_pipe_rpc_context_internal( p->contexts );
-
 	/* Free the handles database. */
 	close_policy_by_pipe(p);
 
+	free_pipe_rpc_context_internal( p->contexts );
+
 	DLIST_REMOVE(InternalPipes, p);
 
 	ZERO_STRUCTP(p);
@@ -125,11 +126,12 @@ struct pipes_struct *make_internal_rpc_pipe_p(TALLOC_CTX *mem_ctx,
 					      struct messaging_context *msg_ctx)
 {
 	struct pipes_struct *p;
+	struct pipe_rpc_fns *context_fns;
 
 	DEBUG(4,("Create pipe requested %s\n",
 		 get_pipe_name_from_syntax(talloc_tos(), syntax)));
 
-	p = TALLOC_ZERO_P(mem_ctx, struct pipes_struct);
+	p = talloc_zero(mem_ctx, struct pipes_struct);
 
 	if (!p) {
 		DEBUG(0,("ERROR! no memory for pipes_struct!\n"));
@@ -167,11 +169,25 @@ struct pipes_struct *make_internal_rpc_pipe_p(TALLOC_CTX *mem_ctx,
 
 	p->endian = RPC_LITTLE_ENDIAN;
 
-	p->syntax = *syntax;
 	p->transport = NCALRPC;
 
-	DEBUG(4,("Created internal pipe %s (pipes_open=%d)\n",
-		 get_pipe_name_from_syntax(talloc_tos(), syntax), pipes_open));
+	context_fns = SMB_MALLOC_P(struct pipe_rpc_fns);
+	if (context_fns == NULL) {
+		DEBUG(0,("malloc() failed!\n"));
+		return False;
+	}
+
+	context_fns->next = context_fns->prev = NULL;
+	context_fns->n_cmds = rpc_srv_get_pipe_num_cmds(syntax);
+	context_fns->cmds = rpc_srv_get_pipe_cmds(syntax);
+	context_fns->context_id = 0;
+	context_fns->syntax = *syntax;
+
+	/* add to the list of open contexts */
+	DLIST_ADD(p->contexts, context_fns);
+
+	DEBUG(4,("Created internal pipe %s\n",
+		 get_pipe_name_from_syntax(talloc_tos(), syntax)));
 
 	talloc_set_destructor(p, close_internal_rpc_pipe_hnd);
 
@@ -184,8 +200,9 @@ static NTSTATUS rpcint_dispatch(struct pipes_struct *p,
 				const DATA_BLOB *in_data,
 				DATA_BLOB *out_data)
 {
-	uint32_t num_cmds = rpc_srv_get_pipe_num_cmds(&p->syntax);
-	const struct api_struct *cmds = rpc_srv_get_pipe_cmds(&p->syntax);
+	struct pipe_rpc_fns *fns = find_pipe_fns_by_context(p->contexts, 0);
+	uint32_t num_cmds = fns->n_cmds;
+	const struct api_struct *cmds = fns->cmds;
 	uint32_t i;
 	bool ok;
 
@@ -517,7 +534,8 @@ NTSTATUS rpcint_binding_handle(TALLOC_CTX *mem_ctx,
  * @brief Create a new RPC client context which uses a local transport.
  *
  * This creates a local transport. It is a shortcut to directly call the server
- * functions and avoid marschalling.
+ * functions and avoid marshalling.
+ * NOTE: this function should be used only by rpc_pipe_open_interface()
  *
  * @param[in]  mem_ctx  The memory context to use.
  *
@@ -534,19 +552,8 @@ NTSTATUS rpcint_binding_handle(TALLOC_CTX *mem_ctx,
  *
  * @return              NT_STATUS_OK on success, a corresponding NT status if an
  *                      error occured.
- *
- * @code
- *   struct rpc_pipe_client *winreg_pipe;
- *   NTSTATUS status;
- *
- *   status = rpc_pipe_open_internal(tmp_ctx,
- *                                   &ndr_table_winreg.syntax_id,
- *                                   p->session_info,
- *                                   client_id,
- *                                   &winreg_pipe);
- * @endcode
  */
-NTSTATUS rpc_pipe_open_internal(TALLOC_CTX *mem_ctx,
+static NTSTATUS rpc_pipe_open_internal(TALLOC_CTX *mem_ctx,
 				const struct ndr_syntax_id *abstract_syntax,
 				const struct auth_serversupplied_info *serversupplied_info,
 				struct client_address *client_id,
@@ -556,7 +563,7 @@ NTSTATUS rpc_pipe_open_internal(TALLOC_CTX *mem_ctx,
 	struct rpc_pipe_client *result;
 	NTSTATUS status;
 
-	result = TALLOC_ZERO_P(mem_ctx, struct rpc_pipe_client);
+	result = talloc_zero(mem_ctx, struct rpc_pipe_client);
 	if (result == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -831,17 +838,36 @@ done:
 }
 
 /**
- * @brief Create a new RPC client context which uses a local dispatch function.
+ * @brief Create a new RPC client context which uses a local dispatch function
+ *	  or a remote transport, depending on rpc_server configuration for the
+ *	  specific service.
+ *
+ * @param[in]  mem_ctx  The memory context to use.
+ *
+ * @param[in]  abstract_syntax Normally the syntax_id of the autogenerated
+ *                             ndr_table_<name>.
+ *
+ * @param[in]  serversupplied_info The server supplied authentication function.
  *
- * @param mem_ctx	The memory context on which thje pipe will ultimately
- *			be allocated
- * @param name		The pipe name to connect to.
- * @param session_info	Credentials to use for the connection.
- * @param pipe		[in|out] Checks if a pipe is connected, and connects it
- *				 if not
+ * @param[in]  client_id The client address information.
  *
- * @return              NT_STATUS_OK on success, a corresponding NT status if
- *			an error occured.
+ * @param[in]  msg_ctx  The messaging context to use.
+ *
+ * @param[out] presult  A pointer to store the connected rpc client pipe.
+ *
+ * @return              NT_STATUS_OK on success, a corresponding NT status if an
+ *                      error occured.
+ *
+ * @code
+ *   struct rpc_pipe_client *winreg_pipe;
+ *   NTSTATUS status;
+ *
+ *   status = rpc_pipe_open_interface(tmp_ctx,
+ *                                    &ndr_table_winreg.syntax_id,
+ *                                    p->session_info,
+ *                                    client_id,
+ *                                    &winreg_pipe);
+ * @endcode
  */
 
 NTSTATUS rpc_pipe_open_interface(TALLOC_CTX *mem_ctx,
@@ -884,7 +910,7 @@ NTSTATUS rpc_pipe_open_interface(TALLOC_CTX *mem_ctx,
 					   "rpc_server", pipe_name,
 					   "embedded");
 
-	if (StrCaseCmp(server_type, "embedded") == 0) {
+	if (strcasecmp_m(server_type, "embedded") == 0) {
 		status = rpc_pipe_open_internal(tmp_ctx,
 						syntax, session_info,
 						client_id, msg_ctx,
@@ -892,8 +918,8 @@ NTSTATUS rpc_pipe_open_interface(TALLOC_CTX *mem_ctx,
 		if (!NT_STATUS_IS_OK(status)) {
 			goto done;
 		}
-	} else if (StrCaseCmp(server_type, "daemon") == 0 ||
-		   StrCaseCmp(server_type, "external") == 0) {
+	} else if (strcasecmp_m(server_type, "daemon") == 0 ||
+		   strcasecmp_m(server_type, "external") == 0) {
 		/* It would be nice to just use rpc_pipe_open_ncalrpc() but
 		 * for now we need to use the special proxy setup to connect
 		 * to spoolssd. */
diff --git a/source3/rpc_server/rpc_ncacn_np.h b/source3/rpc_server/rpc_ncacn_np.h
index 7c8619b46db..3d9a170cdc9 100644
--- a/source3/rpc_server/rpc_ncacn_np.h
+++ b/source3/rpc_server/rpc_ncacn_np.h
@@ -20,6 +20,10 @@
 #ifndef _RPC_NCACN_NP_H_
 #define _RPC_NCACN_NP_H_
 
+struct dcerpc_binding_handle;
+struct ndr_interface_table;
+struct tsocket_address;
+
 struct np_proxy_state {
 	uint16_t file_type;
 	uint16_t device_state;
@@ -45,12 +49,6 @@ NTSTATUS rpcint_binding_handle(TALLOC_CTX *mem_ctx,
 			       const struct auth_serversupplied_info *session_info,
 			       struct messaging_context *msg_ctx,
 			       struct dcerpc_binding_handle **binding_handle);
-NTSTATUS rpc_pipe_open_internal(TALLOC_CTX *mem_ctx,
-				const struct ndr_syntax_id *abstract_syntax,
-				const struct auth_serversupplied_info *serversupplied_info,
-				struct client_address *client_id,
-				struct messaging_context *msg_ctx,
-				struct rpc_pipe_client **presult);
 NTSTATUS rpc_pipe_open_interface(TALLOC_CTX *mem_ctx,
 				 const struct ndr_syntax_id *syntax,
 				 const struct auth_serversupplied_info *session_info,
@@ -58,4 +56,9 @@ NTSTATUS rpc_pipe_open_interface(TALLOC_CTX *mem_ctx,
 				 struct messaging_context *msg_ctx,
 				 struct rpc_pipe_client **cli_pipe);
 
+struct pipes_struct *get_first_internal_pipe(void);
+struct pipes_struct *get_next_internal_pipe(struct pipes_struct *p);
+bool check_open_pipes(void);
+int close_internal_rpc_pipe_hnd(struct pipes_struct *p);
+
 #endif /* _RPC_NCACN_NP_H_ */
diff --git a/source3/rpc_server/rpc_server.c b/source3/rpc_server/rpc_server.c
index c7c77f03757..44c1b104c0b 100644
--- a/source3/rpc_server/rpc_server.c
+++ b/source3/rpc_server/rpc_server.c
@@ -18,6 +18,7 @@
 */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "rpc_server/rpc_server.h"
 #include "rpc_dce.h"
 #include "librpc/gen_ndr/netlogon.h"
@@ -26,7 +27,9 @@
 #include "libcli/named_pipe_auth/npa_tstream.h"
 #include "../auth/auth_sam_reply.h"
 #include "auth.h"
-#include "ntdomain.h"
+#include "rpc_server/rpc_ncacn_np.h"
+#include "rpc_server/srv_pipe_hnd.h"
+#include "rpc_server/srv_pipe.h"
 
 #define SERVER_TCP_LOW_PORT  1024
 #define SERVER_TCP_HIGH_PORT 1300
@@ -76,7 +79,6 @@ static NTSTATUS auth_anonymous_session_info(TALLOC_CTX *mem_ctx,
  * sent from the client */
 static int make_server_pipes_struct(TALLOC_CTX *mem_ctx,
 				    const char *pipe_name,
-				    const struct ndr_syntax_id id,
 				    enum dcerpc_transport_t transport,
 				    bool ncalrpc_as_system,
 				    const char *client_address,
@@ -90,14 +92,13 @@ static int make_server_pipes_struct(TALLOC_CTX *mem_ctx,
 	struct pipes_struct *p;
 	struct auth_serversupplied_info *server_info;
 	NTSTATUS status;
-	bool ok;
 
 	p = talloc_zero(mem_ctx, struct pipes_struct);
 	if (!p) {
 		*perrno = ENOMEM;
 		return -1;
 	}
-	p->syntax = id;
+
 	p->transport = transport;
 	p->ncalrpc_as_system = ncalrpc_as_system;
 
@@ -108,15 +109,6 @@ static int make_server_pipes_struct(TALLOC_CTX *mem_ctx,
 		return -1;
 	}
 
-	ok = init_pipe_handles(p, &id);
-	if (!ok) {
-		DEBUG(1, ("Failed to init handles\n"));
-		TALLOC_FREE(p);
-		*perrno = EINVAL;
-		return -1;
-	}
-
-
 	data_blob_free(&p->in_data.data);
 	data_blob_free(&p->in_data.pdu);
 
@@ -258,6 +250,17 @@ bool setup_named_pipe_socket(const char *pipe_name,
 	}
 	state->fd = -1;
 
+	/*
+	 * As lp_ncalrpc_dir() should have 0755, but
+	 * lp_ncalrpc_dir()/np should have 0700, we need to
+	 * create lp_ncalrpc_dir() first.
+	 */
+	if (!directory_create_or_exist(lp_ncalrpc_dir(), geteuid(), 0755)) {
+		DEBUG(0, ("Failed to create pipe directory %s - %s\n",
+			  lp_ncalrpc_dir(), strerror(errno)));
+		goto out;
+	}
+
 	np_dir = talloc_asprintf(state, "%s/np", lp_ncalrpc_dir());
 	if (!np_dir) {
 		DEBUG(0, ("Out of memory\n"));
@@ -340,7 +343,6 @@ static void named_pipe_listener(struct tevent_context *ev,
 
 struct named_pipe_client {
 	const char *pipe_name;
-	struct ndr_syntax_id pipe_id;
 
 	struct tevent_context *ev;
 	struct messaging_context *msg_ctx;
@@ -369,20 +371,11 @@ static void named_pipe_accept_done(struct tevent_req *subreq);
 
 static void named_pipe_accept_function(const char *pipe_name, int fd)
 {
-	struct ndr_syntax_id syntax;
 	struct named_pipe_client *npc;
 	struct tstream_context *plain;
 	struct tevent_req *subreq;
-	bool ok;
 	int ret;
 
-	ok = is_known_pipename(pipe_name, &syntax);
-	if (!ok) {
-		DEBUG(1, ("Unknown pipe [%s]\n", pipe_name));
-		close(fd);
-		return;
-	}
-
 	npc = talloc_zero(NULL, struct named_pipe_client);
 	if (!npc) {
 		DEBUG(0, ("Out of memory!\n"));
@@ -390,7 +383,6 @@ static void named_pipe_accept_function(const char *pipe_name, int fd)
 		return;
 	}
 	npc->pipe_name = pipe_name;
-	npc->pipe_id = syntax;
 	npc->ev = server_event_context();
 	npc->msg_ctx = server_messaging_context();
 
@@ -470,7 +462,7 @@ static void named_pipe_accept_done(struct tevent_req *subreq)
 	}
 
 	ret = make_server_pipes_struct(npc,
-					npc->pipe_name, npc->pipe_id, NCACN_NP,
+				       npc->pipe_name, NCACN_NP,
 					false, cli_addr, NULL, npc->session_info,
 					&npc->p, &error);
 	if (ret != 0) {
@@ -679,7 +671,6 @@ fail:
 
 static void dcerpc_ncacn_accept(struct tevent_context *ev_ctx,
 				struct messaging_context *msg_ctx,
-				struct ndr_syntax_id syntax_id,
 				enum dcerpc_transport_t transport,
 				const char *name,
 				uint16_t port,
@@ -699,7 +690,6 @@ static void dcerpc_ncacn_tcpip_listener(struct tevent_context *ev,
 
 uint16_t setup_dcerpc_ncacn_tcpip_socket(struct tevent_context *ev_ctx,
 					 struct messaging_context *msg_ctx,
-					 struct ndr_syntax_id syntax_id,
 					 const struct sockaddr_storage *ifss,
 					 uint16_t port)
 {
@@ -713,7 +703,6 @@ uint16_t setup_dcerpc_ncacn_tcpip_socket(struct tevent_context *ev_ctx,
 		return 0;
 	}
 
-	state->syntax_id = syntax_id;
 	state->fd = -1;
 	state->ep.port = port;
 	state->disconnect_fn = NULL;
@@ -839,7 +828,6 @@ static void dcerpc_ncacn_tcpip_listener(struct tevent_context *ev,
 
 	dcerpc_ncacn_accept(state->ev_ctx,
 			    state->msg_ctx,
-			    state->syntax_id,
 			    NCACN_IP_TCP,
 			    NULL,
 			    state->ep.port,
@@ -860,7 +848,6 @@ static void dcerpc_ncalrpc_listener(struct tevent_context *ev,
 
 bool setup_dcerpc_ncalrpc_socket(struct tevent_context *ev_ctx,
 				 struct messaging_context *msg_ctx,
-				 struct ndr_syntax_id syntax_id,
 				 const char *name,
 				 dcerpc_ncacn_disconnect_fn fn)
 {
@@ -873,7 +860,6 @@ bool setup_dcerpc_ncalrpc_socket(struct tevent_context *ev_ctx,
 		return false;
 	}
 
-	state->syntax_id = syntax_id;
 	state->fd = -1;
 	state->disconnect_fn = fn;
 
@@ -888,13 +874,13 @@ bool setup_dcerpc_ncalrpc_socket(struct tevent_context *ev_ctx,
 		return false;
 	}
 
-	if (!directory_create_or_exist(lp_ncalrpc_dir(), geteuid(), 0700)) {
+	if (!directory_create_or_exist(lp_ncalrpc_dir(), geteuid(), 0755)) {
 		DEBUG(0, ("Failed to create pipe directory %s - %s\n",
 			  lp_ncalrpc_dir(), strerror(errno)));
 		goto out;
 	}
 
-	state->fd = create_pipe_sock(lp_ncalrpc_dir(), name, 0700);
+	state->fd = create_pipe_sock(lp_ncalrpc_dir(), name, 0755);
 	if (state->fd == -1) {
 		DEBUG(0, ("Failed to create pipe socket! [%s/%s]\n",
 			  lp_ncalrpc_dir(), name));
@@ -969,15 +955,13 @@ static void dcerpc_ncalrpc_listener(struct tevent_context *ev,
 
 	dcerpc_ncacn_accept(state->ev_ctx,
 			    state->msg_ctx,
-			    state->syntax_id, NCALRPC,
+			    NCALRPC,
 			    state->ep.name, 0,
 			    cli_addr, NULL, sd,
 			    state->disconnect_fn);
 }
 
 struct dcerpc_ncacn_conn {
-	struct ndr_syntax_id syntax_id;
-
 	enum dcerpc_transport_t transport;
 
 	union {
@@ -1011,7 +995,6 @@ static void dcerpc_ncacn_packet_done(struct tevent_req *subreq);
 
 static void dcerpc_ncacn_accept(struct tevent_context *ev_ctx,
 				struct messaging_context *msg_ctx,
-				struct ndr_syntax_id syntax_id,
 				enum dcerpc_transport_t transport,
 				const char *name,
 				uint16_t port,
@@ -1040,7 +1023,6 @@ static void dcerpc_ncacn_accept(struct tevent_context *ev_ctx,
 	}
 
 	ncacn_conn->transport = transport;
-	ncacn_conn->syntax_id = syntax_id;
 	ncacn_conn->ev_ctx = ev_ctx;
 	ncacn_conn->msg_ctx = msg_ctx;
 	ncacn_conn->sock = s;
@@ -1171,7 +1153,6 @@ static void dcerpc_ncacn_accept(struct tevent_context *ev_ctx,
 
 	rc = make_server_pipes_struct(ncacn_conn,
 				      pipe_name,
-				      ncacn_conn->syntax_id,
 				      ncacn_conn->transport,
 				      system_user,
 				      cli_str,
diff --git a/source3/rpc_server/rpc_server.h b/source3/rpc_server/rpc_server.h
index 3791589bb64..aafc96238a5 100644
--- a/source3/rpc_server/rpc_server.h
+++ b/source3/rpc_server/rpc_server.h
@@ -20,6 +20,8 @@
 #ifndef _RPC_SERVER_H_
 #define _RPC_SERVER_H_
 
+struct pipes_struct;
+
 typedef bool (*dcerpc_ncacn_disconnect_fn)(struct pipes_struct *p);
 
 void set_incoming_fault(struct pipes_struct *p);
@@ -29,13 +31,11 @@ bool setup_named_pipe_socket(const char *pipe_name,
 
 uint16_t setup_dcerpc_ncacn_tcpip_socket(struct tevent_context *ev_ctx,
 					 struct messaging_context *msg_ctx,
-					 struct ndr_syntax_id syntax_id,
 					 const struct sockaddr_storage *ifss,
 					 uint16_t port);
 
 bool setup_dcerpc_ncalrpc_socket(struct tevent_context *ev_ctx,
 				 struct messaging_context *msg_ctx,
-				 struct ndr_syntax_id syntax_id,
 				 const char *name,
 				 dcerpc_ncacn_disconnect_fn fn);
 
diff --git a/source3/rpc_server/samr/srv_samr_nt.c b/source3/rpc_server/samr/srv_samr_nt.c
index 99dd3f4d585..a253f307ac6 100644
--- a/source3/rpc_server/samr/srv_samr_nt.c
+++ b/source3/rpc_server/samr/srv_samr_nt.c
@@ -33,8 +33,8 @@
 
 #include "includes.h"
 #include "system/passwd.h"
-#include "smbd/globals.h"
 #include "../libcli/auth/libcli_auth.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_samr.h"
 #include "rpc_server/samr/srv_samr_util.h"
 #include "../lib/crypto/arcfour.h"
@@ -43,7 +43,6 @@
 #include "../libcli/security/security.h"
 #include "passdb.h"
 #include "auth.h"
-#include "ntdomain.h"
 #include "rpc_server/srv_access_check.h"
 
 #undef DBGC_CLASS
@@ -785,7 +784,7 @@ static NTSTATUS make_user_sam_entry_list(TALLOC_CTX *ctx,
 		return NT_STATUS_OK;
 	}
 
-	sam = TALLOC_ZERO_ARRAY(ctx, struct samr_SamEntry, num_entries);
+	sam = talloc_zero_array(ctx, struct samr_SamEntry, num_entries);
 	if (sam == NULL) {
 		DEBUG(0, ("make_user_sam_entry_list: TALLOC_ZERO failed!\n"));
 		return NT_STATUS_NO_MEMORY;
@@ -843,7 +842,7 @@ NTSTATUS _samr_EnumDomainUsers(struct pipes_struct *p,
 		return status;
 	}
 
-	samr_array = TALLOC_ZERO_P(p->mem_ctx, struct samr_SamArray);
+	samr_array = talloc_zero(p->mem_ctx, struct samr_SamArray);
 	if (!samr_array) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -939,7 +938,7 @@ static void make_group_sam_entry_list(TALLOC_CTX *ctx,
 		return;
 	}
 
-	sam = TALLOC_ZERO_ARRAY(ctx, struct samr_SamEntry, num_sam_entries);
+	sam = talloc_zero_array(ctx, struct samr_SamEntry, num_sam_entries);
 	if (sam == NULL) {
 		return;
 	}
@@ -978,7 +977,7 @@ NTSTATUS _samr_EnumDomainGroups(struct pipes_struct *p,
 
 	DEBUG(5,("_samr_EnumDomainGroups: %d\n", __LINE__));
 
-	samr_array = TALLOC_ZERO_P(p->mem_ctx, struct samr_SamArray);
+	samr_array = talloc_zero(p->mem_ctx, struct samr_SamArray);
 	if (!samr_array) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1056,7 +1055,7 @@ NTSTATUS _samr_EnumDomainAliases(struct pipes_struct *p,
 	DEBUG(5,("_samr_EnumDomainAliases: sid %s\n",
 		 sid_string_dbg(&dinfo->sid)));
 
-	samr_array = TALLOC_ZERO_P(p->mem_ctx, struct samr_SamArray);
+	samr_array = talloc_zero(p->mem_ctx, struct samr_SamArray);
 	if (!samr_array) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1121,7 +1120,7 @@ static NTSTATUS init_samr_dispinfo_1(TALLOC_CTX *ctx,
 
 	r->count = num_entries;
 
-	r->entries = TALLOC_ZERO_ARRAY(ctx, struct samr_DispEntryGeneral, num_entries);
+	r->entries = talloc_zero_array(ctx, struct samr_DispEntryGeneral, num_entries);
 	if (!r->entries) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1165,7 +1164,7 @@ static NTSTATUS init_samr_dispinfo_2(TALLOC_CTX *ctx,
 
 	r->count = num_entries;
 
-	r->entries = TALLOC_ZERO_ARRAY(ctx, struct samr_DispEntryFull, num_entries);
+	r->entries = talloc_zero_array(ctx, struct samr_DispEntryFull, num_entries);
 	if (!r->entries) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1206,7 +1205,7 @@ static NTSTATUS init_samr_dispinfo_3(TALLOC_CTX *ctx,
 
 	r->count = num_entries;
 
-	r->entries = TALLOC_ZERO_ARRAY(ctx, struct samr_DispEntryFullGroup, num_entries);
+	r->entries = talloc_zero_array(ctx, struct samr_DispEntryFullGroup, num_entries);
 	if (!r->entries) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1247,7 +1246,7 @@ static NTSTATUS init_samr_dispinfo_4(TALLOC_CTX *ctx,
 
 	r->count = num_entries;
 
-	r->entries = TALLOC_ZERO_ARRAY(ctx, struct samr_DispEntryAscii, num_entries);
+	r->entries = talloc_zero_array(ctx, struct samr_DispEntryAscii, num_entries);
 	if (!r->entries) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1283,7 +1282,7 @@ static NTSTATUS init_samr_dispinfo_5(TALLOC_CTX *ctx,
 
 	r->count = num_entries;
 
-	r->entries = TALLOC_ZERO_ARRAY(ctx, struct samr_DispEntryAscii, num_entries);
+	r->entries = talloc_zero_array(ctx, struct samr_DispEntryAscii, num_entries);
 	if (!r->entries) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1579,7 +1578,7 @@ NTSTATUS _samr_QueryAliasInfo(struct pipes_struct *p,
 		return status;
 	}
 
-	alias_info = TALLOC_ZERO_P(p->mem_ctx, union samr_AliasInfo);
+	alias_info = talloc_zero(p->mem_ctx, union samr_AliasInfo);
 	if (!alias_info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1985,12 +1984,12 @@ NTSTATUS _samr_ChangePasswordUser3(struct pipes_struct *p,
 		time_t u_expire, u_min_age;
 		uint32 account_policy_temp;
 
-		dominfo = TALLOC_ZERO_P(p->mem_ctx, struct samr_DomInfo1);
+		dominfo = talloc_zero(p->mem_ctx, struct samr_DomInfo1);
 		if (!dominfo) {
 			return NT_STATUS_NO_MEMORY;
 		}
 
-		reject = TALLOC_ZERO_P(p->mem_ctx,
+		reject = talloc_zero(p->mem_ctx,
 				struct userPwdChangeFailureInformation);
 		if (!reject) {
 			return NT_STATUS_NO_MEMORY;
@@ -2051,7 +2050,7 @@ static bool make_samr_lookup_rids(TALLOC_CTX *ctx, uint32 num_names,
 	*lsa_name_array_p = NULL;
 
 	if (num_names != 0) {
-		lsa_name_array = TALLOC_ZERO_ARRAY(ctx, struct lsa_String, num_names);
+		lsa_name_array = talloc_zero_array(ctx, struct lsa_String, num_names);
 		if (!lsa_name_array) {
 			return false;
 		}
@@ -2101,9 +2100,9 @@ NTSTATUS _samr_LookupRids(struct pipes_struct *p,
 	}
 
 	if (num_rids) {
-		names = TALLOC_ZERO_ARRAY(p->mem_ctx, const char *, num_rids);
-		attrs = TALLOC_ZERO_ARRAY(p->mem_ctx, enum lsa_SidType, num_rids);
-		wire_attrs = TALLOC_ZERO_ARRAY(p->mem_ctx, uint32, num_rids);
+		names = talloc_zero_array(p->mem_ctx, const char *, num_rids);
+		attrs = talloc_zero_array(p->mem_ctx, enum lsa_SidType, num_rids);
+		wire_attrs = talloc_zero_array(p->mem_ctx, uint32, num_rids);
 
 		if ((names == NULL) || (attrs == NULL) || (wire_attrs==NULL))
 			return NT_STATUS_NO_MEMORY;
@@ -2289,12 +2288,12 @@ static NTSTATUS init_samr_parameters_string(TALLOC_CTX *mem_ctx,
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	r = TALLOC_ZERO_P(mem_ctx, struct lsa_BinaryString);
+	r = talloc_zero(mem_ctx, struct lsa_BinaryString);
 	if (!r) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	r->array = TALLOC_ZERO_ARRAY(mem_ctx, uint16_t, blob->length/2);
+	r->array = talloc_zero_array(mem_ctx, uint16_t, blob->length/2);
 	if (!r->array) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -2988,7 +2987,7 @@ NTSTATUS _samr_QueryUserInfo(struct pipes_struct *p,
 	DEBUG(5,("_samr_QueryUserInfo: sid:%s\n",
 		 sid_string_dbg(&uinfo->sid)));
 
-	user_info = TALLOC_ZERO_P(p->mem_ctx, union samr_UserInfo);
+	user_info = talloc_zero(p->mem_ctx, union samr_UserInfo);
 	if (!user_info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -3150,7 +3149,7 @@ NTSTATUS _samr_GetGroupsForUser(struct pipes_struct *p,
 		return result;
 	}
 
-	rids = TALLOC_ZERO_P(p->mem_ctx, struct samr_RidWithAttributeArray);
+	rids = talloc_zero(p->mem_ctx, struct samr_RidWithAttributeArray);
 	if (!rids) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -3324,7 +3323,7 @@ static NTSTATUS query_dom_info_2(TALLOC_CTX *mem_ctx,
 
 	r->oem_information.string	= lp_serverstring();
 	r->domain_name.string		= lp_workgroup();
-	r->primary.string		= global_myname();
+	r->primary.string		= lp_netbios_name();
 	r->sequence_num			= seq_num;
 	r->domain_server_state		= DOMAIN_SERVER_ENABLED;
 	r->role				= (enum samr_Role) samr_get_server_role();
@@ -3391,7 +3390,7 @@ static NTSTATUS query_dom_info_6(TALLOC_CTX *mem_ctx,
 	/* NT returns its own name when a PDC. win2k and later
 	 * only the name of the PDC if itself is a BDC (samba4
 	 * idl) */
-	r->primary.string = global_myname();
+	r->primary.string = lp_netbios_name();
 
 	return NT_STATUS_OK;
 }
@@ -3599,7 +3598,7 @@ NTSTATUS _samr_QueryDomainInfo(struct pipes_struct *p,
 		return status;
 	}
 
-	dom_info = TALLOC_ZERO_P(p->mem_ctx, union samr_DomainInfo);
+	dom_info = talloc_zero(p->mem_ctx, union samr_DomainInfo);
 	if (!dom_info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -4055,7 +4054,7 @@ NTSTATUS _samr_LookupDomain(struct pipes_struct *p,
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	sid = TALLOC_ZERO_P(p->mem_ctx, struct dom_sid2);
+	sid = talloc_zero(p->mem_ctx, struct dom_sid2);
 	if (!sid) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -4096,12 +4095,12 @@ NTSTATUS _samr_EnumDomains(struct pipes_struct *p,
 		return status;
 	}
 
-	sam = TALLOC_ZERO_P(p->mem_ctx, struct samr_SamArray);
+	sam = talloc_zero(p->mem_ctx, struct samr_SamArray);
 	if (!sam) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
-	entry_array = TALLOC_ZERO_ARRAY(p->mem_ctx,
+	entry_array = talloc_zero_array(p->mem_ctx,
 					struct samr_SamEntry,
 					num_entries);
 	if (!entry_array) {
@@ -5299,7 +5298,7 @@ NTSTATUS _samr_GetAliasMembership(struct pipes_struct *p,
 		return NT_STATUS_OBJECT_TYPE_MISMATCH;
 
 	if (r->in.sids->num_sids) {
-		members = TALLOC_ARRAY(p->mem_ctx, struct dom_sid, r->in.sids->num_sids);
+		members = talloc_array(p->mem_ctx, struct dom_sid, r->in.sids->num_sids);
 
 		if (members == NULL)
 			return NT_STATUS_NO_MEMORY;
@@ -5370,7 +5369,7 @@ NTSTATUS _samr_GetMembersInAlias(struct pipes_struct *p,
 	}
 
 	if (num_sids) {
-		sids = TALLOC_ZERO_ARRAY(p->mem_ctx, struct lsa_SidPtr, num_sids);
+		sids = talloc_zero_array(p->mem_ctx, struct lsa_SidPtr, num_sids);
 		if (sids == NULL) {
 			TALLOC_FREE(pdb_sids);
 			return NT_STATUS_NO_MEMORY;
@@ -5416,7 +5415,7 @@ NTSTATUS _samr_QueryGroupMember(struct pipes_struct *p,
 		return status;
 	}
 
-	rids = TALLOC_ZERO_P(p->mem_ctx, struct samr_RidAttrArray);
+	rids = talloc_zero(p->mem_ctx, struct samr_RidAttrArray);
 	if (!rids) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -5440,7 +5439,7 @@ NTSTATUS _samr_QueryGroupMember(struct pipes_struct *p,
 		return status;
 
 	if (num_members) {
-		attr=TALLOC_ZERO_ARRAY(p->mem_ctx, uint32, num_members);
+		attr=talloc_zero_array(p->mem_ctx, uint32, num_members);
 		if (attr == NULL) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -5955,7 +5954,7 @@ NTSTATUS _samr_QueryGroupInfo(struct pipes_struct *p,
 	group_name = talloc_strdup(r, map.nt_name);
 	group_description = talloc_strdup(r, map.comment);
 
-	info = TALLOC_ZERO_P(p->mem_ctx, union samr_GroupInfo);
+	info = talloc_zero(p->mem_ctx, union samr_GroupInfo);
 	if (!info) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -6133,7 +6132,7 @@ NTSTATUS _samr_SetAliasInfo(struct pipes_struct *p,
 			/* make sure the name doesn't already exist as a user
 			   or local group */
 
-			fstr_sprintf( group_name, "%s\\%s", global_myname(), info.acct_name );
+			fstr_sprintf( group_name, "%s\\%s", lp_netbios_name(), info.acct_name );
 			status = can_create( p->mem_ctx, group_name );
 			if ( !NT_STATUS_IS_OK( status ) )
 				return status;
diff --git a/source3/rpc_server/samr/srv_samr_util.c b/source3/rpc_server/samr/srv_samr_util.c
index 29123321f86..d052846b2e5 100644
--- a/source3/rpc_server/samr/srv_samr_util.c
+++ b/source3/rpc_server/samr/srv_samr_util.c
@@ -612,7 +612,16 @@ void copy_id21_to_sam_passwd(const char *log_prefix,
 		DEBUG(10,("%s SAMR_FIELD_EXPIRED_FLAG: %02X\n", l,
 			from->password_expired));
 		if (from->password_expired != 0) {
-			pdb_set_pass_last_set_time(to, 0, PDB_CHANGED);
+			/* Only allow the set_time to zero (which means
+			   "User Must Change Password on Next Login"
+			   if the user object allows password change. */
+			if (pdb_get_pass_can_change(to)) {
+				pdb_set_pass_last_set_time(to, 0, PDB_CHANGED);
+			} else {
+				DEBUG(10,("%s Disallowing set of 'User Must "
+					"Change Password on Next Login' as "
+					"user object disallows this.\n", l));
+			}
 		} else {
 			/* A subtlety here: some windows commands will
 			   clear the expired flag even though it's not
diff --git a/source3/rpc_server/spoolss/srv_spoolss_nt.c b/source3/rpc_server/spoolss/srv_spoolss_nt.c
index 312f50965b7..8216388905f 100644
--- a/source3/rpc_server/spoolss/srv_spoolss_nt.c
+++ b/source3/rpc_server/spoolss/srv_spoolss_nt.c
@@ -28,6 +28,7 @@
    up, all the errors returned are DOS errors, not NT status codes. */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "nt_printing.h"
 #include "srv_spoolss_util.h"
 #include "../librpc/gen_ndr/srv_spoolss.h"
@@ -46,9 +47,13 @@
 #include "serverid.h"
 #include "../libcli/registry/util_reg.h"
 #include "smbd/smbd.h"
+#include "smbd/globals.h"
 #include "auth.h"
 #include "messages.h"
-#include "ntdomain.h"
+#include "rpc_server/spoolss/srv_spoolss_nt.h"
+#include "util_tdb.h"
+#include "libsmb/libsmb.h"
+#include "printing/printer_list.h"
 
 /* macros stolen from s4 spoolss server */
 #define SPOOLSS_BUFFER_UNION(fn,info,level) \
@@ -574,7 +579,7 @@ static WERROR set_printer_hnd_name(TALLOC_CTX *mem_ctx,
 	DEBUG(4,("Setting printer name=%s (len=%lu)\n", handlename,
 		(unsigned long)strlen(handlename)));
 
-	aprinter = CONST_DISCARD(char *, handlename);
+	aprinter = discard_const_p(char, handlename);
 	if ( *handlename == '\\' ) {
 		servername = canon_servername(handlename);
 		if ( (aprinter = strchr_m( servername, '\\' )) != NULL ) {
@@ -726,7 +731,7 @@ static WERROR set_printer_hnd_name(TALLOC_CTX *mem_ctx,
 
 	DEBUGADD(4,("set_printer_hnd_name: Printer found: %s -> %s\n", aprinter, sname));
 
-	fstrcpy(Printer->sharename, sname);
+	strlcpy(Printer->sharename, sname, sizeof(Printer->sharename));
 
 	return WERR_OK;
 }
@@ -1089,7 +1094,7 @@ static int notify_msg_ctr_addmsg( SPOOLSS_NOTIFY_MSG_CTR *ctr, SPOOLSS_NOTIFY_MS
 	if ( i == ctr->num_groups ) {
 		ctr->num_groups++;
 
-		if ( !(groups = TALLOC_REALLOC_ARRAY( ctr->ctx, ctr->msg_groups, SPOOLSS_NOTIFY_MSG_GROUP, ctr->num_groups)) ) {
+		if ( !(groups = talloc_realloc( ctr->ctx, ctr->msg_groups, SPOOLSS_NOTIFY_MSG_GROUP, ctr->num_groups)) ) {
 			DEBUG(0,("notify_msg_ctr_addmsg: talloc_realloc() failed!\n"));
 			return 0;
 		}
@@ -1107,7 +1112,7 @@ static int notify_msg_ctr_addmsg( SPOOLSS_NOTIFY_MSG_CTR *ctr, SPOOLSS_NOTIFY_MS
 
 	msg_grp->num_msgs++;
 
-	if ( !(msg_list = TALLOC_REALLOC_ARRAY( ctr->ctx, msg_grp->msgs, SPOOLSS_NOTIFY_MSG, msg_grp->num_msgs )) ) {
+	if ( !(msg_list = talloc_realloc( ctr->ctx, msg_grp->msgs, SPOOLSS_NOTIFY_MSG, msg_grp->num_msgs )) ) {
 		DEBUG(0,("notify_msg_ctr_addmsg: talloc_realloc() failed for new message [%d]!\n", msg_grp->num_msgs));
 		return 0;
 	}
@@ -1120,7 +1125,7 @@ static int notify_msg_ctr_addmsg( SPOOLSS_NOTIFY_MSG_CTR *ctr, SPOOLSS_NOTIFY_MS
 
 	if ( msg->len != 0 )
 		msg_grp->msgs[new_slot].notify.data = (char *)
-			TALLOC_MEMDUP( ctr->ctx, msg->notify.data, msg->len );
+			talloc_memdup( ctr->ctx, msg->notify.data, msg->len );
 
 	return ctr->num_groups;
 }
@@ -1500,7 +1505,7 @@ static bool srv_spoolss_drv_upgrade_printer(const char *drivername,
 
 	messaging_send_buf(msg_ctx, messaging_server_id(msg_ctx),
 			   MSG_PRINTER_DRVUPGRADE,
-			   (uint8_t *)drivername, len+1);
+			   (const uint8_t *)drivername, len+1);
 
 	return true;
 }
@@ -2335,32 +2340,32 @@ static WERROR getprinterdata_printer_server(TALLOC_CTX *mem_ctx,
 {
 	DEBUG(8,("getprinterdata_printer_server:%s\n", value));
 
-	if (!StrCaseCmp(value, "W3SvcInstalled")) {
+	if (!strcasecmp_m(value, "W3SvcInstalled")) {
 		*type = REG_DWORD;
 		data->value = 0x00;
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "BeepEnabled")) {
+	if (!strcasecmp_m(value, "BeepEnabled")) {
 		*type = REG_DWORD;
 		data->value = 0x00;
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "EventLog")) {
+	if (!strcasecmp_m(value, "EventLog")) {
 		*type = REG_DWORD;
 		/* formally was 0x1b */
 		data->value = 0x00;
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "NetPopup")) {
+	if (!strcasecmp_m(value, "NetPopup")) {
 		*type = REG_DWORD;
 		data->value = 0x00;
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "MajorVersion")) {
+	if (!strcasecmp_m(value, "MajorVersion")) {
 		*type = REG_DWORD;
 
 		/* Windows NT 4.0 seems to not allow uploading of drivers
@@ -2377,7 +2382,7 @@ static WERROR getprinterdata_printer_server(TALLOC_CTX *mem_ctx,
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "MinorVersion")) {
+	if (!strcasecmp_m(value, "MinorVersion")) {
 		*type = REG_DWORD;
 		data->value = 0x00;
 		return WERR_OK;
@@ -2390,7 +2395,7 @@ static WERROR getprinterdata_printer_server(TALLOC_CTX *mem_ctx,
 	 *  uint32_t build 	 = [2195|2600]
 	 *  extra unicode string = e.g. "Service Pack 3"
 	 */
-	if (!StrCaseCmp(value, "OSVersion")) {
+	if (!strcasecmp_m(value, "OSVersion")) {
 		DATA_BLOB blob;
 		enum ndr_err_code ndr_err;
 		struct spoolss_OSVersion os;
@@ -2413,7 +2418,7 @@ static WERROR getprinterdata_printer_server(TALLOC_CTX *mem_ctx,
 	}
 
 
-	if (!StrCaseCmp(value, "DefaultSpoolDirectory")) {
+	if (!strcasecmp_m(value, "DefaultSpoolDirectory")) {
 		*type = REG_SZ;
 
 		data->string = talloc_strdup(mem_ctx, "C:\\PRINTERS");
@@ -2422,7 +2427,7 @@ static WERROR getprinterdata_printer_server(TALLOC_CTX *mem_ctx,
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "Architecture")) {
+	if (!strcasecmp_m(value, "Architecture")) {
 		*type = REG_SZ;
 		data->string = talloc_strdup(mem_ctx,
 			lp_parm_const_string(GLOBAL_SECTION_SNUM, "spoolss", "architecture", SPOOLSS_ARCHITECTURE_NT_X86));
@@ -2431,7 +2436,7 @@ static WERROR getprinterdata_printer_server(TALLOC_CTX *mem_ctx,
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "DsPresent")) {
+	if (!strcasecmp_m(value, "DsPresent")) {
 		*type = REG_DWORD;
 
 		/* only show the publish check box if we are a
@@ -2445,7 +2450,7 @@ static WERROR getprinterdata_printer_server(TALLOC_CTX *mem_ctx,
 		return WERR_OK;
 	}
 
-	if (!StrCaseCmp(value, "DNSMachineName")) {
+	if (!strcasecmp_m(value, "DNSMachineName")) {
 		const char *hostname = get_mydnsfullname();
 
 		if (!hostname) {
@@ -2518,7 +2523,7 @@ static bool spoolss_connect_to_client(struct rpc_pipe_client **pp_pipe,
 	}
 
 	/* setup the connection */
-	ret = cli_full_connection( &the_cli, global_myname(), remote_machine,
+	ret = cli_full_connection( &the_cli, lp_netbios_name(), remote_machine,
 		&rm_addr, 0, "IPC$", "IPC",
 		"", /* username */
 		"", /* domain */
@@ -2876,7 +2881,21 @@ static void spoolss_notify_location(struct messaging_context *msg_ctx,
 				    struct spoolss_PrinterInfo2 *pinfo2,
 				    TALLOC_CTX *mem_ctx)
 {
-	SETUP_SPOOLSS_NOTIFY_DATA_STRING(data, pinfo2->location);
+	const char *loc = pinfo2->location;
+	NTSTATUS status;
+
+	status = printer_list_get_printer(mem_ctx,
+					  pinfo2->sharename,
+					  NULL,
+					  &loc,
+					  NULL);
+	if (NT_STATUS_IS_OK(status)) {
+		if (loc == NULL) {
+			loc = pinfo2->location;
+		}
+	}
+
+	SETUP_SPOOLSS_NOTIFY_DATA_STRING(data, loc);
 }
 
 /*******************************************************************
@@ -3404,7 +3423,6 @@ static bool construct_notify_printer_info(struct messaging_context *msg_ctx,
 	uint16_t field;
 
 	struct spoolss_Notify *current_data;
-	print_queue_struct *queue=NULL;
 
 	type = option_type->type;
 
@@ -3420,7 +3438,7 @@ static bool construct_notify_printer_info(struct messaging_context *msg_ctx,
 		if (!search_notify(type, field, &j) )
 			continue;
 
-		info->notifies = TALLOC_REALLOC_ARRAY(info, info->notifies,
+		info->notifies = talloc_realloc(info, info->notifies,
 						      struct spoolss_Notify,
 						      info->count + 1);
 		if (info->notifies == NULL) {
@@ -3438,7 +3456,7 @@ static bool construct_notify_printer_info(struct messaging_context *msg_ctx,
 			   pinfo2->printername));
 
 		notify_info_data_table[j].fn(msg_ctx, snum, current_data,
-					     queue, pinfo2, mem_ctx);
+					     NULL, pinfo2, mem_ctx);
 
 		info->count++;
 	}
@@ -3480,7 +3498,7 @@ static bool construct_notify_jobs_info(struct messaging_context *msg_ctx,
 		if (!search_notify(type, field, &j) )
 			continue;
 
-		info->notifies = TALLOC_REALLOC_ARRAY(info, info->notifies,
+		info->notifies = talloc_realloc(info, info->notifies,
 						      struct spoolss_Notify,
 						      info->count + 1);
 		if (info->notifies == NULL) {
@@ -4009,8 +4027,24 @@ static WERROR construct_printer_info2(TALLOC_CTX *mem_ctx,
 	}
 	W_ERROR_HAVE_NO_MEMORY(r->comment);
 
-	r->location		= talloc_strdup(mem_ctx, info2->location);
+	r->location	= talloc_strdup(mem_ctx, info2->location);
+	if (info2->location[0] == '\0') {
+		const char *loc = NULL;
+		NTSTATUS nt_status;
+
+		nt_status = printer_list_get_printer(mem_ctx,
+						     info2->sharename,
+						     NULL,
+						     &loc,
+						     NULL);
+		if (NT_STATUS_IS_OK(nt_status)) {
+			if (loc != NULL) {
+				r->location = talloc_strdup(mem_ctx, loc);
+			}
+		}
+	}
 	W_ERROR_HAVE_NO_MEMORY(r->location);
+
 	r->sepfile		= talloc_strdup(mem_ctx, info2->sepfile);
 	W_ERROR_HAVE_NO_MEMORY(r->sepfile);
 	r->printprocessor	= talloc_strdup(mem_ctx, info2->printprocessor);
@@ -4145,10 +4179,9 @@ static WERROR construct_printer_info6(TALLOC_CTX *mem_ctx,
 				      struct spoolss_PrinterInfo6 *r,
 				      int snum)
 {
-	int count;
 	print_status_struct status;
 
-	count = print_queue_length(msg_ctx, snum, &status);
+	print_queue_length(msg_ctx, snum, &status);
 
 	r->status = nt_printq_status(status.status);
 
@@ -4274,7 +4307,7 @@ static WERROR enum_all_printers_info_level(TALLOC_CTX *mem_ctx,
 			goto out;
 		}
 
-		info = TALLOC_REALLOC_ARRAY(mem_ctx, info,
+		info = talloc_realloc(mem_ctx, info,
 					    union spoolss_PrinterInfo,
 					    count + 1);
 		if (!info) {
@@ -5314,7 +5347,7 @@ static WERROR spoolss_DriverFileInfo_from_driver(TALLOC_CTX *mem_ctx,
 	*count_p = 0;
 
 	if (strlen(driver->driver_path)) {
-		info = TALLOC_REALLOC_ARRAY(mem_ctx, info,
+		info = talloc_realloc(mem_ctx, info,
 					    struct spoolss_DriverFileInfo,
 					    count + 1);
 		W_ERROR_HAVE_NO_MEMORY(info);
@@ -5329,7 +5362,7 @@ static WERROR spoolss_DriverFileInfo_from_driver(TALLOC_CTX *mem_ctx,
 	}
 
 	if (strlen(driver->config_file)) {
-		info = TALLOC_REALLOC_ARRAY(mem_ctx, info,
+		info = talloc_realloc(mem_ctx, info,
 					    struct spoolss_DriverFileInfo,
 					    count + 1);
 		W_ERROR_HAVE_NO_MEMORY(info);
@@ -5344,7 +5377,7 @@ static WERROR spoolss_DriverFileInfo_from_driver(TALLOC_CTX *mem_ctx,
 	}
 
 	if (strlen(driver->data_file)) {
-		info = TALLOC_REALLOC_ARRAY(mem_ctx, info,
+		info = talloc_realloc(mem_ctx, info,
 					    struct spoolss_DriverFileInfo,
 					    count + 1);
 		W_ERROR_HAVE_NO_MEMORY(info);
@@ -5359,7 +5392,7 @@ static WERROR spoolss_DriverFileInfo_from_driver(TALLOC_CTX *mem_ctx,
 	}
 
 	if (strlen(driver->help_file)) {
-		info = TALLOC_REALLOC_ARRAY(mem_ctx, info,
+		info = talloc_realloc(mem_ctx, info,
 					    struct spoolss_DriverFileInfo,
 					    count + 1);
 		W_ERROR_HAVE_NO_MEMORY(info);
@@ -5374,7 +5407,7 @@ static WERROR spoolss_DriverFileInfo_from_driver(TALLOC_CTX *mem_ctx,
 	}
 
 	for (i=0; driver->dependent_files[i] && driver->dependent_files[i][0] != '\0'; i++) {
-		info = TALLOC_REALLOC_ARRAY(mem_ctx, info,
+		info = talloc_realloc(mem_ctx, info,
 					    struct spoolss_DriverFileInfo,
 					    count + 1);
 		W_ERROR_HAVE_NO_MEMORY(info);
@@ -6004,7 +6037,7 @@ static bool check_printer_ok(TALLOC_CTX *mem_ctx,
 		info2->location));
 
 	/* we force some elements to "correct" values */
-	info2->servername = talloc_asprintf(mem_ctx, "\\\\%s", global_myname());
+	info2->servername = talloc_asprintf(mem_ctx, "\\\\%s", lp_netbios_name());
 	if (info2->servername == NULL) {
 		return false;
 	}
@@ -6016,7 +6049,7 @@ static bool check_printer_ok(TALLOC_CTX *mem_ctx,
 	/* check to see if we allow printername != sharename */
 	if (lp_force_printername(snum)) {
 		info2->printername = talloc_asprintf(mem_ctx, "\\\\%s\\%s",
-					global_myname(), info2->sharename);
+					lp_netbios_name(), info2->sharename);
 	} else {
 		/* make sure printername is in \\server\printername format */
 		fstrcpy(printername, info2->printername);
@@ -6027,7 +6060,7 @@ static bool check_printer_ok(TALLOC_CTX *mem_ctx,
 		}
 
 		info2->printername = talloc_asprintf(mem_ctx, "\\\\%s\\%s",
-					global_myname(), p);
+					lp_netbios_name(), p);
 	}
 	if (info2->printername == NULL) {
 		return false;
@@ -6413,7 +6446,7 @@ static WERROR update_dsspooler(TALLOC_CTX *mem_ctx,
 					  buffer.length);
 	}
 
-	push_reg_sz(mem_ctx, &buffer, global_myname());
+	push_reg_sz(mem_ctx, &buffer, lp_netbios_name());
 	winreg_set_printer_dataex(mem_ctx,
 				  session_info,
 				  msg_ctx,
@@ -6428,7 +6461,7 @@ static WERROR update_dsspooler(TALLOC_CTX *mem_ctx,
 	if (dnsdomname != NULL && dnsdomname[0] != '\0') {
 		longname = talloc_strdup(mem_ctx, dnsdomname);
 	} else {
-		longname = talloc_strdup(mem_ctx, global_myname());
+		longname = talloc_strdup(mem_ctx, lp_netbios_name());
 	}
 	if (longname == NULL) {
 		result = WERR_NOMEM;
@@ -6447,7 +6480,7 @@ static WERROR update_dsspooler(TALLOC_CTX *mem_ctx,
 				  buffer.length);
 
 	uncname = talloc_asprintf(mem_ctx, "\\\\%s\\%s",
-				  global_myname(), printer->sharename);
+				  lp_netbios_name(), printer->sharename);
 	push_reg_sz(mem_ctx, &buffer, uncname);
 	winreg_set_printer_dataex(mem_ctx,
 				  session_info,
@@ -6891,7 +6924,7 @@ static WERROR enumjobs_level1(TALLOC_CTX *mem_ctx,
 	int i;
 	WERROR result = WERR_OK;
 
-	info = TALLOC_ARRAY(mem_ctx, union spoolss_JobInfo, num_queues);
+	info = talloc_array(mem_ctx, union spoolss_JobInfo, num_queues);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	*count = num_queues;
@@ -6935,7 +6968,7 @@ static WERROR enumjobs_level2(TALLOC_CTX *mem_ctx,
 	int i;
 	WERROR result = WERR_OK;
 
-	info = TALLOC_ARRAY(mem_ctx, union spoolss_JobInfo, num_queues);
+	info = talloc_array(mem_ctx, union spoolss_JobInfo, num_queues);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	*count = num_queues;
@@ -6990,7 +7023,7 @@ static WERROR enumjobs_level3(TALLOC_CTX *mem_ctx,
 	int i;
 	WERROR result = WERR_OK;
 
-	info = TALLOC_ARRAY(mem_ctx, union spoolss_JobInfo, num_queues);
+	info = talloc_array(mem_ctx, union spoolss_JobInfo, num_queues);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	*count = num_queues;
@@ -7259,7 +7292,7 @@ static WERROR enumprinterdrivers_level_by_architecture(TALLOC_CTX *mem_ctx,
 			  num_drivers, architecture, version));
 
 		if (num_drivers != 0) {
-			info = TALLOC_REALLOC_ARRAY(mem_ctx, info,
+			info = talloc_realloc(mem_ctx, info,
 						    union spoolss_DriverInfo,
 						    count + num_drivers);
 			if (!info) {
@@ -7599,7 +7632,7 @@ static WERROR enumports_hook(TALLOC_CTX *ctx, int *count, char ***lines)
 	/* if no hook then just fill in the default port */
 
 	if ( !*cmd ) {
-		if (!(qlines = TALLOC_ARRAY( NULL, char*, 2 ))) {
+		if (!(qlines = talloc_array( NULL, char*, 2 ))) {
 			return WERR_NOMEM;
 		}
 		if (!(qlines[0] = talloc_strdup(qlines, SAMBA_PRINTER_PORT_NAME ))) {
@@ -7660,7 +7693,7 @@ static WERROR enumports_level_1(TALLOC_CTX *mem_ctx,
 	}
 
 	if (numlines) {
-		info = TALLOC_ARRAY(mem_ctx, union spoolss_PortInfo, numlines);
+		info = talloc_array(mem_ctx, union spoolss_PortInfo, numlines);
 		if (!info) {
 			DEBUG(10,("Returning WERR_NOMEM\n"));
 			result = WERR_NOMEM;
@@ -7712,7 +7745,7 @@ static WERROR enumports_level_2(TALLOC_CTX *mem_ctx,
 	}
 
 	if (numlines) {
-		info = TALLOC_ARRAY(mem_ctx, union spoolss_PortInfo, numlines);
+		info = talloc_array(mem_ctx, union spoolss_PortInfo, numlines);
 		if (!info) {
 			DEBUG(10,("Returning WERR_NOMEM\n"));
 			result = WERR_NOMEM;
@@ -8011,12 +8044,12 @@ WERROR _spoolss_AddPrinterDriverEx(struct pipes_struct *p,
 	}
 
 	DEBUG(5,("Cleaning driver's information\n"));
-	err = clean_up_driver_struct(p->mem_ctx, p, r->in.info_ctr);
+	err = clean_up_driver_struct(p->mem_ctx, p->session_info, r->in.info_ctr);
 	if (!W_ERROR_IS_OK(err))
 		goto done;
 
 	DEBUG(5,("Moving driver to final destination\n"));
-	err = move_driver_to_download_area(p, r->in.info_ctr);
+	err = move_driver_to_download_area(p->session_info, r->in.info_ctr);
 	if (!W_ERROR_IS_OK(err)) {
 		goto done;
 	}
@@ -8639,7 +8672,7 @@ static WERROR enumprintprocessors_level_1(TALLOC_CTX *mem_ctx,
 	union spoolss_PrintProcessorInfo *info;
 	WERROR result;
 
-	info = TALLOC_ARRAY(mem_ctx, union spoolss_PrintProcessorInfo, 1);
+	info = talloc_array(mem_ctx, union spoolss_PrintProcessorInfo, 1);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	*count = 1;
@@ -8741,7 +8774,7 @@ static WERROR enumprintprocdatatypes_level_1(TALLOC_CTX *mem_ctx,
 	WERROR result;
 	union spoolss_PrintProcDataTypesInfo *info;
 
-	info = TALLOC_ARRAY(mem_ctx, union spoolss_PrintProcDataTypesInfo, 1);
+	info = talloc_array(mem_ctx, union spoolss_PrintProcDataTypesInfo, 1);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	*count = 1;
@@ -8798,6 +8831,10 @@ WERROR _spoolss_EnumPrintProcDataTypes(struct pipes_struct *p,
 		return WERR_UNKNOWN_LEVEL;
 	}
 
+	if (!W_ERROR_IS_OK(result)) {
+		return result;
+	}
+
 	*r->out.needed	= SPOOLSS_BUFFER_UNION_ARRAY(p->mem_ctx,
 						     spoolss_EnumPrintProcDataTypes,
 						     *r->out.info, r->in.level,
@@ -8853,7 +8890,7 @@ static WERROR enumprintmonitors_level_1(TALLOC_CTX *mem_ctx,
 	union spoolss_MonitorInfo *info;
 	WERROR result = WERR_OK;
 
-	info = TALLOC_ARRAY(mem_ctx, union spoolss_MonitorInfo, 2);
+	info = talloc_array(mem_ctx, union spoolss_MonitorInfo, 2);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	*count = 2;
@@ -8893,7 +8930,7 @@ static WERROR enumprintmonitors_level_2(TALLOC_CTX *mem_ctx,
 	union spoolss_MonitorInfo *info;
 	WERROR result = WERR_OK;
 
-	info = TALLOC_ARRAY(mem_ctx, union spoolss_MonitorInfo, 2);
+	info = talloc_array(mem_ctx, union spoolss_MonitorInfo, 2);
 	W_ERROR_HAVE_NO_MEMORY(info);
 
 	*count = 2;
@@ -9154,9 +9191,9 @@ WERROR _spoolss_GetPrinterDataEx(struct pipes_struct *p,
 	int 			snum = 0;
 	WERROR result = WERR_OK;
 	DATA_BLOB blob;
-	enum winreg_Type val_type;
-	uint8_t *val_data;
-	uint32_t val_size;
+	enum winreg_Type val_type = REG_NONE;
+	uint8_t *val_data = NULL;
+	uint32_t val_size = 0;
 
 
 	DEBUG(4,("_spoolss_GetPrinterDataEx\n"));
diff --git a/source3/rpc_server/spoolss/srv_spoolss_nt.h b/source3/rpc_server/spoolss/srv_spoolss_nt.h
new file mode 100644
index 00000000000..d6d141a5e8f
--- /dev/null
+++ b/source3/rpc_server/spoolss/srv_spoolss_nt.h
@@ -0,0 +1,40 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Copyright (C) Andrew Tridgell              1992-2000,
+ *  Copyright (C) Luke Kenneth Casson Leighton 1996-2000,
+ *  Copyright (C) Jean François Micouleau      1998-2000,
+ *  Copyright (C) Jeremy Allison               2001-2002,
+ *  Copyright (C) Gerald Carter		       2000-2004,
+ *  Copyright (C) Tim Potter                   2001-2002.
+ *  Copyright (C) Guenther Deschner            2009-2010.
+ *  Copyright (C) Andreas Schneider            2010.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_SERVER_SPOOLSS_SRV_SPOOLSS_NT_H_
+#define _RPC_SERVER_SPOOLSS_SRV_SPOOLSS_NT_H_
+
+/* The following definitions come from rpc_server/srv_spoolss_nt.c  */
+void srv_spoolss_cleanup(void);
+
+void do_drv_upgrade_printer(struct messaging_context *msg,
+			    void *private_data,
+			    uint32_t msg_type,
+			    struct server_id server_id,
+			    DATA_BLOB *data);
+void update_monitored_printq_cache(struct messaging_context *msg_ctx);
+
+#endif /* _RPC_SERVER_SPOOLSS_SRV_SPOOLSS_NT_H_ */
diff --git a/source3/rpc_server/spoolss/srv_spoolss_util.c b/source3/rpc_server/spoolss/srv_spoolss_util.c
index 9e9e253dd71..5201571a372 100644
--- a/source3/rpc_server/spoolss/srv_spoolss_util.c
+++ b/source3/rpc_server/spoolss/srv_spoolss_util.c
@@ -23,7 +23,6 @@
 #include "nt_printing.h"
 #include "srv_spoolss_util.h"
 #include "../librpc/gen_ndr/ndr_spoolss.h"
-#include "../librpc/gen_ndr/srv_winreg.h"
 #include "../librpc/gen_ndr/ndr_winreg_c.h"
 #include "../librpc/gen_ndr/ndr_security.h"
 #include "secrets.h"
@@ -450,7 +449,7 @@ static WERROR winreg_printer_enumvalues(TALLOC_CTX *mem_ctx,
 		return WERR_OK;
 	}
 
-	enum_values = TALLOC_ARRAY(tmp_ctx, struct spoolss_PrinterEnumValues, num_values);
+	enum_values = talloc_array(tmp_ctx, struct spoolss_PrinterEnumValues, num_values);
 	if (enum_values == NULL) {
 		result = WERR_NOMEM;
 		goto error;
@@ -472,7 +471,7 @@ static WERROR winreg_printer_enumvalues(TALLOC_CTX *mem_ctx,
 		data_size = max_valbufsize;
 		data = NULL;
 		if (data_size) {
-			data = (uint8_t *) TALLOC(tmp_ctx, data_size);
+			data = (uint8_t *) talloc_zero_size(tmp_ctx, data_size);
 		}
 		length = 0;
 
@@ -1048,7 +1047,7 @@ WERROR winreg_create_printer(TALLOC_CTX *mem_ctx,
 						      winreg_handle,
 						      &key_hnd,
 						      SPOOL_REG_SHORTSERVERNAME,
-						      global_myname(),
+						      lp_netbios_name(),
 						      &result);
 			if (!NT_STATUS_IS_OK(status)) {
 				result = ntstatus_to_werror(status);
@@ -1063,9 +1062,9 @@ WERROR winreg_create_printer(TALLOC_CTX *mem_ctx,
 			 */
 			dnssuffix = get_mydnsdomname(tmp_ctx);
 			if (dnssuffix != NULL && dnssuffix[0] != '\0') {
-				longname = talloc_asprintf(tmp_ctx, "%s.%s", global_myname(), dnssuffix);
+				longname = talloc_asprintf(tmp_ctx, "%s.%s", lp_netbios_name(), dnssuffix);
 			} else {
-				longname = talloc_strdup(tmp_ctx, global_myname());
+				longname = talloc_strdup(tmp_ctx, lp_netbios_name());
 			}
 			if (longname == NULL) {
 				result = WERR_NOMEM;
@@ -2316,9 +2315,9 @@ WERROR winreg_get_printer_dataex(TALLOC_CTX *mem_ctx,
 	struct dcerpc_binding_handle *winreg_handle = NULL;
 	struct policy_handle hive_hnd, key_hnd;
 	struct winreg_String wvalue;
-	enum winreg_Type type_in;
+	enum winreg_Type type_in = REG_NONE;
 	char *path;
-	uint8_t *data_in;
+	uint8_t *data_in = NULL;
 	uint32_t data_in_size = 0;
 	uint32_t value_len = 0;
 	WERROR result = WERR_OK;
@@ -3085,7 +3084,7 @@ WERROR winreg_printer_enumforms1(TALLOC_CTX *mem_ctx,
 		goto done;
 	}
 
-	info = TALLOC_ARRAY(tmp_ctx, union spoolss_FormInfo, num_builtin + num_values);
+	info = talloc_array(tmp_ctx, union spoolss_FormInfo, num_builtin + num_values);
 	if (info == NULL) {
 		result = WERR_NOMEM;
 		goto done;
@@ -3338,8 +3337,8 @@ WERROR winreg_printer_getform1(TALLOC_CTX *mem_ctx,
 	struct dcerpc_binding_handle *winreg_handle = NULL;
 	struct policy_handle hive_hnd, key_hnd;
 	struct winreg_String wvalue;
-	enum winreg_Type type_in;
-	uint8_t *data_in;
+	enum winreg_Type type_in = REG_NONE;
+	uint8_t *data_in = NULL;
 	uint32_t data_in_size = 0;
 	uint32_t value_len = 0;
 	uint32_t num_builtin = ARRAY_SIZE(builtin_forms1);
diff --git a/source3/rpc_server/srv_access_check.h b/source3/rpc_server/srv_access_check.h
index 27a09bd0074..72ce539568e 100644
--- a/source3/rpc_server/srv_access_check.h
+++ b/source3/rpc_server/srv_access_check.h
@@ -1,3 +1,35 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Copyright (C) Andrew Tridgell                   1992-1997,
+ *  Copyright (C) Luke Kenneth Casson Leighton      1996-1997,
+ *  Copyright (C) Paul Ashton                       1997,
+ *  Copyright (C) Marc Jacobsen			    1999,
+ *  Copyright (C) Jeremy Allison                    2001-2008,
+ *  Copyright (C) Jean François Micouleau           1998-2001,
+ *  Copyright (C) Jim McDonough <jmcd@us.ibm.com>   2002,
+ *  Copyright (C) Gerald (Jerry) Carter             2003-2004,
+ *  Copyright (C) Simo Sorce                        2003.
+ *  Copyright (C) Volker Lendecke		    2005.
+ *  Copyright (C) Guenther Deschner		    2008.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_SERVER_SRV_ACCESS_CHECK_H_
+#define _RPC_SERVER_SRV_ACCESS_CHECK_H_
+
 /* The following definitions come from rpc_server/srv_access_check.c */
 
 NTSTATUS access_check_object( struct security_descriptor *psd, struct security_token *token,
@@ -8,3 +40,5 @@ NTSTATUS access_check_object( struct security_descriptor *psd, struct security_t
 void map_max_allowed_access(const struct security_token *nt_token,
 			    const struct security_unix_token *unix_token,
 			    uint32_t *pacc_requested);
+
+#endif /* _RPC_SERVER_SRV_ACCESS_CHECK_H_ */
diff --git a/source3/rpc_server/srv_pipe.c b/source3/rpc_server/srv_pipe.c
index 27a7aaeaf47..fc08d56b1e9 100644
--- a/source3/rpc_server/srv_pipe.c
+++ b/source3/rpc_server/srv_pipe.c
@@ -41,6 +41,8 @@
 #include "smbd/smbd.h"
 #include "auth.h"
 #include "ntdomain.h"
+#include "rpc_server/srv_pipe.h"
+#include "rpc_server/rpc_contexts.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -330,23 +332,30 @@ bool setup_fault_pdu(struct pipes_struct *p, NTSTATUS fault_status)
 static bool check_bind_req(struct pipes_struct *p,
 			   struct ndr_syntax_id* abstract,
 			   struct ndr_syntax_id* transfer,
-			   uint32 context_id)
+			   uint32_t context_id)
 {
 	struct pipe_rpc_fns *context_fns;
+	bool ok;
 
 	DEBUG(3,("check_bind_req for %s\n",
-		 get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+		 get_pipe_name_from_syntax(talloc_tos(), abstract)));
 
 	/* we have to check all now since win2k introduced a new UUID on the lsaprpc pipe */
 	if (rpc_srv_pipe_exists_by_id(abstract) &&
 	   ndr_syntax_id_equal(transfer, &ndr_transfer_syntax)) {
-		DEBUG(3, ("check_bind_req: \\PIPE\\%s -> \\PIPE\\%s\n",
-			rpc_srv_get_pipe_cli_name(abstract),
-			rpc_srv_get_pipe_srv_name(abstract)));
+		DEBUG(3, ("check_bind_req: %s -> %s rpc service\n",
+			  rpc_srv_get_pipe_cli_name(abstract),
+			  rpc_srv_get_pipe_srv_name(abstract)));
 	} else {
 		return false;
 	}
 
+	ok = init_pipe_handles(p, abstract);
+	if (!ok) {
+		DEBUG(1, ("Failed to init pipe handles!\n"));
+		return false;
+	}
+
 	context_fns = SMB_MALLOC_P(struct pipe_rpc_fns);
 	if (context_fns == NULL) {
 		DEBUG(0,("check_bind_req: malloc() failed!\n"));
@@ -357,6 +366,7 @@ static bool check_bind_req(struct pipes_struct *p,
 	context_fns->n_cmds = rpc_srv_get_pipe_num_cmds(abstract);
 	context_fns->cmds = rpc_srv_get_pipe_cmds(abstract);
 	context_fns->context_id = context_id;
+	context_fns->syntax = *abstract;
 
 	/* add to the list of open contexts */
 
@@ -615,14 +625,12 @@ static bool pipe_ntlmssp_verify_final(TALLOC_CTX *mem_ctx,
 				struct auth_ntlmssp_state *ntlmssp_ctx,
 				enum dcerpc_AuthLevel auth_level,
 				struct client_address *client_id,
-				struct ndr_syntax_id *syntax,
 				struct auth_serversupplied_info **session_info)
 {
 	NTSTATUS status;
 	bool ret;
 
-	DEBUG(5, (__location__ ": pipe %s checking user details\n",
-		 get_pipe_name_from_syntax(talloc_tos(), syntax)));
+	DEBUG(5, (__location__ ": checking user details\n"));
 
 	/* Finally - if the pipe negotiated integrity (sign) or privacy (seal)
 	   ensure the underlying NTLMSSP flags are also set. If not we should
@@ -635,8 +643,7 @@ static bool pipe_ntlmssp_verify_final(TALLOC_CTX *mem_ctx,
 						DCERPC_AUTH_LEVEL_PRIVACY));
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0, (__location__ ": Client failed to negotatie proper "
-			  "security for pipe %s\n",
-			  get_pipe_name_from_syntax(talloc_tos(), syntax)));
+			  "security for rpc connection\n"));
 		return false;
 	}
 
@@ -767,7 +774,7 @@ static NTSTATUS pipe_auth_verify_final(struct pipes_struct *p)
 						    struct auth_ntlmssp_state);
 		if (!pipe_ntlmssp_verify_final(p, ntlmssp_ctx,
 						p->auth.auth_level,
-						p->client_id, &p->syntax,
+						p->client_id,
 						&p->session_info)) {
 			return NT_STATUS_ACCESS_DENIED;
 		}
@@ -813,7 +820,6 @@ static NTSTATUS pipe_auth_verify_final(struct pipes_struct *p)
 			if (!pipe_ntlmssp_verify_final(p, ntlmssp_ctx,
 							p->auth.auth_level,
 							p->client_id,
-							&p->syntax,
 							&p->session_info)) {
 				return NT_STATUS_ACCESS_DENIED;
 			}
@@ -854,9 +860,7 @@ static bool api_pipe_bind_req(struct pipes_struct *p,
 
 	/* No rebinds on a bound pipe - use alter context. */
 	if (p->pipe_bound) {
-		DEBUG(2,("api_pipe_bind_req: rejecting bind request on bound "
-			 "pipe %s.\n",
-			 get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+		DEBUG(2,("Rejecting bind request on bound rpc connection\n"));
 		return setup_bind_nak(p, pkt);
 	}
 
@@ -871,38 +875,35 @@ static bool api_pipe_bind_req(struct pipes_struct *p,
 	 */
 	id = pkt->u.bind.ctx_list[0].abstract_syntax;
 	if (rpc_srv_pipe_exists_by_id(&id)) {
-		DEBUG(3, ("api_pipe_bind_req: \\PIPE\\%s -> \\PIPE\\%s\n",
-			rpc_srv_get_pipe_cli_name(&id),
-			rpc_srv_get_pipe_srv_name(&id)));
+		DEBUG(3, ("api_pipe_bind_req: %s -> %s rpc service\n",
+			  rpc_srv_get_pipe_cli_name(&id),
+			  rpc_srv_get_pipe_srv_name(&id)));
 	} else {
 		status = smb_probe_module(
 			"rpc", get_pipe_name_from_syntax(
 				talloc_tos(),
-				&pkt->u.bind.ctx_list[0].abstract_syntax));
+				&id));
 
 		if (NT_STATUS_IS_ERR(status)) {
-                       DEBUG(3,("api_pipe_bind_req: Unknown pipe name %s in bind request.\n",
-                                get_pipe_name_from_syntax(
-					talloc_tos(),
-					&pkt->u.bind.ctx_list[0].abstract_syntax)));
+			DEBUG(3,("api_pipe_bind_req: Unknown rpc service name "
+                                 "%s in bind request.\n",
+				 get_pipe_name_from_syntax(talloc_tos(), &id)));
 
 			return setup_bind_nak(p, pkt);
 		}
 
 		if (rpc_srv_get_pipe_interface_by_cli_name(
 				get_pipe_name_from_syntax(talloc_tos(),
-							  &p->syntax),
+							  &id),
 				&id)) {
-			DEBUG(3, ("api_pipe_bind_req: \\PIPE\\%s -> \\PIPE\\%s\n",
-				rpc_srv_get_pipe_cli_name(&id),
-				rpc_srv_get_pipe_srv_name(&id)));
+			DEBUG(3, ("api_pipe_bind_req: %s -> %s rpc service\n",
+				  rpc_srv_get_pipe_cli_name(&id),
+				  rpc_srv_get_pipe_srv_name(&id)));
 		} else {
 			DEBUG(0, ("module %s doesn't provide functions for "
 				  "pipe %s!\n",
-				  get_pipe_name_from_syntax(talloc_tos(),
-							    &p->syntax),
-				  get_pipe_name_from_syntax(talloc_tos(),
-							    &p->syntax)));
+				  get_pipe_name_from_syntax(talloc_tos(), &id),
+				  get_pipe_name_from_syntax(talloc_tos(), &id)));
 			return setup_bind_nak(p, pkt);
 		}
 	}
@@ -1507,28 +1508,9 @@ static bool api_pipe_alter_context(struct pipes_struct *p,
 	return setup_bind_nak(p, pkt);
 }
 
-/****************************************************************************
- Find the set of RPC functions associated with this context_id
-****************************************************************************/
-
-static PIPE_RPC_FNS* find_pipe_fns_by_context( PIPE_RPC_FNS *list, uint32 context_id )
-{
-	PIPE_RPC_FNS *fns = NULL;
-
-	if ( !list ) {
-		DEBUG(0,("find_pipe_fns_by_context: ERROR!  No context list for pipe!\n"));
-		return NULL;
-	}
-
-	for (fns=list; fns; fns=fns->next ) {
-		if ( fns->context_id == context_id )
-			return fns;
-	}
-	return NULL;
-}
-
 static bool api_rpcTNP(struct pipes_struct *p, struct ncacn_packet *pkt,
-		       const struct api_struct *api_rpc_cmds, int n_cmds);
+		       const struct api_struct *api_rpc_cmds, int n_cmds,
+		       const struct ndr_syntax_id *syntax);
 
 /****************************************************************************
  Find the correct RPC function to call for this request.
@@ -1547,16 +1529,13 @@ static bool api_pipe_request(struct pipes_struct *p,
 	    ((p->auth.auth_type == DCERPC_AUTH_TYPE_NTLMSSP) ||
 	     (p->auth.auth_type == DCERPC_AUTH_TYPE_KRB5) ||
 	     (p->auth.auth_type == DCERPC_AUTH_TYPE_SPNEGO))) {
-		if(!become_authenticated_pipe_user(p)) {
+		if(!become_authenticated_pipe_user(p->session_info)) {
 			data_blob_free(&p->out_data.rdata);
 			return False;
 		}
 		changed_user = True;
 	}
 
-	DEBUG(5, ("Requested \\PIPE\\%s\n",
-		  get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
-
 	/* get the set of RPC functions for this context */
 
 	pipe_fns = find_pipe_fns_by_context(p->contexts,
@@ -1564,15 +1543,19 @@ static bool api_pipe_request(struct pipes_struct *p,
 
 	if ( pipe_fns ) {
 		TALLOC_CTX *frame = talloc_stackframe();
-		ret = api_rpcTNP(p, pkt, pipe_fns->cmds, pipe_fns->n_cmds);
+
+		DEBUG(5, ("Requested %s rpc service\n",
+			  get_pipe_name_from_syntax(talloc_tos(), &pipe_fns->syntax)));
+
+		ret = api_rpcTNP(p, pkt, pipe_fns->cmds, pipe_fns->n_cmds,
+				 &pipe_fns->syntax);
+
 		TALLOC_FREE(frame);
 	}
 	else {
 		DEBUG(0, ("No rpc function table associated with context "
-			  "[%d] on pipe [%s]\n",
-			  pkt->u.request.context_id,
-			  get_pipe_name_from_syntax(talloc_tos(),
-						    &p->syntax)));
+			  "[%d]\n",
+			  pkt->u.request.context_id));
 	}
 
 	if (changed_user) {
@@ -1587,20 +1570,21 @@ static bool api_pipe_request(struct pipes_struct *p,
  ********************************************************************/
 
 static bool api_rpcTNP(struct pipes_struct *p, struct ncacn_packet *pkt,
-		       const struct api_struct *api_rpc_cmds, int n_cmds)
+		       const struct api_struct *api_rpc_cmds, int n_cmds,
+		       const struct ndr_syntax_id *syntax)
 {
 	int fn_num;
 	uint32_t offset1;
 
 	/* interpret the command */
 	DEBUG(4,("api_rpcTNP: %s op 0x%x - ",
-		 get_pipe_name_from_syntax(talloc_tos(), &p->syntax),
+		 get_pipe_name_from_syntax(talloc_tos(), syntax),
 		 pkt->u.request.opnum));
 
 	if (DEBUGLEVEL >= 50) {
 		fstring name;
 		slprintf(name, sizeof(name)-1, "in_%s",
-			 get_pipe_name_from_syntax(talloc_tos(), &p->syntax));
+			 get_pipe_name_from_syntax(talloc_tos(), syntax));
 		dump_pdu_region(name, pkt->u.request.opnum,
 				&p->in_data.data, 0,
 				p->in_data.data.length);
@@ -1633,7 +1617,7 @@ static bool api_rpcTNP(struct pipes_struct *p, struct ncacn_packet *pkt,
 	/* do the actual command */
 	if(!api_rpc_cmds[fn_num].fn(p)) {
 		DEBUG(0,("api_rpcTNP: %s: %s failed.\n",
-			 get_pipe_name_from_syntax(talloc_tos(), &p->syntax),
+			 get_pipe_name_from_syntax(talloc_tos(), syntax),
 			 api_rpc_cmds[fn_num].name));
 		data_blob_free(&p->out_data.rdata);
 		return False;
@@ -1656,14 +1640,14 @@ static bool api_rpcTNP(struct pipes_struct *p, struct ncacn_packet *pkt,
 	if (DEBUGLEVEL >= 50) {
 		fstring name;
 		slprintf(name, sizeof(name)-1, "out_%s",
-			 get_pipe_name_from_syntax(talloc_tos(), &p->syntax));
+			 get_pipe_name_from_syntax(talloc_tos(), syntax));
 		dump_pdu_region(name, pkt->u.request.opnum,
 				&p->out_data.rdata, offset1,
 				p->out_data.rdata.length);
 	}
 
 	DEBUG(5,("api_rpcTNP: called %s successfully\n",
-		 get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+		 get_pipe_name_from_syntax(talloc_tos(), syntax)));
 
 	/* Check for buffer underflow in rpc parsing */
 	if ((DEBUGLEVEL >= 10) &&
@@ -1706,8 +1690,8 @@ void set_incoming_fault(struct pipes_struct *p)
 	p->in_data.pdu_needed_len = 0;
 	p->in_data.pdu.length = 0;
 	p->fault_state = True;
-	DEBUG(10, ("set_incoming_fault: Setting fault state on pipe %s\n",
-		   get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+
+	DEBUG(10, ("Setting fault state\n"));
 }
 
 static NTSTATUS dcesrv_auth_request(struct pipe_auth_data *auth,
@@ -1841,8 +1825,7 @@ void process_complete_pdu(struct pipes_struct *p)
 	bool reply = False;
 
 	if(p->fault_state) {
-		DEBUG(10,("process_complete_pdu: pipe %s in fault state.\n",
-			  get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+		DEBUG(10,("RPC connection in fault state.\n"));
 		goto done;
 	}
 
@@ -1874,7 +1857,7 @@ void process_complete_pdu(struct pipes_struct *p)
 	/* Store the call_id */
 	p->call_id = pkt->call_id;
 
-	DEBUG(10, ("Processing packet type %d\n", (int)pkt->ptype));
+	DEBUG(10, ("Processing packet type %u\n", (unsigned int)pkt->ptype));
 
 	switch (pkt->ptype) {
 	case DCERPC_PKT_REQUEST:
@@ -1882,19 +1865,12 @@ void process_complete_pdu(struct pipes_struct *p)
 		break;
 
 	case DCERPC_PKT_PING: /* CL request - ignore... */
-		DEBUG(0, ("process_complete_pdu: Error. "
-			  "Connectionless packet type %d received on "
-			  "pipe %s.\n", (int)pkt->ptype,
-			 get_pipe_name_from_syntax(talloc_tos(),
-						   &p->syntax)));
+		DEBUG(0, ("Error - Connectionless packet type %u received\n",
+			  (unsigned int)pkt->ptype));
 		break;
 
 	case DCERPC_PKT_RESPONSE: /* No responses here. */
-		DEBUG(0, ("process_complete_pdu: Error. "
-			  "DCERPC_PKT_RESPONSE received from client "
-			  "on pipe %s.\n",
-			 get_pipe_name_from_syntax(talloc_tos(),
-						   &p->syntax)));
+		DEBUG(0, ("Error - DCERPC_PKT_RESPONSE received from client"));
 		break;
 
 	case DCERPC_PKT_FAULT:
@@ -1907,11 +1883,8 @@ void process_complete_pdu(struct pipes_struct *p)
 	case DCERPC_PKT_CL_CANCEL:
 	case DCERPC_PKT_FACK:
 	case DCERPC_PKT_CANCEL_ACK:
-		DEBUG(0, ("process_complete_pdu: Error. "
-			  "Connectionless packet type %u received on "
-			  "pipe %s.\n", (unsigned int)pkt->ptype,
-			 get_pipe_name_from_syntax(talloc_tos(),
-						   &p->syntax)));
+		DEBUG(0, ("Error - Connectionless packet type %u received\n",
+			  (unsigned int)pkt->ptype));
 		break;
 
 	case DCERPC_PKT_BIND:
@@ -1925,12 +1898,9 @@ void process_complete_pdu(struct pipes_struct *p)
 
 	case DCERPC_PKT_BIND_ACK:
 	case DCERPC_PKT_BIND_NAK:
-		DEBUG(0, ("process_complete_pdu: Error. "
-			  "DCERPC_PKT_BINDACK/DCERPC_PKT_BINDNACK "
-			  "packet type %u received on pipe %s.\n",
-			  (unsigned int)pkt->ptype,
-			 get_pipe_name_from_syntax(talloc_tos(),
-						   &p->syntax)));
+		DEBUG(0, ("Error - DCERPC_PKT_BINDACK/DCERPC_PKT_BINDNACK "
+			  "packet type %u received.\n",
+			  (unsigned int)pkt->ptype));
 		break;
 
 
@@ -1944,11 +1914,8 @@ void process_complete_pdu(struct pipes_struct *p)
 		break;
 
 	case DCERPC_PKT_ALTER_RESP:
-		DEBUG(0, ("process_complete_pdu: Error. "
-			  "DCERPC_PKT_ALTER_RESP on pipe %s: "
-			  "Should only be server -> client.\n",
-			 get_pipe_name_from_syntax(talloc_tos(),
-						   &p->syntax)));
+		DEBUG(0, ("Error - DCERPC_PKT_ALTER_RESP received: "
+			  "Should only be server -> client.\n"));
 		break;
 
 	case DCERPC_PKT_AUTH3:
@@ -1961,11 +1928,8 @@ void process_complete_pdu(struct pipes_struct *p)
 		break;
 
 	case DCERPC_PKT_SHUTDOWN:
-		DEBUG(0, ("process_complete_pdu: Error. "
-			  "DCERPC_PKT_SHUTDOWN on pipe %s: "
-			  "Should only be server -> client.\n",
-			 get_pipe_name_from_syntax(talloc_tos(),
-						   &p->syntax)));
+		DEBUG(0, ("Error - DCERPC_PKT_SHUTDOWN received: "
+			  "Should only be server -> client.\n"));
 		break;
 
 	case DCERPC_PKT_CO_CANCEL:
@@ -2010,9 +1974,7 @@ void process_complete_pdu(struct pipes_struct *p)
 
 done:
 	if (!reply) {
-		DEBUG(3,("process_complete_pdu: DCE/RPC fault sent on "
-			 "pipe %s\n", get_pipe_name_from_syntax(talloc_tos(),
-								&p->syntax)));
+		DEBUG(3,("DCE/RPC fault sent!"));
 		set_incoming_fault(p);
 		setup_fault_pdu(p, NT_STATUS(DCERPC_FAULT_OP_RNG_ERROR));
 		TALLOC_FREE(pkt);
diff --git a/source3/rpc_server/srv_pipe.h b/source3/rpc_server/srv_pipe.h
new file mode 100644
index 00000000000..453cca18d91
--- /dev/null
+++ b/source3/rpc_server/srv_pipe.h
@@ -0,0 +1,33 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Almost completely rewritten by (C) Jeremy Allison 2005 - 2010
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_SERVER_SRV_PIPE_H_
+#define _RPC_SERVER_SRV_PIPE_H_
+
+struct ncacn_packet;
+struct pipes_struct;
+
+/* The following definitions come from rpc_server/srv_pipe.c  */
+
+bool create_next_pdu(struct pipes_struct *p);
+bool api_pipe_bind_auth3(struct pipes_struct *p, struct ncacn_packet *pkt);
+bool setup_fault_pdu(struct pipes_struct *p, NTSTATUS status);
+bool is_known_pipename(const char *cli_filename, struct ndr_syntax_id *syntax);
+
+#endif /* _RPC_SERVER_SRV_PIPE_H_ */
diff --git a/source3/rpc_server/srv_pipe_hnd.c b/source3/rpc_server/srv_pipe_hnd.c
index 52525987269..501bb1efc77 100644
--- a/source3/rpc_server/srv_pipe_hnd.c
+++ b/source3/rpc_server/srv_pipe_hnd.c
@@ -20,15 +20,15 @@
  */
 
 #include "includes.h"
-#include "../librpc/gen_ndr/srv_spoolss.h"
-#include "librpc/gen_ndr/ndr_named_pipe_auth.h"
-#include "../libcli/named_pipe_auth/npa_tstream.h"
 #include "rpc_server.h"
-#include "smbd/globals.h"
 #include "fake_file.h"
 #include "rpc_dce.h"
-#include "rpc_server/rpc_ncacn_np.h"
 #include "ntdomain.h"
+#include "rpc_server/rpc_ncacn_np.h"
+#include "rpc_server/srv_pipe_hnd.h"
+#include "rpc_server/srv_pipe.h"
+#include "../lib/tsocket/tsocket.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -37,7 +37,7 @@
  Ensures we have at least RPC_HEADER_LEN amount of data in the incoming buffer.
 ****************************************************************************/
 
-static ssize_t fill_rpc_header(struct pipes_struct *p, char *data, size_t data_to_copy)
+static ssize_t fill_rpc_header(struct pipes_struct *p, const char *data, size_t data_to_copy)
 {
 	size_t len_needed_to_complete_hdr =
 		MIN(data_to_copy, RPC_HEADER_LEN - p->in_data.pdu.length);
@@ -126,7 +126,7 @@ static void free_pipe_context(struct pipes_struct *p)
  Accepts incoming data on an rpc pipe. Processes the data in pdu sized units.
 ****************************************************************************/
 
-ssize_t process_incoming_data(struct pipes_struct *p, char *data, size_t n)
+ssize_t process_incoming_data(struct pipes_struct *p, const char *data, size_t n)
 {
 	size_t data_to_copy = MIN(n, RPC_MAX_PDU_FRAG_LEN
 					- p->in_data.pdu.length);
@@ -231,7 +231,7 @@ ssize_t process_incoming_data(struct pipes_struct *p, char *data, size_t n)
  Accepts incoming data on an internal rpc pipe.
 ****************************************************************************/
 
-static ssize_t write_to_internal_pipe(struct pipes_struct *p, char *data, size_t n)
+static ssize_t write_to_internal_pipe(struct pipes_struct *p, const char *data, size_t n)
 {
 	size_t data_left = n;
 
@@ -280,7 +280,7 @@ static ssize_t read_from_internal_pipe(struct pipes_struct *p, char *data,
 	}
 
 	DEBUG(6,(" name: %s len: %u\n",
-		 get_pipe_name_from_syntax(talloc_tos(), &p->syntax),
+		 get_pipe_name_from_syntax(talloc_tos(), &p->contexts->syntax),
 		 (unsigned int)n));
 
 	/*
@@ -298,7 +298,7 @@ static ssize_t read_from_internal_pipe(struct pipes_struct *p, char *data,
                 DEBUG(5,("read_from_pipe: too large read (%u) requested on "
 			 "pipe %s. We can only service %d sized reads.\n",
 			 (unsigned int)n,
-			 get_pipe_name_from_syntax(talloc_tos(), &p->syntax),
+			 get_pipe_name_from_syntax(talloc_tos(), &p->contexts->syntax),
 			 RPC_MAX_PDU_FRAG_LEN ));
 		n = RPC_MAX_PDU_FRAG_LEN;
 	}
@@ -319,7 +319,7 @@ static ssize_t read_from_internal_pipe(struct pipes_struct *p, char *data,
 
 		DEBUG(10,("read_from_pipe: %s: current_pdu_len = %u, "
 			  "current_pdu_sent = %u returning %d bytes.\n",
-			  get_pipe_name_from_syntax(talloc_tos(), &p->syntax),
+			  get_pipe_name_from_syntax(talloc_tos(), &p->contexts->syntax),
 			  (unsigned int)p->out_data.frag.length,
 			  (unsigned int)p->out_data.current_pdu_sent,
 			  (int)data_returned));
@@ -340,7 +340,7 @@ static ssize_t read_from_internal_pipe(struct pipes_struct *p, char *data,
 
 	DEBUG(10,("read_from_pipe: %s: fault_state = %d : data_sent_length "
 		  "= %u, p->out_data.rdata.length = %u.\n",
-		  get_pipe_name_from_syntax(talloc_tos(), &p->syntax),
+		  get_pipe_name_from_syntax(talloc_tos(), &p->contexts->syntax),
 		  (int)p->fault_state,
 		  (unsigned int)p->out_data.data_sent_length,
 		  (unsigned int)p->out_data.rdata.length));
@@ -362,7 +362,7 @@ static ssize_t read_from_internal_pipe(struct pipes_struct *p, char *data,
 
 	if(!create_next_pdu(p)) {
 		DEBUG(0,("read_from_pipe: %s: create_next_pdu failed.\n",
-			 get_pipe_name_from_syntax(talloc_tos(), &p->syntax)));
+			 get_pipe_name_from_syntax(talloc_tos(), &p->contexts->syntax)));
 		return -1;
 	}
 
@@ -433,7 +433,7 @@ NTSTATUS np_open(TALLOC_CTX *mem_ctx, const char *name,
 	rpcsrv_type = lp_parm_const_string(GLOBAL_SECTION_SNUM,
 					   "rpc_server", name,
 					   "embedded");
-	if (StrCaseCmp(rpcsrv_type, "embedded") != 0) {
+	if (strcasecmp_m(rpcsrv_type, "embedded") != 0) {
 		external = true;
 	}
 
@@ -535,7 +535,7 @@ struct tevent_req *np_write_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
 		struct pipes_struct *p = talloc_get_type_abort(
 			handle->private_data, struct pipes_struct);
 
-		state->nwritten = write_to_internal_pipe(p, (char *)data, len);
+		state->nwritten = write_to_internal_pipe(p, (const char *)data, len);
 
 		status = (state->nwritten >= 0)
 			? NT_STATUS_OK : NT_STATUS_UNEXPECTED_IO_ERROR;
@@ -549,7 +549,7 @@ struct tevent_req *np_write_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
 
 		state->ev = ev;
 		state->p = p;
-		state->iov.iov_base = CONST_DISCARD(void *, data);
+		state->iov.iov_base = discard_const_p(void, data);
 		state->iov.iov_len = len;
 
 		subreq = tstream_writev_queue_send(state, ev,
diff --git a/source3/rpc_server/srv_pipe_hnd.h b/source3/rpc_server/srv_pipe_hnd.h
new file mode 100644
index 00000000000..680add469b0
--- /dev/null
+++ b/source3/rpc_server/srv_pipe_hnd.h
@@ -0,0 +1,51 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Copyright (C) Andrew Tridgell              1992-1998,
+ *  Largely re-written : 2005
+ *  Copyright (C) Jeremy Allison		1998 - 2005
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_SERVER_SRV_PIPE_HND_H_
+#define _RPC_SERVER_SRV_PIPE_HND_H_
+
+struct tsocket_address;
+struct pipes_struct;
+
+/* The following definitions come from rpc_server/srv_pipe_hnd.c  */
+
+bool fsp_is_np(struct files_struct *fsp);
+NTSTATUS np_open(TALLOC_CTX *mem_ctx, const char *name,
+		 const struct tsocket_address *local_address,
+		 const struct tsocket_address *remote_address,
+		 struct client_address *client_id,
+		 struct auth_serversupplied_info *session_info,
+		 struct messaging_context *msg_ctx,
+		 struct fake_file_handle **phandle);
+bool np_read_in_progress(struct fake_file_handle *handle);
+struct tevent_req *np_write_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
+				 struct fake_file_handle *handle,
+				 const uint8_t *data, size_t len);
+NTSTATUS np_write_recv(struct tevent_req *req, ssize_t *pnwritten);
+struct tevent_req *np_read_send(TALLOC_CTX *mem_ctx, struct event_context *ev,
+				struct fake_file_handle *handle,
+				uint8_t *data, size_t len);
+NTSTATUS np_read_recv(struct tevent_req *req, ssize_t *nread,
+		      bool *is_data_outstanding);
+
+ssize_t process_incoming_data(struct pipes_struct *p, const char *data, size_t n);
+
+#endif /* _RPC_SERVER_SRV_PIPE_HND_H_ */
diff --git a/source3/rpc_server/srv_pipe_register.c b/source3/rpc_server/srv_pipe_register.c
index a6d654277ed..c3500c97be6 100644
--- a/source3/rpc_server/srv_pipe_register.c
+++ b/source3/rpc_server/srv_pipe_register.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "librpc/rpc/dcerpc.h"
 #include "srv_pipe_internal.h"
+#include "rpc_server/srv_pipe_register.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -197,7 +198,7 @@ NTSTATUS rpc_srv_register(int version, const char *clnt, const char *srv,
 	rpc_entry = SMB_REALLOC_ARRAY_KEEP_OLD_ON_ERROR(rpc_lookup, struct rpc_table, rpc_lookup_size);
 	if (NULL == rpc_entry) {
 		rpc_lookup_size--;
-		DEBUG(0, ("rpc_pipe_register_commands: memory allocation failed\n"));
+		DEBUG(0, ("rpc_srv_register: memory allocation failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	} else {
 		rpc_lookup = rpc_entry;
diff --git a/source3/rpc_server/srv_pipe_register.h b/source3/rpc_server/srv_pipe_register.h
new file mode 100644
index 00000000000..b1b454e0bd5
--- /dev/null
+++ b/source3/rpc_server/srv_pipe_register.h
@@ -0,0 +1,39 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *  Almost completely rewritten by (C) Jeremy Allison 2005 - 2010
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_SERVER_SRV_PIPE_REGISTER_H_
+#define _RPC_SERVER_SRV_PIPE_REGISTER_H_
+
+struct rpc_srv_callbacks {
+	bool (*init)(void *private_data);
+	bool (*shutdown)(void *private_data);
+	void *private_data;
+};
+
+/* The following definitions come from rpc_server/srv_rpc_register.c  */
+
+NTSTATUS rpc_srv_register(int version, const char *clnt,
+			  const char *srv,
+			  const struct ndr_interface_table *iface,
+			  const struct api_struct *cmds, int size,
+			  const struct rpc_srv_callbacks *rpc_srv_cb);
+
+NTSTATUS rpc_srv_unregister(const struct ndr_interface_table *iface);
+
+#endif /* _RPC_SERVER_SRV_PIPE_REGISTER_H_ */
diff --git a/source3/rpc_server/srvsvc/srv_srvsvc_nt.c b/source3/rpc_server/srvsvc/srv_srvsvc_nt.c
index e51fee89c61..752b8576676 100644
--- a/source3/rpc_server/srvsvc/srv_srvsvc_nt.c
+++ b/source3/rpc_server/srvsvc/srv_srvsvc_nt.c
@@ -25,6 +25,7 @@
 
 #include "includes.h"
 #include "system/passwd.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_srvsvc.h"
 #include "../libcli/security/security.h"
 #include "../librpc/gen_ndr/ndr_security.h"
@@ -32,9 +33,9 @@
 #include "session.h"
 #include "../lib/util/util_pw.h"
 #include "smbd/smbd.h"
+#include "smbd/globals.h"
 #include "auth.h"
 #include "messages.h"
-#include "ntdomain.h"
 
 extern const struct generic_mapping file_generic_mapping;
 
@@ -100,7 +101,7 @@ static int pipe_enum_fn( struct db_record *rec, void *p)
 		return 1;
 	}
 
-	f = TALLOC_REALLOC_ARRAY(fenum->ctx, fenum->ctr3->array,
+	f = talloc_realloc(fenum->ctx, fenum->ctr3->array,
 				 struct srvsvc_NetFileInfo3, i+1);
 	if ( !f ) {
 		DEBUG(0,("conn_enum_fn: realloc failed for %d items\n", i+1));
@@ -135,7 +136,7 @@ static WERROR net_enum_pipes(TALLOC_CTX *ctx,
 	fenum.username = username;
 	fenum.ctr3 = *ctr3;
 
-	if (connections_traverse(pipe_enum_fn, &fenum) == -1) {
+	if (connections_traverse(pipe_enum_fn, &fenum) < 0) {
 		DEBUG(0,("net_enum_pipes: traverse of connections.tdb "
 			 "failed\n"));
 		return WERR_NOMEM;
@@ -178,7 +179,7 @@ static void enum_file_fn( const struct share_mode_entry *e,
 		return;
 	}
 
-	f = TALLOC_REALLOC_ARRAY(fenum->ctx, fenum->ctr3->array,
+	f = talloc_realloc(fenum->ctx, fenum->ctr3->array,
 				 struct srvsvc_NetFileInfo3, i+1);
 	if ( !f ) {
 		DEBUG(0,("conn_enum_fn: realloc failed for %d items\n", i+1));
@@ -568,12 +569,12 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 
 	/* Ensure all the usershares are loaded. */
 	become_root();
-	load_usershare_shares();
+	load_usershare_shares(msg_ctx_to_sconn(p->msg_ctx));
 	load_registry_shares();
 	num_services = lp_numservices();
 	unbecome_root();
 
-        allowed = TALLOC_ZERO_ARRAY(ctx, bool, num_services);
+        allowed = talloc_zero_array(ctx, bool, num_services);
         W_ERROR_HAVE_NO_MEMORY(allowed);
 
         /* Count the number of entries. */
@@ -599,11 +600,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 	alloc_entries = num_entries - resume_handle;
 	switch (info_ctr->level) {
 	case 0:
-		ctr.ctr0 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr0);
+		ctr.ctr0 = talloc_zero(ctx, struct srvsvc_NetShareCtr0);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr0);
 
 		ctr.ctr0->count = alloc_entries;
-		ctr.ctr0->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo0, alloc_entries);
+		ctr.ctr0->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo0, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr0->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -616,11 +617,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 1:
-		ctr.ctr1 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr1);
+		ctr.ctr1 = talloc_zero(ctx, struct srvsvc_NetShareCtr1);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1);
 
 		ctr.ctr1->count = alloc_entries;
-		ctr.ctr1->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo1, alloc_entries);
+		ctr.ctr1->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo1, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -633,11 +634,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 2:
-		ctr.ctr2 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr2);
+		ctr.ctr2 = talloc_zero(ctx, struct srvsvc_NetShareCtr2);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr2);
 
 		ctr.ctr2->count = alloc_entries;
-		ctr.ctr2->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo2, alloc_entries);
+		ctr.ctr2->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo2, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr2->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -650,11 +651,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 501:
-		ctr.ctr501 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr501);
+		ctr.ctr501 = talloc_zero(ctx, struct srvsvc_NetShareCtr501);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr501);
 
 		ctr.ctr501->count = alloc_entries;
-		ctr.ctr501->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo501, alloc_entries);
+		ctr.ctr501->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo501, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr501->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -667,11 +668,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 502:
-		ctr.ctr502 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr502);
+		ctr.ctr502 = talloc_zero(ctx, struct srvsvc_NetShareCtr502);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr502);
 
 		ctr.ctr502->count = alloc_entries;
-		ctr.ctr502->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo502, alloc_entries);
+		ctr.ctr502->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo502, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr502->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -684,11 +685,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 1004:
-		ctr.ctr1004 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr1004);
+		ctr.ctr1004 = talloc_zero(ctx, struct srvsvc_NetShareCtr1004);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1004);
 
 		ctr.ctr1004->count = alloc_entries;
-		ctr.ctr1004->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo1004, alloc_entries);
+		ctr.ctr1004->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo1004, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1004->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -701,11 +702,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 1005:
-		ctr.ctr1005 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr1005);
+		ctr.ctr1005 = talloc_zero(ctx, struct srvsvc_NetShareCtr1005);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1005);
 
 		ctr.ctr1005->count = alloc_entries;
-		ctr.ctr1005->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo1005, alloc_entries);
+		ctr.ctr1005->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo1005, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1005->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -718,11 +719,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 1006:
-		ctr.ctr1006 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr1006);
+		ctr.ctr1006 = talloc_zero(ctx, struct srvsvc_NetShareCtr1006);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1006);
 
 		ctr.ctr1006->count = alloc_entries;
-		ctr.ctr1006->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo1006, alloc_entries);
+		ctr.ctr1006->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo1006, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1006->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -735,11 +736,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 1007:
-		ctr.ctr1007 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr1007);
+		ctr.ctr1007 = talloc_zero(ctx, struct srvsvc_NetShareCtr1007);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1007);
 
 		ctr.ctr1007->count = alloc_entries;
-		ctr.ctr1007->array = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetShareInfo1007, alloc_entries);
+		ctr.ctr1007->array = talloc_zero_array(ctx, struct srvsvc_NetShareInfo1007, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1007->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -752,11 +753,11 @@ static WERROR init_srv_share_info_ctr(struct pipes_struct *p,
 		break;
 
 	case 1501:
-		ctr.ctr1501 = TALLOC_ZERO_P(ctx, struct srvsvc_NetShareCtr1501);
+		ctr.ctr1501 = talloc_zero(ctx, struct srvsvc_NetShareCtr1501);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1501);
 
 		ctr.ctr1501->count = alloc_entries;
-		ctr.ctr1501->array = TALLOC_ZERO_ARRAY(ctx, struct sec_desc_buf, alloc_entries);
+		ctr.ctr1501->array = talloc_zero_array(ctx, struct sec_desc_buf, alloc_entries);
 		W_ERROR_HAVE_NO_MEMORY(ctr.ctr1501->array);
 
 		for (snum = 0; snum < num_services; snum++) {
@@ -815,7 +816,7 @@ static WERROR init_srv_sess_info_0(struct pipes_struct *p,
 
 	for (; resume_handle < *total_entries; resume_handle++) {
 
-		ctr0->array = TALLOC_REALLOC_ARRAY(p->mem_ctx,
+		ctr0->array = talloc_realloc(p->mem_ctx,
 						   ctr0->array,
 						   struct srvsvc_NetSessInfo0,
 						   num_entries+1);
@@ -913,7 +914,7 @@ static WERROR init_srv_sess_info_1(struct pipes_struct *p,
 		num_files = net_count_files(pw->pw_uid, session_list[resume_handle].pid);
 		guest = strequal( session_list[resume_handle].username, lp_guestaccount() );
 
-		ctr1->array = TALLOC_REALLOC_ARRAY(p->mem_ctx,
+		ctr1->array = talloc_realloc(p->mem_ctx,
 						   ctr1->array,
 						   struct srvsvc_NetSessInfo1,
 						   num_entries+1);
@@ -968,7 +969,7 @@ static WERROR init_srv_conn_info_0(struct srvsvc_NetConnCtr0 *ctr0,
 
 	for (; resume_handle < *total_entries; resume_handle++) {
 
-		ctr0->array = TALLOC_REALLOC_ARRAY(talloc_tos(),
+		ctr0->array = talloc_realloc(talloc_tos(),
 						   ctr0->array,
 						   struct srvsvc_NetConnInfo0,
 						   num_entries+1);
@@ -1022,7 +1023,7 @@ static WERROR init_srv_conn_info_1(struct srvsvc_NetConnCtr1 *ctr1,
 
 	for (; resume_handle < *total_entries; resume_handle++) {
 
-		ctr1->array = TALLOC_REALLOC_ARRAY(talloc_tos(),
+		ctr1->array = talloc_realloc(talloc_tos(),
 						   ctr1->array,
 						   struct srvsvc_NetConnInfo1,
 						   num_entries+1);
@@ -1138,15 +1139,15 @@ WERROR _srvsvc_NetSrvGetInfo(struct pipes_struct *p,
 	case 102: {
 		struct srvsvc_NetSrvInfo102 *info102;
 
-		info102 = TALLOC_P(p->mem_ctx, struct srvsvc_NetSrvInfo102);
+		info102 = talloc(p->mem_ctx, struct srvsvc_NetSrvInfo102);
 		if (!info102) {
 			return WERR_NOMEM;
 		}
 
 		info102->platform_id	= PLATFORM_ID_NT;
-		info102->server_name	= global_myname();
-		info102->version_major	= lp_major_announce_version();
-		info102->version_minor	= lp_minor_announce_version();
+		info102->server_name	= lp_netbios_name();
+		info102->version_major	= SAMBA_MAJOR_NBT_ANNOUNCE_VERSION;
+		info102->version_minor	= SAMBA_MINOR_NBT_ANNOUNCE_VERSION;
 		info102->server_type	= lp_default_server_announce();
 		info102->comment	= string_truncate(lp_serverstring(),
 						MAX_SERVER_STRING_LENGTH);
@@ -1164,15 +1165,15 @@ WERROR _srvsvc_NetSrvGetInfo(struct pipes_struct *p,
 	case 101: {
 		struct srvsvc_NetSrvInfo101 *info101;
 
-		info101 = TALLOC_P(p->mem_ctx, struct srvsvc_NetSrvInfo101);
+		info101 = talloc(p->mem_ctx, struct srvsvc_NetSrvInfo101);
 		if (!info101) {
 			return WERR_NOMEM;
 		}
 
 		info101->platform_id	= PLATFORM_ID_NT;
-		info101->server_name	= global_myname();
-		info101->version_major	= lp_major_announce_version();
-		info101->version_minor	= lp_minor_announce_version();
+		info101->server_name	= lp_netbios_name();
+		info101->version_major	= SAMBA_MAJOR_NBT_ANNOUNCE_VERSION;
+		info101->version_minor	= SAMBA_MINOR_NBT_ANNOUNCE_VERSION;
 		info101->server_type	= lp_default_server_announce();
 		info101->comment	= string_truncate(lp_serverstring(),
 						MAX_SERVER_STRING_LENGTH);
@@ -1183,13 +1184,13 @@ WERROR _srvsvc_NetSrvGetInfo(struct pipes_struct *p,
 	case 100: {
 		struct srvsvc_NetSrvInfo100 *info100;
 
-		info100 = TALLOC_P(p->mem_ctx, struct srvsvc_NetSrvInfo100);
+		info100 = talloc(p->mem_ctx, struct srvsvc_NetSrvInfo100);
 		if (!info100) {
 			return WERR_NOMEM;
 		}
 
 		info100->platform_id	= PLATFORM_ID_NT;
-		info100->server_name	= global_myname();
+		info100->server_name	= lp_netbios_name();
 
 		r->out.info->info100 = info100;
 
@@ -1453,47 +1454,47 @@ WERROR _srvsvc_NetShareGetInfo(struct pipes_struct *p,
 
 	switch (r->in.level) {
 		case 0:
-			info->info0 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo0);
+			info->info0 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo0);
 			W_ERROR_HAVE_NO_MEMORY(info->info0);
 			init_srv_share_info_0(p, info->info0, snum);
 			break;
 		case 1:
-			info->info1 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo1);
+			info->info1 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo1);
 			W_ERROR_HAVE_NO_MEMORY(info->info1);
 			init_srv_share_info_1(p, info->info1, snum);
 			break;
 		case 2:
-			info->info2 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo2);
+			info->info2 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo2);
 			W_ERROR_HAVE_NO_MEMORY(info->info2);
 			init_srv_share_info_2(p, info->info2, snum);
 			break;
 		case 501:
-			info->info501 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo501);
+			info->info501 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo501);
 			W_ERROR_HAVE_NO_MEMORY(info->info501);
 			init_srv_share_info_501(p, info->info501, snum);
 			break;
 		case 502:
-			info->info502 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo502);
+			info->info502 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo502);
 			W_ERROR_HAVE_NO_MEMORY(info->info502);
 			init_srv_share_info_502(p, info->info502, snum);
 			break;
 		case 1004:
-			info->info1004 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo1004);
+			info->info1004 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo1004);
 			W_ERROR_HAVE_NO_MEMORY(info->info1004);
 			init_srv_share_info_1004(p, info->info1004, snum);
 			break;
 		case 1005:
-			info->info1005 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo1005);
+			info->info1005 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo1005);
 			W_ERROR_HAVE_NO_MEMORY(info->info1005);
 			init_srv_share_info_1005(p, info->info1005, snum);
 			break;
 		case 1006:
-			info->info1006 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo1006);
+			info->info1006 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo1006);
 			W_ERROR_HAVE_NO_MEMORY(info->info1006);
 			init_srv_share_info_1006(p, info->info1006, snum);
 			break;
 		case 1007:
-			info->info1007 = TALLOC_P(p->mem_ctx, struct srvsvc_NetShareInfo1007);
+			info->info1007 = talloc(p->mem_ctx, struct srvsvc_NetShareInfo1007);
 			W_ERROR_HAVE_NO_MEMORY(info->info1007);
 			init_srv_share_info_1007(p, info->info1007, snum);
 			break;
@@ -1513,40 +1514,6 @@ WERROR _srvsvc_NetShareGetInfo(struct pipes_struct *p,
 }
 
 /*******************************************************************
- Check a given DOS pathname is valid for a share.
-********************************************************************/
-
-char *valid_share_pathname(TALLOC_CTX *ctx, const char *dos_pathname)
-{
-	char *ptr = NULL;
-
-	if (!dos_pathname) {
-		return NULL;
-	}
-
-	ptr = talloc_strdup(ctx, dos_pathname);
-	if (!ptr) {
-		return NULL;
-	}
-	/* Convert any '\' paths to '/' */
-	unix_format(ptr);
-	ptr = unix_clean_name(ctx, ptr);
-	if (!ptr) {
-		return NULL;
-	}
-
-	/* NT is braindead - it wants a C: prefix to a pathname ! So strip it. */
-	if (strlen(ptr) > 2 && ptr[1] == ':' && ptr[0] != '/')
-		ptr += 2;
-
-	/* Only absolute paths allowed. */
-	if (*ptr != '/')
-		return NULL;
-
-	return ptr;
-}
-
-/*******************************************************************
  _srvsvc_NetShareSetInfo. Modify share details.
 ********************************************************************/
 
@@ -2095,7 +2062,7 @@ WERROR _srvsvc_NetRemoteTOD(struct pipes_struct *p,
 
 	DEBUG(5,("_srvsvc_NetRemoteTOD: %d\n", __LINE__));
 
-	if ( !(tod = TALLOC_ZERO_P(p->mem_ctx, struct srvsvc_NetRemoteTODInfo)) )
+	if ( !(tod = talloc_zero(p->mem_ctx, struct srvsvc_NetRemoteTODInfo)) )
 		return WERR_NOMEM;
 
 	*r->out.info = tod;
@@ -2222,7 +2189,7 @@ WERROR _srvsvc_NetGetFileSecurity(struct pipes_struct *p,
 
 	sd_size = ndr_size_security_descriptor(psd, 0);
 
-	sd_buf = TALLOC_ZERO_P(p->mem_ctx, struct sec_desc_buf);
+	sd_buf = talloc_zero(p->mem_ctx, struct sec_desc_buf);
 	if (!sd_buf) {
 		werr = WERR_NOMEM;
 		goto error_exit;
@@ -2471,7 +2438,7 @@ WERROR _srvsvc_NetDiskEnum(struct pipes_struct *p,
 
 	*r->out.totalentries = init_server_disk_enum(&resume);
 
-	r->out.info->disks = TALLOC_ZERO_ARRAY(ctx, struct srvsvc_NetDiskInfo0,
+	r->out.info->disks = talloc_zero_array(ctx, struct srvsvc_NetDiskInfo0,
 					       MAX_SERVER_DISK_ENTRIES);
 	W_ERROR_HAVE_NO_MEMORY(r->out.info->disks);
 
diff --git a/source3/rpc_server/svcctl/srv_svcctl_nt.c b/source3/rpc_server/svcctl/srv_svcctl_nt.c
index a4eb282de7c..f515906339f 100644
--- a/source3/rpc_server/svcctl/srv_svcctl_nt.c
+++ b/source3/rpc_server/svcctl/srv_svcctl_nt.c
@@ -23,13 +23,14 @@
  */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_svcctl.h"
 #include "../libcli/security/security.h"
 #include "../librpc/gen_ndr/ndr_security.h"
 #include "services/services.h"
 #include "services/svc_winreg_glue.h"
 #include "auth.h"
-#include "ntdomain.h"
+#include "rpc_server/svcctl/srv_svcctl_nt.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -70,7 +71,7 @@ bool init_service_op_table( void )
 	int num_services = SVCCTL_NUM_INTERNAL_SERVICES + str_list_length( service_list );
 	int i;
 
-	if ( !(svcctl_ops = TALLOC_ARRAY( NULL, struct service_control_op, num_services+1)) ) {
+	if ( !(svcctl_ops = talloc_array( NULL, struct service_control_op, num_services+1)) ) {
 		DEBUG(0,("init_service_op_table: talloc() failed!\n"));
 		return False;
 	}
@@ -208,7 +209,7 @@ static WERROR create_open_service_handle(struct pipes_struct *p,
 	WERROR result = WERR_OK;
 	struct service_control_op *s_op;
 
-	if ( !(info = TALLOC_ZERO_P( NULL, SERVICE_INFO )) )
+	if ( !(info = talloc_zero( NULL, SERVICE_INFO )) )
 		return WERR_NOMEM;
 
 	/* the Service Manager has a NULL name */
@@ -420,7 +421,7 @@ static int enumerate_status(TALLOC_CTX *ctx,
 	while ( svcctl_ops[num_services].name )
 		num_services++;
 
-	if ( !(st = TALLOC_ARRAY( ctx, struct ENUM_SERVICE_STATUSW, num_services )) ) {
+	if ( !(st = talloc_array( ctx, struct ENUM_SERVICE_STATUSW, num_services )) ) {
 		DEBUG(0,("enumerate_status: talloc() failed!\n"));
 		return -1;
 	}
@@ -667,17 +668,18 @@ WERROR _svcctl_QueryServiceStatusEx(struct pipes_struct *p,
 /********************************************************************
 ********************************************************************/
 
-static WERROR fill_svc_config(TALLOC_CTX *ctx,
+static WERROR fill_svc_config(TALLOC_CTX *mem_ctx,
 			      struct messaging_context *msg_ctx,
 			      struct auth_serversupplied_info *session_info,
 			      const char *name,
 			      struct QUERY_SERVICE_CONFIG *config)
 {
-	TALLOC_CTX *mem_ctx = talloc_stackframe();
 	const char *result = NULL;
 
 	/* now fill in the individual values */
 
+	ZERO_STRUCTP(config);
+
 	config->displayname = svcctl_lookup_dispname(mem_ctx,
 						     msg_ctx,
 						     session_info,
@@ -719,9 +721,6 @@ static WERROR fill_svc_config(TALLOC_CTX *ctx,
 	else
 		config->start_type = SVCCTL_DEMAND_START;
 
-
-	talloc_free(mem_ctx);
-
 	return WERR_OK;
 }
 
@@ -776,7 +775,8 @@ WERROR _svcctl_QueryServiceConfig2W(struct pipes_struct *p,
 				    struct svcctl_QueryServiceConfig2W *r)
 {
 	SERVICE_INFO *info = find_service_info_by_hnd( p, r->in.handle );
-	uint32 buffer_size;
+	uint32_t buffer_size;
+	DATA_BLOB blob = data_blob_null;
 
 	/* perform access checks */
 
@@ -796,7 +796,6 @@ WERROR _svcctl_QueryServiceConfig2W(struct pipes_struct *p,
 			struct SERVICE_DESCRIPTION desc_buf;
 			const char *description;
 			enum ndr_err_code ndr_err;
-			DATA_BLOB blob;
 
 			description = svcctl_lookup_description(p->mem_ctx,
 								p->msg_ctx,
@@ -811,9 +810,6 @@ WERROR _svcctl_QueryServiceConfig2W(struct pipes_struct *p,
 				return WERR_INVALID_PARAM;
 			}
 
-			buffer_size = ndr_size_SERVICE_DESCRIPTION(&desc_buf, 0);
-			r->out.buffer = blob.data;
-
 			break;
 		}
 		break;
@@ -821,7 +817,6 @@ WERROR _svcctl_QueryServiceConfig2W(struct pipes_struct *p,
 		{
 			struct SERVICE_FAILURE_ACTIONS actions;
 			enum ndr_err_code ndr_err;
-			DATA_BLOB blob;
 
 			/* nothing to say...just service the request */
 
@@ -833,9 +828,6 @@ WERROR _svcctl_QueryServiceConfig2W(struct pipes_struct *p,
 				return WERR_INVALID_PARAM;
 			}
 
-			buffer_size = ndr_size_SERVICE_FAILURE_ACTIONS(&actions, 0);
-			r->out.buffer = blob.data;
-
 			break;
 		}
 		break;
@@ -844,12 +836,15 @@ WERROR _svcctl_QueryServiceConfig2W(struct pipes_struct *p,
 		return WERR_UNKNOWN_LEVEL;
 	}
 
+	buffer_size = blob.length;
 	buffer_size += buffer_size % 4;
 	*r->out.needed = (buffer_size > r->in.offered) ? buffer_size : r->in.offered;
 
         if (buffer_size > r->in.offered)
                 return WERR_INSUFFICIENT_BUFFER;
 
+	memcpy(r->out.buffer, blob.data, blob.length);
+
 	return WERR_OK;
 }
 
@@ -941,7 +936,7 @@ WERROR _svcctl_QueryServiceObjectSecurity(struct pipes_struct *p,
 	}
 
 	*r->out.needed = len;
-	r->out.buffer = buffer;
+	memcpy(r->out.buffer, buffer, len);
 
 	return WERR_OK;
 }
diff --git a/source3/rpc_server/svcctl/srv_svcctl_nt.h b/source3/rpc_server/svcctl/srv_svcctl_nt.h
new file mode 100644
index 00000000000..dd049272882
--- /dev/null
+++ b/source3/rpc_server/svcctl/srv_svcctl_nt.h
@@ -0,0 +1,33 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  RPC Pipe client / server routines
+ *
+ *  Copyright (C) Marcin Krzysztof Porwit           2005.
+ *
+ *  Largely Rewritten (Again) by:
+ *  Copyright (C) Gerald (Jerry) Carter             2005.
+ *  Copyright (C) Guenther Deschner                 2008,2009.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _RPC_SERVER_SVCCTL_SRV_SVCCTL_NT_H_
+#define _RPC_SERVER_SVCCTL_SRV_SVCCTL_NT_H_
+
+/* The following definitions come from rpc_server/srv_svcctl_nt.c  */
+
+bool init_service_op_table( void );
+bool shutdown_service_op_table(void);
+
+#endif /* _RPC_SERVER_SVCCTL_SRV_SVCCTL_NT_H_ */
diff --git a/source3/rpc_server/winreg/srv_winreg_nt.c b/source3/rpc_server/winreg/srv_winreg_nt.c
index fedb665a30b..6fee5b6acd9 100644
--- a/source3/rpc_server/winreg/srv_winreg_nt.c
+++ b/source3/rpc_server/winreg/srv_winreg_nt.c
@@ -21,6 +21,7 @@
 /* Implementation of registry functions. */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_winreg.h"
 #include "registry.h"
 #include "registry/reg_api.h"
@@ -28,8 +29,8 @@
 #include "registry/reg_perfcount.h"
 #include "rpc_misc.h"
 #include "auth.h"
-#include "ntdomain.h"
 #include "lib/privileges.h"
+#include "libcli/security/secdesc.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -541,7 +542,7 @@ WERROR _winreg_InitiateSystemShutdownEx(struct pipes_struct *p,
 		if ( (msg = talloc_strdup(p->mem_ctx, r->in.message->string )) == NULL ) {
 			return WERR_NOMEM;
 		}
-		chkmsg = TALLOC_ARRAY(p->mem_ctx, char, strlen(msg)+1);
+		chkmsg = talloc_array(p->mem_ctx, char, strlen(msg)+1);
 		if (!chkmsg) {
 			return WERR_NOMEM;
 		}
diff --git a/source3/rpc_server/wkssvc/srv_wkssvc_nt.c b/source3/rpc_server/wkssvc/srv_wkssvc_nt.c
index d44414f7767..49c71b8d6eb 100644
--- a/source3/rpc_server/wkssvc/srv_wkssvc_nt.c
+++ b/source3/rpc_server/wkssvc/srv_wkssvc_nt.c
@@ -23,6 +23,7 @@
 /* This is the implementation of the wks interface. */
 
 #include "includes.h"
+#include "ntdomain.h"
 #include "librpc/gen_ndr/libnet_join.h"
 #include "libnet/libnet_join.h"
 #include "../libcli/auth/libcli_auth.h"
@@ -31,7 +32,6 @@
 #include "session.h"
 #include "smbd/smbd.h"
 #include "auth.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_RPC_SRV
@@ -196,7 +196,7 @@ static struct dom_usr *get_domain_userlist(TALLOC_CTX *mem_ctx)
 			DEBUG(10, ("talloc_asprintf failed\n"));
 			continue;
 		}
-		if (strcmp(machine_name, global_myname()) == 0) {
+		if (strcmp(machine_name, lp_netbios_name()) == 0) {
 			p = session_list[i].username;
 			nm = strstr(p, sep);
 			if (nm) {
@@ -260,11 +260,11 @@ static struct wkssvc_NetWkstaInfo100 *create_wks_info_100(TALLOC_CTX *mem_ctx)
 	}
 
 	info100->platform_id	 = PLATFORM_ID_NT;	/* unknown */
-	info100->version_major	 = lp_major_announce_version();
-	info100->version_minor	 = lp_minor_announce_version();
+	info100->version_major	 = SAMBA_MAJOR_NBT_ANNOUNCE_VERSION;
+	info100->version_minor	 = SAMBA_MINOR_NBT_ANNOUNCE_VERSION;
 
 	info100->server_name = talloc_asprintf_strupper_m(
-		info100, "%s", global_myname());
+		info100, "%s", lp_netbios_name());
 	info100->domain_name = talloc_asprintf_strupper_m(
 		info100, "%s", lp_workgroup());
 
@@ -289,11 +289,11 @@ static struct wkssvc_NetWkstaInfo101 *create_wks_info_101(TALLOC_CTX *mem_ctx)
 	}
 
 	info101->platform_id	 = PLATFORM_ID_NT;	/* unknown */
-	info101->version_major	 = lp_major_announce_version();
-	info101->version_minor	 = lp_minor_announce_version();
+	info101->version_major	 = SAMBA_MAJOR_NBT_ANNOUNCE_VERSION;
+	info101->version_minor	 = SAMBA_MINOR_NBT_ANNOUNCE_VERSION;
 
 	info101->server_name = talloc_asprintf_strupper_m(
-		info101, "%s", global_myname());
+		info101, "%s", lp_netbios_name());
 	info101->domain_name = talloc_asprintf_strupper_m(
 		info101, "%s", lp_workgroup());
 	info101->lan_root = "";
@@ -320,11 +320,11 @@ static struct wkssvc_NetWkstaInfo102 *create_wks_info_102(TALLOC_CTX *mem_ctx)
 	}
 
 	info102->platform_id	 = PLATFORM_ID_NT;	/* unknown */
-	info102->version_major	 = lp_major_announce_version();
-	info102->version_minor	 = lp_minor_announce_version();
+	info102->version_major	 = SAMBA_MAJOR_NBT_ANNOUNCE_VERSION;
+	info102->version_minor	 = SAMBA_MINOR_NBT_ANNOUNCE_VERSION;
 
 	info102->server_name = talloc_asprintf_strupper_m(
-		info102, "%s", global_myname());
+		info102, "%s", lp_netbios_name());
 	info102->domain_name = talloc_asprintf_strupper_m(
 		info102, "%s", lp_workgroup());
 	info102->lan_root = "";
@@ -528,7 +528,7 @@ static struct wkssvc_NetWkstaEnumUsersCtr1 *create_enum_users1(
 		/* For a local user the domain name and logon server are
 		 * both returned as the local machine's NetBIOS name */
 		ctr1->user1[i].logon_domain = ctr1->user1[i].logon_server =
-			talloc_asprintf_strupper_m(ctr1->user1, "%s", global_myname());
+			talloc_asprintf_strupper_m(ctr1->user1, "%s", lp_netbios_name());
 
 		ctr1->user1[i].other_domains = NULL;	/* Maybe in future? */
 	}
diff --git a/source3/rpc_server/wscript_build b/source3/rpc_server/wscript_build
index 346d770385e..5d21d5e3408 100644
--- a/source3/rpc_server/wscript_build
+++ b/source3/rpc_server/wscript_build
@@ -26,26 +26,32 @@ bld.SAMBA3_SUBSYSTEM('rpc',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_NCACN_NP',
-                    source='rpc_ncacn_np.c rpc_handles.c',
-                    deps='auth_sam_reply')
+                    source='rpc_ncacn_np.c rpc_handles.c rpc_contexts.c',
+                    deps='auth_sam_reply RPC_PIPE_REGISTER AUTH_COMMON npa_tstream')
 
 bld.SAMBA3_SUBSYSTEM('RPC_SERVICE',
-                    source='rpc_server.c')
+                    source='rpc_server.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('RPC_CRYPTO',
-                    source='dcesrv_ntlmssp.c dcesrv_gssapi.c dcesrv_spnego.c')
+                     source='dcesrv_ntlmssp.c dcesrv_gssapi.c dcesrv_spnego.c',
+                     deps = 'KRB5_PAC')
 
 bld.SAMBA3_SUBSYSTEM('RPC_PIPE_REGISTER',
-                    source='srv_pipe_register.c')
+                    source='srv_pipe_register.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('RPC_SERVER_REGISTER',
-                    source='rpc_ep_setup.c ../librpc/rpc/dcerpc_ep.c')
+                    source='rpc_ep_setup.c ../librpc/rpc/dcerpc_ep.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('EPMD',
-		     source='epmd.c')
+                    source='epmd.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('SRV_ACCESS_CHECK',
-                    source='srv_access_check.c')
+                    source='srv_access_check.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('RPC_SAMR',
                     source=RPC_SAMR_SRC,
@@ -64,10 +70,12 @@ bld.SAMBA3_SUBSYSTEM('RPC_WINREG',
 
 bld.SAMBA3_SUBSYSTEM('RPC_INITSHUTDOWN',
                     source=RPC_INITSHUTDOWN_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_DSSETUP',
                     source=RPC_DSSETUP_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_WKSSVC',
@@ -82,23 +90,27 @@ bld.SAMBA3_SUBSYSTEM('RPC_SVCCTL',
 
 bld.SAMBA3_SUBSYSTEM('RPC_NTSVCS',
                     source=RPC_NTSVCS_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_NETLOGON',
                     source=RPC_NETLOGON_SRC,
+                     deps='RPC_NCACN_NP',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_NETDFS',
                     source=RPC_NETDFS_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_SRVSVC',
                     source=RPC_SRVSVC_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_SPOOLSS',
                     source=RPC_SPOOLSS_SRC,
-                    deps='cups PRINTING PRINTBACKEND LIBCLI_WINREG',
+                    deps='cups PRINTING PRINTBACKEND LIBCLI_WINREG RPC_NCACN_NP',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_EVENTLOG',
@@ -108,10 +120,12 @@ bld.SAMBA3_SUBSYSTEM('RPC_EVENTLOG',
 
 bld.SAMBA3_SUBSYSTEM('RPC_RPCECHO',
                     source=RPC_RPCECHO_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_EPMAPPER',
                     source=RPC_EPMAPPER_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('RPC_SERVER',
diff --git a/source3/rpcclient/cmd_drsuapi.c b/source3/rpcclient/cmd_drsuapi.c
index 3f1ecab7503..292fa8838b2 100644
--- a/source3/rpcclient/cmd_drsuapi.c
+++ b/source3/rpcclient/cmd_drsuapi.c
@@ -40,7 +40,7 @@ static WERROR cracknames(struct rpc_pipe_client *cli,
 	struct drsuapi_DsNameString *names;
 	struct dcerpc_binding_handle *b = cli->binding_handle;
 
-	names = TALLOC_ZERO_ARRAY(mem_ctx, struct drsuapi_DsNameString, argc);
+	names = talloc_zero_array(mem_ctx, struct drsuapi_DsNameString, argc);
 	W_ERROR_HAVE_NO_MEMORY(names);
 
 	for (i=0; i<argc; i++) {
diff --git a/source3/rpcclient/cmd_lsarpc.c b/source3/rpcclient/cmd_lsarpc.c
index 4b065d0dd2d..3382eb1a5c7 100644
--- a/source3/rpcclient/cmd_lsarpc.c
+++ b/source3/rpcclient/cmd_lsarpc.c
@@ -400,7 +400,7 @@ static NTSTATUS cmd_lsa_lookup_sids(struct rpc_pipe_client *cli, TALLOC_CTX *mem
 
 	/* Convert arguments to sids */
 
-	sids = TALLOC_ARRAY(mem_ctx, struct dom_sid, argc - 1);
+	sids = talloc_array(mem_ctx, struct dom_sid, argc - 1);
 
 	if (!sids) {
 		printf("could not allocate memory for %d sids\n", argc - 1);
@@ -978,7 +978,7 @@ static NTSTATUS cmd_lsa_add_acct_rights(struct rpc_pipe_client *cli,
 		goto done;
 
 	rights.count = argc-2;
-	rights.names = TALLOC_ARRAY(mem_ctx, struct lsa_StringLarge,
+	rights.names = talloc_array(mem_ctx, struct lsa_StringLarge,
 				    rights.count);
 	if (!rights.names) {
 		return NT_STATUS_NO_MEMORY;
@@ -1036,7 +1036,7 @@ static NTSTATUS cmd_lsa_remove_acct_rights(struct rpc_pipe_client *cli,
 		goto done;
 
 	rights.count = argc-2;
-	rights.names = TALLOC_ARRAY(mem_ctx, struct lsa_StringLarge,
+	rights.names = talloc_array(mem_ctx, struct lsa_StringLarge,
 				    rights.count);
 	if (!rights.names) {
 		return NT_STATUS_NO_MEMORY;
@@ -1494,7 +1494,7 @@ static NTSTATUS cmd_lsa_add_priv(struct rpc_pipe_client *cli,
 		}
 
 		privs.count++;
-		set = TALLOC_REALLOC_ARRAY(mem_ctx, set,
+		set = talloc_realloc(mem_ctx, set,
 					   struct lsa_LUIDAttribute,
 					   privs.count);
 		if (!set) {
@@ -1592,7 +1592,7 @@ static NTSTATUS cmd_lsa_del_priv(struct rpc_pipe_client *cli,
 		}
 
 		privs.count++;
-		set = TALLOC_REALLOC_ARRAY(mem_ctx, set,
+		set = talloc_realloc(mem_ctx, set,
 					   struct lsa_LUIDAttribute,
 					   privs.count);
 		if (!set) {
diff --git a/source3/rpcclient/cmd_netlogon.c b/source3/rpcclient/cmd_netlogon.c
index 63057ac3689..ad43452bf11 100644
--- a/source3/rpcclient/cmd_netlogon.c
+++ b/source3/rpcclient/cmd_netlogon.c
@@ -583,7 +583,7 @@ static NTSTATUS cmd_netlogon_sam_sync(struct rpc_pipe_client *cli,
 	NTSTATUS result = NT_STATUS_UNSUCCESSFUL;
 	NTSTATUS status;
 	const char *logon_server = cli->desthost;
-	const char *computername = global_myname();
+	const char *computername = lp_netbios_name();
 	struct netr_Authenticator credential;
 	struct netr_Authenticator return_authenticator;
 	enum netr_SamDatabaseID database_id = SAM_DATABASE_DOMAIN;
@@ -654,7 +654,7 @@ static NTSTATUS cmd_netlogon_sam_deltas(struct rpc_pipe_client *cli,
 	NTSTATUS status;
 	uint32_t tmp;
 	const char *logon_server = cli->desthost;
-	const char *computername = global_myname();
+	const char *computername = lp_netbios_name();
 	struct netr_Authenticator credential;
 	struct netr_Authenticator return_authenticator;
 	enum netr_SamDatabaseID database_id = SAM_DATABASE_DOMAIN;
@@ -1121,8 +1121,8 @@ static NTSTATUS cmd_netlogon_database_redo(struct rpc_pipe_client *cli,
 	status = rpccli_netlogon_setup_creds(cli,
 					     server_name, /* server name */
 					     lp_workgroup(), /* domain */
-					     global_myname(), /* client name */
-					     global_myname(), /* machine account name */
+					     lp_netbios_name(), /* client name */
+					     lp_netbios_name(), /* machine account name */
 					     trust_passwd_hash,
 					     sec_channel_type,
 					     &neg_flags);
@@ -1141,7 +1141,7 @@ static NTSTATUS cmd_netlogon_database_redo(struct rpc_pipe_client *cli,
 
 	status = dcerpc_netr_DatabaseRedo(b, mem_ctx,
 					  server_name,
-					  global_myname(),
+					  lp_netbios_name(),
 					  &clnt_creds,
 					  &srv_cred,
 					  e,
@@ -1187,7 +1187,7 @@ static NTSTATUS cmd_netlogon_capabilities(struct rpc_pipe_client *cli,
 
 	status = dcerpc_netr_LogonGetCapabilities(b, mem_ctx,
 						  cli->desthost,
-						  global_myname(),
+						  lp_netbios_name(),
 						  &credential,
 						  &return_authenticator,
 						  level,
diff --git a/source3/rpcclient/cmd_samr.c b/source3/rpcclient/cmd_samr.c
index 727c9d14560..24bd55abf32 100644
--- a/source3/rpcclient/cmd_samr.c
+++ b/source3/rpcclient/cmd_samr.c
@@ -284,18 +284,18 @@ static NTSTATUS get_domain_handle(struct rpc_pipe_client *cli,
 	struct dcerpc_binding_handle *b = cli->binding_handle;
 	NTSTATUS status = NT_STATUS_INVALID_PARAMETER, result;
 
-	if (StrCaseCmp(sam, "domain") == 0) {
+	if (strcasecmp_m(sam, "domain") == 0) {
 		status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					      connect_pol,
 					      access_mask,
 					      _domain_sid,
 					      domain_pol,
 					      &result);
-	} else if (StrCaseCmp(sam, "builtin") == 0) {
+	} else if (strcasecmp_m(sam, "builtin") == 0) {
 		status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					      connect_pol,
 					      access_mask,
-					      CONST_DISCARD(struct dom_sid2 *, &global_sid_Builtin),
+					      discard_const_p(struct dom_sid2, &global_sid_Builtin),
 					      domain_pol,
 					      &result);
 	}
@@ -728,7 +728,7 @@ static NTSTATUS cmd_samr_query_useraliases(struct rpc_pipe_client *cli,
 	}
 
 	if (num_sids) {
-		sid_array.sids = TALLOC_ZERO_ARRAY(mem_ctx, struct lsa_SidPtr, num_sids);
+		sid_array.sids = talloc_zero_array(mem_ctx, struct lsa_SidPtr, num_sids);
 		if (sid_array.sids == NULL)
 			return NT_STATUS_NO_MEMORY;
 	} else {
@@ -2087,7 +2087,7 @@ static NTSTATUS cmd_samr_lookup_names(struct rpc_pipe_client *cli,
 
 	num_names = argc - 2;
 
-	if ((names = TALLOC_ARRAY(mem_ctx, struct lsa_String, num_names)) == NULL) {
+	if ((names = talloc_array(mem_ctx, struct lsa_String, num_names)) == NULL) {
 		dcerpc_samr_Close(b, mem_ctx, &domain_pol, &result);
 		dcerpc_samr_Close(b, mem_ctx, &connect_pol, &result);
 		status = NT_STATUS_NO_MEMORY;
@@ -2167,7 +2167,7 @@ static NTSTATUS cmd_samr_lookup_rids(struct rpc_pipe_client *cli,
 
 	num_rids = argc - 2;
 
-	if ((rids = TALLOC_ARRAY(mem_ctx, uint32, num_rids)) == NULL) {
+	if ((rids = talloc_array(mem_ctx, uint32, num_rids)) == NULL) {
 		dcerpc_samr_Close(b, mem_ctx, &domain_pol, &result);
 		dcerpc_samr_Close(b, mem_ctx, &connect_pol, &result);
 		status = NT_STATUS_NO_MEMORY;
diff --git a/source3/rpcclient/cmd_spoolss.c b/source3/rpcclient/cmd_spoolss.c
index 712b23724ec..0e12201f927 100644
--- a/source3/rpcclient/cmd_spoolss.c
+++ b/source3/rpcclient/cmd_spoolss.c
@@ -27,12 +27,12 @@
 #include "../librpc/gen_ndr/ndr_spoolss_c.h"
 #include "rpc_client/cli_spoolss.h"
 #include "rpc_client/init_spoolss.h"
-#include "registry.h"
 #include "registry/reg_objects.h"
 #include "nt_printing.h"
 #include "../libcli/security/display_sec.h"
 #include "../libcli/security/security_descriptor.h"
 #include "../libcli/registry/util_reg.h"
+#include "libsmb/libsmb.h"
 
 #define RPCCLIENT_PRINTERNAME(_printername, _cli, _arg) \
 { \
@@ -83,7 +83,7 @@ static const char *cmd_spoolss_get_short_archi(const char *long_archi)
         do {
                 i++;
         } while ( (archi_table[i].long_archi!=NULL ) &&
-                  StrCaseCmp(long_archi, archi_table[i].long_archi) );
+                  strcasecmp_m(long_archi, archi_table[i].long_archi) );
 
         if (archi_table[i].long_archi==NULL) {
                 DEBUGADD(10,("Unknown architecture [%s] !\n", long_archi));
@@ -1616,7 +1616,7 @@ static char *get_driver_3_param(TALLOC_CTX *mem_ctx, char *str,
 	   parameter because two consecutive delimiters
 	   will not return an empty string.  See man strtok(3)
 	   for details */
-	if (ptr && (StrCaseCmp(ptr, "NULL") == 0)) {
+	if (ptr && (strcasecmp_m(ptr, "NULL") == 0)) {
 		ptr = NULL;
 	}
 
@@ -3263,7 +3263,7 @@ static WERROR cmd_spoolss_rffpcnex(struct rpc_pipe_client *cli,
 	}
 	option.types[1].fields[0].field = JOB_NOTIFY_FIELD_PRINTER_NAME;
 
-	clientname = talloc_asprintf(mem_ctx, "\\\\%s", global_myname());
+	clientname = talloc_asprintf(mem_ctx, "\\\\%s", lp_netbios_name());
 	if (!clientname) {
 		result = WERR_NOMEM;
 		goto done;
@@ -3429,7 +3429,7 @@ static WERROR cmd_spoolss_printercmp(struct rpc_pipe_client *cli,
 
 	/* first get the connection to the remote server */
 
-	nt_status = cli_full_connection(&cli_server2, global_myname(), argv[2],
+	nt_status = cli_full_connection(&cli_server2, lp_netbios_name(), argv[2],
 					NULL, 0,
 					"IPC$", "IPC",
 					get_cmdline_auth_info_username(rpcclient_auth_info),
diff --git a/source3/rpcclient/rpcclient.c b/source3/rpcclient/rpcclient.c
index cebe2a9c6ae..542862606b2 100644
--- a/source3/rpcclient/rpcclient.c
+++ b/source3/rpcclient/rpcclient.c
@@ -30,7 +30,7 @@
 #include "../libcli/smbreadline/smbreadline.h"
 #include "../libcli/security/security.h"
 #include "passdb.h"
-#include "ntdomain.h"
+#include "libsmb/libsmb.h"
 
 enum pipe_auth_type_spnego {
 	PIPE_AUTH_TYPE_SPNEGO_NONE = 0,
@@ -240,7 +240,7 @@ static NTSTATUS cmd_listcommands(struct rpc_pipe_client *cli, TALLOC_CTX *mem_ct
 	{
 		tmp_set = tmp->cmd_set;
 
-		if (!StrCaseCmp(argv[1], tmp_set->name))
+		if (!strcasecmp_m(argv[1], tmp_set->name))
 		{
 			printf("Available commands on the %s pipe:\n\n", tmp_set->name);
 
@@ -775,7 +775,7 @@ static NTSTATUS do_cmd(struct cli_state *cli,
 			ntresult = rpccli_netlogon_setup_creds(cmd_entry->rpc_pipe,
 						cli->desthost,   /* server name */
 						get_cmdline_auth_info_domain(auth_info),  /* domain */
-						global_myname(), /* client name */
+						lp_netbios_name(), /* client name */
 						machine_account, /* machine account name */
 						trust_password,
 						sec_channel_type,
@@ -892,7 +892,6 @@ out_free:
 	struct sockaddr_storage server_ss;
 	NTSTATUS 		nt_status;
 	static int		opt_port = 0;
-	fstring new_workgroup;
 	int result = 0;
 	TALLOC_CTX *frame = talloc_stackframe();
 	uint32_t flags = 0;
@@ -975,22 +974,11 @@ out_free:
 		goto done;
 	}
 
-	/* save the workgroup...
-
-	   FIXME!! do we need to do this for other options as well
-	   (or maybe a generic way to keep lp_load() from overwriting
-	   everything)?  */
-
-	fstrcpy( new_workgroup, lp_workgroup() );
-
 	/* Load smb.conf file */
 
 	if (!lp_load(get_dyn_CONFIGFILE(),True,False,False,True))
 		fprintf(stderr, "Can't load %s\n", get_dyn_CONFIGFILE());
 
-	if ( strlen(new_workgroup) != 0 )
-		set_global_myworkgroup( new_workgroup );
-
 	/*
 	 * Get password
 	 * from stdin if necessary
@@ -1091,7 +1079,7 @@ out_free:
 	}
 
 
-	nt_status = cli_full_connection(&cli, global_myname(), binding->host,
+	nt_status = cli_full_connection(&cli, lp_netbios_name(), binding->host,
 					opt_ipaddr ? &server_ss : NULL, opt_port,
 					"IPC$", "IPC",
 					get_cmdline_auth_info_username(rpcclient_auth_info),
diff --git a/source3/script/mkbuildoptions-waf.awk b/source3/script/mkbuildoptions-waf.awk
index ddb3f6527a9..2d7063a79e6 100644
--- a/source3/script/mkbuildoptions-waf.awk
+++ b/source3/script/mkbuildoptions-waf.awk
@@ -21,7 +21,7 @@ BEGIN {
 	print "";
 	print "#include \"includes.h\"";
 	print "#include \"build_env.h\"";
-	print "#include \"dynconfig.h\"";
+	print "#include \"dynconfig/dynconfig.h\"";
 	print "";
 	print "static int output(bool screen, const char *format, ...) PRINTF_ATTRIBUTE(2,3);";
 	print "void build_options(bool screen);";
diff --git a/source3/script/mkbuildoptions.awk b/source3/script/mkbuildoptions.awk
index 1955a80b46c..0b1b2ef3ab8 100644
--- a/source3/script/mkbuildoptions.awk
+++ b/source3/script/mkbuildoptions.awk
@@ -21,7 +21,7 @@ BEGIN {
 	print "";
 	print "#include \"includes.h\"";
 	print "#include \"build_env.h\"";
-	print "#include \"dynconfig.h\"";
+	print "#include \"dynconfig/dynconfig.h\"";
 	print "";
 	print "static int output(bool screen, const char *format, ...) PRINTF_ATTRIBUTE(2,3);";
 	print "void build_options(bool screen);";
@@ -235,7 +235,7 @@ function output(ARRAY, ELEMENTS, TITLE) {
 END {
 	##################################################
 	# add code to show various options
-	print "/* Output various other options (as gleaned from include/config.h.in) */";
+	print "/* Output various other options (as gleaned from include/autoconf/config.h.in) */";
 	output(sys_ary,     sys_i,     "System Headers");
 	output(headers_ary, headers_i, "Headers");
 	output(utmp_ary,    utmp_i,    "UTMP Options");
diff --git a/source3/script/tests/test_net_registry_roundtrip.sh b/source3/script/tests/test_net_registry_roundtrip.sh
index f35a7524281..33114ccd4c0 100755
--- a/source3/script/tests/test_net_registry_roundtrip.sh
+++ b/source3/script/tests/test_net_registry_roundtrip.sh
@@ -7,7 +7,7 @@
 
 if [ $# -lt 3 ]; then
 cat <<EOF
-Usage: test_net_registry_roundtrip.sh SCRIPTDIR SERVERCONFFILE CONFIGURATION
+Usage: test_net_registry_roundtrip.sh SCRIPTDIR SERVERCONFFILE PREFIX CONFIGURATION
 EOF
 exit 1;
 fi
@@ -144,7 +144,7 @@ conf_roundtrip()
     rm -r $DIR
 }
 
-CONF_FILES=${CONF_FILES:-$(find $SRCDIR/source3 -name '*.conf' | grep -v examples/logon | xargs grep -l "\[global\]")}
+CONF_FILES=$SERVERCONFFILE
 
 # remove old logs:
 for OLDDIR in $(find ${PREFIX} -type d -name "${LOGDIR_PREFIX}_*") ; do
diff --git a/source3/script/tests/test_pthreadpool.sh b/source3/script/tests/test_pthreadpool.sh
new file mode 100755
index 00000000000..b1c7c6da11c
--- /dev/null
+++ b/source3/script/tests/test_pthreadpool.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+incdir=`dirname $0`/../../../testprogs/blackbox
+. $incdir/subunit.sh
+
+if [ ! -x $BINDIR/pthreadpooltest ] ; then
+    # Some machines don't have /bin/true, simulate it
+    cat >$BINDIR/pthreadpooltest <<EOF
+#!/bin/sh
+exit 0
+EOF
+    chmod +x $BINDIR/pthreadpooltest
+fi
+
+failed=0
+
+testit "pthreadpool" $VALGRIND $BINDIR/pthreadpooltest ||
+	failed=`expr $failed + 1`
+
+testok $0 $failed
diff --git a/source3/script/tests/test_smbclient_s3.sh b/source3/script/tests/test_smbclient_s3.sh
index 0395f3e12c6..8937c97bbef 100755
--- a/source3/script/tests/test_smbclient_s3.sh
+++ b/source3/script/tests/test_smbclient_s3.sh
@@ -4,22 +4,23 @@
 
 if [ $# -lt 7 ]; then
 cat <<EOF
-Usage: test_smbclient_s3.sh SERVER SERVER_IP USERNAME PASSWORD USERID LOCAL_PATH PREFIX SMBCLIENT
+Usage: test_smbclient_s3.sh SERVER SERVER_IP DOMAIN USERNAME PASSWORD USERID LOCAL_PATH PREFIX SMBCLIENT
 EOF
 exit 1;
 fi
 
 SERVER="$1"
 SERVER_IP="$2"
-USERNAME="$3"
-PASSWORD="$4"
-USERID="$5"
-LOCAL_PATH="$6"
-PREFIX="$7"
-SMBCLIENT="$8"
+DOMAIN="$3"
+USERNAME="$4"
+PASSWORD="$5"
+USERID="$6"
+LOCAL_PATH="$7"
+PREFIX="$8"
+SMBCLIENT="$9"
 SMBCLIENT="$VALGRIND ${SMBCLIENT}"
 WBINFO="$VALGRIND ${WBINFO:-$BINDIR/wbinfo}"
-shift 8
+shift 9
 ADDARGS="$*"
 
 incdir=`dirname $0`/../../../testprogs/blackbox
@@ -218,6 +219,7 @@ EOF
     if [ $ret != 0 ] ; then
 	echo "$out"
 	echo "failed writing into read-only directory with error $ret"
+
 	false
 	return
     fi
@@ -235,6 +237,49 @@ EOF
     fi
 }
 
+
+# Test sending a message
+test_message()
+{
+    tmpfile=$PREFIX/message_in.$$
+
+    cat > $tmpfile <<EOF
+Test message from pid $$
+EOF
+
+    cmd='$SMBCLIENT "$@" -U$USERNAME%$PASSWORD -M $SERVER -p 139 $ADDARGS -n msgtest < $tmpfile 2>&1'
+    eval echo "$cmd"
+    out=`eval $cmd`
+    ret=$?
+
+    if [ $ret != 0 ] ; then
+	echo "$out"
+	echo "failed sending message to $SERVER with error $ret"
+	false
+	rm -f $tmpfile
+	return
+    fi
+
+    cmd='$SMBCLIENT "$@" -U$USERNAME%$PASSWORD //$SERVER/tmpguest -p 139 $ADDARGS -c "get message.msgtest $PREFIX/message_out.$$" 2>&1'
+    eval echo "$cmd"
+    out=`eval $cmd`
+    ret=$?
+
+    if [ $ret != 0 ] ; then
+	echo "$out"
+	echo "failed getting sent message from $SERVER with error $ret"
+	false
+	return
+    fi
+
+    if [ cmp $PREFIX/message_out.$$ $tmpfile != 0 ] ; then
+	echo "failed comparison of message from $SERVER"
+	false
+	return
+    fi
+    true
+}
+
 # Test reading an owner-only file (logon as guest) fails.
 test_owner_only_file()
 {
@@ -389,6 +434,43 @@ test_ccache_access()
     $WBINFO --logoff
 }
 
+# Test authenticating using the winbind ccache
+test_auth_file()
+{
+    tmpfile=$PREFIX/smbclient.in.$$
+    cat > $tmpfile <<EOF
+username=${USERNAME}
+password=${PASSWORD}
+domain=${DOMAIN}
+EOF
+    $SMBCLIENT //$SERVER_IP/tmp --authentication-file=$tmpfile \
+	-c quit 2>&1
+    ret=$?
+    rm $tmpfile
+
+    if [ $ret != 0 ] ; then
+	echo "smbclient failed to use auth file"
+	false
+	return
+    fi
+
+    cat > $tmpfile <<EOF
+username=${USERNAME}
+password=xxxx
+domain=${DOMAIN}
+EOF
+    $SMBCLIENT //$SERVER_IP/tmp --authentication-file=$tmpfile\
+	-c quit 2>&1
+    ret=$?
+    rm $tmpfile
+
+    if [ $ret -eq 0 ] ; then
+	echo "smbclient succeeded with wrong auth file credentials"
+	false
+	return
+    fi
+}
+
 LOGDIR_PREFIX=test_smbclient_s3
 
 # possibly remove old logdirs:
@@ -444,6 +526,14 @@ testit "ccache access works for smbclient" \
     test_ccache_access || \
     failed=`expr $failed + 1`
 
+testit "sending a message to the remote server" \
+    test_message || \
+    failed=`expr $failed + 1`
+
+testit "using an authentication file" \
+    test_auth_file || \
+    failed=`expr $failed + 1`
+
 testit "rm -rf $LOGDIR" \
     rm -rf $LOGDIR || \
     failed=`expr $failed + 1`
diff --git a/source3/selftest/knownfail b/source3/selftest/knownfail
index 95f01f64dcf..4f746c8d706 100644
--- a/source3/selftest/knownfail
+++ b/source3/selftest/knownfail
@@ -1,14 +1,16 @@
-samba3.blackbox.failure # this is designed to fail, for testing our test infrastructure
+^samba3.blackbox.failure # this is designed to fail, for testing our test infrastructure
 .*printer.*print_test_extended # fails on some hosts due to timing issues ?
 .*printer.*print_test # fails on some hosts due to timing issues ?
-samba3.posix_s3.rap.printing # fails sometimes on sn-devel
-samba3.posix_s3.rpc.spoolss.*printserver.enum_printers_old # fails on some hosts due to timing issues ?
-samba3.posix_s3.rpc.spoolss.printer.*addprinterex.print_test # another intermittent failure
-samba3.posix_s3.smb2.lock.*.rw-exclusive # another intermittent failure
+^samba3.posix_s3.rap.printing # fails sometimes on sn-devel
+^samba3.posix_s3.rpc.spoolss.*printserver.enum_printers_old # fails on some hosts due to timing issues ?
+^samba3.posix_s3.rpc.spoolss.printer.*addprinterex.print_test # another intermittent failure
+^samba3.posix_s3.smb2.lock.*.rw-exclusive # another intermittent failure
 .*driver.add_driver_timestamps # we only can store dates, not timestamps
-samba3.raw.mux.* #This test is flaky on the async lock time
-samba3.smbtorture_s3.*OPLOCK4 # fails sometimes on sn-devel
-samba3.posix_s3.nbt.dgram.*netlogon2
-samba3.*rap.sam.*.useradd # Not provided by Samba 3
-samba3.*rap.sam.*.userdelete # Not provided by Samba 3
-samba3.*rap.basic.*.netsessiongetinfo # Not provided by Samba 3
+^samba3.raw.mux.* #This test is flaky on the async lock time
+^samba3.smbtorture_s3.*OPLOCK4 # fails sometimes on sn-devel
+^samba3.posix_s3.nbt.dgram.*netlogon2
+^samba3.*rap.sam.*.useradd # Not provided by Samba 3
+^samba3.*rap.sam.*.userdelete # Not provided by Samba 3
+^samba3.*rap.basic.*.netsessiongetinfo # Not provided by Samba 3
+^samba3.posix_s3.libsmbclient .opendir # This requires a workgroup called 'WORKGROUP' and for netbios browse lists to have been registered
+
diff --git a/source3/selftest/ktest-secrets.tdb2 b/source3/selftest/ktest-secrets.tdb2
new file mode 100644
index 00000000000..2af58b140f1
--- /dev/null
+++ b/source3/selftest/ktest-secrets.tdb2
diff --git a/source3/selftest/quick b/source3/selftest/quick
new file mode 100644
index 00000000000..bb3b066b498
--- /dev/null
+++ b/source3/selftest/quick
@@ -0,0 +1 @@
+#none
diff --git a/source3/selftest/skip b/source3/selftest/skip
index 02166a14578..4366ae68726 100644
--- a/source3/selftest/skip
+++ b/source3/selftest/skip
@@ -1,23 +1,23 @@
-samba3.smbtorture_s3.*.randomipc
-samba3.smbtorture_s3.*.negnowait
-samba3.smbtorture_s3.*.nbench
-samba3.smbtorture_s3.*.errmapextract
-samba3.smbtorture_s3.*.trans2scan
-samba3.smbtorture_s3.*.nttransscan
-samba3.smbtorture_s3.*.deny1
-samba3.smbtorture_s3.*.deny2
-samba3.smbtorture_s3.*.openattr
-samba3.smbtorture_s3.*.casetable
-samba3.smbtorture_s3.*.eatest
-samba3.smbtorture_s3.*.mangle
-samba3.smbtorture_s3.*.utable
-samba3.smbtorture_s3.*.pipe_number
-samba3.smbtorture_s3.*.CHAIN1
-samba3.*base.charset
-samba3.*raw.acls
-samba3.*raw.composite
-samba3.*raw.context
-samba3.*raw.ioctl
-samba3.*raw.qfileinfo
-samba3.*raw.qfsinfo
-samba3.*raw.sfileinfo.base
+^samba3.smbtorture_s3.*.randomipc
+^samba3.smbtorture_s3.*.negnowait
+^samba3.smbtorture_s3.*.nbench
+^samba3.smbtorture_s3.*.errmapextract
+^samba3.smbtorture_s3.*.trans2scan
+^samba3.smbtorture_s3.*.nttransscan
+^samba3.smbtorture_s3.*.deny1
+^samba3.smbtorture_s3.*.deny2
+^samba3.smbtorture_s3.*.openattr
+^samba3.smbtorture_s3.*.casetable
+^samba3.smbtorture_s3.*.eatest
+^samba3.smbtorture_s3.*.mangle
+^samba3.smbtorture_s3.*.utable
+^samba3.smbtorture_s3.*.pipe_number
+^samba3.smbtorture_s3.*.CHAIN1
+^samba3.*base.charset
+^samba3.*raw.acls
+^samba3.*raw.composite
+^samba3.*raw.context
+^samba3.*raw.ioctl
+^samba3.*raw.qfileinfo
+^samba3.*raw.qfsinfo
+^samba3.*raw.sfileinfo.base
diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py
index f05e00537d9..2a3002189c0 100755
--- a/source3/selftest/tests.py
+++ b/source3/selftest/tests.py
@@ -53,13 +53,14 @@ plantestsuite("samba3.local_s3", "s3dc:local", [os.path.join(samba3srcdir, "scri
 
 tests=[ "FDPASS", "LOCK1", "LOCK2", "LOCK3", "LOCK4", "LOCK5", "LOCK6", "LOCK7", "LOCK9",
         "UNLINK", "BROWSE", "ATTR", "TRANS2", "TORTURE",
-        "OPLOCK1", "OPLOCK2", "OPLOCK3", "OPLOCK4", "STREAMERROR",
+        "OPLOCK1", "OPLOCK2", "OPLOCK4", "STREAMERROR",
         "DIR", "DIR1", "DIR-CREATETIME", "TCON", "TCONDEV", "RW1", "RW2", "RW3", "RW-SIGNING",
         "OPEN", "XCOPY", "RENAME", "DELETE", "DELETE-LN", "PROPERTIES", "W2K",
         "TCON2", "IOCTL", "CHKPATH", "FDSESS", "LOCAL-SUBSTITUTE", "CHAIN1",
         "GETADDRINFO", "POSIX", "UID-REGRESSION-TEST", "SHORTNAME-TEST",
         "LOCAL-BASE64", "LOCAL-GENCACHE", "POSIX-APPEND",
         "CASE-INSENSITIVE-CREATE",
+        "BAD-NBT-SESSION",
         "LOCAL-string_to_sid", "LOCAL-CONVERT-STRING" ]
 
 for t in tests:
@@ -112,20 +113,20 @@ for env in ["secshare", "secserver"]:
 
 # plain
 for env in ["s3dc"]:
-    plantestsuite("samba3.blackbox.smbclient_s3.plain (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration])
+    plantestsuite("samba3.blackbox.smbclient_s3.plain (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration])
 
 for env in ["member"]:
-    plantestsuite("samba3.blackbox.smbclient_s3.plain (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration])
+    plantestsuite("samba3.blackbox.smbclient_s3.plain (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration])
 
 for env in ["s3dc"]:
-    plantestsuite("samba3.blackbox.smbclient_s3.sign (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration, "--signing=required"])
+    plantestsuite("samba3.blackbox.smbclient_s3.sign (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration, "--signing=required"])
 
 for env in ["member"]:
-    plantestsuite("samba3.blackbox.smbclient_s3.sign (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration, "--signing=required"])
+    plantestsuite("samba3.blackbox.smbclient_s3.sign (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration, "--signing=required"])
 
 # encrypted
 for env in ["s3dc"]:
-    plantestsuite("samba3.blackbox.smbclient_s3.crypt (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration, "-e"])
+    plantestsuite("samba3.blackbox.smbclient_s3.crypt (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), configuration, "-e"])
 
 #TODO encrypted against member, with member creds, and with DC creds
 plantestsuite("samba3.blackbox.net.misc", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_misc.sh"),
@@ -140,6 +141,9 @@ plantestsuite("samba3.blackbox.net.local.registry.roundtrip", "s3dc:local", [os.
 plantestsuite("samba3.blackbox.testparm", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_testparm_s3.sh"),
                                                        "$LOCAL_PATH"])
 
+plantestsuite(
+    "samba3.pthreadpool", "s3dc",
+    [os.path.join(samba3srcdir, "script/tests/test_pthreadpool.sh")])
 
 #smbtorture4 tests
 
@@ -157,7 +161,9 @@ raw = ["raw.acls", "raw.chkpath", "raw.close", "raw.composite", "raw.context", "
        "raw.sfileinfo.base", "raw.sfileinfo.bug", "raw.streams", "raw.unlink", "raw.write",
        "raw.samba3hide", "raw.samba3badpath", "raw.sfileinfo.rename",
        "raw.samba3caseinsensitive", "raw.samba3posixtimedlock",
-       "raw.samba3rootdirfid", "raw.sfileinfo.end-of-file"]
+       "raw.samba3rootdirfid", "raw.sfileinfo.end-of-file",
+       "raw.bench-oplock", "raw.bench-lock", "raw.bench-open", "raw.bench-tcon",
+       "raw.samba3checkfsp", "raw.samba3closeerr", "raw.samba3oplocklogoff"]
 
 smb2 = ["smb2.lock", "smb2.read", "smb2.compound", "smb2.connect", "smb2.scan", "smb2.scanfind",
         "smb2.bench-oplock"]
@@ -186,7 +192,9 @@ unix = ["unix.info2", "unix.whoami"]
 
 nbt = ["nbt.dgram" ]
 
-tests= base + raw + smb2 + rpc + unix + local + winbind + rap + nbt
+libsmbclient = ["libsmbclient"]
+
+tests= base + raw + smb2 + rpc + unix + local + winbind + rap + nbt + libsmbclient
 
 sub = subprocess.Popen("%s --version 2> /dev/null" % smb4torture, stdout=subprocess.PIPE, stdin=subprocess.PIPE, shell=True)
 sub.communicate("")
@@ -195,17 +203,17 @@ if sub.returncode == 0:
     for t in tests:
         if t == "base.delaywrite":
             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --maximum-runtime=900')
-        if t == "rap.sam":
+        elif t == "rap.sam":
             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --option=doscharset=ISO-8859-1')
         elif t == "unix.whoami":
             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpguest -U$USERNAME%$PASSWORD')
         elif t == "raw.samba3posixtimedlock":
             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpguest -U$USERNAME%$PASSWORD --option=torture:localdir=$SELFTEST_PREFIX/s3dc/share')
+        elif t == "raw.chkpath":
+            plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpcase -U$USERNAME%$PASSWORD')
         else:
             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD')
 
-        if t == "raw.chkpath":
-            plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpcase -U$USERNAME%$PASSWORD')
 
     test = 'rpc.lsa.lookupsids'
     auth_options = ["", "ntlm", "spnego" ]
diff --git a/source3/smbd/aio.c b/source3/smbd/aio.c
index 682d042806a..db2926b4a4f 100644
--- a/source3/smbd/aio.c
+++ b/source3/smbd/aio.c
@@ -21,6 +21,7 @@
 #include "includes.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 #if defined(WITH_AIO)
 
@@ -80,8 +81,8 @@ static bool initialize_async_io_handler(void)
 	}
 	tried_signal_setup = true;
 
-	aio_signal_event = tevent_add_signal(smbd_event_context(),
-					     smbd_event_context(),
+	aio_signal_event = tevent_add_signal(server_event_context(),
+					     server_event_context(),
 					     RT_SIGNAL_AIO, SA_SIGINFO,
 					     smbd_aio_signal_handler,
 					     NULL);
@@ -115,7 +116,7 @@ static struct aio_extra *create_aio_extra(TALLOC_CTX *mem_ctx,
 					files_struct *fsp,
 					size_t buflen)
 {
-	struct aio_extra *aio_ex = TALLOC_ZERO_P(mem_ctx, struct aio_extra);
+	struct aio_extra *aio_ex = talloc_zero(mem_ctx, struct aio_extra);
 
 	if (!aio_ex) {
 		return NULL;
@@ -250,7 +251,7 @@ NTSTATUS schedule_aio_read_and_X(connection_struct *conn,
 
 NTSTATUS schedule_aio_write_and_X(connection_struct *conn,
 			      struct smb_request *smbreq,
-			      files_struct *fsp, char *data,
+			      files_struct *fsp, const char *data,
 			      SMB_OFF_T startpos,
 			      size_t numtowrite)
 {
@@ -328,7 +329,7 @@ NTSTATUS schedule_aio_write_and_X(connection_struct *conn,
 	/* Now set up the aio record for the write call. */
 
 	a->aio_fildes = fsp->fh->fd;
-	a->aio_buf = data;
+	a->aio_buf = discard_const_p(char, data);
 	a->aio_nbytes = numtowrite;
 	a->aio_offset = startpos;
 	a->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
@@ -1029,7 +1030,7 @@ NTSTATUS schedule_aio_read_and_X(connection_struct *conn,
 
 NTSTATUS schedule_aio_write_and_X(connection_struct *conn,
 			      struct smb_request *smbreq,
-			      files_struct *fsp, char *data,
+			      files_struct *fsp, const char *data,
 			      SMB_OFF_T startpos,
 			      size_t numtowrite)
 {
diff --git a/source3/smbd/avahi_register.c b/source3/smbd/avahi_register.c
index 05bd6038f46..368168d41dc 100644
--- a/source3/smbd/avahi_register.c
+++ b/source3/smbd/avahi_register.c
@@ -88,7 +88,7 @@ static void avahi_client_callback(AvahiClient *c, AvahiClientState status,
 		}
 		if (avahi_entry_group_add_service(
 			    state->entry_group, AVAHI_IF_UNSPEC,
-			    AVAHI_PROTO_UNSPEC, 0, global_myname(),
+			    AVAHI_PROTO_UNSPEC, 0, lp_netbios_name(),
 			    "_smb._tcp", NULL, NULL, state->port, NULL) < 0) {
 			error = avahi_client_errno(c);
 			DEBUG(10, ("avahi_entry_group_add_service failed: "
diff --git a/source3/smbd/blocking.c b/source3/smbd/blocking.c
index c208234bdcf..d9c1fb2d3b3 100644
--- a/source3/smbd/blocking.c
+++ b/source3/smbd/blocking.c
@@ -139,7 +139,7 @@ static bool recalc_brl_timeout(struct smbd_server_connection *sconn)
 		    (int)from_now.tv_sec, (int)from_now.tv_usec));
 	}
 
-	sconn->smb1.locks.brl_timeout = event_add_timed(smbd_event_context(),
+	sconn->smb1.locks.brl_timeout = event_add_timed(server_event_context(),
 							NULL, next_timeout,
 							brl_timeout_fn, sconn);
 	if (sconn->smb1.locks.brl_timeout == NULL) {
@@ -208,8 +208,7 @@ bool push_blocking_lock_request( struct byte_range_lock *br_lck,
 		blr->expire_time.tv_sec = 0;
 		blr->expire_time.tv_usec = 0; /* Never expire. */
 	} else {
-		blr->expire_time = timeval_current_ofs(lock_timeout/1000,
-					(lock_timeout % 1000) * 1000);
+		blr->expire_time = timeval_current_ofs_msec(lock_timeout);
 	}
 	blr->lock_num = lock_num;
 	blr->smblctx = smblctx;
@@ -337,7 +336,7 @@ static void reply_lockingX_error(struct blocking_lock_record *blr, NTSTATUS stat
 	uint8_t *data;
 	int i;
 
-	data = (uint8_t *)blr->req->buf
+	data = discard_const_p(uint8_t, blr->req->buf)
 		+ ((large_file_format ? 20 : 10)*num_ulocks);
 
 	/* 
@@ -429,7 +428,7 @@ static bool process_lockingX(struct blocking_lock_record *blr)
 	uint8_t *data;
 	NTSTATUS status = NT_STATUS_OK;
 
-	data = (uint8_t *)blr->req->buf
+	data = discard_const_p(uint8_t, blr->req->buf)
 		+ ((large_file_format ? 20 : 10)*num_ulocks);
 
 	/* 
@@ -575,9 +574,9 @@ static bool blocking_lock_record_process(struct blocking_lock_record *blr)
  Called when a file is closed.
 *****************************************************************************/
 
-void cancel_pending_lock_requests_by_fid(files_struct *fsp,
-			struct byte_range_lock *br_lck,
-			enum file_close_type close_type)
+void smbd_cancel_pending_lock_requests_by_fid(files_struct *fsp,
+					      struct byte_range_lock *br_lck,
+					      enum file_close_type close_type)
 {
 	struct smbd_server_connection *sconn = fsp->conn->sconn;
 	struct blocking_lock_record *blr, *blr_cancelled, *next = NULL;
diff --git a/source3/smbd/close.c b/source3/smbd/close.c
index aeed4e3c9d7..52cfc111fbe 100644
--- a/source3/smbd/close.c
+++ b/source3/smbd/close.c
@@ -283,7 +283,7 @@ static NTSTATUS close_remove_share_mode(files_struct *fsp,
 
 	/* Ensure any pending write time updates are done. */
 	if (fsp->update_write_time_event) {
-		update_write_time_handler(smbd_event_context(),
+		update_write_time_handler(server_event_context(),
 					fsp->update_write_time_event,
 					timeval_current(),
 					(void *)fsp);
diff --git a/source3/smbd/conn.c b/source3/smbd/conn.c
index 16a0ee053e9..a3f66b36be7 100644
--- a/source3/smbd/conn.c
+++ b/source3/smbd/conn.c
@@ -53,10 +53,9 @@ int conn_num_open(struct smbd_server_connection *sconn)
  Check if a snum is in use.
 ****************************************************************************/
 
-bool conn_snum_used(int snum)
+bool conn_snum_used(struct smbd_server_connection *sconn,
+		    int snum)
 {
-	struct smbd_server_connection *sconn = smbd_server_conn;
-
 	if (sconn->using_smb2) {
 		/* SMB2 */
 		struct smbd_smb2_session *sess;
@@ -134,8 +133,8 @@ connection_struct *conn_new(struct smbd_server_connection *sconn)
 
 	if (sconn->using_smb2) {
 		/* SMB2 */
-		if (!(conn=TALLOC_ZERO_P(NULL, connection_struct)) ||
-		    !(conn->params = TALLOC_P(conn, struct share_params))) {
+		if (!(conn=talloc_zero(NULL, connection_struct)) ||
+		    !(conn->params = talloc(conn, struct share_params))) {
 			DEBUG(0,("TALLOC_ZERO() failed!\n"));
 			TALLOC_FREE(conn);
 			return NULL;
@@ -189,8 +188,8 @@ find_again:
 		return NULL;
 	}
 
-	if (!(conn=TALLOC_ZERO_P(NULL, connection_struct)) ||
-	    !(conn->params = TALLOC_P(conn, struct share_params))) {
+	if (!(conn=talloc_zero(NULL, connection_struct)) ||
+	    !(conn->params = talloc(conn, struct share_params))) {
 		DEBUG(0,("TALLOC_ZERO() failed!\n"));
 		TALLOC_FREE(conn);
 		return NULL;
@@ -212,142 +211,52 @@ find_again:
 }
 
 /****************************************************************************
- Close all conn structures.
- Return true if any were closed.
-****************************************************************************/
-
-bool conn_close_all(struct smbd_server_connection *sconn)
-{
-	bool ret = false;
-	if (sconn->using_smb2) {
-		/* SMB2 */
-		struct smbd_smb2_session *sess;
-		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
-			struct smbd_smb2_tcon *tcon, *tc_next;
-
-			for (tcon = sess->tcons.list; tcon; tcon = tc_next) {
-				tc_next = tcon->next;
-				TALLOC_FREE(tcon);
-				ret = true;
-			}
-		}
-	} else {
-		/* SMB1 */
-		connection_struct *conn, *next;
-
-		for (conn=sconn->smb1.tcons.Connections;conn;conn=next) {
-			next=conn->next;
-			set_current_service(conn, 0, True);
-			close_cnum(conn, conn->vuid);
-			ret = true;
-		}
-	}
-	return ret;
-}
-
-/****************************************************************************
- Update last used timestamps.
-****************************************************************************/
-
-static void conn_lastused_update(struct smbd_server_connection *sconn,time_t t)
-{
-	if (sconn->using_smb2) {
-		/* SMB2 */
-		struct smbd_smb2_session *sess;
-		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
-			struct smbd_smb2_tcon *ptcon;
-
-			for (ptcon = sess->tcons.list; ptcon; ptcon = ptcon->next) {
-				connection_struct *conn = ptcon->compat_conn;
-				/* Update if connection wasn't idle. */
-				if (conn && conn->lastused != conn->lastused_count) {
-					conn->lastused = t;
-					conn->lastused_count = t;
-				}
-			}
-		}
-	} else {
-		/* SMB1 */
-		connection_struct *conn;
-		for (conn=sconn->smb1.tcons.Connections;conn;conn=conn->next) {
-			/* Update if connection wasn't idle. */
-			if (conn->lastused != conn->lastused_count) {
-				conn->lastused = t;
-				conn->lastused_count = t;
-			}
-		}
-	}
-}
-
-/****************************************************************************
- Idle inactive connections.
+ Clear a vuid out of the connection's vuid cache
 ****************************************************************************/
 
-bool conn_idle_all(struct smbd_server_connection *sconn, time_t t)
+static void conn_clear_vuid_cache(connection_struct *conn, uint16_t vuid)
 {
-	int deadtime = lp_deadtime()*60;
-
-	conn_lastused_update(sconn, t);
-
-	if (deadtime <= 0) {
-		deadtime = DEFAULT_SMBD_TIMEOUT;
-	}
-
-	if (sconn->using_smb2) {
-		/* SMB2 */
-		struct smbd_smb2_session *sess;
-		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
-			struct smbd_smb2_tcon *ptcon;
-
-			for (ptcon = sess->tcons.list; ptcon; ptcon = ptcon->next) {
-				time_t age;
-				connection_struct *conn = ptcon->compat_conn;
-
-				if (conn == NULL) {
-					continue;
-				}
-
-				age = t - conn->lastused;
-				/* close dirptrs on connections that are idle */
-				if (age > DPTR_IDLE_TIMEOUT) {
-					dptr_idlecnum(conn);
-				}
-
-				if (conn->num_files_open > 0 || age < deadtime) {
-					return false;
-				}
-			}
-		}
-	} else {
-		/* SMB1 */
-		connection_struct *conn;
-		for (conn=sconn->smb1.tcons.Connections;conn;conn=conn->next) {
-			time_t age = t - conn->lastused;
-
-			/* close dirptrs on connections that are idle */
-			if (age > DPTR_IDLE_TIMEOUT) {
-				dptr_idlecnum(conn);
-			}
+	int i;
 
-			if (conn->num_files_open > 0 || age < deadtime) {
-				return false;
+	for (i=0; i<VUID_CACHE_SIZE; i++) {
+		struct vuid_cache_entry *ent;
+
+		ent = &conn->vuid_cache.array[i];
+
+		if (ent->vuid == vuid) {
+			ent->vuid = UID_FIELD_INVALID;
+			/*
+			 * We need to keep conn->session_info around
+			 * if it's equal to ent->session_info as a SMBulogoff
+			 * is often followed by a SMBtdis (with an invalid
+			 * vuid). The debug code (or regular code in
+			 * vfs_full_audit) wants to refer to the
+			 * conn->session_info pointer to print debug
+			 * statements. Theoretically this is a bug,
+			 * as once the vuid is gone the session_info
+			 * on the conn struct isn't valid any more,
+			 * but there's enough code that assumes
+			 * conn->session_info is never null that
+			 * it's easier to hold onto the old pointer
+			 * until we get a new sessionsetupX.
+			 * As everything is hung off the
+			 * conn pointer as a talloc context we're not
+			 * leaking memory here. See bug #6315. JRA.
+			 */
+			if (conn->session_info == ent->session_info) {
+				ent->session_info = NULL;
+			} else {
+				TALLOC_FREE(ent->session_info);
 			}
+			ent->read_only = False;
 		}
 	}
-
-	/*
-	 * Check all pipes for any open handles. We cannot
-	 * idle with a handle open.
-	 */
-	if (check_open_pipes()) {
-		return false;
-	}
-
-	return true;
 }
 
 /****************************************************************************
  Clear a vuid out of the validity cache, and as the 'owner' of a connection.
+
+ Called from invalidate_vuid()
 ****************************************************************************/
 
 void conn_clear_vuid_caches(struct smbd_server_connection *sconn,uint16_t vuid)
@@ -451,64 +360,3 @@ void conn_free(connection_struct *conn)
 
 	conn_free_internal(conn);
 }
-
-/****************************************************************************
- Receive a smbcontrol message to forcibly unmount a share.
- The message contains just a share name and all instances of that
- share are unmounted.
- The special sharename '*' forces unmount of all shares.
-****************************************************************************/
-
-void msg_force_tdis(struct messaging_context *msg,
-		    void *private_data,
-		    uint32_t msg_type,
-		    struct server_id server_id,
-		    DATA_BLOB *data)
-{
-	struct smbd_server_connection *sconn;
-	connection_struct *conn, *next;
-	fstring sharename;
-
-	sconn = msg_ctx_to_sconn(msg);
-	if (sconn == NULL) {
-		DEBUG(1, ("could not find sconn\n"));
-		return;
-	}
-
-	fstrcpy(sharename, (const char *)data->data);
-
-	if (strcmp(sharename, "*") == 0) {
-		DEBUG(1,("Forcing close of all shares\n"));
-		conn_close_all(sconn);
-		return;
-	}
-
-	if (sconn->using_smb2) {
-		/* SMB2 */
-		struct smbd_smb2_session *sess;
-		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
-			struct smbd_smb2_tcon *tcon, *tc_next;
-
-			for (tcon = sess->tcons.list; tcon; tcon = tc_next) {
-				tc_next = tcon->next;
-				if (tcon->compat_conn &&
-						strequal(lp_servicename(SNUM(tcon->compat_conn)),
-								sharename)) {
-					DEBUG(1,("Forcing close of share %s cnum=%d\n",
-						sharename, tcon->compat_conn->cnum));
-					TALLOC_FREE(tcon);
-				}
-			}
-		}
-	} else {
-		/* SMB1 */
-		for (conn=sconn->smb1.tcons.Connections;conn;conn=next) {
-			next=conn->next;
-			if (strequal(lp_servicename(SNUM(conn)), sharename)) {
-				DEBUG(1,("Forcing close of share %s cnum=%d\n",
-					sharename, conn->cnum));
-				close_cnum(conn, (uint16)-1);
-			}
-		}
-	}
-}
diff --git a/source3/smbd/conn_idle.c b/source3/smbd/conn_idle.c
new file mode 100644
index 00000000000..46f5b7b111c
--- /dev/null
+++ b/source3/smbd/conn_idle.c
@@ -0,0 +1,207 @@
+/*
+   Unix SMB/CIFS implementation.
+   Manage connections_struct structures
+   Copyright (C) Andrew Tridgell 1998
+   Copyright (C) Alexander Bokovoy 2002
+   Copyright (C) Jeremy Allison 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "smbd/smbd.h"
+#include "smbd/globals.h"
+#include "rpc_server/rpc_ncacn_np.h"
+
+/****************************************************************************
+ Update last used timestamps.
+****************************************************************************/
+
+static void conn_lastused_update(struct smbd_server_connection *sconn,time_t t)
+{
+	if (sconn->using_smb2) {
+		/* SMB2 */
+		struct smbd_smb2_session *sess;
+		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
+			struct smbd_smb2_tcon *ptcon;
+
+			for (ptcon = sess->tcons.list; ptcon; ptcon = ptcon->next) {
+				connection_struct *conn = ptcon->compat_conn;
+				/* Update if connection wasn't idle. */
+				if (conn && conn->lastused != conn->lastused_count) {
+					conn->lastused = t;
+					conn->lastused_count = t;
+				}
+			}
+		}
+	} else {
+		/* SMB1 */
+		connection_struct *conn;
+		for (conn=sconn->smb1.tcons.Connections;conn;conn=conn->next) {
+			/* Update if connection wasn't idle. */
+			if (conn->lastused != conn->lastused_count) {
+				conn->lastused = t;
+				conn->lastused_count = t;
+			}
+		}
+	}
+}
+
+/****************************************************************************
+ Idle inactive connections.
+****************************************************************************/
+
+bool conn_idle_all(struct smbd_server_connection *sconn, time_t t)
+{
+	int deadtime = lp_deadtime()*60;
+
+	conn_lastused_update(sconn, t);
+
+	if (deadtime <= 0) {
+		deadtime = DEFAULT_SMBD_TIMEOUT;
+	}
+
+	if (sconn->using_smb2) {
+		/* SMB2 */
+		struct smbd_smb2_session *sess;
+		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
+			struct smbd_smb2_tcon *ptcon;
+
+			for (ptcon = sess->tcons.list; ptcon; ptcon = ptcon->next) {
+				time_t age;
+				connection_struct *conn = ptcon->compat_conn;
+
+				if (conn == NULL) {
+					continue;
+				}
+
+				age = t - conn->lastused;
+				/* close dirptrs on connections that are idle */
+				if (age > DPTR_IDLE_TIMEOUT) {
+					dptr_idlecnum(conn);
+				}
+
+				if (conn->num_files_open > 0 || age < deadtime) {
+					return false;
+				}
+			}
+		}
+	} else {
+		/* SMB1 */
+		connection_struct *conn;
+		for (conn=sconn->smb1.tcons.Connections;conn;conn=conn->next) {
+			time_t age = t - conn->lastused;
+
+			/* close dirptrs on connections that are idle */
+			if (age > DPTR_IDLE_TIMEOUT) {
+				dptr_idlecnum(conn);
+			}
+
+			if (conn->num_files_open > 0 || age < deadtime) {
+				return false;
+			}
+		}
+	}
+
+	/*
+	 * Check all pipes for any open handles. We cannot
+	 * idle with a handle open.
+	 */
+	if (check_open_pipes()) {
+		return false;
+	}
+
+	return true;
+}
+
+/****************************************************************************
+ Close all conn structures.
+ Return true if any were closed.
+****************************************************************************/
+
+bool conn_close_all(struct smbd_server_connection *sconn)
+{
+	bool ret = false;
+	if (sconn->using_smb2) {
+		/* SMB2 */
+		struct smbd_smb2_session *sess;
+		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
+			struct smbd_smb2_tcon *tcon, *tc_next;
+
+			for (tcon = sess->tcons.list; tcon; tcon = tc_next) {
+				tc_next = tcon->next;
+				TALLOC_FREE(tcon);
+				ret = true;
+			}
+		}
+	} else {
+		/* SMB1 */
+		connection_struct *conn, *next;
+
+		for (conn=sconn->smb1.tcons.Connections;conn;conn=next) {
+			next=conn->next;
+			set_current_service(conn, 0, True);
+			close_cnum(conn, conn->vuid);
+			ret = true;
+		}
+	}
+	return ret;
+}
+
+
+/****************************************************************************
+ Forcibly unmount a share.
+ All instances of the parameter 'sharename' share are unmounted.
+ The special sharename '*' forces unmount of all shares.
+****************************************************************************/
+
+void conn_force_tdis(struct smbd_server_connection *sconn, const char *sharename)
+{
+	connection_struct *conn, *next;
+
+	if (strcmp(sharename, "*") == 0) {
+		DEBUG(1,("Forcing close of all shares\n"));
+		conn_close_all(sconn);
+		return;
+	}
+
+	if (sconn->using_smb2) {
+		/* SMB2 */
+		struct smbd_smb2_session *sess;
+		for (sess = sconn->smb2.sessions.list; sess; sess = sess->next) {
+			struct smbd_smb2_tcon *tcon, *tc_next;
+
+			for (tcon = sess->tcons.list; tcon; tcon = tc_next) {
+				tc_next = tcon->next;
+				if (tcon->compat_conn &&
+						strequal(lp_servicename(SNUM(tcon->compat_conn)),
+								sharename)) {
+					DEBUG(1,("Forcing close of share %s cnum=%d\n",
+						sharename, tcon->compat_conn->cnum));
+					TALLOC_FREE(tcon);
+				}
+			}
+		}
+	} else {
+		/* SMB1 */
+		for (conn=sconn->smb1.tcons.Connections;conn;conn=next) {
+			next=conn->next;
+			if (strequal(lp_servicename(SNUM(conn)), sharename)) {
+				DEBUG(1,("Forcing close of share %s cnum=%d\n",
+					sharename, conn->cnum));
+				close_cnum(conn, (uint16)-1);
+			}
+		}
+	}
+}
diff --git a/source3/smbd/conn_msg.c b/source3/smbd/conn_msg.c
new file mode 100644
index 00000000000..b0c968a6807
--- /dev/null
+++ b/source3/smbd/conn_msg.c
@@ -0,0 +1,49 @@
+/*
+   Unix SMB/CIFS implementation.
+   Manage connections_struct structures
+   Copyright (C) Andrew Tridgell 1998
+   Copyright (C) Alexander Bokovoy 2002
+   Copyright (C) Jeremy Allison 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "smbd/smbd.h"
+#include "smbd/globals.h"
+
+/****************************************************************************
+ Receive a smbcontrol message to forcibly unmount a share.
+ The message contains just a share name and all instances of that
+ share are unmounted.
+ The special sharename '*' forces unmount of all shares.
+****************************************************************************/
+
+void msg_force_tdis(struct messaging_context *msg,
+		    void *private_data,
+		    uint32_t msg_type,
+		    struct server_id server_id,
+		    DATA_BLOB *data)
+{
+	struct smbd_server_connection *sconn;
+	const char *sharename = (const char *)data->data;
+
+	sconn = msg_ctx_to_sconn(msg);
+	if (sconn == NULL) {
+		DEBUG(1, ("could not find sconn\n"));
+		return;
+	}
+
+	conn_force_tdis(sconn, sharename);
+}
diff --git a/source3/smbd/connection.c b/source3/smbd/connection.c
index 8560a5d211b..7ed9518756b 100644
--- a/source3/smbd/connection.c
+++ b/source3/smbd/connection.c
@@ -101,6 +101,7 @@ static int count_fn(struct db_record *rec,
 int count_current_connections( const char *sharename, bool clear  )
 {
 	struct count_stat cs;
+	int ret;
 
 	cs.curr_connections = 0;
 	cs.name = sharename;
@@ -111,10 +112,18 @@ int count_current_connections( const char *sharename, bool clear  )
 	 * as it leads to deadlock.
 	 */
 
-	if (connections_forall(count_fn, &cs) == -1) {
+	/*
+	 * become_root() because we might have to open connections.tdb
+	 * via ctdb, which is not possible without root.
+	 */
+	become_root();
+	ret = connections_forall(count_fn, &cs);
+	unbecome_root();
+
+	if (ret < 0) {
 		DEBUG(0,("count_current_connections: traverse of "
 			 "connections.tdb failed\n"));
-		return False;
+		return 0;
 	}
 
 	return cs.curr_connections;
diff --git a/source3/smbd/dfree.c b/source3/smbd/dfree.c
index 63c287cd417..03541880e15 100644
--- a/source3/smbd/dfree.c
+++ b/source3/smbd/dfree.c
@@ -207,7 +207,7 @@ uint64_t get_dfree_info(connection_struct *conn,
 
 	/* No cached info or time to refresh. */
 	if (!dfc) {
-		dfc = TALLOC_P(conn, struct dfree_cached_info);
+		dfc = talloc(conn, struct dfree_cached_info);
 		if (!dfc) {
 			return dfree_ret;
 		}
diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c
index 5fc87a608fd..fda7c34c578 100644
--- a/source3/smbd/dir.c
+++ b/source3/smbd/dir.c
@@ -92,7 +92,7 @@ bool make_dir_struct(TALLOC_CTX *ctx,
 		return False;
 	}
 
-	if ((mode & aDIR) != 0) {
+	if ((mode & FILE_ATTRIBUTE_DIRECTORY) != 0) {
 		size = 0;
 	}
 
@@ -872,15 +872,15 @@ bool dir_check_ftype(connection_struct *conn, uint32 mode, uint32 dirtype)
 	uint32 mask;
 
 	/* Check the "may have" search bits. */
-	if (((mode & ~dirtype) & (aHIDDEN | aSYSTEM | aDIR)) != 0)
+	if (((mode & ~dirtype) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_DIRECTORY)) != 0)
 		return False;
 
 	/* Check the "must have" bits, which are the may have bits shifted eight */
 	/* If must have bit is set, the file/dir can not be returned in search unless the matching
 		file attribute is set */
-	mask = ((dirtype >> 8) & (aDIR|aARCH|aRONLY|aHIDDEN|aSYSTEM)); /* & 0x37 */
+	mask = ((dirtype >> 8) & (FILE_ATTRIBUTE_DIRECTORY|FILE_ATTRIBUTE_ARCHIVE|FILE_ATTRIBUTE_READONLY|FILE_ATTRIBUTE_HIDDEN|FILE_ATTRIBUTE_SYSTEM)); /* & 0x37 */
 	if(mask) {
-		if((mask & (mode & (aDIR|aARCH|aRONLY|aHIDDEN|aSYSTEM))) == mask)   /* check if matching attribute present */
+		if((mask & (mode & (FILE_ATTRIBUTE_DIRECTORY|FILE_ATTRIBUTE_ARCHIVE|FILE_ATTRIBUTE_READONLY|FILE_ATTRIBUTE_HIDDEN|FILE_ATTRIBUTE_SYSTEM))) == mask)   /* check if matching attribute present */
 			return True;
 		else
 			return False;
@@ -1342,7 +1342,7 @@ struct smb_Dir *OpenDir(TALLOC_CTX *mem_ctx, connection_struct *conn,
 			const char *mask,
 			uint32 attr)
 {
-	struct smb_Dir *dirp = TALLOC_ZERO_P(mem_ctx, struct smb_Dir);
+	struct smb_Dir *dirp = talloc_zero(mem_ctx, struct smb_Dir);
 	struct smbd_server_connection *sconn = conn->sconn;
 
 	if (!dirp) {
@@ -1386,7 +1386,7 @@ static struct smb_Dir *OpenDir_fsp(TALLOC_CTX *mem_ctx, connection_struct *conn,
 			const char *mask,
 			uint32 attr)
 {
-	struct smb_Dir *dirp = TALLOC_ZERO_P(mem_ctx, struct smb_Dir);
+	struct smb_Dir *dirp = talloc_zero(mem_ctx, struct smb_Dir);
 	struct smbd_server_connection *sconn = conn->sconn;
 
 	if (!dirp) {
@@ -1559,7 +1559,7 @@ void DirCacheAdd(struct smb_Dir *dirp, const char *name, long offset)
 	}
 
 	if (dirp->name_cache == NULL) {
-		dirp->name_cache = TALLOC_ZERO_ARRAY(
+		dirp->name_cache = talloc_zero_array(
 			dirp, struct name_cache_entry, dirp->name_cache_size);
 
 		if (dirp->name_cache == NULL) {
@@ -1625,8 +1625,8 @@ bool SearchDir(struct smb_Dir *dirp, const char *name, long *poffset)
  Is this directory empty ?
 *****************************************************************/
 
-NTSTATUS can_delete_directory(struct connection_struct *conn,
-				const char *dirname)
+NTSTATUS smbd_can_delete_directory(struct connection_struct *conn,
+				   const char *dirname)
 {
 	NTSTATUS status = NT_STATUS_OK;
 	long dirpos = 0;
diff --git a/source3/smbd/dmapi.c b/source3/smbd/dmapi.c
index 2869b6d781c..8e80a588c5f 100644
--- a/source3/smbd/dmapi.c
+++ b/source3/smbd/dmapi.c
@@ -96,7 +96,7 @@ static int dmapi_init_session(struct smbd_dmapi_context *ctx)
 	do {
 		dm_sessid_t *new_sessions;
 		nsessions *= 2;
-		new_sessions = TALLOC_REALLOC_ARRAY(tmp_ctx, sessions, 
+		new_sessions = talloc_realloc(tmp_ctx, sessions, 
 						    dm_sessid_t, nsessions);
 		if (new_sessions == NULL) {
 			talloc_free(tmp_ctx);
@@ -289,7 +289,7 @@ uint32 dmapi_file_flags(const char * const path)
 	become_root();
 #endif
 
-	err = dm_path_to_handle(CONST_DISCARD(char *, path),
+	err = dm_path_to_handle(discard_const_p(char, path),
 		&dm_handle, &dm_handle_len);
 	if (err < 0) {
 		DEBUG(DMAPI_TRACE, ("dm_path_to_handle(%s): %s\n",
@@ -308,7 +308,7 @@ uint32 dmapi_file_flags(const char * const path)
 
 		set_effective_capability(DMAPI_ACCESS_CAPABILITY);
 
-		err = dm_path_to_handle(CONST_DISCARD(char *, path),
+		err = dm_path_to_handle(discard_const_p(char, path),
 			&dm_handle, &dm_handle_len);
 		if (err < 0) {
 			DEBUG(DMAPI_TRACE,
diff --git a/source3/smbd/dosmode.c b/source3/smbd/dosmode.c
index a9b6dfef78e..823d44d0aaf 100644
--- a/source3/smbd/dosmode.c
+++ b/source3/smbd/dosmode.c
@@ -41,7 +41,7 @@ static int set_link_read_only_flag(const SMB_STRUCT_STAT *const sbuf)
 #ifdef S_ISLNK
 #if LINKS_READ_ONLY
 	if (S_ISLNK(sbuf->st_mode) && S_ISDIR(sbuf->st_mode))
-		return aRONLY;
+		return FILE_ATTRIBUTE_READONLY;
 #endif
 #endif
 	return 0;
@@ -173,36 +173,36 @@ static uint32 dos_mode_from_sbuf(connection_struct *conn,
 	if (ro_opts == MAP_READONLY_YES) {
 		/* Original Samba method - map inverse of user "w" bit. */
 		if ((smb_fname->st.st_ex_mode & S_IWUSR) == 0) {
-			result |= aRONLY;
+			result |= FILE_ATTRIBUTE_READONLY;
 		}
 	} else if (ro_opts == MAP_READONLY_PERMISSIONS) {
 		/* Check actual permissions for read-only. */
 		if (!can_write_to_file(conn, smb_fname)) {
-			result |= aRONLY;
+			result |= FILE_ATTRIBUTE_READONLY;
 		}
 	} /* Else never set the readonly bit. */
 
 	if (MAP_ARCHIVE(conn) && ((smb_fname->st.st_ex_mode & S_IXUSR) != 0))
-		result |= aARCH;
+		result |= FILE_ATTRIBUTE_ARCHIVE;
 
 	if (MAP_SYSTEM(conn) && ((smb_fname->st.st_ex_mode & S_IXGRP) != 0))
-		result |= aSYSTEM;
+		result |= FILE_ATTRIBUTE_SYSTEM;
 
 	if (MAP_HIDDEN(conn) && ((smb_fname->st.st_ex_mode & S_IXOTH) != 0))
-		result |= aHIDDEN;   
+		result |= FILE_ATTRIBUTE_HIDDEN;
 
 	if (S_ISDIR(smb_fname->st.st_ex_mode))
-		result = aDIR | (result & aRONLY);
+		result = FILE_ATTRIBUTE_DIRECTORY | (result & FILE_ATTRIBUTE_READONLY);
 
 	result |= set_link_read_only_flag(&smb_fname->st);
 
 	DEBUG(8,("dos_mode_from_sbuf returning "));
 
-	if (result & aHIDDEN) DEBUG(8, ("h"));
-	if (result & aRONLY ) DEBUG(8, ("r"));
-	if (result & aSYSTEM) DEBUG(8, ("s"));
-	if (result & aDIR   ) DEBUG(8, ("d"));
-	if (result & aARCH  ) DEBUG(8, ("a"));
+	if (result & FILE_ATTRIBUTE_HIDDEN) DEBUG(8, ("h"));
+	if (result & FILE_ATTRIBUTE_READONLY ) DEBUG(8, ("r"));
+	if (result & FILE_ATTRIBUTE_SYSTEM) DEBUG(8, ("s"));
+	if (result & FILE_ATTRIBUTE_DIRECTORY   ) DEBUG(8, ("d"));
+	if (result & FILE_ATTRIBUTE_ARCHIVE  ) DEBUG(8, ("a"));
 
 	DEBUG(8,("\n"));
 	return result;
@@ -318,18 +318,18 @@ static bool get_ea_dos_attribute(connection_struct *conn,
 	}
 
 	if (S_ISDIR(smb_fname->st.st_ex_mode)) {
-		dosattr |= aDIR;
+		dosattr |= FILE_ATTRIBUTE_DIRECTORY;
 	}
 	/* FILE_ATTRIBUTE_SPARSE is valid on get but not on set. */
 	*pattr = (uint32)(dosattr & (SAMBA_ATTRIBUTES_MASK|FILE_ATTRIBUTE_SPARSE));
 
 	DEBUG(8,("get_ea_dos_attribute returning (0x%x)", dosattr));
 
-	if (dosattr & aHIDDEN) DEBUG(8, ("h"));
-	if (dosattr & aRONLY ) DEBUG(8, ("r"));
-	if (dosattr & aSYSTEM) DEBUG(8, ("s"));
-	if (dosattr & aDIR   ) DEBUG(8, ("d"));
-	if (dosattr & aARCH  ) DEBUG(8, ("a"));
+	if (dosattr & FILE_ATTRIBUTE_HIDDEN) DEBUG(8, ("h"));
+	if (dosattr & FILE_ATTRIBUTE_READONLY ) DEBUG(8, ("r"));
+	if (dosattr & FILE_ATTRIBUTE_SYSTEM) DEBUG(8, ("s"));
+	if (dosattr & FILE_ATTRIBUTE_DIRECTORY   ) DEBUG(8, ("d"));
+	if (dosattr & FILE_ATTRIBUTE_ARCHIVE  ) DEBUG(8, ("a"));
 
 	DEBUG(8,("\n"));
 
@@ -465,7 +465,7 @@ uint32 dos_mode_msdfs(connection_struct *conn,
 		/* Only . and .. are not hidden. */
 		if (p[0] == '.' && !((p[1] == '\0') ||
 				(p[1] == '.' && p[2] == '\0'))) {
-			result |= aHIDDEN;
+			result |= FILE_ATTRIBUTE_HIDDEN;
 		}
 	}
 
@@ -473,9 +473,9 @@ uint32 dos_mode_msdfs(connection_struct *conn,
 
 	/* Optimization : Only call is_hidden_path if it's not already
 	   hidden. */
-	if (!(result & aHIDDEN) &&
+	if (!(result & FILE_ATTRIBUTE_HIDDEN) &&
 	    IS_HIDDEN_PATH(conn, smb_fname->base_name)) {
-		result |= aHIDDEN;
+		result |= FILE_ATTRIBUTE_HIDDEN;
 	}
 
 	if (result == 0) {
@@ -486,11 +486,11 @@ uint32 dos_mode_msdfs(connection_struct *conn,
 
 	DEBUG(8,("dos_mode_msdfs returning "));
 
-	if (result & aHIDDEN) DEBUG(8, ("h"));
-	if (result & aRONLY ) DEBUG(8, ("r"));
-	if (result & aSYSTEM) DEBUG(8, ("s"));
-	if (result & aDIR   ) DEBUG(8, ("d"));
-	if (result & aARCH  ) DEBUG(8, ("a"));
+	if (result & FILE_ATTRIBUTE_HIDDEN) DEBUG(8, ("h"));
+	if (result & FILE_ATTRIBUTE_READONLY ) DEBUG(8, ("r"));
+	if (result & FILE_ATTRIBUTE_SYSTEM) DEBUG(8, ("s"));
+	if (result & FILE_ATTRIBUTE_DIRECTORY   ) DEBUG(8, ("d"));
+	if (result & FILE_ATTRIBUTE_ARCHIVE  ) DEBUG(8, ("a"));
 	if (result & FILE_ATTRIBUTE_SPARSE ) DEBUG(8, ("[sparse]"));
 
 	DEBUG(8,("\n"));
@@ -507,13 +507,13 @@ int dos_attributes_to_stat_dos_flags(uint32_t dosmode)
 {
 	uint32_t dos_stat_flags = 0;
 
-	if (dosmode & aARCH)
+	if (dosmode & FILE_ATTRIBUTE_ARCHIVE)
 		dos_stat_flags |= UF_DOS_ARCHIVE;
-	if (dosmode & aHIDDEN)
+	if (dosmode & FILE_ATTRIBUTE_HIDDEN)
 		dos_stat_flags |= UF_DOS_HIDDEN;
-	if (dosmode & aRONLY)
+	if (dosmode & FILE_ATTRIBUTE_READONLY)
 		dos_stat_flags |= UF_DOS_RO;
-	if (dosmode & aSYSTEM)
+	if (dosmode & FILE_ATTRIBUTE_SYSTEM)
 		dos_stat_flags |= UF_DOS_SYSTEM;
 	if (dosmode & FILE_ATTRIBUTE_NONINDEXED)
 		dos_stat_flags |= UF_DOS_NOINDEX;
@@ -540,19 +540,19 @@ static bool get_stat_dos_flags(connection_struct *conn,
 		  smb_fname_str_dbg(smb_fname)));
 
 	if (smb_fname->st.st_ex_flags & UF_DOS_ARCHIVE)
-		*dosmode |= aARCH;
+		*dosmode |= FILE_ATTRIBUTE_ARCHIVE;
 	if (smb_fname->st.st_ex_flags & UF_DOS_HIDDEN)
-		*dosmode |= aHIDDEN;
+		*dosmode |= FILE_ATTRIBUTE_HIDDEN;
 	if (smb_fname->st.st_ex_flags & UF_DOS_RO)
-		*dosmode |= aRONLY;
+		*dosmode |= FILE_ATTRIBUTE_READONLY;
 	if (smb_fname->st.st_ex_flags & UF_DOS_SYSTEM)
-		*dosmode |= aSYSTEM;
+		*dosmode |= FILE_ATTRIBUTE_SYSTEM;
 	if (smb_fname->st.st_ex_flags & UF_DOS_NOINDEX)
 		*dosmode |= FILE_ATTRIBUTE_NONINDEXED;
 	if (smb_fname->st.st_ex_flags & FILE_ATTRIBUTE_SPARSE)
 		*dosmode |= FILE_ATTRIBUTE_SPARSE;
 	if (S_ISDIR(smb_fname->st.st_ex_mode))
-		*dosmode |= aDIR;
+		*dosmode |= FILE_ATTRIBUTE_DIRECTORY;
 
 	*dosmode |= set_link_read_only_flag(&smb_fname->st);
 
@@ -637,7 +637,7 @@ uint32 dos_mode(connection_struct *conn, struct smb_filename *smb_fname)
 		/* Only . and .. are not hidden. */
 		if (p[0] == '.' && !((p[1] == '\0') ||
 				(p[1] == '.' && p[2] == '\0'))) {
-			result |= aHIDDEN;
+			result |= FILE_ATTRIBUTE_HIDDEN;
 		}
 	}
 
@@ -658,9 +658,9 @@ uint32 dos_mode(connection_struct *conn, struct smb_filename *smb_fname)
 
 	/* Optimization : Only call is_hidden_path if it's not already
 	   hidden. */
-	if (!(result & aHIDDEN) &&
+	if (!(result & FILE_ATTRIBUTE_HIDDEN) &&
 	    IS_HIDDEN_PATH(conn, smb_fname->base_name)) {
-		result |= aHIDDEN;
+		result |= FILE_ATTRIBUTE_HIDDEN;
 	}
 
 	if (result == 0) {
@@ -671,11 +671,11 @@ uint32 dos_mode(connection_struct *conn, struct smb_filename *smb_fname)
 
 	DEBUG(8,("dos_mode returning "));
 
-	if (result & aHIDDEN) DEBUG(8, ("h"));
-	if (result & aRONLY ) DEBUG(8, ("r"));
-	if (result & aSYSTEM) DEBUG(8, ("s"));
-	if (result & aDIR   ) DEBUG(8, ("d"));
-	if (result & aARCH  ) DEBUG(8, ("a"));
+	if (result & FILE_ATTRIBUTE_HIDDEN) DEBUG(8, ("h"));
+	if (result & FILE_ATTRIBUTE_READONLY ) DEBUG(8, ("r"));
+	if (result & FILE_ATTRIBUTE_SYSTEM) DEBUG(8, ("s"));
+	if (result & FILE_ATTRIBUTE_DIRECTORY   ) DEBUG(8, ("d"));
+	if (result & FILE_ATTRIBUTE_ARCHIVE  ) DEBUG(8, ("a"));
 	if (result & FILE_ATTRIBUTE_SPARSE ) DEBUG(8, ("[sparse]"));
 
 	DEBUG(8,("\n"));
@@ -712,9 +712,9 @@ int file_set_dosmode(connection_struct *conn, struct smb_filename *smb_fname,
 			   &smb_fname->st.st_ex_mode);
 
 	if (S_ISDIR(smb_fname->st.st_ex_mode))
-		dosmode |= aDIR;
+		dosmode |= FILE_ATTRIBUTE_DIRECTORY;
 	else
-		dosmode &= ~aDIR;
+		dosmode &= ~FILE_ATTRIBUTE_DIRECTORY;
 
 	new_create_timespec = smb_fname->st.st_ex_btime;
 
diff --git a/source3/smbd/error.c b/source3/smbd/error.c
index 6655a6fb082..2fb4241968d 100644
--- a/source3/smbd/error.c
+++ b/source3/smbd/error.c
@@ -21,9 +21,6 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 
-/* From lib/error.c */
-extern struct unix_error_map unix_dos_nt_errmap[];
-
 bool use_nt_status(void)
 {
 	return lp_nt_status_support() && (global_client_caps & CAP_STATUS32);
@@ -80,11 +77,10 @@ void error_packet_set(char *outbuf, uint8 eclass, uint32 ecode, NTSTATUS ntstatu
 			 smb_fn_name(CVAL(outbuf,smb_com)),
 			 nt_errstr(ntstatus)));
 	} else {
-		/* We're returning a DOS error only. */
-		if (NT_STATUS_IS_DOS(ntstatus)) {
-			eclass = NT_STATUS_DOS_CLASS(ntstatus);
-			ecode = NT_STATUS_DOS_CODE(ntstatus);
-		} else 	if (eclass == 0 && NT_STATUS_V(ntstatus)) {
+		/* We're returning a DOS error only,
+		 * nt_status_to_dos() pulls DOS error codes out of the
+		 * NTSTATUS */
+		if (NT_STATUS_IS_DOS(ntstatus) || (eclass == 0 && NT_STATUS_V(ntstatus))) {
 			ntstatus_to_dos(ntstatus, &eclass, &ecode);
 		}
 
diff --git a/source3/smbd/file_access.c b/source3/smbd/file_access.c
index 960dcb703a5..74855649ea2 100644
--- a/source3/smbd/file_access.c
+++ b/source3/smbd/file_access.c
@@ -92,6 +92,11 @@ bool can_delete_file_in_directory(connection_struct *conn,
 		return False;
 	}
 
+	if (!lp_acl_check_permissions(SNUM(conn))) {
+		/* This option means don't check. */
+		return true;
+	}
+
 	/* Get the parent directory permission mask and owners. */
 	if (!parent_dirname(ctx, smb_fname->base_name, &dname, NULL)) {
 		return False;
diff --git a/source3/smbd/fileio.c b/source3/smbd/fileio.c
index 16d8853b54c..3b317f9a863 100644
--- a/source3/smbd/fileio.c
+++ b/source3/smbd/fileio.c
@@ -240,8 +240,8 @@ void trigger_write_time_update(struct files_struct *fsp)
 
 	/* trigger the update 2 seconds later */
 	fsp->update_write_time_event =
-		event_add_timed(smbd_event_context(), NULL,
-				timeval_current_ofs(0, delay),
+		event_add_timed(server_event_context(), NULL,
+				timeval_current_ofs_usec(delay),
 				update_write_time_handler, fsp);
 }
 
@@ -322,7 +322,7 @@ ssize_t write_file(struct smb_request *req,
 				int dosmode = dos_mode(fsp->conn, fsp->fsp_name);
 				if (!IS_DOS_ARCHIVE(dosmode)) {
 					file_set_dosmode(fsp->conn, fsp->fsp_name,
-						 dosmode | aARCH, NULL, false);
+						 dosmode | FILE_ATTRIBUTE_ARCHIVE, NULL, false);
 				}
 			}
 
diff --git a/source3/smbd/filename.c b/source3/smbd/filename.c
index 08bc79dfd22..ec8d1139faa 100644
--- a/source3/smbd/filename.c
+++ b/source3/smbd/filename.c
@@ -163,11 +163,11 @@ static NTSTATUS check_parent_exists(TALLOC_CTX *ctx,
 	 * last compnent to shorten the tree walk. */
 
 	/*
-	 * Safe to use CONST_DISCARD
+	 * Safe to use discard_const_p
 	 * here as last_component points
 	 * into our smb_fname->base_name.
 	 */
-	*pp_start = CONST_DISCARD(char *,last_component);
+	*pp_start = discard_const_p(char, last_component);
 
 	/* Update dirpath. */
 	TALLOC_FREE(*pp_dirpath);
diff --git a/source3/smbd/files.c b/source3/smbd/files.c
index b9063ccbb89..b8a25c1d5b6 100644
--- a/source3/smbd/files.c
+++ b/source3/smbd/files.c
@@ -21,6 +21,8 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "libcli/security/security.h"
+#include "util_tdb.h"
+#include <ccan/hash/hash.h>
 
 #define VALID_FNUM(fnum)   (((fnum) >= 0) && ((fnum) < real_max_open_files))
 
@@ -598,7 +600,6 @@ NTSTATUS dup_file_fsp(struct smb_request *req, files_struct *from,
 NTSTATUS file_name_hash(connection_struct *conn,
 			const char *name, uint32_t *p_name_hash)
 {
-	TDB_DATA key;
 	char *fullpath = NULL;
 
 	/* Set the hash of the full pathname. */
@@ -609,8 +610,7 @@ NTSTATUS file_name_hash(connection_struct *conn,
 	if (!fullpath) {
 		return NT_STATUS_NO_MEMORY;
 	}
-	key = string_term_tdb_data(fullpath);
-	*p_name_hash = tdb_jenkins_hash(&key);
+	*p_name_hash = hash(fullpath, strlen(fullpath) + 1, 0);
 
 	DEBUG(10,("file_name_hash: %s hash 0x%x\n",
 		fullpath,
diff --git a/source3/smbd/globals.c b/source3/smbd/globals.c
index 63c1f5dcb5d..2736fa69d0b 100644
--- a/source3/smbd/globals.c
+++ b/source3/smbd/globals.c
@@ -20,8 +20,10 @@
 #include "includes.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
+#include "lib/smbd_shim.h"
 #include "memcache.h"
 #include "messages.h"
+#include "tdb_compat.h"
 
 #if defined(WITH_AIO)
 struct aio_extra *aio_list_head = NULL;
@@ -144,14 +146,29 @@ struct memcache *smbd_memcache(void)
 	return smbd_memcache_ctx;
 }
 
+static const struct smbd_shim smbd_shim_fns = 
+{
+	.cancel_pending_lock_requests_by_fid = smbd_cancel_pending_lock_requests_by_fid,
+	.send_stat_cache_delete_message = smbd_send_stat_cache_delete_message,
+	.can_delete_directory = smbd_can_delete_directory,
+	.change_to_root_user = smbd_change_to_root_user,
+	
+	.contend_level2_oplocks_begin = smbd_contend_level2_oplocks_begin,
+	.contend_level2_oplocks_end = smbd_contend_level2_oplocks_end,
+
+	.become_root = smbd_become_root,
+	.unbecome_root = smbd_unbecome_root
+};
 
 void smbd_init_globals(void)
 {
+	set_smbd_shim(&smbd_shim_fns);
+
 	ZERO_STRUCT(conn_ctx_stack);
 
 	ZERO_STRUCT(sec_ctx_stack);
 
-	smbd_server_conn = talloc_zero(smbd_event_context(), struct smbd_server_connection);
+	smbd_server_conn = talloc_zero(server_event_context(), struct smbd_server_connection);
 	if (!smbd_server_conn) {
 		exit_server("failed to create smbd_server_connection");
 	}
@@ -159,3 +176,32 @@ void smbd_init_globals(void)
 	smbd_server_conn->smb1.echo_handler.trusted_fd = -1;
 	smbd_server_conn->smb1.echo_handler.socket_lock_fd = -1;
 }
+
+void smbd_set_server_fd(int fd)
+{
+	struct smbd_server_connection *sconn = smbd_server_conn;
+	char addr[INET6_ADDRSTRLEN];
+	const char *name;
+
+	sconn->sock = fd;
+
+	/*
+	 * Initialize sconn->client_id: If we can't find the client's
+	 * name, default to its address.
+	 */
+
+	client_addr(fd, sconn->client_id.addr, sizeof(sconn->client_id.addr));
+
+	name = client_name(sconn->sock);
+	if (strcmp(name, "UNKNOWN") != 0) {
+		name = talloc_strdup(sconn, name);
+	} else {
+		name = NULL;
+	}
+	sconn->client_id.name =
+		(name != NULL) ? name : sconn->client_id.addr;
+
+	sub_set_socket_ids(sconn->client_id.addr, sconn->client_id.name,
+			   client_socket_addr(sconn->sock, addr,
+					      sizeof(addr)));
+}
diff --git a/source3/smbd/globals.h b/source3/smbd/globals.h
index 01be31a57f9..97d75fd4137 100644
--- a/source3/smbd/globals.h
+++ b/source3/smbd/globals.h
@@ -47,7 +47,8 @@ struct fsp_singleton_cache {
 extern const struct mangle_fns *mangle_fns;
 
 extern unsigned char *chartest;
-extern TDB_CONTEXT *tdb_mangled_cache;
+struct tdb_context;
+extern struct tdb_context *tdb_mangled_cache;
 
 /*
   this determines how many characters are used from the original filename
diff --git a/source3/smbd/ipc.c b/source3/smbd/ipc.c
index 4f2fea52658..669e28e715c 100644
--- a/source3/smbd/ipc.c
+++ b/source3/smbd/ipc.c
@@ -28,6 +28,7 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "smbprofile.h"
+#include "rpc_server/srv_pipe_hnd.h"
 
 #define NERR_notsupported 50
 
@@ -267,7 +268,7 @@ static void api_dcerpc_cmd(connection_struct *conn, struct smb_request *req,
 	state->num_data = length;
 	state->max_read = max_read;
 
-	subreq = np_write_send(state, smbd_event_context(), state->handle,
+	subreq = np_write_send(state, server_event_context(), state->handle,
 			       state->data, length);
 	if (subreq == NULL) {
 		TALLOC_FREE(state);
@@ -297,14 +298,14 @@ static void api_dcerpc_cmd_write_done(struct tevent_req *subreq)
 		goto send;
 	}
 
-	state->data = TALLOC_REALLOC_ARRAY(state, state->data, uint8_t,
+	state->data = talloc_realloc(state, state->data, uint8_t,
 					   state->max_read);
 	if (state->data == NULL) {
 		reply_nterror(req, NT_STATUS_NO_MEMORY);
 		goto send;
 	}
 
-	subreq = np_read_send(req->conn, smbd_event_context(),
+	subreq = np_read_send(req->conn, server_event_context(),
 			      state->handle, state->data, state->max_read);
 	if (subreq == NULL) {
 		reply_nterror(req, NT_STATUS_NO_MEMORY);
@@ -644,7 +645,7 @@ void reply_trans(struct smb_request *req)
 		return;
 	}
 
-	if ((state = TALLOC_P(conn, struct trans_state)) == NULL) {
+	if ((state = talloc(conn, struct trans_state)) == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		reply_nterror(req, NT_STATUS_NO_MEMORY);
 		END_PROFILE(SMBtrans);
@@ -741,7 +742,7 @@ void reply_trans(struct smb_request *req)
 			goto bad_param;
 		}
 
-		if((state->setup = TALLOC_ARRAY(
+		if((state->setup = talloc_array(
 			    state, uint16, state->setup_count)) == NULL) {
 			DEBUG(0,("reply_trans: setup malloc fail for %u "
 				 "bytes !\n", (unsigned int)
@@ -807,7 +808,7 @@ void reply_transs(struct smb_request *req)
 
 	START_PROFILE(SMBtranss);
 
-	show_msg((char *)req->inbuf);
+	show_msg((const char *)req->inbuf);
 
 	if (req->wct < 8) {
 		reply_nterror(req, NT_STATUS_INVALID_PARAMETER);
diff --git a/source3/smbd/lanman.c b/source3/smbd/lanman.c
index 13297c02221..dac91b2a40b 100644
--- a/source3/smbd/lanman.c
+++ b/source3/smbd/lanman.c
@@ -34,17 +34,15 @@
 #include "rpc_client/cli_spoolss.h"
 #include "rpc_client/init_spoolss.h"
 #include "../librpc/gen_ndr/ndr_srvsvc_c.h"
-#include "../librpc/gen_ndr/srv_samr.h"
-#include "../librpc/gen_ndr/srv_srvsvc.h"
 #include "../librpc/gen_ndr/rap.h"
 #include "../lib/util/binsearch.h"
 #include "../libcli/auth/libcli_auth.h"
 #include "rpc_client/init_lsa.h"
-#include "rpc_server/rpc_ncacn_np.h"
 #include "../libcli/security/security.h"
 #include "printing.h"
 #include "passdb/machine_sid.h"
 #include "auth.h"
+#include "rpc_server/rpc_ncacn_np.h"
 
 #ifdef CHECK_TYPES
 #undef CHECK_TYPES
@@ -614,9 +612,9 @@ static void fill_printq_info_52(struct spoolss_DriverInfo3 *driver,
 {
 	int 				i;
 	fstring 			location;
-	trim_string((char *)driver->driver_path, "\\print$\\WIN40\\0\\", 0);
-	trim_string((char *)driver->data_file, "\\print$\\WIN40\\0\\", 0);
-	trim_string((char *)driver->help_file, "\\print$\\WIN40\\0\\", 0);
+	trim_string(discard_const_p(char, driver->driver_path), "\\print$\\WIN40\\0\\", 0);
+	trim_string(discard_const_p(char, driver->data_file), "\\print$\\WIN40\\0\\", 0);
+	trim_string(discard_const_p(char, driver->help_file), "\\print$\\WIN40\\0\\", 0);
 
 	PACKI(desc, "W", 0x0400);                     /* don't know */
 	PACKS(desc, "z", driver->driver_name);        /* long printer name */
@@ -643,7 +641,7 @@ static void fill_printq_info_52(struct spoolss_DriverInfo3 *driver,
 
 	for ( i=0; i<count && driver->dependent_files && *driver->dependent_files[i]; i++)
 	{
-		trim_string((char *)driver->dependent_files[i], "\\print$\\WIN40\\0\\", 0);
+		trim_string(discard_const_p(char, driver->dependent_files[i]), "\\print$\\WIN40\\0\\", 0);
 		PACKS(desc,"z",driver->dependent_files[i]);         /* driver files to copy */
 		DEBUG(3,("Dependent File: %s:\n", driver->dependent_files[i]));
 	}
@@ -1429,7 +1427,7 @@ static int fill_srv_info(struct srv_info_struct *service,
 
 static int srv_comp(struct srv_info_struct *s1,struct srv_info_struct *s2)
 {
-	return StrCaseCmp(s1->name,s2->name);
+	return strcasecmp_m(s1->name,s2->name);
 }
 
 /****************************************************************************
@@ -1598,7 +1596,7 @@ static int srv_name_match(const char *n1, const char *n2)
 	 *  the server will return a list of servers that exist on
 	 *  the network greater than or equal to the FirstNameToReturn.
 	 */
-	int ret = StrCaseCmp(n1, n2);
+	int ret = strcasecmp_m(n1, n2);
 
 	if (ret <= 0) {
 		return 0;
@@ -1710,7 +1708,7 @@ static bool api_RNetServerEnum3(struct smbd_server_connection *sconn,
 			 */
 			for (;first > 0;) {
 				int ret;
-				ret = StrCaseCmp(first_name,
+				ret = strcasecmp_m(first_name,
 						 servers[first-1].name);
 				if (ret > 0) {
 					break;
@@ -2094,7 +2092,7 @@ static bool api_RNetShareEnum(struct smbd_server_connection *sconn,
 	/* Ensure all the usershares are loaded. */
 	become_root();
 	load_registry_shares();
-	count = load_usershare_shares();
+	count = load_usershare_shares(sconn);
 	unbecome_root();
 
 	data_len = fixed_len = string_len = 0;
@@ -2256,7 +2254,7 @@ static bool api_RNetShareAdd(struct smbd_server_connection *sconn,
 		return false;
 	}
 
-	status = rpc_pipe_open_internal(mem_ctx, &ndr_table_srvsvc.syntax_id,
+	status = rpc_pipe_open_interface(mem_ctx, &ndr_table_srvsvc.syntax_id,
 					conn->session_info,
 					&conn->sconn->client_id,
 					conn->sconn->msg_ctx,
@@ -2367,7 +2365,7 @@ static bool api_RNetGroupEnum(struct smbd_server_connection *sconn,
 		return False;
 	}
 
-	status = rpc_pipe_open_internal(
+	status = rpc_pipe_open_interface(
 		talloc_tos(), &ndr_table_samr.syntax_id,
 		conn->session_info, &conn->sconn->client_id,
 		conn->sconn->msg_ctx, &samr_pipe);
@@ -2379,7 +2377,7 @@ static bool api_RNetGroupEnum(struct smbd_server_connection *sconn,
 
 	b = samr_pipe->binding_handle;
 
-	status = dcerpc_samr_Connect2(b, talloc_tos(), global_myname(),
+	status = dcerpc_samr_Connect2(b, talloc_tos(), lp_netbios_name(),
 				      SAMR_ACCESS_LOOKUP_DOMAIN, &samr_handle,
 				      &result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -2573,7 +2571,7 @@ static bool api_NetUserGetGroups(struct smbd_server_connection *sconn,
 	p = *rdata;
 	endp = *rdata + *rdata_len;
 
-	status = rpc_pipe_open_internal(
+	status = rpc_pipe_open_interface(
 		talloc_tos(), &ndr_table_samr.syntax_id,
 		conn->session_info, &conn->sconn->client_id,
 		conn->sconn->msg_ctx, &samr_pipe);
@@ -2585,7 +2583,7 @@ static bool api_NetUserGetGroups(struct smbd_server_connection *sconn,
 
 	b = samr_pipe->binding_handle;
 
-	status = dcerpc_samr_Connect2(b, talloc_tos(), global_myname(),
+	status = dcerpc_samr_Connect2(b, talloc_tos(), lp_netbios_name(),
 				      SAMR_ACCESS_LOOKUP_DOMAIN, &samr_handle,
 				      &result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -2765,7 +2763,7 @@ static bool api_RNetUserEnum(struct smbd_server_connection *sconn,
 	p = *rdata;
 	endp = *rdata + *rdata_len;
 
-	status = rpc_pipe_open_internal(
+	status = rpc_pipe_open_interface(
 		talloc_tos(), &ndr_table_samr.syntax_id,
 		conn->session_info, &conn->sconn->client_id,
 		conn->sconn->msg_ctx, &samr_pipe);
@@ -2777,7 +2775,7 @@ static bool api_RNetUserEnum(struct smbd_server_connection *sconn,
 
 	b = samr_pipe->binding_handle;
 
-	status = dcerpc_samr_Connect2(b, talloc_tos(), global_myname(),
+	status = dcerpc_samr_Connect2(b, talloc_tos(), lp_netbios_name(),
 				      SAMR_ACCESS_LOOKUP_DOMAIN, &samr_handle,
 				      &result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -3031,7 +3029,7 @@ static bool api_SetUserPassword(struct smbd_server_connection *sconn,
 	ZERO_STRUCT(domain_handle);
 	ZERO_STRUCT(user_handle);
 
-	status = rpc_pipe_open_internal(mem_ctx, &ndr_table_samr.syntax_id,
+	status = rpc_pipe_open_interface(mem_ctx, &ndr_table_samr.syntax_id,
 					conn->session_info,
 					&conn->sconn->client_id,
 					conn->sconn->msg_ctx,
@@ -3046,7 +3044,7 @@ static bool api_SetUserPassword(struct smbd_server_connection *sconn,
 	b = cli->binding_handle;
 
 	status = dcerpc_samr_Connect2(b, mem_ctx,
-				      global_myname(),
+				      lp_netbios_name(),
 				      SAMR_ACCESS_CONNECT_TO_SERVER |
 				      SAMR_ACCESS_ENUM_DOMAINS |
 				      SAMR_ACCESS_LOOKUP_DOMAIN,
@@ -3282,7 +3280,7 @@ static bool api_SamOEMChangePassword(struct smbd_server_connection *sconn,
 	memcpy(password.data, data, 516);
 	memcpy(hash.hash, data+516, 16);
 
-	status = rpc_pipe_open_internal(mem_ctx, &ndr_table_samr.syntax_id,
+	status = rpc_pipe_open_interface(mem_ctx, &ndr_table_samr.syntax_id,
 					conn->session_info,
 					&conn->sconn->client_id,
 					conn->sconn->msg_ctx,
@@ -3296,7 +3294,7 @@ static bool api_SamOEMChangePassword(struct smbd_server_connection *sconn,
 
 	b = cli->binding_handle;
 
-	init_lsa_AsciiString(&server, global_myname());
+	init_lsa_AsciiString(&server, lp_netbios_name());
 	init_lsa_AsciiString(&account, user);
 
 	status = dcerpc_samr_OemChangePasswordUser2(b, mem_ctx,
@@ -3866,7 +3864,7 @@ static bool api_RNetServerGetInfo(struct smbd_server_connection *sconn,
 	p = *rdata;
 	p2 = p + struct_len;
 
-	status = rpc_pipe_open_internal(mem_ctx, &ndr_table_srvsvc.syntax_id,
+	status = rpc_pipe_open_interface(mem_ctx, &ndr_table_srvsvc.syntax_id,
 					conn->session_info,
 					&conn->sconn->client_id,
 					conn->sconn->msg_ctx,
@@ -4029,8 +4027,8 @@ static bool api_NetWkstaGetInfo(struct smbd_server_connection *sconn,
 	}
 	p += 4;
 
-	SCVAL(p,0,lp_major_announce_version()); /* system version - e.g 4 in 4.1 */
-	SCVAL(p,1,lp_minor_announce_version()); /* system version - e.g .1 in 4.1 */
+	SCVAL(p,0,SAMBA_MAJOR_NBT_ANNOUNCE_VERSION); /* system version - e.g 4 in 4.1 */
+	SCVAL(p,1,SAMBA_MINOR_NBT_ANNOUNCE_VERSION); /* system version - e.g .1 in 4.1 */
 	p += 2;
 
 	SIVAL(p,0,PTR_DIFF(p2,*rdata));
@@ -4293,7 +4291,7 @@ static bool api_RNetUserGetInfo(struct smbd_server_connection *sconn,
 	ZERO_STRUCT(domain_handle);
 	ZERO_STRUCT(user_handle);
 
-	status = rpc_pipe_open_internal(mem_ctx, &ndr_table_samr.syntax_id,
+	status = rpc_pipe_open_interface(mem_ctx, &ndr_table_samr.syntax_id,
 					conn->session_info,
 					&conn->sconn->client_id,
 					conn->sconn->msg_ctx,
@@ -4308,7 +4306,7 @@ static bool api_RNetUserGetInfo(struct smbd_server_connection *sconn,
 	b = cli->binding_handle;
 
 	status = dcerpc_samr_Connect2(b, mem_ctx,
-				      global_myname(),
+				      lp_netbios_name(),
 				      SAMR_ACCESS_CONNECT_TO_SERVER |
 				      SAMR_ACCESS_ENUM_DOMAINS |
 				      SAMR_ACCESS_LOOKUP_DOMAIN,
diff --git a/source3/smbd/mangle_hash.c b/source3/smbd/mangle_hash.c
index d98b350a37e..c196a13ed02 100644
--- a/source3/smbd/mangle_hash.c
+++ b/source3/smbd/mangle_hash.c
@@ -25,6 +25,7 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "mangle.h"
+#include "util_tdb.h"
 
 /* -------------------------------------------------------------------------- **
  * Other stuff...
@@ -86,7 +87,7 @@ static void init_valid_table(void)
 		return;
 	}
 
-	valid_table = (uint8 *)map_file(data_path("valid.dat"), 0x10000);
+	valid_table = (uint8 *)map_file(data_path(talloc_tos(), "valid.dat"), 0x10000);
 	if (!valid_table) {
 		smb_panic("Could not load valid.dat file required for mangle method=hash");
 		return;
@@ -473,7 +474,7 @@ static void cache_mangled_name( const char mangled_name[13],
 		return;
 
 	/* Init the string lengths. */
-	safe_strcpy(mangled_name_key, mangled_name, sizeof(mangled_name_key)-1);
+	strlcpy(mangled_name_key, mangled_name, sizeof(mangled_name_key));
 
 	/* See if the extensions are unmangled.  If so, store the entry
 	 * without the extension, thus creating a "group" reverse map.
@@ -638,7 +639,7 @@ static bool to_8_3(char magic_char, const char *in, char out[13], int default_ca
 
 	if( p ) {
 		if( p == s )
-			safe_strcpy( extension, "___", 3 );
+			strlcpy( extension, "___", 4);
 		else {
 			*p++ = 0;
 			while( *p && extlen < 3 ) {
@@ -670,7 +671,7 @@ static bool to_8_3(char magic_char, const char *in, char out[13], int default_ca
 
 	if( *extension ) {
 		out[baselen+3] = '.';
-		safe_strcpy(&out[baselen+4], extension, 3);
+		strlcpy(&out[baselen+4], extension, 4);
 	}
 
 	SAFE_FREE(s);
@@ -738,7 +739,7 @@ static bool hash_name_to_8_3(const char *in,
 	if (NT_STATUS_IS_OK(is_valid_name(in_ucs2, False, False)) &&
 				NT_STATUS_IS_OK(is_8_3_w(in_ucs2, False))) {
 		TALLOC_FREE(in_ucs2);
-		safe_strcpy(out, in, 12);
+		strlcpy(out, in, 13);
 		return True;
 	}
 
@@ -772,8 +773,13 @@ const struct mangle_fns *mangle_hash_init(void)
 	mangle_reset();
 
 	/* Create the in-memory tdb using our custom hash function. */
+#ifndef BUILD_TDB2
 	tdb_mangled_cache = tdb_open_ex("mangled_cache", 1031, TDB_INTERNAL,
 				(O_RDWR|O_CREAT), 0644, NULL, fast_string_hash);
+#else
+	/* FIXME: We should *never* open a tdb without logging! */
+	tdb_mangled_cache = tdb_open("mangled_cache", TDB_INTERNAL, 0, 0, NULL);
+#endif
 
 	return &mangle_hash_fns;
 }
diff --git a/source3/smbd/mangle_hash2.c b/source3/smbd/mangle_hash2.c
index a39bb895c28..4de80cbe31b 100644
--- a/source3/smbd/mangle_hash2.c
+++ b/source3/smbd/mangle_hash2.c
@@ -705,7 +705,7 @@ static bool hash2_name_to_8_3(const char *name,
 		/* if the name is already a valid 8.3 name then we don't need to
 		 * change anything */
 		if (is_legal_name(name) && is_8_3(name, False, False, p)) {
-			safe_strcpy(new_name, name, 12);
+			strlcpy(new_name, name, 13);
 			return True;
 		}
 	}
diff --git a/source3/smbd/message.c b/source3/smbd/message.c
index 34997a13dc5..63b08e12620 100644
--- a/source3/smbd/message.c
+++ b/source3/smbd/message.c
@@ -202,7 +202,7 @@ void reply_sendstrt(struct smb_request *req)
 
 	TALLOC_FREE(smbd_msg_state);
 
-	smbd_msg_state = TALLOC_ZERO_P(NULL, struct msg_state);
+	smbd_msg_state = talloc_zero(NULL, struct msg_state);
 
 	if (smbd_msg_state == NULL) {
 		reply_nterror(req, NT_STATUS_NO_MEMORY);
@@ -259,7 +259,7 @@ void reply_sendtxt(struct smb_request *req)
 
 	len = MIN(SVAL(msg, 0), smbreq_bufrem(req, msg+2));
 
-	tmp = TALLOC_REALLOC_ARRAY(smbd_msg_state, smbd_msg_state->msg,
+	tmp = talloc_realloc(smbd_msg_state, smbd_msg_state->msg,
 				   char, old_len + len);
 
 	if (tmp == NULL) {
diff --git a/source3/smbd/msdfs.c b/source3/smbd/msdfs.c
index ab67ac8596d..47c2b1ee786 100644
--- a/source3/smbd/msdfs.c
+++ b/source3/smbd/msdfs.c
@@ -231,7 +231,7 @@ NTSTATUS create_conn_struct(TALLOC_CTX *ctx,
 	char *oldcwd;
 	const char *vfs_user;
 
-	conn = TALLOC_ZERO_P(ctx, connection_struct);
+	conn = talloc_zero(ctx, connection_struct);
 	if (conn == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -252,7 +252,7 @@ NTSTATUS create_conn_struct(TALLOC_CTX *ctx,
 
 	/* needed for smbd_vfs_init() */
 
-	if (!(conn->params = TALLOC_ZERO_P(conn, struct share_params))) {
+	if (!(conn->params = talloc_zero(conn, struct share_params))) {
 		DEBUG(0, ("TALLOC failed\n"));
 		TALLOC_FREE(conn);
 		return NT_STATUS_NO_MEMORY;
@@ -364,7 +364,7 @@ static bool parse_msdfs_symlink(TALLOC_CTX *ctx,
 		return False;
 	}
 
-	alt_path = TALLOC_ARRAY(ctx, char *, MAX_REFERRAL_COUNT);
+	alt_path = talloc_array(ctx, char *, MAX_REFERRAL_COUNT);
 	if (!alt_path) {
 		return False;
 	}
@@ -378,7 +378,7 @@ static bool parse_msdfs_symlink(TALLOC_CTX *ctx,
 	DEBUG(10,("parse_msdfs_symlink: count=%d\n", count));
 
 	if (count) {
-		reflist = *preflist = TALLOC_ZERO_ARRAY(ctx,
+		reflist = *preflist = talloc_zero_array(ctx,
 				struct referral, count);
 		if(reflist == NULL) {
 			TALLOC_FREE(alt_path);
@@ -443,7 +443,7 @@ static bool is_msdfs_link_internal(TALLOC_CTX *ctx,
 
 	if (pp_link_target) {
 		bufsize = 1024;
-		link_target = TALLOC_ARRAY(ctx, char, bufsize);
+		link_target = talloc_array(ctx, char, bufsize);
 		if (!link_target) {
 			return False;
 		}
@@ -692,7 +692,7 @@ static NTSTATUS dfs_redirect(TALLOC_CTX *ctx,
 			bool *ppath_contains_wcard)
 {
 	NTSTATUS status;
-	struct dfs_path *pdp = TALLOC_P(ctx, struct dfs_path);
+	struct dfs_path *pdp = talloc(ctx, struct dfs_path);
 
 	if (!pdp) {
 		return NT_STATUS_NO_MEMORY;
@@ -794,7 +794,7 @@ static NTSTATUS self_ref(TALLOC_CTX *ctx,
 	*self_referralp = True;
 
 	jucn->referral_count = 1;
-	if((ref = TALLOC_ZERO_P(ctx, struct referral)) == NULL) {
+	if((ref = talloc_zero(ctx, struct referral)) == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
 
@@ -825,7 +825,7 @@ NTSTATUS get_referred_path(TALLOC_CTX *ctx,
 	int snum;
 	NTSTATUS status = NT_STATUS_NOT_FOUND;
 	bool dummy;
-	struct dfs_path *pdp = TALLOC_P(ctx, struct dfs_path);
+	struct dfs_path *pdp = talloc(ctx, struct dfs_path);
 	char *oldpath;
 
 	if (!pdp) {
@@ -899,7 +899,7 @@ NTSTATUS get_referred_path(TALLOC_CTX *ctx,
  		 */
 
 		jucn->referral_count = 1;
-		if ((ref = TALLOC_ZERO_P(ctx, struct referral)) == NULL) {
+		if ((ref = talloc_zero(ctx, struct referral)) == NULL) {
 			TALLOC_FREE(pdp);
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -1233,7 +1233,7 @@ int setup_dfs_referral(connection_struct *orig_conn,
 		pathnamep++;
 	}
 
-	junction = TALLOC_ZERO_P(ctx, struct junction_map);
+	junction = talloc_zero(ctx, struct junction_map);
 	if (!junction) {
 		*pstatus = NT_STATUS_NO_MEMORY;
 		talloc_destroy(ctx);
@@ -1318,7 +1318,7 @@ bool create_junction(TALLOC_CTX *ctx,
 {
 	int snum;
 	bool dummy;
-	struct dfs_path *pdp = TALLOC_P(ctx,struct dfs_path);
+	struct dfs_path *pdp = talloc(ctx,struct dfs_path);
 	NTSTATUS status;
 
 	if (!pdp) {
@@ -1630,7 +1630,7 @@ static int form_junctions(TALLOC_CTX *ctx,
 	jucn[cnt].comment = "";
 	jucn[cnt].referral_count = 1;
 
-	ref = jucn[cnt].referral_list = TALLOC_ZERO_P(ctx, struct referral);
+	ref = jucn[cnt].referral_list = talloc_zero(ctx, struct referral);
 	if (jucn[cnt].referral_list == NULL) {
 		goto out;
 	}
@@ -1709,7 +1709,8 @@ out:
 	return cnt;
 }
 
-struct junction_map *enum_msdfs_links(TALLOC_CTX *ctx, size_t *p_num_jn)
+struct junction_map *enum_msdfs_links(struct smbd_server_connection *sconn,
+				      TALLOC_CTX *ctx, size_t *p_num_jn)
 {
 	struct junction_map *jn = NULL;
 	int i=0;
@@ -1724,7 +1725,7 @@ struct junction_map *enum_msdfs_links(TALLOC_CTX *ctx, size_t *p_num_jn)
 	/* Ensure all the usershares are loaded. */
 	become_root();
 	load_registry_shares();
-	sharecount = load_usershare_shares();
+	sharecount = load_usershare_shares(sconn);
 	unbecome_root();
 
 	for(i=0;i < sharecount;i++) {
@@ -1735,7 +1736,7 @@ struct junction_map *enum_msdfs_links(TALLOC_CTX *ctx, size_t *p_num_jn)
 	if (jn_count == 0) {
 		return NULL;
 	}
-	jn = TALLOC_ARRAY(ctx,  struct junction_map, jn_count);
+	jn = talloc_array(ctx,  struct junction_map, jn_count);
 	if (!jn) {
 		return NULL;
 	}
@@ -1786,7 +1787,7 @@ NTSTATUS resolve_dfspath_wcard(TALLOC_CTX *ctx,
 		 * Once srvstr_get_path() uses talloc it'll
 		 * be a talloced ptr anyway.
 		 */
-		*pp_name_out = CONST_DISCARD(char *,name_in);
+		*pp_name_out = discard_const_p(char, name_in);
 	}
 	return status;
 }
diff --git a/source3/smbd/msg_idmap.c b/source3/smbd/msg_idmap.c
index b1ee60ff7f7..2a00f1bbb91 100644
--- a/source3/smbd/msg_idmap.c
+++ b/source3/smbd/msg_idmap.c
@@ -27,10 +27,8 @@
 
 #include "includes.h"
 #include "smbd/smbd.h"
-#include "memcache.h"
 #include "globals.h"
 #include "../libcli/security/dom_sid.h"
-#include "../librpc/gen_ndr/ndr_security.h"
 #include "idmap_cache.h"
 #include "passdb/lookup_sid.h"
 #include "auth.h"
diff --git a/source3/smbd/negprot.c b/source3/smbd/negprot.c
index 6877ccc861f..a6d7cc0ab3e 100644
--- a/source3/smbd/negprot.c
+++ b/source3/smbd/negprot.c
@@ -224,7 +224,7 @@ DATA_BLOB negprot_spnego(TALLOC_CTX *ctx, struct smbd_server_connection *sconn)
 	} else {
 		fstring myname;
 		char *host_princ_s = NULL;
-		name_to_fqdn(myname, global_myname());
+		name_to_fqdn(myname, lp_netbios_name());
 		strlower_m(myname);
 		if (asprintf(&host_princ_s, "cifs/%s@%s", myname, lp_realm())
 		    == -1) {
@@ -234,6 +234,10 @@ DATA_BLOB negprot_spnego(TALLOC_CTX *ctx, struct smbd_server_connection *sconn)
 		SAFE_FREE(host_princ_s);
 	}
 
+	if (blob.length == 0 || blob.data == NULL) {
+		return data_blob_null;
+	}
+
 	blob_out = data_blob_talloc(ctx, NULL, 16 + blob.length);
 	if (blob_out.data == NULL) {
 		data_blob_free(&blob);
@@ -242,10 +246,10 @@ DATA_BLOB negprot_spnego(TALLOC_CTX *ctx, struct smbd_server_connection *sconn)
 
 	memset(blob_out.data, '\0', 16);
 
-	checked_strlcpy(unix_name, global_myname(), sizeof(unix_name));
+	checked_strlcpy(unix_name, lp_netbios_name(), sizeof(unix_name));
 	strlower_m(unix_name);
 	push_ascii_nstring(dos_name, unix_name);
-	safe_strcpy((char *)blob_out.data, dos_name, 16);
+	strlcpy((char *)blob_out.data, dos_name, 17);
 
 #ifdef DEVELOPER
 	/* Fix valgrind 'uninitialized bytes' issue. */
@@ -569,7 +573,7 @@ void reply_negprot(struct smb_request *req)
 
 		char **tmp;
 
-		tmp = TALLOC_REALLOC_ARRAY(talloc_tos(), cliprotos, char *,
+		tmp = talloc_realloc(talloc_tos(), cliprotos, char *,
 					   num_cliprotos+1);
 		if (tmp == NULL) {
 			DEBUG(0, ("talloc failed\n"));
@@ -712,6 +716,11 @@ void reply_negprot(struct smb_request *req)
 	}
 
 	TALLOC_FREE(cliprotos);
+
+	if (lp_async_smb_echo_handler() && !fork_echo_handler(sconn)) {
+		exit_server("Failed to fork echo handler");
+	}
+
 	END_PROFILE(SMBnegprot);
 	return;
 }
diff --git a/source3/smbd/notify.c b/source3/smbd/notify.c
index 24385c94a0d..8160d5ac638 100644
--- a/source3/smbd/notify.c
+++ b/source3/smbd/notify.c
@@ -183,7 +183,7 @@ NTSTATUS change_notify_create(struct files_struct *fsp, uint32 filter,
 
 	SMB_ASSERT(fsp->notify == NULL);
 
-	if (!(fsp->notify = TALLOC_ZERO_P(NULL, struct notify_change_buf))) {
+	if (!(fsp->notify = talloc_zero(NULL, struct notify_change_buf))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -424,7 +424,7 @@ static void notify_fsp(files_struct *fsp, uint32 action, const char *name)
 		return;
 	}
 
-	if (!(changes = TALLOC_REALLOC_ARRAY(
+	if (!(changes = talloc_realloc(
 		      fsp->notify, fsp->notify->changes,
 		      struct notify_change, fsp->notify->num_changes+1))) {
 		DEBUG(0, ("talloc_realloc failed\n"));
@@ -520,7 +520,7 @@ struct sys_notify_context *sys_notify_context_create(connection_struct *conn,
 {
 	struct sys_notify_context *ctx;
 
-	if (!(ctx = TALLOC_P(mem_ctx, struct sys_notify_context))) {
+	if (!(ctx = talloc(mem_ctx, struct sys_notify_context))) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
diff --git a/source3/smbd/notify_internal.c b/source3/smbd/notify_internal.c
index a8eaf793921..a0193a44679 100644
--- a/source3/smbd/notify_internal.c
+++ b/source3/smbd/notify_internal.c
@@ -29,6 +29,8 @@
 #include "dbwrap.h"
 #include "smbd/smbd.h"
 #include "messages.h"
+#include "lib/util/tdb_wrap.h"
+#include "util_tdb.h"
 
 struct notify_context {
 	struct db_context *db_recursive;
@@ -196,7 +198,7 @@ static NTSTATUS notify_load(struct notify_context *notify, struct db_record *rec
 	notify->seqnum = seqnum;
 
 	talloc_free(notify->array);
-	notify->array = TALLOC_ZERO_P(notify, struct notify_array);
+	notify->array = talloc_zero(notify, struct notify_array);
 	NT_STATUS_HAVE_NO_MEMORY(notify->array);
 
 	if (!rec) {
@@ -529,7 +531,7 @@ NTSTATUS notify_add(struct notify_context *notify, struct notify_entry *e0,
 
 	depth = count_chars(e.path, '/');
 
-	listel = TALLOC_ZERO_P(notify, struct notify_list);
+	listel = talloc_zero(notify, struct notify_list);
 	if (listel == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -597,7 +599,7 @@ NTSTATUS notify_remove_onelevel(struct notify_context *notify,
 
 	rec = notify->db_onelevel->fetch_locked(
 		notify->db_onelevel, array,
-		make_tdb_data((uint8_t *)fid, sizeof(*fid)));
+		make_tdb_data((const uint8_t *)fid, sizeof(*fid)));
 	if (rec == NULL) {
 		DEBUG(10, ("notify_remove_onelevel: fetch_locked for %s failed"
 			   "\n", file_id_string_tos(fid)));
@@ -841,7 +843,7 @@ void notify_onelevel(struct notify_context *notify, uint32_t action,
 	if (notify->db_onelevel->fetch(
 		    notify->db_onelevel, array,
 		    make_tdb_data((uint8_t *)&fid, sizeof(fid)),
-		    &dbuf) == -1) {
+		    &dbuf) != 0) {
 		TALLOC_FREE(array);
 		return;
 	}
diff --git a/source3/smbd/ntquotas.c b/source3/smbd/ntquotas.c
index d0056756ac0..1f4fc2d0a21 100644
--- a/source3/smbd/ntquotas.c
+++ b/source3/smbd/ntquotas.c
@@ -22,6 +22,7 @@
 #include "../lib/util/util_pw.h"
 #include "system/passwd.h"
 #include "passdb/lookup_sid.h"
+#include "libsmb/libsmb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_QUOTA
@@ -200,14 +201,14 @@ int vfs_get_user_ntquota_list(files_struct *fsp, SMB_NTQUOTA_LIST **qt_list)
 		DEBUG(15,("quota entry for id[%s] path[%s]\n",
 			  sid_string_dbg(&sid), fsp->conn->connectpath));
 
-		if ((tmp_list_ent=TALLOC_ZERO_P(mem_ctx,SMB_NTQUOTA_LIST))==NULL) {
+		if ((tmp_list_ent=talloc_zero(mem_ctx,SMB_NTQUOTA_LIST))==NULL) {
 			DEBUG(0,("TALLOC_ZERO() failed\n"));
 			*qt_list = NULL;
 			talloc_destroy(mem_ctx);
 			return (-1);
 		}
 
-		if ((tmp_list_ent->quotas=TALLOC_ZERO_P(mem_ctx,SMB_NTQUOTA_STRUCT))==NULL) {
+		if ((tmp_list_ent->quotas=talloc_zero(mem_ctx,SMB_NTQUOTA_STRUCT))==NULL) {
 			DEBUG(0,("TALLOC_ZERO() failed\n"));
 			*qt_list = NULL;
 			talloc_destroy(mem_ctx);
@@ -240,7 +241,7 @@ void *init_quota_handle(TALLOC_CTX *mem_ctx)
 	if (!mem_ctx)
 		return False;
 
-	qt_handle = TALLOC_ZERO_P(mem_ctx,SMB_NTQUOTA_HANDLE);
+	qt_handle = talloc_zero(mem_ctx,SMB_NTQUOTA_HANDLE);
 	if (qt_handle==NULL) {
 		DEBUG(0,("TALLOC_ZERO() failed\n"));
 		return NULL;
diff --git a/source3/smbd/nttrans.c b/source3/smbd/nttrans.c
index 71f20ac77e0..5fdb07d769a 100644
--- a/source3/smbd/nttrans.c
+++ b/source3/smbd/nttrans.c
@@ -29,6 +29,7 @@
 #include "auth.h"
 #include "ntioctl.h"
 #include "smbprofile.h"
+#include "libsmb/libsmb.h"
 
 extern const struct generic_mapping file_generic_mapping;
 
@@ -1363,7 +1364,7 @@ static NTSTATUS copy_internals(TALLOC_CTX *ctx,
 
 	/* Ensure attributes match. */
 	fattr = dos_mode(conn, smb_fname_src);
-	if ((fattr & ~attrs) & (aHIDDEN | aSYSTEM)) {
+	if ((fattr & ~attrs) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) {
 		status = NT_STATUS_NO_SUCH_FILE;
 		goto out;
 	}
@@ -1450,7 +1451,7 @@ static NTSTATUS copy_internals(TALLOC_CTX *ctx,
 
 	status = close_file(NULL, fsp2, NORMAL_CLOSE);
 
-	/* Grrr. We have to do this as open_file_ntcreate adds aARCH when it
+	/* Grrr. We have to do this as open_file_ntcreate adds FILE_ATTRIBUTE_ARCHIVE when it
 	   creates the file. This isn't the correct thing to do in the copy
 	   case. JRA */
 	if (!parent_dirname(talloc_tos(), smb_fname_dst->base_name, &parent,
@@ -2213,8 +2214,7 @@ static void call_nt_transact_ioctl(connection_struct *conn,
 		 * Allocate the correct amount and return the pointer to let
 		 * it be deallocated when we return.
 		 */
-		SHADOW_COPY_DATA *shadow_data = NULL;
-		TALLOC_CTX *shadow_mem_ctx = NULL;
+		struct shadow_copy_data *shadow_data = NULL;
 		bool labels = False;
 		uint32 labels_data_count = 0;
 		uint32 i;
@@ -2235,28 +2235,19 @@ static void call_nt_transact_ioctl(connection_struct *conn,
 			labels = True;
 		}
 
-		shadow_mem_ctx = talloc_init("SHADOW_COPY_DATA");
-		if (shadow_mem_ctx == NULL) {
-			DEBUG(0,("talloc_init(SHADOW_COPY_DATA) failed!\n"));
-			reply_nterror(req, NT_STATUS_NO_MEMORY);
-			return;
-		}
-
-		shadow_data = TALLOC_ZERO_P(shadow_mem_ctx,SHADOW_COPY_DATA);
+		shadow_data = talloc_zero(talloc_tos(),
+					    struct shadow_copy_data);
 		if (shadow_data == NULL) {
 			DEBUG(0,("TALLOC_ZERO() failed!\n"));
-			talloc_destroy(shadow_mem_ctx);
 			reply_nterror(req, NT_STATUS_NO_MEMORY);
 			return;
 		}
 
-		shadow_data->mem_ctx = shadow_mem_ctx;
-
 		/*
 		 * Call the VFS routine to actually do the work.
 		 */
 		if (SMB_VFS_GET_SHADOW_COPY_DATA(fsp, shadow_data, labels)!=0) {
-			talloc_destroy(shadow_data->mem_ctx);
+			TALLOC_FREE(shadow_data);
 			if (errno == ENOSYS) {
 				DEBUG(5,("FSCTL_GET_SHADOW_COPY_DATA: connectpath %s, not supported.\n", 
 					conn->connectpath));
@@ -2281,14 +2272,14 @@ static void call_nt_transact_ioctl(connection_struct *conn,
 		if (max_data_count<data_count) {
 			DEBUG(0,("FSCTL_GET_SHADOW_COPY_DATA: max_data_count(%u) too small (%u) bytes needed!\n",
 				max_data_count,data_count));
-			talloc_destroy(shadow_data->mem_ctx);
+			TALLOC_FREE(shadow_data);
 			reply_nterror(req, NT_STATUS_BUFFER_TOO_SMALL);
 			return;
 		}
 
 		pdata = nttrans_realloc(ppdata, data_count);
 		if (pdata == NULL) {
-			talloc_destroy(shadow_data->mem_ctx);
+			TALLOC_FREE(shadow_data);
 			reply_nterror(req, NT_STATUS_NO_MEMORY);
 			return;
 		}
@@ -2321,7 +2312,7 @@ static void call_nt_transact_ioctl(connection_struct *conn,
 			}
 		}
 
-		talloc_destroy(shadow_data->mem_ctx);
+		TALLOC_FREE(shadow_data);
 
 		send_nt_replies(conn, req, NT_STATUS_OK, NULL, 0,
 				pdata, data_count);
@@ -2861,7 +2852,7 @@ static void handle_nttrans(connection_struct *conn,
 {
 	if (get_Protocol() >= PROTOCOL_NT1) {
 		req->flags2 |= 0x40; /* IS_LONG_NAME */
-		SSVAL(req->inbuf,smb_flg2,req->flags2);
+		SSVAL(discard_const_p(uint8_t, req->inbuf),smb_flg2,req->flags2);
 	}
 
 
@@ -3029,7 +3020,7 @@ void reply_nttrans(struct smb_request *req)
 		return;
 	}
 
-	if ((state = TALLOC_P(conn, struct trans_state)) == NULL) {
+	if ((state = talloc(conn, struct trans_state)) == NULL) {
 		reply_nterror(req, NT_STATUS_NO_MEMORY);
 		END_PROFILE(SMBnttrans);
 		return;
@@ -3203,7 +3194,7 @@ void reply_nttranss(struct smb_request *req)
 
 	START_PROFILE(SMBnttranss);
 
-	show_msg((char *)req->inbuf);
+	show_msg((const char *)req->inbuf);
 
 	if (req->wct < 18) {
 		reply_nterror(req, NT_STATUS_INVALID_PARAMETER);
diff --git a/source3/smbd/open.c b/source3/smbd/open.c
index 14e6bf9b4e6..86a5924f16e 100644
--- a/source3/smbd/open.c
+++ b/source3/smbd/open.c
@@ -77,6 +77,16 @@ NTSTATUS smbd_check_open_rights(struct connection_struct *conn,
 	NTSTATUS status;
 	struct security_descriptor *sd = NULL;
 
+	if ((access_mask & DELETE_ACCESS) && !lp_acl_check_permissions(SNUM(conn))) {
+		*access_granted = access_mask;
+
+		DEBUG(10,("smbd_check_open_rights: not checking ACL "
+			"on DELETE_ACCESS on file %s. Granting 0x%x\n",
+			smb_fname_str_dbg(smb_fname),
+			(unsigned int)*access_granted ));
+		return NT_STATUS_OK;
+	}
+
 	status = SMB_VFS_GET_NT_ACL(conn, smb_fname->base_name,
 			(SECINFO_OWNER |
 			SECINFO_GROUP |
@@ -241,6 +251,8 @@ void change_file_owner_to_parent(connection_struct *conn,
 		DEBUG(10,("change_file_owner_to_parent: changed new file %s to "
 			"parent directory uid %u.\n", fsp_str_dbg(fsp),
 			(unsigned int)smb_fname_parent->st.st_ex_uid));
+		/* Ensure the uid entry is updated. */
+		fsp->fsp_name->st.st_ex_uid = smb_fname_parent->st.st_ex_uid;
 	}
 
 	TALLOC_FREE(smb_fname_parent);
@@ -316,10 +328,9 @@ NTSTATUS change_dir_owner_to_parent(connection_struct *conn,
 
 	/* Ensure we're pointing at the same place. */
 	if (smb_fname_cwd->st.st_ex_dev != psbuf->st_ex_dev ||
-	    smb_fname_cwd->st.st_ex_ino != psbuf->st_ex_ino ||
-	    smb_fname_cwd->st.st_ex_mode != psbuf->st_ex_mode ) {
+	    smb_fname_cwd->st.st_ex_ino != psbuf->st_ex_ino) {
 		DEBUG(0,("change_dir_owner_to_parent: "
-			 "device/inode/mode on directory %s changed. "
+			 "device/inode on directory %s changed. "
 			 "Refusing to chown !\n", fname ));
 		status = NT_STATUS_ACCESS_DENIED;
 		goto chdir;
@@ -350,6 +361,8 @@ NTSTATUS change_dir_owner_to_parent(connection_struct *conn,
 		DEBUG(10,("change_dir_owner_to_parent: changed ownership of new "
 			"directory %s to parent directory uid %u.\n",
 			fname, (unsigned int)smb_fname_parent->st.st_ex_uid ));
+		/* Ensure the uid entry is updated. */
+		psbuf->st_ex_uid = smb_fname_parent->st.st_ex_uid;
 	}
 
  chdir:
@@ -378,6 +391,7 @@ static NTSTATUS open_file(files_struct *fsp,
 	int accmode = (flags & O_ACCMODE);
 	int local_flags = flags;
 	bool file_existed = VALID_STAT(fsp->fsp_name->st);
+	bool file_created = false;
 
 	fsp->fh->fd = -1;
 	errno = EPERM;
@@ -477,23 +491,7 @@ static NTSTATUS open_file(files_struct *fsp,
 		}
 
 		if ((local_flags & O_CREAT) && !file_existed) {
-
-			/* Inherit the ACL if required */
-			if (lp_inherit_perms(SNUM(conn))) {
-				inherit_access_posix_acl(conn, parent_dir,
-							 smb_fname->base_name,
-							 unx_mode);
-			}
-
-			/* Change the owner if required. */
-			if (lp_inherit_owner(SNUM(conn))) {
-				change_file_owner_to_parent(conn, parent_dir,
-							    fsp);
-			}
-
-			notify_fname(conn, NOTIFY_ACTION_ADDED,
-				     FILE_NOTIFY_CHANGE_FILE_NAME,
-				     smb_fname->base_name);
+			file_created = true;
 		}
 
 	} else {
@@ -603,6 +601,47 @@ static NTSTATUS open_file(files_struct *fsp,
 			fd_close(fsp);
 			return status;
 		}
+
+		if (file_created) {
+			bool need_re_stat = false;
+			/* Do all inheritance work after we've
+			   done a successful stat call and filled
+			   in the stat struct in fsp->fsp_name. */
+
+			/* Inherit the ACL if required */
+			if (lp_inherit_perms(SNUM(conn))) {
+				inherit_access_posix_acl(conn, parent_dir,
+							 smb_fname->base_name,
+							 unx_mode);
+				need_re_stat = true;
+			}
+
+			/* Change the owner if required. */
+			if (lp_inherit_owner(SNUM(conn))) {
+				change_file_owner_to_parent(conn, parent_dir,
+							    fsp);
+				need_re_stat = true;
+			}
+
+			if (need_re_stat) {
+				if (fsp->fh->fd == -1) {
+					ret = SMB_VFS_STAT(conn, smb_fname);
+				} else {
+					ret = SMB_VFS_FSTAT(fsp, &smb_fname->st);
+					/* If we have an fd, this stat should succeed. */
+					if (ret == -1) {
+						DEBUG(0,("Error doing fstat on open file %s "
+							 "(%s)\n",
+							 smb_fname_str_dbg(smb_fname),
+							 strerror(errno) ));
+					}
+				}
+			}
+
+			notify_fname(conn, NOTIFY_ACTION_ADDED,
+				     FILE_NOTIFY_CHANGE_FILE_NAME,
+				     smb_fname->base_name);
+		}
 	}
 
 	/*
@@ -949,7 +988,9 @@ static NTSTATUS send_break_message(files_struct *fsp,
  * Do internal consistency checks on the share mode for a file.
  */
 
-static void find_oplock_types(struct share_mode_lock *lck,
+static void find_oplock_types(files_struct *fsp,
+				int oplock_request,
+				struct share_mode_lock *lck,
 				struct share_mode_entry **pp_batch,
 				struct share_mode_entry **pp_ex_or_batch,
 				bool *got_level2,
@@ -962,11 +1003,27 @@ static void find_oplock_types(struct share_mode_lock *lck,
 	*got_level2 = false;
 	*got_no_oplock = false;
 
+	/* Ignore stat or internal opens, as is done in
+		delay_for_batch_oplocks() and
+		delay_for_exclusive_oplocks().
+	 */
+	if ((oplock_request & INTERNAL_OPEN_ONLY) || is_stat_open(fsp->access_mask)) {
+		return;
+	}
+
 	for (i=0; i<lck->num_share_modes; i++) {
 		if (!is_valid_share_mode_entry(&lck->share_modes[i])) {
 			continue;
 		}
 
+		if (lck->share_modes[i].op_type == NO_OPLOCK &&
+				is_stat_open(lck->share_modes[i].access_mask)) {
+			/* We ignore stat opens in the table - they
+			   always have NO_OPLOCK and never get or
+			   cause breaks. JRA. */
+			continue;
+		}
+
 		if (BATCH_OPLOCK_TYPE(lck->share_modes[i].op_type)) {
 			/* batch - can only be one. */
 			if (*pp_ex_or_batch || *pp_batch || *got_level2 || *got_no_oplock) {
@@ -1037,18 +1094,8 @@ static bool delay_for_exclusive_oplocks(files_struct *fsp,
 	return false;
 }
 
-static bool file_has_brlocks(files_struct *fsp)
-{
-	struct byte_range_lock *br_lck;
-
-	br_lck = brl_get_locks_readonly(fsp);
-	if (!br_lck)
-		return false;
-
-	return br_lck->num_locks > 0 ? true : false;
-}
-
 static void grant_fsp_oplock_type(files_struct *fsp,
+				const struct byte_range_lock *br_lck,
 				int oplock_request,
 				bool got_level2_oplock,
 				bool got_a_none_oplock)
@@ -1066,7 +1113,7 @@ static void grant_fsp_oplock_type(files_struct *fsp,
 		DEBUG(10,("grant_fsp_oplock_type: oplock type 0x%x on file %s\n",
 			fsp->oplock_type, fsp_str_dbg(fsp)));
 		return;
-	} else if (lp_locking(fsp->conn->params) && file_has_brlocks(fsp)) {
+	} else if (br_lck && br_lck->num_locks > 0) {
 		DEBUG(10,("grant_fsp_oplock_type: file %s has byte range locks\n",
 			fsp_str_dbg(fsp)));
 		fsp->oplock_type = NO_OPLOCK;
@@ -1544,6 +1591,55 @@ void remove_deferred_open_entry(struct file_id id, uint64_t mid,
 	}
 }
 
+/****************************************************************
+ Ensure we get the brlock lock followed by the share mode lock
+ in the correct order to prevent deadlocks if other smbd's are
+ using the brlock database on this file simultaneously with this open
+ (that code also gets the locks in brlock -> share mode lock order).
+****************************************************************/
+
+static bool acquire_ordered_locks(TALLOC_CTX *mem_ctx,
+				files_struct *fsp,
+				const struct file_id id,
+				const char *connectpath,
+				const struct smb_filename *smb_fname,
+				const struct timespec *p_old_write_time,
+				struct share_mode_lock **p_lck,
+				struct byte_range_lock **p_br_lck)
+{
+	/* Ordering - we must get the br_lck for this
+	   file before the share mode. */
+	if (lp_locking(fsp->conn->params)) {
+		*p_br_lck = brl_get_locks_readonly(fsp);
+		if (*p_br_lck == NULL) {
+			DEBUG(0, ("Could not get br_lock\n"));
+			return false;
+		}
+		/* Note - we don't need to free the returned
+		   br_lck explicitly as it was allocated on talloc_tos()
+		   and so will be autofreed (and release the lock)
+		   once the frame context disappears.
+
+		   If it was set to fsp->brlock_rec then it was
+		   talloc_move'd to hang off the fsp pointer and
+		   in this case is guarenteed to not be holding the
+		   lock on the brlock database. */
+	}
+
+	*p_lck = get_share_mode_lock(mem_ctx,
+				id,
+				connectpath,
+				smb_fname,
+				p_old_write_time);
+
+	if (*p_lck == NULL) {
+		DEBUG(0, ("Could not get share mode lock\n"));
+		TALLOC_FREE(*p_br_lck);
+		return false;
+	}
+	return true;
+}
+
 /****************************************************************************
  Open a file with a share mode. Passed in an already created files_struct *.
 ****************************************************************************/
@@ -1622,9 +1718,9 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 		unx_mode = (mode_t)(new_dos_attributes & ~FILE_FLAG_POSIX_SEMANTICS);
 		new_dos_attributes = 0;
 	} else {
-		/* We add aARCH to this as this mode is only used if the file is
+		/* We add FILE_ATTRIBUTE_ARCHIVE to this as this mode is only used if the file is
 		 * created new. */
-		unx_mode = unix_mode(conn, new_dos_attributes | aARCH,
+		unx_mode = unix_mode(conn, new_dos_attributes | FILE_ATTRIBUTE_ARCHIVE,
 				     smb_fname, parent_dir);
 	}
 
@@ -1888,6 +1984,7 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 	}
 
 	if (file_existed) {
+		struct byte_range_lock *br_lck = NULL;
 		struct share_mode_entry *batch_entry = NULL;
 		struct share_mode_entry *exclusive_entry = NULL;
 		bool got_level2_oplock = false;
@@ -1896,17 +1993,21 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 		struct timespec old_write_time = smb_fname->st.st_ex_mtime;
 		id = vfs_file_id_from_sbuf(conn, &smb_fname->st);
 
-		lck = get_share_mode_lock(talloc_tos(), id,
-					  conn->connectpath,
-					  smb_fname, &old_write_time);
-
-		if (lck == NULL) {
-			DEBUG(0, ("Could not get share mode lock\n"));
+		if (!acquire_ordered_locks(talloc_tos(),
+					fsp,
+					id,
+					conn->connectpath,
+					smb_fname,
+					&old_write_time,
+					&lck,
+					&br_lck)) {
 			return NT_STATUS_SHARING_VIOLATION;
 		}
 
 		/* Get the types we need to examine. */
-		find_oplock_types(lck,
+		find_oplock_types(fsp,
+				oplock_request,
+				lck,
 				&batch_entry,
 				&exclusive_entry,
 				&got_level2_oplock,
@@ -1953,6 +2054,7 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 		}
 
 		grant_fsp_oplock_type(fsp,
+				br_lck,
                                 oplock_request,
                                 got_level2_oplock,
                                 got_a_none_oplock);
@@ -2109,13 +2211,12 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 			     open_access_mask);
 
 	if (!NT_STATUS_IS_OK(fsp_open)) {
-		if (lck != NULL) {
-			TALLOC_FREE(lck);
-		}
+		TALLOC_FREE(lck);
 		return fsp_open;
 	}
 
 	if (!file_existed) {
+		struct byte_range_lock *br_lck = NULL;
 		struct share_mode_entry *batch_entry = NULL;
 		struct share_mode_entry *exclusive_entry = NULL;
 		bool got_level2_oplock = false;
@@ -2138,20 +2239,21 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 
 		id = fsp->file_id;
 
-		lck = get_share_mode_lock(talloc_tos(), id,
-					  conn->connectpath,
-					  smb_fname, &old_write_time);
-
-		if (lck == NULL) {
-			DEBUG(0, ("open_file_ntcreate: Could not get share "
-				  "mode lock for %s\n",
-				  smb_fname_str_dbg(smb_fname)));
-			fd_close(fsp);
+		if (!acquire_ordered_locks(talloc_tos(),
+					fsp,
+					id,
+					conn->connectpath,
+					smb_fname,
+					&old_write_time,
+					&lck,
+					&br_lck)) {
 			return NT_STATUS_SHARING_VIOLATION;
 		}
 
 		/* Get the types we need to examine. */
-		find_oplock_types(lck,
+		find_oplock_types(fsp,
+				oplock_request,
+				lck,
 				&batch_entry,
 				&exclusive_entry,
 				&got_level2_oplock,
@@ -2194,8 +2296,6 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 		if (!NT_STATUS_IS_OK(status)) {
 			struct deferred_open_record state;
 
-			fd_close(fsp);
-
 			state.delayed_for_oplocks = False;
 			state.id = id;
 
@@ -2211,10 +2311,12 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 					   req, &state);
 			}
 			TALLOC_FREE(lck);
+			fd_close(fsp);
 			return status;
 		}
 
 		grant_fsp_oplock_type(fsp,
+				br_lck,
                                 oplock_request,
                                 got_level2_oplock,
                                 got_a_none_oplock);
@@ -2335,7 +2437,8 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 		new_file_created = True;
 	}
 
-	set_share_mode(lck, fsp, get_current_uid(conn), 0,
+	set_share_mode(lck, fsp, get_current_uid(conn),
+			req ? req->mid : 0,
 		       fsp->oplock_type);
 
 	/* Handle strange delete on close create semantics. */
@@ -2361,7 +2464,7 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn,
 		    lp_store_dos_attributes(SNUM(conn))) {
 			if (!posix_open) {
 				if (file_set_dosmode(conn, smb_fname,
-					    new_dos_attributes | aARCH,
+					    new_dos_attributes | FILE_ATTRIBUTE_ARCHIVE,
 					    parent_dir, true) == 0) {
 					unx_mode = smb_fname->st.st_ex_mode;
 				}
@@ -2475,6 +2578,7 @@ static NTSTATUS mkdir_internal(connection_struct *conn,
 	char *parent_dir;
 	NTSTATUS status;
 	bool posix_open = false;
+	bool need_re_stat = false;
 
 	if(!CAN_WRITE(conn)) {
 		DEBUG(5,("mkdir_internal: failing create on read-only share "
@@ -2496,7 +2600,7 @@ static NTSTATUS mkdir_internal(connection_struct *conn,
 		posix_open = true;
 		mode = (mode_t)(file_attributes & ~FILE_FLAG_POSIX_SEMANTICS);
 	} else {
-		mode = unix_mode(conn, aDIR, smb_dname, parent_dir);
+		mode = unix_mode(conn, FILE_ATTRIBUTE_DIRECTORY, smb_dname, parent_dir);
 	}
 
 	if (SMB_VFS_MKDIR(conn, smb_dname->base_name, mode) != 0) {
@@ -2521,7 +2625,7 @@ static NTSTATUS mkdir_internal(connection_struct *conn,
 	if (lp_store_dos_attributes(SNUM(conn))) {
 		if (!posix_open) {
 			file_set_dosmode(conn, smb_dname,
-					 file_attributes | aDIR,
+					 file_attributes | FILE_ATTRIBUTE_DIRECTORY,
 					 parent_dir, true);
 		}
 	}
@@ -2529,6 +2633,7 @@ static NTSTATUS mkdir_internal(connection_struct *conn,
 	if (lp_inherit_perms(SNUM(conn))) {
 		inherit_access_posix_acl(conn, parent_dir,
 					 smb_dname->base_name, mode);
+		need_re_stat = true;
 	}
 
 	if (!posix_open) {
@@ -2543,6 +2648,7 @@ static NTSTATUS mkdir_internal(connection_struct *conn,
 			SMB_VFS_CHMOD(conn, smb_dname->base_name,
 				      (smb_dname->st.st_ex_mode |
 					  (mode & ~smb_dname->st.st_ex_mode)));
+			need_re_stat = true;
 		}
 	}
 
@@ -2551,6 +2657,15 @@ static NTSTATUS mkdir_internal(connection_struct *conn,
 		change_dir_owner_to_parent(conn, parent_dir,
 					   smb_dname->base_name,
 					   &smb_dname->st);
+		need_re_stat = true;
+	}
+
+	if (need_re_stat) {
+		if (SMB_VFS_LSTAT(conn, smb_dname) == -1) {
+			DEBUG(2, ("Could not stat directory '%s' just created: %s\n",
+			  smb_fname_str_dbg(smb_dname), strerror(errno)));
+			return map_nt_error_from_unix(errno);
+		}
 	}
 
 	notify_fname(conn, NOTIFY_ACTION_ADDED, FILE_NOTIFY_CHANGE_DIR_NAME,
@@ -2828,7 +2943,8 @@ static NTSTATUS open_directory(connection_struct *conn,
 		return status;
 	}
 
-	set_share_mode(lck, fsp, get_current_uid(conn), 0, NO_OPLOCK);
+	set_share_mode(lck, fsp, get_current_uid(conn),
+			req ? req->mid : 0, NO_OPLOCK);
 
 	/* For directories the delete on close bit at open time seems
 	   always to be honored on close... See test 19 in Samba4 BASE-DELETE. */
@@ -3018,7 +3134,7 @@ NTSTATUS open_streams_for_delete(connection_struct *conn,
 		return NT_STATUS_OK;
 	}
 
-	streams = TALLOC_ARRAY(talloc_tos(), files_struct *, num_streams);
+	streams = talloc_array(talloc_tos(), files_struct *, num_streams);
 	if (streams == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		status = NT_STATUS_NO_MEMORY;
@@ -3172,8 +3288,7 @@ static NTSTATUS create_file_unixpath(connection_struct *conn,
 
 	/* Setting FILE_SHARE_DELETE is the hint. */
 
-	if (lp_acl_check_permissions(SNUM(conn))
-	    && (create_disposition != FILE_CREATE)
+	if ((create_disposition != FILE_CREATE)
 	    && (access_mask & DELETE_ACCESS)
 	    && (!(can_delete_file_in_directory(conn, smb_fname) ||
 		 can_access_file_acl(conn, smb_fname, DELETE_ACCESS)))) {
@@ -3531,7 +3646,7 @@ NTSTATUS get_relative_fid_filename(connection_struct *conn,
 		 * Copy in the base directory name.
 		 */
 
-		parent_fname = TALLOC_ARRAY(talloc_tos(), char,
+		parent_fname = talloc_array(talloc_tos(), char,
 		    dir_name_len+2);
 		if (parent_fname == NULL) {
 			status = NT_STATUS_NO_MEMORY;
diff --git a/source3/smbd/oplock.c b/source3/smbd/oplock.c
index 087d6ef045a..284b0321aa8 100644
--- a/source3/smbd/oplock.c
+++ b/source3/smbd/oplock.c
@@ -224,7 +224,7 @@ bool should_notify_deferred_opens()
 static char *new_break_message_smb1(TALLOC_CTX *mem_ctx,
 				   files_struct *fsp, int cmd)
 {
-	char *result = TALLOC_ARRAY(mem_ctx, char, smb_size + 8*2 + 0);
+	char *result = talloc_array(mem_ctx, char, smb_size + 8*2 + 0);
 
 	if (result == NULL) {
 		DEBUG(0, ("talloc failed\n"));
@@ -353,7 +353,7 @@ static void add_oplock_timeout_handler(files_struct *fsp)
 	}
 
 	fsp->oplock_timeout =
-		event_add_timed(smbd_event_context(), fsp,
+		event_add_timed(server_event_context(), fsp,
 				timeval_current_ofs(OPLOCK_BREAK_TIMEOUT, 0),
 				oplock_timeout_handler, fsp);
 
@@ -459,7 +459,7 @@ void process_oplock_async_level2_break_message(struct messaging_context *msg_ctx
 	message_to_share_mode_entry(&msg, (char *)data->data);
 
 	DEBUG(10, ("Got oplock async level 2 break message from pid %s: "
-		   "%s/%lu\n", procid_str(talloc_tos(), &src),
+		   "%s/%lu\n", server_id_str(talloc_tos(), &src),
 		   file_id_string_tos(&msg.id), msg.share_file_id));
 
 	fsp = initial_break_processing(sconn, msg.id, msg.share_file_id);
@@ -510,7 +510,7 @@ static void process_oplock_break_message(struct messaging_context *msg_ctx,
 	message_to_share_mode_entry(&msg, (char *)data->data);
 
 	DEBUG(10, ("Got oplock break message from pid %s: %s/%lu\n",
-		   procid_str(talloc_tos(), &src), file_id_string_tos(&msg.id),
+		   server_id_str(talloc_tos(), &src), file_id_string_tos(&msg.id),
 		   msg.share_file_id));
 
 	fsp = initial_break_processing(sconn, msg.id, msg.share_file_id);
@@ -617,7 +617,7 @@ static void process_kernel_oplock_break(struct messaging_context *msg_ctx,
 	file_id = (unsigned long)IVAL(data->data, 24);
 
 	DEBUG(10, ("Got kernel oplock break message from pid %s: %s/%u\n",
-		   procid_str(talloc_tos(), &src), file_id_string_tos(&id),
+		   server_id_str(talloc_tos(), &src), file_id_string_tos(&id),
 		   (unsigned int)file_id));
 
 	fsp = initial_break_processing(sconn, id, file_id);
@@ -704,7 +704,7 @@ static void process_oplock_break_response(struct messaging_context *msg_ctx,
 	message_to_share_mode_entry(&msg, (char *)data->data);
 
 	DEBUG(10, ("Got oplock break response from pid %s: %s/%lu mid %llu\n",
-		   procid_str(talloc_tos(), &src), file_id_string_tos(&msg.id),
+		   server_id_str(talloc_tos(), &src), file_id_string_tos(&msg.id),
 		   msg.share_file_id, (unsigned long long)msg.op_mid));
 
 	schedule_deferred_open_message_smb(msg.op_mid);
@@ -732,7 +732,7 @@ static void process_open_retry_message(struct messaging_context *msg_ctx,
 	message_to_share_mode_entry(&msg, (char *)data->data);
 
 	DEBUG(10, ("Got open retry msg from pid %s: %s mid %llu\n",
-		   procid_str(talloc_tos(), &src), file_id_string_tos(&msg.id),
+		   server_id_str(talloc_tos(), &src), file_id_string_tos(&msg.id),
 		   (unsigned long long)msg.op_mid));
 
 	schedule_deferred_open_message_smb(msg.op_mid);
@@ -849,7 +849,7 @@ static void contend_level2_oplocks_begin_default(files_struct *fsp,
 	TALLOC_FREE(lck);
 }
 
-void contend_level2_oplocks_begin(files_struct *fsp,
+void smbd_contend_level2_oplocks_begin(files_struct *fsp,
 				  enum level2_contention_type type)
 {
 	if (koplocks && koplocks->ops->contend_level2_oplocks_begin) {
@@ -860,7 +860,7 @@ void contend_level2_oplocks_begin(files_struct *fsp,
 	contend_level2_oplocks_begin_default(fsp, type);
 }
 
-void contend_level2_oplocks_end(files_struct *fsp,
+void smbd_contend_level2_oplocks_end(files_struct *fsp,
 				enum level2_contention_type type)
 {
 	/* Only kernel oplocks implement this so far */
diff --git a/source3/smbd/oplock_irix.c b/source3/smbd/oplock_irix.c
index 207952d040c..ef737efd223 100644
--- a/source3/smbd/oplock_irix.c
+++ b/source3/smbd/oplock_irix.c
@@ -332,7 +332,7 @@ struct kernel_oplocks *irix_init_kernel_oplocks(TALLOC_CTX *mem_ctx)
 	ctx->read_fd = pfd[0];
 	ctx->write_fd = pfd[1];
 
-	ctx->read_fde = event_add_fd(smbd_event_context(),
+	ctx->read_fde = event_add_fd(server_event_context(),
 				     ctx,
 				     ctx->read_fd,
 				     EVENT_FD_READ,
diff --git a/source3/smbd/oplock_linux.c b/source3/smbd/oplock_linux.c
index 4f243e2e9ff..19f4d6e06d8 100644
--- a/source3/smbd/oplock_linux.c
+++ b/source3/smbd/oplock_linux.c
@@ -210,7 +210,7 @@ struct kernel_oplocks *linux_init_kernel_oplocks(TALLOC_CTX *mem_ctx)
 
 	ctx->ops = &linux_koplocks;
 
-	se = tevent_add_signal(smbd_event_context(),
+	se = tevent_add_signal(server_event_context(),
 			       ctx,
 			       RT_SIGNAL_LEASE, SA_SIGINFO,
 			       linux_oplock_signal_handler,
diff --git a/source3/smbd/oplock_onefs.c b/source3/smbd/oplock_onefs.c
index dbf465ca63a..4717b1faf7d 100644
--- a/source3/smbd/oplock_onefs.c
+++ b/source3/smbd/oplock_onefs.c
@@ -709,7 +709,7 @@ struct kernel_oplocks *onefs_init_kernel_oplocks(TALLOC_CTX *mem_ctx)
 	DEBUG(10, ("oplock event_fd = %d\n", ctx->onefs_event_fd));
 
 	/* Register the oplock event_fd with samba's event system */
-	ctx->read_fde = event_add_fd(smbd_event_context(),
+	ctx->read_fde = event_add_fd(server_event_context(),
 				     ctx,
 				     ctx->onefs_event_fd,
 				     EVENT_FD_READ,
diff --git a/source3/smbd/password.c b/source3/smbd/password.c
index 720b1bf4c20..69b37596c5d 100644
--- a/source3/smbd/password.c
+++ b/source3/smbd/password.c
@@ -549,7 +549,7 @@ static char *validate_group(struct smbd_server_connection *sconn,
 				DEBUG(10,("validate_group: = gr_mem = "
 					  "%s\n", gptr->gr_mem[i]));
 
-				safe_strcpy(member, gptr->gr_mem[i],
+				strlcpy(member, gptr->gr_mem[i],
 					list_len - (member-member_list));
 				member += member_len;
 			}
@@ -644,7 +644,7 @@ bool authorise_login(struct smbd_server_connection *sconn,
 					get_session_workgroup(sconn),
 					user2,password)) {
 				ok = True;
-				fstrcpy(user,user2);
+				strlcpy(user,user2,sizeof(fstring));
 				DEBUG(3,("authorise_login: ACCEPTED: session "
 					 "list username (%s) and given "
 					 "password ok\n", user));
@@ -695,7 +695,7 @@ bool authorise_login(struct smbd_server_connection *sconn,
 						get_session_workgroup(sconn),
 						user2,password)) {
 					ok = True;
-					fstrcpy(user,user2);
+					strlcpy(user,user2,sizeof(fstring));
 					DEBUG(3,("authorise_login: ACCEPTED: "
 						 "user list username and "
 						 "given password ok (%s)\n",
@@ -714,7 +714,7 @@ bool authorise_login(struct smbd_server_connection *sconn,
 		fstrcpy(guestname,lp_guestaccount());
 		guest_pw = Get_Pwnam_alloc(talloc_tos(), guestname);
 		if (guest_pw != NULL) {
-			fstrcpy(user,guestname);
+			strlcpy(user,guestname,sizeof(fstring));
 			ok = True;
 			DEBUG(3,("authorise_login: ACCEPTED: guest account "
 				 "and guest ok (%s)\n",	user));
diff --git a/source3/smbd/perfcount.c b/source3/smbd/perfcount.c
index 3bc83b98d5f..15618937582 100644
--- a/source3/smbd/perfcount.c
+++ b/source3/smbd/perfcount.c
@@ -85,7 +85,7 @@ NTSTATUS smb_register_perfcounter(int interface_version, const char *name,
 
 	entry = SMB_XMALLOC_P(struct smb_perfcount_module);
 	entry->name = smb_xstrdup(name);
-	entry->handlers = (struct smb_perfcount_handlers*) handlers;
+	entry->handlers = discard_const_p(struct smb_perfcount_handlers, handlers);
 
 	DLIST_ADD(modules, entry);
 	DEBUG(3, ("Successfully added perfcounter module '%s'\n", name));
diff --git a/source3/smbd/pipes.c b/source3/smbd/pipes.c
index 89d6c8dfca1..2de48adcbcf 100644
--- a/source3/smbd/pipes.c
+++ b/source3/smbd/pipes.c
@@ -29,6 +29,7 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "libcli/security/security.h"
+#include "rpc_server/srv_pipe_hnd.h"
 
 #define	PIPE		"\\PIPE\\"
 #define	PIPELEN		strlen(PIPE)
@@ -202,7 +203,7 @@ void reply_pipe_write(struct smb_request *req)
 	DEBUG(6, ("reply_pipe_write: %x name: %s len: %d\n", (int)fsp->fnum,
 		  fsp_str_dbg(fsp), (int)state->numtowrite));
 
-	subreq = np_write_send(state, smbd_event_context(),
+	subreq = np_write_send(state, server_event_context(),
 			       fsp->fake_file_handle, data, state->numtowrite);
 	if (subreq == NULL) {
 		TALLOC_FREE(state);
@@ -269,7 +270,7 @@ void reply_pipe_write_and_X(struct smb_request *req)
 {
 	files_struct *fsp = file_fsp(req, SVAL(req->vwv+2, 0));
 	int smb_doff = SVAL(req->vwv+11, 0);
-	uint8_t *data;
+	const uint8_t *data;
 	struct pipe_write_andx_state *state;
 	struct tevent_req *subreq;
 
@@ -298,7 +299,7 @@ void reply_pipe_write_and_X(struct smb_request *req)
 	DEBUG(6, ("reply_pipe_write_and_X: %x name: %s len: %d\n",
 		  (int)fsp->fnum, fsp_str_dbg(fsp), (int)state->numtowrite));
 
-	data = (uint8_t *)smb_base(req->inbuf) + smb_doff;
+	data = (const uint8_t *)smb_base(req->inbuf) + smb_doff;
 
 	if (state->pipe_start_message_raw) {
 		/*
@@ -318,7 +319,7 @@ void reply_pipe_write_and_X(struct smb_request *req)
 		state->numtowrite -= 2;
 	}
 
-	subreq = np_write_send(state, smbd_event_context(),
+	subreq = np_write_send(state, server_event_context(),
 			       fsp->fake_file_handle, data, state->numtowrite);
 	if (subreq == NULL) {
 		TALLOC_FREE(state);
@@ -425,7 +426,7 @@ void reply_pipe_read_and_X(struct smb_request *req)
 	state->outbuf = req->outbuf;
 	req->outbuf = NULL;
 
-	subreq = np_read_send(state, smbd_event_context(),
+	subreq = np_read_send(state, server_event_context(),
 			      fsp->fake_file_handle, data,
 			      state->smb_maxcnt);
 	if (subreq == NULL) {
diff --git a/source3/smbd/posix_acls.c b/source3/smbd/posix_acls.c
index 9252ee639fd..da25a52b8db 100644
--- a/source3/smbd/posix_acls.c
+++ b/source3/smbd/posix_acls.c
@@ -3687,7 +3687,7 @@ NTSTATUS append_parent_acl(files_struct *fsp,
 
 	num_aces += parent_sd->dacl->num_aces;
 
-	if((new_ace = TALLOC_ZERO_ARRAY(mem_ctx, struct security_ace,
+	if((new_ace = talloc_zero_array(mem_ctx, struct security_ace,
 					num_aces)) == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -4744,13 +4744,13 @@ struct security_descriptor *get_nt_acl_no_snum( TALLOC_CTX *ctx, const char *fna
 	struct fd_handle fh;
 	NTSTATUS status;
 
-	conn = TALLOC_ZERO_P(ctx, connection_struct);
+	conn = talloc_zero(ctx, connection_struct);
 	if (conn == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
 	}
 
-	if (!(conn->params = TALLOC_P(conn, struct share_params))) {
+	if (!(conn->params = talloc(conn, struct share_params))) {
 		DEBUG(0,("get_nt_acl_no_snum: talloc() failed!\n"));
 		TALLOC_FREE(conn);
 		return NULL;
diff --git a/source3/smbd/process.c b/source3/smbd/process.c
index 248b3e403d0..f3f532fbaec 100644
--- a/source3/smbd/process.c
+++ b/source3/smbd/process.c
@@ -19,6 +19,7 @@
 */
 
 #include "includes.h"
+#include "../lib/tsocket/tsocket.h"
 #include "system/filesys.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
@@ -32,6 +33,8 @@
 #include "auth.h"
 #include "messages.h"
 #include "smbprofile.h"
+#include "rpc_server/spoolss/srv_spoolss_nt.h"
+#include "libsmb/libsmb.h"
 
 extern bool global_machine_password_needs_changing;
 
@@ -318,7 +321,7 @@ static NTSTATUS receive_smb_raw_talloc_partial_read(TALLOC_CTX *mem_ctx,
 
 		/* Copy the header we've written. */
 
-		*buffer = (char *)TALLOC_MEMDUP(mem_ctx,
+		*buffer = (char *)talloc_memdup(mem_ctx,
 				writeX_header,
 				sizeof(writeX_header));
 
@@ -343,7 +346,7 @@ static NTSTATUS receive_smb_raw_talloc_partial_read(TALLOC_CTX *mem_ctx,
 	 * talloc and return.
 	 */
 
-	*buffer = TALLOC_ARRAY(mem_ctx, char, len+4);
+	*buffer = talloc_array(mem_ctx, char, len+4);
 
 	if (*buffer == NULL) {
 		DEBUG(0, ("Could not allocate inbuf of length %d\n",
@@ -412,7 +415,7 @@ static NTSTATUS receive_smb_raw_talloc(TALLOC_CTX *mem_ctx,
 	 * The +4 here can't wrap, we've checked the length above already.
 	 */
 
-	*buffer = TALLOC_ARRAY(mem_ctx, char, len+4);
+	*buffer = talloc_array(mem_ctx, char, len+4);
 
 	if (*buffer == NULL) {
 		DEBUG(0, ("Could not allocate inbuf of length %d\n",
@@ -501,9 +504,9 @@ static bool init_smb_request(struct smb_request *req,
 	req->vuid   = SVAL(inbuf, smb_uid);
 	req->tid    = SVAL(inbuf, smb_tid);
 	req->wct    = CVAL(inbuf, smb_wct);
-	req->vwv    = (uint16_t *)(inbuf+smb_vwv);
+	req->vwv    = discard_const_p(uint16_t, (inbuf+smb_vwv));
 	req->buflen = smb_buflen(inbuf);
-	req->buf    = (const uint8_t *)smb_buf(inbuf);
+	req->buf    = (const uint8_t *)smb_buf_const(inbuf);
 	req->unread_bytes = unread_bytes;
 	req->encrypted = encrypted;
 	req->sconn = sconn;
@@ -522,7 +525,7 @@ static bool init_smb_request(struct smb_request *req,
 		return false;
 	}
 	/* Ensure bcc is correct. */
-	if (((uint8 *)smb_buf(inbuf)) + req->buflen > inbuf + req_size) {
+	if (((const uint8_t *)smb_buf_const(inbuf)) + req->buflen > inbuf + req_size) {
 		DEBUG(0,("init_smb_request: invalid bcc number %u "
 			"(wct = %u, size %u)\n",
 			(unsigned int)req->buflen,
@@ -591,7 +594,7 @@ static bool push_queued_message(struct smb_request *req,
 	int msg_len = smb_len(req->inbuf) + 4;
 	struct pending_message_list *msg;
 
-	msg = TALLOC_ZERO_P(NULL, struct pending_message_list);
+	msg = talloc_zero(NULL, struct pending_message_list);
 
 	if(msg == NULL) {
 		DEBUG(0,("push_message: malloc fail (1)\n"));
@@ -621,7 +624,7 @@ static bool push_queued_message(struct smb_request *req,
 		}
 	}
 
-	msg->te = event_add_timed(smbd_event_context(),
+	msg->te = event_add_timed(server_event_context(),
 				  msg,
 				  end_time,
 				  smbd_deferred_open_timer,
@@ -705,7 +708,7 @@ void schedule_deferred_open_message_smb(uint64_t mid)
 				"scheduling mid %llu\n",
 				(unsigned long long)mid ));
 
-			te = event_add_timed(smbd_event_context(),
+			te = event_add_timed(server_event_context(),
 					     pml,
 					     timeval_zero(),
 					     smbd_deferred_open_timer,
@@ -888,7 +891,7 @@ struct idle_event *event_add_idle(struct event_context *event_ctx,
 	struct idle_event *result;
 	struct timeval now = timeval_current();
 
-	result = TALLOC_P(mem_ctx, struct idle_event);
+	result = talloc(mem_ctx, struct idle_event);
 	if (result == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		return NULL;
@@ -931,8 +934,8 @@ void smbd_setup_sig_term_handler(void)
 {
 	struct tevent_signal *se;
 
-	se = tevent_add_signal(smbd_event_context(),
-			       smbd_event_context(),
+	se = tevent_add_signal(server_event_context(),
+			       server_event_context(),
 			       SIGTERM, 0,
 			       smbd_sig_term_handler,
 			       NULL);
@@ -984,11 +987,11 @@ static NTSTATUS smbd_server_connection_loop_once(struct smbd_server_connection *
 	 * select for longer than it would take to wait for them.
 	 */
 
-	event_add_to_poll_args(smbd_event_context(), conn,
+	event_add_to_poll_args(server_event_context(), conn,
 			       &conn->pfds, &num_pfds, &timeout);
 
 	/* Process a signal and timed events now... */
-	if (run_events_poll(smbd_event_context(), 0, NULL, 0)) {
+	if (run_events_poll(server_event_context(), 0, NULL, 0)) {
 		return NT_STATUS_RETRY;
 	}
 
@@ -1003,11 +1006,14 @@ static NTSTATUS smbd_server_connection_loop_once(struct smbd_server_connection *
 		errno = sav;
 	}
 
-	if (ret == -1 && errno != EINTR) {
+	if (ret == -1) {
+		if (errno == EINTR) {
+			return NT_STATUS_RETRY;
+		}
 		return map_nt_error_from_unix(errno);
 	}
 
-	retry = run_events_poll(smbd_event_context(), ret, conn->pfds,
+	retry = run_events_poll(server_event_context(), ret, conn->pfds,
 				num_pfds);
 	if (retry) {
 		return NT_STATUS_RETRY;
@@ -1345,12 +1351,12 @@ static bool create_outbuf(TALLOC_CTX *mem_ctx, struct smb_request *req,
 		char *msg;
 		if (asprintf(&msg, "num_bytes too large: %u",
 			     (unsigned)num_bytes) == -1) {
-			msg = CONST_DISCARD(char *, "num_bytes too large");
+			msg = discard_const_p(char, "num_bytes too large");
 		}
 		smb_panic(msg);
 	}
 
-	*outbuf = TALLOC_ARRAY(mem_ctx, char,
+	*outbuf = talloc_array(mem_ctx, char,
 			       smb_size + num_words*2 + num_bytes);
 	if (*outbuf == NULL) {
 		return false;
@@ -1372,7 +1378,7 @@ static bool create_outbuf(TALLOC_CTX *mem_ctx, struct smb_request *req,
 void reply_outbuf(struct smb_request *req, uint8 num_words, uint32 num_bytes)
 {
 	char *outbuf;
-	if (!create_outbuf(req, req, (char *)req->inbuf, &outbuf, num_words,
+	if (!create_outbuf(req, req, (const char *)req->inbuf, &outbuf, num_words,
 			   num_bytes)) {
 		smb_panic("could not allocate output buffer\n");
 	}
@@ -1444,7 +1450,7 @@ static connection_struct *switch_message(uint8 type, struct smb_request *req, in
 
 	if (smb_messages[type].fn == NULL) {
 		DEBUG(0,("Unknown message type %d!\n",type));
-		smb_dump("Unknown", 1, (char *)req->inbuf, size);
+		smb_dump("Unknown", 1, (const char *)req->inbuf, size);
 		reply_unknown_new(req, type);
 		return NULL;
 	}
@@ -1459,10 +1465,10 @@ static connection_struct *switch_message(uint8 type, struct smb_request *req, in
 	DEBUG(3,("switch message %s (pid %d) conn 0x%lx\n", smb_fn_name(type),
 		 (int)sys_getpid(), (unsigned long)conn));
 
-	smb_dump(smb_fn_name(type), 1, (char *)req->inbuf, size);
+	smb_dump(smb_fn_name(type), 1, (const char *)req->inbuf, size);
 
 	/* Ensure this value is replaced in the incoming packet. */
-	SSVAL(req->inbuf,smb_uid,session_tag);
+	SSVAL(discard_const_p(uint8_t, req->inbuf),smb_uid,session_tag);
 
 	/*
 	 * Ensure the correct username is in current_user_info.  This is a
@@ -1748,7 +1754,7 @@ static void construct_reply_common(struct smb_request *req, const char *inbuf,
 
 void construct_reply_common_req(struct smb_request *req, char *outbuf)
 {
-	construct_reply_common(req, (char *)req->inbuf, outbuf);
+	construct_reply_common(req, (const char *)req->inbuf, outbuf);
 }
 
 /*
@@ -1888,7 +1894,7 @@ static bool smb_splice_chain(uint8_t **poutbuf, uint8_t smb_command,
 		return false;
 	}
 
-	outbuf = TALLOC_REALLOC_ARRAY(NULL, *poutbuf, uint8_t, new_size);
+	outbuf = talloc_realloc(NULL, *poutbuf, uint8_t, new_size);
 	if (outbuf == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		return false;
@@ -1902,7 +1908,7 @@ static bool smb_splice_chain(uint8_t **poutbuf, uint8_t smb_command,
 
 		if (!find_andx_cmd_ofs(outbuf, &andx_cmd_ofs)) {
 			DEBUG(1, ("invalid command chain\n"));
-			*poutbuf = TALLOC_REALLOC_ARRAY(
+			*poutbuf = talloc_realloc(
 				NULL, *poutbuf, uint8_t, old_size);
 			return false;
 		}
@@ -1971,9 +1977,9 @@ void chain_reply(struct smb_request *req)
 	uint32_t chain_offset;	/* uint32_t to avoid overflow */
 
 	uint8_t wct;
-	uint16_t *vwv;
+	const uint16_t *vwv;
 	uint16_t buflen;
-	uint8_t *buf;
+	const uint8_t *buf;
 
 	if (IVAL(req->outbuf, smb_rcls) != 0) {
 		fixup_chain_error_packet(req);
@@ -1990,7 +1996,7 @@ void chain_reply(struct smb_request *req)
 
 	if ((req->wct < 2) || (CVAL(req->outbuf, smb_wct) < 2)) {
 		if (req->chain_outbuf == NULL) {
-			req->chain_outbuf = TALLOC_REALLOC_ARRAY(
+			req->chain_outbuf = talloc_realloc(
 				req, req->outbuf, uint8_t,
 				smb_len(req->outbuf) + 4);
 			if (req->chain_outbuf == NULL) {
@@ -2022,7 +2028,7 @@ void chain_reply(struct smb_request *req)
 		 * over-allocated (reply_pipe_read_and_X used to be such an
 		 * example).
 		 */
-		req->chain_outbuf = TALLOC_REALLOC_ARRAY(
+		req->chain_outbuf = talloc_realloc(
 			req, req->outbuf, uint8_t, smb_len(req->outbuf) + 4);
 		if (req->chain_outbuf == NULL) {
 			smb_panic("talloc failed");
@@ -2122,7 +2128,7 @@ void chain_reply(struct smb_request *req)
 	if (length_needed > smblen) {
 		goto error;
 	}
-	vwv = (uint16_t *)(smb_base(req->inbuf) + chain_offset + 1);
+	vwv = (const uint16_t *)(smb_base(req->inbuf) + chain_offset + 1);
 
 	/*
 	 * Now grab the new byte buffer....
@@ -2138,11 +2144,11 @@ void chain_reply(struct smb_request *req)
 	if (length_needed > smblen) {
 		goto error;
 	}
-	buf = (uint8_t *)(vwv+wct+1);
+	buf = (const uint8_t *)(vwv+wct+1);
 
 	req->cmd = chain_cmd;
 	req->wct = wct;
-	req->vwv = vwv;
+	req->vwv = discard_const_p(uint16_t, vwv);
 	req->buflen = buflen;
 	req->buf = buf;
 
@@ -2240,13 +2246,14 @@ static bool fd_is_readable(int fd)
 
 }
 
-static void smbd_server_connection_write_handler(struct smbd_server_connection *conn)
+static void smbd_server_connection_write_handler(
+	struct smbd_server_connection *sconn)
 {
 	/* TODO: make write nonblocking */
 }
 
 static void smbd_server_connection_read_handler(
-	struct smbd_server_connection *conn, int fd)
+	struct smbd_server_connection *sconn, int fd)
 {
 	uint8_t *inbuf = NULL;
 	size_t inbuf_len = 0;
@@ -2256,29 +2263,29 @@ static void smbd_server_connection_read_handler(
 	NTSTATUS status;
 	uint32_t seqnum;
 
-	bool from_client = (conn->sock == fd);
+	bool from_client = (sconn->sock == fd);
 
 	if (from_client) {
-		smbd_lock_socket(conn);
+		smbd_lock_socket(sconn);
 
 		if (lp_async_smb_echo_handler() && !fd_is_readable(fd)) {
 			DEBUG(10,("the echo listener was faster\n"));
-			smbd_unlock_socket(conn);
+			smbd_unlock_socket(sconn);
 			return;
 		}
 
 		/* TODO: make this completely nonblocking */
-		status = receive_smb_talloc(mem_ctx, conn, fd,
+		status = receive_smb_talloc(mem_ctx, sconn, fd,
 					    (char **)(void *)&inbuf,
 					    0, /* timeout */
 					    &unread_bytes,
 					    &encrypted,
 					    &inbuf_len, &seqnum,
 					    false /* trusted channel */);
-		smbd_unlock_socket(conn);
+		smbd_unlock_socket(sconn);
 	} else {
 		/* TODO: make this completely nonblocking */
-		status = receive_smb_talloc(mem_ctx, conn, fd,
+		status = receive_smb_talloc(mem_ctx, sconn, fd,
 					    (char **)(void *)&inbuf,
 					    0, /* timeout */
 					    &unread_bytes,
@@ -2298,7 +2305,7 @@ static void smbd_server_connection_read_handler(
 	}
 
 process:
-	process_smb(conn, inbuf, inbuf_len, unread_bytes,
+	process_smb(sconn, inbuf, inbuf_len, unread_bytes,
 		    seqnum, encrypted, NULL);
 }
 
@@ -2339,6 +2346,7 @@ static void smbd_server_echo_handler(struct event_context *ev,
 	}
 }
 
+#ifdef CLUSTER_SUPPORT
 /****************************************************************************
 received when we should release a specific IP
 ****************************************************************************/
@@ -2351,6 +2359,9 @@ static void release_ip(const char *ip, void *priv)
 		p = addr + 7;
 	}
 
+	DEBUG(10, ("Got release IP message for %s, "
+		   "our address is %s\n", ip, p));
+
 	if ((strcmp(p, ip) == 0) || ((p != addr) && strcmp(addr, ip) == 0)) {
 		/* we can't afford to do a clean exit - that involves
 		   database writes, which would potentially mean we
@@ -2365,16 +2376,6 @@ static void release_ip(const char *ip, void *priv)
 	}
 }
 
-static void msg_release_ip(struct messaging_context *msg_ctx, void *private_data,
-			   uint32_t msg_type, struct server_id server_id, DATA_BLOB *data)
-{
-	struct smbd_server_connection *sconn = talloc_get_type_abort(
-		private_data, struct smbd_server_connection);
-
-	release_ip((char *)data->data, sconn->client_id.addr);
-}
-
-#ifdef CLUSTER_SUPPORT
 static int client_get_tcp_info(int sock, struct sockaddr_storage *server,
 			       struct sockaddr_storage *client)
 {
@@ -2608,7 +2609,7 @@ static bool smbd_echo_reply(uint8_t *inbuf, size_t inbuf_len,
 		return false;
 	}
 
-	if (!create_outbuf(talloc_tos(), &req, (char *)req.inbuf, &outbuf,
+	if (!create_outbuf(talloc_tos(), &req, (const char *)req.inbuf, &outbuf,
 			   1, req.buflen)) {
 		DEBUG(10, ("create_outbuf failed\n"));
 		return false;
@@ -2788,7 +2789,7 @@ static void smbd_echo_loop(struct smbd_server_connection *sconn,
 /*
  * Handle SMBecho requests in a forked child process
  */
-static bool fork_echo_handler(struct smbd_server_connection *sconn)
+bool fork_echo_handler(struct smbd_server_connection *sconn)
 {
 	int listener_pipe[2];
 	int res;
@@ -2813,7 +2814,7 @@ static bool fork_echo_handler(struct smbd_server_connection *sconn)
 		set_blocking(listener_pipe[1], false);
 
 		status = reinit_after_fork(sconn->msg_ctx,
-					   smbd_event_context(),
+					   server_event_context(),
 					   procid_self(), false);
 		if (!NT_STATUS_IS_OK(status)) {
 			DEBUG(1, ("reinit_after_fork failed: %s\n",
@@ -2833,7 +2834,7 @@ static bool fork_echo_handler(struct smbd_server_connection *sconn)
 	 * Without smb signing this is the same as the normal smbd
 	 * listener. This needs to change once signing comes in.
 	 */
-	sconn->smb1.echo_handler.trusted_fde = event_add_fd(smbd_event_context(),
+	sconn->smb1.echo_handler.trusted_fde = event_add_fd(server_event_context(),
 					sconn,
 					sconn->smb1.echo_handler.trusted_fd,
 					EVENT_FD_READ,
@@ -2902,8 +2903,7 @@ void smbd_process(struct smbd_server_connection *sconn)
 	const char *remaddr = NULL;
 	int ret;
 
-	if (lp_maxprotocol() == PROTOCOL_SMB2 &&
-	    !lp_async_smb_echo_handler()) {
+	if (lp_maxprotocol() == PROTOCOL_SMB2) {
 		/*
 		 * We're not making the decision here,
 		 * we're just allowing the client
@@ -3024,10 +3024,6 @@ void smbd_process(struct smbd_server_connection *sconn)
 		exit_server("Failed to init smb_signing");
 	}
 
-	if (lp_async_smb_echo_handler() && !fork_echo_handler(sconn)) {
-		exit_server("Failed to fork echo handler");
-	}
-
 	/* Setup oplocks */
 	if (!init_oplocks(sconn->msg_ctx))
 		exit_server("Failed to init oplocks");
@@ -3035,8 +3031,6 @@ void smbd_process(struct smbd_server_connection *sconn)
 	/* register our message handlers */
 	messaging_register(sconn->msg_ctx, NULL,
 			   MSG_SMB_FORCE_TDIS, msg_force_tdis);
-	messaging_register(sconn->msg_ctx, sconn,
-			   MSG_SMB_RELEASE_IP, msg_release_ip);
 	messaging_register(sconn->msg_ctx, NULL,
 			   MSG_SMB_CLOSE_FILE, msg_close_file);
 
@@ -3050,7 +3044,7 @@ void smbd_process(struct smbd_server_connection *sconn)
 			   MSG_DEBUG, debug_message);
 
 	if ((lp_keepalive() != 0)
-	    && !(event_add_idle(smbd_event_context(), NULL,
+	    && !(event_add_idle(server_event_context(), NULL,
 				timeval_set(lp_keepalive(), 0),
 				"keepalive", keepalive_fn,
 				NULL))) {
@@ -3058,14 +3052,14 @@ void smbd_process(struct smbd_server_connection *sconn)
 		exit(1);
 	}
 
-	if (!(event_add_idle(smbd_event_context(), NULL,
+	if (!(event_add_idle(server_event_context(), NULL,
 			     timeval_set(IDLE_CLOSED_TIMEOUT, 0),
 			     "deadtime", deadtime_fn, sconn))) {
 		DEBUG(0, ("Could not add deadtime event\n"));
 		exit(1);
 	}
 
-	if (!(event_add_idle(smbd_event_context(), NULL,
+	if (!(event_add_idle(server_event_context(), NULL,
 			     timeval_set(SMBD_HOUSEKEEPING_INTERVAL, 0),
 			     "housekeeping", housekeeping_fn, sconn))) {
 		DEBUG(0, ("Could not add housekeeping event\n"));
@@ -3122,7 +3116,7 @@ void smbd_process(struct smbd_server_connection *sconn)
 		exit_server("init_dptrs() failed");
 	}
 
-	sconn->smb1.fde = event_add_fd(smbd_event_context(),
+	sconn->smb1.fde = event_add_fd(server_event_context(),
 						  sconn,
 						  sconn->sock,
 						  EVENT_FD_READ,
@@ -3157,7 +3151,7 @@ void smbd_process(struct smbd_server_connection *sconn)
 
 bool req_is_in_chain(struct smb_request *req)
 {
-	if (req->vwv != (uint16_t *)(req->inbuf+smb_vwv)) {
+	if (req->vwv != (const uint16_t *)(req->inbuf+smb_vwv)) {
 		/*
 		 * We're right now handling a subsequent request, so we must
 		 * be in a chain
diff --git a/source3/smbd/proto.h b/source3/smbd/proto.h
index 36eea5e16ee..adeaf688996 100644
--- a/source3/smbd/proto.h
+++ b/source3/smbd/proto.h
@@ -1,3 +1,50 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  Main SMB server routines
+ *
+ *  Copyright (C) Andrew Tridgell			1992-2002,2006
+ *  Copyright (C) Jeremy Allison			1992-2010
+ *  Copyright (C) Volker Lendecke			1993-2009
+ *  Copyright (C) John H Terpstra			1995-1998
+ *  Copyright (C) Luke Kenneth Casson Leighton		1996-1998
+ *  Copyright (C) Paul Ashton				1997-1998
+ *  Copyright (C) Tim Potter				1999-2000
+ *  Copyright (C) T.D.Lee@durham.ac.uk			1999
+ *  Copyright (C) Ying Chen				2000
+ *  Copyright (C) Shirish Kalele			2000
+ *  Copyright (C) Andrew Bartlett			2001-2003
+ *  Copyright (C) Alexander Bokovoy			2002,2005
+ *  Copyright (C) Simo Sorce				2001-2002,2009
+ *  Copyright (C) Andreas Gruenbacher			2002
+ *  Copyright (C) Jim McDonough <jmcd@us.ibm.com>	2002
+ *  Copyright (C) Martin Pool				2002
+ *  Copyright (C) Luke Howard				2003
+ *  Copyright (C) Stefan (metze) Metzmacher		2003,2009
+ *  Copyright (C) Steve French				2005
+ *  Copyright (C) Gerald (Jerry) Carter			2006
+ *  Copyright (C) James Peach				2006-2007
+ *  Copyright (C) Jelmer Vernooij			2002-2003
+ *  Copyright (C) Michael Adam				2007
+ *  Copyright (C) Rishi Srivatsavai			2007
+ *  Copyright (C) Tim Prouty				2009
+ *  Copyright (C) Gregor Beck				2011
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _SMBD_PROTO_H_
+#define _SMBD_PROTO_H_
 
 /* The following definitions come from smbd/signing.c  */
 
@@ -23,7 +70,7 @@ NTSTATUS schedule_aio_read_and_X(connection_struct *conn,
 			     size_t smb_maxcnt);
 NTSTATUS schedule_aio_write_and_X(connection_struct *conn,
 			      struct smb_request *req,
-			      files_struct *fsp, char *data,
+			      files_struct *fsp, const char *data,
 			      SMB_OFF_T startpos,
 			      size_t numtowrite);
 NTSTATUS schedule_smb2_aio_read(connection_struct *conn,
@@ -63,6 +110,9 @@ bool push_blocking_lock_request( struct byte_range_lock *br_lck,
 		uint64_t offset,
 		uint64_t count,
 		uint64_t blocking_smblctx);
+void smbd_cancel_pending_lock_requests_by_fid(files_struct *fsp,
+			struct byte_range_lock *br_lck,
+			enum file_close_type close_type);
 void cancel_pending_lock_requests_by_fid(files_struct *fsp,
 			struct byte_range_lock *br_lck,
 			enum file_close_type close_type);
@@ -94,7 +144,7 @@ NTSTATUS delete_all_streams(connection_struct *conn, const char *fname);
 
 void conn_init(struct smbd_server_connection *sconn);
 int conn_num_open(struct smbd_server_connection *sconn);
-bool conn_snum_used(int snum);
+bool conn_snum_used(struct smbd_server_connection *sconn, int snum);
 connection_struct *conn_find(struct smbd_server_connection *sconn,
 			     unsigned cnum);
 connection_struct *conn_new(struct smbd_server_connection *sconn);
@@ -102,6 +152,7 @@ bool conn_close_all(struct smbd_server_connection *sconn);
 bool conn_idle_all(struct smbd_server_connection *sconn, time_t t);
 void conn_clear_vuid_caches(struct smbd_server_connection *sconn, uint16 vuid);
 void conn_free(connection_struct *conn);
+void conn_force_tdis(struct smbd_server_connection *sconn, const char *sharename);
 void msg_force_tdis(struct messaging_context *msg,
 		    void *private_data,
 		    uint32_t msg_type,
@@ -188,6 +239,8 @@ void DirCacheAdd(struct smb_Dir *dirp, const char *name, long offset);
 bool SearchDir(struct smb_Dir *dirp, const char *name, long *poffset);
 NTSTATUS can_delete_directory(struct connection_struct *conn,
 				const char *dirname);
+NTSTATUS smbd_can_delete_directory(struct connection_struct *conn,
+				const char *dirname);
 
 /* The following definitions come from smbd/dmapi.c  */
 
@@ -407,7 +460,8 @@ bool create_junction(TALLOC_CTX *ctx,
 		struct junction_map *jucn);
 bool create_msdfs_link(const struct junction_map *jucn);
 bool remove_msdfs_link(const struct junction_map *jucn);
-struct junction_map *enum_msdfs_links(TALLOC_CTX *ctx, size_t *p_num_jn);
+struct junction_map *enum_msdfs_links(struct smbd_server_connection *sconn,
+				      TALLOC_CTX *ctx, size_t *p_num_jn);
 NTSTATUS resolve_dfspath(TALLOC_CTX *ctx,
 			connection_struct *conn,
 			bool dfs_pathnames,
@@ -621,6 +675,10 @@ void contend_level2_oplocks_begin(files_struct *fsp,
 				  enum level2_contention_type type);
 void contend_level2_oplocks_end(files_struct *fsp,
 				enum level2_contention_type type);
+void smbd_contend_level2_oplocks_begin(files_struct *fsp,
+				  enum level2_contention_type type);
+void smbd_contend_level2_oplocks_end(files_struct *fsp,
+				enum level2_contention_type type);
 void share_mode_entry_to_message(char *msg, const struct share_mode_entry *e);
 void message_to_share_mode_entry(struct share_mode_entry *e, char *msg);
 bool init_oplocks(struct messaging_context *msg_ctx);
@@ -750,6 +808,7 @@ size_t req_wct_ofs(struct smb_request *req);
 void chain_reply(struct smb_request *req);
 bool req_is_in_chain(struct smb_request *req);
 void smbd_process(struct smbd_server_connection *sconn);
+bool fork_echo_handler(struct smbd_server_connection *sconn);
 
 /* The following definitions come from smbd/quotas.c  */
 
@@ -996,10 +1055,18 @@ bool stat_cache_lookup(connection_struct *conn,
 			char **pp_dirpath,
 			char **pp_start,
 			SMB_STRUCT_STAT *pst);
+void smbd_send_stat_cache_delete_message(struct messaging_context *msg_ctx,
+				    const char *name);
 void send_stat_cache_delete_message(struct messaging_context *msg_ctx,
 				    const char *name);
 void stat_cache_delete(const char *name);
-unsigned int fast_string_hash(TDB_DATA *key);
+#if BUILD_TDB2
+struct tdb_data;
+unsigned int fast_string_hash(struct tdb_data *key);
+#else
+struct TDB_DATA;
+unsigned int fast_string_hash(struct TDB_DATA *key);
+#endif
 bool reset_stat_cache( void );
 
 /* The following definitions come from smbd/statvfs.c  */
@@ -1046,15 +1113,17 @@ void reply_transs2(struct smb_request *req);
 /* The following definitions come from smbd/uid.c  */
 
 bool change_to_guest(void);
-void conn_clear_vuid_cache(connection_struct *conn, uint16_t vuid);
 bool change_to_user(connection_struct *conn, uint16 vuid);
 bool change_to_user_by_session(connection_struct *conn,
 			       const struct auth_serversupplied_info *session_info);
 bool change_to_root_user(void);
-bool become_authenticated_pipe_user(struct pipes_struct *p);
+bool smbd_change_to_root_user(void);
+bool become_authenticated_pipe_user(struct auth_serversupplied_info *session_info);
 bool unbecome_authenticated_pipe_user(void);
 void become_root(void);
 void unbecome_root(void);
+void smbd_become_root(void);
+void smbd_unbecome_root(void);
 bool become_user(connection_struct *conn, uint16 vuid);
 bool become_user_by_session(connection_struct *conn,
 			    const struct auth_serversupplied_info *session_info);
@@ -1130,3 +1199,5 @@ void *avahi_start_register(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
 /* The following definitions come from smbd/msg_idmap.c */
 
 void msg_idmap_register_msgs(struct messaging_context *ctx);
+
+#endif /* _SMBD_PROTO_H_ */
diff --git a/source3/smbd/quotas.c b/source3/smbd/quotas.c
index c9f85b6ad09..41516621f0f 100644
--- a/source3/smbd/quotas.c
+++ b/source3/smbd/quotas.c
@@ -26,6 +26,7 @@
 
 #include "includes.h"
 #include "smbd/smbd.h"
+#include "system/filesys.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_QUOTA
diff --git a/source3/smbd/reply.c b/source3/smbd/reply.c
index 01d4332b7e7..a804884ef7b 100644
--- a/source3/smbd/reply.c
+++ b/source3/smbd/reply.c
@@ -314,7 +314,7 @@ size_t srvstr_get_path_req_wcard(TALLOC_CTX *mem_ctx, struct smb_request *req,
 				 char **pp_dest, const char *src, int flags,
 				 NTSTATUS *err, bool *contains_wcard)
 {
-	return srvstr_get_path_wcard(mem_ctx, (char *)req->inbuf, req->flags2,
+	return srvstr_get_path_wcard(mem_ctx, (const char *)req->inbuf, req->flags2,
 				     pp_dest, src, smbreq_bufrem(req, src),
 				     flags, err, contains_wcard);
 }
@@ -491,6 +491,13 @@ static bool netbios_session_retarget(struct smbd_server_connection *sconn,
 	return ret;
 }
 
+static void reply_called_name_not_present(char *outbuf)
+{
+	smb_setlen(outbuf, 1);
+	SCVAL(outbuf, 0, 0x83);
+	SCVAL(outbuf, 4, 0x82);
+}
+
 /****************************************************************************
  Reply to a (netbios-level) special message. 
 ****************************************************************************/
@@ -531,11 +538,13 @@ void reply_special(struct smbd_server_connection *sconn, char *inbuf, size_t inb
 		name_len1 = name_len((unsigned char *)(inbuf+4),inbuf_size - 4);
 		if (name_len1 <= 0 || name_len1 > inbuf_size - 4) {
 			DEBUG(0,("Invalid name length in session request\n"));
+			reply_called_name_not_present(outbuf);
 			break;
 		}
 		name_len2 = name_len((unsigned char *)(inbuf+4+name_len1),inbuf_size - 4 - name_len1);
 		if (name_len2 <= 0 || name_len2 > inbuf_size - 4 - name_len1) {
 			DEBUG(0,("Invalid name length in session request\n"));
+			reply_called_name_not_present(outbuf);
 			break;
 		}
 
@@ -546,6 +555,7 @@ void reply_special(struct smbd_server_connection *sconn, char *inbuf, size_t inb
 
 		if (name_type1 == -1 || name_type2 == -1) {
 			DEBUG(0,("Invalid name type in session request\n"));
+			reply_called_name_not_present(outbuf);
 			break;
 		}
 
@@ -575,7 +585,7 @@ void reply_special(struct smbd_server_connection *sconn, char *inbuf, size_t inb
 		if (name_type2 == 'R') {
 			/* We are being asked for a pathworks session --- 
 			   no thanks! */
-			SCVAL(outbuf, 0,0x83);
+			reply_called_name_not_present(outbuf);
 			break;
 		}
 
@@ -614,6 +624,10 @@ void reply_special(struct smbd_server_connection *sconn, char *inbuf, size_t inb
 		    msg_type, msg_flags));
 
 	srv_send_smb(sconn, outbuf, false, 0, false, NULL);
+
+	if (CVAL(outbuf, 0) != 0x82) {
+		exit_server_cleanly("invalid netbios session");
+	}
 	return;
 }
 
@@ -873,7 +887,7 @@ void reply_tcon_and_X(struct smb_request *req)
 		 service));
 
 	/* set the incoming and outgoing tid to the just created one */
-	SSVAL(req->inbuf,smb_tid,conn->cnum);
+	SSVAL(discard_const_p(uint8_t, req->inbuf),smb_tid,conn->cnum);
 	SSVAL(req->outbuf,smb_tid,conn->cnum);
 
 	END_PROFILE(SMBtconX);
@@ -958,7 +972,7 @@ void reply_ioctl(struct smb_request *req)
 				SSVAL(p, 0, 0);
 			}
 			srvstr_push((char *)req->outbuf, req->flags2, p+2,
-				    global_myname(), 15,
+				    lp_netbios_name(), 15,
 				    STR_TERMINATE|STR_ASCII);
 			if (conn) {
 				srvstr_push((char *)req->outbuf, req->flags2,
@@ -1108,9 +1122,9 @@ void reply_getatr(struct smb_request *req)
 	/* dos smetimes asks for a stat of "" - it returns a "hidden directory"
 		under WfWg - weird! */
 	if (*fname == '\0') {
-		mode = aHIDDEN | aDIR;
+		mode = FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY;
 		if (!CAN_WRITE(conn)) {
-			mode |= aRONLY;
+			mode |= FILE_ATTRIBUTE_READONLY;
 		}
 		size = 0;
 		mtime = 0;
@@ -1156,7 +1170,7 @@ void reply_getatr(struct smb_request *req)
 		}
 
 		mtime = convert_timespec_to_time_t(smb_fname->st.st_ex_mtime);
-		if (mode & aDIR) {
+		if (mode & FILE_ATTRIBUTE_DIRECTORY) {
 			size = 0;
 		}
 	}
@@ -1257,9 +1271,9 @@ void reply_setatr(struct smb_request *req)
 
 	if (mode != FILE_ATTRIBUTE_NORMAL) {
 		if (VALID_STAT_OF_DIR(smb_fname->st))
-			mode |= aDIR;
+			mode |= FILE_ATTRIBUTE_DIRECTORY;
 		else
-			mode &= ~aDIR;
+			mode &= ~FILE_ATTRIBUTE_DIRECTORY;
 
 		if (file_set_dosmode(conn, smb_fname, mode, NULL,
 				     false) != 0) {
@@ -1428,7 +1442,7 @@ void reply_search(struct smb_request *req)
 	status_len = SVAL(p, 0);
 	p += 2;
 
-	/* dirtype &= ~aDIR; */
+	/* dirtype &= ~FILE_ATTRIBUTE_DIRECTORY; */
 
 	if (status_len == 0) {
 		nt_status = filename_convert(ctx, conn,
@@ -1520,11 +1534,11 @@ void reply_search(struct smb_request *req)
 	/* Initialize per SMBsearch/SMBffirst/SMBfunique operation data */
 	dptr_init_search_op(dirptr);
 
-	if ((dirtype&0x1F) == aVOLID) {
+	if ((dirtype&0x1F) == FILE_ATTRIBUTE_VOLUME) {
 		char buf[DIR_STRUCT_SIZE];
 		memcpy(buf,status,21);
 		if (!make_dir_struct(ctx,buf,"???????????",volume_label(SNUM(conn)),
-				0,aVOLID,0,!allow_long_path_components)) {
+				0,FILE_ATTRIBUTE_VOLUME,0,!allow_long_path_components)) {
 			reply_nterror(req, NT_STATUS_NO_MEMORY);
 			goto out;
 		}
@@ -1820,7 +1834,7 @@ void reply_open(struct smb_request *req)
 
 	mtime = convert_timespec_to_time_t(smb_fname->st.st_ex_mtime);
 
-	if (fattr & aDIR) {
+	if (fattr & FILE_ATTRIBUTE_DIRECTORY) {
 		DEBUG(3,("attempt to open a directory %s\n",
 			 fsp_str_dbg(fsp)));
 		close_file(req, fsp, ERROR_CLOSE);
@@ -2004,7 +2018,7 @@ void reply_open_and_X(struct smb_request *req)
 
 	fattr = dos_mode(conn, fsp->fsp_name);
 	mtime = convert_timespec_to_time_t(fsp->fsp_name->st.st_ex_mtime);
-	if (fattr & aDIR) {
+	if (fattr & FILE_ATTRIBUTE_DIRECTORY) {
 		close_file(req, fsp, ERROR_CLOSE);
 		reply_nterror(req, NT_STATUS_ACCESS_DENIED);
 		goto out;
@@ -2160,7 +2174,7 @@ void reply_mknew(struct smb_request *req)
 		goto out;
 	}
 
-	if (fattr & aVOLID) {
+	if (fattr & FILE_ATTRIBUTE_VOLUME) {
 		DEBUG(0,("Attempt to create file (%s) with volid set - "
 			 "please report this\n",
 			 smb_fname_str_dbg(smb_fname)));
@@ -2381,15 +2395,18 @@ void reply_ctemp(struct smb_request *req)
 static NTSTATUS can_rename(connection_struct *conn, files_struct *fsp,
 			uint16 dirtype)
 {
-	uint32 fmode;
-
 	if (!CAN_WRITE(conn)) {
 		return NT_STATUS_MEDIA_WRITE_PROTECTED;
 	}
 
-	fmode = dos_mode(conn, fsp->fsp_name);
-	if ((fmode & ~dirtype) & (aHIDDEN | aSYSTEM)) {
-		return NT_STATUS_NO_SUCH_FILE;
+	if ((dirtype & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) !=
+			(FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) {
+		/* Only bother to read the DOS attribute if we might deny the
+		   rename on the grounds of attribute missmatch. */
+		uint32_t fmode = dos_mode(conn, fsp->fsp_name);
+		if ((fmode & ~dirtype) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) {
+			return NT_STATUS_NO_SUCH_FILE;
+		}
 	}
 
 	if (S_ISDIR(fsp->fsp_name->st.st_ex_mode)) {
@@ -2449,16 +2466,16 @@ static NTSTATUS do_unlink(connection_struct *conn,
 	fattr = dos_mode(conn, smb_fname);
 
 	if (dirtype & FILE_ATTRIBUTE_NORMAL) {
-		dirtype = aDIR|aARCH|aRONLY;
+		dirtype = FILE_ATTRIBUTE_DIRECTORY|FILE_ATTRIBUTE_ARCHIVE|FILE_ATTRIBUTE_READONLY;
 	}
 
-	dirtype &= (aDIR|aARCH|aRONLY|aHIDDEN|aSYSTEM);
+	dirtype &= (FILE_ATTRIBUTE_DIRECTORY|FILE_ATTRIBUTE_ARCHIVE|FILE_ATTRIBUTE_READONLY|FILE_ATTRIBUTE_HIDDEN|FILE_ATTRIBUTE_SYSTEM);
 	if (!dirtype) {
 		return NT_STATUS_NO_SUCH_FILE;
 	}
 
 	if (!dir_check_ftype(conn, fattr, dirtype)) {
-		if (fattr & aDIR) {
+		if (fattr & FILE_ATTRIBUTE_DIRECTORY) {
 			return NT_STATUS_FILE_IS_A_DIRECTORY;
 		}
 		return NT_STATUS_NO_SUCH_FILE;
@@ -2489,13 +2506,13 @@ static NTSTATUS do_unlink(connection_struct *conn,
 	}
 
 	/* Can't delete a directory. */
-	if (fattr & aDIR) {
+	if (fattr & FILE_ATTRIBUTE_DIRECTORY) {
 		return NT_STATUS_FILE_IS_A_DIRECTORY;
 	}
 #endif
 
 #if 0 /* JRATEST */
-	else if (dirtype & aDIR) /* Asked for a directory and it isn't. */
+	else if (dirtype & FILE_ATTRIBUTE_DIRECTORY) /* Asked for a directory and it isn't. */
 		return NT_STATUS_OBJECT_NAME_INVALID;
 #endif /* JRATEST */
 
@@ -2632,7 +2649,7 @@ NTSTATUS unlink_internals(connection_struct *conn, struct smb_request *req,
 		const char *dname = NULL;
 		char *talloced = NULL;
 
-		if ((dirtype & SAMBA_ATTRIBUTES_MASK) == aDIR) {
+		if ((dirtype & SAMBA_ATTRIBUTES_MASK) == FILE_ATTRIBUTE_DIRECTORY) {
 			status = NT_STATUS_OBJECT_NAME_INVALID;
 			goto out;
 		}
@@ -3077,9 +3094,9 @@ static void send_file_readbraw(connection_struct *conn,
 
 normal_readbraw:
 
-	outbuf = TALLOC_ARRAY(NULL, char, nread+4);
+	outbuf = talloc_array(NULL, char, nread+4);
 	if (!outbuf) {
-		DEBUG(0,("send_file_readbraw: TALLOC_ARRAY failed for size %u.\n",
+		DEBUG(0,("send_file_readbraw: talloc_array failed for size %u.\n",
 			(unsigned)(nread+4)));
 		reply_readbraw_error(sconn);
 		return;
@@ -3213,7 +3230,7 @@ void reply_readbraw(struct smb_request *req)
 				"(%x << 32) used and we don't support "
 				"64 bit offsets.\n",
 			(unsigned int)IVAL(req->vwv+8, 0) ));
-			reply_readbraw_error();
+			reply_readbraw_error(sconn);
 			END_PROFILE(SMBreadbraw);
 			return;
 		}
@@ -3896,7 +3913,7 @@ void reply_writebraw(struct smb_request *req)
 	size_t numtowrite=0;
 	size_t tcount;
 	SMB_OFF_T startpos;
-	char *data=NULL;
+	const char *data=NULL;
 	bool write_through;
 	files_struct *fsp;
 	struct lock_struct lock;
@@ -3909,7 +3926,7 @@ void reply_writebraw(struct smb_request *req)
 	 * type of SMBwritec, not SMBwriteBraw, as this tells the client
 	 * we're finished.
 	 */
-	SCVAL(req->inbuf,smb_com,SMBwritec);
+	SCVAL(discard_const_p(uint8_t, req->inbuf),smb_com,SMBwritec);
 
 	if (srv_is_signing_active(req->sconn)) {
 		END_PROFILE(SMBwritebraw);
@@ -3955,8 +3972,8 @@ void reply_writebraw(struct smb_request *req)
 		on whether we are using the core+ or lanman1.0 protocol */
 
 	if(get_Protocol() <= PROTOCOL_COREPLUS) {
-		numtowrite = SVAL(smb_buf(req->inbuf),-2);
-		data = smb_buf(req->inbuf);
+		numtowrite = SVAL(smb_buf_const(req->inbuf),-2);
+		data = smb_buf_const(req->inbuf);
 	} else {
 		numtowrite = SVAL(req->vwv+10, 0);
 		data = smb_base(req->inbuf) + SVAL(req->vwv+11, 0);
@@ -4001,7 +4018,7 @@ void reply_writebraw(struct smb_request *req)
 	total_written = nwritten;
 
 	/* Allocate a buffer of 64k + length. */
-	buf = TALLOC_ARRAY(NULL, char, 65540);
+	buf = talloc_array(NULL, char, 65540);
 	if (!buf) {
 		reply_nterror(req, NT_STATUS_NO_MEMORY);
 		error_to_writebrawerr(req);
@@ -4474,7 +4491,7 @@ void reply_write_and_X(struct smb_request *req)
 	ssize_t nwritten;
 	unsigned int smb_doff;
 	unsigned int smblen;
-	char *data;
+	const char *data;
 	NTSTATUS status;
 	int saved_errno = 0;
 
@@ -5815,7 +5832,7 @@ static void rename_open_files(connection_struct *conn,
 	files_struct *fsp;
 	bool did_rename = False;
 	NTSTATUS status;
-	uint32_t new_name_hash;
+	uint32_t new_name_hash = 0;
 
 	for(fsp = file_find_di_first(conn->sconn, lck->id); fsp;
 	    fsp = file_find_di_next(fsp)) {
@@ -6405,7 +6422,7 @@ NTSTATUS rename_internals(TALLOC_CTX *ctx,
 
 		/* Quick check for "." and ".." */
 		if (ISDOT(dname) || ISDOTDOT(dname)) {
-			if (attrs & aDIR) {
+			if (attrs & FILE_ATTRIBUTE_DIRECTORY) {
 				sysdir_entry = True;
 			} else {
 				TALLOC_FREE(talloced);
@@ -7971,7 +7988,7 @@ void reply_getattrE(struct smb_request *req)
 	srv_put_dos_date2((char *)req->outbuf, smb_vwv4,
 			  convert_timespec_to_time_t(fsp->fsp_name->st.st_ex_mtime));
 
-	if (mode & aDIR) {
+	if (mode & FILE_ATTRIBUTE_DIRECTORY) {
 		SIVAL(req->outbuf, smb_vwv6, 0);
 		SIVAL(req->outbuf, smb_vwv8, 0);
 	} else {
diff --git a/source3/smbd/seal.c b/source3/smbd/seal.c
index b3dd84264a8..26185062a6f 100644
--- a/source3/smbd/seal.c
+++ b/source3/smbd/seal.c
@@ -26,6 +26,7 @@
 #include "smb_crypt.h"
 #include "../lib/util/asn1.h"
 #include "auth.h"
+#include "libsmb/libsmb.h"
 
 /******************************************************************************
  Server side encryption.
@@ -133,7 +134,7 @@ static NTSTATUS get_srv_gss_creds(const char *service,
 	NTSTATUS status = NT_STATUS_OK;
 
 	gss_OID_desc nt_hostbased_service =
-	{10, CONST_DISCARD(char *,"\x2a\x86\x48\x86\xf7\x12\x01\x02\x01\x04")};
+	{10, discard_const_p(char, "\x2a\x86\x48\x86\xf7\x12\x01\x02\x01\x04")};
 
 	if (asprintf(&host_princ_s, "%s@%s", service, name) == -1) {
 		return NT_STATUS_NO_MEMORY;
@@ -195,7 +196,7 @@ static NTSTATUS make_auth_gss(struct smb_srv_trans_enc_ctx *ec)
 	gss_cred_id_t srv_cred;
 	fstring fqdn;
 
-	name_to_fqdn(fqdn, global_myname());
+	name_to_fqdn(fqdn, lp_netbios_name());
 	strlower_m(fqdn);
 
 	status = get_srv_gss_creds("cifs", fqdn, GSS_C_ACCEPT, &srv_cred);
diff --git a/source3/smbd/server.c b/source3/smbd/server.c
index d3c6c1884f0..5aa3ddbdc9a 100644
--- a/source3/smbd/server.c
+++ b/source3/smbd/server.c
@@ -26,7 +26,6 @@
 #include "popt_common.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
-#include "registry.h"
 #include "registry/reg_init_full.h"
 #include "libcli/auth/schannel.h"
 #include "secrets.h"
@@ -52,40 +51,6 @@ extern void start_spoolssd(struct event_context *ev_ctx,
 extern int dcelogin_atmost_once;
 #endif /* WITH_DFS */
 
-static void smbd_set_server_fd(int fd)
-{
-	struct smbd_server_connection *sconn = smbd_server_conn;
-	char addr[INET6_ADDRSTRLEN];
-	const char *name;
-
-	sconn->sock = fd;
-
-	/*
-	 * Initialize sconn->client_id: If we can't find the client's
-	 * name, default to its address.
-	 */
-
-	client_addr(fd, sconn->client_id.addr, sizeof(sconn->client_id.addr));
-
-	name = client_name(sconn->sock);
-	if (strcmp(name, "UNKNOWN") != 0) {
-		name = talloc_strdup(sconn, name);
-	} else {
-		name = NULL;
-	}
-	sconn->client_id.name =
-		(name != NULL) ? name : sconn->client_id.addr;
-
-	sub_set_socket_ids(sconn->client_id.addr, sconn->client_id.name,
-			   client_socket_addr(sconn->sock, addr,
-					      sizeof(addr)));
-}
-
-struct event_context *smbd_event_context(void)
-{
-	return server_event_context();
-}
-
 /*******************************************************************
  What to do when smb.conf is updated.
  ********************************************************************/
@@ -288,7 +253,7 @@ static void remove_child_pid(pid_t pid, bool unclean_shutdown)
 		if (!cleanup_te) {
 			/* call the cleanup timer, but not too often */
 			int cleanup_time = lp_parm_int(-1, "smbd", "cleanuptime", 20);
-			cleanup_te = event_add_timed(smbd_event_context(), NULL,
+			cleanup_te = event_add_timed(server_event_context(), NULL,
 						timeval_current_ofs(cleanup_time, 0),
 						cleanup_timeout_fn,
 						&cleanup_te);
@@ -366,8 +331,8 @@ static void smbd_setup_sig_chld_handler(void)
 {
 	struct tevent_signal *se;
 
-	se = tevent_add_signal(smbd_event_context(),
-			       smbd_event_context(),
+	se = tevent_add_signal(server_event_context(),
+			       server_event_context(),
 			       SIGCHLD, 0,
 			       smbd_sig_chld_handler,
 			       NULL);
@@ -469,7 +434,7 @@ static void smbd_accept_connection(struct tevent_context *ev,
 		s = NULL;
 
 		status = reinit_after_fork(smbd_messaging_context(),
-					   smbd_event_context(), procid_self(),
+					   server_event_context(), procid_self(),
 					   true);
 		if (!NT_STATUS_IS_OK(status)) {
 			if (NT_STATUS_EQUAL(status,
@@ -582,7 +547,7 @@ static bool smbd_open_one_socket(struct smbd_parent_context *parent,
 		return false;
 	}
 
-	s->fde = tevent_add_fd(smbd_event_context(),
+	s->fde = tevent_add_fd(server_event_context(),
 			       s,
 			       s->fd, TEVENT_FD_READ,
 			       smbd_accept_connection,
@@ -632,7 +597,7 @@ static bool open_sockets_smbd(struct smbd_parent_context *parent,
 {
 	int num_interfaces = iface_count();
 	int i;
-	char *ports;
+	const char *ports;
 	unsigned dns_port = 0;
 
 #ifdef HAVE_ATEXIT
@@ -763,7 +728,7 @@ static bool open_sockets_smbd(struct smbd_parent_context *parent,
 		return false;
 	}
 
-	if (!(event_add_idle(smbd_event_context(), NULL,
+	if (!(event_add_idle(server_event_context(), NULL,
 			     timeval_set(SMBD_HOUSEKEEPING_INTERVAL, 0),
 			     "parent_housekeeping", smbd_parent_housekeeping,
 			     NULL))) {
@@ -801,14 +766,14 @@ static bool open_sockets_smbd(struct smbd_parent_context *parent,
 
 	if (lp_multicast_dns_register() && (dns_port != 0)) {
 #ifdef WITH_DNSSD_SUPPORT
-		smbd_setup_mdns_registration(smbd_event_context(),
+		smbd_setup_mdns_registration(server_event_context(),
 					     parent, dns_port);
 #endif
 #ifdef WITH_AVAHI_SUPPORT
 		void *avahi_conn;
 
 		avahi_conn = avahi_start_register(
-			smbd_event_context(), smbd_event_context(), dns_port);
+			server_event_context(), server_event_context(), dns_port);
 		if (avahi_conn == NULL) {
 			DEBUG(10, ("avahi_start_register failed\n"));
 		}
@@ -827,7 +792,7 @@ static void smbd_parent_loop(struct smbd_parent_context *parent)
 		int ret;
 		TALLOC_CTX *frame = talloc_stackframe();
 
-		ret = tevent_loop_once(smbd_event_context());
+		ret = tevent_loop_once(server_event_context());
 		if (ret != 0) {
 			exit_server_cleanly("tevent_loop_once() error");
 		}
@@ -917,7 +882,7 @@ extern void build_options(bool screen);
 	load_case_tables();
 
 	/* Initialize the event context, it will panic on error */
-	smbd_event_context();
+	server_event_context();
 
 	smbd_init_globals();
 
@@ -1063,7 +1028,7 @@ extern void build_options(bool screen);
 	 * Reloading of the printers will not work here as we don't have a
 	 * server info and rpc services set up. It will be called later.
 	 */
-	if (!reload_services(smbd_messaging_context(), -1, False)) {
+	if (!reload_services(NULL, -1, False)) {
 		exit(1);
 	}
 
@@ -1124,7 +1089,7 @@ extern void build_options(bool screen);
 		pidfile_create("smbd");
 
 	status = reinit_after_fork(smbd_messaging_context(),
-				   smbd_event_context(),
+				   server_event_context(),
 				   procid_self(), false);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0,("reinit_after_fork() failed\n"));
@@ -1134,7 +1099,7 @@ extern void build_options(bool screen);
 	smbd_server_conn->msg_ctx = smbd_messaging_context();
 
 	smbd_setup_sig_term_handler();
-	smbd_setup_sig_hup_handler(smbd_event_context(),
+	smbd_setup_sig_hup_handler(server_event_context(),
 				   smbd_server_conn->msg_ctx);
 
 	/* Setup all the TDB's - including CLEAR_IF_FIRST tdb's. */
@@ -1148,7 +1113,7 @@ extern void build_options(bool screen);
 	/* Initialise the password backed before the global_sam_sid
 	   to ensure that we fetch from ldap before we make a domain sid up */
 
-	if(!initialize_password_db(False, smbd_event_context()))
+	if(!initialize_password_db(False, server_event_context()))
 		exit(1);
 
 	if (!secrets_init()) {
@@ -1178,15 +1143,15 @@ extern void build_options(bool screen);
 	if (!locking_init())
 		exit(1);
 
-	if (!messaging_tdb_parent_init(smbd_event_context())) {
+	if (!messaging_tdb_parent_init(server_event_context())) {
 		exit(1);
 	}
 
-	if (!notify_internal_parent_init(smbd_event_context())) {
+	if (!notify_internal_parent_init(server_event_context())) {
 		exit(1);
 	}
 
-	if (!serverid_parent_init(smbd_event_context())) {
+	if (!serverid_parent_init(server_event_context())) {
 		exit(1);
 	}
 
@@ -1232,13 +1197,13 @@ extern void build_options(bool screen);
 		rpcsrv_type = lp_parm_const_string(GLOBAL_SECTION_SNUM,
 						   "rpc_server", "epmapper",
 						   "none");
-		if (StrCaseCmp(rpcsrv_type, "daemon") == 0) {
-			start_epmd(smbd_event_context(),
+		if (strcasecmp_m(rpcsrv_type, "daemon") == 0) {
+			start_epmd(server_event_context(),
 				   smbd_server_conn->msg_ctx);
 		}
 	}
 
-	if (!dcesrv_ep_setup(smbd_event_context(), smbd_server_conn->msg_ctx)) {
+	if (!dcesrv_ep_setup(server_event_context(), smbd_server_conn->msg_ctx)) {
 		exit(1);
 	}
 
@@ -1253,7 +1218,7 @@ extern void build_options(bool screen);
 
 	if (is_daemon && !interactive
 	    && lp_parm_bool(-1, "smbd", "backgroundqueue", true)) {
-		start_background_queue(smbd_event_context(),
+		start_background_queue(server_event_context(),
 				       smbd_messaging_context());
 	}
 
@@ -1265,8 +1230,8 @@ extern void build_options(bool screen);
 		rpcsrv_type = lp_parm_const_string(GLOBAL_SECTION_SNUM,
 						   "rpc_server", "spoolss",
 						   "embedded");
-		if (StrCaseCmp(rpcsrv_type, "daemon") == 0) {
-			start_spoolssd(smbd_event_context(),
+		if (strcasecmp_m(rpcsrv_type, "daemon") == 0) {
+			start_spoolssd(server_event_context(),
 				       smbd_messaging_context());
 		}
 	}
@@ -1296,7 +1261,7 @@ extern void build_options(bool screen);
 		return(0);
 	}
 
-	parent = talloc_zero(smbd_event_context(), struct smbd_parent_context);
+	parent = talloc_zero(server_event_context(), struct smbd_parent_context);
 	if (!parent) {
 		exit_server("talloc(struct smbd_parent_context) failed");
 	}
diff --git a/source3/smbd/server_exit.c b/source3/smbd/server_exit.c
index 554affc9849..c7a318d891f 100644
--- a/source3/smbd/server_exit.c
+++ b/source3/smbd/server_exit.c
@@ -25,7 +25,7 @@
 #include "includes.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
-
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_dfs.h"
 #include "../librpc/gen_ndr/srv_dssetup.h"
 #include "../librpc/gen_ndr/srv_echo.h"
diff --git a/source3/smbd/server_reload.c b/source3/smbd/server_reload.c
index 82b0cb03e95..fef66dac6a5 100644
--- a/source3/smbd/server_reload.c
+++ b/source3/smbd/server_reload.c
@@ -113,7 +113,11 @@ bool reload_services(struct messaging_context *msg_ctx, int smb_sock,
 	if (test && !lp_file_list_changed())
 		return(True);
 
-	lp_killunused(conn_snum_used);
+	if (msg_ctx) {
+		lp_killunused(msg_ctx_to_sconn(msg_ctx), conn_snum_used);
+	} else {
+		lp_killunused(NULL, NULL);
+	}
 
 	ret = lp_load(get_dyn_CONFIGFILE(), False, False, True, True);
 
diff --git a/source3/smbd/service.c b/source3/smbd/service.c
index b29713d21d2..0857645d1c6 100644
--- a/source3/smbd/service.c
+++ b/source3/smbd/service.c
@@ -19,6 +19,7 @@
 
 #include "includes.h"
 #include "system/filesys.h"
+#include "../lib/tsocket/tsocket.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../librpc/gen_ndr/netlogon.h"
@@ -221,232 +222,6 @@ bool set_current_service(connection_struct *conn, uint16 flags, bool do_chdir)
 	return(True);
 }
 
-static int load_registry_service(const char *servicename)
-{
-	if (!lp_registry_shares()) {
-		return -1;
-	}
-
-	if ((servicename == NULL) || (*servicename == '\0')) {
-		return -1;
-	}
-
-	if (strequal(servicename, GLOBAL_NAME)) {
-		return -2;
-	}
-
-	if (!process_registry_service(servicename)) {
-		return -1;
-	}
-
-	return lp_servicenumber(servicename);
-}
-
-void load_registry_shares(void)
-{
-	DEBUG(8, ("load_registry_shares()\n"));
-	if (!lp_registry_shares()) {
-		return;
-	}
-
-	process_registry_shares();
-
-	return;
-}
-
-/****************************************************************************
- Add a home service. Returns the new service number or -1 if fail.
-****************************************************************************/
-
-int add_home_service(const char *service, const char *username, const char *homedir)
-{
-	int iHomeService;
-
-	if (!service || !homedir || homedir[0] == '\0')
-		return -1;
-
-	if ((iHomeService = lp_servicenumber(HOMES_NAME)) < 0) {
-		if ((iHomeService = load_registry_service(HOMES_NAME)) < 0) {
-			return -1;
-		}
-	}
-
-	/*
-	 * If this is a winbindd provided username, remove
-	 * the domain component before adding the service.
-	 * Log a warning if the "path=" parameter does not
-	 * include any macros.
-	 */
-
-	{
-		const char *p = strchr(service,*lp_winbind_separator());
-
-		/* We only want the 'user' part of the string */
-		if (p) {
-			service = p + 1;
-		}
-	}
-
-	if (!lp_add_home(service, iHomeService, username, homedir)) {
-		return -1;
-	}
-
-	return lp_servicenumber(service);
-
-}
-
-/**
- * Find a service entry.
- *
- * @param service is modified (to canonical form??)
- **/
-
-int find_service(TALLOC_CTX *ctx, const char *service_in, char **p_service_out)
-{
-	int iService;
-
-	if (!service_in) {
-		return -1;
-	}
-
-	/* First make a copy. */
-	*p_service_out = talloc_strdup(ctx, service_in);
-	if (!*p_service_out) {
-		return -1;
-	}
-
-	all_string_sub(*p_service_out,"\\","/",0);
-
-	iService = lp_servicenumber(*p_service_out);
-
-	/* now handle the special case of a home directory */
-	if (iService < 0) {
-		char *phome_dir = get_user_home_dir(ctx, *p_service_out);
-
-		if(!phome_dir) {
-			/*
-			 * Try mapping the servicename, it may
-			 * be a Windows to unix mapped user name.
-			 */
-			if(map_username(ctx, *p_service_out, p_service_out)) {
-				if (*p_service_out == NULL) {
-					/* Out of memory. */
-					return -1;
-				}
-				phome_dir = get_user_home_dir(
-						ctx, *p_service_out);
-			}
-		}
-
-		DEBUG(3,("checking for home directory %s gave %s\n",*p_service_out,
-			phome_dir?phome_dir:"(NULL)"));
-
-		iService = add_home_service(*p_service_out,*p_service_out /* 'username' */, phome_dir);
-	}
-
-	/* If we still don't have a service, attempt to add it as a printer. */
-	if (iService < 0) {
-		int iPrinterService;
-
-		if ((iPrinterService = lp_servicenumber(PRINTERS_NAME)) < 0) {
-			iPrinterService = load_registry_service(PRINTERS_NAME);
-		}
-		if (iPrinterService >= 0) {
-			DEBUG(3,("checking whether %s is a valid printer name...\n",
-				*p_service_out));
-			if (pcap_printername_ok(*p_service_out)) {
-				DEBUG(3,("%s is a valid printer name\n",
-					*p_service_out));
-				DEBUG(3,("adding %s as a printer service\n",
-					*p_service_out));
-				lp_add_printer(*p_service_out, iPrinterService);
-				iService = lp_servicenumber(*p_service_out);
-				if (iService < 0) {
-					DEBUG(0,("failed to add %s as a printer service!\n",
-						*p_service_out));
-				}
-			} else {
-				DEBUG(3,("%s is not a valid printer name\n",
-					*p_service_out));
-			}
-		}
-	}
-
-	/* Check for default vfs service?  Unsure whether to implement this */
-	if (iService < 0) {
-	}
-
-	if (iService < 0) {
-		iService = load_registry_service(*p_service_out);
-	}
-
-	/* Is it a usershare service ? */
-	if (iService < 0 && *lp_usershare_path()) {
-		/* Ensure the name is canonicalized. */
-		strlower_m(*p_service_out);
-		iService = load_usershare_service(*p_service_out);
-	}
-
-	/* just possibly it's a default service? */
-	if (iService < 0) {
-		char *pdefservice = lp_defaultservice();
-		if (pdefservice &&
-				*pdefservice &&
-				!strequal(pdefservice, *p_service_out)
-				&& !strstr_m(*p_service_out,"..")) {
-			/*
-			 * We need to do a local copy here as lp_defaultservice() 
-			 * returns one of the rotating lp_string buffers that
-			 * could get overwritten by the recursive find_service() call
-			 * below. Fix from Josef Hinteregger <joehtg@joehtg.co.at>.
-			 */
-			char *defservice = talloc_strdup(ctx, pdefservice);
-
-			if (!defservice) {
-				goto fail;
-			}
-
-			/* Disallow anything except explicit share names. */
-			if (strequal(defservice,HOMES_NAME) ||
-					strequal(defservice, PRINTERS_NAME) ||
-					strequal(defservice, "IPC$")) {
-				TALLOC_FREE(defservice);
-				goto fail;
-			}
-
-			iService = find_service(ctx, defservice, p_service_out);
-			if (!*p_service_out) {
-				TALLOC_FREE(defservice);
-				iService = -1;
-				goto fail;
-			}
-			if (iService >= 0) {
-				all_string_sub(*p_service_out, "_","/",0);
-				iService = lp_add_service(*p_service_out, iService);
-			}
-			TALLOC_FREE(defservice);
-		}
-	}
-
-	if (iService >= 0) {
-		if (!VALID_SNUM(iService)) {
-			DEBUG(0,("Invalid snum %d for %s\n",iService,
-				*p_service_out));
-			iService = -1;
-		}
-	}
-
-  fail:
-
-	if (iService < 0) {
-		DEBUG(3,("find_service() failed to find service %s\n",
-			*p_service_out));
-	}
-
-	return (iService);
-}
-
-
 /****************************************************************************
  do some basic sainity checks on the share.  
  This function modifies dev, ecode.
@@ -930,7 +705,7 @@ connection_struct *make_connection_snum(struct smbd_server_connection *sconn,
 		conn->notify_ctx = notify_init(conn,
 					       sconn_server_id(sconn),
 					       sconn->msg_ctx,
-					       smbd_event_context(),
+					       server_event_context(),
 					       conn);
 	}
 
diff --git a/source3/smbd/session.c b/source3/smbd/session.c
index f3a678b2055..48afb5389f9 100644
--- a/source3/smbd/session.c
+++ b/source3/smbd/session.c
@@ -231,7 +231,7 @@ static int gather_sessioninfo(const char *key, struct sessionid *session,
 {
 	struct session_list *sesslist = (struct session_list *)private_data;
 
-	sesslist->sessions = TALLOC_REALLOC_ARRAY(
+	sesslist->sessions = talloc_realloc(
 		sesslist->mem_ctx, sesslist->sessions, struct sessionid,
 		sesslist->count+1);
 
@@ -264,7 +264,7 @@ int list_sessions(TALLOC_CTX *mem_ctx, struct sessionid **session_list)
 	sesslist.sessions = NULL;
 
 	ret = sessionid_traverse_read(gather_sessioninfo, (void *) &sesslist);
-	if (ret == -1) {
+	if (ret < 0) {
 		DEBUG(3, ("Session traverse failed\n"));
 		SAFE_FREE(sesslist.sessions);
 		*session_list = NULL;
diff --git a/source3/smbd/sesssetup.c b/source3/smbd/sesssetup.c
index 3cf6758e296..dda4ef296e4 100644
--- a/source3/smbd/sesssetup.c
+++ b/source3/smbd/sesssetup.c
@@ -23,6 +23,7 @@
 */
 
 #include "includes.h"
+#include "../lib/tsocket/tsocket.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/auth/spnego.h"
@@ -193,7 +194,7 @@ static bool make_krb5_skew_error(DATA_BLOB *pblob_out)
 		return False;
 	}
 	/* Create server principal. */
-	asprintf(&host_princ_s, "%s$@%s", global_myname(), lp_realm());
+	asprintf(&host_princ_s, "%s$@%s", lp_netbios_name(), lp_realm());
 	if (!host_princ_s) {
 		goto out;
 	}
@@ -929,13 +930,28 @@ static NTSTATUS check_spnego_blob_complete(struct smbd_server_connection *sconn,
 	}
 
 	asn1_load(data, *pblob);
-	asn1_start_tag(data, pblob->data[0]);
-	if (data->has_error || data->nesting == NULL) {
+	if (asn1_start_tag(data, pblob->data[0])) {
+		/* asn1_start_tag checks if the given
+		   length of the blob is enough to complete
+		   the tag. If it returns true we know
+		   there is nothing to do - the blob is
+		   complete. */
 		asn1_free(data);
-		/* Let caller catch. */
 		return NT_STATUS_OK;
 	}
 
+	if (data->nesting == NULL) {
+		/* Incorrect tag, allocation failed,
+		   or reading the tag length failed.
+		   Let the caller catch. */
+		asn1_free(data);
+		return NT_STATUS_OK;
+	}
+
+	/* Here we know asn1_start_tag() has set data->has_error to true.
+	   asn1_tag_remaining() will have failed due to the given blob
+	   being too short. We need to work out how short. */
+
 	/* Integer wrap paranoia.... */
 
 	if (data->nesting->taglen + data->nesting->start < data->nesting->taglen ||
@@ -964,6 +980,13 @@ static NTSTATUS check_spnego_blob_complete(struct smbd_server_connection *sconn,
 
 	if (needed_len <= pblob->length) {
 		/* Nothing to do - blob is complete. */
+		/* THIS SHOULD NOT HAPPEN - asn1_start_tag()
+		   above should have caught this !!! */
+		DEBUG(0,("check_spnego_blob_complete: logic "
+			"error (needed_len = %u, "
+			"pblob->length = %u).\n",
+			(unsigned int)needed_len,
+			(unsigned int)pblob->length ));
 		return NT_STATUS_OK;
 	}
 
@@ -1042,7 +1065,7 @@ static void reply_sesssetup_and_X_spnego(struct smb_request *req)
 	file_save("negotiate.dat", blob1.data, blob1.length);
 #endif
 
-	p2 = (char *)req->buf + blob1.length;
+	p2 = (const char *)req->buf + blob1.length;
 
 	p2 += srvstr_pull_req_talloc(talloc_tos(), req, &tmp, p2,
 				     STR_TERMINATE);
@@ -1200,7 +1223,7 @@ static int shutdown_other_smbds(const struct connections_key *key,
 	struct shutdown_state *state = (struct shutdown_state *)private_data;
 
 	DEBUG(10, ("shutdown_other_smbds: %s, %s\n",
-		   procid_str(talloc_tos(), &crec->pid), crec->addr));
+		   server_id_str(talloc_tos(), &crec->pid), crec->addr));
 
 	if (!process_exists(crec->pid)) {
 		DEBUG(10, ("process does not exist\n"));
@@ -1684,7 +1707,7 @@ void reply_sesssetup_and_X(struct smb_request *req)
 	data_blob_free(&lm_resp);
 
 	SSVAL(req->outbuf,smb_uid,sess_vuid);
-	SSVAL(req->inbuf,smb_uid,sess_vuid);
+	SSVAL(discard_const_p(char, req->inbuf),smb_uid,sess_vuid);
 	req->vuid = sess_vuid;
 
 	if (!sconn->smb1.sessions.done_sesssetup) {
diff --git a/source3/smbd/signing.c b/source3/smbd/signing.c
index bea24098839..25b3c40d7d2 100644
--- a/source3/smbd/signing.c
+++ b/source3/smbd/signing.c
@@ -176,7 +176,7 @@ bool srv_init_signing(struct smbd_server_connection *conn)
 		struct smbd_shm_signing *s;
 
 		/* setup the signing state in shared memory */
-		s = talloc_zero(smbd_event_context(), struct smbd_shm_signing);
+		s = talloc_zero(server_event_context(), struct smbd_shm_signing);
 		if (s == NULL) {
 			return false;
 		}
@@ -198,7 +198,7 @@ bool srv_init_signing(struct smbd_server_connection *conn)
 		return true;
 	}
 
-	conn->smb1.signing_state = smb_signing_init(smbd_event_context(),
+	conn->smb1.signing_state = smb_signing_init(server_event_context(),
 						    allowed, mandatory);
 	if (!conn->smb1.signing_state) {
 		return false;
diff --git a/source3/smbd/smb2_break.c b/source3/smbd/smb2_break.c
index 925384c3bfe..5d5ab4139d9 100644
--- a/source3/smbd/smb2_break.c
+++ b/source3/smbd/smb2_break.c
@@ -23,6 +23,7 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 static struct tevent_req *smbd_smb2_oplock_break_send(TALLOC_CTX *mem_ctx,
 						      struct tevent_context *ev,
diff --git a/source3/smbd/smb2_create.c b/source3/smbd/smb2_create.c
index 6e292468f9e..73b8ade5bd3 100644
--- a/source3/smbd/smb2_create.c
+++ b/source3/smbd/smb2_create.c
@@ -25,6 +25,7 @@
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
 #include "../librpc/gen_ndr/ndr_security.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 int map_smb2_oplock_levels_to_samba(uint8_t in_oplock_level)
 {
diff --git a/source3/smbd/smb2_find.c b/source3/smbd/smb2_find.c
index 3ac573c2137..9fc8f1fef26 100644
--- a/source3/smbd/smb2_find.c
+++ b/source3/smbd/smb2_find.c
@@ -23,6 +23,7 @@
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
 #include "trans2.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 static struct tevent_req *smbd_smb2_find_send(TALLOC_CTX *mem_ctx,
 					      struct tevent_context *ev,
@@ -228,7 +229,7 @@ static struct tevent_req *smbd_smb2_find_send(TALLOC_CTX *mem_ctx,
 	int last_entry_off = 0;
 	int off = 0;
 	uint32_t num = 0;
-	uint32_t dirtype = aHIDDEN | aSYSTEM | aDIR;
+	uint32_t dirtype = FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_DIRECTORY;
 	bool dont_descend = false;
 	bool ask_sharemode = true;
 
diff --git a/source3/smbd/smb2_flush.c b/source3/smbd/smb2_flush.c
index 93955929077..c3f5a301290 100644
--- a/source3/smbd/smb2_flush.c
+++ b/source3/smbd/smb2_flush.c
@@ -22,6 +22,7 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 static struct tevent_req *smbd_smb2_flush_send(TALLOC_CTX *mem_ctx,
 					       struct tevent_context *ev,
diff --git a/source3/smbd/smb2_getinfo.c b/source3/smbd/smb2_getinfo.c
index 1d6b862d6ae..3c8c690342e 100644
--- a/source3/smbd/smb2_getinfo.c
+++ b/source3/smbd/smb2_getinfo.c
@@ -24,6 +24,7 @@
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
 #include "trans2.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 static struct tevent_req *smbd_smb2_getinfo_send(TALLOC_CTX *mem_ctx,
 						 struct tevent_context *ev,
diff --git a/source3/smbd/smb2_ioctl.c b/source3/smbd/smb2_ioctl.c
index 6ee9318212e..bce02b0a3a8 100644
--- a/source3/smbd/smb2_ioctl.c
+++ b/source3/smbd/smb2_ioctl.c
@@ -22,6 +22,9 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
+#include "../lib/util/tevent_ntstatus.h"
+#include "rpc_server/srv_pipe_hnd.h"
+#include "include/ntioctl.h"
 
 static struct tevent_req *smbd_smb2_ioctl_send(TALLOC_CTX *mem_ctx,
 					       struct tevent_context *ev,
@@ -242,8 +245,9 @@ static struct tevent_req *smbd_smb2_ioctl_send(TALLOC_CTX *mem_ctx,
 	state->in_max_output = in_max_output;
 	state->out_output = data_blob_null;
 
-	DEBUG(10,("smbd_smb2_ioctl: file_id[0x%016llX]\n",
-		  (unsigned long long)in_file_id_volatile));
+	DEBUG(10, ("smbd_smb2_ioctl: ctl_code[0x%08x] file_id[0x%016llX]\n",
+		   (unsigned)in_ctl_code,
+		   (unsigned long long)in_file_id_volatile));
 
 	smbreq = smbd_smb2_fake_smb_request(smb2req);
 	if (tevent_req_nomem(smbreq, req)) {
@@ -375,6 +379,131 @@ static struct tevent_req *smbd_smb2_ioctl_send(TALLOC_CTX *mem_ctx,
 					req);
 		return req;
 
+	case 0x00144064:	/* FSCTL_SRV_ENUMERATE_SNAPSHOTS */
+	{
+		/*
+		 * This is called to retrieve the number of Shadow Copies (a.k.a. snapshots)
+		 * and return their volume names.  If max_data_count is 16, then it is just
+		 * asking for the number of volumes and length of the combined names.
+		 *
+		 * pdata is the data allocated by our caller, but that uses
+		 * total_data_count (which is 0 in our case) rather than max_data_count.
+		 * Allocate the correct amount and return the pointer to let
+		 * it be deallocated when we return.
+		 */
+		struct shadow_copy_data *shadow_data = NULL;
+		bool labels = False;
+		uint32_t labels_data_count = 0;
+		uint32_t data_count;
+		uint32_t i;
+		char *pdata;
+		NTSTATUS status;
+
+		if (fsp == NULL) {
+			tevent_req_nterror(req, NT_STATUS_FILE_CLOSED);
+			return tevent_req_post(req, ev);
+		}
+
+		if (in_max_output < 16) {
+			DEBUG(0,("FSCTL_GET_SHADOW_COPY_DATA: "
+				 "in_max_output(%u) < 16 is invalid!\n",
+				 in_max_output));
+			tevent_req_nterror(req, NT_STATUS_INVALID_PARAMETER);
+			return tevent_req_post(req, ev);
+		}
+
+		if (in_max_output > 16) {
+			labels = True;
+		}
+
+		shadow_data = talloc_zero(talloc_tos(),
+					    struct shadow_copy_data);
+		if (tevent_req_nomem(shadow_data, req)) {
+			DEBUG(0,("TALLOC_ZERO() failed!\n"));
+			return tevent_req_post(req, ev);
+		}
+
+		/*
+		 * Call the VFS routine to actually do the work.
+		 */
+		if (SMB_VFS_GET_SHADOW_COPY_DATA(fsp, shadow_data, labels)
+		    != 0) {
+			if (errno == ENOSYS) {
+				DEBUG(5, ("FSCTL_GET_SHADOW_COPY_DATA: "
+					  "connectpath %s, not supported.\n",
+					  smbreq->conn->connectpath));
+				status = NT_STATUS_NOT_SUPPORTED;
+			} else {
+				DEBUG(0,("FSCTL_GET_SHADOW_COPY_DATA: "
+					 "connectpath %s, failed.\n",
+					 smbreq->conn->connectpath));
+				status = map_nt_error_from_unix(errno);
+			}
+			TALLOC_FREE(shadow_data);
+			tevent_req_nterror(req, status);
+			return tevent_req_post(req, ev);
+		}
+
+		labels_data_count =
+			(shadow_data->num_volumes*2*sizeof(SHADOW_COPY_LABEL))
+			+ 2;
+
+		if (labels) {
+			data_count = 12+labels_data_count+4;
+		} else {
+			data_count = 16;
+		}
+
+		if (labels && (in_max_output < data_count)) {
+			DEBUG(0, ("FSCTL_GET_SHADOW_COPY_DATA: "
+				  "in_max_output(%u) too small (%u) bytes "
+				  "needed!\n", in_max_output, data_count));
+			TALLOC_FREE(shadow_data);
+			tevent_req_nterror(req, NT_STATUS_BUFFER_TOO_SMALL);
+			return tevent_req_post(req, ev);
+		}
+
+		state->out_output = data_blob_talloc(state, NULL, data_count);
+		if (tevent_req_nomem(state->out_output.data, req)) {
+			return tevent_req_post(req, ev);
+		}
+
+		pdata = (char *)state->out_output.data;
+
+		/* num_volumes 4 bytes */
+		SIVAL(pdata, 0, shadow_data->num_volumes);
+
+		if (labels) {
+			/* num_labels 4 bytes */
+			SIVAL(pdata, 4, shadow_data->num_volumes);
+		}
+
+		/* needed_data_count 4 bytes */
+		SIVAL(pdata, 8, labels_data_count+4);
+
+		pdata += 12;
+
+		DEBUG(10,("FSCTL_GET_SHADOW_COPY_DATA: %u volumes for "
+			  "path[%s].\n",
+			  shadow_data->num_volumes, fsp_str_dbg(fsp)));
+		if (labels && shadow_data->labels) {
+			for (i=0; i<shadow_data->num_volumes; i++) {
+				srvstr_push(pdata, smbreq->flags2,
+					    pdata, shadow_data->labels[i],
+					    2*sizeof(SHADOW_COPY_LABEL),
+					    STR_UNICODE|STR_TERMINATE);
+				pdata += 2*sizeof(SHADOW_COPY_LABEL);
+				DEBUGADD(10, ("Label[%u]: '%s'\n", i,
+					      shadow_data->labels[i]));
+			}
+		}
+
+		TALLOC_FREE(shadow_data);
+
+		tevent_req_done(req);
+		return tevent_req_post(req, ev);
+        }
+
 	default:
 		if (IS_IPC(smbreq->conn)) {
 			tevent_req_nterror(req, NT_STATUS_FS_DRIVER_REQUIRED);
diff --git a/source3/smbd/smb2_lock.c b/source3/smbd/smb2_lock.c
index dcf1e0c6a53..5d615e1bed5 100644
--- a/source3/smbd/smb2_lock.c
+++ b/source3/smbd/smb2_lock.c
@@ -23,6 +23,7 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "messages.h"
 
 struct smbd_smb2_lock_element {
@@ -567,7 +568,7 @@ static bool recalc_smb2_brl_timeout(struct smbd_server_connection *sconn)
 	}
 
 	sconn->smb2.locks.brl_timeout = event_add_timed(
-				smbd_event_context(),
+				server_event_context(),
 				NULL,
 				next_timeout,
 				brl_timeout_fn,
@@ -627,9 +628,7 @@ bool push_blocking_lock_request_smb2( struct byte_range_lock *br_lck,
 		blr->expire_time.tv_sec = 0;
 		blr->expire_time.tv_usec = 0; /* Never expire. */
 	} else {
-		blr->expire_time = timeval_current_ofs(
-			lock_timeout/1000,
-			(lock_timeout % 1000) * 1000);
+		blr->expire_time = timeval_current_ofs_msec(lock_timeout);
 	}
 
 	blr->lock_num = lock_num;
diff --git a/source3/smbd/smb2_negprot.c b/source3/smbd/smb2_negprot.c
index 1e02c88d3dc..f639503ad4c 100644
--- a/source3/smbd/smb2_negprot.c
+++ b/source3/smbd/smb2_negprot.c
@@ -78,7 +78,7 @@ NTSTATUS smbd_smb2_request_process_negprot(struct smbd_smb2_request *req)
 	uint16_t dialect = 0;
 	uint32_t capabilities;
 
-/* TODO: drop the connection with INVALI_PARAMETER */
+/* TODO: drop the connection with INVALID_PARAMETER */
 
 	if (req->in.vector[i+1].iov_len != (expected_body_size & 0xFFFFFFFE)) {
 		return smbd_smb2_request_error(req, NT_STATUS_INVALID_PARAMETER);
@@ -177,5 +177,7 @@ NTSTATUS smbd_smb2_request_process_negprot(struct smbd_smb2_request *req)
 
 	outdyn = security_buffer;
 
+	req->sconn->using_smb2 = true;
+
 	return smbd_smb2_request_done(req, outbody, &outdyn);
 }
diff --git a/source3/smbd/smb2_notify.c b/source3/smbd/smb2_notify.c
index d52b2878eca..9e377ceda42 100644
--- a/source3/smbd/smb2_notify.c
+++ b/source3/smbd/smb2_notify.c
@@ -23,6 +23,7 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 struct smbd_smb2_notify_state {
 	struct smbd_smb2_request *smb2req;
diff --git a/source3/smbd/smb2_read.c b/source3/smbd/smb2_read.c
index bffda890106..6e686ef2738 100644
--- a/source3/smbd/smb2_read.c
+++ b/source3/smbd/smb2_read.c
@@ -24,6 +24,8 @@
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
 #include "libcli/security/security.h"
+#include "../lib/util/tevent_ntstatus.h"
+#include "rpc_server/srv_pipe_hnd.h"
 
 static struct tevent_req *smbd_smb2_read_send(TALLOC_CTX *mem_ctx,
 					      struct tevent_context *ev,
@@ -306,7 +308,7 @@ static NTSTATUS schedule_smb2_sendfile_read(struct smbd_smb2_request *smb2req,
 	/* Make a copy of state attached to the smb2req. Attach
 	   the destructor here as this will trigger the sendfile
 	   call when the request is destroyed. */
-	state_copy = TALLOC_P(smb2req, struct smbd_smb2_read_state);
+	state_copy = talloc(smb2req, struct smbd_smb2_read_state);
 	if (!state_copy) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -442,7 +444,7 @@ static struct tevent_req *smbd_smb2_read_send(TALLOC_CTX *mem_ctx,
 			return tevent_req_post(req, ev);
 		}
 
-		subreq = np_read_send(state, smbd_event_context(),
+		subreq = np_read_send(state, server_event_context(),
 				      fsp->fake_file_handle,
 				      state->out_data.data,
 				      state->out_data.length);
diff --git a/source3/smbd/smb2_server.c b/source3/smbd/smb2_server.c
index 0004e7ca8ca..65bb5b9eb0a 100644
--- a/source3/smbd/smb2_server.c
+++ b/source3/smbd/smb2_server.c
@@ -24,6 +24,7 @@
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
 #include "../lib/tsocket/tsocket.h"
+#include "../lib/util/tevent_ntstatus.h"
 #include "smbprofile.h"
 
 #define OUTVEC_ALLOC_SIZE (SMB2_HDR_BODY + 9)
@@ -94,7 +95,7 @@ static NTSTATUS smbd_initialize_smb2(struct smbd_server_connection *sconn)
 
 	TALLOC_FREE(sconn->smb1.fde);
 
-	sconn->smb2.event_ctx = smbd_event_context();
+	sconn->smb2.event_ctx = server_event_context();
 
 	sconn->smb2.recv_queue = tevent_queue_create(sconn, "smb2 recv queue");
 	if (sconn->smb2.recv_queue == NULL) {
@@ -268,15 +269,15 @@ static NTSTATUS smbd_smb2_request_create(struct smbd_server_connection *sconn,
 	memcpy(req->in.nbt_hdr, inbuf, 4);
 
 	ofs = 0;
-	req->in.vector[0].iov_base	= (void *)req->in.nbt_hdr;
+	req->in.vector[0].iov_base	= discard_const_p(void, req->in.nbt_hdr);
 	req->in.vector[0].iov_len	= 4;
 	ofs += req->in.vector[0].iov_len;
 
-	req->in.vector[1].iov_base	= (void *)(inbuf + ofs);
+	req->in.vector[1].iov_base	= discard_const_p(void, (inbuf + ofs));
 	req->in.vector[1].iov_len	= SMB2_HDR_BODY;
 	ofs += req->in.vector[1].iov_len;
 
-	req->in.vector[2].iov_base	= (void *)(inbuf + ofs);
+	req->in.vector[2].iov_base	= discard_const_p(void, (inbuf + ofs));
 	req->in.vector[2].iov_len	= SVAL(inbuf, ofs) & 0xFFFE;
 	ofs += req->in.vector[2].iov_len;
 
@@ -284,7 +285,7 @@ static NTSTATUS smbd_smb2_request_create(struct smbd_server_connection *sconn,
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	req->in.vector[3].iov_base	= (void *)(inbuf + ofs);
+	req->in.vector[3].iov_base	= discard_const_p(void, (inbuf + ofs));
 	req->in.vector[3].iov_len	= size - ofs;
 	ofs += req->in.vector[3].iov_len;
 
@@ -607,7 +608,7 @@ static bool dup_smb2_vec3(TALLOC_CTX *ctx,
 			srcvec[1].iov_base ==
 				((uint8_t *)srcvec[0].iov_base) +
 					SMB2_HDR_BODY) {
-		outvec[1].iov_base = ((uint8_t *)outvec[1].iov_base) +
+		outvec[1].iov_base = ((uint8_t *)outvec[0].iov_base) +
 					SMB2_HDR_BODY;
 		outvec[1].iov_len = 8;
 	} else {
@@ -2017,9 +2018,11 @@ static int smbd_smb2_request_next_vector(struct tstream_context *stream,
 				invalid = true;
 			}
 
-			if ((body_size % 2) != 0) {
-				body_size -= 1;
-			}
+			/*
+			 * Mask out the lowest bit, the "dynamic" part
+			 * of body_size.
+			 */
+			body_size &= ~1;
 
 			if (body_size > (full_size - SMB2_HDR_BODY)) {
 				/*
diff --git a/source3/smbd/smb2_sesssetup.c b/source3/smbd/smb2_sesssetup.c
index d727ee98b02..c33d1c61be9 100644
--- a/source3/smbd/smb2_sesssetup.c
+++ b/source3/smbd/smb2_sesssetup.c
@@ -844,7 +844,7 @@ NTSTATUS smbd_smb2_request_check_session(struct smbd_smb2_request *req)
 	if (chained_fixup) {
 		/* Fix up our own outhdr. */
 		outhdr = (const uint8_t *)req->out.vector[i].iov_base;
-		SBVAL(outhdr, SMB2_HDR_SESSION_ID, in_session_id);
+		SBVAL(discard_const_p(uint8_t, outhdr), SMB2_HDR_SESSION_ID, in_session_id);
 	}
 	return NT_STATUS_OK;
 }
diff --git a/source3/smbd/smb2_setinfo.c b/source3/smbd/smb2_setinfo.c
index 39b7aee76b5..e1149408d19 100644
--- a/source3/smbd/smb2_setinfo.c
+++ b/source3/smbd/smb2_setinfo.c
@@ -24,6 +24,7 @@
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
 #include "trans2.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 static struct tevent_req *smbd_smb2_setinfo_send(TALLOC_CTX *mem_ctx,
 						 struct tevent_context *ev,
diff --git a/source3/smbd/smb2_tcon.c b/source3/smbd/smb2_tcon.c
index bdd18d5d07e..1b4aa79070a 100644
--- a/source3/smbd/smb2_tcon.c
+++ b/source3/smbd/smb2_tcon.c
@@ -331,7 +331,7 @@ NTSTATUS smbd_smb2_request_check_tcon(struct smbd_smb2_request *req)
 	if (chained_fixup) {
 		/* Fix up our own outhdr. */
 		outhdr = (const uint8_t *)req->out.vector[i].iov_base;
-		SIVAL(outhdr, SMB2_HDR_TID, in_tid);
+		SIVAL(discard_const_p(uint8_t, outhdr), SMB2_HDR_TID, in_tid);
 	}
 
 	return NT_STATUS_OK;
diff --git a/source3/smbd/smb2_write.c b/source3/smbd/smb2_write.c
index 413fe0c7d17..057abcc8f7f 100644
--- a/source3/smbd/smb2_write.c
+++ b/source3/smbd/smb2_write.c
@@ -22,6 +22,8 @@
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
 #include "../libcli/smb/smb_common.h"
+#include "../lib/util/tevent_ntstatus.h"
+#include "rpc_server/srv_pipe_hnd.h"
 
 static struct tevent_req *smbd_smb2_write_send(TALLOC_CTX *mem_ctx,
 					       struct tevent_context *ev,
@@ -281,7 +283,7 @@ static struct tevent_req *smbd_smb2_write_send(TALLOC_CTX *mem_ctx,
 			return tevent_req_post(req, ev);
 		}
 
-		subreq = np_write_send(state, smbd_event_context(),
+		subreq = np_write_send(state, server_event_context(),
 				       fsp->fake_file_handle,
 				       in_data.data,
 				       in_data.length);
diff --git a/source3/smbd/srvstr.c b/source3/smbd/srvstr.c
index 050de599a57..c069dd4a5ec 100644
--- a/source3/smbd/srvstr.c
+++ b/source3/smbd/srvstr.c
@@ -56,7 +56,7 @@ ssize_t message_push_string(uint8 **outbuf, const char *str, int flags)
 	 */
 	grow_size = (strlen(str) + 2) * 4;
 
-	if (!(tmp = TALLOC_REALLOC_ARRAY(NULL, *outbuf, uint8,
+	if (!(tmp = talloc_realloc(NULL, *outbuf, uint8,
 					 buf_size + grow_size))) {
 		DEBUG(0, ("talloc failed\n"));
 		return -1;
diff --git a/source3/smbd/statcache.c b/source3/smbd/statcache.c
index 5c74b928550..963b7c4bc13 100644
--- a/source3/smbd/statcache.c
+++ b/source3/smbd/statcache.c
@@ -25,6 +25,7 @@
 #include "smbd/smbd.h"
 #include "messages.h"
 #include "smbprofile.h"
+#include "tdb_compat.h"
 
 /****************************************************************************
  Stat cache code used in unix_convert.
@@ -339,8 +340,8 @@ bool stat_cache_lookup(connection_struct *conn,
  Tell all smbd's to delete an entry.
 **************************************************************************/
 
-void send_stat_cache_delete_message(struct messaging_context *msg_ctx,
-				    const char *name)
+void smbd_send_stat_cache_delete_message(struct messaging_context *msg_ctx,
+					 const char *name)
 {
 #ifdef DEVELOPER
 	message_send_all(msg_ctx,
diff --git a/source3/smbd/trans2.c b/source3/smbd/trans2.c
index 507ae9ba93a..b853722eae0 100644
--- a/source3/smbd/trans2.c
+++ b/source3/smbd/trans2.c
@@ -35,6 +35,7 @@
 #include "trans2.h"
 #include "auth.h"
 #include "smbprofile.h"
+#include "rpc_server/srv_pipe_hnd.h"
 
 #define DIR_ENTRY_SAFETY_MARGIN 4096
 
@@ -106,7 +107,7 @@ static bool samba_private_attr_name(const char *unix_ea_name)
 		if (strequal( prohibited_ea_names[i], unix_ea_name))
 			return true;
 	}
-	if (StrnCaseCmp(unix_ea_name, SAMBA_XATTR_DOSSTREAM_PREFIX,
+	if (strncasecmp_m(unix_ea_name, SAMBA_XATTR_DOSSTREAM_PREFIX,
 			strlen(SAMBA_XATTR_DOSSTREAM_PREFIX)) == 0) {
 		return true;
 	}
@@ -128,7 +129,7 @@ NTSTATUS get_ea_value(TALLOC_CTX *mem_ctx, connection_struct *conn,
 
  again:
 
-	val = TALLOC_REALLOC_ARRAY(mem_ctx, val, char, attr_size);
+	val = talloc_realloc(mem_ctx, val, char, attr_size);
 	if (!val) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -191,7 +192,7 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 	 * TALLOC the result early to get the talloc hierarchy right.
 	 */
 
-	names = TALLOC_ARRAY(mem_ctx, char *, 1);
+	names = talloc_array(mem_ctx, char *, 1);
 	if (names == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		return NT_STATUS_NO_MEMORY;
@@ -199,7 +200,7 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 
 	while (ea_namelist_size <= 65536) {
 
-		ea_namelist = TALLOC_REALLOC_ARRAY(
+		ea_namelist = talloc_realloc(
 			names, ea_namelist, char, ea_namelist_size);
 		if (ea_namelist == NULL) {
 			DEBUG(0, ("talloc failed\n"));
@@ -258,7 +259,7 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
 		num_names += 1;
 	}
 
-	tmp = TALLOC_REALLOC_ARRAY(mem_ctx, names, char *, num_names);
+	tmp = talloc_realloc(mem_ctx, names, char *, num_names);
 	if (tmp == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		TALLOC_FREE(names);
@@ -315,7 +316,7 @@ static struct ea_list *get_ea_list_from_file(TALLOC_CTX *mem_ctx, connection_str
 		    || samba_private_attr_name(names[i]))
 			continue;
 
-		listp = TALLOC_P(mem_ctx, struct ea_list);
+		listp = talloc(mem_ctx, struct ea_list);
 		if (listp == NULL) {
 			return NULL;
 		}
@@ -387,7 +388,7 @@ static unsigned int fill_ea_buffer(TALLOC_CTX *mem_ctx, char *pdata, unsigned in
 		SCVAL(p,0,ea_list->ea.flags);
 		SCVAL(p,1,dos_namelen);
 		SSVAL(p,2,ea_list->ea.value.length);
-		fstrcpy(p+4, dos_ea_name);
+		strlcpy(p+4, dos_ea_name, dos_namelen+1);
 		memcpy( p + 4 + dos_namelen + 1, ea_list->ea.value.data, ea_list->ea.value.length);
 
 		total_data_size -= 4 + dos_namelen + 1 + ea_list->ea.value.length;
@@ -451,7 +452,7 @@ static NTSTATUS fill_ea_chained_buffer(TALLOC_CTX *mem_ctx,
 		SCVAL(p, 0x04, ea_list->ea.flags);
 		SCVAL(p, 0x05, dos_namelen);
 		SSVAL(p, 0x06, ea_list->ea.value.length);
-		fstrcpy((char *)(p+0x08), dos_ea_name);
+		strlcpy((char *)(p+0x08), dos_ea_name, dos_namelen+1);
 		memcpy(p + 0x08 + dos_namelen + 1, ea_list->ea.value.data, ea_list->ea.value.length);
 
 		total_data_size -= this_size;
@@ -490,7 +491,7 @@ static void canonicalize_ea_name(connection_struct *conn, files_struct *fsp, con
 		if (strequal(&unix_ea_name[5], ea_list->ea.name)) {
 			DEBUG(10,("canonicalize_ea_name: %s -> %s\n",
 				&unix_ea_name[5], ea_list->ea.name));
-			safe_strcpy(&unix_ea_name[5], ea_list->ea.name, sizeof(fstring)-6);
+			strlcpy(&unix_ea_name[5], ea_list->ea.name, sizeof(fstring)-5);
 			break;
 		}
 	}
@@ -589,7 +590,7 @@ static struct ea_list *read_ea_name_list(TALLOC_CTX *ctx, const char *pdata, siz
 	size_t converted_size, offset = 0;
 
 	while (offset + 2 < data_size) {
-		struct ea_list *eal = TALLOC_ZERO_P(ctx, struct ea_list);
+		struct ea_list *eal = talloc_zero(ctx, struct ea_list);
 		unsigned int namelen = CVAL(pdata,offset);
 
 		offset++; /* Go past the namelen byte. */
@@ -627,7 +628,7 @@ static struct ea_list *read_ea_name_list(TALLOC_CTX *ctx, const char *pdata, siz
 
 struct ea_list *read_ea_list_entry(TALLOC_CTX *ctx, const char *pdata, size_t data_size, size_t *pbytes_used)
 {
-	struct ea_list *eal = TALLOC_ZERO_P(ctx, struct ea_list);
+	struct ea_list *eal = talloc_zero(ctx, struct ea_list);
 	uint16 val_len;
 	unsigned int namelen;
 	size_t converted_size;
@@ -1148,7 +1149,7 @@ static void call_trans2open(connection_struct *conn,
 	fattr = dos_mode(conn, smb_fname);
 	mtime = convert_timespec_to_time_t(smb_fname->st.st_ex_mtime);
 	inode = smb_fname->st.st_ex_ino;
-	if (fattr & aDIR) {
+	if (fattr & FILE_ATTRIBUTE_DIRECTORY) {
 		close_file(req, fsp, ERROR_CLOSE);
 		reply_nterror(req, NT_STATUS_ACCESS_DENIED);
 		goto out;
@@ -1218,7 +1219,7 @@ static bool exact_match(bool has_wild,
 	if (case_sensitive) {
 		return strcmp(str,mask)==0;
 	} else {
-		return StrCaseCmp(str,mask) == 0;
+		return strcasecmp_m(str,mask) == 0;
 	}
 }
 
@@ -1524,7 +1525,7 @@ static bool smbd_marshall_dir_entry(TALLOC_CTX *ctx,
 	ZERO_STRUCT(create_date_ts);
 	ZERO_STRUCT(cdate_ts);
 
-	if (!(mode & aDIR)) {
+	if (!(mode & FILE_ATTRIBUTE_DIRECTORY)) {
 		file_size = get_file_size_stat(&smb_fname->st);
 	}
 	allocation_size = SMB_VFS_GET_ALLOC_SIZE(conn, NULL, &smb_fname->st);
@@ -3733,36 +3734,10 @@ cap_low = 0x%x, cap_high = 0x%x\n",
 				/* unknown_1 24 NULL bytes in pdata*/
 
 				/* the soft quotas 8 bytes (uint64_t)*/
-				quotas.softlim = (uint64_t)IVAL(pdata,24);
-#ifdef LARGE_SMB_OFF_T
-				quotas.softlim |= (((uint64_t)IVAL(pdata,28)) << 32);
-#else /* LARGE_SMB_OFF_T */
-				if ((IVAL(pdata,28) != 0)&&
-					((quotas.softlim != 0xFFFFFFFF)||
-					(IVAL(pdata,28)!=0xFFFFFFFF))) {
-					/* more than 32 bits? */
-					reply_nterror(
-						req,
-						NT_STATUS_INVALID_PARAMETER);
-					return;
-				}
-#endif /* LARGE_SMB_OFF_T */
+				quotas.softlim = BVAL(pdata,24);
 
 				/* the hard quotas 8 bytes (uint64_t)*/
-				quotas.hardlim = (uint64_t)IVAL(pdata,32);
-#ifdef LARGE_SMB_OFF_T
-				quotas.hardlim |= (((uint64_t)IVAL(pdata,36)) << 32);
-#else /* LARGE_SMB_OFF_T */
-				if ((IVAL(pdata,36) != 0)&&
-					((quotas.hardlim != 0xFFFFFFFF)||
-					(IVAL(pdata,36)!=0xFFFFFFFF))) {
-					/* more than 32 bits? */
-					reply_nterror(
-						req,
-						NT_STATUS_INVALID_PARAMETER);
-					return;
-				}
-#endif /* LARGE_SMB_OFF_T */
+				quotas.hardlim = BVAL(pdata,32);
 
 				/* quota_flags 2 bytes **/
 				quotas.qflags = SVAL(pdata,40);
@@ -4253,7 +4228,7 @@ NTSTATUS smbd_do_qfilepathinfo(connection_struct *conn,
 	mode = dos_mode(conn, smb_fname);
 	nlink = psbuf->st_ex_nlink;
 
-	if (nlink && (mode&aDIR)) {
+	if (nlink && (mode&FILE_ATTRIBUTE_DIRECTORY)) {
 		nlink = 1;
 	}
 
@@ -4334,7 +4309,7 @@ NTSTATUS smbd_do_qfilepathinfo(connection_struct *conn,
 		}
 	}
 
-	if (!(mode & aDIR)) {
+	if (!(mode & FILE_ATTRIBUTE_DIRECTORY)) {
 		file_size = get_file_size_stat(psbuf);
 	}
 
@@ -4504,7 +4479,7 @@ NTSTATUS smbd_do_qfilepathinfo(connection_struct *conn,
 			SOFF_T(pdata,8,file_size);
 			SIVAL(pdata,16,nlink);
 			SCVAL(pdata,20,delete_pending?1:0);
-			SCVAL(pdata,21,(mode&aDIR)?1:0);
+			SCVAL(pdata,21,(mode&FILE_ATTRIBUTE_DIRECTORY)?1:0);
 			SSVAL(pdata,22,0); /* Padding. */
 			break;
 
@@ -4587,7 +4562,7 @@ NTSTATUS smbd_do_qfilepathinfo(connection_struct *conn,
 			SOFF_T(pdata,8,file_size);
 			SIVAL(pdata,16,nlink);
 			SCVAL(pdata,20,delete_pending);
-			SCVAL(pdata,21,(mode&aDIR)?1:0);
+			SCVAL(pdata,21,(mode&FILE_ATTRIBUTE_DIRECTORY)?1:0);
 			SSVAL(pdata,22,0);
 			pdata += 24;
 			SIVAL(pdata,0,ea_size);
@@ -4618,7 +4593,7 @@ NTSTATUS smbd_do_qfilepathinfo(connection_struct *conn,
 			SBVAL(pdata,	0x30, file_size);
 			SIVAL(pdata,	0x38, nlink);
 			SCVAL(pdata,	0x3C, delete_pending);
-			SCVAL(pdata,	0x3D, (mode&aDIR)?1:0);
+			SCVAL(pdata,	0x3D, (mode&FILE_ATTRIBUTE_DIRECTORY)?1:0);
 			SSVAL(pdata,	0x3E, 0); /* padding */
 			SBVAL(pdata,	0x40, file_index);
 			SIVAL(pdata,	0x48, ea_size);
@@ -4793,7 +4768,7 @@ NTSTATUS smbd_do_qfilepathinfo(connection_struct *conn,
 		case SMB_QUERY_FILE_UNIX_LINK:
 			{
 				int len;
-				char *buffer = TALLOC_ARRAY(mem_ctx, char, PATH_MAX+1);
+				char *buffer = talloc_array(mem_ctx, char, PATH_MAX+1);
 
 				if (!buffer) {
 					return NT_STATUS_NO_MEMORY;
@@ -5326,7 +5301,7 @@ total_data=%u (should be %u)\n", (unsigned int)total_data, (unsigned int)IVAL(pd
 			}
 
 			/* Copy the lock range data. */
-			lock_data = (char *)TALLOC_MEMDUP(
+			lock_data = (char *)talloc_memdup(
 				req, pdata, total_data);
 			if (!lock_data) {
 				reply_nterror(req, NT_STATUS_NO_MEMORY);
@@ -5580,9 +5555,9 @@ static NTSTATUS smb_set_file_dosmode(connection_struct *conn,
 
 	if (dosmode) {
 		if (S_ISDIR(smb_fname_base->st.st_ex_mode)) {
-			dosmode |= aDIR;
+			dosmode |= FILE_ATTRIBUTE_DIRECTORY;
 		} else {
-			dosmode &= ~aDIR;
+			dosmode &= ~FILE_ATTRIBUTE_DIRECTORY;
 		}
 	}
 
@@ -6075,8 +6050,9 @@ static NTSTATUS smb2_file_rename_information(connection_struct *conn,
 		  "SMB_FILE_RENAME_INFORMATION (fnum %d) %s -> %s\n",
 		  fsp->fnum, fsp_str_dbg(fsp),
 		  smb_fname_str_dbg(smb_fname_dst)));
-	status = rename_internals_fsp(conn, fsp, smb_fname_dst, 0,
-				      overwrite);
+	status = rename_internals_fsp(conn, fsp, smb_fname_dst,
+				(FILE_ATTRIBUTE_HIDDEN|FILE_ATTRIBUTE_SYSTEM),
+				overwrite);
 
  out:
 	TALLOC_FREE(smb_fname_dst);
@@ -7251,6 +7227,9 @@ static NTSTATUS smb_posix_mkdir(connection_struct *conn,
  Open/Create a file with POSIX semantics.
 ****************************************************************************/
 
+#define SMB_O_RDONLY_MAPPING (FILE_READ_DATA|FILE_READ_ATTRIBUTES|FILE_READ_EA)
+#define SMB_O_WRONLY_MAPPING (FILE_WRITE_DATA|FILE_WRITE_ATTRIBUTES|FILE_WRITE_EA)
+
 static NTSTATUS smb_posix_open(connection_struct *conn,
 			       struct smb_request *req,
 				char **ppdata,
@@ -7266,7 +7245,7 @@ static NTSTATUS smb_posix_open(connection_struct *conn,
 	uint32 mod_unixmode = 0;
 	uint32 create_disp = 0;
 	uint32 access_mask = 0;
-	uint32 create_options = 0;
+	uint32 create_options = FILE_NON_DIRECTORY_FILE;
 	NTSTATUS status = NT_STATUS_OK;
 	mode_t unixmode = (mode_t)0;
 	files_struct *fsp = NULL;
@@ -7296,13 +7275,14 @@ static NTSTATUS smb_posix_open(connection_struct *conn,
 
 	switch (wire_open_mode & SMB_ACCMODE) {
 		case SMB_O_RDONLY:
-			access_mask = FILE_READ_DATA;
+			access_mask = SMB_O_RDONLY_MAPPING;
 			break;
 		case SMB_O_WRONLY:
-			access_mask = FILE_WRITE_DATA;
+			access_mask = SMB_O_WRONLY_MAPPING;
 			break;
 		case SMB_O_RDWR:
-			access_mask = FILE_READ_DATA|FILE_WRITE_DATA;
+			access_mask = (SMB_O_RDONLY_MAPPING|
+					SMB_O_WRONLY_MAPPING);
 			break;
 		default:
 			DEBUG(5,("smb_posix_open: invalid open mode 0x%x\n",
@@ -7312,18 +7292,55 @@ static NTSTATUS smb_posix_open(connection_struct *conn,
 
 	wire_open_mode &= ~SMB_ACCMODE;
 
-	if((wire_open_mode & (SMB_O_CREAT | SMB_O_EXCL)) == (SMB_O_CREAT | SMB_O_EXCL)) {
-		create_disp = FILE_CREATE;
-	} else if((wire_open_mode & (SMB_O_CREAT | SMB_O_TRUNC)) == (SMB_O_CREAT | SMB_O_TRUNC)) {
-		create_disp = FILE_OVERWRITE_IF;
-	} else if((wire_open_mode & SMB_O_CREAT) == SMB_O_CREAT) {
-		create_disp = FILE_OPEN_IF;
-	} else if ((wire_open_mode & (SMB_O_CREAT | SMB_O_EXCL | SMB_O_TRUNC)) == 0) {
-		create_disp = FILE_OPEN;
-	} else {
-		DEBUG(5,("smb_posix_open: invalid create mode 0x%x\n",
-			(unsigned int)wire_open_mode ));
-		return NT_STATUS_INVALID_PARAMETER;
+	/* First take care of O_CREAT|O_EXCL interactions. */
+	switch (wire_open_mode & (SMB_O_CREAT | SMB_O_EXCL)) {
+		case (SMB_O_CREAT | SMB_O_EXCL):
+			/* File exists fail. File not exist create. */
+			create_disp = FILE_CREATE;
+			break;
+		case SMB_O_CREAT:
+			/* File exists open. File not exist create. */
+			create_disp = FILE_OPEN_IF;
+			break;
+		case 0:
+			/* File exists open. File not exist fail. */
+			create_disp = FILE_OPEN;
+			break;
+		case SMB_O_EXCL:
+			/* O_EXCL on its own without O_CREAT is undefined. */
+		default:
+			DEBUG(5,("smb_posix_open: invalid create mode 0x%x\n",
+				(unsigned int)wire_open_mode ));
+			return NT_STATUS_INVALID_PARAMETER;
+	}
+
+	/* Next factor in the effects of O_TRUNC. */
+	wire_open_mode &= ~(SMB_O_CREAT | SMB_O_EXCL);
+
+	if (wire_open_mode & SMB_O_TRUNC) {
+		switch (create_disp) {
+			case FILE_CREATE:
+				/* (SMB_O_CREAT | SMB_O_EXCL | O_TRUNC) */
+				/* Leave create_disp alone as
+				   (O_CREAT|O_EXCL|O_TRUNC) == (O_CREAT|O_EXCL)
+				*/
+				/* File exists fail. File not exist create. */
+				break;
+			case FILE_OPEN_IF:
+				/* SMB_O_CREAT | SMB_O_TRUNC */
+				/* File exists overwrite. File not exist create. */
+				create_disp = FILE_OVERWRITE_IF;
+				break;
+			case FILE_OPEN:
+				/* SMB_O_TRUNC */
+				/* File exists overwrite. File not exist fail. */
+				create_disp = FILE_OVERWRITE;
+				break;
+			default:
+				/* Cannot get here. */
+				smb_panic("smb_posix_open: logic error");
+				return NT_STATUS_INVALID_PARAMETER;
+		}
 	}
 
 	raw_unixmode = IVAL(pdata,8);
@@ -7350,6 +7367,15 @@ static NTSTATUS smb_posix_open(connection_struct *conn,
 		mod_unixmode |= FILE_FLAG_NO_BUFFERING;
 	}
 
+	if ((wire_open_mode & SMB_O_DIRECTORY) ||
+			VALID_STAT_OF_DIR(smb_fname->st)) {
+		if (access_mask != SMB_O_RDONLY_MAPPING) {
+			return NT_STATUS_FILE_IS_A_DIRECTORY;
+		}
+		create_options &= ~FILE_NON_DIRECTORY_FILE;
+		create_options |= FILE_DIRECTORY_FILE;
+	}
+
 	DEBUG(10,("smb_posix_open: file %s, smb_posix_flags = %u, mode 0%o\n",
 		smb_fname_str_dbg(smb_fname),
 		(unsigned int)wire_open_mode,
@@ -7364,7 +7390,7 @@ static NTSTATUS smb_posix_open(connection_struct *conn,
 		(FILE_SHARE_READ | FILE_SHARE_WRITE |	/* share_access */
 		    FILE_SHARE_DELETE),
 		create_disp,				/* create_disposition*/
-		FILE_NON_DIRECTORY_FILE,		/* create_options */
+		create_options,				/* create_options */
 		mod_unixmode,				/* file_attributes */
 		oplock_request,				/* oplock_request */
 		0,					/* allocation_size */
@@ -8302,7 +8328,7 @@ static void call_trans2getdfsreferral(connection_struct *conn,
 		return;
 	}
 
-	SSVAL(req->inbuf, smb_flg2,
+	SSVAL((discard_const_p(uint8_t, req->inbuf)), smb_flg2,
 	      SVAL(req->inbuf,smb_flg2) | FLAGS2_DFS_PATHNAMES);
 	send_trans2_replies(conn, req,0,0,*ppdata,reply_size, max_data_bytes);
 
@@ -8351,7 +8377,7 @@ static void call_trans2ioctl(connection_struct *conn,
 			SSVAL(pdata, 0, 0);
 		}
 		srvstr_push(pdata, req->flags2, pdata + 2,
-			    global_myname(), 15,
+			    lp_netbios_name(), 15,
 			    STR_ASCII|STR_TERMINATE); /* Our NetBIOS name */
 		srvstr_push(pdata, req->flags2, pdata+18,
 			    lp_servicename(SNUM(conn)), 13,
@@ -8433,7 +8459,7 @@ static void handle_trans2(connection_struct *conn, struct smb_request *req,
 {
 	if (get_Protocol() >= PROTOCOL_NT1) {
 		req->flags2 |= 0x40; /* IS_LONG_NAME */
-		SSVAL(req->inbuf,smb_flg2,req->flags2);
+		SSVAL((discard_const_p(uint8_t, req->inbuf)),smb_flg2,req->flags2);
 	}
 
 	if (conn->encrypt_level == Required && !req->encrypted) {
@@ -8646,7 +8672,7 @@ void reply_trans2(struct smb_request *req)
 		}
 	}
 
-	if ((state = TALLOC_P(conn, struct trans_state)) == NULL) {
+	if ((state = talloc(conn, struct trans_state)) == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		reply_nterror(req, NT_STATUS_NO_MEMORY);
 		END_PROFILE(SMBtrans2);
@@ -8792,7 +8818,7 @@ void reply_transs2(struct smb_request *req)
 
 	START_PROFILE(SMBtranss2);
 
-	show_msg((char *)req->inbuf);
+	show_msg((const char *)req->inbuf);
 
 	if (req->wct < 8) {
 		reply_nterror(req, NT_STATUS_INVALID_PARAMETER);
diff --git a/source3/smbd/uid.c b/source3/smbd/uid.c
index b554b36054b..285b158a191 100644
--- a/source3/smbd/uid.c
+++ b/source3/smbd/uid.c
@@ -25,7 +25,6 @@
 #include "libcli/security/security.h"
 #include "passdb/lookup_sid.h"
 #include "auth.h"
-#include "ntdomain.h"
 
 /* what user is current? */
 extern struct current_user current_user;
@@ -183,50 +182,6 @@ static bool check_user_ok(connection_struct *conn,
 }
 
 /****************************************************************************
- Clear a vuid out of the connection's vuid cache
- This is only called on SMBulogoff.
-****************************************************************************/
-
-void conn_clear_vuid_cache(connection_struct *conn, uint16_t vuid)
-{
-	int i;
-
-	for (i=0; i<VUID_CACHE_SIZE; i++) {
-		struct vuid_cache_entry *ent;
-
-		ent = &conn->vuid_cache.array[i];
-
-		if (ent->vuid == vuid) {
-			ent->vuid = UID_FIELD_INVALID;
-			/*
-			 * We need to keep conn->session_info around
-			 * if it's equal to ent->session_info as a SMBulogoff
-			 * is often followed by a SMBtdis (with an invalid
-			 * vuid). The debug code (or regular code in
-			 * vfs_full_audit) wants to refer to the
-			 * conn->session_info pointer to print debug
-			 * statements. Theoretically this is a bug,
-			 * as once the vuid is gone the session_info
-			 * on the conn struct isn't valid any more,
-			 * but there's enough code that assumes
-			 * conn->session_info is never null that
-			 * it's easier to hold onto the old pointer
-			 * until we get a new sessionsetupX.
-			 * As everything is hung off the
-			 * conn pointer as a talloc context we're not
-			 * leaking memory here. See bug #6315. JRA.
-			 */
-			if (conn->session_info == ent->session_info) {
-				ent->session_info = NULL;
-			} else {
-				TALLOC_FREE(ent->session_info);
-			}
-			ent->read_only = False;
-		}
-	}
-}
-
-/****************************************************************************
  Become the user of a connection number without changing the security context
  stack, but modify the current_user entries.
 ****************************************************************************/
@@ -390,7 +345,7 @@ bool change_to_user_by_session(connection_struct *conn,
  but modify the current_user entries.
 ****************************************************************************/
 
-bool change_to_root_user(void)
+bool smbd_change_to_root_user(void)
 {
 	set_root_sec_ctx();
 
@@ -409,14 +364,14 @@ bool change_to_root_user(void)
  user. Doesn't modify current_user.
 ****************************************************************************/
 
-bool become_authenticated_pipe_user(struct pipes_struct *p)
+bool become_authenticated_pipe_user(struct auth_serversupplied_info *session_info)
 {
 	if (!push_sec_ctx())
 		return False;
 
-	set_sec_ctx(p->session_info->utok.uid, p->session_info->utok.gid,
-		    p->session_info->utok.ngroups, p->session_info->utok.groups,
-		    p->session_info->security_token);
+	set_sec_ctx(session_info->utok.uid, session_info->utok.gid,
+		    session_info->utok.ngroups, session_info->utok.groups,
+		    session_info->security_token);
 
 	return True;
 }
@@ -486,7 +441,7 @@ static void pop_conn_ctx(void)
  restores the connection context.
 ****************************************************************************/
 
-void become_root(void)
+void smbd_become_root(void)
 {
 	 /*
 	  * no good way to handle push_sec_ctx() failing without changing
@@ -501,7 +456,7 @@ void become_root(void)
 
 /* Unbecome the root user */
 
-void unbecome_root(void)
+void smbd_unbecome_root(void)
 {
 	pop_sec_ctx();
 	pop_conn_ctx();
diff --git a/source3/smbd/vfs.c b/source3/smbd/vfs.c
index 3b482e7f0cd..7be38672678 100644
--- a/source3/smbd/vfs.c
+++ b/source3/smbd/vfs.c
@@ -183,7 +183,7 @@ bool vfs_init_custom(connection_struct *conn, const char *vfs_object)
 
 	DEBUGADD(5,("Successfully loaded vfs module [%s] with the new modules system\n", vfs_object));
 
-	handle = TALLOC_ZERO_P(conn, vfs_handle_struct);
+	handle = talloc_zero(conn, vfs_handle_struct);
 	if (!handle) {
 		DEBUG(0,("TALLOC_ZERO() failed!\n"));
 		goto fail;
@@ -795,7 +795,7 @@ int vfs_ChDir(connection_struct *conn, const char *path)
 
 char *vfs_GetWd(TALLOC_CTX *ctx, connection_struct *conn)
 {
-        char s[PATH_MAX+1];
+        char *current_dir = NULL;
 	char *result = NULL;
 	DATA_BLOB cache_value;
 	struct file_id key;
@@ -803,8 +803,6 @@ char *vfs_GetWd(TALLOC_CTX *ctx, connection_struct *conn)
 	struct smb_filename *smb_fname_full = NULL;
 	NTSTATUS status;
 
-	*s = 0;
-
 	if (!lp_getwd_cache()) {
 		goto nocache;
 	}
@@ -866,7 +864,8 @@ char *vfs_GetWd(TALLOC_CTX *ctx, connection_struct *conn)
 	 * systems, or the not quite so bad getwd.
 	 */
 
-	if (!SMB_VFS_GETWD(conn,s)) {
+	current_dir = SMB_VFS_GETWD(conn);
+	if (current_dir == NULL) {
 		DEBUG(0, ("vfs_GetWd: SMB_VFS_GETWD call failed: %s\n",
 			  strerror(errno)));
 		goto out;
@@ -877,10 +876,11 @@ char *vfs_GetWd(TALLOC_CTX *ctx, connection_struct *conn)
 
 		memcache_add(smbd_memcache(), GETWD_CACHE,
 			     data_blob_const(&key, sizeof(key)),
-			     data_blob_const(s, strlen(s)+1));
+			     data_blob_const(current_dir,
+						strlen(current_dir)+1));
 	}
 
-	result = talloc_strdup(ctx, s);
+	result = talloc_strdup(ctx, current_dir);
 	if (result == NULL) {
 		errno = ENOMEM;
 	}
@@ -888,6 +888,7 @@ char *vfs_GetWd(TALLOC_CTX *ctx, connection_struct *conn)
  out:
 	TALLOC_FREE(smb_fname_dot);
 	TALLOC_FREE(smb_fname_full);
+	SAFE_FREE(current_dir);
 	return result;
 }
 
@@ -899,6 +900,8 @@ char *vfs_GetWd(TALLOC_CTX *ctx, connection_struct *conn)
 NTSTATUS check_reduced_name(connection_struct *conn, const char *fname)
 {
 	char *resolved_name = NULL;
+	bool allow_symlinks = true;
+	bool allow_widelinks = false;
 
 	DEBUG(3,("check_reduced_name [%s] [%s]\n", fname, conn->connectpath));
 
@@ -973,9 +976,13 @@ NTSTATUS check_reduced_name(connection_struct *conn, const char *fname)
 		return NT_STATUS_OBJECT_NAME_INVALID;
 	}
 
-	/* Check for widelinks allowed. */
-	if (!lp_widelinks(SNUM(conn))) {
+	allow_widelinks = lp_widelinks(SNUM(conn));
+	allow_symlinks = lp_symlinks(SNUM(conn));
+
+	/* Common widelinks and symlinks checks. */
+	if (!allow_widelinks || !allow_symlinks) {
 		const char *conn_rootdir;
+		size_t rootdir_len;
 
 		conn_rootdir = SMB_VFS_CONNECTPATH(conn, fname);
 		if (conn_rootdir == NULL) {
@@ -985,8 +992,9 @@ NTSTATUS check_reduced_name(connection_struct *conn, const char *fname)
 			return NT_STATUS_ACCESS_DENIED;
 		}
 
+		rootdir_len = strlen(conn_rootdir);
 		if (strncmp(conn_rootdir, resolved_name,
-				strlen(conn_rootdir)) != 0) {
+				rootdir_len) != 0) {
 			DEBUG(2, ("check_reduced_name: Bad access "
 				"attempt: %s is a symlink outside the "
 				"share path\n", fname));
@@ -995,35 +1003,38 @@ NTSTATUS check_reduced_name(connection_struct *conn, const char *fname)
 			SAFE_FREE(resolved_name);
 			return NT_STATUS_ACCESS_DENIED;
 		}
-	}
 
-        /* Check if we are allowing users to follow symlinks */
-        /* Patch from David Clerc <David.Clerc@cui.unige.ch>
-                University of Geneva */
+		/* Extra checks if all symlinks are disallowed. */
+		if (!allow_symlinks) {
+			/* fname can't have changed in resolved_path. */
+			const char *p = &resolved_name[rootdir_len];
 
-#ifdef S_ISLNK
-        if (!lp_symlinks(SNUM(conn))) {
-		struct smb_filename *smb_fname = NULL;
-		NTSTATUS status;
+			/* *p can be '\0' if fname was "." */
+			if (*p == '\0' && ISDOT(fname)) {
+				goto out;
+			}
 
-		status = create_synthetic_smb_fname(talloc_tos(), fname, NULL,
-						    NULL, &smb_fname);
-		if (!NT_STATUS_IS_OK(status)) {
-			SAFE_FREE(resolved_name);
-                        return status;
+			if (*p != '/') {
+				DEBUG(2, ("check_reduced_name: logic error (%c) "
+					"in resolved_name: %s\n",
+					*p,
+					fname));
+				SAFE_FREE(resolved_name);
+				return NT_STATUS_ACCESS_DENIED;
+			}
+
+			p++;
+			if (strcmp(fname, p)!=0) {
+				DEBUG(2, ("check_reduced_name: Bad access "
+					"attempt: %s is a symlink\n",
+					fname));
+				SAFE_FREE(resolved_name);
+				return NT_STATUS_ACCESS_DENIED;
+			}
 		}
+	}
 
-		if ( (SMB_VFS_LSTAT(conn, smb_fname) != -1) &&
-                                (S_ISLNK(smb_fname->st.st_ex_mode)) ) {
-			SAFE_FREE(resolved_name);
-                        DEBUG(3,("check_reduced_name: denied: file path name "
-				 "%s is a symlink\n",resolved_name));
-			TALLOC_FREE(smb_fname);
-			return NT_STATUS_ACCESS_DENIED;
-                }
-		TALLOC_FREE(smb_fname);
-        }
-#endif
+  out:
 
 	DEBUG(3,("check_reduced_name: %s reduced to %s\n", fname,
 		 resolved_name));
@@ -1164,7 +1175,7 @@ int smb_vfs_call_set_quota(struct vfs_handle_struct *handle,
 
 int smb_vfs_call_get_shadow_copy_data(struct vfs_handle_struct *handle,
 				      struct files_struct *fsp,
-				      SHADOW_COPY_DATA *shadow_copy_data,
+				      struct shadow_copy_data *shadow_copy_data,
 				      bool labels)
 {
 	VFS_FIND(get_shadow_copy_data);
@@ -1496,7 +1507,7 @@ NTSTATUS vfs_chown_fsp(files_struct *fsp, uid_t uid, gid_t gid)
 		}
 
 		ZERO_STRUCT(local_fname);
-		local_fname.base_name = CONST_DISCARD(char *,final_component);
+		local_fname.base_name = discard_const_p(char, final_component);
 
 		/* Must use lstat here. */
 		ret = SMB_VFS_LSTAT(fsp->conn, &local_fname);
@@ -1543,10 +1554,10 @@ int smb_vfs_call_chdir(struct vfs_handle_struct *handle, const char *path)
 	return handle->fns->chdir(handle, path);
 }
 
-char *smb_vfs_call_getwd(struct vfs_handle_struct *handle, char *buf)
+char *smb_vfs_call_getwd(struct vfs_handle_struct *handle)
 {
 	VFS_FIND(getwd);
-	return handle->fns->getwd(handle, buf);
+	return handle->fns->getwd(handle);
 }
 
 int smb_vfs_call_ntimes(struct vfs_handle_struct *handle,
diff --git a/source3/stf/info3cache.py b/source3/stf/info3cache.py
deleted file mode 100755
index 96d5a1d4596..00000000000
--- a/source3/stf/info3cache.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python
-#
-# Upon a winbindd authentication, test that an info3 record is cached in
-# netsamlogon_cache.tdb and cache records are removed from winbindd_cache.tdb
-#
-
-import comfychair, stf
-from samba import tdb, winbind
-
-#
-# We want to implement the following test on a win2k native mode domain.
-#
-# 1. trash netsamlogon_cache.tdb
-# 2. wbinfo -r DOMAIN\Administrator                    [FAIL]
-# 3. wbinfo --auth-crap DOMAIN\Administrator%password  [PASS]
-# 4. wbinfo -r DOMAIN\Administrator                    [PASS]
-#
-# Also for step 3 we want to try 'wbinfo --auth-smbd' and
-# 'wbinfo --auth-plaintext'
-#
-
-#
-# TODO: To implement this test we need to be able to
-#
-#  - pass username%password combination for an invidivual winbindd request
-#    (so we can get the administrator SID so we can clear the info3 cache)
-#
-#  - start/restart winbindd (to trash the winbind cache)
-#
-#  - from samba import dynconfig (to find location of info3 cache)
-#
-#  - be able to modify the winbindd cache (to set/reset individual winbind
-#    cache entries)
-#
-#  - have --auth-crap present in HEAD
-#
-
-class WinbindAuthCrap(comfychair.TestCase):
-    def runtest(self):
-        raise comfychair.NotRunError, "not implemented"
-        
-class WinbindAuthSmbd(comfychair.TestCase):
-    def runtest(self):
-        # Grr - winbindd in HEAD doesn't contain the auth_smbd function
-        raise comfychair.NotRunError, "no auth_smbd in HEAD"
-
-class WinbindAuthPlaintext(comfychair.TestCase):
-    def runtest(self):
-        raise comfychair.NotRunError, "not implemented"
-
-tests = [WinbindAuthCrap, WinbindAuthSmbd, WinbindAuthPlaintext]
-
-if __name__ == "__main__":
-    comfychair.main(tests)
diff --git a/source3/stf/sambalib.py b/source3/stf/sambalib.py
deleted file mode 100644
index 552280ed292..00000000000
--- a/source3/stf/sambalib.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#! /usr/bin/python     
-
-# Comfychair test cases for Samba string functions.
-
-# Copyright (C) 2003 by Martin Pool <mbp@samba.org>
-# 
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; either version 3 of the
-# License, or (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-"""Tests for Samba library functions."""
-
-import sys, re, comfychair
-from unicodenames import *
-
-class snprintf_Test(comfychair.TestCase):
-    def runtest(self):
-        # Everything is built in to the test
-        out, err = self.runcmd('t_snprintf')
-        
-# Define the tests exported by this module
-tests = [snprintf_Test]
-
-# Handle execution of this file as a main program
-if __name__ == '__main__':
-    comfychair.main(tests)
-
-# Local variables:
-# coding: utf-8
-# End:
diff --git a/source3/torture/cmd_vfs.c b/source3/torture/cmd_vfs.c
index 2c3a4160024..9527a34af4d 100644
--- a/source3/torture/cmd_vfs.c
+++ b/source3/torture/cmd_vfs.c
@@ -57,7 +57,7 @@ static NTSTATUS cmd_populate(struct vfs_state *vfs, TALLOC_CTX *mem_ctx, int arg
 	}
 	c = argv[1][0];
 	size = atoi(argv[2]);
-	vfs->data = TALLOC_ARRAY(mem_ctx, char, size);
+	vfs->data = talloc_array(mem_ctx, char, size);
 	if (vfs->data == NULL) {
 		printf("populate: error=-1 (not enough memory)");
 		return NT_STATUS_UNSUCCESSFUL;
@@ -435,7 +435,7 @@ static NTSTATUS cmd_read(struct vfs_state *vfs, TALLOC_CTX *mem_ctx, int argc, c
 	/* do some error checking on these */
 	fd = atoi(argv[1]);
 	size = atoi(argv[2]);
-	vfs->data = TALLOC_ARRAY(mem_ctx, char, size);
+	vfs->data = talloc_array(mem_ctx, char, size);
 	if (vfs->data == NULL) {
 		printf("read: error=-1 (not enough memory)");
 		return NT_STATUS_UNSUCCESSFUL;
@@ -889,13 +889,14 @@ static NTSTATUS cmd_fchown(struct vfs_state *vfs, TALLOC_CTX *mem_ctx, int argc,
 
 static NTSTATUS cmd_getwd(struct vfs_state *vfs, TALLOC_CTX *mem_ctx, int argc, const char **argv)
 {
-	char buf[PATH_MAX];
-	if (SMB_VFS_GETWD(vfs->conn, buf) == NULL) {
+	char *buf = SMB_VFS_GETWD(vfs->conn);
+	if (buf == NULL) {
 		printf("getwd: error=%d (%s)\n", errno, strerror(errno));
 		return NT_STATUS_UNSUCCESSFUL;
 	}
 
 	printf("getwd: %s\n", buf);
+	SAFE_FREE(buf);
 	return NT_STATUS_OK;
 }
 
diff --git a/source3/torture/denytest.c b/source3/torture/denytest.c
index 1f60734fa7a..be01e829dc9 100644
--- a/source3/torture/denytest.c
+++ b/source3/torture/denytest.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "torture/proto.h"
+#include "libsmb/libsmb.h"
 
 extern bool torture_showall;
 
@@ -1421,9 +1422,10 @@ bool torture_denytest1(int dummy)
 	printf("starting denytest1\n");
 
 	for (i=0;i<2;i++) {
-		cli_unlink(cli1, fnames[i], aSYSTEM | aHIDDEN);
+		cli_unlink(cli1, fnames[i], FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 		cli_open(cli1, fnames[i], O_RDWR|O_CREAT, DENY_NONE, &fnum1);
-		cli_write(cli1, fnum1, 0, fnames[i], 0, strlen(fnames[i]));
+		cli_writeall(cli1, fnum1, 0, (const uint8_t *)fnames[i], 0,
+			     strlen(fnames[i]), NULL);
 		cli_close(cli1, fnum1);
 	}
 
@@ -1452,7 +1454,9 @@ bool torture_denytest1(int dummy)
 			if (cli_read(cli1, fnum2, (char *)&x, 0, 1) == 1) {
 				res += A_R;
 			}
-			if (cli_write(cli1, fnum2, 0, (char *)&x, 0, 1) == 1) {
+			if (NT_STATUS_IS_OK(cli_writeall(cli1, fnum2, 0,
+							 (uint8_t *)&x, 0, 1,
+							 NULL))) {
 				res += A_W;
 			}
 		}
@@ -1481,7 +1485,7 @@ bool torture_denytest1(int dummy)
 	}
 
 	for (i=0;i<2;i++) {
-		cli_unlink(cli1, fnames[i], aSYSTEM | aHIDDEN);
+		cli_unlink(cli1, fnames[i], FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	}
 		
 	if (!torture_close_connection(cli1)) {
@@ -1512,9 +1516,10 @@ bool torture_denytest2(int dummy)
 	printf("starting denytest2\n");
 
 	for (i=0;i<2;i++) {
-		cli_unlink(cli1, fnames[i], aSYSTEM | aHIDDEN);
+		cli_unlink(cli1, fnames[i], FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 		cli_open(cli1, fnames[i], O_RDWR|O_CREAT, DENY_NONE, &fnum1);
-		cli_write(cli1, fnum1, 0, fnames[i], 0, strlen(fnames[i]));
+		cli_writeall(cli1, fnum1, 0, (const uint8_t *)fnames[i], 0,
+			     strlen(fnames[i]), NULL);
 		cli_close(cli1, fnum1);
 	}
 
@@ -1541,7 +1546,9 @@ bool torture_denytest2(int dummy)
 			if (cli_read(cli2, fnum2, (char *)&x, 0, 1) == 1) {
 				res += A_R;
 			}
-			if (cli_write(cli2, fnum2, 0, (char *)&x, 0, 1) == 1) {
+			if (NT_STATUS_IS_OK(cli_writeall(cli2, fnum2, 0,
+							 (uint8_t *)&x, 0, 1,
+							 NULL))) {
 				res += A_W;
 			}
 		}
@@ -1570,7 +1577,7 @@ bool torture_denytest2(int dummy)
 	}
 		
 	for (i=0;i<2;i++) {
-		cli_unlink(cli1, fnames[i], aSYSTEM | aHIDDEN);
+		cli_unlink(cli1, fnames[i], FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	}
 
 	if (!torture_close_connection(cli1)) {
diff --git a/source3/torture/locktest.c b/source3/torture/locktest.c
index d52a8de4605..d27eb5fa548 100644
--- a/source3/torture/locktest.c
+++ b/source3/torture/locktest.c
@@ -18,6 +18,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "system/filesys.h"
 #include "locking/proto.h"
 #include "libsmb/nmblib.h"
@@ -163,10 +164,8 @@ return a connection to a server
 static struct cli_state *connect_one(char *share, int snum)
 {
 	struct cli_state *c;
-	struct nmb_name called, calling;
 	char *server_n;
 	fstring server;
-	struct sockaddr_storage ss;
 	fstring myname;
 	static int count;
 	NTSTATUS status;
@@ -179,42 +178,20 @@ static struct cli_state *connect_one(char *share, int snum)
 
 	server_n = server;
 
-	zero_sockaddr(&ss);
-
 	slprintf(myname,sizeof(myname), "lock-%lu-%u", (unsigned long)getpid(), count++);
 
-	make_nmb_name(&calling, myname, 0x0);
-	make_nmb_name(&called , server, 0x20);
-
- again:
-        zero_sockaddr(&ss);
-
 	/* have to open a new connection */
-	if (!(c=cli_initialise())) {
-		DEBUG(0,("Connection to %s failed\n", server_n));
-		return NULL;
-	}
 
-	status = cli_connect(c, server_n, &ss);
+	status = cli_connect_nb(server_n, NULL, 0, 0x20, myname, Undefined,
+				&c);
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(0,("Connection to %s failed. Error %s\n", server_n, nt_errstr(status) ));
+		DEBUG(0, ("Connection to %s failed. Error %s\n", server_n,
+			  nt_errstr(status)));
 		return NULL;
 	}
 
 	c->use_kerberos = use_kerberos;
 
-	if (!cli_session_request(c, &calling, &called)) {
-		DEBUG(0,("session request to %s failed\n", called.name));
-		cli_shutdown(c);
-		if (strcmp(called.name, "*SMBSERVER")) {
-			make_nmb_name(&called , "*SMBSERVER", 0x20);
-			goto again;
-		}
-		return NULL;
-	}
-
-	DEBUG(4,(" session request ok\n"));
-
 	status = cli_negprot(c);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0, ("protocol negotiation failed: %s\n",
@@ -232,8 +209,8 @@ static struct cli_state *connect_one(char *share, int snum)
 	}
 
 	if (got_pass == 1) {
-		fstrcpy(password[1], password[0]);
-		fstrcpy(username[1], username[0]);
+		strlcpy(password[1], password[0],sizeof(password[1]));
+		strlcpy(username[1], username[0],sizeof(username[1]));
 	}
 
 	status = cli_session_setup(c, username[snum],
@@ -400,7 +377,7 @@ static void close_files(struct cli_state *cli[NSERVERS][NCONNECTIONS],
 		}
 	}
 	for (server=0;server<NSERVERS;server++) {
-		cli_unlink(cli[server][0], FILENAME, aSYSTEM | aHIDDEN);
+		cli_unlink(cli[server][0], FILENAME, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	}
 }
 
diff --git a/source3/torture/locktest2.c b/source3/torture/locktest2.c
index 55dc1d52b78..aba6d2ff134 100644
--- a/source3/torture/locktest2.c
+++ b/source3/torture/locktest2.c
@@ -18,6 +18,7 @@
 */
 
 #include "includes.h"
+#include "libsmb/libsmb.h"
 #include "system/filesys.h"
 #include "locking/proto.h"
 
@@ -325,7 +326,7 @@ static void close_files(struct cli_state *cli[NSERVERS][NCONNECTIONS],
 		}
 	}
 	for (server=0;server<NSERVERS;server++) {
-		cli_unlink(cli[server][0], FILENAME, aSYSTEM | aHIDDEN);
+		cli_unlink(cli[server][0], FILENAME, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	}
 }
 
diff --git a/source3/torture/mangle_test.c b/source3/torture/mangle_test.c
index dd684671409..4fc91dc4b3a 100644
--- a/source3/torture/mangle_test.c
+++ b/source3/torture/mangle_test.c
@@ -20,7 +20,9 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "torture/proto.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
+#include "util_tdb.h"
 
 extern int torture_numops;
 
@@ -40,46 +42,53 @@ static bool test_one(struct cli_state *cli, const char *name)
 
 	total++;
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, name, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum))) {
-		printf("open of %s failed (%s)\n", name, cli_errstr(cli));
+	status = cli_open(cli, name, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", name, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum))) {
-		printf("close of %s failed (%s)\n", name, cli_errstr(cli));
+	status = cli_close(cli, fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close of %s failed (%s)\n", name, nt_errstr(status));
 		return False;
 	}
 
 	/* get the short name */
 	status = cli_qpathinfo_alt_name(cli, name, shortname);
 	if (!NT_STATUS_IS_OK(status)) {
-		printf("query altname of %s failed (%s)\n", name, cli_errstr(cli));
+		printf("query altname of %s failed (%s)\n", name, nt_errstr(status));
 		return False;
 	}
 
 	fstr_sprintf(name2, "\\mangle_test\\%s", shortname);
-	if (!NT_STATUS_IS_OK(cli_unlink(cli, name2, aSYSTEM | aHIDDEN))) {
+	status = cli_unlink(cli, name2, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
 		printf("unlink of %s  (%s) failed (%s)\n", 
-		       name2, name, cli_errstr(cli));
+		       name2, name, nt_errstr(status));
 		return False;
 	}
 
 	/* recreate by short name */
-	if (!NT_STATUS_IS_OK(cli_open(cli, name2, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum))) {
-		printf("open2 of %s failed (%s)\n", name2, cli_errstr(cli));
+	status = cli_open(cli, name2, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open2 of %s failed (%s)\n", name2, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum))) {
-		printf("close of %s failed (%s)\n", name, cli_errstr(cli));
+
+	status = cli_close(cli, fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close of %s failed (%s)\n", name, nt_errstr(status));
 		return False;
 	}
 
 	/* and unlink by long name */
-	if (!NT_STATUS_IS_OK(cli_unlink(cli, name, aSYSTEM | aHIDDEN))) {
+	status = cli_unlink(cli, name, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
 		printf("unlink2 of %s  (%s) failed (%s)\n", 
-		       name, name2, cli_errstr(cli));
+		       name, name2, nt_errstr(status));
 		failures++;
-		cli_unlink(cli, name2, aSYSTEM | aHIDDEN);
+		cli_unlink(cli, name2, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 		return True;
 	}
 
@@ -98,7 +107,7 @@ static bool test_one(struct cli_state *cli, const char *name)
 	} else {
 		TDB_DATA namedata;
 		/* store it for later */
-		namedata.dptr = CONST_DISCARD(uint8 *, name);
+		namedata.dptr = discard_const_p(uint8, name);
 		namedata.dsize = strlen(name)+1;
 		tdb_store_bystring(tdb, shortname, namedata, TDB_REPLACE);
 	}
@@ -172,13 +181,13 @@ bool torture_mangle(int dummy)
 	}
 
 	/* we will use an internal tdb to store the names we have used */
-	tdb = tdb_open(NULL, 100000, TDB_INTERNAL, 0, 0);
+	tdb = tdb_open_compat(NULL, 100000, TDB_INTERNAL, 0, 0, NULL, NULL);
 	if (!tdb) {
 		printf("ERROR: Failed to open tdb\n");
 		return False;
 	}
 
-	cli_unlink(cli, "\\mangle_test\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\mangle_test\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, "\\mangle_test");
 
 	if (!NT_STATUS_IS_OK(cli_mkdir(cli, "\\mangle_test"))) {
@@ -202,7 +211,7 @@ bool torture_mangle(int dummy)
 		}
 	}
 
-	cli_unlink(cli, "\\mangle_test\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\mangle_test\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	if (!NT_STATUS_IS_OK(cli_rmdir(cli, "\\mangle_test"))) {
 		printf("ERROR: Failed to remove directory\n");
 		return False;
diff --git a/source3/torture/masktest.c b/source3/torture/masktest.c
index c8f346435af..9bb34dfe459 100644
--- a/source3/torture/masktest.c
+++ b/source3/torture/masktest.c
@@ -2,17 +2,17 @@
    Unix SMB/CIFS implementation.
    mask_match tester
    Copyright (C) Andrew Tridgell 1999
-   
+
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "trans2.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/nmblib.h"
 
 static fstring password;
@@ -166,10 +167,8 @@ return a connection to a server
 static struct cli_state *connect_one(char *share)
 {
 	struct cli_state *c;
-	struct nmb_name called, calling;
 	char *server_n;
 	char *server;
-	struct sockaddr_storage ss;
 	NTSTATUS status;
 
 	server = share+2;
@@ -180,40 +179,16 @@ static struct cli_state *connect_one(char *share)
 
 	server_n = server;
 
-	zero_sockaddr(&ss);
-
-	make_nmb_name(&calling, "masktest", 0x0);
-	make_nmb_name(&called , server, 0x20);
-
- again:
-        zero_sockaddr(&ss);
-
-	/* have to open a new connection */
-	if (!(c=cli_initialise())) {
-		DEBUG(0,("Connection to %s failed\n", server_n));
-		return NULL;
-	}
-
-	status = cli_connect(c, server_n, &ss);
+	status = cli_connect_nb(server, NULL, 0, 0x20, "masktest", Undefined,
+				&c);
 	if (!NT_STATUS_IS_OK(status)) {
-		DEBUG(0,("Connection to %s failed. Error %s\n", server_n, nt_errstr(status) ));
+		DEBUG(0,("Connection to %s failed. Error %s\n", server_n,
+			 nt_errstr(status)));
 		return NULL;
 	}
 
 	c->protocol = max_protocol;
 
-	if (!cli_session_request(c, &calling, &called)) {
-		DEBUG(0,("session request to %s failed\n", called.name));
-		cli_shutdown(c);
-		if (strcmp(called.name, "*SMBSERVER")) {
-			make_nmb_name(&called , "*SMBSERVER", 0x20);
-			goto again;
-		}
-		return NULL;
-	}
-
-	DEBUG(4,(" session request ok\n"));
-
 	status = cli_negprot(c);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0, ("protocol negotiation failed: %s\n",
@@ -313,11 +288,11 @@ static void get_real_name(struct cli_state *cli,
 	*pp_long_name = NULL;
 	/* nasty hack to force level 260 listings - tridge */
 	if (max_protocol <= PROTOCOL_LANMAN1) {
-		cli_list_trans(cli, "\\masktest\\*.*", aHIDDEN | aDIR,
+		cli_list_trans(cli, "\\masktest\\*.*", FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY,
 			       SMB_FIND_FILE_BOTH_DIRECTORY_INFO, listfn,
 			       &state);
 	} else {
-		cli_list_trans(cli, "\\masktest\\*", aHIDDEN | aDIR,
+		cli_list_trans(cli, "\\masktest\\*", FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY,
 			       SMB_FIND_FILE_BOTH_DIRECTORY_INFO,
 			       listfn, &state);
 	}
@@ -359,7 +334,7 @@ static void testpair(struct cli_state *cli, const char *mask, const char *file)
 		return;
 	}
 	fstrcpy(res1, "---");
-	cli_list(cli, mask, aHIDDEN | aDIR, listfn, NULL);
+	cli_list(cli, mask, FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY, listfn, NULL);
 
 	res2 = reg_test(cli, mask, long_name, short_name);
 
@@ -371,7 +346,7 @@ static void testpair(struct cli_state *cli, const char *mask, const char *file)
 		if (die_on_error) exit(1);
 	}
 
-	cli_unlink(cli, file, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, file, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (count % 100 == 0) DEBUG(0,("%d\n", count));
 	SAFE_FREE(long_name);
@@ -388,7 +363,7 @@ static void test_mask(int argc, char *argv[],
 
 	cli_mkdir(cli, "\\masktest");
 
-	cli_unlink(cli, "\\masktest\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\masktest\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (argc >= 2) {
 		while (argc >= 2) {
@@ -411,8 +386,8 @@ static void test_mask(int argc, char *argv[],
 	while (1) {
 		l1 = 1 + random() % 20;
 		l2 = 1 + random() % 20;
-		mask = TALLOC_ARRAY(ctx, char, strlen("\\masktest\\")+1+22);
-		file = TALLOC_ARRAY(ctx, char, strlen("\\masktest\\")+1+22);
+		mask = talloc_array(ctx, char, strlen("\\masktest\\")+1+22);
+		file = talloc_array(ctx, char, strlen("\\masktest\\")+1+22);
 		if (!mask || !file) {
 			goto finished;
 		}
diff --git a/source3/torture/msgtest.c b/source3/torture/msgtest.c
index 607d4c3bb1c..c7fa3153d8a 100644
--- a/source3/torture/msgtest.c
+++ b/source3/torture/msgtest.c
@@ -85,7 +85,7 @@ static void pong_message(struct messaging_context *msg_ctx,
 	/* Now test that the duplicate filtering code works. */
 	pong_count = 0;
 
-	safe_strcpy(buf, "1234567890", sizeof(buf)-1);
+	strlcpy(buf, "1234567890", sizeof(buf));
 
 	for (i=0;i<n;i++) {
 		messaging_send(msg_ctx, messaging_server_id(msg_ctx), MSG_PING,
diff --git a/source3/torture/nbench.c b/source3/torture/nbench.c
index 182f6f7cd91..d2fddf00945 100644
--- a/source3/torture/nbench.c
+++ b/source3/torture/nbench.c
@@ -19,7 +19,9 @@
 
 #include "includes.h"
 #include "torture/proto.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
+#include "../lib/util/tevent_ntstatus.h"
 
 static long long int ival(const char *str)
 {
@@ -94,7 +96,7 @@ static struct nbench_cmd_struct *nbench_parse(TALLOC_CTX *mem_ctx,
 	char *cmd;
 	char *status;
 
-	result = TALLOC_P(mem_ctx, struct nbench_cmd_struct);
+	result = talloc(mem_ctx, struct nbench_cmd_struct);
 	if (result == NULL) {
 		return NULL;
 	}
diff --git a/source3/torture/nbio.c b/source3/torture/nbio.c
index 664943b681b..0da6bc084e2 100644
--- a/source3/torture/nbio.c
+++ b/source3/torture/nbio.c
@@ -22,6 +22,7 @@
 #include "includes.h"
 #include "torture/proto.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 
 #define MAX_FILES 1000
@@ -134,10 +135,13 @@ void nb_setup(struct cli_state *cli)
 
 void nb_unlink(const char *fname)
 {
-	if (!NT_STATUS_IS_OK(cli_unlink(c, fname, aSYSTEM | aHIDDEN))) {
+	NTSTATUS status;
+
+	status = cli_unlink(c, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
 #if NBDEBUG
 		printf("(%d) unlink %s failed (%s)\n", 
-		       line_count, fname, cli_errstr(c));
+		       line_count, fname, nt_errstr(status));
 #endif
 	}
 }
@@ -165,7 +169,7 @@ void nb_createx(const char *fname,
 				create_options, 0, &fd);
 	if (!NT_STATUS_IS_OK(status) && handle != -1) {
 		printf("ERROR: cli_ntcreate failed for %s - %s\n",
-		       fname, cli_errstr(c));
+		       fname, nt_errstr(status));
 		exit(1);
 	}
 	if (NT_STATUS_IS_OK(status) && handle == -1) {
@@ -189,13 +193,17 @@ void nb_createx(const char *fname,
 void nb_writex(int handle, int offset, int size, int ret_size)
 {
 	int i;
+	NTSTATUS status;
 
 	if (buf[0] == 0) memset(buf, 1, sizeof(buf));
 
 	i = find_handle(handle);
-	if (cli_write(c, ftable[i].fd, 0, buf, offset, size) != ret_size) {
-		printf("(%d) ERROR: write failed on handle %d, fd %d \
-errno %d (%s)\n", line_count, handle, ftable[i].fd, errno, strerror(errno));
+	status = cli_writeall(c, ftable[i].fd, 0, (uint8_t *)buf, offset, size,
+			      NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("(%d) ERROR: write failed on handle %d, fd %d "
+		       "error %s\n", line_count, handle, ftable[i].fd,
+		       nt_errstr(status));
 		exit(1);
 	}
 
@@ -228,18 +236,24 @@ void nb_close(int handle)
 
 void nb_rmdir(const char *fname)
 {
-	if (!NT_STATUS_IS_OK(cli_rmdir(c, fname))) {
+	NTSTATUS status;
+
+	status = cli_rmdir(c, fname);
+	if (!NT_STATUS_IS_OK(status)) {
 		printf("ERROR: rmdir %s failed (%s)\n", 
-		       fname, cli_errstr(c));
+		       fname, nt_errstr(status));
 		exit(1);
 	}
 }
 
 void nb_rename(const char *oldname, const char *newname)
 {
-	if (!NT_STATUS_IS_OK(cli_rename(c, oldname, newname))) {
+	NTSTATUS status;
+
+	status = cli_rename(c, oldname, newname);
+	if (!NT_STATUS_IS_OK(status)) {
 		printf("ERROR: rename %s %s failed (%s)\n", 
-		       oldname, newname, cli_errstr(c));
+		       oldname, newname, nt_errstr(status));
 		exit(1);
 	}
 }
@@ -301,13 +315,13 @@ static NTSTATUS delete_fn(const char *mnt, struct file_info *finfo,
 		printf("asprintf failed\n");
 		return NT_STATUS_NO_MEMORY;
 	}
-	if (finfo->mode & aDIR) {
+	if (finfo->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		char *s2;
 		if (asprintf(&s2, "%s\\*", s) == -1) {
 			printf("asprintf failed\n");
 			return NT_STATUS_NO_MEMORY;
 		}
-		status = cli_list(c, s2, aDIR, delete_fn, NULL);
+		status = cli_list(c, s2, FILE_ATTRIBUTE_DIRECTORY, delete_fn, NULL);
 		if (!NT_STATUS_IS_OK(status)) {
 			free(n);
 			free(s2);
@@ -332,7 +346,7 @@ void nb_deltree(const char *dname)
 	}
 
 	total_deleted = 0;
-	cli_list(c, mask, aDIR, delete_fn, NULL);
+	cli_list(c, mask, FILE_ATTRIBUTE_DIRECTORY, delete_fn, NULL);
 	free(mask);
 	cli_rmdir(c, dname);
 
diff --git a/source3/torture/pdbtest.c b/source3/torture/pdbtest.c
index 7ab20c76926..33419db2753 100644
--- a/source3/torture/pdbtest.c
+++ b/source3/torture/pdbtest.c
@@ -24,6 +24,14 @@
 #include "popt_common.h"
 #include "passdb.h"
 
+#include "../librpc/gen_ndr/drsblobs.h"
+#include "../librpc/gen_ndr/ndr_drsblobs.h"
+#include "../libcli/security/dom_sid.h"
+
+#define TRUST_DOM "trustdom"
+#define TRUST_PWD "trustpwd1232"
+#define TRUST_SID "S-1-5-21-1111111111-2222222222-3333333333"
+
 static bool samu_correct(struct samu *s1, struct samu *s2)
 {
 	bool ret = True;
@@ -366,6 +374,81 @@ int main(int argc, char **argv)
 					get_friendly_nt_error_msg(rv));
 	}
 
+	/* test trustdom calls */
+	struct pdb_trusted_domain *td;
+	struct pdb_trusted_domain *new_td;
+	struct trustAuthInOutBlob taiob;
+	struct AuthenticationInformation aia;
+	enum ndr_err_code ndr_err;
+
+	td = talloc_zero(ctx ,struct pdb_trusted_domain);
+	if (!td) {
+		fprintf(stderr, "talloc failed\n");
+		exit(1);
+	}
+
+	td->domain_name = talloc_strdup(td, TRUST_DOM);
+	td->netbios_name = talloc_strdup(td, TRUST_DOM);
+	if (!td->domain_name || !td->netbios_name) {
+		fprintf(stderr, "talloc failed\n");
+		exit(1);
+	}
+
+	td->trust_auth_incoming = data_blob_null;
+
+	ZERO_STRUCT(taiob);
+	ZERO_STRUCT(aia);
+	taiob.count = 1;
+	taiob.current.count = 1;
+	taiob.current.array = &aia;
+	unix_to_nt_time(&aia.LastUpdateTime, time(NULL));
+	aia.AuthType = TRUST_AUTH_TYPE_CLEAR;
+	aia.AuthInfo.clear.password = (uint8_t *) talloc_strdup(ctx, TRUST_PWD);
+	aia.AuthInfo.clear.size = strlen(TRUST_PWD);
+
+	taiob.previous.count = 0;
+	taiob.previous.array = NULL;
+
+	ndr_err = ndr_push_struct_blob(&td->trust_auth_outgoing,
+					td, &taiob,
+			(ndr_push_flags_fn_t) ndr_push_trustAuthInOutBlob);
+	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+		fprintf(stderr, "ndr_push_struct_blob failed.\n");
+		exit(1);
+	}
+
+	td->trust_direction = LSA_TRUST_DIRECTION_OUTBOUND;
+	td->trust_type = LSA_TRUST_TYPE_DOWNLEVEL;
+	td->trust_attributes = 0;
+	td->trust_forest_trust_info = data_blob_null;
+
+	rv = pdb->set_trusted_domain(pdb, TRUST_DOM, td);
+	if (!NT_STATUS_IS_OK(rv)) {
+		fprintf(stderr, "Error in set_trusted_domain %s\n",
+				get_friendly_nt_error_msg(rv));
+		error = True;
+	}
+
+	rv = pdb->get_trusted_domain(pdb, ctx, TRUST_DOM, &new_td);
+	if (!NT_STATUS_IS_OK(rv)) {
+		fprintf(stderr, "Error in set_trusted_domain %s\n",
+				get_friendly_nt_error_msg(rv));
+		error = True;
+	}
+
+	if (!strequal(td->domain_name, new_td->domain_name) ||
+	    !strequal(td->netbios_name, new_td->netbios_name) ||
+	    !sid_equal(&td->security_identifier, &new_td->security_identifier) ||
+	    td->trust_direction != new_td->trust_direction ||
+	    td->trust_type != new_td->trust_type ||
+	    td->trust_attributes != new_td->trust_attributes ||
+	    td->trust_auth_incoming.length != new_td->trust_auth_incoming.length ||
+	    td->trust_forest_trust_info.length != new_td->trust_forest_trust_info.length ||
+	    data_blob_cmp(&td->trust_auth_outgoing, &new_td->trust_auth_outgoing) != 0) {
+		fprintf(stderr, "Old and new trusdet domain data do not match\n");
+		error = True;
+	}
+
 	TALLOC_FREE(ctx);
 
 	if (error) {
diff --git a/source3/torture/proto.h b/source3/torture/proto.h
index a45aa153c43..6a47a18cd7c 100644
--- a/source3/torture/proto.h
+++ b/source3/torture/proto.h
@@ -23,6 +23,8 @@
 #ifndef __TORTURE_H__
 #define __TORTURE_H__
 
+struct cli_state;
+
 /* The following definitions come from torture/denytest.c  */
 
 bool torture_denytest1(int dummy);
@@ -87,5 +89,7 @@ bool run_nbench2(int dummy);
 bool run_async_echo(int dummy);
 bool run_smb_any_connect(int dummy);
 bool run_addrchange(int dummy);
+bool run_notify_online(int dummy);
+bool run_nttrans_create(int dummy);
 
 #endif /* __TORTURE_H__ */
diff --git a/source3/torture/scanner.c b/source3/torture/scanner.c
index c7d3ebc2913..1ca4ca45956 100644
--- a/source3/torture/scanner.c
+++ b/source3/torture/scanner.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "torture/proto.h"
+#include "libsmb/libsmb.h"
 
 #define VERBOSE 0
 #define OP_MIN 0
@@ -172,7 +173,7 @@ static bool scan_trans2(struct cli_state *cli, int op, int level,
 
 	status = try_trans2_len(cli, "newfile", op, level, param, data, param_len, &data_len, 
 				&rparam_len, &rdata_len);
-	cli_unlink(cli, "\\newfile.dat", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\newfile.dat", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, "\\newfile.dat");
 	if (NT_STATUS_IS_OK(status)) return True;
 
@@ -379,7 +380,7 @@ static bool scan_nttrans(struct cli_state *cli, int op, int level,
 
 	status = try_nttrans_len(cli, "newfile", op, level, param, data, param_len, &data_len, 
 				&rparam_len, &rdata_len);
-	cli_unlink(cli, "\\newfile.dat", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\newfile.dat", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, "\\newfile.dat");
 	if (NT_STATUS_IS_OK(status)) return True;
 
diff --git a/source3/torture/t_strcmp.c b/source3/torture/t_strcmp.c
index 318423b8be0..5be26cdc79f 100644
--- a/source3/torture/t_strcmp.c
+++ b/source3/torture/t_strcmp.c
@@ -1,7 +1,7 @@
 /*
  * Copyright (C) 2003 by Martin Pool
  *
- * Test harness for StrCaseCmp
+ * Test harness for strcasecmp_m
  */
 
 #include "includes.h"
@@ -16,7 +16,7 @@ int main(int argc, char *argv[])
 
 	if (argc < 3) {
 		fprintf(stderr, "usage: %s STRING1 STRING2 [ITERS]\n"
-			"Compares two strings, prints the results of StrCaseCmp\n",
+			"Compares two strings, prints the results of strcasecmp_m\n",
 			argv[0]);
 		return 2;
 	}
@@ -24,7 +24,7 @@ int main(int argc, char *argv[])
 		iters = atoi(argv[3]);
 
 	for (i = 0; i < iters; i++)
-		ret = StrCaseCmp(argv[1], argv[2]);
+		ret = strcasecmp_m(argv[1], argv[2]);
 
 	printf("%d\n", ret);
 	
diff --git a/source3/torture/test_async_echo.c b/source3/torture/test_async_echo.c
index e443d4f8a7d..d097f49ca5e 100644
--- a/source3/torture/test_async_echo.c
+++ b/source3/torture/test_async_echo.c
@@ -19,6 +19,7 @@
 
 #include "includes.h"
 #include "torture/proto.h"
+#include "libsmb/libsmb.h"
 #include "rpc_client/cli_pipe.h"
 #include "librpc/gen_ndr/ndr_echo_c.h"
 
diff --git a/source3/torture/test_case_insensitive.c b/source3/torture/test_case_insensitive.c
index df9d1498618..26b3d65d7ec 100644
--- a/source3/torture/test_case_insensitive.c
+++ b/source3/torture/test_case_insensitive.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "torture/proto.h"
 #include "system/filesys.h"
+#include "libsmb/libsmb.h"
 
 /*
  * Regression test file creates on case insensitive file systems (e.g. OS/X)
diff --git a/source3/torture/test_notify_online.c b/source3/torture/test_notify_online.c
new file mode 100644
index 00000000000..b9ebc00b54b
--- /dev/null
+++ b/source3/torture/test_notify_online.c
@@ -0,0 +1,294 @@
+/*
+   Unix SMB/CIFS implementation.
+   Make sure that for offline files pread and pwrite trigger a notify
+   Copyright (C) Volker Lendecke 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "torture/proto.h"
+#include "libcli/security/security.h"
+#include "lib/util/tevent_ntstatus.h"
+#include "libsmb/libsmb.h"
+
+extern char *test_filename;
+
+struct notify_online_state {
+	struct tevent_context *ev;
+	struct cli_state *cli;
+	uint16_t dnum;
+	const char *fname;
+	uint16_t fnum;
+	bool got_notify;
+};
+
+static void notify_online_opened_dir(struct tevent_req *subreq);
+static void notify_online_notify_callback(struct tevent_req *subreq);
+static void notify_online_opened_file(struct tevent_req *subreq);
+static void notify_online_sent_read(struct tevent_req *subreq);
+static void notify_online_sent_closefile(struct tevent_req *subreq);
+static void notify_online_waited(struct tevent_req *subreq);
+static void notify_online_sent_closedir(struct tevent_req *subreq);
+
+static struct tevent_req *notify_online_send(
+	TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+	struct cli_state *cli, const char *dname, const char *fname)
+{
+	struct tevent_req *req, *subreq;
+	struct notify_online_state *state;
+
+	req = tevent_req_create(mem_ctx, &state, struct notify_online_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->ev = ev;
+	state->cli = cli;
+	state->fname = fname;
+
+	subreq = cli_ntcreate_send(
+		state, ev, cli, dname, EXTENDED_RESPONSE_REQUIRED,
+		SEC_FILE_READ_DATA, 0,
+		FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+		FILE_OPEN, 0, 0);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, notify_online_opened_dir, req);
+	return req;
+}
+
+static void notify_online_opened_dir(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct notify_online_state *state = tevent_req_data(
+		req, struct notify_online_state);
+	NTSTATUS status;
+
+	status = cli_ntcreate_recv(subreq, &state->dnum);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	subreq = cli_notify_send(state, state->ev, state->cli, state->dnum,
+				 128, FILE_NOTIFY_CHANGE_ATTRIBUTES, false);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, notify_online_notify_callback, req);
+
+	subreq = cli_ntcreate_send(
+		state, state->ev, state->cli, state->fname, 0,
+		GENERIC_READ_ACCESS, FILE_ATTRIBUTE_NORMAL,
+		FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+		FILE_OPEN, FILE_NON_DIRECTORY_FILE, 0);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, notify_online_opened_file, req);
+}
+
+static void notify_online_notify_callback(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct notify_online_state *state = tevent_req_data(
+		req, struct notify_online_state);
+	NTSTATUS status;
+	uint32_t num_changes;
+	struct notify_change *changes;
+
+	status = cli_notify_recv(subreq, state, &num_changes, &changes);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	if ((num_changes == 1)
+	    && (changes[0].action == NOTIFY_ACTION_MODIFIED)
+	    && (strcmp(changes[0].name, state->fname) == 0)) {
+		state->got_notify = true;
+	}
+	tevent_req_done(req);
+}
+
+static void notify_online_opened_file(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct notify_online_state *state = tevent_req_data(
+		req, struct notify_online_state);
+	NTSTATUS status;
+
+	status = cli_ntcreate_recv(subreq, &state->fnum);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	subreq = cli_read_andx_send(
+		state, state->ev, state->cli, state->fnum, 0, 1);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, notify_online_sent_read, req);
+}
+
+static void notify_online_sent_read(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct notify_online_state *state = tevent_req_data(
+		req, struct notify_online_state);
+	NTSTATUS status;
+	ssize_t received;
+	uint8_t *buf;
+
+	status = cli_read_andx_recv(subreq, &received, &buf);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	subreq = cli_close_send(
+		state, state->ev, state->cli, state->fnum);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, notify_online_sent_closefile, req);
+}
+
+static void notify_online_sent_closefile(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct notify_online_state *state = tevent_req_data(
+		req, struct notify_online_state);
+	NTSTATUS status;
+
+	status = cli_close_recv(subreq);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	subreq = tevent_wakeup_send(
+		state, state->ev, timeval_current_ofs(10, 0));
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, notify_online_waited, req);
+}
+
+static void notify_online_waited(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct notify_online_state *state = tevent_req_data(
+		req, struct notify_online_state);
+
+	tevent_wakeup_recv(subreq);
+	TALLOC_FREE(subreq);
+	subreq = cli_close_send(
+		state, state->ev, state->cli, state->dnum);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, notify_online_sent_closedir, req);
+}
+
+static void notify_online_sent_closedir(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	NTSTATUS status;
+
+	status = cli_close_recv(subreq);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+}
+
+static NTSTATUS notify_online_recv(struct tevent_req *req, bool *got_notify)
+{
+	struct notify_online_state *state = tevent_req_data(
+		req, struct notify_online_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+	*got_notify = state->got_notify;
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS notify_online(struct cli_state *cli,
+			      const char *dirname, const char *filename,
+			      bool *got_notify)
+{
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct event_context *ev;
+	struct tevent_req *req;
+	NTSTATUS status = NT_STATUS_NO_MEMORY;
+
+	ev = event_context_init(frame);
+	if (ev == NULL) {
+		goto fail;
+	}
+	req = notify_online_send(frame, ev, cli, dirname, filename);
+	if (req == NULL) {
+		goto fail;
+	}
+	if (!tevent_req_poll_ntstatus(req, ev, &status)) {
+		goto fail;
+	}
+	status = notify_online_recv(req, got_notify);
+ fail:
+	TALLOC_FREE(frame);
+	return status;
+}
+
+bool run_notify_online(int dummy)
+{
+	struct cli_state *cli;
+	NTSTATUS status;
+	char *p;
+	const char *dir;
+	const char *file;
+	bool got_notify = false;
+
+	printf("Starting NOTIFY_ONLINE\n");
+
+	if (test_filename == NULL) {
+		fprintf(stderr, "<-f filename> missing\n");
+		return false;
+	}
+
+	if (!torture_open_connection(&cli, 0)) {
+		return false;
+	}
+
+	p = strrchr(test_filename, '/');
+	if (p != NULL) {
+		dir = SMB_STRNDUP(test_filename, p-test_filename);
+		file = SMB_STRDUP(p+1);
+	} else {
+		dir = "";
+		file = test_filename;
+	}
+
+	status = notify_online(cli, dir, file, &got_notify);
+	d_printf("notify_online returned %s (%d)\n", nt_errstr(status),
+		 (int)got_notify);
+	torture_close_connection(cli);
+	return NT_STATUS_IS_OK(status) && got_notify;
+}
diff --git a/source3/torture/test_nttrans_create.c b/source3/torture/test_nttrans_create.c
new file mode 100644
index 00000000000..9869f020e9b
--- /dev/null
+++ b/source3/torture/test_nttrans_create.c
@@ -0,0 +1,108 @@
+/*
+   Unix SMB/CIFS implementation.
+   Basic test for share secdescs vs nttrans_create
+   Copyright (C) Volker Lendecke 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "torture/proto.h"
+#include "libsmb/libsmb.h"
+#include "libcli/security/dom_sid.h"
+#include "libcli/security/secdesc.h"
+#include "libcli/security/security.h"
+
+bool run_nttrans_create(int dummy)
+{
+	struct cli_state *cli = NULL;
+	NTSTATUS status, status2;
+	bool ret = false;
+	struct security_ace ace;
+	struct security_acl acl;
+	struct security_descriptor *sd;
+	const char *fname = "transtest";
+	uint16_t fnum, fnum2;
+	struct dom_sid owner;
+
+	printf("Starting NTTRANS_CREATE\n");
+
+	if (!torture_open_connection(&cli, 0)) {
+		printf("torture_open_connection failed\n");
+		goto fail;
+	}
+
+	ZERO_STRUCT(ace);
+	ace.type = SEC_ACE_TYPE_ACCESS_ALLOWED;
+	ace.access_mask = SEC_RIGHTS_FILE_ALL & ~SEC_STD_WRITE_DAC;
+	sid_copy(&ace.trustee, &global_sid_World);
+
+	acl.revision = SECURITY_ACL_REVISION_NT4;
+	acl.size = 0;
+	acl.num_aces = 1;
+	acl.aces = &ace;
+
+	dom_sid_parse("S-1-22-1-1000", &owner);
+
+	sd = make_sec_desc(talloc_tos(),
+			   SECURITY_DESCRIPTOR_REVISION_1,
+			   SEC_DESC_SELF_RELATIVE|
+			   SEC_DESC_DACL_PRESENT|SEC_DESC_OWNER_DEFAULTED|
+			   SEC_DESC_GROUP_DEFAULTED,
+			   NULL, NULL, NULL, &acl, NULL);
+	if (sd == NULL) {
+		d_fprintf(stderr, "make_sec_desc failed\n");
+		goto fail;
+	}
+
+	status = cli_nttrans_create(
+		cli, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS|
+		READ_CONTROL_ACCESS,
+		FILE_ATTRIBUTE_NORMAL,
+		FILE_SHARE_READ|FILE_SHARE_WRITE| FILE_SHARE_DELETE,
+		FILE_CREATE, 0, 0, sd, NULL, 0, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		d_fprintf(stderr, "cli_nttrans_create returned %s\n",
+			  nt_errstr(status));
+		goto fail;
+	}
+
+	cli_query_secdesc(cli, fnum, talloc_tos());
+
+	status2 = cli_ntcreate(cli, fname, 0, WRITE_DAC_ACCESS,
+			       FILE_ATTRIBUTE_NORMAL,
+			       FILE_SHARE_READ|FILE_SHARE_WRITE|
+			       FILE_SHARE_DELETE,
+			       FILE_OPEN, 0, 0, &fnum2);
+
+	status = cli_nt_delete_on_close(cli, fnum, true);
+	if (!NT_STATUS_IS_OK(status)) {
+		d_fprintf(stderr, "cli_nt_delete_on_close returned %s\n",
+			  nt_errstr(status));
+		goto fail;
+	}
+
+	if (!NT_STATUS_EQUAL(status2, NT_STATUS_ACCESS_DENIED)) {
+		d_fprintf(stderr, "cli_ntcreate returned %s\n",
+			  nt_errstr(status));
+		goto fail;
+	}
+
+	ret = true;
+fail:
+	if (cli != NULL) {
+		torture_close_connection(cli);
+	}
+	return ret;
+}
diff --git a/source3/torture/test_posix_append.c b/source3/torture/test_posix_append.c
index 6c8317e5aad..2f562633e50 100644
--- a/source3/torture/test_posix_append.c
+++ b/source3/torture/test_posix_append.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "torture/proto.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 
 /*
  * Make sure that GENERIC_WRITE does not trigger append. See
@@ -32,9 +33,8 @@ bool run_posix_append(int dummy)
 	const char *fname = "append";
 	NTSTATUS status;
 	uint16_t fnum;
-	ssize_t written;
 	SMB_OFF_T size;
-	char c = '\0';
+	uint8_t c = '\0';
 	bool ret = false;
 
 	printf("Starting POSIX_APPEND\n");
@@ -69,14 +69,14 @@ bool run_posix_append(int dummy)
 	 * with a file of 2 byte length.
 	 */
 
-	written = cli_write(cli, fnum, 0, &c, 0, sizeof(c));
-	if (written != sizeof(c)) {
-		printf("cli_write failed\n");
+	status = cli_writeall(cli, fnum, 0, &c, 0, sizeof(c), NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("cli_write failed: %s\n", nt_errstr(status));
 		goto fail;
 	}
-	written = cli_write(cli, fnum, 0, &c, 0, sizeof(c));
-	if (written != sizeof(c)) {
-		printf("cli_write failed\n");
+	status = cli_writeall(cli, fnum, 0, &c, 0, sizeof(c), NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("cli_write failed: %s\n", nt_errstr(status));
 		goto fail;
 	}
 
diff --git a/source3/torture/test_smbsock_any_connect.c b/source3/torture/test_smbsock_any_connect.c
index 28a99815e77..a964e0fd831 100644
--- a/source3/torture/test_smbsock_any_connect.c
+++ b/source3/torture/test_smbsock_any_connect.c
@@ -35,7 +35,7 @@ bool run_smb_any_connect(int dummy)
 	interpret_string_addr(&addrs[4], "192.168.99.9", 0);
 
 	status = smbsock_any_connect(addrs, NULL, NULL, NULL, NULL,
-				     ARRAY_SIZE(addrs), 0,
+				     ARRAY_SIZE(addrs), 0, 0,
 				     &fd, &chosen_index, &port);
 
 	d_printf("smbsock_any_connect returned %s (fd %d)\n",
diff --git a/source3/torture/torture.c b/source3/torture/torture.c
index 6319d6194d7..faa4326489a 100644
--- a/source3/torture/torture.c
+++ b/source3/torture/torture.c
@@ -31,9 +31,13 @@
 #include "dbwrap.h"
 #include "talloc_dict.h"
 #include "async_smb.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "trans2.h"
 #include "libsmb/nmblib.h"
+#include "../lib/util/tevent_ntstatus.h"
+#include "util_tdb.h"
+#include "libsmb/read_smb.h"
 
 extern char *optarg;
 extern int optind;
@@ -57,6 +61,7 @@ static bool use_multishare_conn = False;
 static bool do_encrypt;
 static const char *local_path = NULL;
 static int signing_state = Undefined;
+char *test_filename;
 
 bool torture_showall = False;
 
@@ -173,24 +178,11 @@ static bool force_cli_encryption(struct cli_state *c,
 
 static struct cli_state *open_nbt_connection(void)
 {
-	struct nmb_name called, calling;
-	struct sockaddr_storage ss;
 	struct cli_state *c;
 	NTSTATUS status;
 
-	make_nmb_name(&calling, myname, 0x0);
-	make_nmb_name(&called , host, 0x20);
-
-        zero_sockaddr(&ss);
-
-	if (!(c = cli_initialise_ex(signing_state))) {
-		printf("Failed initialize cli_struct to connect with %s\n", host);
-		return NULL;
-	}
-
-	c->port = port_to_use;
-
-	status = cli_connect(c, host, &ss);
+	status = cli_connect_nb(host, NULL, port_to_use, 0x20, myname,
+				signing_state, &c);
 	if (!NT_STATUS_IS_OK(status)) {
 		printf("Failed to connect with %s. Error %s\n", host, nt_errstr(status) );
 		return NULL;
@@ -202,27 +194,6 @@ static struct cli_state *open_nbt_connection(void)
 	if (use_oplocks) c->use_oplocks = True;
 	if (use_level_II_oplocks) c->use_level_II_oplocks = True;
 
-	if (!cli_session_request(c, &calling, &called)) {
-		/*
-		 * Well, that failed, try *SMBSERVER ...
-		 * However, we must reconnect as well ...
-		 */
-		status = cli_connect(c, host, &ss);
-		if (!NT_STATUS_IS_OK(status)) {
-			printf("Failed to connect with %s. Error %s\n", host, nt_errstr(status) );
-			return NULL;
-		}
-
-		make_nmb_name(&called, "*SMBSERVER", 0x20);
-		if (!cli_session_request(c, &calling, &called)) {
-			printf("%s rejected the session\n",host);
-			printf("We tried with a called name of %s & %s\n",
-				host, "*SMBSERVER");
-			cli_shutdown(c);
-			return NULL;
-		}
-	}
-
 	return c;
 }
 
@@ -230,115 +201,94 @@ static struct cli_state *open_nbt_connection(void)
  Send a corrupt session request. See rfc1002.txt 4.3 and 4.3.2.
 ****************************************************************************/
 
-static bool cli_bad_session_request(struct cli_state *cli,
+static bool cli_bad_session_request(int fd,
                          struct nmb_name *calling, struct nmb_name *called)
 {
-        char *p;
-        int len = 4;
-        int namelen = 0;
-        char *tmp;
+	TALLOC_CTX *frame;
+	uint8_t len_buf[4];
+	struct iovec iov[3];
+	ssize_t len;
+	uint8_t *inbuf;
+	int err;
+	bool ret = false;
+	uint8_t message_type;
+	uint8_t error;
 
-        memcpy(&(cli->calling), calling, sizeof(*calling));
-        memcpy(&(cli->called ), called , sizeof(*called ));
+	frame = talloc_stackframe();
 
-        /* put in the destination name */
+	iov[0].iov_base = len_buf;
+	iov[0].iov_len  = sizeof(len_buf);
 
-        tmp = name_mangle(talloc_tos(), cli->called.name,
-                          cli->called.name_type);
-        if (tmp == NULL) {
-                return false;
-        }
+	/* put in the destination name */
 
-        p = cli->outbuf+len;
-        namelen = name_len((unsigned char *)tmp, talloc_get_size(tmp));
-        if (namelen > 0) {
-                memcpy(p, tmp, namelen);
-                len += namelen;
-        }
-        TALLOC_FREE(tmp);
-
-	/* Deliberately corrupt the name len (first byte) */
-	*p = 100;
+	iov[1].iov_base = name_mangle(talloc_tos(), called->name,
+				      called->name_type);
+	if (iov[1].iov_base == NULL) {
+		goto fail;
+	}
+	iov[1].iov_len = name_len((unsigned char *)iov[1].iov_base,
+				  talloc_get_size(iov[1].iov_base));
 
-        /* and my name */
+	/* and my name */
 
-        tmp = name_mangle(talloc_tos(), cli->calling.name,
-                          cli->calling.name_type);
-        if (tmp == NULL) {
-                return false;
-        }
+	iov[2].iov_base = name_mangle(talloc_tos(), calling->name,
+				      calling->name_type);
+	if (iov[2].iov_base == NULL) {
+		goto fail;
+	}
+	iov[2].iov_len = name_len((unsigned char *)iov[2].iov_base,
+				  talloc_get_size(iov[2].iov_base));
 
-        p = cli->outbuf+len;
-        namelen = name_len((unsigned char *)tmp, talloc_get_size(tmp));
-        if (namelen > 0) {
-                memcpy(p, tmp, namelen);
-                len += namelen;
-        }
-        TALLOC_FREE(tmp);
 	/* Deliberately corrupt the name len (first byte) */
-	*p = 100;
+	*((uint8_t *)iov[2].iov_base) = 100;
 
-        /* send a session request (RFC 1002) */
-        /* setup the packet length
+	/* send a session request (RFC 1002) */
+	/* setup the packet length
          * Remove four bytes from the length count, since the length
          * field in the NBT Session Service header counts the number
          * of bytes which follow.  The cli_send_smb() function knows
          * about this and accounts for those four bytes.
          * CRH.
          */
-        len -= 4;
-        _smb_setlen(cli->outbuf,len);
-        SCVAL(cli->outbuf,0,0x81);
-
-        cli_send_smb(cli);
-        DEBUG(5,("Sent session request\n"));
-
-        if (!cli_receive_smb(cli))
-                return False;
-
-        if (CVAL(cli->inbuf,0) != 0x82) {
-                /* This is the wrong place to put the error... JRA. */
-                cli->rap_error = CVAL(cli->inbuf,4);
-                return False;
-        }
-        return(True);
-}
-
-static struct cli_state *open_bad_nbt_connection(void)
-{
-	struct nmb_name called, calling;
-	struct sockaddr_storage ss;
-	struct cli_state *c;
-	NTSTATUS status;
-
-	make_nmb_name(&calling, myname, 0x0);
-	make_nmb_name(&called , host, 0x20);
 
-        zero_sockaddr(&ss);
+	_smb_setlen(len_buf, iov[1].iov_len + iov[2].iov_len);
+	SCVAL(len_buf,0,0x81);
 
-	if (!(c = cli_initialise_ex(signing_state))) {
-		printf("Failed initialize cli_struct to connect with %s\n", host);
-		return NULL;
+	len = write_data_iov(fd, iov, 3);
+	if (len == -1) {
+		goto fail;
 	}
-
-	c->port = 139;
-
-	status = cli_connect(c, host, &ss);
-	if (!NT_STATUS_IS_OK(status)) {
-		printf("Failed to connect with %s. Error %s\n", host, nt_errstr(status) );
-		return NULL;
+	len = read_smb(fd, talloc_tos(), &inbuf, &err);
+	if (len == -1) {
+		errno = err;
+		goto fail;
 	}
 
-	c->timeout = 4000; /* set a short timeout (4 seconds) */
+	message_type = CVAL(inbuf, 0);
+	if (message_type != 0x83) {
+		d_fprintf(stderr, "Expected msg type 0x83, got 0x%2.2x\n",
+			  message_type);
+		goto fail;
+        }
 
-	if (!cli_bad_session_request(c, &calling, &called)) {
-		printf("Failed to connect with %s. Error %s\n", host, nt_errstr(status) );
-		return NULL;
-	}
+	if (smb_len(inbuf) != 1) {
+		d_fprintf(stderr, "Expected smb_len 1, got %d\n",
+			  (int)smb_len(inbuf));
+		goto fail;
+        }
 
-	return c;
-}
+	error = CVAL(inbuf, 4);
+	if (error !=  0x82) {
+		d_fprintf(stderr, "Expected error 0x82, got %d\n",
+			  (int)error);
+		goto fail;
+        }
 
+	ret = true;
+fail:
+	TALLOC_FREE(frame);
+        return ret;
+}
 
 /* Insert a NULL at the first separator of the given path and return a pointer
  * to the remainder of the string.
@@ -572,12 +522,14 @@ static bool rw_torture(struct cli_state *c)
 		status = cli_open(c, lockfname, O_RDWR, DENY_NONE, &fnum2);
 	}
 	if (!NT_STATUS_IS_OK(status)) {
-		printf("open of %s failed (%s)\n", lockfname, cli_errstr(c));
+		printf("open of %s failed (%s)\n",
+		       lockfname, nt_errstr(status));
 		return False;
 	}
 
 	for (i=0;i<torture_numops;i++) {
 		unsigned n = (unsigned)sys_random()%10;
+
 		if (i % 10 == 0) {
 			printf("%d\r", i); fflush(stdout);
 		}
@@ -587,22 +539,28 @@ static bool rw_torture(struct cli_state *c)
 			return False;
 		}
 
-		if (!NT_STATUS_IS_OK(cli_open(c, fname, O_RDWR | O_CREAT | O_TRUNC, DENY_ALL, &fnum))) {
-			printf("open failed (%s)\n", cli_errstr(c));
+		status = cli_open(c, fname, O_RDWR | O_CREAT | O_TRUNC,
+                                  DENY_ALL, &fnum);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("open failed (%s)\n", nt_errstr(status));
 			correct = False;
 			break;
 		}
 
-		if (cli_write(c, fnum, 0, (char *)&pid, 0, sizeof(pid)) != sizeof(pid)) {
-			printf("write failed (%s)\n", cli_errstr(c));
+		status = cli_writeall(c, fnum, 0, (uint8_t *)&pid, 0,
+				      sizeof(pid), NULL);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("write failed (%s)\n", nt_errstr(status));
 			correct = False;
 		}
 
 		for (j=0;j<50;j++) {
-			if (cli_write(c, fnum, 0, (char *)buf, 
-				      sizeof(pid)+(j*sizeof(buf)), 
-				      sizeof(buf)) != sizeof(buf)) {
-				printf("write failed (%s)\n", cli_errstr(c));
+			status = cli_writeall(c, fnum, 0, (uint8_t *)buf,
+					      sizeof(pid)+(j*sizeof(buf)),
+					      sizeof(buf), NULL);
+			if (!NT_STATUS_IS_OK(status)) {
+				printf("write failed (%s)\n",
+				       nt_errstr(status));
 				correct = False;
 			}
 		}
@@ -619,24 +577,27 @@ static bool rw_torture(struct cli_state *c)
 			correct = False;
 		}
 
-		if (!NT_STATUS_IS_OK(cli_close(c, fnum))) {
-			printf("close failed (%s)\n", cli_errstr(c));
+		status = cli_close(c, fnum);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("close failed (%s)\n", nt_errstr(status));
 			correct = False;
 		}
 
-		if (!NT_STATUS_IS_OK(cli_unlink(c, fname, aSYSTEM | aHIDDEN))) {
-			printf("unlink failed (%s)\n", cli_errstr(c));
+		status = cli_unlink(c, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("unlink failed (%s)\n", nt_errstr(status));
 			correct = False;
 		}
 
-		if (!NT_STATUS_IS_OK(cli_unlock(c, fnum2, n*sizeof(int), sizeof(int)))) {
-			printf("unlock failed (%s)\n", cli_errstr(c));
+		status = cli_unlock(c, fnum2, n*sizeof(int), sizeof(int));
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("unlock failed (%s)\n", nt_errstr(status));
 			correct = False;
 		}
 	}
 
 	cli_close(c, fnum2);
-	cli_unlink(c, lockfname, aSYSTEM | aHIDDEN);
+	cli_unlink(c, lockfname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("%d\n", i);
 
@@ -681,14 +642,15 @@ static bool rw_torture3(struct cli_state *c, char *lockfname)
 
 	if (procnum == 0)
 	{
-		if (!NT_STATUS_IS_OK(cli_unlink(c, lockfname, aSYSTEM | aHIDDEN))) {
+		if (!NT_STATUS_IS_OK(cli_unlink(c, lockfname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN))) {
 			printf("unlink failed (%s) (normal, this file should not exist)\n", cli_errstr(c));
 		}
 
-		if (!NT_STATUS_IS_OK(cli_open(c, lockfname, O_RDWR | O_CREAT | O_EXCL, 
-				 DENY_NONE, &fnum))) {
+		status = cli_open(c, lockfname, O_RDWR | O_CREAT | O_EXCL,
+		                  DENY_NONE, &fnum);
+		if (!NT_STATUS_IS_OK(status)) {
 			printf("first open read/write of %s failed (%s)\n",
-					lockfname, cli_errstr(c));
+					lockfname, nt_errstr(status));
 			return False;
 		}
 	}
@@ -705,7 +667,7 @@ static bool rw_torture3(struct cli_state *c, char *lockfname)
 		}
 		if (!NT_STATUS_IS_OK(status)) {
 			printf("second open read-only of %s failed (%s)\n",
-					lockfname, cli_errstr(c));
+					lockfname, nt_errstr(status));
 			return False;
 		}
 	}
@@ -728,8 +690,11 @@ static bool rw_torture3(struct cli_state *c, char *lockfname)
 				sent = sizeof(buf) - count;
 			}
 
-			if (cli_write(c, fnum, 0, buf+count, count, (size_t)sent) != sent) {
-				printf("write failed (%s)\n", cli_errstr(c));
+			status = cli_writeall(c, fnum, 0, (uint8_t *)buf+count,
+					      count, (size_t)sent, NULL);
+			if (!NT_STATUS_IS_OK(status)) {
+				printf("write failed (%s)\n",
+				       nt_errstr(status));
 				correct = False;
 			}
 		}
@@ -759,8 +724,9 @@ static bool rw_torture3(struct cli_state *c, char *lockfname)
 
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(c, fnum))) {
-		printf("close failed (%s)\n", cli_errstr(c));
+	status = cli_close(c, fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -777,26 +743,30 @@ static bool rw_torture2(struct cli_state *c1, struct cli_state *c2)
 	char buf_rd[131072];
 	bool correct = True;
 	ssize_t bytes_read;
+	NTSTATUS status;
 
-	if (!NT_STATUS_IS_OK(cli_unlink(c1, lockfname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s) (normal, this file should not exist)\n", cli_errstr(c1));
+	status = cli_unlink(c1, lockfname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s) (normal, this file should not exist)\n", nt_errstr(status));
 	}
 
-	if (!NT_STATUS_IS_OK(cli_open(c1, lockfname, O_RDWR | O_CREAT | O_EXCL, 
-			 DENY_NONE, &fnum1))) {
+	status = cli_open(c1, lockfname, O_RDWR | O_CREAT | O_EXCL,
+	                  DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
 		printf("first open read/write of %s failed (%s)\n",
-				lockfname, cli_errstr(c1));
+				lockfname, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_open(c2, lockfname, O_RDONLY, 
-			 DENY_NONE, &fnum2))) {
+
+	status = cli_open(c2, lockfname, O_RDONLY, DENY_NONE, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
 		printf("second open read-only of %s failed (%s)\n",
-				lockfname, cli_errstr(c2));
+				lockfname, nt_errstr(status));
 		cli_close(c1, fnum1);
 		return False;
 	}
 
-	for (i=0;i<torture_numops;i++)
+	for (i = 0; i < torture_numops; i++)
 	{
 		size_t buf_size = ((unsigned)sys_random()%(sizeof(buf)-1))+ 1;
 		if (i % 10 == 0) {
@@ -805,8 +775,10 @@ static bool rw_torture2(struct cli_state *c1, struct cli_state *c2)
 
 		generate_random_buffer((unsigned char *)buf, buf_size);
 
-		if (cli_write(c1, fnum1, 0, buf, 0, buf_size) != buf_size) {
-			printf("write failed (%s)\n", cli_errstr(c1));
+		status = cli_writeall(c1, fnum1, 0, (uint8_t *)buf, 0,
+				      buf_size, NULL);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("write failed (%s)\n", nt_errstr(status));
 			correct = False;
 			break;
 		}
@@ -827,17 +799,21 @@ static bool rw_torture2(struct cli_state *c1, struct cli_state *c2)
 		}
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(c2, fnum2))) {
-		printf("close failed (%s)\n", cli_errstr(c2));
+	status = cli_close(c2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
-	if (!NT_STATUS_IS_OK(cli_close(c1, fnum1))) {
-		printf("close failed (%s)\n", cli_errstr(c1));
+
+	status = cli_close(c1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_unlink(c1, lockfname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(c1));
+	status = cli_unlink(c1, lockfname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -903,6 +879,7 @@ static bool run_readwritelarge_internal(int max_xmit_k)
 	SMB_OFF_T fsize;
 	char buf[126*1024];
 	bool correct = True;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli1, 0)) {
 		return False;
@@ -922,19 +899,21 @@ static bool run_readwritelarge_internal(int max_xmit_k)
 
 	printf("starting readwritelarge_internal\n");
 
-	cli_unlink(cli1, lockfname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, lockfname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, lockfname, O_RDWR | O_CREAT | O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open read/write of %s failed (%s)\n", lockfname, cli_errstr(cli1));
+	status = cli_open(cli1, lockfname, O_RDWR | O_CREAT | O_EXCL,
+	                  DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open read/write of %s failed (%s)\n", lockfname, nt_errstr(status));
 		return False;
 	}
 
-	cli_write(cli1, fnum1, 0, buf, 0, sizeof(buf));
+	cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 0, sizeof(buf), NULL);
 
-	if (!NT_STATUS_IS_OK(cli_qfileinfo_basic(
-				     cli1, fnum1, NULL, &fsize, NULL, NULL,
-				     NULL, NULL, NULL))) {
-		printf("qfileinfo failed (%s)\n", cli_errstr(cli1));
+	status = cli_qfileinfo_basic(cli1, fnum1, NULL, &fsize, NULL, NULL,
+				     NULL, NULL, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("qfileinfo failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -947,18 +926,22 @@ static bool run_readwritelarge_internal(int max_xmit_k)
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, lockfname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(cli1));
+	status = cli_unlink(cli1, lockfname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, lockfname, O_RDWR | O_CREAT | O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open read/write of %s failed (%s)\n", lockfname, cli_errstr(cli1));
+	status = cli_open(cli1, lockfname, O_RDWR | O_CREAT | O_EXCL,
+	                  DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open read/write of %s failed (%s)\n", lockfname, nt_errstr(status));
 		return False;
 	}
 
@@ -966,10 +949,10 @@ static bool run_readwritelarge_internal(int max_xmit_k)
 
 	cli_smbwrite(cli1, fnum1, buf, 0, sizeof(buf), NULL);
 
-	if (!NT_STATUS_IS_OK(cli_qfileinfo_basic(
-				     cli1, fnum1, NULL, &fsize, NULL, NULL,
-				     NULL, NULL, NULL))) {
-		printf("qfileinfo failed (%s)\n", cli_errstr(cli1));
+	status = cli_qfileinfo_basic(cli1, fnum1, NULL, &fsize, NULL, NULL,
+				     NULL, NULL, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("qfileinfo failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -997,8 +980,9 @@ static bool run_readwritelarge_internal(int max_xmit_k)
 		printf("readwritelarge test 3 (truncate test) succeeded (size = %x)\n", fsize);
 #endif
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -1160,6 +1144,7 @@ static bool run_locktest1(int dummy)
 	uint16_t fnum1, fnum2, fnum3;
 	time_t t1, t2;
 	unsigned lock_timeout;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli1, 0) || !torture_open_connection(&cli2, 1)) {
 		return False;
@@ -1169,18 +1154,24 @@ static bool run_locktest1(int dummy)
 
 	printf("starting locktest1\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE,
+	                  &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR, DENY_NONE, &fnum2))) {
-		printf("open2 of %s failed (%s)\n", fname, cli_errstr(cli1));
+
+	status = cli_open(cli1, fname, O_RDWR, DENY_NONE, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum3))) {
-		printf("open3 of %s failed (%s)\n", fname, cli_errstr(cli2));
+
+	status = cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum3);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open3 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -1218,8 +1209,9 @@ static bool run_locktest1(int dummy)
 	printf("server slept for %u seconds for a %u second timeout\n",
 	       (unsigned int)(t2-t1), lock_timeout);
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum2))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -1231,18 +1223,21 @@ static bool run_locktest1(int dummy)
 				 NT_STATUS_FILE_LOCK_CONFLICT)) return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum3))) {
-		printf("close3 failed (%s)\n", cli_errstr(cli2));
+	status = cli_close(cli2, fnum3);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close3 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, fname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(cli1));
+	status = cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -1283,18 +1278,20 @@ static bool run_tcon_test(int dummy)
 
 	printf("starting tcontest\n");
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	cnum1 = cli->cnum;
 	vuid1 = cli->vuid;
 
-	if (cli_write(cli, fnum1, 0, buf, 130, 4) != 4) {
-		printf("initial write failed (%s)", cli_errstr(cli));
+	status = cli_writeall(cli, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("initial write failed (%s)", nt_errstr(status));
 		return False;
 	}
 
@@ -1314,40 +1311,47 @@ static bool run_tcon_test(int dummy)
 	/* try a write with the wrong tid */
 	cli->cnum = cnum2;
 
-	if (cli_write(cli, fnum1, 0, buf, 130, 4) == 4) {
+	status = cli_writeall(cli, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (NT_STATUS_IS_OK(status)) {
 		printf("* server allows write with wrong TID\n");
 		ret = False;
 	} else {
-		printf("server fails write with wrong TID : %s\n", cli_errstr(cli));
+		printf("server fails write with wrong TID : %s\n",
+		       nt_errstr(status));
 	}
 
 
 	/* try a write with an invalid tid */
 	cli->cnum = cnum3;
 
-	if (cli_write(cli, fnum1, 0, buf, 130, 4) == 4) {
+	status = cli_writeall(cli, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (NT_STATUS_IS_OK(status)) {
 		printf("* server allows write with invalid TID\n");
 		ret = False;
 	} else {
-		printf("server fails write with invalid TID : %s\n", cli_errstr(cli));
+		printf("server fails write with invalid TID : %s\n",
+		       nt_errstr(status));
 	}
 
 	/* try a write with an invalid vuid */
 	cli->vuid = vuid2;
 	cli->cnum = cnum1;
 
-	if (cli_write(cli, fnum1, 0, buf, 130, 4) == 4) {
+	status = cli_writeall(cli, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (NT_STATUS_IS_OK(status)) {
 		printf("* server allows write with invalid VUID\n");
 		ret = False;
 	} else {
-		printf("server fails write with invalid VUID : %s\n", cli_errstr(cli));
+		printf("server fails write with invalid VUID : %s\n",
+		       nt_errstr(status));
 	}
 
 	cli->cnum = cnum1;
 	cli->vuid = vuid1;
 
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum1))) {
-		printf("close failed (%s)\n", cli_errstr(cli));
+	status = cli_close(cli, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -1534,6 +1538,7 @@ static bool run_locktest2(int dummy)
 	const char *fname = "\\lockt2.lck";
 	uint16_t fnum1, fnum2, fnum3;
 	bool correct = True;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli, 0)) {
 		return False;
@@ -1543,24 +1548,27 @@ static bool run_locktest2(int dummy)
 
 	printf("starting locktest2\n");
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_setpid(cli, 1);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR, DENY_NONE, &fnum2))) {
-		printf("open2 of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR, DENY_NONE, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	cli_setpid(cli, 2);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR, DENY_NONE, &fnum3))) {
-		printf("open3 of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR, DENY_NONE, &fnum3);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open3 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -1631,18 +1639,21 @@ static bool run_locktest2(int dummy)
 
 	cli_setpid(cli, 1);
 
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum1))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli));
+	status = cli_close(cli, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum2))) {
-		printf("close2 failed (%s)\n", cli_errstr(cli));
+	status = cli_close(cli, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum3))) {
-		printf("close3 failed (%s)\n", cli_errstr(cli));
+	status = cli_close(cli, fnum3);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close3 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -1669,6 +1680,7 @@ static bool run_locktest3(int dummy)
 	int i;
 	uint32 offset;
 	bool correct = True;
+	NTSTATUS status;
 
 #define NEXT_OFFSET offset += (~(uint32)0) / torture_numops
 
@@ -1680,14 +1692,18 @@ static bool run_locktest3(int dummy)
 
 	printf("starting locktest3\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE,
+	                 &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum2))) {
-		printf("open2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+
+	status = cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -1735,33 +1751,38 @@ static bool run_locktest3(int dummy)
 	for (offset=i=0;i<torture_numops;i++) {
 		NEXT_OFFSET;
 
-		if (!NT_STATUS_IS_OK(cli_unlock(cli1, fnum1, offset-1, 1))) {
+		status = cli_unlock(cli1, fnum1, offset-1, 1);
+		if (!NT_STATUS_IS_OK(status)) {
 			printf("unlock1 %d failed (%s)\n", 
 			       i,
-			       cli_errstr(cli1));
+			       nt_errstr(status));
 			return False;
 		}
 
-		if (!NT_STATUS_IS_OK(cli_unlock(cli2, fnum2, offset-2, 1))) {
+		status = cli_unlock(cli2, fnum2, offset-2, 1);
+		if (!NT_STATUS_IS_OK(status)) {
 			printf("unlock2 %d failed (%s)\n", 
 			       i,
-			       cli_errstr(cli1));
+			       nt_errstr(status));
 			return False;
 		}
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-		printf("close2 failed (%s)\n", cli_errstr(cli2));
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, fname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(cli1));
+	status = cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -1793,6 +1814,7 @@ static bool run_locktest4(int dummy)
 	bool ret;
 	char buf[1000];
 	bool correct = True;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli1, 0) || !torture_open_connection(&cli2, 1)) {
 		return False;
@@ -1803,15 +1825,17 @@ static bool run_locktest4(int dummy)
 
 	printf("starting locktest4\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
 	cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum2);
 
 	memset(buf, 0, sizeof(buf));
 
-	if (cli_write(cli1, fnum1, 0, buf, 0, sizeof(buf)) != sizeof(buf)) {
-		printf("Failed to create file\n");
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 0, sizeof(buf),
+			      NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Failed to create file: %s\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -1883,8 +1907,12 @@ static bool run_locktest4(int dummy)
 	EXPECTED(ret, False);
 	printf("this server %s strict write locking\n", ret?"doesn't do":"does");
 
-	ret = cli_lock(cli1, fnum1, 130, 4, 0, READ_LOCK) &&
-	      (cli_write(cli2, fnum2, 0, buf, 130, 4) == 4);
+	ret = cli_lock(cli1, fnum1, 130, 4, 0, READ_LOCK);
+	if (ret) {
+		status = cli_writeall(cli2, fnum2, 0, (uint8_t *)buf, 130, 4,
+				      NULL);
+		ret = NT_STATUS_IS_OK(status);
+	}
 	EXPECTED(ret, False);
 	printf("this server %s strict read locking\n", ret?"doesn't do":"does");
 
@@ -1901,21 +1929,24 @@ static bool run_locktest4(int dummy)
 	      cli_lock(cli1, fnum1, 150, 4, 0, READ_LOCK) &&
 	      NT_STATUS_IS_OK(cli_unlock(cli1, fnum1, 150, 4)) &&
 	      (cli_read(cli2, fnum2, buf, 150, 4) == 4) &&
-	      !(cli_write(cli2, fnum2, 0, buf, 150, 4) == 4) &&
+	      !(NT_STATUS_IS_OK(cli_writeall(cli2, fnum2, 0, (uint8_t *)buf,
+					     150, 4, NULL))) &&
 	      NT_STATUS_IS_OK(cli_unlock(cli1, fnum1, 150, 4));
 	EXPECTED(ret, True);
 	printf("this server %s do recursive lock overlays\n", ret?"does":"doesn't");
 
 	ret = cli_lock(cli1, fnum1, 160, 4, 0, READ_LOCK) &&
 	      NT_STATUS_IS_OK(cli_unlock(cli1, fnum1, 160, 4)) &&
-	      (cli_write(cli2, fnum2, 0, buf, 160, 4) == 4) &&		
+	      NT_STATUS_IS_OK(cli_writeall(cli2, fnum2, 0, (uint8_t *)buf,
+					   160, 4, NULL)) &&
 	      (cli_read(cli2, fnum2, buf, 160, 4) == 4);		
 	EXPECTED(ret, True);
 	printf("the same process %s remove a read lock using write locking\n", ret?"can":"cannot");
 
 	ret = cli_lock(cli1, fnum1, 170, 4, 0, WRITE_LOCK) &&
 	      NT_STATUS_IS_OK(cli_unlock(cli1, fnum1, 170, 4)) &&
-	      (cli_write(cli2, fnum2, 0, buf, 170, 4) == 4) &&		
+	      NT_STATUS_IS_OK(cli_writeall(cli2, fnum2, 0, (uint8_t *)buf,
+					   170, 4, NULL)) &&
 	      (cli_read(cli2, fnum2, buf, 170, 4) == 4);		
 	EXPECTED(ret, True);
 	printf("the same process %s remove a write lock using read locking\n", ret?"can":"cannot");
@@ -1923,7 +1954,8 @@ static bool run_locktest4(int dummy)
 	ret = cli_lock(cli1, fnum1, 190, 4, 0, WRITE_LOCK) &&
 	      cli_lock(cli1, fnum1, 190, 4, 0, READ_LOCK) &&
 	      NT_STATUS_IS_OK(cli_unlock(cli1, fnum1, 190, 4)) &&
-	      !(cli_write(cli2, fnum2, 0, buf, 190, 4) == 4) &&		
+	      !NT_STATUS_IS_OK(cli_writeall(cli2, fnum2, 0, (uint8_t *)buf,
+					    190, 4, NULL)) &&
 	      (cli_read(cli2, fnum2, buf, 190, 4) == 4);		
 	EXPECTED(ret, True);
 	printf("the same process %s remove the first lock first\n", ret?"does":"doesn't");
@@ -1945,7 +1977,7 @@ static bool run_locktest4(int dummy)
  fail:
 	cli_close(cli1, fnum1);
 	cli_close(cli2, fnum2);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	torture_close_connection(cli1);
 	torture_close_connection(cli2);
 
@@ -1964,6 +1996,7 @@ static bool run_locktest5(int dummy)
 	bool ret;
 	char buf[1000];
 	bool correct = True;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli1, 0) || !torture_open_connection(&cli2, 1)) {
 		return False;
@@ -1974,7 +2007,7 @@ static bool run_locktest5(int dummy)
 
 	printf("starting locktest5\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
 	cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum2);
@@ -1982,8 +2015,10 @@ static bool run_locktest5(int dummy)
 
 	memset(buf, 0, sizeof(buf));
 
-	if (cli_write(cli1, fnum1, 0, buf, 0, sizeof(buf)) != sizeof(buf)) {
-		printf("Failed to create file\n");
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 0, sizeof(buf),
+			      NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Failed to create file: %s\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -2064,7 +2099,7 @@ static bool run_locktest5(int dummy)
  fail:
 	cli_close(cli1, fnum1);
 	cli_close(cli2, fnum2);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	if (!torture_close_connection(cli1)) {
 		correct = False;
 	}
@@ -2099,7 +2134,7 @@ static bool run_locktest6(int dummy)
 	for (i=0;i<1;i++) {
 		printf("Testing %s\n", fname[i]);
 
-		cli_unlink(cli, fname[i], aSYSTEM | aHIDDEN);
+		cli_unlink(cli, fname[i], FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 		cli_open(cli, fname[i], O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum);
 		status = cli_locktype(cli, fnum, 0, 8, 0, LOCKING_ANDX_CHANGE_LOCKTYPE);
@@ -2111,7 +2146,7 @@ static bool run_locktest6(int dummy)
 		cli_close(cli, fnum);
 		printf("CANCEL_LOCK gave %s\n", nt_errstr(status));
 
-		cli_unlink(cli, fname[i], aSYSTEM | aHIDDEN);
+		cli_unlink(cli, fname[i], FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	}
 
 	torture_close_connection(cli);
@@ -2127,6 +2162,7 @@ static bool run_locktest7(int dummy)
 	uint16_t fnum1;
 	char buf[200];
 	bool correct = False;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli1, 0)) {
 		return False;
@@ -2136,14 +2172,16 @@ static bool run_locktest7(int dummy)
 
 	printf("starting locktest7\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
 
 	memset(buf, 0, sizeof(buf));
 
-	if (cli_write(cli1, fnum1, 0, buf, 0, sizeof(buf)) != sizeof(buf)) {
-		printf("Failed to create file\n");
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 0, sizeof(buf),
+			      NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Failed to create file: %s\n", nt_errstr(status));
 		goto fail;
 	}
 
@@ -2163,9 +2201,11 @@ static bool run_locktest7(int dummy)
 		printf("pid1 successfully read the range 130:4\n");
 	}
 
-	if (cli_write(cli1, fnum1, 0, buf, 130, 4) != 4) {
-		printf("pid1 unable to write to the range 130:4, error was %s\n", cli_errstr(cli1));
-		if (NT_STATUS_V(cli_nt_error(cli1)) != NT_STATUS_V(NT_STATUS_FILE_LOCK_CONFLICT)) {
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("pid1 unable to write to the range 130:4, error was "
+		       "%s\n", nt_errstr(status));
+		if (!NT_STATUS_EQUAL(status, NT_STATUS_FILE_LOCK_CONFLICT)) {
 			printf("Incorrect error (should be NT_STATUS_FILE_LOCK_CONFLICT)\n");
 			goto fail;
 		}
@@ -2182,9 +2222,11 @@ static bool run_locktest7(int dummy)
 		printf("pid2 successfully read the range 130:4\n");
 	}
 
-	if (cli_write(cli1, fnum1, 0, buf, 130, 4) != 4) {
-		printf("pid2 unable to write to the range 130:4, error was %s\n", cli_errstr(cli1));
-		if (NT_STATUS_V(cli_nt_error(cli1)) != NT_STATUS_V(NT_STATUS_FILE_LOCK_CONFLICT)) {
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("pid2 unable to write to the range 130:4, error was "
+		       "%s\n", nt_errstr(status));
+		if (!NT_STATUS_EQUAL(status, NT_STATUS_FILE_LOCK_CONFLICT)) {
 			printf("Incorrect error (should be NT_STATUS_FILE_LOCK_CONFLICT)\n");
 			goto fail;
 		}
@@ -2210,8 +2252,10 @@ static bool run_locktest7(int dummy)
 		printf("pid1 successfully read the range 130:4\n");
 	}
 
-	if (cli_write(cli1, fnum1, 0, buf, 130, 4) != 4) {
-		printf("pid1 unable to write to the range 130:4, error was %s\n", cli_errstr(cli1));
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("pid1 unable to write to the range 130:4, error was "
+		       "%s\n", nt_errstr(status));
 		goto fail;
 	} else {
 		printf("pid1 successfully wrote to the range 130:4\n");
@@ -2230,9 +2274,11 @@ static bool run_locktest7(int dummy)
 		goto fail;
 	}
 
-	if (cli_write(cli1, fnum1, 0, buf, 130, 4) != 4) {
-		printf("pid2 unable to write to the range 130:4, error was %s\n", cli_errstr(cli1));
-		if (NT_STATUS_V(cli_nt_error(cli1)) != NT_STATUS_V(NT_STATUS_FILE_LOCK_CONFLICT)) {
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 130, 4, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("pid2 unable to write to the range 130:4, error was "
+		       "%s\n", nt_errstr(status));
+		if (!NT_STATUS_EQUAL(status, NT_STATUS_FILE_LOCK_CONFLICT)) {
 			printf("Incorrect error (should be NT_STATUS_FILE_LOCK_CONFLICT)\n");
 			goto fail;
 		}
@@ -2246,7 +2292,7 @@ static bool run_locktest7(int dummy)
 
 fail:
 	cli_close(cli1, fnum1);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	torture_close_connection(cli1);
 
 	printf("finished locktest7\n");
@@ -2278,12 +2324,12 @@ static bool run_locktest8(int dummy)
 
 	printf("starting locktest8\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_WRITE,
 			  &fnum1);
 	if (!NT_STATUS_IS_OK(status)) {
-		d_fprintf(stderr, "cli_open returned %s\n", cli_errstr(cli1));
+		d_fprintf(stderr, "cli_open returned %s\n", nt_errstr(status));
 		return false;
 	}
 
@@ -2292,7 +2338,7 @@ static bool run_locktest8(int dummy)
 	status = cli_open(cli1, fname, O_RDONLY, DENY_NONE, &fnum2);
 	if (!NT_STATUS_IS_OK(status)) {
 		d_fprintf(stderr, "cli_open second time returned %s\n",
-			  cli_errstr(cli1));
+			  nt_errstr(status));
 		goto fail;
 	}
 
@@ -2304,14 +2350,14 @@ static bool run_locktest8(int dummy)
 
 	status = cli_close(cli1, fnum1);
 	if (!NT_STATUS_IS_OK(status)) {
-		d_fprintf(stderr, "cli_close(fnum1) %s\n", cli_errstr(cli1));
+		d_fprintf(stderr, "cli_close(fnum1) %s\n", nt_errstr(status));
 		goto fail;
 	}
 
 	status = cli_open(cli1, fname, O_RDWR, DENY_NONE, &fnum1);
 	if (!NT_STATUS_IS_OK(status)) {
 		d_fprintf(stderr, "cli_open third time returned %s\n",
-                          cli_errstr(cli1));
+                          nt_errstr(status));
                 goto fail;
         }
 
@@ -2320,7 +2366,7 @@ static bool run_locktest8(int dummy)
 fail:
 	cli_close(cli1, fnum1);
 	cli_close(cli1, fnum2);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	torture_close_connection(cli1);
 
 	printf("finished locktest8\n");
@@ -2473,7 +2519,7 @@ static bool run_locktest9(int dummy)
 	status = cli_open(cli1, fname, O_RDWR, DENY_NONE,
 			  &fnum);
 	if (!NT_STATUS_IS_OK(status)) {
-		d_fprintf(stderr, "cli_open returned %s\n", cli_errstr(cli1));
+		d_fprintf(stderr, "cli_open returned %s\n", nt_errstr(status));
 		return false;
 	}
 
@@ -2514,7 +2560,7 @@ static bool run_locktest9(int dummy)
 
 	status = cli_close(cli1, fnum);
 	if (!NT_STATUS_IS_OK(status)) {
-		d_fprintf(stderr, "cli_close(fnum1) %s\n", cli_errstr(cli1));
+		d_fprintf(stderr, "cli_close(fnum1) %s\n", nt_errstr(status));
 		goto fail;
 	}
 
@@ -2540,6 +2586,7 @@ static bool run_fdpasstest(int dummy)
 	const char *fname = "\\fdpass.tst";
 	uint16_t fnum1;
 	char buf[1024];
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli1, 0) || !torture_open_connection(&cli2, 1)) {
 		return False;
@@ -2549,15 +2596,19 @@ static bool run_fdpasstest(int dummy)
 
 	printf("starting fdpasstest\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE,
+	                  &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (cli_write(cli1, fnum1, 0, "hello world\n", 0, 13) != 13) {
-		printf("write failed (%s)\n", cli_errstr(cli1));
+	status = cli_writeall(cli1, fnum1, 0, (const uint8_t *)"hello world\n", 0,
+			      13, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("write failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -2572,7 +2623,7 @@ static bool run_fdpasstest(int dummy)
 	}
 
 	cli_close(cli1, fnum1);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	torture_close_connection(cli1);
 	torture_close_connection(cli2);
@@ -2594,6 +2645,7 @@ static bool run_fdsesstest(int dummy)
 	uint16_t fnum2;
 	char buf[1024];
 	bool ret = True;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli, 0))
 		return False;
@@ -2610,16 +2662,19 @@ static bool run_fdsesstest(int dummy)
 
 	printf("starting fdsesstest\n");
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli, fname1, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (cli_write(cli, fnum1, 0, "hello world\n", 0, 13) != 13) {
-		printf("write failed (%s)\n", cli_errstr(cli));
+	status = cli_writeall(cli, fnum1, 0, (const uint8_t *)"hello world\n", 0, 13,
+			      NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("write failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -2635,7 +2690,7 @@ static bool run_fdsesstest(int dummy)
 	if (NT_STATUS_IS_OK(cli_open(cli, fname1, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum2))) {
 		printf("create with different vuid, same cnum succeeded.\n");
 		cli_close(cli, fnum2);
-		cli_unlink(cli, fname1, aSYSTEM | aHIDDEN);
+		cli_unlink(cli, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	} else {
 		printf("create with different vuid, same cnum failed.\n");
 		printf("This will cause problems with service clients.\n");
@@ -2655,7 +2710,7 @@ static bool run_fdsesstest(int dummy)
 
 	cli->cnum = saved_cnum;
 	cli_close(cli, fnum1);
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	torture_close_connection(cli);
 
@@ -2674,6 +2729,7 @@ static bool run_unlinktest(int dummy)
 	const char *fname = "\\unlink.tst";
 	uint16_t fnum;
 	bool correct = True;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli, 0)) {
 		return False;
@@ -2683,16 +2739,17 @@ static bool run_unlinktest(int dummy)
 
 	printf("starting unlink test\n");
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_setpid(cli, 1);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (NT_STATUS_IS_OK(cli_unlink(cli, fname, aSYSTEM | aHIDDEN))) {
+	if (NT_STATUS_IS_OK(cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN))) {
 		printf("error: server allowed unlink on an open file\n");
 		correct = False;
 	} else {
@@ -2701,7 +2758,7 @@ static bool run_unlinktest(int dummy)
 	}
 
 	cli_close(cli, fnum);
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (!torture_close_connection(cli)) {
 		correct = False;
@@ -2719,12 +2776,12 @@ test how many open files this server supports on the one socket
 static bool run_maxfidtest(int dummy)
 {
 	struct cli_state *cli;
-	const char *ftemplate = "\\maxfid.%d.%d";
 	fstring fname;
 	uint16_t fnums[0x11000];
 	int i;
 	int retries=4;
 	bool correct = True;
+	NTSTATUS status;
 
 	cli = current_cli;
 
@@ -2736,11 +2793,12 @@ static bool run_maxfidtest(int dummy)
 	cli_sockopt(cli, sockops);
 
 	for (i=0; i<0x11000; i++) {
-		slprintf(fname,sizeof(fname)-1,ftemplate, i,(int)getpid());
-		if (!NT_STATUS_IS_OK(cli_open(cli, fname, 
-					O_RDWR|O_CREAT|O_TRUNC, DENY_NONE, &fnums[i]))) {
+		slprintf(fname,sizeof(fname)-1,"\\maxfid.%d.%d", i,(int)getpid());
+		status = cli_open(cli, fname, O_RDWR|O_CREAT|O_TRUNC, DENY_NONE,
+		                  &fnums[i]);
+		if (!NT_STATUS_IS_OK(status)) {
 			printf("open of %s failed (%s)\n", 
-			       fname, cli_errstr(cli));
+			       fname, nt_errstr(status));
 			printf("maximum fnum is %d\n", i);
 			break;
 		}
@@ -2751,11 +2809,13 @@ static bool run_maxfidtest(int dummy)
 
 	printf("cleaning up\n");
 	for (;i>=0;i--) {
-		slprintf(fname,sizeof(fname)-1,ftemplate, i,(int)getpid());
+		slprintf(fname,sizeof(fname)-1,"\\maxfid.%d.%d", i,(int)getpid());
 		cli_close(cli, fnums[i]);
-		if (!NT_STATUS_IS_OK(cli_unlink(cli, fname, aSYSTEM | aHIDDEN))) {
+
+		status = cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+		if (!NT_STATUS_IS_OK(status)) {
 			printf("unlink of %s failed (%s)\n", 
-			       fname, cli_errstr(cli));
+			       fname, nt_errstr(status));
 			correct = False;
 		}
 		printf("%6d\r", i);
@@ -2827,15 +2887,37 @@ static bool run_negprot_nowait(int dummy)
 /* send smb negprot commands, not reading the response */
 static bool run_bad_nbt_session(int dummy)
 {
-	static struct cli_state *cli;
+	struct nmb_name called, calling;
+	struct sockaddr_storage ss;
+	NTSTATUS status;
+	int fd;
+	bool ret;
 
 	printf("starting bad nbt session test\n");
 
-	if (!(cli = open_bad_nbt_connection())) {
-		return False;
+	make_nmb_name(&calling, myname, 0x0);
+	make_nmb_name(&called , host, 0x20);
+
+	if (!resolve_name(host, &ss, 0x20, true)) {
+		d_fprintf(stderr, "Could not resolve name %s\n", host);
+		return false;
+	}
+
+	status = open_socket_out(&ss, 139, 10000, &fd);
+	if (!NT_STATUS_IS_OK(status)) {
+		d_fprintf(stderr, "open_socket_out failed: %s\n",
+			  nt_errstr(status));
+		return false;
+	}
+
+	ret = cli_bad_session_request(fd, &calling, &called);
+	close(fd);
+	if (!ret) {
+		d_fprintf(stderr, "open_socket_out failed: %s\n",
+			  nt_errstr(status));
+		return false;
 	}
 
-	cli_shutdown(cli);
 	printf("finished bad nbt session test\n");
 	return true;
 }
@@ -2942,6 +3024,7 @@ static bool run_attrtest(int dummy)
 	time_t t, t2;
 	const char *fname = "\\attrib123456789.tst";
 	bool correct = True;
+	NTSTATUS status;
 
 	printf("starting attrib test\n");
 
@@ -2949,12 +3032,14 @@ static bool run_attrtest(int dummy)
 		return False;
 	}
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_open(cli, fname, 
 			O_RDWR | O_CREAT | O_TRUNC, DENY_NONE, &fnum);
 	cli_close(cli, fnum);
-	if (!NT_STATUS_IS_OK(cli_getatr(cli, fname, NULL, NULL, &t))) {
-		printf("getatr failed (%s)\n", cli_errstr(cli));
+
+	status = cli_getatr(cli, fname, NULL, NULL, &t);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("getatr failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -2967,13 +3052,15 @@ static bool run_attrtest(int dummy)
 
 	t2 = t-60*60*24; /* 1 day ago */
 
-	if (!NT_STATUS_IS_OK(cli_setatr(cli, fname, 0, t2))) {
-		printf("setatr failed (%s)\n", cli_errstr(cli));
+	status = cli_setatr(cli, fname, 0, t2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("setatr failed (%s)\n", nt_errstr(status));
 		correct = True;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_getatr(cli, fname, NULL, NULL, &t))) {
-		printf("getatr failed (%s)\n", cli_errstr(cli));
+	status = cli_getatr(cli, fname, NULL, NULL, &t);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("getatr failed (%s)\n", nt_errstr(status));
 		correct = True;
 	}
 
@@ -2984,7 +3071,7 @@ static bool run_attrtest(int dummy)
 		correct = True;
 	}
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (!torture_close_connection(cli)) {
 		correct = False;
@@ -3027,19 +3114,18 @@ static bool run_trans2test(int dummy)
 		correct = false;
 	}
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
-	cli_open(cli, fname, 
-			O_RDWR | O_CREAT | O_TRUNC, DENY_NONE, &fnum);
-	if (!NT_STATUS_IS_OK(cli_qfileinfo_basic(
-				     cli, fnum, NULL, &size, &c_time_ts,
-				     &a_time_ts, &w_time_ts,
-				     &m_time_ts, NULL))) {
-		printf("ERROR: qfileinfo failed (%s)\n", cli_errstr(cli));
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_open(cli, fname, O_RDWR | O_CREAT | O_TRUNC, DENY_NONE, &fnum);
+	status = cli_qfileinfo_basic(cli, fnum, NULL, &size, &c_time_ts,
+	                             &a_time_ts, &w_time_ts, &m_time_ts, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("ERROR: qfileinfo failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_qfilename(cli, fnum, pname, sizeof(pname)))) {
-		printf("ERROR: qfilename failed (%s)\n", cli_errstr(cli));
+	status = cli_qfilename(cli, fnum, pname, sizeof(pname));
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("ERROR: qfilename failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -3053,10 +3139,11 @@ static bool run_trans2test(int dummy)
 
 	sleep(2);
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, 
-			O_RDWR | O_CREAT | O_TRUNC, DENY_NONE, &fnum))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli));
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	status = cli_open(cli, fname, O_RDWR | O_CREAT | O_TRUNC, DENY_NONE,
+	                  &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 	cli_close(cli, fnum);
@@ -3085,7 +3172,7 @@ static bool run_trans2test(int dummy)
 	}
 
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_open(cli, fname, 
 			O_RDWR | O_CREAT | O_TRUNC, DENY_NONE, &fnum);
 	cli_close(cli, fnum);
@@ -3102,13 +3189,14 @@ static bool run_trans2test(int dummy)
 		}
 	}
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 
 	/* check if the server updates the directory modification time
            when creating a new file */
-	if (!NT_STATUS_IS_OK(cli_mkdir(cli, dname))) {
-		printf("ERROR: mkdir failed (%s)\n", cli_errstr(cli));
+	status = cli_mkdir(cli, dname);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("ERROR: mkdir failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 	sleep(3);
@@ -3121,7 +3209,7 @@ static bool run_trans2test(int dummy)
 
 	cli_open(cli, fname2, 
 			O_RDWR | O_CREAT | O_TRUNC, DENY_NONE, &fnum);
-	cli_write(cli, fnum,  0, (char *)&fnum, 0, sizeof(fnum));
+	cli_writeall(cli, fnum,  0, (uint8_t *)&fnum, 0, sizeof(fnum), NULL);
 	cli_close(cli, fnum);
 	status = cli_qpathinfo2(cli, "\\trans2\\", &c_time_ts, &a_time_ts,
 				&w_time_ts, &m_time2_ts, &size, NULL, NULL);
@@ -3135,7 +3223,7 @@ static bool run_trans2test(int dummy)
 			correct = False;
 		}
 	}
-	cli_unlink(cli, fname2, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname2, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, dname);
 
 	if (!torture_close_connection(cli)) {
@@ -3213,6 +3301,7 @@ static bool run_oplock1(int dummy)
 	const char *fname = "\\lockt1.lck";
 	uint16_t fnum1;
 	bool correct = True;
+	NTSTATUS status;
 
 	printf("starting oplock test 1\n");
 
@@ -3220,29 +3309,33 @@ static bool run_oplock1(int dummy)
 		return False;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_sockopt(cli1, sockops);
 
 	cli1->use_oplocks = True;
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE,
+			  &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	cli1->use_oplocks = False;
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, fname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(cli1));
+	status = cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -3264,6 +3357,7 @@ static bool run_oplock2(int dummy)
 	char buf[4];
 	bool correct = True;
 	volatile bool *shared_correct;
+	NTSTATUS status;
 
 	shared_correct = (volatile bool *)shm_setup(sizeof(bool));
 	*shared_correct = True;
@@ -3291,13 +3385,15 @@ static bool run_oplock2(int dummy)
 	cli2->use_oplocks = True;
 	cli2->use_level_II_oplocks = True;
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_sockopt(cli1, sockops);
 	cli_sockopt(cli2, sockops);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE,
+	                  &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -3307,16 +3403,18 @@ static bool run_oplock2(int dummy)
 
 	if (fork() == 0) {
 		/* Child code */
-		if (!NT_STATUS_IS_OK(cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum2))) {
-			printf("second open of %s failed (%s)\n", fname, cli_errstr(cli1));
+		status = cli_open(cli2, fname, O_RDWR, DENY_NONE, &fnum2);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("second open of %s failed (%s)\n", fname, nt_errstr(status));
 			*shared_correct = False;
 			exit(0);
 		}
 
 		sleep(2);
 
-		if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-			printf("close2 failed (%s)\n", cli_errstr(cli1));
+		status = cli_close(cli2, fnum2);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("close2 failed (%s)\n", nt_errstr(status));
 			*shared_correct = False;
 		}
 
@@ -3356,22 +3454,17 @@ static bool run_oplock2(int dummy)
 
 	cli_read(cli1, fnum1, buf, 0, 4);
 
-#if 0
-	if (cli_write(cli1, fnum1, 0, buf, 0, 4) != 4) {
-		printf("write on fnum1 failed (%s)\n", cli_errstr(cli1));
-		correct = False;
-	}
-#endif
-
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
 	sleep(4);
 
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, fname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(cli1));
+	status = cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
@@ -3388,78 +3481,21 @@ static bool run_oplock2(int dummy)
 	return correct;
 }
 
-/* handler for oplock 3 tests */
-static NTSTATUS oplock3_handler(struct cli_state *cli, uint16_t fnum, unsigned char level)
-{
-	printf("got oplock break fnum=%d level=%d\n",
-	       fnum, level);
-	return cli_oplock_ack(cli, fnum, level);
-}
-
-static bool run_oplock3(int dummy)
-{
+struct oplock4_state {
+	struct tevent_context *ev;
 	struct cli_state *cli;
-	const char *fname = "\\oplockt3.dat";
-	uint16_t fnum;
-	char buf[4] = "abcd";
-	bool correct = True;
-	volatile bool *shared_correct;
-
-	shared_correct = (volatile bool *)shm_setup(sizeof(bool));
-	*shared_correct = True;
-
-	printf("starting oplock test 3\n");
-
-	if (fork() == 0) {
-		/* Child code */
-		use_oplocks = True;
-		use_level_II_oplocks = True;
-		if (!torture_open_connection(&cli, 0)) {
-			*shared_correct = False;
-			exit(0);
-		} 
-		sleep(2);
-		/* try to trigger a oplock break in parent */
-		cli_open(cli, fname, O_RDWR, DENY_NONE, &fnum);
-		cli_write(cli, fnum, 0, buf, 0, 4);
-		exit(0);
-	}
-
-	/* parent code */
-	use_oplocks = True;
-	use_level_II_oplocks = True;
-	if (!torture_open_connection(&cli, 1)) { /* other is forked */
-		return False;
-	}
-	cli_oplock_handler(cli, oplock3_handler);
-	cli_open(cli, fname, O_RDWR|O_CREAT, DENY_NONE, &fnum);
-	cli_write(cli, fnum, 0, buf, 0, 4);
-	cli_close(cli, fnum);
-	cli_open(cli, fname, O_RDWR, DENY_NONE, &fnum);
-	cli->timeout = 20000;
-	cli_receive_smb(cli);
-	printf("finished oplock test 3\n");
-
-	return (correct && *shared_correct);
-
-/* What are we looking for here?  What's sucess and what's FAILURE? */
-}
+	bool *got_break;
+	uint16_t *fnum2;
+};
 
-/* handler for oplock 4 tests */
-bool *oplock4_shared_correct;
-
-static NTSTATUS oplock4_handler(struct cli_state *cli, uint16_t fnum, unsigned char level)
-{
-	printf("got oplock break fnum=%d level=%d\n",
-	       fnum, level);
-	*oplock4_shared_correct = true;
-	cli_oplock_ack(cli, fnum, level);
-	return NT_STATUS_UNSUCCESSFUL; /* Cause cli_receive_smb to return. */
-}
+static void oplock4_got_break(struct tevent_req *req);
+static void oplock4_got_open(struct tevent_req *req);
 
 static bool run_oplock4(int dummy)
 {
+	struct tevent_context *ev;
 	struct cli_state *cli1, *cli2;
+	struct tevent_req *oplock_req, *open_req;
 	const char *fname = "\\lockt4.lck";
 	const char *fname_ln = "\\lockt4_ln.lck";
 	uint16_t fnum1, fnum2;
@@ -3467,8 +3503,9 @@ static bool run_oplock4(int dummy)
 	NTSTATUS status;
 	bool correct = true;
 
-	oplock4_shared_correct = (bool *)shm_setup(sizeof(bool));
-	*oplock4_shared_correct = false;
+	bool got_break;
+
+	struct oplock4_state *state;
 
 	printf("starting oplock test 4\n");
 
@@ -3484,32 +3521,37 @@ static bool run_oplock4(int dummy)
 		return false;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname_ln, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname_ln, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_sockopt(cli1, sockops);
 	cli_sockopt(cli2, sockops);
 
 	/* Create the file. */
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE,
+	                  &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return false;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		return false;
 	}
 
 	/* Now create a hardlink. */
-	if (!NT_STATUS_IS_OK(cli_nt_hardlink(cli1, fname, fname_ln))) {
-		printf("nt hardlink failed (%s)\n", cli_errstr(cli1));
+	status = cli_nt_hardlink(cli1, fname, fname_ln);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("nt hardlink failed (%s)\n", nt_errstr(status));
 		return false;
 	}
 
 	/* Prove that opening hardlinks cause deny modes to conflict. */
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR, DENY_ALL, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR, DENY_ALL, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return false;
 	}
 
@@ -3526,8 +3568,9 @@ static bool run_oplock4(int dummy)
 		return false;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		return false;
 	}
 
@@ -3537,44 +3580,78 @@ static bool run_oplock4(int dummy)
 	cli2->use_oplocks = true;
 	cli2->use_level_II_oplocks = true;
 
-	cli_oplock_handler(cli1, oplock4_handler);
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return false;
 	}
 
-	if (fork() == 0) {
-		/* Child code */
-		if (!NT_STATUS_IS_OK(cli_open(cli2, fname_ln, O_RDWR, DENY_NONE, &fnum2))) {
-			printf("open of %s failed (%s)\n", fname_ln, cli_errstr(cli1));
-			*oplock4_shared_correct = false;
-			exit(0);
-		}
+	ev = tevent_context_init(talloc_tos());
+	if (ev == NULL) {
+		printf("tevent_req_create failed\n");
+		return false;
+	}
 
-		if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-			printf("close2 failed (%s)\n", cli_errstr(cli1));
-			*oplock4_shared_correct = false;
-		}
+	state = talloc(ev, struct oplock4_state);
+	if (state == NULL) {
+		printf("talloc failed\n");
+		return false;
+	}
+	state->ev = ev;
+	state->cli = cli1;
+	state->got_break = &got_break;
+	state->fnum2 = &fnum2;
+
+	oplock_req = cli_smb_oplock_break_waiter_send(
+		talloc_tos(), ev, cli1);
+	if (oplock_req == NULL) {
+		printf("cli_smb_oplock_break_waiter_send failed\n");
+		return false;
+	}
+	tevent_req_set_callback(oplock_req, oplock4_got_break, state);
 
-		exit(0);
+	open_req = cli_open_send(
+		talloc_tos(), ev, cli2, fname_ln, O_RDWR, DENY_NONE);
+	if (oplock_req == NULL) {
+		printf("cli_open_send failed\n");
+		return false;
 	}
+	tevent_req_set_callback(open_req, oplock4_got_open, state);
 
-	sleep(2);
+	got_break = false;
+	fnum2 = 0xffff;
 
-	/* Process the oplock break. */
-	cli_receive_smb(cli1);
+	while (!got_break || fnum2 == 0xffff) {
+		int ret;
+		ret = tevent_loop_once(ev);
+		if (ret == -1) {
+			printf("tevent_loop_once failed: %s\n",
+			       strerror(errno));
+			return false;
+		}
+	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
+		correct = false;
+	}
+
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		correct = false;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, fname, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(cli1));
+	status = cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		correct = false;
 	}
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, fname_ln, aSYSTEM | aHIDDEN))) {
-		printf("unlink failed (%s)\n", cli_errstr(cli1));
+
+	status = cli_unlink(cli1, fname_ln, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink failed (%s)\n", nt_errstr(status));
 		correct = false;
 	}
 
@@ -3582,7 +3659,7 @@ static bool run_oplock4(int dummy)
 		correct = false;
 	}
 
-	if (!*oplock4_shared_correct) {
+	if (!got_break) {
 		correct = false;
 	}
 
@@ -3591,6 +3668,43 @@ static bool run_oplock4(int dummy)
 	return correct;
 }
 
+static void oplock4_got_break(struct tevent_req *req)
+{
+	struct oplock4_state *state = tevent_req_callback_data(
+		req, struct oplock4_state);
+	uint16_t fnum;
+	uint8_t level;
+	NTSTATUS status;
+
+	status = cli_smb_oplock_break_waiter_recv(req, &fnum, &level);
+	TALLOC_FREE(req);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("cli_smb_oplock_break_waiter_recv returned %s\n",
+		       nt_errstr(status));
+		return;
+	}
+	*state->got_break = true;
+
+	req = cli_oplock_ack_send(state, state->ev, state->cli, fnum,
+				  NO_OPLOCK);
+	if (req == NULL) {
+		printf("cli_oplock_ack_send failed\n");
+		return;
+	}
+}
+
+static void oplock4_got_open(struct tevent_req *req)
+{
+	struct oplock4_state *state = tevent_req_callback_data(
+		req, struct oplock4_state);
+	NTSTATUS status;
+
+	status = cli_open_recv(req, state->fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("cli_open_recv returned %s\n", nt_errstr(status));
+		*state->fnum2 = 0xffff;
+	}
+}
 
 /*
   Test delete on close semantics.
@@ -3603,6 +3717,7 @@ static bool run_deletetest(int dummy)
 	uint16_t fnum1 = (uint16_t)-1;
 	uint16_t fnum2 = (uint16_t)-1;
 	bool correct = True;
+	NTSTATUS status;
 
 	printf("starting delete test\n");
 
@@ -3615,18 +3730,20 @@ static bool run_deletetest(int dummy)
 	/* Test 1 - this should delete the file on close. */
 
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_ALL_ACCESS|DELETE_ACCESS, FILE_ATTRIBUTE_NORMAL,
-				   0, FILE_OVERWRITE_IF, 
-				   FILE_DELETE_ON_CLOSE, 0, &fnum1))) {
-		printf("[1] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_ALL_ACCESS|DELETE_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF,
+			      FILE_DELETE_ON_CLOSE, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[1] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[1] close failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[1] close failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3642,46 +3759,53 @@ static bool run_deletetest(int dummy)
 	/* Test 2 - this should delete the file on close. */
 
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_ALL_ACCESS,
-				   FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE, 
-				   FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("[2] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_ALL_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[2] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_nt_delete_on_close(cli1, fnum1, true))) {
-		printf("[2] setting delete_on_close failed (%s)\n", cli_errstr(cli1));
+	status = cli_nt_delete_on_close(cli1, fnum1, true);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[2] setting delete_on_close failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[2] close failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[2] close failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
 	if (NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDONLY, DENY_NONE, &fnum1))) {
 		printf("[2] open of %s succeeded should have been deleted on close !\n", fname);
-		if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-			printf("[2] close failed (%s)\n", cli_errstr(cli1));
+		status = cli_close(cli1, fnum1);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("[2] close failed (%s)\n", nt_errstr(status));
 			correct = False;
 			goto fail;
 		}
-		cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+		cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	} else
 		printf("second delete on close test succeeded.\n");
 
 	/* Test 3 - ... */
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_ALL_ACCESS, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("[3] open - 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_ALL_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL,
+			      FILE_SHARE_READ|FILE_SHARE_WRITE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[3] open - 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3697,28 +3821,33 @@ static bool run_deletetest(int dummy)
 	}
 
 	/* This should succeed. */
-
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS, FILE_ATTRIBUTE_NORMAL,
-			FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, FILE_OPEN, 0, 0, &fnum2))) {
-		printf("[3] open  - 2 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS,
+			     FILE_ATTRIBUTE_NORMAL,
+			     FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+			     FILE_OPEN, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[3] open  - 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_nt_delete_on_close(cli1, fnum1, true))) {
-		printf("[3] setting delete_on_close failed (%s)\n", cli_errstr(cli1));
+	status = cli_nt_delete_on_close(cli1, fnum1, true);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[3] setting delete_on_close failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[3] close 1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[3] close 1 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum2))) {
-		printf("[3] close 2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[3] close 2 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3727,10 +3856,11 @@ static bool run_deletetest(int dummy)
 
 	if (NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDONLY, DENY_NONE, &fnum1))) {
 		printf("[3] open of %s succeeded should have been deleted on close !\n", fname);
-		if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-			printf("[3] close failed (%s)\n", cli_errstr(cli1));
+		status = cli_close(cli1, fnum1);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("[3] close failed (%s)\n", nt_errstr(status));
 		}
-		cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+		cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 		correct = False;
 		goto fail;
 	} else
@@ -3738,31 +3868,40 @@ static bool run_deletetest(int dummy)
 
 	/* Test 4 ... */
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
-			FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("[4] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0,
+	                      FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL,
+			      FILE_SHARE_READ|FILE_SHARE_WRITE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[4] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
 	/* This should succeed. */
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS,
-			FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, FILE_OPEN, 0, 0, &fnum2))) {
-		printf("[4] open  - 2 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS,
+	                     FILE_ATTRIBUTE_NORMAL,
+			     FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+			     FILE_OPEN, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[4] open  - 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum2))) {
-		printf("[4] close - 1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[4] close - 1 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_nt_delete_on_close(cli1, fnum1, true))) {
-		printf("[4] setting delete_on_close failed (%s)\n", cli_errstr(cli1));
+	status = cli_nt_delete_on_close(cli1, fnum1, true);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[4] setting delete_on_close failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3777,18 +3916,20 @@ static bool run_deletetest(int dummy)
 	} else
 		printf("fourth delete on close test succeeded.\n");
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[4] close - 2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[4] close - 2 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
 	/* Test 5 ... */
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT, DENY_NONE, &fnum1))) {
-		printf("[5] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[5] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3801,8 +3942,9 @@ static bool run_deletetest(int dummy)
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[5] close - 2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[5] close - 2 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3811,12 +3953,15 @@ static bool run_deletetest(int dummy)
 
 	/* Test 6 ... */
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA,
-				   FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
-				   FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("[6] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA,
+			     FILE_ATTRIBUTE_NORMAL,
+			     FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+			     FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[6] open of %s failed (%s)\n", fname,
+		       nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3829,8 +3974,9 @@ static bool run_deletetest(int dummy)
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[6] close - 2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[6] close - 2 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3839,11 +3985,14 @@ static bool run_deletetest(int dummy)
 
 	/* Test 7 ... */
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
-				   FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("[7] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0,
+	                      FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF,
+			      0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[7] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3860,22 +4009,24 @@ static bool run_deletetest(int dummy)
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[7] close - 2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[7] close - 2 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
 	/* This next open should succeed - we reset the flag. */
-
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDONLY, DENY_NONE, &fnum1))) {
-		printf("[5] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDONLY, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[5] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[7] close - 2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[7] close - 2 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3884,7 +4035,7 @@ static bool run_deletetest(int dummy)
 
 	/* Test 7 ... */
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (!torture_open_connection(&cli2, 1)) {
 		printf("[8] failed to open second connection.\n");
@@ -3894,18 +4045,24 @@ static bool run_deletetest(int dummy)
 
 	cli_sockopt(cli1, sockops);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
-				   FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
-				   FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("[8] open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0,
+	                     FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
+			     FILE_ATTRIBUTE_NORMAL,
+			     FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+			     FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[8] open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
-				   FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
-				   FILE_OPEN, 0, 0, &fnum2))) {
-		printf("[8] open 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0,
+			     FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
+			     FILE_ATTRIBUTE_NORMAL,
+			     FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+			     FILE_OPEN, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[8] open 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3916,20 +4073,23 @@ static bool run_deletetest(int dummy)
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[8] close - 1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[8] close - 1 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-		printf("[8] close - 2 failed (%s)\n", cli_errstr(cli2));
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[8] close - 2 failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
 	/* This should fail.. */
-	if (NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDONLY, DENY_NONE, &fnum1))) {
+	status = cli_open(cli1, fname, O_RDONLY, DENY_NONE, &fnum1);
+	if (NT_STATUS_IS_OK(status)) {
 		printf("[8] open of %s succeeded should have been deleted on close !\n", fname);
 		goto fail;
 		correct = False;
@@ -3946,16 +4106,21 @@ static bool run_deletetest(int dummy)
 
 	printf("ninth delete on close test succeeded.\n");
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
-				   FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE, FILE_OVERWRITE_IF, FILE_DELETE_ON_CLOSE, 0, &fnum1))) {
-		printf("[10] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0,
+			     FILE_READ_DATA|FILE_WRITE_DATA|DELETE_ACCESS,
+			     FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			     FILE_OVERWRITE_IF, FILE_DELETE_ON_CLOSE,
+			     0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[10] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
 	/* This should delete the file. */
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[10] close failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[10] close failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -3969,21 +4134,24 @@ static bool run_deletetest(int dummy)
 		printf("tenth delete on close test succeeded.\n");
 
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	/* What error do we get when attempting to open a read-only file with
 	   delete access ? */
 
 	/* Create a readonly file. */
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA,
-				   FILE_ATTRIBUTE_READONLY, FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("[11] open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA,
+	                      FILE_ATTRIBUTE_READONLY, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[11] open of %s failed (%s)\n", fname, nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("[11] close failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("[11] close failed (%s)\n", nt_errstr(status));
 		correct = False;
 		goto fail;
 	}
@@ -4017,7 +4185,7 @@ static bool run_deletetest(int dummy)
 	if (fnum1 != (uint16_t)-1) cli_close(cli1, fnum1);
 	if (fnum2 != (uint16_t)-1) cli_close(cli1, fnum2);
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (cli1 && !torture_close_connection(cli1)) {
 		correct = False;
@@ -4045,25 +4213,28 @@ static bool run_deletetest_ln(int dummy)
 		return false;
 	}
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli, fname_ln, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli, fname_ln, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_sockopt(cli, sockops);
 
 	/* Create the file. */
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return false;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli, fnum))) {
-		printf("close1 failed (%s)\n", cli_errstr(cli));
+	status = cli_close(cli, fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close1 failed (%s)\n", nt_errstr(status));
 		return false;
 	}
 
 	/* Now create a hardlink. */
-	if (!NT_STATUS_IS_OK(cli_nt_hardlink(cli, fname, fname_ln))) {
-		printf("nt hardlink failed (%s)\n", cli_errstr(cli));
+	status = cli_nt_hardlink(cli, fname, fname_ln);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("nt hardlink failed (%s)\n", nt_errstr(status));
 		return false;
 	}
 
@@ -4127,8 +4298,8 @@ static bool run_deletetest_ln(int dummy)
                 correct = False;
         }
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli, fname_ln, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli, fname_ln, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (!torture_close_connection(cli)) {
 		correct = false;
@@ -4196,6 +4367,7 @@ static bool run_xcopy(int dummy)
 	const char *fname = "\\test.txt";
 	bool correct = True;
 	uint16_t fnum1, fnum2;
+	NTSTATUS status;
 
 	printf("starting xcopy test\n");
 
@@ -4203,19 +4375,19 @@ static bool run_xcopy(int dummy)
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0,
-				   FIRST_DESIRED_ACCESS, FILE_ATTRIBUTE_ARCHIVE,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 
-				   0x4044, 0, &fnum1))) {
-		printf("First open failed - %s\n", cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, FIRST_DESIRED_ACCESS,
+			      FILE_ATTRIBUTE_ARCHIVE, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0x4044, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("First open failed - %s\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0,
-				   SECOND_DESIRED_ACCESS, 0,
-				   FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, FILE_OPEN, 
-				   0x200000, 0, &fnum2))) {
-		printf("second open failed - %s\n", cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, SECOND_DESIRED_ACCESS, 0,
+	                     FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+			     FILE_OPEN, 0x200000, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("second open failed - %s\n", nt_errstr(status));
 		return False;
 	}
 
@@ -4245,28 +4417,33 @@ static bool run_rename(int dummy)
 		return False;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname1, aSYSTEM | aHIDDEN);
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_READ, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("First open failed - %s\n", cli_errstr(cli1));
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("First open failed - %s\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_rename(cli1, fname, fname1))) {
-		printf("First rename failed (SHARE_READ) (this is correct) - %s\n", cli_errstr(cli1));
+	status = cli_rename(cli1, fname, fname1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("First rename failed (SHARE_READ) (this is correct) - %s\n", nt_errstr(status));
 	} else {
 		printf("First rename succeeded (SHARE_READ) - this should have failed !\n");
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close - 1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close - 1 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname1, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	status = cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS, FILE_ATTRIBUTE_NORMAL,
 #if 0
 			      FILE_SHARE_DELETE|FILE_SHARE_NONE,
@@ -4275,28 +4452,32 @@ static bool run_rename(int dummy)
 #endif
 			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
 	if (!NT_STATUS_IS_OK(status)) {
-		printf("Second open failed - %s\n", cli_errstr(cli1));
+		printf("Second open failed - %s\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_rename(cli1, fname, fname1))) {
-		printf("Second rename failed (SHARE_DELETE | SHARE_READ) - this should have succeeded - %s\n", cli_errstr(cli1));
+	status = cli_rename(cli1, fname, fname1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Second rename failed (SHARE_DELETE | SHARE_READ) - this should have succeeded - %s\n", nt_errstr(status));
 		correct = False;
 	} else {
 		printf("Second rename succeeded (SHARE_DELETE | SHARE_READ)\n");
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close - 2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close - 2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname1, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, READ_CONTROL_ACCESS, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("Third open failed - %s\n", cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, READ_CONTROL_ACCESS,
+	                      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Third open failed - %s\n", nt_errstr(status));
 		return False;
 	}
 
@@ -4322,58 +4503,68 @@ static bool run_rename(int dummy)
   }
 #endif
 
-	if (!NT_STATUS_IS_OK(cli_rename(cli1, fname, fname1))) {
-		printf("Third rename failed (SHARE_NONE) - this should have succeeded - %s\n", cli_errstr(cli1));
+	status = cli_rename(cli1, fname, fname1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Third rename failed (SHARE_NONE) - this should have succeeded - %s\n", nt_errstr(status));
 		correct = False;
 	} else {
 		printf("Third rename succeeded (SHARE_NONE)\n");
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close - 3 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close - 3 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname1, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
         /*----*/
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_READ | FILE_SHARE_WRITE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("Fourth open failed - %s\n", cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS,
+	                      FILE_ATTRIBUTE_NORMAL,
+			      FILE_SHARE_READ | FILE_SHARE_WRITE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Fourth open failed - %s\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_rename(cli1, fname, fname1))) {
-		printf("Fourth rename failed (SHARE_READ | SHARE_WRITE) (this is correct) - %s\n", cli_errstr(cli1));
+	status = cli_rename(cli1, fname, fname1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Fourth rename failed (SHARE_READ | SHARE_WRITE) (this is correct) - %s\n", nt_errstr(status));
 	} else {
 		printf("Fourth rename succeeded (SHARE_READ | SHARE_WRITE) - this should have failed !\n");
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close - 4 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close - 4 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname1, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
         /*--*/
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("Fifth open failed - %s\n", cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, GENERIC_READ_ACCESS,
+	                 FILE_ATTRIBUTE_NORMAL,
+			 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+			 FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Fifth open failed - %s\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_rename(cli1, fname, fname1))) {
-		printf("Fifth rename failed (SHARE_READ | SHARE_WRITE | SHARE_DELETE) - this should have succeeded - %s ! \n",
-			cli_errstr(cli1));
+	status = cli_rename(cli1, fname, fname1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("Fifth rename failed (SHARE_READ | SHARE_WRITE | SHARE_DELETE) - this should have succeeded - %s ! \n", nt_errstr(status));
 		correct = False;
 	} else {
-		printf("Fifth rename succeeded (SHARE_READ | SHARE_WRITE | SHARE_DELETE) (this is correct) - %s\n", cli_errstr(cli1));
+		printf("Fifth rename succeeded (SHARE_READ | SHARE_WRITE | SHARE_DELETE) (this is correct) - %s\n", nt_errstr(status));
 	}
 
         /*
@@ -4391,16 +4582,17 @@ static bool run_rename(int dummy)
           } */
 
         /*--*/
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close - 5 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close - 5 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
 	/* Check that the renamed file has FILE_ATTRIBUTE_ARCHIVE. */
-	if (!NT_STATUS_IS_OK(cli_getatr(cli1, fname1, &attr, NULL, NULL))) {
+	status = cli_getatr(cli1, fname1, &attr, NULL, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
 		printf("getatr on file %s failed - %s ! \n",
-			fname1,
-			cli_errstr(cli1));
+			fname1, nt_errstr(status));
 		correct = False;
 	} else {
 		if (attr != FILE_ATTRIBUTE_ARCHIVE) {
@@ -4415,8 +4607,8 @@ static bool run_rename(int dummy)
 		}
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-	cli_unlink(cli1, fname1, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	cli_unlink(cli1, fname1, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	if (!torture_close_connection(cli1)) {
 		correct = False;
@@ -4431,6 +4623,7 @@ static bool run_pipe_number(int dummy)
 	const char *pipe_name = "\\SPOOLSS";
 	uint16_t fnum;
 	int num_pipes = 0;
+	NTSTATUS status;
 
 	printf("starting pipenumber test\n");
 	if (!torture_open_connection(&cli1, 0)) {
@@ -4439,9 +4632,12 @@ static bool run_pipe_number(int dummy)
 
 	cli_sockopt(cli1, sockops);
 	while(1) {
-		if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, pipe_name, 0, FILE_READ_DATA, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OPEN_IF, 0, 0, &fnum))) {
-			printf("Open of pipe %s failed with error (%s)\n", pipe_name, cli_errstr(cli1));
+		status = cli_ntcreate(cli1, pipe_name, 0, FILE_READ_DATA,
+				      FILE_ATTRIBUTE_NORMAL,
+				      FILE_SHARE_READ|FILE_SHARE_WRITE,
+				      FILE_OPEN_IF, 0, 0, &fnum);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("Open of pipe %s failed with error (%s)\n", pipe_name, nt_errstr(status));
 			break;
 		}
 		num_pipes++;
@@ -4475,27 +4671,31 @@ static bool run_opentest(int dummy)
 	}
 
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_sockopt(cli1, sockops);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_setatr(cli1, fname, aRONLY, 0))) {
-		printf("cli_setatr failed (%s)\n", cli_errstr(cli1));
+	status = cli_setatr(cli1, fname, FILE_ATTRIBUTE_READONLY, 0);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("cli_setatr failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDONLY, DENY_WRITE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDONLY, DENY_WRITE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -4515,8 +4715,9 @@ static bool run_opentest(int dummy)
 
 	cli_setatr(cli1, fname, 0, 0);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDONLY, DENY_WRITE, &fnum1))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDONLY, DENY_WRITE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -4528,19 +4729,20 @@ static bool run_opentest(int dummy)
 		printf("correct error code ERRDOS/ERRbadshare returned\n");
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("finished open test 2\n");
 
 	/* Test truncate open disposition on file opened for read. */
-
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1))) {
-		printf("(3) open (1) of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("(3) open (1) of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -4548,19 +4750,22 @@ static bool run_opentest(int dummy)
 
 	memset(buf, '\0', 20);
 
-	if (cli_write(cli1, fnum1, 0, buf, 0, 20) != 20) {
-		printf("write failed (%s)\n", cli_errstr(cli1));
+	status = cli_writeall(cli1, fnum1, 0, (uint8_t *)buf, 0, 20, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("write failed (%s)\n", nt_errstr(status));
 		correct = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("(3) close1 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("(3) close1 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
 	/* Ensure size == 20. */
-	if (!NT_STATUS_IS_OK(cli_getatr(cli1, fname, NULL, &fsize, NULL))) {
-		printf("(3) getatr failed (%s)\n", cli_errstr(cli1));
+	status = cli_getatr(cli1, fname, NULL, &fsize, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("(3) getatr failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -4570,20 +4775,22 @@ static bool run_opentest(int dummy)
 	}
 
 	/* Now test if we can truncate a file opened for readonly. */
-
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDONLY|O_TRUNC, DENY_NONE, &fnum1))) {
-		printf("(3) open (2) of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDONLY|O_TRUNC, DENY_NONE, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("(3) open (2) of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close2 failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close2 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
 	/* Ensure size == 0. */
-	if (!NT_STATUS_IS_OK(cli_getatr(cli1, fname, NULL, &fsize, NULL))) {
-		printf("(3) getatr failed (%s)\n", cli_errstr(cli1));
+	status = cli_getatr(cli1, fname, NULL, &fsize, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("(3) getatr failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -4593,19 +4800,24 @@ static bool run_opentest(int dummy)
 	}
 	printf("finished open test 3\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("Do ctemp tests\n");
-	if (!NT_STATUS_IS_OK(cli_ctemp(cli1, talloc_tos(), "\\", &fnum1, &tmp_path))) {
-		printf("ctemp failed (%s)\n", cli_errstr(cli1));
+	status = cli_ctemp(cli1, talloc_tos(), "\\", &fnum1, &tmp_path);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("ctemp failed (%s)\n", nt_errstr(status));
 		return False;
 	}
+
 	printf("ctemp gave path %s\n", tmp_path);
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close of temp failed (%s)\n", cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close of temp failed (%s)\n", nt_errstr(status));
 	}
-	if (!NT_STATUS_IS_OK(cli_unlink(cli1, tmp_path, aSYSTEM | aHIDDEN))) {
-		printf("unlink of temp failed (%s)\n", cli_errstr(cli1));
+
+	status = cli_unlink(cli1, tmp_path, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("unlink of temp failed (%s)\n", nt_errstr(status));
 	}
 
 	/* Test the non-io opens... */
@@ -4615,137 +4827,174 @@ static bool run_opentest(int dummy)
 	}
 
 	cli_setatr(cli2, fname, 0, 0);
-	cli_unlink(cli2, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli2, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	cli_sockopt(cli2, sockops);
 
 	printf("TEST #1 testing 2 non-io opens (no delete)\n");
-
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("TEST #1 open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #1 open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OPEN_IF, 0, 0, &fnum2))) {
-		printf("TEST #1 open 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0, FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OPEN_IF, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #1 open 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("TEST #1 close 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #1 close 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-		printf("TEST #1 close 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #1 close 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	printf("non-io open test #1 passed.\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("TEST #2 testing 2 non-io opens (first with delete)\n");
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, DELETE_ACCESS|FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("TEST #2 open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0,
+	                      DELETE_ACCESS|FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #2 open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OPEN_IF, 0, 0, &fnum2))) {
-		printf("TEST #2 open 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0, FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OPEN_IF, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #2 open 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("TEST #2 close 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #2 close 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-		printf("TEST #2 close 2 of %s failed (%s)\n", fname, cli_errstr(cli1));
+
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #2 close 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	printf("non-io open test #2 passed.\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("TEST #3 testing 2 non-io opens (second with delete)\n");
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("TEST #3 open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, FILE_READ_ATTRIBUTES,
+	                      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #3 open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, DELETE_ACCESS|FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OPEN_IF, 0, 0, &fnum2))) {
-		printf("TEST #3 open 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0,
+	                      DELETE_ACCESS|FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OPEN_IF, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #3 open 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("TEST #3 close 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #3 close 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-		printf("TEST #3 close 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #3 close 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	printf("non-io open test #3 passed.\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("TEST #4 testing 2 non-io opens (both with delete)\n");
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, DELETE_ACCESS|FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("TEST #4 open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0,
+			       DELETE_ACCESS|FILE_READ_ATTRIBUTES,
+			       FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			       FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #4 open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, DELETE_ACCESS|FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OPEN_IF, 0, 0, &fnum2))) {
-		printf("TEST #4 open 2 of %s SUCCEEDED - should have failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0,
+			      DELETE_ACCESS|FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OPEN_IF, 0, 0, &fnum2);
+	if (NT_STATUS_IS_OK(status)) {
+		printf("TEST #4 open 2 of %s SUCCEEDED - should have failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	printf("TEST #4 open 2 of %s gave %s (correct error should be %s)\n", fname, cli_errstr(cli2), "sharing violation");
+	printf("TEST #4 open 2 of %s gave %s (correct error should be %s)\n", fname, nt_errstr(status), "sharing violation");
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("TEST #4 close 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #4 close 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	printf("non-io open test #4 passed.\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("TEST #5 testing 2 non-io opens (both with delete - both with file share delete)\n");
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, DELETE_ACCESS|FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_DELETE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("TEST #5 open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0,
+	                      DELETE_ACCESS|FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_DELETE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #5 open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, DELETE_ACCESS|FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_DELETE, FILE_OPEN_IF, 0, 0, &fnum2))) {
-		printf("TEST #5 open 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0,
+			      DELETE_ACCESS|FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_DELETE,
+			      FILE_OPEN_IF, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #5 open 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("TEST #5 close 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #5 close 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-		printf("TEST #5 close 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #5 close 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -4753,27 +5002,33 @@ static bool run_opentest(int dummy)
 
 	printf("TEST #6 testing 1 non-io open, one io open\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("TEST #6 open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, FILE_READ_DATA,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #6 open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_READ, FILE_OPEN_IF, 0, 0, &fnum2))) {
-		printf("TEST #6 open 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0, FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ,
+			      FILE_OPEN_IF, 0, 0, &fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #6 open 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("TEST #6 close 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #6 close 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli2, fnum2))) {
-		printf("TEST #6 close 2 of %s failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_close(cli2, fnum2);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #6 close 2 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -4781,30 +5036,37 @@ static bool run_opentest(int dummy)
 
 	printf("TEST #7 testing 1 non-io open, one io open with delete\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-		printf("TEST #7 open 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_ntcreate(cli1, fname, 0, FILE_READ_DATA,
+			      FILE_ATTRIBUTE_NORMAL, FILE_SHARE_NONE,
+			      FILE_OVERWRITE_IF, 0, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #7 open 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	if (NT_STATUS_IS_OK(cli_ntcreate(cli2, fname, 0, DELETE_ACCESS|FILE_READ_ATTRIBUTES, FILE_ATTRIBUTE_NORMAL,
-				   FILE_SHARE_READ|FILE_SHARE_DELETE, FILE_OPEN_IF, 0, 0, &fnum2))) {
-		printf("TEST #7 open 2 of %s SUCCEEDED - should have failed (%s)\n", fname, cli_errstr(cli2));
+	status = cli_ntcreate(cli2, fname, 0,
+			      DELETE_ACCESS|FILE_READ_ATTRIBUTES,
+			      FILE_ATTRIBUTE_NORMAL,
+			      FILE_SHARE_READ|FILE_SHARE_DELETE,
+			      FILE_OPEN_IF, 0, 0, &fnum2);
+	if (NT_STATUS_IS_OK(status)) {
+		printf("TEST #7 open 2 of %s SUCCEEDED - should have failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
-	printf("TEST #7 open 2 of %s gave %s (correct error should be %s)\n", fname, cli_errstr(cli2), "sharing violation");
+	printf("TEST #7 open 2 of %s gave %s (correct error should be %s)\n", fname, nt_errstr(status), "sharing violation");
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("TEST #7 close 1 of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #7 close 1 of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
 	printf("non-io open test #7 passed.\n");
 
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("TEST #8 testing open without WRITE_ATTRIBUTES, updating close write time.\n");
 	status = cli_ntcreate(cli1, fname, 0, FILE_WRITE_DATA, FILE_ATTRIBUTE_NORMAL,
@@ -4817,8 +5079,10 @@ static bool run_opentest(int dummy)
 	}
 
 	/* Write to ensure we have to update the file time. */
-	if (cli_write(cli1, fnum1, 0, "TEST DATA\n", 0, 10) != 10) {
-		printf("TEST #8 cli_write failed: %s\n", cli_errstr(cli1));
+	status = cli_writeall(cli1, fnum1, 0, (const uint8_t *)"TEST DATA\n", 0, 10,
+			      NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("TEST #8 cli_write failed: %s\n", nt_errstr(status));
 		correct = false;
 		goto out;
 	}
@@ -4911,11 +5175,79 @@ static bool run_simple_posix_open_test(int dummy)
 	cli_posix_unlink(cli1, sname);
 
 	/* Create a directory. */
-	if (!NT_STATUS_IS_OK(cli_posix_mkdir(cli1, dname, 0777))) {
-		printf("POSIX mkdir of %s failed (%s)\n", dname, cli_errstr(cli1));
+	status = cli_posix_mkdir(cli1, dname, 0777);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX mkdir of %s failed (%s)\n", dname, nt_errstr(status));
+		goto out;
+	}
+
+	status = cli_posix_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL,
+				0600, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX create of %s failed (%s)\n", fname, nt_errstr(status));
 		goto out;
 	}
 
+	/* Test ftruncate - set file size. */
+	status = cli_ftruncate(cli1, fnum1, 1000);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("ftruncate failed (%s)\n", nt_errstr(status));
+		goto out;
+	}
+
+	/* Ensure st_size == 1000 */
+	status = cli_posix_stat(cli1, fname, &sbuf);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("stat failed (%s)\n", nt_errstr(status));
+		goto out;
+	}
+
+	if (sbuf.st_ex_size != 1000) {
+		printf("ftruncate - stat size (%u) != 1000\n", (unsigned int)sbuf.st_ex_size);
+		goto out;
+	}
+
+	/* Test ftruncate - set file size back to zero. */
+	status = cli_ftruncate(cli1, fnum1, 0);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("ftruncate failed (%s)\n", nt_errstr(status));
+		goto out;
+	}
+
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close failed (%s)\n", nt_errstr(status));
+		goto out;
+	}
+
+	/* Now open the file again for read only. */
+	status = cli_posix_open(cli1, fname, O_RDONLY, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX open of %s failed (%s)\n", fname, nt_errstr(status));
+		goto out;
+	}
+
+	/* Now unlink while open. */
+	status = cli_posix_unlink(cli1, fname);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX unlink of %s failed (%s)\n", fname, nt_errstr(status));
+		goto out;
+	}
+
+	status = cli_close(cli1, fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("close(2) failed (%s)\n", nt_errstr(status));
+		goto out;
+	}
+
+	/* Ensure the file has gone. */
+	status = cli_posix_open(cli1, fname, O_RDONLY, 0, &fnum1);
+	if (NT_STATUS_IS_OK(status)) {
+		printf("POSIX open of %s succeeded, should have been deleted.\n", fname);
+		goto out;
+	}
+
+	/* Create again to test open with O_TRUNC. */
 	if (!NT_STATUS_IS_OK(cli_posix_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, 0600, &fnum1))) {
 		printf("POSIX create of %s failed (%s)\n", fname, cli_errstr(cli1));
 		goto out;
@@ -4938,42 +5270,48 @@ static bool run_simple_posix_open_test(int dummy)
 		goto out;
 	}
 
-	/* Test ftruncate - set file size back to zero. */
-	if (!NT_STATUS_IS_OK(cli_ftruncate(cli1, fnum1, 0))) {
-		printf("ftruncate failed (%s)\n", cli_errstr(cli1));
+	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
+		printf("close(2) failed (%s)\n", cli_errstr(cli1));
 		goto out;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close failed (%s)\n", cli_errstr(cli1));
+	/* Re-open with O_TRUNC. */
+	if (!NT_STATUS_IS_OK(cli_posix_open(cli1, fname, O_WRONLY|O_TRUNC, 0600, &fnum1))) {
+		printf("POSIX create of %s failed (%s)\n", fname, cli_errstr(cli1));
 		goto out;
 	}
 
-	/* Now open the file again for read only. */
-	if (!NT_STATUS_IS_OK(cli_posix_open(cli1, fname, O_RDONLY, 0, &fnum1))) {
-		printf("POSIX open of %s failed (%s)\n", fname, cli_errstr(cli1));
+	/* Ensure st_size == 0 */
+	if (!NT_STATUS_IS_OK(cli_posix_stat(cli1, fname, &sbuf))) {
+		printf("stat failed (%s)\n", cli_errstr(cli1));
 		goto out;
 	}
 
-	/* Now unlink while open. */
-	if (!NT_STATUS_IS_OK(cli_posix_unlink(cli1, fname))) {
-		printf("POSIX unlink of %s failed (%s)\n", fname, cli_errstr(cli1));
+	if (sbuf.st_ex_size != 0) {
+		printf("O_TRUNC - stat size (%u) != 0\n", (unsigned int)sbuf.st_ex_size);
 		goto out;
 	}
 
 	if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-		printf("close(2) failed (%s)\n", cli_errstr(cli1));
+		printf("close failed (%s)\n", cli_errstr(cli1));
 		goto out;
 	}
 
-	/* Ensure the file has gone. */
-	if (NT_STATUS_IS_OK(cli_posix_open(cli1, fname, O_RDONLY, 0, &fnum1))) {
-		printf("POSIX open of %s succeeded, should have been deleted.\n", fname);
+	if (!NT_STATUS_IS_OK(cli_posix_unlink(cli1, fname))) {
+		printf("POSIX unlink of %s failed (%s)\n", fname, cli_errstr(cli1));
+		goto out;
+	}
+
+	if (!NT_STATUS_IS_OK(cli_posix_open(cli1, dname, O_RDONLY, 0, &fnum1))) {
+		printf("POSIX open directory O_RDONLY of %s failed (%s)\n",
+			dname, cli_errstr(cli1));
 		goto out;
 	}
 
-	/* What happens when we try and POSIX open a directory ? */
-	if (NT_STATUS_IS_OK(cli_posix_open(cli1, dname, O_RDONLY, 0, &fnum1))) {
+	cli_close(cli1, fnum1);
+
+	/* What happens when we try and POSIX open a directory for write ? */
+	if (NT_STATUS_IS_OK(cli_posix_open(cli1, dname, O_RDWR, 0, &fnum1))) {
 		printf("POSIX open of directory %s succeeded, should have failed.\n", fname);
 		goto out;
 	} else {
@@ -4984,34 +5322,41 @@ static bool run_simple_posix_open_test(int dummy)
 	}
 
 	/* Create the file. */
-	if (!NT_STATUS_IS_OK(cli_posix_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, 0600, &fnum1))) {
-		printf("POSIX create of %s failed (%s)\n", fname, cli_errstr(cli1));
+	status = cli_posix_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL,
+				0600, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX create of %s failed (%s)\n", fname, nt_errstr(status));
 		goto out;
 	}
 
 	/* Write some data into it. */
-	if (cli_write(cli1, fnum1, 0, "TEST DATA\n", 0, 10) != 10) {
-		printf("cli_write failed: %s\n", cli_errstr(cli1));
+	status = cli_writeall(cli1, fnum1, 0, (const uint8_t *)"TEST DATA\n", 0, 10,
+			      NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("cli_write failed: %s\n", nt_errstr(status));
 		goto out;
 	}
 
 	cli_close(cli1, fnum1);
 
 	/* Now create a hardlink. */
-	if (!NT_STATUS_IS_OK(cli_posix_hardlink(cli1, fname, hname))) {
-		printf("POSIX hardlink of %s failed (%s)\n", hname, cli_errstr(cli1));
+	status = cli_posix_hardlink(cli1, fname, hname);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX hardlink of %s failed (%s)\n", hname, nt_errstr(status));
 		goto out;
 	}
 
 	/* Now create a symlink. */
-	if (!NT_STATUS_IS_OK(cli_posix_symlink(cli1, fname, sname))) {
-		printf("POSIX symlink of %s failed (%s)\n", sname, cli_errstr(cli1));
+	status = cli_posix_symlink(cli1, fname, sname);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX symlink of %s failed (%s)\n", sname, nt_errstr(status));
 		goto out;
 	}
 
 	/* Open the hardlink for read. */
-	if (!NT_STATUS_IS_OK(cli_posix_open(cli1, hname, O_RDONLY, 0, &fnum1))) {
-		printf("POSIX open of %s failed (%s)\n", hname, cli_errstr(cli1));
+	status = cli_posix_open(cli1, hname, O_RDONLY, 0, &fnum1);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX open of %s failed (%s)\n", hname, nt_errstr(status));
 		goto out;
 	}
 
@@ -5026,14 +5371,16 @@ static bool run_simple_posix_open_test(int dummy)
 	}
 
 	/* Do a POSIX lock/unlock. */
-	if (!NT_STATUS_IS_OK(cli_posix_lock(cli1, fnum1, 0, 100, true, READ_LOCK))) {
-		printf("POSIX lock failed %s\n", cli_errstr(cli1));
+	status = cli_posix_lock(cli1, fnum1, 0, 100, true, READ_LOCK);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX lock failed %s\n", nt_errstr(status));
 		goto out;
 	}
 
 	/* Punch a hole in the locked area. */
-	if (!NT_STATUS_IS_OK(cli_posix_unlock(cli1, fnum1, 10, 80))) {
-		printf("POSIX unlock failed %s\n", cli_errstr(cli1));
+	status = cli_posix_unlock(cli1, fnum1, 10, 80);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX unlock failed %s\n", nt_errstr(status));
 		goto out;
 	}
 
@@ -5041,7 +5388,8 @@ static bool run_simple_posix_open_test(int dummy)
 
 	/* Open the symlink for read - this should fail. A POSIX
 	   client should not be doing opens on a symlink. */
-	if (NT_STATUS_IS_OK(cli_posix_open(cli1, sname, O_RDONLY, 0, &fnum1))) {
+	status = cli_posix_open(cli1, sname, O_RDONLY, 0, &fnum1);
+	if (NT_STATUS_IS_OK(status)) {
 		printf("POSIX open of %s succeeded (should have failed)\n", sname);
 		goto out;
 	} else {
@@ -5050,13 +5398,14 @@ static bool run_simple_posix_open_test(int dummy)
 			printf("POSIX open of %s should have failed "
 				"with NT_STATUS_OBJECT_PATH_NOT_FOUND, "
 				"failed with %s instead.\n",
-				sname, cli_errstr(cli1));
+				sname, nt_errstr(status));
 			goto out;
 		}
 	}
 
-	if (!NT_STATUS_IS_OK(cli_posix_readlink(cli1, sname, namebuf, sizeof(namebuf)))) {
-		printf("POSIX readlink on %s failed (%s)\n", sname, cli_errstr(cli1));
+	status = cli_posix_readlink(cli1, sname, namebuf, sizeof(namebuf));
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX readlink on %s failed (%s)\n", sname, nt_errstr(status));
 		goto out;
 	}
 
@@ -5066,8 +5415,9 @@ static bool run_simple_posix_open_test(int dummy)
 		goto out;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_posix_rmdir(cli1, dname))) {
-		printf("POSIX rmdir failed (%s)\n", cli_errstr(cli1));
+	status = cli_posix_rmdir(cli1, dname);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("POSIX rmdir failed (%s)\n", nt_errstr(status));
 		goto out;
 	}
 
@@ -5162,6 +5512,7 @@ static bool run_openattrtest(int dummy)
 	bool correct = True;
 	uint16 attr;
 	unsigned int i, j, k, l;
+	NTSTATUS status;
 
 	printf("starting open attr test\n");
 
@@ -5173,34 +5524,43 @@ static bool run_openattrtest(int dummy)
 
 	for (k = 0, i = 0; i < sizeof(open_attrs_table)/sizeof(uint32); i++) {
 		cli_setatr(cli1, fname, 0, 0);
-		cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
-		if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_WRITE_DATA, open_attrs_table[i],
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 0, 0, &fnum1))) {
-			printf("open %d (1) of %s failed (%s)\n", i, fname, cli_errstr(cli1));
+		cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+
+		status = cli_ntcreate(cli1, fname, 0, FILE_WRITE_DATA,
+				       open_attrs_table[i], FILE_SHARE_NONE,
+				       FILE_OVERWRITE_IF, 0, 0, &fnum1);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("open %d (1) of %s failed (%s)\n", i, fname, nt_errstr(status));
 			return False;
 		}
 
-		if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-			printf("close %d (1) of %s failed (%s)\n", i, fname, cli_errstr(cli1));
+		status = cli_close(cli1, fnum1);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("close %d (1) of %s failed (%s)\n", i, fname, nt_errstr(status));
 			return False;
 		}
 
 		for (j = 0; j < sizeof(open_attrs_table)/sizeof(uint32); j++) {
-			if (!NT_STATUS_IS_OK(cli_ntcreate(cli1, fname, 0, FILE_READ_DATA|FILE_WRITE_DATA, open_attrs_table[j],
-					   FILE_SHARE_NONE, FILE_OVERWRITE, 0, 0, &fnum1))) {
+			status = cli_ntcreate(cli1, fname, 0,
+					      FILE_READ_DATA|FILE_WRITE_DATA,
+					      open_attrs_table[j],
+					      FILE_SHARE_NONE, FILE_OVERWRITE,
+					      0, 0, &fnum1);
+			if (!NT_STATUS_IS_OK(status)) {
 				for (l = 0; l < sizeof(attr_results)/sizeof(struct trunc_open_results); l++) {
 					if (attr_results[l].num == k) {
 						printf("[%d] trunc open 0x%x -> 0x%x of %s failed - should have succeeded !(0x%x:%s)\n",
 								k, open_attrs_table[i],
 								open_attrs_table[j],
-								fname, NT_STATUS_V(cli_nt_error(cli1)), cli_errstr(cli1));
+								fname, NT_STATUS_V(status), nt_errstr(status));
 						correct = False;
 					}
 				}
-				if (NT_STATUS_V(cli_nt_error(cli1)) != NT_STATUS_V(NT_STATUS_ACCESS_DENIED)) {
+
+				if (!NT_STATUS_EQUAL(status, NT_STATUS_ACCESS_DENIED)) {
 					printf("[%d] trunc open 0x%x -> 0x%x failed with wrong error code %s\n",
 							k, open_attrs_table[i], open_attrs_table[j],
-							cli_errstr(cli1));
+							nt_errstr(status));
 					correct = False;
 				}
 #if 0
@@ -5210,13 +5570,15 @@ static bool run_openattrtest(int dummy)
 				continue;
 			}
 
-			if (!NT_STATUS_IS_OK(cli_close(cli1, fnum1))) {
-				printf("close %d (2) of %s failed (%s)\n", j, fname, cli_errstr(cli1));
+			status = cli_close(cli1, fnum1);
+			if (!NT_STATUS_IS_OK(status)) {
+				printf("close %d (2) of %s failed (%s)\n", j, fname, nt_errstr(status));
 				return False;
 			}
 
-			if (!NT_STATUS_IS_OK(cli_getatr(cli1, fname, &attr, NULL, NULL))) {
-				printf("getatr(2) failed (%s)\n", cli_errstr(cli1));
+			status = cli_getatr(cli1, fname, &attr, NULL, NULL);
+			if (!NT_STATUS_IS_OK(status)) {
+				printf("getatr(2) failed (%s)\n", nt_errstr(status));
 				return False;
 			}
 
@@ -5245,7 +5607,7 @@ static bool run_openattrtest(int dummy)
 	}
 
 	cli_setatr(cli1, fname, 0, 0);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	printf("open attr test %s.\n", correct ? "passed" : "failed");
 
@@ -5316,7 +5678,7 @@ static bool run_dirtest(int dummy)
 	for (i=0;i<torture_numops;i++) {
 		fstring fname;
 		slprintf(fname, sizeof(fname), "\\%x", (int)random());
-		cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+		cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	}
 
 	if (!torture_close_connection(cli)) {
@@ -5338,11 +5700,11 @@ static NTSTATUS del_fn(const char *mnt, struct file_info *finfo, const char *mas
 	if (strcmp(finfo->name, ".") == 0 || strcmp(finfo->name, "..") == 0)
 		return NT_STATUS_OK;
 
-	if (finfo->mode & aDIR) {
+	if (finfo->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		if (!NT_STATUS_IS_OK(cli_rmdir(pcli, fname)))
 			printf("del_fn: failed to rmdir %s\n,", fname );
 	} else {
-		if (!NT_STATUS_IS_OK(cli_unlink(pcli, fname, aSYSTEM | aHIDDEN)))
+		if (!NT_STATUS_IS_OK(cli_unlink(pcli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN)))
 			printf("del_fn: failed to unlink %s\n,", fname );
 	}
 	return NT_STATUS_OK;
@@ -5367,10 +5729,11 @@ bool torture_ioctl_test(int dummy)
 
 	printf("starting ioctl test\n");
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum))) {
-		printf("open of %s failed (%s)\n", fname, cli_errstr(cli));
+	status = cli_open(cli, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open of %s failed (%s)\n", fname, nt_errstr(status));
 		return False;
 	}
 
@@ -5411,6 +5774,7 @@ bool torture_chkpath_test(int dummy)
 	static struct cli_state *cli;
 	uint16_t fnum;
 	bool ret;
+	NTSTATUS status;
 
 	if (!torture_open_connection(&cli, 0)) {
 		return False;
@@ -5420,36 +5784,43 @@ bool torture_chkpath_test(int dummy)
 
 	/* cleanup from an old run */
 	cli_rmdir(cli, "\\chkpath.dir\\dir2");
-	cli_unlink(cli, "\\chkpath.dir\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\chkpath.dir\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, "\\chkpath.dir");
 
-	if (!NT_STATUS_IS_OK(cli_mkdir(cli, "\\chkpath.dir"))) {
-		printf("mkdir1 failed : %s\n", cli_errstr(cli));
+	status = cli_mkdir(cli, "\\chkpath.dir");
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("mkdir1 failed : %s\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_mkdir(cli, "\\chkpath.dir\\dir2"))) {
-		printf("mkdir2 failed : %s\n", cli_errstr(cli));
+	status = cli_mkdir(cli, "\\chkpath.dir\\dir2");
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("mkdir2 failed : %s\n", nt_errstr(status));
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, "\\chkpath.dir\\foo.txt", O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum))) {
-		printf("open1 failed (%s)\n", cli_errstr(cli));
+	status = cli_open(cli, "\\chkpath.dir\\foo.txt", O_RDWR|O_CREAT|O_EXCL,
+			  DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open1 failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 	cli_close(cli, fnum);
 
-	if (!NT_STATUS_IS_OK(cli_chkpath(cli, "\\chkpath.dir"))) {
-		printf("chkpath1 failed: %s\n", cli_errstr(cli));
+	status = cli_chkpath(cli, "\\chkpath.dir");
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("chkpath1 failed: %s\n", nt_errstr(status));
 		ret = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_chkpath(cli, "\\chkpath.dir\\dir2"))) {
-		printf("chkpath2 failed: %s\n", cli_errstr(cli));
+	status = cli_chkpath(cli, "\\chkpath.dir\\dir2");
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("chkpath2 failed: %s\n", nt_errstr(status));
 		ret = False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_chkpath(cli, "\\chkpath.dir\\foo.txt"))) {
+	status = cli_chkpath(cli, "\\chkpath.dir\\foo.txt");
+	if (!NT_STATUS_IS_OK(status)) {
 		ret = check_error(__LINE__, cli, ERRDOS, ERRbadpath, 
 				  NT_STATUS_NOT_A_DIRECTORY);
 	} else {
@@ -5474,7 +5845,7 @@ bool torture_chkpath_test(int dummy)
 	}
 
 	cli_rmdir(cli, "\\chkpath.dir\\dir2");
-	cli_unlink(cli, "\\chkpath.dir\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\chkpath.dir\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, "\\chkpath.dir");
 
 	if (!torture_close_connection(cli)) {
@@ -5503,12 +5874,14 @@ static bool run_eatest(int dummy)
 		return False;
 	}
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli, fname, 0,
-				   FIRST_DESIRED_ACCESS, FILE_ATTRIBUTE_ARCHIVE,
-				   FILE_SHARE_NONE, FILE_OVERWRITE_IF, 
-				   0x4044, 0, &fnum))) {
-		printf("open failed - %s\n", cli_errstr(cli));
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+
+	status = cli_ntcreate(cli, fname, 0,
+                              FIRST_DESIRED_ACCESS, FILE_ATTRIBUTE_ARCHIVE,
+                              FILE_SHARE_NONE, FILE_OVERWRITE_IF,
+                              0x4044, 0, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open failed - %s\n", nt_errstr(status));
 		talloc_destroy(mem_ctx);
 		return False;
 	}
@@ -5631,7 +6004,7 @@ static bool run_dirtest1(int dummy)
 	cli_sockopt(cli, sockops);
 
 	cli_list(cli, "\\LISTDIR\\*", 0, del_fn, cli);
-	cli_list(cli, "\\LISTDIR\\*", aDIR, del_fn, cli);
+	cli_list(cli, "\\LISTDIR\\*", FILE_ATTRIBUTE_DIRECTORY, del_fn, cli);
 	cli_rmdir(cli, "\\LISTDIR");
 	cli_mkdir(cli, "\\LISTDIR");
 
@@ -5657,7 +6030,7 @@ static bool run_dirtest1(int dummy)
 
 	/* Now ensure that doing an old list sees both files and directories. */
 	num_seen = 0;
-	cli_list_old(cli, "\\LISTDIR\\*", aDIR, list_fn, &num_seen);
+	cli_list_old(cli, "\\LISTDIR\\*", FILE_ATTRIBUTE_DIRECTORY, list_fn, &num_seen);
 	printf("num_seen = %d\n", num_seen );
 	/* We should see 100 files + 1000 directories + . and .. */
 	if (num_seen != 2002)
@@ -5667,20 +6040,20 @@ static bool run_dirtest1(int dummy)
 	 * relevent entries.
 	 */
 	num_seen = 0;
-	cli_list_old(cli, "\\LISTDIR\\*", (aDIR<<8)|aDIR, list_fn, &num_seen);
+	cli_list_old(cli, "\\LISTDIR\\*", (FILE_ATTRIBUTE_DIRECTORY<<8)|FILE_ATTRIBUTE_DIRECTORY, list_fn, &num_seen);
 	printf("num_seen = %d\n", num_seen );
 	if (num_seen != 1002)
 		correct = False;
 
 	num_seen = 0;
-	cli_list_old(cli, "\\LISTDIR\\*", (aARCH<<8)|aDIR, list_fn, &num_seen);
+	cli_list_old(cli, "\\LISTDIR\\*", (FILE_ATTRIBUTE_ARCHIVE<<8)|FILE_ATTRIBUTE_DIRECTORY, list_fn, &num_seen);
 	printf("num_seen = %d\n", num_seen );
 	if (num_seen != 1000)
 		correct = False;
 
 	/* Delete everything. */
 	cli_list(cli, "\\LISTDIR\\*", 0, del_fn, cli);
-	cli_list(cli, "\\LISTDIR\\*", aDIR, del_fn, cli);
+	cli_list(cli, "\\LISTDIR\\*", FILE_ATTRIBUTE_DIRECTORY, del_fn, cli);
 	cli_rmdir(cli, "\\LISTDIR");
 
 #if 0
@@ -5730,9 +6103,9 @@ static bool run_error_map_extract(int dummy) {
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_session_setup(c_nt, "", "", 0, "", 0,
-					       workgroup))) {
-		printf("%s rejected the NT-error initial session setup (%s)\n",host, cli_errstr(c_nt));
+	status = cli_session_setup(c_nt, "", "", 0, "", 0, workgroup);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("%s rejected the NT-error initial session setup (%s)\n",host, nt_errstr(status));
 		return False;
 	}
 
@@ -5753,19 +6126,21 @@ static bool run_error_map_extract(int dummy) {
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_session_setup(c_dos, "", "", 0, "", 0,
-					       workgroup))) {
-		printf("%s rejected the DOS-error initial session setup (%s)\n",host, cli_errstr(c_dos));
+	status = cli_session_setup(c_dos, "", "", 0, "", 0, workgroup);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("%s rejected the DOS-error initial session setup (%s)\n",
+			host, nt_errstr(status));
 		return False;
 	}
 
 	for (error=(0xc0000000 | 0x1); error < (0xc0000000| 0xFFF); error++) {
 		fstr_sprintf(user, "%X", error);
 
-		if (NT_STATUS_IS_OK(cli_session_setup(c_nt, user, 
-						      password, strlen(password),
-						      password, strlen(password),
-						      workgroup))) {
+		status = cli_session_setup(c_nt, user,
+					   password, strlen(password),
+					   password, strlen(password),
+					   workgroup);
+		if (NT_STATUS_IS_OK(status)) {
 			printf("/** Session setup succeeded.  This shouldn't happen...*/\n");
 		}
 
@@ -5799,14 +6174,14 @@ static bool run_error_map_extract(int dummy) {
 
 		if (NT_STATUS_V(nt_status) != error) { 
 			printf("/*\t{ This NT error code was 'sqashed'\n\t from %s to %s \n\t during the session setup }\n*/\n", 
-			       get_nt_error_c_code(NT_STATUS(error)), 
-			       get_nt_error_c_code(nt_status));
+			       get_nt_error_c_code(talloc_tos(), NT_STATUS(error)), 
+			       get_nt_error_c_code(talloc_tos(), nt_status));
 		}
 
 		printf("\t{%s,\t%s,\t%s},\n", 
 		       smb_dos_err_class(errclass), 
 		       smb_dos_err_name(errclass, errnum), 
-		       get_nt_error_c_code(NT_STATUS(error)));
+		       get_nt_error_c_code(talloc_tos(), NT_STATUS(error)));
 	}
 	return True;
 }
@@ -5823,11 +6198,11 @@ static bool run_sesssetup_bench(int dummy)
 		return false;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(
-			c, fname, 0, GENERIC_ALL_ACCESS|DELETE_ACCESS,
-			FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF,
-			FILE_DELETE_ON_CLOSE, 0, &fnum))) {
-		d_printf("open %s failed: %s\n", fname, cli_errstr(c));
+	status = cli_ntcreate(c, fname, 0, GENERIC_ALL_ACCESS|DELETE_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF,
+			      FILE_DELETE_ON_CLOSE, 0, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		d_printf("open %s failed: %s\n", fname, nt_errstr(status));
 		return false;
 	}
 
@@ -5936,7 +6311,7 @@ static bool run_chain1(int dummy)
 
 
 	reqs[1] = cli_write_andx_create(talloc_tos(), evt, cli1, 0, 0,
-					(uint8_t *)str, 0, strlen(str)+1,
+					(const uint8_t *)str, 0, strlen(str)+1,
 					smbreqs, 1, &smbreqs[1]);
 	if (reqs[1] == NULL) return false;
 	tevent_req_set_callback(reqs[1], chain1_write_completion, NULL);
@@ -5983,7 +6358,7 @@ static bool run_chain2(int dummy)
 	NTSTATUS status;
 
 	printf("starting chain2 test\n");
-	status = cli_start_connection(&cli1, global_myname(), host, NULL,
+	status = cli_start_connection(&cli1, lp_netbios_name(), host, NULL,
 				      port_to_use, Undefined, 0);
 	if (!NT_STATUS_IS_OK(status)) {
 		return False;
@@ -6435,10 +6810,11 @@ static bool run_mangle1(int dummy)
 
 	cli_sockopt(cli, sockops);
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(
-			cli, fname, 0, GENERIC_ALL_ACCESS|DELETE_ACCESS,
-			FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF, 0, 0, &fnum))) {
-		d_printf("open %s failed: %s\n", fname, cli_errstr(cli));
+	status = cli_ntcreate(cli, fname, 0, GENERIC_ALL_ACCESS|DELETE_ACCESS,
+			      FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF,
+			      0, 0, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		d_printf("open %s failed: %s\n", fname, nt_errstr(status));
 		return false;
 	}
 	cli_close(cli, fnum);
@@ -6451,9 +6827,10 @@ static bool run_mangle1(int dummy)
 	}
 	d_printf("alt_name: %s\n", alt_name);
 
-	if (!NT_STATUS_IS_OK(cli_open(cli, alt_name, O_RDONLY, DENY_NONE, &fnum))) {
+	status = cli_open(cli, alt_name, O_RDONLY, DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
 		d_printf("cli_open(%s) failed: %s\n", alt_name,
-			 cli_errstr(cli));
+			 nt_errstr(status));
 		return false;
 	}
 	cli_close(cli, fnum);
@@ -6494,14 +6871,16 @@ static bool run_windows_write(int dummy)
 	struct timeval start_time;
 	double seconds;
 	double kbytes;
+	NTSTATUS status;
 
 	printf("starting windows_write test\n");
 	if (!torture_open_connection(&cli1, 0)) {
 		return False;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum))) {
-		printf("open failed (%s)\n", cli_errstr(cli1));
+	status = cli_open(cli1, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE, &fnum);
+	if (!NT_STATUS_IS_OK(status)) {
+		printf("open failed (%s)\n", nt_errstr(status));
 		return False;
 	}
 
@@ -6510,14 +6889,14 @@ static bool run_windows_write(int dummy)
 	start_time = timeval_current();
 
 	for (i=0; i<torture_numops; i++) {
-		char c = 0;
+		uint8_t c = 0;
 		off_t start = i * torture_blocksize;
-		NTSTATUS status;
 		size_t to_pull = torture_blocksize - 1;
 
-		if (cli_write(cli1, fnum, 0, &c,
-			      start + torture_blocksize - 1, 1) != 1) {
-			printf("cli_write failed: %s\n", cli_errstr(cli1));
+		status = cli_writeall(cli1, fnum, 0, &c,
+				      start + torture_blocksize - 1, 1, NULL);
+		if (!NT_STATUS_IS_OK(status)) {
+			printf("cli_write failed: %s\n", nt_errstr(status));
 			goto fail;
 		}
 
@@ -6539,7 +6918,7 @@ static bool run_windows_write(int dummy)
 	ret = true;
  fail:
 	cli_close(cli1, fnum);
-	cli_unlink(cli1, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli1, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	torture_close_connection(cli1);
 	return ret;
 }
@@ -6666,13 +7045,13 @@ static NTSTATUS shortname_del_fn(const char *mnt, struct file_info *finfo,
 	if (strcmp(finfo->name, ".") == 0 || strcmp(finfo->name, "..") == 0)
 		return NT_STATUS_OK;
 
-	if (finfo->mode & aDIR) {
+	if (finfo->mode & FILE_ATTRIBUTE_DIRECTORY) {
 		status = cli_rmdir(pcli, fname);
 		if (!NT_STATUS_IS_OK(status)) {
 			printf("del_fn: failed to rmdir %s\n,", fname );
 		}
 	} else {
-		status = cli_unlink(pcli, fname, aSYSTEM | aHIDDEN);
+		status = cli_unlink(pcli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 		if (!NT_STATUS_IS_OK(status)) {
 			printf("del_fn: failed to unlink %s\n,", fname );
 		}
@@ -6721,6 +7100,7 @@ static bool run_shortname_test(int dummy)
 	int i;
 	struct sn_state s;
 	char fname[20];
+	NTSTATUS status;
 
 	printf("starting shortname test\n");
 
@@ -6731,12 +7111,13 @@ static bool run_shortname_test(int dummy)
 	cli_sockopt(cli, sockops);
 
 	cli_list(cli, "\\shortname\\*", 0, shortname_del_fn, cli);
-	cli_list(cli, "\\shortname\\*", aDIR, shortname_del_fn, cli);
+	cli_list(cli, "\\shortname\\*", FILE_ATTRIBUTE_DIRECTORY, shortname_del_fn, cli);
 	cli_rmdir(cli, "\\shortname");
 
-	if (!NT_STATUS_IS_OK(cli_mkdir(cli, "\\shortname"))) {
+	status = cli_mkdir(cli, "\\shortname");
+	if (!NT_STATUS_IS_OK(status)) {
 		d_printf("(%s) cli_mkdir of \\shortname failed: %s\n",
-			__location__, cli_errstr(cli));
+			__location__, nt_errstr(status));
 		correct = false;
 		goto out;
 	}
@@ -6747,7 +7128,6 @@ static bool run_shortname_test(int dummy)
 	s.val = false;
 
 	for (i = 32; i < 128; i++) {
-		NTSTATUS status;
 		uint16_t fnum = (uint16_t)-1;
 
 		s.i = i;
@@ -6761,7 +7141,7 @@ static bool run_shortname_test(int dummy)
                                    FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OVERWRITE_IF, 0, 0, &fnum);
 		if (!NT_STATUS_IS_OK(status)) {
 			d_printf("(%s) cli_nt_create of %s failed: %s\n",
-				__location__, fname, cli_errstr(cli));
+				__location__, fname, nt_errstr(status));
 			correct = false;
 			goto out;
 		}
@@ -6776,9 +7156,11 @@ static bool run_shortname_test(int dummy)
 			correct = false;
 			goto out;
 		}
-		if (!NT_STATUS_IS_OK(cli_unlink(cli, fname, aSYSTEM | aHIDDEN))) {
+
+		status = cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
+		if (!NT_STATUS_IS_OK(status)) {
 			d_printf("(%s) failed to delete %s: %s\n",
-				__location__, fname, cli_errstr(cli));
+				__location__, fname, nt_errstr(status));
 			correct = false;
 			goto out;
 		}
@@ -6792,7 +7174,7 @@ static bool run_shortname_test(int dummy)
   out:
 
 	cli_list(cli, "\\shortname\\*", 0, shortname_del_fn, cli);
-	cli_list(cli, "\\shortname\\*", aDIR, shortname_del_fn, cli);
+	cli_list(cli, "\\shortname\\*", FILE_ATTRIBUTE_DIRECTORY, shortname_del_fn, cli);
 	cli_rmdir(cli, "\\shortname");
 	torture_close_connection(cli);
 	return correct;
@@ -6921,7 +7303,7 @@ static bool run_dir_createtime(int dummy)
 		return false;
 	}
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, dname);
 
 	status = cli_mkdir(cli, dname);
@@ -6965,7 +7347,7 @@ static bool run_dir_createtime(int dummy)
 
   out:
 
-	cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+	cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, dname);
 	if (!torture_close_connection(cli)) {
 		ret = false;
@@ -6990,7 +7372,7 @@ static bool run_streamerror(int dummy)
 		return false;
 	}
 
-	cli_unlink(cli, "\\testdir\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\testdir\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, dname);
 
 	status = cli_mkdir(cli, dname);
@@ -7573,7 +7955,7 @@ static NTSTATUS split_ntfs_stream_name(TALLOC_CTX *mem_ctx, const char *fname,
 		stype = "$DATA";
 	}
 	else {
-		if (StrCaseCmp(stype, ":$DATA") != 0) {
+		if (strcasecmp_m(stype, ":$DATA") != 0) {
 			/*
 			 * If there is an explicit stream type, so far we only
 			 * allow $DATA. Is there anything else allowed? -- vl
@@ -7822,7 +8204,7 @@ static bool run_local_wbclient(int dummy)
 		goto fail;
 	}
 
-	wb_ctx = TALLOC_ARRAY(ev, struct wb_context *, nprocs);
+	wb_ctx = talloc_array(ev, struct wb_context *, nprocs);
 	if (wb_ctx == NULL) {
 		goto fail;
 	}
@@ -7900,7 +8282,7 @@ static bool run_getaddrinfo_send(int dummy)
 			goto fail;
 		}
 		tevent_req_set_callback(reqs[i], getaddrinfo_finished,
-					(void *)names[i]);
+					discard_const_p(void, names[i]));
 	}
 
 	for (i=0; i<ARRAY_SIZE(reqs); i++) {
@@ -7966,7 +8348,7 @@ static bool run_local_dbtrans(int dummy)
 	}
 
 	res = db->transaction_start(db);
-	if (res == -1) {
+	if (res != 0) {
 		printf(__location__ "transaction_start failed\n");
 		return false;
 	}
@@ -7993,7 +8375,7 @@ static bool run_local_dbtrans(int dummy)
 	TALLOC_FREE(rec);
 
 	res = db->transaction_commit(db);
-	if (res == -1) {
+	if (res != 0) {
 		printf(__location__ "transaction_commit failed\n");
 		return false;
 	}
@@ -8003,7 +8385,7 @@ static bool run_local_dbtrans(int dummy)
 		int i;
 
 		res = db->transaction_start(db);
-		if (res == -1) {
+		if (res != 0) {
 			printf(__location__ "transaction_start failed\n");
 			break;
 		}
@@ -8033,7 +8415,7 @@ static bool run_local_dbtrans(int dummy)
 		printf("val2=%d\r", val2);
 
 		res = db->transaction_commit(db);
-		if (res == -1) {
+		if (res != 0) {
 			printf(__location__ "transaction_commit failed\n");
 			break;
 		}
@@ -8208,7 +8590,6 @@ static struct {
 	{"NBENCH2", run_nbench2, 0},
 	{"OPLOCK1",  run_oplock1, 0},
 	{"OPLOCK2",  run_oplock2, 0},
-	{"OPLOCK3",  run_oplock3, 0},
 	{"OPLOCK4",  run_oplock4, 0},
 	{"DIR",  run_dirtest, 0},
 	{"DIR1",  run_dirtest1, 0},
@@ -8255,6 +8636,7 @@ static struct {
 	{ "CHAIN1", run_chain1, 0},
 	{ "CHAIN2", run_chain2, 0},
 	{ "WINDOWS-WRITE", run_windows_write, 0},
+	{ "NTTRANS-CREATE", run_nttrans_create, 0},
 	{ "CLI_ECHO", run_cli_echo, 0},
 	{ "GETADDRINFO", run_getaddrinfo_send, 0},
 	{ "TLDAP", run_tldap },
@@ -8262,6 +8644,7 @@ static struct {
 	{ "NOTIFY-BENCH", run_notify_bench },
 	{ "BAD-NBT-SESSION", run_bad_nbt_session },
 	{ "SMB-ANY-CONNECT", run_smb_any_connect },
+	{ "NOTIFY-ONLINE", run_notify_online },
 	{ "LOCAL-SUBSTITUTE", run_local_substitute, 0},
 	{ "LOCAL-GENCACHE", run_local_gencache, 0},
 	{ "LOCAL-TALLOC-DICT", run_local_talloc_dict, 0},
@@ -8355,6 +8738,7 @@ static void usage(void)
 	printf("\t-p port\n");
 	printf("\t-s seed\n");
 	printf("\t-b unclist_filename   specify multiple shares for multiple connections\n");
+	printf("\t-f filename   filename to test\n");
 	printf("\n\n");
 
 	printf("tests are:");
@@ -8432,7 +8816,8 @@ static void usage(void)
 
 	fstrcpy(workgroup, lp_workgroup());
 
-	while ((opt = getopt(argc, argv, "p:hW:U:n:N:O:o:m:Ll:d:Aec:ks:b:B:")) != EOF) {
+	while ((opt = getopt(argc, argv, "p:hW:U:n:N:O:o:m:Ll:d:Aec:ks:b:B:f:"))
+	       != EOF) {
 		switch (opt) {
 		case 'p':
 			port_to_use = atoi(optarg);
@@ -8501,6 +8886,9 @@ static void usage(void)
 		case 'B':
 			torture_blocksize = atoi(optarg);
 			break;
+		case 'f':
+			test_filename = SMB_STRDUP(optarg);
+			break;
 		default:
 			printf("Unknown option %c (%d)\n", (char)opt, opt);
 			usage();
diff --git a/source3/torture/utable.c b/source3/torture/utable.c
index e7ee00a8a36..4c6605d9210 100644
--- a/source3/torture/utable.c
+++ b/source3/torture/utable.c
@@ -21,6 +21,7 @@
 #include "system/filesys.h"
 #include "torture/proto.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 
 bool torture_utable(int dummy)
@@ -42,7 +43,7 @@ bool torture_utable(int dummy)
 	memset(valid, 0, sizeof(valid));
 
 	cli_mkdir(cli, "\\utable");
-	cli_unlink(cli, "\\utable\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\utable\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 	for (c=1; c < 0x10000; c++) {
 		size_t size = 0;
@@ -77,7 +78,7 @@ bool torture_utable(int dummy)
 		}
 
 		cli_close(cli, fnum);
-		cli_unlink(cli, fname, aSYSTEM | aHIDDEN);
+		cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 
 		if (c % 100 == 0) {
 			printf("%d (%d/%d)\r", c, chars_allowed, alt_allowed);
@@ -144,7 +145,7 @@ bool torture_casetable(int dummy)
 
 	memset(equiv, 0, sizeof(equiv));
 
-	cli_unlink(cli, "\\utable\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\utable\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, "\\utable");
 	if (!NT_STATUS_IS_OK(cli_mkdir(cli, "\\utable"))) {
 		printf("Failed to create utable directory!\n");
@@ -198,11 +199,12 @@ bool torture_casetable(int dummy)
 			fflush(stdout);
 		}
 
-		cli_write(cli, fnum, 0, (char *)&c, size, sizeof(c));
+		cli_writeall(cli, fnum, 0, (uint8_t *)&c, size, sizeof(c),
+			     NULL);
 		cli_close(cli, fnum);
 	}
 
-	cli_unlink(cli, "\\utable\\*", aSYSTEM | aHIDDEN);
+	cli_unlink(cli, "\\utable\\*", FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN);
 	cli_rmdir(cli, "\\utable");
 
 	return True;
diff --git a/source3/torture/vfstest.c b/source3/torture/vfstest.c
index 584f6e30ef3..490dd2bd529 100644
--- a/source3/torture/vfstest.c
+++ b/source3/torture/vfstest.c
@@ -414,16 +414,6 @@ void exit_server_cleanly(const char *const reason)
 
 int last_message = -1;
 
-struct event_context *smbd_event_context(void)
-{
-	static struct event_context *ctx;
-
-	if (!ctx && !(ctx = event_context_init(NULL))) {
-		smb_panic("Could not init smbd event context\n");
-	}
-	return ctx;
-}
-
 /* Main function */
 
 int main(int argc, char *argv[])
@@ -479,8 +469,8 @@ int main(int argc, char *argv[])
 
 	/* some basic initialization stuff */
 	sec_init();
-	vfs.conn = TALLOC_ZERO_P(NULL, connection_struct);
-	vfs.conn->params = TALLOC_P(vfs.conn, struct share_params);
+	vfs.conn = talloc_zero(NULL, connection_struct);
+	vfs.conn->params = talloc(vfs.conn, struct share_params);
 	for (i=0; i < 1024; i++)
 		vfs.files[i] = NULL;
 
diff --git a/source3/torture/wbc_async.c b/source3/torture/wbc_async.c
index 182474c2167..9252b58bc98 100644
--- a/source3/torture/wbc_async.c
+++ b/source3/torture/wbc_async.c
@@ -551,7 +551,7 @@ struct tevent_req *wb_trans_send(TALLOC_CTX *mem_ctx,
 
 	if (!tevent_queue_add(wb_ctx->queue, ev, req, wb_trans_trigger,
 			      NULL)) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return tevent_req_post(req, ev);
 	}
 	return req;
diff --git a/source3/utils/eventlogadm.c b/source3/utils/eventlogadm.c
index 9ddd9c7df84..d170604a1bf 100644
--- a/source3/utils/eventlogadm.c
+++ b/source3/utils/eventlogadm.c
@@ -171,14 +171,14 @@ static bool eventlog_add_source( const char *eventlog, const char *sourcename,
 
 	if ( !already_in ) {
 		/* make a new list with an additional entry; copy values, add another */
-		wp = TALLOC_ARRAY(ctx, const char *, numsources + 2 );
+		wp = talloc_array(ctx, const char *, numsources + 2 );
 
 		if ( !wp ) {
 			d_printf("talloc() failed \n");
 			return False;
 		}
 		memcpy( wp, wrklist, sizeof( char * ) * numsources );
-		*( wp + numsources ) = ( char * ) sourcename;
+		*( wp + numsources ) = (const char * ) sourcename;
 		*( wp + numsources + 1 ) = NULL;
 		if (!push_reg_multi_sz(ctx, &blob, wp)) {
 			return false;
@@ -365,15 +365,12 @@ static int DoDumpCommand(int argc, char **argv, bool debugflag, char *exename)
 {
 	ELOG_TDB *etdb;
 	TALLOC_CTX *mem_ctx = talloc_tos();
-	const char *tdb_filename;
 	uint32_t count = 1;
 
 	if (argc > 2) {
 		return -1;
 	}
 
-	tdb_filename = argv[0];
-
 	if (argc > 1) {
 		count = atoi(argv[1]);
 	}
@@ -475,16 +472,16 @@ int main( int argc, char *argv[] )
 
 	/*  note that the separate command types should call usage if they need to... */
 	while ( 1 ) {
-		if ( !StrCaseCmp( opname, "addsource" ) ) {
+		if ( !strcasecmp_m( opname, "addsource" ) ) {
 			rc = DoAddSourceCommand( argc, argv, opt_debug,
 						 exename );
 			break;
 		}
-		if ( !StrCaseCmp( opname, "write" ) ) {
+		if ( !strcasecmp_m( opname, "write" ) ) {
 			rc = DoWriteCommand( argc, argv, opt_debug, exename );
 			break;
 		}
-		if ( !StrCaseCmp( opname, "dump" ) ) {
+		if ( !strcasecmp_m( opname, "dump" ) ) {
 			rc = DoDumpCommand( argc, argv, opt_debug, exename );
 			break;
 		}
diff --git a/source3/utils/net.c b/source3/utils/net.c
index c53b2859b09..95b937444ec 100644
--- a/source3/utils/net.c
+++ b/source3/utils/net.c
@@ -243,7 +243,7 @@ static int net_getlocalsid(struct net_context *c, int argc, const char **argv)
 		name = argv[0];
         }
 	else {
-		name = global_myname();
+		name = lp_netbios_name();
 	}
 
 	if(!initialize_password_db(false, NULL)) {
@@ -286,7 +286,7 @@ static int net_setlocalsid(struct net_context *c, int argc, const char **argv)
 		return 1;
 	}
 
-	if (!secrets_store_domain_sid(global_myname(), &sid)) {
+	if (!secrets_store_domain_sid(lp_netbios_name(), &sid)) {
 		DEBUG(0,("Can't store domain SID as a pdc/bdc.\n"));
 		return 1;
 	}
@@ -346,13 +346,13 @@ static int net_getdomainsid(struct net_context *c, int argc, const char **argv)
 	/* Generate one, if it doesn't exist */
 	get_global_sam_sid();
 
-	if (!secrets_fetch_domain_sid(global_myname(), &domain_sid)) {
+	if (!secrets_fetch_domain_sid(lp_netbios_name(), &domain_sid)) {
 		d_fprintf(stderr, _("Could not fetch local SID\n"));
 		return 1;
 	}
 	sid_to_fstring(sid_str, &domain_sid);
 	d_printf(_("SID for local machine %s is: %s\n"),
-		 global_myname(), sid_str);
+		 lp_netbios_name(), sid_str);
 
 	if (!secrets_fetch_domain_sid(c->opt_workgroup, &domain_sid)) {
 		d_fprintf(stderr, _("Could not fetch domain SID\n"));
@@ -880,6 +880,16 @@ static struct functable net_func[] = {
 
 	lp_load(get_dyn_CONFIGFILE(), true, false, false, true);
 
+#if defined(HAVE_BIND_TEXTDOMAIN_CODESET)
+	/* Bind our gettext results to 'unix charset'
+	   
+	   This ensures that the translations and any embedded strings are in the
+	   same charset.  It won't be the one from the user's locale (we no
+	   longer auto-detect that), but it will be self-consistent.
+	*/
+	bind_textdomain_codeset(MODULE_NAME, lp_unix_charset());
+#endif
+
  	argv_new = (const char **)poptGetArgs(pc);
 
 	argc_new = argc;
@@ -895,7 +905,7 @@ static struct functable net_func[] = {
 	}
 
 	if (c->opt_requester_name) {
-		set_global_myname(c->opt_requester_name);
+		lp_set_cmdline("netbios name", c->opt_requester_name);
 	}
 
 	if (!c->opt_user_name && getenv("LOGNAME")) {
diff --git a/source3/utils/net.h b/source3/utils/net.h
index ac43068c12f..1d1559f12d2 100644
--- a/source3/utils/net.h
+++ b/source3/utils/net.h
@@ -22,6 +22,8 @@
  * include
  */
 
+struct cli_state;
+
 #include "../librpc/gen_ndr/lsa.h"
 
 #include "intl.h"
diff --git a/source3/utils/net_ads.c b/source3/utils/net_ads.c
index 4b4b0a80318..bb78ecbf12e 100644
--- a/source3/utils/net_ads.c
+++ b/source3/utils/net_ads.c
@@ -36,6 +36,7 @@
 #include "secrets.h"
 #include "krb5_env.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 
 #ifdef HAVE_ADS
 
@@ -60,7 +61,8 @@ static int net_ads_cldap_netlogon(struct net_context *c, ADS_STRUCT *ads)
 	struct NETLOGON_SAM_LOGON_RESPONSE_EX reply;
 
 	print_sockaddr(addr, sizeof(addr), &ads->ldap.ss);
-	if ( !ads_cldap_netlogon_5(talloc_tos(), addr, ads->server.realm, &reply ) ) {
+
+	if ( !ads_cldap_netlogon_5(talloc_tos(), &ads->ldap.ss, ads->server.realm, &reply ) ) {
 		d_fprintf(stderr, _("CLDAP query failed!\n"));
 		return -1;
 	}
@@ -154,7 +156,7 @@ static int net_ads_lookup(struct net_context *c, int argc, const char **argv)
 	}
 
 	if (!ads->config.realm) {
-		ads->config.realm = CONST_DISCARD(char *, c->opt_target_workgroup);
+		ads->config.realm = discard_const_p(char, c->opt_target_workgroup);
 		ads->ldap.port = 389;
 	}
 
@@ -383,7 +385,6 @@ int net_ads_check(struct net_context *c)
 static int net_ads_workgroup(struct net_context *c, int argc, const char **argv)
 {
 	ADS_STRUCT *ads;
-	char addr[INET6_ADDRSTRLEN];
 	struct NETLOGON_SAM_LOGON_RESPONSE_EX reply;
 
 	if (c->display_usage) {
@@ -401,12 +402,11 @@ static int net_ads_workgroup(struct net_context *c, int argc, const char **argv)
 	}
 
 	if (!ads->config.realm) {
-		ads->config.realm = CONST_DISCARD(char *, c->opt_target_workgroup);
+		ads->config.realm = discard_const_p(char, c->opt_target_workgroup);
 		ads->ldap.port = 389;
 	}
 
-	print_sockaddr(addr, sizeof(addr), &ads->ldap.ss);
-	if ( !ads_cldap_netlogon_5(talloc_tos(), addr, ads->server.realm, &reply ) ) {
+	if ( !ads_cldap_netlogon_5(talloc_tos(), &ads->ldap.ss, ads->server.realm, &reply ) ) {
 		d_fprintf(stderr, _("CLDAP query failed!\n"));
 		ads_destroy(&ads);
 		return -1;
@@ -441,10 +441,10 @@ static bool usergrp_display(ADS_STRUCT *ads, char *field, void **values, void *d
 	}
 	if (!values) /* must be new field, indicate string field */
 		return true;
-	if (StrCaseCmp(field, "sAMAccountName") == 0) {
+	if (strcasecmp_m(field, "sAMAccountName") == 0) {
 		disp_fields[0] = SMB_STRDUP((char *) values[0]);
 	}
-	if (StrCaseCmp(field, "description") == 0)
+	if (strcasecmp_m(field, "description") == 0)
 		disp_fields[1] = SMB_STRDUP((char *) values[0]);
 	return true;
 }
@@ -915,7 +915,7 @@ static int net_ads_status(struct net_context *c, int argc, const char **argv)
 		return -1;
 	}
 
-	rc = ads_find_machine_acct(ads, &res, global_myname());
+	rc = ads_find_machine_acct(ads, &res, lp_netbios_name());
 	if (!ADS_ERR_OK(rc)) {
 		d_fprintf(stderr, _("ads_find_machine_acct: %s\n"), ads_errstr(rc));
 		ads_destroy(&ads);
@@ -923,7 +923,7 @@ static int net_ads_status(struct net_context *c, int argc, const char **argv)
 	}
 
 	if (ads_count_replies(ads, res) == 0) {
-		d_fprintf(stderr, _("No machine account for '%s' found\n"), global_myname());
+		d_fprintf(stderr, _("No machine account for '%s' found\n"), lp_netbios_name());
 		ads_destroy(&ads);
 		return -1;
 	}
@@ -1100,10 +1100,10 @@ static WERROR check_ads_config( void )
 		return WERR_INVALID_DOMAIN_ROLE;
 	}
 
-	if (strlen(global_myname()) > 15) {
+	if (strlen(lp_netbios_name()) > 15) {
 		d_printf(_("Our netbios name can be at most 15 chars long, "
-			   "\"%s\" is %u chars long\n"), global_myname(),
-			 (unsigned int)strlen(global_myname()));
+			   "\"%s\" is %u chars long\n"), lp_netbios_name(),
+			 (unsigned int)strlen(lp_netbios_name()));
 		return WERR_INVALID_COMPUTERNAME;
 	}
 
@@ -1244,7 +1244,7 @@ static NTSTATUS net_update_dns_ext(TALLOC_CTX *mem_ctx, ADS_STRUCT *ads,
 	if (hostname) {
 		fstrcpy(machine_name, hostname);
 	} else {
-		name_to_fqdn( machine_name, global_myname() );
+		name_to_fqdn( machine_name, lp_netbios_name() );
 	}
 	strlower_m( machine_name );
 
@@ -1349,25 +1349,25 @@ int net_ads_join(struct net_context *c, int argc, const char **argv)
 	/* process additional command line args */
 
 	for ( i=0; i<argc; i++ ) {
-		if ( !StrnCaseCmp(argv[i], "createupn", strlen("createupn")) ) {
+		if ( !strncasecmp_m(argv[i], "createupn", strlen("createupn")) ) {
 			createupn = true;
 			machineupn = get_string_param(argv[i]);
 		}
-		else if ( !StrnCaseCmp(argv[i], "createcomputer", strlen("createcomputer")) ) {
+		else if ( !strncasecmp_m(argv[i], "createcomputer", strlen("createcomputer")) ) {
 			if ( (create_in_ou = get_string_param(argv[i])) == NULL ) {
 				d_fprintf(stderr, _("Please supply a valid OU path.\n"));
 				werr = WERR_INVALID_PARAM;
 				goto fail;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "osName", strlen("osName")) ) {
+		else if ( !strncasecmp_m(argv[i], "osName", strlen("osName")) ) {
 			if ( (os_name = get_string_param(argv[i])) == NULL ) {
 				d_fprintf(stderr, _("Please supply a operating system name.\n"));
 				werr = WERR_INVALID_PARAM;
 				goto fail;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "osVer", strlen("osVer")) ) {
+		else if ( !strncasecmp_m(argv[i], "osVer", strlen("osVer")) ) {
 			if ( (os_version = get_string_param(argv[i])) == NULL ) {
 				d_fprintf(stderr, _("Please supply a valid operating system version.\n"));
 				werr = WERR_INVALID_PARAM;
@@ -1468,7 +1468,7 @@ int net_ads_join(struct net_context *c, int argc, const char **argv)
 			/* kinit with the machine password */
 
 			use_in_memory_ccache();
-			if (asprintf( &ads_dns->auth.user_name, "%s$", global_myname()) == -1) {
+			if (asprintf( &ads_dns->auth.user_name, "%s$", lp_netbios_name()) == -1) {
 				goto fail;
 			}
 			ads_dns->auth.password = secrets_fetch_machine_password(
@@ -1756,7 +1756,7 @@ static int net_ads_printer_info(struct net_context *c, int argc, const char **ar
 	if (argc > 1) {
 		servername =  argv[1];
 	} else {
-		servername = global_myname();
+		servername = lp_netbios_name();
 	}
 
 	rc = ads_find_printer_on_server(ads, &res, printername, servername);
@@ -1819,14 +1819,14 @@ static int net_ads_printer_publish(struct net_context *c, int argc, const char *
 	if (argc == 2) {
 		servername = argv[1];
 	} else {
-		servername = global_myname();
+		servername = lp_netbios_name();
 	}
 
 	/* Get printer data from SPOOLSS */
 
 	resolve_name(servername, &server_ss, 0x20, false);
 
-	nt_status = cli_full_connection(&cli, global_myname(), servername,
+	nt_status = cli_full_connection(&cli, lp_netbios_name(), servername,
 					&server_ss, 0,
 					"IPC$", "IPC",
 					c->opt_user_name, c->opt_workgroup,
@@ -1942,7 +1942,7 @@ static int net_ads_printer_remove(struct net_context *c, int argc, const char **
 	if (argc > 1) {
 		servername = argv[1];
 	} else {
-		servername = global_myname();
+		servername = lp_netbios_name();
 	}
 
 	rc = ads_find_printer_on_server(ads, &res, argv[0], servername);
@@ -2023,8 +2023,8 @@ static int net_ads_password(struct net_context *c, int argc, const char **argv)
 	ADS_STRUCT *ads;
 	const char *auth_principal = c->opt_user_name;
 	const char *auth_password = c->opt_password;
-	char *realm = NULL;
-	char *new_password = NULL;
+	const char *realm = NULL;
+	const char *new_password = NULL;
 	char *chr, *prompt;
 	const char *user;
 	ADS_STATUS ret;
@@ -2083,7 +2083,7 @@ static int net_ads_password(struct net_context *c, int argc, const char **argv)
 	}
 
 	if (argv[1]) {
-		new_password = (char *)argv[1];
+		new_password = (const char *)argv[1];
 	} else {
 		if (asprintf(&prompt, _("Enter new password for %s:"), user) == -1) {
 			return -1;
@@ -2135,7 +2135,7 @@ int net_ads_changetrustpw(struct net_context *c, int argc, const char **argv)
 		return -1;
 	}
 
-	fstrcpy(my_name, global_myname());
+	fstrcpy(my_name, lp_netbios_name());
 	strlower_m(my_name);
 	if (asprintf(&host_principal, "%s$@%s", my_name, ads->config.realm) == -1) {
 		ads_destroy(&ads);
diff --git a/source3/utils/net_cache.c b/source3/utils/net_cache.c
index 88aff4e16ec..afcb7a1874f 100644
--- a/source3/utils/net_cache.c
+++ b/source3/utils/net_cache.c
@@ -244,7 +244,7 @@ static int net_cache_get(struct net_context *c, int argc, const char **argv)
 
 	if (gencache_get_data_blob(keystr, &value, &timeout, NULL)) {
 		print_cache_entry(keystr, value, timeout, NULL);
-		SAFE_FREE(value.data);
+		data_blob_free(&value);
 		return 0;
 	}
 
diff --git a/source3/utils/net_conf.c b/source3/utils/net_conf.c
index fd00c008dd6..93811598140 100644
--- a/source3/utils/net_conf.c
+++ b/source3/utils/net_conf.c
@@ -174,12 +174,12 @@ static int net_conf_delincludes_usage(struct net_context *c, int argc,
 /**
  * This functions process a service previously loaded with libsmbconf.
  */
-static WERROR import_process_service(struct net_context *c,
+static sbcErr import_process_service(struct net_context *c,
 				     struct smbconf_ctx *conf_ctx,
 				     struct smbconf_service *service)
 {
 	uint32_t idx;
-	WERROR werr = WERR_OK;
+	sbcErr err = SBC_ERR_OK;
 	uint32_t num_includes = 0;
 	char **includes = NULL;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
@@ -200,54 +200,58 @@ static WERROR import_process_service(struct net_context *c,
 	}
 
 	if (smbconf_share_exists(conf_ctx, service->name)) {
-		werr = smbconf_delete_share(conf_ctx, service->name);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_delete_share(conf_ctx, service->name);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto done;
 		}
 	}
-	werr = smbconf_create_share(conf_ctx, service->name);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_create_share(conf_ctx, service->name);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
 	for (idx = 0; idx < service->num_params; idx ++) {
 		if (strequal(service->param_names[idx], "include")) {
-			includes = TALLOC_REALLOC_ARRAY(mem_ctx,
+			includes = talloc_realloc(mem_ctx,
 							includes,
 							char *,
 							num_includes+1);
 			if (includes == NULL) {
-				werr = WERR_NOMEM;
+				err = SBC_ERR_NOMEM;
 				goto done;
 			}
 			includes[num_includes] = talloc_strdup(includes,
 						service->param_values[idx]);
 			if (includes[num_includes] == NULL) {
-				werr = WERR_NOMEM;
+				err = SBC_ERR_NOMEM;
 				goto done;
 			}
 			num_includes++;
 		} else {
-			werr = smbconf_set_parameter(conf_ctx,
+			err = smbconf_set_parameter(conf_ctx,
 						     service->name,
 						     service->param_names[idx],
 						     service->param_values[idx]);
-			if (!W_ERROR_IS_OK(werr)) {
+			if (!SBC_ERROR_IS_OK(err)) {
 				d_fprintf(stderr,
 					  _("Error in section [%s], parameter \"%s\": %s\n"),
 					  service->name, service->param_names[idx],
-					  win_errstr(werr));
+					  sbcErrorString(err));
 				goto done;
 			}
 		}
 	}
 
-	werr = smbconf_set_includes(conf_ctx, service->name, num_includes,
-				    (const char **)includes);
+	err = smbconf_set_includes(conf_ctx, service->name, num_includes,
+				   (const char **)includes);
+	if (!SBC_ERROR_IS_OK(err)) {
+		goto done;
+	}
 
+	err = SBC_ERR_OK;
 done:
 	TALLOC_FREE(mem_ctx);
-	return werr;
+	return err;
 }
 
 
@@ -260,7 +264,7 @@ done:
 static int net_conf_list(struct net_context *c, struct smbconf_ctx *conf_ctx,
 			 int argc, const char **argv)
 {
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	int ret = -1;
 	TALLOC_CTX *mem_ctx;
 	uint32_t num_shares;
@@ -274,10 +278,10 @@ static int net_conf_list(struct net_context *c, struct smbconf_ctx *conf_ctx,
 		goto done;
 	}
 
-	werr = smbconf_get_config(conf_ctx, mem_ctx, &num_shares, &shares);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_config(conf_ctx, mem_ctx, &num_shares, &shares);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error getting config: %s\n"),
-			  win_errstr(werr));
+			  sbcErrorString(err));
 		goto done;
 	}
 
@@ -315,7 +319,7 @@ static int net_conf_import(struct net_context *c, struct smbconf_ctx *conf_ctx,
 	char *conf_source = NULL;
 	TALLOC_CTX *mem_ctx;
 	struct smbconf_ctx *txt_ctx;
-	WERROR werr;
+	sbcErr err;
 
 	if (c->display_usage)
 		return net_conf_import_usage(c, argc, argv);
@@ -347,10 +351,10 @@ static int net_conf_import(struct net_context *c, struct smbconf_ctx *conf_ctx,
 		goto done;
 	}
 
-	werr = smbconf_init(mem_ctx, &txt_ctx, conf_source);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_init(mem_ctx, &txt_ctx, conf_source);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf(_("error loading file '%s': %s\n"), filename,
-			 win_errstr(werr));
+			 sbcErrorString(err));
 		goto done;
 	}
 
@@ -362,37 +366,36 @@ static int net_conf_import(struct net_context *c, struct smbconf_ctx *conf_ctx,
 	if (servicename != NULL) {
 		struct smbconf_service *service = NULL;
 
-		werr = smbconf_get_share(txt_ctx, mem_ctx,
-					 servicename,
-					 &service);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_get_share(txt_ctx, mem_ctx,
+					servicename,
+					&service);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto cancel;
 		}
 
-		werr = smbconf_transaction_start(conf_ctx);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_transaction_start(conf_ctx);
+		if (!SBC_ERROR_IS_OK(err)) {
 			d_printf(_("error starting transaction: %s\n"),
-				 win_errstr(werr));
+				 sbcErrorString(err));
 			goto done;
 		}
 
-		werr = import_process_service(c, conf_ctx, service);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = import_process_service(c, conf_ctx, service);
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto cancel;
 		}
 	} else {
 		struct smbconf_service **services = NULL;
 		uint32_t num_shares, sidx;
 
-		werr = smbconf_get_config(txt_ctx, mem_ctx,
+		err = smbconf_get_config(txt_ctx, mem_ctx,
 					  &num_shares,
 					  &services);
-		if (!W_ERROR_IS_OK(werr)) {
+		if (!SBC_ERROR_IS_OK(err)) {
 			goto cancel;
 		}
 		if (!c->opt_testmode) {
-			werr = smbconf_drop(conf_ctx);
-			if (!W_ERROR_IS_OK(werr)) {
+			if (!SBC_ERROR_IS_OK(smbconf_drop(conf_ctx))) {
 				goto cancel;
 			}
 		}
@@ -405,17 +408,17 @@ static int net_conf_import(struct net_context *c, struct smbconf_ctx *conf_ctx,
 		 * imported shares, the MAX_TALLOC_SIZE of 256 MB
 		 * was exceeded.
 		 */
-		werr = smbconf_transaction_start(conf_ctx);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_transaction_start(conf_ctx);
+		if (!SBC_ERROR_IS_OK(err)) {
 			d_printf(_("error starting transaction: %s\n"),
-				 win_errstr(werr));
+				 sbcErrorString(err));
 			goto done;
 		}
 
 		for (sidx = 0; sidx < num_shares; sidx++) {
-			werr = import_process_service(c, conf_ctx,
-						      services[sidx]);
-			if (!W_ERROR_IS_OK(werr)) {
+			err = import_process_service(c, conf_ctx,
+						     services[sidx]);
+			if (!SBC_ERROR_IS_OK(err)) {
 				goto cancel;
 			}
 
@@ -423,26 +426,26 @@ static int net_conf_import(struct net_context *c, struct smbconf_ctx *conf_ctx,
 				continue;
 			}
 
-			werr = smbconf_transaction_commit(conf_ctx);
-			if (!W_ERROR_IS_OK(werr)) {
+			err = smbconf_transaction_commit(conf_ctx);
+			if (!SBC_ERROR_IS_OK(err)) {
 				d_printf(_("error committing transaction: "
 					   "%s\n"),
-					 win_errstr(werr));
+					 sbcErrorString(err));
 				goto done;
 			}
-			werr = smbconf_transaction_start(conf_ctx);
-			if (!W_ERROR_IS_OK(werr)) {
+			err = smbconf_transaction_start(conf_ctx);
+			if (!SBC_ERROR_IS_OK(err)) {
 				d_printf(_("error starting transaction: %s\n"),
-					 win_errstr(werr));
+					 sbcErrorString(err));
 				goto done;
 			}
 		}
 	}
 
-	werr = smbconf_transaction_commit(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_commit(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf(_("error committing transaction: %s\n"),
-			 win_errstr(werr));
+			 sbcErrorString(err));
 	} else {
 		ret = 0;
 	}
@@ -450,10 +453,10 @@ static int net_conf_import(struct net_context *c, struct smbconf_ctx *conf_ctx,
 	goto done;
 
 cancel:
-	werr = smbconf_transaction_cancel(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_cancel(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf(_("error cancelling transaction: %s\n"),
-			 win_errstr(werr));
+			 sbcErrorString(err));
 	}
 
 done:
@@ -465,7 +468,7 @@ static int net_conf_listshares(struct net_context *c,
 			       struct smbconf_ctx *conf_ctx, int argc,
 			       const char **argv)
 {
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	int ret = -1;
 	uint32_t count, num_shares = 0;
 	char **share_names = NULL;
@@ -478,9 +481,9 @@ static int net_conf_listshares(struct net_context *c,
 		goto done;
 	}
 
-	werr = smbconf_get_share_names(conf_ctx, mem_ctx, &num_shares,
-				       &share_names);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_share_names(conf_ctx, mem_ctx, &num_shares,
+				      &share_names);
+	if (!SBC_ERROR_IS_OK(err)) {
 		goto done;
 	}
 
@@ -500,17 +503,17 @@ static int net_conf_drop(struct net_context *c, struct smbconf_ctx *conf_ctx,
 			 int argc, const char **argv)
 {
 	int ret = -1;
-	WERROR werr;
+	sbcErr err;
 
 	if (argc != 0 || c->display_usage) {
 		net_conf_drop_usage(c, argc, argv);
 		goto done;
 	}
 
-	werr = smbconf_drop(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_drop(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error deleting configuration: %s\n"),
-			  win_errstr(werr));
+			  sbcErrorString(err));
 		goto done;
 	}
 
@@ -525,7 +528,7 @@ static int net_conf_showshare(struct net_context *c,
 			      const char **argv)
 {
 	int ret = -1;
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	const char *sharename = NULL;
 	TALLOC_CTX *mem_ctx;
 	uint32_t count;
@@ -544,10 +547,10 @@ static int net_conf_showshare(struct net_context *c,
 		goto done;
 	}
 
-	werr = smbconf_get_share(conf_ctx, mem_ctx, sharename, &service);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_get_share(conf_ctx, mem_ctx, sharename, &service);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf(_("error getting share parameters: %s\n"),
-			 win_errstr(werr));
+			 sbcErrorString(err));
 		goto done;
 	}
 
@@ -576,13 +579,12 @@ static int net_conf_addshare(struct net_context *c,
 			     const char **argv)
 {
 	int ret = -1;
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	char *sharename = NULL;
 	const char *path = NULL;
 	const char *comment = NULL;
 	const char *guest_ok = "no";
 	const char *writeable = "no";
-	SMB_STRUCT_STAT sbuf;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 
 	if (c->display_usage) {
@@ -682,30 +684,14 @@ static int net_conf_addshare(struct net_context *c,
 		goto done;
 	}
 
-	if (sys_stat(path, &sbuf, false) != 0) {
-		d_fprintf(stderr,
-			  _("ERROR: cannot stat path '%s' to ensure "
-			    "this is a directory.\n"
-			    "Error was '%s'.\n"),
-			  path, strerror(errno));
-		goto done;
-	}
-
-	if (!S_ISDIR(sbuf.st_ex_mode)) {
-		d_fprintf(stderr,
-			  _("ERROR: path '%s' is not a directory.\n"),
-			  path);
-		goto done;
-	}
-
 	/*
 	 * start a transaction
 	 */
 
-	werr = smbconf_transaction_start(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_start(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf("error starting transaction: %s\n",
-			 win_errstr(werr));
+			 sbcErrorString(err));
 		goto done;
 	}
 
@@ -713,10 +699,10 @@ static int net_conf_addshare(struct net_context *c,
 	 * create the share
 	 */
 
-	werr = smbconf_create_share(conf_ctx, sharename);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_create_share(conf_ctx, sharename);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error creating share %s: %s\n"),
-			  sharename, win_errstr(werr));
+			  sharename, sbcErrorString(err));
 		goto cancel;
 	}
 
@@ -724,35 +710,35 @@ static int net_conf_addshare(struct net_context *c,
 	 * fill the share with parameters
 	 */
 
-	werr = smbconf_set_parameter(conf_ctx, sharename, "path", path);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_set_parameter(conf_ctx, sharename, "path", path);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error setting parameter %s: %s\n"),
-			  "path", win_errstr(werr));
+			  "path", sbcErrorString(err));
 		goto cancel;
 	}
 
 	if (comment != NULL) {
-		werr = smbconf_set_parameter(conf_ctx, sharename, "comment",
-					     comment);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_set_parameter(conf_ctx, sharename, "comment",
+					    comment);
+		if (!SBC_ERROR_IS_OK(err)) {
 			d_fprintf(stderr, _("Error setting parameter %s: %s\n"),
-				  "comment", win_errstr(werr));
+				  "comment", sbcErrorString(err));
 			goto cancel;
 		}
 	}
 
-	werr = smbconf_set_parameter(conf_ctx, sharename, "guest ok", guest_ok);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_set_parameter(conf_ctx, sharename, "guest ok", guest_ok);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error setting parameter %s: %s\n"),
-			  "'guest ok'", win_errstr(werr));
+			  "'guest ok'", sbcErrorString(err));
 		goto cancel;
 	}
 
-	werr = smbconf_set_parameter(conf_ctx, sharename, "writeable",
-				     writeable);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_set_parameter(conf_ctx, sharename, "writeable",
+				    writeable);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error setting parameter %s: %s\n"),
-			  "writeable", win_errstr(werr));
+			  "writeable", sbcErrorString(err));
 		goto cancel;
 	}
 
@@ -760,10 +746,10 @@ static int net_conf_addshare(struct net_context *c,
 	 * commit the whole thing
 	 */
 
-	werr = smbconf_transaction_commit(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_commit(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf("error committing transaction: %s\n",
-			 win_errstr(werr));
+			 sbcErrorString(err));
 	} else {
 		ret = 0;
 	}
@@ -771,10 +757,10 @@ static int net_conf_addshare(struct net_context *c,
 	goto done;
 
 cancel:
-	werr = smbconf_transaction_cancel(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_cancel(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf("error cancelling transaction: %s\n",
-			 win_errstr(werr));
+			 sbcErrorString(err));
 	}
 
 done:
@@ -788,7 +774,7 @@ static int net_conf_delshare(struct net_context *c,
 {
 	int ret = -1;
 	const char *sharename = NULL;
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 
 	if (argc != 1 || c->display_usage) {
@@ -801,10 +787,10 @@ static int net_conf_delshare(struct net_context *c,
 		goto done;
 	}
 
-	werr = smbconf_delete_share(conf_ctx, sharename);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_delete_share(conf_ctx, sharename);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error deleting share %s: %s\n"),
-			  sharename, win_errstr(werr));
+			  sharename, sbcErrorString(err));
 		goto done;
 	}
 
@@ -818,7 +804,7 @@ static int net_conf_setparm(struct net_context *c, struct smbconf_ctx *conf_ctx,
 			    int argc, const char **argv)
 {
 	int ret = -1;
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	char *service = NULL;
 	char *param = NULL;
 	const char *value_str = NULL;
@@ -846,34 +832,33 @@ static int net_conf_setparm(struct net_context *c, struct smbconf_ctx *conf_ctx,
 	}
 	value_str = argv[2];
 
-	werr = smbconf_transaction_start(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_start(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf(_("error starting transaction: %s\n"),
-			 win_errstr(werr));
+			 sbcErrorString(err));
 		goto done;
 	}
 
 	if (!smbconf_share_exists(conf_ctx, service)) {
-		werr = smbconf_create_share(conf_ctx, service);
-		if (!W_ERROR_IS_OK(werr)) {
+		err = smbconf_create_share(conf_ctx, service);
+		if (!SBC_ERROR_IS_OK(err)) {
 			d_fprintf(stderr, _("Error creating share '%s': %s\n"),
-				  service, win_errstr(werr));
+				  service, sbcErrorString(err));
 			goto cancel;
 		}
 	}
 
-	werr = smbconf_set_parameter(conf_ctx, service, param, value_str);
-
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_set_parameter(conf_ctx, service, param, value_str);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error setting value '%s': %s\n"),
-			  param, win_errstr(werr));
+			  param, sbcErrorString(err));
 		goto cancel;
 	}
 
-	werr = smbconf_transaction_commit(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_commit(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf(_("error committing transaction: %s\n"),
-			 win_errstr(werr));
+			 sbcErrorString(err));
 	} else {
 		ret = 0;
 	}
@@ -881,10 +866,10 @@ static int net_conf_setparm(struct net_context *c, struct smbconf_ctx *conf_ctx,
 	goto done;
 
 cancel:
-	werr = smbconf_transaction_cancel(conf_ctx);
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_transaction_cancel(conf_ctx);
+	if (!SBC_ERROR_IS_OK(err)) {
 		d_printf(_("error cancelling transaction: %s\n"),
-			 win_errstr(werr));
+			 sbcErrorString(err));
 	}
 
 done:
@@ -896,7 +881,7 @@ static int net_conf_getparm(struct net_context *c, struct smbconf_ctx *conf_ctx,
 			    int argc, const char **argv)
 {
 	int ret = -1;
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	char *service = NULL;
 	char *param = NULL;
 	char *valstr = NULL;
@@ -925,21 +910,20 @@ static int net_conf_getparm(struct net_context *c, struct smbconf_ctx *conf_ctx,
 		goto done;
 	}
 
-	werr = smbconf_get_parameter(conf_ctx, mem_ctx, service, param, &valstr);
-
-	if (W_ERROR_EQUAL(werr, WERR_NO_SUCH_SERVICE)) {
+	err = smbconf_get_parameter(conf_ctx, mem_ctx, service, param, &valstr);
+	if (SBC_ERROR_EQUAL(err, SBC_ERR_NO_SUCH_SERVICE)) {
 		d_fprintf(stderr,
 			  _("Error: given service '%s' does not exist.\n"),
 			  service);
 		goto done;
-	} else if (W_ERROR_EQUAL(werr, WERR_INVALID_PARAM)) {
+	} else if (SBC_ERROR_EQUAL(err, SBC_ERR_INVALID_PARAM)) {
 		d_fprintf(stderr,
 			  _("Error: given parameter '%s' is not set.\n"),
 			  param);
 		goto done;
-	} else if (!W_ERROR_IS_OK(werr)) {
+	} else if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error getting value '%s': %s.\n"),
-			  param, win_errstr(werr));
+			  param, sbcErrorString(err));
 		goto done;
 	}
 
@@ -955,7 +939,7 @@ static int net_conf_delparm(struct net_context *c, struct smbconf_ctx *conf_ctx,
 			    int argc, const char **argv)
 {
 	int ret = -1;
-	WERROR werr = WERR_OK;
+	sbcErr err;
 	char *service = NULL;
 	char *param = NULL;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
@@ -981,21 +965,20 @@ static int net_conf_delparm(struct net_context *c, struct smbconf_ctx *conf_ctx,
 		goto done;
 	}
 
-	werr = smbconf_delete_parameter(conf_ctx, service, param);
-
-	if (W_ERROR_EQUAL(werr, WERR_NO_SUCH_SERVICE)) {
+	err = smbconf_delete_parameter(conf_ctx, service, param);
+	if (SBC_ERROR_EQUAL(err, SBC_ERR_NO_SUCH_SERVICE)) {
 		d_fprintf(stderr,
 			  _("Error: given service '%s' does not exist.\n"),
 			  service);
 		goto done;
-	} else if (W_ERROR_EQUAL(werr, WERR_INVALID_PARAM)) {
+	} else if (SBC_ERROR_EQUAL(err, SBC_ERR_INVALID_PARAM)) {
 		d_fprintf(stderr,
 			  _("Error: given parameter '%s' is not set.\n"),
 			  param);
 		goto done;
-	} else if (!W_ERROR_IS_OK(werr)) {
+	} else if (!SBC_ERROR_IS_OK(err)) {
 		d_fprintf(stderr, _("Error deleting value '%s': %s.\n"),
-			  param, win_errstr(werr));
+			  param, sbcErrorString(err));
 		goto done;
 	}
 
@@ -1010,7 +993,7 @@ static int net_conf_getincludes(struct net_context *c,
 				struct smbconf_ctx *conf_ctx,
 				int argc, const char **argv)
 {
-	WERROR werr;
+	sbcErr err;
 	uint32_t num_includes;
 	uint32_t count;
 	char *service;
@@ -1029,10 +1012,10 @@ static int net_conf_getincludes(struct net_context *c,
 		goto done;
 	}
 
-	werr = smbconf_get_includes(conf_ctx, mem_ctx, service,
+	err = smbconf_get_includes(conf_ctx, mem_ctx, service,
 				    &num_includes, &includes);
-	if (!W_ERROR_IS_OK(werr)) {
-		d_printf(_("error getting includes: %s\n"), win_errstr(werr));
+	if (!SBC_ERROR_IS_OK(err)) {
+		d_printf(_("error getting includes: %s\n"), sbcErrorString(err));
 		goto done;
 	}
 
@@ -1051,7 +1034,7 @@ static int net_conf_setincludes(struct net_context *c,
 				struct smbconf_ctx *conf_ctx,
 				int argc, const char **argv)
 {
-	WERROR werr;
+	sbcErr err;
 	char *service;
 	uint32_t num_includes;
 	const char **includes;
@@ -1076,9 +1059,9 @@ static int net_conf_setincludes(struct net_context *c,
 		includes = argv + 1;
 	}
 
-	werr = smbconf_set_includes(conf_ctx, service, num_includes, includes);
-	if (!W_ERROR_IS_OK(werr)) {
-		d_printf(_("error setting includes: %s\n"), win_errstr(werr));
+	err = smbconf_set_includes(conf_ctx, service, num_includes, includes);
+	if (!SBC_ERROR_IS_OK(err)) {
+		d_printf(_("error setting includes: %s\n"), sbcErrorString(err));
 		goto done;
 	}
 
@@ -1093,7 +1076,7 @@ static int net_conf_delincludes(struct net_context *c,
 				struct smbconf_ctx *conf_ctx,
 				int argc, const char **argv)
 {
-	WERROR werr;
+	sbcErr err;
 	char *service;
 	int ret = -1;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
@@ -1109,9 +1092,9 @@ static int net_conf_delincludes(struct net_context *c,
 		goto done;
 	}
 
-	werr = smbconf_delete_includes(conf_ctx, service);
-	if (!W_ERROR_IS_OK(werr)) {
-		d_printf(_("error deleting includes: %s\n"), win_errstr(werr));
+	err = smbconf_delete_includes(conf_ctx, service);
+	if (!SBC_ERROR_IS_OK(err)) {
+		d_printf(_("error deleting includes: %s\n"), sbcErrorString(err));
 		goto done;
 	}
 
@@ -1140,14 +1123,13 @@ static int net_conf_wrap_function(struct net_context *c,
 					    int, const char **),
 				  int argc, const char **argv)
 {
-	WERROR werr;
+	sbcErr err;
 	TALLOC_CTX *mem_ctx = talloc_stackframe();
 	struct smbconf_ctx *conf_ctx;
 	int ret = -1;
 
-	werr = smbconf_init(mem_ctx, &conf_ctx, "registry:");
-
-	if (!W_ERROR_IS_OK(werr)) {
+	err = smbconf_init(mem_ctx, &conf_ctx, "registry:");
+	if (!SBC_ERROR_IS_OK(err)) {
 		return -1;
 	}
 
@@ -1184,7 +1166,7 @@ static int net_conf_run_function(struct net_context *c, int argc,
 
 	if (argc != 0) {
 		for (i=0; table[i].funcname; i++) {
-			if (StrCaseCmp(argv[0], table[i].funcname) == 0)
+			if (strcasecmp_m(argv[0], table[i].funcname) == 0)
 				return net_conf_wrap_function(c, table[i].fn,
 							      argc-1,
 							      argv+1);
diff --git a/source3/utils/net_dom.c b/source3/utils/net_dom.c
index 4378d01fe7b..2b67b7b2fe0 100644
--- a/source3/utils/net_dom.c
+++ b/source3/utils/net_dom.c
@@ -23,6 +23,7 @@
 #include "../librpc/gen_ndr/ndr_winreg.h"
 #include "lib/netapi/netapi.h"
 #include "lib/netapi/netapi_net.h"
+#include "libsmb/libsmb.h"
 
 int net_dom_usage(struct net_context *c, int argc, const char **argv)
 {
diff --git a/source3/utils/net_file.c b/source3/utils/net_file.c
index 27f2d24096d..71a7e05fe77 100644
--- a/source3/utils/net_file.c
+++ b/source3/utils/net_file.c
@@ -44,7 +44,7 @@ int net_file(struct net_context *c, int argc, const char **argv)
 	if (argc < 1)
 		return net_file_usage(c, argc, argv);
 
-	if (StrCaseCmp(argv[0], "HELP") == 0) {
+	if (strcasecmp_m(argv[0], "HELP") == 0) {
 		net_file_usage(c, argc, argv);
 		return 0;
 	}
diff --git a/source3/utils/net_g_lock.c b/source3/utils/net_g_lock.c
index 6ed5d1776a6..f8a7a8b6202 100644
--- a/source3/utils/net_g_lock.c
+++ b/source3/utils/net_g_lock.c
@@ -113,7 +113,7 @@ static int net_g_lock_dump_fn(struct server_id pid, enum g_lock_type lock_type,
 {
 	char *pidstr;
 
-	pidstr = procid_str(talloc_tos(), &pid);
+	pidstr = server_id_str(talloc_tos(), &pid);
 	d_printf("%s: %s (%s)\n", pidstr,
 		 (lock_type & 1) ? "WRITE" : "READ",
 		 (lock_type & G_LOCK_PENDING) ? "pending" : "holder");
@@ -175,7 +175,7 @@ done:
 	TALLOC_FREE(g_ctx);
 	TALLOC_FREE(msg);
 	TALLOC_FREE(ev);
-	return ret;
+	return ret < 0 ? -1 : ret;
 }
 
 int net_g_lock(struct net_context *c, int argc, const char **argv)
diff --git a/source3/utils/net_group.c b/source3/utils/net_group.c
index 7114f1feeaa..505856a0080 100644
--- a/source3/utils/net_group.c
+++ b/source3/utils/net_group.c
@@ -55,7 +55,7 @@ int net_group(struct net_context *c, int argc, const char **argv)
 	if (argc < 1)
 		return net_group_usage(c, argc, argv);
 
-	if (StrCaseCmp(argv[0], "HELP") == 0) {
+	if (strcasecmp_m(argv[0], "HELP") == 0) {
 		net_group_usage(c, argc, argv);
 		return 0;
 	}
diff --git a/source3/utils/net_groupmap.c b/source3/utils/net_groupmap.c
index 818905e122f..09f4bfd33a5 100644
--- a/source3/utils/net_groupmap.c
+++ b/source3/utils/net_groupmap.c
@@ -35,7 +35,7 @@ static bool get_sid_from_input(struct dom_sid *sid, char *input)
 {
 	GROUP_MAP map;
 
-	if (StrnCaseCmp( input, "S-", 2)) {
+	if (strncasecmp_m( input, "S-", 2)) {
 		/* Perhaps its the NT group name? */
 		if (!pdb_getgrnam(&map, input)) {
 			printf(_("NT Group %s doesn't exist in mapping DB\n"),
@@ -100,17 +100,17 @@ static int net_groupmap_list(struct net_context *c, int argc, const char **argv)
 
 	/* get the options */
 	for ( i=0; i<argc; i++ ) {
-		if ( !StrCaseCmp(argv[i], "verbose")) {
+		if ( !strcasecmp_m(argv[i], "verbose")) {
 			long_list = true;
 		}
-		else if ( !StrnCaseCmp(argv[i], "ntgroup", strlen("ntgroup")) ) {
+		else if ( !strncasecmp_m(argv[i], "ntgroup", strlen("ntgroup")) ) {
 			fstrcpy( ntgroup, get_string_param( argv[i] ) );
 			if ( !ntgroup[0] ) {
 				d_fprintf(stderr, _("must supply a name\n"));
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "sid", strlen("sid")) ) {
+		else if ( !strncasecmp_m(argv[i], "sid", strlen("sid")) ) {
 			fstrcpy( sid_string, get_string_param( argv[i] ) );
 			if ( !sid_string[0] ) {
 				d_fprintf(stderr, _("must supply a SID\n"));
@@ -130,7 +130,7 @@ static int net_groupmap_list(struct net_context *c, int argc, const char **argv)
 		GROUP_MAP map;
 
 		if ( sid_string[0] )
-			fstrcpy( ntgroup, sid_string);
+			strlcpy(ntgroup, sid_string, sizeof(ntgroup));
 
 		if (!get_sid_from_input(&sid, ntgroup)) {
 			return -1;
@@ -201,7 +201,7 @@ static int net_groupmap_add(struct net_context *c, int argc, const char **argv)
 
 	/* get the options */
 	for ( i=0; i<argc; i++ ) {
-		if ( !StrnCaseCmp(argv[i], "rid", strlen("rid")) ) {
+		if ( !strncasecmp_m(argv[i], "rid", strlen("rid")) ) {
 			rid = get_int_param(argv[i]);
 			if ( rid < DOMAIN_RID_ADMINS ) {
 				d_fprintf(stderr,
@@ -210,28 +210,28 @@ static int net_groupmap_add(struct net_context *c, int argc, const char **argv)
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "unixgroup", strlen("unixgroup")) ) {
+		else if ( !strncasecmp_m(argv[i], "unixgroup", strlen("unixgroup")) ) {
 			fstrcpy( unixgrp, get_string_param( argv[i] ) );
 			if ( !unixgrp[0] ) {
 				d_fprintf(stderr,_( "must supply a name\n"));
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "ntgroup", strlen("ntgroup")) ) {
+		else if ( !strncasecmp_m(argv[i], "ntgroup", strlen("ntgroup")) ) {
 			fstrcpy( ntgroup, get_string_param( argv[i] ) );
 			if ( !ntgroup[0] ) {
 				d_fprintf(stderr, _("must supply a name\n"));
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "sid", strlen("sid")) ) {
+		else if ( !strncasecmp_m(argv[i], "sid", strlen("sid")) ) {
 			fstrcpy( string_sid, get_string_param( argv[i] ) );
 			if ( !string_sid[0] ) {
 				d_fprintf(stderr, _("must supply a SID\n"));
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "comment", strlen("comment")) ) {
+		else if ( !strncasecmp_m(argv[i], "comment", strlen("comment")) ) {
 			fstrcpy( ntcomment, get_string_param( argv[i] ) );
 			if ( !ntcomment[0] ) {
 				d_fprintf(stderr,
@@ -239,7 +239,7 @@ static int net_groupmap_add(struct net_context *c, int argc, const char **argv)
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "type", strlen("type")) )  {
+		else if ( !strncasecmp_m(argv[i], "type", strlen("type")) )  {
 			fstrcpy( type, get_string_param( argv[i] ) );
 			switch ( type[0] ) {
 				case 'b':
@@ -324,7 +324,7 @@ static int net_groupmap_add(struct net_context *c, int argc, const char **argv)
 	}
 
 	if (!ntgroup[0] )
-		fstrcpy( ntgroup, unixgrp );
+		strlcpy(ntgroup, unixgrp, sizeof(ntgroup));
 
 	if (!NT_STATUS_IS_OK(add_initial_entry(gid, string_sid, sid_type, ntgroup, ntcomment))) {
 		d_fprintf(stderr, _("adding entry for group %s failed!\n"), ntgroup);
@@ -361,21 +361,21 @@ static int net_groupmap_modify(struct net_context *c, int argc, const char **arg
 
 	/* get the options */
 	for ( i=0; i<argc; i++ ) {
-		if ( !StrnCaseCmp(argv[i], "ntgroup", strlen("ntgroup")) ) {
+		if ( !strncasecmp_m(argv[i], "ntgroup", strlen("ntgroup")) ) {
 			fstrcpy( ntgroup, get_string_param( argv[i] ) );
 			if ( !ntgroup[0] ) {
 				d_fprintf(stderr, _("must supply a name\n"));
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "sid", strlen("sid")) ) {
+		else if ( !strncasecmp_m(argv[i], "sid", strlen("sid")) ) {
 			fstrcpy( sid_string, get_string_param( argv[i] ) );
 			if ( !sid_string[0] ) {
 				d_fprintf(stderr, _("must supply a name\n"));
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "comment", strlen("comment")) ) {
+		else if ( !strncasecmp_m(argv[i], "comment", strlen("comment")) ) {
 			fstrcpy( ntcomment, get_string_param( argv[i] ) );
 			if ( !ntcomment[0] ) {
 				d_fprintf(stderr,
@@ -383,7 +383,7 @@ static int net_groupmap_modify(struct net_context *c, int argc, const char **arg
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "unixgroup", strlen("unixgroup")) ) {
+		else if ( !strncasecmp_m(argv[i], "unixgroup", strlen("unixgroup")) ) {
 			fstrcpy( unixgrp, get_string_param( argv[i] ) );
 			if ( !unixgrp[0] ) {
 				d_fprintf(stderr,
@@ -391,7 +391,7 @@ static int net_groupmap_modify(struct net_context *c, int argc, const char **arg
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "type", strlen("type")) )  {
+		else if ( !strncasecmp_m(argv[i], "type", strlen("type")) )  {
 			fstrcpy( type, get_string_param( argv[i] ) );
 			switch ( type[0] ) {
 				case 'd':
@@ -457,10 +457,10 @@ static int net_groupmap_modify(struct net_context *c, int argc, const char **arg
 
 	/* Change comment if new one */
 	if ( ntcomment[0] )
-		fstrcpy( map.comment, ntcomment );
+		strlcpy(map.comment, ntcomment, sizeof(map.comment));
 
 	if ( ntgroup[0] )
-		fstrcpy( map.nt_name, ntgroup );
+		strlcpy(map.nt_name, ntgroup, sizeof(map.nt_name));
 
 	if ( unixgrp[0] ) {
 		gid = nametogid( unixgrp );
@@ -500,14 +500,14 @@ static int net_groupmap_delete(struct net_context *c, int argc, const char **arg
 
 	/* get the options */
 	for ( i=0; i<argc; i++ ) {
-		if ( !StrnCaseCmp(argv[i], "ntgroup", strlen("ntgroup")) ) {
+		if ( !strncasecmp_m(argv[i], "ntgroup", strlen("ntgroup")) ) {
 			fstrcpy( ntgroup, get_string_param( argv[i] ) );
 			if ( !ntgroup[0] ) {
 				d_fprintf(stderr, _("must supply a name\n"));
 				return -1;
 			}
 		}
-		else if ( !StrnCaseCmp(argv[i], "sid", strlen("sid")) ) {
+		else if ( !strncasecmp_m(argv[i], "sid", strlen("sid")) ) {
 			fstrcpy( sid_string, get_string_param( argv[i] ) );
 			if ( !sid_string[0] ) {
 				d_fprintf(stderr, _("must supply a SID\n"));
@@ -528,7 +528,7 @@ static int net_groupmap_delete(struct net_context *c, int argc, const char **arg
 	/* give preference to the SID if we have that */
 
 	if ( sid_string[0] )
-		fstrcpy( ntgroup, sid_string );
+		strlcpy(ntgroup, sid_string, sizeof(ntgroup));
 
 	if ( !get_sid_from_input(&sid, ntgroup) ) {
 		d_fprintf(stderr, _("Unable to resolve group %s to a SID\n"),
diff --git a/source3/utils/net_help.c b/source3/utils/net_help.c
index 4e326ba9020..4aba1c57037 100644
--- a/source3/utils/net_help.c
+++ b/source3/utils/net_help.c
@@ -60,7 +60,7 @@ int net_help(struct net_context *c, int argc, const char **argv)
 		return net_usage(c, argc, argv);
 	}
 
-	if (StrCaseCmp(argv[0], "help") == 0) {
+	if (strcasecmp_m(argv[0], "help") == 0) {
 		return net_help_usage(c, argc, argv);
 	}
 
diff --git a/source3/utils/net_idmap.c b/source3/utils/net_idmap.c
index e059a6432c0..383035e906c 100644
--- a/source3/utils/net_idmap.c
+++ b/source3/utils/net_idmap.c
@@ -25,6 +25,7 @@
 #include "dbwrap.h"
 #include "../libcli/security/security.h"
 #include "net_idmap_check.h"
+#include "util_tdb.h"
 
 #define ALLOC_CHECK(mem) do { \
 	if (!mem) { \
diff --git a/source3/utils/net_idmap_check.c b/source3/utils/net_idmap_check.c
index 88725e18a61..3adc060a7c6 100644
--- a/source3/utils/net_idmap_check.c
+++ b/source3/utils/net_idmap_check.c
@@ -32,6 +32,7 @@
 #include "cbuf.h"
 #include "srprs.h"
 #include <termios.h>
+#include "util_tdb.h"
 
 static int traverse_commit(struct db_record *diff_rec, void* data);
 static int traverse_check(struct db_record *rec, void* data);
@@ -345,7 +346,7 @@ fetch_record(struct check_ctx* ctx, TALLOC_CTX* mem_ctx, TDB_DATA key)
 {
 	TDB_DATA tmp;
 
-	if (ctx->diff->fetch(ctx->diff, mem_ctx, key, &tmp) == -1) {
+	if (ctx->diff->fetch(ctx->diff, mem_ctx, key, &tmp) != 0) {
 		DEBUG(0, ("Out of memory!\n"));
 		return tdb_null;
 	}
@@ -356,7 +357,7 @@ fetch_record(struct check_ctx* ctx, TALLOC_CTX* mem_ctx, TDB_DATA key)
 		return ret;
 	}
 
-	if (ctx->db->fetch(ctx->db, mem_ctx, key, &tmp) == -1) {
+	if (ctx->db->fetch(ctx->db, mem_ctx, key, &tmp) != 0) {
 		DEBUG(0, ("Out of memory!\n"));
 		return tdb_null;
 	}
diff --git a/source3/utils/net_join.c b/source3/utils/net_join.c
index 9e9ef2af065..1493dff74d7 100644
--- a/source3/utils/net_join.c
+++ b/source3/utils/net_join.c
@@ -34,7 +34,7 @@ int net_join_usage(struct net_context *c, int argc, const char **argv)
 
 int net_join(struct net_context *c, int argc, const char **argv)
 {
-	if ((argc > 0) && (StrCaseCmp(argv[0], "HELP") == 0)) {
+	if ((argc > 0) && (strcasecmp_m(argv[0], "HELP") == 0)) {
 		net_join_usage(c, argc, argv);
 		return 0;
 	}
diff --git a/source3/utils/net_lookup.c b/source3/utils/net_lookup.c
index 06aedbd4730..febf481e552 100644
--- a/source3/utils/net_lookup.c
+++ b/source3/utils/net_lookup.c
@@ -290,10 +290,10 @@ static int net_lookup_kdc(struct net_context *c, int argc, const char **argv)
 	}
 
 	if (argc>0) {
-                realm.data = CONST_DISCARD(char *, argv[0]);
+                realm.data = discard_const_p(char, argv[0]);
 		realm.length = strlen(argv[0]);
 	} else if (lp_realm() && *lp_realm()) {
-		realm.data = lp_realm();
+		realm.data = discard_const_p(char, lp_realm());
 		realm.length = strlen((const char *)realm.data);
 	} else {
 		rc = krb5_get_host_realm(ctx, NULL, &realms);
@@ -457,7 +457,7 @@ int net_lookup(struct net_context *c, int argc, const char **argv)
 		return net_lookup_usage(c, argc, argv);
 	}
 	for (i=0; table[i].funcname; i++) {
-		if (StrCaseCmp(argv[0], table[i].funcname) == 0)
+		if (strcasecmp_m(argv[0], table[i].funcname) == 0)
 			return table[i].fn(c, argc-1, argv+1);
 	}
 
diff --git a/source3/utils/net_printing.c b/source3/utils/net_printing.c
index 11c87bb5b4f..2f8caad8fda 100644
--- a/source3/utils/net_printing.c
+++ b/source3/utils/net_printing.c
@@ -28,6 +28,7 @@
 #include "rpc_client/cli_spoolss.h"
 #include "../libcli/security/security.h"
 #include "../librpc/gen_ndr/ndr_security.h"
+#include "util_tdb.h"
 
 #define FORMS_PREFIX "FORMS/"
 #define DRIVERS_PREFIX "DRIVERS/"
@@ -161,7 +162,7 @@ static int net_printing_dump(struct net_context *c, int argc,
 	int ret = -1;
 	TALLOC_CTX *ctx = talloc_stackframe();
 	TDB_CONTEXT *tdb;
-	TDB_DATA kbuf, newkey, dbuf;
+	TDB_DATA kbuf, dbuf;
 
 	if (argc < 1 || c->display_usage) {
 		d_fprintf(stderr, "%s\nnet printing dump <file.tdb>\n",
@@ -175,11 +176,11 @@ static int net_printing_dump(struct net_context *c, int argc,
 		goto done;
 	}
 
-	for (kbuf = tdb_firstkey(tdb);
+	for (kbuf = tdb_firstkey_compat(tdb);
 	     kbuf.dptr;
-	     newkey = tdb_nextkey(tdb, kbuf), free(kbuf.dptr), kbuf=newkey)
+	     kbuf = tdb_nextkey_compat(tdb, kbuf))
 	{
-		dbuf = tdb_fetch(tdb, kbuf);
+		dbuf = tdb_fetch_compat(tdb, kbuf);
 		if (!dbuf.dptr) {
 			continue;
 		}
@@ -512,7 +513,7 @@ static NTSTATUS migrate_printer(TALLOC_CTX *mem_ctx,
 			continue;
 		}
 
-		keyname = CONST_DISCARD(char *, r.printer_data[j].name);
+		keyname = discard_const_p(char, r.printer_data[j].name);
 		valuename = strchr(keyname, '\\');
 		if (valuename == NULL) {
 			continue;
@@ -636,7 +637,7 @@ static NTSTATUS printing_migrate_internal(struct net_context *c,
 {
 	TALLOC_CTX *tmp_ctx;
 	TDB_CONTEXT *tdb;
-	TDB_DATA kbuf, newkey, dbuf;
+	TDB_DATA kbuf, dbuf;
 	NTSTATUS status;
 
 	tmp_ctx = talloc_new(mem_ctx);
@@ -651,11 +652,11 @@ static NTSTATUS printing_migrate_internal(struct net_context *c,
 		goto done;
 	}
 
-	for (kbuf = tdb_firstkey(tdb);
+	for (kbuf = tdb_firstkey_compat(tdb);
 	     kbuf.dptr;
-	     newkey = tdb_nextkey(tdb, kbuf), free(kbuf.dptr), kbuf = newkey)
+	     kbuf = tdb_nextkey_compat(tdb, kbuf))
 	{
-		dbuf = tdb_fetch(tdb, kbuf);
+		dbuf = tdb_fetch_compat(tdb, kbuf);
 		if (!dbuf.dptr) {
 			continue;
 		}
diff --git a/source3/utils/net_proto.h b/source3/utils/net_proto.h
index ba3860a88d8..ad20f6e0743 100644
--- a/source3/utils/net_proto.h
+++ b/source3/utils/net_proto.h
@@ -23,6 +23,7 @@
 #ifndef _NET_PROTO_H_
 #define _NET_PROTO_H_
 
+#include "libads/ads_status.h"
 
 /* The following definitions come from utils/net.c  */
 
diff --git a/source3/utils/net_rap.c b/source3/utils/net_rap.c
index afda2227406..36f6d331970 100644
--- a/source3/utils/net_rap.c
+++ b/source3/utils/net_rap.c
@@ -26,6 +26,7 @@
 #include "../librpc/gen_ndr/rap.h"
 #include "../librpc/gen_ndr/svcctl.h"
 #include "utils/net.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 
 /* The following messages were for error checking that is not properly
@@ -824,7 +825,7 @@ static int rap_user_add(struct net_context *c, int argc, const char **argv)
 	if (!NT_STATUS_IS_OK(net_make_ipc_connection(c, 0, &cli)))
                 return -1;
 
-	safe_strcpy((char *)userinfo.user_name, argv[0], sizeof(userinfo.user_name)-1);
+	strlcpy((char *)userinfo.user_name, argv[0], sizeof(userinfo.user_name));
 	if (c->opt_flags == 0)
                 c->opt_flags = 0x21;
 
@@ -969,7 +970,7 @@ static int rap_group_add(struct net_context *c, int argc, const char **argv)
                 return -1;
 
 	/* BB check for length 21 or smaller explicitly ? BB */
-	safe_strcpy((char *)grinfo.group_name, argv[0], sizeof(grinfo.group_name)-1);
+	strlcpy((char *)grinfo.group_name, argv[0], sizeof(grinfo.group_name));
 	grinfo.reserved1 = '\0';
 	grinfo.comment = smb_xstrdup(c->opt_comment ? c->opt_comment : "");
 
diff --git a/source3/utils/net_registry.c b/source3/utils/net_registry.c
index 19405e2c085..086c4e7ccc3 100644
--- a/source3/utils/net_registry.c
+++ b/source3/utils/net_registry.c
@@ -122,23 +122,88 @@ done:
 	return werr;
 }
 
+static WERROR registry_enumkey(struct registry_key* parent, const char* keyname, bool recursive)
+{
+	WERROR werr;
+	TALLOC_CTX *ctx = talloc_stackframe();
+	char*  subkey_name;
+	NTTIME modtime;
+	uint32_t count;
+	char* valname = NULL;
+	struct registry_value *valvalue = NULL;
+	struct registry_key* key = NULL;
+
+	werr = reg_openkey(ctx, parent, keyname, REG_KEY_READ, &key);
+	if (!W_ERROR_IS_OK(werr)) {
+		goto done;
+	}
+
+	if (recursive) {
+		printf("[%s]\n\n", key->key->name);
+	} else {
+		for (count = 0;
+		     werr = reg_enumkey(ctx, key, count, &subkey_name, &modtime),
+		     W_ERROR_IS_OK(werr);
+		     count++)
+		{
+			print_registry_key(subkey_name, &modtime);
+		}
+		if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
+			goto done;
+		}
+	}
+
+	for (count = 0;
+	     werr = reg_enumvalue(ctx, key, count, &valname, &valvalue),
+	     W_ERROR_IS_OK(werr);
+	     count++)
+	{
+		print_registry_value_with_name(valname, valvalue);
+	}
+	if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
+		goto done;
+	}
+
+	if (!recursive) {
+		werr = WERR_OK;
+		goto done;
+	}
+
+	for (count = 0;
+	     werr = reg_enumkey(ctx, key, count, &subkey_name, &modtime),
+	     W_ERROR_IS_OK(werr);
+	     count++)
+	{
+		werr = registry_enumkey(key, subkey_name, recursive);
+		if (!W_ERROR_IS_OK(werr)) {
+			goto done;
+		}
+	}
+	if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
+		goto done;
+	}
+
+	werr = WERR_OK;
+
+done:
+	TALLOC_FREE(ctx);
+	return werr;
+}
+
+
+
 /*
  *
  * the main "net registry" function implementations
  *
  */
-
 static int net_registry_enumerate(struct net_context *c, int argc,
 				  const char **argv)
 {
 	WERROR werr;
 	struct registry_key *key = NULL;
+	char* name = NULL;
 	TALLOC_CTX *ctx = talloc_stackframe();
-	char *subkey_name;
-	NTTIME modtime;
-	uint32_t count;
-	char *valname = NULL;
-	struct registry_value *valvalue = NULL;
 	int ret = -1;
 
 	if (argc != 1 || c->display_usage) {
@@ -151,40 +216,56 @@ static int net_registry_enumerate(struct net_context *c, int argc,
 		goto done;
 	}
 
-	werr = open_key(ctx, argv[0], REG_KEY_READ, &key);
+	werr = open_hive(ctx, argv[0], REG_KEY_READ, &key, &name);
 	if (!W_ERROR_IS_OK(werr)) {
 		d_fprintf(stderr, _("open_key failed: %s\n"), win_errstr(werr));
 		goto done;
 	}
 
-	for (count = 0;
-	     werr = reg_enumkey(ctx, key, count, &subkey_name, &modtime),
-	     W_ERROR_IS_OK(werr);
-	     count++)
-	{
-		print_registry_key(subkey_name, &modtime);
+	werr = registry_enumkey(key, name, c->opt_reboot);
+	if (W_ERROR_IS_OK(werr)) {
+		ret = 0;
 	}
-	if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
+done:
+	TALLOC_FREE(ctx);
+	return ret;
+}
+
+static int net_registry_enumerate_recursive(struct net_context *c, int argc,
+					    const char **argv)
+{
+	WERROR werr;
+	struct registry_key *key = NULL;
+	char* name = NULL;
+	TALLOC_CTX *ctx = talloc_stackframe();
+	int ret = -1;
+
+	if (argc != 1 || c->display_usage) {
+		d_printf("%s\n%s",
+			 _("Usage:"),
+			 _("net registry enumerate <path>\n"));
+		d_printf("%s\n%s",
+			 _("Example:"),
+			 _("net registry enumerate 'HKLM\\Software\\Samba'\n"));
 		goto done;
 	}
 
-	for (count = 0;
-	     werr = reg_enumvalue(ctx, key, count, &valname, &valvalue),
-	     W_ERROR_IS_OK(werr);
-	     count++)
-	{
-		print_registry_value_with_name(valname, valvalue);
-	}
-	if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
+	werr = open_hive(ctx, argv[0], REG_KEY_READ, &key, &name);
+	if (!W_ERROR_IS_OK(werr)) {
+		d_fprintf(stderr, _("open_key failed: %s\n"), win_errstr(werr));
 		goto done;
 	}
 
-	ret = 0;
+	werr = registry_enumkey(key, name, true);
+	if (W_ERROR_IS_OK(werr)) {
+		ret = 0;
+	}
 done:
 	TALLOC_FREE(ctx);
 	return ret;
 }
 
+
 static int net_registry_createkey(struct net_context *c, int argc,
 				  const char **argv)
 {
@@ -958,9 +1039,11 @@ static int net_registry_import(struct net_context *c, int argc,
 		.createkey   = (reg_import_callback_createkey_t)&import_create_key,
 		.deletekey   = (reg_import_callback_deletekey_t)&import_delete_key,
 		.deleteval   = (reg_import_callback_deleteval_t)&import_delete_val,
-		.setval.registry_value = (reg_import_callback_setval_registry_value_t)
-		&import_create_val,
-		.setval_type           = REGISTRY_VALUE,
+		.setval      = {
+			.registry_value = (reg_import_callback_setval_registry_value_t)
+					  &import_create_val,
+		},
+		.setval_type = REGISTRY_VALUE,
 		.data        = &import_ctx
 	};
 
@@ -1019,7 +1102,6 @@ static int registry_export(TALLOC_CTX *ctx, /*const*/ struct registry_key* key,
 	struct registry_value *valvalue = NULL;
 	char *valname = NULL;
 
-	struct registry_key* subkey = NULL;
 	char *subkey_name = NULL;
 	NTTIME modtime = 0;
 
@@ -1045,6 +1127,8 @@ static int registry_export(TALLOC_CTX *ctx, /*const*/ struct registry_key* key,
 		     W_ERROR_IS_OK(werr);
 	     count++)
 	{
+		struct registry_key* subkey = NULL;
+
 		werr = reg_openkey(ctx, key, subkey_name, REG_KEY_READ,
 				   &subkey);
 		if (!W_ERROR_IS_OK(werr)) {
@@ -1054,6 +1138,7 @@ static int registry_export(TALLOC_CTX *ctx, /*const*/ struct registry_key* key,
 		}
 
 		registry_export(ctx, subkey, f);
+		TALLOC_FREE(subkey);
 	}
 	if (!W_ERROR_EQUAL(WERR_NO_MORE_ITEMS, werr)) {
 		d_fprintf(stderr, _("reg_enumkey failed: %s\n"),
@@ -1174,6 +1259,14 @@ int net_registry(struct net_context *c, int argc, const char **argv)
 			   "    Enumerate registry keys and values")
 		},
 		{
+			"enumerate_recursive",
+			net_registry_enumerate_recursive,
+			NET_TRANSPORT_LOCAL,
+			N_("Enumerate registry keys and values"),
+			N_("net registry enumerate_recursive\n"
+			   "    Enumerate registry keys and values")
+		},
+		{
 			"createkey",
 			net_registry_createkey,
 			NET_TRANSPORT_LOCAL,
diff --git a/source3/utils/net_rpc.c b/source3/utils/net_rpc.c
index 4003b88d51c..2ecf1aaa5a4 100644
--- a/source3/utils/net_rpc.c
+++ b/source3/utils/net_rpc.c
@@ -39,6 +39,7 @@
 #include "lib/netapi/netapi_net.h"
 #include "rpc_client/init_lsa.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "nsswitch/libwbclient/wbclient.h"
 #include "passdb.h"
@@ -365,7 +366,7 @@ static NTSTATUS rpc_oldjoin_internals(struct net_context *c,
 		sec_channel_type = get_sec_channel_type(NULL);
 	}
 
-	fstrcpy(trust_passwd, global_myname());
+	fstrcpy(trust_passwd, lp_netbios_name());
 	strlower_m(trust_passwd);
 
 	/*
@@ -378,7 +379,7 @@ static NTSTATUS rpc_oldjoin_internals(struct net_context *c,
 	E_md4hash(trust_passwd, orig_trust_passwd_hash);
 
 	result = trust_pw_change_and_store_it(pipe_hnd, mem_ctx, c->opt_target_workgroup,
-					      global_myname(),
+					      lp_netbios_name(),
 					      orig_trust_passwd_hash,
 					      sec_channel_type);
 
@@ -480,10 +481,10 @@ int net_rpc_join(struct net_context *c, int argc, const char **argv)
 		return -1;
 	}
 
-	if (strlen(global_myname()) > 15) {
+	if (strlen(lp_netbios_name()) > 15) {
 		d_printf(_("Our netbios name can be at most 15 chars long, "
 			   "\"%s\" is %u chars long\n"),
-			 global_myname(), (unsigned int)strlen(global_myname()));
+			 lp_netbios_name(), (unsigned int)strlen(lp_netbios_name()));
 		return -1;
 	}
 
@@ -549,7 +550,7 @@ NTSTATUS rpc_info_internals(struct net_context *c,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					&connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, domain_sid),
+					discard_const_p(struct dom_sid2, domain_sid),
 					&domain_pol,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -1622,7 +1623,7 @@ static NTSTATUS rpc_group_delete_internals(struct net_context *c,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					&connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, domain_sid),
+					discard_const_p(struct dom_sid2, domain_sid),
 					&domain_pol,
 					&result);
         if (!NT_STATUS_IS_OK(status)) {
@@ -1976,7 +1977,7 @@ static NTSTATUS get_sid_from_name(struct cli_state *cli,
 		TALLOC_FREE(pipe_hnd);
 	}
 
-	if (!NT_STATUS_IS_OK(status) && (StrnCaseCmp(name, "S-", 2) == 0)) {
+	if (!NT_STATUS_IS_OK(status) && (strncasecmp_m(name, "S-", 2) == 0)) {
 
 		/* Try as S-1-5-whatever */
 
@@ -2584,7 +2585,7 @@ static NTSTATUS rpc_group_list_internals(struct net_context *c,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					&connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, domain_sid),
+					discard_const_p(struct dom_sid2, domain_sid),
 					&domain_pol,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -2710,7 +2711,7 @@ static NTSTATUS rpc_group_list_internals(struct net_context *c,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					&connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, &global_sid_Builtin),
+					discard_const_p(struct dom_sid2, &global_sid_Builtin),
 					&domain_pol,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -2956,7 +2957,7 @@ static NTSTATUS rpc_list_alias_members(struct net_context *c,
 		return result;
 	}
 
-	alias_sids = TALLOC_ZERO_ARRAY(mem_ctx, struct dom_sid, num_members);
+	alias_sids = talloc_zero_array(mem_ctx, struct dom_sid, num_members);
 	if (!alias_sids) {
 		d_fprintf(stderr, _("Out of memory\n"));
 		TALLOC_FREE(lsa_pipe);
@@ -3032,7 +3033,7 @@ static NTSTATUS rpc_group_members_internals(struct net_context *c,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					&connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, domain_sid),
+					discard_const_p(struct dom_sid2, domain_sid),
 					&domain_pol,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -3430,7 +3431,7 @@ static WERROR get_share_info(struct net_context *c,
 	{
 		struct srvsvc_NetShareCtr1 *ctr1;
 
-		ctr1 = TALLOC_ZERO_P(mem_ctx, struct srvsvc_NetShareCtr1);
+		ctr1 = talloc_zero(mem_ctx, struct srvsvc_NetShareCtr1);
 		W_ERROR_HAVE_NO_MEMORY(ctr1);
 
 		ctr1->count = 1;
@@ -3444,7 +3445,7 @@ static WERROR get_share_info(struct net_context *c,
 	{
 		struct srvsvc_NetShareCtr2 *ctr2;
 
-		ctr2 = TALLOC_ZERO_P(mem_ctx, struct srvsvc_NetShareCtr2);
+		ctr2 = talloc_zero(mem_ctx, struct srvsvc_NetShareCtr2);
 		W_ERROR_HAVE_NO_MEMORY(ctr2);
 
 		ctr2->count = 1;
@@ -3458,7 +3459,7 @@ static WERROR get_share_info(struct net_context *c,
 	{
 		struct srvsvc_NetShareCtr502 *ctr502;
 
-		ctr502 = TALLOC_ZERO_P(mem_ctx, struct srvsvc_NetShareCtr502);
+		ctr502 = talloc_zero(mem_ctx, struct srvsvc_NetShareCtr502);
 		W_ERROR_HAVE_NO_MEMORY(ctr502);
 
 		ctr502->count = 1;
@@ -3730,7 +3731,7 @@ static NTSTATUS copy_fn(const char *mnt, struct file_info *f,
 	DEBUG(3,("got mask: %s, name: %s\n", mask, f->name));
 
 	/* DIRECTORY */
-	if (f->mode & aDIR) {
+	if (f->mode & FILE_ATTRIBUTE_DIRECTORY) {
 
 		DEBUG(3,("got dir: %s\n", f->name));
 
@@ -3763,8 +3764,8 @@ static NTSTATUS copy_fn(const char *mnt, struct file_info *f,
 		}
 
 		/* search below that directory */
-		fstrcpy(new_mask, dir);
-		fstrcat(new_mask, "\\*");
+		strlcpy(new_mask, dir, sizeof(new_mask));
+		strlcat(new_mask, "\\*", sizeof(new_mask));
 
 		old_dir = local_state->cwd;
 		local_state->cwd = dir;
@@ -3968,7 +3969,7 @@ static NTSTATUS rpc_share_migrate_files_internals(struct net_context *c,
 		cp_clistate.cli_share_src = NULL;
 		cp_clistate.cli_share_dst = NULL;
 		cp_clistate.cwd = NULL;
-		cp_clistate.attribute = aSYSTEM | aHIDDEN | aDIR;
+		cp_clistate.attribute = FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY;
 		cp_clistate.c = c;
 
 	        /* open share source */
@@ -4318,7 +4319,7 @@ static NTSTATUS rpc_fetch_domain_aliases(struct rpc_pipe_client *pipe_hnd,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, domain_sid),
+					discard_const_p(struct dom_sid2, domain_sid),
 					&domain_pol,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -4802,7 +4803,7 @@ static bool get_user_tokens_from_file(FILE *f,
 
 		token = &((*tokens)[*num_tokens-1]);
 
-		fstrcpy(token->name, line);
+		strlcpy(token->name, line, sizeof(token->name));
 		token->token.num_sids = 0;
 		token->token.sids = NULL;
 		continue;
@@ -5803,7 +5804,7 @@ static NTSTATUS rpc_trustdom_add_internals(struct net_context *c,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					&connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, domain_sid),
+					discard_const_p(struct dom_sid2, domain_sid),
 					&domain_pol,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -5977,7 +5978,7 @@ static NTSTATUS rpc_trustdom_del_internals(struct net_context *c,
 	status = dcerpc_samr_OpenDomain(b, mem_ctx,
 					&connect_pol,
 					MAXIMUM_ALLOWED_ACCESS,
-					CONST_DISCARD(struct dom_sid2 *, domain_sid),
+					discard_const_p(struct dom_sid2, domain_sid),
 					&domain_pol,
 					&result);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -6500,7 +6501,6 @@ static int rpc_trustdom_vampire(struct net_context *c, int argc,
 	struct rpc_pipe_client *pipe_hnd = NULL;
 	NTSTATUS nt_status, result;
 	const char *domain_name = NULL;
-	struct dom_sid *queried_dom_sid;
 	struct policy_handle connect_hnd;
 	union lsa_PolicyInformation *info = NULL;
 
@@ -6531,11 +6531,11 @@ static int rpc_trustdom_vampire(struct net_context *c, int argc,
 	 * or to remote one given in command line
 	 */
 
-	if (StrCaseCmp(c->opt_workgroup, lp_workgroup())) {
+	if (strcasecmp_m(c->opt_workgroup, lp_workgroup())) {
 		domain_name = c->opt_workgroup;
 		c->opt_target_workgroup = c->opt_workgroup;
 	} else {
-		fstrcpy(pdc_name, global_myname());
+		fstrcpy(pdc_name, lp_netbios_name());
 		domain_name = talloc_strdup(mem_ctx, lp_workgroup());
 		c->opt_target_workgroup = domain_name;
 	};
@@ -6593,8 +6593,6 @@ static int rpc_trustdom_vampire(struct net_context *c, int argc,
 		return -1;
 	}
 
-	queried_dom_sid = info->account_domain.sid;
-
 	/*
 	 * Keep calling LsaEnumTrustdom over opened pipe until
 	 * the end of enumeration is reached
@@ -6711,11 +6709,11 @@ static int rpc_trustdom_list(struct net_context *c, int argc, const char **argv)
 	 * or to remote one given in command line
 	 */
 
-	if (StrCaseCmp(c->opt_workgroup, lp_workgroup())) {
+	if (strcasecmp_m(c->opt_workgroup, lp_workgroup())) {
 		domain_name = c->opt_workgroup;
 		c->opt_target_workgroup = c->opt_workgroup;
 	} else {
-		fstrcpy(pdc_name, global_myname());
+		fstrcpy(pdc_name, lp_netbios_name());
 		domain_name = talloc_strdup(mem_ctx, lp_workgroup());
 		c->opt_target_workgroup = domain_name;
 	};
@@ -6937,7 +6935,7 @@ static int rpc_trustdom_list(struct net_context *c, int argc, const char **argv)
 
 		for (i = 0; i < num_domains; i++) {
 
-			char *str = CONST_DISCARD(char *, trusts->entries[i].name.string);
+			char *str = discard_const_p(char, trusts->entries[i].name.string);
 
 			found_domain = true;
 
@@ -7090,16 +7088,11 @@ bool net_rpc_check(struct net_context *c, unsigned flags)
 	if (!net_find_server(c, NULL, flags, &server_ss, &server_name))
 		return false;
 
-	if ((cli = cli_initialise()) == NULL) {
+	status = cli_connect_nb(server_name, &server_ss, 0, 0x20,
+				lp_netbios_name(), Undefined, &cli);
+	if (!NT_STATUS_IS_OK(status)) {
 		return false;
 	}
-
-	status = cli_connect(cli, server_name, &server_ss);
-	if (!NT_STATUS_IS_OK(status))
-		goto done;
-	if (!attempt_netbios_session_request(&cli, global_myname(),
-					     server_name, &server_ss))
-		goto done;
 	status = cli_negprot(cli);
 	if (!NT_STATUS_IS_OK(status))
 		goto done;
diff --git a/source3/utils/net_rpc_join.c b/source3/utils/net_rpc_join.c
index 1ca213d155b..d228f928047 100644
--- a/source3/utils/net_rpc_join.c
+++ b/source3/utils/net_rpc_join.c
@@ -30,6 +30,7 @@
 #include "rpc_client/cli_netlogon.h"
 #include "secrets.h"
 #include "rpc_client/init_lsa.h"
+#include "libsmb/libsmb.h"
 
 /* Macro for checking RPC error codes to make things more readable */
 
@@ -306,7 +307,7 @@ int net_rpc_join_newstyle(struct net_context *c, int argc, const char **argv)
 		      "could not open domain");
 
 	/* Create domain user */
-	if ((acct_name = talloc_asprintf(mem_ctx, "%s$", global_myname())) == NULL) {
+	if ((acct_name = talloc_asprintf(mem_ctx, "%s$", lp_netbios_name())) == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
 	}
@@ -442,8 +443,8 @@ int net_rpc_join_newstyle(struct net_context *c, int argc, const char **argv)
 	status = rpccli_netlogon_setup_creds(pipe_hnd,
 					cli->desthost, /* server name */
 					domain,        /* domain */
-					global_myname(), /* client name */
-					global_myname(), /* machine account name */
+					lp_netbios_name(), /* client name */
+					lp_netbios_name(), /* machine account name */
                                         md4_trust_password,
                                         sec_channel_type,
                                         &neg_flags);
@@ -457,7 +458,7 @@ int net_rpc_join_newstyle(struct net_context *c, int argc, const char **argv)
 			d_fprintf(stderr, _("Please make sure that no computer "
 					    "account\nnamed like this machine "
 					    "(%s) exists in the domain\n"),
-				 global_myname());
+				 lp_netbios_name());
 		}
 
 		goto done;
@@ -486,7 +487,7 @@ int net_rpc_join_newstyle(struct net_context *c, int argc, const char **argv)
 						    "computer account\nnamed "
 						    "like this machine (%s) "
 						    "exists in the domain\n"),
-					 global_myname());
+					 lp_netbios_name());
 			}
 
 			goto done;
@@ -498,7 +499,7 @@ int net_rpc_join_newstyle(struct net_context *c, int argc, const char **argv)
 
 	/* Now store the secret in the secrets database */
 
-	strupper_m(CONST_DISCARD(char *, domain));
+	strupper_m(discard_const_p(char, domain));
 
 	if (!secrets_store_domain_sid(domain, domain_sid)) {
 		DEBUG(0, ("error storing domain sid for %s\n", domain));
diff --git a/source3/utils/net_rpc_printer.c b/source3/utils/net_rpc_printer.c
index 7a225871744..272e0c9dfa1 100644
--- a/source3/utils/net_rpc_printer.c
+++ b/source3/utils/net_rpc_printer.c
@@ -27,7 +27,7 @@
 #include "registry/reg_objects.h"
 #include "../libcli/security/security.h"
 #include "../libcli/registry/util_reg.h"
-#include "ntdomain.h"
+#include "libsmb/libsmb.h"
 
 /* support itanium as well */
 static const struct print_architecture_table_node archi_table[]= {
@@ -70,7 +70,7 @@ static void display_print_driver3(struct spoolss_DriverInfo3 *r)
 	printf(_("\tConfigfile: [%s]\n\n"), r->config_file);
 	printf(_("\tHelpfile: [%s]\n\n"), r->help_file);
 
-	for (i=0; r->dependent_files[i] != NULL; i++) {
+	for (i=0; r->dependent_files && r->dependent_files[i] != NULL; i++) {
 		printf(_("\tDependentfiles: [%s]\n"), r->dependent_files[i]);
 	}
 
@@ -158,14 +158,13 @@ NTSTATUS net_copy_fileattr(struct net_context *c,
 		  bool copy_acls, bool copy_attrs,
 		  bool copy_timestamps, bool is_file)
 {
-	NTSTATUS nt_status = NT_STATUS_UNSUCCESSFUL;
+	NTSTATUS nt_status;
 	uint16_t fnum_src = 0;
 	uint16_t fnum_dst = 0;
 	struct security_descriptor *sd = NULL;
 	uint16_t attr;
 	time_t f_atime, f_ctime, f_mtime;
 
-
 	if (!copy_timestamps && !copy_acls && !copy_attrs)
 		return NT_STATUS_OK;
 
@@ -174,17 +173,17 @@ NTSTATUS net_copy_fileattr(struct net_context *c,
 	DEBUGADD(3,("opening %s %s on originating server\n",
 		is_file?"file":"dir", src_name));
 
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli_share_src, src_name, 0, READ_CONTROL_ACCESS, 0,
-				FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OPEN, 0x0, 0x0, &fnum_src))) {
+	nt_status = cli_ntcreate(cli_share_src, src_name, 0,
+				 READ_CONTROL_ACCESS, 0,
+				 FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OPEN,
+				 0x0, 0x0, &fnum_src);
+	if (!NT_STATUS_IS_OK(nt_status)) {
 		DEBUGADD(0,("cannot open %s %s on originating server %s\n",
-			is_file?"file":"dir", src_name, cli_errstr(cli_share_src)));
-		nt_status = cli_nt_error(cli_share_src);
+			is_file?"file":"dir", src_name, nt_errstr(nt_status)));
 		goto out;
 	}
 
-
 	if (copy_acls) {
-
 		/* get the security descriptor */
 		sd = cli_query_secdesc(cli_share_src, fnum_src, mem_ctx);
 		if (!sd) {
@@ -198,81 +197,74 @@ NTSTATUS net_copy_fileattr(struct net_context *c,
 			display_sec_desc(sd);
 	}
 
-
 	if (copy_attrs || copy_timestamps) {
 
 		/* get file attributes */
-		if (!NT_STATUS_IS_OK(cli_getattrE(cli_share_src, fnum_src, &attr, NULL,
-				 &f_ctime, &f_atime, &f_mtime))) {
+		nt_status = cli_getattrE(cli_share_src, fnum_src, &attr, NULL,
+		                      &f_ctime, &f_atime, &f_mtime);
+		if (!NT_STATUS_IS_OK(nt_status)) {
 			DEBUG(0,("failed to get file-attrs: %s\n",
-				cli_errstr(cli_share_src)));
-			nt_status = cli_nt_error(cli_share_src);
+				nt_errstr(nt_status)));
 			goto out;
 		}
 	}
 
-
 	/* open the file/dir on the destination server */
-
-	if (!NT_STATUS_IS_OK(cli_ntcreate(cli_share_dst, dst_name, 0, WRITE_DAC_ACCESS | WRITE_OWNER_ACCESS, 0,
-				FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OPEN, 0x0, 0x0, &fnum_dst))) {
+	nt_status = cli_ntcreate(cli_share_dst, dst_name, 0,
+				 WRITE_DAC_ACCESS | WRITE_OWNER_ACCESS, 0,
+				 FILE_SHARE_READ|FILE_SHARE_WRITE, FILE_OPEN,
+				 0x0, 0x0, &fnum_dst);
+	if (!NT_STATUS_IS_OK(nt_status)) {
 		DEBUG(0,("failed to open %s on the destination server: %s: %s\n",
-			is_file?"file":"dir", dst_name, cli_errstr(cli_share_dst)));
-		nt_status = cli_nt_error(cli_share_dst);
+			is_file?"file":"dir", dst_name, nt_errstr(nt_status)));
 		goto out;
 	}
 
 	if (copy_timestamps) {
-
 		/* set timestamps */
-		if (!NT_STATUS_IS_OK(cli_setattrE(cli_share_dst, fnum_dst, f_ctime, f_atime, f_mtime))) {
+		nt_status = cli_setattrE(cli_share_dst, fnum_dst, f_ctime, f_atime, f_mtime);
+		if (!NT_STATUS_IS_OK(nt_status)) {
 			DEBUG(0,("failed to set file-attrs (timestamps): %s\n",
-				cli_errstr(cli_share_dst)));
-			nt_status = cli_nt_error(cli_share_dst);
+				nt_errstr(nt_status)));
 			goto out;
 		}
 	}
 
 	if (copy_acls) {
-		NTSTATUS status;
-
 		/* set acls */
-		status = cli_set_secdesc(cli_share_dst, fnum_dst, sd);
-		if (!NT_STATUS_IS_OK(status)) {
+		nt_status = cli_set_secdesc(cli_share_dst, fnum_dst, sd);
+		if (!NT_STATUS_IS_OK(nt_status)) {
 			DEBUG(0, ("could not set secdesc on %s: %s\n",
-				  dst_name, nt_errstr(status)));
-			nt_status = status;
+				  dst_name, nt_errstr(nt_status)));
 			goto out;
 		}
 	}
 
 	if (copy_attrs) {
-
 		/* set attrs */
-		if (!NT_STATUS_IS_OK(cli_setatr(cli_share_dst, dst_name, attr, 0))) {
+		nt_status = cli_setatr(cli_share_dst, dst_name, attr, 0);
+		if (!NT_STATUS_IS_OK(nt_status)) {
 			DEBUG(0,("failed to set file-attrs: %s\n",
-				cli_errstr(cli_share_dst)));
-			nt_status = cli_nt_error(cli_share_dst);
+				nt_errstr(nt_status)));
 			goto out;
 		}
 	}
 
 
 	/* closing files */
-
-	if (!NT_STATUS_IS_OK(cli_close(cli_share_src, fnum_src))) {
+	nt_status = cli_close(cli_share_src, fnum_src);
+	if (!NT_STATUS_IS_OK(nt_status)) {
 		d_fprintf(stderr,
 			_("could not close %s on originating server: %s\n"),
-			is_file?"file":"dir", cli_errstr(cli_share_src));
-		nt_status = cli_nt_error(cli_share_src);
+			is_file?"file":"dir", nt_errstr(nt_status));
 		goto out;
 	}
 
-	if (!NT_STATUS_IS_OK(cli_close(cli_share_dst, fnum_dst))) {
+	nt_status = cli_close(cli_share_dst, fnum_dst);
+	if (!NT_STATUS_IS_OK(nt_status)) {
 		d_fprintf(stderr,
 			_("could not close %s on destination server: %s\n"),
-			is_file?"file":"dir", cli_errstr(cli_share_dst));
-		nt_status = cli_nt_error(cli_share_dst);
+			is_file?"file":"dir", nt_errstr(nt_status));
 		goto out;
 	}
 
@@ -343,7 +335,7 @@ NTSTATUS net_copy_file(struct net_context *c,
 	if (!NT_STATUS_IS_OK(nt_status)) {
 		DEBUGADD(0,("cannot open %s %s on originating server %s\n",
 			is_file ? "file":"dir",
-			src_name, cli_errstr(cli_share_src)));
+			src_name, nt_errstr(nt_status)));
 		goto out;
 	}
 
@@ -357,7 +349,7 @@ NTSTATUS net_copy_file(struct net_context *c,
 
 		if (!NT_STATUS_IS_OK(nt_status)) {
 			DEBUGADD(1,("cannot create file %s on destination server: %s\n", 
-				dst_name, cli_errstr(cli_share_dst)));
+				dst_name, nt_errstr(nt_status)));
 			goto out;
 		}
 
@@ -387,20 +379,19 @@ NTSTATUS net_copy_file(struct net_context *c,
 	while (is_file) {
 
 		/* copying file */
-		int n, ret;
+		int n;
 		n = cli_read(cli_share_src, fnum_src, data, nread,
 				read_size);
 
 		if (n <= 0)
 			break;
 
-		ret = cli_write(cli_share_dst, fnum_dst, 0, data,
-			nread, n);
+		nt_status = cli_writeall(cli_share_dst, fnum_dst, 0,
+					 (uint8_t *)data, nread, n, NULL);
 
-		if (n != ret) {
+		if (!NT_STATUS_IS_OK(nt_status)) {
 			d_fprintf(stderr, _("Error writing file: %s\n"),
-				cli_errstr(cli_share_dst));
-			nt_status = cli_nt_error(cli_share_dst);
+				  nt_errstr(nt_status));
 			goto out;
 		}
 
@@ -414,36 +405,41 @@ NTSTATUS net_copy_file(struct net_context *c,
 		DEBUGADD(3,("creating dir %s on the destination server\n",
 			dst_name));
 
-		if (!NT_STATUS_IS_OK(cli_mkdir(cli_share_dst, dst_name))) {
+		nt_status = cli_mkdir(cli_share_dst, dst_name);
+		if (!NT_STATUS_IS_OK(nt_status)) {
 			DEBUG(0,("cannot create directory %s: %s\n",
-				dst_name, cli_errstr(cli_share_dst)));
+				dst_name, nt_errstr(nt_status)));
 			nt_status = NT_STATUS_NO_SUCH_FILE;
 		}
 
-		if (!NT_STATUS_IS_OK(cli_chkpath(cli_share_dst, dst_name))) {
+
+		nt_status = cli_chkpath(cli_share_dst, dst_name);
+		if (!NT_STATUS_IS_OK(nt_status)) {
 			d_fprintf(stderr,
 				_("cannot check for directory %s: %s\n"),
-				dst_name, cli_errstr(cli_share_dst));
+				dst_name, nt_errstr(nt_status));
 			goto out;
 		}
 	}
 
 
 	/* closing files */
-	if (!NT_STATUS_IS_OK(cli_close(cli_share_src, fnum_src))) {
+	nt_status = cli_close(cli_share_src, fnum_src);
+	if (!NT_STATUS_IS_OK(nt_status)) {
 		d_fprintf(stderr,
 			_("could not close file on originating server: %s\n"),
-			cli_errstr(cli_share_src));
-		nt_status = cli_nt_error(cli_share_src);
+			nt_errstr(nt_status));
 		goto out;
 	}
 
-	if (is_file && !NT_STATUS_IS_OK(cli_close(cli_share_dst, fnum_dst))) {
-		d_fprintf(stderr,
+	if (is_file) {
+		nt_status = cli_close(cli_share_dst, fnum_dst);
+		if (!NT_STATUS_IS_OK(nt_status)) {
+			d_fprintf(stderr,
 			_("could not close file on destination server: %s\n"),
-			cli_errstr(cli_share_dst));
-		nt_status = cli_nt_error(cli_share_dst);
-		goto out;
+			nt_errstr(nt_status));
+			goto out;
+		}
 	}
 
 	/* possibly we have to copy some file-attributes / acls / sd */
@@ -563,15 +559,16 @@ static NTSTATUS check_arch_dir(struct cli_state *cli_share, const char *short_ar
 	DEBUG(10,("creating print-driver dir for architecture: %s\n",
 		short_archi));
 
-	if (!NT_STATUS_IS_OK(cli_mkdir(cli_share, dir))) {
+	nt_status = cli_mkdir(cli_share, dir);
+	if (!NT_STATUS_IS_OK(nt_status)) {
                 DEBUG(1,("cannot create directory %s: %s\n",
-                         dir, cli_errstr(cli_share)));
-                nt_status = NT_STATUS_NO_SUCH_FILE;
+                         dir, nt_errstr(nt_status)));
         }
 
-	if (!NT_STATUS_IS_OK(cli_chkpath(cli_share, dir))) {
+	nt_status = cli_chkpath(cli_share, dir);
+	if (!NT_STATUS_IS_OK(nt_status)) {
 		d_fprintf(stderr, _("cannot check %s: %s\n"),
-			dir, cli_errstr(cli_share));
+			dir, nt_errstr(nt_status));
 		goto out;
 	}
 
@@ -2547,7 +2544,7 @@ NTSTATUS rpc_printer_migrate_settings_internals(struct net_context *c,
 
 					if (strequal(info[j].value_name, SPOOL_REG_SHORTSERVERNAME)) {
 
-						push_reg_sz(mem_ctx, &blob, global_myname());
+						push_reg_sz(mem_ctx, &blob, lp_netbios_name());
 					}
 
 					value = regval_compose(talloc_tos(),
diff --git a/source3/utils/net_rpc_registry.c b/source3/utils/net_rpc_registry.c
index 31f4e3ec5fd..9195bd477da 100644
--- a/source3/utils/net_rpc_registry.c
+++ b/source3/utils/net_rpc_registry.c
@@ -31,6 +31,7 @@
 #include <assert.h>
 #include "../libcli/security/display_sec.h"
 #include "../libcli/registry/util_reg.h"
+#include "client.h"
 
 
 /*******************************************************************
@@ -210,9 +211,9 @@ static NTSTATUS registry_enumkeys(TALLOC_CTX *ctx,
 		return NT_STATUS_OK;
 	}
 
-	if ((!(names = TALLOC_ZERO_ARRAY(mem_ctx, char *, num_subkeys))) ||
-	    (!(classes = TALLOC_ZERO_ARRAY(mem_ctx, char *, num_subkeys))) ||
-	    (!(modtimes = TALLOC_ZERO_ARRAY(mem_ctx, NTTIME *,
+	if ((!(names = talloc_zero_array(mem_ctx, char *, num_subkeys))) ||
+	    (!(classes = talloc_zero_array(mem_ctx, char *, num_subkeys))) ||
+	    (!(modtimes = talloc_zero_array(mem_ctx, NTTIME *,
 					    num_subkeys)))) {
 		status = NT_STATUS_NO_MEMORY;
 		goto error;
@@ -332,8 +333,8 @@ static NTSTATUS registry_enumvalues(TALLOC_CTX *ctx,
 		return NT_STATUS_OK;
 	}
 
-	if ((!(names = TALLOC_ARRAY(mem_ctx, char *, num_values))) ||
-	    (!(values = TALLOC_ARRAY(mem_ctx, struct registry_value *,
+	if ((!(names = talloc_array(mem_ctx, char *, num_values))) ||
+	    (!(values = talloc_array(mem_ctx, struct registry_value *,
 				     num_values)))) {
 		status = NT_STATUS_NO_MEMORY;
 		goto error;
@@ -455,8 +456,8 @@ static NTSTATUS registry_enumvalues2(TALLOC_CTX *ctx,
 		return NT_STATUS_OK;
 	}
 
-	if ((!(names = TALLOC_ARRAY(mem_ctx, char *, num_values))) ||
-	    (!(values = TALLOC_ARRAY(mem_ctx, struct regval_blob *,
+	if ((!(names = talloc_array(mem_ctx, char *, num_values))) ||
+	    (!(values = talloc_array(mem_ctx, struct regval_blob *,
 				     num_values)))) {
 		status = NT_STATUS_NO_MEMORY;
 		goto error;
@@ -1450,7 +1451,7 @@ static NTSTATUS rpc_registry_getsd_internal(struct net_context *c,
 		return status;
 	}
 
-	sd = TALLOC_ZERO_P(mem_ctx, struct KeySecurityData);
+	sd = talloc_zero(mem_ctx, struct KeySecurityData);
 	if (!sd) {
 		status = NT_STATUS_NO_MEMORY;
 		goto out;
@@ -1958,7 +1959,9 @@ static NTSTATUS rpc_registry_import_internal(struct net_context *c,
 		.createkey   = (reg_import_callback_createkey_t)&import_create_key,
 		.deletekey   = (reg_import_callback_deletekey_t)&import_delete_key,
 		.deleteval   = (reg_import_callback_deleteval_t)&import_delete_val,
-		.setval.blob = (reg_import_callback_setval_blob_t)&import_create_val,
+		.setval      = {
+			.blob = (reg_import_callback_setval_blob_t)&import_create_val,
+		},
 		.setval_type = BLOB,
 		.data = &import_ctx
 	};
diff --git a/source3/utils/net_rpc_rights.c b/source3/utils/net_rpc_rights.c
index a2847749b70..737bfb0e868 100644
--- a/source3/utils/net_rpc_rights.c
+++ b/source3/utils/net_rpc_rights.c
@@ -190,7 +190,7 @@ static NTSTATUS check_privilege_for_user(struct rpc_pipe_client *pipe_hnd,
 	}
 
 	for (i = 0; i < rights.count; i++) {
-		if (StrCaseCmp(rights.names[i].string, right) == 0) {
+		if (strcasecmp_m(rights.names[i].string, right) == 0) {
 			return NT_STATUS_OK;
 		}
 	}
@@ -506,7 +506,7 @@ static NTSTATUS rpc_rights_grant_internal(struct net_context *c,
 		return status;
 
 	rights.count = argc-1;
-	rights.names = TALLOC_ARRAY(mem_ctx, struct lsa_StringLarge,
+	rights.names = talloc_array(mem_ctx, struct lsa_StringLarge,
 				    rights.count);
 	if (!rights.names) {
 		return NT_STATUS_NO_MEMORY;
@@ -579,7 +579,7 @@ static NTSTATUS rpc_rights_revoke_internal(struct net_context *c,
 		return status;
 
 	rights.count = argc-1;
-	rights.names = TALLOC_ARRAY(mem_ctx, struct lsa_StringLarge,
+	rights.names = talloc_array(mem_ctx, struct lsa_StringLarge,
 				    rights.count);
 	if (!rights.names) {
 		return NT_STATUS_NO_MEMORY;
diff --git a/source3/utils/net_rpc_samsync.c b/source3/utils/net_rpc_samsync.c
index 861f429559f..04aefbb13f2 100644
--- a/source3/utils/net_rpc_samsync.c
+++ b/source3/utils/net_rpc_samsync.c
@@ -51,50 +51,50 @@ static void parse_samsync_partial_replication_objects(TALLOC_CTX *mem_ctx,
 
 		ZERO_STRUCT(o);
 
-		if (!StrnCaseCmp(argv[i], "user_rid=", strlen("user_rid="))) {
+		if (!strncasecmp_m(argv[i], "user_rid=", strlen("user_rid="))) {
 			o.object_identifier.rid		= get_int_param(argv[i]);
 			o.object_type			= NETR_DELTA_USER;
 			o.database_id			= SAM_DATABASE_DOMAIN;
 		}
-		if (!StrnCaseCmp(argv[i], "group_rid=", strlen("group_rid="))) {
+		if (!strncasecmp_m(argv[i], "group_rid=", strlen("group_rid="))) {
 			o.object_identifier.rid		= get_int_param(argv[i]);
 			o.object_type			= NETR_DELTA_GROUP;
 			o.database_id			= SAM_DATABASE_DOMAIN;
 		}
-		if (!StrnCaseCmp(argv[i], "group_member_rid=", strlen("group_member_rid="))) {
+		if (!strncasecmp_m(argv[i], "group_member_rid=", strlen("group_member_rid="))) {
 			o.object_identifier.rid		= get_int_param(argv[i]);
 			o.object_type			= NETR_DELTA_GROUP_MEMBER;
 			o.database_id			= SAM_DATABASE_DOMAIN;
 		}
-		if (!StrnCaseCmp(argv[i], "alias_rid=", strlen("alias_rid="))) {
+		if (!strncasecmp_m(argv[i], "alias_rid=", strlen("alias_rid="))) {
 			o.object_identifier.rid		= get_int_param(argv[i]);
 			o.object_type			= NETR_DELTA_ALIAS;
 			o.database_id			= SAM_DATABASE_BUILTIN;
 		}
-		if (!StrnCaseCmp(argv[i], "alias_member_rid=", strlen("alias_member_rid="))) {
+		if (!strncasecmp_m(argv[i], "alias_member_rid=", strlen("alias_member_rid="))) {
 			o.object_identifier.rid		= get_int_param(argv[i]);
 			o.object_type			= NETR_DELTA_ALIAS_MEMBER;
 			o.database_id			= SAM_DATABASE_BUILTIN;
 		}
-		if (!StrnCaseCmp(argv[i], "account_sid=", strlen("account_sid="))) {
+		if (!strncasecmp_m(argv[i], "account_sid=", strlen("account_sid="))) {
 			const char *sid_str = get_string_param(argv[i]);
 			string_to_sid(&o.object_identifier.sid, sid_str);
 			o.object_type			= NETR_DELTA_ACCOUNT;
 			o.database_id			= SAM_DATABASE_PRIVS;
 		}
-		if (!StrnCaseCmp(argv[i], "policy_sid=", strlen("policy_sid="))) {
+		if (!strncasecmp_m(argv[i], "policy_sid=", strlen("policy_sid="))) {
 			const char *sid_str = get_string_param(argv[i]);
 			string_to_sid(&o.object_identifier.sid, sid_str);
 			o.object_type			= NETR_DELTA_POLICY;
 			o.database_id			= SAM_DATABASE_PRIVS;
 		}
-		if (!StrnCaseCmp(argv[i], "trustdom_sid=", strlen("trustdom_sid="))) {
+		if (!strncasecmp_m(argv[i], "trustdom_sid=", strlen("trustdom_sid="))) {
 			const char *sid_str = get_string_param(argv[i]);
 			string_to_sid(&o.object_identifier.sid, sid_str);
 			o.object_type			= NETR_DELTA_TRUSTED_DOMAIN;
 			o.database_id			= SAM_DATABASE_PRIVS;
 		}
-		if (!StrnCaseCmp(argv[i], "secret_name=", strlen("secret_name="))) {
+		if (!strncasecmp_m(argv[i], "secret_name=", strlen("secret_name="))) {
 			o.object_identifier.name	= get_string_param(argv[i]);
 			o.object_type			= NETR_DELTA_SECRET;
 			o.database_id			= SAM_DATABASE_PRIVS;
diff --git a/source3/utils/net_rpc_shell.c b/source3/utils/net_rpc_shell.c
index 1985bf8b0b6..42dfc48d4a4 100644
--- a/source3/utils/net_rpc_shell.c
+++ b/source3/utils/net_rpc_shell.c
@@ -26,6 +26,7 @@
 #include "lib/netapi/netapi.h"
 #include "lib/netapi/netapi_net.h"
 #include "../libcli/smbreadline/smbreadline.h"
+#include "libsmb/libsmb.h"
 
 static NTSTATUS rpc_sh_info(struct net_context *c,
 			    TALLOC_CTX *mem_ctx, struct rpc_sh_ctx *ctx,
@@ -153,7 +154,7 @@ static bool net_sh_process(struct net_context *c,
 		return true;
 	}
 
-	new_ctx = TALLOC_P(ctx, struct rpc_sh_ctx);
+	new_ctx = talloc(ctx, struct rpc_sh_ctx);
 	if (new_ctx == NULL) {
 		d_fprintf(stderr, _("talloc failed\n"));
 		return false;
@@ -233,7 +234,7 @@ int net_rpc_shell(struct net_context *c, int argc, const char **argv)
 		libnetapi_set_use_kerberos(c->netapi_ctx);
 	}
 
-	ctx = TALLOC_P(NULL, struct rpc_sh_ctx);
+	ctx = talloc(NULL, struct rpc_sh_ctx);
 	if (ctx == NULL) {
 		d_fprintf(stderr, _("talloc failed\n"));
 		return -1;
diff --git a/source3/utils/net_rpc_trust.c b/source3/utils/net_rpc_trust.c
index da0a7cca67d..318c06ffe99 100644
--- a/source3/utils/net_rpc_trust.c
+++ b/source3/utils/net_rpc_trust.c
@@ -25,6 +25,7 @@
 #include "../librpc/gen_ndr/ndr_lsa_c.h"
 #include "../lib/crypto/crypto.h"
 #include "../libcli/security/dom_sid.h"
+#include "libsmb/libsmb.h"
 
 
 #define ARG_OTHERSERVER "otherserver="
@@ -418,19 +419,22 @@ static int rpc_trust_common(struct net_context *net_ctx, int argc,
 	struct other_dom_data *other_dom_data;
 	struct net_context *other_net_ctx = NULL;
 	struct dom_data dom_data[2];
+	void (*usage)(void);
+
+	switch (op) {
+		case TRUST_CREATE:
+			usage = print_trust_usage;
+			break;
+		case TRUST_DELETE:
+			usage = print_trust_delete_usage;
+			break;
+		default:
+			DEBUG(0, ("Unsupported trust operation.\n"));
+			return -1;
+	}
 
 	if (net_ctx->display_usage) {
-		switch (op) {
-			case TRUST_CREATE:
-				print_trust_usage();
-				break;
-			case TRUST_DELETE:
-				print_trust_delete_usage();
-				break;
-			default:
-				DEBUG(0, ("Unsupported trust operation.\n"));
-				return -1;
-		}
+		usage();
 		return 0;
 	}
 
@@ -443,7 +447,7 @@ static int rpc_trust_common(struct net_context *net_ctx, int argc,
 	ret = parse_trust_args(mem_ctx, argc, argv, &other_dom_data, &trust_pw);
 	if (ret != 0) {
 		if (ret == EINVAL) {
-			print_trust_usage();
+			usage();
 		} else {
 			DEBUG(0, ("Failed to parse arguments.\n"));
 		}
@@ -470,7 +474,7 @@ static int rpc_trust_common(struct net_context *net_ctx, int argc,
 		     (dom_data[1].domain_name == NULL ||
 		      dom_data[1].dns_domain_name == NULL))) {
 			DEBUG(0, ("Missing required argument.\n"));
-			print_trust_usage();
+			usage();
 			goto done;
 		}
 	}
diff --git a/source3/utils/net_serverid.c b/source3/utils/net_serverid.c
index 08a1a7f298b..4159eead7a7 100644
--- a/source3/utils/net_serverid.c
+++ b/source3/utils/net_serverid.c
@@ -26,7 +26,7 @@
 static int net_serverid_list_fn(const struct server_id *id,
 				uint32_t msg_flags, void *priv)
 {
-	char *str = procid_str(talloc_tos(), id);
+	char *str = server_id_str(talloc_tos(), id);
 	d_printf("%s %llu 0x%x\n", str, (unsigned long long)id->unique_id,
 		 (unsigned int)msg_flags);
 	TALLOC_FREE(str);
@@ -37,7 +37,7 @@ static int net_serverid_list(struct net_context *c, int argc,
 			     const char **argv)
 {
 	d_printf("pid unique_id msg_flags\n");
-	return serverid_traverse_read(net_serverid_list_fn, NULL) ? 0 : -1;
+	return serverid_traverse_read(net_serverid_list_fn, NULL) > 0 ? 0 : -1;
 }
 
 static int net_serverid_wipe_fn(struct db_record *rec,
@@ -51,7 +51,7 @@ static int net_serverid_wipe_fn(struct db_record *rec,
 	}
 	status = rec->delete_rec(rec);
 	if (!NT_STATUS_IS_OK(status)) {
-		char *str = procid_str(talloc_tos(), id);
+		char *str = server_id_str(talloc_tos(), id);
 		DEBUG(1, ("Could not delete serverid.tdb record %s: %s\n",
 			  str, nt_errstr(status)));
 		TALLOC_FREE(str);
@@ -62,7 +62,7 @@ static int net_serverid_wipe_fn(struct db_record *rec,
 static int net_serverid_wipe(struct net_context *c, int argc,
 			     const char **argv)
 {
-	return serverid_traverse(net_serverid_wipe_fn, NULL) ? 0 : -1;
+	return serverid_traverse(net_serverid_wipe_fn, NULL) > 0 ? 0 : -1;
 }
 
 static int net_serverid_wipedbs_conn(
@@ -75,13 +75,13 @@ static int net_serverid_wipedbs_conn(
 		NTSTATUS status;
 
 		DEBUG(10, ("Deleting connections.tdb record for pid %s\n",
-			   procid_str(talloc_tos(), &key->pid)));
+			   server_id_str(talloc_tos(), &key->pid)));
 
 		status = rec->delete_rec(rec);
 		if (!NT_STATUS_IS_OK(status)) {
 			DEBUG(1, ("Could not delete connections.tdb record "
 				  "for pid %s: %s\n",
-				  procid_str(talloc_tos(), &key->pid),
+				  server_id_str(talloc_tos(), &key->pid),
 				  nt_errstr(status)));
 		}
 	}
@@ -97,13 +97,13 @@ static int net_serverid_wipedbs_sessionid(struct db_record *rec,
 		NTSTATUS status;
 
 		DEBUG(10, ("Deleting sessionid.tdb record for pid %s\n",
-			   procid_str(talloc_tos(), &session->pid)));
+			   server_id_str(talloc_tos(), &session->pid)));
 
 		status = rec->delete_rec(rec);
 		if (!NT_STATUS_IS_OK(status)) {
 			DEBUG(1, ("Could not delete session.tdb record "
 				  "for pid %s: %s\n",
-				  procid_str(talloc_tos(), &session->pid),
+				  server_id_str(talloc_tos(), &session->pid),
 				  nt_errstr(status)));
 		}
 	}
diff --git a/source3/utils/net_share.c b/source3/utils/net_share.c
index 156b27b572a..6eca9d64ceb 100644
--- a/source3/utils/net_share.c
+++ b/source3/utils/net_share.c
@@ -62,7 +62,7 @@ int net_share_usage(struct net_context *c, int argc, const char **argv)
 
 int net_share(struct net_context *c, int argc, const char **argv)
 {
-	if (argc > 0 && StrCaseCmp(argv[0], "HELP") == 0) {
+	if (argc > 0 && strcasecmp_m(argv[0], "HELP") == 0) {
 			net_share_usage(c, argc, argv);
 			return 0;
 	}
diff --git a/source3/utils/net_time.c b/source3/utils/net_time.c
index fdb46dbba50..9b6293ed499 100644
--- a/source3/utils/net_time.c
+++ b/source3/utils/net_time.c
@@ -19,40 +19,25 @@
 #include "includes.h"
 #include "utils/net.h"
 #include "libsmb/nmblib.h"
+#include "libsmb/libsmb.h"
 
 /*
   return the time on a server. This does not require any authentication
 */
 static time_t cli_servertime(const char *host, struct sockaddr_storage *pss, int *zone)
 {
-	struct nmb_name calling, called;
 	time_t ret = 0;
 	struct cli_state *cli = NULL;
 	NTSTATUS status;
 
-	cli = cli_initialise();
-	if (!cli) {
-		goto done;
-	}
-
-	status = cli_connect(cli, host, pss);
+	status = cli_connect_nb(host, pss, 0, 0x20, lp_netbios_name(),
+				Undefined, &cli);
 	if (!NT_STATUS_IS_OK(status)) {
 		fprintf(stderr, _("Can't contact server %s. Error %s\n"),
 			host, nt_errstr(status));
 		goto done;
 	}
 
-	make_nmb_name(&calling, global_myname(), 0x0);
-	if (host) {
-		make_nmb_name(&called, host, 0x20);
-	} else {
-		make_nmb_name(&called, "*SMBSERVER", 0x20);
-	}
-
-	if (!cli_session_request(cli, &calling, &called)) {
-		fprintf(stderr, _("Session request failed\n"));
-		goto done;
-	}
 	status = cli_negprot(cli);
 	if (!NT_STATUS_IS_OK(status)) {
 		fprintf(stderr, _("Protocol negotiation failed: %s\n"),
diff --git a/source3/utils/net_user.c b/source3/utils/net_user.c
index 2cf00170de4..9fb6f80f929 100644
--- a/source3/utils/net_user.c
+++ b/source3/utils/net_user.c
@@ -50,7 +50,7 @@ int net_user(struct net_context *c, int argc, const char **argv)
 	if (argc < 1)
 		return net_user_usage(c, argc, argv);
 
-	if (StrCaseCmp(argv[0], "HELP") == 0) {
+	if (strcasecmp_m(argv[0], "HELP") == 0) {
 		net_user_usage(c, argc, argv);
 		return 0;
 	}
diff --git a/source3/utils/net_usershare.c b/source3/utils/net_usershare.c
index 06ed4533d55..295f9ce4d9b 100644
--- a/source3/utils/net_usershare.c
+++ b/source3/utils/net_usershare.c
@@ -278,7 +278,7 @@ static int get_share_list(TALLOC_CTX *ctx, const char *wcard, bool only_ours)
 		}
 
 		/* (Finally) - add to list. */
-		fl = TALLOC_P(ctx, struct file_list);
+		fl = talloc(ctx, struct file_list);
 		if (!fl) {
 			sys_closedir(dp);
 			return -1;
diff --git a/source3/utils/net_util.c b/source3/utils/net_util.c
index ffcda14e7fd..0b8bd839d1b 100644
--- a/source3/utils/net_util.c
+++ b/source3/utils/net_util.c
@@ -27,6 +27,7 @@
 #include "../librpc/gen_ndr/ndr_dssetup_c.h"
 #include "secrets.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 
 NTSTATUS net_rpc_lookup_name(struct net_context *c,
 			     TALLOC_CTX *mem_ctx, struct cli_state *cli,
@@ -359,7 +360,7 @@ int net_use_krb_machine_account(struct net_context *c)
 
 	c->opt_password = secrets_fetch_machine_password(
 				c->opt_target_workgroup, NULL, NULL);
-	if (asprintf(&user_name, "%s$@%s", global_myname(), lp_realm()) == -1) {
+	if (asprintf(&user_name, "%s$@%s", lp_netbios_name(), lp_realm()) == -1) {
 		return -1;
 	}
 	c->opt_user_name = user_name;
@@ -381,7 +382,7 @@ int net_use_machine_account(struct net_context *c)
 
 	c->opt_password = secrets_fetch_machine_password(
 				c->opt_target_workgroup, NULL, NULL);
-	if (asprintf(&user_name, "%s$", global_myname()) == -1) {
+	if (asprintf(&user_name, "%s$", lp_netbios_name()) == -1) {
 		return -1;
 	}
 	c->opt_user_name = user_name;
@@ -585,7 +586,7 @@ int net_run_function(struct net_context *c, int argc, const char **argv,
 
 	if (argc != 0) {
 		for (i=0; table[i].funcname != NULL; i++) {
-			if (StrCaseCmp(argv[0], table[i].funcname) == 0)
+			if (strcasecmp_m(argv[0], table[i].funcname) == 0)
 				return table[i].fn(c, argc-1, argv+1);
 		}
 	}
diff --git a/source3/utils/netlookup.c b/source3/utils/netlookup.c
index b160ee55863..5d81d43634e 100644
--- a/source3/utils/netlookup.c
+++ b/source3/utils/netlookup.c
@@ -24,6 +24,7 @@
 #include "rpc_client/cli_pipe.h"
 #include "../librpc/gen_ndr/ndr_lsa.h"
 #include "rpc_client/cli_lsarpc.h"
+#include "libsmb/libsmb.h"
 
 /********************************************************
  Connection cachine struct. Goes away when ctx destroyed.
@@ -77,7 +78,7 @@ static struct con_struct *create_cs(struct net_context *c,
 		return cs;
 	}
 
-	cs = TALLOC_P(ctx, struct con_struct);
+	cs = talloc(ctx, struct con_struct);
 	if (!cs) {
 		*perr = NT_STATUS_NO_MEMORY;
 		return NULL;
@@ -97,7 +98,7 @@ static struct con_struct *create_cs(struct net_context *c,
 	}
 #endif
 
-	nt_status = cli_full_connection(&cs->cli, global_myname(), global_myname(),
+	nt_status = cli_full_connection(&cs->cli, lp_netbios_name(), lp_netbios_name(),
 					&loopback_ss, 0,
 					"IPC$", "IPC",
 #if 0
diff --git a/source3/utils/nmblookup.c b/source3/utils/nmblookup.c
index f8545401e6a..73db278e364 100644
--- a/source3/utils/nmblookup.c
+++ b/source3/utils/nmblookup.c
@@ -168,27 +168,9 @@ static bool query_one(const char *lookup, unsigned int lookup_type)
 				    &bcast_addr, talloc_tos(),
 				    &ip_list, &count, &flags);
 	} else {
-		const struct in_addr *bcast;
-		for (j=iface_count() - 1;
-		     !ip_list && j >= 0;
-		     j--) {
-			char addr[INET6_ADDRSTRLEN];
-			struct sockaddr_storage bcast_ss;
-
-			bcast = iface_n_bcast_v4(j);
-			if (!bcast) {
-				continue;
-			}
-			in_addr_to_sockaddr_storage(&bcast_ss, *bcast);
-			print_sockaddr(addr, sizeof(addr), &bcast_ss);
-			d_printf("querying %s on %s\n",
-			       lookup, addr);
-			status = name_query(lookup,lookup_type,
-					    use_bcast,
-					    use_bcast?True:recursion_desired,
-					    &bcast_ss, talloc_tos(),
-					    &ip_list, &count, &flags);
-		}
+		status = name_resolve_bcast(
+			lookup, lookup_type,
+			talloc_tos(), &ip_list, &count);
 	}
 
 	if (!NT_STATUS_IS_OK(status)) {
diff --git a/source3/utils/ntlm_auth.c b/source3/utils/ntlm_auth.c
index 73f41a7fe66..ba11ec73685 100644
--- a/source3/utils/ntlm_auth.c
+++ b/source3/utils/ntlm_auth.c
@@ -217,7 +217,7 @@ const char *get_winbind_netbios_name(void)
 	if (winbindd_request_response(WINBINDD_NETBIOS_NAME, NULL, &response) !=
 	    NSS_STATUS_SUCCESS) {
 		DEBUG(0, ("could not obtain winbind netbios name!\n"));
-		return global_myname();
+		return lp_netbios_name();
 	}
 
 	fstrcpy(winbind_netbios_name, response.data.netbios_name);
@@ -313,13 +313,13 @@ int get_pam_winbind_config()
 		opt_pam_winbind_conf = PAM_WINBIND_CONFIG_FILE;
 	}
 
-	d = iniparser_load(CONST_DISCARD(char *, opt_pam_winbind_conf));
+	d = iniparser_load(discard_const_p(char, opt_pam_winbind_conf));
 
 	if (!d) {
 		return 0;
 	}
 
-	if (iniparser_getboolean(d, CONST_DISCARD(char *, "global:krb5_auth"), false)) {
+	if (iniparser_getboolean(d, discard_const_p(char, "global:krb5_auth"), false)) {
 		ctrl |= WINBIND_KRB5_AUTH;
 	}
 
@@ -657,7 +657,7 @@ static NTSTATUS ntlm_auth_start_ntlmssp_client(struct ntlmssp_state **client_ntl
 	}
 
 	status = ntlmssp_client_start(NULL,
-				      global_myname(),
+				      lp_netbios_name(),
 				      lp_workgroup(),
 				      lp_client_ntlmv2_auth(),
 				      client_ntlmssp_state);
@@ -711,7 +711,7 @@ static NTSTATUS ntlm_auth_start_ntlmssp_server(struct ntlmssp_state **ntlmssp_st
 	bool is_standalone = false;
 
 	if (opt_password) {
-		netbios_name = global_myname();
+		netbios_name = lp_netbios_name();
 		netbios_domain = lp_workgroup();
 	} else {
 		netbios_name = get_winbind_netbios_name();
@@ -1182,7 +1182,7 @@ static void offer_gss_spnego_mechs(void) {
 
 	ZERO_STRUCT(spnego);
 
-	myname_lower = talloc_strdup(ctx, global_myname());
+	myname_lower = talloc_strdup(ctx, lp_netbios_name());
 	if (!myname_lower) {
 		return;
 	}
@@ -1722,7 +1722,7 @@ static void manage_client_ntlmssp_targ(struct spnego_data spnego)
 
 	spnego.type = SPNEGO_NEG_TOKEN_TARG;
 	spnego.negTokenTarg.negResult = SPNEGO_ACCEPT_INCOMPLETE;
-	spnego.negTokenTarg.supportedMech = (char *)OID_NTLMSSP;
+	spnego.negTokenTarg.supportedMech = (const char *)OID_NTLMSSP;
 	spnego.negTokenTarg.responseToken = request;
 	spnego.negTokenTarg.mechListMIC = null_blob;
 
@@ -2063,7 +2063,7 @@ static void manage_ntlm_server_1_request(struct ntlm_auth_state *state,
 			if (!NT_STATUS_IS_OK(
 				    contact_winbind_auth_crap(username, 
 							      domain, 
-							      global_myname(),
+							      lp_netbios_name(),
 							      &challenge, 
 							      &lm_response, 
 							      &nt_response, 
@@ -2680,7 +2680,7 @@ enum {
 			break;
 
                 case OPT_REQUIRE_MEMBERSHIP:
-			if (StrnCaseCmp("S-", require_membership_of, 2) == 0) {
+			if (strncasecmp_m("S-", require_membership_of, 2) == 0) {
 				require_membership_of_sid = require_membership_of;
 			}
 			break;
diff --git a/source3/utils/sharesec.c b/source3/utils/sharesec.c
index 1d7c6c12943..ab52e4727a2 100644
--- a/source3/utils/sharesec.c
+++ b/source3/utils/sharesec.c
@@ -312,7 +312,7 @@ static struct security_descriptor* parse_acl_string(TALLOC_CTX *mem_ctx, const c
 	pacl = szACL;
 	num_ace = count_chars( pacl, ',' ) + 1;
 
-	if ( !(ace = TALLOC_ZERO_ARRAY( mem_ctx, struct security_ace, num_ace )) )
+	if ( !(ace = talloc_zero_array( mem_ctx, struct security_ace, num_ace )) )
 		return NULL;
 
 	for ( i=0; i<num_ace; i++ ) {
diff --git a/source3/utils/smbcacls.c b/source3/utils/smbcacls.c
index 45b4163c960..d226c84b9a7 100644
--- a/source3/utils/smbcacls.c
+++ b/source3/utils/smbcacls.c
@@ -27,6 +27,7 @@
 #include "../librpc/gen_ndr/ndr_lsa.h"
 #include "rpc_client/cli_lsarpc.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 #include "passdb/machine_sid.h"
 
@@ -359,7 +360,12 @@ static bool parse_ace_flags(const char *str, unsigned int *pflags)
 			return false;
 		}
 
-		if (*p != '|' && *p != '\0') {
+		switch (*p) {
+		case '|':
+			p++;
+		case '\0':
+			continue;
+		default:
 			return false;
 		}
 	}
@@ -1044,7 +1050,7 @@ static int inherit(struct cli_state *cli, const char *filename,
 				/* Add inherited flag to all aces */
 				ace->flags=ace->flags|
 				           SEC_ACE_FLAG_INHERITED_ACE;
-				if ((oldattr & aDIR) == aDIR) {
+				if ((oldattr & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY) {
 					if ((ace->flags & SEC_ACE_FLAG_CONTAINER_INHERIT) ==
 					    SEC_ACE_FLAG_CONTAINER_INHERIT) {
 						add_ace(&old->dacl, ace);
@@ -1144,7 +1150,7 @@ static struct cli_state *connect_one(struct user_auth_info *auth_info,
 
 	set_cmdline_auth_info_getpass(auth_info);
 
-	nt_status = cli_full_connection(&c, global_myname(), server, 
+	nt_status = cli_full_connection(&c, lp_netbios_name(), server,
 				&ss, 0,
 				share, "?????",
 				get_cmdline_auth_info_username(auth_info),
diff --git a/source3/utils/smbcontrol.c b/source3/utils/smbcontrol.c
index ea1b6815887..5520dcf608a 100644
--- a/source3/utils/smbcontrol.c
+++ b/source3/utils/smbcontrol.c
@@ -31,6 +31,7 @@
 #include "printing/notify.h"
 #include "libsmb/nmblib.h"
 #include "messages.h"
+#include "util_tdb.h"
 
 #if HAVE_LIBUNWIND_H
 #include <libunwind.h>
@@ -63,7 +64,7 @@ static bool send_message(struct messaging_context *msg_ctx,
 	if (procid_to_pid(&pid) != 0)
 		return NT_STATUS_IS_OK(
 			messaging_send_buf(msg_ctx, pid, msg_type,
-					   (uint8 *)buf, len));
+					   (const uint8 *)buf, len));
 
 	ret = message_send_all(msg_ctx, msg_type, buf, len, &n_sent);
 	DEBUG(10,("smbcontrol/send_message: broadcast message to "
@@ -118,7 +119,7 @@ static void print_pid_string_cb(struct messaging_context *msg,
 {
 	char *pidstr;
 
-	pidstr = procid_str(talloc_tos(), &pid);
+	pidstr = server_id_str(talloc_tos(), &pid);
 	printf("PID %s: %.*s", pidstr, (int)data->length,
 	       (const char *)data->data);
 	TALLOC_FREE(pidstr);
@@ -432,7 +433,7 @@ static void pong_cb(struct messaging_context *msg,
 		    struct server_id pid,
 		    DATA_BLOB *data)
 {
-	char *src_string = procid_str(NULL, &pid);
+	char *src_string = server_id_str(NULL, &pid);
 	printf("PONG from pid %s\n", src_string);
 	TALLOC_FREE(src_string);
 	num_replies++;
@@ -748,7 +749,7 @@ static bool do_printnotify(struct messaging_context *msg_ctx,
 
 		notify_printer_byname(messaging_event_context(msg_ctx),
 				      msg_ctx, argv[2], attribute,
-				      CONST_DISCARD(char *, argv[4]));
+				      discard_const_p(char, argv[4]));
 
 		goto send;
 	}
@@ -1142,7 +1143,7 @@ static void winbind_validate_cache_cb(struct messaging_context *msg,
 				      struct server_id pid,
 				      DATA_BLOB *data)
 {
-	char *src_string = procid_str(NULL, &pid);
+	char *src_string = server_id_str(NULL, &pid);
 	printf("Winbindd cache is %svalid. (answer from pid %s)\n",
 	       (*(data->data) == 0 ? "" : "NOT "), src_string);
 	TALLOC_FREE(src_string);
@@ -1201,7 +1202,7 @@ static void my_make_nmb_name( struct nmb_name *n, const char *name, int type)
 	strupper_m(unix_name);
 	push_ascii(n->name, unix_name, sizeof(n->name), STR_TERMINATE);
 	n->name_type = (unsigned int)type & 0xFF;
-	push_ascii(n->scope,  global_scope(), 64, STR_TERMINATE);
+	push_ascii(n->scope,  lp_netbios_scope(), 64, STR_TERMINATE);
 }
 
 static bool do_nodestatus(struct messaging_context *msg_ctx,
diff --git a/source3/utils/smbcquotas.c b/source3/utils/smbcquotas.c
index 0efa60608b0..9858e0a3a87 100644
--- a/source3/utils/smbcquotas.c
+++ b/source3/utils/smbcquotas.c
@@ -28,6 +28,7 @@
 #include "rpc_client/cli_lsarpc.h"
 #include "fake_file.h"
 #include "../libcli/security/security.h"
+#include "libsmb/libsmb.h"
 
 static char *server;
 
@@ -522,7 +523,7 @@ static struct cli_state *connect_one(const char *share)
 
 	set_cmdline_auth_info_getpass(smbcquotas_auth_info);
 
-	nt_status = cli_full_connection(&c, global_myname(), server, 
+	nt_status = cli_full_connection(&c, lp_netbios_name(), server,
 					    &ss, 0,
 					    share, "?????",
 					    get_cmdline_auth_info_username(smbcquotas_auth_info),
diff --git a/source3/utils/smbget.c b/source3/utils/smbget.c
index fc65b3b570f..d60a13fd69b 100644
--- a/source3/utils/smbget.c
+++ b/source3/utils/smbget.c
@@ -17,6 +17,7 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
 
 #include "includes.h"
+#include "system/filesys.h"
 #include "popt_common.h"
 #include "libsmbclient.h"
 
@@ -669,7 +670,7 @@ int main(int argc, const char **argv)
 	if (smb_encrypt) {
 		SMBCCTX *smb_ctx = smbc_set_context(NULL);
 		smbc_option_set(smb_ctx,
-			CONST_DISCARD(char *, "smb_encrypt_level"),
+			discard_const_p(char, "smb_encrypt_level"),
 			"require");
 	}
 
diff --git a/source3/utils/smbpasswd.c b/source3/utils/smbpasswd.c
index ada057c2479..eeab4ef7de0 100644
--- a/source3/utils/smbpasswd.c
+++ b/source3/utils/smbpasswd.c
@@ -107,6 +107,7 @@ static int process_options(int argc, char **argv, int local_flags)
 			break;
 		case 'c':
 			configfile = optarg;
+			set_dyn_CONFIGFILE(optarg);
 			break;
 		case 'a':
 			local_flags |= LOCAL_ADD_USER;
@@ -378,7 +379,7 @@ static int process_root(int local_flags)
 		 */
 
 		slprintf(buf, sizeof(buf)-1, "%s$", user_name);
-		fstrcpy(user_name, buf);
+		strlcpy(user_name, buf, sizeof(user_name));
 	} else if (local_flags & LOCAL_INTERDOM_ACCOUNT) {
 		static fstring buf;
 
@@ -395,7 +396,7 @@ static int process_root(int local_flags)
 
 		/* prepare uppercased and '$' terminated username */
 		slprintf(buf, sizeof(buf) - 1, "%s$", user_name);
-		fstrcpy(user_name, buf);
+		strlcpy(user_name, buf, sizeof(user_name));
 
 	} else {
 
diff --git a/source3/utils/smbtree.c b/source3/utils/smbtree.c
index 3f7fc976986..41f4103e5a1 100644
--- a/source3/utils/smbtree.c
+++ b/source3/utils/smbtree.c
@@ -23,6 +23,7 @@
 #include "popt_common.h"
 #include "rpc_client/cli_pipe.h"
 #include "../librpc/gen_ndr/ndr_srvsvc_c.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clirap.h"
 
 static int use_bcast;
diff --git a/source3/utils/status.c b/source3/utils/status.c
index cf625fb73fa..4d37f7e66e5 100644
--- a/source3/utils/status.c
+++ b/source3/utils/status.c
@@ -474,7 +474,7 @@ static int traverse_sessionid(const char *key, struct sessionid *session,
 
 		if (result == 0) {
 			d_printf("No locked files\n");
-		} else if (result == -1) {
+		} else if (result < 0) {
 			d_printf("locked file list truncated\n");
 		}
 
diff --git a/source3/utils/testparm.c b/source3/utils/testparm.c
index 978ada26c65..721034dedff 100644
--- a/source3/utils/testparm.c
+++ b/source3/utils/testparm.c
@@ -39,7 +39,7 @@
  Check if a directory exists.
 ********************************************************************/
 
-static bool directory_exist_stat(char *dname,SMB_STRUCT_STAT *st)
+static bool directory_exist_stat(const char *dname,SMB_STRUCT_STAT *st)
 {
 	SMB_STRUCT_STAT st2;
 	bool ret;
@@ -77,7 +77,7 @@ cannot be set in the smb.conf file. nmbd will abort with this setting.\n");
 		ret = 1;
 	}
 
-	if (strequal(lp_workgroup(), global_myname())) {
+	if (strequal(lp_workgroup(), lp_netbios_name())) {
 		fprintf(stderr, "WARNING: 'workgroup' and 'netbios name' " \
 			"must differ.\n");
 		ret = 1;
@@ -128,20 +128,35 @@ cannot be set in the smb.conf file. nmbd will abort with this setting.\n");
 	 * Password server sanity checks.
 	 */
 
-	if((lp_security() == SEC_SERVER || lp_security() >= SEC_DOMAIN) && !lp_passwordserver()) {
+	if((lp_security() == SEC_SERVER || lp_security() >= SEC_DOMAIN) && !*lp_passwordserver()) {
 		const char *sec_setting;
 		if(lp_security() == SEC_SERVER)
 			sec_setting = "server";
 		else if(lp_security() == SEC_DOMAIN)
 			sec_setting = "domain";
+		else if(lp_security() == SEC_ADS)
+			sec_setting = "ads";
 		else
 			sec_setting = "";
 
-		fprintf(stderr, "ERROR: The setting 'security=%s' requires the 'password server' parameter be set \
-to a valid password server.\n", sec_setting );
+		fprintf(stderr, "ERROR: The setting 'security=%s' requires the 'password server' parameter be set\n"
+			"to the default value * or a valid password server.\n", sec_setting );
 		ret = 1;
 	}
 
+	if((lp_security() >= SEC_DOMAIN) && (strcmp(lp_passwordserver(), "*") != 0)) {
+		const char *sec_setting;
+		if(lp_security() == SEC_DOMAIN)
+			sec_setting = "domain";
+		else if(lp_security() == SEC_ADS)
+			sec_setting = "ads";
+		else
+			sec_setting = "";
+
+		fprintf(stderr, "WARNING: The setting 'security=%s' should NOT be combined with the 'password server' parameter.\n"
+			"(by default Samba will discover the correct DC to contact automatically).\n", sec_setting );
+	}
+
 	/*
 	 * Password chat sanity checks.
 	 */
@@ -246,6 +261,11 @@ via the %%o substitution. With encrypted passwords this is not possible.\n", lp_
 		fprintf(stderr,"WARNING: Maximum value for 'os level' is 255!\n");	
 	}
 
+	if (strequal(lp_dos_charset(), "UTF8") || strequal(lp_dos_charset(), "UTF-8")) {
+		fprintf(stderr, "ERROR: 'dos charset' must not be UTF8\n");
+		ret = 1;
+	}
+
 	return ret;
 }   
 
diff --git a/source3/web/cgi.c b/source3/web/cgi.c
index fa2304016ec..cc31eef97e5 100644
--- a/source3/web/cgi.c
+++ b/source3/web/cgi.c
@@ -118,7 +118,7 @@ void cgi_load_variables(void)
 #ifdef DEBUG_COMMENTS
 	char dummy[100]="";
 	print_title(dummy);
-	d_printf("<!== Start dump in cgi_load_variables() %s ==>\n",__FILE__);
+	printf("<!== Start dump in cgi_load_variables() %s ==>\n",__FILE__);
 #endif
 
 	if (!content_length) {
@@ -275,7 +275,7 @@ static void cgi_setup_error(const char *err, const char *header, const char *inf
 		}
 	}
 
-	d_printf("HTTP/1.0 %s\r\n%sConnection: close\r\nContent-Type: text/html\r\n\r\n<HTML><HEAD><TITLE>%s</TITLE></HEAD><BODY><H1>%s</H1>%s<p></BODY></HTML>\r\n\r\n", err, header, err, err, info);
+	printf("HTTP/1.0 %s\r\n%sConnection: close\r\nContent-Type: text/html\r\n\r\n<HTML><HEAD><TITLE>%s</TITLE></HEAD><BODY><H1>%s</H1>%s<p></BODY></HTML>\r\n\r\n", err, header, err, err, info);
 	fclose(stdin);
 	fclose(stdout);
 	exit(0);
diff --git a/source3/web/diagnose.c b/source3/web/diagnose.c
index cf93991f94c..abb4d79b9a7 100644
--- a/source3/web/diagnose.c
+++ b/source3/web/diagnose.c
@@ -20,6 +20,7 @@
 #include "includes.h"
 #include "web/swat_proto.h"
 #include "lib/winbind_util.h"
+#include "libsmb/libsmb.h"
 
 #ifdef WITH_WINBIND
 
@@ -69,15 +70,11 @@ bool smbd_running(void)
 	loopback_ip.s_addr = htonl(INADDR_LOOPBACK);
 	in_addr_to_sockaddr_storage(&ss, loopback_ip);
 
-	if ((cli = cli_initialise()) == NULL)
-		return False;
-
-	status = cli_connect(cli, global_myname(), &ss);
+	status = cli_connect_nb("localhost", &ss, 0, 0x20, lp_netbios_name(),
+				Undefined, &cli);
 	if (!NT_STATUS_IS_OK(status)) {
-		cli_shutdown(cli);
-		return False;
+		return false;
 	}
-
 	cli_shutdown(cli);
 	return True;
 }
diff --git a/source3/web/startstop.c b/source3/web/startstop.c
index 81350e62f5c..e23acf89313 100644
--- a/source3/web/startstop.c
+++ b/source3/web/startstop.c
@@ -19,7 +19,7 @@
 
 #include "includes.h"
 #include "web/swat_proto.h"
-#include "dynconfig.h"
+#include "dynconfig/dynconfig.h"
 
 
 /** Startup smbd from web interface. */
diff --git a/source3/web/swat.c b/source3/web/swat.c
index f06bf5ed26c..277b25c4268 100644
--- a/source3/web/swat.c
+++ b/source3/web/swat.c
@@ -83,7 +83,7 @@ static char *fix_backslash(const char *str)
 	return newstring;
 }
 
-static const char *fix_quotes(TALLOC_CTX *ctx, const char *str)
+static const char *fix_quotes(TALLOC_CTX *ctx, char *str)
 {
 	char *newstring = NULL;
 	char *p = NULL;
@@ -101,7 +101,7 @@ static const char *fix_quotes(TALLOC_CTX *ctx, const char *str)
 		}
 		++p;
 	}
-	newstring = TALLOC_ARRAY(ctx, char, newstring_len);
+	newstring = talloc_array(ctx, char, newstring_len);
 	if (!newstring) {
 		return "";
 	}
diff --git a/source3/winbindd/idmap.c b/source3/winbindd/idmap.c
index 2eb655f7800..6ae10115b78 100644
--- a/source3/winbindd/idmap.c
+++ b/source3/winbindd/idmap.c
@@ -401,7 +401,7 @@ struct idmap_domain *idmap_find_domain(const char *domname)
 		/*
 		 * talloc context for all idmap domains
 		 */
-		idmap_domains = TALLOC_ARRAY(NULL, struct idmap_domain *, 1);
+		idmap_domains = talloc_array(NULL, struct idmap_domain *, 1);
 	}
 
 	if (idmap_domains == NULL) {
diff --git a/source3/winbindd/idmap_ad.c b/source3/winbindd/idmap_ad.c
index cf15f038ca0..65b518074b5 100644
--- a/source3/winbindd/idmap_ad.c
+++ b/source3/winbindd/idmap_ad.c
@@ -216,7 +216,7 @@ static NTSTATUS idmap_ad_initialize(struct idmap_domain *dom)
 	char *config_option;
 	const char *schema_mode = NULL;	
 
-	ctx = TALLOC_ZERO_P(dom, struct idmap_ad_context);
+	ctx = talloc_zero(dom, struct idmap_ad_context);
 	if (ctx == NULL) {
 		DEBUG(0, ("Out of memory!\n"));
 		return NT_STATUS_NO_MEMORY;
@@ -736,7 +736,7 @@ static NTSTATUS nss_ad_generic_init(struct nss_domain_entry *e,
 	if (e->state != NULL) {
 		dom = talloc_get_type(e->state, struct idmap_domain);
 	} else {
-		dom = TALLOC_ZERO_P(e, struct idmap_domain);
+		dom = talloc_zero(e, struct idmap_domain);
 		if (dom == NULL) {
 			DEBUG(0, ("Out of memory!\n"));
 			return NT_STATUS_NO_MEMORY;
@@ -756,7 +756,7 @@ static NTSTATUS nss_ad_generic_init(struct nss_domain_entry *e,
 		ctx = talloc_get_type(dom->private_data,
 				      struct idmap_ad_context);
 	} else {
-		ctx = TALLOC_ZERO_P(dom, struct idmap_ad_context);
+		ctx = talloc_zero(dom, struct idmap_ad_context);
 		if (ctx == NULL) {
 			DEBUG(0, ("Out of memory!\n"));
 			return NT_STATUS_NO_MEMORY;
diff --git a/source3/winbindd/idmap_adex/cell_util.c b/source3/winbindd/idmap_adex/cell_util.c
index c82407c0f48..194544086be 100644
--- a/source3/winbindd/idmap_adex/cell_util.c
+++ b/source3/winbindd/idmap_adex/cell_util.c
@@ -38,7 +38,7 @@
 	for (i = 0; i < num_lines; i++) {
 		/* make sure to avoid substring matches like uid
 		   and uidNumber */
-		if ((StrnCaseCmp(list[i], substr, cmplen) == 0) &&
+		if ((strncasecmp_m(list[i], substr, cmplen) == 0) &&
 		    (list[i][cmplen] == '=')) {
 			/* Don't return an empty string */
 			if (list[i][cmplen + 1] != '\0')
@@ -110,7 +110,7 @@ static NTSTATUS cell_lookup_forest(struct likewise_cell *c)
 		return NT_STATUS_INVALID_PARAMETER;
 	}
 
-	if ((gc = TALLOC_ZERO_P(NULL, struct gc_info)) == NULL) {
+	if ((gc = talloc_zero(NULL, struct gc_info)) == NULL) {
 		nt_status = NT_STATUS_NO_MEMORY;
 		BAIL_ON_NTSTATUS_ERROR(nt_status);
 	}
@@ -230,7 +230,7 @@ done:
 	while (next_token_talloc(frame, &tmp_dn, &buffer, ",")) {
 
 		/* skip everything up the where DC=... begins */
-		if (StrnCaseCmp(buffer, "DC=", 3) != 0)
+		if (strncasecmp_m(buffer, "DC=", 3) != 0)
 			continue;
 
 		if (!domain) {
diff --git a/source3/winbindd/idmap_adex/domain_util.c b/source3/winbindd/idmap_adex/domain_util.c
index c3731064469..99dc0cca658 100644
--- a/source3/winbindd/idmap_adex/domain_util.c
+++ b/source3/winbindd/idmap_adex/domain_util.c
@@ -71,7 +71,7 @@ static NTSTATUS dc_add_domain(const char *domain)
 		return NT_STATUS_OK;
 	}
 
-	dc = TALLOC_ZERO_P(NULL, struct dc_info);
+	dc = talloc_zero(NULL, struct dc_info);
 	BAIL_ON_PTR_ERROR(dc, nt_status);
 
 	dc->dns_name = talloc_strdup(dc, domain);
@@ -281,7 +281,7 @@ static struct dc_info *dc_find_domain(const char *dns_domain)
 	*cell = dc->domain_cell;
 
 done:
-	talloc_destroy(CONST_DISCARD(char*, base));
+	talloc_destroy(discard_const_p(char, base));
 	talloc_destroy(frame);
 
 	return nt_status;
diff --git a/source3/winbindd/idmap_adex/gc_util.c b/source3/winbindd/idmap_adex/gc_util.c
index 77b318cb2d7..461a2e679b7 100644
--- a/source3/winbindd/idmap_adex/gc_util.c
+++ b/source3/winbindd/idmap_adex/gc_util.c
@@ -107,6 +107,7 @@ done:
 	NTSTATUS nt_status = NT_STATUS_UNSUCCESSFUL;
 	struct NETLOGON_SAM_LOGON_RESPONSE_EX cldap_reply;
 	TALLOC_CTX *frame = talloc_stackframe();
+	struct sockaddr_storage ss;
 
 	if (!gc || !domain) {
 		return NT_STATUS_INVALID_PARAMETER;
@@ -126,8 +127,17 @@ done:
 	nt_status = ads_ntstatus(ads_status);
 	BAIL_ON_NTSTATUS_ERROR(nt_status);
 
+	if (!resolve_name(ads->config.ldap_server_name, &ss, 0x20, true)) {
+		DEBUG(5,("gc_find_forest_root: unable to resolve name %s\n",
+			 ads->config.ldap_server_name));
+		nt_status = NT_STATUS_IO_TIMEOUT;
+		/* This matches the old code which did the resolve in
+		 * ads_cldap_netlogon_5 */
+		BAIL_ON_NTSTATUS_ERROR(nt_status);
+	}
+
 	if (!ads_cldap_netlogon_5(frame,
-				  ads->config.ldap_server_name,
+				  &ss,
 				  ads->config.realm,
 				  &cldap_reply))
 	{
@@ -181,7 +191,7 @@ static NTSTATUS gc_add_forest(const char *domain)
 		return NT_STATUS_OK;
 	}
 
-	if ((gc = TALLOC_ZERO_P(NULL, struct gc_info)) == NULL) {
+	if ((gc = talloc_zero(NULL, struct gc_info)) == NULL) {
 		nt_status = NT_STATUS_NO_MEMORY;
 		BAIL_ON_NTSTATUS_ERROR(nt_status);
 	}
@@ -771,17 +781,17 @@ done:
 #endif
 
 	if (count == 0) {
-		ads_tmp = TALLOC_ARRAY(NULL, ADS_STRUCT*, 1);
+		ads_tmp = talloc_array(NULL, ADS_STRUCT*, 1);
 		BAIL_ON_PTR_ERROR(ads_tmp, nt_status);
 
-		msg_tmp = TALLOC_ARRAY(NULL, LDAPMessage*, 1);
+		msg_tmp = talloc_array(NULL, LDAPMessage*, 1);
 		BAIL_ON_PTR_ERROR(msg_tmp, nt_status);
 	} else {
-		ads_tmp = TALLOC_REALLOC_ARRAY(*ads_list, *ads_list, ADS_STRUCT*,
+		ads_tmp = talloc_realloc(*ads_list, *ads_list, ADS_STRUCT*,
 					       count+1);
 		BAIL_ON_PTR_ERROR(ads_tmp, nt_status);
 
-		msg_tmp = TALLOC_REALLOC_ARRAY(*msg_list, *msg_list, LDAPMessage*,
+		msg_tmp = talloc_realloc(*msg_list, *msg_list, LDAPMessage*,
 					       count+1);
 		BAIL_ON_PTR_ERROR(msg_tmp, nt_status);
 	}
diff --git a/source3/winbindd/idmap_adex/likewise_cell.c b/source3/winbindd/idmap_adex/likewise_cell.c
index 0914132b933..0e544e90f76 100644
--- a/source3/winbindd/idmap_adex/likewise_cell.c
+++ b/source3/winbindd/idmap_adex/likewise_cell.c
@@ -85,7 +85,7 @@ static struct likewise_cell *_lw_cell_list = NULL;
 
 	/* Each cell struct is a TALLOC_CTX* */
 
-	c = TALLOC_ZERO_P(NULL, struct likewise_cell);
+	c = talloc_zero(NULL, struct likewise_cell);
 	if (!c) {
 		DEBUG(0,("cell_new: memory allocation failure!\n"));
 		return NULL;
diff --git a/source3/winbindd/idmap_adex/provider_unified.c b/source3/winbindd/idmap_adex/provider_unified.c
index b61666fa502..9e271a0db9e 100644
--- a/source3/winbindd/idmap_adex/provider_unified.c
+++ b/source3/winbindd/idmap_adex/provider_unified.c
@@ -237,7 +237,7 @@ static NTSTATUS search_cell(struct likewise_cell *c,
 done:
 	PRINT_NTSTATUS_ERROR(nt_status, "search_cell", 4);
 
-	talloc_destroy(CONST_DISCARD(char*, base));
+	talloc_destroy(discard_const_p(char, base));
 	talloc_destroy(frame);
 
 	return nt_status;
@@ -310,7 +310,7 @@ static bool check_forest_scope(const char *dn)
 	}
 
 	q++;
-	if (StrnCaseCmp(q, "dc=", 3) != 0) {
+	if (strncasecmp_m(q, "dc=", 3) != 0) {
 		nt_status = NT_STATUS_OBJECT_PATH_NOT_FOUND;
 		BAIL_ON_NTSTATUS_ERROR(nt_status);
 	}
diff --git a/source3/winbindd/idmap_autorid.c b/source3/winbindd/idmap_autorid.c
index 5e3dacf6240..cddbff2d478 100644
--- a/source3/winbindd/idmap_autorid.c
+++ b/source3/winbindd/idmap_autorid.c
@@ -28,6 +28,7 @@
 #include "dbwrap.h"
 #include "idmap.h"
 #include "../libcli/security/dom_sid.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_IDMAP
@@ -376,7 +377,7 @@ static struct autorid_global_config *idmap_autorid_loadconfig(TALLOC_CTX * ctx)
 		return NULL;
 	}
 
-	cfg = TALLOC_ZERO_P(ctx, struct autorid_global_config);
+	cfg = talloc_zero(ctx, struct autorid_global_config);
 	if (!cfg) {
 		return NULL;
 	}
@@ -435,7 +436,14 @@ static NTSTATUS idmap_autorid_initialize(struct idmap_domain *dom)
 	NTSTATUS status;
 	uint32_t hwm;
 
-	config = TALLOC_ZERO_P(dom, struct autorid_global_config);
+	if (!strequal(dom->name, "*")) {
+		DEBUG(0, ("idmap_autorid_initialize: Error: autorid configured "
+			  "for domain '%s'. But autorid can only be used for "
+			  "the default idmap configuration.\n", dom->name));
+		return NT_STATUS_INVALID_PARAMETER;
+	}
+
+	config = talloc_zero(dom, struct autorid_global_config);
 	if (!config) {
 		DEBUG(0, ("Out of memory!\n"));
 		return NT_STATUS_NO_MEMORY;
@@ -447,7 +455,7 @@ static NTSTATUS idmap_autorid_initialize(struct idmap_domain *dom)
 	}
 
 	config->minvalue = dom->low_id;
-	config->rangesize = lp_parm_int(-1, "autorid", "rangesize", 100000);
+	config->rangesize = lp_parm_int(-1, "idmap config *", "rangesize", 100000);
 
 	if (config->rangesize < 2000) {
 		DEBUG(1, ("autorid rangesize must be at least 2000\n"));
@@ -522,14 +530,12 @@ static NTSTATUS idmap_autorid_initialize(struct idmap_domain *dom)
 
 	dom->private_data = config;
 
-	if (!NT_STATUS_IS_OK(status)) {
-		goto error;
-	}
+	goto done;
 
-	return NT_STATUS_OK;
-
-      error:
+error:
 	talloc_free(config);
+
+done:
 	talloc_free(storedconfig);
 
 	return status;
diff --git a/source3/winbindd/idmap_hash/idmap_hash.c b/source3/winbindd/idmap_hash/idmap_hash.c
index 474387918e4..a6e883c4746 100644
--- a/source3/winbindd/idmap_hash/idmap_hash.c
+++ b/source3/winbindd/idmap_hash/idmap_hash.c
@@ -127,7 +127,7 @@ static NTSTATUS be_init(struct idmap_domain *dom)
 
 	/* Create the hash table of domain SIDs */
 
-	hashed_domains = TALLOC_ZERO_ARRAY(dom, struct sid_hash_table, 4096);
+	hashed_domains = talloc_zero_array(dom, struct sid_hash_table, 4096);
 	BAIL_ON_PTR_NT_ERROR(hashed_domains, nt_status);
 
 	/* create the hash table of domain SIDs */
diff --git a/source3/winbindd/idmap_hash/mapfile.c b/source3/winbindd/idmap_hash/mapfile.c
index 28280590934..075f0f25cc4 100644
--- a/source3/winbindd/idmap_hash/mapfile.c
+++ b/source3/winbindd/idmap_hash/mapfile.c
@@ -87,8 +87,8 @@ static bool mapfile_read_line(fstring key, fstring value)
 	*p = '\0';
 	p++;
 
-	fstrcpy(key, buffer);
-	fstrcpy(value, p);
+	strlcpy(key, buffer, sizeof(fstring));
+	strlcpy(value, p, sizeof(fstring));
 
 	/* Eat whitespace */
 
diff --git a/source3/winbindd/idmap_ldap.c b/source3/winbindd/idmap_ldap.c
index 7195912fc38..10d9d2e8b63 100644
--- a/source3/winbindd/idmap_ldap.c
+++ b/source3/winbindd/idmap_ldap.c
@@ -232,8 +232,8 @@ done:
  Allocate a new uid or gid
 ********************************/
 
-static NTSTATUS idmap_ldap_allocate_id(struct idmap_domain *dom,
-				       struct unixid *xid)
+static NTSTATUS idmap_ldap_allocate_id_internal(struct idmap_domain *dom,
+						struct unixid *xid)
 {
 	TALLOC_CTX *mem_ctx;
 	NTSTATUS ret = NT_STATUS_UNSUCCESSFUL;
@@ -391,21 +391,21 @@ done:
  * For now this is for the default idmap domain only.
  * Should be extended later on.
  */
-static NTSTATUS idmap_ldap_get_new_id(struct idmap_domain *dom,
-				      struct unixid *id)
+static NTSTATUS idmap_ldap_allocate_id(struct idmap_domain *dom,
+				       struct unixid *id)
 {
 	NTSTATUS ret;
 
 	if (!strequal(dom->name, "*")) {
-		DEBUG(3, ("idmap_ldap_get_new_id: "
+		DEBUG(3, ("idmap_ldap_allocate_id: "
 			  "Refusing allocation of a new unixid for domain'%s'. "
-			  "Currently only supported for the default "
+			  "This is only supported for the default "
 			  "domain \"*\".\n",
 			   dom->name));
 		return NT_STATUS_NOT_IMPLEMENTED;
 	}
 
-	ret = idmap_ldap_allocate_id(dom, id);
+	ret = idmap_ldap_allocate_id_internal(dom, id);
 
 	return ret;
 }
@@ -443,7 +443,7 @@ static NTSTATUS idmap_ldap_db_init(struct idmap_domain *dom)
 		return NT_STATUS_FILE_IS_OFFLINE;
 	}
 
-	ctx = TALLOC_ZERO_P(dom, struct idmap_ldap_context);
+	ctx = talloc_zero(dom, struct idmap_ldap_context);
 	if ( ! ctx) {
 		DEBUG(0, ("Out of memory!\n"));
 		return NT_STATUS_NO_MEMORY;
@@ -484,7 +484,7 @@ static NTSTATUS idmap_ldap_db_init(struct idmap_domain *dom)
 	ctx->rw_ops = talloc_zero(ctx, struct idmap_rw_ops);
 	CHECK_ALLOC_DONE(ctx->rw_ops);
 
-	ctx->rw_ops->get_new_id = idmap_ldap_get_new_id;
+	ctx->rw_ops->get_new_id = idmap_ldap_allocate_id_internal;
 	ctx->rw_ops->set_mapping = idmap_ldap_set_mapping;
 
 	ret = smbldap_init(ctx, winbind_event_context(), ctx->url,
@@ -1144,7 +1144,7 @@ static struct idmap_methods idmap_ldap_methods = {
 	.init = idmap_ldap_db_init,
 	.unixids_to_sids = idmap_ldap_unixids_to_sids,
 	.sids_to_unixids = idmap_ldap_sids_to_unixids,
-	.allocate_id = idmap_ldap_get_new_id,
+	.allocate_id = idmap_ldap_allocate_id,
 };
 
 NTSTATUS idmap_ldap_init(void);
diff --git a/source3/winbindd/idmap_proto.h b/source3/winbindd/idmap_proto.h
index fa7f7120246..4fa9ed1a268 100644
--- a/source3/winbindd/idmap_proto.h
+++ b/source3/winbindd/idmap_proto.h
@@ -1,3 +1,30 @@
+/*
+ *  Unix SMB/CIFS implementation.
+ *  ID Mapping
+ *
+ *  Copyright (C) Tim Potter 2000
+ *  Copyright (C) Jim McDonough <jmcd@us.ibm.com> 2003
+ *  Copyright (C) Simo Sorce 2003-2007
+ *  Copyright (C) Jeremy Allison 2006
+ *  Copyright (C) Michael Adam 2009-2010
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _WINBINDD_IDMAP_PROTO_H_
+#define _WINBINDD_IDMAP_PROTO_H_
+
 /* The following definitions come from winbindd/idmap.c  */
 
 bool idmap_is_offline(void);
@@ -31,3 +58,5 @@ NTSTATUS idmap_gid_to_sid(const char *domname, struct dom_sid *sid, gid_t gid);
 NTSTATUS idmap_sid_to_uid(const char *dom_name, struct dom_sid *sid, uid_t *uid);
 NTSTATUS idmap_sid_to_gid(const char *domname, struct dom_sid *sid, gid_t *gid);
 bool idmap_unix_id_is_in_range(uint32_t id, struct idmap_domain *dom);
+
+#endif /* _WINBINDD_IDMAP_PROTO_H_ */
diff --git a/source3/winbindd/idmap_rid.c b/source3/winbindd/idmap_rid.c
index 8bb63fd534a..edc5e167740 100644
--- a/source3/winbindd/idmap_rid.c
+++ b/source3/winbindd/idmap_rid.c
@@ -41,7 +41,7 @@ static NTSTATUS idmap_rid_initialize(struct idmap_domain *dom)
 	struct idmap_rid_context *ctx;
 	char *config_option = NULL;
 
-	ctx = TALLOC_ZERO_P(dom, struct idmap_rid_context);
+	ctx = talloc_zero(dom, struct idmap_rid_context);
 	if (ctx == NULL) {
 		DEBUG(0, ("Out of memory!\n"));
 		return NT_STATUS_NO_MEMORY;
diff --git a/source3/winbindd/idmap_tdb.c b/source3/winbindd/idmap_tdb.c
index 19e7f0e1d5f..d99d2f0e4e9 100644
--- a/source3/winbindd/idmap_tdb.c
+++ b/source3/winbindd/idmap_tdb.c
@@ -30,6 +30,7 @@
 #include "idmap_rw.h"
 #include "dbwrap.h"
 #include "../libcli/security/security.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_IDMAP
@@ -145,9 +146,23 @@ static bool idmap_tdb_upgrade(struct idmap_domain *dom, struct db_context *db)
 	bool bigendianheader;
 	struct convert_fn_state s;
 
-	DEBUG(0, ("Upgrading winbindd_idmap.tdb from an old version\n"));
-
+#if BUILD_TDB2
+	/* If we are bigendian, tdb is bigendian if NOT converted. */
+	union {
+		uint16 large;
+		unsigned char small[2];
+	} u;
+	u.large = 0x0102;
+	if (u.small[0] == 0x01)
+		bigendianheader = !(db->get_flags(db) & TDB_CONVERT);
+	else {
+		assert(u.small[0] == 0x02);
+		bigendianheader = (db->get_flags(db) & TDB_CONVERT);
+	}
+#else
 	bigendianheader = (db->get_flags(db) & TDB_BIGENDIAN) ? True : False;
+#endif
+	DEBUG(0, ("Upgrading winbindd_idmap.tdb from an old version\n"));
 
 	vers = dbwrap_fetch_int32(db, "IDMAP_VERSION");
 
diff --git a/source3/winbindd/idmap_tdb2.c b/source3/winbindd/idmap_tdb2.c
index 1a5a51a5b44..2a152444595 100644
--- a/source3/winbindd/idmap_tdb2.c
+++ b/source3/winbindd/idmap_tdb2.c
@@ -38,6 +38,7 @@
 #include "idmap_rw.h"
 #include "dbwrap.h"
 #include "../libcli/security/dom_sid.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_IDMAP
diff --git a/source3/winbindd/nss_info.c b/source3/winbindd/nss_info.c
index 77ac421d551..a3f95c64935 100644
--- a/source3/winbindd/nss_info.c
+++ b/source3/winbindd/nss_info.c
@@ -118,7 +118,7 @@ static NTSTATUS nss_domain_list_add_domain(const char *domain,
 {
 	struct nss_domain_entry *nss_domain;
 
-	nss_domain = TALLOC_ZERO_P(nss_domain_list, struct nss_domain_entry);
+	nss_domain = talloc_zero(nss_domain_list, struct nss_domain_entry);
 	if (!nss_domain) {
 		DEBUG(0, ("nss_domain_list_add_domain: talloc() failure!\n"));
 		return NT_STATUS_NO_MEMORY;
diff --git a/source3/winbindd/wb_fill_pwent.c b/source3/winbindd/wb_fill_pwent.c
index 23057384c2b..a716245ce85 100644
--- a/source3/winbindd/wb_fill_pwent.c
+++ b/source3/winbindd/wb_fill_pwent.c
@@ -72,8 +72,7 @@ static void wb_fill_pwent_sid2uid_done(struct tevent_req *subreq)
 
 	status = wb_sid2uid_recv(subreq, &state->pw->pw_uid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -98,8 +97,7 @@ static void wb_fill_pwent_sid2gid_done(struct tevent_req *subreq)
 
 	status = wb_sid2gid_recv(subreq, &state->pw->pw_gid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -131,7 +129,9 @@ static void wb_fill_pwent_sid2gid_done(struct tevent_req *subreq)
 				     true);
 	}
 
-	fstrcpy(state->pw->pw_name, output_username);
+	strlcpy(state->pw->pw_name,
+		output_username,
+		sizeof(state->pw->pw_name));
 	fstrcpy(state->pw->pw_gecos, state->info->full_name);
 
 	/* Home directory and shell */
@@ -194,7 +194,7 @@ static bool fillup_pw_field(const char *lp_template,
 	if (!templ)
 		return False;
 
-	safe_strcpy(out, templ, sizeof(fstring) - 1);
+	strlcpy(out, templ, sizeof(fstring));
 	TALLOC_FREE(templ);
 
 	return True;
diff --git a/source3/winbindd/wb_getgrsid.c b/source3/winbindd/wb_getgrsid.c
index e55c1b0021b..7d7e56a5ca2 100644
--- a/source3/winbindd/wb_getgrsid.c
+++ b/source3/winbindd/wb_getgrsid.c
@@ -83,8 +83,7 @@ static void wb_getgrsid_lookupsid_done(struct tevent_req *subreq)
 	status = wb_lookupsid_recv(subreq, state, &state->type,
 				   &state->domname, &state->name);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -115,8 +114,7 @@ static void wb_getgrsid_sid2gid_done(struct tevent_req *subreq)
 
 	status = wb_sid2gid_recv(subreq, &state->gid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	subreq = wb_group_members_send(state, state->ev, &state->sid,
@@ -137,8 +135,7 @@ static void wb_getgrsid_got_members(struct tevent_req *subreq)
 
 	status = wb_group_members_recv(subreq, state, &state->members);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/wb_getpwsid.c b/source3/winbindd/wb_getpwsid.c
index 24ce7dc3f4f..3c942f0857c 100644
--- a/source3/winbindd/wb_getpwsid.c
+++ b/source3/winbindd/wb_getpwsid.c
@@ -74,8 +74,7 @@ static void wb_getpwsid_queryuser_done(struct tevent_req *subreq)
 
 	status = wb_queryuser_recv(subreq, state, &state->userinfo);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -118,8 +117,7 @@ static void wb_getpwsid_lookupsid_done(struct tevent_req *subreq)
 	status = wb_lookupsid_recv(subreq, state->userinfo, &type, &domain,
 				   &state->userinfo->acct_name);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	subreq = wb_fill_pwent_send(state, state->ev, state->userinfo,
@@ -137,8 +135,7 @@ static void wb_getpwsid_done(struct tevent_req *subreq)
 	NTSTATUS status;
 
 	status = wb_fill_pwent_recv(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/wb_gettoken.c b/source3/winbindd/wb_gettoken.c
index 5c2af4abd64..aaaffa0bf20 100644
--- a/source3/winbindd/wb_gettoken.c
+++ b/source3/winbindd/wb_gettoken.c
@@ -90,8 +90,7 @@ static void wb_gettoken_gotgroups(struct tevent_req *subreq)
 	status = wb_lookupusergroups_recv(subreq, state, &state->num_sids,
 					  &state->sids);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -135,8 +134,7 @@ static void wb_gettoken_gotlocalgroups(struct tevent_req *subreq)
 
 	status = wb_lookupuseraliases_recv(subreq, state, &num_rids, &rids);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	domain = find_domain_from_sid_noinit(get_global_sam_sid());
@@ -181,8 +179,7 @@ static void wb_gettoken_gotbuiltins(struct tevent_req *subreq)
 
 	status = wb_lookupuseraliases_recv(subreq, state, &num_rids, &rids);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	if (!wb_add_rids_to_sids(state, &state->num_sids, &state->sids,
diff --git a/source3/winbindd/wb_group_members.c b/source3/winbindd/wb_group_members.c
index c9603e73f8b..e6efcabfad4 100644
--- a/source3/winbindd/wb_group_members.c
+++ b/source3/winbindd/wb_group_members.c
@@ -155,8 +155,7 @@ static struct tevent_req *wb_groups_members_send(TALLOC_CTX *mem_ctx,
 	state->all_members = NULL;
 
 	status = wb_groups_members_next_subreq(state, state, &subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return tevent_req_post(req, ev);
 	}
 	if (subreq == NULL) {
@@ -210,8 +209,7 @@ static void wb_groups_members_done(struct tevent_req *subreq)
 	 * and just continue if an error occured.
 	 */
 
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -235,8 +233,7 @@ static void wb_groups_members_done(struct tevent_req *subreq)
 	TALLOC_FREE(members);
 
 	status = wb_groups_members_next_subreq(state, state, &subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	if (subreq == NULL) {
@@ -313,8 +310,7 @@ struct tevent_req *wb_group_members_send(TALLOC_CTX *mem_ctx,
 	state->groups->type = type;
 
 	status = wb_group_members_next_subreq(state, state, &subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return tevent_req_post(req, ev);
 	}
 	if (subreq == NULL) {
@@ -361,8 +357,7 @@ static void wb_group_members_done(struct tevent_req *subreq)
 
 	status = wb_groups_members_recv(subreq, state, &num_members, &members);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -440,8 +435,7 @@ static void wb_group_members_done(struct tevent_req *subreq)
 	}
 
 	status = wb_group_members_next_subreq(state, state, &subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	if (subreq == NULL) {
diff --git a/source3/winbindd/wb_lookupname.c b/source3/winbindd/wb_lookupname.c
index a9b4dfa586d..2ebaba03d3f 100644
--- a/source3/winbindd/wb_lookupname.c
+++ b/source3/winbindd/wb_lookupname.c
@@ -91,8 +91,7 @@ static void wb_lookupname_done(struct tevent_req *subreq)
 
 	status = dcerpc_wbint_LookupName_recv(subreq, state, &result);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	if (NT_STATUS_IS_OK(result)) {
diff --git a/source3/winbindd/wb_lookupsid.c b/source3/winbindd/wb_lookupsid.c
index 2e1b21a8d88..2ab3917ceb7 100644
--- a/source3/winbindd/wb_lookupsid.c
+++ b/source3/winbindd/wb_lookupsid.c
@@ -76,8 +76,7 @@ static void wb_lookupsid_done(struct tevent_req *subreq)
 
 	status = dcerpc_wbint_LookupSid_recv(subreq, state, &result);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	if (NT_STATUS_IS_OK(result)) {
diff --git a/source3/winbindd/wb_lookupsids.c b/source3/winbindd/wb_lookupsids.c
index 05601ad192b..03b2ca9029b 100644
--- a/source3/winbindd/wb_lookupsids.c
+++ b/source3/winbindd/wb_lookupsids.c
@@ -128,26 +128,26 @@ struct tevent_req *wb_lookupsids_send(TALLOC_CTX *mem_ctx,
 		return tevent_req_post(req, ev);
 	}
 
-	state->single_sids = TALLOC_ARRAY(state, uint32_t, num_sids);
+	state->single_sids = talloc_array(state, uint32_t, num_sids);
 	if (tevent_req_nomem(state->single_sids, req)) {
 		return tevent_req_post(req, ev);
 	}
 
-	state->res_domains = TALLOC_ZERO_P(state, struct lsa_RefDomainList);
+	state->res_domains = talloc_zero(state, struct lsa_RefDomainList);
 	if (tevent_req_nomem(state->res_domains, req)) {
 		return tevent_req_post(req, ev);
 	}
-	state->res_domains->domains = TALLOC_ARRAY(
+	state->res_domains->domains = talloc_array(
 		state->res_domains, struct lsa_DomainInfo, num_sids);
 	if (tevent_req_nomem(state->res_domains->domains, req)) {
 		return tevent_req_post(req, ev);
 	}
 
-	state->res_names = TALLOC_ZERO_P(state, struct lsa_TransNameArray);
+	state->res_names = talloc_zero(state, struct lsa_TransNameArray);
 	if (tevent_req_nomem(state->res_names, req)) {
 		return tevent_req_post(req, ev);
 	}
-	state->res_names->names = TALLOC_ARRAY(
+	state->res_names->names = talloc_array(
 		state->res_names, struct lsa_TranslatedName, num_sids);
 	if (tevent_req_nomem(state->res_names->names, req)) {
 		return tevent_req_post(req, ev);
@@ -187,7 +187,7 @@ static bool wb_lookupsids_next(struct tevent_req *req,
 
 		if (sid_check_is_domain(&d->sid)) {
 			state->rids.num_rids = d->sids.num_sids;
-			state->rids.rids = TALLOC_ARRAY(state, uint32_t,
+			state->rids.rids = talloc_array(state, uint32_t,
 							state->rids.num_rids);
 			if (tevent_req_nomem(state->rids.rids, req)) {
 				return false;
@@ -325,7 +325,7 @@ static struct wb_lookupsids_domain *wb_lookupsids_get_domain(
 		return NULL;
 	}
 
-	domains = TALLOC_REALLOC_ARRAY(
+	domains = talloc_realloc(
 		mem_ctx, domains, struct wb_lookupsids_domain, num_domains+1);
 	if (domains == NULL) {
 		return NULL;
@@ -337,13 +337,13 @@ static struct wb_lookupsids_domain *wb_lookupsids_get_domain(
 	sid_split_rid(&domain->sid, NULL);
 	domain->domain = wb_domain;
 
-	domain->sids.sids = TALLOC_ARRAY(domains, struct lsa_SidPtr, num_sids);
+	domain->sids.sids = talloc_array(domains, struct lsa_SidPtr, num_sids);
 	if (domains->sids.sids == NULL) {
 		goto fail;
 	}
 	domain->sids.num_sids = 0;
 
-	domain->sid_indexes = TALLOC_ARRAY(domains, uint32_t, num_sids);
+	domain->sid_indexes = talloc_array(domains, uint32_t, num_sids);
 	if (domain->sid_indexes == NULL) {
 		TALLOC_FREE(domain->sids.sids);
 		goto fail;
@@ -354,7 +354,7 @@ fail:
 	/*
 	 * Realloc to the state it was in before
 	 */
-	*pdomains = TALLOC_REALLOC_ARRAY(
+	*pdomains = talloc_realloc(
 		mem_ctx, domains, struct wb_lookupsids_domain, num_domains);
 	return NULL;
 }
@@ -428,6 +428,7 @@ static void wb_lookupsids_done(struct tevent_req *subreq)
 		req, struct wb_lookupsids_state);
 	struct wb_lookupsids_domain *d;
 	uint32_t i;
+	bool fallback = false;
 
 	NTSTATUS status, result;
 
@@ -437,13 +438,31 @@ static void wb_lookupsids_done(struct tevent_req *subreq)
 		return;
 	}
 
+	d = &state->domains[state->domains_done];
+
+	if (NT_STATUS_IS_ERR(result)) {
+		fallback = true;
+	} else if (state->tmp_names.count != d->sids.num_sids) {
+		fallback = true;
+	}
+
+	if (fallback) {
+		for (i=0; i < d->sids.num_sids; i++) {
+			uint32_t res_sid_index = d->sid_indexes[i];
+
+			state->single_sids[state->num_single_sids] =
+				res_sid_index;
+			state->num_single_sids += 1;
+		}
+		state->domains_done += 1;
+		wb_lookupsids_next(req, state);
+		return;
+	}
+
 	/*
-	 * Ignore "result" here. We depend on the individual states in
-	 * the translated names.
+	 * Look at the individual states in the translated names.
 	 */
 
-	d = &state->domains[state->domains_done];
-
 	for (i=0; i<state->tmp_names.count; i++) {
 
 		uint32_t res_sid_index = d->sid_indexes[i];
@@ -462,7 +481,7 @@ static void wb_lookupsids_done(struct tevent_req *subreq)
 			    &state->tmp_domains, &state->tmp_names.names[i],
 			    state->res_domains, state->res_names,
 			    res_sid_index)) {
-			tevent_req_nomem(NULL, req);
+			tevent_req_oom(req);
 			return;
 		}
 	}
@@ -525,7 +544,7 @@ static void wb_lookupsids_single_done(struct tevent_req *subreq)
 		    &src_domains, &src_name,
 		    state->res_domains, state->res_names,
 		    res_sid_index)) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return;
 	}
 	state->single_sids_done += 1;
@@ -544,6 +563,7 @@ static void wb_lookupsids_lookuprids_done(struct tevent_req *subreq)
 	NTSTATUS status, result;
 	struct wb_lookupsids_domain *d;
 	uint32_t i;
+	bool fallback = false;
 
 	status = dcerpc_wbint_LookupRids_recv(subreq, state, &result);
 	TALLOC_FREE(subreq);
@@ -552,6 +572,30 @@ static void wb_lookupsids_lookuprids_done(struct tevent_req *subreq)
 	}
 
 	d = &state->domains[state->domains_done];
+
+	if (NT_STATUS_IS_ERR(result)) {
+		fallback = true;
+	} else if (state->rid_names.num_principals != d->sids.num_sids) {
+		fallback = true;
+	}
+
+	if (fallback) {
+		for (i=0; i < d->sids.num_sids; i++) {
+			uint32_t res_sid_index = d->sid_indexes[i];
+
+			state->single_sids[state->num_single_sids] =
+				res_sid_index;
+			state->num_single_sids += 1;
+		}
+		state->domains_done += 1;
+		wb_lookupsids_next(req, state);
+		return;
+	}
+
+	/*
+	 * Look at the individual states in the translated names.
+	 */
+
 	sid_copy(&src_domain_sid, get_global_sam_sid());
 	src_domain.name.string = get_global_sam_name();
 	src_domain.sid = &src_domain_sid;
@@ -575,7 +619,7 @@ static void wb_lookupsids_lookuprids_done(struct tevent_req *subreq)
 			    &src_domains, &src_name,
 			    state->res_domains, state->res_names,
 			    res_sid_index)) {
-			tevent_req_nomem(NULL, req);
+			tevent_req_oom(req);
 			return;
 		}
 	}
@@ -595,6 +639,24 @@ NTSTATUS wb_lookupsids_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
 	if (tevent_req_is_nterror(req, &status)) {
 		return status;
 	}
+
+	/*
+	 * The returned names need to match the given sids,
+	 * if not we have a bug in the code!
+	 *
+	 */
+	SMB_ASSERT(state->res_names->count == state->num_sids);
+
+	/*
+	 * Not strictly needed, but it might make debugging in the callers
+	 * easier in future, if the talloc_array_length() returns the
+	 * expected result...
+	 */
+	state->res_domains->domains = talloc_realloc(state->res_domains,
+						     state->res_domains->domains,
+						     struct lsa_DomainInfo,
+						     state->res_domains->count);
+
 	*domains = talloc_move(mem_ctx, &state->res_domains);
 	*names = talloc_move(mem_ctx, &state->res_names);
 	return NT_STATUS_OK;
diff --git a/source3/winbindd/wb_lookupuseraliases.c b/source3/winbindd/wb_lookupuseraliases.c
index f6fe855a856..5e7eb1c33c2 100644
--- a/source3/winbindd/wb_lookupuseraliases.c
+++ b/source3/winbindd/wb_lookupuseraliases.c
@@ -44,7 +44,7 @@ struct tevent_req *wb_lookupuseraliases_send(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 	state->sids.num_sids = num_sids;
-	state->sids.sids = CONST_DISCARD(struct dom_sid *, sids);
+	state->sids.sids = discard_const_p(struct dom_sid, sids);
 
 	subreq = dcerpc_wbint_LookupUserAliases_send(
 		state, ev, dom_child_handle(domain), &state->sids, &state->rids);
diff --git a/source3/winbindd/wb_next_grent.c b/source3/winbindd/wb_next_grent.c
index c7af6416c49..2b3799ab93e 100644
--- a/source3/winbindd/wb_next_grent.c
+++ b/source3/winbindd/wb_next_grent.c
@@ -101,12 +101,11 @@ static void wb_next_grent_fetch_done(struct tevent_req *subreq)
 
 	status = dcerpc_wbint_QueryGroupList_recv(subreq, state, &result);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
+	if (tevent_req_nterror(req, status)) {
 		/* Ignore errors here, just log it */
 		DEBUG(10, ("query_user_list for domain %s returned %s\n",
 			   state->gstate->domain->name,
 			   nt_errstr(status)));
-		tevent_req_nterror(req, status);
 		return;
 	}
 	if (!NT_STATUS_IS_OK(result)) {
@@ -169,8 +168,7 @@ static void wb_next_grent_getgrsid_done(struct tevent_req *subreq)
 	status = wb_getgrsid_recv(subreq, talloc_tos(), &domname, &name,
 				  &state->gr->gr_gid, &state->members);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	if (!fill_grent(talloc_tos(), state->gr, domname, name,
diff --git a/source3/winbindd/wb_next_pwent.c b/source3/winbindd/wb_next_pwent.c
index 4595fbc6533..da4754cc9fc 100644
--- a/source3/winbindd/wb_next_pwent.c
+++ b/source3/winbindd/wb_next_pwent.c
@@ -148,8 +148,7 @@ static void wb_next_pwent_fill_done(struct tevent_req *subreq)
 
 	status = wb_fill_pwent_recv(subreq);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	state->gstate->next_user += 1;
diff --git a/source3/winbindd/wb_sid2gid.c b/source3/winbindd/wb_sid2gid.c
index 6c102a205c5..cb95191e7e3 100644
--- a/source3/winbindd/wb_sid2gid.c
+++ b/source3/winbindd/wb_sid2gid.c
@@ -94,8 +94,7 @@ static void wb_sid2gid_lookup_done(struct tevent_req *subreq)
 
 	status = wb_lookupsid_recv(subreq, talloc_tos(), &type, &domname,
 				   &name);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
diff --git a/source3/winbindd/wb_sid2uid.c b/source3/winbindd/wb_sid2uid.c
index 5dd4b82eef3..a2e0f9712f9 100644
--- a/source3/winbindd/wb_sid2uid.c
+++ b/source3/winbindd/wb_sid2uid.c
@@ -93,8 +93,7 @@ static void wb_sid2uid_lookup_done(struct tevent_req *subreq)
 
 	status = wb_lookupsid_recv(subreq, talloc_tos(), &type, &domname,
 				   &name);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
diff --git a/source3/winbindd/winbindd.c b/source3/winbindd/winbindd.c
index c5a124a628f..469d64e41d5 100644
--- a/source3/winbindd/winbindd.c
+++ b/source3/winbindd/winbindd.c
@@ -27,6 +27,7 @@
 #include "winbindd.h"
 #include "nsswitch/winbind_client.h"
 #include "nsswitch/wb_reqtrans.h"
+#include "ntdomain.h"
 #include "../librpc/gen_ndr/srv_lsa.h"
 #include "../librpc/gen_ndr/srv_samr.h"
 #include "secrets.h"
@@ -378,6 +379,7 @@ static void winbind_msg_validate_cache(struct messaging_context *msg_ctx,
 {
 	uint8 ret;
 	pid_t child_pid;
+	NTSTATUS status;
 
 	DEBUG(10, ("winbindd_msg_validate_cache: got validate-cache "
 		   "message.\n"));
@@ -404,7 +406,10 @@ static void winbind_msg_validate_cache(struct messaging_context *msg_ctx,
 
 	/* child */
 
-	if (!winbindd_reinit_after_fork(NULL)) {
+	status = winbindd_reinit_after_fork(NULL, NULL);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(1, ("winbindd_reinit_after_fork failed: %s\n",
+			  nt_errstr(status)));
 		_exit(0);
 	}
 
@@ -445,11 +450,6 @@ static struct winbindd_dispatch_table {
 	{ WINBINDD_CCACHE_NTLMAUTH, winbindd_ccache_ntlm_auth, "NTLMAUTH" },
 	{ WINBINDD_CCACHE_SAVE, winbindd_ccache_save, "CCACHE_SAVE" },
 
-	/* WINS functions */
-
-	{ WINBINDD_WINS_BYNAME, winbindd_wins_byname, "WINS_BYNAME" },
-	{ WINBINDD_WINS_BYIP, winbindd_wins_byip, "WINS_BYIP" },
-
 	/* End of list */
 
 	{ WINBINDD_NUM_CMDS, NULL, "NONE" }
@@ -540,6 +540,10 @@ static struct winbindd_async_dispatch_table async_nonpriv_table[] = {
 	{ WINBINDD_PAM_CHNG_PSWD_AUTH_CRAP, "PAM_CHNG_PSWD_AUTH_CRAP",
 	  winbindd_pam_chng_pswd_auth_crap_send,
 	  winbindd_pam_chng_pswd_auth_crap_recv },
+	{ WINBINDD_WINS_BYIP, "WINS_BYIP",
+	  winbindd_wins_byip_send, winbindd_wins_byip_recv },
+	{ WINBINDD_WINS_BYNAME, "WINS_BYNAME",
+	  winbindd_wins_byname_send, winbindd_wins_byname_recv },
 
 	{ 0, NULL, NULL, NULL }
 };
@@ -779,7 +783,7 @@ static void new_connection(int listen_sock, bool privileged)
 
 	/* Create new connection structure */
 
-	if ((state = TALLOC_ZERO_P(NULL, struct winbindd_cli_state)) == NULL) {
+	if ((state = talloc_zero(NULL, struct winbindd_cli_state)) == NULL) {
 		close(sock);
 		return;
 	}
@@ -940,12 +944,12 @@ static void winbindd_listen_fde_handler(struct tevent_context *ev,
 
 const char *get_winbind_pipe_dir(void)
 {
-	return lp_parm_const_string(-1, "winbindd", "socket dir", WINBINDD_SOCKET_DIR);
+	return lp_parm_const_string(-1, "winbindd", "socket dir", get_dyn_WINBINDD_SOCKET_DIR());
 }
 
 char *get_winbind_priv_pipe_dir(void)
 {
-	return lock_path(WINBINDD_PRIV_SOCKET_SUBDIR);
+	return state_path(WINBINDD_PRIV_SOCKET_SUBDIR);
 }
 
 static bool winbindd_setup_listeners(void)
diff --git a/source3/winbindd/winbindd.h b/source3/winbindd/winbindd.h
index 4f0f624d5be..3f1c7f558e2 100644
--- a/source3/winbindd/winbindd.h
+++ b/source3/winbindd/winbindd.h
@@ -30,6 +30,8 @@
 #include "talloc_dict.h"
 #include "smb_ldap.h"
 
+#include "../lib/util/tevent_ntstatus.h"
+
 #ifdef HAVE_LIBNSCD
 #include <libnscd.h>
 #endif
diff --git a/source3/winbindd/winbindd_ads.c b/source3/winbindd/winbindd_ads.c
index dde8e3f53d2..610db7a62b6 100644
--- a/source3/winbindd/winbindd_ads.c
+++ b/source3/winbindd/winbindd_ads.c
@@ -199,7 +199,7 @@ static NTSTATUS query_user_list(struct winbindd_domain *domain,
 		goto done;
 	}
 
-	(*pinfo) = TALLOC_ZERO_ARRAY(mem_ctx, struct wbint_userinfo, count);
+	(*pinfo) = talloc_zero_array(mem_ctx, struct wbint_userinfo, count);
 	if (!*pinfo) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -351,7 +351,7 @@ static NTSTATUS enum_dom_groups(struct winbindd_domain *domain,
 		goto done;
 	}
 
-	(*info) = TALLOC_ZERO_ARRAY(mem_ctx, struct wb_acct_info, count);
+	(*info) = talloc_zero_array(mem_ctx, struct wb_acct_info, count);
 	if (!*info) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -489,7 +489,7 @@ static NTSTATUS query_user(struct winbindd_domain *domain,
 
 	/* try netsamlogon cache first */
 
-	if ( (user = netsamlogon_cache_get( mem_ctx, sid )) != NULL ) 
+	if (winbindd_use_cache() && (user = netsamlogon_cache_get( mem_ctx, sid )) != NULL )
 	{
 		DEBUG(5,("query_user: Cache lookup succeeded for %s\n", 
 			 sid_string_dbg(sid)));
@@ -769,7 +769,7 @@ static NTSTATUS lookup_usergroups_memberof(struct winbindd_domain *domain,
 		goto done;
 	}
 
-	group_sids = TALLOC_ZERO_ARRAY(mem_ctx, struct dom_sid, num_strings + 1);
+	group_sids = talloc_zero_array(mem_ctx, struct dom_sid, num_strings + 1);
 	if (!group_sids) {
 		status = NT_STATUS_NO_MEMORY;
 		goto done;
@@ -1085,10 +1085,10 @@ static NTSTATUS lookup_groupmem(struct winbindd_domain *domain,
 	 * cache. Only the rest is passed to the lsa_lookup_sids call. */
 
 	if (num_members) {
-		(*sid_mem) = TALLOC_ZERO_ARRAY(mem_ctx, struct dom_sid, num_members);
-		(*names) = TALLOC_ZERO_ARRAY(mem_ctx, char *, num_members);
-		(*name_types) = TALLOC_ZERO_ARRAY(mem_ctx, uint32, num_members);
-		(sid_mem_nocache) = TALLOC_ZERO_ARRAY(tmp_ctx, struct dom_sid, num_members);
+		(*sid_mem) = talloc_zero_array(mem_ctx, struct dom_sid, num_members);
+		(*names) = talloc_zero_array(mem_ctx, char *, num_members);
+		(*name_types) = talloc_zero_array(mem_ctx, uint32, num_members);
+		(sid_mem_nocache) = talloc_zero_array(tmp_ctx, struct dom_sid, num_members);
 
 		if ((members == NULL) || (*sid_mem == NULL) ||
 		    (*names == NULL) || (*name_types == NULL) ||
diff --git a/source3/winbindd/winbindd_cache.c b/source3/winbindd/winbindd_cache.c
index 68a86257da2..58a8e710f6f 100644
--- a/source3/winbindd/winbindd_cache.c
+++ b/source3/winbindd/winbindd_cache.c
@@ -33,6 +33,7 @@
 #include "nss_info.h"
 #include "../libcli/security/security.h"
 #include "passdb/machine_sid.h"
+#include "util_tdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
@@ -308,7 +309,7 @@ static char *centry_string(struct cache_entry *centry, TALLOC_CTX *mem_ctx)
 		smb_panic_fn("centry_string");
 	}
 
-	ret = TALLOC_ARRAY(mem_ctx, char, len+1);
+	ret = talloc_array(mem_ctx, char, len+1);
 	if (!ret) {
 		smb_panic_fn("centry_string out of memory\n");
 	}
@@ -338,7 +339,7 @@ static char *centry_hash16(struct cache_entry *centry, TALLOC_CTX *mem_ctx)
 		return NULL;
 	}
 
-	ret = TALLOC_ARRAY(mem_ctx, char, 16);
+	ret = talloc_array(mem_ctx, char, 16);
 	if (!ret) {
 		smb_panic_fn("centry_hash out of memory\n");
 	}
@@ -483,9 +484,9 @@ bool wcache_store_seqnum(const char *domain_name, uint32_t seqnum,
 	ret = tdb_store_bystring(wcache->tdb, key_str,
 				 make_tdb_data(buf, sizeof(buf)), TDB_REPLACE);
 	TALLOC_FREE(key_str);
-	if (ret == -1) {
+	if (ret != 0) {
 		DEBUG(10, ("tdb_store_bystring failed: %s\n",
-			   tdb_errorstr(wcache->tdb)));
+			   tdb_errorstr_compat(wcache->tdb)));
 		TALLOC_FREE(key_str);
 		return false;
 	}
@@ -637,7 +638,7 @@ static struct cache_entry *wcache_fetch_raw(char *kstr)
 	TDB_DATA key;
 
 	key = string_tdb_data(kstr);
-	data = tdb_fetch(wcache->tdb, key);
+	data = tdb_fetch_compat(wcache->tdb, key);
 	if (!data.dptr) {
 		/* a cache miss */
 		return NULL;
@@ -1270,7 +1271,7 @@ NTSTATUS wcache_cached_creds_exist(struct winbindd_domain *domain, const struct
 
 	fstr_sprintf(key_str, "CRED/%s", sid_to_fstring(tmp, sid));
 
-	data = tdb_fetch(cache->tdb, string_tdb_data(key_str));
+	data = tdb_fetch_compat(cache->tdb, string_tdb_data(key_str));
 	if (!data.dptr) {
 		return NT_STATUS_OBJECT_NAME_NOT_FOUND;
 	}
@@ -1295,6 +1296,10 @@ NTSTATUS wcache_get_creds(struct winbindd_domain *domain,
 	uint32 rid;
 	fstring tmp;
 
+	if (!winbindd_use_cache()) {
+		return NT_STATUS_OBJECT_NAME_NOT_FOUND;
+	}
+
 	if (!cache->tdb) {
 		return NT_STATUS_INTERNAL_DB_ERROR;
 	}
@@ -1431,7 +1436,7 @@ do_fetch_cache:
 	if (*num_entries == 0)
 		goto do_cached;
 
-	(*info) = TALLOC_ARRAY(mem_ctx, struct wbint_userinfo, *num_entries);
+	(*info) = talloc_array(mem_ctx, struct wbint_userinfo, *num_entries);
 	if (! (*info)) {
 		smb_panic_fn("query_user_list out of memory");
 	}
@@ -1583,7 +1588,7 @@ do_fetch_cache:
 	if (*num_entries == 0)
 		goto do_cached;
 
-	(*info) = TALLOC_ARRAY(mem_ctx, struct wb_acct_info, *num_entries);
+	(*info) = talloc_array(mem_ctx, struct wb_acct_info, *num_entries);
 	if (! (*info)) {
 		smb_panic_fn("enum_dom_groups out of memory");
 	}
@@ -1678,7 +1683,7 @@ do_fetch_cache:
 	if (*num_entries == 0)
 		goto do_cached;
 
-	(*info) = TALLOC_ARRAY(mem_ctx, struct wb_acct_info, *num_entries);
+	(*info) = talloc_array(mem_ctx, struct wb_acct_info, *num_entries);
 	if (! (*info)) {
 		smb_panic_fn("enum_dom_groups out of memory");
 	}
@@ -1856,8 +1861,8 @@ static NTSTATUS name_to_sid(struct winbindd_domain *domain,
 
 		/* Only save the reverse mapping if this was not a UPN */
 		if (!strchr(name, '@')) {
-			strupper_m(CONST_DISCARD(char *,domain_name));
-			strlower_m(CONST_DISCARD(char *,name));
+			strupper_m(discard_const_p(char, domain_name));
+			strlower_m(discard_const_p(char, name));
 			wcache_save_sid_to_name(domain, status, sid, domain_name, name, *type);
 		}
 	}
@@ -2001,8 +2006,8 @@ static NTSTATUS rids_to_names(struct winbindd_domain *domain,
 		return NT_STATUS_OK;
 	}
 
-	*names = TALLOC_ARRAY(mem_ctx, char *, num_rids);
-	*types = TALLOC_ARRAY(mem_ctx, enum lsa_SidType, num_rids);
+	*names = talloc_array(mem_ctx, char *, num_rids);
+	*types = talloc_array(mem_ctx, enum lsa_SidType, num_rids);
 
 	if ((*names == NULL) || (*types == NULL)) {
 		result = NT_STATUS_NO_MEMORY;
@@ -2769,7 +2774,7 @@ static NTSTATUS trusted_domains(struct winbindd_domain *domain,
 	}
 
 do_fetch_cache:
-	trusts->array = TALLOC_ZERO_ARRAY(mem_ctx, struct netr_DomainTrust, num_domains);
+	trusts->array = talloc_zero_array(mem_ctx, struct netr_DomainTrust, num_domains);
 	if (!trusts->array) {
 		TALLOC_FREE(dom_list);
 		return NT_STATUS_NO_MEMORY;
@@ -3160,7 +3165,7 @@ bool initialize_winbindd_cache(void)
 		/* Write the version. */
 		if (!tdb_store_uint32(wcache->tdb, WINBINDD_CACHE_VERSION_KEYSTR, WINBINDD_CACHE_VERSION)) {
 			DEBUG(0,("initialize_winbindd_cache: version number store failed %s\n",
-				tdb_errorstr(wcache->tdb) ));
+				tdb_errorstr_compat(wcache->tdb) ));
 			return false;
 		}
 	}
@@ -3378,7 +3383,7 @@ NTSTATUS wcache_remove_oldest_cached_creds(struct winbindd_domain *domain, const
 	ret = tdb_traverse(cache->tdb, traverse_fn_get_credlist, NULL);
 	if (ret == 0) {
 		return NT_STATUS_OK;
-	} else if ((ret == -1) || (wcache_cred_list == NULL)) {
+	} else if ((ret < 0) || (wcache_cred_list == NULL)) {
 		return NT_STATUS_OBJECT_NAME_NOT_FOUND;
 	}
 
@@ -3389,7 +3394,7 @@ NTSTATUS wcache_remove_oldest_cached_creds(struct winbindd_domain *domain, const
 		TDB_DATA data;
 		time_t t;
 
-		data = tdb_fetch(cache->tdb, string_tdb_data(cred->name));
+		data = tdb_fetch_compat(cache->tdb, string_tdb_data(cred->name));
 		if (!data.dptr) {
 			DEBUG(10,("wcache_remove_oldest_cached_creds: entry for [%s] not found\n", 
 				cred->name));
@@ -4198,10 +4203,10 @@ static bool add_wbdomain_to_tdc_array( struct winbindd_domain *new_dom,
 
 	if ( !set_only ) {
 		if ( !*domains ) {
-			list = TALLOC_ARRAY( NULL, struct winbindd_tdc_domain, 1 );
+			list = talloc_array( NULL, struct winbindd_tdc_domain, 1 );
 			idx = 0;
 		} else {
-			list = TALLOC_REALLOC_ARRAY( *domains, *domains, 
+			list = talloc_realloc( *domains, *domains, 
 						     struct winbindd_tdc_domain,  
 						     (*num_domains)+1);
 			idx = *num_domains;		
@@ -4341,7 +4346,7 @@ static size_t unpack_tdc_domains( unsigned char *buf, int buflen,
 		return 0;
 	}
 
-	list = TALLOC_ARRAY( NULL, struct winbindd_tdc_domain, num_domains );
+	list = talloc_array( NULL, struct winbindd_tdc_domain, num_domains );
 	if ( !list ) {
 		DEBUG(0,("unpack_tdc_domains: Failed to talloc() domain list!\n"));
 		return 0;		
@@ -4415,7 +4420,7 @@ static bool wcache_tdc_store_list( struct winbindd_tdc_domain *domains, size_t n
 	SAFE_FREE( data.dptr );
 	SAFE_FREE( key.dptr );
 
-	return ( ret != -1 );	
+	return ( ret == 0 );
 }
 
 /*********************************************************************
@@ -4432,7 +4437,7 @@ bool wcache_tdc_fetch_list( struct winbindd_tdc_domain **domains, size_t *num_do
 	if ( !key.dptr )
 		return false;
 
-	data = tdb_fetch( wcache->tdb, key );
+	data = tdb_fetch_compat( wcache->tdb, key );
 
 	SAFE_FREE( key.dptr );
 
@@ -4522,7 +4527,7 @@ struct winbindd_tdc_domain * wcache_tdc_fetch_domain( TALLOC_CTX *ctx, const cha
 			DEBUG(10,("wcache_tdc_fetch_domain: Found domain %s\n",
 				  name));
 
-			d = TALLOC_P( ctx, struct winbindd_tdc_domain );
+			d = talloc( ctx, struct winbindd_tdc_domain );
 			if ( !d )
 				break;			
 
@@ -4572,7 +4577,7 @@ struct winbindd_tdc_domain*
 				   dom_list[i].domain_name,
 				   sid_string_dbg(sid)));
 
-			d = TALLOC_P(ctx, struct winbindd_tdc_domain);
+			d = talloc(ctx, struct winbindd_tdc_domain);
 			if (!d)
 				break;
 
@@ -4777,7 +4782,7 @@ bool wcache_fetch_ndr(TALLOC_CTX *mem_ctx, struct winbindd_domain *domain,
 	if (!wcache_ndr_key(talloc_tos(), domain->name, opnum, req, &key)) {
 		return false;
 	}
-	data = tdb_fetch(wcache->tdb, key);
+	data = tdb_fetch_compat(wcache->tdb, key);
 	TALLOC_FREE(key.dptr);
 
 	if (data.dptr == NULL) {
diff --git a/source3/winbindd/winbindd_ccache_access.c b/source3/winbindd/winbindd_ccache_access.c
index 6a265ccaf07..0b52ae2eb9a 100644
--- a/source3/winbindd/winbindd_ccache_access.c
+++ b/source3/winbindd/winbindd_ccache_access.c
@@ -55,7 +55,7 @@ static NTSTATUS do_ntlm_auth_with_hashes(const char *username,
 	DATA_BLOB dummy_msg, reply;
 
 	status = ntlmssp_client_start(NULL,
-				      global_myname(),
+				      lp_netbios_name(),
 				      lp_workgroup(),
 				      lp_client_ntlmv2_auth(),
 				      &ntlmssp_state);
diff --git a/source3/winbindd/winbindd_cm.c b/source3/winbindd/winbindd_cm.c
index 1473d337218..86d6662e5a5 100644
--- a/source3/winbindd/winbindd_cm.c
+++ b/source3/winbindd/winbindd_cm.c
@@ -69,13 +69,13 @@
 #include "rpc_client/cli_lsarpc.h"
 #include "../librpc/gen_ndr/ndr_dssetup_c.h"
 #include "libads/sitename_cache.h"
+#include "libsmb/libsmb.h"
 #include "libsmb/clidgram.h"
 #include "ads.h"
 #include "secrets.h"
 #include "../libcli/security/security.h"
 #include "passdb.h"
 #include "messages.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
@@ -189,6 +189,7 @@ static bool fork_child_dc_connect(struct winbindd_domain *domain)
 	TALLOC_CTX *mem_ctx = NULL;
 	pid_t parent_pid = sys_getpid();
 	char *lfile = NULL;
+	NTSTATUS status;
 
 	if (domain->dc_probe_pid != (pid_t)-1) {
 		/*
@@ -233,7 +234,10 @@ static bool fork_child_dc_connect(struct winbindd_domain *domain)
 		}
 	}
 
-	if (!winbindd_reinit_after_fork(lfile)) {
+	status = winbindd_reinit_after_fork(NULL, lfile);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(1, ("winbindd_reinit_after_fork failed: %s\n",
+			  nt_errstr(status)));
 		messaging_send_buf(winbind_messaging_context(),
 				   pid_to_procid(parent_pid),
 				   MSG_WINBIND_FAILED_TO_GO_ONLINE,
@@ -816,7 +820,12 @@ static NTSTATUS cm_prepare_connection(const struct winbindd_domain *domain,
 
 	(*cli)->timeout = 10000; 	/* 10 seconds */
 	(*cli)->fd = sockfd;
-	fstrcpy((*cli)->desthost, controller);
+	(*cli)->desthost = talloc_strdup((*cli), controller);
+	if ((*cli)->desthost == NULL) {
+		result = NT_STATUS_NO_MEMORY;
+		goto done;
+	}
+
 	(*cli)->use_kerberos = True;
 
 	peeraddr_len = sizeof(peeraddr);
@@ -876,7 +885,7 @@ static NTSTATUS cm_prepare_connection(const struct winbindd_domain *domain,
 
 			(*cli)->use_kerberos = True;
 			DEBUG(5, ("connecting to %s from %s with kerberos principal "
-				  "[%s] and realm [%s]\n", controller, global_myname(),
+				  "[%s] and realm [%s]\n", controller, lp_netbios_name(),
 				  machine_krb5_principal, domain->alt_name));
 
 			winbindd_set_locator_kdc_envs(domain);
@@ -907,7 +916,7 @@ static NTSTATUS cm_prepare_connection(const struct winbindd_domain *domain,
 		(*cli)->use_kerberos = False;
 
 		DEBUG(5, ("connecting to %s from %s with username "
-			  "[%s]\\[%s]\n",  controller, global_myname(),
+			  "[%s]\\[%s]\n",  controller, lp_netbios_name(),
 			  lp_workgroup(), machine_account));
 
 		ads_status = cli_session_setup_spnego(*cli,
@@ -943,7 +952,7 @@ static NTSTATUS cm_prepare_connection(const struct winbindd_domain *domain,
 		/* Only try authenticated if we have a username */
 
 		DEBUG(5, ("connecting to %s from %s with username "
-			  "[%s]\\[%s]\n",  controller, global_myname(),
+			  "[%s]\\[%s]\n",  controller, lp_netbios_name(),
 			  ipc_domain, ipc_username));
 
 		if (NT_STATUS_IS_OK(cli_session_setup(
@@ -1073,7 +1082,7 @@ static bool add_one_dc_unique(TALLOC_CTX *mem_ctx, const char *domain_name,
 			    (struct sockaddr *)(void *)pss))
 			return False;
 
-	*dcs = TALLOC_REALLOC_ARRAY(mem_ctx, *dcs, struct dc_name_ip, (*num)+1);
+	*dcs = talloc_realloc(mem_ctx, *dcs, struct dc_name_ip, (*num)+1);
 
 	if (*dcs == NULL)
 		return False;
@@ -1088,7 +1097,7 @@ static bool add_sockaddr_to_array(TALLOC_CTX *mem_ctx,
 				  struct sockaddr_storage *pss, uint16 port,
 				  struct sockaddr_storage **addrs, int *num)
 {
-	*addrs = TALLOC_REALLOC_ARRAY(mem_ctx, *addrs, struct sockaddr_storage, (*num)+1);
+	*addrs = talloc_realloc(mem_ctx, *addrs, struct sockaddr_storage, (*num)+1);
 
 	if (*addrs == NULL) {
 		*num = 0;
@@ -1375,7 +1384,7 @@ static bool find_new_dc(TALLOC_CTX *mem_ctx,
 		return False;
 
 	status = smbsock_any_connect(addrs, dcnames, NULL, NULL, NULL,
-				     num_addrs, 0, fd, &fd_index, NULL);
+				     num_addrs, 0, 10, fd, &fd_index, NULL);
 	if (!NT_STATUS_IS_OK(status)) {
 		for (i=0; i<num_dcs; i++) {
 			char ab[INET6_ADDRSTRLEN];
@@ -1541,7 +1550,7 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 				return NT_STATUS_UNSUCCESSFUL;
 			}
 			if (dcip_to_name(mem_ctx, domain, &ss, saf_name )) {
-				fstrcpy( domain->dcname, saf_name );
+				strlcpy(domain->dcname, saf_name, sizeof(domain->dcname));
 			} else {
 				winbind_add_failed_connection_entry(
 					domain, saf_servername,
@@ -1571,7 +1580,7 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 
 			status = smbsock_connect(&domain->dcaddr, 0,
 						 NULL, -1, NULL, -1,
-						 &fd, NULL);
+						 &fd, NULL, 10);
 			if (!NT_STATUS_IS_OK(status)) {
 				fd = -1;
 			}
@@ -1699,6 +1708,7 @@ void invalidate_cm_connection(struct winbindd_cm_conn *conn)
 void close_conns_after_fork(void)
 {
 	struct winbindd_domain *domain;
+	struct winbindd_cli_state *cli_state;
 
 	for (domain = domain_list(); domain; domain = domain->next) {
 		struct cli_state *cli = domain->conn.cli;
@@ -1715,6 +1725,15 @@ void close_conns_after_fork(void)
 
 		invalidate_cm_connection(&domain->conn);
 	}
+
+	for (cli_state = winbindd_client_list();
+	     cli_state != NULL;
+	     cli_state = cli_state->next) {
+		if (cli_state->sock >= 0) {
+			close(cli_state->sock);
+			cli_state->sock = -1;
+		}
+	}
 }
 
 static bool connection_ok(struct winbindd_domain *domain)
@@ -1985,7 +2004,7 @@ static void set_dc_type_and_flags_connect( struct winbindd_domain *domain )
 		 * no_dssetup mode here as well to get domain->initialized
 		 * set - gd */
 
-		if (NT_STATUS_V(status) == DCERPC_FAULT_OP_RNG_ERROR) {
+		if (NT_STATUS_EQUAL(status, NT_STATUS_RPC_PROCNUM_OUT_OF_RANGE)) {
 			goto no_dssetup;
 		}
 
@@ -2608,7 +2627,7 @@ NTSTATUS cm_connect_netlogon(struct winbindd_domain *domain,
 		 netlogon_pipe,
 		 domain->dcname, /* server name. */
 		 domain->name,   /* domain name */
-		 global_myname(), /* client name */
+		 lp_netbios_name(), /* client name */
 		 account_name,   /* machine account */
 		 mach_pwd,       /* machine password */
 		 sec_chan_type,  /* from get_trust_pw */
diff --git a/source3/winbindd/winbindd_cred_cache.c b/source3/winbindd/winbindd_cred_cache.c
index ab8934bfd4a..5e79a96ca76 100644
--- a/source3/winbindd/winbindd_cred_cache.c
+++ b/source3/winbindd/winbindd_cred_cache.c
@@ -596,7 +596,7 @@ NTSTATUS add_ccache_to_list(const char *princ_name,
 		return NT_STATUS_OK;
 	}
 
-	entry = TALLOC_P(NULL, struct WINBINDD_CCACHE_ENTRY);
+	entry = talloc(NULL, struct WINBINDD_CCACHE_ENTRY);
 	if (!entry) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -908,7 +908,7 @@ static NTSTATUS winbindd_add_memory_creds_internal(const char *username,
 		return winbindd_replace_memory_creds_internal(memcredp, pass);
 	}
 
-	memcredp = TALLOC_ZERO_P(NULL, struct WINBINDD_MEMORY_CREDS);
+	memcredp = talloc_zero(NULL, struct WINBINDD_MEMORY_CREDS);
 	if (!memcredp) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/winbindd/winbindd_creds.c b/source3/winbindd/winbindd_creds.c
index 6bbd0ffd9d1..a160f7a6b5b 100644
--- a/source3/winbindd/winbindd_creds.c
+++ b/source3/winbindd/winbindd_creds.c
@@ -38,6 +38,10 @@ NTSTATUS winbindd_get_creds(struct winbindd_domain *domain,
 	struct netr_SamInfo3 *info;
 	NTSTATUS status;
 
+	if (!winbindd_use_cache()) {
+		return NT_STATUS_OBJECT_NAME_NOT_FOUND;
+	}
+
 	status = wcache_get_creds(domain, mem_ctx, sid, cached_nt_pass, cred_salt);
 	if (!NT_STATUS_IS_OK(status)) {
 		return status;
diff --git a/source3/winbindd/winbindd_dual.c b/source3/winbindd/winbindd_dual.c
index 0a1534cae1a..5bf90b12223 100644
--- a/source3/winbindd/winbindd_dual.c
+++ b/source3/winbindd/winbindd_dual.c
@@ -35,7 +35,7 @@
 #include "../libcli/security/security.h"
 #include "system/select.h"
 #include "messages.h"
-#include "ntdomain.h"
+#include "../lib/util/tevent_unix.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
@@ -127,7 +127,7 @@ struct tevent_req *wb_child_request_send(TALLOC_CTX *mem_ctx,
 
 	if (!tevent_queue_add(child->queue, ev, req,
 			      wb_child_request_trigger, NULL)) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return tevent_req_post(req, ev);
 	}
 	return req;
@@ -510,6 +510,11 @@ void winbind_child_died(pid_t pid)
 
 	DLIST_REMOVE(winbindd_children, child);
 	child->pid = 0;
+
+	if (child->sock != -1) {
+		close(child->sock);
+		child->sock = -1;
+	}
 }
 
 /* Ensure any negative cache entries with the netbios or realm names are removed. */
@@ -741,7 +746,7 @@ void winbind_msg_onlinestatus(struct messaging_context *msg_ctx,
 	}
 
 	messaging_send_buf(msg_ctx, *sender, MSG_WINBIND_ONLINESTATUS, 
-			   (uint8 *)message, strlen(message) + 1);
+			   (const uint8 *)message, strlen(message) + 1);
 
 	talloc_destroy(mem_ctx);
 }
@@ -818,7 +823,7 @@ void winbind_msg_dump_domain_list(struct messaging_context *msg_ctx,
 
 		messaging_send_buf(msg_ctx, *sender,
 				   MSG_WINBIND_DUMP_DOMAIN_LIST,
-				   (uint8_t *)message, strlen(message) + 1);
+				   (const uint8_t *)message, strlen(message) + 1);
 
 		talloc_destroy(mem_ctx);
 
@@ -1166,7 +1171,8 @@ static void child_msg_dump_event_list(struct messaging_context *msg,
 	dump_event_list(winbind_event_context());
 }
 
-bool winbindd_reinit_after_fork(const char *logfilename)
+NTSTATUS winbindd_reinit_after_fork(const struct winbindd_child *myself,
+				    const char *logfilename)
 {
 	struct winbindd_domain *domain;
 	struct winbindd_child *cl;
@@ -1179,7 +1185,7 @@ bool winbindd_reinit_after_fork(const char *logfilename)
 		true);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(0,("reinit_after_fork() failed\n"));
-		return false;
+		return status;
 	}
 
 	close_conns_after_fork();
@@ -1190,10 +1196,10 @@ bool winbindd_reinit_after_fork(const char *logfilename)
 	}
 
 	if (!winbindd_setup_sig_term_handler(false))
-		return false;
+		return NT_STATUS_NO_MEMORY;
 	if (!winbindd_setup_sig_hup_handler(override_logfile ? NULL :
 					    logfilename))
-		return false;
+		return NT_STATUS_NO_MEMORY;
 
 	/* Stop zombies in children */
 	CatchChild();
@@ -1241,6 +1247,14 @@ bool winbindd_reinit_after_fork(const char *logfilename)
 		 * go through the parent.
 		 */
 		cl->pid = (pid_t)0;
+
+		/*
+		 * Close service sockets to all other children
+		 */
+		if ((cl != myself) && (cl->sock != -1)) {
+			close(cl->sock);
+			cl->sock = -1;
+		}
         }
 	/*
 	 * This is a little tricky, children must not
@@ -1261,7 +1275,7 @@ bool winbindd_reinit_after_fork(const char *logfilename)
 	cl = idmap_child();
 	cl->pid = (pid_t)0;
 
-	return true;
+	return NT_STATUS_OK;
 }
 
 /*
@@ -1281,6 +1295,8 @@ static bool fork_domain_child(struct winbindd_child *child)
 	struct winbindd_request request;
 	struct winbindd_response response;
 	struct winbindd_domain *primary_domain = NULL;
+	NTSTATUS status;
+	ssize_t nwritten;
 
 	if (child->domain) {
 		DEBUG(10, ("fork_domain_child called for domain '%s'\n",
@@ -1309,7 +1325,25 @@ static bool fork_domain_child(struct winbindd_child *child)
 
 	if (child->pid != 0) {
 		/* Parent */
+		ssize_t nread;
+
 		close(fdpair[0]);
+
+		nread = read(fdpair[1], &status, sizeof(status));
+		if (nread != sizeof(status)) {
+			DEBUG(1, ("fork_domain_child: Could not read child status: "
+				  "nread=%d, error=%s\n", (int)nread,
+				  strerror(errno)));
+			close(fdpair[1]);
+			return false;
+		}
+		if (!NT_STATUS_IS_OK(status)) {
+			DEBUG(1, ("fork_domain_child: Child status is %s\n",
+				  nt_errstr(status)));
+			close(fdpair[1]);
+			return false;
+		}
+
 		child->next = child->prev = NULL;
 		DLIST_ADD(winbindd_children, child);
 		child->sock = fdpair[1];
@@ -1324,7 +1358,18 @@ static bool fork_domain_child(struct winbindd_child *child)
 	state.sock = fdpair[0];
 	close(fdpair[1]);
 
-	if (!winbindd_reinit_after_fork(child->logfilename)) {
+	status = winbindd_reinit_after_fork(child, child->logfilename);
+
+	nwritten = write(state.sock, &status, sizeof(status));
+	if (nwritten != sizeof(status)) {
+		DEBUG(1, ("fork_domain_child: Could not write status: "
+			  "nwritten=%d, error=%s\n", (int)nwritten,
+			  strerror(errno)));
+		_exit(0);
+	}
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(1, ("winbindd_reinit_after_fork failed: %s\n",
+			  nt_errstr(status)));
 		_exit(0);
 	}
 
@@ -1424,7 +1469,6 @@ static bool fork_domain_child(struct winbindd_child *child)
 		TALLOC_CTX *frame = talloc_stackframe();
 		struct iovec iov[2];
 		int iov_count;
-		NTSTATUS status;
 
 		if (run_events_poll(winbind_event_context(), 0, NULL, 0)) {
 			TALLOC_FREE(frame);
@@ -1439,7 +1483,7 @@ static bool fork_domain_child(struct winbindd_child *child)
 			child->domain->startup = False;
 		}
 
-		pfds = TALLOC_ZERO_P(talloc_tos(), struct pollfd);
+		pfds = talloc_zero(talloc_tos(), struct pollfd);
 		if (pfds == NULL) {
 			DEBUG(1, ("talloc failed\n"));
 			_exit(1);
diff --git a/source3/winbindd/winbindd_dual_ndr.c b/source3/winbindd/winbindd_dual_ndr.c
index 003d3cded7a..f5f143f6126 100644
--- a/source3/winbindd/winbindd_dual_ndr.c
+++ b/source3/winbindd/winbindd_dual_ndr.c
@@ -29,8 +29,8 @@
 #include "includes.h"
 #include "winbindd/winbindd.h"
 #include "winbindd/winbindd_proto.h"
-#include "librpc/gen_ndr/srv_wbint.h"
 #include "ntdomain.h"
+#include "librpc/gen_ndr/srv_wbint.h"
 
 struct wbint_bh_state {
 	struct winbindd_domain *domain;
@@ -144,7 +144,7 @@ static void wbint_bh_raw_call_done(struct tevent_req *subreq)
 		state->response->extra_data.data,
 		state->response->length - sizeof(struct winbindd_response));
 	if (state->response->extra_data.data && !state->out_data.data) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return;
 	}
 
diff --git a/source3/winbindd/winbindd_dual_srv.c b/source3/winbindd/winbindd_dual_srv.c
index 230edae9080..c2510bc84d9 100644
--- a/source3/winbindd/winbindd_dual_srv.c
+++ b/source3/winbindd/winbindd_dual_srv.c
@@ -24,11 +24,11 @@
 #include "winbindd/winbindd.h"
 #include "winbindd/winbindd_proto.h"
 #include "rpc_client/cli_pipe.h"
+#include "ntdomain.h"
 #include "librpc/gen_ndr/srv_wbint.h"
 #include "../librpc/gen_ndr/ndr_netlogon_c.h"
 #include "idmap.h"
 #include "../libcli/security/security.h"
-#include "ntdomain.h"
 
 void _wbint_Ping(struct pipes_struct *p, struct wbint_Ping *r)
 {
@@ -148,22 +148,22 @@ NTSTATUS _wbint_Sids2UnixIDs(struct pipes_struct *p,
 			}
 		}
 
-		ids = TALLOC_REALLOC_ARRAY(talloc_tos(), ids,
+		ids = talloc_realloc(talloc_tos(), ids,
 					   struct id_map, num_ids);
 		if (ids == NULL) {
 			goto nomem;
 		}
-		id_ptrs = TALLOC_REALLOC_ARRAY(talloc_tos(), id_ptrs,
+		id_ptrs = talloc_realloc(talloc_tos(), id_ptrs,
 					       struct id_map *, num_ids+1);
 		if (id_ptrs == NULL) {
 			goto nomem;
 		}
-		id_idx = TALLOC_REALLOC_ARRAY(talloc_tos(), id_idx,
+		id_idx = talloc_realloc(talloc_tos(), id_idx,
 					      uint32_t, num_ids);
 		if (id_idx == NULL) {
 			goto nomem;
 		}
-		sids = TALLOC_REALLOC_ARRAY(talloc_tos(), sids,
+		sids = talloc_realloc(talloc_tos(), sids,
 					    struct dom_sid, num_ids);
 		if (sids == NULL) {
 			goto nomem;
diff --git a/source3/winbindd/winbindd_getdcname.c b/source3/winbindd/winbindd_getdcname.c
index 1c5f69e50d7..cbf7f22eb88 100644
--- a/source3/winbindd/winbindd_getdcname.c
+++ b/source3/winbindd/winbindd_getdcname.c
@@ -65,8 +65,7 @@ static void winbindd_getdcname_done(struct tevent_req *subreq)
 
 	status = wb_dsgetdcname_recv(subreq, state, &state->dcinfo);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getgrgid.c b/source3/winbindd/winbindd_getgrgid.c
index 80db3090607..5edecffcb2b 100644
--- a/source3/winbindd/winbindd_getgrgid.c
+++ b/source3/winbindd/winbindd_getgrgid.c
@@ -68,8 +68,7 @@ static void winbindd_getgrgid_gid2sid_done(struct tevent_req *subreq)
 
 	status = wb_gid2sid_recv(subreq, &state->sid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -92,8 +91,7 @@ static void winbindd_getgrgid_done(struct tevent_req *subreq)
 	status = wb_getgrsid_recv(subreq, state, &state->domname, &state->name,
 				  &state->gid, &state->members);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getgrnam.c b/source3/winbindd/winbindd_getgrnam.c
index 3ca1aa61115..9460ff44ad1 100644
--- a/source3/winbindd/winbindd_getgrnam.c
+++ b/source3/winbindd/winbindd_getgrnam.c
@@ -97,8 +97,7 @@ static void winbindd_getgrnam_lookupsid_done(struct tevent_req *subreq)
 
 	status = wb_lookupname_recv(subreq, &state->sid, &type);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -127,8 +126,7 @@ static void winbindd_getgrnam_done(struct tevent_req *subreq)
 	status = wb_getgrsid_recv(subreq, state, &state->domname, &state->name,
 				  &state->gid, &state->members);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getgroups.c b/source3/winbindd/winbindd_getgroups.c
index c9b0f195ce5..a42986f15f5 100644
--- a/source3/winbindd/winbindd_getgroups.c
+++ b/source3/winbindd/winbindd_getgroups.c
@@ -96,8 +96,7 @@ static void winbindd_getgroups_lookupname_done(struct tevent_req *subreq)
 
 	status = wb_lookupname_recv(subreq, &state->sid, &state->type);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -119,8 +118,7 @@ static void winbindd_getgroups_gettoken_done(struct tevent_req *subreq)
 	status = wb_gettoken_recv(subreq, state, &state->num_sids,
 				  &state->sids);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
diff --git a/source3/winbindd/winbindd_getpwent.c b/source3/winbindd/winbindd_getpwent.c
index ef905b48bb2..3c035eac8a6 100644
--- a/source3/winbindd/winbindd_getpwent.c
+++ b/source3/winbindd/winbindd_getpwent.c
@@ -97,8 +97,7 @@ static void winbindd_getpwent_done(struct tevent_req *subreq)
 		tevent_req_done(req);
 		return;
 	}
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	state->num_users += 1;
diff --git a/source3/winbindd/winbindd_getpwnam.c b/source3/winbindd/winbindd_getpwnam.c
index bc771b2e41b..f2c93362ca8 100644
--- a/source3/winbindd/winbindd_getpwnam.c
+++ b/source3/winbindd/winbindd_getpwnam.c
@@ -100,8 +100,7 @@ static void winbindd_getpwnam_lookupname_done(struct tevent_req *subreq)
 
 	status = wb_lookupname_recv(subreq, &state->sid, &state->type);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -120,8 +119,7 @@ static void winbindd_getpwnam_done(struct tevent_req *subreq)
 
 	status = wb_getpwsid_recv(subreq);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getpwsid.c b/source3/winbindd/winbindd_getpwsid.c
index efea3fd2b36..52481700756 100644
--- a/source3/winbindd/winbindd_getpwsid.c
+++ b/source3/winbindd/winbindd_getpwsid.c
@@ -70,8 +70,7 @@ static void winbindd_getpwsid_done(struct tevent_req *subreq)
 
 	status = wb_getpwsid_recv(subreq);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getpwuid.c b/source3/winbindd/winbindd_getpwuid.c
index d47c0852143..30f3a04efdd 100644
--- a/source3/winbindd/winbindd_getpwuid.c
+++ b/source3/winbindd/winbindd_getpwuid.c
@@ -65,8 +65,7 @@ static void winbindd_getpwuid_uid2sid_done(struct tevent_req *subreq)
 
 	status = wb_uid2sid_recv(subreq, &state->sid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
@@ -85,8 +84,7 @@ static void winbindd_getpwuid_done(struct tevent_req *subreq)
 
 	status = wb_getpwsid_recv(subreq);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getsidaliases.c b/source3/winbindd/winbindd_getsidaliases.c
index f01dac9c4ce..e40a51dbd6d 100644
--- a/source3/winbindd/winbindd_getsidaliases.c
+++ b/source3/winbindd/winbindd_getsidaliases.c
@@ -112,8 +112,7 @@ static void winbindd_getsidaliases_done(struct tevent_req *subreq)
 	status = wb_lookupuseraliases_recv(subreq, state, &state->num_aliases,
 					   &state->aliases);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getuserdomgroups.c b/source3/winbindd/winbindd_getuserdomgroups.c
index 9fdff4fc66f..55094bb99a2 100644
--- a/source3/winbindd/winbindd_getuserdomgroups.c
+++ b/source3/winbindd/winbindd_getuserdomgroups.c
@@ -83,8 +83,7 @@ static void winbindd_getuserdomgroups_done(struct tevent_req *subreq)
 	status = wb_lookupusergroups_recv(subreq, state, &state->num_sids,
 					  &state->sids);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_getusersids.c b/source3/winbindd/winbindd_getusersids.c
index 157c2ab644b..6b5510ad5a5 100644
--- a/source3/winbindd/winbindd_getusersids.c
+++ b/source3/winbindd/winbindd_getusersids.c
@@ -74,8 +74,7 @@ static void winbindd_getusersids_done(struct tevent_req *subreq)
 	status = wb_gettoken_recv(subreq, state, &state->num_sids,
 				  &state->sids);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_gid_to_sid.c b/source3/winbindd/winbindd_gid_to_sid.c
index f8494d2df23..4a378d3d26c 100644
--- a/source3/winbindd/winbindd_gid_to_sid.c
+++ b/source3/winbindd/winbindd_gid_to_sid.c
@@ -63,8 +63,7 @@ static void winbindd_gid_to_sid_done(struct tevent_req *subreq)
 
 	status = wb_gid2sid_recv(subreq, &state->sid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_group.c b/source3/winbindd/winbindd_group.c
index a985fa254fe..1e4ad5fa8ac 100644
--- a/source3/winbindd/winbindd_group.c
+++ b/source3/winbindd/winbindd_group.c
@@ -60,8 +60,8 @@ bool fill_grent(TALLOC_CTX *mem_ctx, struct winbindd_gr *gr,
 
 	/* Group name and password */
 
-	safe_strcpy(gr->gr_name, full_group_name, sizeof(gr->gr_name) - 1);
-	safe_strcpy(gr->gr_passwd, "x", sizeof(gr->gr_passwd) - 1);
+	strlcpy(gr->gr_name, full_group_name, sizeof(gr->gr_name));
+	strlcpy(gr->gr_passwd, "x", sizeof(gr->gr_passwd));
 
 	return True;
 }
diff --git a/source3/winbindd/winbindd_lookupname.c b/source3/winbindd/winbindd_lookupname.c
index c1d0bf04c1b..1be29fd85c8 100644
--- a/source3/winbindd/winbindd_lookupname.c
+++ b/source3/winbindd/winbindd_lookupname.c
@@ -86,8 +86,7 @@ static void winbindd_lookupname_done(struct tevent_req *subreq)
 
 	status = wb_lookupname_recv(subreq, &state->sid, &state->type);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_lookupsids.c b/source3/winbindd/winbindd_lookupsids.c
index aeaf21dfb53..4a1b83fd34a 100644
--- a/source3/winbindd/winbindd_lookupsids.c
+++ b/source3/winbindd/winbindd_lookupsids.c
@@ -80,8 +80,7 @@ static void winbindd_lookupsids_done(struct tevent_req *subreq)
 	status = wb_lookupsids_recv(subreq, state, &state->domains,
 				    &state->names);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_misc.c b/source3/winbindd/winbindd_misc.c
index 7d2516770bc..3fb14368209 100644
--- a/source3/winbindd/winbindd_misc.c
+++ b/source3/winbindd/winbindd_misc.c
@@ -404,7 +404,7 @@ void winbindd_netbios_name(struct winbindd_cli_state *state)
 	DEBUG(3, ("[%5lu]: request netbios name\n",
 		  (unsigned long)state->pid));
 
-	fstrcpy(state->response->data.netbios_name, global_myname());
+	fstrcpy(state->response->data.netbios_name, lp_netbios_name());
 	request_ok(state);
 }
 
diff --git a/source3/winbindd/winbindd_msrpc.c b/source3/winbindd/winbindd_msrpc.c
index 8986598daa0..9ef0d87f5ad 100644
--- a/source3/winbindd/winbindd_msrpc.c
+++ b/source3/winbindd/winbindd_msrpc.c
@@ -342,7 +342,7 @@ static NTSTATUS msrpc_rids_to_names(struct winbindd_domain *domain,
 	DEBUG(3, ("msrpc_rids_to_names: domain %s\n", domain->name ));
 
 	if (num_rids) {
-		sids = TALLOC_ARRAY(mem_ctx, struct dom_sid, num_rids);
+		sids = talloc_array(mem_ctx, struct dom_sid, num_rids);
 		if (sids == NULL) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -400,7 +400,7 @@ static NTSTATUS msrpc_query_user(struct winbindd_domain *domain,
 {
 	struct rpc_pipe_client *samr_pipe;
 	struct policy_handle dom_pol;
-	struct netr_SamInfo3 *user;
+	struct netr_SamInfo3 *user = NULL;
 	TALLOC_CTX *tmp_ctx;
 	NTSTATUS status;
 
@@ -418,7 +418,9 @@ static NTSTATUS msrpc_query_user(struct winbindd_domain *domain,
 	}
 
 	/* try netsamlogon cache first */
-	user = netsamlogon_cache_get(tmp_ctx, user_sid);
+	if (winbindd_use_cache()) {
+		user = netsamlogon_cache_get(tmp_ctx, user_sid);
+	}
 	if (user != NULL) {
 		DEBUG(5,("msrpc_query_user: Cache lookup succeeded for %s\n",
 			sid_string_dbg(user_sid)));
@@ -699,9 +701,9 @@ static NTSTATUS msrpc_lookup_groupmem(struct winbindd_domain *domain,
 
 #define MAX_LOOKUP_RIDS 900
 
-        *names = TALLOC_ZERO_ARRAY(mem_ctx, char *, *num_names);
-        *name_types = TALLOC_ZERO_ARRAY(mem_ctx, uint32, *num_names);
-        *sid_mem = TALLOC_ZERO_ARRAY(mem_ctx, struct dom_sid, *num_names);
+        *names = talloc_zero_array(mem_ctx, char *, *num_names);
+        *name_types = talloc_zero_array(mem_ctx, uint32, *num_names);
+        *sid_mem = talloc_zero_array(mem_ctx, struct dom_sid, *num_names);
 
 	for (j=0;j<(*num_names);j++)
 		sid_compose(&(*sid_mem)[j], &domain->sid, rid_mem[j]);
@@ -762,7 +764,7 @@ static NTSTATUS msrpc_lookup_groupmem(struct winbindd_domain *domain,
 
 #include <ldap.h>
 
-static int get_ldap_seq(const char *server, int port, uint32 *seq)
+static int get_ldap_seq(const char *server, struct sockaddr_storage *ss, int port, uint32 *seq)
 {
 	int ret = -1;
 	struct timeval to;
@@ -778,7 +780,7 @@ static int get_ldap_seq(const char *server, int port, uint32 *seq)
 	 * search timeout doesn't seem to apply to doing an open as well. JRA.
 	 */
 
-	ldp = ldap_open_with_timeout(server, port, lp_ldap_timeout());
+	ldp = ldap_open_with_timeout(server, ss, port, lp_ldap_timeout());
 	if (ldp == NULL)
 		return -1;
 
@@ -787,7 +789,7 @@ static int get_ldap_seq(const char *server, int port, uint32 *seq)
 	to.tv_usec = 0;
 
 	if (ldap_search_st(ldp, "", LDAP_SCOPE_BASE, "(objectclass=*)",
-			   CONST_DISCARD(char **, attrs), 0, &to, &res))
+			   discard_const_p(char *, attrs), 0, &to, &res))
 		goto done;
 
 	if (ldap_count_entries(ldp, res) != 1)
@@ -822,7 +824,7 @@ static int get_ldap_sequence_number(struct winbindd_domain *domain, uint32 *seq)
 	char addr[INET6_ADDRSTRLEN];
 
 	print_sockaddr(addr, sizeof(addr), &domain->dcaddr);
-	if ((ret = get_ldap_seq(addr, LDAP_PORT, seq)) == 0) {
+	if ((ret = get_ldap_seq(addr, &domain->dcaddr, LDAP_PORT, seq)) == 0) {
 		DEBUG(3, ("get_ldap_sequence_number: Retrieved sequence "
 			  "number for Domain (%s) from DC (%s)\n",
 			domain->name, addr));
@@ -1057,14 +1059,15 @@ static NTSTATUS msrpc_password_policy(struct winbindd_domain *domain,
 	return status;
 }
 
-typedef NTSTATUS (*lookup_sids_fn_t)(struct rpc_pipe_client *cli,
+typedef NTSTATUS (*lookup_sids_fn_t)(struct dcerpc_binding_handle *h,
 				     TALLOC_CTX *mem_ctx,
 				     struct policy_handle *pol,
 				     int num_sids,
 				     const struct dom_sid *sids,
 				     char ***pdomains,
 				     char ***pnames,
-				     enum lsa_SidType **ptypes);
+				     enum lsa_SidType **ptypes,
+				     NTSTATUS *result);
 
 NTSTATUS winbindd_lookup_sids(TALLOC_CTX *mem_ctx,
 			      struct winbindd_domain *domain,
@@ -1075,15 +1078,17 @@ NTSTATUS winbindd_lookup_sids(TALLOC_CTX *mem_ctx,
 			      enum lsa_SidType **types)
 {
 	NTSTATUS status;
+	NTSTATUS result;
 	struct rpc_pipe_client *cli = NULL;
+	struct dcerpc_binding_handle *b = NULL;
 	struct policy_handle lsa_policy;
 	unsigned int orig_timeout;
-	lookup_sids_fn_t lookup_sids_fn = rpccli_lsa_lookup_sids;
+	lookup_sids_fn_t lookup_sids_fn = dcerpc_lsa_lookup_sids;
 
 	if (domain->can_do_ncacn_ip_tcp) {
 		status = cm_connect_lsa_tcp(domain, mem_ctx, &cli);
 		if (NT_STATUS_IS_OK(status)) {
-			lookup_sids_fn = rpccli_lsa_lookup_sids3;
+			lookup_sids_fn = dcerpc_lsa_lookup_sids3;
 			goto lookup;
 		}
 		domain->can_do_ncacn_ip_tcp = false;
@@ -1095,27 +1100,30 @@ NTSTATUS winbindd_lookup_sids(TALLOC_CTX *mem_ctx,
 	}
 
  lookup:
+	b = cli->binding_handle;
+
 	/*
 	 * This call can take a long time
 	 * allow the server to time out.
 	 * 35 seconds should do it.
 	 */
-	orig_timeout = rpccli_set_timeout(cli, 35000);
+	orig_timeout = dcerpc_binding_handle_set_timeout(b, 35000);
 
-	status = lookup_sids_fn(cli,
+	status = lookup_sids_fn(b,
 				mem_ctx,
 				&lsa_policy,
 				num_sids,
 				sids,
 				domains,
 				names,
-				types);
+				types,
+				&result);
 
 	/* And restore our original timeout. */
-	rpccli_set_timeout(cli, orig_timeout);
+	dcerpc_binding_handle_set_timeout(b, orig_timeout);
 
-	if (NT_STATUS_V(status) == DCERPC_FAULT_ACCESS_DENIED ||
-	    NT_STATUS_V(status) == DCERPC_FAULT_SEC_PKG_ERROR) {
+	if (NT_STATUS_EQUAL(status, NT_STATUS_ACCESS_DENIED) ||
+	    NT_STATUS_EQUAL(status, NT_STATUS_RPC_SEC_PKG_ERROR)) {
 		/*
 		 * This can happen if the schannel key is not
 		 * valid anymore, we need to invalidate the
@@ -1130,18 +1138,23 @@ NTSTATUS winbindd_lookup_sids(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	return status;
+	if (!NT_STATUS_IS_OK(result)) {
+		return result;
+	}
+
+	return NT_STATUS_OK;
 }
 
-typedef NTSTATUS (*lookup_names_fn_t)(struct rpc_pipe_client *cli,
+typedef NTSTATUS (*lookup_names_fn_t)(struct dcerpc_binding_handle *h,
 				      TALLOC_CTX *mem_ctx,
 				      struct policy_handle *pol,
-				      int num_names,
+				      uint32_t num_names,
 				      const char **names,
 				      const char ***dom_names,
-				      int level,
+				      enum lsa_LookupNamesLevel level,
 				      struct dom_sid **sids,
-				      enum lsa_SidType **types);
+				      enum lsa_SidType **types,
+				      NTSTATUS *result);
 
 NTSTATUS winbindd_lookup_names(TALLOC_CTX *mem_ctx,
 			       struct winbindd_domain *domain,
@@ -1152,15 +1165,17 @@ NTSTATUS winbindd_lookup_names(TALLOC_CTX *mem_ctx,
 			       enum lsa_SidType **types)
 {
 	NTSTATUS status;
+	NTSTATUS result;
 	struct rpc_pipe_client *cli = NULL;
+	struct dcerpc_binding_handle *b = NULL;
 	struct policy_handle lsa_policy;
 	unsigned int orig_timeout = 0;
-	lookup_names_fn_t lookup_names_fn = rpccli_lsa_lookup_names;
+	lookup_names_fn_t lookup_names_fn = dcerpc_lsa_lookup_names;
 
 	if (domain->can_do_ncacn_ip_tcp) {
 		status = cm_connect_lsa_tcp(domain, mem_ctx, &cli);
 		if (NT_STATUS_IS_OK(status)) {
-			lookup_names_fn = rpccli_lsa_lookup_names4;
+			lookup_names_fn = dcerpc_lsa_lookup_names4;
 			goto lookup;
 		}
 		domain->can_do_ncacn_ip_tcp = false;
@@ -1172,15 +1187,16 @@ NTSTATUS winbindd_lookup_names(TALLOC_CTX *mem_ctx,
 	}
 
  lookup:
+	b = cli->binding_handle;
 
 	/*
 	 * This call can take a long time
 	 * allow the server to time out.
 	 * 35 seconds should do it.
 	 */
-	orig_timeout = rpccli_set_timeout(cli, 35000);
+	orig_timeout = dcerpc_binding_handle_set_timeout(b, 35000);
 
-	status = lookup_names_fn(cli,
+	status = lookup_names_fn(b,
 				 mem_ctx,
 				 &lsa_policy,
 				 num_names,
@@ -1188,13 +1204,14 @@ NTSTATUS winbindd_lookup_names(TALLOC_CTX *mem_ctx,
 				 domains,
 				 1,
 				 sids,
-				 types);
+				 types,
+				 &result);
 
 	/* And restore our original timeout. */
-	rpccli_set_timeout(cli, orig_timeout);
+	dcerpc_binding_handle_set_timeout(b, orig_timeout);
 
-	if (NT_STATUS_V(status) == DCERPC_FAULT_ACCESS_DENIED ||
-	    NT_STATUS_V(status) == DCERPC_FAULT_SEC_PKG_ERROR) {
+	if (NT_STATUS_EQUAL(status, NT_STATUS_ACCESS_DENIED) ||
+	    NT_STATUS_EQUAL(status, NT_STATUS_RPC_SEC_PKG_ERROR)) {
 		/*
 		 * This can happen if the schannel key is not
 		 * valid anymore, we need to invalidate the
@@ -1209,7 +1226,11 @@ NTSTATUS winbindd_lookup_names(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	return status;
+	if (!NT_STATUS_IS_OK(result)) {
+		return result;
+	}
+
+	return NT_STATUS_OK;
 }
 
 /* the rpc backend methods are exposed via this structure */
diff --git a/source3/winbindd/winbindd_ndr.c b/source3/winbindd/winbindd_ndr.c
index 2f26d8d9810..b1fd6d715f5 100644
--- a/source3/winbindd/winbindd_ndr.c
+++ b/source3/winbindd/winbindd_ndr.c
@@ -21,7 +21,7 @@
 #include "winbindd.h"
 #include "../librpc/gen_ndr/ndr_netlogon.h"
 #include "../librpc/gen_ndr/ndr_security.h"
-#include "librpc/ndr/util.h"
+#include "../librpc/ndr/libndr.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
diff --git a/source3/winbindd/winbindd_pam.c b/source3/winbindd/winbindd_pam.c
index 76e06d3735a..910e42971a7 100644
--- a/source3/winbindd/winbindd_pam.c
+++ b/source3/winbindd/winbindd_pam.c
@@ -37,7 +37,6 @@
 #include "../librpc/gen_ndr/krb5pac.h"
 #include "passdb/machine_sid.h"
 #include "auth.h"
-#include "ntdomain.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
@@ -386,9 +385,9 @@ static void fill_in_password_policy(struct winbindd_response *r,
 	r->data.auth.policy.password_properties =
 		p->password_properties;
 	r->data.auth.policy.expire	=
-		nt_time_to_unix_abs((NTTIME *)&(p->max_password_age));
+		nt_time_to_unix_abs((const NTTIME *)&(p->max_password_age));
 	r->data.auth.policy.min_passwordage =
-		nt_time_to_unix_abs((NTTIME *)&(p->min_password_age));
+		nt_time_to_unix_abs((const NTTIME *)&(p->min_password_age));
 }
 
 static NTSTATUS fillup_password_policy(struct winbindd_domain *domain,
@@ -994,7 +993,10 @@ static NTSTATUS winbindd_dual_pam_auth_cached(struct winbindd_domain *domain,
 
 	}
 
-	/* User does *NOT* know the correct password, modify info3 accordingly */
+	/* User does *NOT* know the correct password, modify info3 accordingly, but only if online */
+	if (domain->online == false) {
+		goto failed;
+	}
 
 	/* failure of this is not critical */
 	result = get_max_bad_attempts_from_lockout_policy(domain, state->mem_ctx, &max_allowed_bad_attempts);
@@ -1116,7 +1118,7 @@ static NTSTATUS winbindd_dual_auth_passdb(TALLOC_CTX *mem_ctx,
 	NTSTATUS status;
 
 	status = make_user_info(&user_info, user, user, domain, domain,
-				global_myname(), lm_resp, nt_resp, NULL, NULL,
+				lp_netbios_name(), lm_resp, nt_resp, NULL, NULL,
 				NULL, AUTH_PASSWORD_RESPONSE);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(10, ("make_user_info failed: %s\n", nt_errstr(status)));
@@ -1248,7 +1250,7 @@ static NTSTATUS winbind_samlogon_retry_loop(struct winbindd_domain *domain,
 					info3);
 		}
 
-		if ((NT_STATUS_V(result) == DCERPC_FAULT_OP_RNG_ERROR)
+		if (NT_STATUS_EQUAL(result, NT_STATUS_RPC_PROCNUM_OUT_OF_RANGE)
 		    && domain->can_do_samlogon_ex) {
 			DEBUG(3, ("Got a DC that can not do NetSamLogonEx, "
 				  "retrying with NetSamLogon\n"));
@@ -1346,7 +1348,7 @@ static NTSTATUS winbindd_dual_pam_auth_samlogon(TALLOC_CTX *mem_ctx,
 		   'workstation' passed to the actual SamLogon call.
 		*/
 		names_blob = NTLMv2_generate_names_blob(
-			mem_ctx, global_myname(), lp_workgroup());
+			mem_ctx, lp_netbios_name(), lp_workgroup());
 
 		if (!SMBNTLMv2encrypt(mem_ctx, name_user, name_domain,
 				      pass,
@@ -1384,7 +1386,7 @@ static NTSTATUS winbindd_dual_pam_auth_samlogon(TALLOC_CTX *mem_ctx,
 					     domain->dcname,
 					     name_user,
 					     name_domain,
-					     global_myname(),
+					     lp_netbios_name(),
 					     chal,
 					     lm_resp,
 					     nt_resp,
@@ -1515,8 +1517,8 @@ enum winbindd_result winbindd_dual_pam_auth(struct winbindd_domain *domain,
 		fstr_sprintf( domain_user, "%s%c%s", name_domain,
 			*lp_winbind_separator(),
 			name_user );
-		safe_strcpy( state->request->data.auth.user, domain_user,
-			     sizeof(state->request->data.auth.user)-1 );
+		strlcpy( state->request->data.auth.user, domain_user,
+			     sizeof(state->request->data.auth.user));
 	}
 
 	if (!domain->online) {
@@ -1938,10 +1940,10 @@ enum winbindd_result winbindd_dual_pam_chauthtok(struct winbindd_domain *contact
 	 * short to comply with the samr_ChangePasswordUser3 idl - gd */
 
 	/* only fallback when the chgpasswd_user3 call is not supported */
-	if ((NT_STATUS_EQUAL(result, NT_STATUS(DCERPC_FAULT_OP_RNG_ERROR))) ||
-		   (NT_STATUS_EQUAL(result, NT_STATUS_NOT_SUPPORTED)) ||
-		   (NT_STATUS_EQUAL(result, NT_STATUS_BUFFER_TOO_SMALL)) ||
-		   (NT_STATUS_EQUAL(result, NT_STATUS_NOT_IMPLEMENTED))) {
+	if (NT_STATUS_EQUAL(result, NT_STATUS_RPC_PROCNUM_OUT_OF_RANGE) ||
+	    NT_STATUS_EQUAL(result, NT_STATUS_NOT_SUPPORTED) ||
+	    NT_STATUS_EQUAL(result, NT_STATUS_BUFFER_TOO_SMALL) ||
+	    NT_STATUS_EQUAL(result, NT_STATUS_NOT_IMPLEMENTED)) {
 
 		DEBUG(10,("Password change with chgpasswd_user3 failed with: %s, retrying chgpasswd_user2\n",
 			nt_errstr(result)));
@@ -2136,7 +2138,7 @@ enum winbindd_result winbindd_dual_pam_chng_pswd_auth_crap(struct winbindd_domai
 	}
 
 	if (!*domain && lp_winbind_use_default_domain()) {
-		fstrcpy(domain,(char *)lp_workgroup());
+		fstrcpy(domain,lp_workgroup());
 	}
 
 	if(!*user) {
diff --git a/source3/winbindd/winbindd_pam_auth_crap.c b/source3/winbindd/winbindd_pam_auth_crap.c
index 186e510aa3e..2fb5111510c 100644
--- a/source3/winbindd/winbindd_pam_auth_crap.c
+++ b/source3/winbindd/winbindd_pam_auth_crap.c
@@ -74,7 +74,7 @@ struct tevent_req *winbindd_pam_auth_crap_send(
 	}
 
 	if (request->data.auth_crap.workstation[0] == '\0') {
-		fstrcpy(request->data.auth_crap.workstation, global_myname());
+		fstrcpy(request->data.auth_crap.workstation, lp_netbios_name());
 	}
 
 	subreq = wb_domain_request_send(state, winbind_event_context(), domain,
diff --git a/source3/winbindd/winbindd_proto.h b/source3/winbindd/winbindd_proto.h
index 5b48a99a8fb..ce66964ace4 100644
--- a/source3/winbindd/winbindd_proto.h
+++ b/source3/winbindd/winbindd_proto.h
@@ -291,7 +291,8 @@ void winbind_msg_ip_dropped_parent(struct messaging_context *msg_ctx,
 				   uint32_t msg_type,
 				   struct server_id server_id,
 				   DATA_BLOB *data);
-bool winbindd_reinit_after_fork(const char *logfilename);
+NTSTATUS winbindd_reinit_after_fork(const struct winbindd_child *myself,
+				    const char *logfilename);
 struct winbindd_domain *wb_child_domain(void);
 
 /* The following definitions come from winbindd/winbindd_group.c  */
@@ -391,7 +392,6 @@ struct winbindd_domain *find_lookup_domain_from_name(const char *domain_name);
 bool parse_domain_user(const char *domuser, fstring domain, fstring user);
 bool parse_domain_user_talloc(TALLOC_CTX *mem_ctx, const char *domuser,
 			      char **domain, char **user);
-void parse_add_domuser(void *buf, char *domuser, int *len);
 bool canonicalize_username(fstring username_inout, fstring domain, fstring user);
 void fill_domain_username(fstring name, const char *domain, const char *user, bool can_assume);
 char *fill_domain_username_talloc(TALLOC_CTX *ctx,
@@ -436,7 +436,6 @@ bool parse_sidlist(TALLOC_CTX *mem_ctx, const char *sidstr,
 
 /* The following definitions come from winbindd/winbindd_wins.c  */
 
-void winbindd_wins_byip(struct winbindd_cli_state *state);
 void winbindd_wins_byname(struct winbindd_cli_state *state);
 
 struct tevent_req *wb_ping_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
@@ -861,6 +860,18 @@ struct tevent_req *winbindd_sids_to_xids_send(TALLOC_CTX *mem_ctx,
 					      struct winbindd_request *request);
 NTSTATUS winbindd_sids_to_xids_recv(struct tevent_req *req,
 				    struct winbindd_response *response);
+struct tevent_req *winbindd_wins_byip_send(TALLOC_CTX *mem_ctx,
+					   struct tevent_context *ev,
+					   struct winbindd_cli_state *cli,
+					   struct winbindd_request *request);
+NTSTATUS winbindd_wins_byip_recv(struct tevent_req *req,
+				 struct winbindd_response *presp);
+struct tevent_req *winbindd_wins_byname_send(TALLOC_CTX *mem_ctx,
+					     struct tevent_context *ev,
+					     struct winbindd_cli_state *cli,
+					     struct winbindd_request *request);
+NTSTATUS winbindd_wins_byname_recv(struct tevent_req *req,
+				   struct winbindd_response *presp);
 
 
 /* The following definitions come from winbindd/winbindd_samr.c  */
diff --git a/source3/winbindd/winbindd_rpc.c b/source3/winbindd/winbindd_rpc.c
index 82599e78782..e9114873242 100644
--- a/source3/winbindd/winbindd_rpc.c
+++ b/source3/winbindd/winbindd_rpc.c
@@ -26,12 +26,9 @@
 #include "includes.h"
 #include "winbindd.h"
 #include "winbindd_rpc.h"
-
 #include "rpc_client/rpc_client.h"
 #include "librpc/gen_ndr/ndr_samr_c.h"
-#include "librpc/gen_ndr/srv_samr.h"
 #include "librpc/gen_ndr/ndr_lsa_c.h"
-#include "librpc/gen_ndr/srv_lsa.h"
 #include "rpc_client/cli_samr.h"
 #include "rpc_client/cli_lsarpc.h"
 #include "../libcli/security/security.h"
@@ -92,7 +89,7 @@ NTSTATUS rpc_query_user_list(TALLOC_CTX *mem_ctx,
 
 		num_info += num_dom_users;
 
-		info = TALLOC_REALLOC_ARRAY(mem_ctx,
+		info = talloc_realloc(mem_ctx,
 					    info,
 					    struct wbint_userinfo,
 					    num_info);
@@ -184,7 +181,7 @@ NTSTATUS rpc_enum_dom_groups(TALLOC_CTX *mem_ctx,
 			}
 		}
 
-		info = TALLOC_REALLOC_ARRAY(mem_ctx,
+		info = talloc_realloc(mem_ctx,
 					    info,
 					    struct wb_acct_info,
 					    num_info + count);
@@ -244,7 +241,7 @@ NTSTATUS rpc_enum_local_groups(TALLOC_CTX *mem_ctx,
 			}
 		}
 
-		info = TALLOC_REALLOC_ARRAY(mem_ctx,
+		info = talloc_realloc(mem_ctx,
 					    info,
 					    struct wb_acct_info,
 					    num_info + count);
@@ -407,7 +404,7 @@ NTSTATUS rpc_rids_to_names(TALLOC_CTX *mem_ctx,
 	NTSTATUS status;
 
 	if (num_rids > 0) {
-		sids = TALLOC_ARRAY(mem_ctx, struct dom_sid, num_rids);
+		sids = talloc_array(mem_ctx, struct dom_sid, num_rids);
 		if (sids == NULL) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -597,7 +594,7 @@ NTSTATUS rpc_lookup_usergroups(TALLOC_CTX *mem_ctx,
 		return result;
 	}
 
-	user_grpsids = TALLOC_ARRAY(mem_ctx, struct dom_sid, num_groups);
+	user_grpsids = talloc_array(mem_ctx, struct dom_sid, num_groups);
 	if (user_grpsids == NULL) {
 		status = NT_STATUS_NO_MEMORY;
 		return status;
@@ -647,7 +644,7 @@ NTSTATUS rpc_lookup_useraliases(TALLOC_CTX *mem_ctx,
 			num_queries, num_query_sids));
 
 		if (num_query_sids) {
-			sid_array.sids = TALLOC_ZERO_ARRAY(mem_ctx, struct lsa_SidPtr, num_query_sids);
+			sid_array.sids = talloc_zero_array(mem_ctx, struct lsa_SidPtr, num_query_sids);
 			if (sid_array.sids == NULL) {
 				return NT_STATUS_NO_MEMORY;
 			}
@@ -842,9 +839,9 @@ NTSTATUS rpc_lookup_groupmem(TALLOC_CTX *mem_ctx,
 	 * Step #2: Convert list of rids into list of usernames.
 	 */
 	if (num_names > 0) {
-		names = TALLOC_ZERO_ARRAY(mem_ctx, char *, num_names);
-		name_types = TALLOC_ZERO_ARRAY(mem_ctx, uint32_t, num_names);
-		sid_mem = TALLOC_ZERO_ARRAY(mem_ctx, struct dom_sid, num_names);
+		names = talloc_zero_array(mem_ctx, char *, num_names);
+		name_types = talloc_zero_array(mem_ctx, uint32_t, num_names);
+		sid_mem = talloc_zero_array(mem_ctx, struct dom_sid, num_names);
 		if (names == NULL || name_types == NULL || sid_mem == NULL) {
 			return NT_STATUS_NO_MEMORY;
 		}
@@ -1069,7 +1066,7 @@ static NTSTATUS rpc_try_lookup_sids3(TALLOC_CTX *mem_ctx,
 	if (NT_STATUS_IS_ERR(result)) {
 		return result;
 	}
-	names = TALLOC_ZERO_P(mem_ctx, struct lsa_TransNameArray);
+	names = talloc_zero(mem_ctx, struct lsa_TransNameArray);
 	if (names == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1114,7 +1111,7 @@ NTSTATUS rpc_lookup_sids(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	names = TALLOC_ZERO_P(mem_ctx, struct lsa_TransNameArray);
+	names = talloc_zero(mem_ctx, struct lsa_TransNameArray);
 	if (names == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
diff --git a/source3/winbindd/winbindd_samr.c b/source3/winbindd/winbindd_samr.c
index f24a14391b6..3b9377f7299 100644
--- a/source3/winbindd/winbindd_samr.c
+++ b/source3/winbindd/winbindd_samr.c
@@ -26,14 +26,11 @@
 #include "includes.h"
 #include "winbindd.h"
 #include "winbindd_rpc.h"
-
 #include "rpc_client/rpc_client.h"
 #include "../librpc/gen_ndr/ndr_samr_c.h"
 #include "rpc_client/cli_samr.h"
-#include "../librpc/gen_ndr/srv_samr.h"
 #include "../librpc/gen_ndr/ndr_lsa_c.h"
 #include "rpc_client/cli_lsarpc.h"
-#include "../librpc/gen_ndr/srv_lsa.h"
 #include "rpc_server/rpc_ncacn_np.h"
 #include "../libcli/security/security.h"
 #include "passdb/machine_sid.h"
diff --git a/source3/winbindd/winbindd_show_sequence.c b/source3/winbindd/winbindd_show_sequence.c
index 447706685e7..8e53781e5af 100644
--- a/source3/winbindd/winbindd_show_sequence.c
+++ b/source3/winbindd/winbindd_show_sequence.c
@@ -97,8 +97,7 @@ static void winbindd_show_sequence_done_one(struct tevent_req *subreq)
 
 	status = wb_seqnum_recv(subreq, &state->seqnum);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
@@ -116,8 +115,7 @@ static void winbindd_show_sequence_done_all(struct tevent_req *subreq)
 				 &state->domains, &state->stati,
 				 &state->seqnums);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_sid_to_gid.c b/source3/winbindd/winbindd_sid_to_gid.c
index b0b0e6e211e..df44ed8d6b6 100644
--- a/source3/winbindd/winbindd_sid_to_gid.c
+++ b/source3/winbindd/winbindd_sid_to_gid.c
@@ -72,8 +72,7 @@ static void winbindd_sid_to_gid_done(struct tevent_req *subreq)
 
 	status = wb_sid2gid_recv(subreq, &state->gid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_sid_to_uid.c b/source3/winbindd/winbindd_sid_to_uid.c
index bf2824f9d8d..9ce564f02e8 100644
--- a/source3/winbindd/winbindd_sid_to_uid.c
+++ b/source3/winbindd/winbindd_sid_to_uid.c
@@ -72,8 +72,7 @@ static void winbindd_sid_to_uid_done(struct tevent_req *subreq)
 
 	status = wb_sid2uid_recv(subreq, &state->uid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_sids_to_xids.c b/source3/winbindd/winbindd_sids_to_xids.c
index 52d473b58f8..d08064fbf38 100644
--- a/source3/winbindd/winbindd_sids_to_xids.c
+++ b/source3/winbindd/winbindd_sids_to_xids.c
@@ -80,12 +80,12 @@ struct tevent_req *winbindd_sids_to_xids_send(TALLOC_CTX *mem_ctx,
 
 	DEBUG(10, ("num_sids: %d\n", (int)state->num_sids));
 
-	state->cached = TALLOC_ZERO_ARRAY(state, struct id_map,
+	state->cached = talloc_zero_array(state, struct id_map,
 					  state->num_sids);
 	if (tevent_req_nomem(state->cached, req)) {
 		return tevent_req_post(req, ev);
 	}
-	state->non_cached = TALLOC_ARRAY(state, struct dom_sid,
+	state->non_cached = talloc_array(state, struct dom_sid,
 					 state->num_sids);
 	if (tevent_req_nomem(state->non_cached, req)) {
 		return tevent_req_post(req, ev);
@@ -171,13 +171,12 @@ static void winbindd_sids_to_xids_lookupsids_done(struct tevent_req *subreq)
 	status = wb_lookupsids_recv(subreq, state, &state->domains,
 				    &state->names);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
 	state->ids.num_ids = state->num_non_cached;
-	state->ids.ids = TALLOC_ARRAY(state, struct wbint_TransID,
+	state->ids.ids = talloc_array(state, struct wbint_TransID,
 				      state->num_non_cached);
 	if (tevent_req_nomem(state->ids.ids, req)) {
 		return;
@@ -202,7 +201,7 @@ static void winbindd_sids_to_xids_lookupsids_done(struct tevent_req *subreq)
 			break;
 		};
 		t->domain_index = n->sid_index;
-		sid_peek_rid(&state->sids[i], &t->rid);
+		sid_peek_rid(&state->non_cached[i], &t->rid);
 		t->unix_id = (uint64_t)-1;
 	}
 
@@ -269,6 +268,9 @@ NTSTATUS winbindd_sids_to_xids_recv(struct tevent_req *req,
 			}
 		} else {
 			unix_id = state->ids.ids[num_non_cached].unix_id;
+			if (unix_id == -1) {
+				found = false;
+			}
 			switch(state->ids.ids[num_non_cached].type) {
 			case WBC_ID_TYPE_UID:
 				type = 'U';
diff --git a/source3/winbindd/winbindd_uid_to_sid.c b/source3/winbindd/winbindd_uid_to_sid.c
index 467a1af96a3..653a96f72ae 100644
--- a/source3/winbindd/winbindd_uid_to_sid.c
+++ b/source3/winbindd/winbindd_uid_to_sid.c
@@ -63,8 +63,7 @@ static void winbindd_uid_to_sid_done(struct tevent_req *subreq)
 
 	status = wb_uid2sid_recv(subreq, &state->sid);
 	TALLOC_FREE(subreq);
-	if (!NT_STATUS_IS_OK(status)) {
-		tevent_req_nterror(req, status);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 	tevent_req_done(req);
diff --git a/source3/winbindd/winbindd_util.c b/source3/winbindd/winbindd_util.c
index 9fbc778e048..7dff138fc13 100644
--- a/source3/winbindd/winbindd_util.c
+++ b/source3/winbindd/winbindd_util.c
@@ -261,7 +261,7 @@ static void add_trusted_domains( struct winbindd_domain *domain )
 	struct trustdom_state *state;
 	struct tevent_req *req;
 
-	state = TALLOC_ZERO_P(NULL, struct trustdom_state);
+	state = talloc_zero(NULL, struct trustdom_state);
 	if (state == NULL) {
 		DEBUG(0, ("talloc failed\n"));
 		return;
@@ -894,31 +894,6 @@ bool parse_domain_user_talloc(TALLOC_CTX *mem_ctx, const char *domuser,
 	return ((*domain != NULL) && (*user != NULL));
 }
 
-/* add a domain user name to a buffer */
-void parse_add_domuser(void *buf, char *domuser, int *len)
-{
-	fstring domain;
-	char *p, *user;
-
-	user = domuser;
-	p = strchr(domuser, *lp_winbind_separator());
-
-	if (p) {
-
-		fstrcpy(domain, domuser);
-		domain[PTR_DIFF(p, domuser)] = 0;
-		p++;
-
-		if (assume_domain(domain)) {
-
-			user = p;
-			*len -= (PTR_DIFF(p, domuser));
-		}
-	}
-
-	safe_strcpy((char *)buf, user, *len);
-}
-
 /* Ensure an incoming username from NSS is fully qualified. Replace the
    incoming fstring with DOMAIN <separator> user. Returns the same
    values as parse_domain_user() but also replaces the incoming username.
diff --git a/source3/winbindd/winbindd_wins.c b/source3/winbindd/winbindd_wins.c
deleted file mode 100644
index e1beb1d80bf..00000000000
--- a/source3/winbindd/winbindd_wins.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/* 
-   Unix SMB/CIFS implementation.
-
-   Winbind daemon - WINS related functions
-
-   Copyright (C) Andrew Tridgell 1999
-   Copyright (C) Herb Lewis 2002
-   
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "winbindd.h"
-#include "libsmb/nmblib.h"
-
-#undef DBGC_CLASS
-#define DBGC_CLASS DBGC_WINBIND
-
-static struct node_status *lookup_byaddr_backend(TALLOC_CTX *mem_ctx,
-						 const char *addr, int *count)
-{
-	struct sockaddr_storage ss;
-	struct nmb_name nname;
-	struct node_status *result;
-	NTSTATUS status;
-
-	make_nmb_name(&nname, "*", 0);
-	if (!interpret_string_addr(&ss, addr, AI_NUMERICHOST)) {
-		return NULL;
-	}
-	status = node_status_query(mem_ctx, &nname, &ss,
-				   &result, count, NULL);
-	if (!NT_STATUS_IS_OK(status)) {
-		return NULL;
-	}
-	return result;
-}
-
-static struct sockaddr_storage *lookup_byname_backend(TALLOC_CTX *mem_ctx,
-						      const char *name,
-						      int *count)
-{
-	struct ip_service *ret = NULL;
-	struct sockaddr_storage *return_ss = NULL;
-	int j, i;
-	NTSTATUS status;
-
-	*count = 0;
-
-	/* always try with wins first */
-	if (NT_STATUS_IS_OK(resolve_wins(name,0x20,&ret,count))) {
-		if ( *count == 0 )
-			return NULL;
-		return_ss = TALLOC_ARRAY(mem_ctx, struct sockaddr_storage,
-					 *count);
-		if (return_ss == NULL ) {
-			free( ret );
-			return NULL;
-		}
-
-		/* copy the IP addresses */
-		for ( i=0; i<(*count); i++ )
-			return_ss[i] = ret[i].ss;
-
-		free( ret );
-		return return_ss;
-	}
-
-	/* uggh, we have to broadcast to each interface in turn */
-	for (j=iface_count() - 1;
-	     j >= 0;
-	     j--) {
-		const struct sockaddr_storage *bcast_ss = iface_n_bcast(j);
-		if (!bcast_ss) {
-			continue;
-		}
-		status = name_query(name, 0x20, True, True,bcast_ss,
-				    mem_ctx, &return_ss, count, NULL);
-		if (NT_STATUS_IS_OK(status)) {
-			break;
-		}
-	}
-
-	return return_ss;
-}
-
-/* Get hostname from IP  */
-
-void winbindd_wins_byip(struct winbindd_cli_state *state)
-{
-	fstring response;
-	int i, count, maxlen, size;
-	struct node_status *status;
-
-	/* Ensure null termination */
-	state->request->data.winsreq[sizeof(state->request->data.winsreq)-1]='\0';
-
-	DEBUG(3, ("[%5lu]: wins_byip %s\n", (unsigned long)state->pid,
-		state->request->data.winsreq));
-
-	*response = '\0';
-	maxlen = sizeof(response) - 1;
-
-	if ((status = lookup_byaddr_backend(
-		     state->mem_ctx, state->request->data.winsreq, &count))) {
-	    size = strlen(state->request->data.winsreq);
-	    if (size > maxlen) {
-		TALLOC_FREE(status);
-		request_error(state);
-		return;
-	    }
-	    fstrcat(response,state->request->data.winsreq);
-	    fstrcat(response,"\t");
-	    for (i = 0; i < count; i++) {
-		/* ignore group names */
-		if (status[i].flags & 0x80) continue;
-		if (status[i].type == 0x20) {
-			size = sizeof(status[i].name) + strlen(response);
-			if (size > maxlen) {
-			    TALLOC_FREE(status);
-			    request_error(state);
-			    return;
-			}
-			fstrcat(response, status[i].name);
-			fstrcat(response, " ");
-		}
-	    }
-	    /* make last character a newline */
-	    response[strlen(response)-1] = '\n';
-	    TALLOC_FREE(status);
-	}
-	fstrcpy(state->response->data.winsresp,response);
-	request_ok(state);
-}
-
-/* Get IP from hostname */
-
-void winbindd_wins_byname(struct winbindd_cli_state *state)
-{
-	struct sockaddr_storage *ip_list = NULL;
-	int i, count, maxlen, size;
-	fstring response;
-	char addr[INET6_ADDRSTRLEN];
-
-	/* Ensure null termination */
-	state->request->data.winsreq[sizeof(state->request->data.winsreq)-1]='\0';
-
-	DEBUG(3, ("[%5lu]: wins_byname %s\n", (unsigned long)state->pid,
-		state->request->data.winsreq));
-
-	*response = '\0';
-	maxlen = sizeof(response) - 1;
-
-	ip_list = lookup_byname_backend(
-		state->mem_ctx, state->request->data.winsreq, &count);
-	if (ip_list != NULL){
-		for (i = count; i ; i--) {
-			print_sockaddr(addr, sizeof(addr), &ip_list[i-1]);
-			size = strlen(addr);
-			if (size > maxlen) {
-				TALLOC_FREE(ip_list);
-				request_error(state);
-				return;
-			}
-			if (i != 0) {
-				/* Clear out the newline character */
-				/* But only if there is something in there,
-				otherwise we clobber something in the stack */
-				if (strlen(response)) {
-					response[strlen(response)-1] = ' ';
-				}
-			}
-			fstrcat(response,addr);
-			fstrcat(response,"\t");
-		}
-		size = strlen(state->request->data.winsreq) + strlen(response);
-		if (size > maxlen) {
-		    TALLOC_FREE(ip_list);
-		    request_error(state);
-		    return;
-		}
-		fstrcat(response,state->request->data.winsreq);
-		fstrcat(response,"\n");
-		TALLOC_FREE(ip_list);
-	} else {
-		request_error(state);
-		return;
-	}
-
-	fstrcpy(state->response->data.winsresp,response);
-
-	request_ok(state);
-}
diff --git a/source3/winbindd/winbindd_wins_byip.c b/source3/winbindd/winbindd_wins_byip.c
new file mode 100644
index 00000000000..4ae1b82ac23
--- /dev/null
+++ b/source3/winbindd/winbindd_wins_byip.c
@@ -0,0 +1,135 @@
+/*
+   Unix SMB/CIFS implementation.
+   async implementation of WINBINDD_WINS_BYIP
+   Copyright (C) Volker Lendecke 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "winbindd.h"
+#include "librpc/gen_ndr/ndr_wbint_c.h"
+#include "libsmb/nmblib.h"
+
+struct winbindd_wins_byip_state {
+	struct nmb_name star;
+	struct sockaddr_storage addr;
+	fstring response;
+};
+
+static void winbindd_wins_byip_done(struct tevent_req *subreq);
+
+struct tevent_req *winbindd_wins_byip_send(TALLOC_CTX *mem_ctx,
+					   struct tevent_context *ev,
+					   struct winbindd_cli_state *cli,
+					   struct winbindd_request *request)
+{
+	struct tevent_req *req, *subreq;
+	struct winbindd_wins_byip_state *state;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct winbindd_wins_byip_state);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	/* Ensure null termination */
+	request->data.winsreq[sizeof(request->data.winsreq)-1]='\0';
+
+	fstr_sprintf(state->response, "%s\t", request->data.winsreq);
+
+	DEBUG(3, ("[%5lu]: wins_byip %s\n", (unsigned long)cli->pid,
+		  request->data.winsreq));
+
+	make_nmb_name(&state->star, "*", 0);
+
+	if (!interpret_string_addr(&state->addr, request->data.winsreq,
+				   AI_NUMERICHOST)) {
+		tevent_req_nterror(req, NT_STATUS_INVALID_PARAMETER);
+		return tevent_req_post(req, ev);
+	}
+
+	subreq = node_status_query_send(state, ev, &state->star,
+					&state->addr);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, winbindd_wins_byip_done, req);
+	return req;
+}
+
+static void winbindd_wins_byip_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct winbindd_wins_byip_state *state = tevent_req_data(
+		req, struct winbindd_wins_byip_state);
+	struct node_status *names;
+	int i, num_names;
+	NTSTATUS status;
+
+	status = node_status_query_recv(subreq, talloc_tos(), &names,
+					&num_names, NULL);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	for (i=0; i<num_names; i++) {
+		size_t size;
+		/*
+		 * ignore group names
+		 */
+		if (names[i].flags & 0x80) {
+			continue;
+		}
+		/*
+		 * Only report 0x20
+		 */
+		if (names[i].type != 0x20) {
+			continue;
+		}
+
+		DEBUG(10, ("got name %s\n", names[i].name));
+
+		size = strlen(names[i].name + strlen(state->response));
+		if (size > sizeof(state->response) - 1) {
+			DEBUG(10, ("To much data\n"));
+			tevent_req_nterror(req, STATUS_BUFFER_OVERFLOW);
+			return;
+		}
+		fstrcat(state->response, names[i].name);
+		fstrcat(state->response, " ");
+	}
+	state->response[strlen(state->response)-1] = '\n';
+
+	DEBUG(10, ("response: %s", state->response));
+
+	TALLOC_FREE(names);
+	tevent_req_done(req);
+}
+
+NTSTATUS winbindd_wins_byip_recv(struct tevent_req *req,
+				 struct winbindd_response *presp)
+{
+	struct winbindd_wins_byip_state *state = tevent_req_data(
+		req, struct winbindd_wins_byip_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+	fstrcpy(presp->data.winsresp, state->response);
+	return NT_STATUS_OK;
+}
diff --git a/source3/winbindd/winbindd_wins_byname.c b/source3/winbindd/winbindd_wins_byname.c
new file mode 100644
index 00000000000..ad642d479d0
--- /dev/null
+++ b/source3/winbindd/winbindd_wins_byname.c
@@ -0,0 +1,149 @@
+/*
+   Unix SMB/CIFS implementation.
+   async implementation of WINBINDD_WINS_BYNAME
+   Copyright (C) Volker Lendecke 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "winbindd.h"
+#include "librpc/gen_ndr/ndr_wbint_c.h"
+#include "libsmb/nmblib.h"
+#include "lib/util/string_wrappers.h"
+
+struct winbindd_wins_byname_state {
+	struct tevent_context *ev;
+	struct winbindd_request *request;
+	struct sockaddr_storage *addrs;
+	int num_addrs;
+};
+
+static void winbindd_wins_byname_wins_done(struct tevent_req *subreq);
+static void winbindd_wins_byname_bcast_done(struct tevent_req *subreq);
+
+struct tevent_req *winbindd_wins_byname_send(TALLOC_CTX *mem_ctx,
+					     struct tevent_context *ev,
+					     struct winbindd_cli_state *cli,
+					     struct winbindd_request *request)
+{
+	struct tevent_req *req, *subreq;
+	struct winbindd_wins_byname_state *state;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct winbindd_wins_byname_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->ev = ev;
+	state->request = request;
+
+	/* Ensure null termination */
+	request->data.winsreq[sizeof(request->data.winsreq)-1]='\0';
+
+	DEBUG(3, ("[%5lu]: wins_byname %s\n", (unsigned long)cli->pid,
+		  request->data.winsreq));
+
+	subreq = resolve_wins_send(state, ev, state->request->data.winsreq,
+				   0x20);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, winbindd_wins_byname_wins_done, req);
+	return req;
+}
+
+static void winbindd_wins_byname_wins_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct winbindd_wins_byname_state *state = tevent_req_data(
+		req, struct winbindd_wins_byname_state);
+	NTSTATUS status;
+
+	status = resolve_wins_recv(subreq, talloc_tos(), &state->addrs,
+				   &state->num_addrs, NULL);
+	TALLOC_FREE(subreq);
+	if (NT_STATUS_IS_OK(status)) {
+		tevent_req_done(req);
+		return;
+	}
+	subreq = name_resolve_bcast_send(state, state->ev,
+					 state->request->data.winsreq, 0x20);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, winbindd_wins_byname_bcast_done, req);
+}
+
+static void winbindd_wins_byname_bcast_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct winbindd_wins_byname_state *state = tevent_req_data(
+		req, struct winbindd_wins_byname_state);
+	NTSTATUS status;
+
+	status = name_resolve_bcast_recv(subreq, talloc_tos(), &state->addrs,
+					 &state->num_addrs);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	tevent_req_done(req);
+}
+
+NTSTATUS winbindd_wins_byname_recv(struct tevent_req *req,
+				   struct winbindd_response *presp)
+{
+	struct winbindd_wins_byname_state *state = tevent_req_data(
+		req, struct winbindd_wins_byname_state);
+	char *response;
+	NTSTATUS status;
+	int i;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		return status;
+	}
+
+	response = talloc_strdup(talloc_tos(), "");
+	if (response == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	for (i=0; i<state->num_addrs; i++) {
+		char addr[INET6_ADDRSTRLEN];
+		print_sockaddr(addr, sizeof(addr), &state->addrs[i]);
+
+		response = talloc_asprintf_append_buffer(
+			response, "%s\t", addr);
+		if (response == NULL) {
+			return NT_STATUS_NO_MEMORY;
+		}
+	}
+
+	response = talloc_asprintf_append_buffer(
+		response, "%s\n", state->request->data.winsreq);
+	if (response == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	if (talloc_get_size(response) > sizeof(presp->data.winsresp)) {
+		TALLOC_FREE(response);
+		return NT_STATUS_MARSHALL_OVERFLOW;
+	}
+	fstrcpy(presp->data.winsresp, response);
+	TALLOC_FREE(response);
+	return NT_STATUS_OK;
+}
diff --git a/source3/winbindd/wscript_build b/source3/winbindd/wscript_build
index dbae11697e5..6754f354108 100644
--- a/source3/winbindd/wscript_build
+++ b/source3/winbindd/wscript_build
@@ -24,6 +24,7 @@ IDMAP_SRC = 'idmap.c idmap_util.c ${IDMAP_RW_SRC}'
 
 bld.SAMBA3_SUBSYSTEM('idmap',
                     source=IDMAP_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('IDMAP_ADEX',
@@ -33,6 +34,7 @@ bld.SAMBA3_SUBSYSTEM('IDMAP_ADEX',
 
 bld.SAMBA3_SUBSYSTEM('IDMAP_HASH',
                     source=IDMAP_HASH_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('IDMAP_AD',
@@ -58,6 +60,7 @@ bld.SAMBA3_MODULE('idmap_rid',
 bld.SAMBA3_MODULE('idmap_passdb',
                  subsystem='idmap',
                  source=IDMAP_PASSDB_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('idmap_passdb'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('idmap_passdb'))
@@ -72,6 +75,7 @@ bld.SAMBA3_MODULE('idmap_ldap',
 bld.SAMBA3_MODULE('idmap_nss',
                  subsystem='idmap',
                  source=IDMAP_NSS_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('idmap_nss'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('idmap_nss'))
@@ -79,6 +83,7 @@ bld.SAMBA3_MODULE('idmap_nss',
 bld.SAMBA3_MODULE('idmap_tdb',
                  subsystem='idmap',
                  source=IDMAP_TDB_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('idmap_tdb'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('idmap_tdb'))
@@ -86,6 +91,7 @@ bld.SAMBA3_MODULE('idmap_tdb',
 bld.SAMBA3_MODULE('idmap_tdb2',
                  subsystem='idmap',
                  source=IDMAP_TDB2_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('idmap_tdb2'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('idmap_tdb2'))
@@ -109,6 +115,7 @@ bld.SAMBA3_MODULE('idmap_adex',
 bld.SAMBA3_MODULE('idmap_autorid',
                  subsystem='idmap',
                  source=IDMAP_AUTORID_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('idmap_autorid'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('idmap_autorid'))
@@ -119,11 +126,13 @@ NSS_INFO_SRC = 'nss_info.c'
 
 bld.SAMBA3_SUBSYSTEM('nss_info',
                     source=NSS_INFO_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_MODULE('nss_info_template',
                  subsystem='nss_info',
                  source=NSS_INFO_TEMPLATE_SRC,
+                 deps='samba-util',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('nss_info_template'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('nss_info_template'))
diff --git a/source3/wscript b/source3/wscript
index 49f11f2e5b2..d88faa21044 100644
--- a/source3/wscript
+++ b/source3/wscript
@@ -25,7 +25,7 @@ def set_options(opt):
         opt.RECURSE('../lib/nss_wrapper')
         opt.RECURSE('../lib/socket_wrapper')
         opt.RECURSE('../lib/tevent')
-        opt.RECURSE('../lib/tdb')
+        opt.RECURSE('../lib/tdb_compat')
 
     opt.add_option('--with-static-modules',
                    help=("Comma-separated list of names of modules to statically link in"),
@@ -59,7 +59,7 @@ def set_options(opt):
     opt.SAMBA3_ADD_OPTION('syslog')
     opt.SAMBA3_ADD_OPTION('automount')
     opt.SAMBA3_ADD_OPTION('aio-support')
-    opt.SAMBA3_ADD_OPTION('profiling-data')
+    opt.SAMBA3_ADD_OPTION('profiling-data', default=False)
 
     opt.SAMBA3_ADD_OPTION('cluster-support')
 
@@ -78,6 +78,7 @@ def configure(conf):
     if not conf.env.toplevel_build:
         version = samba_version.load_version(env=conf.env)
         conf.DEFINE('CONFIG_H_IS_FROM_SAMBA', 1)
+        conf.DEFINE('_SAMBA_WAF_BUILD_', version.MAJOR)
         conf.DEFINE('_SAMBA_BUILD_', version.MAJOR, add_to_cflags=True)
         conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
 
@@ -91,7 +92,8 @@ def configure(conf):
     if not conf.env.toplevel_build:
         conf.RECURSE('../lib/replace')
         conf.RECURSE('build')
-        conf.RECURSE('../lib/tdb')
+        conf.RECURSE('../lib/ccan')
+        conf.RECURSE('../lib/tdb_compat')
         conf.RECURSE('../lib/talloc')
         conf.RECURSE('../lib/tevent')
         conf.RECURSE('../lib/popt')
@@ -101,9 +103,9 @@ def configure(conf):
         conf.RECURSE('../libcli/smbreadline')
         conf.RECURSE('../lib/util')
 
-        conf.ADD_EXTRA_INCLUDES('''#source3 #source3/include #lib/replace''')
+        conf.ADD_EXTRA_INCLUDES('''#source3 #source3/include #lib/replace #lib''')
         if not conf.env.USING_SYSTEM_TDB:
-            conf.ADD_EXTRA_INCLUDES('#lib/tdb/include')
+            conf.ADD_EXTRA_INCLUDES('#lib/tdb_compat')
         if not conf.env.USING_SYSTEM_TEVENT:
             conf.ADD_EXTRA_INCLUDES('#lib/tevent')
         if not conf.env.USING_SYSTEM_TALLOC:
@@ -129,14 +131,14 @@ def configure(conf):
     conf.CHECK_FUNCS('opendir64 readdir64 seekdir64 telldir64 rewinddir64 closedir64')
     conf.CHECK_FUNCS('fdopendir fdopendir64')
     conf.CHECK_FUNCS('getpwent_r getdents64 setenv strcasecmp fcvt fcvtl')
-    conf.CHECK_FUNCS('syslog vsyslog timegm setlocale nl_langinfo')
+    conf.CHECK_FUNCS('syslog vsyslog timegm setlocale')
     conf.CHECK_FUNCS_IN('nanosleep', 'rt')
     conf.CHECK_FUNCS('lutimes futimes utimensat futimens')
     conf.CHECK_FUNCS('mlock munlock mlockall munlockall')
     conf.CHECK_FUNCS('memalign posix_memalign hstrerror')
     conf.CHECK_FUNCS('shmget')
     conf.CHECK_FUNCS_IN('shm_open', 'rt', checklibc=True)
-    conf.CHECK_FUNCS('gettext dgettext bindtextdomain textdomain')
+    conf.CHECK_FUNCS('gettext dgettext bindtextdomain textdomain bind_textdomain_codeset')
     #FIXME: for some reason this one still fails
     conf.CHECK_FUNCS_IN('yp_get_default_domain', 'nsl')
     conf.CHECK_FUNCS_IN('dn_expand _dn_expand __dn_expand', 'resolv')
@@ -287,7 +289,7 @@ inotify_init lgetea lgetxattr listea listxattr llistea llistxattr
 llseek _llseek __llseek lremoveea lremovexattr _lseek __lseek lseek64
 lsetea lsetxattr _lstat __lstat lstat64 _lstat64 __lstat64 lutimes
 __lxstat memalign mknod mknod64 mlock mlockall munlock munlockall
-nl_langinfo _open __open open64 _open64 __open64 _opendir __opendir
+_open __open open64 _open64 __open64 _opendir __opendir
 opendir64 pathconf poll posix_fallocate posix_fallocate64
 posix_memalign prctl pread _pread __pread pread64 _pread64 __pread64
 pwrite _pwrite __pwrite pwrite64 _pwrite64
@@ -574,7 +576,7 @@ msg.msg_acctrightslen = sizeof(fd);
 
     # Check for LDAP
     if Options.options.with_ldap:
-        conf.CHECK_HEADERS('ldap.h lber.h')
+        conf.CHECK_HEADERS('ldap.h lber.h ldap_pvt.h')
         conf.CHECK_TYPE('ber_tag_t', 'unsigned int', headers='ldap.h lber.h')
         conf.CHECK_FUNCS_IN('ber_scanf ber_sockbuf_add_io', 'lber')
         conf.CHECK_VARIABLE('LDAP_OPT_SOCKBUF', headers='ldap.h')
@@ -584,7 +586,7 @@ msg.msg_acctrightslen = sizeof(fd);
         conf.CHECK_VARIABLE('LBER_OPT_LOG_PRINT_FN',
                             define='HAVE_LBER_LOG_PRINT_FN', headers='lber.h')
 
-        conf.CHECK_FUNCS_IN('ldap_init ldap_initialize ldap_set_rebind_proc', 'ldap')
+        conf.CHECK_FUNCS_IN('ldap_init ldap_init_fd ldap_initialize ldap_set_rebind_proc', 'ldap')
         conf.CHECK_FUNCS_IN('ldap_add_result_entry', 'ldap')
 
         # Check if ldap_set_rebind_proc() takes three arguments
@@ -619,7 +621,7 @@ msg.msg_acctrightslen = sizeof(fd);
             conf.check_cfg(path="krb5-config", args="--cflags --libs",
                        package="gssapi", uselib_store="krb5")
         conf.CHECK_HEADERS('krb5.h krb5/locate_plugin.h', lib='krb5')
-        conf.CHECK_HEADERS('gssapi.h gssapi/gssapi_generic.h gssapi/gssapi.h gssapi/gssapi_ext.h com_err.h', lib='krb5')
+        conf.CHECK_HEADERS('gssapi.h gssapi/gssapi_generic.h gssapi/gssapi.h gssapi/gssapi_ext.h gssapi/gssapi_krb5.h com_err.h', lib='krb5')
 
         if conf.CONFIG_SET('HAVE_KRB5_LOCATE_PLUGIN_H'):
             conf.env['WINBIND_KRB5_LOCATOR'] = 'bin/winbind_krb5_locator.so'
@@ -632,7 +634,7 @@ msg.msg_acctrightslen = sizeof(fd);
         if conf.CHECK_FUNCS_IN('gss_display_status', 'gssapi') or \
            conf.CHECK_FUNCS_IN('gss_display_status', 'gssapi_krb5'):
             have_gssapi=True
-        conf.CHECK_FUNCS_IN('gss_wrap_iov gss_krb5_import_cred', 'gssapi gssapi_krb5 krb5')
+        conf.CHECK_FUNCS_IN('gss_wrap_iov gss_krb5_import_cred gss_get_name_attribute gss_mech_krb5 gss_oid_equal gss_inquire_sec_context_by_oid', 'gssapi gssapi_krb5 krb5')
         conf.CHECK_FUNCS_IN('krb5_mk_req_extended krb5_kt_compare', 'krb5')
         conf.CHECK_FUNCS('''
 krb5_set_real_time krb5_set_default_in_tkt_etypes krb5_set_default_tgs_enctypes
@@ -1419,6 +1421,16 @@ main() {
                 conf.DEFINE('AIX_SENDFILE_API', '1')
                 conf.DEFINE('WITH_SENDFILE', '1')
 
+    # Check for getcwd allowing a NULL arg.
+    conf.CHECK_CODE('''
+#include <unistd.h>
+main() {
+	char *s = getcwd(NULL,0);
+        exit(s != NULL ?  0 : 1);
+}''', 'GETCWD_TAKES_NULL', addmain=False, execute=True,
+        msg="getcwd takes a NULL argument")
+
+
     conf.CHECK_CODE('''enum TDB_ERROR err = TDB_ERR_NESTING''',
 		    'HAVE_TDB_ERR_NESTING',
 		    headers='tdb.h',
@@ -1637,6 +1649,47 @@ main() {
     if Options.options.with_profiling_data:
         conf.DEFINE('WITH_PROFILE', 1);
 
+    PTHREAD_CFLAGS='error'
+    PTHREAD_LDFLAGS='error'
+
+    if PTHREAD_LDFLAGS == 'error':
+        if conf.CHECK_FUNCS_IN('pthread_attr_init', 'pthread'):
+            PTHREAD_CFLAGS='-D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS'
+            PTHREAD_LDFLAGS='-lpthread'
+    if PTHREAD_LDFLAGS == 'error':
+        if conf.CHECK_FUNCS_IN('pthread_attr_init', 'pthreads'):
+            PTHREAD_CFLAGS='-D_THREAD_SAFE'
+            PTHREAD_LDFLAGS='-lpthreads'
+    if PTHREAD_LDFLAGS == 'error':
+        if conf.CHECK_FUNCS_IN('pthread_attr_init', 'c_r'):
+            PTHREAD_CFLAGS='-D_THREAD_SAFE -pthread'
+            PTHREAD_LDFLAGS='-pthread'
+    if PTHREAD_LDFLAGS == 'error':
+        if conf.CHECK_FUNC('pthread_attr_init'):
+            PTHREAD_CFLAGS='-D_REENTRANT'
+            PTHREAD_LDFLAGS='-lpthread'
+    # especially for HP-UX, where the CHECK_FUNC macro fails to test for
+    # pthread_attr_init. On pthread_mutex_lock it works there...
+    if PTHREAD_LDFLAGS == 'error':
+        if conf.CHECK_FUNCS_IN('pthread_mutex_lock', 'pthread'):
+            PTHREAD_CFLAGS='-D_REENTRANT'
+            PTHREAD_LDFLAGS='-lpthread'
+
+    if PTHREAD_CFLAGS != 'error' and PTHREAD_LDFLAGS != 'error':
+        conf.ADD_CFLAGS(PTHREAD_CFLAGS)
+        conf.ADD_LDFLAGS(PTHREAD_LDFLAGS)
+        conf.CHECK_HEADERS('pthread.h')
+        conf.DEFINE('HAVE_PTHREAD', '1')
+
+    if Options.options.with_pthreadpool:
+        if conf.CONFIG_SET('HAVE_PTHREAD'):
+            conf.DEFINE('WITH_PTHREADPOOL', '1')
+        else:
+            Logs.warn("pthreadpool support cannot be enabled when pthread support was not found")
+            conf.undefine('WITH_PTHREADPOOL')
+
+    if conf.CHECK_HEADERS('gpfs_gpl.h'):
+        conf.DEFINE('HAVE_GPFS', '1')
 
     default_static_modules=TO_LIST('''pdb_smbpasswd pdb_tdbsam pdb_wbc_sam
                                       auth_sam auth_unix auth_winbind auth_wbc auth_server
@@ -1657,6 +1710,9 @@ main() {
         default_static_modules.extend(TO_LIST('pdb_ads auth_netlogond'))
         default_shared_modules.extend(TO_LIST('charset_weird perfcount_test'))
 
+    if conf.env.toplevel_build:
+        default_static_modules.extend(TO_LIST('pdb_samba4 auth_samba4'))
+
     if Options.options.with_acl_support and conf.CONFIG_SET('HAVE_POSIX_ACLS'):
         default_static_modules.extend(TO_LIST('vfs_posixacl'))
 
@@ -1678,6 +1734,9 @@ main() {
     if conf.CONFIG_SET('DARWINOS'):
 	default_shared_modules.extend(TO_LIST('charset_macosxfs'))
 
+    if conf.CONFIG_SET('HAVE_GPFS'):
+	default_shared_modules.extend(TO_LIST('vfs_gpfs vfs_gpfs_hsm_notify'))
+
     explicit_shared_modules = TO_LIST(Options.options.shared_modules, delimiter=',')
     explicit_static_modules = TO_LIST(Options.options.static_modules, delimiter=',')
 
diff --git a/source3/wscript_build b/source3/wscript_build
index 7a01a7a366f..624fcc8a732 100755
--- a/source3/wscript_build
+++ b/source3/wscript_build
@@ -16,8 +16,7 @@ bld.env.public_headers_allow_broken = True
 TDB_LIB_SRC = '''
           lib/dbwrap.c lib/dbwrap_tdb.c
           lib/dbwrap_ctdb.c
-          lib/g_lock.c
-          lib/dbwrap_rbt.c'''
+          lib/g_lock.c'''
 
 TDB_VALIDATE_SRC = '''lib/tdb_validate.c'''
 
@@ -39,8 +38,6 @@ COMPRESSION_SRC = '../lib/compression/mszip.c'
 
 DRSUAPI_SRC = '''${COMPRESSION_SRC}'''
 
-LIBNDR_SRC = '''librpc/ndr/util.c'''
-
 LIBCLI_SPOOLSS_SRC = '''rpc_client/cli_spoolss.c
                      rpc_client/init_spoolss.c'''
 
@@ -57,12 +54,9 @@ LIBCLI_WINREG_SRC = '''rpc_client/cli_winreg.c
 # that requires knowledge of security contexts
 REG_PARSE_PRS_SRC = '''registry/reg_parse_prs.c'''
 
-PTHREADPOOL_SRC = ''
-
 LIB_SRC = '''
           lib/messages.c lib/messages_local.c
-          lib/messages_ctdbd.c lib/packet.c lib/ctdbd_conn.c
-          lib/interfaces.c lib/memcache.c
+          lib/messages_ctdbd.c lib/ctdb_packet.c lib/ctdbd_conn.c
           lib/talloc_dict.c
           lib/util_sconn.c
           lib/serverid.c
@@ -75,36 +69,31 @@ LIB_SRC = '''
           lib/system.c lib/sendfile.c lib/recvfile.c lib/time.c
           lib/username.c
           lib/access.c lib/smbrun.c
-          lib/bitmap.c lib/dprintf.c
           lib/wins_srv.c
           lib/util_sid.c
           lib/util_file.c
-          lib/util.c lib/util_cmdline.c lib/util_names.c
-          lib/util_sock.c lib/sock_exec.c lib/util_sec.c
-          lib/substitute.c lib/dbwrap_util.c
+          lib/util.c
+          lib/util_sock.c lib/sock_exec.c
+          lib/substitute.c lib/substitute_generic.c
           lib/ms_fnmatch.c
           lib/tallocmsg.c lib/dmallocmsg.c
           libsmb/clisigning.c libsmb/smb_signing.c
           intl/lang_tdb.c
-          lib/conn_tdb.c lib/adt_tree.c lib/gencache.c
+          lib/conn_tdb.c lib/gencache.c
           lib/sessionid_tdb.c
           lib/module.c lib/events.c
           lib/server_contexts.c
           lib/ldap_escape.c
-          lib/secdesc.c ${PTHREADPOOL_SRC}
           lib/fncall.c
           libads/krb5_errs.c lib/system_smbd.c lib/audit.c
           lib/file_id.c lib/idmap_cache.c'''
 
-LIB_DUMMY_SRC = '''lib/dummysmbd.c lib/dummyroot.c'''
-LIB_NONSMBD_SRC = '''${LIB_DUMMY_SRC}'''
-
 POPT_LIB_SRC = '''lib/popt_common.c'''
 
 PARAM_UTIL_SRC = '''param/util.c'''
 
 PARAM_WITHOUT_REG_SRC = '''param/loadparm.c param/loadparm_server_role.c
-                           lib/sharesec.c lib/ldap_debug_handler.c'''
+                           lib/sharesec.c lib/ldap_debug_handler.c lib/util_names.c'''
 
 KRBCLIENT_SRC = '''libads/kerberos.c libads/ads_status.c libsmb/clikrb5.c'''
 
@@ -146,7 +135,7 @@ LIBSMB_SRC = '''libsmb/clientgen.c libsmb/cliconnect.c libsmb/clifile.c
              libsmb/clitrans.c libsmb/clisecdesc.c libsmb/clidgram.c
              libsmb/clistr.c libsmb/cliquota.c libsmb/clifsinfo.c libsmb/clidfs.c
              libsmb/clioplock.c libsmb/clirap2.c
-             libsmb/smb_seal.c libsmb/async_smb.c
+             libsmb/smb_seal.c libsmb/async_smb.c  libsmb/read_smb.c
              libsmb/cli_np_tstream.c
              libsmb/smbsock_connect.c'''
 
@@ -261,7 +250,8 @@ WINBINDD_SRC1 = '''winbindd/winbindd.c
                    winbindd/winbindd_pam.c
                    winbindd/winbindd_misc.c
                    winbindd/winbindd_cm.c
-                   winbindd/winbindd_wins.c
+                   winbindd/winbindd_wins_byip.c
+                   winbindd/winbindd_wins_byname.c
                    winbindd/winbindd_msrpc.c
                    winbindd/winbindd_rpc.c
                    winbindd/winbindd_reconnect.c
@@ -345,13 +335,14 @@ WINBINDD_SRC = '''${WINBINDD_SRC1}
 
 MANGLE_SRC = '''smbd/mangle.c smbd/mangle_hash.c smbd/mangle_hash2.c'''
 
-SMBD_SRC_MAIN = '''smbd/server.c smbd/server_exit.c smbd/msg_idmap.c'''
+SMBD_SRC_MAIN = '''smbd/server.c smbd/msg_idmap.c'''
 
 BUILDOPT_SRC = '''smbd/build_options.c'''
 
 SMBD_SRC_SRV = '''smbd/server_reload.c smbd/files.c smbd/connection.c
                smbd/utmp.c smbd/session.c
-               smbd/dfree.c smbd/dir.c smbd/password.c smbd/conn.c
+               smbd/dfree.c smbd/dir.c smbd/password.c smbd/conn_msg.c
+               smbd/conn_idle.c
                smbd/share_access.c smbd/fileio.c
                smbd/ipc.c smbd/lanman.c smbd/negprot.c
                smbd/message.c smbd/nttrans.c smbd/pipes.c
@@ -390,6 +381,7 @@ SMBD_SRC_SRV = '''smbd/server_reload.c smbd/files.c smbd/connection.c
                smbd/smb2_getinfo.c
                smbd/smb2_setinfo.c
                smbd/smb2_break.c
+               smbd/server_exit.c
                ${MANGLE_SRC}'''
 
 SMBD_SRC_BASE = '''${SMBD_SRC_SRV}
@@ -567,7 +559,9 @@ SMBTORTURE_SRC1 = '''torture/torture.c torture/nbio.c torture/scanner.c torture/
                 torture/test_async_echo.c
                 torture/test_addrchange.c
                 torture/test_posix_append.c
+		torture/test_nttrans_create.c
 		torture/test_case_insensitive.c
+		torture/test_notify_online.c
                 torture/test_smbsock_any_connect.c'''
 
 SMBTORTURE_SRC = '''${SMBTORTURE_SRC1}
@@ -642,13 +636,6 @@ t = bld.SAMBA_GENERATOR('build_env.h',
 t.env.SRCDIR = bld.path.abspath()
 t.env.BUILDDIR = bld.path.abspath()
 
-if not bld.env.toplevel_build:
-    bld.SAMBA_GENERATOR('smbtorture4',
-			source='',
-			target='',
-			rule='cd ../; make -f Makefile-smbtorture4 bin/smbtorture4',
-			always=True)
-
 bld.SETUP_BUILD_GROUPS()
 
 if not bld.env.toplevel_build:
@@ -664,45 +651,48 @@ if not bld.env.toplevel_build:
 
 bld.SAMBA3_LIBRARY('netapi',
                     source=LIBNETAPI_SRC,
-                    public_deps='''talloc tdb cap wbclient LIB_NONSMBD LIBSMB KRBCLIENT
-                    passdb SMBLDAP PARAM_WITHOUT_REG samba-util
+                    public_deps='''talloc tdb_compat cap wbclient smbd_shim LIBSMB KRBCLIENT
+                    passdb SMBLDAP param samba-util
                     LIBMSRPC_GEN msrpc3 ads LIBNET DCUTIL NDR_LIBNETAPI
-                    RPC_CLIENT_SCHANNEL LIB_SMBCONF REG_SMBCONF TOKEN_UTIL
+                    RPC_CLIENT_SCHANNEL smbconf REG_SMBCONF TOKEN_UTIL
                     LIBCLI_SAMR LIBCLI_LSA3 LIBRPCCLI_NETLOGON
                     RPC_NDR_SRVSVC RPC_NDR_WKSSVC RPC_NDR_INITSHUTDOWN
-                    INIT_NETLOGON INIT_SAMR''',
+                    INIT_NETLOGON INIT_SAMR POPT_SAMBA3''',
                     public_headers='../source3/lib/netapi/netapi.h',
                     vnum='0',
                     vars=locals())
 
 bld.SAMBA3_LIBRARY('libsmb/smbclient',
                     source=LIBSMBCLIENT_SRC,
-                    public_deps='''talloc tdb wbclient cap param LIB_NONSMBD LIBSMB KRBCLIENT passdb SMBLDAP
-                    LIBMSRPC_GEN msrpc3 LIBCLI_LSA3 RPC_NDR_SRVSVC''',
+                    public_deps='''talloc tdb_compat wbclient cap param  smbd_shim LIBSMB KRBCLIENT passdb SMBLDAP
+                    LIBMSRPC_GEN msrpc3 LIBCLI_LSA3 RPC_NDR_SRVSVC POPT_SAMBA3''',
                     public_headers='include/libsmbclient.h',
                     vnum='0',
                     vars=locals())
 
 bld.SAMBA3_LIBRARY('smbsharemodes',
                     source=LIBSMBSHAREMODES_SRC,
-                    public_deps='''talloc tdb''',
+                    public_deps='''talloc tdb_compat''',
+                    deps='''ccan''',
                     public_headers='include/smb_share_modes.h',
                     vnum='0',
                     vars=locals())
 
 bld.SAMBA3_LIBRARY('nss_wins',
                   source=WINBIND_WINS_NSS_SRC,
-                  deps='''winbind-client param LIBSMB LIB_NONSMBD LIBTSOCKET KRBCLIENT
+                  deps='''winbind-client param LIBSMB smbd_shim LIBTSOCKET KRBCLIENT
                   cap wbclient''',
                   realname='libnss_wins.so.2',
                   vnum='2')
 
-bld.SAMBA3_SUBSYSTEM('msrpc3',
-                    source='${LIBMSRPC_SRC}',
-                    deps='''ndr ndr-util NDR_SECURITY NDR_DCERPC NDR_SCHANNEL
-                    RPC_NDR_EPMAPPER COMMON_SCHANNEL LIBCLI_AUTH
-                    LIBTSOCKET krb5 rpccommon''',
-                    vars=locals())
+bld.SAMBA3_LIBRARY('msrpc3',
+                   source='${LIBMSRPC_SRC}',
+                   deps='''ndr ndr-standard
+                    RPC_NDR_EPMAPPER NTLMSSP_COMMON COMMON_SCHANNEL LIBCLI_AUTH
+                    LIBTSOCKET KRB5_WRAP rpccommon
+                    LIBSMB''',
+                   vars=locals(),
+                   private_library=True)
 
 bld.SAMBA3_SUBSYSTEM('LIBMSRPC_GEN',
                     source='',
@@ -711,7 +701,7 @@ bld.SAMBA3_SUBSYSTEM('LIBMSRPC_GEN',
 
 bld.SAMBA3_SUBSYSTEM('LIBGPO',
                     source='${LIBGPO_SRC}',
-                    deps='gpext',
+                    deps='gpext talloc',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('AVAHI',
@@ -720,7 +710,8 @@ bld.SAMBA3_SUBSYSTEM('AVAHI',
                     enabled=bld.env.with_avahi)
 
 bld.SAMBA3_SUBSYSTEM('GROUPDB',
-                    source=GROUPDB_SRC)
+                    source=GROUPDB_SRC,
+                    deps='tdb_compat')
 
 bld.SAMBA3_SUBSYSTEM('TLDAP',
                     source=TLDAP_SRC,
@@ -728,39 +719,50 @@ bld.SAMBA3_SUBSYSTEM('TLDAP',
 
 bld.SAMBA3_LIBRARY('passdb',
                    source=PASSDB_SRC,
-                   deps='SECRETS3 SMBLDAP GROUPDB pdb wbclient LIBCLI_AUTH',
+                   deps='SECRETS3 SMBLDAP GROUPDB pdb wbclient LIBCLI_AUTH flag_mapping',
                    private_library=True,
                    vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('PARAM_UTIL',
-                    source=PARAM_UTIL_SRC)
+                    source=PARAM_UTIL_SRC,
+                    deps='samba-util')
 
-bld.SAMBA3_SUBSYSTEM('PARAM_WITHOUT_REG',
-                    source=PARAM_WITHOUT_REG_SRC,
-                    deps='''PARAM_UTIL''',
-                    vars=locals())
+if bld.env.toplevel_build:
+    bld.SAMBA3_SUBSYSTEM('LOADPARM_CTX',
+                         source='param/loadparm_ctx.c',
+                         deps='''s3_param_h param''',
+                         vars=locals())
 
-bld.SAMBA3_LIBRARY('param',
-                   source='',
-                   deps='PARAM_WITHOUT_REG samba-util LIB_SMBCONF',
-                   private_library=True,
-                   vars=locals())
+bld.SAMBA3_SUBSYSTEM('param',
+                   source=PARAM_WITHOUT_REG_SRC,
+                   deps='samba-util PARAM_UTIL smbd_conn ldap lber LOADPARM_CTX samba3core smbconf''')
+
+bld.SAMBA3_SUBSYSTEM('param_service',
+                     source='param/service.c',
+                     deps = 'USER_UTIL param USER_UTIL PRINTING')
 
 bld.SAMBA3_SUBSYSTEM('REGFIO',
                     source=REGFIO_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('REG_API_REGF',
                     source=REG_API_REGF_SRC,
+                    deps='samba-util',
                     vars=locals())
 
-bld.SAMBA3_SUBSYSTEM('REG_BASE',
-                    source=REG_BASE_SRC,
-                    vars=locals())
+bld.SAMBA3_LIBRARY('smbregistry',
+                   source=REG_BASE_SRC,
+                   deps='''smbd_shim tdb-wrap3 NDR_SECURITY UTIL_TDB talloc
+                   replace util_reg samba-util security
+                   errors3 adt_tree dbwrap_util util_str util_sec util_malloc''',
+                   vars=locals(),
+                   allow_undefined_symbols=True,
+                   private_library=True)
 
 bld.SAMBA3_SUBSYSTEM('REG_SMBCONF',
                     source=REG_SMBCONF_SRC,
-                    deps='REG_BASE',
+                    deps='smbregistry',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('REG_FULL',
@@ -768,30 +770,34 @@ bld.SAMBA3_SUBSYSTEM('REG_FULL',
                     deps='REG_SMBCONF',
                     vars=locals())
 
-bld.SAMBA3_SUBSYSTEM('POPT_SAMBA3',
-                     source=POPT_LIB_SRC,
-                     deps='popt',
-                     vars=locals())
+bld.SAMBA3_LIBRARY('POPT_SAMBA3',
+                   source=POPT_LIB_SRC,
+                   deps='popt samba-util UTIL_CMDLINE',
+                   vars=locals(),
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('UTIL_CMDLINE',
+                   source='lib/util_cmdline.c',
+                   deps='SECRETS3',
+                   private_library=True)
 
 bld.SAMBA3_SUBSYSTEM('KRBCLIENT',
                     source=KRBCLIENT_SRC,
-                    public_deps='KRB5_WRAP krb5 k5crypto com_err gssapi gssapi_krb5',
+                    public_deps='KRB5_WRAP k5crypto',
                     vars=locals())
 
-bld.SAMBA3_LIBRARY('samba3core',
+bld.SAMBA3_SUBSYSTEM('samba3core',
                    source=LIB_SRC,
-                   deps='LIBCRYPTO ndr ndr-util security NDR_SECURITY charset NDR_MESSAGING LIBASYNC_REQ tdb-wrap3 CHARSET3 UTIL_TDB UTIL_PW SAMBA_VERSION krb5 flag_mapping util_reg',
-                   private_library=True,
+                   deps='LIBCRYPTO ndr security NDR_SECURITY samba-util NDR_MESSAGING LIBASYNC_REQ tdb-wrap3 UTIL_TDB UTIL_PW SAMBA_VERSION KRB5_WRAP flag_mapping util_reg PTHREADPOOL interfaces cap string_init param util_str CHARSET3 namearray dbwrap_util util_sec util_malloc memcache ccan errors3',
                    vars=locals())
 
-bld.SAMBA3_SUBSYSTEM('LIB_NONSMBD',
-                    source=LIB_NONSMBD_SRC,
-                    deps='samba3core',
-                    vars=locals())
+bld.SAMBA3_LIBRARY('smbd_shim',
+                   source='''lib/smbd_shim.c''', 
+                   private_library=True)
 
 bld.SAMBA3_SUBSYSTEM('LIBSMB_ERR',
                     source='',
-                    deps='errors3 SECRETS3 ldap lber MSRPC_PARSE LIBCLI_AUTH rpccommon',
+                    deps='errors3 passdb ldap lber MSRPC_PARSE LIBCLI_AUTH rpccommon',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LIBNTLMSSP',
@@ -799,10 +805,11 @@ bld.SAMBA3_SUBSYSTEM('LIBNTLMSSP',
                     deps='LIBSMB_ERR NDR_NTLMSSP NTLMSSP_COMMON',
                     vars=locals())
 
-bld.SAMBA3_SUBSYSTEM('LIBSMB',
-                    source=LIBSMB_SRC,
-                    deps='LIBNTLMSSP CLDAP LIBNMB LIBNBT LIBDRSUAPI SPNEGO_PARSE LIBTSOCKET KRBCLIENT',
-                    vars=locals())
+bld.SAMBA3_LIBRARY('LIBSMB',
+                   source=LIBSMB_SRC,
+                   deps='LIBNTLMSSP CLDAP LIBNMB LIBNBT LIBDRSUAPI SPNEGO_PARSE LIBTSOCKET KRBCLIENT LIBCLI_SMB_COMMON UTIL_CMDLINE',
+                   vars=locals(),
+                   private_library=True)
 
 bld.SAMBA3_SUBSYSTEM('LIBDRSUAPI',
                     source=DRSUAPI_SRC,
@@ -815,9 +822,9 @@ bld.SAMBA3_SUBSYSTEM('CLDAP',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('SECRETS3',
-                    source=SECRETS_SRC,
-                    deps='NDR_SECRETS',
-                    vars=locals())
+                   source=SECRETS_SRC,
+                   deps='NDR_SECRETS param samba3core passdb',
+                   vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('SMBLDAP',
                     source=SMBLDAP_SRC,
@@ -827,59 +834,72 @@ bld.SAMBA3_SUBSYSTEM('SMBLDAP',
 
 bld.SAMBA3_LIBRARY('ads',
                    source=LIBADS_SRC,
-                   deps='cli-ldap-common krb5 gssapi gssapi_krb5 ldap lber',
+                   deps='cli-ldap-common KRB5_WRAP ldap lber KRBCLIENT param LIBNTLMSSP LIBNMB LIBSMB DCUTIL',
                    private_library=True,
                    vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LIBADS_SERVER',
                     source=LIBADS_SERVER_SRC,
-                    deps='ndr-krb5pac krb5 KRB5_WRAP',
+                    deps='ndr-krb5pac KRB5_WRAP',
 		    vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LIBADS_PRINTER',
                     source=LIBADS_PRINTER_SRC,
+                    deps='samba-util',
 		    vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LIBAFS',
                     source=AFS_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LIBAFS_SETTOKEN',
                     source=AFS_SETTOKEN_SRC,
+                    deps='samba-util',
                     vars=locals())
 
-bld.SAMBA3_SUBSYSTEM('LIB_SMBCONF',
-                    source=LIB_SMBCONF_SRC,
-                    deps='LIBSMBCONF REG_BASE REG_SMBCONF',
-                    vars=locals())
+bld.SAMBA3_LIBRARY('smbconf',
+                   source=LIB_SMBCONF_SRC,
+                   deps='''LIBSMBCONF smbregistry REG_SMBCONF talloc param
+                   util_reg samba-util errors3 CHARSET SAMBA_VERSION''',
+                   public_headers='../lib/smbconf/smbconf.h',
+                   vnum='0')
+
+bld.SAMBA3_LIBRARY('smbd_conn',
+                   source='smbd/conn.c',
+                   deps='string_init bitmap namearray',
+                   private_library=True)
 
 bld.SAMBA3_LIBRARY('smbd_base',
                     source=SMBD_SRC_BASE,
-                    deps='''tdb tevent dl krb5 ldap gssapi gssapi_krb5
+                    deps='''tdb_compat tevent dl krb5 ldap gssapi gssapi_krb5
                     samba-util wbclient crypt nsl cups cap z passdb
-                    PARAM_WITHOUT_REG samba3core LIBSMB POPT_SAMBA3 KRBCLIENT AVAHI
+                    param samba3core LIBSMB POPT_SAMBA3 KRBCLIENT AVAHI
                     LIBMSRPC_GEN msrpc3 ads LIBADS_SERVER LIBADS_PRINTER
                     vfs vfs_default vfs_posixacl auth rpc LOCKING LIBAFS LIBAFS_SETTOKEN PROFILE
-                    PRINTING PRINTBACKEND NDR_XATTR NDR_NOTIFY3 REGFIO
-                    LIB_SMBCONF REG_FULL FNAME_UTIL
+                    PRINTING PRINTBACKEND NDR_XATTR NDR_NOTIFY REGFIO
+                    smbconf REG_FULL FNAME_UTIL
                     LIBCLI_SAMR LIBCLI_LSA3 LIBRPCCLI_NETLOGON LIBCLI_SPOOLSS
                     RPC_NDR_SRVSVC npa_tstream INIT_NETLOGON INIT_SAMR
-                    LIBCLI_SMB_COMMON RPC_SERVER
+                    LIBCLI_SMB_COMMON RPC_SERVER smbd_conn param_service
+                    smbd_shim
                     ''',
                     private_library=True,
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LOCKING',
                     source='${LOCKING_SRC}',
-                    deps='''tdb talloc''',
+                    deps='''tdb_compat talloc''',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('PROFILE',
                     source='${PROFILE_SRC}',
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('PRINTBASE',
                     source=PRINTBASE_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('PRINTBACKEND',
@@ -894,15 +914,17 @@ bld.SAMBA3_SUBSYSTEM('PRINTING',
 
 bld.SAMBA3_SUBSYSTEM('PASSWD_UTIL',
                     source=PASSWD_UTIL_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('FNAME_UTIL',
                     source=FNAME_UTIL_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LIBNET',
                     source=LIBNET_SRC,
-                    deps='NDR_LIBNET_JOIN krb5',
+                    deps='NDR_LIBNET_JOIN KRB5_WRAP',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('LIBNET_DSSYNC',
@@ -932,6 +954,7 @@ bld.SAMBA3_SUBSYSTEM('LIBNBT',
 
 bld.SAMBA3_SUBSYSTEM('SERVICES',
                     source=SERVICES_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('PLAINTEXT_AUTH',
@@ -946,42 +969,91 @@ bld.SAMBA3_SUBSYSTEM('PASSCHANGE',
 
 bld.SAMBA3_SUBSYSTEM('SAMBA_VERSION',
                     source=VERSION_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('SLCACHE',
                     source=SLCACHE_SRC,
+                    deps='samba-util',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('DCUTIL',
                     source=DCUTIL_SRC,
-                    vars=locals())
-
-bld.SAMBA3_SUBSYSTEM('ndr-util',
-                    source=LIBNDR_SRC,
+                     deps='ads LIBRPCCLI_NETLOGON msrpc3 LIBCLI_LSA3 INIT_NETLOGON',
                     vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('tdb-wrap3',
                     source='lib/util_tdb.c',
+                     deps='tdb-wrap',
                     vars=locals())
 
-bld.SAMBA3_SUBSYSTEM('CHARSET3',
-                    source='''lib/util_str.c lib/charcnv.c  ../lib/util/charset/convert_string.c lib/fstring.c''',
-                    public_deps='ICONV_WRAPPER CODEPOINTS',
-                    deps='samba-util')
+bld.SAMBA3_LIBRARY('dbwrap_util',
+                   source='lib/dbwrap_util.c lib/dbwrap_rbt.c',
+                   deps='samba-util UTIL_TDB errors',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('util_malloc',
+                   source='''lib/util_malloc.c''',
+                   deps='samba-util',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('string_init',
+                   source='''lib/string_init.c''',
+                   deps='samba-util',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('util_str',
+                   source='''lib/util_str.c''',
+                   deps='samba-util CHARSET',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('util_sec',
+                   source='''lib/util_sec.c''',
+                   deps='samba-util',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('bitmap',
+                   source='lib/bitmap.c',
+                   deps='samba-util',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('namearray',
+                   source='lib/namearray.c',
+                   deps='samba-util',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('adt_tree',
+                   source='lib/adt_tree.c',
+                   deps='samba-util',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('memcache',
+                   source='lib/memcache.c',
+                   deps='samba-util util_malloc',
+                   private_library=True)
+
+bld.SAMBA3_LIBRARY('CHARSET3',
+                    source='''lib/charcnv.c lib/fstring.c''',
+                    public_deps='ICONV_WRAPPER CHARSET',
+                    deps='samba-util util_str',
+                   private_library=True)
 
 bld.SAMBA3_SUBSYSTEM('ldb3',
-                    source='lib/ldb_compat.c')
+                    source='lib/ldb_compat.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('errors3',
-                     source='../libcli/util/doserr.c libsmb/nterr.c libsmb/errormap.c libsmb/smberr.c lib/errmap_unix.c')
+                     source='libsmb/errormap.c libsmb/smberr.c lib/errmap_unix.c',
+                     deps='errors')
 
 bld.SAMBA3_SUBSYSTEM('LIBCLI_SAMR',
                     source=LIBCLI_SAMR_SRC,
                     deps='RPC_NDR_SAMR')
 
-bld.SAMBA3_SUBSYSTEM('LIBCLI_LSA3',
-                    source=LIBCLI_LSA_SRC,
-                    deps='RPC_NDR_LSA INIT_LSA')
+bld.SAMBA3_LIBRARY('LIBCLI_LSA3',
+                   source=LIBCLI_LSA_SRC,
+                   deps='RPC_NDR_LSA INIT_LSA', 
+                   private_library=True)
 
 bld.SAMBA3_SUBSYSTEM('LIBRPCCLI_NETLOGON',
                     source=LIBRPCCLI_NETLOGON_SRC,
@@ -993,20 +1065,24 @@ bld.SAMBA3_SUBSYSTEM('LIBCLI_SPOOLSS',
 
 bld.SAMBA3_SUBSYSTEM('LIBCLI_WINREG',
                     source=LIBCLI_WINREG_SRC,
-                    deps='RPC_NDR_WINREG')
+                    deps='RPC_NDR_WINREG RPC_NCACN_NP')
 
 bld.SAMBA3_SUBSYSTEM('RPC_CLIENT_SCHANNEL',
-	source=RPC_CLIENT_SCHANNEL_SRC,
-	vars=locals())
+                    source=RPC_CLIENT_SCHANNEL_SRC,
+                    deps='samba-util',
+                    vars=locals())
 
 bld.SAMBA3_SUBSYSTEM('INIT_LSA',
-                    source='rpc_client/init_lsa.c')
+                    source='rpc_client/init_lsa.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('INIT_NETLOGON',
-                    source='rpc_client/init_netlogon.c')
+                    source='rpc_client/init_netlogon.c',
+                    deps='samba-util')
 
 bld.SAMBA3_SUBSYSTEM('INIT_SAMR',
-                    source='rpc_client/init_samr.c')
+                    source='rpc_client/init_samr.c',
+                    deps='samba-util')
 
 ########################## BINARIES #################################
 
@@ -1018,16 +1094,16 @@ bld.SAMBA3_BINARY('smbd/smbd',
 
 bld.SAMBA3_BINARY('nmbd/nmbd',
                  source=NMBD_SRC,
-                 deps='''talloc tdb tevent z cap wbclient dl
-                 passdb param ldap LIB_NONSMBD LIBSMB
+                 deps='''talloc tdb_compat tevent z cap wbclient dl
+                 passdb param ldap smbd_shim LIBSMB
                  POPT_SAMBA3 KRBCLIENT NDR_SAMR NDR_LSA PROFILE''',
                  install_path='${SBINDIR}',
                  vars=locals())
 
 bld.SAMBA3_BINARY('winbindd/winbindd',
                  source=WINBINDD_SRC,
-                 deps='''talloc tdb tevent cap dl z
-                 wbclient passdb ldap param LIB_NONSMBD LIBSMB
+                 deps='''talloc tdb_compat tevent cap dl z
+                 wbclient passdb ldap param smbd_shim LIBSMB
                  POPT_SAMBA3 KRBCLIENT LIBMSRPC_GEN msrpc3 ads LIBADS_SERVER
                  SRV_NDR_WBINT RPC_NDR_WBINT NDR_WBINT LIBAFS
                  LIBAFS_SETTOKEN PROFILE SLCACHE DCUTIL idmap nss_info
@@ -1044,7 +1120,7 @@ bld.SAMBA3_BINARY('winbindd/winbindd',
 bld.SAMBA3_BINARY('web/swat',
                  source=SWAT_SRC,
                  deps='''talloc tevent cap samba3core LIBSMB wbclient param
-                 LIB_NONSMBD passdb POPT_SAMBA3 KRBCLIENT cups
+                 smbd_shim passdb POPT_SAMBA3 KRBCLIENT cups
                  LIBMSRPC_GEN msrpc3 LOCKING PLAINTEXT_AUTH PRINTBASE PRINTING FNAME_UTIL
                  LIBCLI_SAMR INIT_LSA PASSCHANGE''',
                  enabled=bld.env.build_swat,
@@ -1053,8 +1129,8 @@ bld.SAMBA3_BINARY('web/swat',
 
 bld.SAMBA3_BINARY('rpcclient/rpcclient',
                  source=RPCCLIENT_SRC,
-                 deps='''talloc tdb cap POPT_SAMBA3 passdb LIBSMB LIB_NONSMBD
-                 PARAM_WITHOUT_REG wbclient param KRBCLIENT LIBMSRPC_GEN msrpc3
+                 deps='''talloc tdb_compat cap POPT_SAMBA3 passdb LIBSMB smbd_shim
+                 param wbclient param KRBCLIENT LIBMSRPC_GEN msrpc3
                  ads SMBREADLINE DCUTIL RPC_NDR_WINREG RPC_NDR_ECHO
                  RPC_CLIENT_SCHANNEL
                  LIBCLI_SAMR LIBCLI_LSA3 LIBRPCCLI_NETLOGON LIBCLI_SPOOLSS
@@ -1066,15 +1142,16 @@ bld.SAMBA3_BINARY('rpcclient/rpcclient',
 
 bld.SAMBA3_BINARY('client/smbclient' + bld.env.suffix3,
                  source=CLIENT_SRC,
-                 deps='''talloc tdb cap POPT_SAMBA3 passdb LIBSMB LIB_NONSMBD
-                 PARAM_WITHOUT_REG wbclient param KRBCLIENT LIBMSRPC_GEN
-                 msrpc3 SMBREADLINE libsmb/smbclient RPC_NDR_SRVSVC INIT_LSA''',
+                 deps='''talloc tdb_compat cap POPT_SAMBA3 passdb LIBSMB smbd_shim
+                 param wbclient param KRBCLIENT LIBMSRPC_GEN
+                 msrpc3 SMBREADLINE libsmb/smbclient RPC_NDR_SRVSVC INIT_LSA
+                 LIBCLI_SMB_COMMON''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('net',
                  source=NET_SRC,
-                 deps='''talloc tdb netapi addns cap intl POPT_SAMBA3 passdb LIBSMB LIB_NONSMBD
-                 PARAM_WITHOUT_REG wbclient param KRBCLIENT LIBMSRPC_GEN msrpc3 LIBGPO ads LIBADS_SERVER LIBADS_PRINTER
+                 deps='''talloc tdb_compat netapi addns cap intl POPT_SAMBA3 passdb LIBSMB smbd_shim
+                 param wbclient param KRBCLIENT LIBMSRPC_GEN msrpc3 LIBGPO ads LIBADS_SERVER LIBADS_PRINTER
                  LIBAFS LIBAFS_SETTOKEN SMBREADLINE PASSWD_UTIL LIBNET
                  LIBNET_DSSYNC LIBNET_SAMSYNC LIBEVENTLOG DCUTIL
                  REGFIO NDR_NTPRINTING RPC_NDR_WINREG
@@ -1087,79 +1164,79 @@ bld.SAMBA3_BINARY('net',
 
 bld.SAMBA3_BINARY('profiles',
                  source=PROFILES_SRC,
-                 deps='''talloc tdb cap tevent POPT_SAMBA3 param samba3core LIBSMB_ERR LIB_NONSMBD REGFIO''',
+                 deps='''talloc tdb_compat cap tevent POPT_SAMBA3 param samba3core LIBSMB_ERR smbd_shim REGFIO''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbspool',
                  source=CUPS_SRC,
-                 deps='''talloc tdb tevent cap wbclient POPT_SAMBA3 param LIBSMB LIB_NONSMBD samba3core
+                 deps='''talloc tdb_compat tevent cap wbclient POPT_SAMBA3 param LIBSMB smbd_shim samba3core
                  KRBCLIENT asn1util LIBTSOCKET NDR_SAMR NDR_LSA''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('testparm',
                  source=TESTPARM_SRC,
                  deps='''talloc tevent ldap cap 
-                 wbclient asn1util LIBTSOCKET passdb param LIB_NONSMBD
+                 wbclient asn1util LIBTSOCKET passdb param smbd_shim
                  LIBSMB_ERR POPT_SAMBA3''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbta-util',
                  source=SMBTA_UTIL_SRC,
-                 deps='''talloc tdb tevent cap param POPT_SAMBA3 LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap param POPT_SAMBA3 smbd_shim
                  LIBSMB_ERR''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbstatus',
                  source=STATUS_SRC,
-                 deps='''talloc tdb tevent cap param POPT_SAMBA3 LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap param POPT_SAMBA3 smbd_shim
                  LIBSMB_ERR LOCKING PROFILE FNAME_UTIL''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbcontrol',
                  source=SMBCONTROL_SRC,
-                 deps='''talloc tdb tevent cap param LIB_NONSMBD LIBSMB_ERR POPT_SAMBA3 PRINTBASE''',
+                 deps='''talloc tdb_compat tevent cap param smbd_shim LIBSMB_ERR POPT_SAMBA3 PRINTBASE''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbtree',
                  source=SMBTREE_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap wbclient param smbd_shim
                  LIBSMB LIBSMB_ERR POPT_SAMBA3 KRBCLIENT passdb SMBLDAP LIBMSRPC_GEN msrpc3 PROFILE
                  RPC_NDR_SRVSVC''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbpasswd',
                  source=SMBPASSWD_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap wbclient param smbd_shim
                  LIBSMB LIBSMB_ERR POPT_SAMBA3 KRBCLIENT passdb SMBLDAP LIBMSRPC_GEN msrpc3 PASSWD_UTIL
                  LIBCLI_SAMR INIT_LSA PASSCHANGE''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('pdbedit',
                  source=PDBEDIT_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap wbclient param smbd_shim
                  LIBNTLMSSP LIBSMB_ERR POPT_SAMBA3 passdb SMBLDAP
                  PASSWD_UTIL cli-ldap-common''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbget',
                  source=SMBGET_SRC,
-                 deps='''talloc tdb libsmb/smbclient POPT_SAMBA3''',
+                 deps='''talloc tdb_compat libsmb/smbclient POPT_SAMBA3''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('nmblookup' + bld.env.suffix3,
                  source=NMBLOOKUP_SRC,
-                 deps='''talloc tdb tevent cap param LIB_NONSMBD POPT_SAMBA3 LIBSMB_ERR LIBNMB''',
+                 deps='''talloc tdb_compat tevent cap param smbd_shim POPT_SAMBA3 LIBSMB_ERR LIBNMB''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbtorture' + bld.env.suffix3,
                  source=SMBTORTURE_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIBSMB KRBCLIENT TLDAP
-                 LIB_NONSMBD POPT_SAMBA3 asn1util LIBTSOCKET NDR_LSA msrpc3 LIBMSRPC_GEN RPC_NDR_ECHO WB_REQTRANS''',
+                 deps='''talloc tdb_compat tevent cap wbclient param LIBSMB KRBCLIENT TLDAP
+                 smbd_shim POPT_SAMBA3 asn1util LIBTSOCKET NDR_LSA msrpc3 LIBMSRPC_GEN RPC_NDR_ECHO WB_REQTRANS''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbconftort',
                  source=SMBCONFTORT_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIB_NONSMBD LIBSMB_ERR POPT_SAMBA3''',
+                 deps='''talloc tdb_compat tevent cap wbclient param smbd_shim LIBSMB_ERR POPT_SAMBA3''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('replacetort',
@@ -1169,25 +1246,25 @@ bld.SAMBA3_BINARY('replacetort',
 
 bld.SAMBA3_BINARY('masktest' + bld.env.suffix3,
                  source=MASKTEST_SRC,
-                 deps='''talloc tdb cap wbclient param LIB_NONSMBD LIBSMB KRBCLIENT
+                 deps='''talloc tdb_compat cap wbclient param smbd_shim LIBSMB KRBCLIENT
                  asn1util LIBTSOCKET NDR_SAMR NDR_LSA''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('msgtest',
                  source=MSGTEST_SRC,
-                 deps='''talloc tdb tevent cap param LIBSMB_ERR LIB_NONSMBD''',
+                 deps='''talloc tdb_compat tevent cap param LIBSMB_ERR smbd_shim''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbcacls',
                  source=SMBCACLS_SRC,
-                 deps='''talloc tdb cap wbclient param LIBSMB KRBCLIENT
-                 LIB_NONSMBD passdb POPT_SAMBA3 SMBLDAP LIBMSRPC_GEN
+                 deps='''talloc tdb_compat cap wbclient param LIBSMB KRBCLIENT
+                 smbd_shim passdb POPT_SAMBA3 SMBLDAP LIBMSRPC_GEN
                  msrpc3 LIBCLI_LSA3''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('smbcquotas',
                  source=SMBCQUOTAS_SRC,
-                 deps='''talloc tdb cap wbclient param LIB_NONSMBD LIBSMB KRBCLIENT
+                 deps='''talloc tdb_compat cap wbclient param  smbd_shim LIBSMB KRBCLIENT
                  POPT_SAMBA3 passdb SMBLDAP LIBMSRPC_GEN msrpc3
                  LIBCLI_LSA3''',
                  vars=locals())
@@ -1195,23 +1272,23 @@ bld.SAMBA3_BINARY('smbcquotas',
 bld.SAMBA3_BINARY('eventlogadm',
                  source=EVTLOGADM_SRC,
                  deps='''talloc tevent cap POPT_SAMBA3 samba3core param
-                 LIB_NONSMBD LIBSMB_ERR passdb wbclient LIBEVENTLOG''',
+                 smbd_shim LIBSMB_ERR passdb wbclient LIBEVENTLOG''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('sharesec',
                  source=SHARESEC_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIB_NONSMBD LIBSMB_ERR POPT_SAMBA3''',
+                 deps='''talloc tdb_compat tevent cap wbclient param smbd_shim LIBSMB_ERR POPT_SAMBA3''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('locktest' + bld.env.suffix3,
                  source=LOCKTEST_SRC,
-                 deps='''talloc tdb tevent cap wbclient param KRBCLIENT LIBSMB LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap wbclient param KRBCLIENT LIBSMB smbd_shim
                  asn1util LIBTSOCKET NDR_SAMR NDR_LSA LOCKING FNAME_UTIL''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('pdbtest',
                  source=PDBTEST_SRC,
-                 deps='''talloc tdb cap wbclient param LIBSMB KRBCLIENT LIB_NONSMBD passdb
+                 deps='''talloc tdb_compat cap wbclient param LIBSMB KRBCLIENT smbd_shim passdb
                  SMBLDAP POPT_SAMBA3 NDR_SAMR NDR_LSA''',
                  vars=locals())
 
@@ -1222,7 +1299,7 @@ bld.SAMBA3_BINARY('vfstest',
 
 bld.SAMBA3_BINARY('smbiconv',
                  source=SMBICONV_SRC,
-                 deps='''talloc tdb tevent cap param LIB_NONSMBD POPT_SAMBA3 LIBSMB_ERR''',
+                 deps='''talloc tdb_compat tevent cap param smbd_shim POPT_SAMBA3 LIBSMB_ERR''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('log2pcap',
@@ -1232,7 +1309,7 @@ bld.SAMBA3_BINARY('log2pcap',
 
 bld.SAMBA3_BINARY('locktest2',
                  source=LOCKTEST2_SRC,
-                 deps='''talloc tdb tevent cap wbclient param KRBCLIENT LIBSMB LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap wbclient param KRBCLIENT LIBSMB smbd_shim
                  asn1util LIBTSOCKET NDR_SAMR NDR_LSA LOCKING FNAME_UTIL''',
                  vars=locals())
 
@@ -1243,25 +1320,25 @@ bld.SAMBA3_BINARY('debug2html',
 
 bld.SAMBA3_BINARY('smbfilter',
                  source=SMBFILTER_SRC,
-                 deps='''talloc tevent cap wbclient param LIBSMB LIB_NONSMBD KRBCLIENT asn1util LIBTSOCKET
+                 deps='''talloc tevent cap wbclient param LIBSMB smbd_shim KRBCLIENT asn1util LIBTSOCKET
                  NDR_SAMR NDR_LSA''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('versiontest',
                  source='lib/version_test.c',
-                 deps='SAMBA_VERSION',
+                 deps='SAMBA_VERSION samba3core param',
                  vars=locals())
 
 bld.SAMBA3_BINARY('wbinfo' + bld.env.suffix3,
                  source=WBINFO_SRC,
                  deps='''talloc wbclient tevent cap
-                 asn1util LIBTSOCKET passdb ldap param LIB_NONSMBD
+                 asn1util LIBTSOCKET passdb ldap param smbd_shim
                  LIBNTLMSSP POPT_SAMBA3 LIBAFS_SETTOKEN''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('ntlm_auth' + bld.env.suffix3,
                  source=NTLM_AUTH_SRC,
-                 deps='''tdb talloc cap krb5 k5crypto com_err wbclient param LIB_NONSMBD
+                 deps='''tdb_compat talloc cap KRB5_WRAP k5crypto wbclient param smbd_shim
                  samba3core LIBNTLMSSP POPT_SAMBA3 asn1util LIBTSOCKET
                  passdb SMBLDAP winbind-client LIBINIPARSER LIBADS_SERVER
                  NDR_SAMR NDR_LSA NDR_NETLOGON cli-ldap-common LIBNMB SLCACHE SPNEGO_PARSE KRBCLIENT''',
@@ -1273,35 +1350,35 @@ bld.SAMBA3_BINARY('timelimit',
 
 bld.SAMBA3_BINARY('rpc_open_tcp',
                  source=RPC_OPEN_TCP_SRC,
-                 deps='''talloc tdb tevent cap wbclient KRBCLIENT param samba3core LIBSMB LIB_NONSMBD
+                 deps='''talloc tdb_compat tevent cap wbclient KRBCLIENT param samba3core LIBSMB smbd_shim
                  LIBSMB_ERR asn1util LIBTSOCKET LIBMSRPC_GEN msrpc3''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('test_lp_load',
                  source=TEST_LP_LOAD_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIB_NONSMBD POPT_SAMBA3 LIBNTLMSSP''',
+                 deps='''talloc tdb_compat tevent cap wbclient param  smbd_shim POPT_SAMBA3 LIBNTLMSSP''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('dbwrap_tool',
                  source=DBWRAP_TOOL_SRC,
-                 deps='''talloc tdb tevent cap param LIB_NONSMBD LIBSMB_ERR''',
+                 deps='''talloc tdb_compat tevent cap param smbd_shim LIBSMB_ERR''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('dbwrap_torture',
                  source=DBWRAP_TORTURE_SRC,
-                 deps='''talloc tdb tevent cap param LIB_NONSMBD LIBSMB_ERR
+                 deps='''talloc tdb_compat tevent cap param  smbd_shim LIBSMB_ERR
                  POPT_SAMBA3''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('split_tokens',
                  source=SPLIT_TOKENS_SRC,
-                 deps='''talloc tdb tevent cap wbclient param LIB_NONSMBD POPT_SAMBA3
+                 deps='''talloc tdb_compat tevent cap wbclient param smbd_shim POPT_SAMBA3
                  LIBNTLMSSP''',
                  vars=locals())
 
 bld.SAMBA3_BINARY('vlp',
                  source=VLP_SRC,
-                 deps='''talloc tdb PARAM_UTIL UTIL_TDB''',
+                 deps='''talloc tdb_compat PARAM_UTIL UTIL_TDB samba3core param''',
                  vars=locals())
 
 swat_dir = os.path.join(bld.curdir, '../swat')
@@ -1314,32 +1391,20 @@ bld.INSTALL_FILES('${SWATDIR}', swat_files, base_name='../swat')
 
 if not bld.env.toplevel_build:
     bld.SAMBA3_SUBSYSTEM('POPT_SAMBA', source='', deps='POPT_SAMBA3')
-    bld.SAMBA3_SUBSYSTEM('tdb-wrap', source='', deps='tdb-wrap3')
-    bld.SAMBA3_SUBSYSTEM('errors', source='', deps='errors3')
-    bld.SAMBA3_SUBSYSTEM('samba-util', source='', deps='DYNCONFIG')
-    bld.SAMBA3_SUBSYSTEM('CHARSET', source='', deps='CHARSET3')
     bld.SAMBA3_SUBSYSTEM('ldb', source='', deps='ldb3')
     bld.SAMBA3_SUBSYSTEM('dcerpc', '', deps='UTIL_TEVENT')
     bld.SAMBA3_SUBSYSTEM('cli-ldap', '', deps='UTIL_TEVENT')
-else:
-
-    # point the s3 rules at in-tree heimdal
-    bld.SAMBA3_SUBSYSTEM('gssapi_krb5',
-                         source='',
-                         deps='gssapi krb5')
-    bld.SAMBA3_SUBSYSTEM('k5crypto',
-                         source='',
-                         deps='krb5')
-
-
-
+    bld.SAMBA3_SUBSYSTEM('LOADPARM_CTX', '')
 
 ########################## INCLUDES #################################
 
 bld.RECURSE('../lib/util/charset')
 bld.RECURSE('../auth')
+bld.RECURSE('../auth/kerberos')
 bld.RECURSE('../lib/addns')
 bld.RECURSE('../lib/async_req')
+bld.RECURSE('../lib/ccan')
+bld.RECURSE('../lib/socket')
 bld.RECURSE('../libcli/auth')
 bld.RECURSE('../libcli/drsuapi')
 bld.RECURSE('../libcli/ldap')
@@ -1349,6 +1414,7 @@ bld.RECURSE('../libcli/netlogon')
 bld.RECURSE('../libcli/samsync')
 bld.RECURSE('../libcli/security')
 bld.RECURSE('../libcli/smb')
+bld.RECURSE('../libcli/util')
 bld.RECURSE('../libcli/smbreadline')
 bld.RECURSE('../libcli/nbt')
 bld.RECURSE('../libcli/registry')
@@ -1361,7 +1427,7 @@ bld.RECURSE('../lib/smbconf')
 bld.RECURSE('../lib/socket_wrapper')
 bld.RECURSE('../lib/uid_wrapper')
 bld.RECURSE('../lib/talloc')
-bld.RECURSE('../lib/tdb')
+bld.RECURSE('../lib/tdb_compat')
 bld.RECURSE('../lib/util')
 bld.RECURSE('../lib/tevent')
 bld.RECURSE('../lib/tsocket')
@@ -1371,6 +1437,7 @@ bld.RECURSE('../nsswitch')
 bld.RECURSE('../nsswitch/libwbclient')
 bld.RECURSE('auth')
 bld.RECURSE('libgpo/gpext')
+bld.RECURSE('lib/pthreadpool')
 bld.RECURSE('librpc')
 bld.RECURSE('librpc/idl')
 bld.RECURSE('modules')
diff --git a/source4/NEWS b/source4/NEWS
index b441bc784fa..f7c03c60bfd 100644
--- a/source4/NEWS
+++ b/source4/NEWS
@@ -56,8 +56,7 @@ that can be either 'domain controller', 'member server' or 'standalone'. Note th
 member server support does not work yet.
 
 The following parameters have been removed:
-- passdb backend: accounts are now stored in a LDB-based SAM database, 
-	see 'sam database' below.
+- passdb backend: accounts are now stored in a LDB-based SAM database
 - update encrypted
 - public
 - guest ok
@@ -401,12 +400,6 @@ The following parameters have been added:
 
 	Default: smb rpc nbt wrepl ldap cldap web kdc
 
-+ sam database
-	Location of the SAM (account database) database. This should be a 
-	LDB URL.
-
-	Default: set at compile-time
-
 + spoolss database
 	Spoolss (printer) DCE/RPC server database. This should be a LDB URL.
 
diff --git a/source4/auth/auth.h b/source4/auth/auth.h
index 9eb3e7db6fe..04731af0190 100644
--- a/source4/auth/auth.h
+++ b/source4/auth/auth.h
@@ -43,7 +43,7 @@ struct loadparm_context;
 /* version 3 - subsequent samba4 version - abartlet */
 /* version 4 - subsequent samba4 version - metze */
 /* version 0 - till samba4 is stable - metze */
-#define AUTH_INTERFACE_VERSION 0
+#define AUTH4_INTERFACE_VERSION 0
 
 #define AUTH_SESSION_INFO_DEFAULT_GROUPS     0x01 /* Add the user to the default world and network groups */
 #define AUTH_SESSION_INFO_AUTHENTICATED      0x02 /* Add the user to the 'authenticated users' group */
@@ -51,7 +51,7 @@ struct loadparm_context;
 
 struct auth_method_context;
 struct auth_check_password_request;
-struct auth_context;
+struct auth4_context;
 struct auth_session_info;
 struct ldb_dn;
 
@@ -78,7 +78,7 @@ struct auth_operations {
 
 	/* Lookup a 'session info interim' return based only on the principal or DN */
 	NTSTATUS (*get_user_info_dc_principal)(TALLOC_CTX *mem_ctx,
-						       struct auth_context *auth_context,
+						       struct auth4_context *auth_context,
 						       const char *principal,
 						       struct ldb_dn *user_dn,
 						       struct auth_user_info_dc **interim_info);
@@ -86,13 +86,13 @@ struct auth_operations {
 
 struct auth_method_context {
 	struct auth_method_context *prev, *next;
-	struct auth_context *auth_ctx;
+	struct auth4_context *auth_ctx;
 	const struct auth_operations *ops;
 	int depth;
 	void *private_data;
 };
 
-struct auth_context {
+struct auth4_context {
 	struct {
 		/* Who set this up in the first place? */
 		const char *set_by;
@@ -109,7 +109,7 @@ struct auth_context {
 	struct tevent_context *event_ctx;
 
 	/* the messaging context which can be used by backends */
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 
 	/* loadparm context */
 	struct loadparm_context *lp_ctx;
@@ -117,25 +117,25 @@ struct auth_context {
 	/* SAM database for this local machine - to fill in local groups, or to authenticate local NTLM users */
 	struct ldb_context *sam_ctx;
 
-	NTSTATUS (*check_password)(struct auth_context *auth_ctx,
+	NTSTATUS (*check_password)(struct auth4_context *auth_ctx,
 				   TALLOC_CTX *mem_ctx,
 				   const struct auth_usersupplied_info *user_info,
 				   struct auth_user_info_dc **user_info_dc);
 
-	NTSTATUS (*get_challenge)(struct auth_context *auth_ctx, uint8_t chal[8]);
+	NTSTATUS (*get_challenge)(struct auth4_context *auth_ctx, uint8_t chal[8]);
 
-	bool (*challenge_may_be_modified)(struct auth_context *auth_ctx);
+	bool (*challenge_may_be_modified)(struct auth4_context *auth_ctx);
 
-	NTSTATUS (*set_challenge)(struct auth_context *auth_ctx, const uint8_t chal[8], const char *set_by);
+	NTSTATUS (*set_challenge)(struct auth4_context *auth_ctx, const uint8_t chal[8], const char *set_by);
 
 	NTSTATUS (*get_user_info_dc_principal)(TALLOC_CTX *mem_ctx,
-						       struct auth_context *auth_ctx,
+						       struct auth4_context *auth_ctx,
 						       const char *principal,
 						       struct ldb_dn *user_dn,
 						       struct auth_user_info_dc **user_info_dc);
 
 	NTSTATUS (*generate_session_info)(TALLOC_CTX *mem_ctx,
-					  struct auth_context *auth_context,
+					  struct auth4_context *auth_context,
 					  struct auth_user_info_dc *user_info_dc,
 					  uint32_t session_info_flags,
 					  struct auth_session_info **session_info);
@@ -151,7 +151,7 @@ struct auth_critical_sizes {
 	int sizeof_auth_user_info_dc;
 };
 
- NTSTATUS encrypt_user_info(TALLOC_CTX *mem_ctx, struct auth_context *auth_context,
+ NTSTATUS encrypt_user_info(TALLOC_CTX *mem_ctx, struct auth4_context *auth_context,
 			   enum auth_password_state to_state,
 			   const struct auth_usersupplied_info *user_info_in,
 			   const struct auth_usersupplied_info **user_info_encrypted);
@@ -165,7 +165,7 @@ struct ldb_context;
 struct gensec_security;
 struct cli_credentials;
 
-NTSTATUS auth_get_challenge(struct auth_context *auth_ctx, uint8_t chal[8]);
+NTSTATUS auth_get_challenge(struct auth4_context *auth_ctx, uint8_t chal[8]);
 NTSTATUS authsam_account_ok(TALLOC_CTX *mem_ctx,
 			    struct ldb_context *sam_ctx,
 			    uint32_t logon_parameters,
@@ -175,10 +175,7 @@ NTSTATUS authsam_account_ok(TALLOC_CTX *mem_ctx,
 			    const char *name_for_logs,
 			    bool allow_domain_trust,
 			    bool password_change);
-NTSTATUS authsam_expand_nested_groups(struct ldb_context *sam_ctx,
-				      struct ldb_val *dn_val, const bool only_childs, const char *filter,
-				      TALLOC_CTX *res_sids_ctx, struct dom_sid ***res_sids,
-				      unsigned int *num_res_sids);
+
 struct auth_session_info *system_session(struct loadparm_context *lp_ctx);
 NTSTATUS authsam_make_user_info_dc(TALLOC_CTX *mem_ctx, struct ldb_context *sam_ctx,
 					   const char *netbios_name,
@@ -193,20 +190,20 @@ NTSTATUS auth_system_session_info(TALLOC_CTX *parent_ctx,
 
 NTSTATUS auth_context_create_methods(TALLOC_CTX *mem_ctx, const char **methods,
 				     struct tevent_context *ev,
-				     struct messaging_context *msg,
+				     struct imessaging_context *msg,
 				     struct loadparm_context *lp_ctx,
 				     struct ldb_context *sam_ctx,
-				     struct auth_context **auth_ctx);
+				     struct auth4_context **auth_ctx);
 const char **auth_methods_from_lp(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx);
 
 NTSTATUS auth_context_create(TALLOC_CTX *mem_ctx,
 			     struct tevent_context *ev,
-			     struct messaging_context *msg,
+			     struct imessaging_context *msg,
 			     struct loadparm_context *lp_ctx,
-			     struct auth_context **auth_ctx);
-NTSTATUS auth_context_create_from_ldb(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, struct auth_context **auth_ctx);
+			     struct auth4_context **auth_ctx);
+NTSTATUS auth_context_create_from_ldb(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, struct auth4_context **auth_ctx);
 
-NTSTATUS auth_check_password(struct auth_context *auth_ctx,
+NTSTATUS auth_check_password(struct auth4_context *auth_ctx,
 			     TALLOC_CTX *mem_ctx,
 			     const struct auth_usersupplied_info *user_info,
 			     struct auth_user_info_dc **user_info_dc);
@@ -215,7 +212,7 @@ NTSTATUS auth_register(const struct auth_operations *ops);
 NTSTATUS server_service_auth_init(void);
 NTSTATUS authenticate_username_pw(TALLOC_CTX *mem_ctx,
 				  struct tevent_context *ev,
-				  struct messaging_context *msg,
+				  struct imessaging_context *msg,
 				  struct loadparm_context *lp_ctx,
 				  const char *nt4_domain,
 				  const char *nt4_username,
@@ -225,24 +222,24 @@ NTSTATUS authenticate_username_pw(TALLOC_CTX *mem_ctx,
 
 struct tevent_req *auth_check_password_send(TALLOC_CTX *mem_ctx,
 					    struct tevent_context *ev,
-					    struct auth_context *auth_ctx,
+					    struct auth4_context *auth_ctx,
 					    const struct auth_usersupplied_info *user_info);
 NTSTATUS auth_check_password_recv(struct tevent_req *req,
 				  TALLOC_CTX *mem_ctx,
 				  struct auth_user_info_dc **user_info_dc);
 
-bool auth_challenge_may_be_modified(struct auth_context *auth_ctx);
-NTSTATUS auth_context_set_challenge(struct auth_context *auth_ctx, const uint8_t chal[8], const char *set_by);
+bool auth_challenge_may_be_modified(struct auth4_context *auth_ctx);
+NTSTATUS auth_context_set_challenge(struct auth4_context *auth_ctx, const uint8_t chal[8], const char *set_by);
 
 NTSTATUS auth_get_user_info_dc_principal(TALLOC_CTX *mem_ctx,
-					struct auth_context *auth_ctx,
+					struct auth4_context *auth_ctx,
 					const char *principal,
 					struct ldb_dn *user_dn,
 					struct auth_user_info_dc **user_info_dc);
 
 NTSTATUS samba_server_gensec_start(TALLOC_CTX *mem_ctx,
 				   struct tevent_context *event_ctx,
-				   struct messaging_context *msg_ctx,
+				   struct imessaging_context *msg_ctx,
 				   struct loadparm_context *lp_ctx,
 				   struct cli_credentials *server_credentials,
 				   const char *target_service,
diff --git a/source4/auth/credentials/credentials.c b/source4/auth/credentials/credentials.c
index 015c5496931..83e90344bfc 100644
--- a/source4/auth/credentials/credentials.c
+++ b/source4/auth/credentials/credentials.c
@@ -64,6 +64,7 @@ _PUBLIC_ struct cli_credentials *cli_credentials_init(TALLOC_CTX *mem_ctx)
 	cred->principal = NULL;
 	cred->salt_principal = NULL;
 	cred->impersonate_principal = NULL;
+	cred->self_service = NULL;
 	cred->target_service = NULL;
 
 	cred->bind_dn = NULL;
diff --git a/source4/auth/credentials/credentials.h b/source4/auth/credentials/credentials.h
index 0b0de597523..f8fa2f864b9 100644
--- a/source4/auth/credentials/credentials.h
+++ b/source4/auth/credentials/credentials.h
@@ -84,6 +84,7 @@ struct cli_credentials {
 	const char *principal;
 	char *salt_principal;
 	char *impersonate_principal;
+	char *self_service;
 	char *target_service;
 
 	const char *bind_dn;
@@ -277,10 +278,13 @@ bool cli_credentials_parse_password_fd(struct cli_credentials *credentials,
 void cli_credentials_invalidate_ccache(struct cli_credentials *cred, 
 				       enum credentials_obtained obtained);
 void cli_credentials_set_salt_principal(struct cli_credentials *cred, const char *principal);
-void cli_credentials_set_impersonate_principal(struct cli_credentials *cred, const char *principal);
+void cli_credentials_set_impersonate_principal(struct cli_credentials *cred,
+					       const char *principal,
+					       const char *self_service);
 void cli_credentials_set_target_service(struct cli_credentials *cred, const char *principal);
 const char *cli_credentials_get_salt_principal(struct cli_credentials *cred);
 const char *cli_credentials_get_impersonate_principal(struct cli_credentials *cred);
+const char *cli_credentials_get_self_service(struct cli_credentials *cred);
 const char *cli_credentials_get_target_service(struct cli_credentials *cred);
 enum credentials_use_kerberos cli_credentials_get_kerberos_state(struct cli_credentials *creds);
 enum credentials_krb_forwardable cli_credentials_get_krb_forwardable(struct cli_credentials *creds);
diff --git a/source4/auth/credentials/credentials_krb5.c b/source4/auth/credentials/credentials_krb5.c
index d3925a01f67..1643197004c 100644
--- a/source4/auth/credentials/credentials_krb5.c
+++ b/source4/auth/credentials/credentials_krb5.c
@@ -235,9 +235,15 @@ static int cli_credentials_new_ccache(struct cli_credentials *cred,
 
 	if (!ccache_name) {
 		must_free_cc_name = true;
-		ccache_name = talloc_asprintf(ccc, "MEMORY:%p", 
-					      ccc);
-		
+
+		if (lpcfg_parm_bool(lp_ctx, NULL, "credentials", "krb5_cc_file", false)) {
+			ccache_name = talloc_asprintf(ccc, "FILE:/tmp/krb5_cc_samba_%u_%p", 
+						      (unsigned int)getpid(), ccc);
+		} else {
+			ccache_name = talloc_asprintf(ccc, "MEMORY:%p", 
+						      ccc);
+		}
+
 		if (!ccache_name) {
 			talloc_free(ccc);
 			(*error_string) = strerror(ENOMEM);
@@ -288,8 +294,38 @@ _PUBLIC_ int cli_credentials_get_named_ccache(struct cli_credentials *cred,
 
 	if (cred->ccache_obtained >= cred->ccache_threshold && 
 	    cred->ccache_obtained > CRED_UNINITIALISED) {
-		*ccc = cred->ccache;
-		return 0;
+		time_t lifetime;
+		bool expired = false;
+		ret = krb5_cc_get_lifetime(cred->ccache->smb_krb5_context->krb5_context, 
+					   cred->ccache->ccache, &lifetime);
+		if (ret == KRB5_CC_END) {
+			/* If we have a particular ccache set, without
+			 * an initial ticket, then assume there is a
+			 * good reason */
+		} else if (ret == 0) {
+			if (lifetime == 0) {
+				DEBUG(3, ("Ticket in credentials cache for %s expired, will refresh\n",
+					  cli_credentials_get_principal(cred, cred)));
+				expired = true;
+			} else if (lifetime < 300) {
+				DEBUG(3, ("Ticket in credentials cache for %s will shortly expire (%u secs), will refresh\n", 
+					  cli_credentials_get_principal(cred, cred), (unsigned int)lifetime));
+				expired = true;
+			}
+		} else {
+			(*error_string) = talloc_asprintf(cred, "failed to get ccache lifetime: %s\n",
+							  smb_get_krb5_error_message(cred->ccache->smb_krb5_context->krb5_context,
+										     ret, cred));
+			return ret;
+		}
+
+		DEBUG(5, ("Ticket in credentials cache for %s will expire in %u secs\n", 
+			  cli_credentials_get_principal(cred, cred), (unsigned int)lifetime));
+		
+		if (!expired) {
+			*ccc = cred->ccache;
+			return 0;
+		}
 	}
 	if (cli_credentials_is_anonymous(cred)) {
 		(*error_string) = "Cannot get anonymous kerberos credentials";
@@ -351,7 +387,7 @@ void cli_credentials_invalidate_client_gss_creds(struct cli_credentials *cred,
 	}
 	/* Now that we know that the data is 'this specified', then
 	 * don't allow something less 'known' to be returned as a
-	 * ccache.  Ie, if the username is on the commmand line, we
+	 * ccache.  Ie, if the username is on the command line, we
 	 * don't want to later guess to use a file-based ccache */
 	if (obtained > cred->client_gss_creds_threshold) {
 		cred->client_gss_creds_threshold = obtained;
@@ -384,7 +420,7 @@ _PUBLIC_ void cli_credentials_invalidate_ccache(struct cli_credentials *cred,
 	}
 	/* Now that we know that the data is 'this specified', then
 	 * don't allow something less 'known' to be returned as a
-	 * ccache.  Ie, if the username is on the commmand line, we
+	 * ccache.  i.e, if the username is on the command line, we
 	 * don't want to later guess to use a file-based ccache */
 	if (obtained > cred->ccache_threshold) {
 		cred->ccache_threshold  = obtained;
@@ -416,14 +452,41 @@ _PUBLIC_ int cli_credentials_get_client_gss_creds(struct cli_credentials *cred,
 
 	if (cred->client_gss_creds_obtained >= cred->client_gss_creds_threshold && 
 	    cred->client_gss_creds_obtained > CRED_UNINITIALISED) {
-		*_gcc = cred->client_gss_creds;
-		return 0;
+		bool expired = false;
+		OM_uint32 lifetime = 0;
+		gss_cred_usage_t usage = 0;
+		maj_stat = gss_inquire_cred(&min_stat, cred->client_gss_creds->creds, 
+					    NULL, &lifetime, &usage, NULL);
+		if (maj_stat == GSS_S_CREDENTIALS_EXPIRED) {
+			DEBUG(3, ("Credentials for %s expired, must refresh credentials cache\n", cli_credentials_get_principal(cred, cred)));
+			expired = true;
+		} else if (maj_stat == GSS_S_COMPLETE && lifetime < 300) {
+			DEBUG(3, ("Credentials for %s will expire shortly (%u sec), must refresh credentials cache\n", cli_credentials_get_principal(cred, cred), lifetime));
+			expired = true;
+		} else if (maj_stat != GSS_S_COMPLETE) {
+			*error_string = talloc_asprintf(cred, "inquiry of credential lifefime via GSSAPI gss_inquire_cred failed: %s\n",
+							gssapi_error_string(cred, maj_stat, min_stat, NULL));
+			return EINVAL;
+		}
+		if (expired) {
+			cli_credentials_unconditionally_invalidate_client_gss_creds(cred);
+		} else {
+			DEBUG(5, ("GSSAPI credentials for %s will expire in %u secs\n", 
+				  cli_credentials_get_principal(cred, cred), (unsigned int)lifetime));
+		
+			*_gcc = cred->client_gss_creds;
+			return 0;
+		}
 	}
 
 	ret = cli_credentials_get_ccache(cred, event_ctx, lp_ctx,
 					 &ccache, error_string);
 	if (ret) {
-		DEBUG(1, ("Failed to get CCACHE for GSSAPI client: %s\n", error_message(ret)));
+		if (cli_credentials_get_kerberos_state(cred) == CRED_MUST_USE_KERBEROS) {
+			DEBUG(1, ("Failed to get kerberos credentials (kerberos required): %s\n", error_message(ret)));
+		} else {
+			DEBUG(4, ("Failed to get kerberos credentials: %s\n", error_message(ret)));
+		}
 		return ret;
 	}
 
@@ -781,26 +844,43 @@ _PUBLIC_ void cli_credentials_set_salt_principal(struct cli_credentials *cred, c
 	cred->salt_principal = talloc_strdup(cred, principal);
 }
 
-/* The 'impersonate_principal' is used to allow on Kerberos principal
+/* The 'impersonate_principal' is used to allow one Kerberos principal
  * (and it's associated keytab etc) to impersonate another.  The
  * ability to do this is controlled by the KDC, but it is generally
  * permitted to impersonate anyone to yourself.  This allows any
  * member of the domain to get the groups of a user.  This is also
  * known as S4U2Self */
 
-const char *cli_credentials_get_impersonate_principal(struct cli_credentials *cred)
+_PUBLIC_ const char *cli_credentials_get_impersonate_principal(struct cli_credentials *cred)
 {
 	return cred->impersonate_principal;
 }
 
-_PUBLIC_ void cli_credentials_set_impersonate_principal(struct cli_credentials *cred, const char *principal)
+/*
+ * The 'self_service' is the service principal that
+ * represents the same object (by its objectSid)
+ * as the client principal (typically our machine account).
+ * When trying to impersonate 'impersonate_principal' with
+ * S4U2Self.
+ */
+_PUBLIC_ const char *cli_credentials_get_self_service(struct cli_credentials *cred)
+{
+	return cred->self_service;
+}
+
+_PUBLIC_ void cli_credentials_set_impersonate_principal(struct cli_credentials *cred,
+							const char *principal,
+							const char *self_service)
 {
 	talloc_free(cred->impersonate_principal);
 	cred->impersonate_principal = talloc_strdup(cred, principal);
+	talloc_free(cred->self_service);
+	cred->self_service = talloc_strdup(cred, self_service);
+	cli_credentials_set_kerberos_state(cred, CRED_MUST_USE_KERBEROS);
 }
 
-/* when impersonating for S4U2Self we need to set the target principal
- * to ourself, as otherwise we would need additional rights.
+/*
+ * when impersonating for S4U2proxy we need to set the target principal.
  * Similarly, we may only be authorized to do general impersonation to
  * some particular services.
  *
diff --git a/source4/auth/gensec/cyrus_sasl.c b/source4/auth/gensec/cyrus_sasl.c
index bd7664878c7..4a4422645d8 100644
--- a/source4/auth/gensec/cyrus_sasl.c
+++ b/source4/auth/gensec/cyrus_sasl.c
@@ -99,12 +99,12 @@ static int gensec_sasl_get_password(sasl_conn_t *conn, void *context, int id,
 		*psecret = NULL;
 		return SASL_OK;
 	}
-	secret = talloc_size(gensec_security, sizeof(sasl_secret_t)+strlen(password));
+	secret = talloc_size(gensec_security, sizeof(sasl_secret_t)+strlen(password)+1);
 	if (!secret) {
 		return SASL_NOMEM;
 	}
 	secret->len = strlen(password);
-	safe_strcpy((char*)secret->data, password, secret->len+1);
+	strlcpy((char*)secret->data, password, secret->len+1);
 	*psecret = secret;
 	return SASL_OK;
 }
diff --git a/source4/auth/gensec/gensec.c b/source4/auth/gensec/gensec.c
index e632aec2dc0..7e6a83d51f5 100644
--- a/source4/auth/gensec/gensec.c
+++ b/source4/auth/gensec/gensec.c
@@ -514,7 +514,7 @@ const char **gensec_security_oids(struct gensec_security *gensec_security,
 static NTSTATUS gensec_start(TALLOC_CTX *mem_ctx, 
 			     struct tevent_context *ev,
 			     struct gensec_settings *settings,
- 			     struct auth_context *auth_context,
+ 			     struct auth4_context *auth_context,
 			     struct gensec_security **gensec_security)
 {
 	if (ev == NULL) {
@@ -604,7 +604,7 @@ _PUBLIC_ NTSTATUS gensec_client_start(TALLOC_CTX *mem_ctx,
 _PUBLIC_ NTSTATUS gensec_server_start(TALLOC_CTX *mem_ctx, 
 				      struct tevent_context *ev,
 				      struct gensec_settings *settings,
-				      struct auth_context *auth_context,
+				      struct auth4_context *auth_context,
 				      struct gensec_security **gensec_security)
 {
 	NTSTATUS status;
@@ -639,7 +639,7 @@ static NTSTATUS gensec_start_mech(struct gensec_security *gensec_security)
 		if (gensec_security->ops->client_start) {
 			status = gensec_security->ops->client_start(gensec_security);
 			if (!NT_STATUS_IS_OK(status)) {
-				DEBUG(2, ("Failed to start GENSEC client mech %s: %s\n",
+				DEBUG(gensec_security->subcontext?4:2, ("Failed to start GENSEC client mech %s: %s\n",
 					  gensec_security->ops->name, nt_errstr(status))); 
 			}
 			return status;
@@ -1406,7 +1406,7 @@ bool gensec_setting_bool(struct gensec_settings *settings, const char *mechanism
 /*
   initialise the GENSEC subsystem
 */
-_PUBLIC_ NTSTATUS gensec_init(struct loadparm_context *lp_ctx)
+_PUBLIC_ NTSTATUS gensec_init(void)
 {
 	static bool initialized = false;
 #define _MODULE_PROTO(init) extern NTSTATUS init(void);
@@ -1417,7 +1417,7 @@ _PUBLIC_ NTSTATUS gensec_init(struct loadparm_context *lp_ctx)
 	if (initialized) return NT_STATUS_OK;
 	initialized = true;
 	
-	shared_init = load_samba_modules(NULL, lp_ctx, "gensec");
+	shared_init = load_samba_modules(NULL, "gensec");
 
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
diff --git a/source4/auth/gensec/gensec.h b/source4/auth/gensec/gensec.h
index 48268c421ef..e42b4aa5d2f 100644
--- a/source4/auth/gensec/gensec.h
+++ b/source4/auth/gensec/gensec.h
@@ -165,7 +165,7 @@ struct gensec_security {
 	/* When we are a server, this may be filled in to provide an
 	 * NTLM authentication backend, and user lookup (such as if no
 	 * PAC is found) */
-	struct auth_context *auth_context;
+	struct auth4_context *auth_context;
 };
 
 /* this structure is used by backends to determine the size of some critical types */
@@ -179,7 +179,7 @@ struct gensec_critical_sizes {
 
 struct gensec_security;
 struct socket_context;
-struct auth_context;
+struct auth4_context;
 struct auth_user_info_dc;
 
 NTSTATUS gensec_socket_init(struct gensec_security *gensec_security,
@@ -242,7 +242,7 @@ NTSTATUS gensec_start_mech_by_oid(struct gensec_security *gensec_security,
 				  const char *mech_oid);
 const char *gensec_get_name_by_oid(struct gensec_security *gensec_security, const char *oid_string);
 struct cli_credentials *gensec_get_credentials(struct gensec_security *gensec_security);
-NTSTATUS gensec_init(struct loadparm_context *lp_ctx);
+NTSTATUS gensec_init(void);
 NTSTATUS gensec_unseal_packet(struct gensec_security *gensec_security, 
 			      TALLOC_CTX *mem_ctx, 
 			      uint8_t *data, size_t length, 
@@ -270,7 +270,7 @@ const char *gensec_get_name_by_authtype(struct gensec_security *gensec_security,
 NTSTATUS gensec_server_start(TALLOC_CTX *mem_ctx, 
 			     struct tevent_context *ev,
 			     struct gensec_settings *settings,
-			     struct auth_context *auth_context,
+			     struct auth4_context *auth_context,
 			     struct gensec_security **gensec_security);
 NTSTATUS gensec_session_info(struct gensec_security *gensec_security, 
 			     struct auth_session_info **session_info);
diff --git a/source4/auth/gensec/gensec_gssapi.c b/source4/auth/gensec/gensec_gssapi.c
index 59029e6fc90..72c6b3f991d 100644
--- a/source4/auth/gensec/gensec_gssapi.c
+++ b/source4/auth/gensec/gensec_gssapi.c
@@ -47,44 +47,6 @@ _PUBLIC_ NTSTATUS gensec_gssapi_init(void);
 static size_t gensec_gssapi_max_input_size(struct gensec_security *gensec_security);
 static size_t gensec_gssapi_max_wrapped_size(struct gensec_security *gensec_security);
 
-static char *gssapi_error_string(TALLOC_CTX *mem_ctx, 
-				 OM_uint32 maj_stat, OM_uint32 min_stat, 
-				 const gss_OID mech)
-{
-	OM_uint32 disp_min_stat, disp_maj_stat;
-	gss_buffer_desc maj_error_message;
-	gss_buffer_desc min_error_message;
-	char *maj_error_string, *min_error_string;
-	OM_uint32 msg_ctx = 0;
-
-	char *ret;
-
-	maj_error_message.value = NULL;
-	min_error_message.value = NULL;
-	maj_error_message.length = 0;
-	min_error_message.length = 0;
-	
-	disp_maj_stat = gss_display_status(&disp_min_stat, maj_stat, GSS_C_GSS_CODE,
-			   mech, &msg_ctx, &maj_error_message);
-	disp_maj_stat = gss_display_status(&disp_min_stat, min_stat, GSS_C_MECH_CODE,
-			   mech, &msg_ctx, &min_error_message);
-	
-	maj_error_string = talloc_strndup(mem_ctx, (char *)maj_error_message.value, maj_error_message.length);
-
-	min_error_string = talloc_strndup(mem_ctx, (char *)min_error_message.value, min_error_message.length);
-
-	ret = talloc_asprintf(mem_ctx, "%s: %s", maj_error_string, min_error_string);
-
-	talloc_free(maj_error_string);
-	talloc_free(min_error_string);
-
-	gss_release_buffer(&disp_min_stat, &maj_error_message);
-	gss_release_buffer(&disp_min_stat, &min_error_message);
-
-	return ret;
-}
-
-
 static int gensec_gssapi_destructor(struct gensec_gssapi_state *gensec_gssapi_state)
 {
 	OM_uint32 maj_stat, min_stat;
@@ -340,6 +302,10 @@ static NTSTATUS gensec_gssapi_client_start(struct gensec_security *gensec_securi
 
 	gensec_gssapi_state = talloc_get_type(gensec_security->private_data, struct gensec_gssapi_state);
 
+	if (cli_credentials_get_impersonate_principal(creds)) {
+		gensec_gssapi_state->want_flags &= ~(GSS_C_DELEG_FLAG|GSS_C_DELEG_POLICY_FLAG);
+	}
+
 	gensec_gssapi_state->target_principal = gensec_get_target_principal(gensec_security);
 	if (gensec_gssapi_state->target_principal) {
 		name_type = GSS_C_NULL_OID;
@@ -557,6 +523,65 @@ static NTSTATUS gensec_gssapi_update(struct gensec_security *gensec_security,
 			gss_release_buffer(&min_stat2, &output_token);
 			
 			return NT_STATUS_MORE_PROCESSING_REQUIRED;
+		} else if (maj_stat == GSS_S_CONTEXT_EXPIRED) {
+			gss_cred_id_t creds;
+			gss_name_t name;
+			gss_buffer_desc buffer;
+			OM_uint32 lifetime = 0;
+			gss_cred_usage_t usage;
+			const char *role = NULL;
+			DEBUG(0, ("GSS %s Update(krb5)(%d) Update failed, credentials expired during GSSAPI handshake!\n",
+				  role,
+				  gensec_gssapi_state->gss_exchange_count));
+
+			
+			switch (gensec_security->gensec_role) {
+			case GENSEC_CLIENT:
+				creds = gensec_gssapi_state->client_cred->creds;
+				role = "client";
+			case GENSEC_SERVER:
+				creds = gensec_gssapi_state->server_cred->creds;
+				role = "server";
+			}
+
+			maj_stat = gss_inquire_cred(&min_stat, 
+						    creds,
+						    &name, &lifetime, &usage, NULL);
+
+			if (maj_stat == GSS_S_COMPLETE) {
+				const char *usage_string;
+				switch (usage) {
+				case GSS_C_BOTH:
+					usage_string = "GSS_C_BOTH";
+					break;
+				case GSS_C_ACCEPT:
+					usage_string = "GSS_C_ACCEPT";
+					break;
+				case GSS_C_INITIATE:
+					usage_string = "GSS_C_INITIATE";
+					break;
+				}
+				maj_stat = gss_display_name(&min_stat, name, &buffer, NULL);
+				if (maj_stat) {
+					buffer.value = NULL;
+					buffer.length = 0;
+				}
+				if (lifetime > 0) {
+					DEBUG(0, ("GSSAPI gss_inquire_cred indicates expiry of %*.*s in %u sec for %s\n", 
+						  (int)buffer.length, (int)buffer.length, (char *)buffer.value, 
+						  lifetime, usage_string));
+				} else {
+					DEBUG(0, ("GSSAPI gss_inquire_cred indicates %*.*s has already expired for %s\n", 
+						  (int)buffer.length, (int)buffer.length, (char *)buffer.value, 
+						  usage_string));
+				}
+				gss_release_buffer(&min_stat, &buffer);
+				gss_release_name(&min_stat, &name);
+			} else if (maj_stat != GSS_S_COMPLETE) {
+				DEBUG(0, ("inquiry of credential lifefime via GSSAPI gss_inquire_cred failed: %s\n",
+					  gssapi_error_string(out_mem_ctx, maj_stat, min_stat, gensec_gssapi_state->gss_oid)));
+			}
+			return NT_STATUS_INVALID_PARAMETER;
 		} else if (gss_oid_equal(gensec_gssapi_state->gss_oid, gss_mech_krb5)) {
 			switch (min_stat) {
 			case KRB5KRB_AP_ERR_TKT_NYV:
@@ -1262,7 +1287,6 @@ static NTSTATUS gensec_gssapi_session_info(struct gensec_security *gensec_securi
 	struct auth_user_info_dc *user_info_dc = NULL;
 	struct auth_session_info *session_info = NULL;
 	OM_uint32 maj_stat, min_stat;
-	gss_buffer_desc pac;
 	DATA_BLOB pac_blob;
 	struct PAC_SIGNATURE_DATA *pac_srv_sig = NULL;
 	struct PAC_SIGNATURE_DATA *pac_kdc_sig = NULL;
@@ -1277,25 +1301,15 @@ static NTSTATUS gensec_gssapi_session_info(struct gensec_security *gensec_securi
 	mem_ctx = talloc_named(gensec_gssapi_state, 0, "gensec_gssapi_session_info context"); 
 	NT_STATUS_HAVE_NO_MEMORY(mem_ctx);
 
-	maj_stat = gsskrb5_extract_authz_data_from_sec_context(&min_stat, 
-							       gensec_gssapi_state->gssapi_context, 
-							       KRB5_AUTHDATA_WIN2K_PAC,
-							       &pac);
-	
-	
-	if (maj_stat == 0) {
-		pac_blob = data_blob_talloc(mem_ctx, pac.value, pac.length);
-		gss_release_buffer(&min_stat, &pac);
-
-	} else {
-		pac_blob = data_blob(NULL, 0);
-	}
+	nt_status = gssapi_obtain_pac_blob(mem_ctx,  gensec_gssapi_state->gssapi_context,
+					   gensec_gssapi_state->client_name,
+					   &pac_blob);
 	
 	/* IF we have the PAC - otherwise we need to get this
 	 * data from elsewere - local ldb, or (TODO) lookup of some
 	 * kind... 
 	 */
-	if (pac_blob.length) {
+	if (NT_STATUS_IS_OK(nt_status)) {
 		pac_srv_sig = talloc(mem_ctx, struct PAC_SIGNATURE_DATA);
 		if (!pac_srv_sig) {
 			talloc_free(mem_ctx);
diff --git a/source4/auth/gensec/pygensec.c b/source4/auth/gensec/pygensec.c
index fd9726eb754..503974aaa3d 100644
--- a/source4/auth/gensec/pygensec.c
+++ b/source4/auth/gensec/pygensec.c
@@ -127,7 +127,7 @@ static PyObject *py_gensec_start_client(PyTypeObject *type, PyObject *args, PyOb
 		return NULL;
 	}
 
-	status = gensec_init(settings->lp_ctx);
+	status = gensec_init();
 	if (!NT_STATUS_IS_OK(status)) {
 		PyErr_SetNTSTATUS(status);
 		PyObject_DEL(self);
@@ -156,7 +156,7 @@ static PyObject *py_gensec_start_server(PyTypeObject *type, PyObject *args, PyOb
 	PyObject *py_auth_context = Py_None;
 	struct tevent_context *ev;
 	struct gensec_security *gensec;
-	struct auth_context *auth_context = NULL;
+	struct auth4_context *auth_context = NULL;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OO", discard_const_p(char *, kwnames), &py_settings, &py_auth_context))
 		return NULL;
@@ -201,7 +201,7 @@ static PyObject *py_gensec_start_server(PyTypeObject *type, PyObject *args, PyOb
 	}
 
 	if (py_auth_context != Py_None) {
-		auth_context = py_talloc_get_type(py_auth_context, struct auth_context);
+		auth_context = py_talloc_get_type(py_auth_context, struct auth4_context);
 		if (!auth_context) {
 			PyErr_Format(PyExc_TypeError,
 				     "Expected auth.AuthContext for auth_context argument, got %s",
@@ -210,7 +210,7 @@ static PyObject *py_gensec_start_server(PyTypeObject *type, PyObject *args, PyOb
 		}
 	}
 
-	status = gensec_init(settings->lp_ctx);
+	status = gensec_init();
 	if (!NT_STATUS_IS_OK(status)) {
 		PyErr_SetNTSTATUS(status);
 		PyObject_DEL(self);
diff --git a/source4/auth/kerberos/kerberos.c b/source4/auth/kerberos/kerberos.c
index 0db0dd3ced1..0fc9d143abd 100644
--- a/source4/auth/kerberos/kerberos.c
+++ b/source4/auth/kerberos/kerberos.c
@@ -81,86 +81,387 @@
 
   The impersonate_principal is the principal if NULL, or the principal to impersonate
 
-  The target_service defaults to the krbtgt if NULL, but could be kpasswd/realm or the local service (if we are doing s4u2self)
+  The self_service, should be the local service (for S4U2Self if impersonate_principal is given).
+
+  The target_service defaults to the krbtgt if NULL, but could be kpasswd/realm or a remote service (for S4U2Proxy)
 
 */
- krb5_error_code kerberos_kinit_password_cc(krb5_context ctx, krb5_ccache cc, 
-					    krb5_principal principal, const char *password,
-					    krb5_principal impersonate_principal, const char *target_service,
+ krb5_error_code kerberos_kinit_password_cc(krb5_context ctx, krb5_ccache store_cc,
+					    krb5_principal init_principal,
+					    const char *init_password,
+					    krb5_principal impersonate_principal,
+					    const char *self_service,
+					    const char *target_service,
 					    krb5_get_init_creds_opt *krb_options,
 					    time_t *expire_time, time_t *kdc_time)
 {
 	krb5_error_code code = 0;
-	krb5_creds my_creds;
-	krb5_creds *impersonate_creds;
 	krb5_get_creds_opt options;
+	krb5_principal store_principal;
+	krb5_creds store_creds;
+	krb5_creds *s4u2self_creds;
+	Ticket s4u2self_ticket;
+	size_t s4u2self_ticketlen;
+	krb5_creds *s4u2proxy_creds;
+	krb5_principal self_princ;
+	bool s4u2proxy;
+	krb5_principal target_princ;
+	krb5_ccache tmp_cc;
+	const char *self_realm;
+	krb5_principal blacklist_principal = NULL;
+	krb5_principal whitelist_principal = NULL;
 
-	/* If we are not impersonating, then get this ticket for the
+	if (impersonate_principal && self_service == NULL) {
+		return EINVAL;
+	}
+
+	/*
+	 * If we are not impersonating, then get this ticket for the
 	 * target service, otherwise a krbtgt, and get the next ticket
-	 * for the target */
-	if ((code = krb5_get_init_creds_password(ctx, &my_creds, principal, password, 
-						 NULL, NULL,
-						 0,
-						 impersonate_principal ? NULL : target_service,
-						 krb_options))) {
+	 * for the target
+	 */
+	code = krb5_get_init_creds_password(ctx, &store_creds,
+					    init_principal,
+					    init_password,
+					    NULL, NULL,
+					    0,
+					    impersonate_principal ? NULL : target_service,
+					    krb_options);
+	if (code != 0) {
 		return code;
 	}
 
-	if ((code = krb5_cc_initialize(ctx, cc, principal))) {
-		krb5_free_cred_contents(ctx, &my_creds);
+	store_principal = init_principal;
+
+	if (impersonate_principal == NULL) {
+		goto store;
+	}
+
+	/*
+	 * We are trying S4U2Self now:
+	 *
+	 * As we do not want to expose our TGT in the
+	 * krb5_ccache, which is also holds the impersonated creds.
+	 *
+	 * Some low level krb5/gssapi function might use the TGT
+	 * identity and let the client act as our machine account.
+	 *
+	 * We need to avoid that and use a temporary krb5_ccache
+	 * in order to pass our TGT to the krb5_get_creds() function.
+	 */
+	code = krb5_cc_new_unique(ctx, NULL, NULL, &tmp_cc);
+	if (code != 0) {
+		krb5_free_cred_contents(ctx, &store_creds);
 		return code;
 	}
-	
-	if ((code = krb5_cc_store_cred(ctx, cc, &my_creds))) {
-		krb5_free_cred_contents(ctx, &my_creds);
+
+	code = krb5_cc_initialize(ctx, tmp_cc, store_creds.client);
+	if (code != 0) {
+		krb5_cc_destroy(ctx, tmp_cc);
+		krb5_free_cred_contents(ctx, &store_creds);
 		return code;
 	}
-	
-	if (expire_time) {
-		*expire_time = (time_t) my_creds.times.endtime;
+
+	code = krb5_cc_store_cred(ctx, tmp_cc, &store_creds);
+	if (code != 0) {
+		krb5_free_cred_contents(ctx, &store_creds);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
 	}
 
-	if (kdc_time) {
-		*kdc_time = (time_t) my_creds.times.starttime;
+	/*
+	 * we need to remember the client principal of our
+	 * TGT and make sure the KDC does not return this
+	 * in the impersonated tickets. This can happen
+	 * if the KDC does not support S4U2Self and S4U2Proxy.
+	 */
+	blacklist_principal = store_creds.client;
+	store_creds.client = NULL;
+	krb5_free_cred_contents(ctx, &store_creds);
+
+	/*
+	 * Check if we also need S4U2Proxy or if S4U2Self is
+	 * enough in order to get a ticket for the target.
+	 */
+	if (target_service == NULL) {
+		s4u2proxy = false;
+	} else if (strcmp(target_service, self_service) == 0) {
+		s4u2proxy = false;
+	} else {
+		s4u2proxy = true;
 	}
 
-	krb5_free_cred_contents(ctx, &my_creds);
-	
-	if (code == 0 && impersonate_principal) {
-		krb5_principal target_princ;
-		if ((code = krb5_get_creds_opt_alloc(ctx, &options))) {
-			return code;
-		}
+	/*
+	 * For S4U2Self we need our own service principal,
+	 * which belongs to our own realm (available on
+	 * our client principal).
+	 */
+	self_realm = krb5_principal_get_realm(ctx, init_principal);
 
-		if ((code = krb5_get_creds_opt_set_impersonate(ctx, options, impersonate_principal))) {
-			krb5_get_creds_opt_free(ctx, options);
-			return code;
-		}
+	code = krb5_parse_name(ctx, self_service, &self_princ);
+	if (code != 0) {
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
 
-		if ((code = krb5_parse_name(ctx, target_service, &target_princ))) {
-			krb5_get_creds_opt_free(ctx, options);
-			return code;
-		}
+	code = krb5_principal_set_realm(ctx, self_princ, self_realm);
+	if (code != 0) {
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_free_principal(ctx, self_princ);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
 
-		if ((code = krb5_principal_set_realm(ctx, target_princ, krb5_principal_get_realm(ctx, principal)))) {
-			krb5_get_creds_opt_free(ctx, options);
-			krb5_free_principal(ctx, target_princ);
-			return code;
-		}
+	code = krb5_get_creds_opt_alloc(ctx, &options);
+	if (code != 0) {
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_free_principal(ctx, self_princ);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
 
-		if ((code = krb5_get_creds(ctx, options, cc, target_princ, &impersonate_creds))) {
-			krb5_free_principal(ctx, target_princ);
-			krb5_get_creds_opt_free(ctx, options);
+	if (s4u2proxy) {
+		/*
+		 * If we want S4U2Proxy, we need the forwardable flag
+		 * on the S4U2Self ticket.
+		 */
+		krb5_get_creds_opt_set_options(ctx, options, KRB5_GC_FORWARDABLE);
+	}
+
+	code = krb5_get_creds_opt_set_impersonate(ctx, options,
+						  impersonate_principal);
+	if (code != 0) {
+		krb5_get_creds_opt_free(ctx, options);
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_free_principal(ctx, self_princ);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
+
+	code = krb5_get_creds(ctx, options, tmp_cc,
+			      self_princ, &s4u2self_creds);
+	krb5_get_creds_opt_free(ctx, options);
+	krb5_free_principal(ctx, self_princ);
+	if (code != 0) {
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
+
+	if (!s4u2proxy) {
+		krb5_cc_destroy(ctx, tmp_cc);
+
+		/*
+		 * Now make sure we store the impersonated principal
+		 * and creds instead of the TGT related stuff
+		 * in the krb5_ccache of the caller.
+		 */
+		code = krb5_copy_creds_contents(ctx, s4u2self_creds,
+						&store_creds);
+		krb5_free_creds(ctx, s4u2self_creds);
+		if (code != 0) {
 			return code;
 		}
 
+		/*
+		 * It's important to store the principal the KDC
+		 * returned, as otherwise the caller would not find
+		 * the S4U2Self ticket in the krb5_ccache lookup.
+		 */
+		store_principal = store_creds.client;
+		goto store;
+	}
+
+	/*
+	 * We are trying S4U2Proxy:
+	 *
+	 * We need the ticket from the S4U2Self step
+	 * and our TGT in order to get the delegated ticket.
+	 */
+	code = decode_Ticket((const uint8_t *)s4u2self_creds->ticket.data,
+			     s4u2self_creds->ticket.length,
+			     &s4u2self_ticket,
+			     &s4u2self_ticketlen);
+	if (code != 0) {
+		krb5_free_creds(ctx, s4u2self_creds);
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
+
+	/*
+	 * we need to remember the client principal of the
+	 * S4U2Self stage and as it needs to match the one we
+	 * will get for the S4U2Proxy stage. We need this
+	 * in order to detect KDCs which does not support S4U2Proxy.
+	 */
+	whitelist_principal = s4u2self_creds->client;
+	s4u2self_creds->client = NULL;
+	krb5_free_creds(ctx, s4u2self_creds);
+
+	/*
+	 * For S4U2Proxy we also got a target service principal,
+	 * which also belongs to our own realm (available on
+	 * our client principal).
+	 */
+	code = krb5_parse_name(ctx, target_service, &target_princ);
+	if (code != 0) {
+		free_Ticket(&s4u2self_ticket);
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
+
+	code = krb5_principal_set_realm(ctx, target_princ, self_realm);
+	if (code != 0) {
+		free_Ticket(&s4u2self_ticket);
 		krb5_free_principal(ctx, target_princ);
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
+
+	code = krb5_get_creds_opt_alloc(ctx, &options);
+	if (code != 0) {
+		free_Ticket(&s4u2self_ticket);
+		krb5_free_principal(ctx, target_princ);
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
+	}
+
+	krb5_get_creds_opt_set_options(ctx, options, KRB5_GC_FORWARDABLE);
+	krb5_get_creds_opt_set_options(ctx, options, KRB5_GC_CONSTRAINED_DELEGATION);
 
-		code = krb5_cc_store_cred(ctx, cc, impersonate_creds);
+	code = krb5_get_creds_opt_set_ticket(ctx, options, &s4u2self_ticket);
+	free_Ticket(&s4u2self_ticket);
+	if (code != 0) {
 		krb5_get_creds_opt_free(ctx, options);
-		krb5_free_creds(ctx, impersonate_creds);
+		krb5_free_principal(ctx, target_princ);
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_cc_destroy(ctx, tmp_cc);
+		return code;
 	}
 
+	code = krb5_get_creds(ctx, options, tmp_cc,
+			      target_princ, &s4u2proxy_creds);
+	krb5_get_creds_opt_free(ctx, options);
+	krb5_free_principal(ctx, target_princ);
+	krb5_cc_destroy(ctx, tmp_cc);
+	if (code != 0) {
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_principal(ctx, blacklist_principal);
+		return code;
+	}
+
+	/*
+	 * Now make sure we store the impersonated principal
+	 * and creds instead of the TGT related stuff
+	 * in the krb5_ccache of the caller.
+	 */
+	code = krb5_copy_creds_contents(ctx, s4u2proxy_creds,
+					&store_creds);
+	krb5_free_creds(ctx, s4u2proxy_creds);
+	if (code != 0) {
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_principal(ctx, blacklist_principal);
+		return code;
+	}
+
+	/*
+	 * It's important to store the principal the KDC
+	 * returned, as otherwise the caller would not find
+	 * the S4U2Self ticket in the krb5_ccache lookup.
+	 */
+	store_principal = store_creds.client;
+
+ store:
+	if (blacklist_principal &&
+	    krb5_principal_compare(ctx, store_creds.client, blacklist_principal)) {
+		char *sp = NULL;
+		char *ip = NULL;
+
+		code = krb5_unparse_name(ctx, blacklist_principal, &sp);
+		if (code != 0) {
+			sp = NULL;
+		}
+		code = krb5_unparse_name(ctx, impersonate_principal, &ip);
+		if (code != 0) {
+			ip = NULL;
+		}
+		DEBUG(1, ("kerberos_kinit_password_cc: "
+			  "KDC returned self principal[%s] while impersonating [%s]\n",
+			  sp?sp:"<no memory>",
+			  ip?ip:"<no memory>"));
+
+		SAFE_FREE(sp);
+		SAFE_FREE(ip);
+
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_principal(ctx, blacklist_principal);
+		krb5_free_cred_contents(ctx, &store_creds);
+		return KRB5_FWD_BAD_PRINCIPAL;
+	}
+	if (blacklist_principal) {
+		krb5_free_principal(ctx, blacklist_principal);
+	}
+
+	if (whitelist_principal &&
+	    !krb5_principal_compare(ctx, store_creds.client, whitelist_principal)) {
+		char *sp = NULL;
+		char *ep = NULL;
+
+		code = krb5_unparse_name(ctx, store_creds.client, &sp);
+		if (code != 0) {
+			sp = NULL;
+		}
+		code = krb5_unparse_name(ctx, whitelist_principal, &ep);
+		if (code != 0) {
+			ep = NULL;
+		}
+		DEBUG(1, ("kerberos_kinit_password_cc: "
+			  "KDC returned wrong principal[%s] we expected [%s]\n",
+			  sp?sp:"<no memory>",
+			  ep?ep:"<no memory>"));
+
+		SAFE_FREE(sp);
+		SAFE_FREE(ep);
+
+		krb5_free_principal(ctx, whitelist_principal);
+		krb5_free_cred_contents(ctx, &store_creds);
+		return KRB5_FWD_BAD_PRINCIPAL;
+	}
+	if (whitelist_principal) {
+		krb5_free_principal(ctx, whitelist_principal);
+	}
+
+	code = krb5_cc_initialize(ctx, store_cc, store_principal);
+	if (code != 0) {
+		krb5_free_cred_contents(ctx, &store_creds);
+		return code;
+	}
+
+	code = krb5_cc_store_cred(ctx, store_cc, &store_creds);
+	if (code != 0) {
+		krb5_free_cred_contents(ctx, &store_creds);
+		return code;
+	}
+
+	if (expire_time) {
+		*expire_time = (time_t) store_creds.times.endtime;
+	}
+
+	if (kdc_time) {
+		*kdc_time = (time_t) store_creds.times.starttime;
+	}
+
+	krb5_free_cred_contents(ctx, &store_creds);
+
 	return 0;
 }
 
diff --git a/source4/auth/kerberos/kerberos.h b/source4/auth/kerberos/kerberos.h
index c712569e5de..31794b8e034 100644
--- a/source4/auth/kerberos/kerberos.h
+++ b/source4/auth/kerberos/kerberos.h
@@ -97,7 +97,9 @@ krb5_error_code ads_krb5_mk_req(krb5_context context,
 bool get_auth_data_from_tkt(TALLOC_CTX *mem_ctx, DATA_BLOB *auth_data, krb5_ticket *tkt);
 krb5_error_code kerberos_kinit_password_cc(krb5_context ctx, krb5_ccache cc,
 					   krb5_principal principal, const char *password,
-					   krb5_principal impersonate_principal, const char *target_service,
+					   krb5_principal impersonate_principal,
+					   const char *self_service,
+					   const char *target_service,
 					   krb5_get_init_creds_opt *krb_options,
 					   time_t *expire_time, time_t *kdc_time);
 krb5_error_code kerberos_kinit_keyblock_cc(krb5_context ctx, krb5_ccache cc,
diff --git a/source4/auth/kerberos/kerberos_pac.c b/source4/auth/kerberos/kerberos_pac.c
index 5bc80c9a583..8ce970e4867 100644
--- a/source4/auth/kerberos/kerberos_pac.c
+++ b/source4/auth/kerberos/kerberos_pac.c
@@ -448,14 +448,14 @@ NTSTATUS kerberos_pac_blob_to_user_info_dc(TALLOC_CTX *mem_ctx,
 			     pac_blob.data, pac_blob.length,
 			     &pac);
 	if (ret) {
-		return map_nt_error_from_unix(ret);
+		return map_nt_error_from_unix_common(ret);
 	}
 
 
 	ret = kerberos_pac_to_user_info_dc(mem_ctx, pac, context, user_info_dc, pac_srv_sig, pac_kdc_sig);
 	krb5_pac_free(context, pac);
 	if (ret) {
-		return map_nt_error_from_unix(ret);
+		return map_nt_error_from_unix_common(ret);
 	}
 	return NT_STATUS_OK;
 }
diff --git a/source4/auth/kerberos/kerberos_util.c b/source4/auth/kerberos/kerberos_util.c
index 45b0b07e137..9a48e95c6d2 100644
--- a/source4/auth/kerberos/kerberos_util.c
+++ b/source4/auth/kerberos/kerberos_util.c
@@ -173,7 +173,7 @@ static krb5_error_code principals_from_msg(TALLOC_CTX *parent_ctx,
 			return ret;
 		}
 		
-		/* This song-and-dance effectivly puts the principal
+		/* This song-and-dance effectively puts the principal
 		 * into talloc, so we can't loose it. */
 		talloc_set_destructor(principals[i], free_principal);
 		i++;
@@ -262,7 +262,7 @@ static krb5_error_code salt_principal_from_msg(TALLOC_CTX *parent_ctx,
 					  upper_realm,
 					  "host", salt_body, NULL);
 		if (ret == 0) {
-			/* This song-and-dance effectivly puts the principal
+			/* This song-and-dance effectively puts the principal
 			 * into talloc, so we can't loose it. */
 			mem_ctx->smb_krb5_context = talloc_reference(mem_ctx, smb_krb5_context);
 			mem_ctx->principal = *salt_princ;
@@ -338,7 +338,9 @@ krb5_error_code principal_from_credentials(TALLOC_CTX *parent_ctx,
 				 const char **error_string)
 {
 	krb5_error_code ret;
-	const char *password, *target_service;
+	const char *password;
+	const char *self_service;
+	const char *target_service;
 	time_t kdc_time = 0;
 	krb5_principal princ;
 	krb5_principal impersonate_principal;
@@ -363,6 +365,7 @@ krb5_error_code principal_from_credentials(TALLOC_CTX *parent_ctx,
 		return ret;
 	}
 
+	self_service = cli_credentials_get_self_service(credentials);
 	target_service = cli_credentials_get_target_service(credentials);
 
 	password = cli_credentials_get_password(credentials);
@@ -403,7 +406,9 @@ krb5_error_code principal_from_credentials(TALLOC_CTX *parent_ctx,
 		if (password) {
 			ret = kerberos_kinit_password_cc(smb_krb5_context->krb5_context, ccache, 
 							 princ, password,
-							 impersonate_principal, target_service,
+							 impersonate_principal,
+							 self_service,
+							 target_service,
 							 krb_options,
 							 NULL, &kdc_time);
 		} else if (impersonate_principal) {
@@ -733,7 +738,7 @@ static krb5_error_code remove_old_entries(TALLOC_CTX *parent_ctx,
 			/* Release the enumeration.  We are going to
 			 * have to start this from the top again,
 			 * because deletes during enumeration may not
-			 * always be consistant.
+			 * always be consistent.
 			 *
 			 * Also, the enumeration locks a FILE: keytab
 			 */
diff --git a/source4/auth/kerberos/krb5_init_context.c b/source4/auth/kerberos/krb5_init_context.c
index db3a5375c92..fbcaad29d96 100644
--- a/source4/auth/kerberos/krb5_init_context.c
+++ b/source4/auth/kerberos/krb5_init_context.c
@@ -415,7 +415,7 @@ smb_krb5_init_context_basic(TALLOC_CTX *tmp_ctx,
 		return ret;
 	}
 
-	config_file = config_path(tmp_ctx, lp_ctx, "krb5.conf");
+	config_file = lpcfg_config_path(tmp_ctx, lp_ctx, "krb5.conf");
 	if (!config_file) {
 		krb5_free_context(krb5_ctx);
 		return ENOMEM;
diff --git a/source4/auth/kerberos/wscript_build b/source4/auth/kerberos/wscript_build
index 586366d4226..90e8560573f 100644
--- a/source4/auth/kerberos/wscript_build
+++ b/source4/auth/kerberos/wscript_build
@@ -4,7 +4,7 @@ bld.SAMBA_LIBRARY('authkrb5',
                   source='kerberos.c kerberos_heimdal.c kerberos_pac.c gssapi_parse.c krb5_init_context.c keytab_copy.c',
                   autoproto='proto.h',
                   public_deps='krb5 ndr-krb5pac samba_socket LIBCLI_RESOLVE com_err asn1',
-                  deps='asn1util auth_sam_reply tevent LIBPACKET ndr ldb KRB5_WRAP',
+                  deps='asn1util auth_sam_reply tevent LIBPACKET ndr ldb KRB5_WRAP errors',
                   private_library=True
                   )
 
diff --git a/source4/auth/ntlm/auth.c b/source4/auth/ntlm/auth.c
index e2deab78bcf..d2464c3cbf6 100644
--- a/source4/auth/ntlm/auth.c
+++ b/source4/auth/ntlm/auth.c
@@ -31,7 +31,7 @@
 /***************************************************************************
  Set a fixed challenge
 ***************************************************************************/
-_PUBLIC_ NTSTATUS auth_context_set_challenge(struct auth_context *auth_ctx, const uint8_t chal[8], const char *set_by) 
+_PUBLIC_ NTSTATUS auth_context_set_challenge(struct auth4_context *auth_ctx, const uint8_t chal[8], const char *set_by) 
 {
 	auth_ctx->challenge.set_by = talloc_strdup(auth_ctx, set_by);
 	NT_STATUS_HAVE_NO_MEMORY(auth_ctx->challenge.set_by);
@@ -45,7 +45,7 @@ _PUBLIC_ NTSTATUS auth_context_set_challenge(struct auth_context *auth_ctx, cons
 /***************************************************************************
  Set a fixed challenge
 ***************************************************************************/
-_PUBLIC_ bool auth_challenge_may_be_modified(struct auth_context *auth_ctx)
+_PUBLIC_ bool auth_challenge_may_be_modified(struct auth4_context *auth_ctx)
 {
 	return auth_ctx->challenge.may_be_modified;
 }
@@ -54,7 +54,7 @@ _PUBLIC_ bool auth_challenge_may_be_modified(struct auth_context *auth_ctx)
  Try to get a challenge out of the various authentication modules.
  Returns a const char of length 8 bytes.
 ****************************************************************************/
-_PUBLIC_ NTSTATUS auth_get_challenge(struct auth_context *auth_ctx, uint8_t chal[8])
+_PUBLIC_ NTSTATUS auth_get_challenge(struct auth4_context *auth_ctx, uint8_t chal[8])
 {
 	NTSTATUS nt_status;
 	struct auth_method_context *method;
@@ -104,7 +104,7 @@ PAC isn't available, and for tokenGroups in the DSDB stack.
  Supply either a principal or a DN
 ****************************************************************************/
 _PUBLIC_ NTSTATUS auth_get_user_info_dc_principal(TALLOC_CTX *mem_ctx,
-						 struct auth_context *auth_ctx,
+						 struct auth4_context *auth_ctx,
 						 const char *principal,
 						 struct ldb_dn *user_dn,
 						 struct auth_user_info_dc **user_info_dc)
@@ -155,7 +155,7 @@ _PUBLIC_ NTSTATUS auth_get_user_info_dc_principal(TALLOC_CTX *mem_ctx,
  *
  **/
 
-_PUBLIC_ NTSTATUS auth_check_password(struct auth_context *auth_ctx,
+_PUBLIC_ NTSTATUS auth_check_password(struct auth4_context *auth_ctx,
 			     TALLOC_CTX *mem_ctx,
 			     const struct auth_usersupplied_info *user_info, 
 			     struct auth_user_info_dc **user_info_dc)
@@ -188,7 +188,7 @@ _PUBLIC_ NTSTATUS auth_check_password(struct auth_context *auth_ctx,
 }
 
 struct auth_check_password_state {
-	struct auth_context *auth_ctx;
+	struct auth4_context *auth_ctx;
 	const struct auth_usersupplied_info *user_info;
 	struct auth_user_info_dc *user_info_dc;
 	struct auth_method_context *method;
@@ -225,7 +225,7 @@ static void auth_check_password_async_trigger(struct tevent_context *ev,
 
 _PUBLIC_ struct tevent_req *auth_check_password_send(TALLOC_CTX *mem_ctx,
 				struct tevent_context *ev,
-				struct auth_context *auth_ctx,
+				struct auth4_context *auth_ctx,
 				const struct auth_usersupplied_info *user_info)
 {
 	struct tevent_req *req;
@@ -409,7 +409,7 @@ _PUBLIC_ NTSTATUS auth_check_password_recv(struct tevent_req *req,
 /* Wrapper because we don't want to expose all callers to needing to
  * know that session_info is generated from the main ldb */
 static NTSTATUS auth_generate_session_info_wrapper(TALLOC_CTX *mem_ctx,
-						   struct auth_context *auth_context,
+						   struct auth4_context *auth_context,
 						   struct auth_user_info_dc *user_info_dc,
 						   uint32_t session_info_flags,
 						   struct auth_session_info **session_info)
@@ -425,13 +425,13 @@ static NTSTATUS auth_generate_session_info_wrapper(TALLOC_CTX *mem_ctx,
 ***************************************************************************/
 _PUBLIC_ NTSTATUS auth_context_create_methods(TALLOC_CTX *mem_ctx, const char **methods, 
 					      struct tevent_context *ev,
-					      struct messaging_context *msg,
+					      struct imessaging_context *msg,
 					      struct loadparm_context *lp_ctx,
 					      struct ldb_context *sam_ctx,
-					      struct auth_context **auth_ctx)
+					      struct auth4_context **auth_ctx)
 {
 	int i;
-	struct auth_context *ctx;
+	struct auth4_context *ctx;
 
 	auth4_init();
 
@@ -440,7 +440,7 @@ _PUBLIC_ NTSTATUS auth_context_create_methods(TALLOC_CTX *mem_ctx, const char **
 		return NT_STATUS_INTERNAL_ERROR;
 	}
 
-	ctx = talloc(mem_ctx, struct auth_context);
+	ctx = talloc(mem_ctx, struct auth4_context);
 	NT_STATUS_HAVE_NO_MEMORY(ctx);
 	ctx->challenge.set_by		= NULL;
 	ctx->challenge.may_be_modified	= false;
@@ -487,19 +487,21 @@ _PUBLIC_ NTSTATUS auth_context_create_methods(TALLOC_CTX *mem_ctx, const char **
 
 const char **auth_methods_from_lp(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
 {
-	const char **auth_methods = NULL;
+	char **auth_methods = NULL;
+
 	switch (lpcfg_server_role(lp_ctx)) {
 	case ROLE_STANDALONE:
-		auth_methods = lpcfg_parm_string_list(mem_ctx, lp_ctx, NULL, "auth methods", "standalone", NULL);
+		auth_methods = str_list_make(mem_ctx, "anonymous sam_ignoredomain", NULL);
 		break;
 	case ROLE_DOMAIN_MEMBER:
-		auth_methods = lpcfg_parm_string_list(mem_ctx, lp_ctx, NULL, "auth methods", "member server", NULL);
+		auth_methods = str_list_make(mem_ctx, "anonymous sam winbind", NULL);
 		break;
-	case ROLE_DOMAIN_CONTROLLER:
-		auth_methods = lpcfg_parm_string_list(mem_ctx, lp_ctx, NULL, "auth methods", "domain controller", NULL);
+	case ROLE_DOMAIN_BDC:
+	case ROLE_DOMAIN_PDC:
+		auth_methods = str_list_make(mem_ctx, "anonymous sam_ignoredomain winbind", NULL);
 		break;
 	}
-	return auth_methods;
+	return (const char **) auth_methods;
 }
 
 /***************************************************************************
@@ -508,9 +510,9 @@ const char **auth_methods_from_lp(TALLOC_CTX *mem_ctx, struct loadparm_context *
 ***************************************************************************/
 _PUBLIC_ NTSTATUS auth_context_create(TALLOC_CTX *mem_ctx,
 			     struct tevent_context *ev,
-			     struct messaging_context *msg,
+			     struct imessaging_context *msg,
 			     struct loadparm_context *lp_ctx,
-			     struct auth_context **auth_ctx)
+			     struct auth4_context **auth_ctx)
 {
 	NTSTATUS status;
 	const char **auth_methods;
@@ -533,7 +535,7 @@ _PUBLIC_ NTSTATUS auth_context_create(TALLOC_CTX *mem_ctx,
    This allows us not to re-open the LDB when we need to do a some authentication logic (such as tokenGroups)
 
  */
-NTSTATUS auth_context_create_from_ldb(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, struct auth_context **auth_ctx)
+NTSTATUS auth_context_create_from_ldb(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, struct auth4_context **auth_ctx)
 {
 	NTSTATUS status;
 	const char **auth_methods;
@@ -620,10 +622,10 @@ const struct auth_operations *auth_backend_byname(const char *name)
 const struct auth_critical_sizes *auth_interface_version(void)
 {
 	static const struct auth_critical_sizes critical_sizes = {
-		AUTH_INTERFACE_VERSION,
+		AUTH4_INTERFACE_VERSION,
 		sizeof(struct auth_operations),
 		sizeof(struct auth_method_context),
-		sizeof(struct auth_context),
+		sizeof(struct auth4_context),
 		sizeof(struct auth_usersupplied_info),
 		sizeof(struct auth_user_info_dc)
 	};
diff --git a/source4/auth/ntlm/auth_anonymous.c b/source4/auth/ntlm/auth_anonymous.c
index 6b21225aadf..4b0fff03ccc 100644
--- a/source4/auth/ntlm/auth_anonymous.c
+++ b/source4/auth/ntlm/auth_anonymous.c
@@ -24,7 +24,7 @@
 #include "auth/ntlm/auth_proto.h"
 #include "param/param.h"
 
-_PUBLIC_ NTSTATUS auth_anonymous_init(void);
+_PUBLIC_ NTSTATUS auth4_anonymous_init(void);
 
 /**
  * Return a anonymous logon for anonymous users (username = "")
@@ -66,7 +66,7 @@ static const struct auth_operations anonymous_auth_ops = {
 	.check_password	= anonymous_check_password
 };
 
-_PUBLIC_ NTSTATUS auth_anonymous_init(void)
+_PUBLIC_ NTSTATUS auth4_anonymous_init(void)
 {
 	NTSTATUS ret;
 
diff --git a/source4/auth/ntlm/auth_developer.c b/source4/auth/ntlm/auth_developer.c
index da842c98baf..bc27f27fa28 100644
--- a/source4/auth/ntlm/auth_developer.c
+++ b/source4/auth/ntlm/auth_developer.c
@@ -24,7 +24,7 @@
 #include "auth/ntlm/auth_proto.h"
 #include "libcli/security/security.h"
 
-_PUBLIC_ NTSTATUS auth_developer_init(void);
+_PUBLIC_ NTSTATUS auth4_developer_init(void);
 
 static NTSTATUS name_to_ntstatus_want_check(struct auth_method_context *ctx,
 			      		    TALLOC_CTX *mem_ctx,
@@ -185,7 +185,7 @@ static const struct auth_operations fixed_challenge_auth_ops = {
 	.check_password	= fixed_challenge_check_password
 };
 
-_PUBLIC_ NTSTATUS auth_developer_init(void)
+_PUBLIC_ NTSTATUS auth4_developer_init(void)
 {
 	NTSTATUS ret;
 
diff --git a/source4/auth/ntlm/auth_sam.c b/source4/auth/ntlm/auth_sam.c
index f76057a6df1..87a7d275596 100644
--- a/source4/auth/ntlm/auth_sam.c
+++ b/source4/auth/ntlm/auth_sam.c
@@ -72,7 +72,7 @@ static NTSTATUS authsam_search_account(TALLOC_CTX *mem_ctx, struct ldb_context *
  Do a specific test for an smb password being correct, given a smb_password and
  the lanman and NT responses.
 ****************************************************************************/
-static NTSTATUS authsam_password_ok(struct auth_context *auth_context,
+static NTSTATUS authsam_password_ok(struct auth4_context *auth_context,
 				    TALLOC_CTX *mem_ctx,
 				    uint16_t acct_flags,
 				    const struct samr_Password *lm_pwd, 
@@ -142,7 +142,7 @@ static NTSTATUS authsam_password_ok(struct auth_context *auth_context,
   send a message to the drepl server telling it to initiate a
   REPL_SECRET getncchanges extended op to fetch the users secrets
  */
-static void auth_sam_trigger_repl_secret(TALLOC_CTX *mem_ctx, struct auth_context *auth_context,
+static void auth_sam_trigger_repl_secret(TALLOC_CTX *mem_ctx, struct auth4_context *auth_context,
 					 struct ldb_dn *user_dn)
 {
 	struct dcerpc_binding_handle *irpc_handle;
@@ -170,7 +170,7 @@ static void auth_sam_trigger_repl_secret(TALLOC_CTX *mem_ctx, struct auth_contex
 }
 
 
-static NTSTATUS authsam_authenticate(struct auth_context *auth_context, 
+static NTSTATUS authsam_authenticate(struct auth4_context *auth_context, 
 				     TALLOC_CTX *mem_ctx, struct ldb_context *sam_ctx, 
 				     struct ldb_dn *domain_dn,
 				     struct ldb_message *msg,
@@ -357,7 +357,7 @@ static NTSTATUS authsam_want_check(struct auth_method_context *ctx,
 				   
 /* Wrapper for the auth subsystem pointer */
 static NTSTATUS authsam_get_user_info_dc_principal_wrapper(TALLOC_CTX *mem_ctx,
-							  struct auth_context *auth_context,
+							  struct auth4_context *auth_context,
 							  const char *principal,
 							  struct ldb_dn *user_dn,
 							  struct auth_user_info_dc **user_info_dc)
@@ -381,7 +381,7 @@ static const struct auth_operations sam_ops = {
 	.get_user_info_dc_principal = authsam_get_user_info_dc_principal_wrapper
 };
 
-_PUBLIC_ NTSTATUS auth_sam_init(void)
+_PUBLIC_ NTSTATUS auth4_sam_init(void)
 {
 	NTSTATUS ret;
 
diff --git a/source4/auth/ntlm/auth_server.c b/source4/auth/ntlm/auth_server.c
index 7efeb9242ad..9e1ceae0ca8 100644
--- a/source4/auth/ntlm/auth_server.c
+++ b/source4/auth/ntlm/auth_server.c
@@ -27,7 +27,7 @@
 #include "param/param.h"
 #include "libcli/resolve/resolve.h"
 
-_PUBLIC_ NTSTATUS auth_server_init(void);
+_PUBLIC_ NTSTATUS auth4_server_init(void);
 
 /* This version of 'security=server' rewirtten from scratch for Samba4
  * libraries in 2008 */
@@ -223,7 +223,7 @@ static const struct auth_operations server_auth_ops = {
 	.check_password	= server_check_password
 };
 
-_PUBLIC_ NTSTATUS auth_server_init(void)
+_PUBLIC_ NTSTATUS auth4_server_init(void)
 {
 	NTSTATUS ret;
 
diff --git a/source4/auth/ntlm/auth_simple.c b/source4/auth/ntlm/auth_simple.c
index 75eabe855b5..241906e2814 100644
--- a/source4/auth/ntlm/auth_simple.c
+++ b/source4/auth/ntlm/auth_simple.c
@@ -30,7 +30,7 @@
 */
 _PUBLIC_ NTSTATUS authenticate_username_pw(TALLOC_CTX *mem_ctx,
 					   struct tevent_context *ev,
-					   struct messaging_context *msg,
+					   struct imessaging_context *msg,
 					   struct loadparm_context *lp_ctx,
 					   const char *nt4_domain,
 					   const char *nt4_username,
@@ -38,7 +38,7 @@ _PUBLIC_ NTSTATUS authenticate_username_pw(TALLOC_CTX *mem_ctx,
 					   const uint32_t logon_parameters,
 					   struct auth_session_info **session_info) 
 {
-	struct auth_context *auth_context;
+	struct auth4_context *auth_context;
 	struct auth_usersupplied_info *user_info;
 	struct auth_user_info_dc *user_info_dc;
 	NTSTATUS nt_status;
diff --git a/source4/auth/ntlm/auth_unix.c b/source4/auth/ntlm/auth_unix.c
index 743cb8103d8..d79ebc1772c 100644
--- a/source4/auth/ntlm/auth_unix.c
+++ b/source4/auth/ntlm/auth_unix.c
@@ -28,7 +28,7 @@
 #include "../libcli/auth/pam_errors.h"
 #include "param/param.h"
 
-_PUBLIC_ NTSTATUS auth_unix_init(void);
+_PUBLIC_ NTSTATUS auth4_unix_init(void);
 
 /* TODO: look at how to best fill in parms retrieveing a struct passwd info
  * except in case USER_INFO_DONT_CHECK_UNIX_ACCOUNT is set
@@ -607,12 +607,10 @@ static NTSTATUS check_unix_password(TALLOC_CTX *ctx, struct loadparm_context *lp
 {
 	char *username;
 	char *password;
-	char *pwcopy;
 	char *salt;
 	char *crypted;
 	struct passwd *pws;
 	NTSTATUS nt_status;
-	int level = lpcfg_passwordlevel(lp_ctx);
 
 	*ret_passwd = NULL;
 
@@ -737,46 +735,11 @@ static NTSTATUS check_unix_password(TALLOC_CTX *ctx, struct loadparm_context *lp
 		return nt_status;
 	}
 
-	if ( user_info->flags | USER_INFO_CASE_INSENSITIVE_PASSWORD) {
-		return nt_status;
-	}
-
-	/* if the password was given to us with mixed case then we don't
-	 * need to proceed as we know it hasn't been case modified by the
-	 * client */
-	if (strhasupper(password) && strhaslower(password)) {
-		return nt_status;
-	}
-
-	/* make a copy of it */
-	pwcopy = talloc_strdup(ctx, password);
-	if (!pwcopy)
-		return NT_STATUS_NO_MEMORY;
-
-	/* try all lowercase if it's currently all uppercase */
-	if (strhasupper(pwcopy)) {
-		strlower(pwcopy);
-		nt_status = password_check(username, pwcopy, crypted, salt);
-		if NT_STATUS_IS_OK(nt_status) {
-			*ret_passwd = pws;
-			return nt_status;
-		}
-	}
-
-	/* give up? */
-	if (level < 1) {
-		return NT_STATUS_WRONG_PASSWORD;
-	}
-
-	/* last chance - all combinations of up to level chars upper! */
-	strlower(pwcopy);
+	/* we no longer try different case combinations here. The use
+	 * of this code is now web auth, where trying different case
+	 * combinations makes no sense
+	 */
 
-#if 0
-        if (NT_STATUS_IS_OK(nt_status = string_combinations(pwcopy, password_check, level))) {
-		*ret_passwd = pws;
-		return nt_status;
-	}
-#endif   
 	return NT_STATUS_WRONG_PASSWORD;
 }
 
@@ -839,7 +802,7 @@ static const struct auth_operations unix_ops = {
 	.check_password	= authunix_check_password
 };
 
-_PUBLIC_ NTSTATUS auth_unix_init(void)
+_PUBLIC_ NTSTATUS auth4_unix_init(void)
 {
 	NTSTATUS ret;
 
diff --git a/source4/auth/ntlm/auth_util.c b/source4/auth/ntlm/auth_util.c
index 17bfa321673..c19b5cfd42f 100644
--- a/source4/auth/ntlm/auth_util.c
+++ b/source4/auth/ntlm/auth_util.c
@@ -99,7 +99,7 @@ NTSTATUS map_user_info(TALLOC_CTX *mem_ctx,
  Create an auth_usersupplied_data structure after appropriate mapping.
 ****************************************************************************/
 
-NTSTATUS encrypt_user_info(TALLOC_CTX *mem_ctx, struct auth_context *auth_context, 
+NTSTATUS encrypt_user_info(TALLOC_CTX *mem_ctx, struct auth4_context *auth_context, 
 			   enum auth_password_state to_state,
 			   const struct auth_usersupplied_info *user_info_in,
 			   const struct auth_usersupplied_info **user_info_encrypted)
diff --git a/source4/auth/ntlm/auth_winbind.c b/source4/auth/ntlm/auth_winbind.c
index dfb8fce2a6b..da152e718a8 100644
--- a/source4/auth/ntlm/auth_winbind.c
+++ b/source4/auth/ntlm/auth_winbind.c
@@ -31,7 +31,7 @@
 #include "nsswitch/libwbclient/wbclient.h"
 #include "libcli/security/security.h"
 
-_PUBLIC_ NTSTATUS auth_winbind_init(void);
+_PUBLIC_ NTSTATUS auth4_winbind_init(void);
 
 static NTSTATUS get_info3_from_wbcAuthUserInfo(TALLOC_CTX *mem_ctx,
 					       struct wbcAuthUserInfo *info,
@@ -324,7 +324,7 @@ static const struct auth_operations winbind_wbclient_ops = {
 	.check_password	= winbind_check_password_wbclient
 };
 
-_PUBLIC_ NTSTATUS auth_winbind_init(void)
+_PUBLIC_ NTSTATUS auth4_winbind_init(void)
 {
 	NTSTATUS ret;
 
diff --git a/source4/auth/ntlm/wscript_build b/source4/auth/ntlm/wscript_build
index 2ac2773c854..d954ec00861 100644
--- a/source4/auth/ntlm/wscript_build
+++ b/source4/auth/ntlm/wscript_build
@@ -3,7 +3,7 @@
 bld.SAMBA_MODULE('auth4_sam_module',
 	source='auth_sam.c',
 	subsystem='auth4',
-	init_function='auth_sam_init',
+	init_function='auth4_sam_init',
 	deps='samdb auth4_sam NTLMSSP_COMMON samba-hostconfig'
 	)
 
@@ -11,7 +11,7 @@ bld.SAMBA_MODULE('auth4_sam_module',
 bld.SAMBA_MODULE('auth4_anonymous',
 	source='auth_anonymous.c',
 	subsystem='auth4',
-	init_function='auth_anonymous_init',
+	init_function='auth4_anonymous_init',
 	deps='talloc'
 	)
 
@@ -19,7 +19,7 @@ bld.SAMBA_MODULE('auth4_anonymous',
 bld.SAMBA_MODULE('auth4_server',
 	source='auth_server.c',
 	subsystem='auth4',
-	init_function='auth_server_init',
+	init_function='auth4_server_init',
 	deps='samba-util LIBCLI_SMB CREDENTIALS_NTLM'
 	)
 
@@ -27,7 +27,7 @@ bld.SAMBA_MODULE('auth4_server',
 bld.SAMBA_MODULE('auth4_winbind',
 	source='auth_winbind.c',
 	subsystem='auth4',
-	init_function='auth_winbind_init',
+	init_function='auth4_winbind_init',
 	deps='RPC_NDR_WINBIND MESSAGING wbclient'
 	)
 
@@ -35,7 +35,7 @@ bld.SAMBA_MODULE('auth4_winbind',
 bld.SAMBA_MODULE('auth4_developer',
 	source='auth_developer.c',
 	subsystem='auth4',
-	init_function='auth_developer_init',
+	init_function='auth4_developer_init',
 	deps='talloc'
 	)
 
@@ -43,7 +43,7 @@ bld.SAMBA_MODULE('auth4_developer',
 bld.SAMBA_MODULE('auth4_unix',
 	source='auth_unix.c',
 	subsystem='auth4',
-	init_function='auth_unix_init',
+	init_function='auth4_unix_init',
 	deps='pam PAM_ERRORS LIBTSOCKET'
 	)
 
diff --git a/source4/auth/ntlmssp/ntlmssp.h b/source4/auth/ntlmssp/ntlmssp.h
index ff30317f553..00439e68dd7 100644
--- a/source4/auth/ntlmssp/ntlmssp.h
+++ b/source4/auth/ntlmssp/ntlmssp.h
@@ -26,7 +26,7 @@
 struct gensec_ntlmssp_context {
 	struct gensec_security *gensec_security;
 	struct ntlmssp_state *ntlmssp_state;
-	struct auth_context *auth_context;
+	struct auth4_context *auth_context;
 	struct auth_user_info_dc *user_info_dc;
 };
 
diff --git a/source4/auth/ntlmssp/ntlmssp_server.c b/source4/auth/ntlmssp/ntlmssp_server.c
index 9db3b560c10..240edbeaadb 100644
--- a/source4/auth/ntlmssp/ntlmssp_server.c
+++ b/source4/auth/ntlmssp/ntlmssp_server.c
@@ -88,7 +88,7 @@ static NTSTATUS auth_ntlmssp_get_challenge(const struct ntlmssp_state *ntlmssp_s
 	struct gensec_ntlmssp_context *gensec_ntlmssp =
 		talloc_get_type_abort(ntlmssp_state->callback_private,
 				      struct gensec_ntlmssp_context);
-	struct auth_context *auth_context = gensec_ntlmssp->auth_context;
+	struct auth4_context *auth_context = gensec_ntlmssp->auth_context;
 	NTSTATUS status;
 
 	status = auth_context->get_challenge(auth_context, chal);
@@ -111,7 +111,7 @@ static bool auth_ntlmssp_may_set_challenge(const struct ntlmssp_state *ntlmssp_s
 	struct gensec_ntlmssp_context *gensec_ntlmssp =
 		talloc_get_type_abort(ntlmssp_state->callback_private,
 				      struct gensec_ntlmssp_context);
-	struct auth_context *auth_context = gensec_ntlmssp->auth_context;
+	struct auth4_context *auth_context = gensec_ntlmssp->auth_context;
 
 	return auth_context->challenge_may_be_modified(auth_context);
 }
@@ -125,7 +125,7 @@ static NTSTATUS auth_ntlmssp_set_challenge(struct ntlmssp_state *ntlmssp_state,
 	struct gensec_ntlmssp_context *gensec_ntlmssp =
 		talloc_get_type_abort(ntlmssp_state->callback_private,
 				      struct gensec_ntlmssp_context);
-	struct auth_context *auth_context = gensec_ntlmssp->auth_context;
+	struct auth4_context *auth_context = gensec_ntlmssp->auth_context;
 	NTSTATUS nt_status;
 	const uint8_t *chal;
 
@@ -155,7 +155,7 @@ static NTSTATUS auth_ntlmssp_check_password(struct ntlmssp_state *ntlmssp_state,
 	struct gensec_ntlmssp_context *gensec_ntlmssp =
 		talloc_get_type_abort(ntlmssp_state->callback_private,
 				      struct gensec_ntlmssp_context);
-	struct auth_context *auth_context = gensec_ntlmssp->auth_context;
+	struct auth4_context *auth_context = gensec_ntlmssp->auth_context;
 	NTSTATUS nt_status;
 	struct auth_usersupplied_info *user_info;
 
@@ -308,26 +308,26 @@ NTSTATUS gensec_ntlmssp_server_start(struct gensec_security *gensec_security)
 	ntlmssp_state->server.netbios_domain = lpcfg_workgroup(gensec_security->settings->lp_ctx);
 
 	{
-		char dnsdomname[MAXHOSTNAMELEN], dnsname[MAXHOSTNAMELEN];
-
-		/* Find out the DNS domain name */
-		dnsdomname[0] = '\0';
-		safe_strcpy(dnsdomname, lpcfg_dnsdomain(gensec_security->settings->lp_ctx), sizeof(dnsdomname) - 1);
+		const char *dnsdomain = lpcfg_dnsdomain(gensec_security->settings->lp_ctx);
+		char *dnsname, *lower_netbiosname;
+		lower_netbiosname = strlower_talloc(ntlmssp_state, ntlmssp_state->server.netbios_name);
 
 		/* Find out the DNS host name */
-		safe_strcpy(dnsname, ntlmssp_state->server.netbios_name, sizeof(dnsname) - 1);
-		if (dnsdomname[0] != '\0') {
-			safe_strcat(dnsname, ".", sizeof(dnsname) - 1);
-			safe_strcat(dnsname, dnsdomname, sizeof(dnsname) - 1);
+		if (dnsdomain && dnsdomain[0] != '\0') {
+			dnsname = talloc_asprintf(ntlmssp_state, "%s.%s",
+						  lower_netbiosname,
+						  dnsdomain);
+			talloc_free(lower_netbiosname);
+			ntlmssp_state->server.dns_name = dnsname;
+		} else {
+			ntlmssp_state->server.dns_name = lower_netbiosname;
 		}
-		strlower_m(dnsname);
 
-		ntlmssp_state->server.dns_name = talloc_strdup(ntlmssp_state,
-								      dnsname);
 		NT_STATUS_HAVE_NO_MEMORY(ntlmssp_state->server.dns_name);
 
-		ntlmssp_state->server.dns_domain = talloc_strdup(ntlmssp_state,
-								        dnsdomname);
+		ntlmssp_state->server.dns_domain
+			= talloc_strdup(ntlmssp_state,
+					lpcfg_dnsdomain(gensec_security->settings->lp_ctx));
 		NT_STATUS_HAVE_NO_MEMORY(ntlmssp_state->server.dns_domain);
 	}
 
diff --git a/source4/auth/pyauth.c b/source4/auth/pyauth.c
index a4ba88c581f..6b3948970f0 100644
--- a/source4/auth/pyauth.c
+++ b/source4/auth/pyauth.c
@@ -205,7 +205,7 @@ static const char **PyList_AsStringList(TALLOC_CTX *mem_ctx, PyObject *list,
 	return ret;
 }
 
-static PyObject *PyAuthContext_FromContext(struct auth_context *auth_context)
+static PyObject *PyAuthContext_FromContext(struct auth4_context *auth_context)
 {
 	return py_talloc_reference(&PyAuthContext, auth_context);
 }
@@ -214,12 +214,12 @@ static PyObject *py_auth_context_new(PyTypeObject *type, PyObject *args, PyObjec
 {
 	PyObject *py_lp_ctx = Py_None;
 	PyObject *py_ldb = Py_None;
-	PyObject *py_messaging_ctx = Py_None;
+	PyObject *py_imessaging_ctx = Py_None;
 	PyObject *py_auth_context = Py_None;
 	PyObject *py_methods = Py_None;
 	TALLOC_CTX *mem_ctx;
-	struct auth_context *auth_context;
-	struct messaging_context *messaging_context = NULL;
+	struct auth4_context *auth_context;
+	struct imessaging_context *imessaging_context = NULL;
 	struct loadparm_context *lp_ctx;
 	struct tevent_context *ev;
 	struct ldb_context *ldb;
@@ -230,7 +230,7 @@ static PyObject *py_auth_context_new(PyTypeObject *type, PyObject *args, PyObjec
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOOO",
 					 discard_const_p(char *, kwnames),
-					 &py_lp_ctx, &py_messaging_ctx, &py_ldb, &py_methods))
+					 &py_lp_ctx, &py_imessaging_ctx, &py_ldb, &py_methods))
 		return NULL;
 
 	mem_ctx = talloc_new(NULL);
@@ -251,12 +251,12 @@ static PyObject *py_auth_context_new(PyTypeObject *type, PyObject *args, PyObjec
 		return NULL;
 	}
 
-	if (py_messaging_ctx != Py_None) {
-		messaging_context = py_talloc_get_type(py_messaging_ctx, struct messaging_context);
+	if (py_imessaging_ctx != Py_None) {
+		imessaging_context = py_talloc_get_type(py_imessaging_ctx, struct imessaging_context);
 	}
 
 	if (py_methods == Py_None && py_ldb == Py_None) {
-		nt_status = auth_context_create(mem_ctx, ev, messaging_context, lp_ctx, &auth_context);
+		nt_status = auth_context_create(mem_ctx, ev, imessaging_context, lp_ctx, &auth_context);
 	} else {
 		if (py_methods != Py_None) {
 			methods = PyList_AsStringList(mem_ctx, py_methods, "methods");
@@ -268,7 +268,7 @@ static PyObject *py_auth_context_new(PyTypeObject *type, PyObject *args, PyObjec
 			methods = auth_methods_from_lp(mem_ctx, lp_ctx);
 		}
 		nt_status = auth_context_create_methods(mem_ctx, methods, ev, 
-							messaging_context, lp_ctx, 
+							imessaging_context, lp_ctx,
 							ldb, &auth_context);
 	}
 
diff --git a/source4/auth/samba_server_gensec.c b/source4/auth/samba_server_gensec.c
index 07b9b15e17c..24b658ad321 100644
--- a/source4/auth/samba_server_gensec.c
+++ b/source4/auth/samba_server_gensec.c
@@ -29,7 +29,7 @@
 
 NTSTATUS samba_server_gensec_start(TALLOC_CTX *mem_ctx,
 				   struct tevent_context *event_ctx,
-				   struct messaging_context *msg_ctx,
+				   struct imessaging_context *msg_ctx,
 				   struct loadparm_context *lp_ctx,
 				   struct cli_credentials *server_credentials,
 				   const char *target_service,
@@ -37,7 +37,7 @@ NTSTATUS samba_server_gensec_start(TALLOC_CTX *mem_ctx,
 { 
 	NTSTATUS nt_status;
 	struct gensec_security *gensec_ctx;
-	struct auth_context *auth_context;
+	struct auth4_context *auth_context;
 
 	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
 	if (!tmp_ctx) {
diff --git a/source4/auth/system_session.c b/source4/auth/system_session.c
index 54b8f514cf1..3b9edd779df 100644
--- a/source4/auth/system_session.c
+++ b/source4/auth/system_session.c
@@ -190,7 +190,7 @@ static NTSTATUS auth_domain_admin_user_info_dc(TALLOC_CTX *mem_ctx,
 	sid_append_rid(&user_info_dc->sids[PRIMARY_USER_SID_INDEX], DOMAIN_RID_ADMINISTRATOR);
 
 	user_info_dc->sids[PRIMARY_GROUP_SID_INDEX] = *domain_sid;
-	sid_append_rid(&user_info_dc->sids[PRIMARY_USER_SID_INDEX], DOMAIN_RID_USERS);
+	sid_append_rid(&user_info_dc->sids[PRIMARY_GROUP_SID_INDEX], DOMAIN_RID_USERS);
 
 	user_info_dc->sids[2] = global_sid_Builtin_Administrators;
 
diff --git a/source4/cldap_server/cldap_server.c b/source4/cldap_server/cldap_server.c
index 43eadf5affc..d3e8b7fffa4 100644
--- a/source4/cldap_server/cldap_server.c
+++ b/source4/cldap_server/cldap_server.c
@@ -116,7 +116,7 @@ static NTSTATUS cldapd_add_socket(struct cldapd_server *cldapd, struct loadparm_
 						lpcfg_cldap_port(lp_ctx),
 						&socket_address);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		DEBUG(0,("invalid address %s:%d - %s:%s\n",
 			 address, lpcfg_cldap_port(lp_ctx),
 			 gai_strerror(ret), nt_errstr(status)));
@@ -153,19 +153,24 @@ static NTSTATUS cldapd_startup_interfaces(struct cldapd_server *cldapd, struct l
 	TALLOC_CTX *tmp_ctx = talloc_new(cldapd);
 	NTSTATUS status;
 
-	num_interfaces = iface_count(ifaces);
+	num_interfaces = iface_list_count(ifaces);
 
 	/* if we are allowing incoming packets from any address, then
 	   we need to bind to the wildcard address */
 	if (!lpcfg_bind_interfaces_only(lp_ctx)) {
-		status = cldapd_add_socket(cldapd, lp_ctx, "0.0.0.0");
-		NT_STATUS_NOT_OK_RETURN(status);
+		const char **wcard = iface_list_wildcard(cldapd, lp_ctx);
+		NT_STATUS_HAVE_NO_MEMORY(wcard);
+		for (i=0; wcard[i]; i++) {
+			status = cldapd_add_socket(cldapd, lp_ctx, wcard[i]);
+			NT_STATUS_NOT_OK_RETURN(status);
+		}
+		talloc_free(wcard);
 	}
 
 	/* now we have to also listen on the specific interfaces,
 	   so that replies always come from the right IP */
 	for (i=0; i<num_interfaces; i++) {
-		const char *address = talloc_strdup(tmp_ctx, iface_n_ip(ifaces, i));
+		const char *address = talloc_strdup(tmp_ctx, iface_list_n_ip(ifaces, i));
 		status = cldapd_add_socket(cldapd, lp_ctx, address);
 		NT_STATUS_NOT_OK_RETURN(status);
 	}
@@ -184,9 +189,9 @@ static void cldapd_task_init(struct task_server *task)
 	NTSTATUS status;
 	struct interface *ifaces;
 	
-	load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
+	load_interface_list(task, task->lp_ctx, &ifaces);
 
-	if (iface_count(ifaces) == 0) {
+	if (iface_list_count(ifaces) == 0) {
 		task_server_terminate(task, "cldapd: no network interfaces configured", false);
 		return;
 	}
diff --git a/source4/cldap_server/netlogon.c b/source4/cldap_server/netlogon.c
index 6e455e46bd7..77f50ff3e66 100644
--- a/source4/cldap_server/netlogon.c
+++ b/source4/cldap_server/netlogon.c
@@ -37,6 +37,7 @@
 #include "param/param.h"
 #include "../lib/tsocket/tsocket.h"
 #include "libds/common/flag_mapping.h"
+#include "lib/util/util_net.h"
 
 /*
   fill in the cldap netlogon union for a given version
@@ -291,17 +292,18 @@ NTSTATUS fill_netlogon_samlogon_response(struct ldb_context *sam_ctx,
 	client_site      = samdb_client_site_name(sam_ctx, mem_ctx,
 						  src_address, NULL);
 	NT_STATUS_HAVE_NO_MEMORY(client_site);
-	load_interfaces(mem_ctx, lpcfg_interfaces(lp_ctx), &ifaces);
-	/*
-	 * TODO: the caller should pass the address which the client
-	 * used to trigger this call, as the client is able to reach
-	 * this ip.
-	 */
+	load_interface_list(mem_ctx, lp_ctx, &ifaces);
+
 	if (src_address) {
-		pdc_ip = iface_best_ip(ifaces, src_address);
+		pdc_ip = iface_list_best_ip(ifaces, src_address);
 	} else {
-		pdc_ip = iface_n_ip(ifaces, 0);
+		pdc_ip = iface_list_first_v4(ifaces);
+	}
+	if (pdc_ip == NULL || !is_ipaddress_v4(pdc_ip)) {
+		/* this matches windows behaviour */
+		pdc_ip = "127.0.0.1";
 	}
+
 	ZERO_STRUCTP(netlogon);
 
 	/* check if either of these bits is present */
@@ -325,7 +327,7 @@ NTSTATUS fill_netlogon_samlogon_response(struct ldb_context *sam_ctx,
 		netlogon->data.nt5_ex.server_site  = server_site;
 		netlogon->data.nt5_ex.client_site  = client_site;
 		if (version & NETLOGON_NT_VERSION_5EX_WITH_IP) {
-			/* Clearly this needs to be fixed up for IPv6 */
+			/* note that this is always a IPV4 address */
 			extra_flags = NETLOGON_NT_VERSION_5EX_WITH_IP;
 			netlogon->data.nt5_ex.sockaddr.sockaddr_family    = 2;
 			netlogon->data.nt5_ex.sockaddr.pdc_ip       = pdc_ip;
diff --git a/source4/client/cifsdd.c b/source4/client/cifsdd.c
index adf2d72af24..dd6f1377e82 100644
--- a/source4/client/cifsdd.c
+++ b/source4/client/cifsdd.c
@@ -152,7 +152,7 @@ int set_arg_argv(const char * argv)
 			}
 			break;
 		case ARG_SIZE:
-			if (!conv_str_size(val, &arg->arg_val.nval)) {
+			if (!conv_str_size_error(val, &arg->arg_val.nval)) {
 				goto fail;
 			}
 			break;
@@ -597,7 +597,7 @@ int main(int argc, const char ** argv)
 
 	ev = s4_event_context_init(talloc_autofree_context());
 
-	gensec_init(cmdline_lp_ctx);
+	gensec_init();
 	dump_args();
 
 	if (check_arg_numeric("ibs") == 0 || check_arg_numeric("ibs") == 0) {
diff --git a/source4/client/client.c b/source4/client/client.c
index c6c0088b3c3..2f353aec4ec 100644
--- a/source4/client/client.c
+++ b/source4/client/client.c
@@ -309,12 +309,12 @@ static bool mask_match(struct smbcli_state *c, const char *string,
 		return false;
 	
 	if (is_case_sensitive)
-		return ms_fnmatch(pattern, string, 
+		return ms_fnmatch_protocol(pattern, string, 
 				  c->transport->negotiate.protocol) == 0;
 
 	p2 = strlower_talloc(NULL, pattern);
 	s2 = strlower_talloc(NULL, string);
-	ret = ms_fnmatch(p2, s2, c->transport->negotiate.protocol) == 0;
+	ret = ms_fnmatch_protocol(p2, s2, c->transport->negotiate.protocol) == 0;
 	talloc_free(p2);
 	talloc_free(s2);
 
@@ -473,8 +473,8 @@ static void add_to_do_list_queue(const char* entry)
 	}
 	if (do_list_queue)
 	{
-		safe_strcpy(do_list_queue + do_list_queue_end, entry, 
-			    do_list_queue_size - do_list_queue_end - 1);
+		strlcpy(do_list_queue + do_list_queue_end, entry ? entry : "",
+			    do_list_queue_size - do_list_queue_end);
 		do_list_queue_end = new_end;
 		DEBUG(4,("added %s to do_list_queue (start=%d, end=%d)\n",
 			 entry, (int)do_list_queue_start, (int)do_list_queue_end));
@@ -696,11 +696,12 @@ static int do_get(struct smbclient_context *ctx, char *rname, const char *p_lnam
 	char *lname;
 
 
-	lname = talloc_strdup(ctx, p_lname);
 	GetTimeOfDay(&tp_start);
 
 	if (ctx->lowercase) {
-		strlower(lname);
+		lname = strlower_talloc(ctx, p_lname);
+	} else {
+		lname = talloc_strdup(ctx, p_lname);
 	}
 
 	fnum = smbcli_open(ctx->cli->tree, rname, O_RDONLY, DENY_NONE);
@@ -884,13 +885,14 @@ static void do_mget(struct smbclient_context *ctx, struct clilist_file_info *fin
 
 	ctx->remote_cur_dir = talloc_asprintf_append_buffer(NULL, "%s\\", finfo->name);
 
-	l_fname = talloc_strdup(ctx, finfo->name);
-
-	string_replace(l_fname, '\\', '/');
 	if (ctx->lowercase) {
-		strlower(l_fname);
+		l_fname = strlower_talloc(ctx, finfo->name);
+	} else {
+		l_fname = talloc_strdup(ctx, finfo->name);
 	}
 	
+	string_replace(l_fname, '\\', '/');
+
 	if (!directory_exist(l_fname) &&
 	    mkdir(l_fname, 0777) != 0) {
 		d_printf("failed to create directory %s\n", l_fname);
@@ -3231,7 +3233,7 @@ static int do_message_op(const char *netbios_name, const char *desthost,
 		}
 	}
 
-	gensec_init(cmdline_lp_ctx);
+	gensec_init();
 
 	if(poptPeekArg(pc)) {
 		char *s = strdup(poptGetArg(pc)); 
diff --git a/source4/cluster/cluster.c b/source4/cluster/cluster.c
index 746c0048202..757489ebce6 100644
--- a/source4/cluster/cluster.c
+++ b/source4/cluster/cluster.c
@@ -23,7 +23,7 @@
 #include "cluster/cluster.h"
 #include "cluster/cluster_private.h"
 #include "librpc/gen_ndr/misc.h"
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
 static struct cluster_ops *ops;
 
@@ -50,23 +50,12 @@ static void cluster_init(void)
 /*
   create a server_id for the local node
 */
-struct server_id cluster_id(uint64_t id, uint32_t id2)
+struct server_id cluster_id(uint64_t pid, uint32_t task_id)
 {
 	cluster_init();
-	return ops->cluster_id(ops, id, id2);
+	return ops->cluster_id(ops, pid, task_id);
 }
 
-
-/*
-  return a server_id as a string
-*/
-const char *cluster_id_string(TALLOC_CTX *mem_ctx, struct server_id id)
-{
-	cluster_init();
-	return ops->cluster_id_string(ops, mem_ctx, id);
-}
-
-
 /*
   open a temporary tdb in a cluster friendly manner
 */
@@ -80,7 +69,7 @@ struct tdb_wrap *cluster_tdb_tmp_open(TALLOC_CTX *mem_ctx, struct loadparm_conte
 /*
   register a callback function for a messaging endpoint
 */
-NTSTATUS cluster_message_init(struct messaging_context *msg, struct server_id server,
+NTSTATUS cluster_message_init(struct imessaging_context *msg, struct server_id server,
 			      cluster_message_fn_t handler)
 {
 	cluster_init();
diff --git a/source4/cluster/cluster.h b/source4/cluster/cluster.h
index 6cfcb9b21b3..3dd9f4ce7c5 100644
--- a/source4/cluster/cluster.h
+++ b/source4/cluster/cluster.h
@@ -22,30 +22,29 @@
 #ifndef __CLUSTER_H__
 #define __CLUSTER_H__ 
 
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
 /*
   test for same cluster id
 */
-#define cluster_id_equal(id_1, id_2) ((id_1)->id == (id_2)->id \
-				    && (id_1)->id2 == (id_2)->id2 \
-				    && (id_1)->node == (id_2)->node)
+#define cluster_id_equal(id_1, id_2) ((id_1)->pid == (id_2)->pid \
+				    && (id_1)->task_id == (id_2)->task_id \
+				    && (id_1)->vnn == (id_2)->vnn)
 
 /*
   test for same cluster node
 */
-#define cluster_node_equal(id1, id2) ((id1)->node == (id2)->node)
+#define cluster_node_equal(id1, id2) ((id1)->vnn == (id2)->vnn)
 
-struct messaging_context;
-typedef void (*cluster_message_fn_t)(struct messaging_context *, DATA_BLOB);
+struct imessaging_context;
+typedef void (*cluster_message_fn_t)(struct imessaging_context *, DATA_BLOB);
 
 /* prototypes */
-struct server_id cluster_id(uint64_t id, uint32_t id2);
-const char *cluster_id_string(TALLOC_CTX *mem_ctx, struct server_id id);
+struct server_id cluster_id(uint64_t id, uint32_t task_id);
 struct tdb_wrap *cluster_tdb_tmp_open(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx, const char *dbname, int flags);
 void *cluster_backend_handle(void);
 
-NTSTATUS cluster_message_init(struct messaging_context *msg, struct server_id server,
+NTSTATUS cluster_message_init(struct imessaging_context *msg, struct server_id server,
 			      cluster_message_fn_t handler);
 NTSTATUS cluster_message_send(struct server_id server, DATA_BLOB *data);
 
diff --git a/source4/cluster/cluster_private.h b/source4/cluster/cluster_private.h
index e57c983ed0c..6f68ad62bd7 100644
--- a/source4/cluster/cluster_private.h
+++ b/source4/cluster/cluster_private.h
@@ -24,15 +24,13 @@
 
 struct cluster_ops {
 	struct server_id (*cluster_id)(struct cluster_ops *ops, uint64_t id, uint32_t id2);
-	const char *(*cluster_id_string)(struct cluster_ops *ops, 
-					 TALLOC_CTX *, struct server_id );
 	struct tdb_wrap *(*cluster_tdb_tmp_open)(struct cluster_ops *,
 						 TALLOC_CTX *, 
 						 struct loadparm_context *,
 						 const char *, int);
 	void *(*backend_handle)(struct cluster_ops *);
 	NTSTATUS (*message_init)(struct cluster_ops *ops, 
-				 struct messaging_context *msg, struct server_id server,
+				 struct imessaging_context *msg, struct server_id server,
 				 cluster_message_fn_t handler);
 	NTSTATUS (*message_send)(struct cluster_ops *ops,
 				 struct server_id server, DATA_BLOB *data);	
diff --git a/source4/cluster/local.c b/source4/cluster/local.c
index a93b0a65ede..0a294b4d1d1 100644
--- a/source4/cluster/local.c
+++ b/source4/cluster/local.c
@@ -22,36 +22,26 @@
 #include "includes.h"
 #include "cluster/cluster.h"
 #include "cluster/cluster_private.h"
-#include <tdb.h>
-#include "tdb_wrap.h"
+#include "tdb_compat.h"
+#include "lib/util/tdb_wrap.h"
 #include "system/filesys.h"
 #include "param/param.h"
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
 /*
   server a server_id for the local node
 */
-static struct server_id local_id(struct cluster_ops *ops, uint64_t id, uint32_t id2)
+static struct server_id local_id(struct cluster_ops *ops, uint64_t pid, uint32_t task_id)
 {
 	struct server_id server_id;
 	ZERO_STRUCT(server_id);
-	server_id.id = id;
-	server_id.id2 = id2;
+	server_id.pid = pid;
+	server_id.task_id = task_id;
 	return server_id;
 }
 
 
 /*
-  return a server_id as a string
-*/
-static const char *local_id_string(struct cluster_ops *ops,
-				   TALLOC_CTX *mem_ctx, struct server_id id)
-{
-	return talloc_asprintf(mem_ctx, "%u.%llu.%u", id.node, (unsigned long long)id.id, id.id2);
-}
-
-
-/*
   open a tmp tdb for the local node. By using smbd_tmp_path() we don't need
   TDB_CLEAR_IF_FIRST as the tmp path is wiped at startup
 */
@@ -80,7 +70,7 @@ static void *local_backend_handle(struct cluster_ops *ops)
   dummy message init function - not needed as all messages are local
 */
 static NTSTATUS local_message_init(struct cluster_ops *ops,
-				   struct messaging_context *msg, 
+				   struct imessaging_context *msg,
 				   struct server_id server,
 				   cluster_message_fn_t handler)
 {
@@ -98,7 +88,6 @@ static NTSTATUS local_message_send(struct cluster_ops *ops,
 
 static struct cluster_ops cluster_local_ops = {
 	.cluster_id           = local_id,
-	.cluster_id_string    = local_id_string,
 	.cluster_tdb_tmp_open = local_tdb_tmp_open,
 	.backend_handle       = local_backend_handle,
 	.message_init         = local_message_init,
diff --git a/source4/dns_server/dlz_bind9.c b/source4/dns_server/dlz_bind9.c
index 7dd06c1b2c8..4873112a11e 100644
--- a/source4/dns_server/dlz_bind9.c
+++ b/source4/dns_server/dlz_bind9.c
@@ -525,7 +525,7 @@ _PUBLIC_ isc_result_t dlz_create(const char *dlzname,
 
 	if (options.url == NULL) {
 		options.url = talloc_asprintf(tmp_ctx, "ldapi://%s",
-					      private_path(tmp_ctx, state->lp, "ldap_priv/ldapi"));
+					      lpcfg_private_path(tmp_ctx, state->lp, "ldap_priv/ldapi"));
 		if (options.url == NULL) {
 			result = ISC_R_NOMEMORY;
 			goto failed;
@@ -533,7 +533,7 @@ _PUBLIC_ isc_result_t dlz_create(const char *dlzname,
 	}
 
 	ret = ldb_connect(state->samdb, options.url, 0, NULL);
-	if (ret == -1) {
+	if (ret != LDB_SUCCESS) {
 		state->log(ISC_LOG_ERROR, "samba_dlz: Failed to connect to %s - %s",
 			   options.url, ldb_errstring(state->samdb));
 		result = ISC_R_FAILURE;
diff --git a/source4/dns_server/dns_server.c b/source4/dns_server/dns_server.c
index 0e5def15c00..0c36c20773b 100644
--- a/source4/dns_server/dns_server.c
+++ b/source4/dns_server/dns_server.c
@@ -490,7 +490,7 @@ static NTSTATUS dns_add_socket(struct dns_server *dns,
 						address, port,
 						&dns_socket->local_address);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		return status;
 	}
 
@@ -519,7 +519,7 @@ static NTSTATUS dns_add_socket(struct dns_server *dns,
 				     dns_udp_socket,
 				     &dns_udp_socket->dgram);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		DEBUG(0,("Failed to bind to %s:%u UDP - %s\n",
 			 address, port, nt_errstr(status)));
 		return status;
@@ -559,10 +559,10 @@ static NTSTATUS dns_startup_interfaces(struct dns_server *dns, struct loadparm_c
 		return NT_STATUS_INTERNAL_ERROR;
 	}
 
-	num_interfaces = iface_count(ifaces);
+	num_interfaces = iface_list_count(ifaces);
 
 	for (i=0; i<num_interfaces; i++) {
-		const char *address = talloc_strdup(tmp_ctx, iface_n_ip(ifaces, i));
+		const char *address = talloc_strdup(tmp_ctx, iface_list_n_ip(ifaces, i));
 
 		status = dns_add_socket(dns, model_ops, "dns", address, DNS_SERVICE_PORT);
 		NT_STATUS_NOT_OK_RETURN(status);
@@ -617,9 +617,9 @@ static void dns_task_init(struct task_server *task)
 		break;
 	}
 
-	load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
+	load_interface_list(task, task->lp_ctx, &ifaces);
 
-	if (iface_count(ifaces) == 0) {
+	if (iface_list_count(ifaces) == 0) {
 		task_server_terminate(task, "dns: no network interfaces configured", false);
 		return;
 	}
diff --git a/source4/dsdb/common/util.c b/source4/dsdb/common/util.c
index 0c920d7d855..3fa8f67447c 100644
--- a/source4/dsdb/common/util.c
+++ b/source4/dsdb/common/util.c
@@ -1762,7 +1762,7 @@ const char *samdb_client_site_name(struct ldb_context *ldb, TALLOC_CTX *mem_ctx,
 
 		allow_list[0] = l_subnet_name;
 
-		if (allow_access(mem_ctx, NULL, allow_list, "", ip_address)) {
+		if (socket_allow_access(mem_ctx, NULL, allow_list, "", ip_address)) {
 			sites_dn = ldb_msg_find_attr_as_dn(ldb, mem_ctx,
 							   res->msgs[i],
 							   "siteObject");
@@ -3192,7 +3192,7 @@ bool dsdb_dn_is_deleted_val(const struct ldb_val *val)
  */
 bool dsdb_dn_is_upgraded_link_val(struct ldb_val *val)
 {
-	return memmem(val->data, val->length, "<RMD_ADDTIME=", 13) != NULL;
+	return memmem(val->data, val->length, "<RMD_VERSION=", 13) != NULL;
 }
 
 /*
diff --git a/source4/dsdb/common/util_samr.c b/source4/dsdb/common/util_samr.c
index 7a4f6441236..83a8c385af6 100644
--- a/source4/dsdb/common/util_samr.c
+++ b/source4/dsdb/common/util_samr.c
@@ -342,6 +342,11 @@ NTSTATUS dsdb_add_domain_alias(struct ldb_context *ldb,
 	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
 	NT_STATUS_HAVE_NO_MEMORY(tmp_ctx);
 
+	if (ldb_transaction_start(ldb) != LDB_SUCCESS) {
+		DEBUG(0, ("Failed to start transaction in dsdb_add_domain_alias(): %s\n", ldb_errstring(ldb)));
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
 	/* Check if alias already exists */
 	name = samdb_search_string(ldb, tmp_ctx, NULL,
 				   "sAMAccountName",
@@ -350,12 +355,14 @@ NTSTATUS dsdb_add_domain_alias(struct ldb_context *ldb,
 
 	if (name != NULL) {
 		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(ldb);
 		return NT_STATUS_ALIAS_EXISTS;
 	}
 
 	msg = ldb_msg_new(tmp_ctx);
 	if (msg == NULL) {
 		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(ldb);
 		return NT_STATUS_NO_MEMORY;
 	}
 
@@ -364,6 +371,7 @@ NTSTATUS dsdb_add_domain_alias(struct ldb_context *ldb,
 	ldb_dn_add_child_fmt(msg->dn, "CN=%s,CN=Users", alias_name);
 	if (!msg->dn) {
 		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(ldb);
 		return NT_STATUS_NO_MEMORY;
 	}
 
@@ -378,15 +386,18 @@ NTSTATUS dsdb_add_domain_alias(struct ldb_context *ldb,
 		break;
 	case LDB_ERR_ENTRY_ALREADY_EXISTS:
 		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(ldb);
 		return NT_STATUS_ALIAS_EXISTS;
 	case LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS:
 		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(ldb);
 		return NT_STATUS_ACCESS_DENIED;
 	default:
 		DEBUG(0,("Failed to create alias record %s: %s\n",
 			 ldb_dn_get_linearized(msg->dn),
 			 ldb_errstring(ldb)));
 		talloc_free(tmp_ctx);
+		ldb_transaction_cancel(ldb);
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
@@ -394,10 +405,17 @@ NTSTATUS dsdb_add_domain_alias(struct ldb_context *ldb,
 	alias_sid = samdb_search_dom_sid(ldb, tmp_ctx,
 					 msg->dn, "objectSid", NULL);
 
+	if (ldb_transaction_commit(ldb) != LDB_SUCCESS) {
+		DEBUG(0, ("Failed to commit transaction in dsdb_add_domain_alias(): %s\n",
+			  ldb_errstring(ldb)));
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
 	*dn = talloc_steal(mem_ctx, msg->dn);
 	*sid = talloc_steal(mem_ctx, alias_sid);
 	talloc_free(tmp_ctx);
 
+
 	return NT_STATUS_OK;
 }
 
diff --git a/source4/dsdb/dns/dns_update.c b/source4/dsdb/dns/dns_update.c
index ede730a8a94..6650534d131 100644
--- a/source4/dsdb/dns/dns_update.c
+++ b/source4/dsdb/dns/dns_update.c
@@ -79,7 +79,7 @@ static void dnsupdate_rndc_done(struct tevent_req *subreq)
 	ret = samba_runcmd_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret != 0) {
-		service->confupdate.status = map_nt_error_from_unix(sys_errno);
+		service->confupdate.status = map_nt_error_from_unix_common(sys_errno);
 	} else {
 		service->confupdate.status = NT_STATUS_OK;
 	}
@@ -123,12 +123,12 @@ static void dnsupdate_rebuild(struct dnsupdate_service *service)
 
 	path = lpcfg_parm_string(service->task->lp_ctx, NULL, "dnsupdate", "path");
 	if (path == NULL) {
-		path = private_path(tmp_ctx, service->task->lp_ctx, "named.conf.update");
+		path = lpcfg_private_path(tmp_ctx, service->task->lp_ctx, "named.conf.update");
 	}
 
 	path_static = lpcfg_parm_string(service->task->lp_ctx, NULL, "dnsupdate", "extra_static_grant_rules");
 	if (path_static == NULL) {
-		path_static = private_path(tmp_ctx, service->task->lp_ctx, "named.conf.update.static");
+		path_static = lpcfg_private_path(tmp_ctx, service->task->lp_ctx, "named.conf.update.static");
 	}
 
 	tmp_path = talloc_asprintf(tmp_ctx, "%s.tmp", path);
@@ -242,7 +242,7 @@ static void dnsupdate_nameupdate_done(struct tevent_req *subreq)
 	ret = samba_runcmd_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret != 0) {
-		service->nameupdate.status = map_nt_error_from_unix(sys_errno);
+		service->nameupdate.status = map_nt_error_from_unix_common(sys_errno);
 	} else {
 		service->nameupdate.status = NT_STATUS_OK;
 	}
@@ -271,7 +271,7 @@ static void dnsupdate_spnupdate_done(struct tevent_req *subreq)
 	ret = samba_runcmd_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret != 0) {
-		service->nameupdate.status = map_nt_error_from_unix(sys_errno);
+		service->nameupdate.status = map_nt_error_from_unix_common(sys_errno);
 	} else {
 		service->nameupdate.status = NT_STATUS_OK;
 	}
@@ -381,7 +381,7 @@ static void dnsupdate_RODC_callback(struct tevent_req *req)
 	ret = samba_runcmd_recv(req, &sys_errno);
 	talloc_free(req);
 	if (ret != 0) {
-		st->r->out.result = map_nt_error_from_unix(sys_errno);
+		st->r->out.result = map_nt_error_from_unix_common(sys_errno);
 		DEBUG(2,(__location__ ": RODC DNS Update failed: %s\n", nt_errstr(st->r->out.result)));
 	} else {
 		st->r->out.result = NT_STATUS_OK;
diff --git a/source4/dsdb/pydsdb.c b/source4/dsdb/pydsdb.c
index 895bd9a5605..5ca6b02608a 100644
--- a/source4/dsdb/pydsdb.c
+++ b/source4/dsdb/pydsdb.c
@@ -331,6 +331,38 @@ static PyObject *py_dsdb_get_attid_from_lDAPDisplayName(PyObject *self, PyObject
 }
 
 /*
+  return the attribute syntax oid as a string from the attribute name
+ */
+static PyObject *py_dsdb_get_syntax_oid_from_lDAPDisplayName(PyObject *self, PyObject *args)
+{
+	PyObject *py_ldb;
+	struct ldb_context *ldb;
+	struct dsdb_schema *schema;
+	const char *ldap_display_name;
+	const struct dsdb_attribute *attribute;
+
+	if (!PyArg_ParseTuple(args, "Os", &py_ldb, &ldap_display_name))
+		return NULL;
+
+	PyErr_LDB_OR_RAISE(py_ldb, ldb);
+
+	schema = dsdb_get_schema(ldb, NULL);
+
+	if (!schema) {
+		PyErr_SetString(PyExc_RuntimeError, "Failed to find a schema from ldb");
+		return NULL;
+	}
+
+	attribute = dsdb_attribute_by_lDAPDisplayName(schema, ldap_display_name);
+	if (attribute == NULL) {
+		PyErr_Format(PyExc_RuntimeError, "Failed to find attribute '%s'", ldap_display_name);
+		return NULL;
+	}
+
+	return PyString_FromString(attribute->syntax->ldap_oid);
+}
+
+/*
   convert a python string to a DRSUAPI drsuapi_DsReplicaAttribute attribute
  */
 static PyObject *py_dsdb_DsReplicaAttribute(PyObject *self, PyObject *args)
@@ -423,6 +455,109 @@ static PyObject *py_dsdb_DsReplicaAttribute(PyObject *self, PyObject *args)
 	return ret;
 }
 
+
+/*
+  normalise a ldb attribute list
+ */
+static PyObject *py_dsdb_normalise_attributes(PyObject *self, PyObject *args)
+{
+	PyObject *py_ldb, *el_list, *ret;
+	struct ldb_context *ldb;
+	char *ldap_display_name;
+	const struct dsdb_attribute *a;
+	struct dsdb_schema *schema;
+	struct dsdb_syntax_ctx syntax_ctx;
+	struct ldb_message_element *el;
+	struct drsuapi_DsReplicaAttribute *attr;
+	TALLOC_CTX *tmp_ctx;
+	WERROR werr;
+	Py_ssize_t i;
+
+	if (!PyArg_ParseTuple(args, "OsO", &py_ldb, &ldap_display_name, &el_list)) {
+		return NULL;
+	}
+
+	PyErr_LDB_OR_RAISE(py_ldb, ldb);
+
+	if (!PyList_Check(el_list)) {
+		PyErr_Format(PyExc_TypeError, "ldif_elements must be a list");
+		return NULL;
+	}
+
+	schema = dsdb_get_schema(ldb, NULL);
+	if (!schema) {
+		PyErr_SetString(PyExc_RuntimeError, "Failed to find a schema from ldb");
+		return NULL;
+	}
+
+	a = dsdb_attribute_by_lDAPDisplayName(schema, ldap_display_name);
+	if (a == NULL) {
+		PyErr_Format(PyExc_RuntimeError, "Failed to find attribute '%s'", ldap_display_name);
+		return NULL;
+	}
+
+	dsdb_syntax_ctx_init(&syntax_ctx, ldb, schema);
+	syntax_ctx.is_schema_nc = false;
+
+	tmp_ctx = talloc_new(ldb);
+	if (tmp_ctx == NULL) {
+		PyErr_NoMemory();
+		return NULL;
+	}
+
+	el = talloc_zero(tmp_ctx, struct ldb_message_element);
+	if (el == NULL) {
+		PyErr_NoMemory();
+		talloc_free(tmp_ctx);
+		return NULL;
+	}
+
+	el->name = ldap_display_name;
+	el->num_values = PyList_Size(el_list);
+
+	el->values = talloc_array(el, struct ldb_val, el->num_values);
+	if (el->values == NULL) {
+		PyErr_NoMemory();
+		talloc_free(tmp_ctx);
+		return NULL;
+	}
+
+	for (i = 0; i < el->num_values; i++) {
+		PyObject *item = PyList_GetItem(el_list, i);
+		if (!PyString_Check(item)) {
+			PyErr_Format(PyExc_TypeError, "ldif_elements should be strings");
+			return NULL;
+		}
+		el->values[i].data = (uint8_t *)PyString_AsString(item);
+		el->values[i].length = PyString_Size(item);
+	}
+
+	/* first run ldb_to_drsuapi, then convert back again. This has
+	 * the effect of normalising the attributes
+	 */
+
+	attr = talloc_zero(tmp_ctx, struct drsuapi_DsReplicaAttribute);
+	if (attr == NULL) {
+		PyErr_NoMemory();
+		talloc_free(tmp_ctx);
+		return NULL;
+	}
+
+	werr = a->syntax->ldb_to_drsuapi(&syntax_ctx, a, el, attr, attr);
+	PyErr_WERROR_IS_ERR_RAISE(werr);
+
+	/* now convert back again */
+	werr = a->syntax->drsuapi_to_ldb(&syntax_ctx, a, attr, el, el);
+	PyErr_WERROR_IS_ERR_RAISE(werr);
+
+	ret = py_return_ndr_struct("ldb", "MessageElement", el, el);
+
+	talloc_free(tmp_ctx);
+
+	return ret;
+}
+
+
 static PyObject *py_dsdb_set_ntds_invocation_id(PyObject *self, PyObject *args)
 {
 	PyObject *py_ldb, *py_guid;
@@ -699,6 +834,8 @@ static PyMethodDef py_dsdb_methods[] = {
 		METH_VARARGS, NULL },
 	{ "_dsdb_get_attid_from_lDAPDisplayName", (PyCFunction)py_dsdb_get_attid_from_lDAPDisplayName,
 		METH_VARARGS, NULL },
+	{ "_dsdb_get_syntax_oid_from_lDAPDisplayName", (PyCFunction)py_dsdb_get_syntax_oid_from_lDAPDisplayName,
+		METH_VARARGS, NULL },
 	{ "_dsdb_set_ntds_invocation_id",
 		(PyCFunction)py_dsdb_set_ntds_invocation_id, METH_VARARGS,
 		NULL },
@@ -723,6 +860,7 @@ static PyMethodDef py_dsdb_methods[] = {
 		NULL },
 	{ "_dsdb_get_partitions_dn", (PyCFunction)py_dsdb_get_partitions_dn, METH_VARARGS, NULL },
 	{ "_dsdb_DsReplicaAttribute", (PyCFunction)py_dsdb_DsReplicaAttribute, METH_VARARGS, NULL },
+	{ "_dsdb_normalise_attributes", (PyCFunction)py_dsdb_normalise_attributes, METH_VARARGS, NULL },
 	{ NULL }
 };
 
@@ -862,4 +1000,10 @@ void initdsdb(void)
 	ADD_DSDB_FLAG(GPO_FLAG_MACHINE_DISABLE);
 	ADD_DSDB_FLAG(GPO_INHERIT);
 	ADD_DSDB_FLAG(GPO_BLOCK_INHERITANCE);
+
+#define ADD_DSDB_STRING(val)  PyModule_AddObject(m, #val, PyString_FromString(val))
+
+	ADD_DSDB_STRING(DSDB_SYNTAX_BINARY_DN);
+	ADD_DSDB_STRING(DSDB_SYNTAX_STRING_DN);
+	ADD_DSDB_STRING(DSDB_SYNTAX_OR_NAME);
 }
diff --git a/source4/dsdb/repl/drepl_fsmo.c b/source4/dsdb/repl/drepl_fsmo.c
index f8f4769f1b6..db6385315b4 100644
--- a/source4/dsdb/repl/drepl_fsmo.c
+++ b/source4/dsdb/repl/drepl_fsmo.c
@@ -111,7 +111,7 @@ NTSTATUS drepl_take_FSMO_role(struct irpc_message *msg,
 
 	if (fsmo_master_equal(ntds_dn, role_owner_dn) ||
 	    (extended_op == DRSUAPI_EXOP_NONE)) {
-		DEBUG(0,("FSMO role check failed for DN %s and owner %s ",
+		DEBUG(0,("FSMO role check failed for DN %s and owner %s \n",
 			 ldb_dn_get_linearized(fsmo_role_dn),
 			 ldb_dn_get_linearized(role_owner_dn)));
 		r->out.result = WERR_OK;
diff --git a/source4/dsdb/repl/drepl_ridalloc.c b/source4/dsdb/repl/drepl_ridalloc.c
index 48c208c3cf8..53b56b63d2f 100644
--- a/source4/dsdb/repl/drepl_ridalloc.c
+++ b/source4/dsdb/repl/drepl_ridalloc.c
@@ -236,7 +236,7 @@ WERROR dreplsrv_ridalloc_check_rid_pool(struct dreplsrv_service *service)
 
 /* called by the samldb ldb module to tell us to ask for a new RID
    pool */
-void dreplsrv_allocate_rid(struct messaging_context *msg, void *private_data,
+void dreplsrv_allocate_rid(struct imessaging_context *msg, void *private_data,
 			   uint32_t msg_type,
 			   struct server_id server_id, DATA_BLOB *data)
 {
diff --git a/source4/dsdb/repl/drepl_service.c b/source4/dsdb/repl/drepl_service.c
index 0931a340b1a..ec803f6fdb0 100644
--- a/source4/dsdb/repl/drepl_service.c
+++ b/source4/dsdb/repl/drepl_service.c
@@ -497,7 +497,7 @@ static void dreplsrv_task_init(struct task_server *task)
 	IRPC_REGISTER(task->msg_ctx, drsuapi, DRSUAPI_DSREPLICAMOD, dreplsrv_replica_mod, service);
 	IRPC_REGISTER(task->msg_ctx, irpc, DREPL_TAKEFSMOROLE, drepl_take_FSMO_role, service);
 	IRPC_REGISTER(task->msg_ctx, irpc, DREPL_TRIGGER_REPL_SECRET, drepl_trigger_repl_secret, service);
-	messaging_register(task->msg_ctx, service, MSG_DREPL_ALLOCATE_RID, dreplsrv_allocate_rid);
+	imessaging_register(task->msg_ctx, service, MSG_DREPL_ALLOCATE_RID, dreplsrv_allocate_rid);
 }
 
 /*
diff --git a/source4/dsdb/samdb/ldb_modules/acl_read.c b/source4/dsdb/samdb/ldb_modules/acl_read.c
index 181619ab287..35a840e1f49 100644
--- a/source4/dsdb/samdb/ldb_modules/acl_read.c
+++ b/source4/dsdb/samdb/ldb_modules/acl_read.c
@@ -47,6 +47,7 @@ struct aclread_context {
 	bool sd;
 	bool instance_type;
 	bool object_sid;
+	bool indirsync;
 };
 
 struct aclread_private {
@@ -158,18 +159,41 @@ static int aclread_callback(struct ldb_request *req, struct ldb_reply *ares)
 							     access_mask,
 							     attr);
 
-			 if (ret == LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS) {
-				 /* do not return this entry if attribute is
-				    part of the search filter */
-				 if (dsdb_attr_in_parse_tree(ac->req->op.search.tree,
-							     msg->elements[i].name)) {
-					 talloc_free(tmp_ctx);
-					 return LDB_SUCCESS;
-				 }
-				 aclread_mark_inaccesslible(&msg->elements[i]);
-			 } else if (ret != LDB_SUCCESS) {
-				 goto fail;
-			 }
+			/*
+			 * Dirsync control needs the replpropertymetadata attribute
+			 * so return it as it will be removed by the control
+			 * in anycase.
+			 */
+			if (ret == LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS) {
+				if (!ac->indirsync) {
+					/* do not return this entry if attribute is
+					part of the search filter */
+					if (dsdb_attr_in_parse_tree(ac->req->op.search.tree,
+								msg->elements[i].name)) {
+						talloc_free(tmp_ctx);
+						return LDB_SUCCESS;
+					}
+					aclread_mark_inaccesslible(&msg->elements[i]);
+				} else {
+					/*
+					 * We are doing dirysnc answers
+					 * and the object shouldn't be returned (normally)
+					 * but we will return it without replPropertyMetaData
+					 * so that the dirysync module will do what is needed
+					 * (remove the object if it is not deleted, or return
+					 * just the objectGUID if it's deleted).
+					 */
+					if (dsdb_attr_in_parse_tree(ac->req->op.search.tree,
+								msg->elements[i].name)) {
+						ldb_msg_remove_attr(msg, "replPropertyMetaData");
+						break;
+					} else {
+						aclread_mark_inaccesslible(&msg->elements[i]);
+					}
+				}
+			} else if (ret != LDB_SUCCESS) {
+				goto fail;
+			}
 		 }
 		 for (i=0; i < msg->num_elements; i++) {
 			 if (!aclread_is_inaccessible(&msg->elements[i])) {
@@ -224,6 +248,7 @@ static int aclread_search(struct ldb_module *module, struct ldb_request *req)
 	struct aclread_context *ac;
 	struct ldb_request *down_req;
 	struct ldb_control *as_system = ldb_request_get_control(req, LDB_CONTROL_AS_SYSTEM_OID);
+	uint32_t flags = ldb_req_get_custom_flags(req);
 	struct ldb_result *res;
 	struct aclread_private *p;
 	bool is_untrusted = ldb_req_is_untrusted(req);
@@ -284,6 +309,11 @@ static int aclread_search(struct ldb_module *module, struct ldb_request *req)
 	ac->module = module;
 	ac->req = req;
 	ac->schema = dsdb_get_schema(ldb, req);
+	if (flags & DSDB_ACL_CHECKS_DIRSYNC_FLAG) {
+		ac->indirsync = true;
+	} else {
+		ac->indirsync = false;
+	}
 	if (!ac->schema) {
 		return ldb_operr(ldb);
 	}
diff --git a/source4/dsdb/samdb/ldb_modules/dirsync.c b/source4/dsdb/samdb/ldb_modules/dirsync.c
new file mode 100644
index 00000000000..64c5047798d
--- /dev/null
+++ b/source4/dsdb/samdb/ldb_modules/dirsync.c
@@ -0,0 +1,1359 @@
+/*
+   SAMDB control module
+
+   Copyright (C) Matthieu Patou <mat@matws.net> 2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "includes.h"
+#include "ldb/include/ldb.h"
+#include "ldb/include/ldb_errors.h"
+#include "ldb/include/ldb_module.h"
+#include "libcli/security/security.h"
+#include "librpc/gen_ndr/drsblobs.h"
+#include "librpc/gen_ndr/ndr_drsblobs.h"
+#include "librpc/ndr/libndr.h"
+#include "dsdb/samdb/samdb.h"
+#include "util.h"
+
+#define LDAP_DIRSYNC_OBJECT_SECURITY		0x01
+#define LDAP_DIRSYNC_ANCESTORS_FIRST_ORDER	0x800
+#define LDAP_DIRSYNC_PUBLIC_DATA_ONLY		0x2000
+#define LDAP_DIRSYNC_INCREMENTAL_VALUES		0x80000000
+
+
+struct dirsync_context {
+	struct ldb_module *module;
+	struct ldb_request *req;
+
+	/*
+	 * We keep a track of the number of attributes that we
+	 * add just for the need of the implementation
+	 * it will be usefull to track then entries that needs not to
+	 * be returned because there is no real change
+	 */
+
+	unsigned int nbDefaultAttrs;
+	uint64_t highestUSN;
+	uint64_t fromreqUSN;
+	uint32_t cursor_size;
+	bool noextended;
+	bool linkIncrVal;
+	bool localonly;
+	bool partial;
+	bool assystem;
+	int functional_level;
+	const struct GUID *our_invocation_id;
+	const struct dsdb_schema *schema;
+	struct ldb_dn *nc_root;
+	struct drsuapi_DsReplicaCursor *cursors;
+};
+
+
+static int dirsync_filter_entry(struct ldb_request *req,
+					struct ldb_message *msg,
+					struct ldb_control **controls,
+					struct dirsync_context *dsc,
+					bool referral)
+{
+	struct ldb_context *ldb;
+	uint64_t val;
+	enum ndr_err_code ndr_err;
+	uint32_t n;
+	int i;
+	unsigned int size, j;
+	uint32_t deletedattr;
+	struct ldb_val *replMetaData = NULL;
+	struct replPropertyMetaDataBlob rmd;
+	const struct dsdb_attribute *attr;
+	const char **listAttr = NULL;
+	bool namereturned = false;
+	bool nameasked = false;
+	NTSTATUS status;
+	/* Ajustment for the added attributes, it will reduce the number of
+	 * expected to be here attributes*/
+	unsigned int delta = 0;
+	const char **myaccept = NULL;
+	const char *emptyaccept[] = { NULL };
+	const char *extendedaccept[] = { "GUID", "SID", "WKGUID", NULL };
+	const char *rdn = NULL;
+	struct ldb_message_element *el;
+	struct ldb_message *newmsg;
+	bool keep = false;
+	/*
+	 * Where we asked to do extended dn ?
+	 * if so filter out everything bug GUID, SID, WKGUID,
+	 * if not filter out everything (just keep the dn).
+	 */
+	if ( dsc->noextended == true ) {
+		myaccept = emptyaccept;
+	} else {
+		myaccept = extendedaccept;
+	}
+	ldb = ldb_module_get_ctx(dsc->module);
+
+	if (msg->num_elements == 0) {
+		/*
+			* Entry that we don't really have access to
+			*/
+		return LDB_SUCCESS;
+	}
+	ldb_dn_extended_filter(msg->dn, myaccept);
+
+	/*
+	* If the RDN starts with CN then the CN attribute is never returned
+	*/
+	rdn = ldb_dn_get_rdn_name(msg->dn);
+
+	deletedattr = 0;
+	/*
+	 * if objectGUID is asked and we are dealing for the referrals entries and
+	 * the usn searched is 0 then we didn't count the objectGUID as an automatically
+	 * returned attribute, do to so we increament delta.
+	 */
+	if (referral == true &&
+			ldb_attr_in_list(req->op.search.attrs, "objectGUID") &&
+			dsc->fromreqUSN == 0) {
+		delta++;
+	}
+
+
+	/*
+	 * In terms of big O notation this is not the best algorithm,
+	 * but we try our best not to make the worse one.
+	 * We are obliged to run through the n message's elements
+	 * and through the p elements of the replPropertyMetaData.
+	 *
+	 * It turns out that we are crawling twice the message's elements
+	 * the first crawl is to remove the non replicated and generated
+	 * attributes. The second one is to remove attributes that haven't
+	 * a USN > as the requested one.
+	 *
+	 * In the second crawl we are reading the list of elements in the
+	 * replPropertyMetaData for each remaining replicated attribute.
+	 * In order to keep the list small
+	 *
+	 * We have a O(n'*p') complexity, in worse case n' = n and p' = p
+	 * but in most case n' = n/2 (at least half of returned attributes
+	 * are not replicated or generated) and p' is small as we
+	 * list only the attribute that have been modified since last interogation
+	 *
+	 */
+	newmsg = talloc_zero(dsc->req, struct ldb_message);
+	if (newmsg == NULL) {
+		return ldb_oom(ldb);
+	}
+	for (i = msg->num_elements - 1; i >= 0; i--) {
+		attr = dsdb_attribute_by_lDAPDisplayName(dsc->schema, msg->elements[i].name);
+		if (ldb_attr_cmp(msg->elements[i].name, "uSNChanged") == 0) {
+			/* Read the USN it will used at the end of the filtering
+			 * to update the max USN in the cookie if we
+			 * decide to keep this entry
+			 */
+			val = strtoull((const char*)msg->elements[i].values[0].data, NULL, 0);
+			continue;
+		}
+
+		if (ldb_attr_cmp(msg->elements[i].name,
+						"replPropertyMetaData") == 0) {
+			replMetaData = (talloc_steal(dsc, &msg->elements[i].values[0]));
+			continue;
+		}
+	}
+
+	if (replMetaData == NULL) {
+		bool guidfound = false;
+
+		/*
+		 * We are in the case of deleted object where we don't have the
+		 * right to read it.
+		 */
+		if (!ldb_msg_find_attr_as_uint(msg, "isDeleted", 0)) {
+			/*
+			 * This is not a deleted item and we don't
+			 * have the replPropertyMetaData.
+			 * Do not return it
+			 */
+			return LDB_SUCCESS;
+		}
+		newmsg->dn = ldb_dn_new(newmsg, ldb, "");
+		if (newmsg->dn == NULL) {
+			return ldb_oom(ldb);
+		}
+
+		el = ldb_msg_find_element(msg, "objectGUID");
+		if ( el != NULL) {
+			guidfound = true;
+		}
+		/*
+		 * We expect to find the GUID in the object,
+		 * if it turns out not to be the case sometime
+		 * well will uncomment the code bellow
+		 */
+		SMB_ASSERT(guidfound == true);
+		/*
+		if (guidfound == false) {
+			struct GUID guid;
+			struct ldb_val *new_val;
+			DATA_BLOB guid_blob;
+
+			tmp[0] = '\0';
+			txt = strrchr(txt, ':');
+			if (txt == NULL) {
+				return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+			}
+			txt++;
+
+			status = GUID_from_string(txt, &guid);
+			if (!NT_STATUS_IS_OK(status)) {
+				return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+			}
+
+			status = GUID_to_ndr_blob(&guid, msg, &guid_blob);
+			if (!NT_STATUS_IS_OK(status)) {
+				return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+			}
+
+			new_val = talloc(msg, struct ldb_val);
+			if (new_val == NULL) {
+				return ldb_oom(ldb);
+			}
+			new_val->data = talloc_steal(new_val, guid_blob.data);
+			new_val->length = guid_blob.length;
+			if (ldb_msg_add_value(msg, "objectGUID", new_val, NULL) != 0) {
+				return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+			}
+		}
+		*/
+		ldb_msg_add(newmsg, el, LDB_FLAG_MOD_ADD);
+		talloc_steal(newmsg->elements, el->name);
+		talloc_steal(newmsg->elements, el->values);
+
+		talloc_free(msg);
+		return ldb_module_send_entry(dsc->req, msg, controls);
+	}
+
+	ndr_err = ndr_pull_struct_blob(replMetaData, dsc, &rmd,
+		(ndr_pull_flags_fn_t)ndr_pull_replPropertyMetaDataBlob);
+	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+		ldb_set_errstring(ldb, "Unable to unmarshall replPropertyMetaData");
+		return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+	}
+	if (ldb_attr_in_list(req->op.search.attrs, "name") ||
+			ldb_attr_in_list(req->op.search.attrs, "*")) {
+		nameasked = true;
+	}
+
+	/*
+		* If we don't have an USN and no updateness array then we skip the
+		* test phase this is an optimisation for the case when you
+		* first query the DC without a cookie.
+		* As this query is most probably the one
+		* that will return the biggest answer, skipping this part
+		* will really save time.
+		*/
+	if (ldb_dn_compare(dsc->nc_root, msg->dn) == 0) {
+		/* If we have name then we expect to have parentGUID,
+		 * it will not be the case for the root of the NC
+		 */
+		delta++;
+	}
+
+	if (dsc->fromreqUSN > 0 || dsc->cursors != NULL) {
+		j = 0;
+		/*
+		* Allocate an array of size(replMetaData) of char*
+		* we know that it will be oversized but it's a short lived element
+		*/
+		listAttr = talloc_array(msg, const char*, rmd.ctr.ctr1.count + 1);
+		if (listAttr == NULL) {
+			return ldb_oom(ldb);
+		}
+		for (n=0; n < rmd.ctr.ctr1.count; n++) {
+			struct replPropertyMetaData1 *omd = &rmd.ctr.ctr1.array[n];
+			if (omd->local_usn > dsc->fromreqUSN) {
+				const struct dsdb_attribute *a = dsdb_attribute_by_attributeID_id(dsc->schema,
+										omd->attid);
+				if (!dsc->localonly) {
+					struct drsuapi_DsReplicaCursor *tab = dsc->cursors;
+					uint32_t l;
+					for (l=0; l < dsc->cursor_size; l++) {
+						if (GUID_equal(&tab[l].source_dsa_invocation_id, &omd->originating_invocation_id) &&
+								tab[l].highest_usn >= omd->originating_usn) {
+							/*
+							 * If we have in the uptodateness vector an entry
+							 * with the same invocation id as the originating invocation
+							 * and if the usn in the vector is greater or equal to
+							 * the one in originating_usn, then it means that this entry
+							 * has already been sent (from another DC) to the client
+							 * no need to resend it one more time.
+							 */
+							goto skip;
+						}
+					}
+					/* If we are here it's because we have a usn > (max(usn of vectors))*/
+				}
+				if (namereturned == false &&
+						nameasked == true &&
+						ldb_attr_cmp(a->lDAPDisplayName, "name") == 0) {
+					namereturned = true;
+					if (ldb_dn_compare(dsc->nc_root, msg->dn) == 0) {
+						delta++;
+					}
+				}
+				listAttr[j] = a->lDAPDisplayName;
+				j++;
+skip:
+				continue;
+			}
+		}
+		size = j;
+	} else {
+		size = 0;
+		if (ldb_attr_in_list(req->op.search.attrs, "*") ||
+				ldb_attr_in_list(req->op.search.attrs, "name")) {
+			namereturned = true;
+		}
+	}
+
+
+	/*
+	 * Let's loop around the remaining elements
+	 * to see which one are in the listAttr.
+	 * If they are in this array it means that
+	 * their localusn > usn from the request (in the cookie)
+	 * if not we remove the attribute.
+	 */
+	for (i = msg->num_elements - 1; i >= 0; i--) {
+		el = &(msg->elements[i]);
+		attr = dsdb_attribute_by_lDAPDisplayName(dsc->schema,
+				el->name);
+		const char *ldapattrname = el->name;
+		keep = false;
+
+		if (attr->linkID & 1) {
+			/*
+			 * Attribute is a backlink so let's remove it
+			 */
+			continue;
+		}
+
+		if (ldb_attr_cmp(msg->elements[i].name,
+						"replPropertyMetaData") == 0) {
+			continue;
+		}
+
+		if ((attr->systemFlags & (DS_FLAG_ATTR_NOT_REPLICATED | DS_FLAG_ATTR_IS_CONSTRUCTED))) {
+			if (ldb_attr_cmp(attr->lDAPDisplayName, "objectGUID") != 0 &&
+					ldb_attr_cmp(attr->lDAPDisplayName, "parentGUID") != 0) {
+				/*
+				 * Attribute is constructed or not replicated, let's get rid of it
+				 */
+				continue;
+			} else {
+				/* Let's keep the attribute that we forced to be added
+				 * even if they are not in the replicationMetaData
+				 * or are just generated
+				 */
+				if (namereturned == false &&
+					(ldb_attr_cmp(attr->lDAPDisplayName, "parentGUID") == 0)) {
+					delta++;
+					continue;
+				}
+				if (ldb_msg_add(newmsg, el, LDB_FLAG_MOD_ADD) != LDB_SUCCESS) {
+					return ldb_error(ldb,
+						LDB_ERR_OPERATIONS_ERROR,
+						"Unable to add attribute");
+				}
+				talloc_steal(newmsg->elements, el->name);
+				talloc_steal(newmsg->elements, el->values);
+				continue;
+			}
+		}
+
+		if (ldb_attr_cmp(msg->elements[i].name, rdn) == 0) {
+			/*
+			 * We have an attribute that is the same as the start of the RDN
+			 * (ie. attribute CN with rdn CN=).
+			 */
+			continue;
+		}
+
+		if (ldb_attr_cmp(attr->lDAPDisplayName, "instanceType") == 0) {
+			if (ldb_msg_add(newmsg, el, LDB_FLAG_MOD_ADD) != LDB_SUCCESS) {
+				return ldb_error(ldb,
+						LDB_ERR_OPERATIONS_ERROR,
+						"Unable to add attribute");
+			}
+			talloc_steal(newmsg->elements, el->name);
+			talloc_steal(newmsg->elements, el->values);
+			continue;
+		}
+		/* For links, when our functional level > windows 2000
+		 * we use the RMD_LOCAL_USN information to decide wether
+		 * we return the attribute or not.
+		 * For windows 2000 this information is in the replPropertyMetaData
+		 * so it will be handled like any other replicated attribute
+		 */
+
+		if (dsc->functional_level > DS_DOMAIN_FUNCTION_2000 &&
+				attr->linkID != 0 ) {
+			int k;
+			/*
+			 * Elements for incremental changes on linked attributes
+			 */
+			struct ldb_message_element *el_incr_add = NULL;
+			struct ldb_message_element *el_incr_del = NULL;
+			/*
+			 * Attribute is a forwardlink so let's remove it
+			 */
+
+			for (k = el->num_values -1; k >= 0; k--) {
+				char *dn_ln;
+				uint32_t flags = 0;
+				uint32_t tmp_usn = 0;
+				uint32_t tmp_usn2 = 0;
+				struct GUID invocation_id = GUID_zero();
+				struct dsdb_dn *dn = dsdb_dn_parse(msg, ldb, &el->values[k], attr->syntax->ldap_oid);
+				if (dn == NULL) {
+					ldb_set_errstring(ldb, "Cannot parse DN");
+					return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+				}
+				status = dsdb_get_extended_dn_uint32(dn->dn, &tmp_usn, "RMD_LOCAL_USN");
+				if (!NT_STATUS_IS_OK(status)) {
+					talloc_free(dn);
+					return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+				}
+				status = dsdb_get_extended_dn_guid(dn->dn,  &invocation_id, "RMD_INVOCID");
+				if (!NT_STATUS_IS_OK(status)) {
+					talloc_free(dn);
+					return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+				}
+
+				status = dsdb_get_extended_dn_uint32(dn->dn, &flags, "RMD_FLAGS");
+				if (!NT_STATUS_IS_OK(status)) {
+					talloc_free(dn);
+					return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+				}
+
+				status = dsdb_get_extended_dn_uint32(dn->dn, &tmp_usn2, "RMD_ORIGINATING_USN");
+				if (!NT_STATUS_IS_OK(status)) {
+					talloc_free(dn);
+					return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+				}
+
+				ldb_dn_extended_filter(dn->dn, myaccept);
+				dn_ln = ldb_dn_get_extended_linearized(dn, dn->dn, 1);
+				if (dn_ln == NULL)
+				{
+					talloc_free(dn);
+					ldb_set_errstring(ldb, "Cannot linearize dn");
+					return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+				}
+
+				talloc_free(el->values[k].data);
+				el->values[k].data = (uint8_t*)talloc_steal(el->values, dn_ln);
+				if (el->values[k].data == NULL) {
+					talloc_free(dn);
+					return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+				}
+				el->values[k].length = strlen(dn_ln);
+
+
+				if (tmp_usn > dsc->fromreqUSN) {
+					if (!dsc->localonly) {
+						struct drsuapi_DsReplicaCursor *tab = dsc->cursors;
+						uint32_t l;
+
+						for (l=0; l < dsc->cursor_size; l++) {
+							if (GUID_equal(&tab[l].source_dsa_invocation_id, &invocation_id) &&
+									tab[l].highest_usn >= tmp_usn2) {
+								/*
+								* If we have in the uptodateness vector an entry
+								* with the same invocation id as the originating invocation
+								* and if the usn in the vector is greater or equal to
+								* the one in originating_usn, then it means that this entry
+								* has already been sent (from another DC) to the client
+								* no need to resend it one more time.
+								*/
+								goto skip_link;
+							}
+						}
+						/* If we are here it's because we have a usn > (max(usn of vectors))*/
+						keep = true;
+					} else {
+						keep = true;
+					}
+				/* If we are here it's because the link is more recent than either any
+				 * originating usn or local usn
+				 */
+
+					if (dsc->linkIncrVal == true) {
+						struct ldb_message_element *tmpel;
+						if (flags & DSDB_RMD_FLAG_DELETED) {
+							tmpel = el_incr_del;
+						} else {
+							tmpel = el_incr_add;
+						}
+
+						if (tmpel == NULL) {
+							tmpel = talloc_zero(newmsg, struct ldb_message_element);
+							if (tmpel == NULL) {
+								return ldb_oom(ldb);
+							}
+							tmpel->values = talloc_array(tmpel, struct ldb_val, 1);
+							if (tmpel->values == NULL) {
+								return ldb_oom(ldb);
+							}
+							if (flags & DSDB_RMD_FLAG_DELETED) {
+								tmpel->name = talloc_asprintf(tmpel,
+										"%s;range=0-0",
+										el->name);
+							}
+							else {
+								tmpel->name = talloc_asprintf(tmpel,
+										"%s;range=1-1",
+										el->name);
+							}
+							if (tmpel->name == NULL) {
+								return ldb_oom(ldb);
+							}
+							tmpel->num_values = 1;
+						} else {
+							tmpel->num_values += 1;
+							tmpel->values = talloc_realloc(tmpel,
+												tmpel->values,
+												struct ldb_val,
+												tmpel->num_values);
+							if (tmpel->values == NULL) {
+								return ldb_oom(ldb);
+							}
+							tmpel = tmpel;
+						}
+						tmpel->values[tmpel->num_values -1].data =talloc_steal(tmpel->values, el->values[k].data);
+						tmpel->values[tmpel->num_values -1].length = el->values[k].length;
+
+						if (flags & DSDB_RMD_FLAG_DELETED) {
+							el_incr_del = tmpel;
+						} else {
+							el_incr_add = tmpel;
+						}
+					}
+				}
+
+				if (dsc->linkIncrVal == false) {
+					if (flags & DSDB_RMD_FLAG_DELETED) {
+						if (k < (el->num_values - 1)) {
+							memmove(el->values + k,
+									el->values + (k + 1),
+									((el->num_values - 1) - k)*sizeof(*el->values));
+						}
+						el->num_values--;
+					}
+				}
+skip_link:
+				talloc_free(dn);
+
+			}
+			if (keep == true) {
+				if (dsc->linkIncrVal == false) {
+					if (ldb_msg_add(newmsg, el, LDB_FLAG_MOD_ADD) != LDB_SUCCESS) {
+						return ldb_error(ldb,
+							LDB_ERR_OPERATIONS_ERROR,
+							"Unable to add attribute");
+					}
+					talloc_steal(newmsg->elements, el->name);
+					talloc_steal(newmsg->elements, el->values);
+				} else {
+					if (el_incr_del) {
+						if (ldb_msg_add(newmsg, el_incr_del, LDB_FLAG_MOD_ADD))
+							return ldb_error(ldb,
+								LDB_ERR_OPERATIONS_ERROR,
+								"Unable to add attribute");
+					}
+					if (el_incr_add) {
+						if (ldb_msg_add(newmsg, el_incr_add, LDB_FLAG_MOD_ADD))
+							return ldb_error(ldb,
+								LDB_ERR_OPERATIONS_ERROR,
+								"Unable to add attribute");
+					}
+				}
+			}
+			continue;
+		}
+
+		if (listAttr) {
+			for (j=0; j<size; j++) {
+			/*
+				* We mark attribute that has already been seen well
+				* as seen. So that after attribute that are still in
+				* listAttr are attributes that has been modified after
+				* the requested USN but not present in the attributes
+				* returned by the ldb search.
+				* That is to say attributes that have been removed
+				*/
+				if (listAttr[j] && ldb_attr_cmp(listAttr[j], ldapattrname) == 0) {
+					listAttr[j] = NULL;
+					keep = true;
+					continue;
+				}
+			}
+		} else {
+			keep = true;
+		}
+
+		if (keep == true) {
+			if (ldb_msg_add(newmsg, el, LDB_FLAG_MOD_ADD) != LDB_SUCCESS) {
+				return ldb_error(ldb,
+					LDB_ERR_OPERATIONS_ERROR,
+					"Unable to add attribute");
+			}
+			talloc_steal(newmsg->elements, el->name);
+			talloc_steal(newmsg->elements, el->values);
+			continue;
+		}
+	}
+
+	/*
+	 * Here we run through the list of attributes returned
+	 * in the propertyMetaData.
+	 * Entries of this list have usn > requested_usn,
+	 * entries that are also present in the message have been
+	 * replaced by NULL, so at this moment the list contains
+	 * only elements that have a usn > requested_usn and that
+	 * haven't been seen. It's attributes that were removed.
+	 * We add them to the message like empty elements.
+	 */
+	for (j=0; j<size; j++) {
+		if (listAttr[j] && (
+				ldb_attr_in_list(req->op.search.attrs, "*") ||
+				ldb_attr_in_list(req->op.search.attrs, listAttr[j])) &&
+				(ldb_attr_cmp(listAttr[j], rdn) != 0) &&
+				(ldb_attr_cmp(listAttr[j], "instanceType") != 0)) {
+			ldb_msg_add_empty(newmsg, listAttr[j], LDB_FLAG_MOD_DELETE, NULL);
+		}
+	}
+	talloc_free(listAttr);
+
+	if ((newmsg->num_elements - ( dsc->nbDefaultAttrs - delta)) > 0) {
+		/*
+		 * After cleaning attributes there is still some attributes that were not added just
+		 * for the purpose of the control (objectGUID, instanceType, ...)
+		 */
+
+		newmsg->dn = talloc_steal(newmsg, msg->dn);
+		if (val > dsc->highestUSN) {
+			dsc->highestUSN = val;
+		}
+		talloc_free(msg);
+		return ldb_module_send_entry(dsc->req, newmsg, controls);
+	} else {
+		talloc_free(msg);
+		return LDB_SUCCESS;
+	}
+}
+
+
+static int dirsync_create_vector(struct ldb_request *req,
+					struct ldb_reply *ares,
+					struct dirsync_context *dsc,
+					struct ldapControlDirSyncCookie *cookie,
+					struct ldb_context *ldb)
+{
+	struct ldb_result *resVector;
+	const char* attrVector[] = {"replUpToDateVector", NULL };
+	uint64_t highest_usn;
+	struct ldb_dn *nc_root;
+	uint32_t count = 1;
+	int ret;
+	struct drsuapi_DsReplicaCursor *tab;
+
+	nc_root = ldb_get_default_basedn(ldb);
+	ret = ldb_sequence_number(ldb, LDB_SEQ_HIGHEST_SEQ, &highest_usn);
+	if (ret != LDB_SUCCESS) {
+		return ldb_error(ldb, LDB_ERR_OPERATIONS_ERROR, "Unable to get highest USN from current NC");
+	}
+
+	/* If we have a full answer then the highest USN
+	 * is not the highest USN from the result set but the
+	 * highest of the naming context, unless the sequence is not updated yet.
+	 */
+	if (highest_usn > dsc->highestUSN) {
+		dsc->highestUSN = highest_usn;
+	}
+
+
+	ret = dsdb_module_search_dn(dsc->module, dsc, &resVector,
+			nc_root,
+			attrVector,
+			DSDB_FLAG_NEXT_MODULE, req);
+
+	if (resVector->count != 0) {
+		DATA_BLOB blob;
+		uint32_t i;
+		struct ldb_message_element *el = ldb_msg_find_element(resVector->msgs[0], "replUpToDateVector");
+		if (el) {
+			enum ndr_err_code ndr_err;
+			struct replUpToDateVectorBlob utd;
+			blob.data = el->values[0].data;
+			blob.length = el->values[0].length;
+			ndr_err = ndr_pull_struct_blob(&blob, dsc, &utd,
+						(ndr_pull_flags_fn_t)ndr_pull_replUpToDateVectorBlob);
+
+			if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+				return ldb_error(ldb, LDB_ERR_OPERATIONS_ERROR,
+						"Unable to pull replUpToDateVectorBlob structure");
+			}
+
+
+			count += utd.ctr.ctr2.count;
+			tab = talloc_array(cookie, struct drsuapi_DsReplicaCursor, count);
+			if (tab == NULL) {
+				return ldb_oom(ldb);
+			}
+			for (i=1; i < count; i++) {
+				memset(&tab[i], 0, sizeof(struct drsuapi_DsReplicaCursor));
+				tab[i].highest_usn = utd.ctr.ctr2.cursors[i-1].highest_usn;
+				tab[i].source_dsa_invocation_id = utd.ctr.ctr2.cursors[i-1].source_dsa_invocation_id;
+			}
+		} else {
+			tab = talloc_array(cookie, struct drsuapi_DsReplicaCursor, count);
+			if (tab == NULL) {
+				return ldb_oom(ldb);
+			}
+		}
+	} else {
+		/*
+		 * No replUpToDateVector ? it happens quite often (1 DC,
+		 * other DCs didn't update ...
+		 */
+		tab = talloc_array(cookie, struct drsuapi_DsReplicaCursor, count);
+		if (tab == NULL) {
+			return ldb_oom(ldb);
+		}
+	}
+	/* Our vector is always the first */
+	tab[0].highest_usn = dsc->highestUSN;
+	tab[0].source_dsa_invocation_id = *(dsc->our_invocation_id);
+
+
+	/* We have to add the updateness vector that we have*/
+	/* Version is always 1 in dirsync cookies */
+	cookie->blob.extra.uptodateness_vector.version = 1;
+	cookie->blob.extra.uptodateness_vector.reserved = 0;
+	cookie->blob.extra.uptodateness_vector.ctr.ctr1.count = count;
+	cookie->blob.extra.uptodateness_vector.ctr.ctr1.reserved = 0;
+	cookie->blob.extra.uptodateness_vector.ctr.ctr1.cursors = tab;
+
+	return LDB_SUCCESS;
+}
+
+static int dirsync_search_callback(struct ldb_request *req, struct ldb_reply *ares)
+{
+	int ret;
+	struct dirsync_context *dsc;
+	struct ldb_result *res, *res2;
+	struct ldb_dirsync_control *control;
+	struct ldapControlDirSyncCookie *cookie;
+	struct ldb_context *ldb;
+	struct ldb_dn *dn;
+	struct ldb_val *val;
+	DATA_BLOB *blob;
+	NTTIME now;
+	const char *attrs[] = { "objectGUID", NULL };
+	enum ndr_err_code ndr_err;
+	char *tmp;
+	uint32_t flags;
+
+	dsc = talloc_get_type_abort(req->context, struct dirsync_context);
+	ldb = ldb_module_get_ctx(dsc->module);
+	if (!ares) {
+		return ldb_module_done(dsc->req, NULL, NULL,
+				       LDB_ERR_OPERATIONS_ERROR);
+	}
+	if (ares->error != LDB_SUCCESS) {
+		return ldb_module_done(dsc->req, ares->controls,
+				       ares->response, ares->error);
+	}
+
+	switch (ares->type) {
+	case LDB_REPLY_ENTRY:
+		return dirsync_filter_entry(req, ares->message, ares->controls, dsc, false);
+
+	case LDB_REPLY_REFERRAL:
+		/* Skip the ldap(s):// so up to 8 chars,
+		 * we don't care to be precise as the goal is to be in
+		 * the name of DC, then we search the next '/'
+		 * as it will be the last char before the DN of the referal
+		 */
+		if (strncmp(ares->referral, "ldap://", 7) == 0) {
+			tmp = ares->referral + 7;
+		} else if (strncmp(ares->referral, "ldaps://", 8) == 0) {
+			tmp = ares->referral + 8;
+		} else {
+			return ldb_operr(ldb);
+		}
+
+		tmp = strchr(tmp, '/');
+		tmp++;
+
+		dn = ldb_dn_new(dsc, ldb, tmp);
+		if (dn == NULL) {
+			return ldb_oom(ldb);
+		}
+
+		flags = DSDB_FLAG_NEXT_MODULE |
+				DSDB_RMD_FLAG_DELETED |
+				DSDB_SEARCH_SHOW_EXTENDED_DN;
+
+		if (dsc->assystem) {
+			flags = flags | DSDB_FLAG_AS_SYSTEM;
+		}
+
+		ret = dsdb_module_search_tree(dsc->module, dsc, &res,
+					dn, LDB_SCOPE_BASE,
+					req->op.search.tree,
+					req->op.search.attrs,
+					flags, req);
+
+		if (ret != LDB_SUCCESS) {
+			talloc_free(dn);
+			return ret;
+		}
+
+		if (res->count > 1) {
+			char *ldbmsg = talloc_asprintf(dn, "LDB returned more than result for dn: %s", tmp);
+			if (ldbmsg) {
+				ldb_set_errstring(ldb, ldbmsg);
+			}
+			talloc_free(dn);
+			return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+		} else if (res->count == 0) {
+			/* if nothing is returned then it means that we don't
+			* have access to it.
+			*/
+			return LDB_SUCCESS;
+		}
+
+		talloc_free(dn);
+		/*
+		 * Fetch the objectGUID of the root of current NC
+		 */
+		ret = dsdb_module_search_dn(dsc->module, dsc, &res2,
+					req->op.search.base,
+					attrs,
+					DSDB_FLAG_NEXT_MODULE, req);
+
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		if (res2->msgs[0]->num_elements != 1) {
+			ldb_set_errstring(ldb,
+					  "More than 1 attribute returned while looking for objectGUID");
+			return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		val = res2->msgs[0]->elements[0].values;
+		ret = ldb_msg_add_value(res->msgs[0], "parentGUID", val, NULL);
+		/*
+		 * It *very* important to steal otherwise as val is in a subcontext
+		 * related to res2, when the value will be one more time stolen
+		 * it's elements[x].values that will be stolen, so it's important to
+		 * recreate the context hierrachy as if it was done from a ldb_request
+		 */
+		talloc_steal(res->msgs[0]->elements[0].values, val);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		return dirsync_filter_entry(req, res->msgs[0], res->controls, dsc, true);
+
+	case LDB_REPLY_DONE:
+		/*
+		 * Let's add our own control
+		 */
+
+		control = talloc_zero(ares->controls, struct ldb_dirsync_control);
+		if (control == NULL) {
+			return ldb_oom(ldb);
+		}
+
+		/*
+		 * When outputing flags is used to say more results.
+		 * For the moment we didn't honnor the size info */
+
+		control->flags = 0;
+
+		/*
+		 * max_attribute is unused cf. 3.1.1.3.4.1.3 LDAP_SERVER_DIRSYNC_OID in MS-ADTS
+		 */
+
+		control->max_attributes = 0;
+		cookie = talloc_zero(control, struct ldapControlDirSyncCookie);
+		if (cookie == NULL) {
+			return ldb_oom(ldb);
+		}
+
+		if (!dsc->partial) {
+			ret = dirsync_create_vector(req, ares, dsc, cookie, ldb);
+			if (ret != LDB_SUCCESS) {
+				return ldb_module_done(dsc->req, NULL, NULL, ret);
+			}
+		}
+
+		unix_to_nt_time(&now, time(NULL));
+		cookie->blob.time = now;
+		cookie->blob.highwatermark.highest_usn = dsc->highestUSN;
+		cookie->blob.highwatermark.tmp_highest_usn = dsc->highestUSN;
+		cookie->blob.guid1 = *(dsc->our_invocation_id);
+
+		blob = talloc_zero(control, DATA_BLOB);
+		if (blob == NULL) {
+			return ldb_oom(ldb);
+		}
+
+		ndr_err = ndr_push_struct_blob(blob, blob, cookie,
+						(ndr_push_flags_fn_t)ndr_push_ldapControlDirSyncCookie);
+
+		if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+			ldb_set_errstring(ldb, "Can't marshall ldapControlDirSyncCookie struct");
+			return ldb_module_done(dsc->req, NULL, NULL, LDB_ERR_OPERATIONS_ERROR);
+		}
+		control->cookie = (char *)blob->data;
+		control->cookie_len = blob->length;
+		ldb_reply_add_control(ares, LDB_CONTROL_DIRSYNC_OID, true, control);
+
+		return ldb_module_done(dsc->req, ares->controls,
+				       ares->response, LDB_SUCCESS);
+
+	}
+	return LDB_SUCCESS;
+}
+
+static int dirsync_ldb_search(struct ldb_module *module, struct ldb_request *req)
+{
+	struct ldb_control *control;
+	struct ldb_result *acl_res;
+	struct ldb_dirsync_control *dirsync_ctl;
+	struct ldb_request *down_req;
+	struct dirsync_context *dsc;
+	struct ldb_context *ldb;
+	struct ldb_parse_tree *new_tree = req->op.search.tree;
+	uint32_t flags = 0;
+	enum ndr_err_code ndr_err;
+	DATA_BLOB blob;
+	const char **attrs;
+	int ret;
+
+
+	if (ldb_dn_is_special(req->op.search.base)) {
+		return ldb_next_request(module, req);
+	}
+
+	/*
+	 * check if there's an extended dn control
+	 */
+	control = ldb_request_get_control(req, LDB_CONTROL_DIRSYNC_OID);
+	if (control == NULL) {
+		/* not found go on */
+		return ldb_next_request(module, req);
+	}
+
+	ldb = ldb_module_get_ctx(module);
+	/*
+	 * This control must always be critical otherwise we return PROTOCOL error
+	 */
+	if (!control->critical) {
+		return ldb_operr(ldb);
+	}
+
+	dsc = talloc_zero(req, struct dirsync_context);
+	if (dsc == NULL) {
+		return ldb_oom(ldb);
+	}
+	dsc->module = module;
+	dsc->req = req;
+	dsc->nbDefaultAttrs = 0;
+
+
+	dirsync_ctl = talloc_get_type(control->data, struct ldb_dirsync_control);
+	if (dirsync_ctl == NULL) {
+		return ldb_error(ldb, LDB_ERR_PROTOCOL_ERROR, "No data in dirsync control");
+	}
+
+	ret = dsdb_find_nc_root(ldb, dsc, req->op.search.base, &dsc->nc_root);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	if (ldb_dn_compare(dsc->nc_root, req->op.search.base) != 0) {
+		if (dirsync_ctl->flags & LDAP_DIRSYNC_OBJECT_SECURITY) {
+			return ldb_error(ldb, LDB_ERR_UNWILLING_TO_PERFORM,
+				 "DN is not one of the naming context");
+		}
+		else {
+			return ldb_error(ldb, LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS,
+				 "dN is not one of the naming context");
+		}
+	}
+
+	if (!(dirsync_ctl->flags & LDAP_DIRSYNC_OBJECT_SECURITY)) {
+		struct dom_sid *sid;
+		struct security_descriptor *sd = NULL;
+		const char *acl_attrs[] = { "nTSecurityDescriptor", "objectSid", NULL };
+		/*
+		 * If we don't have the flag and if we have the "replicate directory change" granted
+		 * then we upgrade ourself to system to not be blocked by the acl
+		 */
+		/* FIXME we won't check the replicate directory change filtered attribute set
+		 * it should be done so that if attr is not empty then we check that the user
+		 * has also this right
+		 */
+
+		/*
+		 * First change to system to get the SD of the root of current NC
+		 * if we don't the acl_read will forbid us the right to read it ...
+		 */
+		ret = dsdb_module_search_dn(module, dsc, &acl_res,
+					req->op.search.base,
+					acl_attrs,
+					DSDB_FLAG_NEXT_MODULE|DSDB_FLAG_AS_SYSTEM, req);
+
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+
+		sid = samdb_result_dom_sid(dsc, acl_res->msgs[0], "objectSid");
+		/* sid can be null ... */
+		ret = dsdb_get_sd_from_ldb_message(ldb_module_get_ctx(module), acl_res, acl_res->msgs[0], &sd);
+
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		ret = acl_check_extended_right(dsc, sd, acl_user_token(module), GUID_DRS_GET_CHANGES, SEC_ADS_CONTROL_ACCESS, sid);
+
+		if (ret == LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS) {
+			return ret;
+		}
+		dsc->assystem = true;
+		ret = ldb_request_add_control(req, LDB_CONTROL_AS_SYSTEM_OID, false, NULL);
+
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		talloc_free(acl_res);
+	} else {
+		flags |= DSDB_ACL_CHECKS_DIRSYNC_FLAG;
+
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+
+	}
+
+	dsc->functional_level = dsdb_functional_level(ldb);
+
+	if (req->op.search.attrs) {
+		attrs = ldb_attr_list_copy(dsc, req->op.search.attrs);
+		if (attrs == NULL) {
+			return ldb_oom(ldb);
+		}
+		/*
+		* Check if we have only "dn" as attribute, if so then
+		* treat as if "*" was requested
+		*/
+		if (attrs && attrs[0]) {
+			if (ldb_attr_cmp(attrs[0], "dn") == 0 && !attrs[1]) {
+				attrs = talloc_array(dsc, const char*, 2);
+				if (attrs == NULL) {
+					return ldb_oom(ldb);
+				}
+				attrs[0] = "*";
+				attrs[1] = NULL;
+			}
+		}
+		/*
+		 * When returning all the attributes return also the SD as
+		 * Windws do so.
+		 */
+		if (ldb_attr_in_list(attrs, "*")) {
+			struct ldb_sd_flags_control *sdctr = talloc_zero(dsc, struct ldb_sd_flags_control);
+			sdctr->secinfo_flags = 0;
+			ret = ldb_request_add_control(req, LDB_CONTROL_SD_FLAGS_OID, false, sdctr);
+			if (ret != LDB_SUCCESS) {
+				return ret;
+			}
+			attrs = ldb_attr_list_copy_add(dsc, attrs, "parentGUID");
+			if (attrs == NULL) {
+				return ldb_oom(ldb);
+			}
+			attrs = ldb_attr_list_copy_add(dsc, attrs, "replPropertyMetaData");
+			if (attrs == NULL) {
+				return ldb_oom(ldb);
+			}
+			/*
+			* When no attributes are asked we in anycase expect at least 3 attributes:
+			* * instanceType
+			* * objectGUID
+			* * parentGUID
+			*/
+
+			dsc->nbDefaultAttrs = 3;
+		} else {
+			/*
+			 * We will need this two attributes in the callback
+			 */
+			attrs = ldb_attr_list_copy_add(dsc, attrs, "usnChanged");
+			if (attrs == NULL) {
+				return ldb_operr(ldb);
+			}
+			attrs = ldb_attr_list_copy_add(dsc, attrs, "replPropertyMetaData");
+			if (attrs == NULL) {
+				return ldb_operr(ldb);
+			}
+
+			if (!ldb_attr_in_list(attrs, "instanceType")) {
+				attrs = ldb_attr_list_copy_add(dsc, attrs, "instanceType");
+				if (attrs == NULL) {
+					return ldb_operr(ldb);
+				}
+				dsc->nbDefaultAttrs++;
+			}
+
+			if (!ldb_attr_in_list(attrs, "objectGUID")) {
+				attrs = ldb_attr_list_copy_add(dsc, attrs, "objectGUID");
+				if (attrs == NULL) {
+					return ldb_operr(ldb);
+				}
+			}
+			/*
+			 * Always increment the number of asked attributes as we don't care if objectGUID was asked
+			 * or not for counting the number of "real" attributes returned.
+			 */
+			dsc->nbDefaultAttrs++;
+
+			if (!ldb_attr_in_list(attrs, "parentGUID")) {
+				attrs = ldb_attr_list_copy_add(dsc, attrs, "parentGUID");
+				if (attrs == NULL) {
+					return ldb_operr(ldb);
+				}
+			}
+			dsc->nbDefaultAttrs++;
+
+		}
+	} else {
+		struct ldb_sd_flags_control *sdctr = talloc_zero(dsc, struct ldb_sd_flags_control);
+		sdctr->secinfo_flags = 0;
+		ret = ldb_request_add_control(req, LDB_CONTROL_SD_FLAGS_OID, false, sdctr);
+		attrs = talloc_array(dsc, const char*, 4);
+		if (attrs == NULL) {
+			return ldb_operr(ldb);
+		}
+		attrs[0] = "*";
+		attrs[1] = "parentGUID";
+		attrs[2] = "replPropertyMetaData";
+		attrs[3] = NULL;
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		/*
+		 * When no attributes are asked we in anycase expect at least 3 attributes:
+		 * * instanceType
+		 * * objectGUID
+		 * * parentGUID
+		 */
+
+		dsc->nbDefaultAttrs = 3;
+	}
+
+	if (!ldb_request_get_control(req, LDB_CONTROL_EXTENDED_DN_OID)) {
+		ret = ldb_request_add_control(req, LDB_CONTROL_EXTENDED_DN_OID, false, NULL);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		dsc->noextended = true;
+	}
+
+	if (ldb_request_get_control(req, LDB_CONTROL_REVEAL_INTERNALS) == NULL) {
+		ret = ldb_request_add_control(req, LDB_CONTROL_REVEAL_INTERNALS, false, NULL);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+	}
+
+	if (ldb_request_get_control(req, LDB_CONTROL_SHOW_RECYCLED_OID) == NULL) {
+		ret = ldb_request_add_control(req, LDB_CONTROL_SHOW_RECYCLED_OID, false, NULL);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+	}
+
+	if (ldb_request_get_control(req, LDB_CONTROL_SHOW_DELETED_OID) == NULL) {
+		ret = ldb_request_add_control(req, LDB_CONTROL_SHOW_DELETED_OID, false, NULL);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+	}
+
+	if (dirsync_ctl->flags & LDAP_DIRSYNC_INCREMENTAL_VALUES) {
+		dsc->linkIncrVal = true;
+	} else {
+		dsc->linkIncrVal = false;
+	}
+
+	dsc->our_invocation_id = samdb_ntds_invocation_id(ldb);
+	if (dsc->our_invocation_id == NULL) {
+		return ldb_operr(ldb);
+	}
+
+	if (dirsync_ctl->cookie_len > 0) {
+		struct ldapControlDirSyncCookie cookie;
+
+		blob.data = (uint8_t *)dirsync_ctl->cookie;
+		blob.length = dirsync_ctl->cookie_len;
+		ndr_err = ndr_pull_struct_blob(&blob, dsc, &cookie,
+						(ndr_pull_flags_fn_t)ndr_pull_ldapControlDirSyncCookie);
+
+		/* If we can't unmarshall the cookie into the correct structure we return
+		* unsupported critical extension
+		*/
+		if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+			return ldb_error(ldb, LDB_ERR_UNSUPPORTED_CRITICAL_EXTENSION,
+					 "Unable to unmarshall cookie as a ldapControlDirSyncCookie structure");
+		}
+
+		/*
+		* Let's search for the max usn withing the cookie
+		*/
+		if (GUID_equal(&(cookie.blob.guid1), dsc->our_invocation_id)) {
+			/*
+			 * Ok, it's our invocation ID so we can treat the demand
+			 * Let's take the highest usn from (tmp)highest_usn
+			 */
+			dsc->fromreqUSN = cookie.blob.highwatermark.tmp_highest_usn;
+			dsc->localonly = true;
+
+			if (cookie.blob.highwatermark.highest_usn > cookie.blob.highwatermark.tmp_highest_usn) {
+				dsc->fromreqUSN = cookie.blob.highwatermark.highest_usn;
+			}
+		} else {
+			dsc->localonly = false;
+		}
+		if (cookie.blob.extra_length > 0 &&
+				cookie.blob.extra.uptodateness_vector.ctr.ctr1.count > 0) {
+			struct drsuapi_DsReplicaCursor cursor;
+			uint32_t p;
+			for (p=0; p < cookie.blob.extra.uptodateness_vector.ctr.ctr1.count; p++) {
+				cursor = cookie.blob.extra.uptodateness_vector.ctr.ctr1.cursors[p];
+				if (GUID_equal( &(cursor.source_dsa_invocation_id), dsc->our_invocation_id)) {
+					if (cursor.highest_usn > dsc->fromreqUSN) {
+						dsc->fromreqUSN = cursor.highest_usn;
+					}
+				}
+			}
+			dsc->cursors = talloc_steal(dsc,
+					cookie.blob.extra.uptodateness_vector.ctr.ctr1.cursors);
+			if (dsc->cursors == NULL) {
+				return ldb_oom(ldb);
+			}
+			dsc->cursor_size = p;
+		}
+	}
+
+	DEBUG(4, ("Dirsync: searching with min usn > %llu\n",
+				(long long unsigned int)dsc->fromreqUSN));
+	if (dsc->fromreqUSN > 0) {
+		/* FIXME it would be better to use PRId64 */
+		char *expression = talloc_asprintf(dsc, "(&%s(uSNChanged>=%llu))",
+				                        ldb_filter_from_tree(dsc,
+				                             req->op.search.tree),
+				                        (long long unsigned int)(dsc->fromreqUSN + 1));
+
+		if (expression == NULL) {
+			return ldb_oom(ldb);
+		}
+		new_tree = ldb_parse_tree(req, expression);
+		if (new_tree == NULL) {
+			return ldb_error(ldb, LDB_ERR_OPERATIONS_ERROR,
+					"Problem while parsing tree");
+		}
+
+	}
+	/*
+	 * Remove our control from the list of controls
+	 */
+	if (!ldb_save_controls(control, req, NULL)) {
+		return ldb_operr(ldb);
+	}
+	dsc->schema = dsdb_get_schema(ldb, dsc);
+	/*
+	 * At the begining we make the hypothesis that we will return a complete
+	 * result set
+	 */
+
+	dsc->partial = false;
+
+	/*
+	 * 3.1.1.3.4.1.3 of MS-ADTS.pdf specify that if the scope is not subtree
+	 * we treat the search as if subtree was specified
+	 */
+
+	ret = ldb_build_search_req_ex(&down_req, ldb, dsc,
+				      req->op.search.base,
+				      LDB_SCOPE_SUBTREE,
+				      new_tree,
+				      attrs,
+				      req->controls,
+				      dsc, dirsync_search_callback,
+				      req);
+	ldb_req_set_custom_flags(down_req, flags);
+	LDB_REQ_SET_LOCATION(down_req);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+	/* perform the search */
+	return ldb_next_request(module, down_req);
+}
+
+static int dirsync_ldb_init(struct ldb_module *module)
+{
+	int ret;
+
+	ret = ldb_mod_register_control(module, LDB_CONTROL_DIRSYNC_OID);
+	if (ret != LDB_SUCCESS) {
+		ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_ERROR,
+			"dirsync: Unable to register control with rootdse!\n");
+		return ldb_operr(ldb_module_get_ctx(module));
+	}
+
+	return ldb_next_init(module);
+}
+
+static const struct ldb_module_ops ldb_dirsync_ldb_module_ops = {
+	.name		   = "dirsync",
+	.search            = dirsync_ldb_search,
+	.init_context	   = dirsync_ldb_init,
+};
+
+/*
+  initialise the module
+ */
+_PUBLIC_ int ldb_dirsync_module_init(const char *version)
+{
+	int ret;
+	LDB_MODULE_CHECK_VERSION(version);
+	ret = ldb_register_module(&ldb_dirsync_ldb_module_ops);
+	return ret;
+}
diff --git a/source4/dsdb/samdb/ldb_modules/extended_dn_in.c b/source4/dsdb/samdb/ldb_modules/extended_dn_in.c
index 3e2004d6f3e..9a70d9a3dbb 100644
--- a/source4/dsdb/samdb/ldb_modules/extended_dn_in.c
+++ b/source4/dsdb/samdb/ldb_modules/extended_dn_in.c
@@ -103,6 +103,18 @@ static int extended_base_callback(struct ldb_request *req, struct ldb_reply *are
 
 	switch (ares->type) {
 	case LDB_REPLY_ENTRY:
+		if (ac->basedn) {
+			/* we have more than one match! This can
+			   happen as S-1-5-17 appears twice in a
+			   normal provision. We need to return
+			   NO_SUCH_OBJECT */
+			const char *str = talloc_asprintf(req, "Duplicate base-DN matches found for '%s'",
+							  ldb_dn_get_extended_linearized(req, ac->req->op.search.base, 1));
+			ldb_set_errstring(ldb_module_get_ctx(ac->module), str);
+			return ldb_module_done(ac->req, NULL, NULL,
+					       LDB_ERR_NO_SUCH_OBJECT);
+		}
+
 		if (!ac->wellknown_object) {
 			ac->basedn = talloc_steal(ac, ares->message->dn);
 			break;
@@ -303,30 +315,33 @@ static int extended_dn_in_fix(struct ldb_module *module, struct ldb_request *req
 		guid_val = ldb_dn_get_extended_component(dn, "GUID");
 		wkguid_val = ldb_dn_get_extended_component(dn, "WKGUID");
 
-		if (sid_val) {
+		/*
+		  prioritise the GUID - we have had instances of
+		  duplicate SIDs in the database in the
+		  ForeignSecurityPrinciples due to provision errors
+		 */
+		if (guid_val) {
 			all_partitions = true;
 			base_dn = ldb_get_default_basedn(ldb_module_get_ctx(module));
-			base_dn_filter = talloc_asprintf(req, "(objectSid=%s)", 
-							 ldb_binary_encode(req, *sid_val));
+			base_dn_filter = talloc_asprintf(req, "(objectGUID=%s)",
+							 ldb_binary_encode(req, *guid_val));
 			if (!base_dn_filter) {
 				return ldb_oom(ldb_module_get_ctx(module));
 			}
 			base_dn_scope = LDB_SCOPE_SUBTREE;
 			base_dn_attrs = no_attr;
 
-		} else if (guid_val) {
-
+		} else if (sid_val) {
 			all_partitions = true;
 			base_dn = ldb_get_default_basedn(ldb_module_get_ctx(module));
-			base_dn_filter = talloc_asprintf(req, "(objectGUID=%s)", 
-							 ldb_binary_encode(req, *guid_val));
+			base_dn_filter = talloc_asprintf(req, "(objectSid=%s)",
+							 ldb_binary_encode(req, *sid_val));
 			if (!base_dn_filter) {
 				return ldb_oom(ldb_module_get_ctx(module));
 			}
 			base_dn_scope = LDB_SCOPE_SUBTREE;
 			base_dn_attrs = no_attr;
 
-
 		} else if (wkguid_val) {
 			char *wkguid_dup;
 			char *tail_str;
diff --git a/source4/dsdb/samdb/ldb_modules/objectclass_attrs.c b/source4/dsdb/samdb/ldb_modules/objectclass_attrs.c
index 9df121002f0..5639a7a3e3e 100644
--- a/source4/dsdb/samdb/ldb_modules/objectclass_attrs.c
+++ b/source4/dsdb/samdb/ldb_modules/objectclass_attrs.c
@@ -140,7 +140,8 @@ static int attr_handler(struct oc_context *ac)
 		if (!(msg->elements[i].flags & LDB_FLAG_INTERNAL_DISABLE_VALIDATION)) {
 			werr = attr->syntax->validate_ldb(&syntax_ctx, attr,
 							  &msg->elements[i]);
-			if (!W_ERROR_IS_OK(werr)) {
+			if (!W_ERROR_IS_OK(werr) &&
+			    !ldb_request_get_control(ac->req, LDB_CONTROL_RELAX_OID)) {
 				ldb_asprintf_errstring(ldb, "objectclass_attrs: attribute '%s' on entry '%s' contains at least one invalid value!",
 						       msg->elements[i].name,
 						       ldb_dn_get_linearized(msg->dn));
diff --git a/source4/dsdb/samdb/ldb_modules/proxy.c b/source4/dsdb/samdb/ldb_modules/proxy.c
index 6fba24fc2d9..5f6e56f9d47 100644
--- a/source4/dsdb/samdb/ldb_modules/proxy.c
+++ b/source4/dsdb/samdb/ldb_modules/proxy.c
@@ -138,7 +138,7 @@ static int load_proxy_info(struct ldb_module *module)
 	ldb_set_opaque(proxy->upstream, "credentials", creds);
 
 	ret = ldb_connect(proxy->upstream, url, 0, NULL);
-	if (ret != 0) {
+	if (ret != LDB_SUCCESS) {
 		ldb_debug(ldb, LDB_DEBUG_FATAL, "proxy failed to connect to %s\n", url);
 		goto failed;
 	}
diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c
index 646abeb771d..9d2e5e2ac3b 100644
--- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c
+++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c
@@ -254,7 +254,16 @@ static int replmd_process_backlink(struct ldb_module *module, struct la_backlink
 	msg->elements[0].flags |= LDB_FLAG_INTERNAL_DISABLE_SINGLE_VALUE_CHECK;
 
 	ret = dsdb_module_modify(module, msg, DSDB_FLAG_NEXT_MODULE, parent);
-	if (ret != LDB_SUCCESS) {
+	if (ret == LDB_ERR_NO_SUCH_ATTRIBUTE && !bl->active) {
+		/* we allow LDB_ERR_NO_SUCH_ATTRIBUTE as success to
+		   cope with possible corruption where the backlink has
+		   already been removed */
+		DEBUG(0,("WARNING: backlink from %s already removed from %s - %s\n",
+			 ldb_dn_get_linearized(target_dn),
+			 ldb_dn_get_linearized(source_dn),
+			 ldb_errstring(ldb)));
+		ret = LDB_SUCCESS;
+	} else if (ret != LDB_SUCCESS) {
 		ldb_asprintf_errstring(ldb, "Failed to %s backlink from %s to %s - %s",
 				       bl->active?"add":"remove",
 				       ldb_dn_get_linearized(source_dn),
@@ -1634,7 +1643,8 @@ static int replmd_update_la_val(TALLOC_CTX *mem_ctx, struct ldb_val *v, struct d
 	if (old_addtime == NULL) {
 		old_addtime = &tval;
 	}
-	if (dsdb_dn != old_dsdb_dn) {
+	if (dsdb_dn != old_dsdb_dn ||
+	    ldb_dn_get_extended_component(dn, "RMD_ADDTIME") == NULL) {
 		ret = ldb_dn_set_extended_component(dn, "RMD_ADDTIME", old_addtime);
 		if (ret != LDB_SUCCESS) return ret;
 	}
@@ -2488,7 +2498,7 @@ static int replmd_rename_callback(struct ldb_request *req, struct ldb_reply *are
 	if (ret == LDB_ERR_REFERRAL) {
 		struct ldb_dn *olddn = ac->req->op.rename.olddn;
 		struct loadparm_context *lp_ctx;
-		const char *referral;
+		char *referral;
 
 		lp_ctx = talloc_get_type(ldb_get_opaque(ldb, "loadparm"),
 					 struct loadparm_context);
diff --git a/source4/dsdb/samdb/ldb_modules/ridalloc.c b/source4/dsdb/samdb/ldb_modules/ridalloc.c
index 50519196728..28fade11b12 100644
--- a/source4/dsdb/samdb/ldb_modules/ridalloc.c
+++ b/source4/dsdb/samdb/ldb_modules/ridalloc.c
@@ -66,14 +66,14 @@
  */
 static void ridalloc_poke_rid_manager(struct ldb_module *module)
 {
-	struct messaging_context *msg;
+	struct imessaging_context *msg;
 	struct server_id *server;
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 	struct loadparm_context *lp_ctx =
 		(struct loadparm_context *)ldb_get_opaque(ldb, "loadparm");
 	TALLOC_CTX *tmp_ctx = talloc_new(module);
 
-	msg = messaging_client_init(tmp_ctx, lpcfg_messaging_path(tmp_ctx, lp_ctx),
+	msg = imessaging_client_init(tmp_ctx, lpcfg_imessaging_path(tmp_ctx, lp_ctx),
 				    ldb_get_event_context(ldb));
 	if (!msg) {
 		DEBUG(3,(__location__ ": Failed to create messaging context\n"));
@@ -88,7 +88,7 @@ static void ridalloc_poke_rid_manager(struct ldb_module *module)
 		return;
 	}
 
-	messaging_send(msg, server[0], MSG_DREPL_ALLOCATE_RID, NULL);
+	imessaging_send(msg, server[0], MSG_DREPL_ALLOCATE_RID, NULL);
 
 	/* we don't care if the message got through */
 	talloc_free(tmp_ctx);
diff --git a/source4/dsdb/samdb/ldb_modules/rootdse.c b/source4/dsdb/samdb/ldb_modules/rootdse.c
index 0fd65f4795c..c584a11b2c6 100644
--- a/source4/dsdb/samdb/ldb_modules/rootdse.c
+++ b/source4/dsdb/samdb/ldb_modules/rootdse.c
@@ -222,11 +222,10 @@ static int rootdse_add_dynamic(struct ldb_module *module, struct ldb_message *ms
 					  struct loadparm_context);
 		char *ldap_service_name, *hostname;
 
-		hostname = talloc_strdup(msg, lpcfg_netbios_name(lp_ctx));
+		hostname = strlower_talloc(msg, lpcfg_netbios_name(lp_ctx));
 		if (hostname == NULL) {
 			goto failed;
 		}
-		strlower_m(hostname);
 
 		ldap_service_name = talloc_asprintf(msg, "%s:%s$@%s",
 						    samdb_forest_name(ldb, msg),
@@ -613,7 +612,11 @@ static int rootdse_filter_controls(struct ldb_module *module, struct ldb_request
 			continue;
 		}
 
-		if (is_registered) {
+		/* If the control is DIRSYNC control then we keep the critical
+		 * flag as the dirsync module will need to act upon it
+		 */
+		if (is_registered && strcmp(req->controls[i]->oid,
+					LDB_CONTROL_DIRSYNC_OID)!= 0) {
 			req->controls[i]->critical = 0;
 		}
 	}
@@ -1195,7 +1198,7 @@ static int rootdse_become_master(struct ldb_module *module,
 				 struct ldb_request *req,
 				 enum drepl_role_master role)
 {
-	struct messaging_context *msg;
+	struct imessaging_context *msg;
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 	TALLOC_CTX *tmp_ctx = talloc_new(req);
 	struct loadparm_context *lp_ctx = ldb_get_opaque(ldb, "loadparm");
@@ -1223,10 +1226,10 @@ static int rootdse_become_master(struct ldb_module *module,
 				 "RODC cannot become a role master.");
 	}
 
-	msg = messaging_client_init(tmp_ctx, lpcfg_messaging_path(tmp_ctx, lp_ctx),
+	msg = imessaging_client_init(tmp_ctx, lpcfg_imessaging_path(tmp_ctx, lp_ctx),
 				    ldb_get_event_context(ldb));
 	if (!msg) {
-		ldb_asprintf_errstring(ldb, "Failed to generate client messaging context in %s", lpcfg_messaging_path(tmp_ctx, lp_ctx));
+		ldb_asprintf_errstring(ldb, "Failed to generate client messaging context in %s", lpcfg_imessaging_path(tmp_ctx, lp_ctx));
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 	irpc_handle = irpc_binding_handle_by_name(tmp_ctx, msg,
diff --git a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
index 35b323b72f8..e4de1524bee 100644
--- a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
+++ b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
@@ -163,6 +163,7 @@ static int samba_dsdb_init(struct ldb_module *module)
 	static const char *modules_list[] = {"resolve_oids",
 					     "rootdse",
 					     "lazy_commit",
+					     "dirsync",
 					     "paged_results",
 					     "ranged_results",
 					     "anr",
diff --git a/source4/dsdb/samdb/ldb_modules/samldb.c b/source4/dsdb/samdb/ldb_modules/samldb.c
index 21341850d94..6533d1006ba 100644
--- a/source4/dsdb/samdb/ldb_modules/samldb.c
+++ b/source4/dsdb/samdb/ldb_modules/samldb.c
@@ -3,7 +3,7 @@
 
    Copyright (C) Andrew Bartlett <abartlet@samba.org> 2005
    Copyright (C) Simo Sorce  2004-2008
-   Copyright (C) Matthias Dieter Wallnöfer 2009-2010
+   Copyright (C) Matthias Dieter Wallnöfer 2009-2011
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -790,6 +790,8 @@ static int samldb_schema_info_update(struct samldb_ctx *ac)
 	return LDB_SUCCESS;
 }
 
+static int samldb_prim_group_tester(struct samldb_ctx *ac, uint32_t rid);
+
 /*
  * "Objectclass" trigger (MS-SAMR 3.1.1.8.1)
  *
@@ -801,10 +803,9 @@ static int samldb_schema_info_update(struct samldb_ctx *ac)
 static int samldb_objectclass_trigger(struct samldb_ctx *ac)
 {
 	struct ldb_context *ldb = ldb_module_get_ctx(ac->module);
-	struct loadparm_context *lp_ctx = talloc_get_type(ldb_get_opaque(ldb,
-					 "loadparm"), struct loadparm_context);
+	void *skip_allocate_sids = ldb_get_opaque(ldb,
+						  "skip_allocate_sids");
 	struct ldb_message_element *el, *el2;
-	enum sid_generator sid_generator;
 	struct dom_sid *sid;
 	int ret;
 
@@ -830,12 +831,9 @@ static int samldb_objectclass_trigger(struct samldb_ctx *ac)
 	}
 
 	/* but generate a new SID when we do have an add operations */
-	if ((sid == NULL) && (ac->req->operation == LDB_ADD)) {
-		sid_generator = lpcfg_sid_generator(lp_ctx);
-		if (sid_generator == SID_GENERATOR_INTERNAL) {
-			ret = samldb_add_step(ac, samldb_allocate_sid);
-			if (ret != LDB_SUCCESS) return ret;
-		}
+	if ((sid == NULL) && (ac->req->operation == LDB_ADD) && !skip_allocate_sids) {
+		ret = samldb_add_step(ac, samldb_allocate_sid);
+		if (ret != LDB_SUCCESS) return ret;
 	}
 
 	if (strcmp(ac->type, "user") == 0) {
@@ -897,6 +895,16 @@ static int samldb_objectclass_trigger(struct samldb_ctx *ac)
 				return LDB_ERR_OTHER;
 			}
 
+			/* Workstation and (read-only) DC objects do need objectclass "computer" */
+			if ((samdb_find_attribute(ldb, ac->msg,
+						  "objectclass", "computer") == NULL) &&
+			    (user_account_control &
+			     (UF_SERVER_TRUST_ACCOUNT | UF_WORKSTATION_TRUST_ACCOUNT))) {
+				ldb_set_errstring(ldb,
+						  "samldb: Requested account type does need objectclass 'computer'!");
+				return LDB_ERR_OBJECT_CLASS_VIOLATION;
+			}
+
 			account_type = ds_uf2atype(user_account_control);
 			if (account_type == 0) {
 				ldb_set_errstring(ldb, "samldb: Unrecognized account type!");
@@ -911,11 +919,20 @@ static int samldb_objectclass_trigger(struct samldb_ctx *ac)
 			el2 = ldb_msg_find_element(ac->msg, "sAMAccountType");
 			el2->flags = LDB_FLAG_MOD_REPLACE;
 
+			/* "isCriticalSystemObject" might be set */
 			if (user_account_control &
 			    (UF_SERVER_TRUST_ACCOUNT | UF_PARTIAL_SECRETS_ACCOUNT)) {
-				ret = samdb_msg_set_string(ldb, ac->msg, ac->msg,
-							   "isCriticalSystemObject",
-							   "TRUE");
+				ret = ldb_msg_add_string(ac->msg, "isCriticalSystemObject",
+							 "TRUE");
+				if (ret != LDB_SUCCESS) {
+					return ret;
+				}
+				el2 = ldb_msg_find_element(ac->msg,
+							   "isCriticalSystemObject");
+				el2->flags = LDB_FLAG_MOD_REPLACE;
+			} else if (user_account_control & UF_WORKSTATION_TRUST_ACCOUNT) {
+				ret = ldb_msg_add_string(ac->msg, "isCriticalSystemObject",
+							 "FALSE");
 				if (ret != LDB_SUCCESS) {
 					return ret;
 				}
@@ -927,6 +944,18 @@ static int samldb_objectclass_trigger(struct samldb_ctx *ac)
 			/* Step 1.4: "userAccountControl" -> "primaryGroupID" mapping */
 			if (!ldb_msg_find_element(ac->msg, "primaryGroupID")) {
 				uint32_t rid = ds_uf2prim_group_rid(user_account_control);
+
+				/*
+				 * Older AD deployments don't know about the
+				 * RODC group
+				 */
+				if (rid == DOMAIN_RID_READONLY_DCS) {
+					ret = samldb_prim_group_tester(ac, rid);
+					if (ret != LDB_SUCCESS) {
+						return ret;
+					}
+				}
+
 				ret = samdb_msg_add_uint(ldb, ac->msg, ac->msg,
 							 "primaryGroupID", rid);
 				if (ret != LDB_SUCCESS) {
@@ -1009,26 +1038,14 @@ static int samldb_objectclass_trigger(struct samldb_ctx *ac)
  * ac->msg contains the "add"/"modify" message
  */
 
-static int samldb_prim_group_set(struct samldb_ctx *ac)
+static int samldb_prim_group_tester(struct samldb_ctx *ac, uint32_t rid)
 {
 	struct ldb_context *ldb = ldb_module_get_ctx(ac->module);
-	uint32_t rid;
 	struct dom_sid *sid;
 	struct ldb_result *res;
 	int ret;
 	const char *noattrs[] = { NULL };
 
-	rid = ldb_msg_find_attr_as_uint(ac->msg, "primaryGroupID", (uint32_t) -1);
-	if (rid == (uint32_t) -1) {
-		/* we aren't affected of any primary group set */
-		return LDB_SUCCESS;
-
-	} else if (!ldb_request_get_control(ac->req, LDB_CONTROL_RELAX_OID)) {
-		ldb_set_errstring(ldb,
-				  "The primary group isn't settable on add operations!");
-		return LDB_ERR_UNWILLING_TO_PERFORM;
-	}
-
 	sid = dom_sid_add_rid(ac, samdb_domain_sid(ldb), rid);
 	if (sid == NULL) {
 		return ldb_operr(ldb);
@@ -1054,6 +1071,25 @@ static int samldb_prim_group_set(struct samldb_ctx *ac)
 	return LDB_SUCCESS;
 }
 
+static int samldb_prim_group_set(struct samldb_ctx *ac)
+{
+	struct ldb_context *ldb = ldb_module_get_ctx(ac->module);
+	uint32_t rid;
+
+	rid = ldb_msg_find_attr_as_uint(ac->msg, "primaryGroupID", (uint32_t) -1);
+	if (rid == (uint32_t) -1) {
+		/* we aren't affected of any primary group set */
+		return LDB_SUCCESS;
+
+	} else if (!ldb_request_get_control(ac->req, LDB_CONTROL_RELAX_OID)) {
+		ldb_set_errstring(ldb,
+				  "The primary group isn't settable on add operations!");
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+
+	return samldb_prim_group_tester(ac, rid);
+}
+
 static int samldb_prim_group_change(struct samldb_ctx *ac)
 {
 	struct ldb_context *ldb = ldb_module_get_ctx(ac->module);
@@ -1076,14 +1112,11 @@ static int samldb_prim_group_change(struct samldb_ctx *ac)
 
 	/* Fetch information from the existing object */
 
-	ret = dsdb_module_search(ac->module, ac, &res, ac->msg->dn, LDB_SCOPE_BASE, attrs,
-				 DSDB_FLAG_NEXT_MODULE, ac->req, NULL);
+	ret = dsdb_module_search_dn(ac->module, ac, &res, ac->msg->dn, attrs,
+				    DSDB_FLAG_NEXT_MODULE, ac->req);
 	if (ret != LDB_SUCCESS) {
 		return ret;
 	}
-	if (res->count != 1) {
-		return ldb_operr(ldb);
-	}
 
 	/* Finds out the DN of the old primary group */
 
@@ -1219,13 +1252,22 @@ static int samldb_prim_group_trigger(struct samldb_ctx *ac)
 	return ret;
 }
 
+
+/**
+ * This function is called on LDB modify operations. It performs some additions/
+ * replaces on the current LDB message when "userAccountControl" changes.
+ */
 static int samldb_user_account_control_change(struct samldb_ctx *ac)
 {
 	struct ldb_context *ldb = ldb_module_get_ctx(ac->module);
-	uint32_t user_account_control, account_type;
+	uint32_t user_account_control, old_user_account_control, account_type;
 	struct ldb_message_element *el;
 	struct ldb_message *tmp_msg;
 	int ret;
+	struct ldb_result *res;
+	const char *attrs[] = { "userAccountControl", "objectClass", NULL };
+	unsigned int i;
+	bool is_computer = false;
 
 	el = dsdb_get_single_valued_attr(ac->msg, "userAccountControl",
 					 ac->req->operation);
@@ -1253,6 +1295,49 @@ static int samldb_user_account_control_change(struct samldb_ctx *ac)
 		return LDB_ERR_OTHER;
 	}
 
+	/* Fetch the old "userAccountControl" and "objectClass" */
+	ret = dsdb_module_search_dn(ac->module, ac, &res, ac->msg->dn, attrs,
+				    DSDB_FLAG_NEXT_MODULE, ac->req);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+	old_user_account_control = ldb_msg_find_attr_as_uint(res->msgs[0], "userAccountControl", 0);
+	if (old_user_account_control == 0) {
+		return ldb_operr(ldb);
+	}
+	el = ldb_msg_find_element(res->msgs[0], "objectClass");
+	if (el == NULL) {
+		return ldb_operr(ldb);
+	}
+
+	/* When we do not have objectclass "computer" we cannot switch to a (read-only) DC */
+	for (i = 0; i < el->num_values; i++) {
+		if (ldb_attr_cmp((char *)el->values[i].data, "computer") == 0) {
+			is_computer = true;
+			break;
+		}
+	}
+	if (!is_computer &&
+	    (user_account_control & (UF_SERVER_TRUST_ACCOUNT | UF_PARTIAL_SECRETS_ACCOUNT))) {
+		ldb_set_errstring(ldb,
+				  "samldb: Requested account type does need objectclass 'computer'!");
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+
+	/*
+	 * The functions "ds_uf2atype" and "ds_uf2prim_group_rid" are used as
+	 * detectors for account type changes.
+	 * So if the account type does change then we need to adjust the
+	 * "sAMAccountType", the "isCriticalSystemObject" and the
+	 * "primaryGroupID" attribute.
+	 */
+	if ((ds_uf2atype(user_account_control)
+	     == ds_uf2atype(old_user_account_control)) &&
+	    (ds_uf2prim_group_rid(user_account_control)
+	     == ds_uf2prim_group_rid(old_user_account_control))) {
+		return LDB_SUCCESS;
+	}
+
 	account_type = ds_uf2atype(user_account_control);
 	if (account_type == 0) {
 		ldb_set_errstring(ldb, "samldb: Unrecognized account type!");
@@ -1266,6 +1351,7 @@ static int samldb_user_account_control_change(struct samldb_ctx *ac)
 	el = ldb_msg_find_element(ac->msg, "sAMAccountType");
 	el->flags = LDB_FLAG_MOD_REPLACE;
 
+	/* "isCriticalSystemObject" might be set/changed */
 	if (user_account_control
 	    & (UF_SERVER_TRUST_ACCOUNT | UF_PARTIAL_SECRETS_ACCOUNT)) {
 		ret = ldb_msg_add_string(ac->msg, "isCriticalSystemObject",
@@ -1276,10 +1362,28 @@ static int samldb_user_account_control_change(struct samldb_ctx *ac)
 		el = ldb_msg_find_element(ac->msg,
 					   "isCriticalSystemObject");
 		el->flags = LDB_FLAG_MOD_REPLACE;
+	} else if (user_account_control & UF_WORKSTATION_TRUST_ACCOUNT) {
+		ret = ldb_msg_add_string(ac->msg, "isCriticalSystemObject",
+					 "FALSE");
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		el = ldb_msg_find_element(ac->msg,
+					   "isCriticalSystemObject");
+		el->flags = LDB_FLAG_MOD_REPLACE;
 	}
 
 	if (!ldb_msg_find_element(ac->msg, "primaryGroupID")) {
 		uint32_t rid = ds_uf2prim_group_rid(user_account_control);
+
+		/* Older AD deployments don't know about the RODC group */
+		if (rid == DOMAIN_RID_READONLY_DCS) {
+			ret = samldb_prim_group_tester(ac, rid);
+			if (ret != LDB_SUCCESS) {
+				return ret;
+			}
+		}
+
 		ret = samdb_msg_add_uint(ldb, ac->msg, ac->msg,
 					 "primaryGroupID", rid);
 		if (ret != LDB_SUCCESS) {
@@ -1977,7 +2081,7 @@ static int samldb_modify(struct ldb_module *module, struct ldb_request *req)
 
 	el = ldb_msg_find_element(ac->msg, "primaryGroupID");
 	if (el != NULL) {
-		ret = samldb_prim_group_change(ac);
+		ret = samldb_prim_group_trigger(ac);
 		if (ret != LDB_SUCCESS) {
 			return ret;
 		}
diff --git a/source4/dsdb/samdb/ldb_modules/util.c b/source4/dsdb/samdb/ldb_modules/util.c
index 49939e2ff45..7dbf233703a 100644
--- a/source4/dsdb/samdb/ldb_modules/util.c
+++ b/source4/dsdb/samdb/ldb_modules/util.c
@@ -109,39 +109,23 @@ int dsdb_module_search_dn(struct ldb_module *module,
 	return ret;
 }
 
-/*
-  search for attrs in the modules below
- */
-int dsdb_module_search(struct ldb_module *module,
+int dsdb_module_search_tree(struct ldb_module *module,
 		       TALLOC_CTX *mem_ctx,
 		       struct ldb_result **_res,
-		       struct ldb_dn *basedn, enum ldb_scope scope, 
+		       struct ldb_dn *basedn,
+		       enum ldb_scope scope,
+		       struct ldb_parse_tree *tree,
 		       const char * const *attrs,
-		       int dsdb_flags, 
-		       struct ldb_request *parent,
-		       const char *format, ...) _PRINTF_ATTRIBUTE(9, 10)
+		       int dsdb_flags,
+		       struct ldb_request *parent)
 {
 	int ret;
 	struct ldb_request *req;
 	TALLOC_CTX *tmp_ctx;
 	struct ldb_result *res;
-	va_list ap;
-	char *expression;
 
 	tmp_ctx = talloc_new(mem_ctx);
 
-	if (format) {
-		va_start(ap, format);
-		expression = talloc_vasprintf(tmp_ctx, format, ap);
-		va_end(ap);
-
-		if (!expression) {
-			talloc_free(tmp_ctx);
-			return ldb_oom(ldb_module_get_ctx(module));
-		}
-	} else {
-		expression = NULL;
-	}
 
 	res = talloc_zero(tmp_ctx, struct ldb_result);
 	if (!res) {
@@ -149,10 +133,10 @@ int dsdb_module_search(struct ldb_module *module,
 		return ldb_oom(ldb_module_get_ctx(module));
 	}
 
-	ret = ldb_build_search_req(&req, ldb_module_get_ctx(module), tmp_ctx,
+	ret = ldb_build_search_req_ex(&req, ldb_module_get_ctx(module), tmp_ctx,
 				   basedn,
 				   scope,
-				   expression,
+				   tree,
 				   attrs,
 				   NULL,
 				   res,
@@ -196,6 +180,61 @@ int dsdb_module_search(struct ldb_module *module,
 }
 
 /*
+  search for attrs in the modules below
+ */
+int dsdb_module_search(struct ldb_module *module,
+		       TALLOC_CTX *mem_ctx,
+		       struct ldb_result **_res,
+		       struct ldb_dn *basedn, enum ldb_scope scope,
+		       const char * const *attrs,
+		       int dsdb_flags,
+		       struct ldb_request *parent,
+		       const char *format, ...) _PRINTF_ATTRIBUTE(9, 10)
+{
+	int ret;
+	TALLOC_CTX *tmp_ctx;
+	va_list ap;
+	char *expression;
+	struct ldb_parse_tree *tree;
+
+	tmp_ctx = talloc_new(mem_ctx);
+
+	if (format) {
+		va_start(ap, format);
+		expression = talloc_vasprintf(tmp_ctx, format, ap);
+		va_end(ap);
+
+		if (!expression) {
+			talloc_free(tmp_ctx);
+			return ldb_oom(ldb_module_get_ctx(module));
+		}
+	} else {
+		expression = NULL;
+	}
+
+	tree = ldb_parse_tree(tmp_ctx, expression);
+	if (tree == NULL) {
+		talloc_free(tmp_ctx);
+		ldb_set_errstring(ldb_module_get_ctx(module),
+				"Unable to parse search expression");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ret = dsdb_module_search_tree(module,
+		       mem_ctx,
+		       _res,
+		       basedn,
+		       scope,
+		       tree,
+		       attrs,
+		       dsdb_flags,
+		       parent);
+
+	talloc_free(tmp_ctx);
+	return ret;
+}
+
+/*
   find a DN given a GUID. This searches across all partitions
  */
 int dsdb_module_dn_by_guid(struct ldb_module *module, TALLOC_CTX *mem_ctx,
diff --git a/source4/dsdb/samdb/ldb_modules/wscript_build b/source4/dsdb/samdb/ldb_modules/wscript_build
index 8ad893c5516..eb9c664c714 100644
--- a/source4/dsdb/samdb/ldb_modules/wscript_build
+++ b/source4/dsdb/samdb/ldb_modules/wscript_build
@@ -390,3 +390,12 @@ bld.SAMBA_MODULE('ldb_simple_dn',
 	internal_module=False,
 	deps='talloc DSDB_MODULE_HELPERS'
 	)
+
+bld.SAMBA_MODULE('ldb_dirsync',
+	source='dirsync.c',
+	subsystem='ldb',
+	init_function='ldb_dirsync_module_init',
+	module_init_name='ldb_init_module',
+	internal_module=False,
+	deps='talloc events security samdb DSDB_MODULE_HELPERS DSDB_MODULE_HELPER_SCHEMA'
+	)
diff --git a/source4/dsdb/samdb/samdb.c b/source4/dsdb/samdb/samdb.c
index 29b454467cc..d761107b156 100644
--- a/source4/dsdb/samdb/samdb.c
+++ b/source4/dsdb/samdb/samdb.c
@@ -96,7 +96,7 @@ struct ldb_context *samdb_connect(TALLOC_CTX *mem_ctx,
 				  struct tevent_context *ev_ctx,
 				  struct loadparm_context *lp_ctx,
 				  struct auth_session_info *session_info,
-				  int flags)
+				  unsigned int flags)
 {
 	struct ldb_context *ldb;
 	struct dsdb_schema *schema;
@@ -104,7 +104,7 @@ struct ldb_context *samdb_connect(TALLOC_CTX *mem_ctx,
 	struct cli_credentials *credentials;
 	int ret;
 
-	url  = lpcfg_sam_url(lp_ctx);
+	url  = "sam.ldb";
 	credentials = samdb_credentials(lp_ctx);
 
 	ldb = ldb_wrap_find(url, ev_ctx, lp_ctx, session_info, credentials, flags);
diff --git a/source4/dsdb/samdb/samdb.h b/source4/dsdb/samdb/samdb.h
index 8efb5e0af25..96f44c356ea 100644
--- a/source4/dsdb/samdb/samdb.h
+++ b/source4/dsdb/samdb/samdb.h
@@ -192,4 +192,5 @@ struct dsdb_fsmo_extended_op {
 	struct GUID destination_dsa_guid;
 };
 
+#define DSDB_ACL_CHECKS_DIRSYNC_FLAG 0x1
 #endif /* __SAMDB_H__ */
diff --git a/source4/dsdb/schema/schema_convert_to_ol.c b/source4/dsdb/schema/schema_convert_to_ol.c
index 77a9b1f291c..0e42f20cf0d 100644
--- a/source4/dsdb/schema/schema_convert_to_ol.c
+++ b/source4/dsdb/schema/schema_convert_to_ol.c
@@ -134,6 +134,7 @@ static char *print_schema_recursive(char *append_to_string, struct dsdb_schema *
 							may,
 							NULL);
 		if (schema_entry == NULL) {
+			talloc_free(mem_ctx);
 			DEBUG(0, ("failed to generate schema description for %s\n", name));
 			return NULL;
 		}
@@ -145,6 +146,10 @@ static char *print_schema_recursive(char *append_to_string, struct dsdb_schema *
 		case TARGET_FEDORA_DS:
 			out = talloc_asprintf_append(out, "objectClasses: %s\n", schema_entry);
 			break;
+		default:
+			talloc_free(mem_ctx);
+			DEBUG(0,(__location__ " Wrong type of target %u!", (unsigned)target));
+			return NULL;
 		}
 		talloc_free(mem_ctx);
 	} while (0);
@@ -199,6 +204,7 @@ char *dsdb_convert_schema_to_openldap(struct ldb_context *ldb, char *target_str,
 	} else if (strcasecmp(target_str, "fedora-ds") == 0) {
 		target = TARGET_FEDORA_DS;
 	} else {
+		talloc_free(mem_ctx);
 		DEBUG(0, ("Invalid target type for schema conversion %s\n", target_str));
 		return NULL;
 	}
@@ -263,6 +269,7 @@ char *dsdb_convert_schema_to_openldap(struct ldb_context *ldb, char *target_str,
 
 	schema = dsdb_get_schema(ldb, mem_ctx);
 	if (!schema) {
+		talloc_free(mem_ctx);
 		DEBUG(0, ("No schema on ldb to convert!\n"));
 		return NULL;
 	}
@@ -274,6 +281,10 @@ char *dsdb_convert_schema_to_openldap(struct ldb_context *ldb, char *target_str,
 	case TARGET_FEDORA_DS:
 		out = talloc_strdup(mem_ctx, "dn: cn=schema\n");
 		break;
+	default:
+		talloc_free(mem_ctx);
+		DEBUG(0,(__location__ " Wrong type of target %u!", (unsigned)target));
+		return NULL;
 	}
 
 	for (attribute=schema->attributes; attribute; attribute = attribute->next) {
@@ -339,6 +350,7 @@ char *dsdb_convert_schema_to_openldap(struct ldb_context *ldb, char *target_str,
 							    false, false);
 
 		if (schema_entry == NULL) {
+			talloc_free(mem_ctx);
 			DEBUG(0, ("failed to generate attribute description for %s\n", name));
 			return NULL;
 		}
@@ -350,11 +362,18 @@ char *dsdb_convert_schema_to_openldap(struct ldb_context *ldb, char *target_str,
 		case TARGET_FEDORA_DS:
 			out = talloc_asprintf_append(out, "attributeTypes: %s\n", schema_entry);
 			break;
+		default:
+			talloc_free(mem_ctx);
+			DEBUG(0,(__location__ " Wrong type of target %u!", (unsigned)target));
+			return NULL;
 		}
 	}
 
 	out = print_schema_recursive(out, schema, "top", target, attrs_skip, attr_map, oid_map);
 
+	talloc_steal(ldb, out);
+	talloc_free(mem_ctx);
+
 	return out;
 }
 
diff --git a/source4/dsdb/schema/schema_syntax.c b/source4/dsdb/schema/schema_syntax.c
index ea582db68bf..a93cdfaaa91 100644
--- a/source4/dsdb/schema/schema_syntax.c
+++ b/source4/dsdb/schema/schema_syntax.c
@@ -1069,8 +1069,14 @@ static WERROR _dsdb_syntax_OID_oid_drsuapi_to_ldb(const struct dsdb_syntax_ctx *
 						  struct ldb_message_element *out)
 {
 	unsigned int i;
+	const struct dsdb_schema_prefixmap *prefixmap;
 
-	SMB_ASSERT(ctx->pfm_remote);
+	if (ctx->pfm_remote != NULL) {
+		prefixmap = ctx->pfm_remote;
+	} else {
+		prefixmap = ctx->schema->prefixmap;
+	}
+	SMB_ASSERT(prefixmap);
 
 	out->flags	= 0;
 	out->name	= talloc_strdup(mem_ctx, attr->lDAPDisplayName);
@@ -1095,7 +1101,7 @@ static WERROR _dsdb_syntax_OID_oid_drsuapi_to_ldb(const struct dsdb_syntax_ctx *
 
 		attid = IVAL(in->value_ctr.values[i].blob->data, 0);
 
-		status = dsdb_schema_pfm_oid_from_attid(ctx->pfm_remote, attid,
+		status = dsdb_schema_pfm_oid_from_attid(prefixmap, attid,
 							out->values, &oid);
 		if (!W_ERROR_IS_OK(status)) {
 			DEBUG(0,(__location__ ": Error: Unknown ATTID 0x%08X\n",
@@ -1977,20 +1983,21 @@ static WERROR dsdb_syntax_DN_BINARY_drsuapi_to_ldb(const struct dsdb_syntax_ctx
 			W_ERROR_HAVE_NO_MEMORY(dn);
 		}
 
-		status = GUID_to_ndr_blob(&id3.guid, tmp_ctx, &guid_blob);
-		if (!NT_STATUS_IS_OK(status)) {
-			talloc_free(tmp_ctx);
-			return ntstatus_to_werror(status);
-		}
+		if (!GUID_all_zero(&id3.guid)) {
+			status = GUID_to_ndr_blob(&id3.guid, tmp_ctx, &guid_blob);
+			if (!NT_STATUS_IS_OK(status)) {
+				talloc_free(tmp_ctx);
+				return ntstatus_to_werror(status);
+			}
 
-		ret = ldb_dn_set_extended_component(dn, "GUID", &guid_blob);
-		if (ret != LDB_SUCCESS) {
-			talloc_free(tmp_ctx);
-			return WERR_FOOBAR;
+			ret = ldb_dn_set_extended_component(dn, "GUID", &guid_blob);
+			if (ret != LDB_SUCCESS) {
+				talloc_free(tmp_ctx);
+				return WERR_FOOBAR;
+			}
+			talloc_free(guid_blob.data);
 		}
 
-		talloc_free(guid_blob.data);
-
 		if (id3.__ndr_size_sid) {
 			DATA_BLOB sid_blob;
 			ndr_err = ndr_push_struct_blob(&sid_blob, tmp_ctx, &id3.sid,
diff --git a/source4/dsdb/tests/python/dirsync.py b/source4/dsdb/tests/python/dirsync.py
new file mode 100755
index 00000000000..96c69508418
--- /dev/null
+++ b/source4/dsdb/tests/python/dirsync.py
@@ -0,0 +1,713 @@
+#!/usr/bin/env python
+#
+# Unit tests for dirsync control
+# Copyright (C) Matthieu Patou <mat@matws.net> 2011
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import optparse
+import sys
+sys.path.insert(0, "bin/python")
+import samba
+samba.ensure_external_module("testtools", "testtools")
+samba.ensure_external_module("subunit", "subunit/python")
+
+import samba.getopt as options
+import base64
+
+from ldb import LdbError, SCOPE_BASE
+from ldb import Message, MessageElement, Dn
+from ldb import FLAG_MOD_ADD, FLAG_MOD_DELETE
+from samba.dcerpc import security, misc, drsblobs
+from samba.ndr import ndr_unpack, ndr_pack
+
+from samba.auth import system_session
+from samba import gensec, sd_utils
+from samba.samdb import SamDB
+from samba.credentials import Credentials
+import samba.tests
+from samba.tests import delete_force
+from subunit.run import SubunitTestRunner
+import unittest
+
+parser = optparse.OptionParser("dirsync.py [options] <host>")
+sambaopts = options.SambaOptions(parser)
+parser.add_option_group(sambaopts)
+parser.add_option_group(options.VersionOptions(parser))
+
+# use command line creds if available
+credopts = options.CredentialsOptions(parser)
+parser.add_option_group(credopts)
+opts, args = parser.parse_args()
+
+if len(args) < 1:
+    parser.print_usage()
+    sys.exit(1)
+
+host = args[0]
+if not "://" in host:
+    ldaphost = "ldap://%s" % host
+    ldapshost = "ldaps://%s" % host
+else:
+    ldaphost = host
+    start = host.rindex("://")
+    host = host.lstrip(start+3)
+
+lp = sambaopts.get_loadparm()
+creds = credopts.get_credentials(lp)
+
+#
+# Tests start here
+#
+
+class DirsyncBaseTests(samba.tests.TestCase):
+
+    def setUp(self):
+        super(DirsyncBaseTests, self).setUp()
+        self.ldb_admin = ldb
+        self.base_dn = ldb.domain_dn()
+        self.domain_sid = security.dom_sid(ldb.get_domain_sid())
+        self.user_pass = "samba123@AAA"
+        self.configuration_dn = self.ldb_admin.get_config_basedn().get_linearized()
+        self.sd_utils = sd_utils.SDUtils(ldb)
+        #used for anonymous login
+        print "baseDN: %s" % self.base_dn
+
+    def get_user_dn(self, name):
+        return "CN=%s,CN=Users,%s" % (name, self.base_dn)
+
+    def get_ldb_connection(self, target_username, target_password):
+        creds_tmp = Credentials()
+        creds_tmp.set_username(target_username)
+        creds_tmp.set_password(target_password)
+        creds_tmp.set_domain(creds.get_domain())
+        creds_tmp.set_realm(creds.get_realm())
+        creds_tmp.set_workstation(creds.get_workstation())
+        creds_tmp.set_gensec_features(creds_tmp.get_gensec_features()
+                                      | gensec.FEATURE_SEAL)
+        ldb_target = SamDB(url=ldaphost, credentials=creds_tmp, lp=lp)
+        return ldb_target
+
+
+#tests on ldap add operations
+class SimpleDirsyncTests(DirsyncBaseTests):
+
+    def setUp(self):
+        super(SimpleDirsyncTests, self).setUp()
+        # Regular user
+        self.dirsync_user = "test_dirsync_user"
+        self.simple_user = "test_simple_user"
+        self.admin_user = "test_admin_user"
+        self.ouname = None
+
+        self.ldb_admin.newuser(self.dirsync_user, self.user_pass)
+        self.ldb_admin.newuser(self.simple_user, self.user_pass)
+        self.ldb_admin.newuser(self.admin_user, self.user_pass)
+        self.desc_sddl = self.sd_utils.get_sd_as_sddl(self.base_dn)
+
+        user_sid = self.sd_utils.get_object_sid(self.get_user_dn(self.dirsync_user))
+        mod = "(A;;CR;1131f6aa-9c07-11d1-f79f-00c04fc2dcd2;;%s)" % str(user_sid)
+        self.sd_utils.dacl_add_ace(self.base_dn, mod)
+
+        # add admins to the Domain Admins group
+        self.ldb_admin.add_remove_group_members("Domain Admins", self.admin_user,
+                       add_members_operation=True)
+
+    def tearDown(self):
+        super(SimpleDirsyncTests, self).tearDown()
+        delete_force(self.ldb_admin, self.get_user_dn(self.dirsync_user))
+        delete_force(self.ldb_admin, self.get_user_dn(self.simple_user))
+        delete_force(self.ldb_admin, self.get_user_dn(self.admin_user))
+        if self.ouname:
+            delete_force(self.ldb_admin, self.ouname)
+        self.sd_utils.modify_sd_on_dn(self.base_dn, self.desc_sddl)
+        try:
+            self.ldb_admin.deletegroup("testgroup")
+        except:
+            pass
+
+    #def test_dirsync_errors(self):
+
+
+    def test_dirsync_supported(self):
+        """Test the basic of the dirsync is supported"""
+        self.ldb_dirsync = self.get_ldb_connection(self.dirsync_user, self.user_pass)
+        self.ldb_simple = self.get_ldb_connection(self.simple_user, self.user_pass)
+        res = self.ldb_admin.search(self.base_dn, expression="samaccountname=*", controls=["dirsync:1:0:1"])
+        res = self.ldb_dirsync.search(self.base_dn, expression="samaccountname=*", controls=["dirsync:1:0:1"])
+        try:
+            self.ldb_simple.search(self.base_dn,
+                expression="samaccountname=*",
+                controls=["dirsync:1:0:1"])
+        except LdbError,l:
+           self.assertTrue(str(l).find("LDAP_INSUFFICIENT_ACCESS_RIGHTS") != -1)
+
+    def test_parentGUID_referrals(self):
+        res2 = self.ldb_admin.search(self.base_dn, scope=SCOPE_BASE, attrs=["objectGUID"])
+
+        res = self.ldb_admin.search(self.base_dn,
+                                        expression="name=Configuration",
+                                        controls=["dirsync:1:0:1"])
+        self.assertEqual(res2[0].get("objectGUID"), res[0].get("parentGUID"))
+
+    def test_ok_not_rootdc(self):
+        """Test if it's ok to do dirsync on another NC that is not the root DC"""
+        try:
+            res = self.ldb_admin.search("CN=Configuration, %s" % self.base_dn,
+                                        expression="samaccountname=*",
+                                        controls=["dirsync:1:0:1"])
+        except:
+            self.assertTrue(False)
+
+    def test_dirsync_errors(self):
+        """Test if dirsync returns the correct LDAP errors in case of pb"""
+        self.ldb_simple = self.get_ldb_connection(self.simple_user, self.user_pass)
+        self.ldb_dirsync = self.get_ldb_connection(self.dirsync_user, self.user_pass)
+        try:
+            self.ldb_simple.search(self.base_dn,
+                expression="samaccountname=*",
+                controls=["dirsync:1:0:1"])
+        except LdbError,l:
+            print l
+            self.assertTrue(str(l).find("LDAP_INSUFFICIENT_ACCESS_RIGHTS") != -1)
+
+        try:
+            self.ldb_simple.search("CN=Users,%s" % self.base_dn,
+                expression="samaccountname=*",
+                controls=["dirsync:1:0:1"])
+        except LdbError,l:
+            print l
+            self.assertTrue(str(l).find("LDAP_INSUFFICIENT_ACCESS_RIGHTS") != -1)
+
+        try:
+            self.ldb_simple.search("CN=Users,%s" % self.base_dn,
+                expression="samaccountname=*",
+                controls=["dirsync:1:1:1"])
+        except LdbError,l:
+            print l
+            self.assertTrue(str(l).find("LDAP_UNWILLING_TO_PERFORM") != -1)
+
+        try:
+            self.ldb_dirsync.search("CN=Users,%s" % self.base_dn,
+                expression="samaccountname=*",
+                controls=["dirsync:1:0:1"])
+        except LdbError,l:
+            print l
+            self.assertTrue(str(l).find("LDAP_INSUFFICIENT_ACCESS_RIGHTS") != -1)
+
+        try:
+            self.ldb_admin.search("CN=Users,%s" % self.base_dn,
+                expression="samaccountname=*",
+                controls=["dirsync:1:0:1"])
+        except LdbError,l:
+            print l
+            self.assertTrue(str(l).find("LDAP_INSUFFICIENT_ACCESS_RIGHTS") != -1)
+
+        try:
+            self.ldb_admin.search("CN=Users,%s" % self.base_dn,
+                expression="samaccountname=*",
+                controls=["dirsync:1:1:1"])
+        except LdbError,l:
+            print l
+            self.assertTrue(str(l).find("LDAP_UNWILLING_TO_PERFORM") != -1)
+
+
+
+
+    def test_dirsync_attributes(self):
+        """Check behavior with some attributes """
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="samaccountname=*",
+                                    controls=["dirsync:1:0:1"])
+        # Check that nTSecurityDescriptor is returned as it's the case when doing dirsync
+        self.assertTrue(res.msgs[0].get("ntsecuritydescriptor") != None)
+        # Check that non replicated attributes are not returned
+        self.assertTrue(res.msgs[0].get("badPwdCount") == None)
+        # Check that non forward link are not returned
+        self.assertTrue(res.msgs[0].get("memberof") == None)
+
+        # Asking for instanceType will return also objectGUID
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="samaccountname=Administrator",
+                                    attrs=["instanceType"],
+                                    controls=["dirsync:1:0:1"])
+        self.assertTrue(res.msgs[0].get("objectGUID") != None)
+        self.assertTrue(res.msgs[0].get("instanceType") != None)
+
+        # We don't return an entry if asked for objectGUID
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="dn=%s" % self.base_dn,
+                                    attrs=["objectGUID"],
+                                    controls=["dirsync:1:0:1"])
+        self.assertEquals(len(res.msgs), 0)
+
+        # a request on the root of a NC didn't return parentGUID
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="dn=%s" % self.base_dn,
+                                    attrs=["name"],
+                                    controls=["dirsync:1:0:1"])
+        self.assertTrue(res.msgs[0].get("objectGUID") != None)
+        self.assertTrue(res.msgs[0].get("name") != None)
+        self.assertTrue(res.msgs[0].get("parentGUID") == None)
+        self.assertTrue(res.msgs[0].get("instanceType") != None)
+
+         # Asking for name will return also objectGUID and parentGUID
+        # and instanceType and of course name
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="samaccountname=Administrator",
+                                    attrs=["name"],
+                                    controls=["dirsync:1:0:1"])
+        self.assertTrue(res.msgs[0].get("objectGUID") != None)
+        self.assertTrue(res.msgs[0].get("name") != None)
+        self.assertTrue(res.msgs[0].get("parentGUID") != None)
+        self.assertTrue(res.msgs[0].get("instanceType") != None)
+
+        # Asking for dn will not return not only DN but more like if attrs=*
+        # parentGUID should be returned
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="samaccountname=Administrator",
+                                    attrs=["dn"],
+                                    controls=["dirsync:1:0:1"])
+        count = len(res.msgs[0])
+        res2 = self.ldb_admin.search(self.base_dn,
+                                    expression="samaccountname=Administrator",
+                                    controls=["dirsync:1:0:1"])
+        count2 = len(res2.msgs[0])
+        self.assertEqual(count, count2)
+
+        # Asking for cn will return nothing on objects that have CN as RDN
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="samaccountname=Administrator",
+                                    attrs=["cn"],
+                                    controls=["dirsync:1:0:1"])
+        self.assertEqual(len(res.msgs), 0)
+        # Asking for parentGUID will return nothing too
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="samaccountname=Administrator",
+                                    attrs=["parentGUID"],
+                                    controls=["dirsync:1:0:1"])
+        self.assertEqual(len(res.msgs), 0)
+        ouname="OU=testou,%s" % self.base_dn
+        self.ouname = ouname
+        self.ldb_admin.create_ou(ouname)
+        delta = Message()
+        delta.dn = Dn(self.ldb_admin, str(ouname))
+        delta["cn"] = MessageElement("test ou",
+                                        FLAG_MOD_ADD,
+                                        "cn" )
+        self.ldb_admin.modify(delta)
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="name=testou",
+                                    attrs=["cn"],
+                                    controls=["dirsync:1:0:1"])
+
+        self.assertEqual(len(res.msgs), 1)
+        self.assertEqual(len(res.msgs[0]), 3)
+        delete_force(self.ldb_admin, ouname)
+
+    def test_dirsync_with_controls(self):
+        """Check that dirsync return correct informations when dealing with the NC"""
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(dn=%s)" % str(self.base_dn),
+                                    attrs=["name"],
+                                    controls=["dirsync:1:0:10000", "extended_dn:1", "show_deleted:1"])
+
+    def test_dirsync_basenc(self):
+        """Check that dirsync return correct informations when dealing with the NC"""
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(dn=%s)" % str(self.base_dn),
+                                    attrs=["name"],
+                                    controls=["dirsync:1:0:10000"])
+        self.assertEqual(len(res.msgs), 1)
+        self.assertEqual(len(res.msgs[0]), 3)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(dn=%s)" % str(self.base_dn),
+                                    attrs=["ntSecurityDescriptor"],
+                                    controls=["dirsync:1:0:10000"])
+        self.assertEqual(len(res.msgs), 1)
+        self.assertEqual(len(res.msgs[0]), 3)
+
+    def test_dirsync_othernc(self):
+        """Check that dirsync return information for entries that are normaly referrals (ie. other NCs)"""
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(objectclass=configuration)",
+                                    attrs=["name"],
+                                    controls=["dirsync:1:0:10000"])
+        self.assertEqual(len(res.msgs), 1)
+        self.assertEqual(len(res.msgs[0]), 4)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(objectclass=configuration)",
+                                    attrs=["ntSecurityDescriptor"],
+                                    controls=["dirsync:1:0:10000"])
+        self.assertEqual(len(res.msgs), 1)
+        self.assertEqual(len(res.msgs[0]), 3)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(objectclass=domaindns)",
+                                    attrs=["ntSecurityDescriptor"],
+                                    controls=["dirsync:1:0:10000"])
+        nb = len(res.msgs)
+
+        # only sub nc returns a result when asked for objectGUID
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(objectclass=domaindns)",
+                                    attrs=["objectGUID"],
+                                    controls=["dirsync:1:0:0"])
+        self.assertEqual(len(res.msgs), nb - 1)
+        if nb > 1:
+            self.assertTrue(res.msgs[0].get("objectGUID") != None)
+        else:
+            res = self.ldb_admin.search(self.base_dn,
+                                        expression="(objectclass=configuration)",
+                                        attrs=["objectGUID"],
+                                        controls=["dirsync:1:0:0"])
+
+
+    def test_dirsync_send_delta(self):
+        """Check that dirsync return correct delta when sending the last cookie"""
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(samaccountname=test*)(!(isDeleted=*)))",
+                                    controls=["dirsync:1:0:10000"])
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "0"
+        ctl[3] = "10000"
+        control = str(":".join(ctl))
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(samaccountname=test*)(!(isDeleted=*)))",
+                                    controls=[control])
+        self.assertEqual(len(res), 0)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=["dirsync:1:0:100000"])
+
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "0"
+        ctl[3] = "10000"
+        control2 = str(":".join(ctl))
+
+        # Let's create an OU
+        ouname="OU=testou2,%s" % self.base_dn
+        self.ouname = ouname
+        self.ldb_admin.create_ou(ouname)
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=[control2])
+        self.assertEqual(len(res), 1)
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "0"
+        ctl[3] = "10000"
+        control3 = str(":".join(ctl))
+
+        delta = Message()
+        delta.dn = Dn(self.ldb_admin, str(ouname))
+
+        delta["cn"] = MessageElement("test ou",
+                                        FLAG_MOD_ADD,
+                                        "cn" )
+        self.ldb_admin.modify(delta)
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=[control3])
+
+        self.assertEqual(len(res.msgs), 1)
+        # 3 attributes: instanceType, cn and objectGUID
+        self.assertEqual(len(res.msgs[0]), 3)
+
+        delta = Message()
+        delta.dn = Dn(self.ldb_admin, str(ouname))
+        delta["cn"] = MessageElement([],
+                                        FLAG_MOD_DELETE,
+                                        "cn" )
+        self.ldb_admin.modify(delta)
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=[control3])
+
+        self.assertEqual(len(res.msgs), 1)
+        # So we won't have much attribute returned but instanceType and GUID
+        # are.
+        # 3 attributes: instanceType and objectGUID and cn but empty
+        self.assertEqual(len(res.msgs[0]), 3)
+        ouname = "OU=newouname,%s" % self.base_dn
+        self.ldb_admin.rename(str(res[0].dn), str(Dn(self.ldb_admin, ouname)))
+        self.ouname = ouname
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "0"
+        ctl[3] = "10000"
+        control4 = str(":".join(ctl))
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=[control3])
+
+        self.assertTrue(res[0].get("parentGUID") != None)
+        self.assertTrue(res[0].get("name") != None)
+        delete_force(self.ldb_admin, ouname)
+
+    def test_dirsync_linkedattributes(self):
+        """Check that dirsync returnd deleted objects too"""
+        # Let's search for members
+        self.ldb_simple = self.get_ldb_connection(self.simple_user, self.user_pass)
+        res = self.ldb_simple.search(self.base_dn,
+                                    expression="(name=Administrators)",
+                                    controls=["dirsync:1:1:1"])
+
+        self.assertTrue(len(res[0].get("member")) > 0)
+        size = len(res[0].get("member"))
+
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "1"
+        ctl[3] = "10000"
+        control1 = str(":".join(ctl))
+        self.ldb_admin.add_remove_group_members("Administrators", self.simple_user,
+                       add_members_operation=True)
+
+        res = self.ldb_simple.search(self.base_dn,
+                                    expression="(name=Administrators)",
+                                    controls=[control1])
+
+        self.assertEqual(len(res[0].get("member")), size + 1)
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "1"
+        ctl[3] = "10000"
+        control1 = str(":".join(ctl))
+
+        # remove the user from the group
+        self.ldb_admin.add_remove_group_members("Administrators", self.simple_user,
+                       add_members_operation=False)
+
+        res = self.ldb_simple.search(self.base_dn,
+                                    expression="(name=Administrators)",
+                                    controls=[control1])
+
+        self.assertEqual(len(res[0].get("member")), size )
+
+        self.ldb_admin.newgroup("testgroup")
+        self.ldb_admin.add_remove_group_members("testgroup", self.simple_user,
+                       add_members_operation=True)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(name=testgroup)",
+                                    controls=["dirsync:1:0:1"])
+
+        self.assertEqual(len(res[0].get("member")), 1)
+        self.assertTrue(res[0].get("member") != "" )
+
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "0"
+        ctl[3] = "1"
+        control1 = str(":".join(ctl))
+
+        # Check that reasking the same question but with an updated cookie
+        # didn't return any results.
+        print control1
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(name=testgroup)",
+                                    controls=[control1])
+        self.assertEqual(len(res), 0)
+
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "1"
+        ctl[3] = "10000"
+        control1 = str(":".join(ctl))
+
+        self.ldb_admin.add_remove_group_members("testgroup", self.simple_user,
+                       add_members_operation=False)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(name=testgroup)",
+                                    attrs=["member"],
+                                    controls=[control1])
+
+        self.ldb_admin.deletegroup("testgroup")
+        self.assertEqual(len(res[0].get("member")), 0)
+
+
+
+    def test_dirsync_deleted_items(self):
+        """Check that dirsync returnd deleted objects too"""
+        # Let's create an OU
+        ouname="OU=testou3,%s" % self.base_dn
+        self.ouname = ouname
+        self.ldb_admin.create_ou(ouname)
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=["dirsync:1:0:1"])
+        guid = None
+        for e in res:
+            if str(e["name"]) == "testou3":
+                guid = str(ndr_unpack(misc.GUID,e.get("objectGUID")[0]))
+
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "0"
+        ctl[3] = "10000"
+        control1 = str(":".join(ctl))
+
+        # So now delete the object and check that
+        # we can see the object but deleted when admin
+        delete_force(self.ldb_admin, ouname)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(objectClass=organizationalUnit)",
+                                    controls=[control1])
+        self.assertEqual(len(res), 1)
+        guid2 = str(ndr_unpack(misc.GUID,res[0].get("objectGUID")[0]))
+        self.assertEqual(guid2, guid)
+        self.assertTrue(res[0].get("isDeleted"))
+        self.assertTrue(res[0].get("name") != None)
+
+    def test_cookie_from_others(self):
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=["dirsync:1:0:1"])
+        ctl = str(res.controls[0]).split(":")
+        cookie = ndr_unpack(drsblobs.ldapControlDirSyncCookie, base64.b64decode(str(ctl[4])))
+        cookie.blob.guid1 = misc.GUID("128a99bf-abcd-1234-abcd-1fb625e530db")
+        controls=["dirsync:1:0:0:%s" % base64.b64encode(ndr_pack(cookie))]
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=controls)
+
+class ExtendedDirsyncTests(SimpleDirsyncTests):
+    def test_dirsync_linkedattributes(self):
+        flag_incr_linked = 2147483648
+        self.ldb_simple = self.get_ldb_connection(self.simple_user, self.user_pass)
+        res = self.ldb_admin.search(self.base_dn,
+                                    attrs=["member"],
+                                    expression="(name=Administrators)",
+                                    controls=["dirsync:1:%d:1" % flag_incr_linked])
+
+        self.assertTrue(res[0].get("member;range=1-1") != None )
+        self.assertTrue(len(res[0].get("member;range=1-1")) > 0)
+        size = len(res[0].get("member;range=1-1"))
+
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "%d" % flag_incr_linked
+        ctl[3] = "10000"
+        control1 = str(":".join(ctl))
+        self.ldb_admin.add_remove_group_members("Administrators", self.simple_user,
+                       add_members_operation=True)
+        self.ldb_admin.add_remove_group_members("Administrators", self.dirsync_user,
+                       add_members_operation=True)
+
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(name=Administrators)",
+                                    controls=[control1])
+
+        self.assertEqual(len(res[0].get("member;range=1-1")), 2)
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "%d" % flag_incr_linked
+        ctl[3] = "10000"
+        control1 = str(":".join(ctl))
+
+        # remove the user from the group
+        self.ldb_admin.add_remove_group_members("Administrators", self.simple_user,
+                       add_members_operation=False)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(name=Administrators)",
+                                    controls=[control1])
+
+        self.assertEqual(res[0].get("member;range=1-1"), None )
+        self.assertEqual(len(res[0].get("member;range=0-0")), 1)
+
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "%d" % flag_incr_linked
+        ctl[3] = "10000"
+        control2 = str(":".join(ctl))
+
+        self.ldb_admin.add_remove_group_members("Administrators", self.dirsync_user,
+                       add_members_operation=False)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(name=Administrators)",
+                                    controls=[control2])
+
+        self.assertEqual(res[0].get("member;range=1-1"), None )
+        self.assertEqual(len(res[0].get("member;range=0-0")), 1)
+
+        res = self.ldb_admin.search(self.base_dn,
+                                    expression="(name=Administrators)",
+                                    controls=[control1])
+
+        self.assertEqual(res[0].get("member;range=1-1"), None )
+        self.assertEqual(len(res[0].get("member;range=0-0")), 2)
+
+    def test_dirsync_deleted_items(self):
+        """Check that dirsync returnd deleted objects too"""
+        # Let's create an OU
+        self.ldb_simple = self.get_ldb_connection(self.simple_user, self.user_pass)
+        ouname="OU=testou3,%s" % self.base_dn
+        self.ouname = ouname
+        self.ldb_admin.create_ou(ouname)
+
+        # Specify LDAP_DIRSYNC_OBJECT_SECURITY
+        res = self.ldb_simple.search(self.base_dn,
+                                    expression="(&(objectClass=organizationalUnit)(!(isDeleted=*)))",
+                                    controls=["dirsync:1:1:1"])
+
+        guid = None
+        for e in res:
+            if str(e["name"]) == "testou3":
+                guid = str(ndr_unpack(misc.GUID,e.get("objectGUID")[0]))
+
+        self.assertTrue(guid != None)
+        ctl = str(res.controls[0]).split(":")
+        ctl[1] = "1"
+        ctl[2] = "1"
+        ctl[3] = "10000"
+        control1 = str(":".join(ctl))
+
+        # So now delete the object and check that
+        # we can see the object but deleted when admin
+        # we just see the objectGUID when simple user
+        delete_force(self.ldb_admin, ouname)
+
+        res = self.ldb_simple.search(self.base_dn,
+                                    expression="(objectClass=organizationalUnit)",
+                                    controls=[control1])
+        self.assertEqual(len(res), 1)
+        guid2 = str(ndr_unpack(misc.GUID,res[0].get("objectGUID")[0]))
+        self.assertEqual(guid2, guid)
+        self.assertEqual(str(res[0].dn), "")
+
+
+ldb = SamDB(ldapshost, credentials=creds, session_info=system_session(lp), lp=lp)
+
+runner = SubunitTestRunner()
+rc = 0
+#
+if not runner.run(unittest.makeSuite(SimpleDirsyncTests)).wasSuccessful():
+    rc = 1
+if not runner.run(unittest.makeSuite(ExtendedDirsyncTests)).wasSuccessful():
+    rc = 1
+
+sys.exit(rc)
diff --git a/source4/dsdb/tests/python/sam.py b/source4/dsdb/tests/python/sam.py
index 5f7c90db237..b08fba5a7d0 100755
--- a/source4/dsdb/tests/python/sam.py
+++ b/source4/dsdb/tests/python/sam.py
@@ -36,8 +36,8 @@ from samba.dsdb import (UF_NORMAL_ACCOUNT, UF_ACCOUNTDISABLE,
     ATYPE_SECURITY_LOCAL_GROUP, ATYPE_DISTRIBUTION_GLOBAL_GROUP,
     ATYPE_DISTRIBUTION_UNIVERSAL_GROUP, ATYPE_DISTRIBUTION_LOCAL_GROUP,
     ATYPE_WORKSTATION_TRUST)
-from samba.dcerpc.security import (DOMAIN_RID_USERS, DOMAIN_RID_DOMAIN_MEMBERS,
-    DOMAIN_RID_DCS, DOMAIN_RID_READONLY_DCS)
+from samba.dcerpc.security import (DOMAIN_RID_USERS, DOMAIN_RID_ADMINS,
+    DOMAIN_RID_DOMAIN_MEMBERS, DOMAIN_RID_DCS, DOMAIN_RID_READONLY_DCS)
 
 from subunit.run import SubunitTestRunner
 import unittest
@@ -1471,25 +1471,33 @@ class SamTests(unittest.TestCase):
             self.assertEquals(num, ERR_OTHER)
         delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
 
-# This isn't supported yet in s4
-#        try:
-#            ldb.add({
-#                "dn": "cn=ldaptestuser,cn=users," + self.base_dn,
-#                "objectclass": "user",
-#                "userAccountControl": str(UF_SERVER_TRUST_ACCOUNT)})
-#            self.fail()
-#        except LdbError, (num, _):
-#            self.assertEquals(num, ERR_OBJECT_CLASS_VIOLATION)
-#        delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
-#
-#        try:
-#            ldb.add({
-#                "dn": "cn=ldaptestuser,cn=users," + self.base_dn,
-#                "objectclass": "user",
-#                "userAccountControl": str(UF_WORKSTATION_TRUST_ACCOUNT)})
-#        except LdbError, (num, _):
-#            self.assertEquals(num, ERR_OBJECT_CLASS_VIOLATION)
-#        delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
+        try:
+            ldb.add({
+                "dn": "cn=ldaptestuser,cn=users," + self.base_dn,
+                "objectclass": "user",
+                "userAccountControl": str(UF_SERVER_TRUST_ACCOUNT)})
+            self.fail()
+        except LdbError, (num, _):
+            self.assertEquals(num, ERR_OBJECT_CLASS_VIOLATION)
+        delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
+
+        try:
+            ldb.add({
+                "dn": "cn=ldaptestuser,cn=users," + self.base_dn,
+                "objectclass": "user",
+                "userAccountControl": str(UF_WORKSTATION_TRUST_ACCOUNT)})
+        except LdbError, (num, _):
+            self.assertEquals(num, ERR_OBJECT_CLASS_VIOLATION)
+        delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
+
+        try:
+            ldb.add({
+                "dn": "cn=ldaptestuser,cn=users," + self.base_dn,
+                "objectclass": "user",
+                "userAccountControl": str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PARTIAL_SECRETS_ACCOUNT)})
+        except LdbError, (num, _):
+            self.assertEquals(num, ERR_OBJECT_CLASS_VIOLATION)
+        delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
 
 # This isn't supported yet in s4 - needs ACL module adaption
 #        try:
@@ -1570,17 +1578,16 @@ class SamTests(unittest.TestCase):
         except LdbError, (num, _):
             self.assertEquals(num, ERR_OTHER)
 
-# This isn't supported yet in s4
-#        try:
-#            m = Message()
-#            m.dn = Dn(ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
-#            m["userAccountControl"] = MessageElement(
-#              str(UF_SERVER_TRUST_ACCOUNT),
-#              FLAG_MOD_REPLACE, "userAccountControl")
-#            ldb.modify(m)
-#            self.fail()
-#        except LdbError, (num, _):
-#            self.assertEquals(num, ERR_UNWILLING_TO_PERFORM)
+        try:
+            m = Message()
+            m.dn = Dn(ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
+            m["userAccountControl"] = MessageElement(
+              str(UF_SERVER_TRUST_ACCOUNT),
+              FLAG_MOD_REPLACE, "userAccountControl")
+            ldb.modify(m)
+            self.fail()
+        except LdbError, (num, _):
+            self.assertEquals(num, ERR_UNWILLING_TO_PERFORM)
 
         m = Message()
         m.dn = Dn(ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
@@ -1589,6 +1596,17 @@ class SamTests(unittest.TestCase):
           FLAG_MOD_REPLACE, "userAccountControl")
         ldb.modify(m)
 
+        try:
+            m = Message()
+            m.dn = Dn(ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
+            m["userAccountControl"] = MessageElement(
+              str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PARTIAL_SECRETS_ACCOUNT),
+              FLAG_MOD_REPLACE, "userAccountControl")
+            ldb.modify(m)
+            self.fail()
+        except LdbError, (num, _):
+            self.assertEquals(num, ERR_UNWILLING_TO_PERFORM)
+
         res1 = ldb.search("cn=ldaptestuser,cn=users," + self.base_dn,
                           scope=SCOPE_BASE, attrs=["sAMAccountType"])
         self.assertTrue(len(res1) == 1)
@@ -1866,7 +1884,215 @@ class SamTests(unittest.TestCase):
 #        except LdbError, (num, _):
 #            self.assertEquals(num, ERR_INSUFFICIENT_ACCESS_RIGHTS)
 
+        # "primaryGroupID" does not change if account type remains the same
+
+        # For a user account
+
+        ldb.add({
+            "dn": "cn=ldaptestuser2,cn=users," + self.base_dn,
+            "objectclass": "user",
+            "userAccountControl": str(UF_NORMAL_ACCOUNT | UF_PASSWD_NOTREQD | UF_ACCOUNTDISABLE)})
+
+        res1 = ldb.search("cn=ldaptestuser2,cn=users," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["userAccountControl"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(int(res1[0]["userAccountControl"][0]),
+           UF_NORMAL_ACCOUNT | UF_PASSWD_NOTREQD | UF_ACCOUNTDISABLE)
+
+        m = Message()
+        m.dn = Dn(ldb, "<SID=" + ldb.get_domain_sid() + "-" + str(DOMAIN_RID_ADMINS) + ">")
+        m["member"] = MessageElement(
+          "cn=ldaptestuser2,cn=users," + self.base_dn, FLAG_MOD_ADD, "member")
+        ldb.modify(m)
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestuser2,cn=users," + self.base_dn)
+        m["primaryGroupID"] = MessageElement(str(DOMAIN_RID_ADMINS),
+          FLAG_MOD_REPLACE, "primaryGroupID")
+        ldb.modify(m)
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestuser2,cn=users," + self.base_dn)
+        m["userAccountControl"] = MessageElement(
+          str(UF_NORMAL_ACCOUNT | UF_PASSWD_NOTREQD),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestuser2,cn=users," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["userAccountControl", "primaryGroupID"])
+        self.assertTrue(len(res1) == 1)
+        self.assertTrue(int(res1[0]["userAccountControl"][0]) & UF_ACCOUNTDISABLE == 0)
+        self.assertEquals(int(res1[0]["primaryGroupID"][0]), DOMAIN_RID_ADMINS)
+
+        # For a workstation account
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["primaryGroupID"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(int(res1[0]["primaryGroupID"][0]), DOMAIN_RID_DOMAIN_MEMBERS)
+
+        m = Message()
+        m.dn = Dn(ldb, "<SID=" + ldb.get_domain_sid() + "-" + str(DOMAIN_RID_USERS) + ">")
+        m["member"] = MessageElement(
+          "cn=ldaptestcomputer,cn=computers," + self.base_dn, FLAG_MOD_ADD, "member")
+        ldb.modify(m)
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["primaryGroupID"] = MessageElement(str(DOMAIN_RID_USERS),
+          FLAG_MOD_REPLACE, "primaryGroupID")
+        ldb.modify(m)
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["userAccountControl"] = MessageElement(
+          str(UF_WORKSTATION_TRUST_ACCOUNT),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["primaryGroupID"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(int(res1[0]["primaryGroupID"][0]), DOMAIN_RID_USERS)
+
         delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
+        delete_force(self.ldb, "cn=ldaptestuser2,cn=users," + self.base_dn)
+        delete_force(self.ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+
+    def test_isCriticalSystemObject(self):
+        """Test the isCriticalSystemObject behaviour"""
+        print "Testing isCriticalSystemObject behaviour\n"
+
+        # Add tests
+
+        ldb.add({
+            "dn": "cn=ldaptestcomputer,cn=computers," + self.base_dn,
+            "objectclass": "computer"})
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertTrue("isCriticalSystemObject" not in res1[0])
+
+        delete_force(self.ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+
+        ldb.add({
+            "dn": "cn=ldaptestcomputer,cn=computers," + self.base_dn,
+            "objectclass": "computer",
+            "userAccountControl": str(UF_WORKSTATION_TRUST_ACCOUNT)})
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "FALSE")
+
+        delete_force(self.ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+
+        ldb.add({
+            "dn": "cn=ldaptestcomputer,cn=computers," + self.base_dn,
+            "objectclass": "computer",
+            "userAccountControl": str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PARTIAL_SECRETS_ACCOUNT)})
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "TRUE")
+
+        delete_force(self.ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+
+        ldb.add({
+            "dn": "cn=ldaptestcomputer,cn=computers," + self.base_dn,
+            "objectclass": "computer",
+            "userAccountControl": str(UF_SERVER_TRUST_ACCOUNT)})
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "TRUE")
+
+        # Modification tests
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["userAccountControl"] = MessageElement(str(UF_NORMAL_ACCOUNT | UF_PASSWD_NOTREQD),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "TRUE")
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["userAccountControl"] = MessageElement(str(UF_WORKSTATION_TRUST_ACCOUNT),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "FALSE")
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["userAccountControl"] = MessageElement(
+          str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PARTIAL_SECRETS_ACCOUNT),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "TRUE")
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["userAccountControl"] = MessageElement(str(UF_NORMAL_ACCOUNT | UF_PASSWD_NOTREQD),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "TRUE")
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["userAccountControl"] = MessageElement(str(UF_SERVER_TRUST_ACCOUNT),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "TRUE")
+
+        m = Message()
+        m.dn = Dn(ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
+        m["userAccountControl"] = MessageElement(str(UF_WORKSTATION_TRUST_ACCOUNT),
+          FLAG_MOD_REPLACE, "userAccountControl")
+        ldb.modify(m)
+
+        res1 = ldb.search("cn=ldaptestcomputer,cn=computers," + self.base_dn,
+                          scope=SCOPE_BASE,
+                          attrs=["isCriticalSystemObject"])
+        self.assertTrue(len(res1) == 1)
+        self.assertEquals(res1[0]["isCriticalSystemObject"][0], "FALSE")
+
         delete_force(self.ldb, "cn=ldaptestcomputer,cn=computers," + self.base_dn)
 
     def test_service_principal_name_updates(self):
diff --git a/source4/dsdb/tests/python/token_group.py b/source4/dsdb/tests/python/token_group.py
index 62bdbd5ee03..fb7654e7e07 100755
--- a/source4/dsdb/tests/python/token_group.py
+++ b/source4/dsdb/tests/python/token_group.py
@@ -78,7 +78,7 @@ class TokenTest(samba.tests.TestCase):
         res = self.ldb.search("", scope=ldb.SCOPE_BASE, attrs=["tokenGroups"])
         self.assertEquals(len(res), 1)
 
-        print("Geting tokenGroups from rootDSE")
+        print("Getting tokenGroups from rootDSE")
         tokengroups = []
         for sid in res[0]['tokenGroups']:
             tokengroups.append(str(ndr_unpack(samba.dcerpc.security.dom_sid, sid)))
@@ -93,7 +93,7 @@ class TokenTest(samba.tests.TestCase):
             self.fail(msg="calculated groups don't match against rootDSE tokenGroups")
 
     def test_dn_tokenGroups(self):
-        print("Geting tokenGroups from user DN")
+        print("Getting tokenGroups from user DN")
         res = self.ldb.search(self.user_sid_dn, scope=ldb.SCOPE_BASE, attrs=["tokenGroups"])
         self.assertEquals(len(res), 1)
 
diff --git a/source4/dsdb/wscript_build b/source4/dsdb/wscript_build
index 9d56e4f1507..fe00059c4b5 100644
--- a/source4/dsdb/wscript_build
+++ b/source4/dsdb/wscript_build
@@ -8,7 +8,7 @@ bld.SAMBA_LIBRARY('samdb',
 	autoproto='samdb/samdb_proto.h',
 	public_deps='krb5',
 	vnum='0.0.1',
-	deps='ndr NDR_DRSUAPI NDR_DRSBLOBS auth_system_session LIBCLI_AUTH ndr SAMDB_SCHEMA ldbsamba samdb-common LIBCLI_DRSUAPI cli-ldap-common samba-util com_err authkrb5 credentials ldbwrap',
+	deps='ndr NDR_DRSUAPI NDR_DRSBLOBS auth_system_session LIBCLI_AUTH ndr SAMDB_SCHEMA ldbsamba samdb-common LIBCLI_DRSUAPI cli-ldap-common samba-util com_err authkrb5 credentials ldbwrap errors',
 	)
 
 
diff --git a/source4/echo_server/echo_server.c b/source4/echo_server/echo_server.c
index 4be6f21af84..60729d8535c 100644
--- a/source4/echo_server/echo_server.c
+++ b/source4/echo_server/echo_server.c
@@ -25,7 +25,7 @@
 #include "param/param.h"
 /* This defines task_server_terminate */
 #include "smbd/process_model.h"
-/* We get load_interfaces from here */
+/* We get load_interface_list from here */
 #include "socket/netif.h"
 /* NTSTATUS-related stuff */
 #include "libcli/util/ntstatus.h"
@@ -197,7 +197,7 @@ static NTSTATUS echo_add_socket(struct echo_server *echo,
 						address, port,
 						&echo_socket->local_address);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		return status;
 	}
 
@@ -212,7 +212,7 @@ static NTSTATUS echo_add_socket(struct echo_server *echo,
 				     echo_udp_socket,
 				     &echo_udp_socket->dgram);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		DEBUG(0, ("Failed to bind to %s:%u UDP - %s\n",
 			  address, port, nt_errstr(status)));
 		return status;
@@ -269,10 +269,10 @@ static NTSTATUS echo_startup_interfaces(struct echo_server *echo,
 		return NT_STATUS_INTERNAL_ERROR;
 	}
 
-	num_interfaces = iface_count(ifaces);
+	num_interfaces = iface_list_count(ifaces);
 
 	for(i=0; i<num_interfaces; i++) {
-		const char *address = talloc_strdup(tmp_ctx, iface_n_ip(ifaces, i));
+		const char *address = talloc_strdup(tmp_ctx, iface_list_n_ip(ifaces, i));
 
 		status = echo_add_socket(echo, model_ops, "echo", address, ECHO_SERVICE_PORT);
 		NT_STATUS_NOT_OK_RETURN(status);
@@ -308,9 +308,9 @@ static void echo_task_init(struct task_server *task)
 		break;
 	}
 
-	load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
+	load_interface_list(task, task->lp_ctx, &ifaces);
 
-	if (iface_count(ifaces) == 0) {
+	if (iface_list_count(ifaces) == 0) {
 		task_server_terminate(task,
 				      "echo: No network interfaces configured",
 				      false);
diff --git a/source4/heimdal/kdc/krb5tgs.c b/source4/heimdal/kdc/krb5tgs.c
index 037934f2a65..66170cb29f1 100644
--- a/source4/heimdal/kdc/krb5tgs.c
+++ b/source4/heimdal/kdc/krb5tgs.c
@@ -282,8 +282,9 @@ check_PAC(krb5_context context,
 	  hdb_entry_ex *client,
 	  hdb_entry_ex *server,
 	  hdb_entry_ex *krbtgt,
-	  const EncryptionKey *server_key,
+	  const EncryptionKey *server_check_key,
 	  const EncryptionKey *krbtgt_check_key,
+	  const EncryptionKey *server_sign_key,
 	  const EncryptionKey *krbtgt_sign_key,
 	  EncTicketPart *tkt,
 	  krb5_data *rspac,
@@ -328,7 +329,7 @@ check_PAC(krb5_context context,
 
 		ret = krb5_pac_verify(context, pac, tkt->authtime,
 				      client_principal,
-				      krbtgt_check_key, NULL);
+				      server_check_key, krbtgt_check_key);
 		if (ret) {
 		    krb5_pac_free(context, pac);
 		    return ret;
@@ -351,7 +352,7 @@ check_PAC(krb5_context context,
 		    *signedpath = 1;
 		    ret = _krb5_pac_sign(context, pac, tkt->authtime,
 					 client_principal,
-					 server_key, krbtgt_sign_key, rspac);
+					 server_sign_key, krbtgt_sign_key, rspac);
 		}
 		krb5_pac_free(context, pac);
 		
@@ -1464,10 +1465,9 @@ tgs_build_reply(krb5_context context,
 		const struct sockaddr *from_addr)
 {
     krb5_error_code ret;
-    krb5_principal cp = NULL, sp = NULL;
-    krb5_principal client_principal = NULL;
+    krb5_principal cp = NULL, sp = NULL, tp = NULL;
     krb5_principal krbtgt_principal = NULL;
-    char *spn = NULL, *cpn = NULL;
+    char *spn = NULL, *cpn = NULL, *tpn = NULL;
     hdb_entry_ex *server = NULL, *client = NULL, *s4u2self_impersonated_client = NULL;
     HDB *clientdb, *s4u2self_impersonated_clientdb;
     krb5_realm ref_realm = NULL;
@@ -1719,16 +1719,16 @@ server_lookup:
     krb5_free_principal(context, krbtgt_principal);
     if (ret) {
 	krb5_error_code ret2;
-	char *tpn, *tpn2;
-	ret = krb5_unparse_name(context, krbtgt->entry.principal, &tpn);
-	ret2 = krb5_unparse_name(context, krbtgt->entry.principal, &tpn2);
+	char *ktpn, *ktpn2;
+	ret = krb5_unparse_name(context, krbtgt->entry.principal, &ktpn);
+	ret2 = krb5_unparse_name(context, krbtgt_principal, &ktpn2);
 	kdc_log(context, config, 0,
 		"Request with wrong krbtgt: %s, %s not found in our database",
-		(ret == 0) ? tpn : "<unknown>", (ret2 == 0) ? tpn2 : "<unknown>");
+		(ret == 0) ? ktpn : "<unknown>", (ret2 == 0) ? ktpn2 : "<unknown>");
 	if(ret == 0)
-	    free(tpn);
+	    free(ktpn);
 	if(ret2 == 0)
-	    free(tpn2);
+	    free(ktpn2);
 	ret = KRB5KRB_AP_ERR_NOT_US;
 	goto out;
     }
@@ -1740,13 +1740,13 @@ server_lookup:
      * this) before the strcmp() */
     if (strcmp(krb5_principal_get_realm(context, server->entry.principal),
 	       krb5_principal_get_realm(context, krbtgt_out->entry.principal)) != 0) {
-	char *tpn;
-	ret = krb5_unparse_name(context, krbtgt_out->entry.principal, &tpn);
+	char *ktpn;
+	ret = krb5_unparse_name(context, krbtgt_out->entry.principal, &ktpn);
 	kdc_log(context, config, 0,
 		"Request with wrong krbtgt: %s",
-		(ret == 0) ? tpn : "<unknown>");
+		(ret == 0) ? ktpn : "<unknown>");
 	if(ret == 0)
-	    free(tpn);
+	    free(ktpn);
 	ret = KRB5KRB_AP_ERR_NOT_US;
     }
 
@@ -1789,7 +1789,9 @@ server_lookup:
     }
 
     ret = check_PAC(context, config, cp,
-		    client, server, krbtgt, ekey, &tkey_check->key, &tkey_sign->key,
+		    client, server, krbtgt,
+		    &tkey_check->key, &tkey_check->key,
+		    ekey, &tkey_sign->key,
 		    tgt, &rspac, &signedpath);
     if (ret) {
 	const char *msg = krb5_get_error_message(context, ret);
@@ -1821,7 +1823,9 @@ server_lookup:
      * Process request
      */
 
-    client_principal = cp;
+    /* by default the tgt principal matches the client principal */
+    tp = cp;
+    tpn = cpn;
 
     if (client) {
 	const PA_DATA *sdata;
@@ -1832,7 +1836,6 @@ server_lookup:
 	    krb5_crypto crypto;
 	    krb5_data datack;
 	    PA_S4U2Self self;
-	    char *selfcpn = NULL;
 	    const char *str;
 
 	    ret = decode_PA_S4U2Self(sdata->padata_value.data,
@@ -1875,14 +1878,14 @@ server_lookup:
 	    }
 
 	    ret = _krb5_principalname2krb5_principal(context,
-						     &client_principal,
+						     &tp,
 						     self.name,
 						     self.realm);
 	    free_PA_S4U2Self(&self);
 	    if (ret)
 		goto out;
 
-	    ret = krb5_unparse_name(context, client_principal, &selfcpn);	
+	    ret = krb5_unparse_name(context, tp, &tpn);
 	    if (ret)
 		goto out;
 
@@ -1890,7 +1893,7 @@ server_lookup:
 	    if(rspac.data) {
 		krb5_pac p = NULL;
 		krb5_data_free(&rspac);
-		ret = _kdc_db_fetch(context, config, client_principal, HDB_F_GET_CLIENT | HDB_F_CANON,
+		ret = _kdc_db_fetch(context, config, tp, HDB_F_GET_CLIENT | HDB_F_CANON,
 				    NULL, &s4u2self_impersonated_clientdb, &s4u2self_impersonated_client);
 		if (ret) {
 		    const char *msg;
@@ -1904,14 +1907,16 @@ server_lookup:
 		    if (ret == HDB_ERR_NOENTRY)
 			ret = KRB5KDC_ERR_C_PRINCIPAL_UNKNOWN;
 		    msg = krb5_get_error_message(context, ret);
-		    kdc_log(context, config, 1, "S2U4Self principal to impersonate %s not found in database: %s", cpn, msg);
+		    kdc_log(context, config, 1,
+			    "S2U4Self principal to impersonate %s not found in database: %s",
+			    tpn, msg);
 		    krb5_free_error_message(context, msg);
 		    goto out;
 		}
 		ret = _kdc_pac_generate(context, s4u2self_impersonated_client, &p);
 		if (ret) {
 		    kdc_log(context, config, 0, "PAC generation failed for -- %s",
-			    selfcpn);
+			    tpn);
 		    goto out;
 		}
 		if (p != NULL) {
@@ -1922,7 +1927,7 @@ server_lookup:
 		    krb5_pac_free(context, p);
 		    if (ret) {
 			kdc_log(context, config, 0, "PAC signing failed for -- %s",
-				selfcpn);
+				tpn);
 			goto out;
 		    }
 		}
@@ -1937,8 +1942,7 @@ server_lookup:
 		kdc_log(context, config, 0, "S4U2Self: %s is not allowed "
 			"to impersonate to service "
 			"(tried for user %s to service %s)",
-			cpn, selfcpn, spn);
-		free(selfcpn);
+			cpn, tpn, spn);
 		goto out;
 	    }
 
@@ -1954,8 +1958,7 @@ server_lookup:
 		str = "";
 	    }
 	    kdc_log(context, config, 0, "s4u2self %s impersonating %s to "
-		    "service %s %s", cpn, selfcpn, spn, str);
-	    free(selfcpn);
+		    "service %s %s", cpn, tpn, spn, str);
 	}
     }
 
@@ -1971,7 +1974,6 @@ server_lookup:
 	int ad_signedpath = 0;
 	Key *clientkey;
 	Ticket *t;
-	char *str;
 
 	/*
 	 * Require that the KDC have issued the service's krbtgt (not
@@ -2002,11 +2004,23 @@ server_lookup:
 	    goto out;
 	}
 
+	ret = _krb5_principalname2krb5_principal(context,
+						 &tp,
+						 adtkt.cname,
+						 adtkt.crealm);
+	if (ret)
+	    goto out;
+
+	ret = krb5_unparse_name(context, tp, &tpn);
+	if (ret)
+	    goto out;
+
 	/* check that ticket is valid */
 	if (adtkt.flags.forwardable == 0) {
 	    kdc_log(context, config, 0,
 		    "Missing forwardable flag on ticket for "
-		    "constrained delegation from %s to %s ", cpn, spn);
+		    "constrained delegation from %s as %s to %s ",
+		    cpn, tpn, spn);
 	    ret = KRB5KDC_ERR_BADOPTION;
 	    goto out;
 	}
@@ -2015,25 +2029,37 @@ server_lookup:
 					   client, sp);
 	if (ret) {
 	    kdc_log(context, config, 0,
-		    "constrained delegation from %s to %s not allowed",
-		    cpn, spn);
+		    "constrained delegation from %s as %s to %s not allowed",
+		    cpn, tpn, spn);
 	    goto out;
 	}
 
-	ret = _krb5_principalname2krb5_principal(context,
-						 &client_principal,
-						 adtkt.cname,
-						 adtkt.crealm);
-	if (ret)
-	    goto out;
-
-	ret = krb5_unparse_name(context, client_principal, &str);
-	if (ret)
+	ret = verify_flags(context, config, &adtkt, tpn);
+	if (ret) {
 	    goto out;
+	}
 
-	ret = verify_flags(context, config, &adtkt, str);
+	krb5_data_free(&rspac);
+	/*
+	 * generate the PAC for the user.
+	 *
+	 * TODO: pass in t->sname and t->realm and build
+	 * a S4U_DELEGATION_INFO blob to the PAC.
+	 */
+	ret = check_PAC(context, config, tp,
+			client, server, krbtgt,
+			&clientkey->key, &tkey_check->key,
+			ekey, &tkey_sign->key,
+			&adtkt, &rspac, &ad_signedpath);
+	if (ret == 0 && !ad_signedpath)
+	    ret = KRB5KDC_ERR_BADOPTION;
 	if (ret) {
-	    free(str);
+	    const char *msg = krb5_get_error_message(context, ret);
+	    kdc_log(context, config, 0,
+		    "Verify delegated PAC failed to %s for client"
+		    "%s as %s from %s with %s",
+		    spn, cpn, tpn, from, msg);
+	    krb5_free_error_message(context, msg);
 	    goto out;
 	}
 
@@ -2043,7 +2069,7 @@ server_lookup:
 	ret = check_KRB5SignedPath(context,
 				   config,
 				   krbtgt,
-				   cp,
+				   tp,
 				   &adtkt,
 				   NULL,
 				   &ad_signedpath);
@@ -2055,15 +2081,13 @@ server_lookup:
 		    "KRB5SignedPath check from service %s failed "
 		    "for delegation to %s for client %s "
 		    "from %s failed with %s",
-		    spn, str, cpn, from, msg);
+		    spn, tpn, cpn, from, msg);
 	    krb5_free_error_message(context, msg);
-	    free(str);
 	    goto out;
 	}
 
 	kdc_log(context, config, 0, "constrained delegation for %s "
-		"from %s to %s", str, cpn, spn);
-	free(str);
+		"from %s to %s", tpn, cpn, spn);
     }
 
     /*
@@ -2134,7 +2158,7 @@ server_lookup:
     ret = tgs_make_reply(context,
 			 config,
 			 b,
-			 client_principal,
+			 tp,
 			 tgt,
 			 replykey,
 			 rk_is_subkey,
@@ -2156,6 +2180,8 @@ server_lookup:
 			 reply);
 	
 out:
+    if (tpn != cpn)
+	    free(tpn);
     free(spn);
     free(cpn);
 	
@@ -2170,8 +2196,8 @@ out:
     if(s4u2self_impersonated_client)
 	_kdc_free_ent(context, s4u2self_impersonated_client);
 
-    if (client_principal && client_principal != cp)
-	krb5_free_principal(context, client_principal);
+    if (tp && tp != cp)
+	krb5_free_principal(context, tp);
     if (cp)
 	krb5_free_principal(context, cp);
     if (sp)
diff --git a/source4/heimdal/lib/roken/getprogname.c b/source4/heimdal/lib/roken/getprogname.c
deleted file mode 100644
index a310208a843..00000000000
--- a/source4/heimdal/lib/roken/getprogname.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright (c) 1995-2004 Kungliga Tekniska Högskolan
- * (Royal Institute of Technology, Stockholm, Sweden).
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the name of the Institute nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include "roken.h"
-
-#ifndef HAVE___PROGNAME
-const char *__progname;
-#endif
-
-#ifndef HAVE_GETPROGNAME
-ROKEN_LIB_FUNCTION const char * ROKEN_LIB_CALL
-getprogname(void)
-{
-    return __progname;
-}
-#endif /* HAVE_GETPROGNAME */
diff --git a/source4/heimdal/lib/roken/setprogname.c b/source4/heimdal/lib/roken/setprogname.c
deleted file mode 100644
index 88a5f9bb449..00000000000
--- a/source4/heimdal/lib/roken/setprogname.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 1995-2004 Kungliga Tekniska Högskolan
- * (Royal Institute of Technology, Stockholm, Sweden).
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the name of the Institute nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include "roken.h"
-
-#ifndef HAVE___PROGNAME
-extern const char *__progname;
-#endif
-
-#ifndef HAVE_SETPROGNAME
-
-ROKEN_LIB_FUNCTION void ROKEN_LIB_CALL
-setprogname(const char *argv0)
-{
-
-#ifndef HAVE___PROGNAME
-
-    const char *p;
-    if(argv0 == NULL)
-	return;
-    p = strrchr(argv0, '/');
-
-#ifdef BACKSLASH_PATH_DELIM
-    {
-        const char * pb;
-
-        pb = strrchr((p != NULL)? p : argv0, '\\');
-        if (pb != NULL)
-            p = pb;
-    }
-#endif
-
-    if(p == NULL)
-	p = argv0;
-    else
-	p++;
-
-#ifdef _WIN32
-    {
-        char * fn = strdup(p);
-        char * ext;
-
-        strlwr(fn);
-        ext = strrchr(fn, '.');
-        if (ext != NULL && !strcmp(ext, ".exe"))
-            *ext = '\0';
-
-        __progname = fn;
-    }
-#else
-
-    __progname = p;
-
-#endif
-
-#endif  /* HAVE___PROGNAME */
-}
-
-#endif /* HAVE_SETPROGNAME */
diff --git a/source4/heimdal_build/replace.c b/source4/heimdal_build/replace.c
index 51393f6e687..e6a74f9ba8d 100644
--- a/source4/heimdal_build/replace.c
+++ b/source4/heimdal_build/replace.c
@@ -83,3 +83,20 @@
 	return -1;
 }
 #endif
+
+#ifndef HAVE_SETPROGNAME
+
+/* We don't want to use a setprogname reimplementation */
+void setprogname(const char *argv0)
+{
+}
+
+#endif /* HAVE_SETPROGNAME */
+
+#ifndef HAVE_GETPROGNAME
+/* We don't want to use a getprogname reimplementation */
+const char *getprogname(void)
+{
+	return "";
+}
+#endif /* HAVE_GETPROGNAME */
diff --git a/source4/heimdal_build/wscript_build b/source4/heimdal_build/wscript_build
index a1bd4783b04..705caa52e2d 100644
--- a/source4/heimdal_build/wscript_build
+++ b/source4/heimdal_build/wscript_build
@@ -400,12 +400,6 @@ if not bld.CONFIG_SET('USING_SYSTEM_ROKEN'):
         ../heimdal_build/replace.c
     '''
 
-    if not bld.CONFIG_SET('HAVE_GETPROGNAME'):
-        ROKEN_HOSTCC_SOURCE += '''
-            lib/roken/getprogname.c
-            lib/roken/setprogname.c
-        '''
-
     if not bld.CONFIG_SET('HAVE_CLOSEFROM'):
         ROKEN_HOSTCC_SOURCE += '''
             lib/roken/closefrom.c
@@ -621,6 +615,13 @@ if not bld.CONFIG_SET("USING_SYSTEM_KRB5"):
                                    version.c warn.c krb5_err.c
                                    heim_err.c k524_err.c krb_err.c''')]  + ["../heimdal_build/krb5-glue.c"]
 
+    # Alias subsystem to allow common kerberos code that will
+    # otherwise link against MIT's gssapi_krb5
+    HEIMDAL_SUBSYSTEM('gssapi_krb5',
+        '',
+        deps='gssapi'
+        )
+
     HEIMDAL_LIBRARY('krb5', KRB5_SOURCE,
         version_script='lib/krb5/version-script.map',
                         includes='../heimdal/lib/krb5 ../heimdal/lib/asn1 ../heimdal/include',
@@ -633,6 +634,12 @@ if not bld.CONFIG_SET("USING_SYSTEM_KRB5"):
     HEIMDAL_AUTOPROTO('lib/krb5/krb5-protos.h', KRB5_PROTO_SOURCE,
         options='-E KRB5_LIB -q -P comment -o')
 
+    # Alias subsystem to allow common kerberos code that will
+    # otherwise link against MIT's k5crypto
+    HEIMDAL_SUBSYSTEM('k5crypto',
+                      '',
+                      deps='krb5')
+
 if not bld.CONFIG_SET("USING_SYSTEM_ASN1"):
     HEIMDAL_HEIM_ASN1_DER_SOURCE = '''
         lib/asn1/der_get.c
diff --git a/source4/heimdal_build/wscript_configure b/source4/heimdal_build/wscript_configure
index f711fe7f286..6552d3a9253 100644
--- a/source4/heimdal_build/wscript_configure
+++ b/source4/heimdal_build/wscript_configure
@@ -73,6 +73,7 @@ conf.DEFINE('SAMBA4_INTERNAL_HEIMDAL', 1)
 # setup the right defines for a in-tree heimdal build
 Logs.info("Using in-tree heimdal kerberos defines")
 conf.define('HAVE_GSSAPI_GSSAPI_H', 1)
+conf.define('HAVE_GSSAPI_GSSAPI_KRB5_H', 1)
 conf.define('HAVE_AP_OPTS_USE_SUBKEY', 1)
 conf.define('HAVE_KRB5_ADDRESSES', 1)
 conf.define('HAVE_KRB5_KEYTAB_ENTRY_KEYBLOCK', 1)
@@ -82,6 +83,8 @@ conf.define('HAVE_ADDR_TYPE_IN_KRB5_ADDRESS', 1)
 conf.define('HAVE_GSS_DISPLAY_STATUS', 1)
 conf.define('HAVE_GSS_WRAP_IOV', 1)
 conf.define('HAVE_GSS_KRB5_IMPORT_CRED', 1)
+conf.define('HAVE_GSS_OID_EQUAL', 1)
+conf.define('HAVE_GSS_INQUIRE_SEC_CONTEXT_BY_OID', 1)
 conf.define('HAVE_GSSKRB5_EXTRACT_AUTHZ_DATA_FROM_SEC_CONTEXT', 1)
 conf.define('HAVE_LIBGSSAPI', 1)
 conf.define('HAVE_ADDR_TYPE_IN_KRB5_ADDRESS', 1)
diff --git a/source4/kdc/db-glue.c b/source4/kdc/db-glue.c
index 732e553ca3d..72262ac18b5 100644
--- a/source4/kdc/db-glue.c
+++ b/source4/kdc/db-glue.c
@@ -159,6 +159,20 @@ static HDBFlags uf2HDBFlags(krb5_context context, uint32_t userAccountControl, e
 	if (userAccountControl & UF_TRUSTED_FOR_DELEGATION) {
 		flags.ok_as_delegate = 1;
 	}
+	if (userAccountControl & UF_TRUSTED_TO_AUTHENTICATE_FOR_DELEGATION) {
+		/*
+		 * this is confusing...
+		 *
+		 * UF_TRUSTED_FOR_DELEGATION
+		 * => ok_as_delegate
+		 *
+		 * and
+		 *
+		 * UF_TRUSTED_TO_AUTHENTICATE_FOR_DELEGATION
+		 * => trusted_for_delegation
+		 */
+		flags.trusted_for_delegation = 1;
+	}
 	if (!(userAccountControl & UF_NOT_DELEGATED)) {
 		flags.forwardable = 1;
 		flags.proxiable = 1;
@@ -1521,14 +1535,12 @@ krb5_error_code samba_kdc_nextkey(krb5_context context,
 /* Check if a given entry may delegate or do s4u2self to this target principal
  *
  * This is currently a very nasty hack - allowing only delegation to itself.
- *
- * This is shared between the constrained delegation and S4U2Self code.
  */
 krb5_error_code
-samba_kdc_check_identical_client_and_server(krb5_context context,
-					    struct samba_kdc_db_context *kdc_db_ctx,
-					    hdb_entry_ex *entry,
-					    krb5_const_principal target_principal)
+samba_kdc_check_s4u2self(krb5_context context,
+			 struct samba_kdc_db_context *kdc_db_ctx,
+			 hdb_entry_ex *entry,
+			 krb5_const_principal target_principal)
 {
 	krb5_error_code ret;
 	krb5_principal enterprise_prinicpal = NULL;
@@ -1541,11 +1553,11 @@ samba_kdc_check_identical_client_and_server(krb5_context context,
 		"objectSid", NULL
 	};
 
-	TALLOC_CTX *mem_ctx = talloc_named(kdc_db_ctx, 0, "samba_kdc_check_constrained_delegation");
+	TALLOC_CTX *mem_ctx = talloc_named(kdc_db_ctx, 0, "samba_kdc_check_s4u2self");
 
 	if (!mem_ctx) {
 		ret = ENOMEM;
-		krb5_set_error_message(context, ret, "samba_kdc_fetch: talloc_named() failed!");
+		krb5_set_error_message(context, ret, "samba_kdc_check_s4u2self: talloc_named() failed!");
 		return ret;
 	}
 
@@ -1553,7 +1565,7 @@ samba_kdc_check_identical_client_and_server(krb5_context context,
 		/* Need to reparse the enterprise principal to find the real target */
 		if (target_principal->name.name_string.len != 1) {
 			ret = KRB5_PARSE_MALFORMED;
-			krb5_set_error_message(context, ret, "samba_kdc_check_constrained_delegation: request for delegation to enterprise principal with wrong (%d) number of components",
+			krb5_set_error_message(context, ret, "samba_kdc_check_s4u2self: request for delegation to enterprise principal with wrong (%d) number of components",
 					       target_principal->name.name_string.len);
 			talloc_free(mem_ctx);
 			return ret;
@@ -1645,6 +1657,19 @@ samba_kdc_check_pkinit_ms_upn_match(krb5_context context,
 	return ret;
 }
 
+/*
+ * Check if a given entry may delegate to this target principal
+ * with S4U2Proxy.
+ */
+krb5_error_code
+samba_kdc_check_s4u2proxy(krb5_context context,
+			  struct samba_kdc_db_context *kdc_db_ctx,
+			  hdb_entry_ex *entry,
+			  krb5_const_principal target_principal)
+{
+	return KRB5KDC_ERR_BADOPTION;
+}
+
 NTSTATUS samba_kdc_setup_db_ctx(TALLOC_CTX *mem_ctx, struct samba_kdc_base_context *base_ctx,
 				struct samba_kdc_db_context **kdc_db_ctx_out)
 {
diff --git a/source4/kdc/db-glue.h b/source4/kdc/db-glue.h
index 4f1e06fa7a5..18d2c07de62 100644
--- a/source4/kdc/db-glue.h
+++ b/source4/kdc/db-glue.h
@@ -37,10 +37,10 @@ krb5_error_code samba_kdc_nextkey(krb5_context context,
 				  hdb_entry_ex *entry);
 
 krb5_error_code
-samba_kdc_check_identical_client_and_server(krb5_context context,
-					    struct samba_kdc_db_context *kdc_db_ctx,
-					    hdb_entry_ex *entry,
-					    krb5_const_principal target_principal);
+samba_kdc_check_s4u2self(krb5_context context,
+			 struct samba_kdc_db_context *kdc_db_ctx,
+			 hdb_entry_ex *entry,
+			 krb5_const_principal target_principal);
 
 krb5_error_code
 samba_kdc_check_pkinit_ms_upn_match(krb5_context context,
@@ -48,5 +48,11 @@ samba_kdc_check_pkinit_ms_upn_match(krb5_context context,
 				    hdb_entry_ex *entry,
 				    krb5_const_principal certificate_principal);
 
+krb5_error_code
+samba_kdc_check_s4u2proxy(krb5_context context,
+			  struct samba_kdc_db_context *kdc_db_ctx,
+			  hdb_entry_ex *entry,
+			  krb5_const_principal target_principal);
+
 NTSTATUS samba_kdc_setup_db_ctx(TALLOC_CTX *mem_ctx, struct samba_kdc_base_context *base_ctx,
 				struct samba_kdc_db_context **kdc_db_ctx_out);
diff --git a/source4/kdc/hdb-samba4.c b/source4/kdc/hdb-samba4.c
index 8511b2f27ba..f82712e2b29 100644
--- a/source4/kdc/hdb-samba4.c
+++ b/source4/kdc/hdb-samba4.c
@@ -121,7 +121,7 @@ static krb5_error_code hdb_samba4_destroy(krb5_context context, HDB *db)
 }
 
 static krb5_error_code
-hdb_samba4_check_identical_client_and_server(krb5_context context, HDB *db,
+hdb_samba4_check_constrained_delegation(krb5_context context, HDB *db,
 					hdb_entry_ex *entry,
 					krb5_const_principal target_principal)
 {
@@ -130,9 +130,9 @@ hdb_samba4_check_identical_client_and_server(krb5_context context, HDB *db,
 	kdc_db_ctx = talloc_get_type_abort(db->hdb_db,
 					   struct samba_kdc_db_context);
 
-	return samba_kdc_check_identical_client_and_server(context, kdc_db_ctx,
-							   entry,
-							   target_principal);
+	return samba_kdc_check_s4u2proxy(context, kdc_db_ctx,
+					 entry,
+					 target_principal);
 }
 
 static krb5_error_code
@@ -150,6 +150,21 @@ hdb_samba4_check_pkinit_ms_upn_match(krb5_context context, HDB *db,
 						   certificate_principal);
 }
 
+static krb5_error_code
+hdb_samba4_check_s4u2self(krb5_context context, HDB *db,
+			  hdb_entry_ex *entry,
+			  krb5_const_principal target_principal)
+{
+	struct samba_kdc_db_context *kdc_db_ctx;
+
+	kdc_db_ctx = talloc_get_type_abort(db->hdb_db,
+					   struct samba_kdc_db_context);
+
+	return samba_kdc_check_s4u2self(context, kdc_db_ctx,
+					entry,
+					target_principal);
+}
+
 /* This interface is to be called by the KDC and libnet_keytab_dump,
  * which is expecting Samba calling conventions.
  * It is also called by a wrapper (hdb_samba4_create) from the
@@ -197,9 +212,9 @@ NTSTATUS hdb_samba4_create_kdc(struct samba_kdc_base_context *base_ctx,
 	(*db)->hdb_destroy = hdb_samba4_destroy;
 
 	(*db)->hdb_auth_status = NULL;
-	(*db)->hdb_check_constrained_delegation = hdb_samba4_check_identical_client_and_server;
+	(*db)->hdb_check_constrained_delegation = hdb_samba4_check_constrained_delegation;
 	(*db)->hdb_check_pkinit_ms_upn_match = hdb_samba4_check_pkinit_ms_upn_match;
-	(*db)->hdb_check_s4u2self = hdb_samba4_check_identical_client_and_server;
+	(*db)->hdb_check_s4u2self = hdb_samba4_check_s4u2self;
 
 	return NT_STATUS_OK;
 }
diff --git a/source4/kdc/kdc.c b/source4/kdc/kdc.c
index 784b98b0727..07234080439 100644
--- a/source4/kdc/kdc.c
+++ b/source4/kdc/kdc.c
@@ -654,7 +654,7 @@ static NTSTATUS kdc_add_socket(struct kdc_server *kdc,
 						address, port,
 						&kdc_socket->local_address);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		return status;
 	}
 
@@ -685,7 +685,7 @@ static NTSTATUS kdc_add_socket(struct kdc_server *kdc,
 				     kdc_udp_socket,
 				     &kdc_udp_socket->dgram);
 	if (ret != 0) {
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		DEBUG(0,("Failed to bind to %s:%u UDP - %s\n",
 			 address, port, nt_errstr(status)));
 		return status;
@@ -729,29 +729,34 @@ static NTSTATUS kdc_startup_interfaces(struct kdc_server *kdc, struct loadparm_c
 		return NT_STATUS_INTERNAL_ERROR;
 	}
 
-	num_interfaces = iface_count(ifaces);
+	num_interfaces = iface_list_count(ifaces);
 
 	/* if we are allowing incoming packets from any address, then
 	   we need to bind to the wildcard address */
 	if (!lpcfg_bind_interfaces_only(lp_ctx)) {
-		if (kdc_port) {
-			status = kdc_add_socket(kdc, model_ops,
-						"kdc", "0.0.0.0", kdc_port,
-						kdc_process, false);
-			NT_STATUS_NOT_OK_RETURN(status);
-		}
-
-		if (kpasswd_port) {
-			status = kdc_add_socket(kdc, model_ops,
-						"kpasswd", "0.0.0.0", kpasswd_port,
-						kpasswdd_process, false);
-			NT_STATUS_NOT_OK_RETURN(status);
+		const char **wcard = iface_list_wildcard(kdc, lp_ctx);
+		NT_STATUS_HAVE_NO_MEMORY(wcard);
+		for (i=0; wcard[i]; i++) {
+			if (kdc_port) {
+				status = kdc_add_socket(kdc, model_ops,
+							"kdc", wcard[i], kdc_port,
+							kdc_process, false);
+				NT_STATUS_NOT_OK_RETURN(status);
+			}
+
+			if (kpasswd_port) {
+				status = kdc_add_socket(kdc, model_ops,
+							"kpasswd", wcard[i], kpasswd_port,
+							kpasswdd_process, false);
+				NT_STATUS_NOT_OK_RETURN(status);
+			}
 		}
+		talloc_free(wcard);
 		done_wildcard = true;
 	}
 
 	for (i=0; i<num_interfaces; i++) {
-		const char *address = talloc_strdup(tmp_ctx, iface_n_ip(ifaces, i));
+		const char *address = talloc_strdup(tmp_ctx, iface_list_n_ip(ifaces, i));
 
 		if (kdc_port) {
 			status = kdc_add_socket(kdc, model_ops,
@@ -895,9 +900,9 @@ static void kdc_task_init(struct task_server *task)
 		break;
 	}
 
-	load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
+	load_interface_list(task, task->lp_ctx, &ifaces);
 
-	if (iface_count(ifaces) == 0) {
+	if (iface_list_count(ifaces) == 0) {
 		task_server_terminate(task, "kdc: no network interfaces configured", false);
 		return;
 	}
diff --git a/source4/kdc/mit_samba.c b/source4/kdc/mit_samba.c
index dcabe39db67..50b5d1d2927 100644
--- a/source4/kdc/mit_samba.c
+++ b/source4/kdc/mit_samba.c
@@ -330,10 +330,10 @@ static int mit_samba_check_s4u2proxy(struct mit_samba_context *ctx,
 		return ret;
 	}
 
-	ret = samba_kdc_check_identical_client_and_server(ctx->context,
-							  ctx->db_ctx,
-							  entry,
-							  target_principal);
+	ret = samba_kdc_check_s4u2proxy(ctx->context,
+					ctx->db_ctx,
+					entry,
+					target_principal);
 
 	krb5_free_principal(ctx->context, target_principal);
 
diff --git a/source4/kdc/proxy.c b/source4/kdc/proxy.c
index 98db956f65e..324bfb8e2e1 100644
--- a/source4/kdc/proxy.c
+++ b/source4/kdc/proxy.c
@@ -556,7 +556,7 @@ static void kdc_tcp_proxy_read_pdu_done(struct tevent_req *subreq)
 	 */
 	state->out = data_blob_talloc(state, raw.data + 4, raw.length - 4);
 	if (state->out.length != raw.length - 4) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return;
 	}
 
diff --git a/source4/ldap_server/ldap_extended.c b/source4/ldap_server/ldap_extended.c
index f70b8084d7f..5cfa2d3b16f 100644
--- a/source4/ldap_server/ldap_extended.c
+++ b/source4/ldap_server/ldap_extended.c
@@ -82,7 +82,7 @@ static void ldapsrv_starttls_postprocess_done(struct tevent_req *subreq)
 				      conn, &conn->sockets.tls);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		NTSTATUS status = map_nt_error_from_unix(sys_errno);
+		NTSTATUS status = map_nt_error_from_unix_common(sys_errno);
 
 		DEBUG(1,("ldapsrv_starttls_postprocess_done: accept_tls_loop: "
 			 "tstream_tls_accept_recv() - %d:%s => %s",
diff --git a/source4/ldap_server/ldap_server.c b/source4/ldap_server/ldap_server.c
index adcf7bc71d7..411be294401 100644
--- a/source4/ldap_server/ldap_server.c
+++ b/source4/ldap_server/ldap_server.c
@@ -708,7 +708,7 @@ static struct tevent_req *ldapsrv_process_call_send(TALLOC_CTX *mem_ctx,
 	ok = tevent_queue_add(call_queue, ev, req,
 			      ldapsrv_process_call_trigger, NULL);
 	if (!ok) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return tevent_req_post(req, ev);
 	}
 
@@ -815,7 +815,7 @@ static NTSTATUS add_socket(struct task_server *task,
 
 	status = stream_setup_socket(task, task->event_ctx, lp_ctx,
 				     model_ops, &ldap_stream_nonpriv_ops,
-				     "ipv4", address, &port, 
+				     "ip", address, &port,
 				     lpcfg_socket_options(lp_ctx),
 				     ldap_service);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -830,7 +830,7 @@ static NTSTATUS add_socket(struct task_server *task,
 		status = stream_setup_socket(task, task->event_ctx, lp_ctx,
 					     model_ops,
 					     &ldap_stream_nonpriv_ops,
-					     "ipv4", address, &port, 
+					     "ip", address, &port,
 					     lpcfg_socket_options(lp_ctx),
 					     ldap_service);
 		if (!NT_STATUS_IS_OK(status)) {
@@ -852,7 +852,7 @@ static NTSTATUS add_socket(struct task_server *task,
 		status = stream_setup_socket(task, task->event_ctx, lp_ctx,
 					     model_ops,
 					     &ldap_stream_nonpriv_ops,
-					     "ipv4", address, &port, 
+					     "ip", address, &port,
 					     lpcfg_socket_options(lp_ctx),
 					     ldap_service);
 		if (!NT_STATUS_IS_OK(status)) {
@@ -866,7 +866,7 @@ static NTSTATUS add_socket(struct task_server *task,
 			status = stream_setup_socket(task, task->event_ctx, lp_ctx,
 						     model_ops,
 						     &ldap_stream_nonpriv_ops,
-						     "ipv4", address, &port,
+						     "ip", address, &port,
 						     lpcfg_socket_options(lp_ctx),
 						     ldap_service);
 			if (!NT_STATUS_IS_OK(status)) {
@@ -951,25 +951,34 @@ static void ldapsrv_task_init(struct task_server *task)
 		int num_interfaces;
 		int i;
 
-		load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
-		num_interfaces = iface_count(ifaces);
+		load_interface_list(task, task->lp_ctx, &ifaces);
+		num_interfaces = iface_list_count(ifaces);
 
 		/* We have been given an interfaces line, and been 
 		   told to only bind to those interfaces. Create a
 		   socket per interface and bind to only these.
 		*/
 		for(i = 0; i < num_interfaces; i++) {
-			const char *address = iface_n_ip(ifaces, i);
+			const char *address = iface_list_n_ip(ifaces, i);
 			status = add_socket(task, task->lp_ctx, model_ops, address, ldap_service);
 			if (!NT_STATUS_IS_OK(status)) goto failed;
 		}
 	} else {
-		status = add_socket(task, task->lp_ctx, model_ops,
-				    lpcfg_socket_address(task->lp_ctx), ldap_service);
-		if (!NT_STATUS_IS_OK(status)) goto failed;
+		const char **wcard;
+		int i;
+		wcard = iface_list_wildcard(task, task->lp_ctx);
+		if (wcard == NULL) {
+			DEBUG(0,("No wildcard addresses available\n"));
+			goto failed;
+		}
+		for (i=0; wcard[i]; i++) {
+			status = add_socket(task, task->lp_ctx, model_ops, wcard[i], ldap_service);
+			if (!NT_STATUS_IS_OK(status)) goto failed;
+		}
+		talloc_free(wcard);
 	}
 
-	ldapi_path = private_path(ldap_service, task->lp_ctx, "ldapi");
+	ldapi_path = lpcfg_private_path(ldap_service, task->lp_ctx, "ldapi");
 	if (!ldapi_path) {
 		goto failed;
 	}
@@ -986,7 +995,7 @@ static void ldapsrv_task_init(struct task_server *task)
 	}
 
 #ifdef WITH_LDAPI_PRIV_SOCKET
-	priv_dir = private_path(ldap_service, task->lp_ctx, "ldap_priv");
+	priv_dir = lpcfg_private_path(ldap_service, task->lp_ctx, "ldap_priv");
 	if (priv_dir == NULL) {
 		goto failed;
 	}
diff --git a/source4/lib/cmdline/popt_common.c b/source4/lib/cmdline/popt_common.c
index 16c6a1b81bb..af1e900092c 100644
--- a/source4/lib/cmdline/popt_common.c
+++ b/source4/lib/cmdline/popt_common.c
@@ -186,7 +186,7 @@ static void popt_common_callback(poptContext con,
 	}
 }
 
-struct poptOption popt_common_connection[] = {
+struct poptOption popt_common_connection4[] = {
 	{ NULL, 0, POPT_ARG_CALLBACK, (void *)popt_common_callback },
 	{ "name-resolve", 'R', POPT_ARG_STRING, NULL, 'R', "Use these name resolution services only", "NAME-RESOLVE-ORDER" },
 	{ "socket-options", 'O', POPT_ARG_STRING, NULL, 'O', "socket options to use", "SOCKETOPTIONS" },
@@ -199,7 +199,7 @@ struct poptOption popt_common_connection[] = {
 	{ NULL }
 };
 
-struct poptOption popt_common_samba[] = {
+struct poptOption popt_common_samba4[] = {
 	{ NULL, 0, POPT_ARG_CALLBACK|POPT_CBFLAG_PRE|POPT_CBFLAG_POST, (void *)popt_samba_callback },
 	{ "debuglevel",   'd', POPT_ARG_STRING, NULL, 'd', "Set debug level", "DEBUGLEVEL" },
 	{ "debug-stderr", 0, POPT_ARG_NONE, NULL, OPT_DEBUG_STDERR, "Send debug output to STDERR", NULL },
@@ -211,7 +211,7 @@ struct poptOption popt_common_samba[] = {
 	{ NULL }
 };
 
-struct poptOption popt_common_version[] = {
+struct poptOption popt_common_version4[] = {
 	{ NULL, 0, POPT_ARG_CALLBACK, (void *)popt_version_callback },
 	{ "version", 'V', POPT_ARG_NONE, NULL, 'V', "Print version" },
 	{ NULL }
diff --git a/source4/lib/cmdline/popt_common.h b/source4/lib/cmdline/popt_common.h
index 2f4ab2c1786..8aed4d04961 100644
--- a/source4/lib/cmdline/popt_common.h
+++ b/source4/lib/cmdline/popt_common.h
@@ -23,19 +23,19 @@
 #include <popt.h>
 
 /* Common popt structures */
-extern struct poptOption popt_common_samba[];
-extern struct poptOption popt_common_connection[];
-extern struct poptOption popt_common_version[];
-extern struct poptOption popt_common_credentials[];
+extern struct poptOption popt_common_samba4[];
+extern struct poptOption popt_common_connection4[];
+extern struct poptOption popt_common_version4[];
+extern struct poptOption popt_common_credentials4[];
 
 #ifndef POPT_TABLEEND
 #define POPT_TABLEEND { NULL, '\0', 0, 0, 0, NULL, NULL }
 #endif
 
-#define POPT_COMMON_SAMBA { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_samba, 0, "Common samba options:", NULL },
-#define POPT_COMMON_CONNECTION { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_connection, 0, "Connection options:", NULL },
-#define POPT_COMMON_VERSION { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_version, 0, "Common samba options:", NULL },
-#define POPT_COMMON_CREDENTIALS { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_credentials, 0, "Authentication options:", NULL },
+#define POPT_COMMON_SAMBA { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_samba4, 0, "Common samba options:", NULL },
+#define POPT_COMMON_CONNECTION { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_connection4, 0, "Connection options:", NULL },
+#define POPT_COMMON_VERSION { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_version4, 0, "Common samba options:", NULL },
+#define POPT_COMMON_CREDENTIALS { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_common_credentials4, 0, "Authentication options:", NULL },
 
 extern struct cli_credentials *cmdline_credentials;
 extern struct loadparm_context *cmdline_lp_ctx;
diff --git a/source4/lib/cmdline/popt_credentials.c b/source4/lib/cmdline/popt_credentials.c
index 6dcef3f22b4..497d2a7228c 100644
--- a/source4/lib/cmdline/popt_credentials.c
+++ b/source4/lib/cmdline/popt_credentials.c
@@ -168,7 +168,7 @@ static void popt_common_credentials_callback(poptContext con,
 
 
 
-struct poptOption popt_common_credentials[] = {
+struct poptOption popt_common_credentials4[] = {
 	{ NULL, 0, POPT_ARG_CALLBACK|POPT_CBFLAG_PRE|POPT_CBFLAG_POST, (void *)popt_common_credentials_callback },
 	{ "user", 'U', POPT_ARG_STRING, NULL, 'U', "Set the network username", "[DOMAIN/]USERNAME[%PASSWORD]" },
 	{ "no-pass", 'N', POPT_ARG_NONE, &dont_ask, 'N', "Don't ask for a password" },
diff --git a/source4/lib/events/events.h b/source4/lib/events/events.h
index f66698838d7..546a5e51588 100644
--- a/source4/lib/events/events.h
+++ b/source4/lib/events/events.h
@@ -1,6 +1,8 @@
 #ifndef __LIB_EVENTS_H__
 #define __LIB_EVENTS_H__
+#ifndef TEVENT_COMPAT_DEFINES
 #define TEVENT_COMPAT_DEFINES 1
+#endif
 #include <tevent.h>
 struct tevent_context *s4_event_context_init(TALLOC_CTX *mem_ctx);
 struct tevent_context *event_context_find(TALLOC_CTX *mem_ctx) _DEPRECATED_;
diff --git a/source4/lib/ldb-samba/ldb_wrap.c b/source4/lib/ldb-samba/ldb_wrap.c
index 7dcf514e23b..66213bf2881 100644
--- a/source4/lib/ldb-samba/ldb_wrap.c
+++ b/source4/lib/ldb-samba/ldb_wrap.c
@@ -35,7 +35,7 @@
 #include "dsdb/samdb/samdb.h"
 #include "param/param.h"
 #include "../lib/util/dlinklist.h"
-#include <tdb.h>
+#include "../lib/tdb_compat/tdb_compat.h"
 
 /*
   this is used to catch debug messages from ldb
@@ -126,10 +126,7 @@ char *wrap_casefold(void *context, void *mem_ctx, const char *s, size_t n)
 		return NULL;
 	}
 
-	ldb_set_modules_dir(ldb,
-			    talloc_asprintf(ldb,
-					    "%s/ldb",
-					    lpcfg_modulesdir(lp_ctx)));
+	ldb_set_modules_dir(ldb, modules_path(ldb, "ldb"));
 
 	ldb_set_debug(ldb, ldb_wrap_debug, NULL);
 
@@ -177,7 +174,7 @@ char *wrap_casefold(void *context, void *mem_ctx, const char *s, size_t n)
 				   struct loadparm_context *lp_ctx,
 				   struct auth_session_info *session_info,
 				   struct cli_credentials *credentials,
-				   int flags)
+				   unsigned int flags)
 {
 	struct ldb_wrap *w;
 	/* see if we can re-use an existing ldb */
@@ -195,7 +192,7 @@ char *wrap_casefold(void *context, void *mem_ctx, const char *s, size_t n)
 }
 
 int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
-		      const char *url, int flags)
+		      const char *url, unsigned int flags)
 {
 	int ret;
 	char *real_url = NULL;
@@ -209,7 +206,7 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 		flags |= LDB_FLG_ENABLE_TRACING;
 	}
 
-	real_url = private_path(ldb, lp_ctx, url);
+	real_url = lpcfg_private_path(ldb, lp_ctx, url);
 	if (real_url == NULL) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
@@ -230,7 +227,7 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 		   struct loadparm_context *lp_ctx,
 		   struct auth_session_info *session_info,
 		   struct cli_credentials *credentials,
-		   int flags,
+		   unsigned int flags,
 		   struct ldb_context *ldb)
 {
 	struct ldb_wrap *w;
diff --git a/source4/lib/ldb-samba/ldb_wrap.h b/source4/lib/ldb-samba/ldb_wrap.h
index 4d2539fff51..aa7ccb3a234 100644
--- a/source4/lib/ldb-samba/ldb_wrap.h
+++ b/source4/lib/ldb-samba/ldb_wrap.h
@@ -53,18 +53,18 @@ struct ldb_context *ldb_wrap_find(const char *url,
 								  struct loadparm_context *lp_ctx,
 								  struct auth_session_info *session_info,
 								  struct cli_credentials *credentials,
-								  int flags);
+								  unsigned int flags);
 bool ldb_wrap_add(const char *url, struct tevent_context *ev,
 				  struct loadparm_context *lp_ctx,
 				  struct auth_session_info *session_info,
 				  struct cli_credentials *credentials,
-				  int flags,
+				  unsigned int flags,
 				  struct ldb_context *ldb);
 char *ldb_relative_path(struct ldb_context *ldb,
 				 TALLOC_CTX *mem_ctx,
 				 const char *name);
 
 int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
-		      const char *url, int flags);
+		      const char *url, unsigned int flags);
 
 #endif /* _LDB_WRAP_H_ */
diff --git a/source4/lib/ldb-samba/samba_extensions.c b/source4/lib/ldb-samba/samba_extensions.c
index 63b0f3df919..be9f36a5a7d 100644
--- a/source4/lib/ldb-samba/samba_extensions.c
+++ b/source4/lib/ldb-samba/samba_extensions.c
@@ -82,7 +82,7 @@ static int extensions_hook(struct ldb_context *ldb, enum ldb_module_hook_type t)
 		if (r != LDB_SUCCESS) {
 			return ldb_operr(ldb);
 		}
-		gensec_init(cmdline_lp_ctx);
+		gensec_init();
 
 		if (ldb_set_opaque(ldb, "sessionInfo", system_session(cmdline_lp_ctx))) {
 			return ldb_operr(ldb);
diff --git a/source4/lib/ldb-samba/wscript_build b/source4/lib/ldb-samba/wscript_build
index a8d4df2ce0f..2e1cacba649 100644
--- a/source4/lib/ldb-samba/wscript_build
+++ b/source4/lib/ldb-samba/wscript_build
@@ -8,7 +8,7 @@ bld.SAMBA_LIBRARY('ldbsamba',
                   source='ldif_handlers.c',
                   autoproto='ldif_handlers_proto.h',
                   public_deps='ldb',
-                  deps='security ndr NDR_DRSBLOBS NDR_DNSP ldbwrap samdb-common SAMDB_SCHEMA tdb pyldb-util',
+                  deps='security ndr NDR_DRSBLOBS NDR_DNSP ldbwrap samdb-common SAMDB_SCHEMA tdb_compat pyldb-util errors',
                   private_library=True
                   )
 
diff --git a/source4/lib/ldb/common/ldb_controls.c b/source4/lib/ldb/common/ldb_controls.c
index 5048b6deac3..b3ef2434932 100644
--- a/source4/lib/ldb/common/ldb_controls.c
+++ b/source4/lib/ldb/common/ldb_controls.c
@@ -482,10 +482,10 @@ struct ldb_control *ldb_parse_control_from_string(struct ldb_context *ldb, TALLO
 		}
 
 		/* w2k3 seems to ignore the parameter,
-			* but w2k sends a wrong cookie when this value is to small
-			* this would cause looping forever, while getting
-			* the same data and same cookie forever
-			*/
+		 * but w2k sends a wrong cookie when this value is to small
+		 * this would cause looping forever, while getting
+		 * the same data and same cookie forever
+		 */
 		if (max_attrs == 0) max_attrs = 0x0FFFFFFF;
 
 		ctrl->oid = LDB_CONTROL_DIRSYNC_OID;
diff --git a/source4/lib/ldb/common/ldb_msg.c b/source4/lib/ldb/common/ldb_msg.c
index 9c5a279b8a7..28c414e6b23 100644
--- a/source4/lib/ldb/common/ldb_msg.c
+++ b/source4/lib/ldb/common/ldb_msg.c
@@ -130,7 +130,6 @@ static int _ldb_msg_add_el(struct ldb_message *msg,
 	els = talloc_realloc(msg, msg->elements,
 			     struct ldb_message_element, msg->num_elements + 1);
 	if (!els) {
-		errno = ENOMEM;
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -164,7 +163,6 @@ int ldb_msg_add_empty(struct ldb_message *msg,
 	el->flags = flags;
 	el->name = talloc_strdup(msg->elements, attr_name);
 	if (!el->name) {
-		errno = ENOMEM;
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -229,7 +227,6 @@ int ldb_msg_add_value(struct ldb_message *msg,
 	vals = talloc_realloc(msg->elements, el->values, struct ldb_val,
 			      el->num_values+1);
 	if (!vals) {
-		errno = ENOMEM;
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 	el->values = vals;
diff --git a/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c b/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
index e5b43fd0c9c..223868a6c0a 100644
--- a/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
+++ b/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
@@ -1136,7 +1136,7 @@ static int lsql_modify(struct lsql_context *ctx)
 	for (i = 0; i < msg->num_elements; i++) {
 		const struct ldb_message_element *el = &msg->elements[i];
 		const struct ldb_schema_attribute *a;
-		int flags = el->flags & LDB_FLAG_MOD_MASK;
+		unsigned int flags = el->flags & LDB_FLAG_MOD_MASK;
 		char *attr;
 		char *mod;
 		unsigned int j;
@@ -1596,7 +1596,8 @@ static const struct ldb_module_ops lsqlite3_ops = {
  */
 
 static int initialize(struct lsqlite3_private *lsqlite3,
-		      struct ldb_context *ldb, const char *url, int flags)
+		      struct ldb_context *ldb, const char *url,
+		      unsigned int flags)
 {
 	TALLOC_CTX *local_ctx;
         long long queryInt;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_cache.c b/source4/lib/ldb/ldb_tdb/ldb_cache.c
index 697f7427a44..e54ceaaa98a 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_cache.c
@@ -345,9 +345,6 @@ int ltdb_cache_load(struct ldb_module *module)
 		ltdb->check_base = false;
 	}
 
-	talloc_free(ltdb->cache->last_attribute.name);
-	memset(&ltdb->cache->last_attribute, 0, sizeof(ltdb->cache->last_attribute));
-
 	talloc_free(ltdb->cache->indexlist);
 	ltdb_attributes_unload(module); /* calls internally "talloc_free" */
 
diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c
index 02e4acbbdef..24cc93feb94 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_index.c
@@ -155,7 +155,7 @@ static int ltdb_dn_list_load(struct ldb_module *module,
 	key.dptr = discard_const_p(unsigned char, ldb_dn_get_linearized(dn));
 	key.dsize = strlen((char *)key.dptr);
 
-	rec = tdb_fetch(ltdb->idxptr->itdb, key);
+	rec = tdb_fetch_compat(ltdb->idxptr->itdb, key);
 	if (rec.dptr == NULL) {
 		goto normal_index;
 	}
@@ -261,7 +261,7 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
 	}
 
 	if (ltdb->idxptr->itdb == NULL) {
-		ltdb->idxptr->itdb = tdb_open(NULL, 1000, TDB_INTERNAL, O_RDWR, 0);
+		ltdb->idxptr->itdb = tdb_open_compat(NULL, 1000, TDB_INTERNAL, O_RDWR, 0, NULL, NULL);
 		if (ltdb->idxptr->itdb == NULL) {
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
@@ -270,7 +270,7 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
 	key.dptr = discard_const_p(unsigned char, ldb_dn_get_linearized(dn));
 	key.dsize = strlen((char *)key.dptr);
 
-	rec = tdb_fetch(ltdb->idxptr->itdb, key);
+	rec = tdb_fetch_compat(ltdb->idxptr->itdb, key);
 	if (rec.dptr != NULL) {
 		list2 = ltdb_index_idxptr(module, rec, false);
 		if (list2 == NULL) {
@@ -294,7 +294,7 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
 	rec.dsize = sizeof(void *);
 
 	ret = tdb_store(ltdb->idxptr->itdb, key, rec, TDB_INSERT);
-	if (ret == -1) {
+	if (ret != 0) {
 		return ltdb_err_map(tdb_error(ltdb->idxptr->itdb));
 	}
 	return LDB_SUCCESS;
@@ -1569,7 +1569,7 @@ int ltdb_reindex(struct ldb_module *module)
 	 * putting NULL entries in the in-memory tdb
 	 */
 	ret = tdb_traverse(ltdb->tdb, delete_index, module);
-	if (ret == -1) {
+	if (ret < 0) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -1583,7 +1583,7 @@ int ltdb_reindex(struct ldb_module *module)
 
 	/* now traverse adding any indexes for normal LDB records */
 	ret = tdb_traverse(ltdb->tdb, re_index, &ctx);
-	if (ret == -1) {
+	if (ret < 0) {
 		struct ldb_context *ldb = ldb_module_get_ctx(module);
 		ldb_asprintf_errstring(ldb, "reindexing traverse failed: %s", ldb_errstring(ldb));
 		return LDB_ERR_OPERATIONS_ERROR;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_pack.c b/source4/lib/ldb/ldb_tdb/ldb_pack.c
index 8ab07cd347c..7c13065aeef 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_pack.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_pack.c
@@ -74,7 +74,7 @@ static int attribute_storable_values(const struct ldb_message_element *el)
 */
 int ltdb_pack_data(struct ldb_module *module,
 		   const struct ldb_message *message,
-		   struct TDB_DATA *data)
+		   TDB_DATA *data)
 {
 	struct ldb_context *ldb;
 	unsigned int i, j, real_elements=0;
@@ -155,7 +155,7 @@ int ltdb_pack_data(struct ldb_module *module,
   Free with ltdb_unpack_data_free()
 */
 int ltdb_unpack_data(struct ldb_module *module,
-		     const struct TDB_DATA *data,
+		     const TDB_DATA *data,
 		     struct ldb_message *message)
 {
 	struct ldb_context *ldb;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_search.c b/source4/lib/ldb/ldb_tdb/ldb_search.c
index a49751de154..46e2d749984 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_search.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_search.c
@@ -32,6 +32,7 @@
  */
 
 #include "ldb_tdb.h"
+#include <lib/tdb_compat/tdb_compat.h>
 
 /*
   add one element to a message
@@ -223,7 +224,7 @@ static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	tdb_data = tdb_fetch(ltdb->tdb, tdb_key);
+	tdb_data = tdb_fetch_compat(ltdb->tdb, tdb_key);
 	talloc_free(tdb_key.dptr);
 	if (!tdb_data.dptr) {
 		return LDB_ERR_NO_SUCH_OBJECT;
@@ -255,7 +256,7 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	tdb_data = tdb_fetch(ltdb->tdb, tdb_key);
+	tdb_data = tdb_fetch_compat(ltdb->tdb, tdb_key);
 	talloc_free(tdb_key.dptr);
 	if (!tdb_data.dptr) {
 		return LDB_ERR_NO_SUCH_OBJECT;
@@ -479,7 +480,7 @@ static int ltdb_search_full(struct ltdb_context *ctx)
 		ret = tdb_traverse_read(ltdb->tdb, search_func, ctx);
 	}
 
-	if (ret == -1) {
+	if (ret < 0) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
index 2f7f2220860..0d4be491235 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -50,6 +50,7 @@
  */
 
 #include "ldb_tdb.h"
+#include <lib/tdb_compat/tdb_compat.h>
 
 
 /*
@@ -67,9 +68,13 @@ int ltdb_err_map(enum TDB_ERROR tdb_code)
 	case TDB_ERR_IO:
 		return LDB_ERR_PROTOCOL_ERROR;
 	case TDB_ERR_LOCK:
+#ifndef BUILD_TDB2
 	case TDB_ERR_NOLOCK:
+#endif
 		return LDB_ERR_BUSY;
+#ifndef BUILD_TDB2
 	case TDB_ERR_LOCK_TIMEOUT:
+#endif
 		return LDB_ERR_TIME_LIMIT_EXCEEDED;
 	case TDB_ERR_EXISTS:
 		return LDB_ERR_ENTRY_ALREADY_EXISTS;
@@ -110,7 +115,8 @@ int ltdb_unlock_read(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
-		return tdb_unlockall_read(ltdb->tdb);
+		tdb_unlockall_read(ltdb->tdb);
+		return 0;
 	}
 	ltdb->read_lock_count--;
 	return 0;
@@ -124,7 +130,7 @@ int ltdb_unlock_read(struct ldb_module *module)
   note that the key for a record can depend on whether the
   dn refers to a case sensitive index record or not
 */
-struct TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn)
+TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn)
 {
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 	TDB_DATA key;
@@ -263,7 +269,7 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
 	}
 
 	ret = tdb_store(ltdb->tdb, tdb_key, tdb_data, flgs);
-	if (ret == -1) {
+	if (ret != 0) {
 		ret = ltdb_err_map(tdb_error(ltdb->tdb));
 		goto done;
 	}
@@ -653,7 +659,7 @@ int ltdb_modify_internal(struct ldb_module *module,
 		return LDB_ERR_OTHER;
 	}
 
-	tdb_data = tdb_fetch(ltdb->tdb, tdb_key);
+	tdb_data = tdb_fetch_compat(ltdb->tdb, tdb_key);
 	if (!tdb_data.dptr) {
 		talloc_free(tdb_key.dptr);
 		return ltdb_err_map(tdb_error(ltdb->tdb));
@@ -1072,10 +1078,7 @@ static int ltdb_del_trans(struct ldb_module *module)
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
 
-	if (tdb_transaction_cancel(ltdb->tdb) != 0) {
-		return ltdb_err_map(tdb_error(ltdb->tdb));
-	}
-
+	tdb_transaction_cancel(ltdb->tdb);
 	return LDB_SUCCESS;
 }
 
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
index 33313b00da9..96ad43fbd6c 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -1,7 +1,7 @@
 #include "replace.h"
 #include "system/filesys.h"
 #include "system/time.h"
-#include "tdb.h"
+#include "tdb_compat.h"
 #include "ldb_module.h"
 
 /* this private structure is used by the ltdb backend in the
@@ -21,11 +21,6 @@ struct ltdb_private {
 		struct ldb_message *attributes;
 		bool one_level_indexes;
 		bool attribute_indexes;
-
-		struct {
-			char *name;
-			int flags;
-		} last_attribute;
 	} *cache;
 
 	int in_transaction;
@@ -107,11 +102,11 @@ int ltdb_index_transaction_cancel(struct ldb_module *module);
 
 int ltdb_pack_data(struct ldb_module *module,
 		   const struct ldb_message *message,
-		   struct TDB_DATA *data);
+		   TDB_DATA *data);
 void ltdb_unpack_data_free(struct ldb_module *module,
 			   struct ldb_message *message);
 int ltdb_unpack_data(struct ldb_module *module,
-		     const struct TDB_DATA *data,
+		     const TDB_DATA *data,
 		     struct ldb_message *message);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_search.c  */
@@ -132,7 +127,7 @@ int ltdb_search(struct ltdb_context *ctx);
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_tdb.c  */
 int ltdb_lock_read(struct ldb_module *module);
 int ltdb_unlock_read(struct ldb_module *module);
-struct TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn);
+TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn);
 int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
 int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
 int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn);
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c b/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
index b9f3e79f205..16a037a6c33 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
@@ -24,29 +24,30 @@
 #include "ldb_tdb.h"
 #include "dlinklist.h"
 
-/*
-  the purpose of this code is to work around the braindead posix locking
-  rules, to allow us to have a ldb open more than once while allowing 
-  locking to work
-*/
-
-struct ltdb_wrap {
-	struct ltdb_wrap *next, *prev;
-	struct tdb_context *tdb;
-	dev_t device;
-	ino_t inode;
-};
-
-static struct ltdb_wrap *tdb_list;
-
-/* destroy the last connection to a tdb */
-static int ltdb_wrap_destructor(struct ltdb_wrap *w)
+/* FIXME: TDB2 does this internally, so no need to wrap multiple opens! */
+#if BUILD_TDB2
+static void ltdb_log_fn(struct tdb_context *tdb,
+			enum tdb_log_level level,
+			const char *message,
+			struct ldb_context *ldb)
 {
-	tdb_close(w->tdb);
-	DLIST_REMOVE(tdb_list, w);
-	return 0;
-}				 
+	enum ldb_debug_level ldb_level;
+	const char *name = tdb_name(tdb);
+
+	switch (level) {
+	case TDB_LOG_WARNING:
+		ldb_level = LDB_DEBUG_WARNING;
+	case TDB_LOG_USE_ERROR:
+	case TDB_LOG_ERROR:
+		ldb_level = LDB_DEBUG_FATAL;
+		break;
+	default:
+		ldb_level = LDB_DEBUG_FATAL;
+	}
 
+	ldb_debug(ldb, ldb_level, "ltdb: tdb(%s): %s", name, message);
+}
+#else /* !TDB2 */
 static void ltdb_log_fn(struct tdb_context *tdb, enum tdb_debug_level level, const char *fmt, ...) PRINTF_ATTRIBUTE(3, 4);
 static void ltdb_log_fn(struct tdb_context *tdb, enum tdb_debug_level level, const char *fmt, ...)
 {
@@ -83,6 +84,32 @@ static void ltdb_log_fn(struct tdb_context *tdb, enum tdb_debug_level level, con
 	ldb_debug(ldb, ldb_level, "ltdb: tdb(%s): %s", name, message);
 	talloc_free(message);
 }
+#endif
+
+/*
+  the purpose of this code is to work around the braindead posix locking
+  rules, to allow us to have a ldb open more than once while allowing
+  locking to work
+
+  TDB2 handles multiple opens, so we don't have this problem there.
+*/
+
+struct ltdb_wrap {
+	struct ltdb_wrap *next, *prev;
+	struct tdb_context *tdb;
+	dev_t device;
+	ino_t inode;
+};
+
+static struct ltdb_wrap *tdb_list;
+
+/* destroy the last connection to a tdb */
+static int ltdb_wrap_destructor(struct ltdb_wrap *w)
+{
+	tdb_close(w->tdb);
+	DLIST_REMOVE(tdb_list, w);
+	return 0;
+}
 
 /*
   wrapped connection to a tdb database. The caller should _not_ free
@@ -98,10 +125,6 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
 {
 	struct ltdb_wrap *w;
 	struct stat st;
-	struct tdb_logging_context log_ctx;
-
-	log_ctx.log_fn = ltdb_log_fn;
-	log_ctx.log_private = ldb;
 
 	if (stat(path, &st) == 0) {
 		for (w=tdb_list;w;w=w->next) {
@@ -119,7 +142,7 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
-	w->tdb = tdb_open_ex(path, hash_size, tdb_flags, open_flags, mode, &log_ctx, NULL);
+	w->tdb = tdb_open_compat(path, hash_size, tdb_flags, open_flags, mode, ltdb_log_fn, ldb);
 	if (w->tdb == NULL) {
 		talloc_free(w);
 		return NULL;
@@ -140,4 +163,3 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
 	
 	return w->tdb;
 }
-
diff --git a/source4/lib/ldb/pyldb.c b/source4/lib/ldb/pyldb.c
index 5fcc5a64b69..e2a2e7180e1 100644
--- a/source4/lib/ldb/pyldb.c
+++ b/source4/lib/ldb/pyldb.c
@@ -7,6 +7,8 @@
    Copyright (C) 2006 Simo Sorce <idra@samba.org>
    Copyright (C) 2007-2010 Jelmer Vernooij <jelmer@samba.org>
    Copyright (C) 2009-2010 Matthias Dieter Wallnöfer
+   Copyright (C) 2009-2011 Andrew Tridgell
+   Copyright (C) 2009-2011 Andrew Bartlett
 
     ** NOTE! The following LGPL license applies to the ldb
     ** library. This does NOT imply that all of Samba is released
@@ -48,7 +50,7 @@ static PyObject *PyLdbModule_FromModule(struct ldb_module *mod);
 static struct ldb_message_element *PyObject_AsMessageElement(
 						      TALLOC_CTX *mem_ctx,
 						      PyObject *set_obj,
-						      int flags,
+						      unsigned int flags,
 						      const char *attr_name);
 
 /* There's no Py_ssize_t in 2.4, apparently */
@@ -384,6 +386,62 @@ static PyObject *py_ldb_dn_canonical_ex_str(PyLdbDnObject *self)
 	return PyString_FromString(ldb_dn_canonical_ex_string(self->dn, self->dn));
 }
 
+static PyObject *py_ldb_dn_extended_str(PyLdbDnObject *self, PyObject *args, PyObject *kwargs)
+{
+	const char * const kwnames[] = { "mode", NULL };
+	int mode = 1;
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i",
+					 discard_const_p(char *, kwnames),
+					 &mode))
+		return NULL;
+	return PyString_FromString(ldb_dn_get_extended_linearized(self->dn, self->dn, mode));
+}
+
+static PyObject *py_ldb_dn_get_extended_component(PyLdbDnObject *self, PyObject *args)
+{
+	char *name;
+	const struct ldb_val *val;
+
+	if (!PyArg_ParseTuple(args, "s", &name))
+		return NULL;
+	val = ldb_dn_get_extended_component(self->dn, name);
+	if (val == NULL) {
+		Py_RETURN_NONE;
+	}
+
+	return PyString_FromStringAndSize((const char *)val->data, val->length);
+}
+
+static PyObject *py_ldb_dn_set_extended_component(PyLdbDnObject *self, PyObject *args)
+{
+	char *name;
+	PyObject *value;
+	int err;
+
+	if (!PyArg_ParseTuple(args, "sO", &name, &value))
+		return NULL;
+
+	if (value == Py_None) {
+		err = ldb_dn_set_extended_component(self->dn, name, NULL);
+	} else {
+		struct ldb_val val;
+		if (!PyString_Check(value)) {
+			PyErr_SetString(PyExc_TypeError, "Expected a string argument");
+			return NULL;
+		}
+		val.data = (uint8_t *)PyString_AsString(value);
+		val.length = PyString_Size(value);
+		err = ldb_dn_set_extended_component(self->dn, name, &val);
+	}
+
+	if (err != LDB_SUCCESS) {
+		PyErr_SetString(PyExc_TypeError, "Failed to set extended component");
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
 static PyObject *py_ldb_dn_repr(PyLdbDnObject *self)
 {
 	return PyString_FromFormat("Dn(%s)", PyObject_REPR(PyString_FromString(ldb_dn_get_linearized(self->dn))));
@@ -485,6 +543,9 @@ static PyMethodDef py_ldb_dn_methods[] = {
 	{ "canonical_ex_str", (PyCFunction)py_ldb_dn_canonical_ex_str, METH_NOARGS,
 		"S.canonical_ex_str() -> string\n"
 		"Canonical version of this DN (like a posix path, with terminating newline)." },
+	{ "extended_str", (PyCFunction)py_ldb_dn_extended_str, METH_VARARGS | METH_KEYWORDS,
+		"S.extended_str(mode=1) -> string\n"
+		"Extended version of this DN" },
 	{ "parent", (PyCFunction)py_ldb_dn_get_parent, METH_NOARGS,
    		"S.parent() -> dn\n"
 		"Get the parent for this DN." },
@@ -497,6 +558,12 @@ static PyMethodDef py_ldb_dn_methods[] = {
 	{ "check_special", (PyCFunction)py_ldb_dn_check_special, METH_VARARGS,
 		"S.check_special(name) -> bool\n\n"
 		"Check if name is a special DN name"},
+	{ "get_extended_component", (PyCFunction)py_ldb_dn_get_extended_component, METH_VARARGS,
+		"S.get_extended_component(name) -> string\n\n"
+		"returns a DN extended component as a binary string"},
+	{ "set_extended_component", (PyCFunction)py_ldb_dn_set_extended_component, METH_VARARGS,
+		"S.set_extended_component(name, value) -> string\n\n"
+		"set a DN extended component as a binary string"},
 	{ NULL }
 };
 
@@ -737,11 +804,11 @@ static int py_ldb_init(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 	char *url = NULL;
 	PyObject *py_options = Py_None;
 	const char **options;
-	int flags = 0;
+	unsigned int flags = 0;
 	int ret;
 	struct ldb_context *ldb;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ziO:Ldb.__init__",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|zIO:Ldb.__init__",
 					 discard_const_p(char *, kwnames),
 					 &url, &flags, &py_options))
 		return -1;
@@ -792,13 +859,13 @@ static PyObject *py_ldb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs
 static PyObject *py_ldb_connect(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	char *url;
-	int flags = 0;
+	unsigned int flags = 0;
 	PyObject *py_options = Py_None;
 	int ret;
 	const char **options;
 	const char * const kwnames[] = { "url", "flags", "options", NULL };
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ziO",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|zIO",
 					 discard_const_p(char *, kwnames),
 					 &url, &flags, &py_options))
 		return NULL;
@@ -819,7 +886,7 @@ static PyObject *py_ldb_connect(PyLdbObject *self, PyObject *args, PyObject *kwa
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_msg;
 	PyObject *py_controls = Py_None;
@@ -829,8 +896,12 @@ static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args)
 	struct ldb_message *msg;
 	int ret;
 	TALLOC_CTX *mem_ctx;
+	bool validate=true;
+	const char * const kwnames[] = { "message", "controls", "validate", NULL };
 
-	if (!PyArg_ParseTuple(args, "O|O", &py_msg, &py_controls))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Ob",
+					 discard_const_p(char *, kwnames),
+					 &py_msg, &py_controls, &validate))
 		return NULL;
 
 	mem_ctx = talloc_new(NULL);
@@ -855,11 +926,13 @@ static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args)
 	}
 	msg = PyLdbMessage_AsMessage(py_msg);
 
-	ret = ldb_msg_sanity_check(ldb_ctx, msg);
-	if (ret != LDB_SUCCESS) {
-		PyErr_LDB_ERROR_IS_ERR_RAISE(PyExc_LdbError, ret, ldb_ctx);
-		talloc_free(mem_ctx);
-		return NULL;
+	if (validate) {
+		ret = ldb_msg_sanity_check(ldb_ctx, msg);
+		if (ret != LDB_SUCCESS) {
+			PyErr_LDB_ERROR_IS_ERR_RAISE(PyExc_LdbError, ret, ldb_ctx);
+			talloc_free(mem_ctx);
+			return NULL;
+		}
 	}
 
 	ret = ldb_build_mod_req(&req, ldb_ctx, mem_ctx, msg, parsed_controls,
@@ -958,7 +1031,7 @@ static struct ldb_message *PyDict_AsMessage(TALLOC_CTX *mem_ctx,
 	return msg;
 }
 
-static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_obj;
 	int ret;
@@ -968,8 +1041,11 @@ static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args)
 	PyObject *py_controls = Py_None;
 	TALLOC_CTX *mem_ctx;
 	struct ldb_control **parsed_controls;
+	const char * const kwnames[] = { "message", "controls", NULL };
 
-	if (!PyArg_ParseTuple(args, "O|O", &py_obj, &py_controls ))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O",
+					 discard_const_p(char *, kwnames),
+					 &py_obj, &py_controls))
 		return NULL;
 
 	mem_ctx = talloc_new(NULL);
@@ -1047,7 +1123,7 @@ static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args)
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_dn;
 	struct ldb_dn *dn;
@@ -1057,8 +1133,11 @@ static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args)
 	PyObject *py_controls = Py_None;
 	TALLOC_CTX *mem_ctx;
 	struct ldb_control **parsed_controls;
+	const char * const kwnames[] = { "dn", "controls", NULL };
 
-	if (!PyArg_ParseTuple(args, "O|O", &py_dn, &py_controls))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O",
+					 discard_const_p(char *, kwnames),
+					 &py_dn, &py_controls))
 		return NULL;
 
 	mem_ctx = talloc_new(NULL);
@@ -1119,7 +1198,7 @@ static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args)
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_ldb_rename(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_rename(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_dn1, *py_dn2;
 	struct ldb_dn *dn1, *dn2;
@@ -1130,10 +1209,13 @@ static PyObject *py_ldb_rename(PyLdbObject *self, PyObject *args)
 	struct ldb_control **parsed_controls;
 	struct ldb_context *ldb_ctx;
 	struct ldb_request *req;
+	const char * const kwnames[] = { "dn1", "dn2", "controls", NULL };
 
 	ldb_ctx = PyLdb_AsLdbContext(self);
 
-	if (!PyArg_ParseTuple(args, "OO|O", &py_dn1, &py_dn2, &py_controls))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|O",
+					 discard_const_p(char *, kwnames),
+					 &py_dn1, &py_dn2, &py_controls))
 		return NULL;
 
 
@@ -1586,17 +1668,17 @@ static PyMethodDef py_ldb_methods[] = {
 	{ "connect", (PyCFunction)py_ldb_connect, METH_VARARGS|METH_KEYWORDS, 
 		"S.connect(url, flags=0, options=None) -> None\n"
 		"Connect to a LDB URL." },
-	{ "modify", (PyCFunction)py_ldb_modify, METH_VARARGS, 
-		"S.modify(message) -> None\n"
+	{ "modify", (PyCFunction)py_ldb_modify, METH_VARARGS|METH_KEYWORDS,
+		"S.modify(message, controls=None, validate=False) -> None\n"
 		"Modify an entry." },
-	{ "add", (PyCFunction)py_ldb_add, METH_VARARGS, 
-		"S.add(message) -> None\n"
+	{ "add", (PyCFunction)py_ldb_add, METH_VARARGS|METH_KEYWORDS,
+		"S.add(message, controls=None) -> None\n"
 		"Add an entry." },
-	{ "delete", (PyCFunction)py_ldb_delete, METH_VARARGS,
-		"S.delete(dn) -> None\n"
+	{ "delete", (PyCFunction)py_ldb_delete, METH_VARARGS|METH_KEYWORDS,
+		"S.delete(dn, controls=None) -> None\n"
 		"Remove an entry." },
-	{ "rename", (PyCFunction)py_ldb_rename, METH_VARARGS,
-		"S.rename(old_dn, new_dn) -> None\n"
+	{ "rename", (PyCFunction)py_ldb_rename, METH_VARARGS|METH_KEYWORDS,
+		"S.rename(old_dn, new_dn, controls=None) -> None\n"
 		"Rename an entry." },
 	{ "search", (PyCFunction)py_ldb_search, METH_VARARGS|METH_KEYWORDS,
 		"S.search(base=None, scope=None, expression=None, attrs=None, controls=None) -> msgs\n"
@@ -2031,7 +2113,7 @@ static PyTypeObject PyLdbModule = {
 static struct ldb_message_element *PyObject_AsMessageElement(
 						      TALLOC_CTX *mem_ctx,
 						      PyObject *set_obj,
-						      int flags,
+						      unsigned int flags,
 						      const char *attr_name)
 {
 	struct ldb_message_element *me;
@@ -2122,9 +2204,9 @@ static PyObject *py_ldb_msg_element_flags(PyLdbMessageElementObject *self, PyObj
 
 static PyObject *py_ldb_msg_element_set_flags(PyLdbMessageElementObject *self, PyObject *args)
 {
-	int flags;
+	unsigned int flags;
 	struct ldb_message_element *el;
-	if (!PyArg_ParseTuple(args, "i", &flags))
+	if (!PyArg_ParseTuple(args, "I", &flags))
 		return NULL;
 
 	el = PyLdbMessageElement_AsMessageElement(self);
@@ -2192,13 +2274,13 @@ static PyObject *py_ldb_msg_element_new(PyTypeObject *type, PyObject *args, PyOb
 {
 	PyObject *py_elements = NULL;
 	struct ldb_message_element *el;
-	int flags = 0;
+	unsigned int flags = 0;
 	char *name = NULL;
 	const char * const kwnames[] = { "elements", "flags", "name", NULL };
 	PyLdbMessageElementObject *ret;
 	TALLOC_CTX *mem_ctx;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|Ois",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OIs",
 					 discard_const_p(char *, kwnames),
 					 &py_elements, &flags, &name))
 		return NULL;
@@ -2427,15 +2509,20 @@ static PyObject *py_ldb_msg_getitem(PyLdbMessageObject *self, PyObject *py_name)
 
 static PyObject *py_ldb_msg_get(PyLdbMessageObject *self, PyObject *args)
 {
-	PyObject *name, *ret;
-	if (!PyArg_ParseTuple(args, "O", &name))
+	PyObject *name, *ret, *retobj;
+	retobj = NULL;
+	if (!PyArg_ParseTuple(args, "O|O", &name, &retobj))
 		return NULL;
 
 	ret = py_ldb_msg_getitem_helper(self, name);
 	if (ret == NULL) {
 		if (PyErr_Occurred())
 			return NULL;
-		Py_RETURN_NONE;
+		if (retobj != NULL) {
+			return retobj;
+		} else {
+			Py_RETURN_NONE;
+		}
 	}
 	return ret;
 }
@@ -3202,4 +3289,14 @@ void initldb(void)
 	PyModule_AddObject(m, "Control", (PyObject *)&PyLdbControl);
 
 	PyModule_AddObject(m, "__version__", PyString_FromString(PACKAGE_VERSION));
+
+#define ADD_LDB_STRING(val)  PyModule_AddObject(m, #val, PyString_FromString(val))
+
+	ADD_LDB_STRING(LDB_SYNTAX_DN);
+	ADD_LDB_STRING(LDB_SYNTAX_DN);
+	ADD_LDB_STRING(LDB_SYNTAX_DIRECTORY_STRING);
+	ADD_LDB_STRING(LDB_SYNTAX_INTEGER);
+	ADD_LDB_STRING(LDB_SYNTAX_BOOLEAN);
+	ADD_LDB_STRING(LDB_SYNTAX_OCTET_STRING);
+	ADD_LDB_STRING(LDB_SYNTAX_UTC_TIME);
 }
diff --git a/source4/lib/ldb/tests/python/api.py b/source4/lib/ldb/tests/python/api.py
index a9f68cbd715..e7658d51ab6 100755
--- a/source4/lib/ldb/tests/python/api.py
+++ b/source4/lib/ldb/tests/python/api.py
@@ -516,6 +516,11 @@ class LdbMsgTests(unittest.TestCase):
         self.msg["foo"] = ["bar"]
         self.assertEquals("bar", self.msg.get("foo")[0])
 
+    def test_get_default(self):
+        self.assertEquals(None, self.msg.get("tatayoyo"))
+
+        self.assertEquals("anniecordie", self.msg.get("tatayoyo", "anniecordie"))
+
     def test_get_unknown(self):
         self.assertEquals(None, self.msg.get("lalalala"))
 
diff --git a/source4/lib/ldb/tools/cmdline.c b/source4/lib/ldb/tools/cmdline.c
index b2be54ebf9b..a06445fc0f4 100644
--- a/source4/lib/ldb/tools/cmdline.c
+++ b/source4/lib/ldb/tools/cmdline.c
@@ -101,7 +101,7 @@ struct ldb_cmdline *ldb_cmdline_process(struct ldb_context *ldb,
 	poptContext pc;
 	int num_options = 0;
 	int opt;
-	int flags = 0;
+	unsigned int flags = 0;
 	int rc;
 	struct poptOption **popt_options;
 
@@ -297,7 +297,7 @@ struct ldb_cmdline *ldb_cmdline_process(struct ldb_context *ldb,
 	}
 
 	/* now connect to the ldb */
-	if (ldb_connect(ldb, ret->url, flags, ret->options) != 0) {
+	if (ldb_connect(ldb, ret->url, flags, ret->options) != LDB_SUCCESS) {
 		fprintf(stderr, "Failed to connect to %s - %s\n", 
 			ret->url, ldb_errstring(ldb));
 		goto failed;
diff --git a/source4/lib/ldb/tools/ldbtest.c b/source4/lib/ldb/tools/ldbtest.c
index a9d8fafe818..4e181af9d55 100644
--- a/source4/lib/ldb/tools/ldbtest.c
+++ b/source4/lib/ldb/tools/ldbtest.c
@@ -126,7 +126,7 @@ static void add_records(struct ldb_context *ldb,
 
 		ldb_delete(ldb, msg.dn);
 
-		if (ldb_add(ldb, &msg) != 0) {
+		if (ldb_add(ldb, &msg) != LDB_SUCCESS) {
 			printf("Add of %s failed - %s\n", name, ldb_errstring(ldb));
 			exit(LDB_ERR_OPERATIONS_ERROR);
 		}
@@ -183,7 +183,7 @@ static void modify_records(struct ldb_context *ldb,
 		vals[2].data = (uint8_t *)talloc_asprintf(tmp_ctx, "%s@other2.example.com", name);
 		vals[2].length = strlen((char *)vals[2].data);
 
-		if (ldb_modify(ldb, &msg) != 0) {
+		if (ldb_modify(ldb, &msg) != LDB_SUCCESS) {
 			printf("Modify of %s failed - %s\n", name, ldb_errstring(ldb));
 			exit(LDB_ERR_OPERATIONS_ERROR);
 		}
@@ -213,7 +213,7 @@ static void delete_records(struct ldb_context *ldb,
 		printf("Deleting uid Test%d\r", i);
 		fflush(stdout);
 
-		if (ldb_delete(ldb, dn) != 0) {
+		if (ldb_delete(ldb, dn) != LDB_SUCCESS) {
 			printf("Delete of %s failed - %s\n", ldb_dn_get_linearized(dn), ldb_errstring(ldb));
 			exit(LDB_ERR_OPERATIONS_ERROR);
 		}
@@ -304,7 +304,7 @@ static void start_test_index(struct ldb_context **ldb)
 	struct ldb_dn *indexlist;
 	struct ldb_dn *basedn;
 	int ret;
-	int flags = 0;
+	unsigned int flags = 0;
 	const char *specials;
 
 	specials = getenv("LDB_SPECIALS");
@@ -343,7 +343,7 @@ static void start_test_index(struct ldb_context **ldb)
 	ldb_msg_add_string(msg, "uid", strdup("test"));
 	ldb_msg_add_string(msg, "objectClass", strdup("OpenLDAPperson"));
 
-	if (ldb_add(*ldb, msg) != 0) {
+	if (ldb_add(*ldb, msg) != LDB_SUCCESS) {
 		printf("Add of %s failed - %s\n", ldb_dn_get_linearized(msg->dn), ldb_errstring(*ldb));
 		exit(LDB_ERR_OPERATIONS_ERROR);
 	}
@@ -356,7 +356,7 @@ static void start_test_index(struct ldb_context **ldb)
 	(*ldb) = ldb_init(options, NULL);
 
 	ret = ldb_connect(*ldb, options->url, flags, NULL);
-	if (ret != 0) {
+	if (ret != LDB_SUCCESS) {
 		printf("failed to connect to %s\n", options->url);
 		exit(LDB_ERR_OPERATIONS_ERROR);
 	}
diff --git a/source4/lib/ldb/wscript b/source4/lib/ldb/wscript
index b59e782e457..7de95494c71 100755
--- a/source4/lib/ldb/wscript
+++ b/source4/lib/ldb/wscript
@@ -16,29 +16,33 @@ sys.path.insert(0, srcdir + '/buildtools/wafsamba')
 import wafsamba, samba_dist, Options
 
 samba_dist.DIST_DIRS('''source4/lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
-                        lib/tdb:lib/tdb lib/tevent:lib/tevent lib/popt:lib/popt
+                        lib/tdb:lib/tdb lib/tdb2:lib/tdb2 lib/tdb_compat:lib/tdb_compat lib/ccan:lib/ccan lib/tevent:lib/tevent lib/popt:lib/popt
                         buildtools:buildtools''')
 
 
 def set_options(opt):
     opt.BUILTIN_DEFAULT('replace')
     opt.PRIVATE_EXTENSION_DEFAULT('ldb', noextension='ldb')
-    opt.RECURSE('lib/tdb')
+    opt.RECURSE('lib/tdb_compat')
     opt.RECURSE('lib/tevent')
     opt.RECURSE('lib/replace')
     opt.tool_options('python') # options for disabling pyc or pyo compilation
 
 def configure(conf):
-    conf.RECURSE('lib/tdb')
+    conf.RECURSE('lib/tdb_compat')
     conf.RECURSE('lib/tevent')
     conf.RECURSE('lib/popt')
     conf.RECURSE('lib/replace')
+    conf.RECURSE('lib/tdb_compat')
     conf.find_program('python', var='PYTHON')
     conf.find_program('xsltproc', var='XSLTPROC')
     conf.check_tool('python')
     conf.check_python_version((2,4,2))
     conf.SAMBA_CHECK_PYTHON_HEADERS(mandatory=True)
 
+    # This make #include <ccan/...> work.
+    conf.ADD_EXTRA_INCLUDES('''#lib''')
+
     # where does the default LIBDIR end up? in conf.env somewhere?
     #
     conf.CONFIG_PATH('LDB_MODULESDIR', conf.SUBST_ENV_VAR('MODULESDIR') + '/ldb')
@@ -72,10 +76,11 @@ def configure(conf):
     conf.SAMBA_CONFIG_H()
 
 def build(bld):
-    bld.RECURSE('lib/tdb')
+    bld.RECURSE('lib/tdb_compat')
     bld.RECURSE('lib/tevent')
     bld.RECURSE('lib/popt')
     bld.RECURSE('lib/replace')
+    bld.RECURSE('lib/tdb_compat')
 
     if bld.env.standalone_ldb:
         private_library = False
@@ -233,14 +238,14 @@ def build(bld):
                          init_function='ldb_tdb_init',
                          module_init_name='ldb_init_module',
                          internal_module=False,
-                         deps='tdb ldb',
+                         deps='tdb_compat ldb',
                          subsystem='ldb')
 
         # have a separate subsystem for common/ldb.c, so it can rebuild
         # for install with a different -DLDB_MODULESDIR=
         bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
                             'common/ldb.c',
-                            deps='tevent',
+                            deps='tevent tdb_compat',
                             includes='include',
                             cflags=['-DLDB_MODULESDIR=\"%s\"' % modules_dir])
 
diff --git a/source4/lib/messaging/irpc.h b/source4/lib/messaging/irpc.h
index bdb1b8fedbf..15f8259e514 100644
--- a/source4/lib/messaging/irpc.h
+++ b/source4/lib/messaging/irpc.h
@@ -35,7 +35,7 @@ struct irpc_message {
 	struct ndr_pull *ndr;
 	bool defer_reply;
 	bool no_reply;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	struct irpc_list *irpc;
 	void *data;
 	struct tevent_context *ev;
@@ -58,24 +58,24 @@ typedef NTSTATUS (*irpc_function_t)(struct irpc_message *, void *r);
 
 struct ndr_interface_table;
 
-NTSTATUS irpc_register(struct messaging_context *msg_ctx, 
+NTSTATUS irpc_register(struct imessaging_context *msg_ctx,
 		       const struct ndr_interface_table *table, 
 		       int call, irpc_function_t fn, void *private_data);
 
 struct dcerpc_binding_handle *irpc_binding_handle(TALLOC_CTX *mem_ctx,
-					struct messaging_context *msg_ctx,
+					struct imessaging_context *msg_ctx,
 					struct server_id server_id,
 					const struct ndr_interface_table *table);
 struct dcerpc_binding_handle *irpc_binding_handle_by_name(TALLOC_CTX *mem_ctx,
-					struct messaging_context *msg_ctx,
+					struct imessaging_context *msg_ctx,
 					const char *dest_task,
 					const struct ndr_interface_table *table);
 void irpc_binding_handle_add_security_token(struct dcerpc_binding_handle *h,
 					    struct security_token *token);
 
-NTSTATUS irpc_add_name(struct messaging_context *msg_ctx, const char *name);
-struct server_id *irpc_servers_byname(struct messaging_context *msg_ctx, TALLOC_CTX *mem_ctx, const char *name);
-void irpc_remove_name(struct messaging_context *msg_ctx, const char *name);
+NTSTATUS irpc_add_name(struct imessaging_context *msg_ctx, const char *name);
+struct server_id *irpc_servers_byname(struct imessaging_context *msg_ctx, TALLOC_CTX *mem_ctx, const char *name);
+void irpc_remove_name(struct imessaging_context *msg_ctx, const char *name);
 NTSTATUS irpc_send_reply(struct irpc_message *m, NTSTATUS status);
 
 #endif
diff --git a/source4/lib/messaging/messaging.c b/source4/lib/messaging/messaging.c
index f9d63203f21..484f22b2ee2 100644
--- a/source4/lib/messaging/messaging.c
+++ b/source4/lib/messaging/messaging.c
@@ -27,22 +27,22 @@
 #include "lib/socket/socket.h"
 #include "librpc/gen_ndr/ndr_irpc.h"
 #include "lib/messaging/irpc.h"
-#include "tdb_wrap.h"
+#include "lib/util/tdb_wrap.h"
 #include "../lib/util/unix_privs.h"
 #include "librpc/rpc/dcerpc.h"
-#include <tdb.h>
+#include "../lib/tdb_compat/tdb_compat.h"
 #include "../lib/util/util_tdb.h"
 #include "cluster/cluster.h"
 #include "../lib/util/tevent_ntstatus.h"
 
 /* change the message version with any incompatible changes in the protocol */
-#define MESSAGING_VERSION 1
+#define IMESSAGING_VERSION 1
 
 /*
   a pending irpc call
 */
 struct irpc_request {
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	int callid;
 	struct {
 		void (*handler)(struct irpc_request *irpc, struct irpc_message *m);
@@ -50,7 +50,7 @@ struct irpc_request {
 	} incoming;
 };
 
-struct messaging_context {
+struct imessaging_context {
 	struct server_id server_id;
 	struct socket_context *sock;
 	const char *base_path;
@@ -58,8 +58,8 @@ struct messaging_context {
 	struct dispatch_fn **dispatch;
 	uint32_t num_types;
 	struct idr_context *dispatch_tree;
-	struct messaging_rec *pending;
-	struct messaging_rec *retry_queue;
+	struct imessaging_rec *pending;
+	struct imessaging_rec *retry_queue;
 	struct irpc_list *irpc;
 	struct idr_context *idr;
 	const char **names;
@@ -81,12 +81,12 @@ struct dispatch_fn {
 };
 
 /* an individual message */
-struct messaging_rec {
-	struct messaging_rec *next, *prev;
-	struct messaging_context *msg;
+struct imessaging_rec {
+	struct imessaging_rec *next, *prev;
+	struct imessaging_context *msg;
 	const char *path;
 
-	struct messaging_header {
+	struct imessaging_header {
 		uint32_t version;
 		uint32_t msg_type;
 		struct server_id from;
@@ -99,20 +99,22 @@ struct messaging_rec {
 };
 
 
-static void irpc_handler(struct messaging_context *, void *, 
+static void irpc_handler(struct imessaging_context *, void *,
 			 uint32_t, struct server_id, DATA_BLOB *);
 
 
 /*
  A useful function for testing the message system.
 */
-static void ping_message(struct messaging_context *msg, void *private_data,
+static void ping_message(struct imessaging_context *msg, void *private_data,
 			 uint32_t msg_type, struct server_id src, DATA_BLOB *data)
 {
-	DEBUG(1,("INFO: Received PING message from server %u.%u [%.*s]\n",
-		 (unsigned int)src.node, (unsigned int)src.id, (int)data->length,
+	char *task_id = server_id_str(NULL, &src);
+	DEBUG(1,("INFO: Received PING message from server %s [%.*s]\n",
+		 task_id, (int)data->length,
 		 data->data?(const char *)data->data:""));
-	messaging_send(msg, src, MSG_PONG, data);
+	talloc_free(task_id);
+	imessaging_send(msg, src, MSG_PONG, data);
 }
 
 /*
@@ -121,7 +123,7 @@ static void ping_message(struct messaging_context *msg, void *private_data,
 static NTSTATUS irpc_uptime(struct irpc_message *msg, 
 			    struct irpc_uptime *r)
 {
-	struct messaging_context *ctx = talloc_get_type(msg->private_data, struct messaging_context);
+	struct imessaging_context *ctx = talloc_get_type(msg->private_data, struct imessaging_context);
 	*r->out.start_time = timeval_to_nttime(&ctx->start_time);
 	return NT_STATUS_OK;
 }
@@ -129,10 +131,10 @@ static NTSTATUS irpc_uptime(struct irpc_message *msg,
 /* 
    return the path to a messaging socket
 */
-static char *messaging_path(struct messaging_context *msg, struct server_id server_id)
+static char *imessaging_path(struct imessaging_context *msg, struct server_id server_id)
 {
 	TALLOC_CTX *tmp_ctx = talloc_new(msg);
-	const char *id = cluster_id_string(tmp_ctx, server_id);
+	const char *id = server_id_str(tmp_ctx, &server_id);
 	char *s;
 	if (id == NULL) {
 		return NULL;
@@ -149,7 +151,7 @@ static char *messaging_path(struct messaging_context *msg, struct server_id serv
   per message. That allows a single messasging context to register
   (for example) a debug handler for more than one piece of code
 */
-static void messaging_dispatch(struct messaging_context *msg, struct messaging_rec *rec)
+static void imessaging_dispatch(struct imessaging_context *msg, struct imessaging_rec *rec)
 {
 	struct dispatch_fn *d, *next;
 
@@ -176,18 +178,18 @@ static void messaging_dispatch(struct messaging_context *msg, struct messaging_r
 /*
   handler for messages that arrive from other nodes in the cluster
 */
-static void cluster_message_handler(struct messaging_context *msg, DATA_BLOB packet)
+static void cluster_message_handler(struct imessaging_context *msg, DATA_BLOB packet)
 {
-	struct messaging_rec *rec;
+	struct imessaging_rec *rec;
 
-	rec = talloc(msg, struct messaging_rec);
+	rec = talloc(msg, struct imessaging_rec);
 	if (rec == NULL) {
-		smb_panic("Unable to allocate messaging_rec");
+		smb_panic("Unable to allocate imessaging_rec");
 	}
 
 	rec->msg           = msg;
 	rec->path          = msg->path;
-	rec->header        = (struct messaging_header *)packet.data;
+	rec->header        = (struct imessaging_header *)packet.data;
 	rec->packet        = packet;
 	rec->retries       = 0;
 
@@ -198,7 +200,7 @@ static void cluster_message_handler(struct messaging_context *msg, DATA_BLOB pac
 		return;
 	}
 
-	messaging_dispatch(msg, rec);
+	imessaging_dispatch(msg, rec);
 	talloc_free(rec);
 }
 
@@ -207,9 +209,9 @@ static void cluster_message_handler(struct messaging_context *msg, DATA_BLOB pac
 /*
   try to send the message
 */
-static NTSTATUS try_send(struct messaging_rec *rec)
+static NTSTATUS try_send(struct imessaging_rec *rec)
 {
-	struct messaging_context *msg = rec->msg;
+	struct imessaging_context *msg = rec->msg;
 	size_t nsent;
 	void *priv;
 	NTSTATUS status;
@@ -238,15 +240,15 @@ static NTSTATUS try_send(struct messaging_rec *rec)
 static void msg_retry_timer(struct tevent_context *ev, struct tevent_timer *te, 
 			    struct timeval t, void *private_data)
 {
-	struct messaging_context *msg = talloc_get_type(private_data,
-							struct messaging_context);
+	struct imessaging_context *msg = talloc_get_type(private_data,
+							struct imessaging_context);
 	msg->retry_te = NULL;
 
 	/* put the messages back on the main queue */
 	while (msg->retry_queue) {
-		struct messaging_rec *rec = msg->retry_queue;
+		struct imessaging_rec *rec = msg->retry_queue;
 		DLIST_REMOVE(msg->retry_queue, rec);
-		DLIST_ADD_END(msg->pending, rec, struct messaging_rec *);
+		DLIST_ADD_END(msg->pending, rec, struct imessaging_rec *);
 	}
 
 	EVENT_FD_WRITEABLE(msg->event.fde);	
@@ -255,10 +257,10 @@ static void msg_retry_timer(struct tevent_context *ev, struct tevent_timer *te,
 /*
   handle a socket write event
 */
-static void messaging_send_handler(struct messaging_context *msg)
+static void imessaging_send_handler(struct imessaging_context *msg)
 {
 	while (msg->pending) {
-		struct messaging_rec *rec = msg->pending;
+		struct imessaging_rec *rec = msg->pending;
 		NTSTATUS status;
 		status = try_send(rec);
 		if (NT_STATUS_EQUAL(status, STATUS_MORE_ENTRIES)) {
@@ -268,7 +270,7 @@ static void messaging_send_handler(struct messaging_context *msg)
 				   backoff this record */
 				DLIST_REMOVE(msg->pending, rec);
 				DLIST_ADD_END(msg->retry_queue, rec, 
-					      struct messaging_rec *);
+					      struct imessaging_rec *);
 				if (msg->retry_te == NULL) {
 					msg->retry_te = 
 						event_add_timed(msg->event.ev, msg, 
@@ -282,8 +284,8 @@ static void messaging_send_handler(struct messaging_context *msg)
 		if (!NT_STATUS_IS_OK(status)) {
 			TALLOC_CTX *tmp_ctx = talloc_new(msg);
 			DEBUG(1,("messaging: Lost message from %s to %s of type %u - %s\n", 
-				 cluster_id_string(tmp_ctx, rec->header->from),
-				 cluster_id_string(tmp_ctx, rec->header->to),
+				 server_id_str(tmp_ctx, &rec->header->from),
+				 server_id_str(tmp_ctx, &rec->header->to),
 				 rec->header->msg_type, 
 				 nt_errstr(status)));
 			talloc_free(tmp_ctx);
@@ -299,9 +301,9 @@ static void messaging_send_handler(struct messaging_context *msg)
 /*
   handle a new incoming packet
 */
-static void messaging_recv_handler(struct messaging_context *msg)
+static void imessaging_recv_handler(struct imessaging_context *msg)
 {
-	struct messaging_rec *rec;
+	struct imessaging_rec *rec;
 	NTSTATUS status;
 	DATA_BLOB packet;
 	size_t msize;
@@ -332,15 +334,15 @@ static void messaging_recv_handler(struct messaging_context *msg)
 		return;
 	}
 
-	rec = talloc(msg, struct messaging_rec);
+	rec = talloc(msg, struct imessaging_rec);
 	if (rec == NULL) {
-		smb_panic("Unable to allocate messaging_rec");
+		smb_panic("Unable to allocate imessaging_rec");
 	}
 
 	talloc_steal(rec, packet.data);
 	rec->msg           = msg;
 	rec->path          = msg->path;
-	rec->header        = (struct messaging_header *)packet.data;
+	rec->header        = (struct imessaging_header *)packet.data;
 	rec->packet        = packet;
 	rec->retries       = 0;
 
@@ -351,7 +353,7 @@ static void messaging_recv_handler(struct messaging_context *msg)
 		return;
 	}
 
-	messaging_dispatch(msg, rec);
+	imessaging_dispatch(msg, rec);
 	talloc_free(rec);
 }
 
@@ -359,16 +361,16 @@ static void messaging_recv_handler(struct messaging_context *msg)
 /*
   handle a socket event
 */
-static void messaging_handler(struct tevent_context *ev, struct tevent_fd *fde, 
+static void imessaging_handler(struct tevent_context *ev, struct tevent_fd *fde,
 			      uint16_t flags, void *private_data)
 {
-	struct messaging_context *msg = talloc_get_type(private_data,
-							struct messaging_context);
+	struct imessaging_context *msg = talloc_get_type(private_data,
+							struct imessaging_context);
 	if (flags & EVENT_FD_WRITE) {
-		messaging_send_handler(msg);
+		imessaging_send_handler(msg);
 	}
 	if (flags & EVENT_FD_READ) {
-		messaging_recv_handler(msg);
+		imessaging_recv_handler(msg);
 	}
 }
 
@@ -376,7 +378,7 @@ static void messaging_handler(struct tevent_context *ev, struct tevent_fd *fde,
 /*
   Register a dispatch function for a particular message type.
 */
-NTSTATUS messaging_register(struct messaging_context *msg, void *private_data,
+NTSTATUS imessaging_register(struct imessaging_context *msg, void *private_data,
 			    uint32_t msg_type, msg_callback_t fn)
 {
 	struct dispatch_fn *d;
@@ -409,7 +411,7 @@ NTSTATUS messaging_register(struct messaging_context *msg, void *private_data,
   register a temporary message handler. The msg_type is allocated
   above MSG_TMP_BASE
 */
-NTSTATUS messaging_register_tmp(struct messaging_context *msg, void *private_data,
+NTSTATUS imessaging_register_tmp(struct imessaging_context *msg, void *private_data,
 				msg_callback_t fn, uint32_t *msg_type)
 {
 	struct dispatch_fn *d;
@@ -435,7 +437,7 @@ NTSTATUS messaging_register_tmp(struct messaging_context *msg, void *private_dat
 /*
   De-register the function for a particular message type.
 */
-void messaging_deregister(struct messaging_context *msg, uint32_t msg_type, void *private_data)
+void imessaging_deregister(struct imessaging_context *msg, uint32_t msg_type, void *private_data)
 {
 	struct dispatch_fn *d, *next;
 
@@ -460,14 +462,14 @@ void messaging_deregister(struct messaging_context *msg, uint32_t msg_type, void
 /*
   Send a message to a particular server
 */
-NTSTATUS messaging_send(struct messaging_context *msg, struct server_id server, 
+NTSTATUS imessaging_send(struct imessaging_context *msg, struct server_id server,
 			uint32_t msg_type, const DATA_BLOB *data)
 {
-	struct messaging_rec *rec;
+	struct imessaging_rec *rec;
 	NTSTATUS status;
 	size_t dlength = data?data->length:0;
 
-	rec = talloc(msg, struct messaging_rec);
+	rec = talloc(msg, struct imessaging_rec);
 	if (rec == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -480,10 +482,10 @@ NTSTATUS messaging_send(struct messaging_context *msg, struct server_id server,
 
 	rec->retries       = 0;
 	rec->msg              = msg;
-	rec->header           = (struct messaging_header *)rec->packet.data;
+	rec->header           = (struct imessaging_header *)rec->packet.data;
 	/* zero padding */
 	ZERO_STRUCTP(rec->header);
-	rec->header->version  = MESSAGING_VERSION;
+	rec->header->version  = IMESSAGING_VERSION;
 	rec->header->msg_type = msg_type;
 	rec->header->from     = msg->server_id;
 	rec->header->to       = server;
@@ -501,7 +503,7 @@ NTSTATUS messaging_send(struct messaging_context *msg, struct server_id server,
 		return status;
 	}
 
-	rec->path = messaging_path(msg, server);
+	rec->path = imessaging_path(msg, server);
 	talloc_steal(rec, rec->path);
 
 	if (msg->pending != NULL) {
@@ -514,7 +516,7 @@ NTSTATUS messaging_send(struct messaging_context *msg, struct server_id server,
 		if (msg->pending == NULL) {
 			EVENT_FD_WRITEABLE(msg->event.fde);
 		}
-		DLIST_ADD_END(msg->pending, rec, struct messaging_rec *);
+		DLIST_ADD_END(msg->pending, rec, struct imessaging_rec *);
 		return NT_STATUS_OK;
 	}
 
@@ -526,7 +528,7 @@ NTSTATUS messaging_send(struct messaging_context *msg, struct server_id server,
 /*
   Send a message to a particular server, with the message containing a single pointer
 */
-NTSTATUS messaging_send_ptr(struct messaging_context *msg, struct server_id server, 
+NTSTATUS imessaging_send_ptr(struct imessaging_context *msg, struct server_id server,
 			    uint32_t msg_type, void *ptr)
 {
 	DATA_BLOB blob;
@@ -534,14 +536,14 @@ NTSTATUS messaging_send_ptr(struct messaging_context *msg, struct server_id serv
 	blob.data = (uint8_t *)&ptr;
 	blob.length = sizeof(void *);
 
-	return messaging_send(msg, server, msg_type, &blob);
+	return imessaging_send(msg, server, msg_type, &blob);
 }
 
 
 /*
   destroy the messaging context
 */
-static int messaging_destructor(struct messaging_context *msg)
+static int imessaging_destructor(struct imessaging_context *msg)
 {
 	unlink(msg->path);
 	while (msg->names && msg->names[0]) {
@@ -553,12 +555,12 @@ static int messaging_destructor(struct messaging_context *msg)
 /*
   create the listening socket and setup the dispatcher
 */
-struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx, 
+struct imessaging_context *imessaging_init(TALLOC_CTX *mem_ctx,
 					 const char *dir,
 					 struct server_id server_id, 
 					 struct tevent_context *ev)
 {
-	struct messaging_context *msg;
+	struct imessaging_context *msg;
 	NTSTATUS status;
 	struct socket_address *path;
 
@@ -566,7 +568,7 @@ struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
-	msg = talloc_zero(mem_ctx, struct messaging_context);
+	msg = talloc_zero(mem_ctx, struct imessaging_context);
 	if (msg == NULL) {
 		return NULL;
 	}
@@ -582,7 +584,7 @@ struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
 	mkdir(dir, 0700);
 
 	msg->base_path     = talloc_reference(msg, dir);
-	msg->path          = messaging_path(msg, server_id);
+	msg->path          = imessaging_path(msg, server_id);
 	msg->server_id     = server_id;
 	msg->idr           = idr_init(msg);
 	msg->dispatch_tree = idr_init(msg);
@@ -617,13 +619,13 @@ struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
 
 	msg->event.ev   = ev;
 	msg->event.fde	= event_add_fd(ev, msg, socket_get_fd(msg->sock), 
-				       EVENT_FD_READ, messaging_handler, msg);
+				       EVENT_FD_READ, imessaging_handler, msg);
 	tevent_fd_set_auto_close(msg->event.fde);
 
-	talloc_set_destructor(msg, messaging_destructor);
+	talloc_set_destructor(msg, imessaging_destructor);
 	
-	messaging_register(msg, NULL, MSG_PING, ping_message);
-	messaging_register(msg, NULL, MSG_IRPC, irpc_handler);
+	imessaging_register(msg, NULL, MSG_PING, ping_message);
+	imessaging_register(msg, NULL, MSG_IRPC, irpc_handler);
 	IRPC_REGISTER(msg, irpc, IRPC_UPTIME, irpc_uptime, msg);
 
 	return msg;
@@ -632,14 +634,14 @@ struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
 /* 
    A hack, for the short term until we get 'client only' messaging in place 
 */
-struct messaging_context *messaging_client_init(TALLOC_CTX *mem_ctx, 
+struct imessaging_context *imessaging_client_init(TALLOC_CTX *mem_ctx,
 						const char *dir,
 						struct tevent_context *ev)
 {
 	struct server_id id;
 	ZERO_STRUCT(id);
-	id.id = random() % 0x10000000;
-	return messaging_init(mem_ctx, dir, id, ev);
+	id.pid = random() % 0x10000000;
+	return imessaging_init(mem_ctx, dir, id, ev);
 }
 /*
   a list of registered irpc server functions
@@ -657,7 +659,7 @@ struct irpc_list {
 /*
   register a irpc server function
 */
-NTSTATUS irpc_register(struct messaging_context *msg_ctx, 
+NTSTATUS irpc_register(struct imessaging_context *msg_ctx,
 		       const struct ndr_interface_table *table, 
 		       int callnum, irpc_function_t fn, void *private_data)
 {
@@ -688,7 +690,7 @@ NTSTATUS irpc_register(struct messaging_context *msg_ctx,
 /*
   handle an incoming irpc reply message
 */
-static void irpc_handler_reply(struct messaging_context *msg_ctx, struct irpc_message *m)
+static void irpc_handler_reply(struct imessaging_context *msg_ctx, struct irpc_message *m)
 {
 	struct irpc_request *irpc;
 
@@ -734,7 +736,7 @@ NTSTATUS irpc_send_reply(struct irpc_message *m, NTSTATUS status)
 
 	/* send the reply message */
 	packet = ndr_push_blob(push);
-	status = messaging_send(m->msg_ctx, m->from, MSG_IRPC, &packet);
+	status = imessaging_send(m->msg_ctx, m->from, MSG_IRPC, &packet);
 	if (!NT_STATUS_IS_OK(status)) goto failed;
 
 failed:
@@ -745,7 +747,7 @@ failed:
 /*
   handle an incoming irpc request message
 */
-static void irpc_handler_request(struct messaging_context *msg_ctx, 
+static void irpc_handler_request(struct imessaging_context *msg_ctx,
 				 struct irpc_message *m)
 {
 	struct irpc_list *i;
@@ -809,7 +811,7 @@ failed:
 /*
   handle an incoming irpc message
 */
-static void irpc_handler(struct messaging_context *msg_ctx, void *private_data,
+static void irpc_handler(struct imessaging_context *msg_ctx, void *private_data,
 			 uint32_t msg_type, struct server_id src, DATA_BLOB *packet)
 {
 	struct irpc_message *m;
@@ -856,7 +858,7 @@ static int irpc_destructor(struct irpc_request *irpc)
 /*
   open the naming database
 */
-static struct tdb_wrap *irpc_namedb_open(struct messaging_context *msg_ctx)
+static struct tdb_wrap *irpc_namedb_open(struct imessaging_context *msg_ctx)
 {
 	struct tdb_wrap *t;
 	char *path = talloc_asprintf(msg_ctx, "%s/names.tdb", msg_ctx->base_path);
@@ -872,7 +874,7 @@ static struct tdb_wrap *irpc_namedb_open(struct messaging_context *msg_ctx)
 /*
   add a string name that this irpc server can be called on
 */
-NTSTATUS irpc_add_name(struct messaging_context *msg_ctx, const char *name)
+NTSTATUS irpc_add_name(struct imessaging_context *msg_ctx, const char *name)
 {
 	struct tdb_wrap *t;
 	TDB_DATA rec;
@@ -912,7 +914,7 @@ NTSTATUS irpc_add_name(struct messaging_context *msg_ctx, const char *name)
 /*
   return a list of server ids for a server name
 */
-struct server_id *irpc_servers_byname(struct messaging_context *msg_ctx,
+struct server_id *irpc_servers_byname(struct imessaging_context *msg_ctx,
 				      TALLOC_CTX *mem_ctx,
 				      const char *name)
 {
@@ -957,7 +959,7 @@ struct server_id *irpc_servers_byname(struct messaging_context *msg_ctx,
 /*
   remove a name from a messaging context
 */
-void irpc_remove_name(struct messaging_context *msg_ctx, const char *name)
+void irpc_remove_name(struct imessaging_context *msg_ctx, const char *name)
 {
 	struct tdb_wrap *t;
 	TDB_DATA rec;
@@ -1005,13 +1007,13 @@ void irpc_remove_name(struct messaging_context *msg_ctx, const char *name)
 	talloc_free(t);
 }
 
-struct server_id messaging_get_server_id(struct messaging_context *msg_ctx)
+struct server_id imessaging_get_server_id(struct imessaging_context *msg_ctx)
 {
 	return msg_ctx->server_id;
 }
 
 struct irpc_bh_state {
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	struct server_id server_id;
 	const struct ndr_interface_table *table;
 	uint32_t timeout;
@@ -1137,7 +1139,7 @@ static struct tevent_req *irpc_bh_raw_call_send(TALLOC_CTX *mem_ctx,
 
 	/* and send it */
 	state->in_packet = ndr_push_blob(ndr);
-	status = messaging_send(hs->msg_ctx, hs->server_id,
+	status = imessaging_send(hs->msg_ctx, hs->server_id,
 				MSG_IRPC, &state->in_packet);
 	if (!NT_STATUS_IS_OK(status)) {
 		tevent_req_nterror(req, status);
@@ -1176,7 +1178,7 @@ static void irpc_bh_raw_call_incoming_handler(struct irpc_request *irpc,
 		m->ndr->data + m->ndr->offset,
 		m->ndr->data_size - m->ndr->offset);
 	if ((m->ndr->data_size - m->ndr->offset) > 0 && !state->out_data.data) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return;
 	}
 
@@ -1270,7 +1272,7 @@ static const struct dcerpc_binding_handle_ops irpc_bh_ops = {
 
 /* initialise a irpc binding handle */
 struct dcerpc_binding_handle *irpc_binding_handle(TALLOC_CTX *mem_ctx,
-					struct messaging_context *msg_ctx,
+					struct imessaging_context *msg_ctx,
 					struct server_id server_id,
 					const struct ndr_interface_table *table)
 {
@@ -1298,7 +1300,7 @@ struct dcerpc_binding_handle *irpc_binding_handle(TALLOC_CTX *mem_ctx,
 }
 
 struct dcerpc_binding_handle *irpc_binding_handle_by_name(TALLOC_CTX *mem_ctx,
-					struct messaging_context *msg_ctx,
+					struct imessaging_context *msg_ctx,
 					const char *dest_task,
 					const struct ndr_interface_table *table)
 {
@@ -1312,7 +1314,7 @@ struct dcerpc_binding_handle *irpc_binding_handle_by_name(TALLOC_CTX *mem_ctx,
 		errno = EADDRNOTAVAIL;
 		return NULL;
 	}
-	if (sids[0].id == 0) {
+	if (sids[0].pid == 0) {
 		talloc_free(sids);
 		errno = EADDRNOTAVAIL;
 		return NULL;
diff --git a/source4/lib/messaging/messaging.h b/source4/lib/messaging/messaging.h
index 4bc6d8c5093..eb8a8abc795 100644
--- a/source4/lib/messaging/messaging.h
+++ b/source4/lib/messaging/messaging.h
@@ -21,9 +21,9 @@
 #ifndef _MESSAGES_H_
 #define _MESSAGES_H_
 
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
-struct messaging_context;
+struct imessaging_context;
 
 /* general messages */
 #define MSG_DEBUG		1
@@ -42,27 +42,27 @@ struct messaging_context;
 /* taskid for messaging of parent process */
 #define SAMBA_PARENT_TASKID     0
 
-typedef void (*msg_callback_t)(struct messaging_context *msg, void *private_data,
+typedef void (*msg_callback_t)(struct imessaging_context *msg, void *private_data,
 			       uint32_t msg_type,
 			       struct server_id server_id, DATA_BLOB *data);
 
-NTSTATUS messaging_send(struct messaging_context *msg, struct server_id server,
+NTSTATUS imessaging_send(struct imessaging_context *msg, struct server_id server,
 			uint32_t msg_type, const DATA_BLOB *data);
-NTSTATUS messaging_register(struct messaging_context *msg, void *private_data,
+NTSTATUS imessaging_register(struct imessaging_context *msg, void *private_data,
 			    uint32_t msg_type,
 			    msg_callback_t fn);
-NTSTATUS messaging_register_tmp(struct messaging_context *msg, void *private_data,
+NTSTATUS imessaging_register_tmp(struct imessaging_context *msg, void *private_data,
 				msg_callback_t fn, uint32_t *msg_type);
-struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
+struct imessaging_context *imessaging_init(TALLOC_CTX *mem_ctx,
 					 const char *dir,
 					 struct server_id server_id,
 					 struct tevent_context *ev);
-struct messaging_context *messaging_client_init(TALLOC_CTX *mem_ctx,
+struct imessaging_context *imessaging_client_init(TALLOC_CTX *mem_ctx,
 					 const char *dir,
 					 struct tevent_context *ev);
-NTSTATUS messaging_send_ptr(struct messaging_context *msg, struct server_id server,
+NTSTATUS imessaging_send_ptr(struct imessaging_context *msg, struct server_id server,
 			    uint32_t msg_type, void *ptr);
-void messaging_deregister(struct messaging_context *msg, uint32_t msg_type, void *private_data);
-struct server_id messaging_get_server_id(struct messaging_context *msg_ctx);
+void imessaging_deregister(struct imessaging_context *msg, uint32_t msg_type, void *private_data);
+struct server_id imessaging_get_server_id(struct imessaging_context *msg_ctx);
 
 #endif
diff --git a/source4/lib/messaging/pymessaging.c b/source4/lib/messaging/pymessaging.c
index 358d205b53b..cafd45beae3 100644
--- a/source4/lib/messaging/pymessaging.c
+++ b/source4/lib/messaging/pymessaging.c
@@ -31,11 +31,11 @@
 #include "param/param.h"
 #include "param/pyparam.h"
 #include "librpc/rpc/dcerpc.h"
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
 void initmessaging(void);
 
-extern PyTypeObject messaging_Type;
+extern PyTypeObject imessaging_Type;
 
 static bool server_id_from_py(PyObject *object, struct server_id *server_id)
 {
@@ -45,12 +45,12 @@ static bool server_id_from_py(PyObject *object, struct server_id *server_id)
 	}
 
 	if (PyTuple_Size(object) == 3) {
-		return PyArg_ParseTuple(object, "iii", &server_id->id, &server_id->id2, &server_id->node);
+		return PyArg_ParseTuple(object, "iii", &server_id->pid, &server_id->task_id, &server_id->vnn);
 	} else {
-		int id, id2;
-		if (!PyArg_ParseTuple(object, "ii", &id, &id2))
+		int pid, task_id;
+		if (!PyArg_ParseTuple(object, "ii", &pid, &task_id))
 			return false;
-		*server_id = cluster_id(id, id2);
+		*server_id = cluster_id(pid, task_id);
 		return true;
 	}
 }
@@ -58,23 +58,23 @@ static bool server_id_from_py(PyObject *object, struct server_id *server_id)
 typedef struct {
 	PyObject_HEAD
 	TALLOC_CTX *mem_ctx;
-	struct messaging_context *msg_ctx;
-} messaging_Object;
+	struct imessaging_context *msg_ctx;
+} imessaging_Object;
 
-static PyObject *py_messaging_connect(PyTypeObject *self, PyObject *args, PyObject *kwargs)
+static PyObject *py_imessaging_connect(PyTypeObject *self, PyObject *args, PyObject *kwargs)
 {
 	struct tevent_context *ev;
 	const char *kwnames[] = { "own_id", "messaging_path", NULL };
 	PyObject *own_id = Py_None;
-	const char *messaging_path = NULL;
-	messaging_Object *ret;
+	const char *imessaging_path = NULL;
+	imessaging_Object *ret;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|Oz:connect", 
-		discard_const_p(char *, kwnames), &own_id, &messaging_path)) {
+		discard_const_p(char *, kwnames), &own_id, &imessaging_path)) {
 		return NULL;
 	}
 
-	ret = PyObject_New(messaging_Object, &messaging_Type);
+	ret = PyObject_New(imessaging_Object, &imessaging_Type);
 	if (ret == NULL)
 		return NULL;
 
@@ -82,11 +82,11 @@ static PyObject *py_messaging_connect(PyTypeObject *self, PyObject *args, PyObje
 
 	ev = s4_event_context_init(ret->mem_ctx);
 
-	if (messaging_path == NULL) {
-		messaging_path = lpcfg_messaging_path(ret->mem_ctx,
+	if (imessaging_path == NULL) {
+		imessaging_path = lpcfg_imessaging_path(ret->mem_ctx,
 								   py_default_loadparm_context(ret->mem_ctx));
 	} else {
-		messaging_path = talloc_strdup(ret->mem_ctx, messaging_path);
+		imessaging_path = talloc_strdup(ret->mem_ctx, imessaging_path);
 	}
 
 	if (own_id != Py_None) {
@@ -95,18 +95,18 @@ static PyObject *py_messaging_connect(PyTypeObject *self, PyObject *args, PyObje
 		if (!server_id_from_py(own_id, &server_id)) 
 			return NULL;
 
-		ret->msg_ctx = messaging_init(ret->mem_ctx, 
-					    messaging_path,
+		ret->msg_ctx = imessaging_init(ret->mem_ctx,
+					    imessaging_path,
 					    server_id,
 					    ev);
 	} else {
-		ret->msg_ctx = messaging_client_init(ret->mem_ctx, 
-					    messaging_path,
+		ret->msg_ctx = imessaging_client_init(ret->mem_ctx,
+					    imessaging_path,
 					    ev);
 	}
 
 	if (ret->msg_ctx == NULL) {
-		PyErr_SetString(PyExc_RuntimeError, "messaging_connect unable to create a messaging context");
+		PyErr_SetString(PyExc_RuntimeError, "imessaging_connect unable to create a messaging context");
 		talloc_free(ret->mem_ctx);
 		return NULL;
 	}
@@ -114,16 +114,16 @@ static PyObject *py_messaging_connect(PyTypeObject *self, PyObject *args, PyObje
 	return (PyObject *)ret;
 }
 
-static void py_messaging_dealloc(PyObject *self)
+static void py_imessaging_dealloc(PyObject *self)
 {
-	messaging_Object *iface = (messaging_Object *)self;
+	imessaging_Object *iface = (imessaging_Object *)self;
 	talloc_free(iface->msg_ctx);
 	self->ob_type->tp_free(self);
 }
 
-static PyObject *py_messaging_send(PyObject *self, PyObject *args, PyObject *kwargs)
+static PyObject *py_imessaging_send(PyObject *self, PyObject *args, PyObject *kwargs)
 {
-	messaging_Object *iface = (messaging_Object *)self;
+	imessaging_Object *iface = (imessaging_Object *)self;
 	uint32_t msg_type;
 	DATA_BLOB data;
 	PyObject *target;
@@ -143,7 +143,7 @@ static PyObject *py_messaging_send(PyObject *self, PyObject *args, PyObject *kwa
 	if (!server_id_from_py(target, &server))
 		return NULL;
 
-	status = messaging_send(iface->msg_ctx, server, msg_type, &data);
+	status = imessaging_send(iface->msg_ctx, server, msg_type, &data);
 	if (NT_STATUS_IS_ERR(status)) {
 		PyErr_SetNTSTATUS(status);
 		return NULL;
@@ -152,20 +152,20 @@ static PyObject *py_messaging_send(PyObject *self, PyObject *args, PyObject *kwa
 	Py_RETURN_NONE;
 }
 
-static void py_msg_callback_wrapper(struct messaging_context *msg, void *private_data,
+static void py_msg_callback_wrapper(struct imessaging_context *msg, void *private_data,
 			       uint32_t msg_type, 
 			       struct server_id server_id, DATA_BLOB *data)
 {
 	PyObject *callback = (PyObject *)private_data;
 
-	PyObject_CallFunction(callback, discard_const_p(char, "i(iii)s#"), msg_type, 
-			      server_id.id, server_id.id2, server_id.node, 
+	PyObject_CallFunction(callback, discard_const_p(char, "i(iii)s#"), msg_type,
+			      server_id.pid, server_id.task_id, server_id.vnn,
 			      data->data, data->length);
 }
 
-static PyObject *py_messaging_register(PyObject *self, PyObject *args, PyObject *kwargs)
+static PyObject *py_imessaging_register(PyObject *self, PyObject *args, PyObject *kwargs)
 {
-	messaging_Object *iface = (messaging_Object *)self;
+	imessaging_Object *iface = (imessaging_Object *)self;
 	int msg_type = -1;
 	PyObject *callback;
 	NTSTATUS status;
@@ -180,11 +180,11 @@ static PyObject *py_messaging_register(PyObject *self, PyObject *args, PyObject
 
 	if (msg_type == -1) {
 		uint32_t msg_type32 = msg_type;
-		status = messaging_register_tmp(iface->msg_ctx, callback,
+		status = imessaging_register_tmp(iface->msg_ctx, callback,
 						py_msg_callback_wrapper, &msg_type32);
 		msg_type = msg_type32;
 	} else {
-		status = messaging_register(iface->msg_ctx, callback,
+		status = imessaging_register(iface->msg_ctx, callback,
 				    msg_type, py_msg_callback_wrapper);
 	}
 	if (NT_STATUS_IS_ERR(status)) {
@@ -195,9 +195,9 @@ static PyObject *py_messaging_register(PyObject *self, PyObject *args, PyObject
 	return PyLong_FromLong(msg_type);
 }
 
-static PyObject *py_messaging_deregister(PyObject *self, PyObject *args, PyObject *kwargs)
+static PyObject *py_imessaging_deregister(PyObject *self, PyObject *args, PyObject *kwargs)
 {
-	messaging_Object *iface = (messaging_Object *)self;
+	imessaging_Object *iface = (imessaging_Object *)self;
 	int msg_type = -1;
 	PyObject *callback;
 	const char *kwnames[] = { "callback", "msg_type", NULL };
@@ -207,49 +207,49 @@ static PyObject *py_messaging_deregister(PyObject *self, PyObject *args, PyObjec
 		return NULL;
 	}
 
-	messaging_deregister(iface->msg_ctx, msg_type, callback);
+	imessaging_deregister(iface->msg_ctx, msg_type, callback);
 
 	Py_DECREF(callback);
 
 	Py_RETURN_NONE;
 }
 
-static PyMethodDef py_messaging_methods[] = {
-	{ "send", (PyCFunction)py_messaging_send, METH_VARARGS|METH_KEYWORDS, 
+static PyMethodDef py_imessaging_methods[] = {
+	{ "send", (PyCFunction)py_imessaging_send, METH_VARARGS|METH_KEYWORDS,
 		"S.send(target, msg_type, data) -> None\nSend a message" },
-	{ "register", (PyCFunction)py_messaging_register, METH_VARARGS|METH_KEYWORDS,
+	{ "register", (PyCFunction)py_imessaging_register, METH_VARARGS|METH_KEYWORDS,
 		"S.register(callback, msg_type=None) -> msg_type\nRegister a message handler" },
-	{ "deregister", (PyCFunction)py_messaging_deregister, METH_VARARGS|METH_KEYWORDS,
+	{ "deregister", (PyCFunction)py_imessaging_deregister, METH_VARARGS|METH_KEYWORDS,
 		"S.deregister(callback, msg_type) -> None\nDeregister a message handler" },
 	{ NULL, NULL, 0, NULL }
 };
 
-static PyObject *py_messaging_server_id(PyObject *obj, void *closure)
+static PyObject *py_imessaging_server_id(PyObject *obj, void *closure)
 {
-	messaging_Object *iface = (messaging_Object *)obj;
-	struct server_id server_id = messaging_get_server_id(iface->msg_ctx);
+	imessaging_Object *iface = (imessaging_Object *)obj;
+	struct server_id server_id = imessaging_get_server_id(iface->msg_ctx);
 
-	return Py_BuildValue("(iii)", server_id.id, server_id.id2, 
-			     server_id.node);
+	return Py_BuildValue("(iii)", server_id.pid, server_id.task_id,
+			     server_id.vnn);
 }
 
-static PyGetSetDef py_messaging_getset[] = {
-	{ discard_const_p(char, "server_id"), py_messaging_server_id, NULL, 
+static PyGetSetDef py_imessaging_getset[] = {
+	{ discard_const_p(char, "server_id"), py_imessaging_server_id, NULL,
 	  discard_const_p(char, "local server id") },
 	{ NULL },
 };
 
 
-PyTypeObject messaging_Type = {
+PyTypeObject imessaging_Type = {
 	PyObject_HEAD_INIT(NULL) 0,
 	.tp_name = "messaging.Messaging",
-	.tp_basicsize = sizeof(messaging_Object),
+	.tp_basicsize = sizeof(imessaging_Object),
 	.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
-	.tp_new = py_messaging_connect,
-	.tp_dealloc = py_messaging_dealloc,
-	.tp_methods = py_messaging_methods,
-	.tp_getset = py_messaging_getset,
-	.tp_doc = "Messaging(own_id=None, messaging_path=None)\n" \
+	.tp_new = py_imessaging_connect,
+	.tp_dealloc = py_imessaging_dealloc,
+	.tp_methods = py_imessaging_methods,
+	.tp_getset = py_imessaging_getset,
+	.tp_doc = "Messaging(own_id=None, imessaging_path=None)\n" \
 		  "Create a new object that can be used to communicate with the peers in the specified messaging path.\n" \
 		  "If no path is specified, the default path from smb.conf will be used."
 };
@@ -258,13 +258,13 @@ void initmessaging(void)
 {
 	PyObject *mod;
 
-	if (PyType_Ready(&messaging_Type) < 0)
+	if (PyType_Ready(&imessaging_Type) < 0)
 		return;
 
 	mod = Py_InitModule3("messaging", NULL, "Internal RPC");
 	if (mod == NULL)
 		return;
 
-	Py_INCREF((PyObject *)&messaging_Type);
-	PyModule_AddObject(mod, "Messaging", (PyObject *)&messaging_Type);
+	Py_INCREF((PyObject *)&imessaging_Type);
+	PyModule_AddObject(mod, "Messaging", (PyObject *)&imessaging_Type);
 }
diff --git a/source4/lib/messaging/tests/irpc.c b/source4/lib/messaging/tests/irpc.c
index 4d0b6b43787..cfa2bcb91e1 100644
--- a/source4/lib/messaging/tests/irpc.c
+++ b/source4/lib/messaging/tests/irpc.c
@@ -34,7 +34,7 @@ static bool test_debug;
 
 struct irpc_test_data
 {
-	struct messaging_context *msg_ctx1, *msg_ctx2;
+	struct imessaging_context *msg_ctx1, *msg_ctx2;
 	struct tevent_context *ev;
 };
 
@@ -246,15 +246,15 @@ static bool irpc_setup(struct torture_context *tctx, void **_data)
 
 	data->ev = tctx->ev;
 	torture_assert(tctx, data->msg_ctx1 = 
-		       messaging_init(tctx, 
-				      lpcfg_messaging_path(tctx, tctx->lp_ctx),
+		       imessaging_init(tctx,
+				      lpcfg_imessaging_path(tctx, tctx->lp_ctx),
 				      cluster_id(0, MSG_ID1),
 				      data->ev),
 		       "Failed to init first messaging context");
 
 	torture_assert(tctx, data->msg_ctx2 = 
-		       messaging_init(tctx, 
-				      lpcfg_messaging_path(tctx, tctx->lp_ctx),
+		       imessaging_init(tctx,
+				      lpcfg_imessaging_path(tctx, tctx->lp_ctx),
 				      cluster_id(0, MSG_ID2), 
 				      data->ev),
 		       "Failed to init second messaging context");
diff --git a/source4/lib/messaging/tests/messaging.c b/source4/lib/messaging/tests/messaging.c
index 82fdf2f73ec..38c34fc52ee 100644
--- a/source4/lib/messaging/tests/messaging.c
+++ b/source4/lib/messaging/tests/messaging.c
@@ -29,24 +29,24 @@
 
 static uint32_t msg_pong;
 
-static void ping_message(struct messaging_context *msg, void *private_data,
+static void ping_message(struct imessaging_context *msg, void *private_data,
 			 uint32_t msg_type, struct server_id src, DATA_BLOB *data)
 {
 	NTSTATUS status;
-	status = messaging_send(msg, src, msg_pong, data);
+	status = imessaging_send(msg, src, msg_pong, data);
 	if (!NT_STATUS_IS_OK(status)) {
 		printf("pong failed - %s\n", nt_errstr(status));
 	}
 }
 
-static void pong_message(struct messaging_context *msg, void *private_data,
+static void pong_message(struct imessaging_context *msg, void *private_data,
 			 uint32_t msg_type, struct server_id src, DATA_BLOB *data)
 {
 	int *count = (int *)private_data;
 	(*count)++;
 }
 
-static void exit_message(struct messaging_context *msg, void *private_data,
+static void exit_message(struct imessaging_context *msg, void *private_data,
 			 uint32_t msg_type, struct server_id src, DATA_BLOB *data)
 {
 	talloc_free(private_data);
@@ -59,8 +59,8 @@ static void exit_message(struct messaging_context *msg, void *private_data,
 static bool test_ping_speed(struct torture_context *tctx)
 {
 	struct tevent_context *ev;
-	struct messaging_context *msg_client_ctx;
-	struct messaging_context *msg_server_ctx;
+	struct imessaging_context *msg_client_ctx;
+	struct imessaging_context *msg_server_ctx;
 	int ping_count = 0;
 	int pong_count = 0;
 	struct timeval tv;
@@ -71,24 +71,24 @@ static bool test_ping_speed(struct torture_context *tctx)
 
 	ev = tctx->ev;
 
-	msg_server_ctx = messaging_init(tctx, 
-					lpcfg_messaging_path(tctx, tctx->lp_ctx), cluster_id(0, 1),
+	msg_server_ctx = imessaging_init(tctx,
+					lpcfg_imessaging_path(tctx, tctx->lp_ctx), cluster_id(0, 1),
 					ev);
 	
 	torture_assert(tctx, msg_server_ctx != NULL, "Failed to init ping messaging context");
 		
-	messaging_register_tmp(msg_server_ctx, NULL, ping_message, &msg_ping);
-	messaging_register_tmp(msg_server_ctx, tctx, exit_message, &msg_exit);
+	imessaging_register_tmp(msg_server_ctx, NULL, ping_message, &msg_ping);
+	imessaging_register_tmp(msg_server_ctx, tctx, exit_message, &msg_exit);
 
-	msg_client_ctx = messaging_init(tctx, 
-					lpcfg_messaging_path(tctx, tctx->lp_ctx),
+	msg_client_ctx = imessaging_init(tctx,
+					lpcfg_imessaging_path(tctx, tctx->lp_ctx),
 					cluster_id(0, 2), 
 					ev);
 
 	torture_assert(tctx, msg_client_ctx != NULL, 
-		       "msg_client_ctx messaging_init() failed");
+		       "msg_client_ctx imessaging_init() failed");
 
-	messaging_register_tmp(msg_client_ctx, &pong_count, pong_message, &msg_pong);
+	imessaging_register_tmp(msg_client_ctx, &pong_count, pong_message, &msg_pong);
 
 	tv = timeval_current();
 
@@ -100,8 +100,8 @@ static bool test_ping_speed(struct torture_context *tctx)
 		data.data = discard_const_p(uint8_t, "testing");
 		data.length = strlen((const char *)data.data);
 
-		status1 = messaging_send(msg_client_ctx, cluster_id(0, 1), msg_ping, &data);
-		status2 = messaging_send(msg_client_ctx, cluster_id(0, 1), msg_ping, NULL);
+		status1 = imessaging_send(msg_client_ctx, cluster_id(0, 1), msg_ping, &data);
+		status2 = imessaging_send(msg_client_ctx, cluster_id(0, 1), msg_ping, NULL);
 
 		torture_assert_ntstatus_ok(tctx, status1, "msg1 failed");
 		ping_count++;
@@ -121,7 +121,7 @@ static bool test_ping_speed(struct torture_context *tctx)
 	}
 
 	torture_comment(tctx, "sending exit\n");
-	messaging_send(msg_client_ctx, cluster_id(0, 1), msg_exit, NULL);
+	imessaging_send(msg_client_ctx, cluster_id(0, 1), msg_exit, NULL);
 
 	torture_assert_int_equal(tctx, ping_count, pong_count, "ping test failed");
 
diff --git a/source4/lib/registry/rpc.c b/source4/lib/registry/rpc.c
index f1e14c1b864..42b7374f7ad 100644
--- a/source4/lib/registry/rpc.c
+++ b/source4/lib/registry/rpc.c
@@ -485,7 +485,7 @@ _PUBLIC_ WERROR reg_open_remote(struct registry_context **ctx,
 	struct dcerpc_pipe *p;
 	struct rpc_registry_context *rctx;
 
-	dcerpc_init(lp_ctx);
+	dcerpc_init();
 
 	rctx = talloc(NULL, struct rpc_registry_context);
 	W_ERROR_HAVE_NO_MEMORY(rctx);
diff --git a/source4/lib/registry/wscript_build b/source4/lib/registry/wscript_build
index 2f0372a9338..5adc941718b 100644
--- a/source4/lib/registry/wscript_build
+++ b/source4/lib/registry/wscript_build
@@ -13,7 +13,7 @@ bld.SAMBA_SUBSYSTEM('TDR_REGF',
 bld.SAMBA_LIBRARY('registry',
 	source='interface.c util.c samba.c patchfile_dotreg.c patchfile_preg.c patchfile.c regf.c hive.c local.c ldb.c dir.c rpc.c',
 	pc_files='registry.pc',
-	public_deps='samba-util TDR_REGF ldb RPC_NDR_WINREG ldbsamba util_reg',
+	public_deps='dcerpc samba-util TDR_REGF ldb RPC_NDR_WINREG ldbsamba util_reg',
 	public_headers='registry.h',
 	vnum='0.0.1'
 	)
diff --git a/source4/lib/socket/access.c b/source4/lib/socket/access.c
index ab39d63ef5b..589797763fc 100644
--- a/source4/lib/socket/access.c
+++ b/source4/lib/socket/access.c
@@ -249,9 +249,9 @@ static bool allow_access_internal(TALLOC_CTX *mem_ctx,
 }
 
 /* return true if access should be allowed */
-bool allow_access(TALLOC_CTX *mem_ctx,
-		  const char **deny_list, const char **allow_list,
-		  const char *cname, const char *caddr)
+bool socket_allow_access(TALLOC_CTX *mem_ctx,
+			 const char **deny_list, const char **allow_list,
+			 const char *cname, const char *caddr)
 {
 	bool ret;
 	char *nc_cname = talloc_strdup(mem_ctx, cname);
@@ -346,7 +346,7 @@ bool socket_check_access(struct socket_context *sock,
 		return false;
 	}
 
-	ret = allow_access(mem_ctx, deny_list, allow_list, name, addr->addr);
+	ret = socket_allow_access(mem_ctx, deny_list, allow_list, name, addr->addr);
 	
 	if (ret) {
 		DEBUG(2,("socket_check_access: Allowed connection to '%s' from %s (%s)\n", 
diff --git a/source4/lib/socket/connect_multi.c b/source4/lib/socket/connect_multi.c
index 300e5fb1e53..4ce5115e97d 100644
--- a/source4/lib/socket/connect_multi.c
+++ b/source4/lib/socket/connect_multi.c
@@ -136,7 +136,7 @@ static void connect_multi_next_socket(struct composite_context *result)
 	if (composite_nomem(state, result)) return;
 
 	state->result = result;
-	result->status = socket_create("ipv4", SOCKET_TYPE_STREAM, &state->sock, 0);
+	result->status = socket_create(multi->server_address->family, SOCKET_TYPE_STREAM, &state->sock, 0);
 	if (!composite_is_ok(result)) return;
 
 	state->addr = socket_address_copy(state, multi->server_address);
@@ -162,7 +162,7 @@ static void connect_multi_next_socket(struct composite_context *result)
 		   connect attempt state, so it will go away when this
 		   request completes */
 		event_add_timed(result->event_ctx, state,
-				timeval_current_ofs(0, MULTI_PORT_DELAY),
+				timeval_current_ofs_usec(MULTI_PORT_DELAY),
 				connect_multi_timer, result);
 	}
 }
diff --git a/source4/lib/socket/interface.c b/source4/lib/socket/interface.c
index c4411b623c0..d5b610fea7f 100644
--- a/source4/lib/socket/interface.c
+++ b/source4/lib/socket/interface.c
@@ -21,15 +21,19 @@
 
 #include "includes.h"
 #include "system/network.h"
+#include "param/param.h"
 #include "lib/socket/netif.h"
 #include "../lib/util/util_net.h"
 #include "../lib/util/dlinklist.h"
 
-/** used for network interfaces */
+/* used for network interfaces */
 struct interface {
 	struct interface *next, *prev;
-	struct in_addr ip;
-	struct in_addr nmask;
+	char *name;
+	int flags;
+	struct sockaddr_storage ip;
+	struct sockaddr_storage netmask;
+	struct sockaddr_storage bcast;
 	const char *ip_s;
 	const char *bcast_s;
 	const char *nmask_s;
@@ -45,31 +49,51 @@ struct interface {
 /****************************************************************************
 Try and find an interface that matches an ip. If we cannot, return NULL
   **************************************************************************/
-static struct interface *iface_find(struct interface *interfaces, 
-				    struct in_addr ip, bool CheckMask)
+static struct interface *iface_list_find(struct interface *interfaces,
+					 const struct sockaddr *ip,
+					 bool check_mask)
 {
 	struct interface *i;
-	if (is_zero_ip_v4(ip)) return interfaces;
 
-	for (i=interfaces;i;i=i->next)
-		if (CheckMask) {
-			if (same_net_v4(i->ip,ip,i->nmask)) return i;
-		} else if (i->ip.s_addr == ip.s_addr) return i;
+	if (is_address_any(ip)) {
+		return interfaces;
+	}
+
+	for (i=interfaces;i;i=i->next) {
+		if (check_mask) {
+			if (same_net(ip, (struct sockaddr *)&i->ip, (struct sockaddr *)&i->netmask)) {
+				return i;
+			}
+		} else if (sockaddr_equal((struct sockaddr *)&i->ip, ip)) {
+			return i;
+		}
+	}
 
 	return NULL;
 }
 
-
 /****************************************************************************
 add an interface to the linked list of interfaces
 ****************************************************************************/
-static void add_interface(TALLOC_CTX *mem_ctx, struct in_addr ip, struct in_addr nmask, struct interface **interfaces)
+static void add_interface(TALLOC_CTX *mem_ctx, const struct iface_struct *ifs, struct interface **interfaces,
+			  bool enable_ipv6)
 {
+	char addr[INET6_ADDRSTRLEN];
 	struct interface *iface;
-	struct in_addr bcast;
 
-	if (iface_find(*interfaces, ip, false)) {
-		DEBUG(3,("not adding duplicate interface %s\n",inet_ntoa(ip)));
+	if (iface_list_find(*interfaces, (const struct sockaddr *)&ifs->ip, false)) {
+		DEBUG(3,("add_interface: not adding duplicate interface %s\n",
+			print_sockaddr(addr, sizeof(addr), &ifs->ip) ));
+		return;
+	}
+
+	if (!(ifs->flags & (IFF_BROADCAST|IFF_LOOPBACK))) {
+		DEBUG(3,("not adding non-broadcast interface %s\n",
+					ifs->name ));
+		return;
+	}
+
+	if (!enable_ipv6 && ifs->ip.ss_family != AF_INET) {
 		return;
 	}
 
@@ -79,26 +103,40 @@ static void add_interface(TALLOC_CTX *mem_ctx, struct in_addr ip, struct in_addr
 	
 	ZERO_STRUCTPN(iface);
 
-	iface->ip = ip;
-	iface->nmask = nmask;
-	bcast.s_addr = MKBCADDR(iface->ip.s_addr, iface->nmask.s_addr);
+	iface->name = talloc_strdup(iface, ifs->name);
+	if (!iface->name) {
+		SAFE_FREE(iface);
+		return;
+	}
+	iface->flags = ifs->flags;
+	iface->ip = ifs->ip;
+	iface->netmask = ifs->netmask;
+	iface->bcast = ifs->bcast;
 
 	/* keep string versions too, to avoid people tripping over the implied
 	   static in inet_ntoa() */
-	iface->ip_s = talloc_strdup(iface, inet_ntoa(iface->ip));
-	iface->nmask_s = talloc_strdup(iface, inet_ntoa(iface->nmask));
-	
-	if (nmask.s_addr != ~0) {
-		iface->bcast_s = talloc_strdup(iface, inet_ntoa(bcast));
-	}
-
-	DLIST_ADD_END(*interfaces, iface, struct interface *);
-
-	DEBUG(3,("added interface ip=%s nmask=%s\n", iface->ip_s, iface->nmask_s));
+	print_sockaddr(addr, sizeof(addr), &iface->ip);
+	DEBUG(4,("added interface %s ip=%s ",
+		 iface->name, addr));
+	iface->ip_s = talloc_strdup(iface, addr);
+
+	print_sockaddr(addr, sizeof(addr),
+		       &iface->bcast);
+	DEBUG(4,("bcast=%s ", addr));
+	iface->bcast_s = talloc_strdup(iface, addr);
+
+	print_sockaddr(addr, sizeof(addr),
+		       &iface->netmask);
+	DEBUG(4,("netmask=%s\n", addr));
+	iface->nmask_s = talloc_strdup(iface, addr);
+
+	/*
+	   this needs to be a ADD_END, as some tests (such as the
+	   spoolss notify test) depend on the interfaces ordering
+	*/
+	DLIST_ADD_END(*interfaces, iface, NULL);
 }
 
-
-
 /**
 interpret a single element from a interfaces= config line 
 
@@ -114,99 +152,155 @@ static void interpret_interface(TALLOC_CTX *mem_ctx,
 				const char *token, 
 				struct iface_struct *probed_ifaces, 
 				int total_probed,
-				struct interface **local_interfaces)
+				struct interface **local_interfaces,
+				bool enable_ipv6)
 {
-	struct in_addr ip, nmask;
+	struct sockaddr_storage ss;
+	struct sockaddr_storage ss_mask;
+	struct sockaddr_storage ss_net;
+	struct sockaddr_storage ss_bcast;
+	struct iface_struct ifs;
 	char *p;
-	char *address;
-	int i, added=0;
+	int i;
+	bool added=false;
+	bool goodaddr = false;
 
-	ip.s_addr = 0;
-	nmask.s_addr = 0;
-	
 	/* first check if it is an interface name */
 	for (i=0;i<total_probed;i++) {
 		if (gen_fnmatch(token, probed_ifaces[i].name) == 0) {
-			add_interface(mem_ctx, probed_ifaces[i].ip,
-				      probed_ifaces[i].netmask,
-				      local_interfaces);
-			added = 1;
+			add_interface(mem_ctx, &probed_ifaces[i],
+				      local_interfaces, enable_ipv6);
+			added = true;
 		}
 	}
-	if (added) return;
+	if (added) {
+		return;
+	}
 
 	/* maybe it is a DNS name */
 	p = strchr_m(token,'/');
-	if (!p) {
-		/* don't try to do dns lookups on wildcard names */
-		if (strpbrk(token, "*?") != NULL) {
+	if (p == NULL) {
+		if (!interpret_string_addr(&ss, token, 0)) {
+			DEBUG(2, ("interpret_interface: Can't find address "
+				  "for %s\n", token));
 			return;
 		}
-		ip.s_addr = interpret_addr2(token).s_addr;
+
 		for (i=0;i<total_probed;i++) {
-			if (ip.s_addr == probed_ifaces[i].ip.s_addr) {
-				add_interface(mem_ctx, probed_ifaces[i].ip,
-					      probed_ifaces[i].netmask,
-					      local_interfaces);
+			if (sockaddr_equal((struct sockaddr *)&ss, (struct sockaddr *)&probed_ifaces[i].ip)) {
+				add_interface(mem_ctx, &probed_ifaces[i],
+					      local_interfaces, enable_ipv6);
 				return;
 			}
 		}
-		DEBUG(2,("can't determine netmask for %s\n", token));
+		DEBUG(2,("interpret_interface: "
+			"can't determine interface for %s\n",
+			token));
 		return;
 	}
 
-	address = talloc_strdup(mem_ctx, token);
-	p = strchr_m(address,'/');
-
 	/* parse it into an IP address/netmasklength pair */
-	*p++ = 0;
-
-	ip.s_addr = interpret_addr2(address).s_addr;
+	*p = 0;
+	goodaddr = interpret_string_addr(&ss, token, 0);
+	*p++ = '/';
+
+	if (!goodaddr) {
+		DEBUG(2,("interpret_interface: "
+			"can't determine interface for %s\n",
+			token));
+		return;
+	}
 
 	if (strlen(p) > 2) {
-		nmask.s_addr = interpret_addr2(p).s_addr;
+		goodaddr = interpret_string_addr(&ss_mask, p, 0);
+		if (!goodaddr) {
+			DEBUG(2,("interpret_interface: "
+				"can't determine netmask from %s\n",
+				p));
+			return;
+		}
 	} else {
-		nmask.s_addr = htonl(((ALLONES >> atoi(p)) ^ ALLONES));
+		char *endp = NULL;
+		unsigned long val = strtoul(p, &endp, 0);
+		if (p == endp || (endp && *endp != '\0')) {
+			DEBUG(2,("interpret_interface: "
+				"can't determine netmask value from %s\n",
+				p));
+			return;
+		}
+		if (!make_netmask(&ss_mask, &ss, val)) {
+			DEBUG(2,("interpret_interface: "
+				"can't apply netmask value %lu from %s\n",
+				val,
+				p));
+			return;
+		}
 	}
 
-	/* maybe the first component was a broadcast address */
-	if (ip.s_addr == MKBCADDR(ip.s_addr, nmask.s_addr) ||
-	    ip.s_addr == MKNETADDR(ip.s_addr, nmask.s_addr)) {
+	make_bcast(&ss_bcast, &ss, &ss_mask);
+	make_net(&ss_net, &ss, &ss_mask);
+
+	/* Maybe the first component was a broadcast address. */
+	if (sockaddr_equal((struct sockaddr *)&ss_bcast, (struct sockaddr *)&ss) ||
+		sockaddr_equal((struct sockaddr *)&ss_net, (struct sockaddr *)&ss)) {
 		for (i=0;i<total_probed;i++) {
-			if (same_net_v4(ip, probed_ifaces[i].ip, nmask)) {
-				add_interface(mem_ctx, probed_ifaces[i].ip, nmask,
-					      local_interfaces);
-				talloc_free(address);
+			if (same_net((struct sockaddr *)&ss,
+						 (struct sockaddr *)&probed_ifaces[i].ip,
+						 (struct sockaddr *)&ss_mask)) {
+				/* Temporarily replace netmask on
+				 * the detected interface - user knows
+				 * best.... */
+				struct sockaddr_storage saved_mask =
+					probed_ifaces[i].netmask;
+				probed_ifaces[i].netmask = ss_mask;
+				DEBUG(2,("interpret_interface: "
+					"using netmask value %s from "
+					"config file on interface %s\n",
+					p,
+					probed_ifaces[i].name));
+				add_interface(mem_ctx, &probed_ifaces[i],
+					      local_interfaces, enable_ipv6);
+				probed_ifaces[i].netmask = saved_mask;
 				return;
 			}
 		}
-		DEBUG(2,("Can't determine ip for broadcast address %s\n", address));
-		talloc_free(address);
+		DEBUG(2,("interpret_interface: Can't determine ip for "
+			"broadcast address %s\n",
+			token));
 		return;
 	}
 
-	add_interface(mem_ctx, ip, nmask, local_interfaces);
-	talloc_free(address);
+	/* Just fake up the interface definition. User knows best. */
+
+	DEBUG(2,("interpret_interface: Adding interface %s\n",
+		token));
+
+	ZERO_STRUCT(ifs);
+	(void)strlcpy(ifs.name, token, sizeof(ifs.name));
+	ifs.flags = IFF_BROADCAST;
+	ifs.ip = ss;
+	ifs.netmask = ss_mask;
+	ifs.bcast = ss_bcast;
+	add_interface(mem_ctx, &ifs,
+		      local_interfaces, enable_ipv6);
 }
 
 
 /**
 load the list of network interfaces
 **/
-void load_interfaces(TALLOC_CTX *mem_ctx, const char **interfaces, struct interface **local_interfaces)
+void load_interface_list(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx, struct interface **local_interfaces)
 {
-	const char **ptr = interfaces;
+	const char **ptr = lpcfg_interfaces(lp_ctx);
 	int i;
-	struct iface_struct ifaces[MAX_INTERFACES];
-	struct in_addr loopback_ip;
+	struct iface_struct *ifaces;
 	int total_probed;
+	bool enable_ipv6 = lpcfg_parm_bool(lp_ctx, NULL, "ipv6", "enable", true);
 
 	*local_interfaces = NULL;
 
-	loopback_ip = interpret_addr2("127.0.0.1");
-
 	/* probe the kernel for interfaces */
-	total_probed = get_interfaces(ifaces, MAX_INTERFACES);
+	total_probed = get_interfaces(mem_ctx, &ifaces);
 
 	/* if we don't have a interfaces line then use all interfaces
 	   except loopback */
@@ -215,27 +309,27 @@ void load_interfaces(TALLOC_CTX *mem_ctx, const char **interfaces, struct interf
 			DEBUG(0,("ERROR: Could not determine network interfaces, you must use a interfaces config line\n"));
 		}
 		for (i=0;i<total_probed;i++) {
-			if (ifaces[i].ip.s_addr != loopback_ip.s_addr) {
-				add_interface(mem_ctx, ifaces[i].ip, 
-					      ifaces[i].netmask, local_interfaces);
+			if (!is_loopback_addr((struct sockaddr *)&ifaces[i].ip)) {
+				add_interface(mem_ctx, &ifaces[i], local_interfaces, enable_ipv6);
 			}
 		}
 	}
 
 	while (ptr && *ptr) {
-		interpret_interface(mem_ctx, *ptr, ifaces, total_probed, local_interfaces);
+		interpret_interface(mem_ctx, *ptr, ifaces, total_probed, local_interfaces, enable_ipv6);
 		ptr++;
 	}
 
 	if (!*local_interfaces) {
 		DEBUG(0,("WARNING: no network interfaces found\n"));
 	}
+	talloc_free(ifaces);
 }
 
 /**
   how many interfaces do we have
   **/
-int iface_count(struct interface *ifaces)
+int iface_list_count(struct interface *ifaces)
 {
 	int ret = 0;
 	struct interface *i;
@@ -248,7 +342,7 @@ int iface_count(struct interface *ifaces)
 /**
   return IP of the Nth interface
   **/
-const char *iface_n_ip(struct interface *ifaces, int n)
+const char *iface_list_n_ip(struct interface *ifaces, int n)
 {
 	struct interface *i;
   
@@ -261,10 +355,59 @@ const char *iface_n_ip(struct interface *ifaces, int n)
 	return NULL;
 }
 
+
+/**
+  return the first IPv4 interface address we have registered
+  **/
+const char *iface_list_first_v4(struct interface *ifaces)
+{
+	struct interface *i;
+
+	for (i=ifaces; i; i=i->next) {
+		if (i->ip.ss_family == AF_INET) {
+			return i->ip_s;
+		}
+	}
+	return NULL;
+}
+
+/**
+  return the first IPv6 interface address we have registered
+  **/
+static const char *iface_list_first_v6(struct interface *ifaces)
+{
+	struct interface *i;
+
+#ifdef HAVE_IPV6
+	for (i=ifaces; i; i=i->next) {
+		if (i->ip.ss_family == AF_INET6) {
+			return i->ip_s;
+		}
+	}
+#endif
+	return NULL;
+}
+
+/**
+   check if an interface is IPv4
+  **/
+bool iface_list_n_is_v4(struct interface *ifaces, int n)
+{
+	struct interface *i;
+
+	for (i=ifaces;i && n;i=i->next)
+		n--;
+
+	if (i) {
+		return i->ip.ss_family == AF_INET;
+	}
+	return false;
+}
+
 /**
   return bcast of the Nth interface
   **/
-const char *iface_n_bcast(struct interface *ifaces, int n)
+const char *iface_list_n_bcast(struct interface *ifaces, int n)
 {
 	struct interface *i;
   
@@ -280,7 +423,7 @@ const char *iface_n_bcast(struct interface *ifaces, int n)
 /**
   return netmask of the Nth interface
   **/
-const char *iface_n_netmask(struct interface *ifaces, int n)
+const char *iface_list_n_netmask(struct interface *ifaces, int n)
 {
 	struct interface *i;
   
@@ -297,28 +440,37 @@ const char *iface_n_netmask(struct interface *ifaces, int n)
   return the local IP address that best matches a destination IP, or
   our first interface if none match
 */
-const char *iface_best_ip(struct interface *ifaces, const char *dest)
+const char *iface_list_best_ip(struct interface *ifaces, const char *dest)
 {
 	struct interface *iface;
-	struct in_addr ip;
+	struct sockaddr_storage ss;
 
-	ip.s_addr = interpret_addr(dest);
-	iface = iface_find(ifaces, ip, true);
+	if (!interpret_string_addr(&ss, dest, AI_NUMERICHOST)) {
+		return iface_list_n_ip(ifaces, 0);
+	}
+	iface = iface_list_find(ifaces, (const struct sockaddr *)&ss, true);
 	if (iface) {
 		return iface->ip_s;
 	}
-	return iface_n_ip(ifaces, 0);
+#ifdef HAVE_IPV6
+	if (ss.ss_family == AF_INET6) {
+		return iface_list_first_v6(ifaces);
+	}
+#endif
+	return iface_list_first_v4(ifaces);
 }
 
 /**
   return true if an IP is one one of our local networks
 */
-bool iface_is_local(struct interface *ifaces, const char *dest)
+bool iface_list_is_local(struct interface *ifaces, const char *dest)
 {
-	struct in_addr ip;
+	struct sockaddr_storage ss;
 
-	ip.s_addr = interpret_addr(dest);
-	if (iface_find(ifaces, ip, true)) {
+	if (!interpret_string_addr(&ss, dest, AI_NUMERICHOST)) {
+		return false;
+	}
+	if (iface_list_find(ifaces, (const struct sockaddr *)&ss, true)) {
 		return true;
 	}
 	return false;
@@ -327,9 +479,50 @@ bool iface_is_local(struct interface *ifaces, const char *dest)
 /**
   return true if a IP matches a IP/netmask pair
 */
-bool iface_same_net(const char *ip1, const char *ip2, const char *netmask)
+bool iface_list_same_net(const char *ip1, const char *ip2, const char *netmask)
 {
-	return same_net_v4(interpret_addr2(ip1),
-			interpret_addr2(ip2),
-			interpret_addr2(netmask));
+	struct sockaddr_storage ip1_ss, ip2_ss, nm_ss;
+
+	if (!interpret_string_addr(&ip1_ss, ip1, AI_NUMERICHOST)) {
+		return false;
+	}
+	if (!interpret_string_addr(&ip2_ss, ip2, AI_NUMERICHOST)) {
+		return false;
+	}
+	if (!interpret_string_addr(&nm_ss, netmask, AI_NUMERICHOST)) {
+		return false;
+	}
+
+	return same_net((struct sockaddr *)&ip1_ss,
+			(struct sockaddr *)&ip2_ss,
+			(struct sockaddr *)&nm_ss);
+}
+
+/**
+   return the list of wildcard interfaces
+   this will include the IPv4 0.0.0.0, and may include IPv6 ::
+   it is overridden by the 'socket address' option in smb.conf
+*/
+const char **iface_list_wildcard(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
+{
+	const char **ret;
+	const char *socket_address;
+
+	/* the user may have configured a specific address */
+	socket_address = lpcfg_socket_address(lp_ctx);
+	if (strcmp(socket_address, "") != 0) {
+		ret = (const char **)str_list_make(mem_ctx, socket_address, NULL);
+		return ret;
+	}
+
+	ret = (const char **)str_list_make(mem_ctx, "0.0.0.0", NULL);
+	if (ret == NULL) return NULL;
+
+#ifdef HAVE_IPV6
+	if (lpcfg_parm_bool(lp_ctx, NULL, "ipv6", "enable", true)) {
+		return str_list_add(ret, "::");
+	}
+#endif
+
+	return ret;
 }
diff --git a/source4/lib/socket/netif.c b/source4/lib/socket/netif.c
deleted file mode 100644
index e36f268bde1..00000000000
--- a/source4/lib/socket/netif.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/* 
-   Unix SMB/CIFS implementation.
-   return a list of network interfaces
-   Copyright (C) Andrew Tridgell 1998
-   Copyright (C) Jeremy Allison 2007
-   Copyright (C) Jelmer Vernooij 2007
-   
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-
-/* working out the interfaces for a OS is an incredibly non-portable
-   thing. We have several possible implementations below, and autoconf
-   tries each of them to see what works
-
-   Note that this file does _not_ include includes.h. That is so this code
-   can be called directly from the autoconf tests. That also means
-   this code cannot use any of the normal Samba debug stuff or defines.
-   This is standalone code.
-
-*/
-
-#include "includes.h"
-#include "system/network.h"
-#include "netif.h"
-#include "lib/util/tsort.h"
-
-/****************************************************************************
- Try the "standard" getifaddrs/freeifaddrs interfaces.
- Also gets IPv6 interfaces.
-****************************************************************************/
-
-/****************************************************************************
- Get the netmask address for a local interface.
-****************************************************************************/
-
-static int _get_interfaces(struct iface_struct *ifaces, int max_interfaces)
-{
-	struct ifaddrs *iflist = NULL;
-	struct ifaddrs *ifptr = NULL;
-	int total = 0;
-
-	if (getifaddrs(&iflist) < 0) {
-		return -1;
-	}
-
-	/* Loop through interfaces, looking for given IP address */
-	for (ifptr = iflist, total = 0;
-			ifptr != NULL && total < max_interfaces;
-			ifptr = ifptr->ifa_next) {
-
-		memset(&ifaces[total], '\0', sizeof(ifaces[total]));
-
-		if (!ifptr->ifa_addr || !ifptr->ifa_netmask) {
-			continue;
-		}
-
-		/* Check the interface is up. */
-		if (!(ifptr->ifa_flags & IFF_UP)) {
-			continue;
-		}
-
-		/* We don't support IPv6 *yet* */
-		if (ifptr->ifa_addr->sa_family != AF_INET) {
-			continue;
-		}
-
-		ifaces[total].ip = ((struct sockaddr_in *)ifptr->ifa_addr)->sin_addr;
-		ifaces[total].netmask = ((struct sockaddr_in *)ifptr->ifa_netmask)->sin_addr;
-
-		strlcpy(ifaces[total].name, ifptr->ifa_name,
-			sizeof(ifaces[total].name));
-		total++;
-	}
-
-	freeifaddrs(iflist);
-
-	return total;
-}
-
-static int iface_comp(struct iface_struct *i1, struct iface_struct *i2)
-{
-	int r;
-	r = strcmp(i1->name, i2->name);
-	if (r) return r;
-	r = ntohl(i1->ip.s_addr) - ntohl(i2->ip.s_addr);
-	if (r) return r;
-	r = ntohl(i1->netmask.s_addr) - ntohl(i2->netmask.s_addr);
-	return r;
-}
-
-/* this wrapper is used to remove duplicates from the interface list generated
-   above */
-int get_interfaces(struct iface_struct *ifaces, int max_interfaces)
-{
-	int total, i, j;
-
-	total = _get_interfaces(ifaces, max_interfaces);
-	if (total <= 0) return total;
-
-	/* now we need to remove duplicates */
-	TYPESAFE_QSORT(ifaces, total, iface_comp);
-
-	for (i=1;i<total;) {
-		if (iface_comp(&ifaces[i-1], &ifaces[i]) == 0) {
-			for (j=i-1;j<total-1;j++) {
-				ifaces[j] = ifaces[j+1];
-			}
-			total--;
-		} else {
-			i++;
-		}
-	}
-
-	return total;
-}
diff --git a/source4/lib/socket/netif.h b/source4/lib/socket/netif.h
index 417c6e074fd..1d90a4fd139 100644
--- a/source4/lib/socket/netif.h
+++ b/source4/lib/socket/netif.h
@@ -20,17 +20,5 @@
 */
 
 #include "system/network.h"
-
-struct iface_struct {
-	char name[16];
-	struct in_addr ip;
-	struct in_addr netmask;
-};
-
-struct interface;
-
-#define MAX_INTERFACES 128
-
-#ifndef AUTOCONF_TEST
+#include "lib/socket/interfaces.h"
 #include "lib/socket/netif_proto.h"
-#endif
diff --git a/source4/lib/socket/socket.c b/source4/lib/socket/socket.c
index 4b5cecab343..2dbdaad11df 100644
--- a/source4/lib/socket/socket.c
+++ b/source4/lib/socket/socket.c
@@ -451,7 +451,7 @@ _PUBLIC_ NTSTATUS socket_dup(struct socket_context *sock)
 	}
 	fd = dup(sock->fd);
 	if (fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	close(sock->fd);
 	sock->fd = fd;
@@ -473,6 +473,11 @@ _PUBLIC_ struct socket_address *socket_address_from_strings(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
+	if (strcmp(family, "ip") == 0 && is_ipaddress_v6(host)) {
+		/* leaving as "ip" would force IPv4 */
+		family = "ipv6";
+	}
+
 	addr->family = family;
 	addr->addr = talloc_strdup(addr, host);
 	if (!addr->addr) {
@@ -498,7 +503,19 @@ _PUBLIC_ struct socket_address *socket_address_from_sockaddr(TALLOC_CTX *mem_ctx
 	if (!addr) {
 		return NULL;
 	}
-	addr->family = NULL; 
+	switch (sockaddr->sa_family) {
+	case AF_INET:
+		addr->family = "ipv4";
+		break;
+#ifdef HAVE_IPV6
+	case AF_INET6:
+		addr->family = "ipv6";
+		break;
+#endif
+	case AF_UNIX:
+		addr->family = "unix";
+		break;
+	}
 	addr->addr = NULL;
 	addr->port = 0;
 	addr->sockaddr = (struct sockaddr *)talloc_memdup(addr, sockaddr, sockaddrlen);
@@ -510,6 +527,50 @@ _PUBLIC_ struct socket_address *socket_address_from_sockaddr(TALLOC_CTX *mem_ctx
 	return addr;
 }
 
+
+/*
+   Create a new socket_address from sockaddr_storage
+ */
+_PUBLIC_ struct socket_address *socket_address_from_sockaddr_storage(TALLOC_CTX *mem_ctx,
+								     const struct sockaddr_storage *sockaddr,
+	uint16_t port)
+{
+	struct socket_address *addr = talloc_zero(mem_ctx, struct socket_address);
+	char addr_str[INET6_ADDRSTRLEN+1];
+	const char *str;
+
+	if (!addr) {
+		return NULL;
+	}
+	addr->port = port;
+	switch (sockaddr->ss_family) {
+	case AF_INET:
+		addr->family = "ipv4";
+		break;
+#ifdef HAVE_IPV6
+	case AF_INET6:
+		addr->family = "ipv6";
+		break;
+#endif
+	default:
+		talloc_free(addr);
+		return NULL;
+	}
+
+	str = print_sockaddr(addr_str, sizeof(addr_str), sockaddr);
+	if (str == NULL) {
+		talloc_free(addr);
+		return NULL;
+	}
+	addr->addr = talloc_strdup(addr, str);
+	if (addr->addr == NULL) {
+		talloc_free(addr);
+		return NULL;
+	}
+
+	return addr;
+}
+
 /* Copy a socket_address structure */
 struct socket_address *socket_address_copy(TALLOC_CTX *mem_ctx,
 					   const struct socket_address *oaddr)
@@ -567,110 +628,6 @@ _PUBLIC_ const struct socket_ops *socket_getops_byname(const char *family, enum
 	return NULL;
 }
 
-enum SOCK_OPT_TYPES {OPT_BOOL,OPT_INT,OPT_ON};
-
-static const struct {
-	const char *name;
-	int level;
-	int option;
-	int value;
-	int opttype;
-} socket_options[] = {
-  {"SO_KEEPALIVE",      SOL_SOCKET,    SO_KEEPALIVE,    0,                 OPT_BOOL},
-  {"SO_REUSEADDR",      SOL_SOCKET,    SO_REUSEADDR,    0,                 OPT_BOOL},
-  {"SO_BROADCAST",      SOL_SOCKET,    SO_BROADCAST,    0,                 OPT_BOOL},
-#ifdef TCP_NODELAY
-  {"TCP_NODELAY",       IPPROTO_TCP,   TCP_NODELAY,     0,                 OPT_BOOL},
-#endif
-#ifdef IPTOS_LOWDELAY
-  {"IPTOS_LOWDELAY",    IPPROTO_IP,    IP_TOS,          IPTOS_LOWDELAY,    OPT_ON},
-#endif
-#ifdef IPTOS_THROUGHPUT
-  {"IPTOS_THROUGHPUT",  IPPROTO_IP,    IP_TOS,          IPTOS_THROUGHPUT,  OPT_ON},
-#endif
-#ifdef SO_REUSEPORT
-  {"SO_REUSEPORT",      SOL_SOCKET,    SO_REUSEPORT,    0,                 OPT_BOOL},
-#endif
-#ifdef SO_SNDBUF
-  {"SO_SNDBUF",         SOL_SOCKET,    SO_SNDBUF,       0,                 OPT_INT},
-#endif
-#ifdef SO_RCVBUF
-  {"SO_RCVBUF",         SOL_SOCKET,    SO_RCVBUF,       0,                 OPT_INT},
-#endif
-#ifdef SO_SNDLOWAT
-  {"SO_SNDLOWAT",       SOL_SOCKET,    SO_SNDLOWAT,     0,                 OPT_INT},
-#endif
-#ifdef SO_RCVLOWAT
-  {"SO_RCVLOWAT",       SOL_SOCKET,    SO_RCVLOWAT,     0,                 OPT_INT},
-#endif
-#ifdef SO_SNDTIMEO
-  {"SO_SNDTIMEO",       SOL_SOCKET,    SO_SNDTIMEO,     0,                 OPT_INT},
-#endif
-#ifdef SO_RCVTIMEO
-  {"SO_RCVTIMEO",       SOL_SOCKET,    SO_RCVTIMEO,     0,                 OPT_INT},
-#endif
-  {NULL,0,0,0,0}};
-
-
-/**
- Set user socket options.
-**/
-_PUBLIC_ void set_socket_options(int fd, const char *options)
-{
-	const char **options_list = (const char **)str_list_make(NULL, options, " \t,");
-	int j;
-
-	if (!options_list)
-		return;
-
-	for (j = 0; options_list[j]; j++) {
-		const char *tok = options_list[j];
-		int ret=0,i;
-		int value = 1;
-		char *p;
-		bool got_value = false;
-
-		if ((p = strchr(tok,'='))) {
-			*p = 0;
-			value = atoi(p+1);
-			got_value = true;
-		}
-
-		for (i=0;socket_options[i].name;i++)
-			if (strequal(socket_options[i].name,tok))
-				break;
-
-		if (!socket_options[i].name) {
-			DEBUG(0,("Unknown socket option %s\n",tok));
-			continue;
-		}
-
-		switch (socket_options[i].opttype) {
-		case OPT_BOOL:
-		case OPT_INT:
-			ret = setsockopt(fd,socket_options[i].level,
-						socket_options[i].option,(char *)&value,sizeof(int));
-			break;
-
-		case OPT_ON:
-			if (got_value)
-				DEBUG(0,("syntax error - %s does not take a value\n",tok));
-
-			{
-				int on = socket_options[i].value;
-				ret = setsockopt(fd,socket_options[i].level,
-							socket_options[i].option,(char *)&on,sizeof(int));
-			}
-			break;	  
-		}
-      
-		if (ret != 0)
-			DEBUG(0,("Failed to set socket option %s (Error %s)\n",tok, strerror(errno) ));
-	}
-
-	talloc_free(options_list);
-}
-
 /*
   set some flags on a socket 
  */
diff --git a/source4/lib/socket/socket.h b/source4/lib/socket/socket.h
index 4a744797b3a..e00b61ba6cd 100644
--- a/source4/lib/socket/socket.h
+++ b/source4/lib/socket/socket.h
@@ -174,14 +174,18 @@ struct socket_address *socket_address_from_strings(TALLOC_CTX *mem_ctx,
 struct socket_address *socket_address_from_sockaddr(TALLOC_CTX *mem_ctx, 
 						    struct sockaddr *sockaddr, 
 						    size_t addrlen);
+struct sockaddr_storage;
+struct socket_address *socket_address_from_sockaddr_storage(TALLOC_CTX *mem_ctx,
+							    const struct sockaddr_storage *sockaddr,
+							    uint16_t port);
 _PUBLIC_ void socket_address_set_port(struct socket_address *a,
 				      uint16_t port);
 struct socket_address *socket_address_copy(TALLOC_CTX *mem_ctx,
 					   const struct socket_address *oaddr);
 const struct socket_ops *socket_getops_byname(const char *name, enum socket_type type);
-bool allow_access(TALLOC_CTX *mem_ctx,
-		  const char **deny_list, const char **allow_list,
-		  const char *cname, const char *caddr);
+bool socket_allow_access(TALLOC_CTX *mem_ctx,
+			 const char **deny_list, const char **allow_list,
+			 const char *cname, const char *caddr);
 bool socket_check_access(struct socket_context *sock, 
 			 const char *service_name,
 			 const char **allow_list, const char **deny_list);
diff --git a/source4/lib/socket/socket_ip.c b/source4/lib/socket/socket_ip.c
index 4e666532520..80f7d333f38 100644
--- a/source4/lib/socket/socket_ip.c
+++ b/source4/lib/socket/socket_ip.c
@@ -47,7 +47,7 @@ static NTSTATUS ipv4_init(struct socket_context *sock)
 
 	sock->fd = socket(PF_INET, type, 0);
 	if (sock->fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	sock->backend_name = "ipv4";
@@ -70,16 +70,16 @@ static NTSTATUS ip_connect_complete(struct socket_context *sock, uint32_t flags)
 	   for non-blocking connect */
 	ret = getsockopt(sock->fd, SOL_SOCKET, SO_ERROR, &error, &len);
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	if (error != 0) {
-		return map_nt_error_from_unix(error);
+		return map_nt_error_from_unix_common(error);
 	}
 
 	if (!(flags & SOCKET_FLAG_BLOCK)) {
 		ret = set_blocking(sock->fd, false);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -102,7 +102,7 @@ static NTSTATUS ipv4_connect(struct socket_context *sock,
 	if (my_address && my_address->sockaddr) {
 		ret = bind(sock->fd, my_address->sockaddr, my_address->sockaddrlen);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	} else if (my_address) {
 		my_ip = interpret_addr2(my_address->addr);
@@ -119,7 +119,7 @@ static NTSTATUS ipv4_connect(struct socket_context *sock,
 			
 			ret = bind(sock->fd, (struct sockaddr *)&my_addr, sizeof(my_addr));
 			if (ret == -1) {
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 		}
 	}
@@ -127,7 +127,7 @@ static NTSTATUS ipv4_connect(struct socket_context *sock,
 	if (srv_address->sockaddr) {
 		ret = connect(sock->fd, srv_address->sockaddr, srv_address->sockaddrlen);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	} else {
 		srv_ip = interpret_addr2(srv_address->addr);
@@ -147,7 +147,7 @@ static NTSTATUS ipv4_connect(struct socket_context *sock,
 
 		ret = connect(sock->fd, (const struct sockaddr *)&srv_addr, sizeof(srv_addr));
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -186,20 +186,20 @@ static NTSTATUS ipv4_listen(struct socket_context *sock,
 	}
 
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (sock->type == SOCKET_TYPE_STREAM) {
 		ret = listen(sock->fd, queue_size);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
 	if (!(flags & SOCKET_FLAG_BLOCK)) {
 		ret = set_blocking(sock->fd, false);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -220,14 +220,14 @@ static NTSTATUS ipv4_accept(struct socket_context *sock, struct socket_context *
 
 	new_fd = accept(sock->fd, (struct sockaddr *)&cli_addr, &cli_addr_len);
 	if (new_fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (!(sock->flags & SOCKET_FLAG_BLOCK)) {
 		int ret = set_blocking(new_fd, false);
 		if (ret == -1) {
 			close(new_fd);
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -268,7 +268,7 @@ static NTSTATUS ip_recv(struct socket_context *sock, void *buf,
 	if (gotlen == 0) {
 		return NT_STATUS_END_OF_FILE;
 	} else if (gotlen == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	*nread = gotlen;
@@ -311,7 +311,7 @@ static NTSTATUS ipv4_recvfrom(struct socket_context *sock, void *buf,
 		return NT_STATUS_END_OF_FILE;
 	} else if (gotlen == -1) {
 		talloc_free(src);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	src->sockaddrlen = from_len;
@@ -342,7 +342,7 @@ static NTSTATUS ip_send(struct socket_context *sock,
 
 	len = send(sock->fd, blob->data, blob->length, 0);
 	if (len == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}	
 
 	*sendlen = len;
@@ -383,7 +383,7 @@ static NTSTATUS ipv4_sendto(struct socket_context *sock,
 			     (struct sockaddr *)&srv_addr, sizeof(srv_addr));
 	}
 	if (len == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}	
 
 	*sendlen = len;
@@ -518,7 +518,7 @@ static NTSTATUS ip_pending(struct socket_context *sock, size_t *npending)
 		*npending = value;
 		return NT_STATUS_OK;
 	}
-	return map_nt_error_from_unix(errno);
+	return map_nt_error_from_unix_common(errno);
 }
 
 static const struct socket_ops ipv4_ops = {
@@ -604,7 +604,7 @@ static NTSTATUS ipv6_init(struct socket_context *sock)
 
 	sock->fd = socket(PF_INET6, type, 0);
 	if (sock->fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	sock->backend_name = "ipv6";
@@ -623,7 +623,7 @@ static NTSTATUS ipv6_tcp_connect(struct socket_context *sock,
 	if (my_address && my_address->sockaddr) {
 		ret = bind(sock->fd, my_address->sockaddr, my_address->sockaddrlen);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	} else if (my_address) {
 		struct in6_addr my_ip;
@@ -638,7 +638,7 @@ static NTSTATUS ipv6_tcp_connect(struct socket_context *sock,
 			
 			ret = bind(sock->fd, (struct sockaddr *)&my_addr, sizeof(my_addr));
 			if (ret == -1) {
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 		}
 	}
@@ -661,15 +661,28 @@ static NTSTATUS ipv6_tcp_connect(struct socket_context *sock,
 		ret = connect(sock->fd, (const struct sockaddr *)&srv_addr, sizeof(srv_addr));
 	}
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return ip_connect_complete(sock, flags);
 }
 
+/*
+  fix the sin6_scope_id based on the address interface
+ */
+static void fix_scope_id(struct sockaddr_in6 *in6,
+			 const char *address)
+{
+	const char *p = strchr(address, '%');
+	if (p != NULL) {
+		in6->sin6_scope_id = if_nametoindex(p+1);
+	}
+}
+
+
 static NTSTATUS ipv6_listen(struct socket_context *sock,
-				const struct socket_address *my_address,
-				int queue_size, uint32_t flags)
+			    const struct socket_address *my_address,
+			    int queue_size, uint32_t flags)
 {
 	struct sockaddr_in6 my_addr;
 	struct in6_addr ip_addr;
@@ -680,31 +693,38 @@ static NTSTATUS ipv6_listen(struct socket_context *sock,
 	if (my_address->sockaddr) {
 		ret = bind(sock->fd, my_address->sockaddr, my_address->sockaddrlen);
 	} else {
+		int one = 1;
 		ip_addr = interpret_addr6(my_address->addr);
 		
 		ZERO_STRUCT(my_addr);
 		my_addr.sin6_addr	= ip_addr;
 		my_addr.sin6_port	= htons(my_address->port);
 		my_addr.sin6_family	= PF_INET6;
-		
-		ret = bind(sock->fd, (struct sockaddr *)&my_addr, sizeof(my_addr));
+		fix_scope_id(&my_addr, my_address->addr);
+
+		/* when binding on ipv6 we always want to only bind on v6 */
+		ret = setsockopt(sock->fd, IPPROTO_IPV6, IPV6_V6ONLY,
+				 (const void *)&one, sizeof(one));
+		if (ret != -1) {
+			ret = bind(sock->fd, (struct sockaddr *)&my_addr, sizeof(my_addr));
+		}
 	}
 
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (sock->type == SOCKET_TYPE_STREAM) {
 		ret = listen(sock->fd, queue_size);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
 	if (!(flags & SOCKET_FLAG_BLOCK)) {
 		ret = set_blocking(sock->fd, false);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -725,14 +745,14 @@ static NTSTATUS ipv6_tcp_accept(struct socket_context *sock, struct socket_conte
 
 	new_fd = accept(sock->fd, (struct sockaddr *)&cli_addr, &cli_addr_len);
 	if (new_fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (!(sock->flags & SOCKET_FLAG_BLOCK)) {
 		int ret = set_blocking(new_fd, false);
 		if (ret == -1) {
 			close(new_fd);
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -796,7 +816,7 @@ static NTSTATUS ipv6_recvfrom(struct socket_context *sock, void *buf,
 		return NT_STATUS_END_OF_FILE;
 	} else if (gotlen == -1) {
 		talloc_free(src);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	src->sockaddrlen = from_len;
@@ -847,7 +867,7 @@ static NTSTATUS ipv6_sendto(struct socket_context *sock,
 			     (struct sockaddr *)&srv_addr, sizeof(srv_addr));
 	}
 	if (len == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}	
 
 	*sendlen = len;
diff --git a/source4/lib/socket/socket_unix.c b/source4/lib/socket/socket_unix.c
index f1fa0a3a30c..d492f012681 100644
--- a/source4/lib/socket/socket_unix.c
+++ b/source4/lib/socket/socket_unix.c
@@ -33,7 +33,7 @@ _PUBLIC_ const struct socket_ops *socket_unixdom_ops(enum socket_type type);
 */
 static NTSTATUS unixdom_error(int ernum)
 {
-	return map_nt_error_from_unix(ernum);
+	return map_nt_error_from_unix_common(ernum);
 }
 
 static NTSTATUS unixdom_init(struct socket_context *sock)
@@ -53,7 +53,7 @@ static NTSTATUS unixdom_init(struct socket_context *sock)
 
 	sock->fd = socket(PF_UNIX, type, 0);
 	if (sock->fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	sock->private_data = NULL;
 
@@ -76,16 +76,16 @@ static NTSTATUS unixdom_connect_complete(struct socket_context *sock, uint32_t f
 	   for non-blocking connect */
 	ret = getsockopt(sock->fd, SOL_SOCKET, SO_ERROR, &error, &len);
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	if (error != 0) {
-		return map_nt_error_from_unix(error);
+		return map_nt_error_from_unix_common(error);
 	}
 
 	if (!(flags & SOCKET_FLAG_BLOCK)) {
 		ret = set_blocking(sock->fd, false);
 		if (ret == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -194,7 +194,7 @@ static NTSTATUS unixdom_accept(struct socket_context *sock,
 		int ret = set_blocking(new_fd, false);
 		if (ret == -1) {
 			close(new_fd);
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 	}
 
@@ -280,7 +280,7 @@ static NTSTATUS unixdom_sendto(struct socket_context *sock,
 			     (struct sockaddr *)&srv_addr, sizeof(srv_addr));
 	}
 	if (len == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}	
 
 	*sendlen = len;
@@ -390,7 +390,7 @@ static NTSTATUS unixdom_pending(struct socket_context *sock, size_t *npending)
 		*npending = value;
 		return NT_STATUS_OK;
 	}
-	return map_nt_error_from_unix(errno);
+	return map_nt_error_from_unix_common(errno);
 }
 
 static const struct socket_ops unixdom_ops = {
diff --git a/source4/lib/socket/testsuite.c b/source4/lib/socket/testsuite.c
index 2489277433f..357e4ae5df4 100644
--- a/source4/lib/socket/testsuite.c
+++ b/source4/lib/socket/testsuite.c
@@ -42,7 +42,7 @@ static bool test_udp(struct torture_context *tctx)
 	TALLOC_CTX *mem_ctx = tctx;
 	struct interface *ifaces;
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
 
 	status = socket_create("ip", SOCKET_TYPE_DGRAM, &sock1, 0);
 	torture_assert_ntstatus_ok(tctx, status, "creating DGRAM IP socket 1");
@@ -53,7 +53,7 @@ static bool test_udp(struct torture_context *tctx)
 	talloc_steal(mem_ctx, sock2);
 
 	localhost = socket_address_from_strings(sock1, sock1->backend_name, 
-						iface_best_ip(ifaces, "127.0.0.1"), 0);
+						iface_list_best_ip(ifaces, "127.0.0.1"), 0);
 
 	torture_assert(tctx, localhost, "Localhost not found");
 
@@ -62,10 +62,10 @@ static bool test_udp(struct torture_context *tctx)
 
 	srv_addr = socket_get_my_addr(sock1, mem_ctx);
 	torture_assert(tctx, srv_addr != NULL && 
-		       strcmp(srv_addr->addr, iface_best_ip(ifaces, "127.0.0.1")) == 0,
+		       strcmp(srv_addr->addr, iface_list_best_ip(ifaces, "127.0.0.1")) == 0,
 				   talloc_asprintf(tctx, 
 		"Expected server address of %s but got %s",
-		      iface_best_ip(ifaces, "127.0.0.1"), srv_addr ? srv_addr->addr : NULL));
+		      iface_list_best_ip(ifaces, "127.0.0.1"), srv_addr ? srv_addr->addr : NULL));
 
 	torture_comment(tctx, "server port is %d\n", srv_addr->port);
 
@@ -135,9 +135,9 @@ static bool test_tcp(struct torture_context *tctx)
 	torture_assert_ntstatus_ok(tctx, status, "creating IP stream socket 1");
 	talloc_steal(mem_ctx, sock2);
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
 	localhost = socket_address_from_strings(sock1, sock1->backend_name, 
-						iface_best_ip(ifaces, "127.0.0.1"), 0);
+						iface_list_best_ip(ifaces, "127.0.0.1"), 0);
 	torture_assert(tctx, localhost, "Localhost not found");
 
 	status = socket_listen(sock1, localhost, 0, 0);
@@ -147,7 +147,7 @@ static bool test_tcp(struct torture_context *tctx)
 	torture_assert(tctx, srv_addr && srv_addr->addr, 
 				   "Unexpected socket_get_my_addr NULL\n");
 
-	torture_assert_str_equal(tctx, srv_addr->addr, iface_best_ip(ifaces, "127.0.0.1"), 
+	torture_assert_str_equal(tctx, srv_addr->addr, iface_list_best_ip(ifaces, "127.0.0.1"),
 			"Unexpected server address");
 
 	torture_comment(tctx, "server port is %d\n", srv_addr->port);
diff --git a/source4/lib/socket/wscript_build b/source4/lib/socket/wscript_build
index e2ff9b078a7..c10970d17aa 100644
--- a/source4/lib/socket/wscript_build
+++ b/source4/lib/socket/wscript_build
@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 
 bld.SAMBA_LIBRARY('netif',
-    source='interface.c netif.c',
-    autoproto='netif_proto.h',
-    deps='samba-util',
-    private_library=True
-    )
+                  source='interface.c',
+                  deps='samba-util interfaces samba-hostconfig',
+                  private_library=True,
+                  autoproto='netif_proto.h'
+                  )
 
 bld.SAMBA_MODULE('socket_ip',
     source='socket_ip.c',
diff --git a/source4/lib/tdb_wrap.c b/source4/lib/tdb_wrap.c
deleted file mode 100644
index 97294e13d30..00000000000
--- a/source4/lib/tdb_wrap.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/* 
-   Unix SMB/CIFS implementation.
-   TDB wrap functions
-
-   Copyright (C) Andrew Tridgell 2004
-   Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2007
-   
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include <tdb.h>
-#include "../lib/util/dlinklist.h"
-#include "tdb_wrap.h"
-#include <tdb.h>
-
-static struct tdb_wrap *tdb_list;
-
-/* destroy the last connection to a tdb */
-static int tdb_wrap_destructor(struct tdb_wrap *w)
-{
-	tdb_close(w->tdb);
-	DLIST_REMOVE(tdb_list, w);
-	return 0;
-}				 
-
-/*
- Log tdb messages via DEBUG().
-*/
-static void tdb_wrap_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, 
-			 const char *format, ...) PRINTF_ATTRIBUTE(3,4);
-
-static void tdb_wrap_log(TDB_CONTEXT *tdb, enum tdb_debug_level level, 
-			 const char *format, ...)
-{
-	va_list ap;
-	char *ptr = NULL;
-	int dl;
-
-	va_start(ap, format);
-	vasprintf(&ptr, format, ap);
-	va_end(ap);
-	
-	switch (level) {
-	case TDB_DEBUG_FATAL:
-		dl = 0;
-		break;
-	case TDB_DEBUG_ERROR:
-		dl = 1;
-		break;
-	case TDB_DEBUG_WARNING:
-		dl = 2;
-		break;
-	case TDB_DEBUG_TRACE:
-		dl = 5;
-		break;
-	default:
-		dl = 0;
-	}		
-
-	if (ptr != NULL) {
-		const char *name = tdb_name(tdb);
-		DEBUG(dl, ("tdb(%s): %s", name ? name : "unnamed", ptr));
-		free(ptr);
-	}
-}
-
-
-/*
-  wrapped connection to a tdb database
-  to close just talloc_free() the tdb_wrap pointer
- */
-struct tdb_wrap *tdb_wrap_open(TALLOC_CTX *mem_ctx,
-			       const char *name, int hash_size, int tdb_flags,
-			       int open_flags, mode_t mode)
-{
-	struct tdb_wrap *w;
-	struct tdb_logging_context log_ctx;
-	log_ctx.log_fn = tdb_wrap_log;
-
-	for (w=tdb_list;w;w=w->next) {
-		if (strcmp(name, w->name) == 0) {
-			return talloc_reference(mem_ctx, w);
-		}
-	}
-
-	w = talloc(mem_ctx, struct tdb_wrap);
-	if (w == NULL) {
-		return NULL;
-	}
-
-	w->name = talloc_strdup(w, name);
-
-	w->tdb = tdb_open_ex(name, hash_size, tdb_flags, 
-			     open_flags, mode, &log_ctx, NULL);
-	if (w->tdb == NULL) {
-		talloc_free(w);
-		return NULL;
-	}
-
-	talloc_set_destructor(w, tdb_wrap_destructor);
-
-	DLIST_ADD(tdb_list, w);
-
-	return w;
-}
diff --git a/source4/lib/wscript_build b/source4/lib/wscript_build
index 872259d6700..cf60820da71 100644
--- a/source4/lib/wscript_build
+++ b/source4/lib/wscript_build
@@ -5,12 +5,3 @@ bld.SAMBA_SUBSYSTEM('GENCACHE',
         enabled=False,
 	deps='tdb-wrap'
 	)
-
-
-bld.SAMBA_LIBRARY('tdb-wrap',
-                  source='tdb_wrap.c',
-                  deps='tdb talloc samba-util',
-                  public_headers='tdb_wrap.h',
-                  private_library=True
-                  )
-
diff --git a/source4/libcli/clifile.c b/source4/libcli/clifile.c
index f521b5f4209..f5e02dd4586 100644
--- a/source4/libcli/clifile.c
+++ b/source4/libcli/clifile.c
@@ -51,34 +51,6 @@ static NTSTATUS smbcli_link_internal(struct smbcli_tree *tree,
 }
 
 /****************************************************************************
- Map standard UNIX permissions onto wire representations.
-****************************************************************************/
-uint32_t unix_perms_to_wire(mode_t perms)
-{
-        unsigned int ret = 0;
-
-        ret |= ((perms & S_IXOTH) ?  UNIX_X_OTH : 0);
-        ret |= ((perms & S_IWOTH) ?  UNIX_W_OTH : 0);
-        ret |= ((perms & S_IROTH) ?  UNIX_R_OTH : 0);
-        ret |= ((perms & S_IXGRP) ?  UNIX_X_GRP : 0);
-        ret |= ((perms & S_IWGRP) ?  UNIX_W_GRP : 0);
-        ret |= ((perms & S_IRGRP) ?  UNIX_R_GRP : 0);
-        ret |= ((perms & S_IXUSR) ?  UNIX_X_USR : 0);
-        ret |= ((perms & S_IWUSR) ?  UNIX_W_USR : 0);
-        ret |= ((perms & S_IRUSR) ?  UNIX_R_USR : 0);
-#ifdef S_ISVTX
-        ret |= ((perms & S_ISVTX) ?  UNIX_STICKY : 0);
-#endif
-#ifdef S_ISGID
-        ret |= ((perms & S_ISGID) ?  UNIX_SET_GID : 0);
-#endif
-#ifdef S_ISUID
-        ret |= ((perms & S_ISUID) ?  UNIX_SET_UID : 0);
-#endif
-        return ret;
-}
-
-/****************************************************************************
  Symlink a file (UNIX extensions).
 ****************************************************************************/
 NTSTATUS smbcli_unix_symlink(struct smbcli_tree *tree, const char *fname_src, 
diff --git a/source4/libcli/finddcs_nbt.c b/source4/libcli/finddcs_nbt.c
index 12b94f7bab3..9579a9e449f 100644
--- a/source4/libcli/finddcs_nbt.c
+++ b/source4/libcli/finddcs_nbt.c
@@ -33,7 +33,7 @@
 struct finddcs_nbt_state {
 	struct tevent_context *ev;
 	struct tevent_req *req;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 
 	const char *my_netbios_name;
 	const char *domain_name;
@@ -70,7 +70,7 @@ struct tevent_req *finddcs_nbt_send(TALLOC_CTX *mem_ctx,
 				struct dom_sid *domain_sid,
 				struct resolve_context *resolve_ctx,
 				struct tevent_context *event_ctx,
-				struct messaging_context *msg_ctx)
+				struct imessaging_context *msg_ctx)
 {
 	struct finddcs_nbt_state *state;
 	struct nbt_name name;
@@ -295,7 +295,7 @@ NTSTATUS finddcs_nbt(TALLOC_CTX *mem_ctx,
 		 struct dom_sid *domain_sid,
 		 struct resolve_context *resolve_ctx,
 		 struct tevent_context *event_ctx,
-		 struct messaging_context *msg_ctx,
+		 struct imessaging_context *msg_ctx,
 		 int *num_dcs, struct nbt_dc_name **dcs)
 {
 	struct tevent_req *req = finddcs_nbt_send(mem_ctx,
diff --git a/source4/libcli/ldap/ldap_bind.c b/source4/libcli/ldap/ldap_bind.c
index 65659794c51..f5cf25ec3f7 100644
--- a/source4/libcli/ldap/ldap_bind.c
+++ b/source4/libcli/ldap/ldap_bind.c
@@ -221,7 +221,7 @@ _PUBLIC_ NTSTATUS ldap_bind_sasl(struct ldap_connection *conn,
 		NULL 
 	};
 
-	gensec_init(lp_ctx);
+	gensec_init();
 
 	status = gensec_client_start(conn, &conn->gensec,
 				     conn->event.event_ctx, 
diff --git a/source4/libcli/rap/rap.c b/source4/libcli/rap/rap.c
index 5ea9e816e81..7743f64c4cc 100644
--- a/source4/libcli/rap/rap.c
+++ b/source4/libcli/rap/rap.c
@@ -1633,3 +1633,50 @@ NTSTATUS smbcli_rap_netuserdelete(struct smbcli_tree *tree,
 	talloc_free(call);
 	return result;
 }
+
+NTSTATUS smbcli_rap_netremotetod(struct smbcli_tree *tree,
+				  TALLOC_CTX *mem_ctx,
+				  struct rap_NetRemoteTOD *r)
+{
+	struct rap_call *call;
+	NTSTATUS result = NT_STATUS_UNSUCCESSFUL;
+
+	if (!(call = new_rap_cli_call(mem_ctx, RAP_NetRemoteTOD))) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	rap_cli_push_rcvbuf(call, r->in.bufsize);
+
+	rap_cli_expect_format(call, "DDBBBBWWBBWB");
+	rap_cli_expect_extra_format(call, "");
+
+	if (DEBUGLEVEL >= 10) {
+		NDR_PRINT_IN_DEBUG(rap_NetRemoteTOD, r);
+	}
+
+	result = rap_cli_do_call(tree, call);
+
+	if (!NT_STATUS_IS_OK(result))
+		goto done;
+
+	result = NT_STATUS_INVALID_PARAMETER;
+
+	NDR_GOTO(ndr_pull_rap_status(call->ndr_pull_param, NDR_SCALARS, &r->out.status));
+	NDR_GOTO(ndr_pull_uint16(call->ndr_pull_param, NDR_SCALARS, &r->out.convert));
+
+	NDR_GOTO(ndr_pull_rap_TimeOfDayInfo(call->ndr_pull_data, NDR_SCALARS|NDR_BUFFERS, &r->out.tod));
+
+	result = NT_STATUS_OK;
+
+	if (!NT_STATUS_IS_OK(result)) {
+		goto done;
+	}
+
+	if (DEBUGLEVEL >= 10) {
+		NDR_PRINT_OUT_DEBUG(rap_NetRemoteTOD, r);
+	}
+
+ done:
+	talloc_free(call);
+	return result;
+}
diff --git a/source4/libcli/raw/clisocket.c b/source4/libcli/raw/clisocket.c
index 84bf250f6a1..bf608f1ec15 100644
--- a/source4/libcli/raw/clisocket.c
+++ b/source4/libcli/raw/clisocket.c
@@ -229,7 +229,7 @@ _PUBLIC_ struct smbcli_socket *smbcli_sock_connect_byname(const char *host, cons
 
 	make_nbt_name(&nbt_name, host, name_type);
 	
-	status = resolve_name(resolve_ctx, &nbt_name, tmp_ctx, &address, event_ctx);
+	status = resolve_name_ex(resolve_ctx, 0, 0, &nbt_name, tmp_ctx, &address, event_ctx);
 	if (!NT_STATUS_IS_OK(status)) {
 		talloc_free(tmp_ctx);
 		return NULL;
diff --git a/source4/libcli/raw/clitransport.c b/source4/libcli/raw/clitransport.c
index 7a3993c79bf..43316692a41 100644
--- a/source4/libcli/raw/clitransport.c
+++ b/source4/libcli/raw/clitransport.c
@@ -338,7 +338,7 @@ _PUBLIC_ void smbcli_transport_idle_handler(struct smbcli_transport *transport,
 
 	transport->socket->event.te = event_add_timed(transport->socket->event.ctx, 
 						      transport,
-						      timeval_current_ofs(0, period),
+						      timeval_current_ofs_usec(period),
 						      idle_handler, transport);
 }
 
diff --git a/source4/libcli/raw/interfaces.h b/source4/libcli/raw/interfaces.h
index f3ce4e9c5f2..d20367474a3 100644
--- a/source4/libcli/raw/interfaces.h
+++ b/source4/libcli/raw/interfaces.h
@@ -924,8 +924,8 @@ enum smb_setfileinfo_level {
 	RAW_SFILEINFO_END_OF_FILE_INFO        = SMB_SFILEINFO_END_OF_FILE_INFO,
 	RAW_SFILEINFO_UNIX_BASIC              = SMB_SFILEINFO_UNIX_BASIC,
 	RAW_SFILEINFO_UNIX_INFO2              = SMB_SFILEINFO_UNIX_INFO2,
-	RAW_SFILEINFO_UNIX_LINK               = SMB_SFILEINFO_UNIX_LINK,
-	RAW_SFILEINFO_UNIX_HLINK	      = SMB_SFILEINFO_UNIX_HLINK,
+	RAW_SFILEINFO_UNIX_LINK               = SMB_SET_FILE_UNIX_LINK,
+	RAW_SFILEINFO_UNIX_HLINK	      = SMB_SET_FILE_UNIX_HLINK,
 	RAW_SFILEINFO_BASIC_INFORMATION       = SMB_SFILEINFO_BASIC_INFORMATION,
 	RAW_SFILEINFO_RENAME_INFORMATION      = SMB_SFILEINFO_RENAME_INFORMATION,
 	RAW_SFILEINFO_LINK_INFORMATION        = SMB_SFILEINFO_LINK_INFORMATION,
diff --git a/source4/libcli/raw/rawfile.c b/source4/libcli/raw/rawfile.c
index 5797540edd0..1cacaab5cfd 100644
--- a/source4/libcli/raw/rawfile.c
+++ b/source4/libcli/raw/rawfile.c
@@ -29,52 +29,6 @@
 	if (!req) return NULL; \
 } while (0)
 
-/**
- Return a string representing a CIFS attribute for a file.
-**/
-char *attrib_string(TALLOC_CTX *mem_ctx, uint32_t attrib)
-{
-	int i, len;
-	const struct {
-		char c;
-		uint16_t attr;
-	} attr_strs[] = {
-		{'V', FILE_ATTRIBUTE_VOLUME},
-		{'D', FILE_ATTRIBUTE_DIRECTORY},
-		{'A', FILE_ATTRIBUTE_ARCHIVE},
-		{'H', FILE_ATTRIBUTE_HIDDEN},
-		{'S', FILE_ATTRIBUTE_SYSTEM},
-		{'N', FILE_ATTRIBUTE_NORMAL},
-		{'R', FILE_ATTRIBUTE_READONLY},
-		{'d', FILE_ATTRIBUTE_DEVICE},
-		{'t', FILE_ATTRIBUTE_TEMPORARY},
-		{'s', FILE_ATTRIBUTE_SPARSE},
-		{'r', FILE_ATTRIBUTE_REPARSE_POINT},
-		{'c', FILE_ATTRIBUTE_COMPRESSED},
-		{'o', FILE_ATTRIBUTE_OFFLINE},
-		{'n', FILE_ATTRIBUTE_NONINDEXED},
-		{'e', FILE_ATTRIBUTE_ENCRYPTED}
-	};
-	char *ret;
-
-	ret = talloc_array(mem_ctx, char, ARRAY_SIZE(attr_strs)+1);
-	if (!ret) {
-		return NULL;
-	}
-
-	for (len=i=0; i<ARRAY_SIZE(attr_strs); i++) {
-		if (attrib & attr_strs[i].attr) {
-			ret[len++] = attr_strs[i].c;
-		}
-	}
-
-	ret[len] = 0;
-
-	talloc_set_name_const(ret, ret);
-
-	return ret;
-}
-
 /****************************************************************************
  Rename a file - async interface
 ****************************************************************************/
@@ -950,6 +904,7 @@ struct smbcli_request *smb_raw_lock_send(struct smbcli_tree *tree, union smb_loc
 		break;
 	}
 	case RAW_LOCK_SMB2:
+	case RAW_LOCK_SMB2_BREAK:
 		return NULL;
 	}
 
diff --git a/source4/libcli/raw/rawsetfileinfo.c b/source4/libcli/raw/rawsetfileinfo.c
index 6ad3e9ee8d9..ff36d50e94e 100644
--- a/source4/libcli/raw/rawsetfileinfo.c
+++ b/source4/libcli/raw/rawsetfileinfo.c
@@ -262,11 +262,11 @@ static bool smb_raw_setinfo_backend(struct smbcli_tree *tree,
 		return smb_raw_setfileinfo_passthru(mem_ctx, parms->generic.level,
 						    parms, blob);
 
-		/* Unhandled levels */
-
+	/* Unhandled levels */
 	case RAW_SFILEINFO_UNIX_LINK:
 	case RAW_SFILEINFO_UNIX_HLINK:
 	case RAW_SFILEINFO_RENAME_INFORMATION_SMB2:
+	case RAW_SFILEINFO_LINK_INFORMATION:
 		break;
 	}
 
diff --git a/source4/libcli/raw/smb.h b/source4/libcli/raw/smb.h
index 05bf91fe3bb..008ab57c0de 100644
--- a/source4/libcli/raw/smb.h
+++ b/source4/libcli/raw/smb.h
@@ -23,8 +23,8 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#ifndef _SMB_H
-#define _SMB_H
+#ifndef _RAW_SMB_H
+#define _RAW_SMB_H
 
 /* deny modes */
 #define DENY_DOS 0
@@ -366,24 +366,6 @@
 #define DESIRED_ACCESS_PIPE 0x2019f
  
 
-/* FileAttributes (search attributes) field */
-#define FILE_ATTRIBUTE_READONLY		0x0001
-#define FILE_ATTRIBUTE_HIDDEN		0x0002
-#define FILE_ATTRIBUTE_SYSTEM		0x0004
-#define FILE_ATTRIBUTE_VOLUME		0x0008
-#define FILE_ATTRIBUTE_DIRECTORY	0x0010
-#define FILE_ATTRIBUTE_ARCHIVE		0x0020
-#define FILE_ATTRIBUTE_DEVICE		0x0040
-#define FILE_ATTRIBUTE_NORMAL		0x0080
-#define FILE_ATTRIBUTE_TEMPORARY	0x0100
-#define FILE_ATTRIBUTE_SPARSE		0x0200
-#define FILE_ATTRIBUTE_REPARSE_POINT	0x0400
-#define FILE_ATTRIBUTE_COMPRESSED	0x0800
-#define FILE_ATTRIBUTE_OFFLINE		0x1000
-#define FILE_ATTRIBUTE_NONINDEXED	0x2000
-#define FILE_ATTRIBUTE_ENCRYPTED	0x4000
-#define FILE_ATTRIBUTE_ALL_MASK 	0x7FFF
-
 /* Flags - combined with attributes. */
 #define FILE_FLAG_WRITE_THROUGH    0x80000000L
 #define FILE_FLAG_NO_BUFFERING     0x20000000L
@@ -472,7 +454,7 @@
 
 /* where to find the base of the SMB packet proper */
 /* REWRITE TODO: smb_base needs to be removed */
-#define smb_base(buf) (((char *)(buf))+4)
+#define smb_base(buf) (((const char *)(buf))+4)
 
 /* we don't allow server strings to be longer than 48 characters as
    otherwise NT will not honour the announce packets */
@@ -600,4 +582,4 @@
 #include "libcli/raw/interfaces.h"
 #include "libcli/smb/smb_common.h"
 
-#endif /* _SMB_H */
+#endif /* _RAW_SMB_H */
diff --git a/source4/libcli/raw/trans2.h b/source4/libcli/raw/trans2.h
index 63632eb5edb..d9a3f5f7d2c 100644
--- a/source4/libcli/raw/trans2.h
+++ b/source4/libcli/raw/trans2.h
@@ -316,161 +316,4 @@ Found 0 aliased levels
 #define MAX_MAC_INFO_LEVEL                      0x3FF
 #define SMB_QFS_MAC_FS_INFO                     0x301
 
-
-
-/* UNIX CIFS Extensions - created by HP */
-/*
- * UNIX CIFS Extensions have the range 0x200 - 0x2FF reserved.
- * Supposedly Microsoft have agreed to this.
- */
-
-#define MIN_UNIX_INFO_LEVEL 0x200
-#define MAX_UNIX_INFO_LEVEL 0x2FF
-
-#define INFO_LEVEL_IS_UNIX(level) (((level) >= MIN_UNIX_INFO_LEVEL) && ((level) <= MAX_UNIX_INFO_LEVEL))
-
-#define SMB_MODE_NO_CHANGE                 0xFFFFFFFF     /* file mode value which */
-                                              /* means "don't change it" */
-#define SMB_UID_NO_CHANGE                  0xFFFFFFFF
-#define SMB_GID_NO_CHANGE                  0xFFFFFFFF
-
-#define SMB_SIZE_NO_CHANGE_LO              0xFFFFFFFF
-#define SMB_SIZE_NO_CHANGE_HI              0xFFFFFFFF
- 
-#define SMB_TIME_NO_CHANGE_LO              0xFFFFFFFF
-#define SMB_TIME_NO_CHANGE_HI              0xFFFFFFFF
-
-/*
-UNIX_BASIC  info level:
-
-Offset Size         Name
-0      LARGE_INTEGER EndOfFile                File size
-8      LARGE_INTEGER Blocks                   Number of bytes used on disk (st_blocks).
-16     LARGE_INTEGER CreationTime             Creation time
-24     LARGE_INTEGER LastAccessTime           Last access time
-32     LARGE_INTEGER LastModificationTime     Last modification time
-40     LARGE_INTEGER Uid                      Numeric user id for the owner
-48     LARGE_INTEGER Gid                      Numeric group id of owner
-56     ULONG Type                             Enumeration specifying the pathname type:
-                                              0 -- File
-                                              1 -- Directory
-                                              2 -- Symbolic link
-                                              3 -- Character device
-                                              4 -- Block device
-                                              5 -- FIFO (named pipe)
-                                              6 -- Unix domain socket
-
-60     LARGE_INTEGER devmajor                 Major device number if type is device
-68     LARGE_INTEGER devminor                 Minor device number if type is device
-76     LARGE_INTEGER uniqueid                 This is a server-assigned unique id for the file. The client
-                                              will typically map this onto an inode number. The scope of
-                                              uniqueness is the share.
-84     LARGE_INTEGER permissions              Standard UNIX file permissions  - see below.
-92     LARGE_INTEGER nlinks                   The number of directory entries that map to this entry
-                                              (number of hard links)
-
-100 - end.
-*/
-
-/*
-SMB_QUERY_FILE_UNIX_INFO2 is SMB_QUERY_FILE_UNIX_BASIC with create
-time and file flags appended. The corresponding info level for
-findfirst/findnext is SMB_FIND_FILE_UNIX_UNIX2.
-
-Size    Offset  Value
----------------------
-0      LARGE_INTEGER EndOfFile  	File size
-8      LARGE_INTEGER Blocks     	Number of blocks used on disk
-16     LARGE_INTEGER ChangeTime 	Attribute change time
-24     LARGE_INTEGER LastAccessTime           Last access time
-32     LARGE_INTEGER LastModificationTime     Last modification time
-40     LARGE_INTEGER Uid        	Numeric user id for the owner
-48     LARGE_INTEGER Gid        	Numeric group id of owner
-56     ULONG Type               	Enumeration specifying the file type
-60     LARGE_INTEGER devmajor   	Major device number if type is device
-68     LARGE_INTEGER devminor   	Minor device number if type is device
-76     LARGE_INTEGER uniqueid   	This is a server-assigned unique id
-84     LARGE_INTEGER permissions        Standard UNIX permissions
-92     LARGE_INTEGER nlinks     	Number of hard link)
-100    LARGE_INTEGER CreationTime       Create/birth time
-108    ULONG FileFlags          	File flags enumeration
-112    ULONG FileFlagsMask      	Mask of valid flags
-*/
-
-/* UNIX filetype mappings. */
-
-#define UNIX_TYPE_FILE      0
-#define UNIX_TYPE_DIR       1
-#define UNIX_TYPE_SYMLINK   2
-#define UNIX_TYPE_CHARDEV   3
-#define UNIX_TYPE_BLKDEV    4
-#define UNIX_TYPE_FIFO      5
-#define UNIX_TYPE_SOCKET    6
-#define UNIX_TYPE_UNKNOWN   0xFFFFFFFF
-
-/*
- * Oh this is fun. "Standard UNIX permissions" has no
- * meaning in POSIX. We need to define the mapping onto
- * and off the wire as this was not done in the original HP
- * spec. JRA.
- */
-
-#define UNIX_X_OTH			0000001
-#define UNIX_W_OTH			0000002
-#define UNIX_R_OTH			0000004
-#define UNIX_X_GRP			0000010
-#define UNIX_W_GRP                      0000020
-#define UNIX_R_GRP                      0000040
-#define UNIX_X_USR                      0000100
-#define UNIX_W_USR                      0000200
-#define UNIX_R_USR                      0000400
-#define UNIX_STICKY                     0001000
-#define UNIX_SET_GID                    0002000
-#define UNIX_SET_UID                    0004000
-
-/* Masks for the above */
-#define UNIX_OTH_MASK                   0000007
-#define UNIX_GRP_MASK                   0000070
-#define UNIX_USR_MASK                   0000700
-#define UNIX_PERM_MASK                  0000777
-#define UNIX_EXTRA_MASK                 0007000
-#define UNIX_ALL_MASK                   0007777
-
-/* Flags for the file_flags field in UNIX_INFO2: */
-#define EXT_SECURE_DELETE               0x00000001
-#define EXT_ENABLE_UNDELETE             0x00000002
-#define EXT_SYNCHRONOUS                 0x00000004
-#define EXT_IMMUTABLE			0x00000008
-#define EXT_OPEN_APPEND_ONLY            0x00000010
-#define EXT_DO_NOT_BACKUP               0x00000020
-#define EXT_NO_UPDATE_ATIME             0x00000040
-#define EXT_HIDDEN                      0x00000080
-
-#define SMB_QFILEINFO_UNIX_LINK         0x201
-#define SMB_SFILEINFO_UNIX_LINK         0x201
-#define SMB_SFILEINFO_UNIX_HLINK        0x203
-
-/*
- Info level for QVOLINFO - returns version of CIFS UNIX extensions, plus
- 64-bits worth of capability fun :-).
-*/
-
-#define SMB_QUERY_CIFS_UNIX_INFO      0x200
-
-/* Returns the following.
-
-  UINT16             major version number
-  UINT16             minor version number
-  LARGE_INTEGER      capability bitfield
-
-*/
-
-#define CIFS_UNIX_MAJOR_VERSION 1
-#define CIFS_UNIX_MINOR_VERSION 0
-
-#define CIFS_UNIX_FCNTL_LOCKS_CAP           0x1
-#define CIFS_UNIX_POSIX_ACLS_CAP            0x2
-
-/* ... more as we think of them :-). */
-
 #endif
diff --git a/source4/libcli/resolve/bcast.c b/source4/libcli/resolve/bcast.c
index 0ad3e1230b5..150705f8d8f 100644
--- a/source4/libcli/resolve/bcast.c
+++ b/source4/libcli/resolve/bcast.c
@@ -47,13 +47,13 @@ struct composite_context *resolve_name_bcast_send(TALLOC_CTX *mem_ctx,
 	int i, count=0;
 	struct resolve_bcast_data *data = talloc_get_type(userdata, struct resolve_bcast_data);
 
-	num_interfaces = iface_count(data->ifaces);
+	num_interfaces = iface_list_count(data->ifaces);
 
 	address_list = talloc_array(mem_ctx, const char *, num_interfaces+1);
 	if (address_list == NULL) return NULL;
 
 	for (i=0;i<num_interfaces;i++) {
-		const char *bcast = iface_n_bcast(data->ifaces, i);
+		const char *bcast = iface_list_n_bcast(data->ifaces, i);
 		if (bcast == NULL) continue;
 		address_list[count] = talloc_strdup(address_list, bcast);
 		if (address_list[count] == NULL) {
@@ -101,6 +101,6 @@ bool resolve_context_add_bcast_method(struct resolve_context *ctx, struct interf
 bool resolve_context_add_bcast_method_lp(struct resolve_context *ctx, struct loadparm_context *lp_ctx)
 {
 	struct interface *ifaces;
-	load_interfaces(ctx, lpcfg_interfaces(lp_ctx), &ifaces);
+	load_interface_list(ctx, lp_ctx, &ifaces);
 	return resolve_context_add_bcast_method(ctx, ifaces, lpcfg_nbt_port(lp_ctx), lpcfg_parm_int(lp_ctx, NULL, "nbt", "timeout", 1));
 }
diff --git a/source4/libcli/resolve/dns_ex.c b/source4/libcli/resolve/dns_ex.c
index 35e2ad74c31..59212125793 100644
--- a/source4/libcli/resolve/dns_ex.c
+++ b/source4/libcli/resolve/dns_ex.c
@@ -37,6 +37,7 @@
 #include "libcli/composite/composite.h"
 #include "librpc/gen_ndr/ndr_nbt.h"
 #include "libcli/resolve/resolve.h"
+#include "lib/util/util_net.h"
 
 #ifdef class
 #undef class
@@ -87,6 +88,7 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 	struct rk_resource_record **srv_rr;
 	uint32_t addrs_valid = 0;
 	struct rk_resource_record **addrs_rr;
+	struct rk_dns_reply **srv_replies = NULL;
 	char *addrs;
 	bool first;
 	uint32_t i;
@@ -135,14 +137,13 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 				continue;
 			}
 		} else {
-			/* we are only interested in A records */
-			/* TODO: add AAAA support */
-			if (rr->type != rk_ns_t_a) {
+			/* we are only interested in A or AAAA records */
+			if (rr->type != rk_ns_t_a && rr->type != rk_ns_t_aaaa) {
 				continue;
 			}
 
-			/* verify we actually have a A record here */
-			if (!rr->u.a) {
+			/* verify we actually have a record here */
+			if (!rr->u.data) {
 				continue;
 			}
 		}
@@ -167,6 +168,13 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 		goto done;
 	}
 
+	srv_replies = talloc_zero_array(state,
+					struct rk_dns_reply *,
+				     	count);
+	if (!srv_replies) {
+		goto done;
+	}
+
 	/* Loop over all returned records and pick the records */
 	for (rr=reply->head;rr;rr=rr->next) {
 		/* we are only interested in the IN class */
@@ -193,14 +201,13 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 			srv_rr[srv_valid] = rr;
 			srv_valid++;
 		} else {
-			/* we are only interested in A records */
-			/* TODO: add AAAA support */
-			if (rr->type != rk_ns_t_a) {
+			/* we are only interested in A or AAAA records */
+			if (rr->type != rk_ns_t_a && rr->type != rk_ns_t_aaaa) {
 				continue;
 			}
 
-			/* verify we actually have a A record here */
-			if (!rr->u.a) {
+			/* verify we actually have a record record here */
+			if (!rr->u.data) {
 				continue;
 			}
 
@@ -210,19 +217,23 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 	}
 
 	for (i=0; i < srv_valid; i++) {
-		for (rr=reply->head;rr;rr=rr->next) {
+		srv_replies[i] = rk_dns_lookup(srv_rr[i]->u.srv->target, "A");
+		if (srv_replies[i] == NULL)
+			continue;
 
+		/* Add first A record to addrs_rr */
+		for (rr=srv_replies[i]->head;rr;rr=rr->next) {
 			if (rr->class != rk_ns_c_in) {
 				continue;
 			}
 
-			/* we are only interested in A records */
-			if (rr->type != rk_ns_t_a) {
+			/* we are only interested in A or AAAA records */
+			if (rr->type != rk_ns_t_a && rr->type != rk_ns_t_aaaa) {
 				continue;
 			}
 
-			/* verify we actually have a srv record here */
-			if (strcmp(&srv_rr[i]->u.srv->target[0], rr->domain) != 0) {
+			/* verify we actually have a record here */
+			if (!rr->u.data) {
 				continue;
 			}
 
@@ -241,8 +252,10 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 		goto done;
 	}
 	first = true;
-	for (i=0; i < count; i++) {
+	for (i=0; i < addrs_valid; i++) {
 		uint16_t port;
+		char addrstr[INET6_ADDRSTRLEN];
+
 		if (!addrs_rr[i]) {
 			continue;
 		}
@@ -254,9 +267,28 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 			port = state->port;
 		}
 
-		addrs = talloc_asprintf_append_buffer(addrs, "%s%s:%u/%s",
+		switch (addrs_rr[i]->type) {
+		case rk_ns_t_a:
+			if (inet_ntop(AF_INET, addrs_rr[i]->u.a,
+				      addrstr, sizeof(addrstr)) == NULL) {
+				continue;
+			}
+			break;
+#ifdef HAVE_IPV6
+		case rk_ns_t_aaaa:
+			if (inet_ntop(AF_INET6, (struct in6_addr *)addrs_rr[i]->u.data,
+				      addrstr, sizeof(addrstr)) == NULL) {
+				continue;
+			}
+			break;
+#endif
+		default:
+			continue;
+		}
+
+		addrs = talloc_asprintf_append_buffer(addrs, "%s%s@%u/%s",
 						      first?"":",",
-						      inet_ntoa(*addrs_rr[i]->u.a),
+						      addrstr,
 						      port,
 						      addrs_rr[i]->domain);
 		if (!addrs) {
@@ -270,6 +302,12 @@ static void run_child_dns_lookup(struct dns_ex_state *state, int fd)
 	}
 
 done:
+	if (reply != NULL)
+		rk_dns_free_data(reply);
+	for (i=0; i < srv_valid; i++) {
+		if (srv_replies[i] != NULL)
+			rk_dns_free_data(srv_replies[i]);
+	}
 	close(fd);
 }
 
@@ -287,7 +325,6 @@ static void run_child_getaddrinfo(struct dns_ex_state *state, int fd)
 
 	ZERO_STRUCT(hints);
 	hints.ai_socktype = SOCK_STREAM;
-	hints.ai_family = AF_INET;/* TODO: add AF_INET6 support */
 	hints.ai_flags = AI_ADDRCONFIG | AI_NUMERICSERV;
 
 	ret = getaddrinfo(state->name.name, "0", &hints, &res_list);
@@ -315,16 +352,13 @@ static void run_child_getaddrinfo(struct dns_ex_state *state, int fd)
 	}
 	first = true;
 	for (res = res_list; res; res = res->ai_next) {
-		struct sockaddr_in *in;
-
-		if (res->ai_family != AF_INET) {
+		char addrstr[INET6_ADDRSTRLEN];
+		if (!print_sockaddr_len(addrstr, sizeof(addrstr), (struct sockaddr *)res->ai_addr, res->ai_addrlen)) {
 			continue;
 		}
-		in = (struct sockaddr_in *)res->ai_addr;
-
-		addrs = talloc_asprintf_append_buffer(addrs, "%s%s:%u/%s",
+		addrs = talloc_asprintf_append_buffer(addrs, "%s%s@%u/%s",
 						      first?"":",",
-						      inet_ntoa(in->sin_addr),
+						      addrstr,
 						      state->port,
 						      state->name.name);
 		if (!addrs) {
@@ -406,7 +440,7 @@ static void pipe_handler(struct tevent_context *ev, struct tevent_fd *fde,
 
 	for (i=0; i < num_addrs; i++) {
 		uint32_t port = 0;
-		char *p = strrchr(addrs[i], ':');
+		char *p = strrchr(addrs[i], '@');
 		char *n;
 
 		if (!p) {
@@ -426,8 +460,7 @@ static void pipe_handler(struct tevent_context *ev, struct tevent_fd *fde,
 		*n = '\0';
 		n++;
 
-		if (strcmp(addrs[i], "0.0.0.0") == 0 ||
-		    inet_addr(addrs[i]) == INADDR_NONE) {
+		if (strcmp(addrs[i], "0.0.0.0") == 0) {
 			composite_error(c, NT_STATUS_OBJECT_NAME_NOT_FOUND);
 			return;
 		}
@@ -437,7 +470,7 @@ static void pipe_handler(struct tevent_context *ev, struct tevent_fd *fde,
 			return;
 		}
 		state->addrs[i] = socket_address_from_strings(state->addrs,
-							      "ipv4",
+							      "ip",
 							      addrs[i],
 							      port);
 		if (composite_nomem(state->addrs[i], c)) return;
@@ -485,7 +518,7 @@ struct composite_context *resolve_name_dns_ex_send(TALLOC_CTX *mem_ctx,
 	/* setup a pipe to chat to our child */
 	ret = pipe(fd);
 	if (ret == -1) {
-		composite_error(c, map_nt_error_from_unix(errno));
+		composite_error(c, map_nt_error_from_unix_common(errno));
 		return c;
 	}
 
@@ -509,7 +542,7 @@ struct composite_context *resolve_name_dns_ex_send(TALLOC_CTX *mem_ctx,
 
 	state->child = fork();
 	if (state->child == (pid_t)-1) {
-		composite_error(c, map_nt_error_from_unix(errno));
+		composite_error(c, map_nt_error_from_unix_common(errno));
 		return c;
 	}
 
diff --git a/source4/libcli/resolve/resolve.c b/source4/libcli/resolve/resolve.c
index c17e93a1f65..b5930aade1f 100644
--- a/source4/libcli/resolve/resolve.c
+++ b/source4/libcli/resolve/resolve.c
@@ -171,12 +171,10 @@ struct composite_context *resolve_name_all_send(struct resolve_context *ctx,
 
 	if (is_ipaddress(state->name.name) || 
 	    strcasecmp(state->name.name, "localhost") == 0) {
-		struct in_addr ip = interpret_addr2(state->name.name);
-
 		state->addrs = talloc_array(state, struct socket_address *, 2);
 		if (composite_nomem(state->addrs, c)) return c;
-		state->addrs[0] = socket_address_from_strings(state->addrs, "ipv4",
-							      inet_ntoa(ip), 0);
+		state->addrs[0] = socket_address_from_strings(state->addrs, "ip",
+							      state->name.name, 0);
 		if (composite_nomem(state->addrs[0], c)) return c;
 		state->addrs[1] = NULL;
 		state->names = talloc_array(state, char *, 2);
@@ -316,18 +314,6 @@ NTSTATUS resolve_name_ex(struct resolve_context *ctx,
 }
 
 
-/*
-  general name resolution - sync call
- */
-NTSTATUS resolve_name(struct resolve_context *ctx,
-		      struct nbt_name *name,
-		      TALLOC_CTX *mem_ctx,
-		      const char **reply_addr,
-		      struct tevent_context *ev)
-{
-	return resolve_name_ex(ctx, 0, 0, name, mem_ctx, reply_addr, ev);
-}
-
 /* Initialise a struct nbt_name with a NULL scope */
 
 void make_nbt_name(struct nbt_name *nbt, const char *name, int type)
diff --git a/source4/libcli/resolve/wins.c b/source4/libcli/resolve/wins.c
index 989f40e6cda..a52b99a23ef 100644
--- a/source4/libcli/resolve/wins.c
+++ b/source4/libcli/resolve/wins.c
@@ -77,6 +77,6 @@ bool resolve_context_add_wins_method(struct resolve_context *ctx, const char **a
 bool resolve_context_add_wins_method_lp(struct resolve_context *ctx, struct loadparm_context *lp_ctx)
 {
 	struct interface *ifaces;
-	load_interfaces(ctx, lpcfg_interfaces(lp_ctx), &ifaces);
+	load_interface_list(ctx, lp_ctx, &ifaces);
 	return resolve_context_add_wins_method(ctx, lpcfg_wins_server_list(lp_ctx), ifaces, lpcfg_nbt_port(lp_ctx), lpcfg_parm_int(lp_ctx, NULL, "nbt", "timeout", 1));
 }
diff --git a/source4/libcli/smb2/connect.c b/source4/libcli/smb2/connect.c
index 64ed6c3acce..0e3bf1512be 100644
--- a/source4/libcli/smb2/connect.c
+++ b/source4/libcli/smb2/connect.c
@@ -20,6 +20,8 @@
 */
 
 #include "includes.h"
+#include <tevent.h>
+#include "lib/util/tevent_ntstatus.h"
 #include "libcli/raw/libcliraw.h"
 #include "libcli/raw/raw_proto.h"
 #include "libcli/smb2/smb2.h"
@@ -29,6 +31,7 @@
 #include "param/param.h"
 
 struct smb2_connect_state {
+	struct tevent_context *ev;
 	struct cli_credentials *credentials;
 	struct resolve_context *resolve_ctx;
 	const char *host;
@@ -43,67 +46,167 @@ struct smb2_connect_state {
 	struct smb2_tree *tree;
 };
 
+static void smb2_connect_resolve_done(struct composite_context *creq);
+
 /*
-  continue after tcon reply
-*/
-static void continue_tcon(struct smb2_request *req)
+  a composite function that does a full negprot/sesssetup/tcon, returning
+  a connected smb2_tree
+ */
+struct tevent_req *smb2_connect_send(TALLOC_CTX *mem_ctx,
+				     struct tevent_context *ev,
+				     const char *host,
+				     const char **ports,
+				     const char *share,
+				     struct resolve_context *resolve_ctx,
+				     struct cli_credentials *credentials,
+				     struct smbcli_options *options,
+				     const char *socket_options,
+				     struct gensec_settings *gensec_settings)
 {
-	struct composite_context *c = talloc_get_type(req->async.private_data, 
-						      struct composite_context);
-	struct smb2_connect_state *state = talloc_get_type(c->private_data, 
-							   struct smb2_connect_state);
-
-	c->status = smb2_tree_connect_recv(req, &state->tcon);
-	if (!composite_is_ok(c)) return;
-	
-	state->tree->tid = state->tcon.out.tid;
+	struct tevent_req *req;
+	struct smb2_connect_state *state;
+	struct nbt_name name;
+	struct composite_context *creq;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct smb2_connect_state);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	state->ev = ev;
+	state->credentials = credentials;
+	state->options = *options;
+	state->host = host;
+	state->ports = ports;
+	state->share = share;
+	state->resolve_ctx = resolve_ctx;
+	state->socket_options = socket_options;
+	state->gensec_settings = gensec_settings;
+
+	ZERO_STRUCT(name);
+	name.name = host;
 
-	composite_done(c);
+	creq = resolve_name_send(resolve_ctx, state, &name, ev);
+	if (tevent_req_nomem(creq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	creq->async.fn = smb2_connect_resolve_done;
+	creq->async.private_data = req;
+	return req;
 }
 
-/*
-  continue after a session setup
-*/
-static void continue_session(struct composite_context *creq)
+static void smb2_connect_socket_done(struct composite_context *creq);
+
+static void smb2_connect_resolve_done(struct composite_context *creq)
 {
-	struct composite_context *c = talloc_get_type(creq->async.private_data, 
-						      struct composite_context);
-	struct smb2_connect_state *state = talloc_get_type(c->private_data, 
-							   struct smb2_connect_state);
-	struct smb2_request *req;
+	struct tevent_req *req =
+		talloc_get_type_abort(creq->async.private_data,
+		struct tevent_req);
+	struct smb2_connect_state *state =
+		tevent_req_data(req,
+		struct smb2_connect_state);
+	NTSTATUS status;
+	const char *addr;
+	const char **ports;
+	const char *default_ports[] = { "445", NULL };
 
-	c->status = smb2_session_setup_spnego_recv(creq);
-	if (!composite_is_ok(c)) return;
+	status = resolve_name_recv(creq, state, &addr);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
 
-	state->tree = smb2_tree_init(state->session, state, true);
-	if (composite_nomem(state->tree, c)) return;
+	if (state->ports == NULL) {
+		ports = default_ports;
+	} else {
+		ports = state->ports;
+	}
 
-	state->tcon.in.reserved = 0;
-	state->tcon.in.path     = talloc_asprintf(state, "\\\\%s\\%s", 
-						  state->host, state->share);
-	if (composite_nomem(state->tcon.in.path, c)) return;
-	
-	req = smb2_tree_connect_send(state->tree, &state->tcon);
-	if (composite_nomem(req, c)) return;
+	creq = smbcli_sock_connect_send(state, addr, ports,
+					state->host, state->resolve_ctx,
+					state->ev, state->socket_options);
+	if (tevent_req_nomem(creq, req)) {
+		return;
+	}
+	creq->async.fn = smb2_connect_socket_done;
+	creq->async.private_data = req;
+}
+
+static void smb2_connect_negprot_done(struct smb2_request *smb2req);
 
-	req->async.fn = continue_tcon;
-	req->async.private_data = c;	
+static void smb2_connect_socket_done(struct composite_context *creq)
+{
+	struct tevent_req *req =
+		talloc_get_type_abort(creq->async.private_data,
+		struct tevent_req);
+	struct smb2_connect_state *state =
+		tevent_req_data(req,
+		struct smb2_connect_state);
+	struct smbcli_socket *sock;
+	struct smb2_transport *transport;
+	struct smb2_request *smb2req;
+	NTSTATUS status;
+	uint16_t dialects[3] = {
+		SMB2_DIALECT_REVISION_000,
+		SMB2_DIALECT_REVISION_202,
+		SMB2_DIALECT_REVISION_210
+	};
+
+	status = smbcli_sock_connect_recv(creq, state, &sock);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	transport = smb2_transport_init(sock, state, &state->options);
+	if (tevent_req_nomem(transport, req)) {
+		return;
+	}
+
+	ZERO_STRUCT(state->negprot);
+	state->negprot.in.dialect_count = ARRAY_SIZE(dialects);
+	switch (transport->options.signing) {
+	case SMB_SIGNING_OFF:
+		state->negprot.in.security_mode = 0;
+		break;
+	case SMB_SIGNING_SUPPORTED:
+	case SMB_SIGNING_AUTO:
+		state->negprot.in.security_mode = SMB2_NEGOTIATE_SIGNING_ENABLED;
+		break;
+	case SMB_SIGNING_REQUIRED:
+		state->negprot.in.security_mode =
+			SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED;
+		break;
+	}
+	state->negprot.in.capabilities  = 0;
+	unix_to_nt_time(&state->negprot.in.start_time, time(NULL));
+	state->negprot.in.dialects = dialects;
+
+	smb2req = smb2_negprot_send(transport, &state->negprot);
+	if (tevent_req_nomem(smb2req, req)) {
+		return;
+	}
+	smb2req->async.fn = smb2_connect_negprot_done;
+	smb2req->async.private_data = req;
 }
 
-/*
-  continue after negprot reply
-*/
-static void continue_negprot(struct smb2_request *req)
+static void smb2_connect_session_done(struct tevent_req *subreq);
+
+static void smb2_connect_negprot_done(struct smb2_request *smb2req)
 {
-	struct composite_context *c = talloc_get_type(req->async.private_data, 
-						      struct composite_context);
-	struct smb2_connect_state *state = talloc_get_type(c->private_data, 
-							   struct smb2_connect_state);
-	struct smb2_transport *transport = req->transport;
-	struct composite_context *creq;
+	struct tevent_req *req =
+		talloc_get_type_abort(smb2req->async.private_data,
+		struct tevent_req);
+	struct smb2_connect_state *state =
+		tevent_req_data(req,
+		struct smb2_connect_state);
+	struct smb2_transport *transport = smb2req->transport;
+	struct tevent_req *subreq;
+	NTSTATUS status;
 
-	c->status = smb2_negprot_recv(req, c, &state->negprot);
-	if (!composite_is_ok(c)) return;
+	status = smb2_negprot_recv(smb2req, state, &state->negprot);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
 
 	transport->negotiate.secblob = state->negprot.out.secblob;
 	talloc_steal(transport, transport->negotiate.secblob.data);
@@ -115,7 +218,7 @@ static void continue_negprot(struct smb2_request *req)
 	switch (transport->options.signing) {
 	case SMB_SIGNING_OFF:
 		if (transport->negotiate.security_mode & SMB2_NEGOTIATE_SIGNING_REQUIRED) {
-			composite_error(c, NT_STATUS_ACCESS_DENIED);
+			tevent_req_nterror(req, NT_STATUS_ACCESS_DENIED);
 			return;
 		}
 		transport->signing_required = false;
@@ -138,179 +241,158 @@ static void continue_negprot(struct smb2_request *req)
 		if (transport->negotiate.security_mode & SMB2_NEGOTIATE_SIGNING_ENABLED) {
 			transport->signing_required = true;
 		} else {
-			composite_error(c, NT_STATUS_ACCESS_DENIED);
+			tevent_req_nterror(req, NT_STATUS_ACCESS_DENIED);
 			return;
 		}
 		break;
 	}
 
 	state->session = smb2_session_init(transport, state->gensec_settings, state, true);
-	if (composite_nomem(state->session, c)) return;
-
-	creq = smb2_session_setup_spnego_send(state->session, state->credentials);
-
-	composite_continue(c, creq, continue_session, c);
-}
-
-/*
-  continue after a socket connect completes
-*/
-static void continue_socket(struct composite_context *creq)
-{
-	struct composite_context *c = talloc_get_type(creq->async.private_data, 
-						      struct composite_context);
-	struct smb2_connect_state *state = talloc_get_type(c->private_data, 
-							   struct smb2_connect_state);
-	struct smbcli_socket *sock;
-	struct smb2_transport *transport;
-	struct smb2_request *req;
-	uint16_t dialects[3] = {
-		SMB2_DIALECT_REVISION_000,
-		SMB2_DIALECT_REVISION_202,
-		SMB2_DIALECT_REVISION_210
-	};
-
-	c->status = smbcli_sock_connect_recv(creq, state, &sock);
-	if (!composite_is_ok(c)) return;
-
-	transport = smb2_transport_init(sock, state, &state->options);
-	if (composite_nomem(transport, c)) return;
-
-	ZERO_STRUCT(state->negprot);
-	state->negprot.in.dialect_count = sizeof(dialects) / sizeof(dialects[0]);
-	switch (transport->options.signing) {
-	case SMB_SIGNING_OFF:
-		state->negprot.in.security_mode = 0;
-		break;
-	case SMB_SIGNING_SUPPORTED:
-	case SMB_SIGNING_AUTO:
-		state->negprot.in.security_mode = SMB2_NEGOTIATE_SIGNING_ENABLED;
-		break;
-	case SMB_SIGNING_REQUIRED:
-		state->negprot.in.security_mode = 
-			SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED;
-		break;
+	if (tevent_req_nomem(state->session, req)) {
+		return;
 	}
-	state->negprot.in.capabilities  = 0;
-	unix_to_nt_time(&state->negprot.in.start_time, time(NULL));
-	state->negprot.in.dialects = dialects;
 
-	req = smb2_negprot_send(transport, &state->negprot);
-	if (composite_nomem(req, c)) return;
-
-	req->async.fn = continue_negprot;
-	req->async.private_data = c;
+	subreq = smb2_session_setup_spnego_send(state, state->ev,
+						state->session,
+						state->credentials);
+	if (tevent_req_nomem(subreq, req)) {
+		return;
+	}
+	tevent_req_set_callback(subreq, smb2_connect_session_done, req);
 }
 
+static void smb2_connect_tcon_done(struct smb2_request *smb2req);
 
-/*
-  continue after a resolve finishes
-*/
-static void continue_resolve(struct composite_context *creq)
+static void smb2_connect_session_done(struct tevent_req *subreq)
 {
-	struct composite_context *c = talloc_get_type(creq->async.private_data, 
-						      struct composite_context);
-	struct smb2_connect_state *state = talloc_get_type(c->private_data, 
-							   struct smb2_connect_state);
-	const char *addr;
-	const char **ports;
-	const char *default_ports[] = { "445", NULL };
+	struct tevent_req *req =
+		tevent_req_callback_data(subreq,
+		struct tevent_req);
+	struct smb2_connect_state *state =
+		tevent_req_data(req,
+		struct smb2_connect_state);
+	struct smb2_request *smb2req;
+	NTSTATUS status;
 
-	c->status = resolve_name_recv(creq, state, &addr);
-	if (!composite_is_ok(c)) return;
+	status = smb2_session_setup_spnego_recv(subreq);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
 
-	if (state->ports == NULL) {
-		ports = default_ports;
-	} else {
-		ports = state->ports;
+	state->tree = smb2_tree_init(state->session, state, true);
+	if (tevent_req_nomem(state->tree, req)) {
+		return;
 	}
 
-	creq = smbcli_sock_connect_send(state, addr, ports, state->host, state->resolve_ctx, c->event_ctx, state->socket_options);
+	state->tcon.in.reserved = 0;
+	state->tcon.in.path     = talloc_asprintf(state, "\\\\%s\\%s",
+						  state->host, state->share);
+	if (tevent_req_nomem(state->tcon.in.path, req)) {
+		return;
+	}
 
-	composite_continue(c, creq, continue_socket, c);
+	smb2req = smb2_tree_connect_send(state->tree, &state->tcon);
+	if (tevent_req_nomem(smb2req, req)) {
+		return;
+	}
+	smb2req->async.fn = smb2_connect_tcon_done;
+	smb2req->async.private_data = req;
 }
 
-/*
-  a composite function that does a full negprot/sesssetup/tcon, returning
-  a connected smb2_tree
- */
-struct composite_context *smb2_connect_send(TALLOC_CTX *mem_ctx,
-					    const char *host,
-						const char **ports,
-					    const char *share,
-					    struct resolve_context *resolve_ctx,
-					    struct cli_credentials *credentials,
-					    struct tevent_context *ev,
-					    struct smbcli_options *options,
-						const char *socket_options,
-						struct gensec_settings *gensec_settings)
+static void smb2_connect_tcon_done(struct smb2_request *smb2req)
 {
-	struct composite_context *c;
-	struct smb2_connect_state *state;
-	struct nbt_name name;
-	struct composite_context *creq;
-
-	c = composite_create(mem_ctx, ev);
-	if (c == NULL) return NULL;
-
-	state = talloc(c, struct smb2_connect_state);
-	if (composite_nomem(state, c)) return c;
-	c->private_data = state;
+	struct tevent_req *req =
+		talloc_get_type_abort(smb2req->async.private_data,
+		struct tevent_req);
+	struct smb2_connect_state *state =
+		tevent_req_data(req,
+		struct smb2_connect_state);
+	NTSTATUS status;
 
-	state->credentials = credentials;
-	state->options = *options;
-	state->host = talloc_strdup(c, host);
-	if (composite_nomem(state->host, c)) return c;
-	state->ports = talloc_reference(state, ports);
-	state->share = talloc_strdup(c, share);
-	if (composite_nomem(state->share, c)) return c;
-	state->resolve_ctx = talloc_reference(state, resolve_ctx);
-	state->socket_options = talloc_reference(state, socket_options);
-	state->gensec_settings = talloc_reference(state, gensec_settings);
+	status = smb2_tree_connect_recv(smb2req, &state->tcon);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
 
-	ZERO_STRUCT(name);
-	name.name = host;
+	state->tree->tid = state->tcon.out.tid;
 
-	creq = resolve_name_send(resolve_ctx, state, &name, c->event_ctx);
-	composite_continue(c, creq, continue_resolve, c);
-	return c;
+	tevent_req_done(req);
 }
 
-/*
-  receive a connect reply
-*/
-NTSTATUS smb2_connect_recv(struct composite_context *c, TALLOC_CTX *mem_ctx,
+NTSTATUS smb2_connect_recv(struct tevent_req *req,
+			   TALLOC_CTX *mem_ctx,
 			   struct smb2_tree **tree)
 {
+	struct smb2_connect_state *state =
+		tevent_req_data(req,
+		struct smb2_connect_state);
 	NTSTATUS status;
-	struct smb2_connect_state *state = talloc_get_type(c->private_data, 
-							   struct smb2_connect_state);
-	status = composite_wait(c);
-	if (NT_STATUS_IS_OK(status)) {
-		*tree = talloc_steal(mem_ctx, state->tree);
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
 	}
-	talloc_free(c);
-	return status;
+
+	*tree = talloc_move(mem_ctx, &state->tree);
+
+	tevent_req_received(req);
+	return NT_STATUS_OK;
 }
 
 /*
   sync version of smb2_connect
 */
-NTSTATUS smb2_connect(TALLOC_CTX *mem_ctx, 
-		      const char *host, const char **ports, 
-			  const char *share,
+NTSTATUS smb2_connect(TALLOC_CTX *mem_ctx,
+		      const char *host,
+		      const char **ports,
+		      const char *share,
 		      struct resolve_context *resolve_ctx,
 		      struct cli_credentials *credentials,
 		      struct smb2_tree **tree,
 		      struct tevent_context *ev,
 		      struct smbcli_options *options,
-			  const char *socket_options,
-			  struct gensec_settings *gensec_settings)
+		      const char *socket_options,
+		      struct gensec_settings *gensec_settings)
 {
-	struct composite_context *c = smb2_connect_send(mem_ctx, host, ports, 
-													share, resolve_ctx, 
-													credentials, ev, options,
-													socket_options,
-													gensec_settings);
-	return smb2_connect_recv(c, mem_ctx, tree);
+	struct tevent_req *subreq;
+	NTSTATUS status;
+	bool ok;
+	TALLOC_CTX *frame = talloc_stackframe();
+
+	if (frame == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	subreq = smb2_connect_send(frame,
+				   ev,
+				   host,
+				   ports,
+				   share,
+				   resolve_ctx,
+				   credentials,
+				   options,
+				   socket_options,
+				   gensec_settings);
+	if (subreq == NULL) {
+		TALLOC_FREE(frame);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	ok = tevent_req_poll(subreq, ev);
+	if (!ok) {
+		status = map_nt_error_from_unix_common(errno);
+		TALLOC_FREE(frame);
+		return status;
+	}
+
+	status = smb2_connect_recv(subreq, mem_ctx, tree);
+	TALLOC_FREE(subreq);
+	if (!NT_STATUS_IS_OK(status)) {
+		TALLOC_FREE(frame);
+		return status;
+	}
+
+	TALLOC_FREE(frame);
+	return NT_STATUS_OK;
 }
diff --git a/source4/libcli/smb2/session.c b/source4/libcli/smb2/session.c
index 12479623e78..d46cdefc69a 100644
--- a/source4/libcli/smb2/session.c
+++ b/source4/libcli/smb2/session.c
@@ -20,10 +20,11 @@
 */
 
 #include "includes.h"
+#include <tevent.h>
+#include "lib/util/tevent_ntstatus.h"
 #include "libcli/raw/libcliraw.h"
 #include "libcli/smb2/smb2.h"
 #include "libcli/smb2/smb2_calls.h"
-#include "libcli/composite/composite.h"
 #include "auth/gensec/gensec.h"
 
 #include <unistd.h>
@@ -136,56 +137,133 @@ NTSTATUS smb2_session_setup(struct smb2_session *session,
 	return smb2_session_setup_recv(req, mem_ctx, io);
 }
 
-
-struct smb2_session_state {
+struct smb2_session_setup_spnego_state {
 	struct smb2_session_setup io;
 	struct smb2_request *req;
 	NTSTATUS gensec_status;
 };
 
+static void smb2_session_setup_spnego_handler(struct smb2_request *req);
+
+/*
+  a composite function that does a full SPNEGO session setup
+ */
+struct tevent_req *smb2_session_setup_spnego_send(TALLOC_CTX *mem_ctx,
+						  struct tevent_context *ev,
+						  struct smb2_session *session,
+						  struct cli_credentials *credentials)
+{
+	struct tevent_req *req;
+	struct smb2_session_setup_spnego_state *state;
+	const char *chosen_oid;
+	struct smb2_request *subreq;
+	NTSTATUS status;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct smb2_session_setup_spnego_state);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	ZERO_STRUCT(state->io);
+	state->io.in.vc_number          = 0;
+	if (session->transport->signing_required) {
+		state->io.in.security_mode =
+			SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED;
+	}
+	state->io.in.capabilities       = 0;
+	state->io.in.channel            = 0;
+	state->io.in.previous_sessionid = 0;
+
+	status = gensec_set_credentials(session->gensec, credentials);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	status = gensec_set_target_hostname(session->gensec,
+					    session->transport->socket->hostname);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	status = gensec_set_target_service(session->gensec, "cifs");
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	if (session->transport->negotiate.secblob.length > 0) {
+		chosen_oid = GENSEC_OID_SPNEGO;
+	} else {
+		chosen_oid = GENSEC_OID_NTLMSSP;
+	}
+
+	status = gensec_start_mech_by_oid(session->gensec, chosen_oid);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	status = gensec_update(session->gensec, state,
+			       session->transport->negotiate.secblob,
+			       &state->io.in.secblob);
+	if (!NT_STATUS_EQUAL(status, NT_STATUS_MORE_PROCESSING_REQUIRED)) {
+		tevent_req_nterror(req, status);
+		return tevent_req_post(req, ev);
+	}
+	state->gensec_status = status;
+
+	subreq = smb2_session_setup_send(session, &state->io);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	subreq->async.fn = smb2_session_setup_spnego_handler;
+	subreq->async.private_data = req;
+
+	return req;
+}
+
 /*
   handle continuations of the spnego session setup
 */
-static void session_request_handler(struct smb2_request *req)
+static void smb2_session_setup_spnego_handler(struct smb2_request *subreq)
 {
-	struct composite_context *c = talloc_get_type(req->async.private_data, 
-						      struct composite_context);
-	struct smb2_session_state *state = talloc_get_type(c->private_data, 
-							   struct smb2_session_state);
-	struct smb2_session *session = req->session;
+	struct tevent_req *req =
+		talloc_get_type_abort(subreq->async.private_data,
+		struct tevent_req);
+	struct smb2_session_setup_spnego_state *state =
+		tevent_req_data(req,
+		struct smb2_session_setup_spnego_state);
+	struct smb2_session *session = subreq->session;
 	NTSTATUS session_key_err;
 	DATA_BLOB session_key;
 	NTSTATUS peer_status;
+	NTSTATUS status;
 
-	c->status = smb2_session_setup_recv(req, c, &state->io);
-	peer_status = c->status;
-
+	status = smb2_session_setup_recv(subreq, state, &state->io);
+	peer_status = status;
 	if (NT_STATUS_EQUAL(peer_status, NT_STATUS_MORE_PROCESSING_REQUIRED) ||
 	    (NT_STATUS_IS_OK(peer_status) &&
 	     NT_STATUS_EQUAL(state->gensec_status, NT_STATUS_MORE_PROCESSING_REQUIRED))) {
-		c->status = gensec_update(session->gensec, c, 
-					  state->io.out.secblob,
-					  &state->io.in.secblob);
-		state->gensec_status = c->status;
-
+		status = gensec_update(session->gensec, state,
+				       state->io.out.secblob,
+				       &state->io.in.secblob);
+		state->gensec_status = status;
 		session->uid = state->io.out.uid;
 	}
 
-	if (!NT_STATUS_IS_OK(c->status) &&
-	    !NT_STATUS_EQUAL(c->status, NT_STATUS_MORE_PROCESSING_REQUIRED)) {
-		composite_error(c, c->status);
+	if (!NT_STATUS_IS_OK(status) &&
+	    !NT_STATUS_EQUAL(status, NT_STATUS_MORE_PROCESSING_REQUIRED)) {
+		tevent_req_nterror(req, status);
 		return;
 	}
 
 	if (NT_STATUS_EQUAL(peer_status, NT_STATUS_MORE_PROCESSING_REQUIRED)) {
-		state->req = smb2_session_setup_send(session, &state->io);
-		if (state->req == NULL) {
-			composite_error(c, NT_STATUS_NO_MEMORY);
+		subreq = smb2_session_setup_send(session, &state->io);
+		if (tevent_req_nomem(subreq, req)) {
 			return;
 		}
 
-		state->req->async.fn = session_request_handler;
-		state->req->async.private_data = c;
+		subreq->async.fn = smb2_session_setup_spnego_handler;
+		subreq->async.private_data = req;
 		return;
 	}
 
@@ -198,84 +276,21 @@ static void session_request_handler(struct smb2_request *req)
 		if (session->session_key.length == 0) {
 			DEBUG(0,("Wrong session key length %u for SMB2 signing\n",
 				 (unsigned)session->session_key.length));
-			composite_error(c, NT_STATUS_ACCESS_DENIED);
+			tevent_req_nterror(req, NT_STATUS_ACCESS_DENIED);
 			return;
 		}
 		session->signing_active = true;
 	}
 
-	composite_done(c);
-}
-
-/*
-  a composite function that does a full SPNEGO session setup
- */
-struct composite_context *smb2_session_setup_spnego_send(struct smb2_session *session, 
-							 struct cli_credentials *credentials)
-{
-	struct composite_context *c;
-	struct smb2_session_state *state;
-	const char *chosen_oid;
-
-	c = composite_create(session, session->transport->socket->event.ctx);
-	if (c == NULL) return NULL;
-
-	state = talloc(c, struct smb2_session_state);
-	if (composite_nomem(state, c)) return c;
-	c->private_data = state;
-
-	ZERO_STRUCT(state->io);
-	state->io.in.vc_number          = 0;
-	if (session->transport->signing_required) {
-		state->io.in.security_mode = 
-			SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED;
-	}
-	state->io.in.capabilities       = 0;
-	state->io.in.channel            = 0;
-	state->io.in.previous_sessionid = 0;
-
-	c->status = gensec_set_credentials(session->gensec, credentials);
-	if (!composite_is_ok(c)) return c;
-
-	c->status = gensec_set_target_hostname(session->gensec, 
-					       session->transport->socket->hostname);
-	if (!composite_is_ok(c)) return c;
-
-	c->status = gensec_set_target_service(session->gensec, "cifs");
-	if (!composite_is_ok(c)) return c;
-
-	if (session->transport->negotiate.secblob.length > 0) {
-		chosen_oid = GENSEC_OID_SPNEGO;
-	} else {
-		chosen_oid = GENSEC_OID_NTLMSSP;
-	}
-
-	c->status = gensec_start_mech_by_oid(session->gensec, chosen_oid);
-	if (!composite_is_ok(c)) return c;
-
-	c->status = gensec_update(session->gensec, c, 
-				  session->transport->negotiate.secblob,
-				  &state->io.in.secblob);
-	if (!NT_STATUS_EQUAL(c->status, NT_STATUS_MORE_PROCESSING_REQUIRED)) {
-		composite_error(c, c->status);
-		return c;
-	}
-	state->gensec_status = c->status;
-		
-	state->req = smb2_session_setup_send(session, &state->io);
-	composite_continue_smb2(c, state->req, session_request_handler, c);
-	return c;
+	tevent_req_done(req);
 }
 
 /*
   receive a composite session setup reply
 */
-NTSTATUS smb2_session_setup_spnego_recv(struct composite_context *c)
+NTSTATUS smb2_session_setup_spnego_recv(struct tevent_req *req)
 {
-	NTSTATUS status;
-	status = composite_wait(c);
-	talloc_free(c);
-	return status;
+	return tevent_req_simple_recv_ntstatus(req);
 }
 
 /*
@@ -284,6 +299,37 @@ NTSTATUS smb2_session_setup_spnego_recv(struct composite_context *c)
 NTSTATUS smb2_session_setup_spnego(struct smb2_session *session, 
 				   struct cli_credentials *credentials)
 {
-	struct composite_context *c = smb2_session_setup_spnego_send(session, credentials);
-	return smb2_session_setup_spnego_recv(c);
+	struct tevent_req *subreq;
+	NTSTATUS status;
+	bool ok;
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct tevent_context *ev = session->transport->socket->event.ctx;
+
+	if (frame == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	subreq = smb2_session_setup_spnego_send(frame, ev,
+						session, credentials);
+	if (subreq == NULL) {
+		TALLOC_FREE(frame);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	ok = tevent_req_poll(subreq, ev);
+	if (!ok) {
+		status = map_nt_error_from_unix_common(errno);
+		TALLOC_FREE(frame);
+		return status;
+	}
+
+	status = smb2_session_setup_spnego_recv(subreq);
+	TALLOC_FREE(subreq);
+	if (!NT_STATUS_IS_OK(status)) {
+		TALLOC_FREE(frame);
+		return status;
+	}
+
+	TALLOC_FREE(frame);
+	return NT_STATUS_OK;
 }
diff --git a/source4/libcli/smb2/wscript_build b/source4/libcli/smb2/wscript_build
index c39cf5fd39a..5b5ac7d4e68 100644
--- a/source4/libcli/smb2/wscript_build
+++ b/source4/libcli/smb2/wscript_build
@@ -3,6 +3,7 @@
 bld.SAMBA_SUBSYSTEM('LIBCLI_SMB2',
 	source='transport.c request.c negprot.c session.c tcon.c create.c close.c connect.c getinfo.c write.c read.c setinfo.c find.c ioctl.c logoff.c tdis.c flush.c lock.c notify.c cancel.c keepalive.c break.c util.c signing.c lease_break.c',
 	autoproto='smb2_proto.h',
-	public_deps='LIBCLI_RAW LIBPACKET gensec'
+	deps='UTIL_TEVENT',
+	public_deps='LIBCLI_RAW LIBPACKET gensec tevent'
 	)
 
diff --git a/source4/libcli/smb_composite/smb2.c b/source4/libcli/smb_composite/smb2.c
index d71708a974e..5c93869b5c8 100644
--- a/source4/libcli/smb_composite/smb2.c
+++ b/source4/libcli/smb_composite/smb2.c
@@ -25,6 +25,8 @@
 
 
 #include "includes.h"
+#include <tevent.h>
+#include "lib/util/tevent_ntstatus.h"
 #include "libcli/raw/libcliraw.h"
 #include "libcli/raw/raw_proto.h"
 #include "libcli/composite/composite.h"
@@ -264,107 +266,182 @@ NTSTATUS smb2_composite_rmdir(struct smb2_tree *tree, struct smb_rmdir *io)
 	return composite_wait_free(c);
 }
 
+struct smb2_composite_setpathinfo_state {
+	struct smb2_tree *tree;
+	union smb_setfileinfo io;
+	NTSTATUS set_status;
+	struct smb2_create cr;
+	struct smb2_close cl;
+};
+
+static void smb2_composite_setpathinfo_create_done(struct smb2_request *smb2req);
 
 /*
-  continue after the setfileinfo in a composite setpathinfo
- */
-static void continue_setpathinfo_close(struct smb2_request *req)
+  composite SMB2 setpathinfo call
+*/
+struct tevent_req *smb2_composite_setpathinfo_send(TALLOC_CTX *mem_ctx,
+						   struct tevent_context *ev,
+						   struct smb2_tree *tree,
+						   const union smb_setfileinfo *io)
 {
-	struct composite_context *ctx = talloc_get_type(req->async.private_data, 
-							struct composite_context);
-	struct smb2_tree *tree = req->tree;
-	struct smb2_close close_parm;
-	NTSTATUS status;
-	union smb_setfileinfo *io2 = talloc_get_type(ctx->private_data, 
-						     union smb_setfileinfo);
+	struct tevent_req *req;
+	struct smb2_composite_setpathinfo_state *state;
+	struct smb2_request *smb2req;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct smb2_composite_setpathinfo_state);
+	if (req == NULL) {
+		return NULL;
+	}
 
-	status = smb2_setinfo_recv(req);
-	if (!NT_STATUS_IS_OK(status)) {
-		composite_error(ctx, status);
-		return;
+	state->tree = tree;
+	state->io = *io;
+
+	state->cr.in.desired_access     = SEC_FLAG_MAXIMUM_ALLOWED;
+	state->cr.in.create_disposition = NTCREATEX_DISP_OPEN;
+	state->cr.in.share_access =
+		NTCREATEX_SHARE_ACCESS_DELETE|
+		NTCREATEX_SHARE_ACCESS_READ|
+		NTCREATEX_SHARE_ACCESS_WRITE;
+	state->cr.in.create_options = 0;
+	state->cr.in.fname = state->io.generic.in.file.path;
+	if (state->cr.in.fname[0] == '\\') {
+		state->cr.in.fname++;
 	}
 
-	ZERO_STRUCT(close_parm);
-	close_parm.in.file.handle = io2->generic.in.file.handle;
-	
-	req = smb2_close_send(tree, &close_parm);
-	composite_continue_smb2(ctx, req, continue_close, ctx);
+	smb2req = smb2_create_send(tree, &state->cr);
+	if (tevent_req_nomem(smb2req, req)) {
+		return tevent_req_post(req, ev);
+	}
+	smb2req->async.fn = smb2_composite_setpathinfo_create_done;
+	smb2req->async.private_data = req;
+
+	return req;
 }
 
+static void smb2_composite_setpathinfo_setinfo_done(struct smb2_request *smb2req);
 
-/*
-  continue after the create in a composite setpathinfo
- */
-static void continue_setpathinfo(struct smb2_request *req)
+static void smb2_composite_setpathinfo_create_done(struct smb2_request *smb2req)
 {
-	struct composite_context *ctx = talloc_get_type(req->async.private_data, 
-							struct composite_context);
-	struct smb2_tree *tree = req->tree;
-	struct smb2_create create_parm;
+	struct tevent_req *req =
+		talloc_get_type_abort(smb2req->async.private_data,
+		struct tevent_req);
+	struct smb2_composite_setpathinfo_state *state =
+		tevent_req_data(req,
+		struct smb2_composite_setpathinfo_state);
 	NTSTATUS status;
-	union smb_setfileinfo *io2 = talloc_get_type(ctx->private_data, 
-						     union smb_setfileinfo);
 
-	status = smb2_create_recv(req, ctx, &create_parm);
-	if (!NT_STATUS_IS_OK(status)) {
-		composite_error(ctx, status);
+	status = smb2_create_recv(smb2req, state, &state->cr);
+	if (tevent_req_nterror(req, status)) {
 		return;
 	}
 
-	io2->generic.in.file.handle = create_parm.out.file.handle;
+	state->io.generic.in.file.handle = state->cr.out.file.handle;
 
-	req = smb2_setinfo_file_send(tree, io2);
-	composite_continue_smb2(ctx, req, continue_setpathinfo_close, ctx);
+	smb2req = smb2_setinfo_file_send(state->tree, &state->io);
+	if (tevent_req_nomem(smb2req, req)) {
+		return;
+	}
+	smb2req->async.fn = smb2_composite_setpathinfo_setinfo_done;
+	smb2req->async.private_data = req;
 }
 
+static void smb2_composite_setpathinfo_close_done(struct smb2_request *smb2req);
 
-/*
-  composite SMB2 setpathinfo call
-*/
-struct composite_context *smb2_composite_setpathinfo_send(struct smb2_tree *tree, 
-							  union smb_setfileinfo *io)
+static void smb2_composite_setpathinfo_setinfo_done(struct smb2_request *smb2req)
 {
-	struct composite_context *ctx;
-	struct smb2_create create_parm;
-	struct smb2_request *req;
-	union smb_setfileinfo *io2;
+	struct tevent_req *req =
+		talloc_get_type_abort(smb2req->async.private_data,
+		struct tevent_req);
+	struct smb2_composite_setpathinfo_state *state =
+		tevent_req_data(req,
+		struct smb2_composite_setpathinfo_state);
+	NTSTATUS status;
 
-	ctx = composite_create(tree, tree->session->transport->socket->event.ctx);
-	if (ctx == NULL) return NULL;
+	status = smb2_setinfo_recv(smb2req);
+	state->set_status = status;
 
-	ZERO_STRUCT(create_parm);
-	create_parm.in.desired_access     = SEC_FLAG_MAXIMUM_ALLOWED;
-	create_parm.in.create_disposition = NTCREATEX_DISP_OPEN;
-	create_parm.in.share_access = 
-		NTCREATEX_SHARE_ACCESS_DELETE|
-		NTCREATEX_SHARE_ACCESS_READ|
-		NTCREATEX_SHARE_ACCESS_WRITE;
-	create_parm.in.create_options = 0;
-	create_parm.in.fname = io->generic.in.file.path;
-	if (create_parm.in.fname[0] == '\\') {
-		create_parm.in.fname++;
+	state->cl.in.file.handle = state->io.generic.in.file.handle;
+
+	smb2req = smb2_close_send(state->tree, &state->cl);
+	if (tevent_req_nomem(smb2req, req)) {
+		return;
 	}
+	smb2req->async.fn = smb2_composite_setpathinfo_close_done;
+	smb2req->async.private_data = req;
+}
 
-	req = smb2_create_send(tree, &create_parm);
+static void smb2_composite_setpathinfo_close_done(struct smb2_request *smb2req)
+{
+	struct tevent_req *req =
+		talloc_get_type_abort(smb2req->async.private_data,
+		struct tevent_req);
+	struct smb2_composite_setpathinfo_state *state =
+		tevent_req_data(req,
+		struct smb2_composite_setpathinfo_state);
+	NTSTATUS status;
 
-	io2 = talloc(ctx, union smb_setfileinfo);
-	if (composite_nomem(io2, ctx)) {
-		return ctx;
+	status = smb2_close_recv(smb2req, &state->cl);
+
+	if (tevent_req_nterror(req, state->set_status)) {
+		return;
 	}
-	*io2 = *io;
 
-	ctx->private_data = io2;
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
 
-	composite_continue_smb2(ctx, req, continue_setpathinfo, ctx);
-	return ctx;
+	tevent_req_done(req);
 }
 
+NTSTATUS smb2_composite_setpathinfo_recv(struct tevent_req *req)
+{
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
+	}
+
+	tevent_req_received(req);
+	return NT_STATUS_OK;
+}
 
 /*
   composite setpathinfo call
  */
 NTSTATUS smb2_composite_setpathinfo(struct smb2_tree *tree, union smb_setfileinfo *io)
 {
-	struct composite_context *c = smb2_composite_setpathinfo_send(tree, io);
-	return composite_wait_free(c);	
+	struct tevent_req *subreq;
+	NTSTATUS status;
+	bool ok;
+	TALLOC_CTX *frame = talloc_stackframe();
+	struct tevent_context *ev = tree->session->transport->socket->event.ctx;
+
+	if (frame == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	subreq = smb2_composite_setpathinfo_send(frame, ev, tree, io);
+	if (subreq == NULL) {
+		TALLOC_FREE(frame);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	ok = tevent_req_poll(subreq, ev);
+	if (!ok) {
+		status = map_nt_error_from_unix_common(errno);
+		TALLOC_FREE(frame);
+		return status;
+	}
+
+	status = smb2_composite_setpathinfo_recv(subreq);
+	TALLOC_FREE(subreq);
+	if (!NT_STATUS_IS_OK(status)) {
+		TALLOC_FREE(frame);
+		return status;
+	}
+
+	TALLOC_FREE(frame);
+	return NT_STATUS_OK;
 }
diff --git a/source4/libcli/wbclient/wbclient.c b/source4/libcli/wbclient/wbclient.c
index 5c4312cc66d..4f50c106947 100644
--- a/source4/libcli/wbclient/wbclient.c
+++ b/source4/libcli/wbclient/wbclient.c
@@ -31,7 +31,7 @@
  * \param
  */
 struct wbc_context *wbc_init(TALLOC_CTX *mem_ctx,
-			     struct messaging_context *msg_ctx,
+			     struct imessaging_context *msg_ctx,
 			     struct tevent_context *event_ctx)
 {
 	struct wbc_context *ctx;
diff --git a/source4/libcli/wbclient/wbclient.h b/source4/libcli/wbclient/wbclient.h
index e7473e3e7b1..1fa2f59c575 100644
--- a/source4/libcli/wbclient/wbclient.h
+++ b/source4/libcli/wbclient/wbclient.h
@@ -28,7 +28,7 @@ struct wbc_context {
 };
 
 struct wbc_context *wbc_init(TALLOC_CTX *mem_ctx,
-			     struct messaging_context *msg_ctx,
+			     struct imessaging_context *msg_ctx,
 			     struct tevent_context *event_ctx);
 
 struct composite_context *wbc_sids_to_xids_send(struct wbc_context *wbc_ctx,
diff --git a/source4/libcli/wrepl/winsrepl.c b/source4/libcli/wrepl/winsrepl.c
index 842a35175ec..c7f1ad07cb8 100644
--- a/source4/libcli/wrepl/winsrepl.c
+++ b/source4/libcli/wrepl/winsrepl.c
@@ -136,8 +136,8 @@ NTSTATUS wrepl_socket_split_stream(struct wrepl_socket *wrepl_socket,
 const char *wrepl_best_ip(struct loadparm_context *lp_ctx, const char *peer_ip)
 {
 	struct interface *ifaces;
-	load_interfaces(lp_ctx, lpcfg_interfaces(lp_ctx), &ifaces);
-	return iface_best_ip(ifaces, peer_ip);
+	load_interface_list(lp_ctx, lp_ctx, &ifaces);
+	return iface_list_best_ip(ifaces, peer_ip);
 }
 
 struct wrepl_connect_state {
@@ -181,7 +181,7 @@ struct tevent_req *wrepl_connect_send(TALLOC_CTX *mem_ctx,
 						our_ip, 0,
 						&state->local_address);
 	if (ret != 0) {
-		NTSTATUS status = map_nt_error_from_unix(errno);
+		NTSTATUS status = map_nt_error_from_unix_common(errno);
 		tevent_req_nterror(req, status);
 		return tevent_req_post(req, ev);
 	}
@@ -190,7 +190,7 @@ struct tevent_req *wrepl_connect_send(TALLOC_CTX *mem_ctx,
 						peer_ip, WINS_REPLICATION_PORT,
 						&state->remote_address);
 	if (ret != 0) {
-		NTSTATUS status = map_nt_error_from_unix(errno);
+		NTSTATUS status = map_nt_error_from_unix_common(errno);
 		tevent_req_nterror(req, status);
 		return tevent_req_post(req, ev);
 	}
@@ -201,7 +201,7 @@ struct tevent_req *wrepl_connect_send(TALLOC_CTX *mem_ctx,
 			      wrepl_connect_trigger,
 			      NULL);
 	if (!ok) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return tevent_req_post(req, ev);
 	}
 
@@ -250,7 +250,7 @@ static void wrepl_connect_done(struct tevent_req *subreq)
 	ret = tstream_inet_tcp_connect_recv(subreq, &sys_errno,
 					    state, &state->stream, NULL);
 	if (ret != 0) {
-		NTSTATUS status = map_nt_error_from_unix(sys_errno);
+		NTSTATUS status = map_nt_error_from_unix_common(sys_errno);
 		tevent_req_nterror(req, status);
 		return;
 	}
@@ -382,7 +382,7 @@ struct tevent_req *wrepl_request_send(TALLOC_CTX *mem_ctx,
 			      wrepl_request_trigger,
 			      NULL);
 	if (!ok) {
-		tevent_req_nomem(NULL, req);
+		tevent_req_oom(req);
 		return tevent_req_post(req, ev);
 	}
 
@@ -443,7 +443,7 @@ static void wrepl_request_writev_done(struct tevent_req *subreq)
 	ret = tstream_writev_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		NTSTATUS status = map_nt_error_from_unix(sys_errno);
+		NTSTATUS status = map_nt_error_from_unix_common(sys_errno);
 		TALLOC_FREE(state->caller.wrepl_socket->stream);
 		tevent_req_nterror(req, status);
 		return;
@@ -494,7 +494,7 @@ static void wrepl_request_disconnect_done(struct tevent_req *subreq)
 	ret = tstream_disconnect_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		NTSTATUS status = map_nt_error_from_unix(sys_errno);
+		NTSTATUS status = map_nt_error_from_unix_common(sys_errno);
 		TALLOC_FREE(state->caller.wrepl_socket->stream);
 		tevent_req_nterror(req, status);
 		return;
diff --git a/source4/libcli/wscript_build b/source4/libcli/wscript_build
index fbe13ea8e9a..1799bcabadc 100644
--- a/source4/libcli/wscript_build
+++ b/source4/libcli/wscript_build
@@ -3,15 +3,6 @@
 bld.RECURSE('ldap')
 bld.RECURSE('wbclient')
 
-bld.SAMBA_LIBRARY('errors',
-                  source='../../libcli/util/doserr.c util/errormap.c util/nterr.c',
-                  public_headers='../../libcli/util/error.h ../../libcli/util/ntstatus.h ../../libcli/util/doserr.h ../../libcli/util/werror.h',
-                  header_path='core',
-                  deps='talloc',
-                  private_library=True
-                  )
-
-
 bld.SAMBA_SUBSYSTEM('LIBSAMBA_TSOCKET',
 	source='../../libcli/util/tstream.c',
 	public_deps='LIBTSOCKET UTIL_TEVENT'
@@ -36,8 +27,8 @@ bld.SAMBA_SUBSYSTEM('LIBCLI_COMPOSITE',
 bld.SAMBA_SUBSYSTEM('LIBCLI_SMB_COMPOSITE',
 	source='smb_composite/loadfile.c smb_composite/savefile.c smb_composite/connect.c smb_composite/sesssetup.c smb_composite/fetchfile.c smb_composite/appendacl.c smb_composite/fsinfo.c smb_composite/smb2.c',
 	autoproto='smb_composite/proto.h',
-	deps='LIBCLI_SMB2',
-	public_deps='LIBCLI_COMPOSITE credentials gensec LIBCLI_RESOLVE'
+	deps='LIBCLI_SMB2 UTIL_TEVENT',
+	public_deps='LIBCLI_COMPOSITE credentials gensec LIBCLI_RESOLVE tevent'
 	)
 
 
@@ -83,11 +74,12 @@ bld.SAMBA_SUBSYSTEM('LIBCLI_SMB',
 	)
 
 
-bld.SAMBA_SUBSYSTEM('LIBCLI_RAW',
+bld.SAMBA_LIBRARY('LIBCLI_RAW',
 	source='raw/rawfile.c raw/smb_signing.c raw/clisocket.c raw/clitransport.c raw/clisession.c raw/clitree.c raw/clierror.c raw/rawrequest.c raw/rawreadwrite.c raw/rawsearch.c raw/rawsetfileinfo.c raw/raweas.c raw/rawtrans.c raw/clioplock.c raw/rawnegotiate.c raw/rawfsinfo.c raw/rawfileinfo.c raw/rawnotify.c raw/rawioctl.c raw/rawacl.c raw/rawdate.c raw/rawlpq.c raw/rawshadow.c',
 	autoproto='raw/raw_proto.h',
 	public_deps='samba_socket LIBPACKET LIBCRYPTO',
-	deps='LIBCLI_COMPOSITE LIBCLI_RESOLVE security ndr samba-util errors CHARSET talloc LIBCLI_SMB_COMPOSITE tevent NDR_NBT_BUF LIBCLI_SMB_COMMON'
+	deps='LIBCLI_COMPOSITE LIBCLI_RESOLVE security ndr samba-util errors CHARSET talloc LIBCLI_SMB_COMPOSITE tevent NDR_NBT_BUF LIBCLI_SMB_COMMON',
+                  private_library=True
 	)
 
 bld.RECURSE('smb2')
diff --git a/source4/libnet/libnet.c b/source4/libnet/libnet.c
index 53ebb9e34aa..32df85d5279 100644
--- a/source4/libnet/libnet.c
+++ b/source4/libnet/libnet.c
@@ -43,7 +43,7 @@ struct libnet_context *libnet_context_init(struct tevent_context *ev,
 	ctx->lp_ctx = lp_ctx;
 
 	/* make sure dcerpc is initialized */
-	dcerpc_init(lp_ctx);
+	dcerpc_init();
 
 	/* name resolution methods */
 	ctx->resolve_ctx = lpcfg_resolve_context(lp_ctx);
diff --git a/source4/libnet/libnet_become_dc.c b/source4/libnet/libnet_become_dc.c
index a68f4c5ea05..4d845ca1f08 100644
--- a/source4/libnet/libnet_become_dc.c
+++ b/source4/libnet/libnet_become_dc.c
@@ -769,7 +769,7 @@ static void becomeDC_send_cldap(struct libnet_BecomeDC_state *s)
 						lpcfg_cldap_port(s->libnet->lp_ctx),
 						&dest_address);
 	if (ret != 0) {
-		c->status = map_nt_error_from_unix(errno);
+		c->status = map_nt_error_from_unix_common(errno);
 		if (!composite_is_ok(c)) return;
 	}
 
diff --git a/source4/libnet/libnet_join.c b/source4/libnet/libnet_join.c
index 6e76df43e3d..a1124fdd623 100644
--- a/source4/libnet/libnet_join.c
+++ b/source4/libnet/libnet_join.c
@@ -889,9 +889,9 @@ NTSTATUS libnet_JoinDomain(struct libnet_context *ctx, TALLOC_CTX *mem_ctx, stru
 	return status;
 }
 
-static NTSTATUS libnet_Join_primary_domain(struct libnet_context *ctx, 
-					   TALLOC_CTX *mem_ctx, 
-					   struct libnet_Join *r)
+NTSTATUS libnet_Join_member(struct libnet_context *ctx,
+			    TALLOC_CTX *mem_ctx,
+			    struct libnet_Join_member *r)
 {
 	NTSTATUS status;
 	TALLOC_CTX *tmp_mem;
@@ -916,15 +916,7 @@ static NTSTATUS libnet_Join_primary_domain(struct libnet_context *ctx,
 		return NT_STATUS_NO_MEMORY;
 	}
 	
-	if (r->in.join_type == SEC_CHAN_BDC) {
-		acct_type = ACB_SVRTRUST;
-	} else if (r->in.join_type == SEC_CHAN_WKSTA) {
-		acct_type = ACB_WSTRUST;
-	} else {
-		r->out.error_string = NULL;
-		talloc_free(tmp_mem);	
-		return NT_STATUS_INVALID_PARAMETER;
-	}
+	acct_type = ACB_WSTRUST;
 
 	if (r->in.netbios_name != NULL) {
 		netbios_name = r->in.netbios_name;
@@ -972,7 +964,7 @@ static NTSTATUS libnet_Join_primary_domain(struct libnet_context *ctx,
 	set_secrets->domain_name = r2->out.domain_name;
 	set_secrets->realm = r2->out.realm;
 	set_secrets->netbios_name = netbios_name;
-	set_secrets->secure_channel_type = r->in.join_type;
+	set_secrets->secure_channel_type = SEC_CHAN_WKSTA;
 	set_secrets->machine_password = r2->out.join_password;
 	set_secrets->key_version_number = r2->out.kvno;
 	set_secrets->domain_sid = r2->out.domain_sid;
@@ -996,21 +988,3 @@ static NTSTATUS libnet_Join_primary_domain(struct libnet_context *ctx,
 	return NT_STATUS_OK;
 }
 
-NTSTATUS libnet_Join(struct libnet_context *ctx, TALLOC_CTX *mem_ctx, struct libnet_Join *r)
-{
-	switch (r->in.join_type) {
-		case SEC_CHAN_WKSTA:
-			return libnet_Join_primary_domain(ctx, mem_ctx, r);
-		case SEC_CHAN_BDC:
-			return libnet_Join_primary_domain(ctx, mem_ctx, r);
-		case SEC_CHAN_DOMAIN:
-		case SEC_CHAN_DNS_DOMAIN:
-		case SEC_CHAN_NULL:
-			break;
-	}
-
-	r->out.error_string = talloc_asprintf(mem_ctx,
-					      "Invalid join type specified (%08X) attempting to join domain %s",
-					      r->in.join_type, r->in.domain_name);
-	return NT_STATUS_INVALID_PARAMETER;
-}
diff --git a/source4/libnet/libnet_join.h b/source4/libnet/libnet_join.h
index 79884130d8a..6acf374b383 100644
--- a/source4/libnet/libnet_join.h
+++ b/source4/libnet/libnet_join.h
@@ -63,11 +63,10 @@ struct libnet_JoinDomain {
 	} out;
 };
 
-struct libnet_Join {
+struct libnet_Join_member {
 	struct {
 		const char *domain_name;
 		const char *netbios_name;
-		enum netr_SchannelType join_type;
 		enum libnet_Join_level level;
 	} in;
 	
diff --git a/source4/libnet/libnet_rpc.c b/source4/libnet/libnet_rpc.c
index e0781c3816d..8aacfc398a1 100644
--- a/source4/libnet/libnet_rpc.c
+++ b/source4/libnet/libnet_rpc.c
@@ -109,6 +109,10 @@ static struct composite_context* libnet_RpcConnectSrv_send(struct libnet_context
 	case LIBNET_RPC_CONNECT_SERVER:
 	case LIBNET_RPC_CONNECT_SERVER_ADDRESS:
 		b->flags = r->in.dcerpc_flags;
+		break;
+	default:
+		/* other types have already been checked before */
+		break;
 	}
 
 	if (DEBUGLEVEL >= 10) {
diff --git a/source4/libnet/libnet_site.c b/source4/libnet/libnet_site.c
index 9bfca74a368..a74dd59a22b 100644
--- a/source4/libnet/libnet_site.c
+++ b/source4/libnet/libnet_site.c
@@ -64,7 +64,7 @@ NTSTATUS libnet_FindSite(TALLOC_CTX *ctx, struct libnet_context *lctx, struct li
 						&dest_address);
 	if (ret != 0) {
 		r->out.error_string = NULL;
-		status = map_nt_error_from_unix(errno);
+		status = map_nt_error_from_unix_common(errno);
 		return status;
 	}
 
@@ -168,7 +168,9 @@ NTSTATUS libnet_JoinSite(struct libnet_context *ctx,
 	}
 
 	make_nbt_name_client(&name, libnet_r->out.samr_binding->host);
-	status = resolve_name(lpcfg_resolve_context(ctx->lp_ctx), &name, r, &dest_addr, ctx->event_ctx);
+	status = resolve_name_ex(lpcfg_resolve_context(ctx->lp_ctx),
+				 0, 0,
+				 &name, r, &dest_addr, ctx->event_ctx);
 	if (!NT_STATUS_IS_OK(status)) {
 		libnet_r->out.error_string = NULL;
 		talloc_free(tmp_ctx);
diff --git a/source4/libnet/libnet_unbecome_dc.c b/source4/libnet/libnet_unbecome_dc.c
index 4bffc5079e7..85d47a91f92 100644
--- a/source4/libnet/libnet_unbecome_dc.c
+++ b/source4/libnet/libnet_unbecome_dc.c
@@ -277,7 +277,7 @@ static void unbecomeDC_send_cldap(struct libnet_UnbecomeDC_state *s)
 						lpcfg_cldap_port(s->libnet->lp_ctx),
 						&dest_address);
 	if (ret != 0) {
-		c->status = map_nt_error_from_unix(errno);
+		c->status = map_nt_error_from_unix_common(errno);
 		if (!composite_is_ok(c)) return;
 	}
 
diff --git a/source4/libnet/py_net.c b/source4/libnet/py_net.c
index b7f7d9eaafe..ffcd60da27d 100644
--- a/source4/libnet/py_net.c
+++ b/source4/libnet/py_net.c
@@ -41,17 +41,17 @@ typedef struct {
 	struct tevent_context *ev;
 } py_net_Object;
 
-static PyObject *py_net_join(py_net_Object *self, PyObject *args, PyObject *kwargs)
+static PyObject *py_net_join_member(py_net_Object *self, PyObject *args, PyObject *kwargs)
 {
-	struct libnet_Join r;
+	struct libnet_Join_member r;
 	NTSTATUS status;
 	PyObject *result;
 	TALLOC_CTX *mem_ctx;
-	const char *kwnames[] = { "domain_name", "netbios_name", "join_type", "level", NULL };
+	const char *kwnames[] = { "domain_name", "netbios_name", "level", NULL };
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssii:Join", discard_const_p(char *, kwnames), 
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssi:Join", discard_const_p(char *, kwnames),
 					 &r.in.domain_name, &r.in.netbios_name, 
-					 &r.in.join_type, &r.in.level))
+					 &r.in.level))
 		return NULL;
 
 	mem_ctx = talloc_new(self->mem_ctx);
@@ -60,7 +60,7 @@ static PyObject *py_net_join(py_net_Object *self, PyObject *args, PyObject *kwar
 		return NULL;
 	}
 
-	status = libnet_Join(self->libnet_ctx, mem_ctx, &r);
+	status = libnet_Join_member(self->libnet_ctx, mem_ctx, &r);
 	if (NT_STATUS_IS_ERR(status)) {
 		PyErr_SetString(PyExc_RuntimeError, r.out.error_string?r.out.error_string:nt_errstr(status));
 		talloc_free(mem_ctx);
@@ -76,7 +76,7 @@ static PyObject *py_net_join(py_net_Object *self, PyObject *args, PyObject *kwar
 	return result;
 }
 
-static const char py_net_join_doc[] = "join(domain_name, netbios_name, join_type, level) -> (join_password, domain_sid, domain_name)\n\n" \
+static const char py_net_join_member_doc[] = "join_member(domain_name, netbios_name, level) -> (join_password, domain_sid, domain_name)\n\n" \
 "Join the domain with the specified name.";
 
 static PyObject *py_net_set_password(py_net_Object *self, PyObject *args, PyObject *kwargs)
@@ -526,7 +526,7 @@ static const char py_net_finddc_doc[] = "finddc(domain, server_type)\n"
 					 "find a DC with the specified server_type bits. Return the DNS name";
 
 static PyMethodDef net_obj_methods[] = {
-	{"join", (PyCFunction)py_net_join, METH_VARARGS|METH_KEYWORDS, py_net_join_doc},
+	{"join_member", (PyCFunction)py_net_join_member, METH_VARARGS|METH_KEYWORDS, py_net_join_member_doc},
 	{"set_password", (PyCFunction)py_net_set_password, METH_VARARGS|METH_KEYWORDS, py_net_set_password_doc},
 	{"export_keytab", (PyCFunction)py_net_export_keytab, METH_VARARGS|METH_KEYWORDS, py_net_export_keytab_doc},
 	{"time", (PyCFunction)py_net_time, METH_VARARGS|METH_KEYWORDS, py_net_time_doc},
diff --git a/source4/librpc/idl/opendb.idl b/source4/librpc/idl/opendb.idl
index 4973cd09826..b76992960af 100644
--- a/source4/librpc/idl/opendb.idl
+++ b/source4/librpc/idl/opendb.idl
@@ -7,7 +7,7 @@
    ntvfs/common/opendb.c
 */
 
-import "server_id4.idl";
+import "server_id.idl";
 
 [
   pointer_default(unique)
diff --git a/source4/librpc/idl/s4_notify.idl b/source4/librpc/idl/s4_notify.idl
deleted file mode 100644
index 89ade2991c9..00000000000
--- a/source4/librpc/idl/s4_notify.idl
+++ /dev/null
@@ -1,58 +0,0 @@
-#include "idl_types.h"
-
-/*
-   IDL structures for notify change code
-
-   this defines the structures used in the notify database code, and
-   the change notify buffers
-*/
-
-import "server_id4.idl";
-
-[
-  pointer_default(unique)
-]
-interface notify
-{
-
-	/* structure used in the notify database */
-	typedef [public] struct {
-		server_id server;
-		uint32 filter; /* filter to apply in this directory */
-		uint32 subdir_filter; /* filter to apply in child directories */
-		utf8string path;
-		uint32 path_len; /* saves some computation on search */
-		pointer private_data;
-	} notify_entry;
-
-	/*
-	  to allow for efficient search for matching entries, we
-	  divide them by the directory depth, with a separate array
-	  per depth. The entries within each depth are sorted by path,
-	  allowing for a bisection search.
-
-	  The max_mask and max_mask_subdir at each depth is the
-	  bitwise or of the filters and subdir filters for all entries
-	  at that depth. This allows a depth to be quickly skipped if
-	  no entries will match the target filter
-	*/
-	typedef struct {
-		uint32 max_mask;
-		uint32 max_mask_subdir;
-		uint32 num_entries;
-		notify_entry entries[num_entries];
-	} notify_depth;
-
-	typedef [public] struct {
-		uint32 num_depths;
-		notify_depth depth[num_depths];
-	} notify_array;
-
-	/* structure sent between servers in notify messages */
-	typedef [public] struct {
-		uint32 action;
-		utf8string path;
-		pointer private_data;
-	} notify_event;
-
-}
diff --git a/source4/librpc/idl/server_id4.idl b/source4/librpc/idl/server_id4.idl
deleted file mode 100644
index 486143546b6..00000000000
--- a/source4/librpc/idl/server_id4.idl
+++ /dev/null
@@ -1,12 +0,0 @@
-[
-	pointer_default(unique)
-]
-interface server_id
-{
-	/* id used to identify a endpoint, possibly in a cluster */
-	typedef [public] struct {
-		hyper id;
-		uint32 id2;
-		uint32 node;
-	} server_id;
-}
diff --git a/source4/librpc/idl/wscript_build b/source4/librpc/idl/wscript_build
index 6fe3690b335..59b16766ea5 100644
--- a/source4/librpc/idl/wscript_build
+++ b/source4/librpc/idl/wscript_build
@@ -5,8 +5,8 @@ import os
 topinclude=os.path.join(bld.srcnode.abspath(), 'librpc/idl')
 
 bld.SAMBA_PIDL_LIST('PIDL',
-                    source='''irpc.idl nfs4acl.idl s4_notify.idl ntp_signd.idl
-                              opendb.idl sasl_helpers.idl server_id4.idl winbind.idl
+                    source='''irpc.idl nfs4acl.idl ntp_signd.idl
+                              opendb.idl sasl_helpers.idl winbind.idl
                               winsif.idl winsrepl.idl winstation.idl''',
                     options="--includedir=%s --header --ndr-parser --client --python --server" % topinclude,
                     output_dir='../gen_ndr')
diff --git a/source4/librpc/rpc/dcerpc.c b/source4/librpc/rpc/dcerpc.c
index cc728668667..110da57c939 100644
--- a/source4/librpc/rpc/dcerpc.c
+++ b/source4/librpc/rpc/dcerpc.c
@@ -76,9 +76,9 @@ struct rpc_request {
 	} async;
 };
 
-_PUBLIC_ NTSTATUS dcerpc_init(struct loadparm_context *lp_ctx)
+_PUBLIC_ NTSTATUS dcerpc_init(void)
 {
-	return gensec_init(lp_ctx);
+	return gensec_init();
 }
 
 static void dcerpc_connection_dead(struct dcecli_connection *conn, NTSTATUS status);
diff --git a/source4/librpc/rpc/dcerpc.h b/source4/librpc/rpc/dcerpc.h
index 5ca6246343d..22afdf880fa 100644
--- a/source4/librpc/rpc/dcerpc.h
+++ b/source4/librpc/rpc/dcerpc.h
@@ -198,7 +198,7 @@ NTSTATUS dcerpc_bind_auth_schannel(TALLOC_CTX *tmp_ctx,
 				   struct loadparm_context *lp_ctx,
 				   uint8_t auth_level);
 struct tevent_context *dcerpc_event_context(struct dcerpc_pipe *p);
-NTSTATUS dcerpc_init(struct loadparm_context *lp_ctx);
+NTSTATUS dcerpc_init(void);
 struct smbcli_tree *dcerpc_smb_tree(struct dcecli_connection *c);
 uint16_t dcerpc_smb_fnum(struct dcecli_connection *c);
 NTSTATUS dcerpc_secondary_context(struct dcerpc_pipe *p, 
diff --git a/source4/librpc/rpc/dcerpc_connect.c b/source4/librpc/rpc/dcerpc_connect.c
index 842ef432063..c236399b529 100644
--- a/source4/librpc/rpc/dcerpc_connect.c
+++ b/source4/librpc/rpc/dcerpc_connect.c
@@ -186,16 +186,18 @@ static void continue_pipe_open_smb2(struct composite_context *ctx)
 /*
   Stage 2 of ncacn_np_smb2: Open a named pipe after successful smb2 connection
 */
-static void continue_smb2_connect(struct composite_context *ctx)
+static void continue_smb2_connect(struct tevent_req *subreq)
 {
 	struct composite_context *open_req;
-	struct composite_context *c = talloc_get_type(ctx->async.private_data,
-						      struct composite_context);
+	struct composite_context *c =
+		tevent_req_callback_data(subreq,
+		struct composite_context);
 	struct pipe_np_smb2_state *s = talloc_get_type(c->private_data,
 						       struct pipe_np_smb2_state);
 
 	/* receive result of smb2 connect request */
-	c->status = smb2_connect_recv(ctx, c, &s->tree);
+	c->status = smb2_connect_recv(subreq, c, &s->tree);
+	TALLOC_FREE(subreq);
 	if (!composite_is_ok(c)) return;
 
 	/* prepare named pipe open parameters */
@@ -220,7 +222,7 @@ static struct composite_context *dcerpc_pipe_connect_ncacn_np_smb2_send(
 {
 	struct composite_context *c;
 	struct pipe_np_smb2_state *s;
-	struct composite_context *conn_req;
+	struct tevent_req *subreq;
 	struct smbcli_options options;
 
 	/* composite context allocation and setup */
@@ -247,17 +249,17 @@ static struct composite_context *dcerpc_pipe_connect_ncacn_np_smb2_send(
 	lpcfg_smbcli_options(lp_ctx, &options);
 
 	/* send smb2 connect request */
-	conn_req = smb2_connect_send(mem_ctx, s->io.binding->host, 
+	subreq = smb2_connect_send(s, c->event_ctx,
+			s->io.binding->host,
 			lpcfg_parm_string_list(mem_ctx, lp_ctx, NULL, "smb2", "ports", NULL),
-					"IPC$", 
-				     s->io.resolve_ctx,
-				     s->io.creds,
-				     c->event_ctx,
-				     &options,
-					 lpcfg_socket_options(lp_ctx),
-					 lpcfg_gensec_settings(mem_ctx, lp_ctx)
-					 );
-	composite_continue(c, conn_req, continue_smb2_connect, c);
+			"IPC$",
+			s->io.resolve_ctx,
+			s->io.creds,
+			&options,
+			lpcfg_socket_options(lp_ctx),
+			lpcfg_gensec_settings(mem_ctx, lp_ctx));
+	if (composite_nomem(subreq, c)) return c;
+	tevent_req_set_callback(subreq, continue_smb2_connect, c);
 	return c;
 }
 
diff --git a/source4/librpc/rpc/dcerpc_sock.c b/source4/librpc/rpc/dcerpc_sock.c
index f0451ac6745..1dd993d94b2 100644
--- a/source4/librpc/rpc/dcerpc_sock.c
+++ b/source4/librpc/rpc/dcerpc_sock.c
@@ -371,10 +371,7 @@ struct pipe_tcp_state {
 };
 
 
-#if 0 /* disabled till we can resolve names to ipv6 addresses */
-static void continue_ipv6_open_socket(struct composite_context *ctx);
-#endif
-static void continue_ipv4_open_socket(struct composite_context *ctx);
+static void continue_ip_open_socket(struct composite_context *ctx);
 static void continue_ip_resolve_name(struct composite_context *ctx);
 
 static void continue_ip_resolve_name(struct composite_context *ctx)
@@ -383,62 +380,28 @@ static void continue_ip_resolve_name(struct composite_context *ctx)
 						      struct composite_context);
 	struct pipe_tcp_state *s = talloc_get_type(c->private_data,
 						   struct pipe_tcp_state);
-	struct composite_context *sock_ipv4_req;
+	struct composite_context *sock_ip_req;
 
 	c->status = resolve_name_recv(ctx, s, &s->address);
 	if (!composite_is_ok(c)) return;
 
 	/* prepare server address using host ip:port and transport name */
-	s->srvaddr = socket_address_from_strings(s->conn, "ipv4", s->address, s->port);
+	s->srvaddr = socket_address_from_strings(s->conn, "ip", s->address, s->port);
 	if (composite_nomem(s->srvaddr, c)) return;
 
-	/* resolve_nbt_name gives only ipv4 ... - send socket open request */
-	sock_ipv4_req = dcerpc_pipe_open_socket_send(c, s->conn, s->localaddr,
+	sock_ip_req = dcerpc_pipe_open_socket_send(c, s->conn, s->localaddr,
 						     s->srvaddr, s->target_hostname,
 						     NULL,
 						     NCACN_IP_TCP);
-	composite_continue(c, sock_ipv4_req, continue_ipv4_open_socket, c);
+	composite_continue(c, sock_ip_req, continue_ip_open_socket, c);
 }
 
-/*
-  Stage 2 of dcerpc_pipe_open_tcp_send: receive result of pipe open request
-  on IPv6 and send the request on IPv4 unless IPv6 transport succeeded.
-*/
-#if 0 /* disabled till we can resolve names to ipv6 addresses */
-static void continue_ipv6_open_socket(struct composite_context *ctx)
-{
-	struct composite_context *c = talloc_get_type(ctx->async.private_data,
-						      struct composite_context);
-	struct pipe_tcp_state *s = talloc_get_type(c->private_data,
-						   struct pipe_tcp_state);
-	struct composite_context *sock_ipv4_req;
-
-	/* receive result of socket open request */
-	c->status = dcerpc_pipe_open_socket_recv(ctx);
-	if (NT_STATUS_IS_OK(c->status)) {
-		composite_done(c);
-		return;
-	}
-
-	talloc_free(s->srvaddr);
-
-	/* prepare server address using host:ip and transport name */
-	s->srvaddr = socket_address_from_strings(s->conn, "ipv4", s->address, s->port);
-	if (composite_nomem(s->srvaddr, c)) return;
-
-	/* try IPv4 if IPv6 fails */
-	sock_ipv4_req = dcerpc_pipe_open_socket_send(c, s->conn, s->localaddr,
-						     s->srvaddr, s->target_hostname, 
-						     NCACN_IP_TCP);
-	composite_continue(c, sock_ipv4_req, continue_ipv4_open_socket, c);
-}
-#endif
 
 /*
   Stage 2 of dcerpc_pipe_open_tcp_send: receive result of pipe open request
-  on IPv4 transport.
+  on IP transport.
 */
-static void continue_ipv4_open_socket(struct composite_context *ctx)
+static void continue_ip_open_socket(struct composite_context *ctx)
 {
 	struct composite_context *c = talloc_get_type(ctx->async.private_data,
 						      struct composite_context);
diff --git a/source4/librpc/rpc/pyrpc_util.c b/source4/librpc/rpc/pyrpc_util.c
index 3821638fb34..385acc87e59 100644
--- a/source4/librpc/rpc/pyrpc_util.c
+++ b/source4/librpc/rpc/pyrpc_util.c
@@ -72,9 +72,9 @@ static NTSTATUS pyrpc_irpc_connect(TALLOC_CTX *mem_ctx, const char *irpc_server,
 				   struct loadparm_context *lp_ctx,
 				   struct dcerpc_binding_handle **binding_handle)
 {
-	struct messaging_context *msg;
+	struct imessaging_context *msg;
 
-	msg = messaging_client_init(mem_ctx, lpcfg_messaging_path(mem_ctx, lp_ctx), event_ctx);
+	msg = imessaging_client_init(mem_ctx, lpcfg_imessaging_path(mem_ctx, lp_ctx), event_ctx);
 	NT_STATUS_HAVE_NO_MEMORY(msg);
 
 	*binding_handle = irpc_binding_handle_by_name(mem_ctx, msg, irpc_server, table);
@@ -119,7 +119,7 @@ PyObject *py_dcerpc_interface_init_helper(PyTypeObject *type, PyObject *args, Py
 		return NULL;
 	}
 
-	status = dcerpc_init(lp_ctx);
+	status = dcerpc_init();
 	if (!NT_STATUS_IS_OK(status)) {
 		PyErr_SetNTSTATUS(status);
 		talloc_free(mem_ctx);
diff --git a/source4/librpc/wscript_build b/source4/librpc/wscript_build
index ce015ccaa52..39541b6a33b 100755
--- a/source4/librpc/wscript_build
+++ b/source4/librpc/wscript_build
@@ -4,13 +4,6 @@ bld.RECURSE('../../librpc/idl')
 bld.RECURSE('../../librpc/tools')
 bld.RECURSE('idl')
 
-bld.SAMBA_SUBSYSTEM('NDR_SERVER_ID4',
-	source='gen_ndr/ndr_server_id4.c',
-	deps='ndr',
-	public_headers='gen_ndr/server_id4.h',
-	header_path='gen_ndr'
-	)
-
 
 bld.SAMBA_SUBSYSTEM('NDR_WINSTATION',
 	source='gen_ndr/ndr_winstation.c',
@@ -50,12 +43,6 @@ bld.SAMBA_SUBSYSTEM('NDR_OPENDB',
 	)
 
 
-bld.SAMBA_SUBSYSTEM('NDR_NOTIFY',
-	source='gen_ndr/ndr_s4_notify.c',
-	public_deps='ndr NDR_SERVER_ID4'
-	)
-
-
 bld.SAMBA_SUBSYSTEM('NDR_NTP_SIGND',
 	source='gen_ndr/ndr_ntp_signd.c',
 	public_deps='ndr'
@@ -76,7 +63,7 @@ bld.SAMBA_SUBSYSTEM('NDR_WINBIND',
 # create a grouping library to consolidate our samba4 specific NDR code
 bld.SAMBA_LIBRARY('ndr-samba4',
 	source=[],
-	deps='NDR_WINBIND NDR_IRPC NDR_NFS4ACL NDR_OPENDB NDR_NOTIFY ndr-table',
+	deps='NDR_WINBIND NDR_IRPC NDR_NFS4ACL NDR_OPENDB ndr-table',
 	private_library=True,
 	grouping_library=True
 	)
@@ -92,11 +79,6 @@ bld.SAMBA_LIBRARY('dcerpc-samba4',
 
 bld.SAMBA_PIDL_TABLES('GEN_NDR_TABLES', 'gen_ndr/tables.c')
 
-if bld.env.enable_s3build:
-    s3_ndr = "NDR_WBINT"
-else:
-    s3_ndr = ""
-
 bld.SAMBA_SUBSYSTEM('ndr-table',
 	source='../../librpc/ndr/ndr_table.c gen_ndr/tables.c',
         public_deps='''ndr-standard NDR_AUDIOSRV NDR_DSBACKUP NDR_EFS
@@ -109,7 +91,7 @@ bld.SAMBA_SUBSYSTEM('ndr-table',
         NDR_SASL_HELPERS NDR_NOTIFY NDR_WINBIND NDR_FRSRPC NDR_FRSAPI
         NDR_FRSTRANS NDR_NFS4ACL NDR_NTP_SIGND NDR_DCOM NDR_WMI
         NDR_NAMED_PIPE_AUTH NDR_NTLMSSP NDR_DFSBLOBS NDR_DNSP
-        NDR_NTPRINTING NDR_DNS NDR_BACKUPKEY NDR_PREG ''' + s3_ndr,
+        NDR_NTPRINTING NDR_DNS NDR_BACKUPKEY NDR_PREG NDR_WBINT''',
         depends_on='GEN_NDR_TABLES'
         )
 
diff --git a/source4/nbt_server/interfaces.c b/source4/nbt_server/interfaces.c
index bd2143e26ce..ccbc89accf1 100644
--- a/source4/nbt_server/interfaces.c
+++ b/source4/nbt_server/interfaces.c
@@ -276,7 +276,7 @@ static NTSTATUS nbtd_add_wins_socket(struct nbtd_server *nbtsrv)
 NTSTATUS nbtd_startup_interfaces(struct nbtd_server *nbtsrv, struct loadparm_context *lp_ctx,
 				 struct interface *ifaces)
 {
-	int num_interfaces = iface_count(ifaces);
+	int num_interfaces = iface_list_count(ifaces);
 	int i;
 	TALLOC_CTX *tmp_ctx = talloc_new(nbtsrv);
 	NTSTATUS status;
@@ -286,15 +286,16 @@ NTSTATUS nbtd_startup_interfaces(struct nbtd_server *nbtsrv, struct loadparm_con
 	if (!lpcfg_bind_interfaces_only(lp_ctx)) {
 		const char *primary_address;
 
+		primary_address = iface_list_first_v4(ifaces);
+
 		/* the primary address is the address we will return
 		   for non-WINS queries not made on a specific
 		   interface */
-		if (num_interfaces > 0) {
-			primary_address = iface_n_ip(ifaces, 0);
-		} else {
+		if (primary_address == NULL) {
 			primary_address = inet_ntoa(interpret_addr2(
-							lpcfg_netbios_name(lp_ctx)));
+							    lpcfg_netbios_name(lp_ctx)));
 		}
+
 		primary_address = talloc_strdup(tmp_ctx, primary_address);
 		NT_STATUS_HAVE_NO_MEMORY(primary_address);
 
@@ -308,15 +309,21 @@ NTSTATUS nbtd_startup_interfaces(struct nbtd_server *nbtsrv, struct loadparm_con
 	}
 
 	for (i=0; i<num_interfaces; i++) {
-		const char *bcast = iface_n_bcast(ifaces, i);
+		const char *bcast;
 		const char *address, *netmask;
 
+		if (!iface_list_n_is_v4(ifaces, i)) {
+			/* v4 only for NBT protocol */
+			continue;
+		}
+
+		bcast = iface_list_n_bcast(ifaces, i);
 		/* we can't assume every interface is broadcast capable */
 		if (bcast == NULL) continue;
 
-		address = talloc_strdup(tmp_ctx, iface_n_ip(ifaces, i));
+		address = talloc_strdup(tmp_ctx, iface_list_n_ip(ifaces, i));
 		bcast   = talloc_strdup(tmp_ctx, bcast);
-		netmask = talloc_strdup(tmp_ctx, iface_n_netmask(ifaces, i));
+		netmask = talloc_strdup(tmp_ctx, iface_list_n_netmask(ifaces, i));
 
 		status = nbtd_add_socket(nbtsrv, lp_ctx,
 					 address, address, bcast, netmask);
@@ -346,7 +353,7 @@ const char **nbtd_address_list(struct nbtd_interface *iface, TALLOC_CTX *mem_ctx
 	bool is_loopback = false;
 
 	if (iface->ip_address) {
-		is_loopback = iface_same_net(iface->ip_address, "127.0.0.1", "255.0.0.0");
+		is_loopback = iface_list_same_net(iface->ip_address, "127.0.0.1", "255.0.0.0");
 		ret = str_list_add(ret, iface->ip_address);
 	}
 
@@ -356,7 +363,7 @@ const char **nbtd_address_list(struct nbtd_interface *iface, TALLOC_CTX *mem_ctx
 		if (!iface2->ip_address) continue;
 
 		if (!is_loopback) {
-			if (iface_same_net(iface2->ip_address, "127.0.0.1", "255.0.0.0")) {
+			if (iface_list_same_net(iface2->ip_address, "127.0.0.1", "255.0.0.0")) {
 				continue;
 			}
 		}
@@ -380,7 +387,7 @@ struct nbtd_interface *nbtd_find_request_iface(struct nbtd_server *nbtd_server,
 
 	/* try to find a exact match */
 	for (cur=nbtd_server->interfaces;cur;cur=cur->next) {
-		if (iface_same_net(address, cur->ip_address, cur->netmask)) {
+		if (iface_list_same_net(address, cur->ip_address, cur->netmask)) {
 			DEBUG(10,("find interface for dst[%s] ip: %s/%s (iface[%p])\n",
 				  address, cur->ip_address, cur->netmask, cur));
 			return cur;
diff --git a/source4/nbt_server/nbt_server.c b/source4/nbt_server/nbt_server.c
index 6dbdaf9f54c..175ad5e4a43 100644
--- a/source4/nbt_server/nbt_server.c
+++ b/source4/nbt_server/nbt_server.c
@@ -41,9 +41,9 @@ static void nbtd_task_init(struct task_server *task)
 	NTSTATUS status;
 	struct interface *ifaces;
 
-	load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
+	load_interface_list(task, task->lp_ctx, &ifaces);
 
-	if (iface_count(ifaces) == 0) {
+	if (iface_list_count(ifaces) == 0) {
 		task_server_terminate(task, "nbtd: no network interfaces configured", false);
 		return;
 	}
diff --git a/source4/nbt_server/wins/wins_ldb.c b/source4/nbt_server/wins/wins_ldb.c
index 0896e1d74e5..304c98d803a 100644
--- a/source4/nbt_server/wins/wins_ldb.c
+++ b/source4/nbt_server/wins/wins_ldb.c
@@ -92,8 +92,8 @@ static int wins_ldb_init(struct ldb_module *module)
 	owner = lpcfg_parm_string(lp_ctx, NULL, "winsdb", "local_owner");
 	if (!owner) {
 		struct interface *ifaces;
-		load_interfaces(module, lpcfg_interfaces(lp_ctx), &ifaces);
-		owner = iface_n_ip(ifaces, 0);
+		load_interface_list(module, lp_ctx, &ifaces);
+		owner = iface_list_first_v4(ifaces);
 		if (!owner) {
 			owner = "0.0.0.0";
 		}
diff --git a/source4/nbt_server/wins/winsdb.c b/source4/nbt_server/wins/winsdb.c
index 791ce957c7b..4a1486092b2 100644
--- a/source4/nbt_server/wins/winsdb.c
+++ b/source4/nbt_server/wins/winsdb.c
@@ -975,7 +975,7 @@ static bool winsdb_check_or_add_module_list(struct tevent_context *ev_ctx,
 		flags |= LDB_FLG_NOSYNC;
 	}
 
-	h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lock_path(h, lp_ctx, lpcfg_wins_url(lp_ctx)),
+	h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lpcfg_lock_path(h, lp_ctx, lpcfg_wins_url(lp_ctx)),
 				  NULL, NULL, flags);
 	if (!h->ldb) goto failed;
 
@@ -1011,7 +1011,7 @@ struct winsdb_handle *winsdb_connect(TALLOC_CTX *mem_ctx,
 		flags |= LDB_FLG_NOSYNC;
 	}
 
-	h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lock_path(h, lp_ctx, lpcfg_wins_url(lp_ctx)),
+	h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lpcfg_lock_path(h, lp_ctx, lpcfg_wins_url(lp_ctx)),
 				  NULL, NULL, flags);
 	if (!h->ldb) goto failed;
 
diff --git a/source4/nbt_server/wins/winsserver.c b/source4/nbt_server/wins/winsserver.c
index 1e6a0cfca87..5f1f41747f4 100644
--- a/source4/nbt_server/wins/winsserver.c
+++ b/source4/nbt_server/wins/winsserver.c
@@ -688,7 +688,7 @@ static void nbtd_wins_randomize1Clist(struct loadparm_context *lp_ctx,
 		bool same;
 
 		/* if the current one is in the same subnet, use it */
-		same = iface_same_net(addresses[idx], src->addr, mask);
+		same = iface_list_same_net(addresses[idx], src->addr, mask);
 		if (same) {
 			sidx = idx;
 			break;
@@ -1057,8 +1057,8 @@ NTSTATUS nbtd_winsserver_init(struct nbtd_server *nbtsrv)
 
 	if (owner == NULL) {
 		struct interface *ifaces;
-		load_interfaces(nbtsrv->task, lpcfg_interfaces(nbtsrv->task->lp_ctx), &ifaces);
-		owner = iface_n_ip(ifaces, 0);
+		load_interface_list(nbtsrv->task, nbtsrv->task->lp_ctx, &ifaces);
+		owner = iface_list_first_v4(ifaces);
 	}
 
 	nbtsrv->winssrv->wins_db     = winsdb_connect(nbtsrv->winssrv, nbtsrv->task->event_ctx, 
diff --git a/source4/ntptr/ntptr_base.c b/source4/ntptr/ntptr_base.c
index 268e84b1270..42e7b10d4a6 100644
--- a/source4/ntptr/ntptr_base.c
+++ b/source4/ntptr/ntptr_base.c
@@ -68,12 +68,12 @@ NTSTATUS ntptr_register(const void *_ops)
 	return NT_STATUS_OK;
 }
 
-NTSTATUS ntptr_init(struct loadparm_context *lp_ctx)
+NTSTATUS ntptr_init(void)
 {
 #define _MODULE_PROTO(init) extern NTSTATUS init(void);
 	STATIC_ntptr_MODULES_PROTO;
 	init_module_fn static_init[] = { STATIC_ntptr_MODULES };
-	init_module_fn *shared_init = load_samba_modules(NULL, lp_ctx, "ntptr");
+	init_module_fn *shared_init = load_samba_modules(NULL, "ntptr");
 
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
diff --git a/source4/ntvfs/cifs/vfs_cifs.c b/source4/ntvfs/cifs/vfs_cifs.c
index 27bf1ead241..91ca08d2de0 100644
--- a/source4/ntvfs/cifs/vfs_cifs.c
+++ b/source4/ntvfs/cifs/vfs_cifs.c
@@ -99,10 +99,12 @@ NTSTATUS ntvfs_cifs_init(void);
 #define CIFS_DOMAIN		"cifs:domain"
 #define CIFS_SHARE		"cifs:share"
 #define CIFS_USE_MACHINE_ACCT	"cifs:use-machine-account"
+#define CIFS_USE_S4U2PROXY	"cifs:use-s4u2proxy"
 #define CIFS_MAP_GENERIC	"cifs:map-generic"
 #define CIFS_MAP_TRANS2		"cifs:map-trans2"
 
 #define CIFS_USE_MACHINE_ACCT_DEFAULT	false
+#define CIFS_USE_S4U2PROXY_DEFAULT	false
 #define CIFS_MAP_GENERIC_DEFAULT	false
 #define CIFS_MAP_TRANS2_DEFAULT		true
 
@@ -150,6 +152,7 @@ static NTSTATUS cvfs_connect(struct ntvfs_module_context *ntvfs,
 
 	struct cli_credentials *credentials;
 	bool machine_account;
+	bool s4u2proxy;
 	const char* sharename;
 
 	switch (tcon->generic.level) {
@@ -187,6 +190,7 @@ static NTSTATUS cvfs_connect(struct ntvfs_module_context *ntvfs,
 	}
 
 	machine_account = share_bool_option(scfg, CIFS_USE_MACHINE_ACCT, CIFS_USE_MACHINE_ACCT_DEFAULT);
+	s4u2proxy = share_bool_option(scfg, CIFS_USE_S4U2PROXY, CIFS_USE_S4U2PROXY_DEFAULT);
 
 	p = talloc_zero(ntvfs, struct cvfs_private);
 	if (!p) {
@@ -226,9 +230,54 @@ static NTSTATUS cvfs_connect(struct ntvfs_module_context *ntvfs,
 	} else if (req->session_info->credentials) {
 		DEBUG(5, ("CIFS backend: Using delegated credentials\n"));
 		credentials = req->session_info->credentials;
+	} else if (s4u2proxy) {
+		struct ccache_container *ccc = NULL;
+		const char *err_str = NULL;
+		int ret;
+		char *impersonate_principal;
+		char *self_service;
+		char *target_service;
+
+		impersonate_principal = talloc_asprintf(req, "%s@%s",
+						req->session_info->info->account_name,
+						req->session_info->info->domain_name);
+
+		self_service = talloc_asprintf(req, "cifs/%s",
+					       lpcfg_netbios_name(ntvfs->ctx->lp_ctx));
+
+		target_service = talloc_asprintf(req, "cifs/%s", host);
+
+		DEBUG(5, ("CIFS backend: Using S4U2Proxy credentials\n"));
+
+		credentials = cli_credentials_init(p);
+		cli_credentials_set_conf(credentials, ntvfs->ctx->lp_ctx);
+		if (domain) {
+			cli_credentials_set_domain(credentials, domain, CRED_SPECIFIED);
+		}
+		status = cli_credentials_set_machine_account(credentials, ntvfs->ctx->lp_ctx);
+		if (!NT_STATUS_IS_OK(status)) {
+			return status;
+		}
+		cli_credentials_invalidate_ccache(credentials, CRED_SPECIFIED);
+		cli_credentials_set_impersonate_principal(credentials,
+							  impersonate_principal,
+							  self_service);
+		cli_credentials_set_target_service(credentials, target_service);
+		ret = cli_credentials_get_ccache(credentials,
+						 ntvfs->ctx->event_ctx,
+						 ntvfs->ctx->lp_ctx,
+						 &ccc,
+						 &err_str);
+		if (ret != 0) {
+			status = NT_STATUS_CROSSREALM_DELEGATION_FAILURE;
+			DEBUG(1,("S4U2Proxy: cli_credentials_get_ccache() gave: ret[%d] str[%s] - %s\n",
+				ret, err_str, nt_errstr(status)));
+			return status;
+		}
+
 	} else {
 		DEBUG(1,("CIFS backend: NO delegated credentials found: You must supply server, user and password or the client must supply delegated credentials\n"));
-		return NT_STATUS_INVALID_PARAMETER;
+		return NT_STATUS_INTERNAL_ERROR;
 	}
 
 	/* connect to the server, using the smbd event context */
diff --git a/source4/ntvfs/cifs_posix_cli/svfs_util.c b/source4/ntvfs/cifs_posix_cli/svfs_util.c
index d8a7909390c..f351c5840ea 100644
--- a/source4/ntvfs/cifs_posix_cli/svfs_util.c
+++ b/source4/ntvfs/cifs_posix_cli/svfs_util.c
@@ -41,16 +41,15 @@ char *cifspsx_unix_path(struct ntvfs_module_context *ntvfs,
 {
 	struct cifspsx_private *p = ntvfs->private_data;
 	char *ret;
+	char *name_lower = strlower_talloc(p, name);
 
 	if (*name != '\\') {
-		ret = talloc_asprintf(req, "%s/%s", p->connectpath, name);
+		ret = talloc_asprintf(req, "%s/%s", p->connectpath, name_lower);
 	} else {
-		ret = talloc_asprintf(req, "%s%s", p->connectpath, name);
+		ret = talloc_asprintf(req, "%s%s", p->connectpath, name_lower);
 	}
 	all_string_sub(ret, "\\", "/", 0);
-
-	strlower(ret + strlen(p->connectpath));
-
+	talloc_free(name_lower);
 	return ret;
 }
 
@@ -85,9 +84,8 @@ struct cifspsx_dir *cifspsx_list_unix(TALLOC_CTX *mem_ctx, struct ntvfs_request
 	/* the wildcard pattern is the last part */
 	mask = p+1;
 
-	low_mask = talloc_strdup(mem_ctx, mask);
+	low_mask = strlower_talloc(mem_ctx, mask);
 	if (!low_mask) { return NULL; }
-	strlower(low_mask);
 
 	odir = opendir(dir->unix_dir);
 	if (!odir) { return NULL; }
@@ -102,12 +100,11 @@ struct cifspsx_dir *cifspsx_list_unix(TALLOC_CTX *mem_ctx, struct ntvfs_request
 			continue;
 		}
 
-		low_name = talloc_strdup(mem_ctx, dent->d_name);
+		low_name = strlower_talloc(mem_ctx, dent->d_name);
 		if (!low_name) { continue; }
-		strlower(low_name);
 
 		/* check it matches the wildcard pattern */
-		if (ms_fnmatch(low_mask, low_name, PROTOCOL_NT1) != 0) {
+		if (ms_fnmatch_protocol(low_mask, low_name, PROTOCOL_NT1) != 0) {
 			continue;
 		}
 		
diff --git a/source4/ntvfs/cifs_posix_cli/vfs_cifs_posix.c b/source4/ntvfs/cifs_posix_cli/vfs_cifs_posix.c
index 2a8c69e2207..949b6dbbb76 100644
--- a/source4/ntvfs/cifs_posix_cli/vfs_cifs_posix.c
+++ b/source4/ntvfs/cifs_posix_cli/vfs_cifs_posix.c
@@ -150,7 +150,7 @@ static NTSTATUS cifspsx_unlink(struct ntvfs_module_context *ntvfs,
 
 	/* ignoring wildcards ... */
 	if (unlink(unix_path) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return NT_STATUS_OK;
@@ -179,7 +179,7 @@ static NTSTATUS cifspsx_chkpath(struct ntvfs_module_context *ntvfs,
 	unix_path = cifspsx_unix_path(ntvfs, req, cp->chkpath.in.path);
 
 	if (stat(unix_path, &st) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (!S_ISDIR(st.st_mode)) {
@@ -294,7 +294,7 @@ static NTSTATUS cifspsx_qpathinfo(struct ntvfs_module_context *ntvfs,
 	DEBUG(19,("cifspsx_qpathinfo: file %s\n", unix_path));
 	if (stat(unix_path, &st) == -1) {
 		DEBUG(19,("cifspsx_qpathinfo: file %s errno=%d\n", unix_path, errno));
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	DEBUG(19,("cifspsx_qpathinfo: file %s, stat done\n", unix_path));
 	return cifspsx_map_fileinfo(ntvfs, req, info, &st, unix_path);
@@ -320,7 +320,7 @@ static NTSTATUS cifspsx_qfileinfo(struct ntvfs_module_context *ntvfs,
 	}
 	
 	if (fstat(f->fd, &st) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return cifspsx_map_fileinfo(ntvfs, req,info, &st, f->name);
@@ -389,13 +389,13 @@ static NTSTATUS cifspsx_open(struct ntvfs_module_context *ntvfs,
 		case NTCREATEX_DISP_CREATE:
 			if (mkdir(unix_path, 0755) == -1) {
 				DEBUG(9,("cifspsx_open: mkdir %s errno=%d\n", unix_path, errno));
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 			break;
 		case NTCREATEX_DISP_OPEN_IF:
 			if (mkdir(unix_path, 0755) == -1 && errno != EEXIST) {
 				DEBUG(9,("cifspsx_open: mkdir %s errno=%d\n", unix_path, errno));
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 			break;
 		}
@@ -404,13 +404,13 @@ static NTSTATUS cifspsx_open(struct ntvfs_module_context *ntvfs,
 do_open:
 	fd = open(unix_path, flags, 0644);
 	if (fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (fstat(fd, &st) == -1) {
 		DEBUG(9,("cifspsx_open: fstat errno=%d\n", errno));
 		close(fd);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	status = ntvfs_handle_new(ntvfs, req, &handle);
@@ -459,7 +459,7 @@ static NTSTATUS cifspsx_mkdir(struct ntvfs_module_context *ntvfs,
 	unix_path = cifspsx_unix_path(ntvfs, req, md->mkdir.in.path);
 
 	if (mkdir(unix_path, 0777) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return NT_STATUS_OK;
@@ -478,7 +478,7 @@ static NTSTATUS cifspsx_rmdir(struct ntvfs_module_context *ntvfs,
 	unix_path = cifspsx_unix_path(ntvfs, req, rd->in.path);
 
 	if (rmdir(unix_path) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return NT_STATUS_OK;
@@ -502,7 +502,7 @@ static NTSTATUS cifspsx_rename(struct ntvfs_module_context *ntvfs,
 	unix_path2 = cifspsx_unix_path(ntvfs, req, ren->rename.in.pattern2);
 
 	if (rename(unix_path1, unix_path2) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	
 	return NT_STATUS_OK;
@@ -541,7 +541,7 @@ static NTSTATUS cifspsx_read(struct ntvfs_module_context *ntvfs,
 		    rd->readx.in.maxcnt,
 		    rd->readx.in.offset);
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	rd->readx.out.nread = ret;
@@ -577,7 +577,7 @@ static NTSTATUS cifspsx_write(struct ntvfs_module_context *ntvfs,
 		     wr->writex.in.count,
 		     wr->writex.in.offset);
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 		
 	wr->writex.out.nwritten = ret;
@@ -648,7 +648,7 @@ static NTSTATUS cifspsx_close(struct ntvfs_module_context *ntvfs,
 	}
 
 	if (close(f->fd) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	DLIST_REMOVE(p->open_files, f);
@@ -738,7 +738,7 @@ static NTSTATUS cifspsx_setfileinfo(struct ntvfs_module_context *ntvfs,
 	case RAW_SFILEINFO_END_OF_FILE_INFORMATION:
 		if (ftruncate(f->fd, 
 			      info->end_of_file_info.in.size) == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 		break;
 	case RAW_SFILEINFO_SETATTRE:
@@ -784,7 +784,7 @@ static NTSTATUS cifspsx_fsinfo(struct ntvfs_module_context *ntvfs,
 	if (sys_fsusage(p->connectpath,
 			&fs->generic.out.blocks_free, 
 			&fs->generic.out.blocks_total) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	fs->generic.out.block_size = 512;
@@ -824,7 +824,7 @@ static NTSTATUS cifspsx_fsattr(struct ntvfs_module_context *ntvfs,
 	}
 
 	if (stat(p->connectpath, &st) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	unix_to_nt_time(&fs->generic.out.create_time, st.st_ctime);
diff --git a/source4/ntvfs/common/brlock.c b/source4/ntvfs/common/brlock.c
index 94041d20141..a6232f85964 100644
--- a/source4/ntvfs/common/brlock.c
+++ b/source4/ntvfs/common/brlock.c
@@ -26,7 +26,7 @@
 
 #include "includes.h"
 #include "system/filesys.h"
-#include <tdb.h>
+#include "tdb_compat.h"
 #include "messaging/messaging.h"
 #include "lib/messaging/irpc.h"
 #include "libcli/libcli.h"
@@ -38,26 +38,26 @@ static const struct brlock_ops *ops;
 /*
   set the brl backend ops
 */
-void brl_set_ops(const struct brlock_ops *new_ops)
+void brlock_set_ops(const struct brlock_ops *new_ops)
 {
 	ops = new_ops;
 }
 
 /*
   Open up the brlock database. Close it down using talloc_free(). We
-  need the messaging_ctx to allow for pending lock notifications.
+  need the imessaging_ctx to allow for pending lock notifications.
 */
-struct brl_context *brl_init(TALLOC_CTX *mem_ctx, struct server_id server, 
+struct brl_context *brlock_init(TALLOC_CTX *mem_ctx, struct server_id server, 
 			     struct loadparm_context *lp_ctx,
-			     struct messaging_context *messaging_ctx)
+			     struct imessaging_context *imessaging_ctx)
 {
 	if (ops == NULL) {
 		brl_tdb_init_ops();
 	}
-	return ops->brl_init(mem_ctx, server, lp_ctx, messaging_ctx);
+	return ops->brl_init(mem_ctx, server, lp_ctx, imessaging_ctx);
 }
 
-struct brl_handle *brl_create_handle(TALLOC_CTX *mem_ctx, struct ntvfs_handle *ntvfs, DATA_BLOB *file_key)
+struct brl_handle *brlock_create_handle(TALLOC_CTX *mem_ctx, struct ntvfs_handle *ntvfs, DATA_BLOB *file_key)
 {
 	return ops->brl_create_handle(mem_ctx, ntvfs, file_key);
 }
@@ -69,7 +69,7 @@ struct brl_handle *brl_create_handle(TALLOC_CTX *mem_ctx, struct ntvfs_handle *n
   someone else closing an overlapping lock range) a messaging
   notification is sent, identified by the notify_ptr
 */
-NTSTATUS brl_lock(struct brl_context *brl,
+NTSTATUS brlock_lock(struct brl_context *brl,
 		  struct brl_handle *brlh,
 		  uint32_t smbpid,
 		  uint64_t start, uint64_t size, 
@@ -83,7 +83,7 @@ NTSTATUS brl_lock(struct brl_context *brl,
 /*
  Unlock a range of bytes.
 */
-NTSTATUS brl_unlock(struct brl_context *brl,
+NTSTATUS brlock_unlock(struct brl_context *brl,
 		    struct brl_handle *brlh, 
 		    uint32_t smbpid,
 		    uint64_t start, uint64_t size)
@@ -96,7 +96,7 @@ NTSTATUS brl_unlock(struct brl_context *brl,
   given up trying to establish a lock or when they have succeeded in
   getting it. In either case they no longer need to be notified.
 */
-NTSTATUS brl_remove_pending(struct brl_context *brl,
+NTSTATUS brlock_remove_pending(struct brl_context *brl,
 			    struct brl_handle *brlh, 
 			    void *notify_ptr)
 {
@@ -107,7 +107,7 @@ NTSTATUS brl_remove_pending(struct brl_context *brl,
 /*
   Test if we are allowed to perform IO on a region of an open file
 */
-NTSTATUS brl_locktest(struct brl_context *brl,
+NTSTATUS brlock_locktest(struct brl_context *brl,
 		      struct brl_handle *brlh,
 		      uint32_t smbpid, 
 		      uint64_t start, uint64_t size, 
@@ -120,7 +120,7 @@ NTSTATUS brl_locktest(struct brl_context *brl,
 /*
  Remove any locks associated with a open file.
 */
-NTSTATUS brl_close(struct brl_context *brl,
+NTSTATUS brlock_close(struct brl_context *brl,
 		   struct brl_handle *brlh)
 {
 	return ops->brl_close(brl, brlh);
@@ -129,7 +129,7 @@ NTSTATUS brl_close(struct brl_context *brl,
 /*
  Get a number of locks associated with a open file.
 */
-NTSTATUS brl_count(struct brl_context *brl,
+NTSTATUS brlock_count(struct brl_context *brl,
 		   struct brl_handle *brlh,
 		   int *count)
 {
diff --git a/source4/ntvfs/common/brlock.h b/source4/ntvfs/common/brlock.h
index e5e618d045d..703538f0738 100644
--- a/source4/ntvfs/common/brlock.h
+++ b/source4/ntvfs/common/brlock.h
@@ -24,7 +24,7 @@
 struct brlock_ops {
 	struct brl_context *(*brl_init)(TALLOC_CTX *, struct server_id , 
 					struct loadparm_context *lp_ctx,
-					struct messaging_context *);
+					struct imessaging_context *);
 	struct brl_handle *(*brl_create_handle)(TALLOC_CTX *, struct ntvfs_handle *, DATA_BLOB *);
 	NTSTATUS (*brl_lock)(struct brl_context *,
 			     struct brl_handle *,
@@ -52,7 +52,7 @@ struct brlock_ops {
 };
 
 
-void brl_set_ops(const struct brlock_ops *new_ops);
+void brlock_set_ops(const struct brlock_ops *new_ops);
 void brl_tdb_init_ops(void);
 void brl_ctdb_init_ops(void);
 
diff --git a/source4/ntvfs/common/brlock_tdb.c b/source4/ntvfs/common/brlock_tdb.c
index 35d4c27d0f0..817448377cf 100644
--- a/source4/ntvfs/common/brlock_tdb.c
+++ b/source4/ntvfs/common/brlock_tdb.c
@@ -26,9 +26,9 @@
 
 #include "includes.h"
 #include "system/filesys.h"
-#include <tdb.h>
+#include "tdb_compat.h"
 #include "messaging/messaging.h"
-#include "tdb_wrap.h"
+#include "lib/util/tdb_wrap.h"
 #include "lib/messaging/irpc.h"
 #include "libcli/libcli.h"
 #include "cluster/cluster.h"
@@ -48,7 +48,7 @@
 struct brl_context {
 	struct tdb_wrap *w;
 	struct server_id server;
-	struct messaging_context *messaging_ctx;
+	struct imessaging_context *imessaging_ctx;
 };
 
 /*
@@ -89,12 +89,12 @@ static bool brl_invalid_lock_range(uint64_t start, uint64_t size)
 
 /*
   Open up the brlock.tdb database. Close it down using
-  talloc_free(). We need the messaging_ctx to allow for
+  talloc_free(). We need the imessaging_ctx to allow for
   pending lock notifications.
 */
 static struct brl_context *brl_tdb_init(TALLOC_CTX *mem_ctx, struct server_id server, 
 					struct loadparm_context *lp_ctx,
-				    struct messaging_context *messaging_ctx)
+				    struct imessaging_context *imessaging_ctx)
 {
 	struct brl_context *brl;
 
@@ -110,7 +110,7 @@ static struct brl_context *brl_tdb_init(TALLOC_CTX *mem_ctx, struct server_id se
 	}
 
 	brl->server = server;
-	brl->messaging_ctx = messaging_ctx;
+	brl->imessaging_ctx = imessaging_ctx;
 
 	return brl;
 }
@@ -333,7 +333,7 @@ static NTSTATUS brl_tdb_lock(struct brl_context *brl,
 		}
 	}
 
-	dbuf = tdb_fetch(brl->w->tdb, kbuf);
+	dbuf = tdb_fetch_compat(brl->w->tdb, kbuf);
 
 	lock.context.smbpid = smbpid;
 	lock.context.server = brl->server;
@@ -419,7 +419,7 @@ static void brl_tdb_notify_unlock(struct brl_context *brl,
 			if (locks[i].lock_type == PENDING_WRITE_LOCK) {
 				last_notice = i;
 			}
-			messaging_send_ptr(brl->messaging_ctx, locks[i].context.server, 
+			imessaging_send_ptr(brl->imessaging_ctx, locks[i].context.server,
 					   MSG_BRL_RETRY, locks[i].notify_ptr);
 		}
 	}
@@ -468,7 +468,7 @@ static NTSTATUS brl_tdb_unlock(struct brl_context *brl,
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
-	dbuf = tdb_fetch(brl->w->tdb, kbuf);
+	dbuf = tdb_fetch_compat(brl->w->tdb, kbuf);
 	if (!dbuf.dptr) {
 		tdb_chainunlock(brl->w->tdb, kbuf);
 		return NT_STATUS_RANGE_NOT_LOCKED;
@@ -568,7 +568,7 @@ static NTSTATUS brl_tdb_remove_pending(struct brl_context *brl,
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
-	dbuf = tdb_fetch(brl->w->tdb, kbuf);
+	dbuf = tdb_fetch_compat(brl->w->tdb, kbuf);
 	if (!dbuf.dptr) {
 		tdb_chainunlock(brl->w->tdb, kbuf);
 		return NT_STATUS_RANGE_NOT_LOCKED;
@@ -639,7 +639,7 @@ static NTSTATUS brl_tdb_locktest(struct brl_context *brl,
 		return NT_STATUS_INVALID_LOCK_RANGE;
 	}
 
-	dbuf = tdb_fetch(brl->w->tdb, kbuf);
+	dbuf = tdb_fetch_compat(brl->w->tdb, kbuf);
 	if (dbuf.dptr == NULL) {
 		return NT_STATUS_OK;
 	}
@@ -686,7 +686,7 @@ static NTSTATUS brl_tdb_close(struct brl_context *brl,
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
-	dbuf = tdb_fetch(brl->w->tdb, kbuf);
+	dbuf = tdb_fetch_compat(brl->w->tdb, kbuf);
 	if (!dbuf.dptr) {
 		tdb_chainunlock(brl->w->tdb, kbuf);
 		return NT_STATUS_OK;
@@ -751,7 +751,7 @@ static NTSTATUS brl_tdb_count(struct brl_context *brl, struct brl_handle *brlh,
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
-	dbuf = tdb_fetch(brl->w->tdb, kbuf);
+	dbuf = tdb_fetch_compat(brl->w->tdb, kbuf);
 	if (!dbuf.dptr) {
 		tdb_chainunlock(brl->w->tdb, kbuf);
 		return NT_STATUS_OK;
@@ -779,5 +779,5 @@ static const struct brlock_ops brlock_tdb_ops = {
 
 void brl_tdb_init_ops(void)
 {
-	brl_set_ops(&brlock_tdb_ops);
+	brlock_set_ops(&brlock_tdb_ops);
 }
diff --git a/source4/ntvfs/common/notify.c b/source4/ntvfs/common/notify.c
index 0b5f91bfe19..98f17a65654 100644
--- a/source4/ntvfs/common/notify.c
+++ b/source4/ntvfs/common/notify.c
@@ -25,12 +25,12 @@
 
 #include "includes.h"
 #include "system/filesys.h"
-#include <tdb.h>
+#include "../lib/tdb_compat/tdb_compat.h"
 #include "../lib/util/util_tdb.h"
 #include "messaging/messaging.h"
-#include "tdb_wrap.h"
+#include "lib/util/tdb_wrap.h"
 #include "lib/messaging/irpc.h"
-#include "librpc/gen_ndr/ndr_s4_notify.h"
+#include "librpc/gen_ndr/ndr_notify.h"
 #include "../lib/util/dlinklist.h"
 #include "ntvfs/common/ntvfs_common.h"
 #include "ntvfs/sysdep/sys_notify.h"
@@ -41,7 +41,7 @@
 struct notify_context {
 	struct tdb_wrap *w;
 	struct server_id server;
-	struct messaging_context *messaging_ctx;
+	struct imessaging_context *imessaging_ctx;
 	struct notify_list *list;
 	struct notify_array *array;
 	int seqnum;
@@ -63,7 +63,7 @@ struct notify_list {
 #define NOTIFY_ENABLE_DEFAULT	true
 
 static NTSTATUS notify_remove_all(struct notify_context *notify);
-static void notify_handler(struct messaging_context *msg_ctx, void *private_data, 
+static void notify_handler(struct imessaging_context *msg_ctx, void *private_data,
 			   uint32_t msg_type, struct server_id server_id, DATA_BLOB *data);
 
 /*
@@ -71,18 +71,18 @@ static void notify_handler(struct messaging_context *msg_ctx, void *private_data
 */
 static int notify_destructor(struct notify_context *notify)
 {
-	messaging_deregister(notify->messaging_ctx, MSG_PVFS_NOTIFY, notify);
+	imessaging_deregister(notify->imessaging_ctx, MSG_PVFS_NOTIFY, notify);
 	notify_remove_all(notify);
 	return 0;
 }
 
 /*
   Open up the notify.tdb database. You should close it down using
-  talloc_free(). We need the messaging_ctx to allow for notifications
+  talloc_free(). We need the imessaging_ctx to allow for notifications
   via internal messages
 */
 struct notify_context *notify_init(TALLOC_CTX *mem_ctx, struct server_id server, 
-				   struct messaging_context *messaging_ctx,
+				   struct imessaging_context *imessaging_ctx,
 				   struct loadparm_context *lp_ctx,
 				   struct tevent_context *ev,
 				   struct share_config *scfg)
@@ -109,7 +109,7 @@ struct notify_context *notify_init(TALLOC_CTX *mem_ctx, struct server_id server,
 	}
 
 	notify->server = server;
-	notify->messaging_ctx = messaging_ctx;
+	notify->imessaging_ctx = imessaging_ctx;
 	notify->list = NULL;
 	notify->array = NULL;
 	notify->seqnum = tdb_get_seqnum(notify->w->tdb);
@@ -118,7 +118,7 @@ struct notify_context *notify_init(TALLOC_CTX *mem_ctx, struct server_id server,
 
 	/* register with the messaging subsystem for the notify
 	   message type */
-	messaging_register(notify->messaging_ctx, notify, 
+	imessaging_register(notify->imessaging_ctx, notify,
 			   MSG_PVFS_NOTIFY, notify_handler);
 
 	notify->sys_notify_ctx = sys_notify_context_create(scfg, notify, ev);
@@ -247,7 +247,7 @@ static NTSTATUS notify_save(struct notify_context *notify)
 /*
   handle incoming notify messages
 */
-static void notify_handler(struct messaging_context *msg_ctx, void *private_data, 
+static void notify_handler(struct imessaging_context *msg_ctx, void *private_data,
 			   uint32_t msg_type, struct server_id server_id, DATA_BLOB *data)
 {
 	struct notify_context *notify = talloc_get_type(private_data, struct notify_context);
@@ -563,7 +563,7 @@ static void notify_send(struct notify_context *notify, struct notify_entry *e,
 		return;
 	}
 
-	status = messaging_send(notify->messaging_ctx, e->server, 
+	status = imessaging_send(notify->imessaging_ctx, e->server,
 				MSG_PVFS_NOTIFY, &data);
 	talloc_free(tmp_ctx);
 }
diff --git a/source4/ntvfs/common/opendb.c b/source4/ntvfs/common/opendb.c
index 12fe7015a75..29081ef7d60 100644
--- a/source4/ntvfs/common/opendb.c
+++ b/source4/ntvfs/common/opendb.c
@@ -55,7 +55,7 @@ void odb_set_ops(const struct opendb_ops *new_ops)
 
 /*
   Open up the openfiles.tdb database. Close it down using
-  talloc_free(). We need the messaging_ctx to allow for pending open
+  talloc_free(). We need the imessaging_ctx to allow for pending open
   notifications.
 */
 struct odb_context *odb_init(TALLOC_CTX *mem_ctx, 
diff --git a/source4/ntvfs/common/opendb_tdb.c b/source4/ntvfs/common/opendb_tdb.c
index 881fd5b57a4..9884e1f8b0d 100644
--- a/source4/ntvfs/common/opendb_tdb.c
+++ b/source4/ntvfs/common/opendb_tdb.c
@@ -40,9 +40,9 @@
 
 #include "includes.h"
 #include "system/filesys.h"
-#include <tdb.h>
+#include "../lib/tdb_compat/tdb_compat.h"
 #include "messaging/messaging.h"
-#include "tdb_wrap.h"
+#include "lib/util/tdb_wrap.h"
 #include "lib/messaging/irpc.h"
 #include "librpc/gen_ndr/ndr_opendb.h"
 #include "ntvfs/ntvfs.h"
@@ -74,13 +74,13 @@ struct odb_lock {
 	} can_open;
 };
 
-static NTSTATUS odb_oplock_break_send(struct messaging_context *msg_ctx,
+static NTSTATUS odb_oplock_break_send(struct imessaging_context *msg_ctx,
 				      struct opendb_entry *e,
 				      uint8_t level);
 
 /*
   Open up the openfiles.tdb database. Close it down using
-  talloc_free(). We need the messaging_ctx to allow for pending open
+  talloc_free(). We need the imessaging_ctx to allow for pending open
   notifications.
 */
 static struct odb_context *odb_tdb_init(TALLOC_CTX *mem_ctx, 
@@ -238,7 +238,7 @@ static NTSTATUS odb_pull_record(struct odb_lock *lck, struct opendb_file *file)
 	DATA_BLOB blob;
 	enum ndr_err_code ndr_err;
 
-	dbuf = tdb_fetch(odb->w->tdb, lck->key);
+	dbuf = tdb_fetch_compat(odb->w->tdb, lck->key);
 	if (dbuf.dptr == NULL) {
 		return NT_STATUS_OBJECT_NAME_NOT_FOUND;
 	}
@@ -294,7 +294,7 @@ static NTSTATUS odb_push_record(struct odb_lock *lck, struct opendb_file *file)
 /*
   send an oplock break to a client
 */
-static NTSTATUS odb_oplock_break_send(struct messaging_context *msg_ctx,
+static NTSTATUS odb_oplock_break_send(struct imessaging_context *msg_ctx,
 				      struct opendb_entry *e,
 				      uint8_t level)
 {
@@ -311,7 +311,7 @@ static NTSTATUS odb_oplock_break_send(struct messaging_context *msg_ctx,
 
 	blob = data_blob_const(&op_break, sizeof(op_break));
 
-	status = messaging_send(msg_ctx, e->server,
+	status = imessaging_send(msg_ctx, e->server,
 				MSG_NTVFS_OPLOCK_BREAK, &blob);
 	NT_STATUS_NOT_OK_RETURN(status);
 
@@ -611,7 +611,7 @@ static NTSTATUS odb_tdb_close_file(struct odb_lock *lck, void *file_handle,
 
 	/* send any pending notifications, removing them once sent */
 	for (i=0;i<lck->file.num_pending;i++) {
-		messaging_send_ptr(odb->ntvfs_ctx->msg_ctx,
+		imessaging_send_ptr(odb->ntvfs_ctx->msg_ctx,
 				   lck->file.pending[i].server,
 				   MSG_PVFS_RETRY_OPEN,
 				   lck->file.pending[i].notify_ptr);
@@ -666,7 +666,7 @@ static NTSTATUS odb_tdb_update_oplock(struct odb_lock *lck, void *file_handle,
 
 	/* send any pending notifications, removing them once sent */
 	for (i=0;i<lck->file.num_pending;i++) {
-		messaging_send_ptr(odb->ntvfs_ctx->msg_ctx,
+		imessaging_send_ptr(odb->ntvfs_ctx->msg_ctx,
 				   lck->file.pending[i].server,
 				   MSG_PVFS_RETRY_OPEN,
 				   lck->file.pending[i].notify_ptr);
diff --git a/source4/ntvfs/ipc/vfs_ipc.c b/source4/ntvfs/ipc/vfs_ipc.c
index 185e72cfb0f..fc9ff4fb12d 100644
--- a/source4/ntvfs/ipc/vfs_ipc.c
+++ b/source4/ntvfs/ipc/vfs_ipc.c
@@ -353,7 +353,7 @@ static void ipc_open_done(struct tevent_req *subreq)
 				       &p->allocation_size);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		goto reply;
 	}
 
@@ -602,7 +602,7 @@ static void ipc_read_done(struct tevent_req *subreq)
 	ret = tstream_readv_pdu_queue_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		goto reply;
 	}
 
@@ -687,7 +687,7 @@ static void ipc_write_done(struct tevent_req *subreq)
 	ret = tstream_writev_queue_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		goto reply;
 	}
 
@@ -1009,7 +1009,7 @@ static void ipc_trans_writev_done(struct tevent_req *subreq)
 		status = NT_STATUS_PIPE_DISCONNECTED;
 		goto reply;
 	} else if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		goto reply;
 	}
 
@@ -1045,7 +1045,7 @@ static void ipc_trans_readv_done(struct tevent_req *subreq)
 	ret = tstream_readv_pdu_queue_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		goto reply;
 	}
 
@@ -1215,7 +1215,7 @@ static void ipc_ioctl_writev_done(struct tevent_req *subreq)
 	ret = tstream_writev_queue_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		goto reply;
 	}
 
@@ -1251,7 +1251,7 @@ static void ipc_ioctl_readv_done(struct tevent_req *subreq)
 	ret = tstream_readv_pdu_queue_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		goto reply;
 	}
 
diff --git a/source4/ntvfs/nbench/vfs_nbench.c b/source4/ntvfs/nbench/vfs_nbench.c
index 35cd86f34db..565f2dafedc 100644
--- a/source4/ntvfs/nbench/vfs_nbench.c
+++ b/source4/ntvfs/nbench/vfs_nbench.c
@@ -165,10 +165,9 @@ static NTSTATUS nbench_disconnect(struct ntvfs_module_context *ntvfs)
 static void nbench_unlink_send(struct ntvfs_request *req)
 {
 	union smb_unlink *unl = req->async_states->private_data;
-
 	nbench_log(req, "Unlink \"%s\" 0x%x %s\n", 
 		   unl->unlink.in.pattern, unl->unlink.in.attrib, 
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -213,7 +212,7 @@ static void nbench_chkpath_send(struct ntvfs_request *req)
 
 	nbench_log(req, "Chkpath \"%s\" %s\n", 
 		   cp->chkpath.in.path, 
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -239,7 +238,7 @@ static void nbench_qpathinfo_send(struct ntvfs_request *req)
 	nbench_log(req, "QUERY_PATH_INFORMATION \"%s\" %d %s\n", 
 		   info->generic.in.file.path, 
 		   info->generic.level,
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -264,7 +263,7 @@ static void nbench_qfileinfo_send(struct ntvfs_request *req)
 	nbench_log(req, "QUERY_FILE_INFORMATION %s %d %s\n", 
 		   nbench_ntvfs_handle_string(req, info->generic.in.file.ntvfs),
 		   info->generic.level,
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -289,7 +288,7 @@ static void nbench_setpathinfo_send(struct ntvfs_request *req)
 	nbench_log(req, "SET_PATH_INFORMATION \"%s\" %d %s\n", 
 		   st->generic.in.file.path, 
 		   st->generic.level,
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -321,7 +320,7 @@ static void nbench_open_send(struct ntvfs_request *req)
 			   io->ntcreatex.in.create_options, 
 			   io->ntcreatex.in.open_disposition, 
 			   nbench_ntvfs_handle_string(req, io->ntcreatex.out.file.ntvfs),
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 
 	default:
@@ -373,7 +372,7 @@ static void nbench_rmdir_send(struct ntvfs_request *req)
 
 	nbench_log(req, "Rmdir \"%s\" %s\n", 
 		   rd->in.path, 
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -400,7 +399,7 @@ static void nbench_rename_send(struct ntvfs_request *req)
 		nbench_log(req, "Rename \"%s\" \"%s\" %s\n", 
 			   ren->rename.in.pattern1, 
 			   ren->rename.in.pattern2, 
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 
 	default:
@@ -459,7 +458,7 @@ static void nbench_read_send(struct ntvfs_request *req)
 			   (int)rd->readx.in.offset,
 			   rd->readx.in.maxcnt,
 			   rd->readx.out.nread,
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 	default:
 		nbench_log(req, "Read-%d - NOT HANDLED\n",
@@ -497,7 +496,7 @@ static void nbench_write_send(struct ntvfs_request *req)
 			   (int)wr->writex.in.offset,
 			   wr->writex.in.count,
 			   wr->writex.out.nwritten,
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 
 	case RAW_WRITE_WRITE:
@@ -509,7 +508,7 @@ static void nbench_write_send(struct ntvfs_request *req)
 			   wr->write.in.offset,
 			   wr->write.in.count,
 			   wr->write.out.nwritten,
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 
 	default:
@@ -563,12 +562,12 @@ static void nbench_flush_send(struct ntvfs_request *req)
 	case RAW_FLUSH_FLUSH:
 		nbench_log(req, "Flush %s %s\n",
 			   nbench_ntvfs_handle_string(req, io->flush.in.file.ntvfs),
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 	case RAW_FLUSH_ALL:
 		nbench_log(req, "Flush %d %s\n",
 			   0xFFFF,
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 	default:
 		nbench_log(req, "Flush-%d - NOT HANDLED\n",
@@ -601,7 +600,7 @@ static void nbench_close_send(struct ntvfs_request *req)
 	case RAW_CLOSE_CLOSE:
 		nbench_log(req, "Close %s %s\n",
 			   nbench_ntvfs_handle_string(req, io->close.in.file.ntvfs),
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 
 	default:
@@ -718,14 +717,14 @@ static void nbench_lock_send(struct ntvfs_request *req)
 			   nbench_ntvfs_handle_string(req, lck->lockx.in.file.ntvfs),
 			   (int)lck->lockx.in.locks[0].offset,
 			   (int)lck->lockx.in.locks[0].count,
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 	} else if (lck->generic.level == RAW_LOCK_LOCKX &&
 		   lck->lockx.in.ulock_cnt == 1) {
 		nbench_log(req, "UnlockX %s %d %d %s\n", 
 			   nbench_ntvfs_handle_string(req, lck->lockx.in.file.ntvfs),
 			   (int)lck->lockx.in.locks[0].offset,
 			   (int)lck->lockx.in.locks[0].count,
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 	} else {
 		nbench_log(req, "Lock-%d - NOT HANDLED\n", lck->generic.level);
 	}
@@ -753,7 +752,7 @@ static void nbench_setfileinfo_send(struct ntvfs_request *req)
 	nbench_log(req, "SET_FILE_INFORMATION %s %d %s\n", 
 		   nbench_ntvfs_handle_string(req, info->generic.in.file.ntvfs),
 		   info->generic.level,
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -778,7 +777,7 @@ static void nbench_fsinfo_send(struct ntvfs_request *req)
 
 	nbench_log(req, "QUERY_FS_INFORMATION %d %s\n", 
 		   fs->generic.level, 
-		   get_nt_error_c_code(req->async_states->status));
+		   get_nt_error_c_code(req, req->async_states->status));
 
 	PASS_THRU_REP_POST(req);
 }
@@ -832,7 +831,7 @@ static void nbench_search_first_send(struct ntvfs_request *req)
 			   io->t2ffirst.data_level,
 			   io->t2ffirst.in.max_count,
 			   io->t2ffirst.out.count,
-			   get_nt_error_c_code(req->async_states->status));
+			   get_nt_error_c_code(req, req->async_states->status));
 		break;
 		
 	default:
diff --git a/source4/ntvfs/ntvfs.h b/source4/ntvfs/ntvfs.h
index 203b588e087..463acc3c801 100644
--- a/source4/ntvfs/ntvfs.h
+++ b/source4/ntvfs/ntvfs.h
@@ -24,7 +24,7 @@
 #include "libcli/raw/interfaces.h"
 #include "param/share.h"
 #include "librpc/gen_ndr/security.h"
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
 /* modules can use the following to determine if the interface has changed */
 /* version 1 -> 0 - make module stacking easier -- metze */
@@ -201,7 +201,7 @@ struct ntvfs_context {
 	struct server_id server_id;
 	struct loadparm_context *lp_ctx;
 	struct tevent_context *event_ctx;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 
 	struct {
 		void *private_data;
@@ -330,9 +330,9 @@ struct ntvfs_critical_sizes {
 	.sizeof_ntvfs_handle_data	= sizeof(struct ntvfs_handle_data), \
     }
 
-struct messaging_context;
+struct imessaging_context;
 #include "librpc/gen_ndr/security.h"
-#include "librpc/gen_ndr/s4_notify.h"
+#include "librpc/gen_ndr/notify.h"
 #include "ntvfs/ntvfs_proto.h"
 
 #endif /* _NTVFS_H_ */
diff --git a/source4/ntvfs/ntvfs_base.c b/source4/ntvfs/ntvfs_base.c
index 1b1a53361be..448d2919df3 100644
--- a/source4/ntvfs/ntvfs_base.c
+++ b/source4/ntvfs/ntvfs_base.c
@@ -153,7 +153,7 @@ bool ntvfs_interface_differs(const struct ntvfs_critical_sizes *const iface)
 NTSTATUS ntvfs_init_connection(TALLOC_CTX *mem_ctx, struct share_config *scfg, enum ntvfs_type type,
 			       enum protocol_types protocol,
 			       uint64_t ntvfs_client_caps,
-			       struct tevent_context *ev, struct messaging_context *msg,
+			       struct tevent_context *ev, struct imessaging_context *msg,
 			       struct loadparm_context *lp_ctx,
 			       struct server_id server_id, struct ntvfs_context **_ctx)
 {
@@ -235,7 +235,7 @@ NTSTATUS ntvfs_init(struct loadparm_context *lp_ctx)
 	if (initialized) return NT_STATUS_OK;
 	initialized = true;
 	
-	shared_init = load_samba_modules(NULL, lp_ctx, "ntvfs");
+	shared_init = load_samba_modules(NULL, "ntvfs");
 
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
diff --git a/source4/ntvfs/ntvfs_generic.c b/source4/ntvfs/ntvfs_generic.c
index 8e1eb0bc668..bed9c9c7556 100644
--- a/source4/ntvfs/ntvfs_generic.c
+++ b/source4/ntvfs/ntvfs_generic.c
@@ -576,9 +576,6 @@ static NTSTATUS ntvfs_map_fsinfo_finish(struct ntvfs_module_context *ntvfs,
 
 	/* and convert it to the required level */
 	switch (fs->generic.level) {
-	case RAW_QFS_GENERIC:
-		return NT_STATUS_INVALID_LEVEL;
-
 	case RAW_QFS_DSKATTR: {
 		/* map from generic to DSKATTR */
 		unsigned int bpunit = 64;
@@ -666,8 +663,11 @@ static NTSTATUS ntvfs_map_fsinfo_finish(struct ntvfs_module_context *ntvfs,
 		fs->objectid_information.out.guid = fs2->generic.out.guid;
 		ZERO_STRUCT(fs->objectid_information.out.unknown);
 		return NT_STATUS_OK;
-	}
 
+	case RAW_QFS_GENERIC:
+	case RAW_QFS_UNIX_INFO:
+		return NT_STATUS_INVALID_LEVEL;
+	}
 
 	return NT_STATUS_INVALID_LEVEL;
 }
@@ -715,8 +715,6 @@ NTSTATUS ntvfs_map_fileinfo(TALLOC_CTX *mem_ctx,
 	int i;
 	/* and convert it to the required level using results in info2 */
 	switch (info->generic.level) {
-		case RAW_FILEINFO_GENERIC:
-		return NT_STATUS_INVALID_LEVEL;
 	case RAW_FILEINFO_GETATTR:
 		info->getattr.out.attrib = info2->generic.out.attrib & 0xff;
 		info->getattr.out.size = info2->generic.out.size;
@@ -931,6 +929,13 @@ NTSTATUS ntvfs_map_fileinfo(TALLOC_CTX *mem_ctx,
 		info->unix_link_info.out.link_dest = info2->generic.out.link_dest;
 		return NT_STATUS_OK;
 #endif
+	case RAW_FILEINFO_GENERIC:
+	case RAW_FILEINFO_SEC_DESC:
+	case RAW_FILEINFO_EA_LIST:
+	case RAW_FILEINFO_UNIX_INFO2:
+	case RAW_FILEINFO_SMB2_ALL_EAS:
+	case RAW_FILEINFO_SMB2_ALL_INFORMATION:
+		return NT_STATUS_INVALID_LEVEL;
 	}
 
 	return NT_STATUS_INVALID_LEVEL;
diff --git a/source4/ntvfs/posix/pvfs_acl.c b/source4/ntvfs/posix/pvfs_acl.c
index 67b544d4dee..d7a778e1f7d 100644
--- a/source4/ntvfs/posix/pvfs_acl.c
+++ b/source4/ntvfs/posix/pvfs_acl.c
@@ -89,7 +89,7 @@ const struct pvfs_acl_ops *pvfs_acl_backend_byname(const char *name)
 	return NULL;
 }
 
-NTSTATUS pvfs_acl_init(struct loadparm_context *lp_ctx)
+NTSTATUS pvfs_acl_init(void)
 {
 	static bool initialized = false;
 #define _MODULE_PROTO(init) extern NTSTATUS init(void);
@@ -100,7 +100,7 @@ NTSTATUS pvfs_acl_init(struct loadparm_context *lp_ctx)
 	if (initialized) return NT_STATUS_OK;
 	initialized = true;
 
-	shared_init = load_samba_modules(NULL, lp_ctx, "pvfs_acl");
+	shared_init = load_samba_modules(NULL, "pvfs_acl");
 
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
diff --git a/source4/ntvfs/posix/pvfs_dirlist.c b/source4/ntvfs/posix/pvfs_dirlist.c
index 77f19c35850..1bc91c1c788 100644
--- a/source4/ntvfs/posix/pvfs_dirlist.c
+++ b/source4/ntvfs/posix/pvfs_dirlist.c
@@ -199,7 +199,7 @@ const char *pvfs_list_next(struct pvfs_dir *dir, off_t *ofs)
 	if (*ofs == DIR_OFFSET_DOT) {
 		(*ofs) = DIR_OFFSET_DOTDOT;
 		dir->offset = *ofs;
-		if (ms_fnmatch(dir->pattern, ".", protocol) == 0) {
+		if (ms_fnmatch_protocol(dir->pattern, ".", protocol) == 0) {
 			dcache_add(dir, ".");
 			return ".";
 		}
@@ -208,7 +208,7 @@ const char *pvfs_list_next(struct pvfs_dir *dir, off_t *ofs)
 	if (*ofs == DIR_OFFSET_DOTDOT) {
 		(*ofs) = DIR_OFFSET_BASE;
 		dir->offset = *ofs;
-		if (ms_fnmatch(dir->pattern, "..", protocol) == 0) {
+		if (ms_fnmatch_protocol(dir->pattern, "..", protocol) == 0) {
 			dcache_add(dir, "..");
 			return "..";
 		}
@@ -228,10 +228,10 @@ const char *pvfs_list_next(struct pvfs_dir *dir, off_t *ofs)
 			continue;
 		}
 
-		if (ms_fnmatch(dir->pattern, dname, protocol) != 0) {
+		if (ms_fnmatch_protocol(dir->pattern, dname, protocol) != 0) {
 			char *short_name = pvfs_short_name_component(dir->pvfs, dname);
 			if (short_name == NULL ||
-			    ms_fnmatch(dir->pattern, short_name, protocol) != 0) {
+			    ms_fnmatch_protocol(dir->pattern, short_name, protocol) != 0) {
 				talloc_free(short_name);
 				continue;
 			}
diff --git a/source4/ntvfs/posix/pvfs_lock.c b/source4/ntvfs/posix/pvfs_lock.c
index 11757deea49..0d99860e59b 100644
--- a/source4/ntvfs/posix/pvfs_lock.c
+++ b/source4/ntvfs/posix/pvfs_lock.c
@@ -39,7 +39,7 @@ NTSTATUS pvfs_check_lock(struct pvfs_state *pvfs,
 		return NT_STATUS_OK;
 	}
 
-	return brl_locktest(pvfs->brl_context,
+	return brlock_locktest(pvfs->brl_context,
 			    f->brl_handle,
 			    smbpid,
 			    offset, count, rw);
@@ -70,7 +70,7 @@ static void pvfs_lock_async_failed(struct pvfs_state *pvfs,
 {
 	/* undo the locks we just did */
 	for (i--;i>=0;i--) {
-		brl_unlock(pvfs->brl_context,
+		brlock_unlock(pvfs->brl_context,
 			   f->brl_handle,
 			   locks[i].pid,
 			   locks[i].offset,
@@ -127,7 +127,7 @@ static void pvfs_pending_lock_continue(void *private_data, enum pvfs_wait_notice
 		 * because with this we'll get the correct error code
 		 * FILE_LOCK_CONFLICT in the error case
 		 */
-		status = brl_lock(pvfs->brl_context,
+		status = brlock_lock(pvfs->brl_context,
 				  f->brl_handle,
 				  locks[pending->pending_lock].pid,
 				  locks[pending->pending_lock].offset,
@@ -143,7 +143,7 @@ static void pvfs_pending_lock_continue(void *private_data, enum pvfs_wait_notice
 	   don't need the pending lock any more */
 	if (NT_STATUS_IS_OK(status) || timed_out) {
 		NTSTATUS status2;
-		status2 = brl_remove_pending(pvfs->brl_context, 
+		status2 = brlock_remove_pending(pvfs->brl_context, 
 					     f->brl_handle, pending);
 		if (!NT_STATUS_IS_OK(status2)) {
 			DEBUG(0,("pvfs_lock: failed to remove pending lock - %s\n", nt_errstr(status2)));
@@ -177,7 +177,7 @@ static void pvfs_pending_lock_continue(void *private_data, enum pvfs_wait_notice
 			pending->pending_lock = i;
 		}
 
-		status = brl_lock(pvfs->brl_context,
+		status = brlock_lock(pvfs->brl_context,
 				  f->brl_handle,
 				  locks[i].pid,
 				  locks[i].offset,
@@ -225,7 +225,7 @@ void pvfs_lock_close(struct pvfs_state *pvfs, struct pvfs_file *f)
 	if (f->lock_count || f->pending_list) {
 		DEBUG(5,("pvfs_lock: removing %.0f locks on close\n", 
 			 (double)f->lock_count));
-		brl_close(f->pvfs->brl_context, f->brl_handle);
+		brlock_close(f->pvfs->brl_context, f->brl_handle);
 		f->lock_count = 0;
 	}
 
@@ -324,8 +324,7 @@ NTSTATUS pvfs_lock(struct ntvfs_module_context *ntvfs,
 		pending->req = req;
 
 		pending->end_time = 
-			timeval_current_ofs(lck->lockx.in.timeout/1000,
-					    1000*(lck->lockx.in.timeout%1000));
+			timeval_current_ofs_msec(lck->lockx.in.timeout);
 	}
 
 	if (lck->lockx.in.mode & LOCKING_ANDX_SHARED_LOCK) {
@@ -350,7 +349,7 @@ NTSTATUS pvfs_lock(struct ntvfs_module_context *ntvfs,
 	locks = lck->lockx.in.locks;
 
 	for (i=0;i<lck->lockx.in.ulock_cnt;i++) {
-		status = brl_unlock(pvfs->brl_context,
+		status = brlock_unlock(pvfs->brl_context,
 				    f->brl_handle,
 				    locks[i].pid,
 				    locks[i].offset,
@@ -369,7 +368,7 @@ NTSTATUS pvfs_lock(struct ntvfs_module_context *ntvfs,
 			pending->pending_lock = i;
 		}
 
-		status = brl_lock(pvfs->brl_context,
+		status = brlock_lock(pvfs->brl_context,
 				  f->brl_handle,
 				  locks[i].pid,
 				  locks[i].offset,
@@ -394,7 +393,7 @@ NTSTATUS pvfs_lock(struct ntvfs_module_context *ntvfs,
 
 			/* undo the locks we just did */
 			for (i--;i>=0;i--) {
-				brl_unlock(pvfs->brl_context,
+				brlock_unlock(pvfs->brl_context,
 					   f->brl_handle,
 					   locks[i].pid,
 					   locks[i].offset,
diff --git a/source4/ntvfs/posix/pvfs_open.c b/source4/ntvfs/posix/pvfs_open.c
index 08a54f8e42b..d56bce58f75 100644
--- a/source4/ntvfs/posix/pvfs_open.c
+++ b/source4/ntvfs/posix/pvfs_open.c
@@ -590,7 +590,7 @@ static NTSTATUS pvfs_brl_locking_handle(TALLOC_CTX *mem_ctx,
 		data_blob_free(&odb_key);
 	}
 
-	h = brl_create_handle(mem_ctx, ntvfs, &key);
+	h = brlock_create_handle(mem_ctx, ntvfs, &key);
 	NT_STATUS_HAVE_NO_MEMORY(h);
 
 	*_h = h;
@@ -1174,7 +1174,7 @@ static NTSTATUS pvfs_open_setup_retry(struct ntvfs_module_context *ntvfs,
 		*final_timeout = timeval_add(&req->statistics.request_time,
 					     pvfs->oplock_break_timeout,
 					     0);
-		end_time = timeval_current_ofs(0, (pvfs->sharing_violation_delay*4)/5);
+		end_time = timeval_current_ofs_usec((pvfs->sharing_violation_delay*4)/5);
 		end_time = timeval_min(final_timeout, &end_time);
 	} else {
 		return NT_STATUS_INTERNAL_ERROR;
@@ -1569,7 +1569,7 @@ NTSTATUS pvfs_open(struct ntvfs_module_context *ntvfs,
 
 	f->handle->fd = fd;
 
-	status = brl_count(f->pvfs->brl_context, f->brl_handle, &count);
+	status = brlock_count(f->pvfs->brl_context, f->brl_handle, &count);
 	if (!NT_STATUS_IS_OK(status)) {
 		talloc_free(lck);
 		return status;
diff --git a/source4/ntvfs/posix/pvfs_oplock.c b/source4/ntvfs/posix/pvfs_oplock.c
index a10188f3eb5..5b9f3a318c8 100644
--- a/source4/ntvfs/posix/pvfs_oplock.c
+++ b/source4/ntvfs/posix/pvfs_oplock.c
@@ -32,7 +32,7 @@ struct pvfs_oplock {
 	uint32_t level;
 	struct timeval break_to_level_II;
 	struct timeval break_to_none;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 };
 
 static NTSTATUS pvfs_oplock_release_internal(struct pvfs_file_handle *h,
@@ -158,7 +158,7 @@ static void pvfs_oplock_break(struct pvfs_oplock *opl, uint8_t level)
 	}
 }
 
-static void pvfs_oplock_break_dispatch(struct messaging_context *msg,
+static void pvfs_oplock_break_dispatch(struct imessaging_context *msg,
 				       void *private_data, uint32_t msg_type,
 				       struct server_id src, DATA_BLOB *data)
 {
@@ -169,7 +169,7 @@ static void pvfs_oplock_break_dispatch(struct messaging_context *msg,
 	ZERO_STRUCT(opb);
 
 	/* we need to check that this one is for us. See
-	   messaging_send_ptr() for the other side of this.
+	   imessaging_send_ptr() for the other side of this.
 	 */
 	if (data->length == sizeof(struct opendb_oplock_break)) {
 		struct opendb_oplock_break *p;
@@ -192,7 +192,7 @@ static void pvfs_oplock_break_dispatch(struct messaging_context *msg,
 
 static int pvfs_oplock_destructor(struct pvfs_oplock *opl)
 {
-	messaging_deregister(opl->msg_ctx, MSG_NTVFS_OPLOCK_BREAK, opl);
+	imessaging_deregister(opl->msg_ctx, MSG_NTVFS_OPLOCK_BREAK, opl);
 	return 0;
 }
 
@@ -228,7 +228,7 @@ NTSTATUS pvfs_setup_oplock(struct pvfs_file *f, uint32_t oplock_granted)
 	opl->level	= level;
 	opl->msg_ctx	= f->pvfs->ntvfs->ctx->msg_ctx;
 
-	status = messaging_register(opl->msg_ctx,
+	status = imessaging_register(opl->msg_ctx,
 				    opl,
 				    MSG_NTVFS_OPLOCK_BREAK,
 				    pvfs_oplock_break_dispatch);
diff --git a/source4/ntvfs/posix/pvfs_qfileinfo.c b/source4/ntvfs/posix/pvfs_qfileinfo.c
index e54fc2d6690..92843067539 100644
--- a/source4/ntvfs/posix/pvfs_qfileinfo.c
+++ b/source4/ntvfs/posix/pvfs_qfileinfo.c
@@ -149,9 +149,6 @@ static NTSTATUS pvfs_map_fileinfo(struct pvfs_state *pvfs,
 				  int fd)
 {
 	switch (info->generic.level) {
-	case RAW_FILEINFO_GENERIC:
-		return NT_STATUS_INVALID_LEVEL;
-
 	case RAW_FILEINFO_GETATTR:
 		info->getattr.out.attrib     = name->dos.attrib;
 		info->getattr.out.size       = name->st.st_size;
@@ -333,6 +330,12 @@ static NTSTATUS pvfs_map_fileinfo(struct pvfs_state *pvfs,
 							      name->original_name);
 		NT_STATUS_HAVE_NO_MEMORY(info->all_info2.out.fname.s);
 		return NT_STATUS_OK;
+
+	case RAW_FILEINFO_GENERIC:
+	case RAW_FILEINFO_UNIX_BASIC:
+	case RAW_FILEINFO_UNIX_INFO2:
+	case RAW_FILEINFO_UNIX_LINK:
+		return NT_STATUS_INVALID_LEVEL;
 	}
 
 	return NT_STATUS_INVALID_LEVEL;
diff --git a/source4/ntvfs/posix/pvfs_search.c b/source4/ntvfs/posix/pvfs_search.c
index a050de1ec38..893f55c5aca 100644
--- a/source4/ntvfs/posix/pvfs_search.c
+++ b/source4/ntvfs/posix/pvfs_search.c
@@ -221,7 +221,9 @@ static NTSTATUS fill_search_info(struct pvfs_state *pvfs,
 		return NT_STATUS_OK;
 
 	case RAW_SEARCH_DATA_GENERIC:
-		break;
+	case RAW_SEARCH_DATA_UNIX_INFO:
+	case RAW_SEARCH_DATA_UNIX_INFO2:
+		return NT_STATUS_INVALID_LEVEL;
 	}
 
 	return NT_STATUS_INVALID_LEVEL;
diff --git a/source4/ntvfs/posix/pvfs_util.c b/source4/ntvfs/posix/pvfs_util.c
index 8dedf13a633..c6c6eaa13c8 100644
--- a/source4/ntvfs/posix/pvfs_util.c
+++ b/source4/ntvfs/posix/pvfs_util.c
@@ -40,7 +40,7 @@ bool pvfs_has_wildcard(const char *str)
 NTSTATUS pvfs_map_errno(struct pvfs_state *pvfs, int unix_errno)
 {
 	NTSTATUS status;
-	status = map_nt_error_from_unix(unix_errno);
+	status = map_nt_error_from_unix_common(unix_errno);
 	DEBUG(10,(__location__ " mapped unix errno %d -> %s\n", unix_errno, nt_errstr(status)));
 	return status;
 }
diff --git a/source4/ntvfs/posix/pvfs_wait.c b/source4/ntvfs/posix/pvfs_wait.c
index 876ce527978..013de69889c 100644
--- a/source4/ntvfs/posix/pvfs_wait.c
+++ b/source4/ntvfs/posix/pvfs_wait.c
@@ -33,7 +33,7 @@ struct pvfs_wait {
 	void (*handler)(void *, enum pvfs_wait_notice);
 	void *private_data;
 	int msg_type;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	struct tevent_context *ev;
 	struct ntvfs_request *req;
 	enum pvfs_wait_notice reason;
@@ -56,7 +56,7 @@ NTSTATUS pvfs_async_setup(struct ntvfs_module_context *ntvfs,
 /*
   receive a completion message for a wait
 */
-static void pvfs_wait_dispatch(struct messaging_context *msg,
+static void pvfs_wait_dispatch(struct imessaging_context *msg,
 			       void *private_data, uint32_t msg_type,
 			       struct server_id src, DATA_BLOB *data)
 {
@@ -66,7 +66,7 @@ static void pvfs_wait_dispatch(struct messaging_context *msg,
 	void *p = NULL;
 
 	/* we need to check that this one is for us. See
-	   messaging_send_ptr() for the other side of this.
+	   imessaging_send_ptr() for the other side of this.
 	 */
 	if (data->length == sizeof(void *)) {
 		void **pp;
@@ -116,7 +116,7 @@ static void pvfs_wait_timeout(struct tevent_context *ev,
 static int pvfs_wait_destructor(struct pvfs_wait *pwait)
 {
 	if (pwait->msg_type != -1) {
-		messaging_deregister(pwait->msg_ctx, pwait->msg_type, pwait);
+		imessaging_deregister(pwait->msg_ctx, pwait->msg_type, pwait);
 	}
 	DLIST_REMOVE(pwait->pvfs->wait_list, pwait);
 	return 0;
@@ -162,7 +162,7 @@ struct pvfs_wait *pvfs_wait_message(struct pvfs_state *pvfs,
 	/* register with the messaging subsystem for this message
 	   type */
 	if (msg_type != -1) {
-		messaging_register(pwait->msg_ctx,
+		imessaging_register(pwait->msg_ctx,
 				   pwait,
 				   msg_type,
 				   pvfs_wait_dispatch);
diff --git a/source4/ntvfs/posix/pvfs_write.c b/source4/ntvfs/posix/pvfs_write.c
index fb629a87fb3..892d3dd7495 100644
--- a/source4/ntvfs/posix/pvfs_write.c
+++ b/source4/ntvfs/posix/pvfs_write.c
@@ -64,7 +64,7 @@ static void pvfs_trigger_write_time_update(struct pvfs_file_handle *h)
 		return;
 	}
 
-	tv = timeval_current_ofs(0, pvfs->writetime_delay);
+	tv = timeval_current_ofs_usec(pvfs->writetime_delay);
 
 	h->write_time.update_triggered = true;
 	h->write_time.update_on_close = true;
diff --git a/source4/ntvfs/posix/python/pyxattr_tdb.c b/source4/ntvfs/posix/python/pyxattr_tdb.c
index 5e72ac9dde5..2b28aca365a 100644
--- a/source4/ntvfs/posix/python/pyxattr_tdb.c
+++ b/source4/ntvfs/posix/python/pyxattr_tdb.c
@@ -20,8 +20,8 @@
 
 #include <Python.h>
 #include "includes.h"
-#include <tdb.h>
-#include "tdb_wrap.h"
+#include "tdb_compat.h"
+#include "lib/util/tdb_wrap.h"
 #include "librpc/ndr/libndr.h"
 #include "lib/util/wrap_xattr.h"
 #include "ntvfs/posix/vfs_posix.h"
diff --git a/source4/ntvfs/posix/vfs_posix.c b/source4/ntvfs/posix/vfs_posix.c
index ca1d1633279..00ed146c965 100644
--- a/source4/ntvfs/posix/vfs_posix.c
+++ b/source4/ntvfs/posix/vfs_posix.c
@@ -26,8 +26,8 @@
 #include "includes.h"
 #include "vfs_posix.h"
 #include "librpc/gen_ndr/security.h"
-#include <tdb.h>
-#include "tdb_wrap.h"
+#include "tdb_compat.h"
+#include "lib/util/tdb_wrap.h"
 #include "libcli/security/security.h"
 #include "lib/events/events.h"
 #include "param/param.h"
@@ -212,7 +212,7 @@ static NTSTATUS pvfs_connect(struct ntvfs_module_context *ntvfs,
 	 * TODO: call this from ntvfs_posix_init()
 	 *       but currently we don't have a lp_ctx there
 	 */
-	status = pvfs_acl_init(ntvfs->ctx->lp_ctx);
+	status = pvfs_acl_init();
 	NT_STATUS_NOT_OK_RETURN(status);
 
 	pvfs = talloc_zero(ntvfs, struct pvfs_state);
@@ -249,7 +249,7 @@ static NTSTATUS pvfs_connect(struct ntvfs_module_context *ntvfs,
 
 	ntvfs->private_data = pvfs;
 
-	pvfs->brl_context = brl_init(pvfs, 
+	pvfs->brl_context = brlock_init(pvfs, 
 				     pvfs->ntvfs->ctx->server_id,
 				     pvfs->ntvfs->ctx->lp_ctx,
 				     pvfs->ntvfs->ctx->msg_ctx);
diff --git a/source4/ntvfs/posix/vfs_posix.h b/source4/ntvfs/posix/vfs_posix.h
index d60369df501..e1593a38cf5 100644
--- a/source4/ntvfs/posix/vfs_posix.h
+++ b/source4/ntvfs/posix/vfs_posix.h
@@ -204,7 +204,7 @@ struct pvfs_file {
 	/* a file handle to be used for byte range locking */
 	struct brl_handle *brl_handle;
 
-	/* a count of active locks - used to avoid calling brl_close on
+	/* a count of active locks - used to avoid calling brlock_close on
 	   file close */
 	uint64_t lock_count;
 
diff --git a/source4/ntvfs/posix/xattr_tdb.c b/source4/ntvfs/posix/xattr_tdb.c
index de7c83677a4..07b37122158 100644
--- a/source4/ntvfs/posix/xattr_tdb.c
+++ b/source4/ntvfs/posix/xattr_tdb.c
@@ -20,8 +20,8 @@
 */
 
 #include "includes.h"
-#include "tdb_wrap.h"
-#include <tdb.h>
+#include "lib/util/tdb_wrap.h"
+#include "tdb_compat.h"
 #include "vfs_posix.h"
 
 #define XATTR_LIST_ATTR ".xattr_list"
@@ -129,7 +129,7 @@ NTSTATUS pull_xattr_blob_tdb_raw(struct tdb_wrap *ea_tdb,
 		return status;
 	}
 
-	tdata = tdb_fetch(ea_tdb->tdb, tkey);
+	tdata = tdb_fetch_compat(ea_tdb->tdb, tkey);
 	if (tdata.dptr == NULL) {
 		return NT_STATUS_NOT_FOUND;
 	}
@@ -185,7 +185,7 @@ NTSTATUS push_xattr_blob_tdb_raw(struct tdb_wrap *ea_tdb,
 		goto done;
 	}
 
-	if (tdb_store(ea_tdb->tdb, tkey, tdata, TDB_REPLACE) == -1) {
+	if (tdb_store(ea_tdb->tdb, tkey, tdata, TDB_REPLACE) != 0) {
 		status = NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
 
@@ -218,7 +218,7 @@ NTSTATUS delete_xattr_tdb(struct pvfs_state *pvfs, const char *attr_name,
 		return status;
 	}
 	
-	if (tdb_delete(pvfs->ea_db->tdb, tkey) == -1) {
+	if (tdb_delete(pvfs->ea_db->tdb, tkey) != 0) {
 		talloc_free(tkey.dptr);
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
diff --git a/source4/ntvfs/simple/svfs_util.c b/source4/ntvfs/simple/svfs_util.c
index 2a01c2d5dec..c4e015668c7 100644
--- a/source4/ntvfs/simple/svfs_util.c
+++ b/source4/ntvfs/simple/svfs_util.c
@@ -38,16 +38,15 @@ char *svfs_unix_path(struct ntvfs_module_context *ntvfs,
 {
 	struct svfs_private *p = ntvfs->private_data;
 	char *ret;
+	char *name_lower = strlower_talloc(p, name);
 
 	if (*name != '\\') {
-		ret = talloc_asprintf(req, "%s/%s", p->connectpath, name);
+		ret = talloc_asprintf(req, "%s/%s", p->connectpath, name_lower);
 	} else {
-		ret = talloc_asprintf(req, "%s%s", p->connectpath, name);
+		ret = talloc_asprintf(req, "%s%s", p->connectpath, name_lower);
 	}
 	all_string_sub(ret, "\\", "/", 0);
-
-	strlower(ret + strlen(p->connectpath));
-
+	talloc_free(name_lower);
 	return ret;
 }
 
@@ -82,9 +81,8 @@ struct svfs_dir *svfs_list_unix(TALLOC_CTX *mem_ctx, struct ntvfs_request *req,
 	/* the wildcard pattern is the last part */
 	mask = p+1;
 
-	low_mask = talloc_strdup(mem_ctx, mask);
+	low_mask = strlower_talloc(mem_ctx, mask);
 	if (!low_mask) { return NULL; }
-	strlower(low_mask);
 
 	odir = opendir(dir->unix_dir);
 	if (!odir) { return NULL; }
@@ -99,12 +97,11 @@ struct svfs_dir *svfs_list_unix(TALLOC_CTX *mem_ctx, struct ntvfs_request *req,
 			continue;
 		}
 
-		low_name = talloc_strdup(mem_ctx, dent->d_name);
+		low_name = strlower_talloc(mem_ctx, dent->d_name);
 		if (!low_name) { continue; }
-		strlower(low_name);
 
 		/* check it matches the wildcard pattern */
-		if (ms_fnmatch(low_mask, low_name, PROTOCOL_NT1) != 0) {
+		if (ms_fnmatch_protocol(low_mask, low_name, PROTOCOL_NT1) != 0) {
 			continue;
 		}
 		
diff --git a/source4/ntvfs/simple/vfs_simple.c b/source4/ntvfs/simple/vfs_simple.c
index 03bf76634d1..7ae41db521b 100644
--- a/source4/ntvfs/simple/vfs_simple.c
+++ b/source4/ntvfs/simple/vfs_simple.c
@@ -147,7 +147,7 @@ static NTSTATUS svfs_unlink(struct ntvfs_module_context *ntvfs,
 
 	/* ignoring wildcards ... */
 	if (unlink(unix_path) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return NT_STATUS_OK;
@@ -176,7 +176,7 @@ static NTSTATUS svfs_chkpath(struct ntvfs_module_context *ntvfs,
 	unix_path = svfs_unix_path(ntvfs, req, cp->chkpath.in.path);
 
 	if (stat(unix_path, &st) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (!S_ISDIR(st.st_mode)) {
@@ -291,7 +291,7 @@ static NTSTATUS svfs_qpathinfo(struct ntvfs_module_context *ntvfs,
 	DEBUG(19,("svfs_qpathinfo: file %s\n", unix_path));
 	if (stat(unix_path, &st) == -1) {
 		DEBUG(19,("svfs_qpathinfo: file %s errno=%d\n", unix_path, errno));
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	DEBUG(19,("svfs_qpathinfo: file %s, stat done\n", unix_path));
 	return svfs_map_fileinfo(ntvfs, req, info, &st, unix_path);
@@ -317,7 +317,7 @@ static NTSTATUS svfs_qfileinfo(struct ntvfs_module_context *ntvfs,
 	}
 	
 	if (fstat(f->fd, &st) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return svfs_map_fileinfo(ntvfs, req,info, &st, f->name);
@@ -386,13 +386,13 @@ static NTSTATUS svfs_open(struct ntvfs_module_context *ntvfs,
 		case NTCREATEX_DISP_CREATE:
 			if (mkdir(unix_path, 0755) == -1) {
 				DEBUG(9,("svfs_open: mkdir %s errno=%d\n", unix_path, errno));
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 			break;
 		case NTCREATEX_DISP_OPEN_IF:
 			if (mkdir(unix_path, 0755) == -1 && errno != EEXIST) {
 				DEBUG(9,("svfs_open: mkdir %s errno=%d\n", unix_path, errno));
-				return map_nt_error_from_unix(errno);
+				return map_nt_error_from_unix_common(errno);
 			}
 			break;
 		}
@@ -401,13 +401,13 @@ static NTSTATUS svfs_open(struct ntvfs_module_context *ntvfs,
 do_open:
 	fd = open(unix_path, flags, 0644);
 	if (fd == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	if (fstat(fd, &st) == -1) {
 		DEBUG(9,("svfs_open: fstat errno=%d\n", errno));
 		close(fd);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	status = ntvfs_handle_new(ntvfs, req, &handle);
@@ -456,7 +456,7 @@ static NTSTATUS svfs_mkdir(struct ntvfs_module_context *ntvfs,
 	unix_path = svfs_unix_path(ntvfs, req, md->mkdir.in.path);
 
 	if (mkdir(unix_path, 0777) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return NT_STATUS_OK;
@@ -475,7 +475,7 @@ static NTSTATUS svfs_rmdir(struct ntvfs_module_context *ntvfs,
 	unix_path = svfs_unix_path(ntvfs, req, rd->in.path);
 
 	if (rmdir(unix_path) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	return NT_STATUS_OK;
@@ -499,7 +499,7 @@ static NTSTATUS svfs_rename(struct ntvfs_module_context *ntvfs,
 	unix_path2 = svfs_unix_path(ntvfs, req, ren->rename.in.pattern2);
 
 	if (rename(unix_path1, unix_path2) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	
 	return NT_STATUS_OK;
@@ -538,7 +538,7 @@ static NTSTATUS svfs_read(struct ntvfs_module_context *ntvfs,
 		    rd->readx.in.maxcnt,
 		    rd->readx.in.offset);
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	rd->readx.out.nread = ret;
@@ -574,7 +574,7 @@ static NTSTATUS svfs_write(struct ntvfs_module_context *ntvfs,
 		     wr->writex.in.count,
 		     wr->writex.in.offset);
 	if (ret == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 		
 	wr->writex.out.nwritten = ret;
@@ -645,7 +645,7 @@ static NTSTATUS svfs_close(struct ntvfs_module_context *ntvfs,
 	}
 
 	if (close(f->fd) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	DLIST_REMOVE(p->open_files, f);
@@ -735,7 +735,7 @@ static NTSTATUS svfs_setfileinfo(struct ntvfs_module_context *ntvfs,
 	case RAW_SFILEINFO_END_OF_FILE_INFORMATION:
 		if (ftruncate(f->fd, 
 			      info->end_of_file_info.in.size) == -1) {
-			return map_nt_error_from_unix(errno);
+			return map_nt_error_from_unix_common(errno);
 		}
 		break;
 	case RAW_SFILEINFO_SETATTRE:
@@ -781,7 +781,7 @@ static NTSTATUS svfs_fsinfo(struct ntvfs_module_context *ntvfs,
 	if (sys_fsusage(p->connectpath,
 			&fs->generic.out.blocks_free, 
 			&fs->generic.out.blocks_total) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	fs->generic.out.block_size = 512;
@@ -821,7 +821,7 @@ static NTSTATUS svfs_fsattr(struct ntvfs_module_context *ntvfs,
 	}
 
 	if (stat(p->connectpath, &st) == -1) {
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	unix_to_nt_time(&fs->generic.out.create_time, st.st_ctime);
diff --git a/source4/ntvfs/smb2/vfs_smb2.c b/source4/ntvfs/smb2/vfs_smb2.c
index 6c789976869..889c534dd0a 100644
--- a/source4/ntvfs/smb2/vfs_smb2.c
+++ b/source4/ntvfs/smb2/vfs_smb2.c
@@ -162,7 +162,6 @@ static NTSTATUS cvfs_connect(struct ntvfs_module_context *ntvfs,
 	NTSTATUS status;
 	struct cvfs_private *p;
 	const char *host, *user, *pass, *domain, *remote_share, *sharename;
-	struct composite_context *creq;
 	struct share_config *scfg = ntvfs->ctx->config;
 	struct smb2_tree *tree;
 	struct cli_credentials *credentials;
@@ -250,17 +249,15 @@ static NTSTATUS cvfs_connect(struct ntvfs_module_context *ntvfs,
 
 	lpcfg_smbcli_options(ntvfs->ctx->lp_ctx, &options);
 
-	creq = smb2_connect_send(p, host,
+	status = smb2_connect(p, host,
 			lpcfg_parm_string_list(p, ntvfs->ctx->lp_ctx, NULL, "smb2", "ports", NULL),
-				remote_share, 
-				 lpcfg_resolve_context(ntvfs->ctx->lp_ctx),
-				 credentials,
-				 ntvfs->ctx->event_ctx, &options,
-				 lpcfg_socket_options(ntvfs->ctx->lp_ctx),
-				 lpcfg_gensec_settings(p, ntvfs->ctx->lp_ctx)
-				 );
-
-	status = smb2_connect_recv(creq, p, &tree);
+			remote_share,
+			lpcfg_resolve_context(ntvfs->ctx->lp_ctx),
+			credentials,
+			&tree,
+			ntvfs->ctx->event_ctx, &options,
+			lpcfg_socket_options(ntvfs->ctx->lp_ctx),
+			lpcfg_gensec_settings(p, ntvfs->ctx->lp_ctx));
 	NT_STATUS_NOT_OK_RETURN(status);
 
 	status = smb2_get_roothandle(tree, &p->roothandle);
diff --git a/source4/ntvfs/sysdep/inotify.c b/source4/ntvfs/sysdep/inotify.c
index 42aac0b0974..28ea32e8a00 100644
--- a/source4/ntvfs/sysdep/inotify.c
+++ b/source4/ntvfs/sysdep/inotify.c
@@ -258,7 +258,7 @@ static NTSTATUS inotify_setup(struct sys_notify_context *ctx)
 	if (in->fd == -1) {
 		DEBUG(0,("Failed to init inotify - %s\n", strerror(errno)));
 		talloc_free(in);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 	in->ctx = ctx;
 	in->watches = NULL;
@@ -274,7 +274,7 @@ static NTSTATUS inotify_setup(struct sys_notify_context *ctx)
 		}
 		DEBUG(0,("Failed to tevent_add_fd() - %s\n", strerror(errno)));
 		talloc_free(in);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	tevent_fd_set_auto_close(fde);
@@ -373,7 +373,7 @@ static NTSTATUS inotify_watch(struct sys_notify_context *ctx,
 	wd = inotify_add_watch(in->fd, e->path, mask);
 	if (wd == -1) {
 		e->filter = filter;
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	w = talloc(in, struct inotify_watch_context);
diff --git a/source4/ntvfs/sysdep/sys_lease.c b/source4/ntvfs/sysdep/sys_lease.c
index 8b0b3a51107..7865f717a42 100644
--- a/source4/ntvfs/sysdep/sys_lease.c
+++ b/source4/ntvfs/sysdep/sys_lease.c
@@ -40,7 +40,7 @@ static uint32_t num_backends;
 _PUBLIC_ struct sys_lease_context *sys_lease_context_create(struct share_config *scfg,
 							    TALLOC_CTX *mem_ctx,
 							    struct tevent_context *ev,
-							    struct messaging_context *msg,
+							    struct imessaging_context *msg,
 							    sys_lease_send_break_fn break_send)
 {
 	struct sys_lease_context *ctx;
diff --git a/source4/ntvfs/sysdep/sys_lease.h b/source4/ntvfs/sysdep/sys_lease.h
index 422797b0b89..57a5e0a35fc 100644
--- a/source4/ntvfs/sysdep/sys_lease.h
+++ b/source4/ntvfs/sysdep/sys_lease.h
@@ -21,10 +21,10 @@
 
 struct sys_lease_context;
 struct opendb_entry;
-struct messaging_context;
+struct imessaging_context;
 struct tevent_context;
 
-typedef NTSTATUS (*sys_lease_send_break_fn)(struct messaging_context *,
+typedef NTSTATUS (*sys_lease_send_break_fn)(struct imessaging_context *,
 					    struct opendb_entry *,
 					    uint8_t level);
 
@@ -41,7 +41,7 @@ struct sys_lease_ops {
 
 struct sys_lease_context {
 	struct tevent_context *event_ctx;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	sys_lease_send_break_fn break_send;
 	void *private_data; /* for use of backend */
 	const struct sys_lease_ops *ops;
@@ -53,7 +53,7 @@ NTSTATUS sys_lease_init(void);
 struct sys_lease_context *sys_lease_context_create(struct share_config *scfg,
 						   TALLOC_CTX *mem_ctx,
 						   struct tevent_context *ev,
-						   struct messaging_context *msg_ctx,
+						   struct imessaging_context *msg_ctx,
 						   sys_lease_send_break_fn break_send);
 
 NTSTATUS sys_lease_setup(struct sys_lease_context *ctx,
diff --git a/source4/ntvfs/sysdep/sys_lease_linux.c b/source4/ntvfs/sysdep/sys_lease_linux.c
index c0fb4335d96..3b0605b73b9 100644
--- a/source4/ntvfs/sysdep/sys_lease_linux.c
+++ b/source4/ntvfs/sysdep/sys_lease_linux.c
@@ -131,13 +131,13 @@ static NTSTATUS linux_lease_setup(struct sys_lease_context *ctx,
 	ret = fcntl(*fd, F_SETSIG, LINUX_LEASE_RT_SIGNAL);
 	if (ret == -1) {
 		talloc_free(p);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	ret = fcntl(*fd, F_SETLEASE, F_WRLCK);
 	if (ret == -1) {
 		talloc_free(p);
-		return map_nt_error_from_unix(errno);
+		return map_nt_error_from_unix_common(errno);
 	}
 
 	DLIST_ADD(leases, p);
diff --git a/source4/ntvfs/sysdep/sys_notify.h b/source4/ntvfs/sysdep/sys_notify.h
index c474d4e6a02..d912a9bdaf6 100644
--- a/source4/ntvfs/sysdep/sys_notify.h
+++ b/source4/ntvfs/sysdep/sys_notify.h
@@ -17,7 +17,7 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "librpc/gen_ndr/s4_notify.h"
+#include "librpc/gen_ndr/notify.h"
 #include "param/share.h"
 
 struct sys_notify_context;
diff --git a/source4/param/loadparm.c b/source4/param/loadparm.c
index 95460c62356..14285be0171 100644
--- a/source4/param/loadparm.c
+++ b/source4/param/loadparm.c
@@ -68,172 +68,39 @@
 #include "rpc_server/common/common.h"
 #include "lib/socket/socket.h"
 #include "auth/gensec/gensec.h"
+#include "s3_param.h"
 
 #define standard_sub_basic talloc_strdup
 
 static bool do_parameter(const char *, const char *, void *);
 static bool defaults_saved = false;
 
-/**
- * This structure describes global (ie., server-wide) parameters.
- */
-struct loadparm_global
-{
-	enum server_role server_role;
-	enum sid_generator sid_generator;
-
-	const char **smb_ports;
-	char *ncalrpc_dir;
-	char *dos_charset;
-	char *unix_charset;
-	char *display_charset;
-	char *szLockDir;
-	char *szModulesDir;
-	char *szPidDir;
-	char *szServerString;
-	char *szAutoServices;
-	char *szPasswdChat;
-	char *szShareBackend;
-	char *szSAM_URL;
-	char *szIDMAP_URL;
-	char *szSECRETS_URL;
-	char *szSPOOLSS_URL;
-	char *szWINS_CONFIG_URL;
-	char *szWINS_URL;
-	char *szPrivateDir;
-	const char **szPasswordServers;
-	char *szSocketOptions;
-	char *szRealm;
-	char *szRealm_upper;
-	char *szRealm_lower;
-	const char **szWINSservers;
-	const char **szInterfaces;
-	char *szSocketAddress;
-	char *szAnnounceVersion;	/* This is initialised in init_globals */
-	char *szWorkgroup;
-	char *szNetbiosName;
-	const char **szNetbiosAliases;
-	char *szNetbiosScope;
-	char *szDomainOtherSIDs;
-	const char **szNameResolveOrder;
-	const char **dcerpc_ep_servers;
-	const char **server_services;
-	char *ntptr_providor;
-	char *szWinbindSeparator;
-	char *szWinbinddPrivilegedSocketDirectory;
-	char *szWinbinddSocketDirectory;
-	char *szTemplateShell;
-	char *szTemplateHomedir;
-	int bWinbindSealedPipes;
-	int bIdmapTrustedOnly;
-	int tls_enabled;
-	char *tls_keyfile;
-	char *tls_certfile;
-	char *tls_cafile;
-	char *tls_crlfile;
-	char *tls_dhpfile;
-	char *logfile;
-	char *loglevel;
-	char *panic_action;
-	int max_mux;
-	int debuglevel;
-	int max_xmit;
-	int pwordlevel;
-	int srv_maxprotocol;
-	int srv_minprotocol;
-	int cli_maxprotocol;
-	int cli_minprotocol;
-	int security;
-	int paranoid_server_security;
-	int max_wins_ttl;
-	int min_wins_ttl;
-	int announce_as;	/* This is initialised in init_globals */
-	int nbt_port;
-	int dgram_port;
-	int cldap_port;
-	int krb5_port;
-	int kpasswd_port;
-	int web_port;
-	char *socket_options;
-	int bWINSsupport;
-	int bWINSdnsProxy;
-	char *szWINSHook;
-	int bLocalMaster;
-	int bPreferredMaster;
-	int bEncryptPasswords;
-	int bNullPasswords;
-	int bObeyPamRestrictions;
-	int bLargeReadwrite;
-	int bReadRaw;
-	int bWriteRaw;
-	int bTimeServer;
-	int bBindInterfacesOnly;
-	int bNTSmbSupport;
-	int bNTStatusSupport;
-	int bLanmanAuth;
-	int bNTLMAuth;
-	int bUseSpnego;
-	int server_signing;
-	int client_signing;
-	int bClientPlaintextAuth;
-	int bClientLanManAuth;
-	int bClientNTLMv2Auth;
-	int client_use_spnego_principal;
-	int bHostMSDfs;
-	int bUnicode;
-	int bUnixExtensions;
-	int bDisableNetbios;
-	int bRpcBigEndian;
-	char *szNTPSignDSocketDirectory;
-	const char **szRNDCCommand;
-	const char **szDNSUpdateCommand;
-	const char **szSPNUpdateCommand;
-	const char **szNSUpdateCommand;
-	struct parmlist_entry *param_opt;
-};
-
-
-/**
- * This structure describes a single service.
- */
-struct loadparm_service
-{
-	char *szService;
-	char *szPath;
-	char *szCopy;
-	char *szInclude;
-	char *szPrintername;
-	char **szHostsallow;
-	char **szHostsdeny;
-	char *comment;
-	char *volume;
-	char *fstype;
-	char **ntvfs_handler;
-	int iMaxPrintJobs;
-	int iMaxConnections;
-	int iCSCPolicy;
-	int bAvailable;
-	int bBrowseable;
-	int bRead_only;
-	int bPrint_ok;
-	int bMap_system;
-	int bMap_hidden;
-	int bMap_archive;
-	int bStrictLocking;
-	int bOplocks;
-	int iCreate_mask;
-	int iCreate_force_mode;
-	int iDir_mask;
-	int iDir_force_mode;
-	int *copymap;
-	int bMSDfsRoot;
-	int bStrictSync;
-	int bCIFileSystem;
-	struct parmlist_entry *param_opt;
-
+#define LOADPARM_EXTRA_GLOBALS \
+	struct parmlist_entry *param_opt;				\
+	char *szRealm;							\
+	char *tls_keyfile;						\
+	char *tls_certfile;						\
+	char *tls_cafile;						\
+	char *tls_crlfile;						\
+	char *tls_dhpfile;						\
+	char *loglevel;							\
+	char *panic_action;						\
+	int bPreferredMaster;						\
+	char *szAnnounceVersion;        /* This is initialised in init_globals */
+#define LOADPARM_EXTRA_LOCALS				\
+	struct parmlist_entry *param_opt;		\
+	char *szService;				\
+	char *szCopy;					\
+	char *szInclude;				\
+	char *szPrintername;				\
+	int bAvailable;					\
+	int iMaxPrintJobs;				\
+	char *volume;					\
+	int *copymap;					\
 	char dummy[3];		/* for alignment */
-};
 
+#include "param_global.h"
+#include "param_local.h"
 
 #define NUMPARAMETERS (sizeof(parm_table) / sizeof(struct parm_struct))
 
@@ -264,15 +131,7 @@ static const struct enum_list enum_protocol[] = {
 static const struct enum_list enum_security[] = {
 	{SEC_SHARE, "SHARE"},
 	{SEC_USER, "USER"},
-	{-1, NULL}
-};
-
-static const struct enum_list enum_announce_as[] = {
-	{ANNOUNCE_AS_NT_SERVER, "NT"},
-	{ANNOUNCE_AS_NT_SERVER, "NT Server"},
-	{ANNOUNCE_AS_NT_WORKSTATION, "NT Workstation"},
-	{ANNOUNCE_AS_WIN95, "win95"},
-	{ANNOUNCE_AS_WFW, "WfW"},
+	{SEC_ADS, "ADS"},
 	{-1, NULL}
 };
 
@@ -333,23 +192,14 @@ static const struct enum_list enum_server_role[] = {
 	{-1, NULL}
 };
 
-static const struct enum_list enum_sid_generator[] = {
-	{SID_GENERATOR_INTERNAL, "internal"},
-	{SID_GENERATOR_BACKEND, "backend"},
-	{-1, NULL}
-};
-
 #define GLOBAL_VAR(name) offsetof(struct loadparm_global, name)
 #define LOCAL_VAR(name) offsetof(struct loadparm_service, name)
 
 static struct parm_struct parm_table[] = {
 	{"server role", P_ENUM, P_GLOBAL, GLOBAL_VAR(server_role), NULL, enum_server_role},
-	{"sid generator", P_ENUM, P_GLOBAL, GLOBAL_VAR(sid_generator), NULL, enum_sid_generator},
-
 	{"dos charset", P_STRING, P_GLOBAL, GLOBAL_VAR(dos_charset), NULL, NULL},
 	{"unix charset", P_STRING, P_GLOBAL, GLOBAL_VAR(unix_charset), NULL, NULL},
 	{"ncalrpc dir", P_STRING, P_GLOBAL, GLOBAL_VAR(ncalrpc_dir), NULL, NULL},
-	{"display charset", P_STRING, P_GLOBAL, GLOBAL_VAR(display_charset), NULL, NULL},
 	{"comment", P_STRING, P_LOCAL, LOCAL_VAR(comment), NULL, NULL},
 	{"path", P_STRING, P_LOCAL, LOCAL_VAR(szPath), NULL, NULL},
 	{"directory", P_STRING, P_LOCAL, LOCAL_VAR(szPath), NULL, NULL},
@@ -371,9 +221,6 @@ static struct parm_struct parm_table[] = {
 	{"null passwords", P_BOOL, P_GLOBAL, GLOBAL_VAR(bNullPasswords), NULL, NULL},
 	{"obey pam restrictions", P_BOOL, P_GLOBAL, GLOBAL_VAR(bObeyPamRestrictions), NULL, NULL},
 	{"password server", P_LIST, P_GLOBAL, GLOBAL_VAR(szPasswordServers), NULL, NULL},
-	{"sam database", P_STRING, P_GLOBAL, GLOBAL_VAR(szSAM_URL), NULL, NULL},
-	{"idmap database", P_STRING, P_GLOBAL, GLOBAL_VAR(szIDMAP_URL), NULL, NULL},
-	{"secrets database", P_STRING, P_GLOBAL, GLOBAL_VAR(szSECRETS_URL), NULL, NULL},
 	{"spoolss database", P_STRING, P_GLOBAL, GLOBAL_VAR(szSPOOLSS_URL), NULL, NULL},
 	{"wins config database", P_STRING, P_GLOBAL, GLOBAL_VAR(szWINS_CONFIG_URL), NULL, NULL},
 	{"wins database", P_STRING, P_GLOBAL, GLOBAL_VAR(szWINS_URL), NULL, NULL},
@@ -427,7 +274,6 @@ static struct parm_struct parm_table[] = {
 	{"nt status support", P_BOOL, P_GLOBAL, GLOBAL_VAR(bNTStatusSupport), NULL, NULL},
 
 	{"announce version", P_STRING, P_GLOBAL, GLOBAL_VAR(szAnnounceVersion), NULL, NULL},
-	{"announce as", P_ENUM, P_GLOBAL, GLOBAL_VAR(announce_as), NULL, enum_announce_as},
 	{"max mux", P_INTEGER, P_GLOBAL, GLOBAL_VAR(max_mux), NULL, NULL},
 	{"max xmit", P_BYTES, P_GLOBAL, GLOBAL_VAR(max_xmit), NULL, NULL},
 
@@ -480,7 +326,6 @@ static struct parm_struct parm_table[] = {
 	{"auto services", P_STRING, P_GLOBAL, GLOBAL_VAR(szAutoServices), NULL, NULL},
 	{"lock dir", P_STRING, P_GLOBAL, GLOBAL_VAR(szLockDir), NULL, NULL}, 
 	{"lock directory", P_STRING, P_GLOBAL, GLOBAL_VAR(szLockDir), NULL, NULL},
-	{"modules dir", P_STRING, P_GLOBAL, GLOBAL_VAR(szModulesDir), NULL, NULL},
 	{"pid directory", P_STRING, P_GLOBAL, GLOBAL_VAR(szPidDir), NULL, NULL}, 
 
 	{"socket address", P_STRING, P_GLOBAL, GLOBAL_VAR(szSocketAddress), NULL, NULL},
@@ -532,6 +377,9 @@ struct loadparm_context {
 	unsigned int flags[NUMPARAMETERS];
 	bool loaded;
 	bool refuse_free;
+	bool global; /* Is this the global context, which may set
+		      * global variables such as debug level etc? */
+	const struct loadparm_s3_context *s3_fns;
 };
 
 
@@ -613,35 +461,80 @@ static struct loadparm_context *global_loadparm_context;
 #define lpcfg_default_service global_loadparm_context->sDefault
 #define lpcfg_global_service(i) global_loadparm_context->services[i]
 
-#define FN_GLOBAL_STRING(fn_name,var_name) \
- _PUBLIC_ const char *lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) {if (lp_ctx == NULL) return NULL; return lp_ctx->globals->var_name ? lp_string(lp_ctx->globals->var_name) : "";}
+#define FN_GLOBAL_STRING(fn_name,var_name)				\
+ _PUBLIC_ const char *lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) { \
+	if (lp_ctx == NULL) return NULL;				\
+	if (lp_ctx->s3_fns) {						\
+		SMB_ASSERT(lp_ctx->s3_fns->fn_name);			\
+		return lp_ctx->s3_fns->fn_name();			\
+	}								\
+	return lp_ctx->globals->var_name ? lp_string(lp_ctx->globals->var_name) : ""; \
+}
 
 #define FN_GLOBAL_CONST_STRING(fn_name,var_name) \
- _PUBLIC_ const char *lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) {if (lp_ctx == NULL) return NULL; return lp_ctx->globals->var_name ? lp_ctx->globals->var_name : "";}
-
-#define FN_GLOBAL_LIST(fn_name,var_name) \
- _PUBLIC_ const char **lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) {if (lp_ctx == NULL) return NULL; return lp_ctx->globals->var_name;}
+ _PUBLIC_ const char *lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) {\
+	 if (lp_ctx == NULL) return NULL;				\
+	 if (lp_ctx->s3_fns) {						\
+		 SMB_ASSERT(lp_ctx->s3_fns->fn_name);			\
+		 return lp_ctx->s3_fns->fn_name();			\
+	 }								\
+	 return lp_ctx->globals->var_name ? lp_string(lp_ctx->globals->var_name) : ""; \
+ }
+
+#define FN_GLOBAL_LIST(fn_name,var_name)				\
+ _PUBLIC_ const char **lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) { \
+	 if (lp_ctx == NULL) return NULL;				\
+	 if (lp_ctx->s3_fns) {						\
+		 SMB_ASSERT(lp_ctx->s3_fns->fn_name);			\
+		 return lp_ctx->s3_fns->fn_name();			\
+	 }								\
+	 return lp_ctx->globals->var_name;				\
+ }
 
 #define FN_GLOBAL_BOOL(fn_name,var_name) \
- _PUBLIC_ bool lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) {if (lp_ctx == NULL) return false; return lp_ctx->globals->var_name;}
+ _PUBLIC_ bool lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) {\
+	 if (lp_ctx == NULL) return false;				\
+	 if (lp_ctx->s3_fns) {						\
+		 SMB_ASSERT(lp_ctx->s3_fns->fn_name);			\
+		 return lp_ctx->s3_fns->fn_name();			\
+	 }								\
+	 return lp_ctx->globals->var_name;				\
+}
 
 #define FN_GLOBAL_INTEGER(fn_name,var_name) \
- _PUBLIC_ int lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) {return lp_ctx->globals->var_name;}
+ _PUBLIC_ int lpcfg_ ## fn_name(struct loadparm_context *lp_ctx) { \
+	 if (lp_ctx->s3_fns) {						\
+		 SMB_ASSERT(lp_ctx->s3_fns->fn_name);			\
+		 return lp_ctx->s3_fns->fn_name();			\
+	 }								\
+	 return lp_ctx->globals->var_name;				\
+ }
 
 #define FN_LOCAL_STRING(fn_name,val) \
- _PUBLIC_ const char *lpcfg_ ## fn_name(struct loadparm_service *service, struct loadparm_service *sDefault) {return(lp_string((const char *)((service != NULL && service->val != NULL) ? service->val : sDefault->val)));}
+ _PUBLIC_ const char *lpcfg_ ## fn_name(struct loadparm_service *service, \
+					struct loadparm_service *sDefault) { \
+	 return(lp_string((const char *)((service != NULL && service->val != NULL) ? service->val : sDefault->val))); \
+ }
 
 #define FN_LOCAL_LIST(fn_name,val) \
- _PUBLIC_ const char **lpcfg_ ## fn_name(struct loadparm_service *service, struct loadparm_service *sDefault) {return(const char **)(service != NULL && service->val != NULL? service->val : sDefault->val);}
+ _PUBLIC_ const char **lpcfg_ ## fn_name(struct loadparm_service *service, \
+					 struct loadparm_service *sDefault) {\
+	 return(const char **)(service != NULL && service->val != NULL? service->val : sDefault->val); \
+ }
 
 #define FN_LOCAL_BOOL(fn_name,val) \
- _PUBLIC_ bool lpcfg_ ## fn_name(struct loadparm_service *service, struct loadparm_service *sDefault) {return((service != NULL)? service->val : sDefault->val);}
+ _PUBLIC_ bool lpcfg_ ## fn_name(struct loadparm_service *service, \
+				 struct loadparm_service *sDefault) {	\
+	 return((service != NULL)? service->val : sDefault->val); \
+ }
 
 #define FN_LOCAL_INTEGER(fn_name,val) \
- _PUBLIC_ int lpcfg_ ## fn_name(struct loadparm_service *service, struct loadparm_service *sDefault) {return((service != NULL)? service->val : sDefault->val);}
+ _PUBLIC_ int lpcfg_ ## fn_name(struct loadparm_service *service, \
+				struct loadparm_service *sDefault) {	\
+	 return((service != NULL)? service->val : sDefault->val); \
+ }
 
 FN_GLOBAL_INTEGER(server_role, server_role)
-FN_GLOBAL_INTEGER(sid_generator, sid_generator)
 FN_GLOBAL_LIST(smb_ports, smb_ports)
 FN_GLOBAL_INTEGER(nbt_port, nbt_port)
 FN_GLOBAL_INTEGER(dgram_port, dgram_port)
@@ -652,9 +545,6 @@ FN_GLOBAL_INTEGER(web_port, web_port)
 FN_GLOBAL_BOOL(tls_enabled, tls_enabled)
 FN_GLOBAL_STRING(logfile, logfile)
 FN_GLOBAL_STRING(share_backend, szShareBackend)
-FN_GLOBAL_STRING(sam_url, szSAM_URL)
-FN_GLOBAL_STRING(idmap_url, szIDMAP_URL)
-FN_GLOBAL_STRING(secrets_url, szSECRETS_URL)
 FN_GLOBAL_STRING(spoolss_url, szSPOOLSS_URL)
 FN_GLOBAL_STRING(wins_config_url, szWINS_CONFIG_URL)
 FN_GLOBAL_STRING(wins_url, szWINS_URL)
@@ -668,11 +558,9 @@ FN_GLOBAL_BOOL(idmap_trusted_only, bIdmapTrustedOnly)
 FN_GLOBAL_STRING(private_dir, szPrivateDir)
 FN_GLOBAL_STRING(serverstring, szServerString)
 FN_GLOBAL_STRING(lockdir, szLockDir)
-FN_GLOBAL_STRING(modulesdir, szModulesDir)
 FN_GLOBAL_STRING(ncalrpc_dir, ncalrpc_dir)
 FN_GLOBAL_STRING(dos_charset, dos_charset)
 FN_GLOBAL_STRING(unix_charset, unix_charset)
-FN_GLOBAL_STRING(display_charset, display_charset)
 FN_GLOBAL_STRING(piddir, szPidDir)
 FN_GLOBAL_LIST(rndc_command, szRNDCCommand)
 FN_GLOBAL_LIST(dns_update_command, szDNSUpdateCommand)
@@ -731,7 +619,6 @@ FN_GLOBAL_INTEGER(cli_maxprotocol, cli_maxprotocol)
 FN_GLOBAL_INTEGER(cli_minprotocol, cli_minprotocol)
 FN_GLOBAL_INTEGER(security, security)
 FN_GLOBAL_BOOL(paranoid_server_security, paranoid_server_security)
-FN_GLOBAL_INTEGER(announce_as, announce_as)
 
 FN_LOCAL_STRING(pathname, szPath)
 FN_LOCAL_LIST(hostsallow, szHostsallow)
@@ -779,21 +666,28 @@ const char *lpcfg_get_parametric(struct loadparm_context *lp_ctx,
 			      struct loadparm_service *service,
 			      const char *type, const char *option)
 {
+	char *vfskey_tmp = NULL;
 	char *vfskey = NULL;
 	struct parmlist_entry *data;
 
 	if (lp_ctx == NULL)
 		return NULL;
 
+	if (lp_ctx->s3_fns) {
+		SMB_ASSERT(service == NULL);
+		return lp_ctx->s3_fns->get_parametric(type, option);
+	}
+
 	data = (service == NULL ? lp_ctx->globals->param_opt : service->param_opt);
 
-	asprintf(&vfskey, "%s:%s", type, option);
-	if (vfskey == NULL) return NULL;
-	strlower(vfskey);
+	vfskey_tmp = talloc_asprintf(NULL, "%s:%s", type, option);
+	if (vfskey_tmp == NULL) return NULL;
+	vfskey = strlower_talloc(NULL, vfskey_tmp);
+	talloc_free(vfskey_tmp);
 
 	while (data) {
 		if (strcmp(data->key, vfskey) == 0) {
-			free(vfskey);
+			talloc_free(vfskey);
 			return data->value;
 		}
 		data = data->next;
@@ -805,13 +699,13 @@ const char *lpcfg_get_parametric(struct loadparm_context *lp_ctx,
 		for (data = lp_ctx->globals->param_opt; data;
 		     data = data->next) {
 			if (strcmp(data->key, vfskey) == 0) {
-				free(vfskey);
+				talloc_free(vfskey);
 				return data->value;
 			}
 		}
 	}
 
-	free(vfskey);
+	talloc_free(vfskey);
 
 	return NULL;
 }
@@ -949,7 +843,7 @@ int lpcfg_parm_bytes(struct loadparm_context *lp_ctx,
 
 	const char *value = lpcfg_get_parametric(lp_ctx, service, type, option);
 
-	if (value && conv_str_size(value, &bval)) {
+	if (value && conv_str_size_error(value, &bval)) {
 		if (bval <= INT_MAX) {
 			return (int)bval;
 		}
@@ -1031,7 +925,27 @@ static bool string_set(TALLOC_CTX *mem_ctx, char **dest, const char *src)
 
 	*dest = talloc_strdup(mem_ctx, src);
 	if ((*dest) == NULL) {
-		DEBUG(0,("Out of memory in string_init\n"));
+		DEBUG(0,("Out of memory in string_set\n"));
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * Set a string value, deallocating any existing space, and allocing the space
+ * for the string
+ */
+static bool string_set_upper(TALLOC_CTX *mem_ctx, char **dest, const char *src)
+{
+	talloc_free(*dest);
+
+	if (src == NULL)
+		src = "";
+
+	*dest = strupper_talloc(mem_ctx, src);
+	if ((*dest) == NULL) {
+		DEBUG(0,("Out of memory in string_set_upper\n"));
 		return false;
 	}
 
@@ -1151,9 +1065,9 @@ bool lpcfg_add_home(struct loadparm_context *lp_ctx,
  * Add a new printer service, with defaults coming from service iFrom.
  */
 
-bool lp_add_printer(struct loadparm_context *lp_ctx,
-		    const char *pszPrintername,
-		    struct loadparm_service *default_service)
+bool lpcfg_add_printer(struct loadparm_context *lp_ctx,
+		       const char *pszPrintername,
+		       struct loadparm_service *default_service)
 {
 	const char *comment = "From Printcap";
 	struct loadparm_service *service;
@@ -1292,10 +1206,9 @@ static void copy_service(struct loadparm_service *pserviceDest,
 					break;
 
 				case P_USTRING:
-					string_set(pserviceDest,
-						   (char **)dest_ptr,
-						   *(char **)src_ptr);
-					strupper(*(char **)dest_ptr);
+					string_set_upper(pserviceDest,
+							 (char **)dest_ptr,
+							 *(char **)src_ptr);
 					break;
 				case P_LIST:
 					*(const char ***)dest_ptr = (const char **)str_list_copy(pserviceDest, 
@@ -1422,7 +1335,7 @@ static void add_to_file_list(struct loadparm_context *lp_ctx,
 /*******************************************************************
  Check if a config file has changed date.
 ********************************************************************/
-bool lp_file_list_changed(struct loadparm_context *lp_ctx)
+bool lpcfg_file_list_changed(struct loadparm_context *lp_ctx)
 {
 	struct file_lists *f;
 	DEBUG(6, ("lp_file_list_changed()\n"));
@@ -1528,14 +1441,19 @@ static bool handle_debuglevel(struct loadparm_context *lp_ctx,
 {
 
 	string_set(lp_ctx, ptr, pszParmValue);
-	return debug_parse_levels(pszParmValue);
+	if (lp_ctx->global) {
+		return debug_parse_levels(pszParmValue);
+	}
+	return true;
 }
 
 static bool handle_logfile(struct loadparm_context *lp_ctx,
 			const char *pszParmValue, char **ptr)
 {
 	debug_set_logfile(pszParmValue);
-	string_set(lp_ctx, ptr, pszParmValue);
+	if (lp_ctx->global) {
+		string_set(lp_ctx, ptr, pszParmValue);
+	}
 	return true;
 }
 
@@ -1574,11 +1492,9 @@ static bool lp_do_parameter_parametric(struct loadparm_context *lp_ctx,
 		pszParmName++;
 	}
 
-	name = strdup(pszParmName);
+	name = strlower_talloc(lp_ctx, pszParmName);
 	if (!name) return false;
 
-	strlower(name);
-
 	if (service == NULL) {
 		data = lp_ctx->globals->param_opt;
 		mem_ctx = lp_ctx->globals;
@@ -1594,13 +1510,14 @@ static bool lp_do_parameter_parametric(struct loadparm_context *lp_ctx,
 		if (strcmp(paramo->key, name) == 0) {
 			if ((paramo->priority & FLAG_CMDLINE) &&
 			    !(flags & FLAG_CMDLINE)) {
+				talloc_free(name);
 				return true;
 			}
 
 			talloc_free(paramo->value);
 			paramo->value = talloc_strdup(paramo, pszParmValue);
 			paramo->priority = flags;
-			free(name);
+			talloc_free(name);
 			return true;
 		}
 	}
@@ -1617,7 +1534,7 @@ static bool lp_do_parameter_parametric(struct loadparm_context *lp_ctx,
 		DLIST_ADD(service->param_opt, paramo);
 	}
 
-	free(name);
+	talloc_free(name);
 
 	return true;
 }
@@ -1662,7 +1579,7 @@ static bool set_variable(TALLOC_CTX *mem_ctx, int parmnum, void *parm_ptr,
 		case P_BYTES:
 		{
 			uint64_t val;
-			if (conv_str_size(pszParmValue, &val)) {
+			if (conv_str_size_error(pszParmValue, &val)) {
 				if (val <= INT_MAX) {
 					*(int *)parm_ptr = (int)val;
 					break;
@@ -1683,17 +1600,23 @@ static bool set_variable(TALLOC_CTX *mem_ctx, int parmnum, void *parm_ptr,
 			char **new_list = str_list_make(mem_ctx,
 							pszParmValue, NULL);
 			for (i=0; new_list[i]; i++) {
-				if (new_list[i][0] == '+' && new_list[i][1]) {
+				if (new_list[i][0] == '+' && new_list[i][1] &&
+				    (!str_list_check(*(const char ***)parm_ptr,
+						     &new_list[i][1]))) {
 					*(const char ***)parm_ptr = str_list_add(*(const char ***)parm_ptr,
 										 &new_list[i][1]);
 				} else if (new_list[i][0] == '-' && new_list[i][1]) {
+#if 0 /* This is commented out because we sometimes parse the list
+       * twice, and so we can't assert on this */
 					if (!str_list_check(*(const char ***)parm_ptr,
 							    &new_list[i][1])) {
-						DEBUG(0, ("Unsupported value for: %s = %s, %s is not in the original list\n",
-							  pszParmName, pszParmValue, new_list[i]));
+						DEBUG(0, ("Unsupported value for: %s = %s, %s is not in the original list [%s]\n",
+							  pszParmName, pszParmValue, new_list[i],
+							  str_list_join_shell(mem_ctx, *(const char ***)parm_ptr, ' ')));
 						return false;
 
 					}
+#endif
 					str_list_remove(*(const char ***)parm_ptr,
 							&new_list[i][1]);
 				} else {
@@ -1713,8 +1636,7 @@ static bool set_variable(TALLOC_CTX *mem_ctx, int parmnum, void *parm_ptr,
 			break;
 
 		case P_USTRING:
-			string_set(mem_ctx, (char **)parm_ptr, pszParmValue);
-			strupper(*(char **)parm_ptr);
+			string_set_upper(mem_ctx, (char **)parm_ptr, pszParmValue);
 			break;
 
 		case P_ENUM:
@@ -2275,7 +2197,7 @@ static void lpcfg_add_auto_services(struct loadparm_context *lp_ctx,
  * Unload unused services.
  */
 
-void lp_killunused(struct loadparm_context *lp_ctx,
+void lpcfg_killunused(struct loadparm_context *lp_ctx,
 		   struct smbsrv_connection *smb,
 		   bool (*snumused) (struct smbsrv_connection *, int))
 {
@@ -2292,7 +2214,7 @@ void lp_killunused(struct loadparm_context *lp_ctx,
 }
 
 
-static int lp_destructor(struct loadparm_context *lp_ctx)
+static int lpcfg_destructor(struct loadparm_context *lp_ctx)
 {
 	struct parmlist_entry *data;
 
@@ -2333,7 +2255,7 @@ struct loadparm_context *loadparm_init(TALLOC_CTX *mem_ctx)
 	if (lp_ctx == NULL)
 		return NULL;
 
-	talloc_set_destructor(lp_ctx, lp_destructor);
+	talloc_set_destructor(lp_ctx, lpcfg_destructor);
 	lp_ctx->bInGlobalSection = true;
 	lp_ctx->globals = talloc_zero(lp_ctx, struct loadparm_global);
 	lp_ctx->sDefault = talloc_zero(lp_ctx, struct loadparm_service);
@@ -2400,13 +2322,7 @@ struct loadparm_context *loadparm_init(TALLOC_CTX *mem_ctx)
 	lpcfg_do_global_parameter(lp_ctx, "ntptr providor", "simple_ldb");
 	/* the winbind method for domain controllers is for both RODC
 	   auth forwarding and for trusted domains */
-	lpcfg_do_global_parameter(lp_ctx, "auth methods:domain controller", "anonymous sam_ignoredomain winbind");
-	lpcfg_do_global_parameter(lp_ctx, "auth methods:member server", "anonymous sam winbind");
-	lpcfg_do_global_parameter(lp_ctx, "auth methods:standalone", "anonymous sam_ignoredomain");
 	lpcfg_do_global_parameter(lp_ctx, "private dir", dyn_PRIVATE_DIR);
-	lpcfg_do_global_parameter(lp_ctx, "sam database", "sam.ldb");
-	lpcfg_do_global_parameter(lp_ctx, "idmap database", "idmap.ldb");
-	lpcfg_do_global_parameter(lp_ctx, "secrets database", "secrets.ldb");
 	lpcfg_do_global_parameter(lp_ctx, "spoolss database", "spoolss.ldb");
 	lpcfg_do_global_parameter(lp_ctx, "wins config database", "wins_config.ldb");
 	lpcfg_do_global_parameter(lp_ctx, "wins database", "wins.ldb");
@@ -2430,10 +2346,9 @@ struct loadparm_context *loadparm_init(TALLOC_CTX *mem_ctx)
 
 	lpcfg_do_global_parameter(lp_ctx, "pid directory", dyn_PIDDIR);
 	lpcfg_do_global_parameter(lp_ctx, "lock dir", dyn_LOCKDIR);
-	lpcfg_do_global_parameter(lp_ctx, "modules dir", dyn_MODULESDIR);
 	lpcfg_do_global_parameter(lp_ctx, "ncalrpc dir", dyn_NCALRPCDIR);
 
-	lpcfg_do_global_parameter(lp_ctx, "socket address", "0.0.0.0");
+	lpcfg_do_global_parameter(lp_ctx, "socket address", "");
 	lpcfg_do_global_parameter_var(lp_ctx, "server string",
 				   "Samba %s", SAMBA_VERSION_STRING);
 
@@ -2541,6 +2456,7 @@ struct loadparm_context *loadparm_init_global(bool load_default)
 	if (global_loadparm_context == NULL) {
 		return NULL;
 	}
+	global_loadparm_context->global = true;
 	if (load_default && !global_loadparm_context->loaded) {
 		lpcfg_load_default(global_loadparm_context);
 	}
@@ -2548,6 +2464,20 @@ struct loadparm_context *loadparm_init_global(bool load_default)
 	return global_loadparm_context;
 }
 
+/**
+ * Initialise the global parameter structure.
+ */
+struct loadparm_context *loadparm_init_s3(TALLOC_CTX *mem_ctx, 
+					  const struct loadparm_s3_context *s3_fns)
+{
+	struct loadparm_context *loadparm_context = loadparm_init(mem_ctx);
+	if (!loadparm_context) {
+		return NULL;
+	}
+	loadparm_context->s3_fns = s3_fns;
+	return loadparm_context;
+}
+
 const char *lpcfg_configfile(struct loadparm_context *lp_ctx)
 {
 	return lp_ctx->szConfigFile;
@@ -2574,6 +2504,10 @@ static bool lpcfg_update(struct loadparm_context *lp_ctx)
 		lpcfg_do_global_parameter(lp_ctx, "wins server", "127.0.0.1");
 	}
 
+	if (!lp_ctx->global) {
+		return true;
+	}
+
 	panic_action = lp_ctx->globals->panic_action;
 
 	reload_charcnv(lp_ctx);
@@ -2584,8 +2518,6 @@ static bool lpcfg_update(struct loadparm_context *lp_ctx)
 	settings.timestamp_logs = true;
 	debug_set_settings(&settings);
 
-	/* FIXME: ntstatus_check_dos_mapping = lpcfg_nt_status_support(lp_ctx); */
-
 	/* FIXME: This is a bit of a hack, but we can't use a global, since 
 	 * not everything that uses lp also uses the socket library */
 	if (lpcfg_parm_bool(lp_ctx, NULL, "socket", "testnonblock", false)) {
@@ -2594,12 +2526,6 @@ static bool lpcfg_update(struct loadparm_context *lp_ctx)
 		unsetenv("SOCKET_TESTNONBLOCK");
 	}
 
-	/* FIXME: Check locale in environment for this: */
-	if (strcmp(lpcfg_display_charset(lp_ctx), lpcfg_unix_charset(lp_ctx)) != 0)
-		d_set_iconv(smb_iconv_open(lpcfg_display_charset(lp_ctx), lpcfg_unix_charset(lp_ctx)));
-	else
-		d_set_iconv((smb_iconv_t)-1);
-
 	return true;
 }
 
@@ -2750,7 +2676,7 @@ const char *lpcfg_servicename(const struct loadparm_service *service)
 /**
  * A useful volume label function.
  */
-const char *volume_label(struct loadparm_service *service, struct loadparm_service *sDefault)
+const char *lpcfg_volume_label(struct loadparm_service *service, struct loadparm_service *sDefault)
 {
 	const char *ret;
 	ret = lp_string((const char *)((service != NULL && service->volume != NULL) ?
@@ -2798,6 +2724,10 @@ struct smb_iconv_handle *lpcfg_iconv_handle(struct loadparm_context *lp_ctx)
 _PUBLIC_ void reload_charcnv(struct loadparm_context *lp_ctx)
 {
 	struct smb_iconv_handle *old_ic = lp_ctx->iconv_handle;
+	if (!lp_ctx->global) {
+		return;
+	}
+
 	if (old_ic == NULL) {
 		old_ic = global_iconv_handle;
 	}
@@ -2830,27 +2760,27 @@ void lpcfg_smbcli_session_options(struct loadparm_context *lp_ctx,
 
 _PUBLIC_ char *lpcfg_tls_keyfile(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
 {
-	return private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_keyfile);
+	return lpcfg_private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_keyfile);
 }
 
 _PUBLIC_ char *lpcfg_tls_certfile(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
 {
-	return private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_certfile);
+	return lpcfg_private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_certfile);
 }
 
 _PUBLIC_ char *lpcfg_tls_cafile(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
 {
-	return private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_cafile);
+	return lpcfg_private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_cafile);
 }
 
 _PUBLIC_ char *lpcfg_tls_crlfile(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
 {
-	return private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_crlfile);
+	return lpcfg_private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_crlfile);
 }
 
 _PUBLIC_ char *lpcfg_tls_dhpfile(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
 {
-	return private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_dhpfile);
+	return lpcfg_private_path(mem_ctx, lp_ctx, lp_ctx->globals->tls_dhpfile);
 }
 
 _PUBLIC_ struct dcerpc_server_info *lpcfg_dcerpc_server_info(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
diff --git a/source4/param/param.h b/source4/param/param.h
index b8fb369bdb5..ebf41f7213d 100644
--- a/source4/param/param.h
+++ b/source4/param/param.h
@@ -20,6 +20,8 @@
 #ifndef _PARAM_H /* _PARAM_H */
 #define _PARAM_H 
 
+struct loadparm_s3_context;
+
 struct parmlist_entry;
 
 struct param_context {
@@ -45,23 +47,7 @@ typedef NTSTATUS (*init_module_fn) (void);
    function init_module() which makes a system call */
 #define SAMBA_INIT_MODULE "samba_init_module"
 
-enum server_role {
-	ROLE_STANDALONE=0,
-	ROLE_DOMAIN_MEMBER=1,
-	ROLE_DOMAIN_CONTROLLER=2,
-};
-
-enum sid_generator {
-	SID_GENERATOR_INTERNAL=0,
-	SID_GENERATOR_BACKEND=1,
-};
-
-enum announce_as {/* Types of machine we can announce as. */
-	ANNOUNCE_AS_NT_SERVER=1,
-	ANNOUNCE_AS_WIN95=2,
-	ANNOUNCE_AS_WFW=3,
-	ANNOUNCE_AS_NT_WORKSTATION=4
-};
+#include "libds/common/roles.h"
 
 struct loadparm_context;
 struct loadparm_service;
@@ -211,7 +197,7 @@ struct loadparm_service *lpcfg_service(struct loadparm_context *lp_ctx,
 /**
  * A useful volume label function.
  */
-const char *volume_label(struct loadparm_service *service, struct loadparm_service *sDefault);
+const char *lp_cfg_volume_label(struct loadparm_service *service, struct loadparm_service *sDefault);
 
 /**
  * If we are PDC then prefer us as DMB
@@ -274,7 +260,7 @@ bool lpcfg_is_myname(struct loadparm_context *lp_ctx, const char *name);
 /**
  A useful function for returning a path in the Samba lock directory.
 **/
-char *lock_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
+char *lpcfg_lock_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
 			 const char *name);
 
 /**
@@ -284,7 +270,7 @@ char *lock_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
  *
  * @retval Pointer to a talloc'ed string containing the full path.
  **/
-char *config_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
+char *lpcfg_config_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
 			   const char *name);
 
 /**
@@ -295,7 +281,7 @@ char *config_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
  *
  * @retval Pointer to a talloc'ed string containing the full path.
  **/
-char *private_path(TALLOC_CTX* mem_ctx, 
+char *lpcfg_private_path(TALLOC_CTX* mem_ctx,
 			    struct loadparm_context *lp_ctx,
 			    const char *name);
 
@@ -331,8 +317,8 @@ bool run_init_functions(init_module_fn *fns);
  *
  * Will return an array of function pointers to initialization functions
  */
-init_module_fn *load_samba_modules(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx, const char *subsystem);
-const char *lpcfg_messaging_path(TALLOC_CTX *mem_ctx,
+init_module_fn *load_samba_modules(TALLOC_CTX *mem_ctx, const char *subsystem);
+const char *lpcfg_imessaging_path(TALLOC_CTX *mem_ctx,
 				       struct loadparm_context *lp_ctx);
 struct smb_iconv_handle *smb_iconv_handle_reinit_lp(TALLOC_CTX *mem_ctx,
 							      struct loadparm_context *lp_ctx,
diff --git a/source4/param/pyparam.c b/source4/param/pyparam.c
index 3ba81468605..e1ba18415d3 100644
--- a/source4/param/pyparam.c
+++ b/source4/param/pyparam.c
@@ -229,7 +229,7 @@ static PyObject *py_lp_ctx_private_path(py_talloc_Object *self, PyObject *args)
 	if (!PyArg_ParseTuple(args, "s", &name))
 		return NULL;
 
-	path = private_path(NULL, PyLoadparmContext_AsLoadparmContext(self), name);
+	path = lpcfg_private_path(NULL, PyLoadparmContext_AsLoadparmContext(self), name);
 	ret = PyString_FromString(path);
 	talloc_free(path);
 
@@ -272,6 +272,12 @@ static PyObject *py_lp_dump(PyObject *self, PyObject *args)
 	Py_RETURN_NONE;
 }
 
+static PyObject *py_samdb_url(PyObject *self)
+{
+	struct loadparm_context *lp_ctx = PyLoadparmContext_AsLoadparmContext(self);
+	return PyString_FromFormat("tdb://%s/sam.ldb", lpcfg_private_dir(lp_ctx));
+}
+
 
 static PyMethodDef py_lp_ctx_methods[] = {
 	{ "load", (PyCFunction)py_lp_ctx_load, METH_VARARGS, 
@@ -298,6 +304,9 @@ static PyMethodDef py_lp_ctx_methods[] = {
 		"S.services() -> list" },
 	{ "dump", (PyCFunction)py_lp_dump, METH_VARARGS, 
 		"S.dump(stream, show_defaults=False)" },
+	{ "samdb_url", (PyCFunction)py_samdb_url, METH_NOARGS,
+	        "S.samdb_url() -> string\n"
+	        "Returns the current URL for sam.ldb." },
 	{ NULL }
 };
 
@@ -431,11 +440,18 @@ static PyObject *py_setup_dir(PyObject *self)
     return PyString_FromString(dyn_SETUPDIR);
 }
 
+static PyObject *py_modules_dir(PyObject *self)
+{
+    return PyString_FromString(dyn_MODULESDIR);
+}
+
 static PyMethodDef pyparam_methods[] = {
     { "default_path", (PyCFunction)py_default_path, METH_NOARGS, 
         "Returns the default smb.conf path." },
     { "setup_dir", (PyCFunction)py_setup_dir, METH_NOARGS,
         "Returns the compiled in location of provision tempates." },
+    { "modules_dir", (PyCFunction)py_modules_dir, METH_NOARGS,
+        "Returns the compiled in location of modules." },
     { NULL }
 };
 
diff --git a/source4/param/secrets.c b/source4/param/secrets.c
index 13bfcc5e96a..55d1aa779b2 100644
--- a/source4/param/secrets.c
+++ b/source4/param/secrets.c
@@ -25,7 +25,7 @@
 #include "secrets.h"
 #include "param/param.h"
 #include "system/filesys.h"
-#include "tdb_wrap.h"
+#include "lib/util/tdb_wrap.h"
 #include "lib/ldb-samba/ldb_wrap.h"
 #include <ldb.h>
 #include "../lib/util/util_tdb.h"
@@ -49,22 +49,22 @@ static void get_rand_seed(struct tdb_wrap *secretsdb, int *new_seed)
 }
 
 /**
- * open up the secrets database
+ * open up the randseed database and set the random number generator callback
  */
-struct tdb_wrap *secrets_init(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
+bool randseed_init(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx)
 {
 	char *fname;
 	uint8_t dummy;
 	struct tdb_wrap *tdb;
 
-	fname = private_path(mem_ctx, lp_ctx, "secrets.tdb");
+	fname = lpcfg_private_path(mem_ctx, lp_ctx, "randseed.tdb");
 
 	tdb = tdb_wrap_open(mem_ctx, fname, 0, TDB_DEFAULT, O_RDWR|O_CREAT, 0600);
 
 	if (!tdb) {
 		DEBUG(0,("Failed to open %s\n", fname));
 		talloc_free(fname);
-		return NULL;
+		return false;
 	}
 	talloc_free(fname);
 
@@ -79,7 +79,7 @@ struct tdb_wrap *secrets_init(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_c
 	/* Ensure that the reseed is done now, while we are root, etc */
 	generate_random_buffer(&dummy, sizeof(dummy));
 
-	return tdb;
+	return true;
 }
 
 /**
@@ -88,7 +88,7 @@ struct tdb_wrap *secrets_init(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_c
 struct ldb_context *secrets_db_connect(TALLOC_CTX *mem_ctx,
 					struct loadparm_context *lp_ctx)
 {
-	return ldb_wrap_connect(mem_ctx, NULL, lp_ctx, lpcfg_secrets_url(lp_ctx),
+	return ldb_wrap_connect(mem_ctx, NULL, lp_ctx, "secrets.ldb",
 			       NULL, NULL, 0);
 }
 
diff --git a/source4/param/secrets.h b/source4/param/secrets.h
index 4cab9ccaddc..6576929a0d6 100644
--- a/source4/param/secrets.h
+++ b/source4/param/secrets.h
@@ -43,7 +43,8 @@ struct ldb_context;
 
 #include "librpc/gen_ndr/misc.h"
 
-struct tdb_wrap *secrets_init(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx);
+bool randseed_init(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx);
+
 struct ldb_context *secrets_db_connect(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx);
 struct dom_sid *secrets_get_domain_sid(TALLOC_CTX *mem_ctx,
 				       struct loadparm_context *lp_ctx,
diff --git a/source4/param/share_classic.c b/source4/param/share_classic.c
index 3d5bef91e17..3442d6bd50d 100644
--- a/source4/param/share_classic.c
+++ b/source4/param/share_classic.c
@@ -84,7 +84,7 @@ static const char *sclassic_string_option(struct share_config *scfg,
 	}
 
 	if (strcmp(opt_name, SHARE_VOLUME) == 0) {
-		return volume_label(s, lpcfg_default_service(lp_ctx));
+		return lpcfg_volume_label(s, lpcfg_default_service(lp_ctx));
 	}
 
 	if (strcmp(opt_name, SHARE_TYPE) == 0) {
diff --git a/source4/param/share_ldb.c b/source4/param/share_ldb.c
index c4ab319398c..f4d02b295a3 100644
--- a/source4/param/share_ldb.c
+++ b/source4/param/share_ldb.c
@@ -43,7 +43,7 @@ static NTSTATUS sldb_init(TALLOC_CTX *mem_ctx, const struct share_ops *ops,
 	}
 	
 	sdb = ldb_wrap_connect(*ctx, ev_ctx, lp_ctx,
-			       private_path(*ctx, lp_ctx, "share.ldb"),
+			       lpcfg_private_path(*ctx, lp_ctx, "share.ldb"),
 			       system_session(lp_ctx),
 			       NULL, 0);
 
diff --git a/source4/param/util.c b/source4/param/util.c
index 3413f25e49e..b1a7a571e65 100644
--- a/source4/param/util.c
+++ b/source4/param/util.c
@@ -75,7 +75,7 @@ bool lpcfg_is_myname(struct loadparm_context *lp_ctx, const char *name)
 /**
  A useful function for returning a path in the Samba lock directory.
 **/
-char *lock_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
+char *lpcfg_lock_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
 			 const char *name)
 {
 	char *fname, *dname;
@@ -108,7 +108,7 @@ char *lock_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
  * @retval Pointer to a talloc'ed string containing the full path.
  **/
 
-char *config_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
+char *lpcfg_config_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
 			   const char *name)
 {
 	char *fname, *config_dir, *p;
@@ -139,7 +139,7 @@ char *config_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
  *
  * @retval Pointer to a talloc'ed string containing the full path.
  **/
-char *private_path(TALLOC_CTX* mem_ctx, 
+char *lpcfg_private_path(TALLOC_CTX* mem_ctx,
 			    struct loadparm_context *lp_ctx,
 			    const char *name)
 {
@@ -165,7 +165,7 @@ char *smbd_tmp_path(TALLOC_CTX *mem_ctx,
 {
 	char *fname, *dname;
 
-	dname = private_path(mem_ctx, lp_ctx, "smbd.tmp");
+	dname = lpcfg_private_path(mem_ctx, lp_ctx, "smbd.tmp");
 	if (!directory_exist(dname)) {
 		mkdir(dname,0755);
 	}
@@ -266,23 +266,15 @@ bool run_init_functions(init_module_fn *fns)
 	return ret;
 }
 
-static char *modules_path(TALLOC_CTX* mem_ctx, struct loadparm_context *lp_ctx,
-			  const char *name)
-{
-	return talloc_asprintf(mem_ctx, "%s/%s", 
-			       lpcfg_modulesdir(lp_ctx),
-			       name);
-}
-
 /**
  * Load the initialization functions from DSO files for a specific subsystem.
  *
  * Will return an array of function pointers to initialization functions
  */
 
-init_module_fn *load_samba_modules(TALLOC_CTX *mem_ctx, struct loadparm_context *lp_ctx, const char *subsystem)
+init_module_fn *load_samba_modules(TALLOC_CTX *mem_ctx, const char *subsystem)
 {
-	char *path = modules_path(mem_ctx, lp_ctx, subsystem);
+	char *path = modules_path(mem_ctx, subsystem);
 	init_module_fn *ret;
 
 	ret = load_modules(mem_ctx, path);
@@ -292,7 +284,7 @@ init_module_fn *load_samba_modules(TALLOC_CTX *mem_ctx, struct loadparm_context
 	return ret;
 }
 
-const char *lpcfg_messaging_path(TALLOC_CTX *mem_ctx,
+const char *lpcfg_imessaging_path(TALLOC_CTX *mem_ctx,
 				       struct loadparm_context *lp_ctx)
 {
 	return smbd_tmp_path(mem_ctx, lp_ctx, "msg");
@@ -304,7 +296,6 @@ struct smb_iconv_handle *smb_iconv_handle_reinit_lp(TALLOC_CTX *mem_ctx,
 {
 	return smb_iconv_handle_reinit(mem_ctx, lpcfg_dos_charset(lp_ctx),
 					    lpcfg_unix_charset(lp_ctx),
-					    lpcfg_display_charset(lp_ctx),
 					    lpcfg_parm_bool(lp_ctx, NULL, "iconv", "native", true),
 					    old_ic);
 }
@@ -313,7 +304,8 @@ struct smb_iconv_handle *smb_iconv_handle_reinit_lp(TALLOC_CTX *mem_ctx,
 const char *lpcfg_sam_name(struct loadparm_context *lp_ctx)
 {
 	switch (lpcfg_server_role(lp_ctx)) {
-	case ROLE_DOMAIN_CONTROLLER:
+	case ROLE_DOMAIN_BDC:
+	case ROLE_DOMAIN_PDC:
 		return lpcfg_workgroup(lp_ctx);
 	default:
 		return lpcfg_netbios_name(lp_ctx);
diff --git a/source4/param/wscript_build b/source4/param/wscript_build
index 72674e55740..da10565c6c8 100644
--- a/source4/param/wscript_build
+++ b/source4/param/wscript_build
@@ -1,5 +1,20 @@
 #!/usr/bin/env python
 
+bld.SAMBA_GENERATOR('s3_param_h',
+                    source= 'loadparm.c ../script/mks3param.pl',
+                    target='s3_param.h',
+                    rule='${PERL} ${SRC[1].abspath(env)} ${SRC[0].abspath(env)} --file ${TGT}')
+
+bld.SAMBA_GENERATOR('param_local_h',
+                    source= 'loadparm.c ../script/mkparamdefs.pl',
+                    target='param_local.h',
+                    rule='${PERL} ${SRC[1].abspath(env)} ${SRC[0].abspath(env)} --file ${TGT} --generate-scope=LOCAL')
+
+bld.SAMBA_GENERATOR('param_global_h',
+                    source= 'loadparm.c ../script/mkparamdefs.pl',
+                    target='param_global.h',
+                    rule='${PERL} ${SRC[1].abspath(env)} ${SRC[0].abspath(env)} --file ${TGT} --generate-scope=GLOBAL')
+
 bld.SAMBA_LIBRARY('samba-hostconfig',
 	source='loadparm.c generic.c util.c',
 	pc_files='samba-hostconfig.pc',
diff --git a/source4/partition-upgrade.txt b/source4/partition-upgrade.txt
deleted file mode 100644
index 0abc810c3af..00000000000
--- a/source4/partition-upgrade.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-PARTITIONS UPGRADE
-==================
-
-Notes on using Samba4 after the partitions code upgrade on 21 Oct 2009
-
-The data previously stored in Samba4 partitions is no longer
-accessible, as we have changed the filenames.  
-
-To access your Samba4 tree again, backup your Samba4 prefix, and
-re-run provision with the same settings as in the past. 
-
-This will create new databases in your PREFIX/private directory, like
-DC=SAMBA,DC=EXAMPLE,DC=COM.ldb.  
-
-Restore your users.ldb to the DC=SAMBA,DC=EXAMPLE,DC=COM.ldb,
-configuration.ldb to CN=CONFIGURATION,DC=SAMBA,DC=EXAMPLE,DC=COM.ldb
-and schema.ldb to
-CN=SCHEMA,CN=CONFIGURATION,DC=SAMBA,DC=EXAMPLE,DC=COM.ldb
-
-Also remember to restore your secrets.ldb.
-
diff --git a/source4/rpc_server/common/server_info.c b/source4/rpc_server/common/server_info.c
index 7f533ecf77f..68985d81aa5 100644
--- a/source4/rpc_server/common/server_info.c
+++ b/source4/rpc_server/common/server_info.c
@@ -68,22 +68,8 @@ uint32_t dcesrv_common_get_server_type(TALLOC_CTX *mem_ctx, struct tevent_contex
 	default_server_announce |= SV_TYPE_SERVER;
 	default_server_announce |= SV_TYPE_SERVER_UNIX;
 
-	switch (lpcfg_announce_as(dce_ctx->lp_ctx)) {
-		case ANNOUNCE_AS_NT_SERVER:
-			default_server_announce |= SV_TYPE_SERVER_NT;
-			/* fall through... */
-		case ANNOUNCE_AS_NT_WORKSTATION:
-			default_server_announce |= SV_TYPE_NT;
-			break;
-		case ANNOUNCE_AS_WIN95:
-			default_server_announce |= SV_TYPE_WIN95_PLUS;
-			break;
-		case ANNOUNCE_AS_WFW:
-			default_server_announce |= SV_TYPE_WFW;
-			break;
-		default:
-			break;
-	}
+	default_server_announce |= SV_TYPE_SERVER_NT;
+	default_server_announce |= SV_TYPE_NT;
 
 	switch (lpcfg_server_role(dce_ctx->lp_ctx)) {
 		case ROLE_DOMAIN_MEMBER:
diff --git a/source4/rpc_server/dcerpc_server.c b/source4/rpc_server/dcerpc_server.c
index cd079da5c96..cbba5e24085 100644
--- a/source4/rpc_server/dcerpc_server.c
+++ b/source4/rpc_server/dcerpc_server.c
@@ -366,7 +366,7 @@ _PUBLIC_ NTSTATUS dcesrv_endpoint_connect(struct dcesrv_context *dce_ctx,
 				 const struct dcesrv_endpoint *ep,
 				 struct auth_session_info *session_info,
 				 struct tevent_context *event_ctx,
-				 struct messaging_context *msg_ctx,
+				 struct imessaging_context *msg_ctx,
 				 struct server_id server_id,
 				 uint32_t state_flags,
 				 struct dcesrv_connection **_p)
@@ -1235,7 +1235,7 @@ void dcerpc_server_init(struct loadparm_context *lp_ctx)
 	}
 	initialized = true;
 
-	shared_init = load_samba_modules(NULL, lp_ctx, "dcerpc_server");
+	shared_init = load_samba_modules(NULL, "dcerpc_server");
 
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
@@ -1353,7 +1353,7 @@ static void dcesrv_sock_reply_done(struct tevent_req *subreq)
 	ret = tstream_writev_queue_recv(subreq, &sys_errno);
 	TALLOC_FREE(subreq);
 	if (ret == -1) {
-		status = map_nt_error_from_unix(sys_errno);
+		status = map_nt_error_from_unix_common(sys_errno);
 		dcesrv_terminate_connection(substate->dce_conn, nt_errstr(status));
 		return;
 	}
@@ -1436,7 +1436,7 @@ static void dcesrv_sock_accept(struct stream_connection *srv_conn)
 						  socket_get_fd(srv_conn->socket),
 						  &dcesrv_conn->stream);
 		if (ret == -1) {
-			status = map_nt_error_from_unix(errno);
+			status = map_nt_error_from_unix_common(errno);
 			DEBUG(0, ("dcesrv_sock_accept: "
 				  "failed to setup tstream: %s\n",
 				  nt_errstr(status)));
@@ -1648,7 +1648,7 @@ static NTSTATUS add_socket_rpc_tcp_iface(struct dcesrv_context *dce_ctx, struct
 
 	status = stream_setup_socket(dcesrv_sock, event_ctx, dce_ctx->lp_ctx,
 				     model_ops, &dcesrv_stream_ops, 
-				     "ipv4", address, &port, 
+				     "ip", address, &port,
 				     lpcfg_socket_options(dce_ctx->lp_ctx),
 				     dcesrv_sock);
 	if (!NT_STATUS_IS_OK(status)) {
@@ -1678,18 +1678,24 @@ static NTSTATUS dcesrv_add_ep_tcp(struct dcesrv_context *dce_ctx,
 		int i;
 		struct interface *ifaces;
 
-		load_interfaces(dce_ctx, lpcfg_interfaces(lp_ctx), &ifaces);
+		load_interface_list(dce_ctx, lp_ctx, &ifaces);
 
-		num_interfaces = iface_count(ifaces);
+		num_interfaces = iface_list_count(ifaces);
 		for(i = 0; i < num_interfaces; i++) {
-			const char *address = iface_n_ip(ifaces, i);
+			const char *address = iface_list_n_ip(ifaces, i);
 			status = add_socket_rpc_tcp_iface(dce_ctx, e, event_ctx, model_ops, address);
 			NT_STATUS_NOT_OK_RETURN(status);
 		}
 	} else {
-		status = add_socket_rpc_tcp_iface(dce_ctx, e, event_ctx, model_ops, 
-						  lpcfg_socket_address(lp_ctx));
-		NT_STATUS_NOT_OK_RETURN(status);
+		const char **wcard;
+		int i;
+		wcard = iface_list_wildcard(dce_ctx, lp_ctx);
+		NT_STATUS_HAVE_NO_MEMORY(wcard);
+		for (i=0; wcard[i]; i++) {
+			status = add_socket_rpc_tcp_iface(dce_ctx, e, event_ctx, model_ops, wcard[i]);
+			NT_STATUS_NOT_OK_RETURN(status);
+		}
+		talloc_free(wcard);
 	}
 
 	return NT_STATUS_OK;
diff --git a/source4/rpc_server/dcerpc_server.h b/source4/rpc_server/dcerpc_server.h
index 024009ab707..4fcb5c50a13 100644
--- a/source4/rpc_server/dcerpc_server.h
+++ b/source4/rpc_server/dcerpc_server.h
@@ -23,7 +23,7 @@
 #ifndef SAMBA_DCERPC_SERVER_H
 #define SAMBA_DCERPC_SERVER_H
 
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 #include "librpc/rpc/dcerpc.h"
 #include "librpc/ndr/libndr.h"
 
@@ -111,7 +111,7 @@ struct dcesrv_call_state {
 	struct tevent_context *event_ctx;
 
 	/* the message_context that will be used for async replies */
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 
 	/* this is the pointer to the allocated function struct */
 	void *r;
@@ -200,7 +200,7 @@ struct dcesrv_connection {
 	struct tevent_context *event_ctx;
 
 	/* the message_context that will be used for this connection */
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 
 	/* the server_id that will be used for this connection */
 	struct server_id server_id;
@@ -319,7 +319,7 @@ NTSTATUS dcesrv_endpoint_connect(struct dcesrv_context *dce_ctx,
 				 const struct dcesrv_endpoint *ep,
 				 struct auth_session_info *session_info,
 				 struct tevent_context *event_ctx,
-				 struct messaging_context *msg_ctx,
+				 struct imessaging_context *msg_ctx,
 				 struct server_id server_id,
 				 uint32_t state_flags,
 				 struct dcesrv_connection **_p);
diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c
index 8ae536823de..c57fec2049c 100644
--- a/source4/rpc_server/drsuapi/getncchanges.c
+++ b/source4/rpc_server/drsuapi/getncchanges.c
@@ -360,27 +360,37 @@ static WERROR get_nc_changes_add_la(TALLOC_CTX *mem_ctx,
 	la->attid = sa->attributeID_id;
 	la->flags = active?DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE:0;
 
-	status = dsdb_get_extended_dn_nttime(dsdb_dn->dn, &la->originating_add_time, "RMD_ADDTIME");
-	if (!NT_STATUS_IS_OK(status)) {
-		return ntstatus_to_werror(status);
-	}
 	status = dsdb_get_extended_dn_uint32(dsdb_dn->dn, &la->meta_data.version, "RMD_VERSION");
 	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(0,(__location__ " No RMD_VERSION in linked attribute '%s' in '%s'\n",
+			 sa->lDAPDisplayName, ldb_dn_get_linearized(msg->dn)));
 		return ntstatus_to_werror(status);
 	}
 	status = dsdb_get_extended_dn_nttime(dsdb_dn->dn, &la->meta_data.originating_change_time, "RMD_CHANGETIME");
 	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(0,(__location__ " No RMD_CHANGETIME in linked attribute '%s' in '%s'\n",
+			 sa->lDAPDisplayName, ldb_dn_get_linearized(msg->dn)));
 		return ntstatus_to_werror(status);
 	}
 	status = dsdb_get_extended_dn_guid(dsdb_dn->dn, &la->meta_data.originating_invocation_id, "RMD_INVOCID");
 	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(0,(__location__ " No RMD_INVOCID in linked attribute '%s' in '%s'\n",
+			 sa->lDAPDisplayName, ldb_dn_get_linearized(msg->dn)));
 		return ntstatus_to_werror(status);
 	}
 	status = dsdb_get_extended_dn_uint64(dsdb_dn->dn, &la->meta_data.originating_usn, "RMD_ORIGINATING_USN");
 	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(0,(__location__ " No RMD_ORIGINATING_USN in linked attribute '%s' in '%s'\n",
+			 sa->lDAPDisplayName, ldb_dn_get_linearized(msg->dn)));
 		return ntstatus_to_werror(status);
 	}
 
+	status = dsdb_get_extended_dn_nttime(dsdb_dn->dn, &la->originating_add_time, "RMD_ADDTIME");
+	if (!NT_STATUS_IS_OK(status)) {
+		/* this is possible for upgraded links */
+		la->originating_add_time = la->meta_data.originating_change_time;
+	}
+
 	werr = dsdb_dn_la_to_blob(sam_ctx, sa, schema, *la_list, dsdb_dn, &la->value.blob);
 	W_ERROR_NOT_OK_RETURN(werr);
 
@@ -993,12 +1003,14 @@ static WERROR getncchanges_change_master(struct drsuapi_bind_state *b_state,
 	msg->dn = drs_ObjectIdentifier_to_dn(msg, ldb, req10->naming_context);
 	W_ERROR_HAVE_NO_MEMORY(msg->dn);
 
+	/* TODO: make sure ntds_dn is a valid nTDSDSA object */
 	ret = dsdb_find_dn_by_guid(ldb, msg, &req10->destination_dsa_guid, &ntds_dn);
 	if (ret != LDB_SUCCESS) {
 		DEBUG(0, (__location__ ": Unable to find NTDS object for guid %s - %s\n",
 			  GUID_string(mem_ctx, &req10->destination_dsa_guid), ldb_errstring(ldb)));
 		talloc_free(msg);
-		return WERR_DS_DRA_INTERNAL_ERROR;
+		ctr6->extended_ret = DRSUAPI_EXOP_ERR_UNKNOWN_CALLER;
+		return WERR_OK;
 	}
 
 	ret = ldb_msg_add_string(msg, "fSMORoleOwner", ldb_dn_get_linearized(ntds_dn));
@@ -1161,6 +1173,91 @@ getncchanges_map_req8(TALLOC_CTX *mem_ctx,
 }
 
 
+/**
+ * Collects object for normal replication cycle.
+ */
+static WERROR getncchanges_collect_objects(struct drsuapi_bind_state *b_state,
+					   TALLOC_CTX *mem_ctx,
+					   struct drsuapi_DsGetNCChangesRequest10 *req10,
+					   struct ldb_dn *search_dn,
+					   const char *extra_filter,
+					   struct ldb_result **search_res)
+{
+	int ret;
+	char* search_filter;
+	enum ldb_scope scope = LDB_SCOPE_SUBTREE;
+	//const char *extra_filter;
+	struct drsuapi_getncchanges_state *getnc_state = b_state->getncchanges_state;
+	const char *attrs[] = { "uSNChanged",
+				"objectGUID" ,
+				NULL };
+
+	if (req10->extended_op == DRSUAPI_EXOP_REPL_OBJ ||
+	    req10->extended_op == DRSUAPI_EXOP_REPL_SECRET) {
+		scope = LDB_SCOPE_BASE;
+	}
+
+	//extra_filter = lpcfg_parm_string(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "object filter");
+
+	//getnc_state->min_usn = req10->highwatermark.highest_usn;
+
+	/* Construct response. */
+	search_filter = talloc_asprintf(mem_ctx,
+					"(uSNChanged>=%llu)",
+					(unsigned long long)(getnc_state->min_usn+1));
+
+	if (extra_filter) {
+		search_filter = talloc_asprintf(mem_ctx, "(&%s(%s))", search_filter, extra_filter);
+	}
+
+	if (req10->replica_flags & DRSUAPI_DRS_CRITICAL_ONLY) {
+		search_filter = talloc_asprintf(mem_ctx,
+						"(&%s(isCriticalSystemObject=TRUE))",
+						search_filter);
+	}
+
+	if (req10->replica_flags & DRSUAPI_DRS_ASYNC_REP) {
+		scope = LDB_SCOPE_BASE;
+	}
+
+	if (!search_dn) {
+		search_dn = getnc_state->ncRoot_dn;
+	}
+
+	DEBUG(2,(__location__ ": getncchanges on %s using filter %s\n",
+		 ldb_dn_get_linearized(getnc_state->ncRoot_dn), search_filter));
+	ret = drsuapi_search_with_extended_dn(b_state->sam_ctx, getnc_state, search_res,
+					      search_dn, scope, attrs,
+					      search_filter);
+	if (ret != LDB_SUCCESS) {
+		return WERR_DS_DRA_INTERNAL_ERROR;
+	}
+
+	return WERR_OK;
+}
+
+/**
+ * Collects object for normal replication cycle.
+ */
+static WERROR getncchanges_collect_objects_exop(struct drsuapi_bind_state *b_state,
+						TALLOC_CTX *mem_ctx,
+						struct drsuapi_DsGetNCChangesRequest10 *req10,
+						struct drsuapi_DsGetNCChangesCtr6 *ctr6,
+						struct ldb_dn *search_dn,
+						const char *extra_filter,
+						struct ldb_result **search_res)
+{
+	/* we have nothing to do in case of ex-op failure */
+	if (ctr6->extended_ret != DRSUAPI_EXOP_ERR_SUCCESS) {
+		return WERR_OK;
+	}
+
+	/* TODO: implement extended op specific collection
+	 * of objects. Right now we just normal procedure
+	 * for collecting objects */
+	return getncchanges_collect_objects(b_state, mem_ctx, req10, search_dn, extra_filter, search_res);
+}
+
 /* 
   drsuapi_DsGetNCChanges
 
@@ -1177,9 +1274,6 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
 	struct drsuapi_DsReplicaObjectListItemEx **currentObject;
 	NTSTATUS status;
 	DATA_BLOB session_key;
-	const char *attrs[] = { "uSNChanged",
-				"objectGUID" ,
-				NULL };
 	WERROR werr;
 	struct dcesrv_handle *h;
 	struct drsuapi_bind_state *b_state;	
@@ -1327,6 +1421,10 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
 				     ldb_get_schema_basedn(b_state->sam_ctx));
 		getnc_state->is_schema_nc = (0 == ret);
 
+		if (req10->extended_op != DRSUAPI_EXOP_NONE) {
+			r->out.ctr->ctr6.extended_ret = DRSUAPI_EXOP_ERR_SUCCESS;
+		}
+
 		/*
 		 * This is the first replication cycle and it is
 		 * a good place to handle extended operations
@@ -1391,64 +1489,39 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
 	   Work out if this is the start of a new cycle */
 
 	if (getnc_state->guids == NULL) {
-		char* search_filter;
-		enum ldb_scope scope = LDB_SCOPE_SUBTREE;
 		const char *extra_filter;
-		struct ldb_result *search_res;
-
-		if (req10->extended_op == DRSUAPI_EXOP_REPL_OBJ ||
-		    req10->extended_op == DRSUAPI_EXOP_REPL_SECRET) {
-			scope = LDB_SCOPE_BASE;
-		}
+		struct ldb_result *search_res = NULL;
 
 		extra_filter = lpcfg_parm_string(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "object filter");
 
 		getnc_state->min_usn = req10->highwatermark.highest_usn;
 
-		/* Construct response. */
-		search_filter = talloc_asprintf(mem_ctx,
-						"(uSNChanged>=%llu)",
-						(unsigned long long)(getnc_state->min_usn+1));
-	
-		if (extra_filter) {
-			search_filter = talloc_asprintf(mem_ctx, "(&%s(%s))", search_filter, extra_filter);
-		}
-
-		if (req10->replica_flags & DRSUAPI_DRS_CRITICAL_ONLY) {
-			search_filter = talloc_asprintf(mem_ctx,
-							"(&%s(isCriticalSystemObject=TRUE))",
-							search_filter);
-		}
-		
-		if (req10->replica_flags & DRSUAPI_DRS_ASYNC_REP) {
-			scope = LDB_SCOPE_BASE;
-		}
-		
-		if (!search_dn) {
-			search_dn = getnc_state->ncRoot_dn;
-		}
-
-		DEBUG(2,(__location__ ": getncchanges on %s using filter %s\n",
-			 ldb_dn_get_linearized(getnc_state->ncRoot_dn), search_filter));
-		ret = drsuapi_search_with_extended_dn(sam_ctx, getnc_state, &search_res,
-						      search_dn, scope, attrs,
-						      search_filter);
-		if (ret != LDB_SUCCESS) {
-			return WERR_DS_DRA_INTERNAL_ERROR;
-		}
-
-		if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) {
-			TYPESAFE_QSORT(search_res->msgs,
-				       search_res->count,
-				       site_res_cmp_parent_order);
+		if (req10->extended_op == DRSUAPI_EXOP_NONE) {
+			werr = getncchanges_collect_objects(b_state, mem_ctx, req10,
+							    search_dn, extra_filter,
+							    &search_res);
 		} else {
-			TYPESAFE_QSORT(search_res->msgs,
-				       search_res->count,
-				       site_res_cmp_usn_order);
+			werr = getncchanges_collect_objects_exop(b_state, mem_ctx, req10,
+								 &r->out.ctr->ctr6,
+								 search_dn, extra_filter,
+								 &search_res);
+		}
+		W_ERROR_NOT_OK_RETURN(werr);
+
+		if (search_res) {
+			if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) {
+				TYPESAFE_QSORT(search_res->msgs,
+					       search_res->count,
+					       site_res_cmp_parent_order);
+			} else {
+				TYPESAFE_QSORT(search_res->msgs,
+					       search_res->count,
+					       site_res_cmp_usn_order);
+			}
 		}
 
 		/* extract out the GUIDs list */
-		getnc_state->num_records = search_res->count;
+		getnc_state->num_records = search_res ? search_res->count : 0;
 		getnc_state->guids = talloc_array(getnc_state, struct GUID, getnc_state->num_records);
 		W_ERROR_HAVE_NO_MEMORY(getnc_state->guids);
 
@@ -1705,7 +1778,6 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
 		r->out.ctr->ctr6.uptodateness_vector = NULL;
 		r->out.ctr->ctr6.nc_object_count = 0;
 		ZERO_STRUCT(r->out.ctr->ctr6.new_highwatermark);
-		r->out.ctr->ctr6.extended_ret = DRSUAPI_EXOP_ERR_SUCCESS;
 	}
 
 	DEBUG(r->out.ctr->ctr6.more_data?4:2,
diff --git a/source4/rpc_server/lsa/dcesrv_lsa.c b/source4/rpc_server/lsa/dcesrv_lsa.c
index b9fe5a3f401..50ce2346adc 100644
--- a/source4/rpc_server/lsa/dcesrv_lsa.c
+++ b/source4/rpc_server/lsa/dcesrv_lsa.c
@@ -1429,7 +1429,7 @@ static NTSTATUS update_uint32_t_value(TALLOC_CTX *mem_ctx,
 {
 	const struct ldb_val *orig_val;
 	uint32_t orig_uint = 0;
-	int flags = 0;
+	unsigned int flags = 0;
 	int ret;
 
 	orig_val = ldb_msg_find_ldb_val(orig, attribute);
diff --git a/source4/rpc_server/netlogon/dcerpc_netlogon.c b/source4/rpc_server/netlogon/dcerpc_netlogon.c
index bf8b4844a08..d5a7eebb55c 100644
--- a/source4/rpc_server/netlogon/dcerpc_netlogon.c
+++ b/source4/rpc_server/netlogon/dcerpc_netlogon.c
@@ -37,6 +37,7 @@
 #include "lib/tsocket/tsocket.h"
 #include "librpc/gen_ndr/ndr_netlogon.h"
 #include "librpc/gen_ndr/ndr_irpc.h"
+#include "lib/socket/netif.h"
 
 struct netlogon_server_pipe_state {
 	struct netr_Credential client_challenge;
@@ -600,7 +601,7 @@ static NTSTATUS dcesrv_netr_LogonSamLogon_check(const struct netr_LogonSamLogonE
 static NTSTATUS dcesrv_netr_LogonSamLogon_base(struct dcesrv_call_state *dce_call, TALLOC_CTX *mem_ctx,
 					struct netr_LogonSamLogonEx *r, struct netlogon_creds_CredentialState *creds)
 {
-	struct auth_context *auth_context;
+	struct auth4_context *auth_context;
 	struct auth_usersupplied_info *user_info;
 	struct auth_user_info_dc *user_info_dc;
 	NTSTATUS nt_status;
@@ -1233,6 +1234,7 @@ static NTSTATUS dcesrv_netr_NetrEnumerateTrustedDomains(struct dcesrv_call_state
 static NTSTATUS dcesrv_netr_LogonGetCapabilities(struct dcesrv_call_state *dce_call, TALLOC_CTX *mem_ctx,
 		       struct netr_LogonGetCapabilities *r)
 {
+
 	/* we don't support AES yet */
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
@@ -1710,6 +1712,8 @@ static WERROR dcesrv_netr_DsRGetDCNameEx2(struct dcesrv_call_state *dce_call,
 	NTSTATUS status;
 	const char *dc_name = NULL;
 	const char *domain_name = NULL;
+	struct interface *ifaces;
+	const char *pdc_ip;
 
 	ZERO_STRUCTP(r->out.info);
 
@@ -1815,10 +1819,15 @@ static WERROR dcesrv_netr_DsRGetDCNameEx2(struct dcesrv_call_state *dce_call,
 	W_ERROR_HAVE_NO_MEMORY(info);
 	info->dc_unc           = talloc_asprintf(mem_ctx, "\\\\%s", dc_name);
 	W_ERROR_HAVE_NO_MEMORY(info->dc_unc);
-	info->dc_address = talloc_asprintf(mem_ctx, "\\\\%s",
-					   response.data.nt5_ex.sockaddr.pdc_ip);
+
+	load_interface_list(mem_ctx, lp_ctx, &ifaces);
+	pdc_ip = iface_list_best_ip(ifaces, addr);
+	if (pdc_ip == NULL) {
+		pdc_ip = "127.0.0.1";
+	}
+	info->dc_address = talloc_asprintf(mem_ctx, "\\\\%s", pdc_ip);
 	W_ERROR_HAVE_NO_MEMORY(info->dc_address);
-	info->dc_address_type  = DS_ADDRESS_TYPE_INET; /* TODO: make this dynamic? for ipv6 */
+	info->dc_address_type  = DS_ADDRESS_TYPE_INET;
 	info->domain_guid      = response.data.nt5_ex.domain_uuid;
 	info->domain_name      = domain_name;
 	info->forest_name      = response.data.nt5_ex.forest;
diff --git a/source4/rpc_server/samr/dcesrv_samr.c b/source4/rpc_server/samr/dcesrv_samr.c
index 1c7416cb7be..32aafcd23fa 100644
--- a/source4/rpc_server/samr/dcesrv_samr.c
+++ b/source4/rpc_server/samr/dcesrv_samr.c
@@ -510,6 +510,9 @@ static NTSTATUS dcesrv_samr_info_DomGeneralInformation(struct samr_domain_state
 			info->role = SAMR_ROLE_DOMAIN_BDC;
 		}
 		break;
+	case ROLE_DOMAIN_PDC:
+		info->role = SAMR_ROLE_DOMAIN_PDC;
+		break;
 	case ROLE_DOMAIN_MEMBER:
 		info->role = SAMR_ROLE_DOMAIN_MEMBER;
 		break;
@@ -613,6 +616,9 @@ static NTSTATUS dcesrv_samr_info_DomInfo7(struct samr_domain_state *state,
 			info->role = SAMR_ROLE_DOMAIN_BDC;
 		}
 		break;
+	case ROLE_DOMAIN_PDC:
+		info->role = SAMR_ROLE_DOMAIN_PDC;
+		break;
 	case ROLE_DOMAIN_MEMBER:
 		info->role = SAMR_ROLE_DOMAIN_MEMBER;
 		break;
@@ -1205,7 +1211,6 @@ static NTSTATUS dcesrv_samr_CreateUser2(struct dcesrv_call_state *dce_call, TALL
 	}
 	a_state = talloc(mem_ctx, struct samr_account_state);
 	if (!a_state) {
-		ldb_transaction_cancel(d_state->sam_ctx);
 		return NT_STATUS_NO_MEMORY;
 	}
 	a_state->sam_ctx = d_state->sam_ctx;
diff --git a/source4/samba_tool/samba_tool.c b/source4/samba_tool/samba_tool.c
index 8dfbb832985..ce229444203 100644
--- a/source4/samba_tool/samba_tool.c
+++ b/source4/samba_tool/samba_tool.c
@@ -299,7 +299,7 @@ static int binary_net(int argc, const char **argv)
 
 	setlinebuf(stdout);
 
-	dcerpc_init(cmdline_lp_ctx);
+	dcerpc_init();
 
 	ev = s4_event_context_init(NULL);
 	if (!ev) {
diff --git a/source4/script/mkparamdefs.pl b/source4/script/mkparamdefs.pl
new file mode 100644
index 00000000000..9b255a86fca
--- /dev/null
+++ b/source4/script/mkparamdefs.pl
@@ -0,0 +1,192 @@
+#!/usr/bin/perl
+# Generate loadparm tables for loadparm.c
+# by Andrew Bartlett
+# based on mkproto.pl Written by Jelmer Vernooij
+# based on the original mkproto.sh by Andrew Tridgell
+
+use strict;
+
+# don't use warnings module as it is not portable enough
+# use warnings;
+
+use Getopt::Long;
+use File::Basename;
+use File::Path;
+
+#####################################################################
+# read a file into a string
+
+my $file = undef;
+my $public_define = undef;
+my $_public = "";
+my $_private = "";
+my $public_data = \$_public;
+my $builddir = ".";
+my $srcdir = ".";
+my $generate_scope = undef;
+
+sub public($)
+{
+	my ($d) = @_;
+	$$public_data .= $d;
+}
+
+sub usage()
+{
+	print "Usage: mkparamdefs.pl [options] [c files]\n";
+	print "OPTIONS:\n";
+	print "  --srcdir=path          Read files relative to this directory\n";
+	print "  --builddir=path        Write file relative to this directory\n";
+	print "  --generate-scope=[GLOBAL|LOCAL]        Filter which definitions to generate\n";
+	print "  --help                 Print this help message\n\n";
+	exit 0;
+}
+
+GetOptions(
+	'file=s' => sub { my ($f,$v) = @_; $file = $v; },
+	'srcdir=s' => sub { my ($f,$v) = @_; $srcdir = $v; },
+	'builddir=s' => sub { my ($f,$v) = @_; $builddir = $v; },
+	'generate-scope=s' => sub { my ($f,$v) = @_; $generate_scope = $v; },
+	'help' => \&usage
+) or exit(1);
+
+sub normalize_define($$)
+{
+	my ($define, $file) = @_;
+
+	if (not defined($define) and defined($file)) {
+		$define = "__" . uc($file) . "__";
+		$define =~ tr{./}{__};
+		$define =~ tr{\-}{_};
+	} elsif (not defined($define)) {
+		$define = '_S3_PARAM_H_';
+	}
+
+	return $define;
+}
+
+$public_define = normalize_define($public_define, $file);
+
+sub file_load($)
+{
+    my($filename) = @_;
+    local(*INPUTFILE);
+    open(INPUTFILE, $filename) or return undef;
+    my($saved_delim) = $/;
+    undef $/;
+    my($data) = <INPUTFILE>;
+    close(INPUTFILE);
+    $/ = $saved_delim;
+    return $data;
+}
+
+sub print_header($$$)
+{
+	my ($file, $header_name,$generate_scope) = @_;
+	$file->("#ifndef $header_name\n");
+	$file->("#define $header_name\n\n");
+$file->("/* This file was automatically generated by mkparmdefs.pl. DO NOT EDIT */\n\n");
+	$file->("/**\n");
+	if ($generate_scope eq "GLOBAL") {
+	    $file->(" * This structure describes global (ie., server-wide) parameters.\n");
+	    $file->(" */\n");
+	    $file->("struct loadparm_global \n");
+	} elsif ($generate_scope eq "LOCAL") {
+	    $file->(" * This structure describes a single service.\n");
+	    $file->(" */\n");
+	    $file->("struct loadparm_service \n");
+	}
+$file->("{\n");
+}
+
+sub print_footer($$$) 
+{
+	my ($file, $header_name, $generate_scope) = @_;
+	$file->("LOADPARM_EXTRA_" . $generate_scope . "S\n");
+	$file->("};\n");
+	$file->("\n#endif /* $header_name */\n\n");
+}
+
+sub handle_loadparm($$$) 
+{
+	my ($file,$line,$generate_scope) = @_;
+
+	if ($line =~ /^FN_(GLOBAL|LOCAL)_(CONST_STRING|STRING|BOOL|bool|CHAR|INTEGER|LIST)\((\w+),(.*)\)/o) {
+		my $scope = $1;
+		my $type = $2;
+		my $name = $3;
+		my $var = $4;
+
+		my %tmap = (
+			    "BOOL" => "int ",
+			    "CONST_STRING" => "char *",
+			    "STRING" => "char *",
+			    "INTEGER" => "int ",
+			    "CHAR" => "char ",
+			    "LIST" => "const char **",
+			    );
+
+		if ($scope eq $generate_scope) {
+		    $file->("\t$tmap{$type} $var;\n");
+		}
+	}
+}
+
+sub process_file($$) 
+{
+	my ($file, $filename) = @_;
+
+	$filename =~ s/\.o$/\.c/g;
+
+	if ($filename =~ /^\//) {
+		open(FH, "<$filename") or die("Failed to open $filename");
+	} elsif (!open(FH, "< $builddir/$filename")) {
+	    open(FH, "< $srcdir/$filename") || die "Failed to open $filename";
+	}
+
+	my $comment = undef;
+	my $incomment = 0;
+	while (my $line = <FH>) {	      
+		if ($line =~ /^\/\*\*/) { 
+			$comment = "";
+			$incomment = 1;
+		}
+
+		if ($incomment) {
+			$comment .= $line;
+			if ($line =~ /\*\//) {
+				$incomment = 0;
+			}
+		} 
+
+		# these are ordered for maximum speed
+		next if ($line =~ /^\s/);
+	      
+		next unless ($line =~ /\(/);
+
+		next if ($line =~ /^\/|[;]/);
+
+		if ($line =~ /^FN_/) {
+			handle_loadparm($file, $line, $generate_scope);
+		}
+		next;
+	}
+
+	close(FH);
+}
+
+
+print_header(\&public, $public_define, $generate_scope);
+
+process_file(\&public, $_) foreach (@ARGV);
+print_footer(\&public, $public_define, $generate_scope);
+
+if (not defined($file)) {
+	print STDOUT $$public_data;
+}
+
+mkpath(dirname($file), 0, 0755);
+open(PUBLIC, ">$file") or die("Can't open `$file': $!"); 
+print PUBLIC "$$public_data";
+close(PUBLIC);
+
diff --git a/source4/script/mkrelease.sh b/source4/script/mkrelease.sh
index 6ad927b9829..19a1ade376b 100755
--- a/source4/script/mkrelease.sh
+++ b/source4/script/mkrelease.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 if [ ! -d ".git" -o `dirname $0` != "./source4/script" ]; then
 	echo "Run this script from the top-level directory in the"
@@ -6,8 +6,7 @@ if [ ! -d ".git" -o `dirname $0` != "./source4/script" ]; then
 	exit 1
 fi
 
-cd source4
-../buildtools/bin/waf dist
+./buildtools/bin/waf dist
 TGZFILE="`echo *.tar.gz`"
 gunzip $TGZFILE
 TARFILE="`echo *.tar`"
diff --git a/source4/script/mks3param.pl b/source4/script/mks3param.pl
new file mode 100644
index 00000000000..2d99ad59a9c
--- /dev/null
+++ b/source4/script/mks3param.pl
@@ -0,0 +1,177 @@
+#!/usr/bin/perl
+# Generate loadparm interfaces tables for Samba3/Samba4 integration
+# by Andrew Bartlett
+# based on mkproto.pl Written by Jelmer Vernooij
+# based on the original mkproto.sh by Andrew Tridgell
+
+use strict;
+
+# don't use warnings module as it is not portable enough
+# use warnings;
+
+use Getopt::Long;
+use File::Basename;
+use File::Path;
+
+#####################################################################
+# read a file into a string
+
+my $file = undef;
+my $public_define = undef;
+my $_public = "";
+my $_private = "";
+my $public_data = \$_public;
+my $builddir = ".";
+my $srcdir = ".";
+
+sub public($)
+{
+	my ($d) = @_;
+	$$public_data .= $d;
+}
+
+sub usage()
+{
+	print "Usage: mks3param.pl [options] [c files]\n";
+	print "OPTIONS:\n";
+	print "  --srcdir=path          Read files relative to this directory\n";
+	print "  --builddir=path        Write file relative to this directory\n";
+	print "  --help                 Print this help message\n\n";
+	exit 0;
+}
+
+GetOptions(
+	'file=s' => sub { my ($f,$v) = @_; $file = $v; },
+	'srcdir=s' => sub { my ($f,$v) = @_; $srcdir = $v; },
+	'builddir=s' => sub { my ($f,$v) = @_; $builddir = $v; },
+	'help' => \&usage
+) or exit(1);
+
+sub normalize_define($$)
+{
+	my ($define, $file) = @_;
+
+	if (not defined($define) and defined($file)) {
+		$define = "__" . uc($file) . "__";
+		$define =~ tr{./}{__};
+		$define =~ tr{\-}{_};
+	} elsif (not defined($define)) {
+		$define = '_S3_PARAM_H_';
+	}
+
+	return $define;
+}
+
+$public_define = normalize_define($public_define, $file);
+
+sub file_load($)
+{
+    my($filename) = @_;
+    local(*INPUTFILE);
+    open(INPUTFILE, $filename) or return undef;
+    my($saved_delim) = $/;
+    undef $/;
+    my($data) = <INPUTFILE>;
+    close(INPUTFILE);
+    $/ = $saved_delim;
+    return $data;
+}
+
+sub print_header($$)
+{
+	my ($file, $header_name) = @_;
+	$file->("#ifndef $header_name\n");
+	$file->("#define $header_name\n\n");
+	$file->("/* This file was automatically generated by mks3param.pl. DO NOT EDIT */\n\n");
+	$file->("struct loadparm_s3_context\n");
+	$file->("{\n");
+	$file->("\tconst char * (*get_parametric)(const char *type, const char *option);");
+}
+
+sub print_footer($$) 
+{
+	my ($file, $header_name) = @_;
+	$file->("};");
+	$file->("\n#endif /* $header_name */\n\n");
+}
+
+sub handle_loadparm($$) 
+{
+	my ($file,$line) = @_;
+
+	if ($line =~ /^FN_(GLOBAL|LOCAL)_(CONST_STRING|STRING|BOOL|bool|CHAR|INTEGER|LIST)\((\w+),.*\)/o) {
+		my $scope = $1;
+		my $type = $2;
+		my $name = $3;
+
+		my %tmap = (
+			    "BOOL" => "bool ",
+			    "CONST_STRING" => "const char *",
+			    "STRING" => "const char *",
+			    "INTEGER" => "int ",
+			    "CHAR" => "char ",
+			    "LIST" => "const char **",
+			    );
+
+		$file->("\t$tmap{$type} (*$name)(void);\n");
+	}
+}
+
+sub process_file($$) 
+{
+	my ($file, $filename) = @_;
+
+	$filename =~ s/\.o$/\.c/g;
+
+	if ($filename =~ /^\//) {
+		open(FH, "<$filename") or die("Failed to open $filename");
+	} elsif (!open(FH, "< $builddir/$filename")) {
+	    open(FH, "< $srcdir/$filename") || die "Failed to open $filename";
+	}
+
+	my $comment = undef;
+	my $incomment = 0;
+	while (my $line = <FH>) {	      
+		if ($line =~ /^\/\*\*/) { 
+			$comment = "";
+			$incomment = 1;
+		}
+
+		if ($incomment) {
+			$comment .= $line;
+			if ($line =~ /\*\//) {
+				$incomment = 0;
+			}
+		} 
+
+		# these are ordered for maximum speed
+		next if ($line =~ /^\s/);
+	      
+		next unless ($line =~ /\(/);
+
+		next if ($line =~ /^\/|[;]/);
+
+		if ($line =~ /^FN_/) {
+			handle_loadparm($file, $line);
+		}
+		next;
+	}
+
+	close(FH);
+}
+
+
+print_header(\&public, $public_define);
+
+process_file(\&public, $_) foreach (@ARGV);
+print_footer(\&public, $public_define);
+
+if (not defined($file)) {
+	print STDOUT $$public_data;
+}
+
+mkpath(dirname($file), 0, 0755);
+open(PUBLIC, ">$file") or die("Can't open `$file': $!"); 
+print PUBLIC "$$public_data";
+close(PUBLIC);
+
diff --git a/source4/scripting/bin/findprovisionusnranges b/source4/scripting/bin/findprovisionusnranges
new file mode 100755
index 00000000000..c91e42e936e
--- /dev/null
+++ b/source4/scripting/bin/findprovisionusnranges
@@ -0,0 +1,174 @@
+#!/usr/bin/python
+#
+# Helper for determining USN ranges created of modified by provision and
+# upgradeprovision.
+# Copyright (C) Matthieu Patou <mat@matws.net> 2009-2011
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+import optparse
+import tempfile
+sys.path.insert(0, "bin/python")
+
+from samba.credentials import DONT_USE_KERBEROS
+from samba.auth import system_session
+from samba import Ldb
+import ldb
+                
+import samba.getopt as options
+from samba import param
+from samba import _glue
+from samba.upgradehelpers import get_paths
+from samba.ndr import ndr_unpack
+from samba.dcerpc import drsblobs, misc
+
+parser = optparse.OptionParser("provision [options]")
+sambaopts = options.SambaOptions(parser)
+parser.add_option_group(sambaopts)
+parser.add_option_group(options.VersionOptions(parser))
+parser.add_option("--storedir", type="string", help="Directory where to store result files")
+credopts = options.CredentialsOptions(parser)
+parser.add_option_group(credopts)
+opts = parser.parse_args()[0]
+lp = sambaopts.get_loadparm()
+smbconf = lp.configfile
+
+creds = credopts.get_credentials(lp)
+creds.set_kerberos_state(DONT_USE_KERBEROS)
+session = system_session()
+paths = get_paths(param, smbconf=smbconf)
+basedn="DC=" + lp.get("realm").replace(".",",DC=")
+samdb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp)
+
+hash_id = {}
+ldif = ""
+nb_obj = 0
+
+res = samdb.search(base="", scope=ldb.SCOPE_BASE, attrs=["dsServiceName"])
+
+invocation = None
+if res and len(res) == 1 and res[0]["dsServiceName"] != None:
+    dn = ldb.Dn(samdb, str(res[0]["dsServiceName"]))
+    res = samdb.search(base=str(dn), scope=ldb.SCOPE_BASE, attrs=["invocationId"],
+                        controls=["search_options:1:2"])
+
+    if res and len(res) == 1 and res[0]["invocationId"]:
+        invocation = str(ndr_unpack(misc.GUID, res[0]["invocationId"][0]))   
+    else:
+        print "Unable to find invocation ID"
+        sys.exit(1)
+else:
+    print "Unable to find attribute dsServiceName in rootDSE"
+    sys.exit(1)
+
+res = samdb.search(base=basedn, expression="objectClass=*",
+                                scope=ldb.SCOPE_SUBTREE,
+                                attrs=["replPropertyMetaData"],
+                                controls=["search_options:1:2"])
+
+for e in res:
+    nb_obj = nb_obj + 1
+    obj = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
+                        str(e["replPropertyMetaData"])).ctr
+
+    for o in obj.array:
+        # like a timestamp but with the resolution of 1 minute
+        minutestamp =_glue.nttime2unix(o.originating_change_time)/60
+        hash_ts = hash_id.get(str(o.originating_invocation_id))
+        if hash_ts == None:
+            ob = {}
+            ob["min"] = o.originating_usn
+            ob["max"] = o.originating_usn
+            ob["num"] = 1
+            ob["list"] = [str(e.dn)]
+            hash_ts = {}
+        else:
+            ob = hash_ts.get(minutestamp)
+            if ob == None:
+                ob = {}
+                ob["min"] = o.originating_usn
+                ob["max"] = o.originating_usn
+                ob["num"] = 1
+                ob["list"] = [str(e.dn)]
+            else:
+                if ob["min"] > o.originating_usn:
+                    ob["min"] = o.originating_usn
+                if ob["max"] < o.originating_usn:
+                    ob["max"] = o.originating_usn
+                if not (str(e.dn) in ob["list"]):
+                    ob["num"] = ob["num"] + 1
+                    ob["list"].append(str(e.dn))
+        hash_ts[minutestamp] = ob
+        hash_id[str(o.originating_invocation_id)] = hash_ts
+
+minobj = 5
+print "Here is a list of changes that modified more than %d objects in 1 minute." % minobj
+print "Usually changes made by provision and upgradeprovision are those who affect a couple"\
+        " of hundred of objects or more"
+print "Total number of objects: %d" % nb_obj
+print 
+
+for id in hash_id:
+    hash_ts = hash_id[id]
+    sorted_keys = []
+    sorted_keys.extend(hash_ts.keys())
+    sorted_keys.sort()
+
+    kept_record = []
+    for k in sorted_keys:
+        obj = hash_ts[k]
+        if obj["num"] > minobj:
+            dt = _glue.nttime2string(_glue.unix2nttime(k*60))
+            print "%s # of modification: %d  \tmin: %d max: %d" % (dt , obj["num"],
+                                                                obj["min"],
+                                                                obj["max"])
+        if hash_ts[k]["num"] > 600:
+            kept_record.append(k)
+
+    # Let's try to concatenate consecutive block if they are in the almost same minutestamp
+    for i in range(0, len(kept_record)):
+        if i != 0:
+            key1 = kept_record[i]
+            key2 = kept_record[i-1]
+            if key1 - key2 == 1:
+                # previous record is just 1 minute away from current
+                if int(hash_ts[key1]["min"]) == int(hash_ts[key2]["max"]) + 1:
+                    # Copy the highest USN in the previous record
+                    # and mark the current as skipped
+                    hash_ts[key2]["max"] = hash_ts[key1]["max"]
+                    hash_ts[key1]["skipped"] = True
+
+    for k in kept_record:
+            obj = hash_ts[k]
+            if obj.get("skipped") == None:
+                ldif = "%slastProvisionUSN: %d-%d;%s\n" % (ldif, obj["min"],
+                            obj["max"], id) 
+
+if ldif != "":
+    dest = opts.storedir
+    if dest == None:
+        dest = "/tmp"
+
+    file = tempfile.mktemp(dir=dest, prefix="usnprov", suffix=".ldif")        
+    print 
+    print "To track the USNs modified/created by provision and upgrade proivsion,"
+    print " the following ranges are proposed to be added to your provision sam.ldb: \n%s" % ldif
+    print "We recommend to review them, and if it's correct to integrate the following ldif: %s in your sam.ldb" % file
+    print "You can load this file like this: ldbadd -H %s %s\n"%(str(paths.samdb),file)
+    ldif = "dn: @PROVISION\nprovisionnerID: %s\n%s" % (invocation, ldif)
+    open(file,'w').write(ldif)
+
diff --git a/source4/scripting/bin/renamedc b/source4/scripting/bin/renamedc
new file mode 100755
index 00000000000..0915b157832
--- /dev/null
+++ b/source4/scripting/bin/renamedc
@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+# vim: expandtab
+#
+# Copyright (C) Matthieu Patou <mat@matws.net> 2011
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import optparse
+import sys
+# Allow to run from s4 source directory (without installing samba)
+sys.path.insert(0, "bin/python")
+
+import ldb
+import samba
+import samba.getopt as options
+import os
+
+from samba.credentials import DONT_USE_KERBEROS
+from samba.auth import system_session
+from samba import param
+from samba.provision import find_provision_key_parameters, secretsdb_self_join
+from samba.upgradehelpers import get_ldbs, get_paths
+
+
+__docformat__ = "restructuredText"
+
+parser = optparse.OptionParser("provision [options]")
+sambaopts = options.SambaOptions(parser)
+parser.add_option_group(sambaopts)
+parser.add_option_group(options.VersionOptions(parser))
+credopts = options.CredentialsOptions(parser)
+parser.add_option_group(credopts)
+parser.add_option("--oldname",
+                  help="Old DC name")
+parser.add_option("--newname",
+                  help="New DC name")
+
+opts = parser.parse_args()[0]
+
+if len(sys.argv) == 1:
+    opts.interactive = True
+lp = sambaopts.get_loadparm()
+smbconf = lp.configfile
+
+creds = credopts.get_credentials(lp)
+creds.set_kerberos_state(DONT_USE_KERBEROS)
+
+
+if __name__ == '__main__':
+    global defSDmodified
+    defSDmodified = False
+   # 1) First get files paths
+    paths = get_paths(param, smbconf=smbconf)
+    # Get ldbs with the system session, it is needed for searching
+    # provision parameters
+    session = system_session()
+
+    ldbs = get_ldbs(paths, creds, session, lp)
+    ldbs.sam.transaction_start()
+    ldbs.secrets.transaction_start()
+
+    if opts.oldname is None or opts.newname is None:
+        raise Exception("Option oldname or newname is missing")
+    res = ldbs.sam.search(expression="(&(name=%s)(serverReferenceBL=*))" % opts.oldname)
+    if res is None or len(res) != 1:
+        raise Exception("Wrong number of result returned, are you sure of the old name %s" %
+                opts.oldname)
+
+    # Ok got it then check that the new name is not used as well
+    res2 = ldbs.sam.search(expression="(&(name=%s)(objectclass=computer))" % opts.newname)
+    if len(res2) != 0:
+        raise Exception("Seems that %s is a name that already exists, pick another one" %
+                opts.newname)
+
+    names = find_provision_key_parameters(ldbs.sam, ldbs.secrets, ldbs.idmap,
+                                            paths, smbconf, lp)
+ 
+    #First rename the entry
+    # provision put the name in upper case so let's do it too !
+    newdn = str(res[0].dn).replace("CN=%s" % opts.oldname, "CN=%s" % opts.newname.upper())
+    dnobj = ldb.Dn(ldbs.sam, newdn)
+    ldbs.sam.rename(res[0].dn, dnobj)
+
+    # Then change password and samaccountname and dnshostname
+    msg = ldb.Message(dnobj)
+    machinepass = samba.generate_random_password(128, 255)
+    mputf16 = machinepass.encode('utf-16-le')
+
+    account = "%s$" % opts.newname.upper()
+    msg["clearTextPassword"] = ldb.MessageElement(mputf16,
+                                            ldb.FLAG_MOD_REPLACE,
+                                            "clearTextPassword")
+
+    msg["sAMAccountName"] = ldb.MessageElement(account,
+                                            ldb.FLAG_MOD_REPLACE,
+                                            "sAMAccountName")
+
+    msg["dNSHostName"] = ldb.MessageElement("%s.%s" % (opts.newname,
+                                            names.dnsdomain),
+                                            ldb.FLAG_MOD_REPLACE,
+                                            "dNSHostName")
+    ldbs.sam.modify(msg)
+
+    # Do a self join one more time to resync the secrets file
+    res = ldbs.sam.search(expression=("dn=%s" % newdn),
+            attrs=["msDs-keyVersionNumber", "serverReferenceBL"])
+    assert(len(res) == 1)
+    kvno = int(str(res[0]["msDs-keyVersionNumber"]))
+    serverbldn = ldb.Dn(ldbs.sam, str(res[0]["serverReferenceBL"]))
+
+    secrets_msg = ldbs.secrets.search(expression="sAMAccountName=%s$" %
+                                            opts.oldname.upper(),
+                                            attrs=["secureChannelType"])
+
+    secChanType = int(secrets_msg[0]["secureChannelType"][0])
+
+    secretsdb_self_join(ldbs.secrets, domain=names.domain,
+                realm=names.realm,
+                domainsid=names.domainsid,
+                dnsdomain=names.dnsdomain,
+                netbiosname=opts.newname.upper(),
+                machinepass=machinepass,
+                key_version_number=kvno,
+                secure_channel_type=secChanType)
+
+    # Update RID set reference as there is no back link for the moment.
+
+    res = ldbs.sam.search(expression="(objectClass=rIDSet)", base=newdn, attrs=[])
+    assert(len(res) == 1)
+    newridset = str(res[0].dn)
+    msg = ldb.Message(dnobj)
+
+    msg["rIDSetReferences"] = ldb.MessageElement(newridset,
+                                            ldb.FLAG_MOD_REPLACE,
+                                            "rIDSetReferences")
+    ldbs.sam.modify(msg)
+
+    # Update the server's sites configuration
+    if False:
+        # Desactivated for the moment we have a couple of issues with site 
+        # renaming first one is that it's currently forbidden
+        # second one is that a lot of links are not backlinked
+        # and so won't be updated when the DN change (ie. fmsowner ...)
+        serverbl = str(serverbldn)
+        dnparts = serverbl.split(",")
+        dnparts[0] = "CN=%s" % opts.newname.upper()
+        newserverref = ",".join(dnparts)
+
+        newserverrefdn = ldb.Dn(ldbs.sam, newserverref)
+
+        ldbs.sam.rename(serverbldn, newserverrefdn)
+
+        msg = ldb.Message(newserverrefdn)
+        msg["dNSHostName"] = ldb.MessageElement("%s.%s" % (opts.newname,
+                                                names.dnsdomain),
+                                                ldb.FLAG_MOD_REPLACE,
+                                                "dNSHostName")
+        ldbs.sam.modify(msg)
+
+    try:
+        ldbs.sam.transaction_prepare_commit()
+        ldbs.secrets.transaction_prepare_commit()
+    except Exception:
+        ldbs.sam.rollback()
+        ldbs.secrets.rollback()
+        sys.exit(1)
+
+    try:
+        ldbs.sam.transaction_commit()
+        ldbs.secrets.transaction_commit()
+    except Exception:
+        ldbs.sam.rollback()
+        ldbs.secrets.rollback()
+
+    # All good so far
+    #print lp.get("private dir")
+    cf = open(lp.configfile)
+    ncfname = "%s.new" % lp.configfile
+    newconf = open(ncfname, 'w')
+    for l in cf.readlines():
+        if l.find("netbios name") > 0:
+             newconf.write("\tnetbios name = %s\n" % opts.newname.upper())
+        else:
+            newconf.write(l)
+    newconf.close()
+    cf.close()
+    os.rename(ncfname, lp.configfile)
+
diff --git a/source4/scripting/bin/samba_dnsupdate b/source4/scripting/bin/samba_dnsupdate
index e86fba29833..78d7dc1712c 100755
--- a/source4/scripting/bin/samba_dnsupdate
+++ b/source4/scripting/bin/samba_dnsupdate
@@ -89,6 +89,17 @@ if len(IPs) == 0:
     print "No IP interfaces - skipping DNS updates"
     sys.exit(0)
 
+IP6s = []
+IP4s = []
+for i in IPs:
+    if i.find(':') != -1:
+        if i.find('%') == -1:
+            # we don't want link local addresses for DNS updates
+            IP6s.append(i)
+    else:
+        IP4s.append(i)
+
+
 if opts.verbose:
     print "IPs: %s" % IPs
 
@@ -122,7 +133,7 @@ class dnsobj(object):
         if self.type == 'SRV':
             self.dest = list[2].lower()
             self.port = list[3]
-        elif self.type == 'A':
+        elif self.type in ['A', 'AAAA']:
             self.ip   = list[2] # usually $IP, which gets replaced
         elif self.type == 'CNAME':
             self.dest = list[2].lower()
@@ -132,6 +143,7 @@ class dnsobj(object):
 
     def __str__(self):
         if d.type == "A":     return "%s %s %s" % (self.type, self.name, self.ip)
+        if d.type == "AAAA":  return "%s %s %s" % (self.type, self.name, self.ip)
         if d.type == "SRV":   return "%s %s %s %s" % (self.type, self.name, self.dest, self.port)
         if d.type == "CNAME": return "%s %s %s" % (self.type, self.name, self.dest)
 
@@ -178,7 +190,7 @@ def check_dns_name(d):
         if opts.verbose:
             print "Failed to find DNS entry %s" % d
         return False
-    if d.type == 'A':
+    if d.type in ['A', 'AAAA']:
         # we need to be sure that our IP is there
         for rdata in ans:
             if str(rdata) == str(d.ip):
@@ -210,7 +222,7 @@ def get_subst_vars():
     global lp, am_rodc
     vars = {}
 
-    samdb = SamDB(url=lp.get("sam database"), session_info=system_session(),
+    samdb = SamDB(url=lp.samdb_url(), session_info=system_session(),
                   lp=lp)
 
     vars['DNSDOMAIN'] = lp.get('realm').lower()
@@ -247,6 +259,8 @@ def call_nsupdate(d):
     f = os.fdopen(tmp_fd, 'w')
     if d.type == "A":
         f.write("update add %s %u A %s\n" % (normalised_name, default_ttl, d.ip))
+    if d.type == "AAAA":
+        f.write("update add %s %u AAAA %s\n" % (normalised_name, default_ttl, d.ip))
     if d.type == "SRV":
         if d.existing_port is not None:
             f.write("update delete %s SRV 0 %s %s %s\n" % (normalised_name, d.existing_weight,
@@ -264,7 +278,7 @@ def call_nsupdate(d):
     try:
         cmd = nsupdate_cmd[:]
         cmd.append(tmpfile)
-        ret = subprocess.call(cmd, shell=False)
+        ret = subprocess.call(cmd, shell=False, env={"KRB5CCNAME": ccachename})
         if ret != 0:
             if opts.fail_immediately:
                 sys.exit(1)
@@ -382,16 +396,28 @@ for line in file:
     if line == '' or line[0] == "#":
         continue
     d = parse_dns_line(line, sub_vars)
+    if d.type == 'A' and len(IP4s) == 0:
+        continue
+    if d.type == 'AAAA' and len(IP6s) == 0:
+        continue
     dns_list.append(d)
 
 # now expand the entries, if any are A record with ip set to $IP
 # then replace with multiple entries, one for each interface IP
 for d in dns_list:
-    if d.type == 'A' and d.ip == "$IP":
-        d.ip = IPs[0]
-        for i in range(len(IPs)-1):
+    if d.ip != "$IP":
+        continue
+    if d.type == 'A':
+        d.ip = IP4s[0]
+        for i in range(len(IP4s)-1):
+            d2 = dnsobj(str(d))
+            d2.ip = IP4s[i+1]
+            dns_list.append(d2)
+    if d.type == 'AAAA':
+        d.ip = IP6s[0]
+        for i in range(len(IP6s)-1):
             d2 = dnsobj(str(d))
-            d2.ip = IPs[i+1]
+            d2.ip = IP6s[i+1]
             dns_list.append(d2)
 
 # now check if the entries already exist on the DNS server
@@ -412,7 +438,7 @@ for d in update_list:
     if am_rodc:
         if d.name.lower() == domain.lower():
             continue
-        if d.type != 'A':
+        if not d.type in [ 'A', 'AAAA' ]:
             call_rodc_update(d)
         else:
             call_nsupdate(d)
diff --git a/source4/scripting/bin/samba_spnupdate b/source4/scripting/bin/samba_spnupdate
index 1794f2bd267..9f8f4073d31 100755
--- a/source4/scripting/bin/samba_spnupdate
+++ b/source4/scripting/bin/samba_spnupdate
@@ -82,7 +82,7 @@ def get_subst_vars(samdb):
 
 try:
     private_dir = lp.get("private dir")
-    secrets_path = os.path.join(private_dir, lp.get("secrets database"))
+    secrets_path = os.path.join(private_dir, "secrets.ldb")
 
     secrets_db = Ldb(url=secrets_path, session_info=system_session(),
                      credentials=creds, lp=lp)
@@ -103,9 +103,9 @@ try:
     else:
         credentials = None
 
-    samdb = SamDB(url=lp.get("sam database"), session_info=system_session(), credentials=credentials, lp=lp)
+    samdb = SamDB(url=lp.samdb_url(), session_info=system_session(), credentials=credentials, lp=lp)
 except ldb.LdbError, (num, msg):
-    print("Unable to open sam database %s : %s" % (lp.get("sam database"), msg))
+    print("Unable to open sam database %s : %s" % (lp.samdb_url(), msg))
     sys.exit(1)
 
 
diff --git a/source4/scripting/bin/setup_dns.sh b/source4/scripting/bin/setup_dns.sh
index de4485fc07a..bc2ae96b845 100755
--- a/source4/scripting/bin/setup_dns.sh
+++ b/source4/scripting/bin/setup_dns.sh
@@ -13,7 +13,7 @@ IP="$3"
 RSUFFIX=$(echo $DOMAIN | sed s/[\.]/,DC=/g)
 
 [ -z "$PRIVATEDIR" ] && {
-    PRIVATEDIR=$(bin/testparm --section-name=global --parameter-name='private dir' --suppress-prompt 2> /dev/null)
+    PRIVATEDIR=$(bin/samba-tool testparm --section-name=global --parameter-name='private dir' --suppress-prompt 2> /dev/null)
 }
 
 OBJECTGUID=$(bin/ldbsearch -s base -H "$PRIVATEDIR/sam.ldb" -b "CN=NTDS Settings,CN=$HOSTNAME,CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=$RSUFFIX" objectguid|grep ^objectGUID| cut -d: -f2)
diff --git a/source4/scripting/bin/testparm b/source4/scripting/bin/testparm
deleted file mode 100755
index c30e46148cc..00000000000
--- a/source4/scripting/bin/testparm
+++ /dev/null
@@ -1,219 +0,0 @@
-#!/usr/bin/env python
-# vim: expandtab ft=python
-#
-#   Unix SMB/CIFS implementation.
-#   Test validity of smb.conf
-#   Copyright (C) Karl Auer 1993, 1994-1998
-#
-#   Extensively modified by Andrew Tridgell, 1995
-#   Converted to popt by Jelmer Vernooij (jelmer@nl.linux.org), 2002
-#   Updated for Samba4 by Andrew Bartlett <abartlet@samba.org> 2006
-#   Converted to Python by Jelmer Vernooij <jelmer@samba.org> 2010
-#   
-#   This program is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published by
-#   the Free Software Foundation; either version 3 of the License, or
-#   (at your option) any later version.
-#   
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU General Public License for more details.
-#   
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Testbed for loadparm.c/params.c
-#
-# This module simply loads a specified configuration file and
-# if successful, dumps it's contents to stdout. Note that the
-# operation is performed with DEBUGLEVEL at 3.
-#
-# Useful for a quick 'syntax check' of a configuration file.
-#
-
-import logging
-import optparse
-import os
-import sys
-
-# Find right directory when running from source tree
-sys.path.insert(0, "bin/python")
-
-import samba
-from samba import getopt as options
-
-# Here we do a set of 'hard coded' checks for bad
-# configuration settings.
-
-def do_global_checks(lp, logger):
-    valid = True
-
-    netbios_name = lp.get("netbios name")
-    if not samba.valid_netbios_name(netbios_name):
-        logger.error("netbios name %s is not a valid netbios name", 
-                     netbios_name)
-        valid = False
-
-    workgroup = lp.get("workgroup")
-    if not samba.valid_netbios_name(workgroup):
-        logger.error("workgroup name %s is not a valid netbios name", 
-                     workgroup)
-        valid = False
-
-    lockdir = lp.get("lockdir")
-
-    if not os.path.isdir(lockdir):
-        logger.error("lock directory %s does not exist", lockdir)
-        valid = False
-
-    piddir = lp.get("pid directory")
-
-    if not os.path.isdir(piddir):
-        logger.error("pid directory %s does not exist", piddir)
-        valid = False
-
-    winbind_separator = lp.get("winbind separator")
-
-    if len(winbind_separator) != 1:
-        logger.error("the 'winbind separator' parameter must be a single "
-                     "character.")
-        valid = False
-
-    if winbind_separator == '+':
-        logger.error("'winbind separator = +' might cause problems with group "
-                     "membership.")
-        valid = False
-
-    return valid
-
-
-def allow_access(deny_list, allow_list, cname, caddr):
-    raise NotImplementedError(allow_access)
-
-
-def do_share_checks(lp, logger):
-    valid = True
-    for s in lp.services():
-        if len(s) > 12:
-            logger.warning("You have some share names that are longer than 12 "
-                "characters. These may not be accessible to some older "
-                "clients. (Eg. Windows9x, WindowsMe, and not listed in "
-                "smbclient in Samba 3.0.)")
-            break
-
-    for s in lp.services():
-        deny_list = lp.get("hosts deny", s)
-        allow_list = lp.get("hosts allow", s)
-        if deny_list:
-            for entry in deny_list:
-                if "*" in entry or "?" in entry:
-                    logger.error("Invalid character (* or ?) in hosts deny "
-                                 "list (%s) for service %s.", entry, s)
-                    valid = False
-
-        if allow_list:
-            for entry in allow_list:
-                if "*" in entry or "?" in entry:
-                    logger.error("Invalid character (* or ?) in hosts allow "
-                                 "list (%s) for service %s.", entry, s)
-                    valid = False
-    return valid
-
-def check_client_access(lp, cname, caddr):
-    # this is totally ugly, a real `quick' hack
-    for s in lp.services():
-        if (allow_access(lp.get("hosts deny"), lp.get("hosts allow"), cname,
-                         caddr) and
-            allow_access(lp.get("hosts deny", s), lp.get("hosts allow", s),
-                         cname, caddr)):
-            logger.info("Allow connection from %s (%s) to %s", cname, caddr, s)
-        else:
-            logger.info("Deny connection from %s (%s) to %s", cname, caddr, s)
-
-
-if __name__ == '__main__':
-    parser = optparse.OptionParser("testparm [OPTION...] [host-name] [host-ip]")
-    parser.add_option("--section-name", type="string", metavar="SECTION",
-            help="Limit testparm to a named section")
-    parser.add_option("--parameter-name", type="string", metavar="PARAMETER",
-            help="Limit testparm to a named parameter")
-    parser.add_option("--client-name", type="string", metavar="HOSTNAME",
-            help="Client DNS name for 'hosts allow' checking "
-                 "(should match reverse lookup)")
-    parser.add_option("--client-ip", type="string", metavar="IP",
-            help="Client IP address for 'hosts allow' checking")
-    parser.add_option("--suppress-prompt", action="store_true", default=False,
-            help="Suppress prompt for enter")
-    parser.add_option("-v", "--verbose", action="store_true", 
-            default=False, help="Show default options too")
-    parser.add_option_group(options.VersionOptions(parser))
-    # We need support for smb.conf macros before this will work again 
-    parser.add_option("--server", type="string",
-            help="Set %%L macro to servername")
-    # These are harder to do with the new code structure
-    parser.add_option("--show-all-parameters", action="store_true",
-            default=False, help="Show the parameters, type, possible values")
-
-    sambaopts = options.SambaOptions(parser)
-    parser.add_option_group(sambaopts)
-
-    opts, args = parser.parse_args()
-
-# 
-#    if (show_all_parameters) {
-#        show_parameter_list()
-#        exit(0)
-#    }
-
-    if len(args) > 0:
-        cname = args[0]
-    else:
-        cname = None
-    if len(args) > 1:
-        caddr = args[1]
-    else:
-        caddr = None
-
-    if cname is not None and caddr is None:
-        print "Both a DNS name and an IP address are required for the host " \
-              "access check."
-        sys.exit(1)
-
-#   FIXME: We need support for smb.conf macros before this will work again 
-#
-#    if (new_local_machine) {
-#        set_local_machine_name(new_local_machine, True)
-#    }
-
-    lp = sambaopts.get_loadparm()
-
-    # We need this to force the output
-    samba.set_debug_level(2)
-
-    logger = logging.getLogger("testparm")
-    logger.addHandler(logging.StreamHandler(sys.stdout))
-
-    logger.info("Loaded smb config files from %s", lp.configfile)
-    logger.info("Loaded services file OK.")
-
-    valid = do_global_checks(lp, logger)
-    valid = valid and do_share_checks(lp, logger)
-    if cname is not None:
-        check_client_access(lp, cname, caddr)
-    else:
-        if opts.section_name is not None or opts.parameter_name is not None:
-            if opts.parameter_name is None:
-                lp[opts.section_name].dump(sys.stdout, lp.default_service,
-                                           opts.verbose)
-            else:
-                print lp.get(opts.parameter_name, opts.section_name)
-        else:
-            if not opts.suppress_prompt:
-                print "Press enter to see a dump of your service definitions\n"
-                sys.stdin.readline()
-            lp.dump(sys.stdout, opts.verbose)
-    if valid:
-        sys.exit(0)
-    else:
-        sys.exit(1)
diff --git a/source4/scripting/bin/upgradeprovision b/source4/scripting/bin/upgradeprovision
index 8c79917d5e2..e98b6427762 100755
--- a/source4/scripting/bin/upgradeprovision
+++ b/source4/scripting/bin/upgradeprovision
@@ -42,9 +42,10 @@ from samba.credentials import DONT_USE_KERBEROS
 from samba.auth import system_session, admin_session
 from ldb import (SCOPE_SUBTREE, SCOPE_BASE,
                 FLAG_MOD_REPLACE, FLAG_MOD_ADD, FLAG_MOD_DELETE,
-                MessageElement, Message, Dn)
+                MessageElement, Message, Dn, LdbError)
 from samba import param, dsdb, Ldb
-from samba.provision import (get_domain_descriptor,
+from samba.common import confirm
+from samba.provision import (get_domain_descriptor, find_provision_key_parameters,
                             get_config_descriptor,
                             ProvisioningError, get_last_provision_usn,
                             get_max_usn, update_provision_usn, setup_path)
@@ -52,7 +53,7 @@ from samba.schema import get_linked_attributes, Schema, get_schema_descriptor
 from samba.dcerpc import security, drsblobs, xattr
 from samba.ndr import ndr_unpack
 from samba.upgradehelpers import (dn_sort, get_paths, newprovision,
-                                 find_provision_key_parameters, get_ldbs,
+                                 get_ldbs,
                                  usn_in_range, identic_rename, get_diff_sddls,
                                  update_secrets, CHANGE, ERROR, SIMPLE,
                                  CHANGEALL, GUESS, CHANGESD, PROVISION,
@@ -81,20 +82,25 @@ __docformat__ = "restructuredText"
 # This is most probably because they are populated automatcally when object is
 # created
 # This also apply to imported object from reference provision
-hashAttrNotCopied = {   "dn": 1, "whenCreated": 1, "whenChanged": 1,
-                        "objectGUID": 1, "uSNCreated": 1,
-                        "replPropertyMetaData": 1, "uSNChanged": 1,
-                        "parentGUID": 1, "objectCategory": 1,
-                        "distinguishedName": 1, "nTMixedDomain": 1,
-                        "showInAdvancedViewOnly": 1, "instanceType": 1,
-                        "msDS-Behavior-Version":1, "nextRid":1, "cn": 1,
-                        "lmPwdHistory":1, "pwdLastSet": 1,
-                        "ntPwdHistory":1, "unicodePwd":1,"dBCSPwd":1,
-                        "supplementalCredentials":1, "gPCUserExtensionNames":1,
-                        "gPCMachineExtensionNames":1,"maxPwdAge":1, "secret":1,
-                        "possibleInferiors":1, "privilege":1,
-                        "sAMAccountType":1 }
+replAttrNotCopied = [   "dn", "whenCreated", "whenChanged", "objectGUID",
+                        "parentGUID", "objectCategory", "distinguishedName",
+                        "nTMixedDomain", "showInAdvancedViewOnly",
+                        "instanceType", "msDS-Behavior-Version", "cn",
+                        "lmPwdHistory", "pwdLastSet", "ntPwdHistory",
+                        "unicodePwd", "dBCSPwd", "supplementalCredentials",
+                        "gPCUserExtensionNames", "gPCMachineExtensionNames",
+                        "maxPwdAge", "secret", "possibleInferiors", "privilege",
+                        "sAMAccountType", "oEMInformation", "creationTime" ]
 
+nonreplAttrNotCopied = ["uSNCreated", "replPropertyMetaData", "uSNChanged",
+                        "nextRid" ,"rIDNextRID", "rIDPreviousAllocationPool"]
+
+nonDSDBAttrNotCopied = ["msDS-KeyVersionNumber", "priorSecret", "priorWhenChanged"]
+
+
+attrNotCopied = replAttrNotCopied
+attrNotCopied.extend(nonreplAttrNotCopied)
+attrNotCopied.extend(nonDSDBAttrNotCopied)
 # Usually for an object that already exists we do not overwrite attributes as
 # they might have been changed for good reasons. Anyway for a few of them it's
 # mandatory to replace them otherwise the provision will be broken somehow.
@@ -108,15 +114,19 @@ hashOverwrittenAtt = {  "prefixMap": replace, "systemMayContain": replace,
                         "wellKnownObjects":replace, "privilege":never,
                         "defaultSecurityDescriptor": replace,
                         "rIDAvailablePool": never,
+                        "versionNumber" : add,
                         "rIDNextRID": add, "rIDUsedPool": never,
                         "defaultSecurityDescriptor": replace + add,
                         "isMemberOfPartialAttributeSet": delete,
                         "attributeDisplayNames": replace + add,
                         "versionNumber": add}
 
+dnNotToRecalculate = []
+dnToRecalculate = []
 backlinked = []
 forwardlinked = set()
 dn_syntax_att = []
+not_replicated = []
 def define_what_to_log(opts):
     what = 0
     if opts.debugchange:
@@ -151,6 +161,8 @@ parser.add_option("--debugall", action="store_true",
                   help="Print all available information (very verbose)")
 parser.add_option("--resetfileacl", action="store_true",
                   help="Force a reset on filesystem acls in sysvol / netlogon share")
+parser.add_option("--fixntacl", action="store_true",
+                  help="Only fix NT ACLs in sysvol / netlogon share")
 parser.add_option("--full", action="store_true",
                   help="Perform full upgrade of the samdb (schema, configuration, new objects, ...")
 
@@ -236,6 +248,26 @@ def populate_links(samdb, schemadn):
     for t in linkedAttHash.keys():
         forwardlinked.add(t)
 
+def isReplicated(att):
+    """ Indicate if the attribute is replicated or not
+
+    :param att: Name of the attribute to be tested
+    :return: True is the attribute is replicated, False otherwise
+    """
+
+    return (att not in not_replicated)
+
+def populateNotReplicated(samdb, schemadn):
+    """Populate an array with all the attributes that are not replicated
+
+    :param samdb: A LDB object for sam.ldb file
+    :param schemadn: DN of the schema for the partition"""
+    res = samdb.search(expression="(&(objectclass=attributeSchema)(systemflags:1.2.840.113556.1.4.803:=1))", base=Dn(samdb,
+                        str(schemadn)), scope=SCOPE_SUBTREE,
+                        attrs=["lDAPDisplayName"])
+    for elem in res:
+        not_replicated.append(str(elem["lDAPDisplayName"]))
+
 
 def populate_dnsyntax(samdb, schemadn):
     """Populate an array with all the attributes that have DN synthax
@@ -295,7 +327,7 @@ def print_provision_key_parameters(names):
     message(GUESS, "domainlevel :" + str(names.domainlevel))
 
 
-def handle_special_case(att, delta, new, old, usn, basedn, aldb):
+def handle_special_case(att, delta, new, old, useReplMetadata, basedn, aldb):
     """Define more complicate update rules for some attributes
 
     :param att: The attribute to be updated
@@ -303,7 +335,8 @@ def handle_special_case(att, delta, new, old, usn, basedn, aldb):
                   between the updated object and the reference one
     :param new: The reference object
     :param old: The Updated object
-    :param usn: The highest usn modified by a previous (upgrade)provision
+    :param useReplMetadata: A boolean that indicate if the update process
+                use replPropertyMetaData to decide what has to be updated.
     :param basedn: The base DN of the provision
     :param aldb: An ldb object used to build DN
     :return: True to indicate that the attribute should be kept, False for
@@ -313,7 +346,7 @@ def handle_special_case(att, delta, new, old, usn, basedn, aldb):
     # We do most of the special case handle if we do not have the
     # highest usn as otherwise the replPropertyMetaData will guide us more
     # correctly
-    if usn is None:
+    if not useReplMetadata:
         if (att == "sPNMappings" and flag == FLAG_MOD_REPLACE and
             ldb.Dn(aldb, "CN=Directory Service,CN=Windows NT,"
                         "CN=Services,CN=Configuration,%s" % basedn)
@@ -389,14 +422,14 @@ def handle_special_case(att, delta, new, old, usn, basedn, aldb):
     if (att == "servicePrincipalName" and flag == FLAG_MOD_REPLACE):
         hash = {}
         newval = []
-        changeDelta=0
+        changeDelta = 0
         for elem in old[0][att]:
             hash[str(elem)]=1
             newval.append(str(elem))
 
         for elem in new[0][att]:
             if not hash.has_key(str(elem)):
-                changeDelta=1
+                changeDelta = 1
                 newval.append(str(elem))
         if changeDelta == 1:
             delta[att] = MessageElement(newval, FLAG_MOD_REPLACE, att)
@@ -583,7 +616,7 @@ def add_missing_object(ref_samdb, samdb, dn, names, basedn, hash, index):
             m = re.match(r".*-(\d+)$", sid)
             if m and int(m.group(1))>999:
                 delta.remove("objectSid")
-        for att in hashAttrNotCopied.keys():
+        for att in attrNotCopied:
             delta.remove(att)
         for att in backlinked:
             delta.remove(att)
@@ -650,7 +683,7 @@ def add_deletedobj_containers(ref_samdb, samdb, names):
             delta = samdb.msg_diff(empty, reference[0])
 
             delta.dn = Dn(samdb, str(reference[0]["dn"]))
-            for att in hashAttrNotCopied.keys():
+            for att in attrNotCopied:
                 delta.remove(att)
 
             modcontrols = ["relax:0", "provision:0"]
@@ -771,8 +804,192 @@ msg_elt_flag_strs = {
     ldb.FLAG_MOD_REPLACE: "MOD_REPLACE",
     ldb.FLAG_MOD_DELETE: "MOD_DELETE" }
 
+def checkKeepAttributeOldMtd(delta, att, reference, current,
+                                    basedn, samdb):
+    """ Check if we should keep the attribute modification or not.
+        This function didn't use replicationMetadata to take a decision.
+
+        :param delta: A message diff object
+        :param att: An attribute
+        :param reference: A message object for the current entry comming from
+                            the reference provision.
+        :param current: A message object for the current entry commin from
+                            the current provision.
+        :param basedn: The DN of the partition
+        :param samdb: A ldb connection to the sam database of the current provision.
+
+        :return: The modified message diff.
+    """
+    # Old school way of handling things for pre alpha12 upgrade
+    global defSDmodified
+    isFirst = False
+    txt = ""
+    dn = current[0].dn
+
+    for att in list(delta):
+        msgElt = delta.get(att)
+
+        if att == "nTSecurityDescriptor":
+            defSDmodified = True
+            delta.remove(att)
+            continue
+
+        if att == "dn":
+            continue
+
+        if not hashOverwrittenAtt.has_key(att):
+            if msgElt.flags() != FLAG_MOD_ADD:
+                if not handle_special_case(att, delta, reference, current,
+                                            False, basedn, samdb):
+                    if opts.debugchange or opts.debugall:
+                        try:
+                            dump_denied_change(dn, att,
+                                msg_elt_flag_strs[msgElt.flags()],
+                                current[0][att], reference[0][att])
+                        except KeyError:
+                            dump_denied_change(dn, att,
+                                msg_elt_flag_strs[msgElt.flags()],
+                                current[0][att], None)
+                    delta.remove(att)
+                continue
+        else:
+            if hashOverwrittenAtt.get(att)&2**msgElt.flags() :
+                continue
+            elif  hashOverwrittenAtt.get(att)==never:
+                delta.remove(att)
+                continue
+
+    return delta
+
+def checkKeepAttributeWithMetadata(delta, att, message, reference, current,
+                                    hash_attr_usn, basedn, usns, samdb):
+    """ Check if we should keep the attribute modification or not
+
+        :param delta: A message diff object
+        :param att: An attribute
+        :param message: A function to print messages
+        :param reference: A message object for the current entry comming from
+                            the reference provision.
+        :param current: A message object for the current entry commin from
+                            the current provision.
+        :param hash_attr_usn: A dictionnary with attribute name as keys,
+                                USN and invocation id as values.
+        :param basedn: The DN of the partition
+        :param usns: A dictionnary with invocation ID as keys and USN ranges
+                     as values.
+        :param samdb: A ldb object pointing to the sam DB
+
+        :return: The modified message diff.
+    """
+    global defSDmodified
+    isFirst = True
+    txt = ""
+    dn = current[0].dn
+
+    for att in list(delta):
+        if att in ["dn", "objectSid"]:
+            delta.remove(att)
+            continue
+
+        # We have updated by provision usn information so let's exploit
+        # replMetadataProperties
+        if att in forwardlinked:
+            curval = current[0].get(att, ())
+            refval = reference[0].get(att, ())
+            handle_links(samdb, att, basedn, current[0]["dn"],
+                            curval, refval, delta)
+            continue
+
+        if isFirst and len(delta.items())>1:
+            isFirst = False
+            txt = "%s\n" % (str(dn))
+
+        if handle_special_case(att, delta, reference, current, True, None, None):
+            # This attribute is "complicated" to handle and handling
+            # was done in handle_special_case
+            continue
+
+        attrUSN = None
+        if hash_attr_usn.get(att):
+            [attrUSN, attInvId] = hash_attr_usn.get(att)
+
+        if attrUSN is None:
+            # If it's a replicated attribute and we don't have any USN
+            # information about it. It means that we never saw it before
+            # so let's add it !
+            # If it is a replicated attribute but we are not master on it
+            # (ie. not initially added in the provision we masterize).
+            # attrUSN will be -1
+            if isReplicated(att):
+                continue
+            else:
+                message(CHANGE, "Non replicated attribute %s changed" % att)
+                continue
 
-def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
+        if att == "nTSecurityDescriptor":
+            cursd = ndr_unpack(security.descriptor,
+                str(current[0]["nTSecurityDescriptor"]))
+            cursddl = cursd.as_sddl(names.domainsid)
+            refsd = ndr_unpack(security.descriptor,
+                str(reference[0]["nTSecurityDescriptor"]))
+            refsddl = refsd.as_sddl(names.domainsid)
+
+            diff = get_diff_sddls(refsddl, cursddl)
+            if diff == "":
+                # FIXME find a way to have it only with huge huge verbose mode
+                # message(CHANGE, "%ssd are identical" % txt)
+                # txt = ""
+                delta.remove(att)
+                continue
+            else:
+                delta.remove(att)
+                message(CHANGESD, "%ssd are not identical:\n%s" % (txt, diff))
+                txt = ""
+                if attrUSN == -1:
+                    message(CHANGESD, "But the SD has been changed by someonelse "\
+                                    "so it's impossible to know if the difference"\
+                                    " cames from the modification or from a previous bug")
+                    dnNotToRecalculate.append(str(dn))
+                else:
+                    dnToRecalculate.append(str(dn))
+                continue
+
+        if attrUSN == -1:
+            # This attribute was last modified by another DC forget
+            # about it
+            message(CHANGE, "%sAttribute: %s has been "
+                    "created/modified/deleted by another DC. "
+                    "Doing nothing" % (txt, att))
+            txt = ""
+            delta.remove(att)
+            continue
+        elif not usn_in_range(int(attrUSN), usns.get(attInvId)):
+            message(CHANGE, "%sAttribute: %s was not "
+                            "created/modified/deleted during a "
+                            "provision or upgradeprovision. Current "
+                            "usn: %d. Doing nothing" % (txt, att,
+                                                        attrUSN))
+            txt = ""
+            delta.remove(att)
+            continue
+        else:
+            if att == "defaultSecurityDescriptor":
+                defSDmodified = True
+            if attrUSN:
+                message(CHANGE, "%sAttribute: %s will be modified"
+                                "/deleted it was last modified "
+                                "during a provision. Current usn: "
+                                "%d" % (txt, att, attrUSN))
+                txt = ""
+            else:
+                message(CHANGE, "%sAttribute: %s will be added because "
+                                "it did not exist before" % (txt, att))
+                txt = ""
+            continue
+
+    return delta
+
+def update_present(ref_samdb, samdb, basedn, listPresent, usns):
     """ This function updates the object that are already present in the
         provision
 
@@ -782,10 +999,9 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
                    (ie. DC=foo, DC=bar)
     :param listPresent: A list of object that is present in the provision
     :param usns: A list of USN range modified by previous provision and
-                 upgradeprovision
-    :param invocationid: The value of the invocationid for the current DC"""
+                 upgradeprovision grouped by invocation ID
+    """
 
-    global defSDmodified
     # This hash is meant to speedup lookup of attribute name from an oid,
     # it's for the replPropertyMetaData handling
     hash_oid_name = {}
@@ -801,7 +1017,9 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
         raise ProvisioningError(msg)
 
     changed = 0
-    controls = ["search_options:1:2", "sd_flags:1:2"]
+    controls = ["search_options:1:2", "sd_flags:1:0"]
+    if usns is not None:
+            message(CHANGE, "Using replPropertyMetadata for change selection")
     for dn in listPresent:
         reference = ref_samdb.search(expression="dn=%s" % (str(dn)), base=basedn,
                                         scope=SCOPE_SUBTREE,
@@ -823,14 +1041,17 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
 
         delta = samdb.msg_diff(current[0], reference[0])
 
-        for att in hashAttrNotCopied.keys():
+        for att in backlinked:
             delta.remove(att)
 
-        for att in backlinked:
+        for att in attrNotCopied:
             delta.remove(att)
 
         delta.remove("name")
 
+        if len(delta.items()) == 1:
+            continue
+
         if len(delta.items()) > 1 and usns is not None:
             # Fetch the replPropertyMetaData
             res = samdb.search(expression="dn=%s" % (str(dn)), base=basedn,
@@ -844,145 +1065,25 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
                 # We put in this hash only modification
                 # made on the current host
                 att = hash_oid_name[samdb.get_oid_from_attid(o.attid)]
-                if str(o.originating_invocation_id) == str(invocationid):
-                # Note we could just use 1 here
-                    hash_attr_usn[att] = o.originating_usn
+                if str(o.originating_invocation_id) in usns.keys():
+                    hash_attr_usn[att] = [o.originating_usn, str(o.originating_invocation_id)]
                 else:
-                    hash_attr_usn[att] = -1
-
-        isFirst = 0
-        txt = ""
-
-        for att in list(delta):
-            if usns is not None:
-                # We have updated by provision usn information so let's exploit
-                # replMetadataProperties
-                if att in forwardlinked:
-                    curval = current[0].get(att, ())
-                    refval = reference[0].get(att, ())
-                    handle_links(samdb, att, basedn, current[0]["dn"],
-                                    curval, refval, delta)
-                    continue
-
-                if isFirst == 0 and len(delta.items())>1:
-                    isFirst = 1
-                    txt = "%s\n" % (str(dn))
-                if att == "dn":
-                    # There is always a dn attribute after a msg_diff
-                    continue
-                if att == "rIDAvailablePool":
-                    delta.remove(att)
-                    continue
-                if att == "objectSid":
-                    delta.remove(att)
-                    continue
-                if att == "creationTime":
-                    delta.remove(att)
-                    continue
-                if att == "oEMInformation":
-                    delta.remove(att)
-                    continue
-                if att == "msDs-KeyVersionNumber":
-                # This is the kvno of the computer/user it's a very bad
-                # idea to change it
-                    delta.remove(att)
-                    continue
-                if handle_special_case(att, delta, reference, current, usns, basedn, samdb):
-                    # This attribute is "complicated" to handle and handling
-                    # was done in handle_special_case
-                    continue
-                attrUSN = hash_attr_usn.get(att)
-                if att == "forceLogoff" and attrUSN is None:
-                    continue
-                if  attrUSN is None:
-                    delta.remove(att)
-                    continue
-                if att == "nTSecurityDescriptor":
-                    cursd = ndr_unpack(security.descriptor,
-                        str(current[0]["nTSecurityDescriptor"]))
-                    cursddl = cursd.as_sddl(names.domainsid)
-                    refsd = ndr_unpack(security.descriptor,
-                        str(reference[0]["nTSecurityDescriptor"]))
-                    refsddl = cursd.as_sddl(names.domainsid)
-
-                    if get_diff_sddls(refsddl, cursddl) == "":
-                       message(CHANGE, "sd are identical")
-                    else:
-                       message(CHANGE, "sd are not identical")
-                if attrUSN == -1:
-                    # This attribute was last modified by another DC forget
-                    # about it
-                    message(CHANGE, "%sAttribute: %s has been "
-                            "created/modified/deleted by another DC. "
-                            "Doing nothing" % (txt, att))
-                    txt = ""
-                    delta.remove(att)
-                    continue
-                elif not usn_in_range(int(attrUSN), usns):
-                    message(CHANGE, "%sAttribute: %s was not "
-                                    "created/modified/deleted during a "
-                                    "provision or upgradeprovision. Current "
-                                    "usn: %d. Doing nothing" % (txt, att,
-                                                                attrUSN))
-                    txt = ""
-                    delta.remove(att)
-                    continue
-                else:
-                    if att == "defaultSecurityDescriptor":
-                        defSDmodified = True
-                    if attrUSN:
-                        message(CHANGE, "%sAttribute: %s will be modified"
-                                        "/deleted it was last modified "
-                                        "during a provision. Current usn: "
-                                        "%d" % (txt, att, attrUSN))
-                        txt = ""
-                    else:
-                        message(CHANGE, "%sAttribute: %s will be added because "
-                                        "it did not exist before" % (txt, att))
-                        txt = ""
-                    continue
+                    hash_attr_usn[att] = [-1, None]
 
-            else:
-            # Old school way of handling things for pre alpha12 upgrade
-                defSDmodified = True
-                msgElt = delta.get(att)
-
-                if att == "nTSecurityDescriptor":
-                    delta.remove(att)
-                    continue
-
-                if att == "dn":
-                    continue
-
-                if not hashOverwrittenAtt.has_key(att):
-                    if msgElt.flags() != FLAG_MOD_ADD:
-                        if not handle_special_case(att, delta, reference, current,
-                                                    usns, basedn, samdb):
-                            if opts.debugchange or opts.debugall:
-                                try:
-                                    dump_denied_change(dn, att,
-                                        msg_elt_flag_strs[msgElt.flags()],
-                                        current[0][att], reference[0][att])
-                                except KeyError:
-                                    dump_denied_change(dn, att,
-                                        msg_elt_flag_strs[msgElt.flags()],
-                                        current[0][att], None)
-                            delta.remove(att)
-                        continue
-                else:
-                    if hashOverwrittenAtt.get(att)&2**msgElt.flags() :
-                        continue
-                    elif  hashOverwrittenAtt.get(att)==never:
-                        delta.remove(att)
-                        continue
+        if usns is not None:
+            delta = checkKeepAttributeWithMetadata(delta, att, message, reference,
+                                                    current, hash_attr_usn,
+                                                    basedn, usns, samdb)
+        else:
+            delta =  checkKeepAttributeOldMtd(delta, att, reference, current, basedn, samdb)
 
         delta.dn = dn
         if len(delta.items()) >1:
-            attributes=", ".join(delta.keys())
+            # Skip dn as the value is not really changed ...
+            attributes=", ".join(delta.keys()[1:])
             modcontrols = []
             relaxedatt = ['iscriticalsystemobject', 'grouptype']
             # Let's try to reduce as much as possible the use of relax control
-            #for checkedatt in relaxedatt:
             for attr in delta.keys():
                 if attr.lower() in relaxedatt:
                     modcontrols = ["relax:0", "provision:0"]
@@ -1033,8 +1134,8 @@ def update_partition(ref_samdb, samdb, basedn, names, schema, provisionUSNs, pre
     :param basedn: String value of the DN of the partition
     :param names: List of key provision parameters
     :param schema: A Schema object
-    :param provisionUSNs:  The USNs modified by provision/upgradeprovision
-                           last time
+    :param provisionUSNs:  A dictionnary with range of USN modified during provision
+                            or upgradeprovision. Ranges are grouped by invocationID.
     :param prereloadfunc: A function that must be executed just before the reload
                   of the schema
     """
@@ -1096,7 +1197,7 @@ def update_partition(ref_samdb, samdb, basedn, names, schema, provisionUSNs, pre
         message(SIMPLE, "Schema reloaded!")
 
         changed = update_present(ref_samdb, samdb, basedn, listPresent,
-                                    provisionUSNs, names.invocation)
+                                    provisionUSNs)
         message(SIMPLE, "There are %d changed objects" % (changed))
         return 1
 
@@ -1138,7 +1239,7 @@ def check_updated_sd(ref_sam, cur_sam, names):
                         str(current[i]["nTSecurityDescriptor"]))
             sddl = cursd.as_sddl(names.domainsid)
             if sddl != hash[key]:
-                txt = get_diff_sddls(hash[key], sddl)
+                txt = get_diff_sddls(hash[key], sddl, False)
                 if txt != "":
                     message(CHANGESD, "On object %s ACL is different"
                                       " \n%s" % (current[i]["dn"], txt))
@@ -1152,37 +1253,38 @@ def fix_partition_sd(samdb, names):
     :param samdb: An LDB object pointing to the sam of the current provision
     :param names: A list of key provision parameters
     """
+    alwaysRecalculate = False
+    if len(dnToRecalculate) == 0 and len(dnNotToRecalculate) == 0:
+        alwaysRecalculate = True
+
+
+    # NC's DN can't be both in dnToRecalculate and dnNotToRecalculate
     # First update the SD for the rootdn
-    res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
-                         scope=SCOPE_BASE, attrs=["dn", "whenCreated"],
-                         controls=["search_options:1:2"])
-    delta = Message()
-    delta.dn = Dn(samdb, str(res[0]["dn"]))
-    descr = get_domain_descriptor(names.domainsid)
-    delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
-                                                    "nTSecurityDescriptor")
-    samdb.modify(delta)
+    if alwaysRecalculate or str(names.rootdn) in dnToRecalculate:
+        delta = Message()
+        delta.dn = Dn(samdb, str(names.rootdn))
+        descr = get_domain_descriptor(names.domainsid)
+        delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
+                                                        "nTSecurityDescriptor")
+        samdb.modify(delta)
+
     # Then the config dn
-    res = samdb.search(expression="objectClass=*", base=str(names.configdn),
-                        scope=SCOPE_BASE, attrs=["dn", "whenCreated"],
-                        controls=["search_options:1:2"])
-    delta = Message()
-    delta.dn = Dn(samdb, str(res[0]["dn"]))
-    descr = get_config_descriptor(names.domainsid)
-    delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
-                                                    "nTSecurityDescriptor" )
-    samdb.modify(delta)
-    # Then the schema dn
-    res = samdb.search(expression="objectClass=*", base=str(names.schemadn),
-                        scope=SCOPE_BASE, attrs=["dn", "whenCreated"],
-                        controls=["search_options:1:2"])
+    if alwaysRecalculate or str(names.configdn) in dnToRecalculate:
+        delta = Message()
+        delta.dn = Dn(samdb, str(names.configdn))
+        descr = get_config_descriptor(names.domainsid)
+        delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
+                                                        "nTSecurityDescriptor" )
+        samdb.modify(delta)
 
-    delta = Message()
-    delta.dn = Dn(samdb, str(res[0]["dn"]))
-    descr = get_schema_descriptor(names.domainsid)
-    delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
-                                                    "nTSecurityDescriptor" )
-    samdb.modify(delta)
+    # Then the schema dn
+    if alwaysRecalculate or str(names.schemadn) in dnToRecalculate:
+        delta = Message()
+        delta.dn = Dn(samdb, str(names.schemadn))
+        descr = get_schema_descriptor(names.domainsid)
+        delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
+                                                        "nTSecurityDescriptor" )
+        samdb.modify(delta)
 
 def rebuild_sd(samdb, names):
     """Rebuild security descriptor of the current provision from scratch
@@ -1195,30 +1297,46 @@ def rebuild_sd(samdb, names):
 
     :param names: List of key provision parameters"""
 
+    fix_partition_sd(samdb, names)
 
+    # List of namming contexts
+    listNC = [str(names.rootdn), str(names.configdn), str(names.schemadn)]
     hash = {}
-    res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
+    if len(dnToRecalculate) == 0:
+        res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
                         scope=SCOPE_SUBTREE, attrs=["dn", "whenCreated"],
                         controls=["search_options:1:2"])
-    for obj in res:
-        if not (str(obj["dn"]) == str(names.rootdn) or
-            str(obj["dn"]) == str(names.configdn) or
-            str(obj["dn"]) == str(names.schemadn)):
-            hash[str(obj["dn"])] = obj["whenCreated"]
-
-    listkeys = hash.keys()
-    listkeys.sort(dn_sort)
-
-    for key in listkeys:
+        for obj in res:
+                hash[str(obj["dn"])] = obj["whenCreated"]
+    else:
+        for dn in dnToRecalculate:
+            if hash.has_key(dn):
+                continue
+            # fetch each dn to recalculate and their child within the same partition
+            res = samdb.search(expression="objectClass=*", base=dn,
+                        scope=SCOPE_SUBTREE, attrs=["dn", "whenCreated"])
+            for obj in res:
+                hash[str(obj["dn"])] = obj["whenCreated"]
+
+    listKeys = list(set(hash.keys()))
+    listKeys.sort(dn_sort)
+
+    if len(dnToRecalculate) != 0:
+        message(CHANGESD, "%d DNs have been marked as needed to be recalculated"\
+                            ", recalculating %d due to inheritance"
+                            % (len(dnToRecalculate), len(listKeys)))
+
+    for key in listKeys:
+        if (key in listNC or
+                    key in dnNotToRecalculate):
+            continue
+        delta = Message()
+        delta.dn = Dn(samdb, key)
         try:
-            delta = Message()
-            delta.dn = Dn(samdb, key)
             delta["whenCreated"] = MessageElement(hash[key], FLAG_MOD_REPLACE,
                                                     "whenCreated" )
-            samdb.modify(delta, ["recalculate_sd:0"])
-        except:
-            # XXX: We should always catch an explicit exception.
-            # What could go wrong here?
+            samdb.modify(delta, ["recalculate_sd:0","relax:0"])
+        except LdbError, e:
             samdb.transaction_cancel()
             res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
                                 scope=SCOPE_SUBTREE,
@@ -1226,14 +1344,13 @@ def rebuild_sd(samdb, names):
                                 controls=["search_options:1:2"])
             badsd = ndr_unpack(security.descriptor,
                         str(res[0]["nTSecurityDescriptor"]))
-            print "bad stuff %s" % badsd.as_sddl(names.domainsid)
+            message(ERROR, "On %s bad stuff %s" % (str(delta.dn),badsd.as_sddl(names.domainsid)))
             return
 
 def removeProvisionUSN(samdb):
         attrs = [samba.provision.LAST_PROVISION_USN_ATTRIBUTE, "dn"]
         entry = samdb.search(expression="dn=@PROVISION", base = "",
                                 scope=SCOPE_SUBTREE,
-                                controls=["search_options:1:2"],
                                 attrs=attrs)
         empty = Message()
         empty.dn = entry[0].dn
@@ -1303,7 +1420,7 @@ def update_privilege(ref_private_path, cur_private_path):
                 os.path.join(cur_private_path, "privilege.ldb"))
 
 
-def update_samdb(ref_samdb, samdb, names, highestUSN, schema, prereloadfunc):
+def update_samdb(ref_samdb, samdb, names, provisionUSNs, schema, prereloadfunc):
     """Upgrade the SAM DB contents for all the provision partitions
 
     :param ref_sambdb: An LDB object conntected to the sam.ldb of the reference
@@ -1311,8 +1428,8 @@ def update_samdb(ref_samdb, samdb, names, highestUSN, schema, prereloadfunc):
     :param samdb: An LDB object connected to the sam.ldb of the update
                   provision
     :param names: List of key provision parameters
-    :param highestUSN:  The highest USN modified by provision/upgradeprovision
-                        last time
+    :param provisionUSNs:  A dictionnary with range of USN modified during provision
+                            or upgradeprovision. Ranges are grouped by invocationID.
     :param schema: A Schema object that represent the schema of the provision
     :param prereloadfunc: A function that must be executed just before the reload
                   of the schema
@@ -1320,7 +1437,7 @@ def update_samdb(ref_samdb, samdb, names, highestUSN, schema, prereloadfunc):
 
     message(SIMPLE, "Starting update of samdb")
     ret = update_partition(ref_samdb, samdb, str(names.rootdn), names,
-                            schema, highestUSN, prereloadfunc)
+                            schema, provisionUSNs, prereloadfunc)
     if ret:
         message(SIMPLE, "Update of samdb finished")
         return 1
@@ -1537,7 +1654,7 @@ def sync_calculated_attributes(samdb, names):
 # This resulting object is filtered to remove all the back link attribute
 # (ie. memberOf) as they will be created by the other linked object (ie.
 # the one with the member attribute)
-# All attributes specified in the hashAttrNotCopied associative array are
+# All attributes specified in the attrNotCopied array are
 # also removed it's most of the time generated attributes
 
 # After missing entries have been added the update_partition function will
@@ -1601,202 +1718,226 @@ if __name__ == '__main__':
         # 4)
         lastProvisionUSNs = get_last_provision_usn(ldbs.sam)
         if lastProvisionUSNs is not None:
+            v = 0
+            for k in lastProvisionUSNs.keys():
+                for r in lastProvisionUSNs[k]:
+                    v = v + 1
+
             message(CHANGE,
-                "Find a last provision USN, %d range(s)" % len(lastProvisionUSNs))
+                "Find last provision USN, %d invocation(s) for a total of %d ranges" % \
+                            (len(lastProvisionUSNs.keys()), v /2 ))
+
+            if lastProvisionUSNs.get("default") != None:
+                message(CHANGE, "Old style for usn ranges used")
+                lastProvisionUSNs[str(names.invocation)] = lastProvisionUSNs["default"]
+                del lastProvisionUSNs["default"]
+        else:
+            message(SIMPLE, "Your provision lacks provision range information")
+            if confirm("Do you want to run findprovisionusnranges to try to find them ?", False):
+                ldbs.groupedRollback()
+                os.system("%s %s %s %s %s" % (os.path.join(os.path.dirname(sys.argv[0]),
+                                            "findprovisionusnranges"),
+                                        "--storedir",
+                                        paths.private_dir,
+                                        "-s",
+                                        smbconf))
+                message(SIMPLE, "Once you applied/adapted the change(s) please restart the upgradeprovision script")
+                sys.exit(0)
 
         # Objects will be created with the admin session
         # (not anymore system session)
         adm_session = admin_session(lp, str(names.domainsid))
         # So we reget handle on objects
         # ldbs = get_ldbs(paths, creds, adm_session, lp)
-
-        if not sanitychecks(ldbs.sam, names):
-            message(SIMPLE, "Sanity checks for the upgrade have failed. "
-                            "Check the messages and correct the errors "
-                            "before rerunning upgradeprovision")
-            sys.exit(1)
-
-        # Let's see provision parameters
-        print_provision_key_parameters(names)
-
-        # 5) With all this information let's create a fresh new provision used as
-        # reference
-        message(SIMPLE, "Creating a reference provision")
-        provisiondir = tempfile.mkdtemp(dir=paths.private_dir,
-                                        prefix="referenceprovision")
-        newprovision(names, creds, session, smbconf, provisiondir,
-                        provision_logger)
-
-        # TODO
-        # 6) and 7)
-        # We need to get a list of object which SD is directly computed from
-        # defaultSecurityDescriptor.
-        # This will allow us to know which object we can rebuild the SD in case
-        # of change of the parent's SD or of the defaultSD.
-        # Get file paths of this new provision
-        newpaths = get_paths(param, targetdir=provisiondir)
-        new_ldbs = get_ldbs(newpaths, creds, session, lp)
-        new_ldbs.startTransactions()
-
-        # 8) Populate some associative array to ease the update process
-        # List of attribute which are link and backlink
-        populate_links(new_ldbs.sam, names.schemadn)
-        # List of attribute with ASN DN synthax)
-        populate_dnsyntax(new_ldbs.sam, names.schemadn)
-        # 9)
-        update_privilege(newpaths.private_dir, paths.private_dir)
-        # 10)
-        oem = getOEMInfo(ldbs.sam, str(names.rootdn))
-        # Do some modification on sam.ldb
-        ldbs.groupedCommit()
-        new_ldbs.groupedCommit()
-        deltaattr = None
-# 11)
-        if re.match(".*alpha((9)|(\d\d+)).*", str(oem)):
-            # 11) A
-            # Starting from alpha9 we can consider that the structure is quite ok
-            # and that we should do only dela
-            deltaattr = delta_update_basesamdb(newpaths.samdb,
-                                                paths.samdb,
-                                                creds,
-                                                session,
-                                                lp,
-                                                message)
-        else:
-            # 11) B
-            simple_update_basesamdb(newpaths, paths, names)
-            ldbs = get_ldbs(paths, creds, session, lp)
-            removeProvisionUSN(ldbs.sam)
-
-        ldbs.startTransactions()
-        minUSN = int(str(get_max_usn(ldbs.sam, str(names.rootdn)))) + 1
-        new_ldbs.startTransactions()
-
-        # 12)
-        schema = Schema(names.domainsid, schemadn=str(names.schemadn))
-        # We create a closure that will be invoked just before schema reload
-        def schemareloadclosure():
-            basesam = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp,
-                            options=["modules:"])
-            doit = False
-            if deltaattr is not None and len(deltaattr) > 1:
-                doit = True
-            if doit:
-                deltaattr.remove("dn")
-                for att in deltaattr:
-                    if att.lower() == "dn":
-                        continue
-                    if (deltaattr.get(att) is not None
-                        and deltaattr.get(att).flags() != FLAG_MOD_ADD):
-                        doit = False
-                    elif deltaattr.get(att) is None:
-                        doit = False
-            if doit:
-                message(CHANGE, "Applying delta to @ATTRIBUTES")
-                deltaattr.dn = ldb.Dn(basesam, "@ATTRIBUTES")
-                basesam.modify(deltaattr)
-            else:
-                message(CHANGE, "Not applying delta to @ATTRIBUTES because "
-                                "there is not only add")
-        # 13)
-        if opts.full:
-            if not update_samdb(new_ldbs.sam, ldbs.sam, names, lastProvisionUSNs,
-                                schema, schemareloadclosure):
-                message(SIMPLE, "Rolling back all changes. Check the cause"
-                                " of the problem")
-                message(SIMPLE, "Your system is as it was before the upgrade")
+        if not opts.fixntacl:
+            if not sanitychecks(ldbs.sam, names):
+                message(SIMPLE, "Sanity checks for the upgrade have failed. "
+                        "Check the messages and correct the errors "
+                        "before rerunning upgradeprovision")
                 ldbs.groupedRollback()
-                new_ldbs.groupedRollback()
-                shutil.rmtree(provisiondir)
                 sys.exit(1)
-        else:
-            # Try to reapply the change also when we do not change the sam
-            # as the delta_upgrade
-            schemareloadclosure()
-            sync_calculated_attributes(ldbs.sam, names)
+
+            # Let's see provision parameters
+            print_provision_key_parameters(names)
+
+            # 5) With all this information let's create a fresh new provision used as
+            # reference
+            message(SIMPLE, "Creating a reference provision")
+            provisiondir = tempfile.mkdtemp(dir=paths.private_dir,
+                            prefix="referenceprovision")
+            newprovision(names, creds, session, smbconf, provisiondir,
+                    provision_logger)
+
+            # TODO
+            # 6) and 7)
+            # We need to get a list of object which SD is directly computed from
+            # defaultSecurityDescriptor.
+            # This will allow us to know which object we can rebuild the SD in case
+            # of change of the parent's SD or of the defaultSD.
+            # Get file paths of this new provision
+            newpaths = get_paths(param, targetdir=provisiondir)
+            new_ldbs = get_ldbs(newpaths, creds, session, lp)
+            new_ldbs.startTransactions()
+
+            populateNotReplicated(new_ldbs.sam, names.schemadn)
+            # 8) Populate some associative array to ease the update process
+            # List of attribute which are link and backlink
+            populate_links(new_ldbs.sam, names.schemadn)
+            # List of attribute with ASN DN synthax)
+            populate_dnsyntax(new_ldbs.sam, names.schemadn)
+            # 9)
+            update_privilege(newpaths.private_dir, paths.private_dir)
+            # 10)
+            oem = getOEMInfo(ldbs.sam, str(names.rootdn))
+            # Do some modification on sam.ldb
+            ldbs.groupedCommit()
+            new_ldbs.groupedCommit()
+            deltaattr = None
+        # 11)
+            if re.match(".*alpha((9)|(\d\d+)).*", str(oem)):
+                # 11) A
+                # Starting from alpha9 we can consider that the structure is quite ok
+                # and that we should do only dela
+                deltaattr = delta_update_basesamdb(newpaths.samdb,
+                                paths.samdb,
+                                creds,
+                                session,
+                                lp,
+                                message)
+            else:
+                # 11) B
+                simple_update_basesamdb(newpaths, paths, names)
+                ldbs = get_ldbs(paths, creds, session, lp)
+                removeProvisionUSN(ldbs.sam)
+
+            ldbs.startTransactions()
+            minUSN = int(str(get_max_usn(ldbs.sam, str(names.rootdn)))) + 1
+            new_ldbs.startTransactions()
+
+            # 12)
+            schema = Schema(names.domainsid, schemadn=str(names.schemadn))
+            # We create a closure that will be invoked just before schema reload
+            def schemareloadclosure():
+                basesam = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp,
+                        options=["modules:"])
+                doit = False
+                if deltaattr is not None and len(deltaattr) > 1:
+                    doit = True
+                if doit:
+                    deltaattr.remove("dn")
+                    for att in deltaattr:
+                        if att.lower() == "dn":
+                            continue
+                        if (deltaattr.get(att) is not None
+                            and deltaattr.get(att).flags() != FLAG_MOD_ADD):
+                            doit = False
+                        elif deltaattr.get(att) is None:
+                            doit = False
+                if doit:
+                    message(CHANGE, "Applying delta to @ATTRIBUTES")
+                    deltaattr.dn = ldb.Dn(basesam, "@ATTRIBUTES")
+                    basesam.modify(deltaattr)
+                else:
+                    message(CHANGE, "Not applying delta to @ATTRIBUTES because "
+                        "there is not only add")
+            # 13)
+            if opts.full:
+                if not update_samdb(new_ldbs.sam, ldbs.sam, names, lastProvisionUSNs,
+                        schema, schemareloadclosure):
+                    message(SIMPLE, "Rolling back all changes. Check the cause"
+                            " of the problem")
+                    message(SIMPLE, "Your system is as it was before the upgrade")
+                    ldbs.groupedRollback()
+                    new_ldbs.groupedRollback()
+                    shutil.rmtree(provisiondir)
+                    sys.exit(1)
+            else:
+                # Try to reapply the change also when we do not change the sam
+                # as the delta_upgrade
+                schemareloadclosure()
+                sync_calculated_attributes(ldbs.sam, names)
+                res = ldbs.sam.search(expression="(samaccountname=dns)",
+                        scope=SCOPE_SUBTREE, attrs=["dn"],
+                        controls=["search_options:1:2"])
+                if len(res) > 0:
+                    message(SIMPLE, "You still have the old DNS object for managing "
+                            "dynamic DNS, but you didn't supply --full so "
+                            "a correct update can't be done")
+                    ldbs.groupedRollback()
+                    new_ldbs.groupedRollback()
+                    shutil.rmtree(provisiondir)
+                    sys.exit(1)
+            # 14)
+            update_secrets(new_ldbs.secrets, ldbs.secrets, message)
+            # 14bis)
             res = ldbs.sam.search(expression="(samaccountname=dns)",
-                                scope=SCOPE_SUBTREE, attrs=["dn"],
-                                controls=["search_options:1:2"])
-            if len(res) > 0:
-                message(SIMPLE, "You still have the old DNS object for managing "
-                                "dynamic DNS, but you didn't supply --full so "
-                                "a correct update can't be done")
-                ldbs.groupedRollback()
-                new_ldbs.groupedRollback()
-                shutil.rmtree(provisiondir)
-                sys.exit(1)
-        # 14)
-        update_secrets(new_ldbs.secrets, ldbs.secrets, message)
-        # 14bis)
-        res = ldbs.sam.search(expression="(samaccountname=dns)",
-                                  scope=SCOPE_SUBTREE, attrs=["dn"],
-                                  controls=["search_options:1:2"])
-
-        if (len(res) == 1):
-            ldbs.sam.delete(res[0]["dn"])
-            res2 = ldbs.secrets.search(expression="(samaccountname=dns)",
-                                  scope=SCOPE_SUBTREE, attrs=["dn"])
-            update_dns_account_password(ldbs.sam, ldbs.secrets, names)
-            message(SIMPLE, "IMPORTANT!!! "
-                            "If you were using Dynamic DNS before you need "
-                            "to update your configuration, so that the "
-                            "tkey-gssapi-credential has the following value: "
-                            "DNS/%s.%s" % (names.netbiosname.lower(),
-                                           names.realm.lower()))
-        # 15)
-        message(SIMPLE, "Update machine account")
-        update_machine_account_password(ldbs.sam, ldbs.secrets, names)
-
-        # 16) SD should be created with admin but as some previous acl were so wrong
-        # that admin can't modify them we have first to recreate them with the good
-        # form but with system account and then give the ownership to admin ...
-        if not re.match(r'.*alpha(9|\d\d+)', str(oem)):
-            message(SIMPLE, "Fixing old povision SD")
-            fix_partition_sd(ldbs.sam, names)
-            rebuild_sd(ldbs.sam, names)
-
-        # We calculate the max USN before recalculating the SD because we might
-        # touch object that have been modified after a provision and we do not
-        # want that the next upgradeprovision thinks that it has a green light
-        # to modify them
-
-        # 17)
-        maxUSN = get_max_usn(ldbs.sam, str(names.rootdn))
-
-        # 18) We rebuild SD only if defaultSecurityDescriptor is modified
-        # But in fact we should do it also if one object has its SD modified as
-        # child might need rebuild
-        if defSDmodified:
-            message(SIMPLE, "Updating SD")
-            ldbs.sam.set_session_info(adm_session)
-            # Alpha10 was a bit broken still
-            if re.match(r'.*alpha(\d|10)', str(oem)):
-                fix_partition_sd(ldbs.sam, names)
-            rebuild_sd(ldbs.sam, names)
-
-        # 19)
-        # Now we are quite confident in the recalculate process of the SD, we make
-        # it optional.
-        # Also the check must be done in a clever way as for the moment we just
-        # compare SDDL
-        if opts.debugchangesd:
-            check_updated_sd(new_ldbs.sam, ldbs.sam, names)
-
-        # 20)
-        updateOEMInfo(ldbs.sam, str(names.rootdn))
-        # 21)
-        check_for_DNS(newpaths.private_dir, paths.private_dir)
-        # 22)
-        if lastProvisionUSNs is not None:
-            update_provision_usn(ldbs.sam, minUSN, maxUSN)
-        if opts.full and (names.policyid is None or names.policyid_dc is None):
-            update_policyids(names, ldbs.sam)
-        if opts.full or opts.resetfileacl:
+                        scope=SCOPE_SUBTREE, attrs=["dn"],
+                        controls=["search_options:1:2"])
+
+            if (len(res) == 1):
+                ldbs.sam.delete(res[0]["dn"])
+                res2 = ldbs.secrets.search(expression="(samaccountname=dns)",
+                        scope=SCOPE_SUBTREE, attrs=["dn"])
+                update_dns_account_password(ldbs.sam, ldbs.secrets, names)
+                message(SIMPLE, "IMPORTANT!!! "
+                        "If you were using Dynamic DNS before you need "
+                        "to update your configuration, so that the "
+                        "tkey-gssapi-credential has the following value: "
+                        "DNS/%s.%s" % (names.netbiosname.lower(),
+                            names.realm.lower()))
+            # 15)
+            message(SIMPLE, "Update machine account")
+            update_machine_account_password(ldbs.sam, ldbs.secrets, names)
+
+            dnToRecalculate.sort(dn_sort)
+            # 16) SD should be created with admin but as some previous acl were so wrong
+            # that admin can't modify them we have first to recreate them with the good
+            # form but with system account and then give the ownership to admin ...
+            if str(oem) != "" and not re.match(r'.*alpha(9|\d\d+)', str(oem)):
+                message(SIMPLE, "Fixing very old provision SD")
+                rebuild_sd(ldbs.sam, names)
+
+            # We calculate the max USN before recalculating the SD because we might
+            # touch object that have been modified after a provision and we do not
+            # want that the next upgradeprovision thinks that it has a green light
+            # to modify them
+
+            # 17)
+            maxUSN = get_max_usn(ldbs.sam, str(names.rootdn))
+
+            # 18) We rebuild SD if a we have a list of DN to recalculate or if the
+            # defSDmodified is set.
+            if defSDmodified or len(dnToRecalculate) >0:
+                message(SIMPLE, "Some defaultSecurityDescriptors and/or"
+                                "securityDescriptor have changed, recalculating SD ")
+                ldbs.sam.set_session_info(adm_session)
+                rebuild_sd(ldbs.sam, names)
+
+            # 19)
+            # Now we are quite confident in the recalculate process of the SD, we make
+            # it optional. And we don't do it if there is DN that we must touch
+            # as we are assured that on this DNs we will have differences !
+            # Also the check must be done in a clever way as for the moment we just
+            # compare SDDL
+            if len(dnNotToRecalculate) == 0 and (opts.debugchangesd or opts.debugall):
+                message(CHANGESD, "Checking recalculated SDs")
+                check_updated_sd(new_ldbs.sam, ldbs.sam, names)
+
+            # 20)
+            updateOEMInfo(ldbs.sam, str(names.rootdn))
+            # 21)
+            check_for_DNS(newpaths.private_dir, paths.private_dir)
+            # 22)
+            if lastProvisionUSNs is not None:
+                update_provision_usn(ldbs.sam, minUSN, maxUSN, names.invocation)
+            if opts.full and (names.policyid is None or names.policyid_dc is None):
+                update_policyids(names, ldbs.sam)
+        if opts.full or opts.resetfileacl or opts.fixntacl:
             try:
                 update_gpo(paths, ldbs.sam, names, lp, message, 1)
             except ProvisioningError, e:
                 message(ERROR, "The policy for domain controller is missing. "
-                               "You should restart upgradeprovision with --full")
+                            "You should restart upgradeprovision with --full")
             except IOError, e:
                 message(ERROR, "Setting ACL not supported on your filesystem")
         else:
@@ -1804,25 +1945,29 @@ if __name__ == '__main__':
                 update_gpo(paths, ldbs.sam, names, lp, message, 0)
             except ProvisioningError, e:
                 message(ERROR, "The policy for domain controller is missing. "
-                               "You should restart upgradeprovision with --full")
-        ldbs.groupedCommit()
-        new_ldbs.groupedCommit()
-        message(SIMPLE, "Upgrade finished!")
-        # remove reference provision now that everything is done !
-        # So we have reindexed first if need when the merged schema was reloaded
-        # (as new attributes could have quick in)
-        # But the second part of the update (when we update existing objects
-        # can also have an influence on indexing as some attribute might have their
-        # searchflag modificated
-        message(SIMPLE, "Reopenning samdb to trigger reindexing if needed "
-                        "after modification")
-        samdb = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp)
-        message(SIMPLE, "Reindexing finished")
-
-        shutil.rmtree(provisiondir)
+                            "You should restart upgradeprovision with --full")
+        if not opts.fixntacl:
+            ldbs.groupedCommit()
+            new_ldbs.groupedCommit()
+            message(SIMPLE, "Upgrade finished!")
+            # remove reference provision now that everything is done !
+            # So we have reindexed first if need when the merged schema was reloaded
+            # (as new attributes could have quick in)
+            # But the second part of the update (when we update existing objects
+            # can also have an influence on indexing as some attribute might have their
+            # searchflag modificated
+            message(SIMPLE, "Reopenning samdb to trigger reindexing if needed "
+                    "after modification")
+            samdb = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp)
+            message(SIMPLE, "Reindexing finished")
+
+            shutil.rmtree(provisiondir)
+        else:
+            ldbs.groupedRollback()
+            message(SIMPLE, "ACLs fixed !")
     except StandardError, err:
         message(ERROR, "A problem occurred while trying to upgrade your "
-                       "provision. A full backup is located at %s" % backupdir)
+                   "provision. A full backup is located at %s" % backupdir)
         if opts.debugall or opts.debugchange:
             (typ, val, tb) = sys.exc_info()
             traceback.print_exception(typ, val, tb)
diff --git a/source4/scripting/bin/wscript_build b/source4/scripting/bin/wscript_build
new file mode 100644
index 00000000000..e52b32bc02a
--- /dev/null
+++ b/source4/scripting/bin/wscript_build
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+bld.SAMBA_SCRIPT('samba_dnsupdate', pattern='samba_dnsupdate', installdir='.')
+bld.SAMBA_SCRIPT('samba_spnupdate', pattern='samba_spnupdate', installdir='.')
+bld.SAMBA_SCRIPT('upgradeprovision', pattern='upgradeprovision', installdir='.')
diff --git a/source4/scripting/devel/chgtdcpass b/source4/scripting/devel/chgtdcpass
index dc249834e00..4f5ea15a80c 100755
--- a/source4/scripting/devel/chgtdcpass
+++ b/source4/scripting/devel/chgtdcpass
@@ -29,8 +29,9 @@ import samba.getopt as options
 from samba.credentials import DONT_USE_KERBEROS
 from samba.auth import system_session
 from samba import param
+from samba.provision import find_provision_key_parameters
 from samba.upgradehelpers import (get_paths,
-                                 find_provision_key_parameters, get_ldbs,
+                                  get_ldbs,
                                  update_machine_account_password)
 
 parser = optparse.OptionParser("chgtdcpass [options]")
diff --git a/source4/scripting/devel/demodirsync.py b/source4/scripting/devel/demodirsync.py
new file mode 100755
index 00000000000..41dac6ff517
--- /dev/null
+++ b/source4/scripting/devel/demodirsync.py
@@ -0,0 +1,156 @@
+#!/usr/bin/python
+
+
+import optparse
+import sys
+import base64
+
+sys.path.insert(0, "bin/python")
+
+import samba.getopt as options
+from samba.dcerpc import drsblobs, misc
+from samba.ndr import ndr_pack, ndr_unpack
+from samba import Ldb
+
+parser = optparse.OptionParser("get-descriptor [options]")
+sambaopts = options.SambaOptions(parser)
+credopts = options.CredentialsOptions(parser)
+parser.add_option_group(credopts)
+
+parser.add_option("-b", type="string", metavar="BASE",
+                  help="set base DN for the search")
+parser.add_option("--host", type="string", metavar="HOST",
+                  help="Ip of the host")
+
+lp = sambaopts.get_loadparm()
+creds = credopts.get_credentials(lp)
+
+opts = parser.parse_args()[0]
+
+def printdirsync(ctl):
+        arr = ctl.split(':')
+        if arr[0] == 'dirsync':
+            print "Need to continue: %s" % arr[1]
+            cookie = ndr_unpack(drsblobs.ldapControlDirSyncCookie, base64.b64decode(arr[3]))
+            print "DC's NTDS guid: %s " % cookie.blob.guid1
+            print "highest usn %s" % cookie.blob.highwatermark.highest_usn
+            print "tmp higest usn %s" % cookie.blob.highwatermark.tmp_highest_usn
+            print "reserved usn %s" % cookie.blob.highwatermark.reserved_usn
+            if cookie.blob.extra_length >0:
+                print "highest usn in extra %s" % cookie.blob.extra.ctr.cursors[0].highest_usn
+        return cookie
+
+remote_ldb= Ldb("ldap://" + opts.host + ":389", credentials=creds, lp=lp)
+tab = []
+if opts.b:
+    base = opts.b
+else:
+    base = None
+
+guid = None
+(msgs, ctrls) = remote_ldb.search(expression="(samaccountname=administrator)", base=base, attrs=["objectClass"], controls=["dirsync:1:1:50"])
+if (len(ctrls)):
+    for ctl in ctrls:
+        arr = ctl.split(':')
+        if arr[0] == 'dirsync':
+            cookie = ndr_unpack(drsblobs.ldapControlDirSyncCookie, base64.b64decode(arr[3]))
+            guid = cookie.blob.guid1
+            pass
+if not guid:
+    print "No dirsync control ... strange"
+    sys.exit(1)
+
+print ""
+print "Getting first guest without any cookie"
+(msgs, ctrls) = remote_ldb.searchex(expression="(samaccountname=guest)", base=base, attrs=["objectClass"], controls=["dirsync:1:1:50"])
+cookie = None
+if (len(ctrls)):
+    for ctl in ctrls:
+        cookie = printdirsync(ctl)
+    print "Returned %d entries" % len(msgs)
+
+savedcookie = cookie
+
+print ""
+print "Getting allusers with cookie"
+controls=["dirsync:1:1:50:%s" % base64.b64encode(ndr_pack(cookie))]
+(msgs, ctrls) = remote_ldb.searchex(expression="(samaccountname=*)", base=base, attrs=["objectClass"], controls=controls)
+if (len(ctrls)):
+    for ctl in ctrls:
+        cookie = printdirsync(ctl)
+    print "Returned %d entries" % len(msgs)
+
+cookie = savedcookie
+cookie.blob.guid1 = misc.GUID("128a99bf-e2df-4832-ac0a-1fb625e530db")
+if cookie.blob.extra_length > 0:
+    cookie.blob.extra.ctr.cursors[0].source_dsa_invocation_id = misc.GUID("128a99bf-e2df-4832-ac0a-1fb625e530db")
+
+print ""
+print "Getting all the entries"
+controls=["dirsync:1:1:50:%s" % base64.b64encode(ndr_pack(cookie))]
+(msgs, ctrls) = remote_ldb.searchex(expression="(objectclass=*)", base=base, controls=controls)
+cont = 0
+if (len(ctrls)):
+    for ctl in ctrls:
+        cookie = printdirsync(ctl)
+    if cookie != None:
+        cont = (ctl.split(':'))[1]
+    print "Returned %d entries" % len(msgs)
+
+usn = cookie.blob.highwatermark.tmp_highest_usn
+if cookie.blob.extra_length > 0:
+    bigusn = cookie.blob.extra.ctr.cursors[0].highest_usn
+else:
+    bigusn  = usn + 1000
+while (cont == "1"):
+    print ""
+    controls=["dirsync:1:1:50:%s" % base64.b64encode(ndr_pack(cookie))]
+    (msgs, ctrls) = remote_ldb.searchex(expression="(objectclass=*)", base=base, controls=controls)
+    if (len(ctrls)):
+        for ctl in ctrls:
+            cookie = printdirsync(ctl)
+        if cookie != None:
+            cont = (ctl.split(':'))[1]
+        print "Returned %d entries" % len(msgs)
+
+print ""
+print "Getting with cookie but usn changed to %d we should use the one in extra" % (bigusn - 1)
+cookie.blob.highwatermark.highest_usn = 0
+cookie.blob.highwatermark.tmp_highest_usn = usn - 2
+if cookie.blob.extra_length > 0:
+    print "here"
+    cookie.blob.extra.ctr.cursors[0].highest_usn = bigusn - 1
+controls=["dirsync:1:1:50:%s" % base64.b64encode(ndr_pack(cookie))]
+(msgs, ctrls) = remote_ldb.searchex(expression="(objectclass=*)", base=base, controls=controls)
+if (len(ctrls)):
+    for ctl in ctrls:
+        cookie = printdirsync(ctl)
+    print "Returned %d entries" % len(msgs)
+
+print ""
+print "Getting with cookie but usn %d changed and extra/cursor GUID too" % (usn - 2)
+print " so that it's (tmp)highest_usn that drives the limit"
+cookie.blob.highwatermark.highest_usn = 0
+cookie.blob.highwatermark.tmp_highest_usn = usn - 2
+if cookie.blob.extra_length > 0:
+    cookie.blob.extra.ctr.cursors[0].source_dsa_invocation_id = misc.GUID("128a99bf-e2df-4832-ac0a-1fb625e530db")
+    cookie.blob.extra.ctr.cursors[0].highest_usn = bigusn - 1
+controls=["dirsync:1:1:50:%s" % base64.b64encode(ndr_pack(cookie))]
+(msgs, ctrls) = remote_ldb.searchex(expression="(objectclass=*)", base=base, controls=controls)
+if (len(ctrls)):
+    for ctl in ctrls:
+        cookie = printdirsync(ctl)
+    print "Returned %d entries" % len(msgs)
+
+print ""
+print "Getting with cookie but usn changed to %d" % (usn - 2)
+cookie.blob.highwatermark.highest_usn = 0
+cookie.blob.highwatermark.tmp_highest_usn = (usn - 2)
+if cookie.blob.extra_length > 0:
+    cookie.blob.extra.ctr.cursors[0].highest_usn = (usn - 2)
+controls=["dirsync:1:1:50:%s" % base64.b64encode(ndr_pack(cookie))]
+(msgs, ctrls) = remote_ldb.searchex(expression="(objectclass=*)", base=base, controls=controls)
+if (len(ctrls)):
+    for ctl in ctrls:
+        cookie = printdirsync(ctl)
+    print "Returned %d entries" % len(msgs)
diff --git a/source4/scripting/devel/selftest-vars.sh b/source4/scripting/devel/selftest-vars.sh
index bc73c054078..a8f323dbda8 100644
--- a/source4/scripting/devel/selftest-vars.sh
+++ b/source4/scripting/devel/selftest-vars.sh
@@ -2,38 +2,42 @@
 # outside the test environment
 
 export UID_WRAPPER=1
-export NSS_WRAPPER_PASSWD=st/dc/passwd
-export NSS_WRAPPER_GROUP=st/dc/group
+export NSS_WRAPPER_PASSWD=$PWD/st/dc/passwd
+export NSS_WRAPPER_GROUP=$PWD/st/dc/group
 export CONFIGURATION="--configfile=$PWD/st/dc/etc/smb.conf"
+export VAMPIRE_DC_SERVER=localvampiredc
+export VAMPIRE_DC_SERVER_IP=127.0.0.22
+export VAMPIRE_DC_NETBIOSNAME=localvampiredc1
+export VAMPIRE_DC_NETBIOSALIAS=localvampiredc
 export MEMBER_SERVER=localmember3
-export MEMBER_SERVER_IP=127.0.0.3
+export MEMBER_SERVER_IP=127.0.0.23
 export MEMBER_NETBIOSNAME=localmember3
 export MEMBER_NETBIOSALIAS=localmember
 export RPC_PROXY_SERVER=localrpcproxy4
-export RPC_PROXY_SERVER_IP=127.0.0.4
+export RPC_PROXY_SERVER_IP=127.0.0.24
 export RPC_PROXY_NETBIOSNAME=localrpcproxy4
 export RPC_PROXY_NETBIOSALIAS=localrpcproxy
 export SELFTEST_MAXTIME=1200
 export NETBIOSNAME=localdc1
 export REALM=SAMBA.EXAMPLE.COM
-export SOCKET_WRAPPER_DEFAULT_IFACE=1
+export SOCKET_WRAPPER_DEFAULT_IFACE=21
 export SERVER=localdc1
 export WINBINDD_SOCKET_DIR=$PWD/st/dc/winbindd_socket
 export SELFTEST_PREFIX=$PWD/st
 export DOMAIN=SAMBADOMAIN
 export BINDIR=./bin
-export DC_SERVER_IP=127.0.0.1
+export DC_SERVER_IP=127.0.0.21
 export SELFTEST_INTERFACES=127.0.0.6/8,127.0.0.7/8,127.0.0.8/8,127.0.0.9/8,127.0.0.10/8,127.0.0.11/8
 export SOCKET_WRAPPER_DIR=$PWD/st/w
 export DC_USERNAME=Administrator
 export USERNAME=Administrator
-export SERVER_IP=127.0.0.1
+export SERVER_IP=127.0.0.21
 export KRB5_CONFIG=$PWD/st/dc/etc/krb5.conf
 export PREFIX_ABS=$PWD/st
 export SRCDIR_ABS=$PWD
-export PREFIX=./st
-export KRB5CCNAME=./st/krb5ticket
-export SRCDIR=.
+export PREFIX=$PWD/st
+export KRB5CCNAME=$PWD/st/krb5ticket
+export SRCDIR=$PWD/
 export TLS_ENABLED=yes
 export DC_NETBIOSALIAS=localdc
 export DC_NETBIOSNAME=localdc1
diff --git a/source4/scripting/python/pyglue.c b/source4/scripting/python/pyglue.c
index f89785f971f..8a82f3502a5 100644
--- a/source4/scripting/python/pyglue.c
+++ b/source4/scripting/python/pyglue.c
@@ -25,6 +25,10 @@
 
 void init_glue(void);
 
+#ifndef Py_RETURN_NONE
+#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
+#endif
+
 static PyObject *py_generate_random_str(PyObject *self, PyObject *args)
 {
 	int len;
@@ -149,22 +153,22 @@ static PyObject *py_interface_ips(PyObject *self, PyObject *args)
 		return NULL;
 	}
 
-	load_interfaces(tmp_ctx, lpcfg_interfaces(lp_ctx), &ifaces);
+	load_interface_list(tmp_ctx, lp_ctx, &ifaces);
 
-	count = iface_count(ifaces);
+	count = iface_list_count(ifaces);
 
 	/* first count how many are not loopback addresses */
 	for (ifcount = i = 0; i<count; i++) {
-		const char *ip = iface_n_ip(ifaces, i);
-		if (!(!all_interfaces && iface_same_net(ip, "127.0.0.1", "255.0.0.0"))) {
+		const char *ip = iface_list_n_ip(ifaces, i);
+		if (!(!all_interfaces && iface_list_same_net(ip, "127.0.0.1", "255.0.0.0"))) {
 			ifcount++;
 		}
 	}
 
 	pylist = PyList_New(ifcount);
 	for (ifcount = i = 0; i<count; i++) {
-		const char *ip = iface_n_ip(ifaces, i);
-		if (!(!all_interfaces && iface_same_net(ip, "127.0.0.1", "255.0.0.0"))) {
+		const char *ip = iface_list_n_ip(ifaces, i);
+		if (!(!all_interfaces && iface_list_same_net(ip, "127.0.0.1", "255.0.0.0"))) {
 			PyList_SetItem(pylist, ifcount, PyString_FromString(ip));
 			ifcount++;
 		}
@@ -173,6 +177,30 @@ static PyObject *py_interface_ips(PyObject *self, PyObject *args)
 	return pylist;
 }
 
+static PyObject *py_strcasecmp_m(PyObject *self, PyObject *args)
+{
+	char *s1, *s2;
+
+	if (!PyArg_ParseTuple(args, "ss", &s1, &s2))
+		return NULL;
+
+	return PyInt_FromLong(strcasecmp_m(s1, s2));
+}
+
+static PyObject *py_strstr_m(PyObject *self, PyObject *args)
+{
+	char *s1, *s2, *ret;
+
+	if (!PyArg_ParseTuple(args, "ss", &s1, &s2))
+		return NULL;
+
+	ret = strstr_m(s1, s2);
+	if (!ret) {
+		Py_RETURN_NONE;
+	}
+	return PyString_FromString(ret);
+}
+
 static PyMethodDef py_misc_methods[] = {
 	{ "generate_random_str", (PyCFunction)py_generate_random_str, METH_VARARGS,
 		"generate_random_str(len) -> string\n"
@@ -192,6 +220,10 @@ static PyMethodDef py_misc_methods[] = {
 		"get debug level" },
 	{ "interface_ips", (PyCFunction)py_interface_ips, METH_VARARGS,
 		"get interface IP address list"},
+	{ "strcasecmp_m", (PyCFunction)py_strcasecmp_m, METH_VARARGS,
+		"(for testing) compare two strings using Samba's strcasecmp_m()"},
+	{ "strstr_m", (PyCFunction)py_strstr_m, METH_VARARGS,
+		"(for testing) find one string in another with Samba's strstr_m()"},
 	{ NULL }
 };
 
diff --git a/source4/scripting/python/samba/__init__.py b/source4/scripting/python/samba/__init__.py
index 2a54f47d2bb..76eb44ce928 100644
--- a/source4/scripting/python/samba/__init__.py
+++ b/source4/scripting/python/samba/__init__.py
@@ -26,6 +26,7 @@ __docformat__ = "restructuredText"
 
 import os
 import sys
+import samba.param
 
 def source_tree_topdir():
     '''return the top level directory (the one containing the source4 directory)'''
@@ -77,8 +78,8 @@ class Ldb(_Ldb):
 
         if modules_dir is not None:
             self.set_modules_dir(modules_dir)
-        elif lp is not None:
-            self.set_modules_dir(os.path.join(lp.get("modules dir"), "ldb"))
+        else:
+            self.set_modules_dir(os.path.join(samba.param.modules_dir(), "ldb"))
 
         if session_info is not None:
             self.set_session_info(session_info)
@@ -348,3 +349,5 @@ nttime2string = _glue.nttime2string
 nttime2unix = _glue.nttime2unix
 unix2nttime = _glue.unix2nttime
 generate_random_password = _glue.generate_random_password
+strcasecmp_m = _glue.strcasecmp_m
+strstr_m = _glue.strstr_m
diff --git a/source4/scripting/python/samba/common.py b/source4/scripting/python/samba/common.py
new file mode 100644
index 00000000000..a2a49627972
--- /dev/null
+++ b/source4/scripting/python/samba/common.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+#
+# Samba common functions
+#
+# Copyright (C) Matthieu Patou <mat@matws.net>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+def confirm(msg, forced = False):
+    """confirm an action with the user
+        :param msg: A string to print to the user
+        :param forced: Are the answer forced
+    """
+    if forced:
+        print("%s [YES]" % msg)
+        return True
+
+    v = raw_input(msg + ' [y/N] ')
+    return v.upper() in ['Y', 'YES']
+
+
diff --git a/source4/scripting/python/samba/dbchecker.py b/source4/scripting/python/samba/dbchecker.py
new file mode 100644
index 00000000000..88fd0edf003
--- /dev/null
+++ b/source4/scripting/python/samba/dbchecker.py
@@ -0,0 +1,317 @@
+#!/usr/bin/env python
+#
+# Samba4 AD database checker
+#
+# Copyright (C) Andrew Tridgell 2011
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import ldb
+from samba import dsdb
+from samba import common
+from samba.dcerpc import misc
+
+
+class dsdb_DN(object):
+    '''a class to manipulate DN components'''
+
+    def __init__(self, samdb, dnstring, syntax_oid):
+        if syntax_oid in [ dsdb.DSDB_SYNTAX_BINARY_DN, dsdb.DSDB_SYNTAX_STRING_DN ]:
+            colons = dnstring.split(':')
+            if len(colons) < 4:
+                raise Exception("invalid DN prefix")
+            prefix_len = 4 + len(colons[1]) + int(colons[1])
+            self.prefix = dnstring[0:prefix_len]
+            self.dnstring = dnstring[prefix_len:]
+        else:
+            self.dnstring = dnstring
+            self.prefix = ''
+        try:
+            self.dn = ldb.Dn(samdb, self.dnstring)
+        except Exception, msg:
+            print("ERROR: bad DN string '%s'" % self.dnstring)
+            raise
+
+    def __str__(self):
+        return self.prefix + str(self.dn.extended_str(mode=1))
+
+class dbcheck(object):
+    """check a SAM database for errors"""
+
+    def __init__(self, samdb, samdb_schema=None, verbose=False, fix=False, yes=False, quiet=False):
+        self.samdb = samdb
+        self.samdb_schema = (samdb_schema or samdb)
+        self.verbose = verbose
+        self.fix = fix
+        self.yes = yes
+        self.quiet = quiet
+
+    def check_database(self, DN=None, scope=ldb.SCOPE_SUBTREE, controls=[], attrs=['*']):
+        '''perform a database check, returning the number of errors found'''
+
+        res = self.samdb.search(base=DN, scope=scope, attrs=['dn'], controls=controls)
+        self.report('Checking %u objects' % len(res))
+        error_count = 0
+        for object in res:
+            error_count += self.check_object(object.dn, attrs=attrs)
+        if error_count != 0 and not self.fix:
+            self.report("Please use --fix to fix these errors")
+        self.report('Checked %u objects (%u errors)' % (len(res), error_count))
+
+        return error_count
+
+
+    def report(self, msg):
+        '''print a message unless quiet is set'''
+        if not self.quiet:
+            print(msg)
+
+
+    ################################################################
+    # a local confirm function that obeys the --fix and --yes options
+    def confirm(self, msg):
+        '''confirm a change'''
+        if not self.fix:
+            return False
+        if self.quiet:
+            return self.yes
+        return common.confirm(msg, forced=self.yes)
+
+
+    ################################################################
+    # handle empty attributes
+    def err_empty_attribute(self, dn, attrname):
+        '''fix empty attributes'''
+        self.report("ERROR: Empty attribute %s in %s" % (attrname, dn))
+        if not self.confirm('Remove empty attribute %s from %s?' % (attrname, dn)):
+            self.report("Not fixing empty attribute %s" % attrname)
+            return
+
+        m = ldb.Message()
+        m.dn = dn
+        m[attrname] = ldb.MessageElement('', ldb.FLAG_MOD_DELETE, attrname)
+        if self.verbose:
+            self.report(self.samdb.write_ldif(m, ldb.CHANGETYPE_MODIFY))
+        try:
+            self.samdb.modify(m, controls=["relax:0"], validate=False)
+        except Exception, msg:
+            self.report("Failed to remove empty attribute %s : %s" % (attrname, msg))
+            return
+        self.report("Removed empty attribute %s" % attrname)
+
+
+    ################################################################
+    # handle normalisation mismatches
+    def err_normalise_mismatch(self, dn, attrname, values):
+        '''fix attribute normalisation errors'''
+        self.report("ERROR: Normalisation error for attribute %s in %s" % (attrname, dn))
+        mod_list = []
+        for val in values:
+            normalised = self.samdb.dsdb_normalise_attributes(self.samdb_schema, attrname, [val])
+            if len(normalised) != 1:
+                self.report("Unable to normalise value '%s'" % val)
+                mod_list.append((val, ''))
+            elif (normalised[0] != val):
+                self.report("value '%s' should be '%s'" % (val, normalised[0]))
+                mod_list.append((val, normalised[0]))
+        if not self.confirm('Fix normalisation for %s from %s?' % (attrname, dn)):
+            self.report("Not fixing attribute %s" % attrname)
+            return
+
+        m = ldb.Message()
+        m.dn = dn
+        for i in range(0, len(mod_list)):
+            (val, nval) = mod_list[i]
+            m['value_%u' % i] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
+            if nval != '':
+                m['normv_%u' % i] = ldb.MessageElement(nval, ldb.FLAG_MOD_ADD, attrname)
+
+        if self.verbose:
+            self.report(self.samdb.write_ldif(m, ldb.CHANGETYPE_MODIFY))
+        try:
+            self.samdb.modify(m, controls=["relax:0"], validate=False)
+        except Exception, msg:
+            self.report("Failed to normalise attribute %s : %s" % (attrname, msg))
+            return
+        self.report("Normalised attribute %s" % attrname)
+
+    def is_deleted_objects_dn(self, dsdb_dn):
+        '''see if a dsdb_DN is the special Deleted Objects DN'''
+        return dsdb_dn.prefix == "B:32:18E2EA80684F11D2B9AA00C04F79F805:"
+
+
+    ################################################################
+    # handle a missing GUID extended DN component
+    def err_incorrect_dn_GUID(self, dn, attrname, val, dsdb_dn, errstr):
+        self.report("ERROR: %s component for %s in object %s - %s" % (errstr, attrname, dn, val))
+        controls=["extended_dn:1:1"]
+        if self.is_deleted_objects_dn(dsdb_dn):
+            controls.append("show_deleted:1")
+        try:
+            res = self.samdb.search(base=str(dsdb_dn.dn), scope=ldb.SCOPE_BASE,
+                                    attrs=[], controls=controls)
+        except ldb.LdbError, (enum, estr):
+            self.report("unable to find object for DN %s - cannot fix (%s)" % (dsdb_dn.dn, estr))
+            return
+        dsdb_dn.dn = res[0].dn
+
+        if not self.confirm('Change DN to %s?' % str(dsdb_dn)):
+            self.report("Not fixing %s" % errstr)
+            return
+        m = ldb.Message()
+        m.dn = dn
+        m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
+        m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
+        if self.verbose:
+            self.report(self.samdb.write_ldif(m, ldb.CHANGETYPE_MODIFY))
+        try:
+            self.samdb.modify(m)
+        except Exception, msg:
+            self.report("Failed to fix %s on attribute %s : %s" % (errstr, attrname, msg))
+            return
+        self.report("Fixed %s on attribute %s" % (errstr, attrname))
+
+
+    ################################################################
+    # handle a DN pointing to a deleted object
+    def err_deleted_dn(self, dn, attrname, val, dsdb_dn, correct_dn):
+        self.report("ERROR: target DN is deleted for %s in object %s - %s" % (attrname, dn, val))
+        self.report("Target GUID points at deleted DN %s" % correct_dn)
+        if not self.confirm('Remove DN?'):
+            self.report("Not removing")
+            return
+        m = ldb.Message()
+        m.dn = dn
+        m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
+        if self.verbose:
+            self.report(self.samdb.write_ldif(m, ldb.CHANGETYPE_MODIFY))
+        try:
+            self.samdb.modify(m)
+        except Exception, msg:
+            self.report("Failed to remove deleted DN attribute %s : %s" % (attrname, msg))
+            return
+        self.report("Removed deleted DN on attribute %s" % attrname)
+
+
+    ################################################################
+    # handle a DN string being incorrect
+    def err_dn_target_mismatch(self, dn, attrname, val, dsdb_dn, correct_dn, errstr):
+        self.report("ERROR: incorrect DN string component for %s in object %s - %s" % (attrname, dn, val))
+        dsdb_dn.dn = correct_dn
+
+        if not self.confirm('Change DN to %s?' % str(dsdb_dn)):
+            self.report("Not fixing %s" % errstr)
+            return
+        m = ldb.Message()
+        m.dn = dn
+        m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
+        m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
+        if self.verbose:
+            self.report(self.samdb.write_ldif(m, ldb.CHANGETYPE_MODIFY))
+        try:
+            self.samdb.modify(m)
+        except Exception, msg:
+            self.report("Failed to fix incorrect DN string on attribute %s : %s" % (attrname, msg))
+            return
+        self.report("Fixed incorrect DN string on attribute %s" % (attrname))
+
+
+    ################################################################
+    # specialised checking for a dn attribute
+    def check_dn(self, obj, attrname, syntax_oid):
+        '''check a DN attribute for correctness'''
+        error_count = 0
+        for val in obj[attrname]:
+            dsdb_dn = dsdb_DN(self.samdb, val, syntax_oid)
+
+            # all DNs should have a GUID component
+            guid = dsdb_dn.dn.get_extended_component("GUID")
+            if guid is None:
+                error_count += 1
+                self.err_incorrect_dn_GUID(obj.dn, attrname, val, dsdb_dn, "missing GUID")
+                continue
+
+            guidstr = str(misc.GUID(guid))
+
+            # check its the right GUID
+            try:
+                res = self.samdb.search(base="<GUID=%s>" % guidstr, scope=ldb.SCOPE_BASE,
+                                        attrs=['isDeleted'], controls=["extended_dn:1:1", "show_deleted:1"])
+            except ldb.LdbError, (enum, estr):
+                error_count += 1
+                self.err_incorrect_dn_GUID(obj.dn, attrname, val, dsdb_dn, "incorrect GUID")
+                continue
+
+            # the target DN might be deleted
+            if ((not self.is_deleted_objects_dn(dsdb_dn)) and
+                'isDeleted' in res[0] and
+                res[0]['isDeleted'][0].upper() == "TRUE"):
+                # note that we don't check this for the special wellKnownObjects prefix
+                # for Deleted Objects, as we expect that to be deleted
+                error_count += 1
+                self.err_deleted_dn(obj.dn, attrname, val, dsdb_dn, res[0].dn)
+                continue
+
+            # check the DN matches in string form
+            if res[0].dn.extended_str() != dsdb_dn.dn.extended_str():
+                error_count += 1
+                self.err_dn_target_mismatch(obj.dn, attrname, val, dsdb_dn,
+                                            res[0].dn, "incorrect string version of DN")
+                continue
+
+        return error_count
+
+
+
+    ################################################################
+    # check one object - calls to individual error handlers above
+    def check_object(self, dn, attrs=['*']):
+        '''check one object'''
+        if self.verbose:
+            self.report("Checking object %s" % dn)
+        res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE, controls=["extended_dn:1:1"], attrs=attrs)
+        if len(res) != 1:
+            self.report("Object %s disappeared during check" % dn)
+            return 1
+        obj = res[0]
+        error_count = 0
+        for attrname in obj:
+            if attrname == 'dn':
+                continue
+
+            # check for empty attributes
+            for val in obj[attrname]:
+                if val == '':
+                    self.err_empty_attribute(dn, attrname)
+                    error_count += 1
+                    continue
+
+            # get the syntax oid for the attribute, so we can can have
+            # special handling for some specific attribute types
+            syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(attrname)
+
+            if syntax_oid in [ dsdb.DSDB_SYNTAX_BINARY_DN, dsdb.DSDB_SYNTAX_OR_NAME,
+                               dsdb.DSDB_SYNTAX_STRING_DN, ldb.LDB_SYNTAX_DN ]:
+                # it's some form of DN, do specialised checking on those
+                error_count += self.check_dn(obj, attrname, syntax_oid)
+
+            # check for incorrectly normalised attributes
+            for val in obj[attrname]:
+                normalised = self.samdb.dsdb_normalise_attributes(self.samdb_schema, attrname, [val])
+                if len(normalised) != 1 or normalised[0] != val:
+                    self.err_normalise_mismatch(dn, attrname, obj[attrname])
+                    error_count += 1
+                    break
+        return error_count
diff --git a/source4/scripting/python/samba/hostconfig.py b/source4/scripting/python/samba/hostconfig.py
index 3e6dc6b1ddd..c50b944c987 100644
--- a/source4/scripting/python/samba/hostconfig.py
+++ b/source4/scripting/python/samba/hostconfig.py
@@ -37,7 +37,7 @@ class Hostconfig(object):
         :param session_info: Session info to use
         :param credentials: Credentials to access the SamDB with
         """
-        return SamDB(url=self.lp.get("sam database"),
+        return SamDB(url=self.lp.samdb_url(),
                      session_info=session_info, credentials=credentials,
                      lp=self.lp)
 
diff --git a/source4/scripting/python/samba/idmap.py b/source4/scripting/python/samba/idmap.py
index 93fca46edd3..9d957341de8 100644
--- a/source4/scripting/python/samba/idmap.py
+++ b/source4/scripting/python/samba/idmap.py
@@ -41,7 +41,7 @@ class IDmapDB(samba.Ldb):
 
         self.lp = lp
         if url is None:
-                url = lp.get("idmap database")
+                url = lp.private_path("idmap.ldb")
 
         super(IDmapDB, self).__init__(url=url, lp=lp, modules_dir=modules_dir,
                 session_info=session_info, credentials=credentials, flags=flags,
diff --git a/source4/scripting/python/samba/join.py b/source4/scripting/python/samba/join.py
index c0aee714070..b586e2cd5b0 100644
--- a/source4/scripting/python/samba/join.py
+++ b/source4/scripting/python/samba/join.py
@@ -36,6 +36,11 @@ import talloc
 # this makes debugging easier
 talloc.enable_null_tracking()
 
+class DCJoinException(Exception):
+
+    def __init__(self, msg):
+        super(DCJoinException, self).__init__("Can't join, error: %s" % msg)
+
 
 class dc_join(object):
     '''perform a DC join'''
@@ -62,6 +67,12 @@ class dc_join(object):
                           session_info=system_session(),
                           credentials=ctx.creds, lp=ctx.lp)
 
+        try:
+            ctx.samdb.search(scope=ldb.SCOPE_ONELEVEL, attrs=["dn"])
+        except ldb.LdbError, (enum, estr):
+            raise DCJoinException(estr)
+
+
         ctx.myname = netbios_name
         ctx.samname = "%s$" % ctx.myname
         ctx.base_dn = str(ctx.samdb.get_default_basedn())
diff --git a/source4/scripting/python/samba/netcmd/__init__.py b/source4/scripting/python/samba/netcmd/__init__.py
index cf514d5c49d..1373cb289b6 100644
--- a/source4/scripting/python/samba/netcmd/__init__.py
+++ b/source4/scripting/python/samba/netcmd/__init__.py
@@ -2,6 +2,7 @@
 
 # Unix SMB/CIFS implementation.
 # Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2009
+# Copyright (C) Theresa Halloran <theresahalloran@gmail.com> 2011
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -179,10 +180,6 @@ from samba.netcmd.domainlevel import cmd_domainlevel
 commands["domainlevel"] = cmd_domainlevel()
 from samba.netcmd.setpassword import cmd_setpassword
 commands["setpassword"] = cmd_setpassword()
-from samba.netcmd.setexpiry import cmd_setexpiry
-commands["setexpiry"] = cmd_setexpiry()
-from samba.netcmd.enableaccount import cmd_enableaccount
-commands["enableaccount"] = cmd_enableaccount()
 from samba.netcmd.newuser import cmd_newuser
 commands["newuser"] = cmd_newuser()
 from samba.netcmd.netacl import cmd_acl
@@ -215,3 +212,5 @@ from samba.netcmd.ldapcmp import cmd_ldapcmp
 commands["ldapcmp"] = cmd_ldapcmp()
 from samba.netcmd.testparm import cmd_testparm
 commands["testparm"] =  cmd_testparm()
+from samba.netcmd.dbcheck import cmd_dbcheck
+commands["dbcheck"] =  cmd_dbcheck()
diff --git a/source4/scripting/python/samba/netcmd/dbcheck.py b/source4/scripting/python/samba/netcmd/dbcheck.py
new file mode 100644
index 00000000000..3cc50eb814a
--- /dev/null
+++ b/source4/scripting/python/samba/netcmd/dbcheck.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+#
+# Samba4 AD database checker
+#
+# Copyright (C) Andrew Tridgell 2011
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import ldb, sys
+import samba.getopt as options
+from samba.auth import system_session
+from samba.samdb import SamDB
+from samba.netcmd import (
+    Command,
+    CommandError,
+    Option
+    )
+from samba.dbchecker import dbcheck
+
+
+class cmd_dbcheck(Command):
+    """check local AD database for errors"""
+    synopsis = "dbcheck <DN> [options]"
+
+    takes_optiongroups = {
+        "sambaopts": options.SambaOptions,
+        "versionopts": options.VersionOptions,
+        "credopts": options.CredentialsOptionsDouble,
+    }
+
+    takes_args = ["DN?"]
+
+    takes_options = [
+        Option("--scope", dest="scope", default="SUB",
+            help="Pass search scope that builds DN list. Options: SUB, ONE, BASE"),
+        Option("--fix", dest="fix", default=False, action='store_true',
+               help='Fix any errors found'),
+        Option("--yes", dest="yes", default=False, action='store_true',
+               help="don't confirm changes, just do them all as a single transaction"),
+        Option("--cross-ncs", dest="cross_ncs", default=False, action='store_true',
+               help="cross naming context boundaries"),
+        Option("-v", "--verbose", dest="verbose", action="store_true", default=False,
+            help="Print more details of checking"),
+        Option("--quiet", dest="quiet", action="store_true", default=False,
+            help="don't print details of checking"),
+        Option("--attrs", dest="attrs", default=None, help="list of attributes to check (space separated)"),
+        Option("-H", help="LDB URL for database or target server (defaults to local SAM database)", type=str),
+        ]
+
+    def run(self, DN=None, H=None, verbose=False, fix=False, yes=False, cross_ncs=False, quiet=False,
+            scope="SUB", credopts=None, sambaopts=None, versionopts=None, attrs=None):
+
+        lp = sambaopts.get_loadparm()
+        creds = credopts.get_credentials(lp, fallback_machine=True)
+
+        samdb = SamDB(session_info=system_session(), url=H,
+                      credentials=creds, lp=lp)
+        if H is None:
+            samdb_schema = samdb
+        else:
+            samdb_schema = SamDB(session_info=system_session(), url=None,
+                                 credentials=creds, lp=lp)
+
+        scope_map = { "SUB": ldb.SCOPE_SUBTREE, "BASE":ldb.SCOPE_BASE, "ONE":ldb.SCOPE_ONELEVEL }
+        scope = scope.upper()
+        if not scope in scope_map:
+            raise CommandError("Unknown scope %s" % scope)
+        search_scope = scope_map[scope]
+
+        controls = []
+        if H is not None:
+            controls.append('paged_results:1:1000')
+        if cross_ncs:
+            controls.append("search_options:1:2")
+
+        if not attrs:
+            attrs = ['*']
+        else:
+            attrs = attrs.split()
+
+        if yes and fix:
+            samdb.transaction_start()
+
+        chk = dbcheck(samdb, samdb_schema=samdb_schema, verbose=verbose, fix=fix, yes=yes, quiet=quiet)
+        error_count = chk.check_database(DN=DN, scope=search_scope, controls=controls, attrs=attrs)
+
+        if yes and fix:
+            samdb.transaction_commit()
+
+        if error_count != 0:
+            sys.exit(1)
+
diff --git a/source4/scripting/python/samba/netcmd/drs.py b/source4/scripting/python/samba/netcmd/drs.py
index 56c0e39a591..61717a70e98 100644
--- a/source4/scripting/python/samba/netcmd/drs.py
+++ b/source4/scripting/python/samba/netcmd/drs.py
@@ -233,6 +233,39 @@ class cmd_drs_kcc(Command):
         self.message("Consistency check on %s successful." % DC)
 
 
+def drs_local_replicate(self, SOURCE_DC, NC):
+    '''replicate from a source DC to the local SAM'''
+    self.server = SOURCE_DC
+    drsuapi_connect(self)
+
+    self.local_samdb = SamDB(session_info=system_session(), url=None,
+                             credentials=self.creds, lp=self.lp)
+
+    self.samdb = SamDB(url="ldap://%s" % self.server,
+                       session_info=system_session(),
+                       credentials=self.creds, lp=self.lp)
+
+    # work out the source and destination GUIDs
+    res = self.local_samdb.search(base="", scope=ldb.SCOPE_BASE, attrs=["dsServiceName"])
+    self.ntds_dn = res[0]["dsServiceName"][0]
+
+    res = self.local_samdb.search(base=self.ntds_dn, scope=ldb.SCOPE_BASE, attrs=["objectGUID"])
+    self.ntds_guid = misc.GUID(self.samdb.schema_format_value("objectGUID", res[0]["objectGUID"][0]))
+
+
+    source_dsa_invocation_id = misc.GUID(self.samdb.get_invocation_id())
+    destination_dsa_guid = self.ntds_guid
+
+    self.samdb.transaction_start()
+    repl = drs_utils.drs_Replicate("ncacn_ip_tcp:%s[seal]" % self.server, self.lp,
+                                   self.creds, self.local_samdb)
+    try:
+        repl.replicate(NC, source_dsa_invocation_id, destination_dsa_guid)
+    except Exception, e:
+        raise CommandError("Error replicating DN %s" % NC, e)
+    self.samdb.transaction_commit()
+
+
 
 class cmd_drs_replicate(Command):
     """replicate a naming context between two DCs"""
@@ -250,9 +283,10 @@ class cmd_drs_replicate(Command):
     takes_options = [
         Option("--add-ref", help="use ADD_REF to add to repsTo on source", action="store_true"),
         Option("--sync-forced", help="use SYNC_FORCED to force inbound replication", action="store_true"),
+        Option("--local", help="pull changes directly into the local database (destination DC is ignored)", action="store_true"),
         ]
 
-    def run(self, DEST_DC, SOURCE_DC, NC, add_ref=False, sync_forced=False,
+    def run(self, DEST_DC, SOURCE_DC, NC, add_ref=False, sync_forced=False, local=False,
             sambaopts=None,
             credopts=None, versionopts=None, server=None):
 
@@ -261,6 +295,10 @@ class cmd_drs_replicate(Command):
 
         self.creds = credopts.get_credentials(self.lp, fallback_machine=True)
 
+        if local:
+            drs_local_replicate(self, SOURCE_DC, NC)
+            return
+
         drsuapi_connect(self)
         samdb_connect(self)
 
diff --git a/source4/scripting/python/samba/netcmd/enableaccount.py b/source4/scripting/python/samba/netcmd/enableaccount.py
deleted file mode 100644
index 3ceddb3fd97..00000000000
--- a/source4/scripting/python/samba/netcmd/enableaccount.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env python
-#
-# Enables an user account on a Samba4 server
-# Copyright Jelmer Vernooij 2008
-#
-# Based on the original in EJS:
-# Copyright Andrew Tridgell 2005
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import samba.getopt as options
-
-from samba.auth import system_session
-from samba.netcmd import Command, CommandError, Option
-from samba.samdb import SamDB
-
-class cmd_enableaccount(Command):
-    """Enables a user"""
-
-    synopsis = "enableaccount [username] [options]"
-
-    takes_optiongroups = {
-        "sambaopts": options.SambaOptions,
-        "versionopts": options.VersionOptions,
-        "credopts": options.CredentialsOptions,
-    }
-
-    takes_options = [
-        Option("-H", help="LDB URL for database or target server", type=str),
-        Option("--filter", help="LDAP Filter to set password on", type=str),
-        ]
-
-    takes_args = ["username?"]
-
-    def run(self, username=None, sambaopts=None, credopts=None,
-            versionopts=None, filter=None, H=None):
-        if username is None and filter is None:
-            raise CommandError("Either the username or '--filter' must be specified!")
-
-        if filter is None:
-            filter = "(&(objectClass=user)(sAMAccountName=%s))" % (username)
-
-        lp = sambaopts.get_loadparm()
-        creds = credopts.get_credentials(lp, fallback_machine=True)
-
-        samdb = SamDB(url=H, session_info=system_session(),
-            credentials=creds, lp=lp)
-        samdb.enable_account(filter)
diff --git a/source4/scripting/python/samba/netcmd/gpo.py b/source4/scripting/python/samba/netcmd/gpo.py
index 19007b361cb..fac91670766 100644
--- a/source4/scripting/python/samba/netcmd/gpo.py
+++ b/source4/scripting/python/samba/netcmd/gpo.py
@@ -126,7 +126,7 @@ class cmd_listall(Command):
             print("display name : %s" % m['displayName'][0])
             print("path         : %s" % m['gPCFileSysPath'][0])
             print("dn           : %s" % m.dn)
-            print("version      : %s" % attr_default(m, 'version', '0'))
+            print("version      : %s" % attr_default(m, 'versionNumber', '0'))
             print("flags        : %s" % flags_string(gpo_flags, int(attr_default(m, 'flags', 0))))
             print("")
 
diff --git a/source4/scripting/python/samba/netcmd/group.py b/source4/scripting/python/samba/netcmd/group.py
index 620a7be8664..95db21adfcf 100644
--- a/source4/scripting/python/samba/netcmd/group.py
+++ b/source4/scripting/python/samba/netcmd/group.py
@@ -85,6 +85,7 @@ class cmd_group_add(Command):
                           description=description, mailaddress=mail_address, notes=notes)
         except Exception, e:
             raise CommandError('Failed to create group "%s"' % groupname, e)
+        print("Added group %s" % groupname)
 
 
 class cmd_group_delete(Command):
@@ -115,6 +116,7 @@ class cmd_group_delete(Command):
             samdb.deletegroup(groupname)
         except Exception, e:
             raise CommandError('Failed to remove group "%s"' % groupname, e)
+        print("Deleted group %s" % groupname)
 
 
 class cmd_group_add_members(Command):
@@ -146,6 +148,7 @@ class cmd_group_add_members(Command):
             samdb.add_remove_group_members(groupname, listofmembers, add_members_operation=True)
         except Exception, e:
             raise CommandError('Failed to add members "%s" to group "%s"' % (listofmembers, groupname), e)
+        print("Added members to group %s" % groupname)
 
 
 class cmd_group_remove_members(Command):
@@ -177,6 +180,7 @@ class cmd_group_remove_members(Command):
             samdb.add_remove_group_members(groupname, listofmembers, add_members_operation=False)
         except Exception, e:
             raise CommandError('Failed to remove members "%s" from group "%s"' % (listofmembers, groupname), e)
+        print("Removed members from group %s" % groupname)
 
 
 class cmd_group(SuperCommand):
diff --git a/source4/scripting/python/samba/netcmd/join.py b/source4/scripting/python/samba/netcmd/join.py
index 507253ab817..820709c9e37 100644
--- a/source4/scripting/python/samba/netcmd/join.py
+++ b/source4/scripting/python/samba/netcmd/join.py
@@ -22,7 +22,7 @@ import samba.getopt as options
 
 from samba.net import Net, LIBNET_JOIN_AUTOMATIC
 from samba.netcmd import Command, CommandError, Option
-from samba.dcerpc.misc import SEC_CHAN_WKSTA, SEC_CHAN_BDC
+from samba.dcerpc.misc import SEC_CHAN_WKSTA
 from samba.join import join_RODC, join_DC
 
 class cmd_join(Command):
@@ -39,12 +39,13 @@ class cmd_join(Command):
     takes_options = [
         Option("--server", help="DC to join", type=str),
         Option("--site", help="site to join", type=str),
+        Option("--targetdir", help="where to store provision", type=str),
         ]
 
     takes_args = ["domain", "role?"]
 
     def run(self, domain, role=None, sambaopts=None, credopts=None,
-            versionopts=None, server=None, site=None):
+            versionopts=None, server=None, site=None, targetdir=None):
         lp = sambaopts.get_loadparm()
         creds = credopts.get_credentials(lp)
         net = Net(creds, lp, server=credopts.ipaddress)
@@ -58,21 +59,20 @@ class cmd_join(Command):
             role = role.upper()
 
         if role is None or role == "MEMBER":
-            secure_channel_type = SEC_CHAN_WKSTA
+            (join_password, sid, domain_name) = net.join_member(domain,
+                                                                netbios_name,
+                                                                LIBNET_JOIN_AUTOMATIC)
+
+            self.outf.write("Joined domain %s (%s)\n" % (domain_name, sid))
+            return
+
         elif role == "DC":
             join_DC(server=server, creds=creds, lp=lp, domain=domain,
-                    site=site, netbios_name=netbios_name)
+                    site=site, netbios_name=netbios_name, targetdir=targetdir)
             return
         elif role == "RODC":
             join_RODC(server=server, creds=creds, lp=lp, domain=domain,
-                      site=site, netbios_name=netbios_name)
+                      site=site, netbios_name=netbios_name, targetdir=targetdir)
             return
         else:
             raise CommandError("Invalid role %s (possible values: MEMBER, BDC, RODC)" % role)
-
-        (join_password, sid, domain_name) = net.join(domain,
-                                                     netbios_name,
-                                                     secure_channel_type,
-                                                     LIBNET_JOIN_AUTOMATIC)
-
-        self.outf.write("Joined domain %s (%s)\n" % (domain_name, sid))
diff --git a/source4/scripting/python/samba/netcmd/setexpiry.py b/source4/scripting/python/samba/netcmd/setexpiry.py
deleted file mode 100644
index bd8ea166fac..00000000000
--- a/source4/scripting/python/samba/netcmd/setexpiry.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-#
-# Sets the user password expiry on a Samba4 server
-# Copyright Jelmer Vernooij 2008
-#
-# Based on the original in EJS:
-# Copyright Andrew Tridgell 2005
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-from samba.netcmd import Command, CommandError, Option
-
-import samba.getopt as options
-
-from samba.auth import system_session
-from samba.samdb import SamDB
-
-class cmd_setexpiry(Command):
-    """Sets the expiration of a user account"""
-
-    synopsis = "setexpiry [username] [options]"
-
-    takes_optiongroups = {
-        "sambaopts": options.SambaOptions,
-        "versionopts": options.VersionOptions,
-        "credopts": options.CredentialsOptions,
-    }
-
-    takes_options = [
-        Option("-H", help="LDB URL for database or target server", type=str),
-        Option("--filter", help="LDAP Filter to set password on", type=str),
-        Option("--days", help="Days to expiry", type=int),
-        Option("--noexpiry", help="Password does never expire", action="store_true"),
-    ]
-
-    takes_args = ["username?"]
-
-    def run(self, username=None, sambaopts=None, credopts=None,
-            versionopts=None, H=None, filter=None, days=None, noexpiry=None):
-        if username is None and filter is None:
-            raise CommandError("Either the username or '--filter' must be specified!")
-
-        if filter is None:
-            filter = "(&(objectClass=user)(sAMAccountName=%s))" % (username)
-
-        lp = sambaopts.get_loadparm()
-        creds = credopts.get_credentials(lp)
-
-        if days is None:
-            days = 0
-
-        samdb = SamDB(url=H, session_info=system_session(),
-            credentials=creds, lp=lp)
-
-        samdb.setexpiry(filter, days*24*3600, no_expiry_req=noexpiry)
diff --git a/source4/scripting/python/samba/netcmd/user.py b/source4/scripting/python/samba/netcmd/user.py
index a5750b50107..6acf52d790f 100644
--- a/source4/scripting/python/samba/netcmd/user.py
+++ b/source4/scripting/python/samba/netcmd/user.py
@@ -3,6 +3,7 @@
 # user management
 #
 # Copyright Jelmer Vernooij 2010 <jelmer@samba.org>
+# Copyright Theresa Halloran 2011 <theresahalloran@gmail.com>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,6 +20,10 @@
 #
 
 import samba.getopt as options
+import sys
+from samba.auth import system_session
+from samba.samdb import SamDB
+
 
 from samba.net import Net
 
@@ -26,6 +31,7 @@ from samba.netcmd import (
     Command,
     CommandError,
     SuperCommand,
+    Option,
     )
 
 class cmd_user_add(Command):
@@ -70,6 +76,86 @@ class cmd_user_delete(Command):
         except RuntimeError, msg:
             raise CommandError("Failed to delete user %s: %s" % (name, msg))
 
+class cmd_user_enable(Command):
+    """Enables a user"""
+
+    synopsis = "%prog user enable <username> [options]"
+
+
+    takes_optiongroups = {
+        "sambaopts": options.SambaOptions,
+        "versionopts": options.VersionOptions,
+        "credopts": options.CredentialsOptions,
+    }
+
+    takes_options = [
+        Option("-H", help="LDB URL for database or target server", type=str),
+        Option("--filter", help="LDAP Filter to set password on", type=str),
+        ]
+
+    takes_args = ["username?"]
+
+    def run(self, username=None, sambaopts=None, credopts=None,
+            versionopts=None, filter=None, H=None):
+        if username is None and filter is None:
+            raise CommandError("Either the username or '--filter' must be specified!")
+
+        if filter is None:
+            filter = "(&(objectClass=user)(sAMAccountName=%s))" % (username)
+
+        lp = sambaopts.get_loadparm()
+        creds = credopts.get_credentials(lp, fallback_machine=True)
+
+        samdb = SamDB(url=H, session_info=system_session(),
+            credentials=creds, lp=lp)
+        try:
+            samdb.enable_account(filter)
+        except Exception, msg:
+            raise CommandError("Failed to enable user %s: %s" % (username or filter, msg))
+        print("Enabled user %s" % (username or filter))
+
+
+class cmd_user_setexpiry(Command):
+    """Sets the expiration of a user account"""
+
+    synopsis = "%prog user setexpiry <username> [options]"
+
+    takes_optiongroups = {
+        "sambaopts": options.SambaOptions,
+        "versionopts": options.VersionOptions,
+        "credopts": options.CredentialsOptions,
+    }
+
+    takes_options = [
+        Option("-H", help="LDB URL for database or target server", type=str),
+        Option("--filter", help="LDAP Filter to set password on", type=str),
+        Option("--days", help="Days to expiry", type=int),
+        Option("--noexpiry", help="Password does never expire", action="store_true"),
+    ]
+
+    takes_args = ["username?"]
+    def run(self, username=None, sambaopts=None, credopts=None,
+            versionopts=None, H=None, filter=None, days=None, noexpiry=None):
+        if username is None and filter is None:
+            raise CommandError("Either the username or '--filter' must be specified!")
+
+        if filter is None:
+            filter = "(&(objectClass=user)(sAMAccountName=%s))" % (username)
+
+        lp = sambaopts.get_loadparm()
+        creds = credopts.get_credentials(lp)
+
+        if days is None:
+            days = 0
+
+        samdb = SamDB(url=H, session_info=system_session(),
+            credentials=creds, lp=lp)
+
+        try:
+            samdb.setexpiry(filter, days*24*3600, no_expiry_req=noexpiry)
+        except Exception, msg:
+            raise CommandError("Failed to set expiry for user %s: %s" % (username or filter, msg))
+        print("Set expiry for user %s to %u days" % (username or filter, days))
 
 class cmd_user(SuperCommand):
     """User management [server connection needed]"""
@@ -77,4 +163,5 @@ class cmd_user(SuperCommand):
     subcommands = {}
     subcommands["add"] = cmd_user_add()
     subcommands["delete"] = cmd_user_delete()
-
+    subcommands["enable"] = cmd_user_enable()
+    subcommands["setexpiry"] = cmd_user_setexpiry()
diff --git a/source4/scripting/python/samba/provision/__init__.py b/source4/scripting/python/samba/provision/__init__.py
index ff9b00122d6..5aabd36c1a8 100644
--- a/source4/scripting/python/samba/provision/__init__.py
+++ b/source4/scripting/python/samba/provision/__init__.py
@@ -38,23 +38,23 @@ import uuid
 import socket
 import urllib
 import shutil
+import string
 
 import ldb
 
 from samba.auth import system_session, admin_session
 import samba
+from samba.dsdb import DS_DOMAIN_FUNCTION_2000
 from samba import (
     Ldb,
     check_all_substituted,
-    in_source_tree,
-    source_tree_topdir,
     read_and_sub_file,
     setup_file,
     substitute_var,
     valid_netbios_name,
     version,
     )
-from samba.dcerpc import security
+from samba.dcerpc import security, misc
 from samba.dcerpc.misc import (
     SEC_CHAN_BDC,
     SEC_CHAN_WKSTA,
@@ -94,19 +94,6 @@ def setup_path(file):
 
 # "get_schema_descriptor" is located in "schema.py"
 
-def get_sites_descriptor(domain_sid):
-    sddl = "D:(A;;RPLCLORC;;;AU)" \
-           "(A;;RPWPCRCCLCLORCWOWDSW;;;EA)" \
-           "(A;;RPWPCRCCDCLCLORCWOWDSDDTSW;;;SY)" \
-           "S:AI(AU;CISA;CCDCSDDT;;;WD)" \
-           "(OU;CIIOSA;CR;;f0f8ffab-1191-11d0-a060-00aa006c33ed;WD)" \
-           "(OU;CIIOSA;WP;f30e3bbe-9ff0-11d1-b603-0000f80367c1;bf967ab3-0de6-11d0-a285-00aa003049e2;WD)" \
-           "(OU;CIIOSA;WP;f30e3bbf-9ff0-11d1-b603-0000f80367c1;bf967ab3-0de6-11d0-a285-00aa003049e2;WD)" \
-           "(OU;CIIOSA;WP;3e10944c-c354-11d0-aff8-0000f80367c1;b7b13124-b82e-11d0-afee-0000f80367c1;WD)"
-    sec = security.descriptor.from_sddl(sddl, domain_sid)
-    return ndr_pack(sec)
-
-
 def get_config_descriptor(domain_sid):
     sddl = "O:EAG:EAD:(OA;;CR;1131f6aa-9c07-11d1-f79f-00c04fc2dcd2;;ED)" \
            "(OA;;CR;1131f6ab-9c07-11d1-f79f-00c04fc2dcd2;;ED)" \
@@ -217,8 +204,112 @@ class ProvisionNames(object):
         self.sitename = None
         self.smbconf = None
 
+def find_provision_key_parameters(samdb, secretsdb, idmapdb, paths, smbconf, lp):
+    """Get key provision parameters (realm, domain, ...) from a given provision
+
+    :param samdb: An LDB object connected to the sam.ldb file
+    :param secretsdb: An LDB object connected to the secrets.ldb file
+    :param idmapdb: An LDB object connected to the idmap.ldb file
+    :param paths: A list of path to provision object
+    :param smbconf: Path to the smb.conf file
+    :param lp: A LoadParm object
+    :return: A list of key provision parameters
+    """
+    names = ProvisionNames()
+    names.adminpass = None
+
+    # NT domain, kerberos realm, root dn, domain dn, domain dns name
+    names.domain = string.upper(lp.get("workgroup"))
+    names.realm = lp.get("realm")
+    basedn = "DC=" + names.realm.replace(".",",DC=")
+    names.dnsdomain = names.realm.lower()
+    names.realm = string.upper(names.realm)
+    # netbiosname
+    # Get the netbiosname first (could be obtained from smb.conf in theory)
+    res = secretsdb.search(expression="(flatname=%s)" %
+                            names.domain,base="CN=Primary Domains",
+                            scope=ldb.SCOPE_SUBTREE, attrs=["sAMAccountName"])
+    names.netbiosname = str(res[0]["sAMAccountName"]).replace("$","")
+
+    names.smbconf = smbconf
+
+    # That's a bit simplistic but it's ok as long as we have only 3
+    # partitions
+    current = samdb.search(expression="(objectClass=*)",
+        base="", scope=ldb.SCOPE_BASE,
+        attrs=["defaultNamingContext", "schemaNamingContext",
+               "configurationNamingContext","rootDomainNamingContext"])
+
+    names.configdn = current[0]["configurationNamingContext"]
+    configdn = str(names.configdn)
+    names.schemadn = current[0]["schemaNamingContext"]
+    if not (ldb.Dn(samdb, basedn) == (ldb.Dn(samdb,
+                                       current[0]["defaultNamingContext"][0]))):
+        raise ProvisioningError(("basedn in %s (%s) and from %s (%s)"
+                                 "is not the same ..." % (paths.samdb,
+                                    str(current[0]["defaultNamingContext"][0]),
+                                    paths.smbconf, basedn)))
+
+    names.domaindn=current[0]["defaultNamingContext"]
+    names.rootdn=current[0]["rootDomainNamingContext"]
+    # default site name
+    res3 = samdb.search(expression="(objectClass=site)",
+        base="CN=Sites," + configdn, scope=ldb.SCOPE_ONELEVEL, attrs=["cn"])
+    names.sitename = str(res3[0]["cn"])
+
+    # dns hostname and server dn
+    res4 = samdb.search(expression="(CN=%s)" % names.netbiosname,
+                            base="OU=Domain Controllers,%s" % basedn,
+                            scope=ldb.SCOPE_ONELEVEL, attrs=["dNSHostName"])
+    names.hostname = str(res4[0]["dNSHostName"]).replace("." + names.dnsdomain,"")
+
+    server_res = samdb.search(expression="serverReference=%s" % res4[0].dn,
+                                attrs=[], base=configdn)
+    names.serverdn = server_res[0].dn
+
+    # invocation id/objectguid
+    res5 = samdb.search(expression="(objectClass=*)",
+            base="CN=NTDS Settings,%s" % str(names.serverdn), scope=ldb.SCOPE_BASE,
+            attrs=["invocationID", "objectGUID"])
+    names.invocation = str(ndr_unpack(misc.GUID, res5[0]["invocationId"][0]))
+    names.ntdsguid = str(ndr_unpack(misc.GUID, res5[0]["objectGUID"][0]))
+
+    # domain guid/sid
+    res6 = samdb.search(expression="(objectClass=*)", base=basedn,
+            scope=ldb.SCOPE_BASE, attrs=["objectGUID",
+                "objectSid","msDS-Behavior-Version" ])
+    names.domainguid = str(ndr_unpack(misc.GUID, res6[0]["objectGUID"][0]))
+    names.domainsid = ndr_unpack( security.dom_sid, res6[0]["objectSid"][0])
+    if res6[0].get("msDS-Behavior-Version") is None or \
+        int(res6[0]["msDS-Behavior-Version"][0]) < DS_DOMAIN_FUNCTION_2000:
+        names.domainlevel = DS_DOMAIN_FUNCTION_2000
+    else:
+        names.domainlevel = int(res6[0]["msDS-Behavior-Version"][0])
+
+    # policy guid
+    res7 = samdb.search(expression="(displayName=Default Domain Policy)",
+                        base="CN=Policies,CN=System," + basedn,
+                        scope=ldb.SCOPE_ONELEVEL, attrs=["cn","displayName"])
+    names.policyid = str(res7[0]["cn"]).replace("{","").replace("}","")
+    # dc policy guid
+    res8 = samdb.search(expression="(displayName=Default Domain Controllers"
+                                   " Policy)",
+                            base="CN=Policies,CN=System," + basedn,
+                            scope=ldb.SCOPE_ONELEVEL, attrs=["cn","displayName"])
+    if len(res8) == 1:
+        names.policyid_dc = str(res8[0]["cn"]).replace("{","").replace("}","")
+    else:
+        names.policyid_dc = None
+    res9 = idmapdb.search(expression="(cn=%s)" %
+                            (security.SID_BUILTIN_ADMINISTRATORS),
+                            attrs=["xidNumber"])
+    if len(res9) == 1:
+        names.wheel_gid = res9[0]["xidNumber"]
+    else:
+        raise ProvisioningError("Unable to find uid/gid for Domain Admins rid")
+    return names
 
-def update_provision_usn(samdb, low, high, replace=False):
+def update_provision_usn(samdb, low, high, id, replace=False):
     """Update the field provisionUSN in sam.ldb
 
     This field is used to track range of USN modified by provision and
@@ -229,6 +320,7 @@ def update_provision_usn(samdb, low, high, replace=False):
     :param samdb: An LDB object connect to sam.ldb
     :param low: The lowest USN modified by this upgrade
     :param high: The highest USN modified by this upgrade
+    :param id: The invocation id of the samba's dc
     :param replace: A boolean indicating if the range should replace any
                     existing one or appended (default)
     """
@@ -240,17 +332,24 @@ def update_provision_usn(samdb, low, high, replace=False):
                                 scope=ldb.SCOPE_SUBTREE,
                                 attrs=[LAST_PROVISION_USN_ATTRIBUTE, "dn"])
         for e in entry[0][LAST_PROVISION_USN_ATTRIBUTE]:
+            if not re.search(';', e):
+                e = "%s;%s" % (e, id)
             tab.append(str(e))
 
-    tab.append("%s-%s" % (low, high))
+    tab.append("%s-%s;%s" % (low, high, id))
     delta = ldb.Message()
     delta.dn = ldb.Dn(samdb, "@PROVISION")
     delta[LAST_PROVISION_USN_ATTRIBUTE] = ldb.MessageElement(tab,
         ldb.FLAG_MOD_REPLACE, LAST_PROVISION_USN_ATTRIBUTE)
+    entry = samdb.search(expression="(&(dn=@PROVISION)(provisionnerID=*))",
+                            base="", scope=ldb.SCOPE_SUBTREE,
+                            attrs=["provisionnerID"])
+    if len(entry) == 0 or len(entry[0]) == 0:
+        delta["provisionnerID"] = ldb.MessageElement(id, ldb.FLAG_MOD_ADD, "provisionnerID")
     samdb.modify(delta)
 
 
-def set_provision_usn(samdb, low, high):
+def set_provision_usn(samdb, low, high, id):
     """Set the field provisionUSN in sam.ldb
     This field is used to track range of USN modified by provision and
     upgradeprovision.
@@ -259,9 +358,12 @@ def set_provision_usn(samdb, low, high):
 
     :param samdb: An LDB object connect to sam.ldb
     :param low: The lowest USN modified by this upgrade
-    :param high: The highest USN modified by this upgrade"""
+    :param high: The highest USN modified by this upgrade
+    :param id: The invocationId of the provision"""
+
     tab = []
-    tab.append("%s-%s" % (low, high))
+    tab.append("%s-%s;%s" % (low, high, id))
+
     delta = ldb.Message()
     delta.dn = ldb.Dn(samdb, "@PROVISION")
     delta[LAST_PROVISION_USN_ATTRIBUTE] = ldb.MessageElement(tab,
@@ -286,25 +388,36 @@ def get_max_usn(samdb,basedn):
 
 
 def get_last_provision_usn(sam):
-    """Get the lastest USN modified by a provision or an upgradeprovision
+    """Get USNs ranges modified by a provision or an upgradeprovision
 
     :param sam: An LDB object pointing to the sam.ldb
-    :return: an integer corresponding to the highest USN modified by
-        (upgrade)provision, 0 is this value is unknown
+    :return: a dictionnary which keys are invocation id and values are an array
+             of integer representing the different ranges
     """
     entry = sam.search(expression="(&(dn=@PROVISION)(%s=*))" %
                         LAST_PROVISION_USN_ATTRIBUTE,
                         base="", scope=ldb.SCOPE_SUBTREE,
-                        attrs=[LAST_PROVISION_USN_ATTRIBUTE])
+                        attrs=[LAST_PROVISION_USN_ATTRIBUTE, "provisionnerID"])
     if len(entry):
-        range = []
-        idx = 0
+        myids = []
+        range = {}
         p = re.compile(r'-')
+        if entry[0].get("provisionnerID"):
+            for e in entry[0]["provisionnerID"]:
+                myids.append(str(e))
         for r in entry[0][LAST_PROVISION_USN_ATTRIBUTE]:
-            tab = p.split(str(r))
-            range.append(tab[0])
-            range.append(tab[1])
-            idx = idx + 1
+            tab1 = str(r).split(';')
+            if len(tab1) == 2:
+                id = tab1[1]
+            else:
+                id = "default"
+            if (len(myids) > 0 and id not in myids):
+                continue
+            tab2 = p.split(tab1[0])
+            if range.get(id) == None:
+                range[id] = []
+            range[id].append(tab2[0])
+            range[id].append(tab2[1])
         return range
     else:
         return None
@@ -328,7 +441,7 @@ def check_install(lp, session_info, credentials):
     """
     if lp.get("realm") == "":
         raise Exception("Realm empty")
-    samdb = Ldb(lp.get("sam database"), session_info=session_info,
+    samdb = Ldb(lp.samdb_url(), session_info=session_info,
             credentials=credentials, lp=lp)
     if len(samdb.search("(cn=Administrator)")) != 1:
         raise ProvisioningError("No administrator account found")
@@ -413,12 +526,9 @@ def provision_paths_from_lp(lp, dnsdomain):
     paths.keytab = "secrets.keytab"
 
     paths.shareconf = os.path.join(paths.private_dir, "share.ldb")
-    paths.samdb = os.path.join(paths.private_dir,
-        lp.get("sam database") or "samdb.ldb")
-    paths.idmapdb = os.path.join(paths.private_dir,
-        lp.get("idmap database") or "idmap.ldb")
-    paths.secrets = os.path.join(paths.private_dir,
-        lp.get("secrets database") or "secrets.ldb")
+    paths.samdb = os.path.join(paths.private_dir, "sam.ldb")
+    paths.idmapdb = os.path.join(paths.private_dir, "idmap.ldb")
+    paths.secrets = os.path.join(paths.private_dir, "secrets.ldb")
     paths.privilege = os.path.join(paths.private_dir, "privilege.ldb")
     paths.dns = os.path.join(paths.private_dir, "dns", dnsdomain + ".zone")
     paths.dns_update_list = os.path.join(paths.private_dir, "dns_update_list")
@@ -608,11 +718,6 @@ def make_smbconf(smbconf, hostname, domain, realm, serverrole,
         privatedir_line = ""
         lockdir_line = ""
 
-    if sid_generator == "internal":
-        sid_generator_line = ""
-    else:
-        sid_generator_line = "sid generator = " + sid_generator
-
     sysvol = os.path.join(lp.get("lock dir"), "sysvol")
     netlogon = os.path.join(sysvol, realm.lower(), "scripts")
 
@@ -624,7 +729,6 @@ def make_smbconf(smbconf, hostname, domain, realm, serverrole,
             "SERVERROLE": serverrole,
             "NETLOGONPATH": netlogon,
             "SYSVOLPATH": sysvol,
-            "SIDGENERATOR_LINE": sid_generator_line,
             "PRIVATEDIR_LINE": privatedir_line,
             "LOCKDIR_LINE": lockdir_line
             })
@@ -1166,6 +1270,11 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
             "DESCRIPTOR": descr,
             })
 
+        # Now register this container in the root of the forest
+        msg = ldb.Message(ldb.Dn(samdb, names.domaindn))
+        msg["subRefs"] = ldb.MessageElement(names.configdn , ldb.FLAG_MOD_ADD,
+                    "subRefs")
+
         # The LDIF here was created when the Schema object was constructed
         logger.info("Setting up sam.ldb schema")
         samdb.add_ldif(schema.schema_dn_add, controls=["relax:0"])
@@ -1196,7 +1305,6 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
         samdb.invocation_id = invocationid
 
         logger.info("Setting up sam.ldb configuration data")
-        descr = b64encode(get_sites_descriptor(domainsid))
         setup_add_ldif(samdb, setup_path("provision_configuration.ldif"), {
             "CONFIGDN": names.configdn,
             "NETBIOSNAME": names.netbiosname,
@@ -1208,7 +1316,6 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
             "SERVERDN": names.serverdn,
             "FOREST_FUNCTIONALITY": str(forestFunctionality),
             "DOMAIN_FUNCTIONALITY": str(domainFunctionality),
-            "SITES_DESCRIPTOR": descr
             })
 
         logger.info("Setting up display specifiers")
@@ -1365,6 +1472,25 @@ def setsysvolacl(samdb, netlogon, sysvol, gid, domainsid, dnsdomain, domaindn,
     set_gpos_acl(sysvol, dnsdomain, domainsid, domaindn, samdb, lp)
 
 
+def interface_ips_v4(lp):
+    '''return only IPv4 IPs'''
+    ips = samba.interface_ips(lp, False)
+    ret = []
+    for i in ips:
+        if i.find(':') == -1:
+            ret.append(i)
+    return ret
+
+def interface_ips_v6(lp, linklocal=False):
+    '''return only IPv6 IPs'''
+    ips = samba.interface_ips(lp, False)
+    ret = []
+    for i in ips:
+        if i.find(':') != -1 and (linklocal or i.find('%') == -1):
+            ret.append(i)
+    return ret
+
+
 def provision(logger, session_info, credentials, smbconf=None,
         targetdir=None, samdb_fill=FILL_FULL, realm=None, rootdn=None,
         domaindn=None, schemadn=None, configdn=None, serverdn=None,
@@ -1465,15 +1591,26 @@ def provision(logger, session_info, credentials, smbconf=None,
 
     if hostip is None:
         logger.info("Looking up IPv4 addresses")
-        hostips = samba.interface_ips(lp, False)
-        if len(hostips) == 0:
-            logger.warning("No external IPv4 address has been found. Using loopback.")
-            hostip = '127.0.0.1'
-        else:
+        hostips = interface_ips_v4(lp)
+        if len(hostips) > 0:
             hostip = hostips[0]
             if len(hostips) > 1:
-                logger.warning("More than one IPv4 address found. Using %s.",
+                logger.warning("More than one IPv4 address found. Using %s",
                     hostip)
+    if hostip == "127.0.0.1":
+        hostip = None
+    if hostip is None:
+        logger.warning("No IPv4 address will be assigned")
+
+    if hostip6 is None:
+        logger.info("Looking up IPv6 addresses")
+        hostips = interface_ips_v6(lp, linklocal=False)
+        if hostips:
+            hostip6 = hostips[0]
+        if len(hostips) > 1:
+            logger.warning("More than one IPv6 address found. Using %s", hostip6)
+    if hostip6 is None:
+        logger.warning("No IPv6 address will be assigned")
 
     if serverrole is None:
         serverrole = lp.get("server role")
@@ -1640,6 +1777,7 @@ def provision(logger, session_info, credentials, smbconf=None,
 
                 create_named_txt(paths.namedtxt,
                     realm=names.realm, dnsdomain=names.dnsdomain,
+                    dnsname = "%s.%s" % (names.hostname, names.dnsdomain),
                     private_dir=paths.private_dir,
                     keytab_name=paths.dns_keytab)
                 logger.info("See %s for an example configuration include file for BIND", paths.namedconf)
@@ -1649,9 +1787,9 @@ def provision(logger, session_info, credentials, smbconf=None,
             lastProvisionUSNs = get_last_provision_usn(samdb)
             maxUSN = get_max_usn(samdb, str(names.rootdn))
             if lastProvisionUSNs is not None:
-                update_provision_usn(samdb, 0, maxUSN, 1)
+                update_provision_usn(samdb, 0, maxUSN, invocationid, 1)
             else:
-                set_provision_usn(samdb, 0, maxUSN)
+                set_provision_usn(samdb, 0, maxUSN, invocationid)
 
         create_krb5_conf(paths.krb5conf,
                          dnsdomain=names.dnsdomain, hostname=names.hostname,
@@ -1740,7 +1878,7 @@ def provision_become_dc(smbconf=None, targetdir=None,
         smbconf=smbconf, targetdir=targetdir, samdb_fill=FILL_DRS,
         realm=realm, rootdn=rootdn, domaindn=domaindn, schemadn=schemadn,
         configdn=configdn, serverdn=serverdn, domain=domain,
-        hostname=hostname, hostip="127.0.0.1", domainsid=domainsid,
+        hostname=hostname, hostip=None, domainsid=domainsid,
         machinepass=machinepass, serverrole="domain controller",
         sitename=sitename)
     res.lp.set("debuglevel", str(debuglevel))
@@ -1876,7 +2014,7 @@ def create_named_conf(paths, realm, dnsdomain,
     setup_file(setup_path("named.conf.update"), paths.namedconf_update)
 
 
-def create_named_txt(path, realm, dnsdomain, private_dir,
+def create_named_txt(path, realm, dnsdomain, dnsname, private_dir,
     keytab_name):
     """Write out a file containing zone statements suitable for inclusion in a
     named.conf file (including GSS-TSIG configuration).
@@ -1889,6 +2027,7 @@ def create_named_txt(path, realm, dnsdomain, private_dir,
     """
     setup_file(setup_path("named.txt"), path, {
             "DNSDOMAIN": dnsdomain,
+            "DNSNAME" : dnsname, 
             "REALM": realm,
             "DNS_KEYTAB": keytab_name,
             "DNS_KEYTAB_ABS": os.path.join(private_dir, keytab_name),
diff --git a/source4/scripting/python/samba/samba3.py b/source4/scripting/python/samba/samba3.py
index 2c323bd0b42..ae5b20edd27 100644
--- a/source4/scripting/python/samba/samba3.py
+++ b/source4/scripting/python/samba/samba3.py
@@ -50,9 +50,12 @@ class TdbDatabase(object):
     def __init__(self, file):
         """Open a file.
 
-        :param file: Path of the file to open.
+        :param file: Path of the file to open (appending "2" if TDB2 enabled).
         """
-        self.tdb = tdb.Tdb(file, flags=os.O_RDONLY)
+        if tdb.__version__.startswith("2"):
+            self.tdb = tdb.Tdb(file + "2", flags=os.O_RDONLY)
+        else:
+            self.tdb = tdb.Tdb(file, flags=os.O_RDONLY)
         self._check_version()
 
     def _check_version(self):
diff --git a/source4/scripting/python/samba/samdb.py b/source4/scripting/python/samba/samdb.py
index 99f141e6642..72ee472764b 100644
--- a/source4/scripting/python/samba/samdb.py
+++ b/source4/scripting/python/samba/samdb.py
@@ -46,7 +46,7 @@ class SamDB(samba.Ldb):
         if not auto_connect:
             url = None
         elif url is None and lp is not None:
-            url = lp.get("sam database")
+            url = lp.samdb_url()
 
         super(SamDB, self).__init__(url=url, lp=lp, modules_dir=modules_dir,
             session_info=session_info, credentials=credentials, flags=flags,
@@ -79,6 +79,8 @@ class SamDB(samba.Ldb):
         """
         res = self.search(base=self.domain_dn(), scope=ldb.SCOPE_SUBTREE,
                           expression=search_filter, attrs=["userAccountControl"])
+        if len(res) == 0:
+                raise Exception('Unable to find user "%s"' % search_filter)
         assert(len(res) == 1)
         user_dn = res[0].dn
 
@@ -106,6 +108,8 @@ userAccountControl: %u
         """
         res = self.search(base=self.domain_dn(), scope=ldb.SCOPE_SUBTREE,
                           expression=search_filter, attrs=[])
+        if len(res) == 0:
+                raise Exception('Unable to find user "%s"' % search_filter)
         assert(len(res) == 1)
         user_dn = res[0].dn
 
@@ -138,7 +142,7 @@ pwdLastSet: 0
             "objectClass": "group"}
 
         if grouptype is not None:
-            ldbmessage["groupType"] = "%d" % grouptype
+            ldbmessage["groupType"] = self.normalise_int32(grouptype)
 
         if description is not None:
             ldbmessage["description"] = description
@@ -409,6 +413,8 @@ unicodePwd:: %s
             res = self.search(base=self.domain_dn(), scope=ldb.SCOPE_SUBTREE,
                           expression=search_filter,
                           attrs=["userAccountControl", "accountExpires"])
+            if len(res) == 0:
+                raise Exception('Unable to find user "%s"' % search_filter)
             assert(len(res) == 1)
             user_dn = res[0].dn
 
@@ -470,9 +476,14 @@ accountExpires: %u
 
     def get_attid_from_lDAPDisplayName(self, ldap_display_name,
             is_schema_nc=False):
+        '''return the attribute ID for a LDAP attribute as an integer as found in DRSUAPI'''
         return dsdb._dsdb_get_attid_from_lDAPDisplayName(self,
             ldap_display_name, is_schema_nc)
 
+    def get_syntax_oid_from_lDAPDisplayName(self, ldap_display_name):
+        '''return the syntax OID for a LDAP attribute as a string'''
+        return dsdb._dsdb_get_syntax_oid_from_lDAPDisplayName(self, ldap_display_name)
+
     def set_ntds_settings_dn(self, ntds_settings_dn):
         """Set the NTDS Settings DN, as would be returned on the dsServiceName
         rootDSE attribute.
@@ -501,8 +512,13 @@ accountExpires: %u
         dsdb._dsdb_set_schema_from_ldb(self, ldb_conn)
 
     def dsdb_DsReplicaAttribute(self, ldb, ldap_display_name, ldif_elements):
+        '''convert a list of attribute values to a DRSUAPI DsReplicaAttribute'''
         return dsdb._dsdb_DsReplicaAttribute(ldb, ldap_display_name, ldif_elements)
 
+    def dsdb_normalise_attributes(self, ldb, ldap_display_name, ldif_elements):
+        '''normalise a list of attribute values'''
+        return dsdb._dsdb_normalise_attributes(ldb, ldap_display_name, ldif_elements)
+
     def get_attribute_from_attid(self, attid):
         """ Get from an attid the associated attribute
 
@@ -711,3 +727,9 @@ accountExpires: %u
         if sd:
             m["nTSecurityDescriptor"] = ndr_pack(sd)
         self.add(m)
+
+    def normalise_int32(self, ivalue):
+        '''normalise a ldap integer to signed 32 bit'''
+        if int(ivalue) & 0x80000000:
+            return str(int(ivalue) - 0x100000000)
+        return str(ivalue)
diff --git a/source4/scripting/python/samba/tests/samba3sam.py b/source4/scripting/python/samba/tests/samba3sam.py
index a34f0f620c5..7353391519f 100644
--- a/source4/scripting/python/samba/tests/samba3sam.py
+++ b/source4/scripting/python/samba/tests/samba3sam.py
@@ -30,6 +30,7 @@ from samba.tests import TestCaseInTempDir, env_loadparm
 import samba.dcerpc.security
 import samba.ndr
 from samba.auth import system_session
+from operator import attrgetter
 
 
 def read_datafile(filename):
@@ -64,7 +65,6 @@ class MapBaseTestCase(TestCaseInTempDir):
 
     def setUp(self):
         self.lp = env_loadparm()
-        self.lp.set("sid generator", "backend")
         self.lp.set("workgroup", "TESTS")
         self.lp.set("netbios name", "TESTS")
         super(MapBaseTestCase, self).setUp()
@@ -86,6 +86,7 @@ class MapBaseTestCase(TestCaseInTempDir):
 
             def __init__(self, basedn, dn, lp):
                 self.db = Ldb(lp=lp, session_info=system_session())
+                self.db.set_opaque("skip_allocate_sids", "true");
                 self.basedn = basedn
                 self.basedn_casefold = ldb.Dn(self.db, basedn).get_casefold()
                 self.substvars = {"BASEDN": self.basedn}
@@ -135,12 +136,14 @@ class Samba3SamTestCase(MapBaseTestCase):
     def setUp(self):
         super(Samba3SamTestCase, self).setUp()
         ldb = Ldb(self.ldburl, lp=self.lp, session_info=system_session())
+        ldb.set_opaque("skip_allocate_sids", "true");
         self.samba3.setup_data("samba3.ldif")
         ldif = read_datafile("provision_samba3sam.ldif")
         ldb.add_ldif(self.samba4.subst(ldif))
         self.setup_modules(ldb, self.samba3, self.samba4)
         del ldb
         self.ldb = Ldb(self.ldburl, lp=self.lp, session_info=system_session())
+        self.ldb.set_opaque("skip_allocate_sids", "true");
 
     def test_search_non_mapped(self):
         """Looking up by non-mapped attribute"""
@@ -302,11 +305,13 @@ class MapTestCase(MapBaseTestCase):
     def setUp(self):
         super(MapTestCase, self).setUp()
         ldb = Ldb(self.ldburl, lp=self.lp, session_info=system_session())
+        ldb.set_opaque("skip_allocate_sids", "true");
         ldif = read_datafile("provision_samba3sam.ldif")
         ldb.add_ldif(self.samba4.subst(ldif))
         self.setup_modules(ldb, self.samba3, self.samba4)
         del ldb
         self.ldb = Ldb(self.ldburl, lp=self.lp, session_info=system_session())
+        self.ldb.set_opaque("skip_allocate_sids", "true");
 
     def test_map_search(self):
         """Running search tests on mapped data."""
@@ -439,34 +444,37 @@ objectSid: S-1-5-21-4231626423-2410014848-2360679739-552
         res = self.ldb.search(expression="(revision=x)", scope=SCOPE_DEFAULT, 
                 attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Y"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "y")
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "x")
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[0]["dnsHostName"]), "x")
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Y"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "y")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
 
         # Search by kept attribute
         res = self.ldb.search(expression="(description=y)", 
                 scope=SCOPE_DEFAULT, attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "z")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[0])
         self.assertEquals(str(res[0]["lastLogon"]), "z")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[1])
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "z")
         self.assertEquals(str(res[1]["lastLogon"]), "z")
 
         # Search by renamed attribute
         res = self.ldb.search(expression="(badPwdCount=x)", scope=SCOPE_DEFAULT,
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
         self.assertTrue(not "dnsHostName" in res[1])
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
 
         # Search by converted attribute
         # TODO:
@@ -475,18 +483,19 @@ objectSid: S-1-5-21-4231626423-2410014848-2360679739-552
         #res = self.ldb.search("(objectSid=S-1-5-21-4231626423-2410014848-2360679739-552)", scope=SCOPE_DEFAULT, attrs)
         res = self.ldb.search(expression="(objectSid=*)", base=None, scope=SCOPE_DEFAULT, attrs=["dnsHostName", "lastLogon", "objectSid"])
         self.assertEquals(len(res), 4)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "x")
-        self.assertEquals(str(res[0]["lastLogon"]), "x")
-        self.assertSidEquals("S-1-5-21-4231626423-2410014848-2360679739-552", 
-                             res[0]["objectSid"])
-        self.assertTrue("objectSid" in res[0])
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
-        self.assertTrue(not "dnsHostName" in res[1])
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "x")
         self.assertEquals(str(res[1]["lastLogon"]), "x")
         self.assertSidEquals("S-1-5-21-4231626423-2410014848-2360679739-552",
                              res[1]["objectSid"])
         self.assertTrue("objectSid" in res[1])
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
+        self.assertTrue(not "dnsHostName" in res[0])
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertSidEquals("S-1-5-21-4231626423-2410014848-2360679739-552",
+                             res[0]["objectSid"])
+        self.assertTrue("objectSid" in res[0])
 
         # Search by generated attribute 
         # In most cases, this even works when the mapping is missing
@@ -519,12 +528,13 @@ objectSid: S-1-5-21-4231626423-2410014848-2360679739-552
         attrs = ["dnsHostName", "lastLogon", "objectClass"]
         res = self.ldb.search(expression="(objectClass=user)", attrs=attrs)
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "x")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
+        self.assertTrue(not "dnsHostName" in res[0])
         self.assertEquals(str(res[0]["lastLogon"]), "x")
         self.assertEquals(str(res[0]["objectClass"][0]), "user")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
-        self.assertTrue(not "dnsHostName" in res[1])
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "x")
         self.assertEquals(str(res[1]["lastLogon"]), "x")
         self.assertEquals(str(res[1]["objectClass"][0]), "user")
 
@@ -532,18 +542,19 @@ objectSid: S-1-5-21-4231626423-2410014848-2360679739-552
         res = self.ldb.search(expression="(|(objectClass=user)(badPwdCount=x))",
                               attrs=attrs)
         self.assertEquals(len(res), 3)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(set(res[0]["objectClass"]), set(["top"]))
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "x")
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
-        self.assertEquals(str(res[1]["objectClass"][0]), "user")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=A"))
-        self.assertTrue(not "dnsHostName" in res[2])
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(res[0]["objectClass"][0], "user")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
+        self.assertTrue(not "dnsHostName" in res[1])
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(set(res[1]["objectClass"]), set(["top"]))
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[2]["dnsHostName"]), "x")
         self.assertEquals(str(res[2]["lastLogon"]), "x")
-        self.assertEquals(res[2]["objectClass"][0], "user")
+        self.assertEquals(str(res[2]["objectClass"][0]), "user")
 
         # Testing search by parse tree
 
@@ -551,34 +562,37 @@ objectSid: S-1-5-21-4231626423-2410014848-2360679739-552
         res = self.ldb.search(expression="(&(codePage=x)(revision=x))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Y"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "y")
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "x")
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[0]["dnsHostName"]), "x")
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Y"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "y")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
 
         # Search by conjunction of remote attributes
         res = self.ldb.search(expression="(&(lastLogon=x)(description=x))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "x")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
+        self.assertTrue(not "dnsHostName" in res[0])
         self.assertEquals(str(res[0]["lastLogon"]), "x")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
-        self.assertTrue(not "dnsHostName" in res[1])
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "x")
         self.assertEquals(str(res[1]["lastLogon"]), "x")
         
         # Search by conjunction of local and remote attribute 
         res = self.ldb.search(expression="(&(codePage=x)(description=x))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Y"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "y")
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "x")
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[0]["dnsHostName"]), "x")
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Y"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "y")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
 
         # Search by conjunction of local and remote attribute w/o match
         attrs = ["dnsHostName", "lastLogon"]
@@ -593,40 +607,43 @@ objectSid: S-1-5-21-4231626423-2410014848-2360679739-552
         res = self.ldb.search(expression="(|(revision=x)(dnsHostName=x))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 2)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Y"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "y")
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "x")
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[0]["dnsHostName"]), "x")
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Y"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "y")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
 
         # Search by disjunction of remote attributes
         res = self.ldb.search(expression="(|(badPwdCount=x)(lastLogon=x))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 3)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertFalse("dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "x")
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=A"))
-        self.assertFalse("dnsHostName" in res[2])
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
+        self.assertFalse("dnsHostName" in res[1])
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[2]["dnsHostName"]), "x")
         self.assertEquals(str(res[2]["lastLogon"]), "x")
 
         # Search by disjunction of local and remote attribute
         res = self.ldb.search(expression="(|(revision=x)(lastLogon=y))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 3)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Y"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "y")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        self.assertFalse("dnsHostName" in res[0])
         self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
-        self.assertFalse("dnsHostName" in res[1])
-        self.assertEquals(str(res[1]["lastLogon"]), "y")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[2]["dnsHostName"]), "x")
-        self.assertEquals(str(res[2]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "x")
+        self.assertEquals(str(res[1]["lastLogon"]), "x")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Y"))
+        self.assertEquals(str(res[2]["dnsHostName"]), "y")
+        self.assertEquals(str(res[2]["lastLogon"]), "y")
 
         # Search by disjunction of local and remote attribute w/o match
         res = self.ldb.search(expression="(|(codePage=y)(nextRid=z))", 
@@ -637,142 +654,151 @@ objectSid: S-1-5-21-4231626423-2410014848-2360679739-552
         res = self.ldb.search(expression="(!(revision=x))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 6)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
         self.assertTrue(not "dnsHostName" in res[1])
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[2]["dnsHostName"]), "z")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[2])
         self.assertEquals(str(res[2]["lastLogon"]), "z")
-        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[3])
+        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[3]["dnsHostName"]), "z")
         self.assertEquals(str(res[3]["lastLogon"]), "z")
 
         # Search by negated remote attribute
         res = self.ldb.search(expression="(!(description=x))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 4)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "z")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[0])
         self.assertEquals(str(res[0]["lastLogon"]), "z")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[1])
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "z")
         self.assertEquals(str(res[1]["lastLogon"]), "z")
 
         # Search by negated conjunction of local attributes
         res = self.ldb.search(expression="(!(&(codePage=x)(revision=x)))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 6)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
         self.assertTrue(not "dnsHostName" in res[1])
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[2]["dnsHostName"]), "z")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[2])
         self.assertEquals(str(res[2]["lastLogon"]), "z")
-        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[3])
+        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[3]["dnsHostName"]), "z")
         self.assertEquals(str(res[3]["lastLogon"]), "z")
 
         # Search by negated conjunction of remote attributes
         res = self.ldb.search(expression="(!(&(lastLogon=x)(description=x)))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 6)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Y"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "y")
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        self.assertTrue(not "dnsHostName" in res[0])
         self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=C"))
         self.assertTrue(not "dnsHostName" in res[1])
-        self.assertEquals(str(res[1]["lastLogon"]), "y")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[2]["dnsHostName"]), "z")
-        self.assertEquals(str(res[2]["lastLogon"]), "z")
-        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[3])
+        self.assertEquals(str(res[1]["lastLogon"]), "z")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Y"))
+        self.assertEquals(str(res[2]["dnsHostName"]), "y")
+        self.assertEquals(str(res[2]["lastLogon"]), "y")
+        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[3]["dnsHostName"]), "z")
         self.assertEquals(str(res[3]["lastLogon"]), "z")
 
         # Search by negated conjunction of local and remote attribute
         res = self.ldb.search(expression="(!(&(codePage=x)(description=x)))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 6)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
         self.assertTrue(not "dnsHostName" in res[1])
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[2]["dnsHostName"]), "z")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[2])
         self.assertEquals(str(res[2]["lastLogon"]), "z")
-        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[3])
+        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[3]["dnsHostName"]), "z")
         self.assertEquals(str(res[3]["lastLogon"]), "z")
 
         # Search by negated disjunction of local attributes
         res = self.ldb.search(expression="(!(|(revision=x)(dnsHostName=x)))", 
                               attrs=["dnsHostName", "lastLogon"])
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=A"))
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
         self.assertTrue(not "dnsHostName" in res[1])
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[2]["dnsHostName"]), "z")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[2])
         self.assertEquals(str(res[2]["lastLogon"]), "z")
-        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[3])
+        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[3]["dnsHostName"]), "z")
         self.assertEquals(str(res[3]["lastLogon"]), "z")
 
         # Search by negated disjunction of remote attributes
         res = self.ldb.search(expression="(!(|(badPwdCount=x)(lastLogon=x)))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 5)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=Y"))
-        self.assertEquals(str(res[0]["dnsHostName"]), "y")
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "z")
-        self.assertEquals(str(res[1]["lastLogon"]), "z")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[2])
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[0])
+        self.assertEquals(str(res[0]["lastLogon"]), "z")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Y"))
+        self.assertEquals(str(res[1]["dnsHostName"]), "y")
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[2]["dnsHostName"]), "z")
         self.assertEquals(str(res[2]["lastLogon"]), "z")
 
         # Search by negated disjunction of local and remote attribute
         res = self.ldb.search(expression="(!(|(revision=x)(lastLogon=y)))", 
                               attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 5)
+        res = sorted(res, key=attrgetter('dn'))
         self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
         self.assertEquals(str(res[0]["lastLogon"]), "x")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "z")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=C"))
+        self.assertTrue(not "dnsHostName" in res[1])
         self.assertEquals(str(res[1]["lastLogon"]), "z")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[2])
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[2]["dnsHostName"]), "z")
         self.assertEquals(str(res[2]["lastLogon"]), "z")
 
         # Search by complex parse tree
         res = self.ldb.search(expression="(|(&(revision=x)(dnsHostName=x))(!(&(description=x)(nextRid=y)))(badPwdCount=y))", attrs=["dnsHostName", "lastLogon"])
         self.assertEquals(len(res), 7)
-        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=B"))
+        res = sorted(res, key=attrgetter('dn'))
+        self.assertEquals(str(res[0].dn), self.samba4.dn("cn=A"))
         self.assertTrue(not "dnsHostName" in res[0])
-        self.assertEquals(str(res[0]["lastLogon"]), "y")
-        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=X"))
-        self.assertEquals(str(res[1]["dnsHostName"]), "x")
-        self.assertEquals(str(res[1]["lastLogon"]), "x")
-        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=A"))
+        self.assertEquals(str(res[0]["lastLogon"]), "x")
+        self.assertEquals(str(res[1].dn), self.samba4.dn("cn=B"))
+        self.assertTrue(not "dnsHostName" in res[1])
+        self.assertEquals(str(res[1]["lastLogon"]), "y")
+        self.assertEquals(str(res[2].dn), self.samba4.dn("cn=C"))
         self.assertTrue(not "dnsHostName" in res[2])
-        self.assertEquals(str(res[2]["lastLogon"]), "x")
-        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=Z"))
-        self.assertEquals(str(res[3]["dnsHostName"]), "z")
-        self.assertEquals(str(res[3]["lastLogon"]), "z")
-        self.assertEquals(str(res[4].dn), self.samba4.dn("cn=C"))
-        self.assertTrue(not "dnsHostName" in res[4])
+        self.assertEquals(str(res[2]["lastLogon"]), "z")
+        self.assertEquals(str(res[3].dn), self.samba4.dn("cn=X"))
+        self.assertEquals(str(res[3]["dnsHostName"]), "x")
+        self.assertEquals(str(res[3]["lastLogon"]), "x")
+        self.assertEquals(str(res[4].dn), self.samba4.dn("cn=Z"))
+        self.assertEquals(str(res[4]["dnsHostName"]), "z")
         self.assertEquals(str(res[4]["lastLogon"]), "z")
 
         # Clean up
diff --git a/source3/stf/strings.py b/source4/scripting/python/samba/tests/strings.py
index 4e00021525f..5f3e5c5bb77 100755..100644
--- a/source3/stf/strings.py
+++ b/source4/scripting/python/samba/tests/strings.py
@@ -1,19 +1,20 @@
-#! /usr/bin/python     
+#!/usr/bin/env python
 
-# Comfychair test cases for Samba string functions.
+# subunit test cases for Samba string functions.
 
 # Copyright (C) 2003 by Martin Pool <mbp@samba.org>
-# 
+# Copyright (C) 2011 Andrew Bartlett
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
 # published by the Free Software Foundation; either version 3 of the
 # License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful, but
 # WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
@@ -21,19 +22,15 @@
 # which is the most common setting.  I guess it would be better to
 # force it to that value while running the tests.  I'm not sure of the
 # best way to do that yet.
-# 
-# Note that this is NOT the case in C code until the loadparm table is
-# intialized -- the default seems to be ASCII, which rather lets Samba
-# off the hook. :-) The best way seems to be to put this in the test
-# harnesses:
-#
-#       lp_load("/dev/null", True, False, False);
 #
 # -- mbp
 
-import sys, re, comfychair
+import sys, re
 from unicodenames import *
 
+import samba.tests
+from samba import strcasecmp_m, strstr_m
+
 def signum(a):
     if a < 0:
         return -1
@@ -43,35 +40,9 @@ def signum(a):
         return 0
 
 
-class PushUCS2_Tests(comfychair.TestCase):
-    """Conversion to/from UCS2"""
-    def runtest(self):
-        OE = LATIN_CAPITAL_LETTER_O_WITH_DIARESIS
-        oe = LATIN_CAPITAL_LETTER_O_WITH_DIARESIS
-        cases = ['hello',
-                 'hello world',
-                 'g' + OE + OE + 'gomobile', 
-                 'g' + OE + oe + 'gomobile', 
-                 u'foo\u0100',
-                 KATAKANA_LETTER_A * 20,
-                 ]
-        for u8str in cases:
-            out, err = self.runcmd("t_push_ucs2 \"%s\"" % u8str.encode('utf-8'))
-            self.assert_equal(out, "0\n")
-    
-
-class StrCaseCmp(comfychair.TestCase):
-    """String comparisons in simple ASCII""" 
-    def run_strcmp(self, a, b, expect):
-        out, err = self.runcmd('t_strcmp \"%s\" \"%s\"' % (a.encode('utf-8'), b.encode('utf-8')))
-        if signum(int(out)) != expect:
-            self.fail("comparison failed:\n"
-                      "  a=%s\n"
-                      "  b=%s\n"
-                      "  expected=%s\n"
-                      "  result=%s\n" % (`a`, `b`, `expect`, `out`))
-
-    def runtest(self):
+class strcasecmp_m_Tests(samba.tests.TestCase):
+    """String comparisons in simple ASCII and unicode"""
+    def test_strcasecmp_m(self):
         # A, B, strcasecmp(A, B)
         cases = [('hello', 'hello', 0),
                  ('hello', 'goodbye', +1),
@@ -90,60 +61,44 @@ class StrCaseCmp(comfychair.TestCase):
                  (KATAKANA_LETTER_A, 'a', 1),
                  ]
         for a, b, expect in cases:
-            self.run_strcmp(a, b, expect)
-        
-class strstr_m(comfychair.TestCase):
-    """String comparisons in simple ASCII""" 
-    def run_strstr(self, a, b, expect):
-        out, err = self.runcmd('t_strstr \"%s\" \"%s\"' % (a.encode('utf-8'), b.encode('utf-8')))
-        if (out != (expect + '\n').encode('utf-8')):
-            self.fail("comparison failed:\n"
-                      "  a=%s\n"
-                      "  b=%s\n"
-                      "  expected=%s\n"
-                      "  result=%s\n" % (`a`, `b`, `expect+'\n'`, `out`))
+            self.assertEquals(signum(strcasecmp_m(a.encode('utf-8'),
+                                                  b.encode('utf-8'))),
+                              expect)
 
-    def runtest(self):
+class strstr_m_Tests(samba.tests.TestCase):
+    """strstr_m tests in simple ASCII and unicode strings"""
+    def test_strstr_m(self):
         # A, B, strstr_m(A, B)
         cases = [('hello', 'hello', 'hello'),
-                 ('hello', 'goodbye', '(null)'),
-                 ('goodbye', 'hello', '(null)'),
-                 ('hell', 'hello', '(null)'),
+                 ('hello', 'goodbye', None),
+                 ('goodbye', 'hello', None),
+                 ('hell', 'hello', None),
                  ('hello', 'hell', 'hello'),
                  ('', '', ''),
                  ('a', '', 'a'),
-                 ('', 'a', '(null)'),
-                 ('a', 'A', '(null)'),
-                 ('aa', 'aA', '(null)'),
-                 ('Aa', 'aa', '(null)'),
+                 ('', 'a', None),
+                 ('a', 'A', None),
+                 ('aa', 'aA', None),
+                 ('Aa', 'aa', None),
                  ('%v foo', '%v', '%v foo'),
                  ('foo %v foo', '%v', '%v foo'),
                  ('foo %v', '%v', '%v'),
                  ('longstring ' * 100, 'longstring ' * 99, 'longstring ' * 100),
-                 ('longstring ' * 99, 'longstring ' * 100, '(null)'),
-                 ('longstring a' * 99, 'longstring ' * 100 + 'a', '(null)'),
+                 ('longstring ' * 99, 'longstring ' * 100, None),
+                 ('longstring a' * 99, 'longstring ' * 100 + 'a', None),
                  ('longstring ' * 100 + 'a', 'longstring ' * 100, 'longstring ' * 100 + 'a'),
-                 (KATAKANA_LETTER_A, KATAKANA_LETTER_A + 'bcd', '(null)'),
+                 (KATAKANA_LETTER_A, KATAKANA_LETTER_A + 'bcd', None),
                  (KATAKANA_LETTER_A + 'bcde', KATAKANA_LETTER_A + 'bcd', KATAKANA_LETTER_A + 'bcde'),
                  ('d'+KATAKANA_LETTER_A + 'bcd', KATAKANA_LETTER_A + 'bcd', KATAKANA_LETTER_A + 'bcd'),
-                 ('d'+KATAKANA_LETTER_A + 'bd', KATAKANA_LETTER_A + 'bcd', '(null)'),
-                 
+                 ('d'+KATAKANA_LETTER_A + 'bd', KATAKANA_LETTER_A + 'bcd', None),
+
                  ('e'+KATAKANA_LETTER_A + 'bcdf', KATAKANA_LETTER_A + 'bcd', KATAKANA_LETTER_A + 'bcdf'),
-                 (KATAKANA_LETTER_A, KATAKANA_LETTER_A + 'bcd', '(null)'),
-                 (KATAKANA_LETTER_A*3, 'a', '(null)'),
+                 (KATAKANA_LETTER_A, KATAKANA_LETTER_A + 'bcd', None),
+                 (KATAKANA_LETTER_A*3, 'a', None),
                  ]
         for a, b, expect in cases:
-            self.run_strstr(a, b, expect)
-        
-# Define the tests exported by this module
-tests = [StrCaseCmp,
-         strstr_m,
-         PushUCS2_Tests]
-
-# Handle execution of this file as a main program
-if __name__ == '__main__':
-    comfychair.main(tests)
-
-# Local variables:
-# coding: utf-8
-# End:
+            if expect is not None:
+                expect = expect.encode('utf-8')
+            self.assertEquals(strstr_m(a.encode('utf-8'),
+                                       b.encode('utf-8')),
+                              expect)
diff --git a/source3/stf/unicodenames.py b/source4/scripting/python/samba/tests/unicodenames.py
index 34d0a192383..fa5d0efc8ca 100644
--- a/source3/stf/unicodenames.py
+++ b/source4/scripting/python/samba/tests/unicodenames.py
@@ -1,17 +1,17 @@
 #! /usr/bin/python
 
 # Copyright (C) 2003 by Martin Pool <mbp@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
 # published by the Free Software Foundation; either version 3 of the
 # License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful, but
 # WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
diff --git a/source4/scripting/python/samba/tests/upgradeprovisionneeddc.py b/source4/scripting/python/samba/tests/upgradeprovisionneeddc.py
index 3a9c78e0dcd..596cff6d3aa 100644
--- a/source4/scripting/python/samba/tests/upgradeprovisionneeddc.py
+++ b/source4/scripting/python/samba/tests/upgradeprovisionneeddc.py
@@ -26,9 +26,9 @@ import shutil
 from samba import param
 from samba.credentials import Credentials
 from samba.auth import system_session
-from samba.provision import getpolicypath
+from samba.provision import getpolicypath,find_provision_key_parameters
 from samba.upgradehelpers import (get_paths, get_ldbs,
-                                 find_provision_key_parameters, identic_rename,
+                                 identic_rename,
                                  updateOEMInfo, getOEMInfo, update_gpo,
                                  delta_update_basesamdb,
                                  update_dns_account_password,
diff --git a/source4/scripting/python/samba/upgradehelpers.py b/source4/scripting/python/samba/upgradehelpers.py
index 48f492a7dc3..e15523033fb 100755
--- a/source4/scripting/python/samba/upgradehelpers.py
+++ b/source4/scripting/python/samba/upgradehelpers.py
@@ -24,22 +24,19 @@
 """Helpers used for upgrading between different database formats."""
 
 import os
-import string
 import re
 import shutil
 import samba
 
 from samba import Ldb, version, ntacls
-from samba.dsdb import DS_DOMAIN_FUNCTION_2000
 from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE
 import ldb
-from samba.provision import (ProvisionNames, provision_paths_from_lp,
+from samba.provision import (provision_paths_from_lp,
                             getpolicypath, set_gpos_acl, create_gpo_struct,
                             FILL_FULL, provision, ProvisioningError,
                             setsysvolacl, secretsdb_self_join)
-from samba.dcerpc import misc, security, xattr
+from samba.dcerpc import xattr
 from samba.dcerpc.misc import SEC_CHAN_BDC
-from samba.ndr import ndr_unpack
 from samba.samdb import SamDB
 
 # All the ldb related to registry are commented because the path for them is
@@ -242,112 +239,6 @@ def update_policyids(names, samdb):
         names.policyid_dc = None
 
 
-def find_provision_key_parameters(samdb, secretsdb, idmapdb, paths, smbconf, lp):
-    """Get key provision parameters (realm, domain, ...) from a given provision
-
-    :param samdb: An LDB object connected to the sam.ldb file
-    :param secretsdb: An LDB object connected to the secrets.ldb file
-    :param idmapdb: An LDB object connected to the idmap.ldb file
-    :param paths: A list of path to provision object
-    :param smbconf: Path to the smb.conf file
-    :param lp: A LoadParm object
-    :return: A list of key provision parameters
-    """
-    names = ProvisionNames()
-    names.adminpass = None
-
-    # NT domain, kerberos realm, root dn, domain dn, domain dns name
-    names.domain = string.upper(lp.get("workgroup"))
-    names.realm = lp.get("realm")
-    basedn = "DC=" + names.realm.replace(".",",DC=")
-    names.dnsdomain = names.realm.lower()
-    names.realm = string.upper(names.realm)
-    # netbiosname
-    # Get the netbiosname first (could be obtained from smb.conf in theory)
-    res = secretsdb.search(expression="(flatname=%s)" %
-                            names.domain,base="CN=Primary Domains",
-                            scope=SCOPE_SUBTREE, attrs=["sAMAccountName"])
-    names.netbiosname = str(res[0]["sAMAccountName"]).replace("$","")
-
-    names.smbconf = smbconf
-
-    # That's a bit simplistic but it's ok as long as we have only 3
-    # partitions
-    current = samdb.search(expression="(objectClass=*)", 
-        base="", scope=SCOPE_BASE,
-        attrs=["defaultNamingContext", "schemaNamingContext",
-               "configurationNamingContext","rootDomainNamingContext"])
-
-    names.configdn = current[0]["configurationNamingContext"]
-    configdn = str(names.configdn)
-    names.schemadn = current[0]["schemaNamingContext"]
-    if not (ldb.Dn(samdb, basedn) == (ldb.Dn(samdb,
-                                       current[0]["defaultNamingContext"][0]))):
-        raise ProvisioningError(("basedn in %s (%s) and from %s (%s)"
-                                 "is not the same ..." % (paths.samdb,
-                                    str(current[0]["defaultNamingContext"][0]),
-                                    paths.smbconf, basedn)))
-
-    names.domaindn=current[0]["defaultNamingContext"]
-    names.rootdn=current[0]["rootDomainNamingContext"]
-    # default site name
-    res3 = samdb.search(expression="(objectClass=*)", 
-        base="CN=Sites," + configdn, scope=SCOPE_ONELEVEL, attrs=["cn"])
-    names.sitename = str(res3[0]["cn"])
-
-    # dns hostname and server dn
-    res4 = samdb.search(expression="(CN=%s)" % names.netbiosname,
-                            base="OU=Domain Controllers,%s" % basedn,
-                            scope=SCOPE_ONELEVEL, attrs=["dNSHostName"])
-    names.hostname = str(res4[0]["dNSHostName"]).replace("." + names.dnsdomain,"")
-
-    server_res = samdb.search(expression="serverReference=%s" % res4[0].dn,
-                                attrs=[], base=configdn)
-    names.serverdn = server_res[0].dn
-
-    # invocation id/objectguid
-    res5 = samdb.search(expression="(objectClass=*)",
-            base="CN=NTDS Settings,%s" % str(names.serverdn), scope=SCOPE_BASE,
-            attrs=["invocationID", "objectGUID"])
-    names.invocation = str(ndr_unpack(misc.GUID, res5[0]["invocationId"][0]))
-    names.ntdsguid = str(ndr_unpack(misc.GUID, res5[0]["objectGUID"][0]))
-
-    # domain guid/sid
-    res6 = samdb.search(expression="(objectClass=*)", base=basedn,
-            scope=SCOPE_BASE, attrs=["objectGUID",
-                "objectSid","msDS-Behavior-Version" ])
-    names.domainguid = str(ndr_unpack(misc.GUID, res6[0]["objectGUID"][0]))
-    names.domainsid = ndr_unpack( security.dom_sid, res6[0]["objectSid"][0])
-    if res6[0].get("msDS-Behavior-Version") is None or \
-        int(res6[0]["msDS-Behavior-Version"][0]) < DS_DOMAIN_FUNCTION_2000:
-        names.domainlevel = DS_DOMAIN_FUNCTION_2000
-    else:
-        names.domainlevel = int(res6[0]["msDS-Behavior-Version"][0])
-
-    # policy guid
-    res7 = samdb.search(expression="(displayName=Default Domain Policy)",
-                        base="CN=Policies,CN=System," + basedn,
-                        scope=SCOPE_ONELEVEL, attrs=["cn","displayName"])
-    names.policyid = str(res7[0]["cn"]).replace("{","").replace("}","")
-    # dc policy guid
-    res8 = samdb.search(expression="(displayName=Default Domain Controllers"
-                                   " Policy)",
-                            base="CN=Policies,CN=System," + basedn,
-                            scope=SCOPE_ONELEVEL, attrs=["cn","displayName"])
-    if len(res8) == 1:
-        names.policyid_dc = str(res8[0]["cn"]).replace("{","").replace("}","")
-    else:
-        names.policyid_dc = None
-    res9 = idmapdb.search(expression="(cn=%s)" %
-                            (security.SID_BUILTIN_ADMINISTRATORS),
-                            attrs=["xidNumber"])
-    if len(res9) == 1:
-        names.wheel_gid = res9[0]["xidNumber"]
-    else:
-        raise ProvisioningError("Unable to find uid/gid for Domain Admins rid")
-    return names
-
-
 def newprovision(names, creds, session, smbconf, provdir, logger):
     """Create a new provision.
 
@@ -469,7 +360,7 @@ def chunck_sddl(sddl):
     return hash
 
 
-def get_diff_sddls(refsddl, cursddl):
+def get_diff_sddls(refsddl, cursddl, checkSacl = True):
     """Get the difference between 2 sddl
 
     This function split the textual representation of ACL into smaller
@@ -477,46 +368,54 @@ def get_diff_sddls(refsddl, cursddl):
 
     :param refsddl: First sddl to compare
     :param cursddl: Second sddl to compare
+    :param checkSacl: If false we skip the sacl checks
     :return: A string that explain difference between sddls
     """
 
     txt = ""
-    hash_new = chunck_sddl(cursddl)
+    hash_cur = chunck_sddl(cursddl)
     hash_ref = chunck_sddl(refsddl)
 
-    if hash_new["owner"] != hash_ref["owner"]:
+    if not hash_cur.has_key("owner"):
+        txt = "\tNo owner in current SD"
+    elif hash_cur["owner"] != hash_ref["owner"]:
         txt = "\tOwner mismatch: %s (in ref) %s" \
-              "(in current)\n" % (hash_ref["owner"], hash_new["owner"])
+              "(in current)\n" % (hash_ref["owner"], hash_cur["owner"])
 
-    if hash_new["group"] != hash_ref["group"]:
+    if not hash_cur.has_key("group"):
+        txt = "%s\tNo group in current SD" % txt
+    elif hash_cur["group"] != hash_ref["group"]:
         txt = "%s\tGroup mismatch: %s (in ref) %s" \
-              "(in current)\n" % (txt, hash_ref["group"], hash_new["group"])
+              "(in current)\n" % (txt, hash_ref["group"], hash_cur["group"])
 
-    for part in ["dacl", "sacl"]:
-        if hash_new.has_key(part) and hash_ref.has_key(part):
+    parts = [ "dacl" ]
+    if checkSacl:
+        parts.append("sacl")
+    for part in parts:
+        if hash_cur.has_key(part) and hash_ref.has_key(part):
 
             # both are present, check if they contain the same ACE
-            h_new = set()
+            h_cur = set()
             h_ref = set()
-            c_new = chunck_acl(hash_new[part])
+            c_cur = chunck_acl(hash_cur[part])
             c_ref = chunck_acl(hash_ref[part])
 
-            for elem in c_new["aces"]:
-                h_new.add(elem)
+            for elem in c_cur["aces"]:
+                h_cur.add(elem)
 
             for elem in c_ref["aces"]:
                 h_ref.add(elem)
 
             for k in set(h_ref):
-                if k in h_new:
-                    h_new.remove(k)
+                if k in h_cur:
+                    h_cur.remove(k)
                     h_ref.remove(k)
 
-            if len(h_new) + len(h_ref) > 0:
+            if len(h_cur) + len(h_ref) > 0:
                 txt = "%s\tPart %s is different between reference" \
                       " and current here is the detail:\n" % (txt, part)
 
-                for item in h_new:
+                for item in h_cur:
                     txt = "%s\t\t%s ACE is not present in the" \
                           " reference\n" % (txt, item)
 
@@ -524,9 +423,9 @@ def get_diff_sddls(refsddl, cursddl):
                     txt = "%s\t\t%s ACE is not present in the" \
                           " current\n" % (txt, item)
 
-        elif hash_new.has_key(part) and not hash_ref.has_key(part):
+        elif hash_cur.has_key(part) and not hash_ref.has_key(part):
             txt = "%s\tReference ACL hasn't a %s part\n" % (txt, part)
-        elif not hash_new.has_key(part) and hash_ref.has_key(part):
+        elif not hash_cur.has_key(part) and hash_ref.has_key(part):
             txt = "%s\tCurrent ACL hasn't a %s part\n" % (txt, part)
 
     return txt
@@ -541,7 +440,7 @@ def update_secrets(newsecrets_ldb, secrets_ldb, messagefunc):
         of the updated provision
     """
 
-    messagefunc(SIMPLE, "update secrets.ldb")
+    messagefunc(SIMPLE, "Update of secrets.ldb")
     reference = newsecrets_ldb.search(expression="dn=@MODULES", base="",
                                         scope=SCOPE_SUBTREE)
     current = secrets_ldb.search(expression="dn=@MODULES", base="",
@@ -649,7 +548,7 @@ def getOEMInfo(samdb, rootdn):
     """
     res = samdb.search(expression="(objectClass=*)", base=str(rootdn),
                             scope=SCOPE_BASE, attrs=["dn", "oEMInformation"])
-    if len(res) > 0:
+    if len(res) > 0 and res[0].get("oEMInformation"):
         info = res[0]["oEMInformation"]
         return info
     else:
@@ -666,7 +565,10 @@ def updateOEMInfo(samdb, rootdn):
     res = samdb.search(expression="(objectClass=*)", base=rootdn,
                             scope=SCOPE_BASE, attrs=["dn", "oEMInformation"])
     if len(res) > 0:
-        info = res[0]["oEMInformation"]
+        if res[0].get("oEMInformation"):
+            info = str(res[0]["oEMInformation"])
+        else:
+            info = ""
         info = "%s, upgrade to %s" % (info, version)
         delta = ldb.Message()
         delta.dn = ldb.Dn(samdb, str(res[0]["dn"]))
diff --git a/source4/scripting/wscript_build b/source4/scripting/wscript_build
index 76ff739c9e7..d94fc4fe9cc 100644
--- a/source4/scripting/wscript_build
+++ b/source4/scripting/wscript_build
@@ -4,5 +4,5 @@ from samba_utils import MODE_755
 
 bld.INSTALL_FILES('${SBINDIR}','bin/upgradeprovision bin/samba_dnsupdate bin/samba_spnupdate',
                   chmod=MODE_755, python_fixup=True, flat=True)
-bld.INSTALL_FILES('${BINDIR}','bin/testparm',
-                  chmod=MODE_755, python_fixup=True, flat=True)
+
+bld.RECURSE('bin')
diff --git a/source4/selftest/knownfail b/source4/selftest/knownfail
index cdd7a2d3069..266148b91f4 100644
--- a/source4/selftest/knownfail
+++ b/source4/selftest/knownfail
@@ -3,82 +3,87 @@
 #
 # "make test" will not report failures for tests listed here and will consider
 # a successful run for any of these tests an error.
-samba4.local.resolve.*.async
-samba4.local.iconv.*.next_codepoint()
-samba4..*base.delete.*.deltest17
-samba4..*base.delete.*.deltest20a
-samba4..*base.delete.*.deltest20b
-samba4.raw.rename.*.osxrename
-samba4.raw.rename.*.directory rename
-samba4.rpc.winreg.*security
-samba4.local.registry.(dir|ldb).check hive security
-samba4.local.registry.local.security
-samba4.rpc.wkssvc
-samba4.rpc.handles.*.lsarpc-shared
-samba4.rpc.handles.*.mixed-shared
-samba4.rpc.epmapper
-samba4.rpc.drsuapi.*
-samba4.rpc.lsalookup
-samba4.rpc.cracknames
-samba4.rpc.netlogon.*.LogonUasLogon
-samba4.rpc.netlogon.*.LogonUasLogoff
-samba4.rpc.netlogon.*.DatabaseSync
-samba4.rpc.netlogon.*.DatabaseSync2
-samba4.rpc.netlogon.*.LogonControl
-samba4.rpc.netlogon.*.LogonControl2
-samba4.rpc.netlogon.*.DsrEnumerateDomainTrusts
-samba4.rpc.netlogon.*.NetrEnumerateTrustedDomains
-samba4.rpc.netlogon.*.NetrEnumerateTrustedDomainsEx
-samba4.rpc.netlogon.*.GetPassword
-samba4.rpc.netlogon.*.GetTrustPasswords
-samba4.rpc.netlogon.*.DatabaseRedo
-samba4.rpc.netlogon.*.ServerGetTrustInfo
-samba4.rpc.netlogon.*.GetForestTrustInformation
-samba4.rpc.samr.passwords.badpwdcount # Not provided by Samba 4 yet
-samba4.rpc.samr.passwords.lockout
-samba4.base.charset.*.Testing partial surrogate
+^samba4.local.resolve.*.async
+^samba4.local.iconv.*.next_codepoint()
+^samba4..*base.delete.*.deltest17
+^samba4..*base.delete.*.deltest20a
+^samba4..*base.delete.*.deltest20b
+^samba4.raw.rename.*.osxrename
+^samba4.raw.rename.*.directory rename
+^samba4.rpc.winreg.*security
+^samba4.local.registry.(dir|ldb).check hive security
+^samba4.local.registry.local.security
+^samba4.rpc.wkssvc
+^samba4.rpc.handles.*.lsarpc-shared
+^samba4.rpc.handles.*.mixed-shared
+^samba4.rpc.epmapper
+^samba4.rpc.drsuapi.*
+^samba4.rpc.lsalookup
+^samba4.rpc.cracknames
+^samba4.rpc.netlogon.*.LogonUasLogon
+^samba4.rpc.netlogon.*.LogonUasLogoff
+^samba4.rpc.netlogon.*.DatabaseSync
+^samba4.rpc.netlogon.*.DatabaseSync2
+^samba4.rpc.netlogon.*.LogonControl
+^samba4.rpc.netlogon.*.LogonControl2
+^samba4.rpc.netlogon.*.DsrEnumerateDomainTrusts
+^samba4.rpc.netlogon.*.NetrEnumerateTrustedDomains
+^samba4.rpc.netlogon.*.NetrEnumerateTrustedDomainsEx
+^samba4.rpc.netlogon.*.GetPassword
+^samba4.rpc.netlogon.*.GetTrustPasswords
+^samba4.rpc.netlogon.*.DatabaseRedo
+^samba4.rpc.netlogon.*.ServerGetTrustInfo
+^samba4.rpc.netlogon.*.GetForestTrustInformation
+^samba4.rpc.samr.passwords.badpwdcount # Not provided by Samba 4 yet
+^samba4.rpc.samr.passwords.lockout
+^samba4.base.charset.*.Testing partial surrogate
 .*net.api.delshare.*				# DelShare isn't implemented yet
-samba4.rap.*netservergetinfo
-samba4.rap.*netsessionenum
-samba4.rap.*netsessiongetinfo
-samba4.smb2.persistent.handles1
-samba4.winbind.struct.*.show_sequence     # Not yet working in winbind
-samba4.winbind.struct.*.getpwent          # Not yet working in winbind
-samba4.winbind.struct.*.setpwent          # Not yet working in winbind
-samba4.winbind.struct.*.lookup_name_sid   # Not yet working in winbind
-samba4.winbind.struct.*.list_groups
-samba4.*base.delaywrite.*update of write time and SMBwrite truncate$
-samba4.*base.delaywrite.*update of write time and SMBwrite truncate expand$
-samba4.*base.delaywrite.*delayed update of write time 3a$
-samba4.*base.delaywrite.*delayed update of write time 3c$
-samba4.*base.delaywrite.*update of write time using SET_END_OF_FILE$
-samba4.*base.delaywrite.*update of write time using SET_ALLOCATION_SIZE$
-samba4.ldap.python \(dc\).Test add_ldif\(\) with BASE64 security descriptor input using WRONG domain SID$
+^samba4.rap.*netservergetinfo
+^samba4.rap.*netsessionenum
+^samba4.rap.*netsessiongetinfo
+^samba4.rap.*netremotetod
+^samba4.smb2.persistent.handles1
+^samba4.winbind.struct.*.show_sequence     # Not yet working in winbind
+^samba4.winbind.struct.*.getpwent          # Not yet working in winbind
+^samba4.winbind.struct.*.setpwent          # Not yet working in winbind
+^samba4.winbind.struct.*.lookup_name_sid   # Not yet working in winbind
+^samba4.winbind.struct.*.list_groups
+^samba4.*base.delaywrite.*update of write time and SMBwrite truncate$
+^samba4.*base.delaywrite.*update of write time and SMBwrite truncate expand$
+^samba4.*base.delaywrite.*delayed update of write time 3a$
+^samba4.*base.delaywrite.*delayed update of write time 3c$
+^samba4.*base.delaywrite.*update of write time using SET_END_OF_FILE$
+^samba4.*base.delaywrite.*update of write time using SET_ALLOCATION_SIZE$
+^samba4.ldap.python \(dc\).Test add_ldif\(\) with BASE64 security descriptor input using WRONG domain SID$
 # some operations don't work over the CIFS NTVFS backend yet (eg. root_fid)
-samba4.ntvfs.cifs.base.createx_access
-samba4.ntvfs.cifs.base.createx_sharemodes_dir
-samba4.ntvfs.cifs.base.maximum_allowed
-samba4.base.createx_access # this test is broken for non-administrator users
-samba4.smb2.oplock # oplocks in the s4 SMB2 server are a mess
-samba4.raw.lock.*.async # bug 6960
-samba4.smb2.lock.*.multiple-unlock # bug 6959
-samba4.raw.sfileinfo.*.end-of-file # bug 6962
-samba4.raw.oplock.*.batch22 # bug 6963
-samba4.raw.oplock.*.brl4 # bug 7928
-samba4.raw.lock.*.zerobyteread # bug 6974
-samba4.smb2.lock.*.zerobyteread # bug 6974
-samba4.raw.streams.*.delete
-samba4.raw.streams.*.createdisp
-samba4.raw.streams.*.sumtab
-samba4.raw.acls.*.create_dir
-samba4.raw.acls.*.create_file
-samba4.smb2.create.*.acldir
-samba4.smb2.acls.*.generic
-samba4.smb2.acls.*.inheritflags
-samba4.smb2.acls.*.owner
-samba4.smb2.compound.*.related1
-samba4.smb2.compound.*.related2
-samba4.smb2.compound.*.invalid2
-samba4.ldap.acl.*.search.* # ACL search behaviour not enabled by default
-samba4.ldap.acl.*.ntSecurityDescriptor.* # ACL extended checks on search not enabled by default
-samba4.nbt.winsreplication.owned # fails sometimes, timing related
+^samba4.ntvfs.cifs.base.createx_access
+^samba4.ntvfs.cifs.base.createx_sharemodes_dir
+^samba4.ntvfs.cifs.base.maximum_allowed
+^samba4.base.createx_access # this test is broken for non-administrator users
+^samba4.smb2.oplock # oplocks in the s4 SMB2 server are a mess
+^samba4.raw.lock.*.async # bug 6960
+^samba4.smb2.lock.*.multiple-unlock # bug 6959
+^samba4.raw.sfileinfo.*.end-of-file # bug 6962
+^samba4.raw.oplock.*.batch22 # bug 6963
+^samba4.raw.oplock.*.brl4 # bug 7928
+^samba4.raw.lock.*.zerobyteread # bug 6974
+^samba4.smb2.lock.*.zerobyteread # bug 6974
+^samba4.raw.streams.*.delete
+^samba4.raw.streams.*.createdisp
+^samba4.raw.streams.*.sumtab
+^samba4.raw.acls.*.create_dir
+^samba4.raw.acls.*.create_file
+^samba4.smb2.create.*.acldir
+^samba4.smb2.acls.*.generic
+^samba4.smb2.acls.*.inheritflags
+^samba4.smb2.acls.*.owner
+^samba4.smb2.compound.*.related1
+^samba4.smb2.compound.*.related2
+^samba4.smb2.compound.*.invalid2
+^samba4.ldap.acl.*.search.* # ACL search behaviour not enabled by default
+^samba4.ldap.acl.*.ntSecurityDescriptor.* # ACL extended checks on search not enabled by default
+^samba4.nbt.winsreplication.owned # fails sometimes, timing related
+^samba4.ldap.dirsync.python.dc..__main__.ExtendedDirsyncTests.test_dirsync_deleted_items
+#^samba4.ldap.dirsync.python.dc..__main__.ExtendedDirsyncTests.*
+^samba4.drs.fsmo.python
+^samba4.libsmbclient.opendir.opendir # This requires netbios browsing
diff --git a/source4/selftest/skip b/source4/selftest/skip
index 671269a8fda..c15375a7094 100644
--- a/source4/selftest/skip
+++ b/source4/selftest/skip
@@ -18,69 +18,76 @@
 #
 # Please add a comment for each testsuite you disable explaining why
 # it is being skipped.
-raw.composite
-base.iometer
-base.casetable
-base.nttrans
-base.scan.maxfid
-hold.oplock		# Not a test, but a way to block other clients for a test
-raw.ping.pong		# Needs second server to test
-rpc.samr.accessmask
-raw.scan.eamax
+^samba4.raw.composite
+^samba4.base.iometer
+^samba4.base.casetable
+^samba4.base.nttrans
+^samba4.base.scan.maxfid
+^samba4.raw.hold-oplock 		# Not a test, but a way to block other clients for a test
+^samba4.smb2.hold-oplock 		# Not a test, but a way to block other clients for a test
+^samba4.raw.ping.pong		# Needs second server to test
+^samba4.rpc.samr.accessmask
+^samba4.raw.scan.eamax
 samba4.ntvfs.cifs.raw.qfileinfo.ipc
-smb2.notify
-smb2.scan
-smb2.lease
-smb2.durable.open
-smb2.dir
-ntvfs.cifs.base.charset
-ntvfs.cifs.base.iometer
-ntvfs.cifs.base.casetable
-ntvfs.cifs.base.nttrans
-ntvfs.cifs.base.scan-maxfid
-ntvfs.cifs.base.utable
-ntvfs.cifs.base.smb
-ntvfs.cifs.raw.composite
-ntvfs.cifs.raw.notify
-ntvfs.cifs.raw.scan-eamax
-ntvfs.cifs.raw.context
-ntvfs.cifs.raw.qfileinfo.ipc
-rpc.samsync
-rpc.remact							# Not provided by Samba 4
-rpc.oxidresolve						# Not provided by Samba 4
-rpc.eventlog						# Not provided by Samba 4
-rpc.initshutdown					# Not provided by Samba 4
-rpc.spoolss						# Not provided by Samba 4
-rpc.svcctl							# Not provided by Samba 4
-rpc.atsvc							# Not provided by Samba 4
-rpc.frsapi							# Not provided by Samba 4
-rpc.ntsvcs							# Not provided by Samba 4
-rpc.dfs								# Not provided by Samba 4
-rpc.lsa.forest							# Not provided by Samba 4
+^samba4.smb2.notify
+^samba4.smb2.scan
+^samba4.smb2.lease
+^samba4.smb2.durable.open
+^samba4.smb2.dir
+^samba4.ntvfs.cifs.base.charset
+^samba4.ntvfs.cifs.base.iometer
+^samba4.ntvfs.cifs.base.casetable
+^samba4.ntvfs.cifs.base.nttrans
+^samba4.ntvfs.cifs.base.scan-maxfid
+^samba4.ntvfs.cifs.base.utable
+^samba4.ntvfs.cifs.base.smb
+^samba4.ntvfs.cifs.raw.composite
+^samba4.ntvfs.cifs.raw.notify
+^samba4.ntvfs.cifs.raw.scan-eamax
+^samba4.ntvfs.cifs.raw.context
+^samba4.ntvfs.cifs.raw.qfileinfo.ipc
+^samba4.rpc.samsync
+^samba4.rpc.remact							# Not provided by Samba 4
+^samba4.rpc.oxidresolve						# Not provided by Samba 4
+^samba4.rpc.eventlog						# Not provided by Samba 4
+^samba4.rpc.initshutdown					# Not provided by Samba 4
+^samba4.rpc.spoolss						# Not provided by Samba 4
+^samba4.rpc.svcctl							# Not provided by Samba 4
+^samba4.rpc.atsvc							# Not provided by Samba 4
+^samba4.rpc.frsapi							# Not provided by Samba 4
+^samba4.rpc.ntsvcs							# Not provided by Samba 4
+^samba4.rpc.dfs								# Not provided by Samba 4
+^samba4.rpc.lsa.forest							# Not provided by Samba 4
 ^samba4.base.samba3.*						# Samba3-specific test
 ^samba4.ntvfs.cifs.base.samba3.*				# Samba3-specific test
 ^samba4.raw.samba3.*						# Samba3-specific test
 ^samba4.ntvfs.cifs.raw.samba3.*					# Samba3-specific test
-samba4.ntvfs.cifs.raw.
+^samba4.ntvfs.cifs.raw.
 ^samba4.rpc..*samba3.*						# Samba3-specific test
 ^samba4.samba-tool.domopen.*$				# Hangs for some reason
-nss.test						# Fails
-raw.offline 			# Samba 4 doesn't have much offline support yet
-rpc.autoidl  # this one just generates a lot of noise, and is no longer useful
-samba4.rpc.countcalls # this is not useful now we have full IDL
-samba4.rap.scan # same thing here - we have docs now
-samba4.rap.printing # Not provided by Samba 4
-samba4.rap.sam # Not provided by Samba 4
-samba4.gensec.python # not finished
+^samba4.nss.test						# Fails
+^samba4.raw.offline 			# Samba 4 doesn't have much offline support yet
+^samba4.rpc.autoidl  # this one just generates a lot of noise, and is no longer useful
+^samba4.rpc.countcalls # this is not useful now we have full IDL
+^samba4.rap.scan # same thing here - we have docs now
+^samba4.rap.printing # Not provided by Samba 4
+^samba4.rap.sam # Not provided by Samba 4
 bench # don't run benchmarks in our selftest
-trans2.scan # uses huge number of file descriptors
-base.scan.ioctl # bad idea in make test
-base.scan.pipe_number # bad idea in make test
-base.secleak # no point on build farm
-base.delaywrite # This is randomly failing, depending on timing and filesystem features
-base.winattr
-base.birthtime
-smb2.acls # new test which doesn't pass yet
+^samba4.ntvfs.cifs.trans2.scan 
+^samba4.trans2.scan # uses huge number of file descriptors
+^samba4.ntvfs.cifs.base.scan.ioctl
+^samba4.base.scan.ioctl # bad idea in make test
+^samba4.ntvfs.cifs.base.scan.pipe_number
+^samba4.base.scan.pipe_number # bad idea in make test
+^samba4.ntvfs.cifs.base.secleak
+^samba4.base.secleak # no point on build farm
+^samba4.ntvfs.cifs.base.delaywrite
+^samba4.base.delaywrite # This is randomly failing, depending on timing and filesystem features
+^samba4.ntvfs.cifs.base.winattr
+^samba4.base.winattr
+^samba4.ntvfs.cifs.base.birthtime
+^samba4.base.birthtime
+^samba4.smb2.acls # new test which doesn't pass yet
 # ktutil might not be installed or from mit...
 # we should build a samba4ktutil and use that instead
-samba4.blackbox.ktpass # this test isn't portable ...
+^samba4.blackbox.ktpass # this test isn't portable ...
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index 92ab28c2be9..bf42c7c303d 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -61,11 +61,20 @@ print "OPTIONS %s" % " ".join(torture_options)
 for options in ['-U"$USERNAME%$PASSWORD" --option=socket:testnonblock=true', '-U"$USERNAME%$PASSWORD"', '-U"$USERNAME%$PASSWORD" -k yes', '-U"$USERNAME%$PASSWORD" -k no', '-U"$USERNAME%$PASSWORD" -k no --sign', '-U"$USERNAME%$PASSWORD" -k no --encrypt', '-U"$USERNAME%$PASSWORD" -k yes --encrypt', '-U"$USERNAME%$PASSWORD" -k yes --sign']:
     plantestsuite("samba4.ldb.ldap with options %s(dc)" % options, "dc", "%s/test_ldb.sh ldap $SERVER %s" % (bbdir, options))
 
-# see if we support ldaps
+# see if we support ADS on the Samba3 side
 try:
     config_h = os.environ["CONFIG_H"]
 except KeyError:
-    config_h = os.path.join(samba4bindir, "default/source4/include/config.h")
+    config_h = os.path.join(samba4bindir, "default/include/config.h")
+
+f = open(config_h, 'r')
+try:
+    # The other parts of the HAVE_ADS test are always supplied by the top level build
+    have_ads_support = ("HAVE_LDAP 1" in f.read())
+finally:
+    f.close()
+
+# see if we support ldaps
 f = open(config_h, 'r')
 try:
     have_tls_support = ("ENABLE_GNUTLS 1" in f.read())
@@ -221,7 +230,11 @@ smb2 = smb4torture_testsuites("smb2.")
 raw = filter(lambda x: "raw.qfileinfo.ipc" not in x, smb4torture_testsuites("raw."))
 base = smb4torture_testsuites("base.")
 
-for t in base + raw + smb2:
+netapi = smb4torture_testsuites("netapi.")
+
+libsmbclient = smb4torture_testsuites("libsmbclient.")
+
+for t in base + raw + smb2 + netapi + libsmbclient:
     plansmbtorturetestsuite(t, "dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD' + " " + " ".join(ntvfsargs))
 
 plansmbtorturetestsuite("raw.qfileinfo.ipc", "dc", '//$SERVER/ipc\$ -U$USERNAME%$PASSWORD')
@@ -306,11 +319,12 @@ for mech in [
     "-k no",
     "-k no --option=usespnego=no",
     "-k no --option=gensec:spengo=no",
-    "-k yes",
-    "-k yes --option=gensec:fake_gssapi_krb5=yes --option=gensec:gssapi_krb5=no"]:
+    "-k yes"]:
     signoptions = "%s --signing=off" % mech
-    name = "smb.signing on with %s" % signoptions
+    name = "smb.signing disabled on with %s" % signoptions
     plantestsuite_loadlist("samba4.%s domain-creds" % name, "s4member", [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', signoptions, '-U$DC_USERNAME%$DC_PASSWORD', 'base.xcopy'])
+    if have_ads_support:
+        plantestsuite_loadlist("samba4.%s domain-creds" % name, "s3member", [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', signoptions, '-U$DC_USERNAME%$DC_PASSWORD', 'base.xcopy'])
 
 for mech in [
     "-k no",
@@ -319,6 +333,9 @@ for mech in [
     signoptions = "%s --signing=off" % mech
     name = "smb.signing on with %s" % signoptions
     plantestsuite_loadlist("samba4.%s local-creds" % name, "s4member", [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', signoptions, '-U$NETBIOSNAME/$USERNAME%$PASSWORD', 'base.xcopy'])
+    if have_ads_support:
+        plantestsuite_loadlist("samba4.%s" % name, "plugin_s4_dc", [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', signoptions, '-U$USERNAME%$PASSWORD', 'base.xcopy'])
+        plantestsuite_loadlist("samba4.%s administrator" % name, "plugin_s4_dc", [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', signoptions, '-U$DC_USERNAME%$DC_PASSWORD', 'base.xcopy'])
 plantestsuite_loadlist("samba4.smb.signing --signing=yes anon", "dc", [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', '-k', 'no', '--signing=yes', '-U%', 'base.xcopy'])
 plantestsuite_loadlist("samba4.smb.signing --signing=required anon", "dc", [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', '-k', 'no', '--signing=required', '-U%', 'base.xcopy'])
 plantestsuite_loadlist("samba4.smb.signing --signing=no anon", "s4member",  [valgrindify(smb4torture), "$LISTOPT", '//$NETBIOSNAME/tmp', '-k', 'no', '--signing=no', '-U%', 'base.xcopy'])
@@ -367,6 +384,7 @@ planpythontestsuite("none", "samba.tests.upgrade")
 planpythontestsuite("none", "samba.tests.core")
 planpythontestsuite("none", "samba.tests.provision")
 planpythontestsuite("none", "samba.tests.samba3")
+planpythontestsuite("none", "samba.tests.strings")
 planpythontestsuite("dc:local", "samba.tests.dcerpc.sam")
 planpythontestsuite("dc:local", "samba.tests.dsdb")
 planpythontestsuite("none", "samba.tests.netcmd")
@@ -385,6 +403,7 @@ plantestsuite("samba4.tokengroups.python(dc)", "dc:local", [python, os.path.join
 plantestsuite("samba4.sam.python(dc)", "dc", [python, os.path.join(samba4srcdir, "dsdb/tests/python/sam.py"), '$SERVER', '-U"$USERNAME%$PASSWORD"', '-W', '$DOMAIN'])
 plansambapythontestsuite("samba4.schemaInfo.python(dc)", "dc", os.path.join(samba4srcdir, 'dsdb/tests/python'), 'dsdb_schema_info', extra_args=['-U"$DOMAIN/$DC_USERNAME%$DC_PASSWORD"'])
 plantestsuite("samba4.urgent_replication.python(dc)", "dc", [python, os.path.join(samba4srcdir, "dsdb/tests/python/urgent_replication.py"), '$PREFIX_ABS/dc/private/sam.ldb'], allow_empty_output=True)
+plantestsuite("samba4.ldap.dirsync.python(dc)", "dc", [python, os.path.join(samba4srcdir, "dsdb/tests/python/dirsync.py"), '$SERVER', '-U"$USERNAME%$PASSWORD"', '-W', '$DOMAIN'])
 for env in ["dc", "fl2000dc", "fl2003dc", "fl2008r2dc"]:
     plantestsuite("samba4.ldap_schema.python(%s)" % env, env, [python, os.path.join(samba4srcdir, "dsdb/tests/python/ldap_schema.py"), '$SERVER', '-U"$USERNAME%$PASSWORD"', '-W', '$DOMAIN'])
     plantestsuite("samba4.ldap.possibleInferiors.python(%s)" % env, env, [python, os.path.join(samba4srcdir, "dsdb/samdb/ldb_modules/tests/possibleinferiors.py"), "ldap://$SERVER", '-U"$USERNAME%$PASSWORD"', "-W", "$DOMAIN"])
@@ -429,3 +448,6 @@ plantestsuite_loadlist("samba4.%s.two" % t, "vampire_dc", [valgrindify(smb4tortu
 plantestsuite_loadlist("samba4.rpc.echo", "rodc", [smb4torture, "$LISTOPT", 'ncacn_np:$SERVER', "-k", "yes", '-U$USERNAME%$PASSWORD', '-W' '$DOMAIN', 'rpc.echo'])
 plantestsuite_loadlist("samba4.rpc.echo", "rodc:local", [smb4torture, "$LISTOPT", 'ncacn_np:$SERVER', "-k", "yes", '-P', '-W' '$DOMAIN', 'rpc.echo'])
 plantestsuite("samba4.blackbox.provision-backend.py", "none", ["PYTHON=%s" % python, os.path.join(samba4srcdir, "setup/tests/blackbox_provision-backend.sh"), '$PREFIX/provision'])
+
+# Test renaming the DC
+plantestsuite("samba4.blackbox.renamedc.sh", "none", ["PYTHON=%s" % python, os.path.join(bbdir, "renamedc.sh"), '$PREFIX/provision'])
diff --git a/source4/setup/dns_update_list b/source4/setup/dns_update_list
index c69e155a804..91b182188b5 100644
--- a/source4/setup/dns_update_list
+++ b/source4/setup/dns_update_list
@@ -2,6 +2,8 @@
 # dynamic DNS update. It is processed by the samba_dnsupdate script
 A                                                        ${DNSDOMAIN} $IP
 A                                                        ${HOSTNAME} $IP
+AAAA                                                     ${DNSDOMAIN} $IP
+AAAA                                                     ${HOSTNAME} $IP
 CNAME ${NTDSGUID}._msdcs.${DNSDOMAIN}                    ${HOSTNAME}
 SRV _kerberos._tcp.${SITE}._sites.dc._msdcs.${DNSDOMAIN} ${HOSTNAME} 88
 SRV _ldap._tcp.${SITE}._sites.dc._msdcs.${DNSDOMAIN}     ${HOSTNAME} 389
diff --git a/source4/setup/named.txt b/source4/setup/named.txt
index c1e6b3a9ee5..97de69d8ebb 100644
--- a/source4/setup/named.txt
+++ b/source4/setup/named.txt
@@ -5,7 +5,7 @@
 
 # - Insert the following lines into the options {} section of your named.conf
 # file:
-tkey-gssapi-credential "DNS/${DNSDOMAIN}";
+tkey-gssapi-credential "DNS/${DNSNAME}";
 tkey-domain "${REALM}";
 
 # - Modify BIND init scripts to pass the location of the generated keytab file.
diff --git a/source4/setup/provision.smb.conf.dc b/source4/setup/provision.smb.conf.dc
index a8e98ba4bc9..f489f59ff9d 100644
--- a/source4/setup/provision.smb.conf.dc
+++ b/source4/setup/provision.smb.conf.dc
@@ -3,7 +3,6 @@
 	workgroup	= ${DOMAIN}
 	realm		= ${REALM}
 	server role     = ${SERVERROLE}
-	${SIDGENERATOR_LINE}
 	${PRIVATEDIR_LINE}
 	${LOCKDIR_LINE}
 
diff --git a/source4/setup/provision.smb.conf.member b/source4/setup/provision.smb.conf.member
index 8241fc28f1c..96e5d0c2e5a 100644
--- a/source4/setup/provision.smb.conf.member
+++ b/source4/setup/provision.smb.conf.member
@@ -3,6 +3,5 @@
 	workgroup	= ${DOMAIN}
 	realm		= ${REALM}
 	server role     = ${SERVERROLE}
-	${SIDGENERATOR_LINE}
 	${PRIVATEDIR_LINE}
 	${LOCKDIR_LINE}
diff --git a/source4/setup/provision.smb.conf.standalone b/source4/setup/provision.smb.conf.standalone
index 8241fc28f1c..96e5d0c2e5a 100644
--- a/source4/setup/provision.smb.conf.standalone
+++ b/source4/setup/provision.smb.conf.standalone
@@ -3,6 +3,5 @@
 	workgroup	= ${DOMAIN}
 	realm		= ${REALM}
 	server role     = ${SERVERROLE}
-	${SIDGENERATOR_LINE}
 	${PRIVATEDIR_LINE}
 	${LOCKDIR_LINE}
diff --git a/source4/setup/provision_basedn_modify.ldif b/source4/setup/provision_basedn_modify.ldif
index d67d674319e..a5e704769db 100644
--- a/source4/setup/provision_basedn_modify.ldif
+++ b/source4/setup/provision_basedn_modify.ldif
@@ -82,15 +82,9 @@ pwdProperties: 1
 replace: pwdHistoryLength
 pwdHistoryLength: 24
 -
-replace: rIDManagerReference
-rIDManagerReference: CN=RID Manager$,CN=System,${DOMAINDN}
--
 replace: serverState
 serverState: 1
 -
-replace: subRefs
-subRefs: ${CONFIGDN}
--
 replace: systemFlags
 systemFlags: -1946157056
 -
diff --git a/source4/setup/provision_configuration.ldif b/source4/setup/provision_configuration.ldif
index 2ccf6eded00..cb049b0c1e1 100644
--- a/source4/setup/provision_configuration.ldif
+++ b/source4/setup/provision_configuration.ldif
@@ -1194,7 +1194,6 @@ dn: CN=Sites,${CONFIGDN}
 objectClass: top
 objectClass: sitesContainer
 systemFlags: -2113929216
-nTSecurityDescriptor:: ${SITES_DESCRIPTOR}
 
 dn: CN=${DEFAULTSITE},CN=Sites,${CONFIGDN}
 objectClass: top
diff --git a/source4/setup/provision_self_join_modify.ldif b/source4/setup/provision_self_join_modify.ldif
index 2fe5a43db6f..aba1b862e1b 100644
--- a/source4/setup/provision_self_join_modify.ldif
+++ b/source4/setup/provision_self_join_modify.ldif
@@ -2,6 +2,8 @@ dn: ${DOMAINDN}
 changetype: modify
 replace: fSMORoleOwner
 fSMORoleOwner: CN=NTDS Settings,${SERVERDN}
+replace: rIDManagerReference
+rIDManagerReference: CN=RID Manager$,CN=System,${DOMAINDN}
 
 dn: ${SCHEMADN}
 changetype: modify
diff --git a/source4/setup/tests/blackbox_newuser.sh b/source4/setup/tests/blackbox_newuser.sh
index fe5d051481a..5ce634a70d8 100755
--- a/source4/setup/tests/blackbox_newuser.sh
+++ b/source4/setup/tests/blackbox_newuser.sh
@@ -26,17 +26,17 @@ testit "newuser" $samba_tool newuser $CONFIG --given-name="User" --surname="Test
 testit "newuser" $samba_tool newuser $CONFIG --use-username-as-cn --given-name="User1" --surname="Tester1" --initials="UT1" --profile-path="\\\\myserver\\my\\profile" --script-path="\\\\myserver\\my\\script" --home-directory="\\\\myserver\\my\\homedir" --job-title="Tester" --department="Testing" --company="Samba.org" --description="Description" --mail-address="tester@samba.org" --internet-address="http://samba.org" --telephone-number="001122334455" --physical-delivery-office="101" --home-drive="H:" NewUser1 testp@ssw0Rd
 
 # check the enable account script
-testit "enableaccount" $samba_tool enableaccount $CONFIG NewUser
-testit "enableaccount" $samba_tool enableaccount $CONFIG NewUser1
+testit "enableaccount" $samba_tool user enable $CONFIG NewUser
+testit "enableaccount" $samba_tool user enable $CONFIG NewUser1
 
 # check the enable account script
 testit "setpassword" $samba_tool setpassword $CONFIG NewUser --newpassword=testp@ssw0Rd2
 testit "setpassword" $samba_tool setpassword $CONFIG NewUser1 --newpassword=testp@ssw0Rd2
 
 # check the setexpiry script
-testit "noexpiry" $samba_tool setexpiry $CONFIG NewUser --noexpiry
-testit "noexpiry" $samba_tool setexpiry $CONFIG NewUser1 --noexpiry
-testit "expiry" $samba_tool setexpiry $CONFIG NewUser --days=7
-testit "expiry" $samba_tool setexpiry $CONFIG NewUser1 --days=7
+testit "noexpiry" $samba_tool user setexpiry $CONFIG NewUser --noexpiry
+testit "noexpiry" $samba_tool user setexpiry $CONFIG NewUser1 --noexpiry
+testit "expiry" $samba_tool user setexpiry $CONFIG NewUser --days=7
+testit "expiry" $samba_tool user setexpiry $CONFIG NewUser1 --days=7
 
 exit $failed
diff --git a/source4/setup/tests/blackbox_upgradeprovision.sh b/source4/setup/tests/blackbox_upgradeprovision.sh
index 05b155d5214..a3d1838f887 100755
--- a/source4/setup/tests/blackbox_upgradeprovision.sh
+++ b/source4/setup/tests/blackbox_upgradeprovision.sh
@@ -24,13 +24,10 @@ upgradeprovision_full() {
   if [ -d $PREFIX/upgradeprovision_full ]; then
     rm -fr $PREFIX/upgradeprovision_full
   fi
-	$PYTHON $SRCDIR/source4/setup/provision --domain=FOO --realm=foo.example.com --targetdir="$PREFIX/upgradeprovision_full" --server-role="dc"
+	$PYTHON $SRCDIR/source4/setup/provision --host-name=bar --domain=FOO --realm=foo.example.com --targetdir="$PREFIX/upgradeprovision_full" --server-role="dc"
 	$PYTHON $SRCDIR/source4/scripting/bin/upgradeprovision -s "$PREFIX/upgradeprovision_full/etc/smb.conf" --full --debugchange
 }
 
-testit "upgradeprovision" upgradeprovision
-testit "upgradeprovision_full" upgradeprovision_full
-
 if [ -d $PREFIX/upgradeprovision ]; then
   rm -fr $PREFIX/upgradeprovision
 fi
@@ -39,4 +36,7 @@ if [ -d $PREFIX/upgradeprovision_full ]; then
   rm -fr $PREFIX/upgradeprovision_full
 fi
 
+testit "upgradeprovision" upgradeprovision
+testit "upgradeprovision_full" upgradeprovision_full
+
 exit $failed
diff --git a/source4/setup/wscript_build b/source4/setup/wscript_build
index 241e8b86861..65cbfc9aeb1 100644
--- a/source4/setup/wscript_build
+++ b/source4/setup/wscript_build
@@ -7,6 +7,8 @@ bld.INSTALL_WILDCARD('${SETUPDIR}', 'display-specifiers/*.txt')
 
 bld.INSTALL_FILES('${SBINDIR}', 'provision', chmod=MODE_755, python_fixup=True)
 
+bld.SAMBA_SCRIPT('provision', pattern='provision', installdir='.')
+
 bld.INSTALL_FILES('${SETUPDIR}', 'dns_update_list')
 bld.INSTALL_FILES('${SETUPDIR}', 'spn_update_list')
 
diff --git a/source4/smb_server/service_smb.c b/source4/smb_server/service_smb.c
index 583360bbe79..cbbd2cd95e2 100644
--- a/source4/smb_server/service_smb.c
+++ b/source4/smb_server/service_smb.c
@@ -48,24 +48,32 @@ static void smbsrv_task_init(struct task_server *task)
 		int i;
 		struct interface *ifaces;
 
-		load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
+		load_interface_list(task, task->lp_ctx, &ifaces);
 
-		num_interfaces = iface_count(ifaces);
+		num_interfaces = iface_list_count(ifaces);
 
 		/* We have been given an interfaces line, and been 
 		   told to only bind to those interfaces. Create a
 		   socket per interface and bind to only these.
 		*/
 		for(i = 0; i < num_interfaces; i++) {
-			const char *address = iface_n_ip(ifaces, i);
+			const char *address = iface_list_n_ip(ifaces, i);
 			status = smbsrv_add_socket(task, task->event_ctx, task->lp_ctx, task->model_ops, address);
 			if (!NT_STATUS_IS_OK(status)) goto failed;
 		}
 	} else {
-		/* Just bind to lpcfg_socket_address() (usually 0.0.0.0) */
-		status = smbsrv_add_socket(task, task->event_ctx, task->lp_ctx, task->model_ops,
-					   lpcfg_socket_address(task->lp_ctx));
-		if (!NT_STATUS_IS_OK(status)) goto failed;
+		const char **wcard;
+		int i;
+		wcard = iface_list_wildcard(task, task->lp_ctx);
+		if (wcard == NULL) {
+			DEBUG(0,("No wildcard addresses available\n"));
+			goto failed;
+		}
+		for (i=0; wcard[i]; i++) {
+			status = smbsrv_add_socket(task, task->event_ctx, task->lp_ctx, task->model_ops, wcard[i]);
+			if (!NT_STATUS_IS_OK(status)) goto failed;
+		}
+		talloc_free(wcard);
 	}
 
 	return;
diff --git a/source4/smb_server/smb/negprot.c b/source4/smb_server/smb/negprot.c
index 656da4df201..0a07ab93e2d 100644
--- a/source4/smb_server/smb/negprot.c
+++ b/source4/smb_server/smb/negprot.c
@@ -145,7 +145,7 @@ static void reply_lanman1(struct smbsrv_request *req, uint16_t choice)
 	SSVAL(req->out.vwv, VWV(3), lpcfg_maxmux(req->smb_conn->lp_ctx));
 	SSVAL(req->out.vwv, VWV(4), 1);
 	SSVAL(req->out.vwv, VWV(5), raw); 
-	SIVAL(req->out.vwv, VWV(6), req->smb_conn->connection->server_id.id);
+	SIVAL(req->out.vwv, VWV(6), req->smb_conn->connection->server_id.pid);
 	srv_push_dos_date(req->smb_conn, req->out.vwv, VWV(8), t);
 	SSVAL(req->out.vwv, VWV(10), req->smb_conn->negotiate.zone_offset/60);
 	SIVAL(req->out.vwv, VWV(11), 0); /* reserved */
@@ -199,7 +199,7 @@ static void reply_lanman2(struct smbsrv_request *req, uint16_t choice)
 	SSVAL(req->out.vwv, VWV(3), lpcfg_maxmux(req->smb_conn->lp_ctx));
 	SSVAL(req->out.vwv, VWV(4), 1);
 	SSVAL(req->out.vwv, VWV(5), raw); 
-	SIVAL(req->out.vwv, VWV(6), req->smb_conn->connection->server_id.id);
+	SIVAL(req->out.vwv, VWV(6), req->smb_conn->connection->server_id.pid);
 	srv_push_dos_date(req->smb_conn, req->out.vwv, VWV(8), t);
 	SSVAL(req->out.vwv, VWV(10), req->smb_conn->negotiate.zone_offset/60);
 	SIVAL(req->out.vwv, VWV(11), 0);
@@ -278,7 +278,7 @@ static void reply_nt1(struct smbsrv_request *req, uint16_t choice)
 		capabilities |= CAP_LARGE_READX | CAP_LARGE_WRITEX | CAP_W2K_SMBS;
 	}
 
-	large_test_path = lock_path(req, req->smb_conn->lp_ctx, "large_test.dat");
+	large_test_path = lpcfg_lock_path(req, req->smb_conn->lp_ctx, "large_test.dat");
 	if (large_file_support(large_test_path)) {
 		capabilities |= CAP_LARGE_FILES;
 	}
@@ -332,7 +332,8 @@ static void reply_nt1(struct smbsrv_request *req, uint16_t choice)
 	SSVAL(req->out.vwv+1, VWV(2), 1); /* num vcs */
 	SIVAL(req->out.vwv+1, VWV(3), req->smb_conn->negotiate.max_recv);
 	SIVAL(req->out.vwv+1, VWV(5), 0x10000); /* raw size. full 64k */
-	SIVAL(req->out.vwv+1, VWV(7), req->smb_conn->connection->server_id.id); /* session key */
+	SIVAL(req->out.vwv+1, VWV(7), req->smb_conn->connection->server_id.pid); /* session key */
+
 	SIVAL(req->out.vwv+1, VWV(9), capabilities);
 	push_nttime(req->out.vwv+1, VWV(11), nttime);
 	SSVALS(req->out.vwv+1,VWV(15), req->smb_conn->negotiate.zone_offset/60);
diff --git a/source4/smb_server/smb/receive.c b/source4/smb_server/smb/receive.c
index c2503efabc8..1379fe31cf5 100644
--- a/source4/smb_server/smb/receive.c
+++ b/source4/smb_server/smb/receive.c
@@ -25,7 +25,7 @@
 #include "smb_server/smb_server.h"
 #include "system/filesys.h"
 #include "param/param.h"
-
+#include "cluster/cluster.h"
 
 /*
   send an oplock break request to a client
@@ -471,6 +471,7 @@ static void switch_message(int type, struct smbsrv_request *req)
 	int flags;
 	struct smbsrv_connection *smb_conn = req->smb_conn;
 	NTSTATUS status;
+	char *task_id;
 
 	type &= 0xff;
 
@@ -501,8 +502,10 @@ static void switch_message(int type, struct smbsrv_request *req)
 		}
 	}
 
-	DEBUG(5,("switch message %s (task_id %u)\n",
-		 smb_fn_name(type), (unsigned)req->smb_conn->connection->server_id.id));
+	task_id = server_id_str(NULL, &req->smb_conn->connection->server_id);
+	DEBUG(5,("switch message %s (task_id %s)\n",
+		 smb_fn_name(type), task_id));
+	talloc_free(task_id);
 
 	/* this must be called before we do any reply */
 	if (flags & SIGNING_NO_REPLY) {
diff --git a/source4/smb_server/smb/sesssetup.c b/source4/smb_server/smb/sesssetup.c
index c4efe3919c3..116f2cd9584 100644
--- a/source4/smb_server/smb/sesssetup.c
+++ b/source4/smb_server/smb/sesssetup.c
@@ -34,7 +34,7 @@
 #include "lib/stream/packet.h"
 
 struct sesssetup_context {
-	struct auth_context *auth_context;
+	struct auth4_context *auth_context;
 	struct smbsrv_request *req;
 };
 
diff --git a/source4/smb_server/smb/trans2.c b/source4/smb_server/smb/trans2.c
index 0a6c014e88a..72babd533bf 100644
--- a/source4/smb_server/smb/trans2.c
+++ b/source4/smb_server/smb/trans2.c
@@ -867,24 +867,14 @@ static NTSTATUS fill_normal_dfs_referraltype(struct dfs_referral_type *ref,
 					     const char *dfs_path,
 					     const char *server_path, int isfirstoffset)
 {
-
+	ZERO_STRUCTP(ref);
 	switch (version) {
-	case 3:
-		ZERO_STRUCTP(ref);
-		ref->version = version;
-		ref->referral.v3.data.server_type = DFS_SERVER_NON_ROOT;
-		/* "normal" referral seems to always include the GUID */
-		ref->referral.v3.size = 34;
-
-		ref->referral.v3.data.entry_flags = 0;
-		ref->referral.v3.data.ttl = 600; /* As w2k3 */
-		ref->referral.v3.data.referrals.r1.DFS_path = dfs_path;
-		ref->referral.v3.data.referrals.r1.DFS_alt_path = dfs_path;
-		ref->referral.v3.data.referrals.r1.netw_address = server_path;
-		return NT_STATUS_OK;
 	case 4:
-		ZERO_STRUCTP(ref);
-		ref->version = version;
+		version = 3;
+# if 0
+		/* For the moment there is a bug with XP that don't seems to appriciate much
+		 * level4 so we return just level 3 for everyone
+		 */
 		ref->referral.v4.server_type = DFS_SERVER_NON_ROOT;
 		/* "normal" referral seems to always include the GUID */
 		ref->referral.v4.size = 34;
@@ -892,11 +882,23 @@ static NTSTATUS fill_normal_dfs_referraltype(struct dfs_referral_type *ref,
 		if (isfirstoffset) {
 			ref->referral.v4.entry_flags =  DFS_HEADER_FLAG_TARGET_BCK;
 		}
-		ref->referral.v4.ttl = 600; /* As w2k3 */
-		ref->referral.v4.r1.DFS_path = dfs_path;
-		ref->referral.v4.r1.DFS_alt_path = dfs_path;
-		ref->referral.v4.r1.netw_address = server_path;
+		ref->referral.v4.ttl = 900; /* As w2k8r2 */
+		ref->referral.v4.referrals.r1.DFS_path = talloc_strdup(ref, dfs_path);
+		ref->referral.v4.referrals.r1.DFS_alt_path = talloc_strdup(ref, dfs_path);
+		ref->referral.v4.referrals.r1.netw_address = talloc_strdup(ref, server_path);
+		return NT_STATUS_OK;
+#endif
+	case 3:
+		ref->version = version;
+		ref->referral.v3.server_type = DFS_SERVER_NON_ROOT;
+		/* "normal" referral seems to always include the GUID */
+		ref->referral.v3.size = 34;
 
+		ref->referral.v3.entry_flags = 0;
+		ref->referral.v3.ttl = 600; /* As w2k3 */
+		ref->referral.v3.referrals.r1.DFS_path = talloc_strdup(ref, dfs_path);
+		ref->referral.v3.referrals.r1.DFS_alt_path = talloc_strdup(ref, dfs_path);
+		ref->referral.v3.referrals.r1.netw_address = talloc_strdup(ref, server_path);
 		return NT_STATUS_OK;
 	}
 	return NT_STATUS_INVALID_LEVEL;
@@ -914,18 +916,25 @@ static NTSTATUS fill_domain_dfs_referraltype(struct dfs_referral_type *ref,
 	switch (version) {
 	case 3:
 		ZERO_STRUCTP(ref);
+		DEBUG(8, ("Called fill_domain_dfs_referraltype\n"));
 		ref->version = version;
-		ref->referral.v3.data.server_type = DFS_SERVER_NON_ROOT;
+		ref->referral.v3.server_type = DFS_SERVER_NON_ROOT;
 		/* It's hard coded ... don't think it's a good way but the sizeof return not the
 		 * correct values
 		 *
 		 * We have 18 if the GUID is not included 34 otherwise
 		*/
-		ref->referral.v3.size = 18;
-		ref->referral.v3.data.entry_flags = DFS_FLAG_REFERRAL_DOMAIN_RESP;
-		ref->referral.v3.data.ttl = 600; /* As w2k3 */
-		ref->referral.v3.data.referrals.r2.special_name = domain;
-		ref->referral.v3.data.referrals.r2.nb_expanded_names = numnames;
+		if (numnames == 0) {
+			/* Windows return without the guid when returning domain list
+			 */
+			ref->referral.v3.size = 18;
+		} else {
+			ref->referral.v3.size = 34;
+		}
+		ref->referral.v3.entry_flags = DFS_FLAG_REFERRAL_DOMAIN_RESP;
+		ref->referral.v3.ttl = 600; /* As w2k3 */
+		ref->referral.v3.referrals.r2.special_name = domain;
+		ref->referral.v3.referrals.r2.nb_expanded_names = numnames;
 		/* Put the final terminator */
 		if (names) {
 			const char **names2 = talloc_array(ref, const char *, numnames+1);
@@ -935,8 +944,8 @@ static NTSTATUS fill_domain_dfs_referraltype(struct dfs_referral_type *ref,
 				names2[i] = talloc_asprintf(names2, "\\%s", names[i]);
 				NT_STATUS_HAVE_NO_MEMORY(names2[i]);
 			}
-			names2[numnames] = 0;
-			ref->referral.v3.data.referrals.r2.expanded_names = names2;
+			names2[numnames] = NULL;
+			ref->referral.v3.referrals.r2.expanded_names = names2;
 		}
 		return NT_STATUS_OK;
 	}
@@ -1098,7 +1107,7 @@ static NTSTATUS get_dcs(TALLOC_CTX *ctx, struct ldb_context *ldb,
 	}
 	talloc_free(r);
 
-	if (searched_site != NULL) {
+	if (searched_site != NULL && searched_site[0] != '\0') {
 		ret = ldb_search(ldb, subctx, &r, configdn, LDB_SCOPE_SUBTREE,
 				 attrs_none, "(&(name=%s)(objectClass=site))", searched_site);
 		if (ret != LDB_SUCCESS) {
@@ -1342,7 +1351,7 @@ static NTSTATUS dodomain_referral(TALLOC_CTX *ctx,
 		}
 
 		if (!ok && resp.nb_referrals == 2) {
-			DEBUG(0, (__location__ "; Not able to fit the domain and realm in DFS a "
+			DEBUG(8, (__location__ "; Not able to fit the domain and realm in DFS a "
 				  " 56K buffer, something must be broken"));
 			talloc_free(context);
 			return NT_STATUS_INTERNAL_ERROR;
@@ -1363,6 +1372,8 @@ static NTSTATUS dodomain_referral(TALLOC_CTX *ctx,
  */
 static NTSTATUS dodc_or_sysvol_referral(TALLOC_CTX *ctx,
 					const struct dfs_GetDFSReferral_in dfsreq,
+					const char* requesteddomain,
+					const char* requestedshare,
 					const char* requestedname,
 					struct ldb_context *ldb,
 					struct smb_trans2 *trans,
@@ -1378,16 +1389,13 @@ static NTSTATUS dodc_or_sysvol_referral(TALLOC_CTX *ctx,
 	NTSTATUS status;
 	unsigned int num_domain = 1;
 	enum ndr_err_code ndr_err;
-	const char *requesteddomain;
 	const char *realm = lpcfg_realm(lp_ctx);
 	const char *domain = lpcfg_workgroup(lp_ctx);
 	const char *site_name = NULL; /* Name of the site where the client is */
-	char *share = NULL;
 	bool found = false;
 	bool need_fqdn = false;
 	bool dc_referral = true;
 	unsigned int i;
-	char *tmp;
 	struct dc_set **set;
 	char const **domain_list;
 	struct tsocket_address *remote_address;
@@ -1407,24 +1415,13 @@ static NTSTATUS dodc_or_sysvol_referral(TALLOC_CTX *ctx,
 	context = talloc_new(ctx);
 	NT_STATUS_HAVE_NO_MEMORY(context);
 
-	if (requestedname[0] == '\\' && !strchr(requestedname+1,'\\')) {
-		requestedname++;
-	}
-	requesteddomain = requestedname;
-
-	if (strchr(requestedname,'\\')) {
-		char *subpart;
-		/* we have a second part */
-		requesteddomain = talloc_strdup(context, requestedname+1);
-		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(requesteddomain, context);
-		subpart = strchr(requesteddomain,'\\');
-		subpart[0] = '\0';
-	}
-	tmp = strchr(requestedname + 1,'\\'); /* To get second \ if any */
+	DEBUG(10, ("in this we have request for %s and share %s requested is %s\n",
+				requesteddomain,
+				requestedshare,
+				requestedname));
 
-	if (tmp != NULL) {
-		/* There was a share */
-		share = tmp+1;
+	if (requestedshare) {
+		DEBUG(10, ("Have a non DC domain referal\n"));
 		dc_referral = false;
 	}
 
@@ -1464,7 +1461,7 @@ static NTSTATUS dodc_or_sysvol_referral(TALLOC_CTX *ctx,
 		client_addr = tsocket_address_inet_addr_string(remote_address, context);
 		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(client_addr, context);
 	}
-
+	site_name = samdb_client_site_name(ldb, context, client_addr, NULL);
 	status = get_dcs(context, ldb, site_name, need_fqdn, &set, 0);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(3,("Unable to get list of DCs\n"));
@@ -1508,8 +1505,13 @@ static NTSTATUS dodc_or_sysvol_referral(TALLOC_CTX *ctx,
 		referral = talloc(context, struct dfs_referral_type);
 		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(referral, context);
 
-		referral_str = talloc_asprintf(referral, "\\%s",
-					       requestedname);
+		if (requestedname[0] == '\\') {
+			referral_str = talloc_asprintf(referral, "%s",
+						       requestedname);
+		} else {
+			referral_str = talloc_asprintf(referral, "\\%s",
+						       requestedname);
+		}
 		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(referral_str, context);
 
 		status = fill_domain_dfs_referraltype(referral,  3,
@@ -1564,12 +1566,14 @@ static NTSTATUS dodc_or_sysvol_referral(TALLOC_CTX *ctx,
 				NT_STATUS_HAVE_NO_MEMORY_AND_FREE(referral, context);
 
 				referral_str = talloc_asprintf(referral, "\\%s\\%s",
-							       set[i]->names[j], share);
+							       set[i]->names[j], requestedshare);
+				DEBUG(8, ("Doing a dfs referral for %s with this value %s requested %s\n",  set[i]->names[j], referral_str, requestedname));
 				NT_STATUS_HAVE_NO_MEMORY_AND_FREE(referral_str, context);
 
 				status = fill_normal_dfs_referraltype(referral,
 						dfsreq.max_referral_level,
 						requestedname, referral_str, j==0);
+
 				if (!NT_STATUS_IS_OK(status)) {
 					DEBUG(2, (__location__ ": Unable to fill a normal dfs referral object"));
 					talloc_free(context);
@@ -1616,7 +1620,7 @@ static NTSTATUS trans2_getdfsreferral(struct smbsrv_request *req,
 	struct ldb_context *ldb;
 	struct loadparm_context *lp_ctx;
 	const char *realm, *nbname, *requestedname;
-	char *fqdn, *tmp;
+	char *fqdn, *share, *domain, *tmp;
 	NTSTATUS status;
 
 	lp_ctx = req->tcon->ntvfs->lp_ctx;
@@ -1645,7 +1649,7 @@ static NTSTATUS trans2_getdfsreferral(struct smbsrv_request *req,
 		return status;
 	}
 
-	DEBUG(10, ("Requested DFS name: %s length: %u\n",
+	DEBUG(8, ("Requested DFS name: %s length: %u\n",
 		   dfsreq.servername, (unsigned int)strlen(dfsreq.servername)));
 
 	/*
@@ -1679,31 +1683,41 @@ static NTSTATUS trans2_getdfsreferral(struct smbsrv_request *req,
 	}
 	talloc_free(fqdn);
 
-	tmp = strchr(requestedname + 1,'\\'); /* To get second \ if any */
+	domain = talloc_strdup(context, requestedname);
+	while(*domain && *domain == '\\') {
+		domain++;
+	}
 
+	tmp = strchr(domain,'\\'); /* To get second \ if any */
+	share = NULL;
+	if (tmp) {
+		/*
+		 * We are finishing properly the domain string
+		 * and the share one will start after the \
+		 */
+		tmp[0] = '\\';
+		tmp++;
+		share = talloc_strdup(context, tmp);
+	}
 	/*
-	 * If we have no slash at the first position or (foo.bar.domain.net)
-	 * a slash at the first position but no other slash (\foo.bar.domain.net)
-	 * or a slash at the first position and another slash
-	 * and netlogon or sysvol after the second slash
-	 * (\foo.bar.domain.net\sysvol) then we will handle it because
-	 * it's either a dc referral or a sysvol/netlogon referral
+	 * Here we have filtered the thing the requested name don't contain our DNS name.
+	 * So if the share == NULL or if share in ("sysvol", "netlogon")
+	 * then we proceed. In the first case it will be a dc refereal in the second it will
+	 * be just a sysvol/netlogon referral.
 	 */
-	if (requestedname[0] != '\\' ||
-	    tmp == NULL ||
-	    strcasecmp(tmp+1, "sysvol") == 0 ||
-	    strcasecmp(tmp+1, "netlogon") == 0) {
-		status = dodc_or_sysvol_referral(op, dfsreq, requestedname,
+	if (share == NULL ||
+	    strcasecmp(share, "sysvol") == 0 ||
+	    strcasecmp(share, "netlogon") == 0) {
+		status = dodc_or_sysvol_referral(op, dfsreq, domain, share, requestedname,
 					       ldb, trans, req, lp_ctx);
 		talloc_free(context);
 		return status;
 	}
 
-	if (requestedname[0] == '\\' &&
-	    tmp &&
-	    strchr(tmp+1, '\\') &&
-	    (strncasecmp(tmp+1, "sysvol", 6) == 0 ||
-	     strncasecmp(tmp+1, "netlogon", 8) == 0)) {
+	tmp = strchr(share, '\\');
+	if (tmp &&
+	    (strncasecmp(share, "sysvol", 6) == 0 ||
+	     strncasecmp(share, "netlogon", 8) == 0)) {
 		/*
 		 * We have more than two \ so it something like
 		 * \domain\sysvol\foobar
diff --git a/source4/smb_server/smb_samba3.c b/source4/smb_server/smb_samba3.c
index b0ed38cf6a8..1a99be644a8 100644
--- a/source4/smb_server/smb_samba3.c
+++ b/source4/smb_server/smb_samba3.c
@@ -135,16 +135,16 @@ static void samba3_smb_task_init(struct task_server *task)
 		int i;
 		struct interface *ifaces;
 
-		load_interfaces(task, lpcfg_interfaces(task->lp_ctx), &ifaces);
+		load_interface_list(task, task->lp_ctx, &ifaces);
 
-		num_interfaces = iface_count(ifaces);
+		num_interfaces = iface_list_count(ifaces);
 
 		/* We have been given an interfaces line, and been
 		   told to only bind to those interfaces. Create a
 		   socket per interface and bind to only these.
 		*/
 		for(i = 0; i < num_interfaces; i++) {
-			const char *address = iface_n_ip(ifaces, i);
+			const char *address = iface_list_n_ip(ifaces, i);
 			status = samba3_add_socket(task,
 						   task->event_ctx,
 						   task->lp_ctx,
@@ -152,12 +152,21 @@ static void samba3_smb_task_init(struct task_server *task)
 			if (!NT_STATUS_IS_OK(status)) goto failed;
 		}
 	} else {
-		/* Just bind to lpcfg_socket_address() (usually 0.0.0.0) */
-		status = samba3_add_socket(task,
-					   task->event_ctx, task->lp_ctx,
-					   model_ops,
-					   lpcfg_socket_address(task->lp_ctx));
-		if (!NT_STATUS_IS_OK(status)) goto failed;
+		const char **wcard;
+		int i;
+		wcard = iface_list_wildcard(task, task->lp_ctx);
+		if (wcard == NULL) {
+			DEBUG(0,("No wildcard addresses available\n"));
+			goto failed;
+		}
+		for (i=0; wcard[i]; i++) {
+			status = samba3_add_socket(task,
+						   task->event_ctx, task->lp_ctx,
+						   model_ops,
+						   wcard[i]);
+			if (!NT_STATUS_IS_OK(status)) goto failed;
+		}
+		talloc_free(wcard);
 	}
 
 	return;
diff --git a/source4/smb_server/smb_server.c b/source4/smb_server/smb_server.c
index d21e5fbdb07..d64a597d31f 100644
--- a/source4/smb_server/smb_server.c
+++ b/source4/smb_server/smb_server.c
@@ -190,7 +190,7 @@ _PUBLIC_ NTSTATUS smbsrv_add_socket(TALLOC_CTX *mem_ctx,
 		if (port == 0) continue;
 		status = stream_setup_socket(mem_ctx, event_context, lp_ctx,
 					     model_ops, &smb_stream_ops, 
-					     "ipv4", address, &port, 
+					     "ip", address, &port,
 					     lpcfg_socket_options(lp_ctx),
 					     NULL);
 		NT_STATUS_NOT_OK_RETURN(status);
diff --git a/source4/smb_server/smb_server.h b/source4/smb_server/smb_server.h
index 6088853743b..6fcd9787bb7 100644
--- a/source4/smb_server/smb_server.h
+++ b/source4/smb_server/smb_server.h
@@ -22,6 +22,7 @@
 #include "libcli/raw/request.h"
 #include "libcli/raw/interfaces.h"
 #include "lib/socket/socket.h"
+#include "libds/common/roles.h"
 #include "../lib/util/dlinklist.h"
 #include "../librpc/gen_ndr/nbt.h"
 
@@ -265,8 +266,6 @@ struct smbsrv_request {
 	struct smb_request_buffer out;
 };
 
-enum security_types {SEC_SHARE,SEC_USER};
-
 /* smb server context structure. This should contain all the state
  * information associated with a SMB server connection 
  */
@@ -300,7 +299,7 @@ struct smbsrv_connection {
 		enum protocol_types protocol;
 
 		/* authentication context for multi-part negprot */
-		struct auth_context *auth_context;
+		struct auth4_context *auth_context;
 	
 		/* reference to the kerberos keytab, or machine trust account */
 		struct cli_credentials *server_credentials;
diff --git a/source4/smbd/process_model.c b/source4/smbd/process_model.c
index af12a7b8cb1..3c4c03ebf2c 100644
--- a/source4/smbd/process_model.c
+++ b/source4/smbd/process_model.c
@@ -111,7 +111,7 @@ _PUBLIC_ NTSTATUS process_model_init(struct loadparm_context *lp_ctx)
 	}
 	initialised = true;
 
-	shared_init = load_samba_modules(NULL, lp_ctx, "process_model");
+	shared_init = load_samba_modules(NULL, "process_model");
 	
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
diff --git a/source4/smbd/server.c b/source4/smbd/server.c
index 0f2f1bb1bb7..1be3b2885f6 100644
--- a/source4/smbd/server.c
+++ b/source4/smbd/server.c
@@ -178,13 +178,8 @@ _NORETURN_ static void max_runtime_handler(struct tevent_context *ev,
 					   struct timeval t, void *private_data)
 {
 	const char *binary_name = (const char *)private_data;
-	struct timeval tv;
-	struct timezone tz;
-	if (gettimeofday(&tv, &tz) == 0) {
-		DEBUG(0,("%s: maximum runtime exceeded - terminating, current ts: %d\n", binary_name, (int)tv.tv_sec));
-	} else {
-		DEBUG(0,("%s: maximum runtime exceeded - terminating\n", binary_name));
-	}
+	DEBUG(0,("%s: maximum runtime exceeded - terminating, current ts: %llu\n",
+	      binary_name, (unsigned long long) time(NULL)));
 	exit(0);
 }
 
@@ -221,11 +216,11 @@ static NTSTATUS samba_terminate(struct irpc_message *msg,
 static NTSTATUS setup_parent_messaging(struct tevent_context *event_ctx, 
 				       struct loadparm_context *lp_ctx)
 {
-	struct messaging_context *msg;
+	struct imessaging_context *msg;
 	NTSTATUS status;
 
-	msg = messaging_init(talloc_autofree_context(), 
-			     lpcfg_messaging_path(event_ctx, lp_ctx),
+	msg = imessaging_init(talloc_autofree_context(),
+			     lpcfg_imessaging_path(event_ctx, lp_ctx),
 			     cluster_id(0, SAMBA_PARENT_TASKID), event_ctx);
 	NT_STATUS_HAVE_NO_MEMORY(msg);
 
@@ -388,10 +383,9 @@ static int binary_smbd_main(const char *binary_name, int argc, const char *argv[
 
 	pidfile_create(lpcfg_piddir(cmdline_lp_ctx), binary_name);
 
-	/* Do *not* remove this, until you have removed
-	 * passdb/secrets.c, and proved that Samba still builds... */
-	/* Setup the SECRETS subsystem */
-	if (secrets_init(talloc_autofree_context(), cmdline_lp_ctx) == NULL) {
+	/* Set up a database to hold a random seed, in case we don't
+	 * have /dev/urandom */
+	if (!randseed_init(talloc_autofree_context(), cmdline_lp_ctx)) {
 		return 1;
 	}
 
@@ -402,9 +396,9 @@ static int binary_smbd_main(const char *binary_name, int argc, const char *argv[
 		}
 	}
 
-	gensec_init(cmdline_lp_ctx); /* FIXME: */
+	gensec_init(); /* FIXME: */
 
-	ntptr_init(cmdline_lp_ctx);	/* FIXME: maybe run this in the initialization function 
+	ntptr_init();	/* FIXME: maybe run this in the initialization function 
 						of the spoolss RPC server instead? */
 
 	ntvfs_init(cmdline_lp_ctx); 	/* FIXME: maybe run this in the initialization functions 
@@ -412,7 +406,7 @@ static int binary_smbd_main(const char *binary_name, int argc, const char *argv[
 
 	process_model_init(cmdline_lp_ctx); 
 
-	shared_init = load_samba_modules(NULL, cmdline_lp_ctx, "service");
+	shared_init = load_samba_modules(NULL, "service");
 
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
@@ -445,14 +439,8 @@ static int binary_smbd_main(const char *binary_name, int argc, const char *argv[
 		      discard_const(binary_name));
 
 	if (max_runtime) {
-		struct timeval tv;
-		struct timezone tz;
-
-		if (gettimeofday(&tv, &tz) == 0) {
-			DEBUG(0,("Called with maxruntime %d - current ts %d\n", max_runtime, (int)tv.tv_sec));
-		} else {
-			DEBUG(0,("Called with maxruntime %d\n", max_runtime));
-		}
+		DEBUG(0,("Called with maxruntime %d - current ts %llu\n",
+		      max_runtime, (unsigned long long) time(NULL)));
 		tevent_add_timer(event_ctx, event_ctx,
 				 timeval_current_ofs(max_runtime, 0),
 				 max_runtime_handler,
diff --git a/source4/smbd/service_named_pipe.c b/source4/smbd/service_named_pipe.c
index 086a037b69d..b000083eecd 100644
--- a/source4/smbd/service_named_pipe.c
+++ b/source4/smbd/service_named_pipe.c
@@ -200,14 +200,21 @@ NTSTATUS tstream_setup_named_pipe(TALLOC_CTX *mem_ctx,
 		goto fail;
 	}
 
+	if (!directory_create_or_exist(lpcfg_ncalrpc_dir(lp_ctx), geteuid(), 0755)) {
+		status = map_nt_error_from_unix_common(errno);
+		DEBUG(0,(__location__ ": Failed to create ncalrpc pipe directory '%s' - %s\n",
+			 lpcfg_ncalrpc_dir(lp_ctx), nt_errstr(status)));
+		goto fail;
+	}
+
 	dirname = talloc_asprintf(pipe_sock, "%s/np", lpcfg_ncalrpc_dir(lp_ctx));
 	if (dirname == NULL) {
 		goto fail;
 	}
 
 	if (!directory_create_or_exist(dirname, geteuid(), 0700)) {
-		status = map_nt_error_from_unix(errno);
-		DEBUG(0,(__location__ ": Failed to create stream pipe directory %s - %s\n",
+		status = map_nt_error_from_unix_common(errno);
+		DEBUG(0,(__location__ ": Failed to create stream pipe directory '%s' - %s\n",
 			 dirname, nt_errstr(status)));
 		goto fail;
 	}
diff --git a/source4/smbd/service_stream.c b/source4/smbd/service_stream.c
index 916393253b8..6e65122063b 100644
--- a/source4/smbd/service_stream.c
+++ b/source4/smbd/service_stream.c
@@ -27,6 +27,7 @@
 #include "cluster/cluster.h"
 #include "param/param.h"
 #include "../lib/tsocket/tsocket.h"
+#include "lib/util/util_net.h"
 
 /* the range of ports to try for dcerpc over tcp endpoints */
 #define SERVER_TCP_LOW_PORT  1024
@@ -122,7 +123,7 @@ NTSTATUS stream_new_connection_merge(struct tevent_context *ev,
 				     struct loadparm_context *lp_ctx,
 				     const struct model_ops *model_ops,
 				     const struct stream_server_ops *stream_ops,
-				     struct messaging_context *msg_ctx,
+				     struct imessaging_context *msg_ctx,
 				     void *private_data,
 				     struct stream_connection **_srv_conn)
 {
@@ -186,11 +187,11 @@ static void stream_new_connection(struct tevent_context *ev,
 	}
 
 	/* setup to receive internal messages on this connection */
-	srv_conn->msg_ctx = messaging_init(srv_conn, 
-					   lpcfg_messaging_path(srv_conn, lp_ctx),
+	srv_conn->msg_ctx = imessaging_init(srv_conn,
+					   lpcfg_imessaging_path(srv_conn, lp_ctx),
 					   srv_conn->server_id, ev);
 	if (!srv_conn->msg_ctx) {
-		stream_terminate_connection(srv_conn, "messaging_init() failed");
+		stream_terminate_connection(srv_conn, "imessaging_init() failed");
 		return;
 	}
 
@@ -216,7 +217,7 @@ static void stream_new_connection(struct tevent_context *ev,
 					stream_socket->ops->name, 
 					tsocket_address_string(srv_conn->remote_address, tmp_ctx),
 					tsocket_address_string(srv_conn->local_address, tmp_ctx),
-					cluster_id_string(tmp_ctx, server_id));
+					server_id_str(tmp_ctx, &server_id));
 		if (title) {
 			stream_connection_set_title(srv_conn, title);
 		}
@@ -271,12 +272,34 @@ NTSTATUS stream_setup_socket(TALLOC_CTX *mem_ctx,
 	struct socket_address *socket_address;
 	struct tevent_fd *fde;
 	int i;
+	struct sockaddr_storage ss;
 
 	stream_socket = talloc_zero(mem_ctx, struct stream_socket);
 	NT_STATUS_HAVE_NO_MEMORY(stream_socket);
 
-	status = socket_create(family, SOCKET_TYPE_STREAM, &stream_socket->sock, 0);
-	NT_STATUS_NOT_OK_RETURN(status);
+	if (strcmp(family, "ip") == 0) {
+		/* we will get the real family from the address itself */
+		if (!interpret_string_addr(&ss, sock_addr, 0)) {
+			talloc_free(stream_socket);
+			return NT_STATUS_INVALID_ADDRESS;
+		}
+
+		socket_address = socket_address_from_sockaddr_storage(stream_socket, &ss, port?*port:0);
+		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(socket_address, stream_socket);
+
+		status = socket_create(socket_address->family, SOCKET_TYPE_STREAM, &stream_socket->sock, 0);
+		NT_STATUS_NOT_OK_RETURN(status);
+	} else {
+		status = socket_create(family, SOCKET_TYPE_STREAM, &stream_socket->sock, 0);
+		NT_STATUS_NOT_OK_RETURN(status);
+
+		/* this is for non-IP sockets, eg. unix domain sockets */
+		socket_address = socket_address_from_strings(stream_socket,
+							     stream_socket->sock->backend_name,
+							     sock_addr, port?*port:0);
+		NT_STATUS_HAVE_NO_MEMORY(socket_address);
+	}
+
 
 	talloc_steal(stream_socket, stream_socket->sock);
 
@@ -297,34 +320,19 @@ NTSTATUS stream_setup_socket(TALLOC_CTX *mem_ctx,
 	/* Some sockets don't have a port, or are just described from
 	 * the string.  We are indicating this by having port == NULL */
 	if (!port) {
-		socket_address = socket_address_from_strings(stream_socket, 
-							     stream_socket->sock->backend_name,
-							     sock_addr, 0);
-		NT_STATUS_HAVE_NO_MEMORY(socket_address);
 		status = socket_listen(stream_socket->sock, socket_address, SERVER_LISTEN_BACKLOG, 0);
-		talloc_free(socket_address);
-
 	} else if (*port == 0) {
 		for (i=SERVER_TCP_LOW_PORT;i<= SERVER_TCP_HIGH_PORT;i++) {
-			socket_address = socket_address_from_strings(stream_socket, 
-								     stream_socket->sock->backend_name,
-								     sock_addr, i);
-			NT_STATUS_HAVE_NO_MEMORY(socket_address);
+			socket_address->port = i;
 			status = socket_listen(stream_socket->sock, socket_address, 
 					       SERVER_LISTEN_BACKLOG, 0);
-			talloc_free(socket_address);
 			if (NT_STATUS_IS_OK(status)) {
 				*port = i;
 				break;
 			}
 		}
 	} else {
-		socket_address = socket_address_from_strings(stream_socket, 
-							     stream_socket->sock->backend_name,
-							     sock_addr, *port);
-		NT_STATUS_HAVE_NO_MEMORY(socket_address);
 		status = socket_listen(stream_socket->sock, socket_address, SERVER_LISTEN_BACKLOG, 0);
-		talloc_free(socket_address);
 	}
 
 	if (!NT_STATUS_IS_OK(status)) {
@@ -362,6 +370,7 @@ NTSTATUS stream_setup_socket(TALLOC_CTX *mem_ctx,
 	return NT_STATUS_OK;
 }
 
+
 /*
   setup a connection title 
 */
diff --git a/source4/smbd/service_stream.h b/source4/smbd/service_stream.h
index 02ade716d28..e098a690f1f 100644
--- a/source4/smbd/service_stream.h
+++ b/source4/smbd/service_stream.h
@@ -23,7 +23,7 @@
 #ifndef __SERVICE_STREAM_H__
 #define __SERVICE_STREAM_H__
 
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
 /* modules can use the following to determine if the interface has changed
  * please increment the version number after each interface change
@@ -47,7 +47,7 @@ struct stream_connection {
 	} event;
 
 	struct socket_context *socket;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	struct loadparm_context *lp_ctx;
 
 	struct tstream_context *tstream;
diff --git a/source4/smbd/service_task.c b/source4/smbd/service_task.c
index 1eb8403e3b1..32c44cf6603 100644
--- a/source4/smbd/service_task.c
+++ b/source4/smbd/service_task.c
@@ -78,12 +78,12 @@ static void task_server_callback(struct tevent_context *event_ctx,
 	task->server_id = server_id;
 	task->lp_ctx = lp_ctx;
 
-	task->msg_ctx = messaging_init(task, 
-				       lpcfg_messaging_path(task, task->lp_ctx),
+	task->msg_ctx = imessaging_init(task,
+				       lpcfg_imessaging_path(task, task->lp_ctx),
 				       task->server_id, 
 				       task->event_ctx);
 	if (!task->msg_ctx) {
-		task_server_terminate(task, "messaging_init() failed", true);
+		task_server_terminate(task, "imessaging_init() failed", true);
 		return;
 	}
 
diff --git a/source4/smbd/service_task.h b/source4/smbd/service_task.h
index b8954073e5a..ded4590daff 100644
--- a/source4/smbd/service_task.h
+++ b/source4/smbd/service_task.h
@@ -22,12 +22,12 @@
 #ifndef __SERVICE_TASK_H__
 #define __SERVICE_TASK_H__ 
 
-#include "librpc/gen_ndr/server_id4.h"
+#include "librpc/gen_ndr/server_id.h"
 
 struct task_server {
 	struct tevent_context *event_ctx;
 	const struct model_ops *model_ops;
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	struct loadparm_context *lp_ctx;
 	struct server_id server_id;
 	void *private_data;
diff --git a/source4/torture/basic/attr.c b/source4/torture/basic/attr.c
index 09d1ae29684..2f0248e20fb 100644
--- a/source4/torture/basic/attr.c
+++ b/source4/torture/basic/attr.c
@@ -374,7 +374,7 @@ error_exit_file:
 				[0x%x], got attr 0x%x, should be 0x%x\n",
 			        open_attrs_table[j],
 			        (uint16_t)attr,
-			        open_attrs_table[j]|FILE_ATTRIBUTE_DIRECTORY);
+			        (unsigned int)(open_attrs_table[j]|FILE_ATTRIBUTE_DIRECTORY));
 			CHECK_MAX_FAILURES(error_exit_dir);
 		}
 
diff --git a/source4/torture/basic/mangle_test.c b/source4/torture/basic/mangle_test.c
index 0b7d696e677..d549a0c1eb3 100644
--- a/source4/torture/basic/mangle_test.c
+++ b/source4/torture/basic/mangle_test.c
@@ -20,7 +20,7 @@
 #include "includes.h"
 #include "system/filesys.h"
 #include "system/dir.h"
-#include <tdb.h>
+#include "../lib/tdb_compat/tdb_compat.h"
 #include "../lib/util/util_tdb.h"
 #include "libcli/libcli.h"
 #include "torture/util.h"
@@ -167,7 +167,7 @@ bool torture_mangle(struct torture_context *torture,
 	int i;
 
 	/* we will use an internal tdb to store the names we have used */
-	tdb = tdb_open(NULL, 100000, TDB_INTERNAL, 0, 0);
+	tdb = tdb_open_compat(NULL, 100000, TDB_INTERNAL, 0, 0, NULL, NULL);
 	if (!tdb) {
 		printf("ERROR: Failed to open tdb\n");
 		return false;
diff --git a/source4/torture/dfs/domaindfs.c b/source4/torture/dfs/domaindfs.c
index 467e104e25a..89e9cc1426a 100644
--- a/source4/torture/dfs/domaindfs.c
+++ b/source4/torture/dfs/domaindfs.c
@@ -55,20 +55,24 @@ static bool test_getdomainreferral(struct torture_context *tctx,
 				 talloc_asprintf(tctx,
 					"Not expected version for referral entry 0 got %d expected 3",
 					resp.referral_entries[0].version));
-	torture_assert_int_equal(tctx, resp.referral_entries[0].referral.v3.data.server_type,
+	torture_assert_int_equal(tctx, resp.referral_entries[0].referral.v3.server_type,
 				 DFS_SERVER_NON_ROOT,
 				 talloc_asprintf(tctx,
 					"Wrong server type, expected non root server and got %d",
-					resp.referral_entries[0].referral.v3.data.server_type));
-	torture_assert_int_equal(tctx, resp.referral_entries[0].referral.v3.data.entry_flags,
+					resp.referral_entries[0].referral.v3.server_type));
+	torture_assert_int_equal(tctx, resp.referral_entries[0].referral.v3.entry_flags,
 				 DFS_FLAG_REFERRAL_DOMAIN_RESP,
 				 talloc_asprintf(tctx,
 					"Wrong entry flag expected to have a domain response and got %d",
-					resp.referral_entries[0].referral.v3.data.entry_flags));
+					resp.referral_entries[0].referral.v3.entry_flags));
 	torture_assert_int_equal(tctx, strlen(
-				 resp.referral_entries[0].referral.v3.data.referrals.r2.special_name) > 0,
+				 resp.referral_entries[0].referral.v3.referrals.r2.special_name) > 0,
 				 1,
 				 "Length of domain is 0 or less");
+	torture_assert_int_equal(tctx,
+				 resp.referral_entries[0].referral.v3.referrals.r2.special_name[0] == '\\',
+				 1,
+				 "domain didn't start with a \\");
 	return true;
 }
 
@@ -88,9 +92,9 @@ static bool test_getdcreferral(struct torture_context *tctx,
 		   dfs_cli_do_call(cli->tree, &r),
 		   "Get Domain referral failed");
 
-	str = resp.referral_entries[0].referral.v3.data.referrals.r2.special_name;
+	str = resp.referral_entries[0].referral.v3.referrals.r2.special_name;
 	if( strchr(str, '.') == NULL ) {
-		str = resp.referral_entries[1].referral.v3.data.referrals.r2.special_name;
+		str = resp.referral_entries[1].referral.v3.referrals.r2.special_name;
 	}
 
 	r2.in.req.max_referral_level = 3;
@@ -112,34 +116,38 @@ static bool test_getdcreferral(struct torture_context *tctx,
 				 talloc_asprintf(tctx,
 					"Not expected version for referral entry 0 got %d expected 3",
 					resp2.referral_entries[0].version));
-	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.data.server_type,
+	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.server_type,
 				 DFS_SERVER_NON_ROOT,
 				 talloc_asprintf(tctx,
 					"Wrong server type, expected non root server and got %d",
-					resp2.referral_entries[0].referral.v3.data.server_type));
-	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.data.entry_flags,
+					resp2.referral_entries[0].referral.v3.server_type));
+	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.entry_flags,
 				 DFS_FLAG_REFERRAL_DOMAIN_RESP,
 				 talloc_asprintf(tctx,
 					"Wrong entry flag expected to have a domain response and got %d",
-					resp2.referral_entries[0].referral.v3.data.entry_flags));
+					resp2.referral_entries[0].referral.v3.entry_flags));
 	torture_assert_int_equal(tctx, strlen(
-				 resp2.referral_entries[0].referral.v3.data.referrals.r2.special_name) > 0,
+				 resp2.referral_entries[0].referral.v3.referrals.r2.special_name) > 0,
 				 1,
 				 "Length of domain is 0 or less");
 	torture_assert_int_equal(tctx, strlen(
-				 resp2.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0]) > 0,
+				 resp2.referral_entries[0].referral.v3.referrals.r2.expanded_names[0]) > 0,
 				 1,
 				 "Length of first dc is less than 0");
-	str = strchr(resp2.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0], '.');
-	str2 = resp2.referral_entries[0].referral.v3.data.referrals.r2.special_name;
+	str = strchr(resp2.referral_entries[0].referral.v3.referrals.r2.expanded_names[0], '.');
+	str2 = resp2.referral_entries[0].referral.v3.referrals.r2.special_name;
 	if (str2[0] == '\\') {
 		str2++;
 	}
 	torture_assert_int_equal(tctx, strlen(str) >0, 1 ,"Length of domain too short");
 	str++;
 	torture_assert_int_equal(tctx, strcmp(str,str2), 0,
-					talloc_asprintf(tctx, "Pb domain of the dc is not"\
-								"the same as the requested: domain = %s got =%s",str2 ,str));
+					talloc_asprintf(tctx, "Pb domain of the dc is not "\
+								"the same as the requested: domain was = %s got =%s",str2 ,str));
+	torture_assert_int_equal(tctx,
+				 resp.referral_entries[0].referral.v3.referrals.r2.special_name[0] == '\\',
+				 1,
+				 "dc name didn't start with a \\");
 
 	r3.in.req.max_referral_level = 3;
 	/*
@@ -164,22 +172,22 @@ static bool test_getdcreferral(struct torture_context *tctx,
 				 talloc_asprintf(tctx,
 					"Not expected version for referral entry 0 got %d expected 3",
 					resp3.referral_entries[0].version));
-	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.data.server_type,
+	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.server_type,
 				 DFS_SERVER_NON_ROOT,
 				 talloc_asprintf(tctx,
 					"Wrong server type, expected non root server and got %d",
-					resp3.referral_entries[0].referral.v3.data.server_type));
-	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.data.entry_flags,
+					resp3.referral_entries[0].referral.v3.server_type));
+	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.entry_flags,
 				 DFS_FLAG_REFERRAL_DOMAIN_RESP,
 				 talloc_asprintf(tctx,
 					"Wrong entry flag expected to have a domain response and got %d",
-					resp3.referral_entries[0].referral.v3.data.entry_flags));
+					resp3.referral_entries[0].referral.v3.entry_flags));
 	torture_assert_int_equal(tctx, strlen(
-				 resp3.referral_entries[0].referral.v3.data.referrals.r2.special_name) > 0,
+				 resp3.referral_entries[0].referral.v3.referrals.r2.special_name) > 0,
 				 1,
 				 "Length of domain is 0 or less");
 	torture_assert_int_equal(tctx, strlen(
-				 resp3.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0]) > 0,
+				 resp3.referral_entries[0].referral.v3.referrals.r2.expanded_names[0]) > 0,
 				 1,
 				 "Length of first dc is less than 0");
 	return true;
@@ -202,9 +210,9 @@ static bool test_getdcreferral_netbios(struct torture_context *tctx,
 
 	r2.in.req.max_referral_level = 3;
 
-	str = resp.referral_entries[0].referral.v3.data.referrals.r2.special_name;
+	str = resp.referral_entries[0].referral.v3.referrals.r2.special_name;
 	if( strchr(str, '.') != NULL ) {
-		str = resp.referral_entries[1].referral.v3.data.referrals.r2.special_name;
+		str = resp.referral_entries[1].referral.v3.referrals.r2.special_name;
 	}
 
 	r2.in.req.servername = str;
@@ -224,26 +232,26 @@ static bool test_getdcreferral_netbios(struct torture_context *tctx,
 				 talloc_asprintf(tctx,
 					"Not expected version for referral entry 0 got %d expected 3",
 					resp2.referral_entries[0].version));
-	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.data.server_type,
+	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.server_type,
 				 DFS_SERVER_NON_ROOT,
 				 talloc_asprintf(tctx,
 					"Wrong server type, expected non root server and got %d",
-					resp2.referral_entries[0].referral.v3.data.server_type));
-	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.data.entry_flags,
+					resp2.referral_entries[0].referral.v3.server_type));
+	torture_assert_int_equal(tctx, resp2.referral_entries[0].referral.v3.entry_flags,
 				 DFS_FLAG_REFERRAL_DOMAIN_RESP,
 				 talloc_asprintf(tctx,
 					"Wrong entry flag expected to have a domain response and got %d",
-					resp2.referral_entries[0].referral.v3.data.entry_flags));
+					resp2.referral_entries[0].referral.v3.entry_flags));
 	torture_assert_int_equal(tctx, strlen(
-				 resp2.referral_entries[0].referral.v3.data.referrals.r2.special_name) > 0,
+				 resp2.referral_entries[0].referral.v3.referrals.r2.special_name) > 0,
 				 1,
 				 "Length of domain is 0 or less");
 	torture_assert_int_equal(tctx, strlen(
-				 resp2.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0]) > 0,
+				 resp2.referral_entries[0].referral.v3.referrals.r2.expanded_names[0]) > 0,
 				 1,
 				 "Length of first dc is less than 0");
 	torture_assert(tctx, strchr(
-		       resp2.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0],'.') == NULL,
+		       resp2.referral_entries[0].referral.v3.referrals.r2.expanded_names[0],'.') == NULL,
 		       "referral contains dots it's not a netbios name");
 
 	r3.in.req.max_referral_level = 3;
@@ -269,26 +277,26 @@ static bool test_getdcreferral_netbios(struct torture_context *tctx,
 				 talloc_asprintf(tctx,
 					"Not expected version for referral entry 0 got %d expected 3",
 					resp3.referral_entries[0].version));
-	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.data.server_type,
+	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.server_type,
 				 DFS_SERVER_NON_ROOT,
 				 talloc_asprintf(tctx,
 					"Wrong server type, expected non root server and got %d",
-					resp3.referral_entries[0].referral.v3.data.server_type));
-	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.data.entry_flags,
+					resp3.referral_entries[0].referral.v3.server_type));
+	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.entry_flags,
 				 DFS_FLAG_REFERRAL_DOMAIN_RESP,
 				 talloc_asprintf(tctx,
 					"Wrong entry flag expected to have a domain response and got %d",
-					resp3.referral_entries[0].referral.v3.data.entry_flags));
+					resp3.referral_entries[0].referral.v3.entry_flags));
 	torture_assert_int_equal(tctx, strlen(
-				 resp3.referral_entries[0].referral.v3.data.referrals.r2.special_name) > 0,
+				 resp3.referral_entries[0].referral.v3.referrals.r2.special_name) > 0,
 				 1,
 				 "Length of domain is 0 or less");
 	torture_assert_int_equal(tctx, strlen(
-				 resp3.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0]) > 0,
+				 resp3.referral_entries[0].referral.v3.referrals.r2.expanded_names[0]) > 0,
 				 1,
 				 "Length of first dc is less than 0");
 	torture_assert(tctx, strchr(
-		       resp3.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0],'.') == NULL,
+		       resp3.referral_entries[0].referral.v3.referrals.r2.expanded_names[0],'.') == NULL,
 		       "referral contains dots it's not a netbios name");
 	return true;
 }
@@ -299,6 +307,9 @@ static bool test_getsysvolreferral(struct torture_context *tctx,
 	const char* str;
 	struct dfs_GetDFSReferral r, r2, r3;
 	struct dfs_referral_resp resp, resp2, resp3;
+	uint8_t zeros[16];
+
+	memset(zeros, 0, sizeof(zeros));
 
 	r.in.req.max_referral_level = 3;
 	r.in.req.servername = "";
@@ -308,9 +319,9 @@ static bool test_getsysvolreferral(struct torture_context *tctx,
 		   dfs_cli_do_call(cli->tree, &r),
 		   "Get Domain referral failed");
 
-	str = resp.referral_entries[0].referral.v3.data.referrals.r2.special_name;
+	str = resp.referral_entries[0].referral.v3.referrals.r2.special_name;
 	if( strchr(str, '.') == NULL ) {
-		str = resp.referral_entries[1].referral.v3.data.referrals.r2.special_name;
+		str = resp.referral_entries[1].referral.v3.referrals.r2.special_name;
 	}
 
 	r2.in.req.max_referral_level = 3;
@@ -339,25 +350,41 @@ static bool test_getsysvolreferral(struct torture_context *tctx,
 				 talloc_asprintf(tctx,
 					"Not expected version for referral entry 0 got %d expected 3",
 					resp3.referral_entries[0].version));
-	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.data.server_type,
+	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.server_type,
 				 DFS_SERVER_NON_ROOT,
 				 talloc_asprintf(tctx,
 					"Wrong server type, expected non root server and got %d",
-					resp3.referral_entries[0].referral.v3.data.server_type));
-	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.data.entry_flags,
+					resp3.referral_entries[0].referral.v3.server_type));
+	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v3.entry_flags,
 				 0,
 				 talloc_asprintf(tctx,
 					"Wrong entry flag expected to have a non domain response and got %d",
-					resp3.referral_entries[0].referral.v3.data.entry_flags));
+					resp3.referral_entries[0].referral.v3.entry_flags));
 	torture_assert_int_equal(tctx, strlen(
-				 resp3.referral_entries[0].referral.v3.data.referrals.r2.special_name) > 0,
+				 resp3.referral_entries[0].referral.v3.referrals.r1.DFS_path) > 0,
 				 1,
 				 "Length of domain is 0 or less");
+	torture_assert_int_equal(tctx, strstr(resp3.referral_entries[0].referral.v3.referrals.r1.DFS_path,
+					str+1) != NULL, 1,
+				 	talloc_asprintf(tctx,
+						"Wrong DFS_path %s unable to find substring %s in it",
+						resp3.referral_entries[0].referral.v3.referrals.r1.DFS_path,
+						str+1));
 	torture_assert_int_equal(tctx, strlen(
-				 resp2.referral_entries[0].referral.v3.data.referrals.r2.expanded_names[0]) > 0,
+				 resp3.referral_entries[0].referral.v3.referrals.r1.netw_address) > 0,
 				 1,
 				 "Length of first referral is less than 0");
-
+	torture_assert_int_equal(tctx, strstr(resp3.referral_entries[0].referral.v3.referrals.r1.netw_address,
+					str+1) != NULL, 1,
+				 	talloc_asprintf(tctx,
+						"Wrong DFS_path %s unable to find substring %s in it",
+						resp3.referral_entries[0].referral.v3.referrals.r1.netw_address,
+						str+1));
+#if 0
+	/*
+	 * Due to strange behavior with XP and level 4
+	 * we are obliged to degrade to level 3 ...
+	 */
 	r3.in.req.max_referral_level = 4;
 
 	torture_assert_ntstatus_ok(tctx,
@@ -368,6 +395,20 @@ static bool test_getsysvolreferral(struct torture_context *tctx,
 				 talloc_asprintf(tctx,
 					"Not expected version for referral entry 0 got %d expected 4",
 					resp3.referral_entries[0].version));
+	torture_assert_int_equal(tctx, memcmp(resp3.referral_entries[0].referral.v3.service_site_guid.value, zeros, 16), 0,
+				 talloc_asprintf(tctx,
+					"Service_site_guid is not NULL as expected"));
+#endif
+	r3.in.req.max_referral_level = 4;
+
+	torture_assert_ntstatus_ok(tctx,
+		   dfs_cli_do_call(cli->tree, &r3),
+		   "Get sysvol Domain referral failed");
+
+	torture_assert_int_equal(tctx, resp3.referral_entries[0].version, 3,
+				 talloc_asprintf(tctx,
+					"Not expected version for referral entry 0 got %d expected 3 in degraded mode",
+					resp3.referral_entries[0].version));
 #if 0
 	/*
 	 * We do not support fallback indication for the moment
@@ -375,12 +416,12 @@ static bool test_getsysvolreferral(struct torture_context *tctx,
 	torture_assert_int_equal(tctx, resp3.header_flags,
 					DFS_HEADER_FLAG_STORAGE_SVR | DFS_HEADER_FLAG_TARGET_BCK,
 					"Header flag different it's not a referral for a storage with fallback");
-#endif
 	torture_assert_int_equal(tctx, resp3.referral_entries[0].referral.v4.entry_flags,
 				 DFS_FLAG_REFERRAL_FIRST_TARGET_SET,
 				 talloc_asprintf(tctx,
 					"Wrong entry flag expected to have a non domain response and got %d",
 					resp3.referral_entries[0].referral.v4.entry_flags));
+#endif
 	return true;
 }
 
@@ -426,14 +467,14 @@ static bool test_getsysvolplusreferral(struct torture_context *tctx,
 		   "Get Domain referral failed");
 
 	r2.in.req.max_referral_level = 3;
-	r2.in.req.servername = resp.referral_entries[0].referral.v3.data.referrals.r2.special_name;
+	r2.in.req.servername = resp.referral_entries[0].referral.v3.referrals.r2.special_name;
 	r2.out.resp = &resp2;
 
 	torture_assert_ntstatus_ok(tctx,
 		   dfs_cli_do_call(cli->tree, &r2),
 		   "Get DC Domain referral failed");
 
-	str = resp2.referral_entries[0].referral.v3.data.referrals.r2.special_name;
+	str = resp2.referral_entries[0].referral.v3.referrals.r2.special_name;
 	r3.in.req.max_referral_level = 3;
 	r3.in.req.servername = talloc_asprintf(tctx, "%s\\sysvol\\foo", str);
 	r3.out.resp = &resp3;
diff --git a/source4/torture/drs/python/getnc_exop.py b/source4/torture/drs/python/getnc_exop.py
new file mode 100644
index 00000000000..904c0133334
--- /dev/null
+++ b/source4/torture/drs/python/getnc_exop.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Tests various schema replication scenarios
+#
+# Copyright (C) Kamen Mazdrashki <kamenim@samba.org> 2011
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+#
+# Usage:
+#  export DC1=dc1_dns_name
+#  export DC2=dc2_dns_name
+#  export SUBUNITRUN=$samba4srcdir/scripting/bin/subunitrun
+#  PYTHONPATH="$PYTHONPATH:$samba4srcdir/torture/drs/python" $SUBUNITRUN getnc_exop -U"$DOMAIN/$DC_USERNAME"%"$DC_PASSWORD"
+#
+
+import drs_base
+import samba.tests
+
+from ldb import SCOPE_BASE
+
+from samba.dcerpc import drsuapi, misc, drsblobs
+from samba.drs_utils import drs_DsBind
+
+
+class DrsReplicaSyncTestCase(drs_base.DrsBaseTestCase):
+    """Intended as a semi-black box test case for DsGetNCChanges
+       implementation for extended operations. It should be testing
+       how DsGetNCChanges handles different input params (mostly invalid).
+       Final goal is to make DsGetNCChanges as binary compatible to
+       Windows implementation as possible"""
+
+    def setUp(self):
+        super(DrsReplicaSyncTestCase, self).setUp()
+
+    def tearDown(self):
+        super(DrsReplicaSyncTestCase, self).tearDown()
+
+    def _exop_req8(self, dest_dsa, invocation_id, nc_dn_str, exop):
+        req8 = drsuapi.DsGetNCChangesRequest8()
+    
+        req8.destination_dsa_guid = misc.GUID(dest_dsa)
+        req8.source_dsa_invocation_id = misc.GUID(invocation_id)
+        req8.naming_context = drsuapi.DsReplicaObjectIdentifier()
+        req8.naming_context.dn = unicode(nc_dn_str)
+        req8.highwatermark = drsuapi.DsReplicaHighWaterMark()
+        req8.highwatermark.tmp_highest_usn = 0
+        req8.highwatermark.reserved_usn = 0
+        req8.highwatermark.highest_usn = 0
+        req8.uptodateness_vector = None
+        req8.replica_flags = 0
+        req8.max_object_count = 0
+        req8.max_ndr_size = 402116
+        req8.extended_op = exop
+        req8.fsmo_info = 0
+        req8.partial_attribute_set = None
+        req8.partial_attribute_set_ex = None
+        req8.mapping_ctr.num_mappings = 0
+        req8.mapping_ctr.mappings = None
+    
+        return req8
+
+    def _ds_bind(self, server_name):
+        binding_str = "ncacn_ip_tcp:%s[print,seal]" % server_name
+
+        drs = drsuapi.drsuapi(binding_str, self.get_loadparm(), self.get_credentials())
+        (drs_handle, supported_extensions) = drs_DsBind(drs)
+        return (drs, drs_handle)
+
+    def _determine_fSMORoleOwner(self, fsmo_obj_dn):
+        """Returns (owner, not_owner) pair where:
+             owner: dns name for FSMO owner
+             not_owner: dns name for DC not owning the FSMO"""
+        # collect info to return later
+        fsmo_info_1 = {"dns_name": self.dnsname_dc1,
+                       "invocation_id": self.ldb_dc1.get_invocation_id(),
+                       "ntds_guid": self.ldb_dc1.get_ntds_GUID()}
+        fsmo_info_2 = {"dns_name": self.dnsname_dc2,
+                       "invocation_id": self.ldb_dc2.get_invocation_id(),
+                       "ntds_guid": self.ldb_dc2.get_ntds_GUID()}
+        # determine the owner dc
+        res = self.ldb_dc1.search(fsmo_obj_dn,
+                                  scope=SCOPE_BASE, attrs=["fSMORoleOwner"])
+        assert len(res) == 1, "Only one fSMORoleOwner value expected for %s!"%fsmo_obj_dn
+        fsmo_owner = res[0]["fSMORoleOwner"][0]
+        if fsmo_owner == self.info_dc1["dsServiceName"][0]:
+            return (fsmo_info_1, fsmo_info_2)
+        return (fsmo_info_2, fsmo_info_1)
+
+    def _check_exop_failed(self, ctr6, expected_failure):
+        self.assertEqual(ctr6.extended_ret, expected_failure)
+        #self.assertEqual(ctr6.object_count, 0)
+        #self.assertEqual(ctr6.first_object, None)
+        self.assertEqual(ctr6.more_data, False)
+        self.assertEqual(ctr6.nc_object_count, 0)
+        self.assertEqual(ctr6.nc_linked_attributes_count, 0)
+        self.assertEqual(ctr6.linked_attributes_count, 0)
+        self.assertEqual(ctr6.linked_attributes, None)
+        self.assertEqual(ctr6.drs_error[0], 0)
+
+    def test_FSMONotOwner(self):
+        """Test role transfer with against DC not owner of the role"""
+        fsmo_dn = self.ldb_dc1.get_schema_basedn()
+        (fsmo_owner, fsmo_not_owner) = self._determine_fSMORoleOwner(fsmo_dn)
+        
+        req8 = self._exop_req8(dest_dsa=fsmo_owner["ntds_guid"],
+                               invocation_id=fsmo_not_owner["invocation_id"],
+                               nc_dn_str=fsmo_dn,
+                               exop=drsuapi.DRSUAPI_EXOP_FSMO_REQ_ROLE)
+
+        (drs, drs_handle) = self._ds_bind(fsmo_not_owner["dns_name"])
+        (level, ctr) = drs.DsGetNCChanges(drs_handle, 8, req8)
+        self.assertEqual(level, 6, "Expected level 6 response!")
+        self._check_exop_failed(ctr, drsuapi.DRSUAPI_EXOP_ERR_FSMO_NOT_OWNER)
+        self.assertEqual(ctr.source_dsa_guid, misc.GUID(fsmo_not_owner["ntds_guid"]))
+        self.assertEqual(ctr.source_dsa_invocation_id, misc.GUID(fsmo_not_owner["invocation_id"]))
+
+    def test_InvalidDestDSA(self):
+        """Test role transfer with invalid destination DSA guid"""
+        fsmo_dn = self.ldb_dc1.get_schema_basedn()
+        (fsmo_owner, fsmo_not_owner) = self._determine_fSMORoleOwner(fsmo_dn)
+
+        req8 = self._exop_req8(dest_dsa="9c637462-5b8c-4467-aef2-bdb1f57bc4ef",
+                               invocation_id=fsmo_owner["invocation_id"],
+                               nc_dn_str=fsmo_dn,
+                               exop=drsuapi.DRSUAPI_EXOP_FSMO_REQ_ROLE)
+
+        (drs, drs_handle) = self._ds_bind(fsmo_owner["dns_name"])
+        (level, ctr) = drs.DsGetNCChanges(drs_handle, 8, req8)
+        self.assertEqual(level, 6, "Expected level 6 response!")
+        self._check_exop_failed(ctr, drsuapi.DRSUAPI_EXOP_ERR_UNKNOWN_CALLER)
+        self.assertEqual(ctr.source_dsa_guid, misc.GUID(fsmo_owner["ntds_guid"]))
+        self.assertEqual(ctr.source_dsa_invocation_id, misc.GUID(fsmo_owner["invocation_id"]))
diff --git a/source4/torture/drs/rpc/dssync.c b/source4/torture/drs/rpc/dssync.c
index 2ec3ded2877..8279e736b19 100644
--- a/source4/torture/drs/rpc/dssync.c
+++ b/source4/torture/drs/rpc/dssync.c
@@ -105,8 +105,9 @@ static struct DsSyncTest *test_create_context(struct torture_context *tctx)
 	make_nbt_name_server(&name, ctx->drsuapi_binding->host);
 
 	/* do an initial name resolution to find its IP */
-	status = resolve_name(lpcfg_resolve_context(tctx->lp_ctx), &name, tctx,
-			      &ctx->dest_address, tctx->ev);
+	status = resolve_name_ex(lpcfg_resolve_context(tctx->lp_ctx),
+				 0, 0, &name, tctx,
+				 &ctx->dest_address, tctx->ev);
 	if (!NT_STATUS_IS_OK(status)) {
 		printf("Failed to resolve %s - %s\n",
 		       name.name, nt_errstr(status));
@@ -270,10 +271,7 @@ static bool test_LDAPBind(struct torture_context *tctx, struct DsSyncTest *ctx,
 		return NULL;
 	}
 
-	ldb_set_modules_dir(ldb,
-			    talloc_asprintf(ldb,
-					    "%s/ldb",
-					    lpcfg_modulesdir(tctx->lp_ctx)));
+	ldb_set_modules_dir(ldb, modules_path(ldb, "ldb"));
 
 	if (ldb_set_opaque(ldb, "credentials", credentials)) {
 		talloc_free(ldb);
diff --git a/source4/torture/drs/rpc/msds_intid.c b/source4/torture/drs/rpc/msds_intid.c
index 53f4992ba29..14c6454abe1 100644
--- a/source4/torture/drs/rpc/msds_intid.c
+++ b/source4/torture/drs/rpc/msds_intid.c
@@ -283,10 +283,7 @@ static bool _test_LDAPBind(struct torture_context *tctx,
 		return NULL;
 	}
 
-	ldb_set_modules_dir(ldb,
-			    talloc_asprintf(ldb,
-					    "%s/ldb",
-					    lpcfg_modulesdir(tctx->lp_ctx)));
+	ldb_set_modules_dir(ldb, modules_path(ldb, "ldb"));
 
 	if (ldb_set_opaque(ldb, "credentials", credentials) != LDB_SUCCESS) {
 		talloc_free(ldb);
diff --git a/source4/torture/gentest.c b/source4/torture/gentest.c
index 6f69460dc8a..9b6e7fc0fd5 100644
--- a/source4/torture/gentest.c
+++ b/source4/torture/gentest.c
@@ -2318,6 +2318,18 @@ static void gen_setfileinfo(int instance, union smb_setfileinfo *info)
 	case RAW_SFILEINFO_UNIX_INFO2:
 	case RAW_SFILEINFO_UNIX_LINK:
 	case RAW_SFILEINFO_UNIX_HLINK:
+	case RAW_SFILEINFO_LINK_INFORMATION:
+	case RAW_SFILEINFO_PIPE_INFORMATION:
+	case RAW_SFILEINFO_VALID_DATA_INFORMATION:
+	case RAW_SFILEINFO_SHORT_NAME_INFORMATION:
+	case RAW_SFILEINFO_1027:
+	case RAW_SFILEINFO_1030:
+	case RAW_SFILEINFO_1031:
+	case RAW_SFILEINFO_1036:
+	case RAW_SFILEINFO_1041:
+	case RAW_SFILEINFO_1042:
+	case RAW_SFILEINFO_1043:
+	case RAW_SFILEINFO_1044:
 		/* Untested */
 		break;
 	}
@@ -3247,7 +3259,7 @@ static bool split_unc_name(const char *unc, char **server, char **share)
 
 	ev = s4_event_context_init(talloc_autofree_context());
 
-	gensec_init(lp_ctx);
+	gensec_init();
 
 	ret = start_gentest(ev, lp_ctx);
 
diff --git a/source4/torture/ldap/cldapbench.c b/source4/torture/ldap/cldapbench.c
index 5c7e56f40bd..7d02a2580b1 100644
--- a/source4/torture/ldap/cldapbench.c
+++ b/source4/torture/ldap/cldapbench.c
@@ -215,7 +215,8 @@ bool torture_bench_cldap(struct torture_context *torture)
 	make_nbt_name_server(&name, torture_setting_string(torture, "host", NULL));
 
 	/* do an initial name resolution to find its IP */
-	status = resolve_name(lpcfg_resolve_context(torture->lp_ctx), &name, torture, &address, torture->ev);
+	status = resolve_name_ex(lpcfg_resolve_context(torture->lp_ctx),
+				 0, 0, &name, torture, &address, torture->ev);
 	if (!NT_STATUS_IS_OK(status)) {
 		printf("Failed to resolve %s - %s\n",
 		       name.name, nt_errstr(status));
diff --git a/source4/torture/libnet/libnet_BecomeDC.c b/source4/torture/libnet/libnet_BecomeDC.c
index 0fbad00d7b8..9f569e6a486 100644
--- a/source4/torture/libnet/libnet_BecomeDC.c
+++ b/source4/torture/libnet/libnet_BecomeDC.c
@@ -44,7 +44,7 @@ bool torture_net_become_dc(struct torture_context *torture)
 	struct ldb_message *msg;
 	int ldb_ret;
 	uint32_t i;
-	char *sam_ldb_path;
+	char *private_dir;
 	const char *address;
 	struct nbt_name name;
 	const char *netbios_name;
@@ -67,8 +67,9 @@ bool torture_net_become_dc(struct torture_context *torture)
 	make_nbt_name_server(&name, torture_setting_string(torture, "host", NULL));
 
 	/* do an initial name resolution to find its IP */
-	status = resolve_name(lpcfg_resolve_context(torture->lp_ctx),
-			      &name, torture, &address, torture->ev);
+	status = resolve_name_ex(lpcfg_resolve_context(torture->lp_ctx),
+				 0, 0,
+				 &name, torture, &address, torture->ev);
 	torture_assert_ntstatus_ok(torture, status, talloc_asprintf(torture,
 				   "Failed to resolve %s - %s\n",
 				   name.name, nt_errstr(status)));
@@ -143,13 +144,13 @@ bool torture_net_become_dc(struct torture_context *torture)
 	talloc_unlink(s, ldb);
 
 	lp_ctx = libnet_vampire_cb_lp_ctx(s);
-	sam_ldb_path = talloc_asprintf(s, "%s/%s", location, "private/sam.ldb");
-	lpcfg_set_cmdline(lp_ctx, "sam database", sam_ldb_path);
-	torture_comment(torture, "Reopen the SAM LDB with system credentials and all replicated data: %s\n", sam_ldb_path);
+	private_dir = talloc_asprintf(s, "%s/%s", location, "private");
+	lpcfg_set_cmdline(lp_ctx, "private dir", private_dir);
+	torture_comment(torture, "Reopen the SAM LDB with system credentials and all replicated data: %s\n", private_dir);
 	ldb = samdb_connect(s, torture->ev, lp_ctx, system_session(lp_ctx), 0);
 	torture_assert_goto(torture, ldb != NULL, ret, cleanup,
 				      talloc_asprintf(torture,
-				      "Failed to open '%s'\n", sam_ldb_path));
+				      "Failed to open '%s/sam.ldb'\n", private_dir));
 
 	torture_assert_goto(torture, dsdb_uses_global_schema(ldb), ret, cleanup,
 						"Uses global schema");
diff --git a/source4/torture/libnetapi/libnetapi.c b/source4/torture/libnetapi/libnetapi.c
index 633dc6cc3e3..92c7b3a394e 100644
--- a/source4/torture/libnetapi/libnetapi.c
+++ b/source4/torture/libnetapi/libnetapi.c
@@ -17,11 +17,13 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "includes.h"
+#include "source3/include/includes.h"
 #include "torture/smbtorture.h"
 #include "auth/credentials/credentials.h"
 #include "lib/cmdline/popt_common.h"
-#include <netapi.h>
+#include "source3/lib/netapi/netapi.h"
+#include "source3/lib/netapi/netapi_private.h"
+#include "source4/param/param.h"
 #include "torture/libnetapi/proto.h"
 
 bool torture_libnetapi_init_context(struct torture_context *tctx,
@@ -30,13 +32,19 @@ bool torture_libnetapi_init_context(struct torture_context *tctx,
 	NET_API_STATUS status;
 	struct libnetapi_ctx *ctx;
 
-	status = libnetapi_init(&ctx);
+	if (!lp_load(lpcfg_configfile(tctx->lp_ctx), true, false, false, true)) {
+		fprintf(stderr, "error loading %s\n", lpcfg_configfile(tctx->lp_ctx));
+		return W_ERROR_V(WERR_GENERAL_FAILURE);
+	}
+
+	init_names();
+	load_interfaces();
+
+	status = libnetapi_net_init(&ctx);
 	if (status != 0) {
 		return false;
 	}
 
-	libnetapi_set_debuglevel(ctx,
-		talloc_asprintf(ctx, "%d", DEBUGLEVEL));
 	libnetapi_set_username(ctx,
 		cli_credentials_get_username(cmdline_credentials));
 	libnetapi_set_password(ctx,
@@ -52,10 +60,16 @@ static bool torture_libnetapi_initialize(struct torture_context *tctx)
         NET_API_STATUS status;
 	struct libnetapi_ctx *ctx;
 
+	/* We must do this first, as otherwise we fail if we don't
+	 * have an smb.conf in the default path (we need to use the
+	 * torture smb.conf */
+	torture_assert(tctx, torture_libnetapi_init_context(tctx, &ctx),
+		       "failed to initialize libnetapi");
+
 	status = libnetapi_init(&ctx);
-	if (status != 0) {
-		return false;
-	}
+
+	torture_assert(tctx, ctx != NULL, "Failed to get a libnetapi_ctx");
+	torture_assert_int_equal(tctx, status, 0, "libnetapi_init failed despite alredy being set up");
 
 	libnetapi_free(ctx);
 
diff --git a/source4/torture/libnetapi/wscript_build b/source4/torture/libnetapi/wscript_build
index a087c96662d..4f579c9820d 100644
--- a/source4/torture/libnetapi/wscript_build
+++ b/source4/torture/libnetapi/wscript_build
@@ -5,8 +5,7 @@ bld.SAMBA_MODULE('TORTURE_LIBNETAPI',
 	autoproto='proto.h',
 	subsystem='smbtorture',
 	init_function='torture_libnetapi_init',
-	deps='POPT_CREDENTIALS NETAPI',
+	deps='POPT_CREDENTIALS netapi',
 	internal_module=True,
-        enabled=False
 	)
 
diff --git a/source4/torture/libsmbclient/wscript_build b/source4/torture/libsmbclient/wscript_build
index 8f93487389f..8b2c516dbfc 100644
--- a/source4/torture/libsmbclient/wscript_build
+++ b/source4/torture/libsmbclient/wscript_build
@@ -6,9 +6,8 @@ bld.SAMBA_MODULE('TORTURE_LIBSMBCLIENT',
 	autoproto='proto.h',
 	subsystem='smbtorture',
 	init_function='torture_libsmbclient_init',
-	deps='POPT_CREDENTIALS SMBCLIENT',
-	internal_module=True,
-        enabled=False
+	deps='POPT_CREDENTIALS libsmb/smbclient',
+	internal_module=True
 	)
 
 
diff --git a/source4/torture/local/dbspeed.c b/source4/torture/local/dbspeed.c
index 8768b349eff..614a9b52ae9 100644
--- a/source4/torture/local/dbspeed.c
+++ b/source4/torture/local/dbspeed.c
@@ -21,11 +21,11 @@
 
 #include "includes.h"
 #include "system/filesys.h"
-#include <tdb.h>
+#include "tdb_compat.h"
 #include <ldb.h>
 #include <ldb_errors.h>
 #include "ldb_wrap.h"
-#include "lib/tdb_wrap.h"
+#include "lib/util/tdb_wrap.h"
 #include "torture/smbtorture.h"
 #include "param/param.h"
 
@@ -97,7 +97,7 @@ static bool test_tdb_speed(struct torture_context *torture, const void *_data)
 		i = random() % torture_entries;
 		key.dptr = (uint8_t *)talloc_asprintf(tmp_ctx, "S-1-5-21-53173311-3623041448-2049097239-%u", i);
 		key.dsize = strlen((char *)key.dptr)+1;
-		data = tdb_fetch(tdbw->tdb, key);
+		data = tdb_fetch_compat(tdbw->tdb, key);
 		talloc_free(key.dptr);
 		if (data.dptr == NULL) {
 			torture_result(torture, TORTURE_FAIL, "Failed to find SID %d!", i);
@@ -106,7 +106,7 @@ static bool test_tdb_speed(struct torture_context *torture, const void *_data)
 		free(data.dptr);
 		key.dptr = (uint8_t *)talloc_asprintf(tmp_ctx, "UID %u", i);
 		key.dsize = strlen((char *)key.dptr)+1;
-		data = tdb_fetch(tdbw->tdb, key);
+		data = tdb_fetch_compat(tdbw->tdb, key);
 		talloc_free(key.dptr);
 		if (data.dptr == NULL) {
 			torture_result(torture, TORTURE_FAIL, "Failed to find UID %d!", i);
diff --git a/source4/torture/locktest.c b/source4/torture/locktest.c
index 445a626e7be..77fcb69eafd 100644
--- a/source4/torture/locktest.c
+++ b/source4/torture/locktest.c
@@ -644,7 +644,7 @@ static void usage(poptContext pc)
 
 	ev = s4_event_context_init(talloc_autofree_context());
 
-	gensec_init(lp_ctx);
+	gensec_init();
 
 	DEBUG(0,("seed=%u base=%d range=%d min_length=%d\n", 
 		 seed, lock_base, lock_range, min_length));
diff --git a/source4/torture/masktest.c b/source4/torture/masktest.c
index 9c66291087f..a711634b24c 100644
--- a/source4/torture/masktest.c
+++ b/source4/torture/masktest.c
@@ -49,7 +49,7 @@ static bool reg_match_one(struct smbcli_state *cli, const char *pattern, const c
 
 	if (ISDOTDOT(file)) file = ".";
 
-	return ms_fnmatch(pattern, file, cli->transport->negotiate.protocol)==0;
+	return ms_fnmatch_protocol(pattern, file, cli->transport->negotiate.protocol)==0;
 }
 
 static char *reg_test(struct smbcli_state *cli, TALLOC_CTX *mem_ctx, const char *pattern, const char *long_name, const char *short_name)
@@ -153,10 +153,8 @@ static void get_real_name(TALLOC_CTX *mem_ctx, struct smbcli_state *cli,
 			listfn, &state);
 
 	if (f_info_hit) {
-		*short_name = talloc_strdup(mem_ctx, last_hit.short_name);
-		strlower(*short_name);
-		*long_name = talloc_strdup(mem_ctx, last_hit.long_name);
-		strlower(*long_name);
+		*short_name = strlower_talloc(mem_ctx, last_hit.short_name);
+		*long_name = strlower_talloc(mem_ctx, last_hit.long_name);
 	}
 
 	if (*short_name == '\0') {
@@ -177,7 +175,7 @@ static void testpair(TALLOC_CTX *mem_ctx, struct smbcli_state *cli, char *mask,
 
 	count++;
 
-	safe_strcpy(res1, "---", sizeof(res1));
+	strlcpy(res1, "---", sizeof(res1));
 
 	state.mem_ctx = mem_ctx;
 
@@ -191,7 +189,7 @@ static void testpair(TALLOC_CTX *mem_ctx, struct smbcli_state *cli, char *mask,
 	resultp = res1;
 	short_name = talloc_strdup(mem_ctx, "");
 	get_real_name(mem_ctx, cli, &long_name, &short_name);
-	safe_strcpy(res1, "---", sizeof(res1));
+	strlcpy(res1, "---", sizeof(res1));
 	smbcli_list_new(cli->tree, mask,
 			FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY,
 			RAW_SEARCH_DATA_BOTH_DIRECTORY_INFO,
@@ -362,7 +360,7 @@ static void usage(poptContext pc)
 
 	ev = s4_event_context_init(mem_ctx);
 
-	gensec_init(lp_ctx);
+	gensec_init();
 
 	lpcfg_smbcli_options(lp_ctx, &options);
 	lpcfg_smbcli_session_options(lp_ctx, &session_options);
diff --git a/source4/torture/nbt/browse.c b/source4/torture/nbt/browse.c
deleted file mode 100644
index dcf96ba3ef3..00000000000
--- a/source4/torture/nbt/browse.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/* 
-   Unix SMB/CIFS implementation.
-
-   Browse service
-
-   (C) Jelmer Vernooij 2005
-   
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "librpc/gen_ndr/nbt.h"
-#include "libcli/resolve/resolve.h"
-#include "torture/torture.h"
-
-/*
-  test nbt dgram operations
-*/
-bool torture_nbt_browse(struct torture_context *torture)
-{
-	const char *address;
-	struct nbt_name name;
-	TALLOC_CTX *mem_ctx = talloc_new(NULL);
-	NTSTATUS status;
-	bool ret = true;
-	
-	name.name = lpcfg_workgroup();
-	name.type = NBT_NAME_BROWSER;
-	name.scope = NULL;
-
-	/* do an initial name resolution to find its IP */
-	status = resolve_name(&name, mem_ctx, &address, torture->ev);
-	if (!NT_STATUS_IS_OK(status)) {
-		printf("Failed to resolve %s - %s\n",
-		       name.name, nt_errstr(status));
-		talloc_free(mem_ctx);
-		return false;
-	}
-
-	talloc_free(mem_ctx);
-
-	return ret;
-}
diff --git a/source4/torture/nbt/dgram.c b/source4/torture/nbt/dgram.c
index 36914bdcc36..aa4759edaf9 100644
--- a/source4/torture/nbt/dgram.c
+++ b/source4/torture/nbt/dgram.c
@@ -86,11 +86,13 @@ static bool nbt_test_netlogon(struct torture_context *tctx)
 
 	/* do an initial name resolution to find its IP */
 	torture_assert_ntstatus_ok(tctx, 
-				   resolve_name(lpcfg_resolve_context(tctx->lp_ctx), &name, tctx, &address, tctx->ev),
+				   resolve_name_ex(lpcfg_resolve_context(tctx->lp_ctx),
+						   0, 0,
+						   &name, tctx, &address, tctx->ev),
 				   talloc_asprintf(tctx, "Failed to resolve %s", name.name));
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
-	myaddress = talloc_strdup(dgmsock, iface_best_ip(ifaces, address));
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
+	myaddress = talloc_strdup(dgmsock, iface_list_best_ip(ifaces, address));
 
 
 	socket_address = socket_address_from_strings(dgmsock, dgmsock->sock->backend_name,
@@ -180,11 +182,13 @@ static bool nbt_test_netlogon2(struct torture_context *tctx)
 
 	/* do an initial name resolution to find its IP */
 	torture_assert_ntstatus_ok(tctx, 
-				   resolve_name(lpcfg_resolve_context(tctx->lp_ctx), &name, tctx, &address, tctx->ev),
+				   resolve_name_ex(lpcfg_resolve_context(tctx->lp_ctx),
+						   0, 0,
+						   &name, tctx, &address, tctx->ev),
 				   talloc_asprintf(tctx, "Failed to resolve %s", name.name));
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
-	myaddress = talloc_strdup(dgmsock, iface_best_ip(ifaces, address));
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
+	myaddress = talloc_strdup(dgmsock, iface_list_best_ip(ifaces, address));
 
 	socket_address = socket_address_from_strings(dgmsock, dgmsock->sock->backend_name,
 						     myaddress, lpcfg_dgram_port(tctx->lp_ctx));
@@ -450,11 +454,12 @@ static bool nbt_test_ntlogon(struct torture_context *tctx)
 
 	/* do an initial name resolution to find its IP */
 	torture_assert_ntstatus_ok(tctx, 
-				   resolve_name(lpcfg_resolve_context(tctx->lp_ctx), &name, tctx, &address, tctx->ev),
+				   resolve_name_ex(lpcfg_resolve_context(tctx->lp_ctx),
+						   0, 0, &name, tctx, &address, tctx->ev),
 				   talloc_asprintf(tctx, "Failed to resolve %s", name.name));
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
-	myaddress = talloc_strdup(dgmsock, iface_best_ip(ifaces, address));
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
+	myaddress = talloc_strdup(dgmsock, iface_list_best_ip(ifaces, address));
 
 	socket_address = socket_address_from_strings(dgmsock, dgmsock->sock->backend_name,
 						     myaddress, lpcfg_dgram_port(tctx->lp_ctx));
diff --git a/source4/torture/nbt/nbt.c b/source4/torture/nbt/nbt.c
index 7c45f867e74..1b7fe49cf84 100644
--- a/source4/torture/nbt/nbt.c
+++ b/source4/torture/nbt/nbt.c
@@ -39,7 +39,9 @@ bool torture_nbt_get_name(struct torture_context *tctx,
 
 	/* do an initial name resolution to find its IP */
 	torture_assert_ntstatus_ok(tctx, 
-				   resolve_name(lpcfg_resolve_context(tctx->lp_ctx), name, tctx, address, tctx->ev),
+				   resolve_name_ex(lpcfg_resolve_context(tctx->lp_ctx),
+						   0, 0,
+						   name, tctx, address, tctx->ev),
 				   talloc_asprintf(tctx, 
 						   "Failed to resolve %s", name->name));
 	
diff --git a/source4/torture/nbt/register.c b/source4/torture/nbt/register.c
index d52ae4181ec..24ca328b30f 100644
--- a/source4/torture/nbt/register.c
+++ b/source4/torture/nbt/register.c
@@ -54,9 +54,9 @@ static bool nbt_register_own(struct torture_context *tctx)
 	if (!torture_nbt_get_name(tctx, &name, &address))
 		return false;
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
 
-	myaddress = iface_best_ip(ifaces, address);
+	myaddress = iface_list_best_ip(ifaces, address);
 
 	socket_address = socket_address_from_strings(tctx, nbtsock->sock->backend_name,
 						     myaddress, 0);
@@ -123,9 +123,9 @@ static bool nbt_refresh_own(struct torture_context *tctx)
 	if (!torture_nbt_get_name(tctx, &name, &address))
 		return false;
 	
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
 
-	myaddress = iface_best_ip(ifaces, address);
+	myaddress = iface_list_best_ip(ifaces, address);
 
 	socket_address = socket_address_from_strings(tctx, nbtsock->sock->backend_name,
 						     myaddress, 0);
diff --git a/source4/torture/nbt/wins.c b/source4/torture/nbt/wins.c
index 0ddaa8a05a1..571249f562d 100644
--- a/source4/torture/nbt/wins.c
+++ b/source4/torture/nbt/wins.c
@@ -65,9 +65,9 @@ static bool nbt_test_wins_name(struct torture_context *tctx, const char *address
 	struct interface *ifaces;
 	bool low_port = try_low_port;
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
 
-	myaddress = talloc_strdup(tctx, iface_best_ip(ifaces, address));
+	myaddress = talloc_strdup(tctx, iface_list_best_ip(ifaces, address));
 
 	socket_address = socket_address_from_strings(tctx, 
 						     nbtsock->sock->backend_name,
diff --git a/source4/torture/nbt/winsbench.c b/source4/torture/nbt/winsbench.c
index aff620878c8..cb71d8755dd 100644
--- a/source4/torture/nbt/winsbench.c
+++ b/source4/torture/nbt/winsbench.c
@@ -246,8 +246,8 @@ static bool bench_wins(struct torture_context *tctx)
 	state->registered = talloc_zero_array(state, bool, state->num_names);
 	state->wins_server = address;
 	state->wins_port = lpcfg_nbt_port(tctx->lp_ctx);
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
-	state->my_ip = talloc_strdup(tctx, iface_best_ip(ifaces, address));
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
+	state->my_ip = talloc_strdup(tctx, iface_list_best_ip(ifaces, address));
 	state->ttl = timelimit;
 
 	my_ip = socket_address_from_strings(nbtsock, nbtsock->sock->backend_name, 
diff --git a/source4/torture/nbt/winsreplication.c b/source4/torture/nbt/winsreplication.c
index cf8db494cab..7165f10dbbb 100644
--- a/source4/torture/nbt/winsreplication.c
+++ b/source4/torture/nbt/winsreplication.c
@@ -615,14 +615,14 @@ static struct test_wrepl_conflict_conn *test_create_conflict_ctx(
 	ctx->nbtsock = nbt_name_socket_init(ctx, tctx->ev);
 	if (!ctx->nbtsock) return NULL;
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
 
-	ctx->myaddr = socket_address_from_strings(tctx, ctx->nbtsock->sock->backend_name, iface_best_ip(ifaces, address), 0);
+	ctx->myaddr = socket_address_from_strings(tctx, ctx->nbtsock->sock->backend_name, iface_list_best_ip(ifaces, address), 0);
 	if (!ctx->myaddr) return NULL;
 
-	for (i = 0; i < iface_count(ifaces); i++) {
-		if (strcmp(ctx->myaddr->addr, iface_n_ip(ifaces, i)) == 0) continue;
-		ctx->myaddr2 = socket_address_from_strings(tctx, ctx->nbtsock->sock->backend_name, iface_n_ip(ifaces, i), 0);
+	for (i = 0; i < iface_list_count(ifaces); i++) {
+		if (strcmp(ctx->myaddr->addr, iface_list_n_ip(ifaces, i)) == 0) continue;
+		ctx->myaddr2 = socket_address_from_strings(tctx, ctx->nbtsock->sock->backend_name, iface_list_n_ip(ifaces, i), 0);
 		if (!ctx->myaddr2) return NULL;
 		break;
 	}
@@ -679,12 +679,12 @@ static struct test_wrepl_conflict_conn *test_create_conflict_ctx(
 	ctx->addresses_best[0].owner	= ctx->b.address;
 	ctx->addresses_best[0].ip	= ctx->myaddr->addr;
 
-	ctx->addresses_all_num = iface_count(ifaces);
+	ctx->addresses_all_num = iface_list_count(ifaces);
 	ctx->addresses_all = talloc_array(ctx, struct wrepl_ip, ctx->addresses_all_num);
 	if (!ctx->addresses_all) return NULL;
 	for (i=0; i < ctx->addresses_all_num; i++) {
 		ctx->addresses_all[i].owner	= ctx->b.address;
-		ctx->addresses_all[i].ip	= talloc_strdup(ctx->addresses_all, iface_n_ip(ifaces, i));
+		ctx->addresses_all[i].ip	= talloc_strdup(ctx->addresses_all, iface_list_n_ip(ifaces, i));
 		if (!ctx->addresses_all[i].ip) return NULL;
 	}
 
@@ -6701,6 +6701,7 @@ static bool test_conflict_owned_released_vs_replica(struct torture_context *tctx
 }
 
 struct test_conflict_owned_active_vs_replica_struct {
+	struct torture_context *tctx;
 	const char *line; /* just better debugging */
 	const char *section; /* just better debugging */
 	struct nbt_name name;
@@ -6760,6 +6761,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. unique,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -6786,6 +6788,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. unique,active with different ip(s), positive response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UA_DI_P", 0x00, NULL),
 		.wins	= {
@@ -6813,6 +6816,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. unique,active with different ip(s), positive response other ips
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UA_DI_O", 0x00, NULL),
 		.wins	= {
@@ -6842,6 +6846,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. unique,active with different ip(s), negative response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UA_DI_N", 0x00, NULL),
 		.wins	= {
@@ -6869,6 +6874,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. unique,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -6895,6 +6901,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. unique,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -6924,6 +6931,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. group,active with same ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_GA_SI_R", 0x00, NULL),
 		.wins	= {
@@ -6951,6 +6959,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. group,active with different ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_GA_DI_R", 0x00, NULL),
 		.wins	= {
@@ -6978,6 +6987,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. group,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_GT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7004,6 +7014,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. group,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_GT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7033,6 +7044,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. sgroup,active with same ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_SA_SI_R", 0x00, NULL),
 		.wins	= {
@@ -7060,6 +7072,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. group,active with different ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_SA_DI_R", 0x00, NULL),
 		.wins	= {
@@ -7087,6 +7100,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. sgroup,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_ST_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7113,6 +7127,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. sgroup,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_ST_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7142,6 +7157,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7168,6 +7184,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,active with superset ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MA_SP_U", 0x00, NULL),
 		.wins	= {
@@ -7194,6 +7211,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,active with different ip(s), positive response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MA_DI_P", 0x00, NULL),
 		.wins	= {
@@ -7221,6 +7239,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,active with different ip(s), positive response other ips
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MA_DI_O", 0x00, NULL),
 		.wins	= {
@@ -7250,6 +7269,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,active with different ip(s), negative response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MA_DI_N", 0x00, NULL),
 		.wins	= {
@@ -7277,6 +7297,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7303,6 +7324,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7332,6 +7354,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. unique,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_UA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7358,6 +7381,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. unique,active with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_UA_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7384,6 +7408,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. unique,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_UT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7410,6 +7435,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. unique,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_UT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7439,6 +7465,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. group,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_GA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7465,6 +7492,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. group,active with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_GA_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7491,6 +7519,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. group,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_GT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7517,6 +7546,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. group,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_GT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7546,6 +7576,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. sgroup,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_SA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7572,6 +7603,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. sgroup,active with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_SA_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7598,6 +7630,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. sgroup,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_ST_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7624,6 +7657,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. sgroup,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_ST_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7653,6 +7687,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. mhomed,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_MA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7679,6 +7714,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. mhomed,active with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_MA_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7705,6 +7741,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. mhomed,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_MT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -7731,6 +7768,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * group,active vs. mhomed,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_GA_MT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -7760,6 +7798,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. unique,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_UA_SI_U", 0x1C, NULL),
 		.wins	= {
@@ -7786,6 +7825,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. unique,active with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_UA_DI_U", 0x1C, NULL),
 		.wins	= {
@@ -7812,6 +7852,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. unique,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_UT_SI_U", 0x1C, NULL),
 		.wins	= {
@@ -7838,6 +7879,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. unique,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_UT_DI_U", 0x1C, NULL),
 		.wins	= {
@@ -7867,6 +7909,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. group,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_GA_SI_U", 0x1C, NULL),
 		.wins	= {
@@ -7893,6 +7936,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. group,active with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_GA_DI_U", 0x1C, NULL),
 		.wins	= {
@@ -7919,6 +7963,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. group,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_GT_SI_U", 0x1C, NULL),
 		.wins	= {
@@ -7945,6 +7990,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. group,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_GT_DI_U", 0x1C, NULL),
 		.wins	= {
@@ -7974,6 +8020,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. mhomed,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_MA_SI_U", 0x1C, NULL),
 		.wins	= {
@@ -8000,6 +8047,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. mhomed,active with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_MA_DI_U", 0x1C, NULL),
 		.wins	= {
@@ -8026,6 +8074,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. mhomed,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_MT_SI_U", 0x1C, NULL),
 		.wins	= {
@@ -8052,6 +8101,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. mhomed,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_MT_DI_U", 0x1C, NULL),
 		.wins	= {
@@ -8081,6 +8131,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. unique,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_UA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -8107,6 +8158,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. unique,active with different ip(s), positive response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_UA_DI_P", 0x00, NULL),
 		.wins	= {
@@ -8134,6 +8186,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. unique,active with different ip(s), positive response other ips
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_UA_DI_O", 0x00, NULL),
 		.wins	= {
@@ -8163,6 +8216,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. unique,active with different ip(s), negative response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_UA_DI_N", 0x00, NULL),
 		.wins	= {
@@ -8190,6 +8244,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. unique,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_UT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -8216,6 +8271,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. unique,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_UT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -8245,6 +8301,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. group,active with same ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_GA_SI_R", 0x00, NULL),
 		.wins	= {
@@ -8272,6 +8329,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. group,active with different ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_GA_DI_R", 0x00, NULL),
 		.wins	= {
@@ -8299,6 +8357,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. group,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_GT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -8325,6 +8384,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. group,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_GT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -8354,6 +8414,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. sgroup,active with same ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_SA_SI_R", 0x00, NULL),
 		.wins	= {
@@ -8381,6 +8442,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. group,active with different ip(s), release expected
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_SA_DI_R", 0x00, NULL),
 		.wins	= {
@@ -8408,6 +8470,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. sgroup,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_ST_SI_U", 0x00, NULL),
 		.wins	= {
@@ -8434,6 +8497,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. sgroup,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_ST_DI_U", 0x00, NULL),
 		.wins	= {
@@ -8463,6 +8527,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SI_U", 0x00, NULL),
 		.wins	= {
@@ -8489,6 +8554,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with superset ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SP_U", 0x00, NULL),
 		.wins	= {
@@ -8515,6 +8581,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with different ip(s), positive response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_DI_P", 0x00, NULL),
 		.wins	= {
@@ -8542,6 +8609,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with different ip(s), positive response other ips
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_DI_O", 0x00, NULL),
 		.wins	= {
@@ -8571,6 +8639,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with different ip(s), negative response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_DI_N", 0x00, NULL),
 		.wins	= {
@@ -8598,6 +8667,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,tombstone with same ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MT_SI_U", 0x00, NULL),
 		.wins	= {
@@ -8624,6 +8694,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,tombstone with different ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MT_DI_U", 0x00, NULL),
 		.wins	= {
@@ -8653,6 +8724,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with superset ip(s), unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.section= "Test Replica vs. owned active: some more MHOMED combinations",
 		.name	= _NBT_NAME("_MA_MA_SP_U", 0x00, NULL),
@@ -8682,6 +8754,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with same ips, unchecked
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SM_U", 0x00, NULL),
 		.comment= "C:MHOMED vs. B:MHOMED => B:MHOMED",
@@ -8710,6 +8783,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with subset ip(s), positive response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SB_P", 0x00, NULL),
 		.comment= "C:MHOMED vs. B:BEST (C:MHOMED) => B:MHOMED",
@@ -8739,6 +8813,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with subset ip(s), positive response, with all addresses
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SB_A", 0x00, NULL),
 		.comment= "C:MHOMED vs. B:BEST (C:ALL) => B:MHOMED",
@@ -8772,6 +8847,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 *       the release demand has no effect to the database record...
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SB_PRA", 0x00, NULL),
 		.comment= "C:MHOMED vs. B:BEST (C:BEST) => C:MHOMED",
@@ -8804,6 +8880,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with subset ip(s), positive response, with other addresses
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SB_O", 0x00, NULL),
 		.comment= "C:MHOMED vs. B:BEST (B:B_3_4) =>C:MHOMED",
@@ -8835,6 +8912,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. mhomed,active with subset ip(s), negative response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_MA_MA_SB_N", 0x00, NULL),
 		.comment= "C:MHOMED vs. B:BEST (NEGATIVE) => B:BEST",
@@ -8867,6 +8945,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * mhomed,active vs. unique,active with subset ip(s), positive response
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.section= "Test Replica vs. owned active: some more UNIQUE,MHOMED combinations",
 		.name	= _NBT_NAME("_MA_UA_SB_P", 0x00, NULL),
@@ -8899,6 +8978,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 *       the release demand has no effect to the database record...
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UA_DI_PRA", 0x00, NULL),
 		.comment= "C:BEST vs. B:BEST2 (C:BEST2,LR:BEST2) => C:BEST",
@@ -8931,6 +9011,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. unique,active with different ip(s), positive response, with all addresses
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_UA_DI_A", 0x00, NULL),
 		.comment= "C:BEST vs. B:BEST2 (C:ALL) => B:MHOMED",
@@ -8962,6 +9043,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * unique,active vs. mhomed,active with different ip(s), positive response, with all addresses
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_UA_MA_DI_A", 0x00, NULL),
 		.comment= "C:BEST vs. B:BEST2 (C:ALL) => B:MHOMED",
@@ -8996,6 +9078,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,active with different ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.section= "Test Replica vs. owned active: SGROUP vs. SGROUP tests",
 		.name	= _NBT_NAME("_SA_SA_DI_U", 0x1C, NULL),
@@ -9024,6 +9107,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,active with same ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_SA_SI_U", 0x1C, NULL),
 		.skip	= (ctx->addresses_all_num < 3),
@@ -9051,6 +9135,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,active with superset ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_SA_SP_U", 0x1C, NULL),
 		.skip	= (ctx->addresses_all_num < 3),
@@ -9078,6 +9163,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,active with subset ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_SA_SB_U", 0x1C, NULL),
 		.skip	= (ctx->addresses_all_num < 3),
@@ -9105,6 +9191,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,tombstone with different ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_ST_DI_U", 0x1C, NULL),
 		.skip	= (ctx->addresses_all_num < 3),
@@ -9132,6 +9219,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,tombstone with same ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_ST_SI_U", 0x1C, NULL),
 		.skip	= (ctx->addresses_all_num < 3),
@@ -9159,6 +9247,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,tombstone with superset ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_ST_SP_U", 0x1C, NULL),
 		.skip	= (ctx->addresses_all_num < 3),
@@ -9186,6 +9275,7 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	 * sgroup,active vs. sgroup,tombstone with subset ip(s)
 	 */
 	{
+		.tctx	= tctx,
 		.line	= __location__,
 		.name	= _NBT_NAME("_SA_ST_SB_U", 0x1C, NULL),
 		.skip	= (ctx->addresses_all_num < 3),
@@ -9489,20 +9579,26 @@ static bool test_conflict_owned_active_vs_replica(struct torture_context *tctx,
 	return ret;
 }
 
+#define __NBT_LABEL_CAT1__(a,b)    a##b
+#define __NBT_LABEL_CAT2__(a,b)    __NBT_LABEL_CAT1__(a,b)
+#define _NBT_LABEL __NBT_LABEL_CAT2__(_label_, __LINE__)
+
 #define _NBT_ASSERT(v, correct) do { \
-	if ((v) != (correct)) { \
-		printf("(%s) Incorrect value %s=%d - should be %s (%d)\n", \
-		       __location__, #v, v, #correct, correct); \
+	bool _ret = true; \
+	torture_assert_int_equal_goto(rec->tctx, v, correct, \
+			_ret, _NBT_LABEL, "Invalid int value"); \
+_NBT_LABEL: \
+	if (!_ret) { \
 		return; \
 	} \
 } while (0)
 
 #define _NBT_ASSERT_STRING(v, correct) do { \
-	if ( ((!v) && (correct)) || \
-	     ((v) && (!correct)) || \
-	     ((v) && (correct) && strcmp(v,correct) != 0)) { \
-		printf("(%s) Incorrect value %s=%s - should be %s\n", \
-		       __location__, #v, v, correct); \
+	bool _ret = true; \
+	torture_assert_str_equal_goto(rec->tctx, v, correct, \
+			_ret, _NBT_LABEL, "Invalid string value"); \
+_NBT_LABEL: \
+	if (!_ret) { \
 		return; \
 	} \
 } while (0)
@@ -9522,8 +9618,8 @@ static void test_conflict_owned_active_vs_replica_handler_query(struct nbt_name_
 
 	name = &req_packet->questions[0].name;
 
-	_NBT_ASSERT(name->type, rec->name.type);
 	_NBT_ASSERT_STRING(name->name, rec->name.name);
+	_NBT_ASSERT(name->type, rec->name.type);
 	_NBT_ASSERT_STRING(name->scope, rec->name.scope);
 
 	_NBT_ASSERT(rec->defend.expect_release, false);
@@ -9621,8 +9717,8 @@ static void test_conflict_owned_active_vs_replica_handler_release(
 
 	name = &req_packet->questions[0].name;
 
-	_NBT_ASSERT(name->type, rec->name.type);
 	_NBT_ASSERT_STRING(name->name, rec->name.name);
+	_NBT_ASSERT(name->type, rec->name.type);
 	_NBT_ASSERT_STRING(name->scope, rec->name.scope);
 
 	_NBT_ASSERT(rec->defend.expect_release, true);
@@ -9668,6 +9764,17 @@ static void test_conflict_owned_active_vs_replica_handler(struct nbt_name_socket
 {
 	struct test_conflict_owned_active_vs_replica_struct *rec = 
 		(struct test_conflict_owned_active_vs_replica_struct *)nbtsock->incoming.private_data;
+	struct nbt_name *name = &req_packet->questions[0].name;
+
+	if (req_packet->operation & NBT_FLAG_BROADCAST) {
+		torture_comment(rec->tctx,
+			"%s: incoming packet name[%s] flags[0x%08X] from[%s]\n",
+			__location__,
+			nbt_name_string(rec->tctx, name),
+			req_packet->operation,
+			src->addr);
+		return;
+	}
 
 	rec->defend.ret = false;
 
@@ -9679,8 +9786,14 @@ static void test_conflict_owned_active_vs_replica_handler(struct nbt_name_socket
 		test_conflict_owned_active_vs_replica_handler_release(nbtsock, req_packet, src);
 		break;
 	default:
-		printf("%s: unexpected incoming packet\n", __location__);
-		return;
+		torture_comment(rec->tctx,
+			"%s: unexpected packet name[%s] flags[0x%08X] from[%s]\n",
+			__location__,
+			nbt_name_string(rec->tctx, name),
+			req_packet->operation,
+			src->addr);
+		_NBT_ASSERT((req_packet->operation & NBT_OPCODE), NBT_OPCODE_QUERY);
+		break;
 	}
 }
 
diff --git a/source4/torture/ndr/dfsblob.c b/source4/torture/ndr/dfsblob.c
index 3c95928a5c0..27ad5e40bfa 100644
--- a/source4/torture/ndr/dfsblob.c
+++ b/source4/torture/ndr/dfsblob.c
@@ -63,7 +63,7 @@ static const uint8_t dfs_get_ref_out2[] = {
 static bool dfs_referral_out_check(struct torture_context *tctx, struct dfs_referral_resp *r)
 {
 	torture_assert_str_equal(tctx,
-		r->referral_entries[0].referral.v3.data.referrals.r2.special_name,
+		r->referral_entries[0].referral.v3.referrals.r2.special_name,
 		"\\msw2k3.tst", "Special name");
 	ndr_push_struct_blob(&blob, tctx, r, (ndr_push_flags_fn_t)ndr_push_dfs_referral_resp);
 	torture_assert_int_equal(tctx, blob.data[blob.length-2], 0, "expanded names not null terminated");
diff --git a/source4/torture/ndr/ndr.c b/source4/torture/ndr/ndr.c
index 36b2b5540c0..6c564d3310a 100644
--- a/source4/torture/ndr/ndr.c
+++ b/source4/torture/ndr/ndr.c
@@ -355,6 +355,7 @@ struct torture_suite *torture_local_ndr(TALLOC_CTX *mem_ctx)
 	torture_suite_add_suite(suite, ndr_nbt_suite(suite));
 	torture_suite_add_suite(suite, ndr_ntlmssp_suite(suite));
 	torture_suite_add_suite(suite, ndr_backupkey_suite(suite));
+	torture_suite_add_suite(suite, ndr_string_suite(suite));
 
 	torture_suite_add_simple_test(suite, "string terminator",
 				      test_check_string_terminator);
diff --git a/source4/torture/ndr/string.c b/source4/torture/ndr/string.c
new file mode 100644
index 00000000000..30ed1e4d1a0
--- /dev/null
+++ b/source4/torture/ndr/string.c
@@ -0,0 +1,198 @@
+#include "includes.h"
+#include "torture/ndr/ndr.h"
+#include "torture/ndr/proto.h"
+#include "../lib/util/dlinklist.h"
+#include "param/param.h"
+
+static const char const *ascii = "ascii";
+/* the following is equivalent to "kamelåså öäüÿéèóò" in latin1 */
+static const char const latin1[] = { 0x6b, 0x61, 0x6d, 0x65, 0x6c, 0xe5, 0x73,
+                                     0xe5, 0x20, 0xF6, 0xE4, 0xFC, 0xFF, 0xE9,
+                                     0xE8, 0xF3, 0xF2, 0x00 };
+/* the following is equivalent to "kamelåså ☺☺☺ öäüÿéèóò" in utf8 */
+static const char const utf8[] = { 0x6b, 0x61, 0x6d, 0x65, 0x6c, 0xc3, 0xa5,
+                                   0x73, 0xc3, 0xa5, 0x20, 0xE2, 0x98, 0xBA,
+                                   0xE2, 0x98, 0xBA, 0xE2, 0x98, 0xBA, 0x20,
+                                   0xc3, 0xb6, 0xc3, 0xa4, 0xc3, 0xbc, 0xc3,
+                                   0xbf, 0xc3, 0xa9, 0xc3, 0xa8, 0xc3, 0xb3,
+                                   0xc3, 0xb2, 0x00 };
+
+/* purely for convenience */
+static int fl_ascii_null = LIBNDR_FLAG_STR_ASCII|LIBNDR_FLAG_STR_NULLTERM;
+static int fl_utf8_null = LIBNDR_FLAG_STR_UTF8|LIBNDR_FLAG_STR_NULLTERM;
+static int fl_raw8_null = LIBNDR_FLAG_STR_RAW8|LIBNDR_FLAG_STR_NULLTERM;
+
+static bool
+test_ndr_push_string (struct torture_context *tctx, const char *string,
+                      int flags, enum ndr_err_code exp_ndr_err,
+                      bool strcmp_pass)
+{
+	TALLOC_CTX *mem_ctx;
+	struct ndr_push *ndr;
+	enum ndr_err_code err;
+
+	torture_comment(tctx,
+                        "test_ndr_push_string %s flags 0x%x expecting "
+	                "err 0x%x and strcmp %s\n", string, flags, exp_ndr_err,
+	                strcmp_pass?"pass":"fail");
+	if (exp_ndr_err != NDR_ERR_SUCCESS) {
+		torture_comment(tctx, "(ignore any Conversion error) ");
+	}
+
+	mem_ctx = talloc_named (NULL, 0, "test_ndr_push_string");
+	ndr = talloc_zero (mem_ctx, struct ndr_push);
+	ndr_set_flags (&ndr->flags, flags);
+
+	err = ndr_push_string (ndr, NDR_SCALARS, string);
+	torture_assert(tctx, err == exp_ndr_err,
+	               "ndr_push_string: unexpected return code");
+
+	if (exp_ndr_err == NDR_ERR_SUCCESS) {
+		torture_assert(tctx, ndr->data != NULL,
+		               "ndr_push_string: succeeded but NULL data");
+
+		torture_assert(tctx,
+			       strcmp_pass == !strcmp(string, (char *)ndr->data),
+		               "ndr_push_string: post-push strcmp");
+	}
+
+	talloc_free(mem_ctx);
+	return true;
+}
+
+static bool
+test_ndr_pull_string (struct torture_context *tctx, const char *string,
+                      int flags, enum ndr_err_code exp_ndr_err,
+                      bool strcmp_pass)
+{
+	TALLOC_CTX *mem_ctx;
+	DATA_BLOB blob;
+	struct ndr_pull *ndr;
+	enum ndr_err_code err;
+	const char *result = NULL;
+
+	torture_comment(tctx,
+                        "test_ndr_pull_string '%s' flags 0x%x expecting "
+	                "err 0x%x and strcmp %s\n", string, flags, exp_ndr_err,
+	                strcmp_pass?"pass":"fail");
+	if (exp_ndr_err != NDR_ERR_SUCCESS) {
+		torture_comment(tctx, "(ignore any Conversion error) ");
+	}
+
+	mem_ctx = talloc_named (NULL, 0, "test_ndr_pull_string");
+
+	blob = data_blob_string_const(string);
+	ndr = ndr_pull_init_blob(&blob, mem_ctx);
+	ndr_set_flags (&ndr->flags, flags);
+
+	err = ndr_pull_string (ndr, NDR_SCALARS, &result);
+	torture_assert(tctx, err == exp_ndr_err,
+	               "ndr_pull_string: unexpected return code");
+
+	if (exp_ndr_err == NDR_ERR_SUCCESS) {
+		torture_assert(tctx, result != NULL,
+		               "ndr_pull_string: NULL data");
+		torture_assert(tctx, strcmp_pass == !strcmp(string, result),
+		               "ndr_pull_string: post-pull strcmp");
+		torture_assert(tctx, result != NULL,
+		               "ndr_pull_string succeeded but result NULL");
+	}
+
+	talloc_free(mem_ctx);
+	return true;
+}
+
+static bool
+torture_ndr_string(struct torture_context *torture)
+{
+	const char *saved_dos_cp = lpcfg_dos_charset(torture->lp_ctx);
+
+	torture_assert(torture,
+	               test_ndr_push_string (torture, ascii, fl_ascii_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_push_string(ASCII, STR_ASCII|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_push_string (torture, utf8, fl_utf8_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_push_string(UTF8, STR_UTF8|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_push_string (torture, utf8, fl_raw8_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_push_string(UTF8, STR_RAW8|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_push_string (torture, latin1, fl_raw8_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_push_string(LATIN1, STR_RAW8|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_push_string (torture, utf8, fl_ascii_null,
+	                                     NDR_ERR_CHARCNV, false),
+	               "test_ndr_push_string(UTF8, STR_ASCII|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_push_string (torture, latin1, fl_ascii_null,
+	                                     NDR_ERR_CHARCNV, false),
+	               "test_ndr_push_string(LATIN1, STR_ASCII|STR_NULL)");
+
+
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, ascii, fl_ascii_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_pull_string(ASCII, STR_ASCII|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, utf8, fl_utf8_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_pull_string(UTF8, STR_UTF8|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, utf8, fl_raw8_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_pull_string(UTF8, STR_RAW8|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, latin1, fl_raw8_null,
+	                                     NDR_ERR_SUCCESS, true),
+	               "test_ndr_pull_string(LATIN1, STR_RAW8|STR_NULL)");
+
+	/* Depending on runtime config, the behavior of ndr_pull_string on
+	 * incorrect combinations of strings and flags (latin1 with ASCII
+	 * flags, for example) may differ; it may return NDR_ERR_CHARCNV, or
+	 * it may return NDR_ERR_SUCCESS but with a string that has been
+	 * mutilated, depending on the value of "dos charset".  We test for
+	 * both cases here. */
+
+	lpcfg_do_global_parameter(torture->lp_ctx, "dos charset", "ASCII");
+	reload_charcnv(torture->lp_ctx);
+
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, latin1, fl_ascii_null,
+	                                     NDR_ERR_CHARCNV, false),
+	               "test_ndr_pull_string(LATIN1, STR_ASCII|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, utf8, fl_ascii_null,
+	                                     NDR_ERR_CHARCNV, false),
+	               "test_ndr_pull_string(UTF8, STR_ASCII|STR_NULL)");
+
+	lpcfg_do_global_parameter(torture->lp_ctx, "dos charset", "CP850");
+	reload_charcnv(torture->lp_ctx);
+
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, latin1, fl_ascii_null,
+	                                     NDR_ERR_SUCCESS, false),
+	               "test_ndr_pull_string(LATIN1, STR_ASCII|STR_NULL)");
+	torture_assert(torture,
+	               test_ndr_pull_string (torture, utf8, fl_ascii_null,
+	                                     NDR_ERR_SUCCESS, false),
+	               "test_ndr_pull_string(UTF8, STR_ASCII|STR_NULL)");
+
+	lpcfg_do_global_parameter(torture->lp_ctx, "dos charset", saved_dos_cp);
+	reload_charcnv(torture->lp_ctx);
+
+	return true;
+}
+
+struct torture_suite *ndr_string_suite(TALLOC_CTX *ctx)
+{
+	struct torture_suite *suite = torture_suite_create(ctx, "ndr_string");
+
+	torture_suite_add_simple_test(suite, "ndr_string", torture_ndr_string);
+	suite->description = talloc_strdup(suite, "NDR - string-conversion focused push/pull tests");
+
+	return suite;
+}
diff --git a/source4/torture/rap/rap.c b/source4/torture/rap/rap.c
index 5c1c5e196c6..ee37158a431 100644
--- a/source4/torture/rap/rap.c
+++ b/source4/torture/rap/rap.c
@@ -206,6 +206,21 @@ static bool test_netsessiongetinfo(struct torture_context *tctx,
 	return true;
 }
 
+static bool test_netremotetod(struct torture_context *tctx,
+			      struct smbcli_state *cli)
+{
+	struct rap_NetRemoteTOD r;
+
+	r.in.bufsize = 8192;
+
+	torture_assert_ntstatus_ok(tctx,
+		smbcli_rap_netremotetod(cli->tree, tctx, &r),
+		"smbcli_rap_netremotetod failed");
+	torture_assert_werr_ok(tctx, W_ERROR(r.out.status),
+		"smbcli_rap_netremotetod failed");
+
+	return true;
+}
 
 bool torture_rap_scan(struct torture_context *torture, struct smbcli_state *cli)
 {
@@ -246,6 +261,8 @@ NTSTATUS torture_rap_init(void)
 				    test_netsessionenum);
 	torture_suite_add_1smb_test(suite_basic, "netsessiongetinfo",
 				    test_netsessiongetinfo);
+	torture_suite_add_1smb_test(suite_basic, "netremotetod",
+				    test_netremotetod);
 
 	torture_suite_add_1smb_test(suite, "scan", torture_rap_scan);
 
diff --git a/source4/torture/raw/acls.c b/source4/torture/raw/acls.c
index 666d164c969..01ee8be6a9f 100644
--- a/source4/torture/raw/acls.c
+++ b/source4/torture/raw/acls.c
@@ -1846,6 +1846,7 @@ done:
 	return ret;
 }
 
+#if 0
 static bool test_inheritance_flags(struct torture_context *tctx,
     struct smbcli_state *cli)
 {
@@ -2073,6 +2074,7 @@ done:
 	smbcli_deltree(cli->tree, BASEDIR);
 	return ret;
 }
+#endif
 
 /*
   test dynamic acl inheritance
@@ -2201,13 +2203,14 @@ static bool test_inheritance_dynamic(struct torture_context *tctx,
 	smbcli_unlink(cli->tree, fname1);
 
 done:
-	torture_comment(tctx, "put back original sd\n");
-	set.set_secdesc.level = RAW_SFILEINFO_SEC_DESC;
-	set.set_secdesc.in.file.fnum = fnum;
-	set.set_secdesc.in.secinfo_flags = SECINFO_DACL;
-	set.set_secdesc.in.sd = sd_orig;
-	status = smb_raw_setfileinfo(cli->tree, &set);
-
+	if (sd_orig != NULL) {
+		torture_comment(tctx, "put back original sd\n");
+		set.set_secdesc.level = RAW_SFILEINFO_SEC_DESC;
+		set.set_secdesc.in.file.fnum = fnum;
+		set.set_secdesc.in.secinfo_flags = SECINFO_DACL;
+		set.set_secdesc.in.sd = sd_orig;
+		status = smb_raw_setfileinfo(cli->tree, &set);
+	}
 	smbcli_close(cli->tree, fnum);
 	smbcli_rmdir(cli->tree, dname);
 	smb_raw_exit(cli->session);
@@ -2244,6 +2247,8 @@ done:
 	CHECK_STATUS_FOR_BIT_ACTION(status, bits, do {} while (0)); \
 } while (0)
 
+#if 0
+
 /* test what access mask is needed for getting and setting security_descriptors
   Test copied to smb2/acls.c for SMB2. */
 static bool test_sd_get_set(struct torture_context *tctx, 
@@ -2433,6 +2438,7 @@ done:
 	return ret;
 }
 
+#endif
 
 /* 
    basic testing of security descriptor calls
@@ -2450,11 +2456,14 @@ struct torture_suite *torture_raw_acls(TALLOC_CTX *mem_ctx)
 	torture_suite_add_1smb_test(suite, "owner", test_owner_bits);
 	torture_suite_add_1smb_test(suite, "inheritance", test_inheritance);
 
-	/* torture_suite_add_1smb_test(suite, "INHERITFLAGS", test_inheritance_flags); */
+#if 0
+	torture_suite_add_1smb_test(suite, "INHERITFLAGS", test_inheritance_flags);
+#endif
 	torture_suite_add_1smb_test(suite, "dynamic", test_inheritance_dynamic);
-	/* XXX This test does not work against XP or Vista.
+#if 0
+	/* XXX This test does not work against XP or Vista. */
 	torture_suite_add_1smb_test(suite, "GETSET", test_sd_get_set);
-	*/
+#endif
 
 	return suite;
 }
diff --git a/source4/torture/raw/open.c b/source4/torture/raw/open.c
index 89042d77dc2..8a66a12cd57 100644
--- a/source4/torture/raw/open.c
+++ b/source4/torture/raw/open.c
@@ -130,7 +130,7 @@ static const char *rdwr_string(enum rdwr_mode m)
 	if ((v) != (finfo.all_info.out.field)) { \
 		torture_result(tctx, TORTURE_FAIL, \
 		       "(%s) wrong value for field %s  0x%x - 0x%x\n", \
-		       __location__, #field, (int)v, (int)(finfo.all_info.out.field)); \
+		       __location__, #field, (unsigned int)(v), (unsigned int)(finfo.all_info.out.field)); \
 		dump_all_info(tctx, &finfo); \
 		ret = false; \
 	}} while (0)
@@ -139,7 +139,7 @@ static const char *rdwr_string(enum rdwr_mode m)
 	if ((v) != (correct)) { \
 		torture_result(tctx, TORTURE_FAIL, \
 		       "(%s) wrong value for %s  0x%x - should be 0x%x\n", \
-		       __location__, #v, (int)(v), (int)correct); \
+		       __location__, #v, (unsigned int)(v), (unsigned int)(correct)); \
 		ret = false; \
 	}} while (0)
 
@@ -152,7 +152,7 @@ static const char *rdwr_string(enum rdwr_mode m)
 	status = smb_raw_setpathinfo(cli->tree, &sfinfo); \
 	if (!NT_STATUS_IS_OK(status)) { \
 		torture_warning(tctx, "(%s) Failed to set attrib 0x%x on %s\n", \
-		       __location__, sattrib, fname); \
+		       __location__, (unsigned int)(sattrib), fname); \
 	}} while (0)
 
 /*
@@ -337,7 +337,7 @@ static bool test_openx(struct torture_context *tctx, struct smbcli_state *cli)
 				__location__, nt_errstr(status),
 				nt_errstr(open_funcs[i].correct_status),
 				i, (int)open_funcs[i].with_file,
-				(int)open_funcs[i].open_func);
+				open_funcs[i].open_func);
 			ret = false;
 		}
 		if (NT_STATUS_IS_OK(status)) {
@@ -602,7 +602,7 @@ static bool test_t2open(struct torture_context *tctx, struct smbcli_state *cli)
 				 __location__, nt_errstr(status),
 				nt_errstr(open_funcs[i].correct_status),
 				i, (int)open_funcs[i].with_file,
-				(int)open_funcs[i].open_func);
+				open_funcs[i].open_func);
 			ret = false;
 		}
 		if (NT_STATUS_IS_OK(status)) {
diff --git a/source4/torture/raw/qfileinfo.c b/source4/torture/raw/qfileinfo.c
index 414084080b9..ca5f66795e6 100644
--- a/source4/torture/raw/qfileinfo.c
+++ b/source4/torture/raw/qfileinfo.c
@@ -779,7 +779,7 @@ static bool torture_raw_qfileinfo_internals(struct torture_context *torture,
 	s1 = fnum_find("BASIC_INFO");
 	if (s1 && is_ipc) {
 		if (s1->basic_info.out.attrib != FILE_ATTRIBUTE_NORMAL) {
-			printf("(%d) attrib basic_info/nlink incorrect - %d should be %d\n", __LINE__, s1->basic_info.out.attrib, FILE_ATTRIBUTE_NORMAL);
+			printf("(%d) attrib basic_info/nlink incorrect - %d should be %d\n", __LINE__, s1->basic_info.out.attrib, (int)FILE_ATTRIBUTE_NORMAL);
 			ret = false;
 		}
 	}
diff --git a/source4/torture/raw/streams.c b/source4/torture/raw/streams.c
index 5ce9b756efc..ae3bc2a7131 100644
--- a/source4/torture/raw/streams.c
+++ b/source4/torture/raw/streams.c
@@ -1629,6 +1629,7 @@ static bool test_stream_create_disposition(struct torture_context *tctx,
 	return ret;
 }
 
+#if 0
 /* Test streaminfo with enough streams on a file to fill up the buffer.  */
 static bool test_stream_large_streaminfo(struct torture_context *tctx,
 					 struct smbcli_state *cli)
@@ -1674,6 +1675,7 @@ static bool test_stream_large_streaminfo(struct torture_context *tctx,
 	smbcli_deltree(cli->tree, BASEDIR);
 	return ret;
 }
+#endif
 
 /* Test the effect of setting attributes on a stream. */
 static bool test_stream_attributes(struct torture_context *tctx,
@@ -1913,8 +1915,10 @@ struct torture_suite *torture_raw_streams(TALLOC_CTX *tctx)
 	torture_suite_add_1smb_test(suite, "attr", test_stream_attributes);
 	torture_suite_add_1smb_test(suite, "sumtab", test_stream_summary_tab);
 
-	/* torture_suite_add_1smb_test(suite, "LARGESTREAMINFO", */
-	/*     test_stream_large_streaminfo); */
+#if 0
+	torture_suite_add_1smb_test(suite, "LARGESTREAMINFO",
+		test_stream_large_streaminfo);
+#endif
 
 	return suite;
 }
diff --git a/source4/torture/rpc/dsgetinfo.c b/source4/torture/rpc/dsgetinfo.c
index 6122ff06037..a0360e84041 100644
--- a/source4/torture/rpc/dsgetinfo.c
+++ b/source4/torture/rpc/dsgetinfo.c
@@ -88,7 +88,7 @@ static const char *torture_get_ldap_base_dn(struct torture_context *tctx, struct
 	}
 
 	ldb_set_modules_dir(ldb,
-		talloc_asprintf(ldb, "%s/ldb", lpcfg_modulesdir(tctx->lp_ctx)));
+			    modules_path(ldb, "ldb"));
 
 	ret = ldb_connect(ldb, ldap_url, 0, NULL);
 	if (ret != LDB_SUCCESS) {
diff --git a/source4/torture/rpc/remote_pac.c b/source4/torture/rpc/remote_pac.c
index c4efabcebc2..70912781a82 100644
--- a/source4/torture/rpc/remote_pac.c
+++ b/source4/torture/rpc/remote_pac.c
@@ -476,8 +476,9 @@ static bool test_S2U4Self(struct torture_context *tctx,
 
 	/* Wipe out any existing ccache */
 	cli_credentials_invalidate_ccache(credentials, CRED_SPECIFIED);
-	cli_credentials_set_target_service(credentials, talloc_asprintf(tmp_ctx, "host/%s", test_machine_name));
-	cli_credentials_set_impersonate_principal(credentials, cli_credentials_get_principal(cmdline_credentials, tmp_ctx));
+	cli_credentials_set_impersonate_principal(credentials,
+			cli_credentials_get_principal(cmdline_credentials, tmp_ctx),
+			talloc_asprintf(tmp_ctx, "host/%s", test_machine_name));
 
 	status = gensec_client_start(tctx, &gensec_client_context, tctx->ev,
 				     lpcfg_gensec_settings(tctx, tctx->lp_ctx));
@@ -525,7 +526,7 @@ static bool test_S2U4Self(struct torture_context *tctx,
 	/* Don't pollute the remaining tests with the changed credentials */
 	cli_credentials_invalidate_ccache(credentials, CRED_SPECIFIED);
 	cli_credentials_set_target_service(credentials, NULL);
-	cli_credentials_set_impersonate_principal(credentials, NULL);
+	cli_credentials_set_impersonate_principal(credentials, NULL, NULL);
 
 	/* Extract the PAC using Samba's code */
 
diff --git a/source4/torture/rpc/rpc.c b/source4/torture/rpc/rpc.c
index 03936f22fff..01ce93f373e 100644
--- a/source4/torture/rpc/rpc.c
+++ b/source4/torture/rpc/rpc.c
@@ -75,7 +75,7 @@ _PUBLIC_ NTSTATUS torture_rpc_connection(struct torture_context *tctx,
 	NTSTATUS status;
 	struct dcerpc_binding *binding;
 
-	dcerpc_init(tctx->lp_ctx);
+	dcerpc_init();
 
 	status = torture_rpc_binding(tctx, &binding);
 	if (NT_STATUS_IS_ERR(status))
diff --git a/source4/torture/rpc/spoolss_notify.c b/source4/torture/rpc/spoolss_notify.c
index ec29c3944fe..ea05d9bd1c3 100644
--- a/source4/torture/rpc/spoolss_notify.c
+++ b/source4/torture/rpc/spoolss_notify.c
@@ -247,20 +247,13 @@ static NTSTATUS spoolss__op_init_server(struct dcesrv_context *dce_ctx, const st
 static bool test_OpenPrinter(struct torture_context *tctx,
 			     struct dcerpc_pipe *p,
 			     struct policy_handle *handle,
-			     const char *name)
+			     const char *printername)
 {
 	struct spoolss_OpenPrinter r;
-	const char *printername;
 	struct dcerpc_binding_handle *b = p->binding_handle;
 
 	ZERO_STRUCT(r);
 
-	if (name) {
-		printername	= talloc_asprintf(tctx, "\\\\%s\\%s", dcerpc_server_name(p), name);
-	} else {
-		printername	= talloc_asprintf(tctx, "\\\\%s", dcerpc_server_name(p));
-	}
-
 	r.in.printername	= printername;
 	r.in.datatype		= NULL;
 	r.in.devmode_ctr.devmode= NULL;
@@ -352,17 +345,17 @@ static bool test_RemoteFindFirstPrinterChangeNotifyEx(struct torture_context *tc
 static bool test_RouterRefreshPrinterChangeNotify(struct torture_context *tctx,
 						  struct dcerpc_binding_handle *b,
 						  struct policy_handle *handle,
-						  struct spoolss_NotifyOption *options)
+						  struct spoolss_NotifyOption *options,
+						  struct spoolss_NotifyInfo **info)
 {
 	struct spoolss_RouterRefreshPrinterChangeNotify r;
-	struct spoolss_NotifyInfo *info;
 
 	torture_comment(tctx, "Testing RouterRefreshPrinterChangeNotify\n");
 
 	r.in.handle = handle;
 	r.in.change_low = 0;
 	r.in.options = options;
-	r.out.info = &info;
+	r.out.info = info;
 
 	torture_assert_ntstatus_ok(tctx, dcerpc_spoolss_RouterRefreshPrinterChangeNotify_r(b, tctx, &r),
 		"RouterRefreshPrinterChangeNotify failed");
@@ -455,8 +448,8 @@ static bool test_start_dcerpc_server(struct torture_context *tctx,
 
 	lpcfg_set_cmdline(tctx->lp_ctx, "dcerpc endpoint servers", "spoolss");
 
-	load_interfaces(tctx, lpcfg_interfaces(tctx->lp_ctx), &ifaces);
-	address = iface_n_ip(ifaces, 0);
+	load_interface_list(tctx, tctx->lp_ctx, &ifaces);
+	address = iface_list_first_v4(ifaces);
 
 	torture_comment(tctx, "Listening for callbacks on %s\n", address);
 
@@ -503,31 +496,37 @@ static bool test_RFFPCNEx(struct torture_context *tctx,
 	struct spoolss_NotifyOption *printer_option = setup_printer_NotifyOption(tctx);
 #endif
 	struct dcerpc_binding_handle *b = p->binding_handle;
+	const char *printername = NULL;
+	struct spoolss_NotifyInfo *info = NULL;
 
 	received_packets = NULL;
 
 	/* Start DCE/RPC server */
 	torture_assert(tctx, test_start_dcerpc_server(tctx, p->conn->event_ctx, &dce_ctx, &address), "");
 
-	torture_assert(tctx, test_OpenPrinter(tctx, p, &handle, NULL), "");
+	printername	= talloc_asprintf(tctx, "\\\\%s", dcerpc_server_name(p));
+
+	torture_assert(tctx, test_OpenPrinter(tctx, p, &handle, printername), "");
 	torture_assert(tctx, test_RemoteFindFirstPrinterChangeNotifyEx(tctx, b, &handle, address, server_option), "");
 	torture_assert(tctx, received_packets, "no packets received");
 	torture_assert_int_equal(tctx, received_packets->opnum, NDR_SPOOLSS_REPLYOPENPRINTER,
 		"no ReplyOpenPrinter packet after RemoteFindFirstPrinterChangeNotifyEx");
-	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, b, &handle, NULL), "");
-	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, b, &handle, server_option), "");
+	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, b, &handle, NULL, &info), "");
+	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, b, &handle, server_option, &info), "");
 	torture_assert(tctx, test_ClosePrinter(tctx, b, &handle), "");
 	tmp = last_packet(received_packets);
 	torture_assert_int_equal(tctx, tmp->opnum, NDR_SPOOLSS_REPLYCLOSEPRINTER,
 		"no ReplyClosePrinter packet after ClosePrinter");
 #if 0
+	printername	= talloc_asprintf(tctx, "\\\\%s\\%s", dcerpc_server_name(p), name);
+
 	torture_assert(tctx, test_OpenPrinter(tctx, p, &handle, "Epson AL-2600"), "");
 	torture_assert(tctx, test_RemoteFindFirstPrinterChangeNotifyEx(tctx, p, &handle, address, printer_option), "");
 	tmp = last_packet(received_packets);
 	torture_assert_int_equal(tctx, tmp->opnum, NDR_SPOOLSS_REPLYOPENPRINTER,
 		"no ReplyOpenPrinter packet after RemoteFindFirstPrinterChangeNotifyEx");
-	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, p, &handle, NULL), "");
-	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, p, &handle, printer_option), "");
+	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, p, &handle, NULL, &info), "");
+	torture_assert(tctx, test_RouterRefreshPrinterChangeNotify(tctx, p, &handle, printer_option, &info), "");
 	torture_assert(tctx, test_SetPrinter(tctx, p, &handle), "");
 	tmp = last_packet(received_packets);
 	torture_assert_int_equal(tctx, tmp->opnum, NDR_SPOOLSS_ROUTERREPLYPRINTEREX,
diff --git a/source4/torture/rpc/testjoin.c b/source4/torture/rpc/testjoin.c
index c4ab0e43ada..76344e0ddf0 100644
--- a/source4/torture/rpc/testjoin.c
+++ b/source4/torture/rpc/testjoin.c
@@ -583,14 +583,14 @@ static NTSTATUS torture_leave_ads_domain(struct torture_context *torture,
 	ldb_set_opaque(ldb_ctx, "loadparm", cmdline_lp_ctx);
 
 	rtn = ldb_connect(ldb_ctx, remote_ldb_url, 0, NULL);
-	if (rtn != 0) {
+	if (rtn != LDB_SUCCESS) {
 		libnet_r->out.error_string = NULL;
 		talloc_free(tmp_ctx);
 		return NT_STATUS_UNSUCCESSFUL;
 	}
 
 	rtn = ldb_delete(ldb_ctx, server_dn);
-	if (rtn != 0) {
+	if (rtn != LDB_SUCCESS) {
 		libnet_r->out.error_string = NULL;
 		talloc_free(tmp_ctx);
 		return NT_STATUS_UNSUCCESSFUL;
diff --git a/source4/torture/smb2/acls.c b/source4/torture/smb2/acls.c
index 3883ae5055f..fa6c002da7b 100644
--- a/source4/torture/smb2/acls.c
+++ b/source4/torture/smb2/acls.c
@@ -1662,6 +1662,7 @@ done:
 	CHECK_STATUS_FOR_BIT_ACTION(status, bits, do {} while (0)); \
 } while (0)
 
+#if 0
 /* test what access mask is needed for getting and setting security_descriptors */
 /* Note: This test was copied from raw/acls.c. */
 static bool test_sd_get_set(struct torture_context *tctx, struct smb2_tree *tree)
@@ -1849,6 +1850,7 @@ done:
 
 	return ret;
 }
+#endif
 
 /*
    basic testing of SMB2 ACLs
@@ -1863,9 +1865,10 @@ struct torture_suite *torture_smb2_acls_init(void)
 	torture_suite_add_1smb2_test(suite, "INHERITANCE", test_inheritance);
 	torture_suite_add_1smb2_test(suite, "INHERITFLAGS", test_inheritance_flags);
 	torture_suite_add_1smb2_test(suite, "DYNAMIC", test_inheritance_dynamic);
-	/* XXX This test does not work against XP or Vista.
+#if 0
+	/* XXX This test does not work against XP or Vista. */
 	torture_suite_add_1smb2_test(suite, "GETSET", test_sd_get_set);
-	*/
+#endif
 
 	suite->description = talloc_strdup(suite, "SMB2-ACLS tests");
 
diff --git a/source4/torture/smbtorture.c b/source4/torture/smbtorture.c
index 62cf0abfb70..83816e8be55 100644
--- a/source4/torture/smbtorture.c
+++ b/source4/torture/smbtorture.c
@@ -686,7 +686,7 @@ int main(int argc,char *argv[])
 
 	torture->lp_ctx = cmdline_lp_ctx;
 
-	gensec_init(cmdline_lp_ctx);
+	gensec_init();
 
 	if (shell) {
 		/* In shell mode, just ignore any remaining test names. */
diff --git a/source4/torture/torture.c b/source4/torture/torture.c
index ffd884a38db..9b1719ed7d6 100644
--- a/source4/torture/torture.c
+++ b/source4/torture/torture.c
@@ -43,26 +43,12 @@ bool torture_register_suite(struct torture_suite *suite)
 	return torture_suite_add_suite(torture_root, suite);
 }
 
-#ifndef ENABLE_LIBNETAPI
-NTSTATUS torture_libnetapi_init(void)
-{
-	return NT_STATUS_OK;
-}
-#endif
-
-#ifndef ENABLE_LIBSMBCLIENT
-NTSTATUS torture_libsmbclient_init(void)
-{
-	return NT_STATUS_OK;
-}
-#endif
-
 _PUBLIC_ int torture_init(void)
 {
 #define _MODULE_PROTO(init) extern NTSTATUS init(void);
 	STATIC_smbtorture_MODULES_PROTO;
 	init_module_fn static_init[] = { STATIC_smbtorture_MODULES };
-	init_module_fn *shared_init = load_samba_modules(NULL, cmdline_lp_ctx, "smbtorture");
+	init_module_fn *shared_init = load_samba_modules(NULL, "smbtorture");
 
 	run_init_functions(static_init);
 	run_init_functions(shared_init);
diff --git a/source4/torture/unix/whoami.c b/source4/torture/unix/whoami.c
index 45b2775646d..1e79d7eeb23 100644
--- a/source4/torture/unix/whoami.c
+++ b/source4/torture/unix/whoami.c
@@ -29,10 +29,6 @@
 /* Size (in bytes) of the required fields in the SMBwhoami response. */
 #define WHOAMI_REQUIRED_SIZE	40
 
-enum smb_whoami_flags {
-    SMB_WHOAMI_GUEST = 0x1 /* Logged in as (or squashed to) guest */
-};
-
 /*
    SMBWhoami - Query the user mapping performed by the server for the
    connected tree. This is a subcommand of the TRANS2_QFSINFO.
diff --git a/source4/torture/winbind/struct_based.c b/source4/torture/winbind/struct_based.c
index aeb81c972ce..2bab94088a3 100644
--- a/source4/torture/winbind/struct_based.c
+++ b/source4/torture/winbind/struct_based.c
@@ -914,7 +914,6 @@ static bool parse_domain_user(struct torture_context *torture,
 	fstrcpy(user, p+1);
 	fstrcpy(domain, domuser);
 	domain[PTR_DIFF(p, domuser)] = 0;
-	strupper_m(domain);
 
 	return true;
 }
diff --git a/source4/torture/wscript_build b/source4/torture/wscript_build
index 68ec4e62209..106cc64280c 100644
--- a/source4/torture/wscript_build
+++ b/source4/torture/wscript_build
@@ -33,7 +33,7 @@ bld.RECURSE('libnetapi')
 bld.RECURSE('libsmbclient')
 
 bld.SAMBA_SUBSYSTEM('TORTURE_NDR',
-	source='ndr/ndr.c ndr/winreg.c ndr/atsvc.c ndr/lsa.c ndr/epmap.c ndr/dfs.c ndr/netlogon.c ndr/drsuapi.c ndr/spoolss.c ndr/samr.c ndr/dfsblob.c ndr/drsblobs.c ndr/nbt.c ndr/ntlmssp.c ndr/backupkey.c',
+	source='ndr/ndr.c ndr/winreg.c ndr/atsvc.c ndr/lsa.c ndr/epmap.c ndr/dfs.c ndr/netlogon.c ndr/drsuapi.c ndr/spoolss.c ndr/samr.c ndr/dfsblob.c ndr/drsblobs.c ndr/nbt.c ndr/ntlmssp.c ndr/backupkey.c ndr/string.c',
 	autoproto='ndr/proto.h',
 	deps='torture'
 	)
diff --git a/source4/utils/ntlm_auth.c b/source4/utils/ntlm_auth.c
index a3701f88224..d5c385cbc68 100644
--- a/source4/utils/ntlm_auth.c
+++ b/source4/utils/ntlm_auth.c
@@ -366,7 +366,7 @@ static void manage_gensec_request(enum stdio_helper_mode stdio_helper_mode,
 	};
 	struct gensec_ntlm_state *state;
 	struct tevent_context *ev;
-	struct messaging_context *msg;
+	struct imessaging_context *msg;
 
 	NTSTATUS nt_status;
 	bool first = false;
@@ -461,9 +461,9 @@ static void manage_gensec_request(enum stdio_helper_mode stdio_helper_mode,
 		case SQUID_2_5_NTLMSSP:
 		{
 			const char *winbind_method[] = { "winbind", NULL };
-			struct auth_context *auth_context;
+			struct auth4_context *auth_context;
 
-			msg = messaging_client_init(state, lpcfg_messaging_path(state, lp_ctx), ev);
+			msg = imessaging_client_init(state, lpcfg_imessaging_path(state, lp_ctx), ev);
 			if (!msg) {
 				talloc_free(mem_ctx);
 				exit(1);
@@ -1103,7 +1103,7 @@ int main(int argc, const char **argv)
 		return 1;
 	}
 
-	gensec_init(cmdline_lp_ctx);
+	gensec_init();
 
 	if (opt_domain == NULL) {
 		opt_domain = lpcfg_workgroup(cmdline_lp_ctx);
diff --git a/source4/web_server/web_server.c b/source4/web_server/web_server.c
index 338cc2b57a3..cdf6f9b7d0e 100644
--- a/source4/web_server/web_server.c
+++ b/source4/web_server/web_server.c
@@ -248,15 +248,14 @@ static void websrv_send(struct stream_connection *conn, uint16_t flags)
 */
 static void websrv_accept(struct stream_connection *conn)
 {
-	struct task_server *task = talloc_get_type(conn->private_data, struct task_server);
-	struct web_server_data *wdata = talloc_get_type(task->private_data, struct web_server_data);
+	struct web_server_data *wdata = talloc_get_type(conn->private_data, struct web_server_data);
 	struct websrv_context *web;
 	struct socket_context *tls_socket;
 
 	web = talloc_zero(conn, struct websrv_context);
 	if (web == NULL) goto failed;
 
-	web->task = task;
+	web->task = wdata->task;
 	web->conn = conn;
 	conn->private_data = web;
 	talloc_set_destructor(web, websrv_destructor);
@@ -312,6 +311,7 @@ static void websrv_task_init(struct task_server *task)
 	wdata = talloc_zero(task, struct web_server_data);
 	if (wdata == NULL) goto failed;
 
+	wdata->task = task;
 	task->private_data = wdata;
 
 	if (lpcfg_interfaces(task->lp_ctx) && lpcfg_bind_interfaces_only(task->lp_ctx)) {
@@ -319,16 +319,16 @@ static void websrv_task_init(struct task_server *task)
 		int i;
 		struct interface *ifaces;
 
-		load_interfaces(NULL, lpcfg_interfaces(task->lp_ctx), &ifaces);
+		load_interface_list(NULL, task->lp_ctx, &ifaces);
 
-		num_interfaces = iface_count(ifaces);
+		num_interfaces = iface_list_count(ifaces);
 		for(i = 0; i < num_interfaces; i++) {
-			const char *address = iface_n_ip(ifaces, i);
+			const char *address = iface_list_n_ip(ifaces, i);
 			status = stream_setup_socket(task,
 						     task->event_ctx,
 						     task->lp_ctx, model_ops,
 						     &web_stream_ops, 
-						     "ipv4", address, 
+						     "ip", address,
 						     &port, lpcfg_socket_options(task->lp_ctx),
 						     task);
 			if (!NT_STATUS_IS_OK(status)) goto failed;
@@ -336,13 +336,23 @@ static void websrv_task_init(struct task_server *task)
 
 		talloc_free(ifaces);
 	} else {
-		status = stream_setup_socket(task, task->event_ctx,
-					     task->lp_ctx, model_ops,
-					     &web_stream_ops,
-					     "ipv4", lpcfg_socket_address(task->lp_ctx),
-					     &port, lpcfg_socket_options(task->lp_ctx),
-					     task);
-		if (!NT_STATUS_IS_OK(status)) goto failed;
+		const char **wcard;
+		int i;
+		wcard = iface_list_wildcard(task, task->lp_ctx);
+		if (wcard == NULL) {
+			DEBUG(0,("No wildcard addresses available\n"));
+			goto failed;
+		}
+		for (i=0; wcard[i]; i++) {
+			status = stream_setup_socket(task, task->event_ctx,
+						     task->lp_ctx, model_ops,
+						     &web_stream_ops,
+						     "ip", wcard[i],
+						     &port, lpcfg_socket_options(task->lp_ctx),
+						     wdata);
+			if (!NT_STATUS_IS_OK(status)) goto failed;
+		}
+		talloc_free(wcard);
 	}
 
 	wdata->tls_params = tls_initialise(wdata, task->lp_ctx);
diff --git a/source4/web_server/web_server.h b/source4/web_server/web_server.h
index aa4d83c17b5..274a54a5c3d 100644
--- a/source4/web_server/web_server.h
+++ b/source4/web_server/web_server.h
@@ -29,6 +29,7 @@ struct web_server_data {
 	void (*http_process_input)(struct web_server_data *wdata, 
 				   struct websrv_context *web);
 	void *private_data;
+	struct task_server *task;
 };
 
 struct http_header {
diff --git a/source4/winbind/idmap.c b/source4/winbind/idmap.c
index ff57064d48b..4f2a54384c7 100644
--- a/source4/winbind/idmap.c
+++ b/source4/winbind/idmap.c
@@ -164,7 +164,7 @@ struct idmap_context *idmap_init(TALLOC_CTX *mem_ctx,
 	idmap_ctx->lp_ctx = lp_ctx;
 
 	idmap_ctx->ldb_ctx = ldb_wrap_connect(mem_ctx, ev_ctx, lp_ctx,
-					      lpcfg_idmap_url(lp_ctx),
+					      "idmap.ldb",
 					      system_session(lp_ctx),
 					      NULL, 0);
 	if (idmap_ctx->ldb_ctx == NULL) {
diff --git a/source4/winbind/wb_init_domain.c b/source4/winbind/wb_init_domain.c
index 50a6af05fd6..9847afbba05 100644
--- a/source4/winbind/wb_init_domain.c
+++ b/source4/winbind/wb_init_domain.c
@@ -154,7 +154,7 @@ struct composite_context *wb_init_domain_send(TALLOC_CTX *mem_ctx,
 	     (lpcfg_server_role(service->task->lp_ctx) == ROLE_DOMAIN_CONTROLLER)) &&
 	    (dom_sid_equal(state->domain->info->sid,
 			   state->service->primary_sid))) {
-		state->domain->netlogon_binding->flags |= DCERPC_SCHANNEL;
+		state->domain->netlogon_binding->flags |= DCERPC_SCHANNEL | DCERPC_SCHANNEL_128;
 
 		/* For debugging, it can be a real pain if all the traffic is encrypted */
 		if (lpcfg_winbind_sealed_pipes(service->task->lp_ctx)) {
diff --git a/source4/winbind/wb_samba3_protocol.c b/source4/winbind/wb_samba3_protocol.c
index 4bb0582cd22..f0f803dac5d 100644
--- a/source4/winbind/wb_samba3_protocol.c
+++ b/source4/winbind/wb_samba3_protocol.c
@@ -183,6 +183,7 @@ NTSTATUS wbsrv_samba3_handle_call(struct wbsrv_samba3_call *s3call)
 	case WINBINDD_PAM_CHAUTHTOK:
 	case WINBINDD_PAM_CHNG_PSWD_AUTH_CRAP:
 	case WINBINDD_LOOKUPRIDS:
+	case WINBINDD_LOOKUPSIDS:
 	case WINBINDD_SIDS_TO_XIDS:
 	case WINBINDD_ALLOCATE_UID:
 	case WINBINDD_ALLOCATE_GID:
@@ -199,7 +200,11 @@ NTSTATUS wbsrv_samba3_handle_call(struct wbsrv_samba3_call *s3call)
 	case WINBINDD_DUAL_NDRCMD:
 	case WINBINDD_CCACHE_NTLMAUTH:
 	case WINBINDD_NUM_CMDS:
-		DEBUG(10, ("Unimplemented winbind samba3 request %d\n", 
+	case WINBINDD_CHANGE_MACHACC:
+	case WINBINDD_PING_DC:
+	case WINBINDD_DC_INFO:
+	case WINBINDD_CCACHE_SAVE:
+		DEBUG(10, ("Unimplemented winbind samba3 request %d\n",
 			   s3call->request->cmd));
 		break;
 	}
diff --git a/source4/winbind/wb_server.h b/source4/winbind/wb_server.h
index f20bc0aa510..12dd1888ed1 100644
--- a/source4/winbind/wb_server.h
+++ b/source4/winbind/wb_server.h
@@ -104,7 +104,7 @@ struct wbsrv_connection {
 
 #define WBSRV_SAMBA3_SET_STRING(dest, src) do { \
 	memset(dest, 0, sizeof(dest));\
-	safe_strcpy(dest, src, sizeof(dest)-1);\
+	strlcpy((dest), (src) ? (src) : "", sizeof(dest));\
 } while(0)
 
 /*
diff --git a/source4/wrepl_server/wrepl_in_connection.c b/source4/wrepl_server/wrepl_in_connection.c
index 9d3ddbf0da7..962a1cb7fa4 100644
--- a/source4/wrepl_server/wrepl_in_connection.c
+++ b/source4/wrepl_server/wrepl_in_connection.c
@@ -441,16 +441,19 @@ NTSTATUS wreplsrv_setup_sockets(struct wreplsrv_service *service, struct loadpar
 		int i;
 		struct interface *ifaces;
 
-		load_interfaces(task, lpcfg_interfaces(lp_ctx), &ifaces);
+		load_interface_list(task, lp_ctx, &ifaces);
 
-		num_interfaces = iface_count(ifaces);
+		num_interfaces = iface_list_count(ifaces);
 
 		/* We have been given an interfaces line, and been 
 		   told to only bind to those interfaces. Create a
 		   socket per interface and bind to only these.
 		*/
 		for(i = 0; i < num_interfaces; i++) {
-			address = iface_n_ip(ifaces, i);
+			if (!iface_list_n_is_v4(ifaces, i)) {
+				continue;
+			}
+			address = iface_list_n_ip(ifaces, i);
 			status = stream_setup_socket(task, task->event_ctx,
 						     task->lp_ctx, model_ops,
 						     &wreplsrv_stream_ops,
@@ -465,6 +468,9 @@ NTSTATUS wreplsrv_setup_sockets(struct wreplsrv_service *service, struct loadpar
 		}
 	} else {
 		address = lpcfg_socket_address(lp_ctx);
+		if (strcmp(address, "") == 0) {
+			address = "0.0.0.0";
+		}
 		status = stream_setup_socket(task, task->event_ctx, task->lp_ctx,
 					     model_ops, &wreplsrv_stream_ops,
 					     "ipv4", address, &port, lpcfg_socket_options(task->lp_ctx),
diff --git a/source4/wrepl_server/wrepl_scavenging.c b/source4/wrepl_server/wrepl_scavenging.c
index 18d4780c0bf..d04064970c9 100644
--- a/source4/wrepl_server/wrepl_scavenging.c
+++ b/source4/wrepl_server/wrepl_scavenging.c
@@ -306,7 +306,7 @@ static NTSTATUS wreplsrv_scavenging_replica_non_active_records(struct wreplsrv_s
 }
 
 struct verify_state {
-	struct messaging_context *msg_ctx;
+	struct imessaging_context *msg_ctx;
 	struct wreplsrv_service *service;
 	struct winsdb_record *rec;
 	struct nbtd_proxy_wins_challenge r;
diff --git a/source4/wrepl_server/wrepl_server.c b/source4/wrepl_server/wrepl_server.c
index 83510c73a8e..79e0cfc24d7 100644
--- a/source4/wrepl_server/wrepl_server.c
+++ b/source4/wrepl_server/wrepl_server.c
@@ -38,7 +38,7 @@ static struct ldb_context *wins_config_db_connect(TALLOC_CTX *mem_ctx,
 						  struct tevent_context *ev_ctx,
 						  struct loadparm_context *lp_ctx)
 {
-	return ldb_wrap_connect(mem_ctx, ev_ctx, lp_ctx, private_path(mem_ctx,
+	return ldb_wrap_connect(mem_ctx, ev_ctx, lp_ctx, lpcfg_private_path(mem_ctx,
 			        lp_ctx, lpcfg_wins_config_url(lp_ctx)),
 				system_session(lp_ctx), NULL, 0);
 }
@@ -78,8 +78,8 @@ static NTSTATUS wreplsrv_open_winsdb(struct wreplsrv_service *service,
 
 	if (owner == NULL) {
 		struct interface *ifaces;
-		load_interfaces(service, lpcfg_interfaces(lp_ctx), &ifaces);
-		owner = iface_n_ip(ifaces, 0);
+		load_interface_list(service, lp_ctx, &ifaces);
+		owner = iface_list_first_v4(ifaces);
 	}
 
 	service->wins_db     = winsdb_connect(service, service->task->event_ctx, lp_ctx, owner, WINSDB_HANDLE_CALLER_WREPL);
diff --git a/testdata/samba3/account_policy.tdb2 b/testdata/samba3/account_policy.tdb2
new file mode 100644
index 00000000000..434787e070c
--- /dev/null
+++ b/testdata/samba3/account_policy.tdb2
diff --git a/testdata/samba3/group_mapping.tdb2 b/testdata/samba3/group_mapping.tdb2
new file mode 100644
index 00000000000..1a6466ea042
--- /dev/null
+++ b/testdata/samba3/group_mapping.tdb2
diff --git a/testdata/samba3/passdb.tdb2 b/testdata/samba3/passdb.tdb2
new file mode 100644
index 00000000000..4ef87ed0017
--- /dev/null
+++ b/testdata/samba3/passdb.tdb2
diff --git a/testdata/samba3/registry.tdb2 b/testdata/samba3/registry.tdb2
new file mode 100644
index 00000000000..d747c3d055a
--- /dev/null
+++ b/testdata/samba3/registry.tdb2
diff --git a/testdata/samba3/secrets.tdb2 b/testdata/samba3/secrets.tdb2
new file mode 100644
index 00000000000..5f88d6ac80f
--- /dev/null
+++ b/testdata/samba3/secrets.tdb2
diff --git a/testdata/samba3/share_info.tdb2 b/testdata/samba3/share_info.tdb2
new file mode 100644
index 00000000000..96e4c7aaf89
--- /dev/null
+++ b/testdata/samba3/share_info.tdb2
diff --git a/testdata/samba3/winbindd_idmap.tdb2 b/testdata/samba3/winbindd_idmap.tdb2
new file mode 100644
index 00000000000..a63d875055e
--- /dev/null
+++ b/testdata/samba3/winbindd_idmap.tdb2
diff --git a/testprogs/blackbox/renamedc.sh b/testprogs/blackbox/renamedc.sh
new file mode 100755
index 00000000000..d0615b319c7
--- /dev/null
+++ b/testprogs/blackbox/renamedc.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+if [ $# -lt 1 ]; then
+cat <<EOF
+Usage: blackbox_upgradeprovision.sh PREFIX
+EOF
+exit 1;
+fi
+
+PREFIX="$1"
+shift 1
+
+. `dirname $0`/subunit.sh
+
+if [ ! -d $PREFIX/upgradeprovision_full ]; then
+	$PYTHON $SRCDIR/source4/setup/provision --host-name=bar --domain=FOO --realm=foo.example.com --targetdir="$PREFIX/upgradeprovision_full" --server-role="dc"
+fi
+
+
+testrenamedc() {
+	$PYTHON $SRCDIR/source4/scripting/bin/renamedc \
+		--oldname="BAR" \
+		--newname="RAYMONBAR" \
+		-s $PREFIX/upgradeprovision_full/etc/smb.conf
+}
+
+
+testrenamedc2() {
+	$PYTHON $SRCDIR/source4/scripting/bin/renamedc \
+		--oldname="RAYMONBAR" \
+		--newname="BAR" \
+		-s $PREFIX/upgradeprovision_full/etc/smb.conf
+}
+
+testit "renamedc" testrenamedc
+testit "renamedc2" testrenamedc2
+
+if [ $failed -eq 0 ]; then
+	rm -rf $PREFIX/upgradeprovision_full
+fi
+
+exit $failed
diff --git a/testprogs/blackbox/test_kinit.sh b/testprogs/blackbox/test_kinit.sh
index a00f67e8c61..5ef92acae57 100755
--- a/testprogs/blackbox/test_kinit.sh
+++ b/testprogs/blackbox/test_kinit.sh
@@ -29,7 +29,7 @@ ldbmodify="$samba4bindir/ldbmodify$EXEEXT"
 ldbsearch="$samba4bindir/ldbsearch$EXEEXT"
 rkpty="$samba4bindir/rkpty$EXEEXT"
 samba4kpasswd="$samba4bindir/samba4kpasswd$EXEEXT"
-enableaccount="$samba_tool enableaccount"
+enableaccount="$samba_tool user enable"
 machineaccountccache="$samba4srcdir/scripting/bin/machineaccountccache"
 
 . `dirname $0`/subunit.sh
diff --git a/testprogs/blackbox/test_pkinit.sh b/testprogs/blackbox/test_pkinit.sh
index 8d5c7994039..55cb6e3e5d8 100755
--- a/testprogs/blackbox/test_pkinit.sh
+++ b/testprogs/blackbox/test_pkinit.sh
@@ -29,7 +29,7 @@ ldbmodify="$samba4bindir/ldbmodify$EXEEXT"
 ldbsearch="$samba4bindir/ldbsearch$EXEEXT"
 rkpty="$samba4bindir/rkpty$EXEEXT"
 samba4kpasswd="$samba4bindir/samba4kpasswd$EXEEXT"
-enableaccount="$samba_tool enableaccount"
+enableaccount="$samba_tool user enable"
 machineaccountccache="$samba4srcdir/scripting/bin/machineaccountccache"
 
 . `dirname $0`/subunit.sh
diff --git a/testsuite/build_farm/runlist b/testsuite/build_farm/runlist
index 594c172b6ff..f9456b74ec4 100644
--- a/testsuite/build_farm/runlist
+++ b/testsuite/build_farm/runlist
@@ -9,7 +9,7 @@ torture-LOCK3 torture-LOCK4 torture-LOCK5 \
 torture-LOCK6 torture-LOCK7 \
 torture-UNLINK torture-BROWSE torture-ATTR \
 torture-TRANS2 torture-TORTURE torture-OPLOCK1 \
-torture-OPLOCK3 torture-DIR torture-DIR1 torture-DENY1 \
+torture-DIR torture-DIR1 torture-DENY1 \
 torture-DENY2 torture-TCON torture-TCON2 torture-TCONDEV \
 torture-RW1 torture-RW2 torture-OPEN torture-XCOPY \
 torture-RENAME torture-DELETE torture-PROPERTIES \
diff --git a/testsuite/build_farm/torture-OPLOCK3.test b/testsuite/build_farm/torture-OPLOCK3.test
deleted file mode 100644
index f8dfb3f8e9b..00000000000
--- a/testsuite/build_farm/torture-OPLOCK3.test
+++ /dev/null
@@ -1,2 +0,0 @@
-. torture_setup.fns
-test_torture "OPLOCK3"
diff --git a/testsuite/printing/psec.c b/testsuite/printing/psec.c
index 24386149966..59205446d05 100644
--- a/testsuite/printing/psec.c
+++ b/testsuite/printing/psec.c
@@ -171,7 +171,7 @@ int psec_getsec(char *printer)
 	slprintf(tdb_path, sizeof(tdb_path) - 1, "%s/ntdrivers.tdb", 
 		 lp_lockdir());
 
-	tdb = tdb_open(tdb_path, 0, 0, O_RDONLY, 0600);
+	tdb = tdb_open_compat(tdb_path, 0, 0, O_RDONLY, 0600, NULL, NULL);
 
 	if (!tdb) {
 		printf("psec: failed to open nt drivers database: %s\n",
@@ -275,7 +275,7 @@ int psec_setsec(char *printer)
 	slprintf(tdb_path, sizeof(tdb_path) - 1, "%s/ntdrivers.tdb", 
 		 lp_lockdir());
 
-	tdb = tdb_open(tdb_path, 0, 0, O_RDWR, 0600);
+	tdb = tdb_open_compat(tdb_path, 0, 0, O_RDWR, 0600, NULL, NULL);
 
 	if (!tdb) {
 		printf("psec: failed to open nt drivers database: %s\n",
diff --git a/wscript b/wscript
index 6961eb84af7..a4fb4700872 100755
--- a/wscript
+++ b/wscript
@@ -13,16 +13,6 @@ import wafsamba, Options, samba_dist, Scripting, Utils, samba_version
 
 samba_dist.DIST_DIRS('.')
 
-#This is a list of files that we don't want in the package, for
-#whatever reason.  Directories should be listed with a trailing / to
-#avoid over-exclusion.
-
-#This list includes files that would confuse the recipient of a
-#samba-4.0.0 branded tarball (until the merge is complete) and the
-#core elements of the autotools build system (which is known to
-#produce buggy binaries).
-samba_dist.DIST_BLACKLIST('README Manifest Read-Manifest-Now Roadmap ' +
-                          'packaging/ docs-xml/ examples/ swat/ WHATSNEW.txt MAINTAINERS ')
 # install in /usr/local/samba by default
 Options.default_prefix = '/usr/local/samba'
 
@@ -32,7 +22,7 @@ def set_options(opt):
     opt.BUILTIN_DEFAULT('NONE')
     opt.PRIVATE_EXTENSION_DEFAULT('samba4')
     opt.RECURSE('lib/replace')
-    opt.RECURSE('source4/dynconfig')
+    opt.RECURSE('dynconfig')
     opt.RECURSE('source4/lib/ldb')
     opt.RECURSE('selftest')
     opt.RECURSE('source4/lib/tls')
@@ -47,10 +37,6 @@ def set_options(opt):
                    help='enable special build farm options',
                    action='store_true', dest='BUILD_FARM')
 
-    gr.add_option('--enable-s3build',
-                   help='enable build of s3 binaries',
-                   action='store_true', dest='S3BUILD')
-
     opt.tool_options('python') # options for disabling pyc or pyo compilation
     # enable options related to building python extensions
 
@@ -60,19 +46,17 @@ def configure(conf):
     version = samba_version.load_version(env=conf.env)
 
     conf.DEFINE('CONFIG_H_IS_FROM_SAMBA', 1)
+    conf.DEFINE('_SAMBA_WAF_BUILD_', version.MAJOR)
     conf.DEFINE('_SAMBA_BUILD_', version.MAJOR, add_to_cflags=True)
     conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
 
     if Options.options.developer:
         conf.ADD_CFLAGS('-DDEVELOPER -DDEBUG_PASSWORD')
 
-    if Options.options.S3BUILD:
-        conf.env.enable_s3build = True
-
     # this enables smbtorture.static for s3 in the build farm
     conf.env.BUILD_FARM = Options.options.BUILD_FARM or os.environ.get('RUN_FROM_BUILD_FARM')
 
-    conf.ADD_EXTRA_INCLUDES('#include/public #source4 #lib #source4/lib #source4/include #include')
+    conf.ADD_EXTRA_INCLUDES('#include/public #source4 #lib #source4/lib #source4/include #include #lib/replace')
 
     conf.RECURSE('lib/replace')
 
@@ -95,12 +79,13 @@ def configure(conf):
     if int(conf.env['PYTHON_VERSION'][0]) >= 3:
         raise Utils.WafError('Python version 3.x is not supported by Samba yet')
 
-    conf.RECURSE('source4/dynconfig')
+    conf.RECURSE('dynconfig')
     conf.RECURSE('source4/lib/ldb')
     conf.RECURSE('source4/heimdal_build')
     conf.RECURSE('source4/lib/tls')
     conf.RECURSE('source4/ntvfs/sysdep')
     conf.RECURSE('lib/util')
+    conf.RECURSE('lib/ccan')
     conf.RECURSE('lib/zlib')
     conf.RECURSE('lib/util/charset')
     conf.RECURSE('source4/auth')
@@ -113,20 +98,29 @@ def configure(conf):
     conf.RECURSE('libcli/smbreadline')
     conf.RECURSE('pidl')
     conf.RECURSE('selftest')
-    if conf.env.enable_s3build:
-        conf.RECURSE('source3')
+    conf.RECURSE('source3')
 
     # we don't want any libraries or modules to rely on runtime
     # resolution of symbols
     if sys.platform != "openbsd4":
         conf.env.undefined_ldflags = conf.ADD_LDFLAGS('-Wl,-no-undefined', testflags=True)
 
+    if sys.platform != "openbsd4" and conf.env.undefined_ignore_ldflags == []:
+        if conf.CHECK_LDFLAGS(['-undefined', 'dynamic_lookup']):
+            conf.env.undefined_ignore_ldflags = ['-undefined', 'dynamic_lookup']
+
+
     # gentoo always adds this. We want our normal build to be as
     # strict as the strictest OS we support, so adding this here
     # allows us to find problems on our development hosts faster.
     # It also results in faster load time.
-    if sys.platform != "openbsd4":
-        conf.env.asneeded_ldflags = conf.ADD_LDFLAGS('-Wl,--as-needed', testflags=True)
+
+    # However, until the source3 waf build settles down, this needs to
+    # be disabled, as the bugs mentioned above are hitting too many of
+    # our users
+
+    #if sys.platform != "openbsd4":
+    #    conf.env.asneeded_ldflags = conf.ADD_LDFLAGS('-Wl,--as-needed', testflags=True)
 
     if not conf.CHECK_NEED_LC("-lc not needed"):
         conf.ADD_LDFLAGS('-lc', testflags=False)
@@ -159,7 +153,7 @@ def ctags(ctx):
 # of commands in --help
 def build(bld):
     '''build all targets'''
-    samba_version.load_version(env=bld.env)
+    samba_version.load_version(env=bld.env, is_install=bld.is_install)
     pass
 
 
diff --git a/wscript_build b/wscript_build
index d955f90dc2c..aa37c4bbc70 100644
--- a/wscript_build
+++ b/wscript_build
@@ -27,17 +27,19 @@ bld.env.public_headers_skip = ['param/param_proto.h', 'lib/ldb_compat.h']
 # force headers to use SAMBA4 rules
 bld.env.public_headers_replace = { '#if _SAMBA_BUILD_ == 4' : '#if 1 /* _SAMBA_BUILD_ == 4 */' }
 
-samba_version.load_version(bld.env)
+samba_version.load_version(bld.env, is_install=bld.is_install)
 bld.SAMBA_MKVERSION('version.h')
 
 # bld.ENABLE_MAGIC_ORDERING()
 
 bld.RECURSE('lib/replace')
+bld.RECURSE('lib/socket')
 bld.RECURSE('lib/talloc')
-bld.RECURSE('lib/tdb')
 bld.RECURSE('lib/tevent')
+bld.RECURSE('lib/ccan')
+bld.RECURSE('lib/tdb_compat')
 bld.RECURSE('source4/lib/ldb')
-bld.RECURSE('source4/dynconfig')
+bld.RECURSE('dynconfig')
 bld.RECURSE('lib/util/charset')
 bld.RECURSE('source4/scripting/python')
 bld.RECURSE('source4/param')
@@ -49,6 +51,7 @@ bld.RECURSE('source4/smbd')
 bld.RECURSE('source4/libnet')
 bld.RECURSE('source4/auth')
 bld.RECURSE('auth')
+bld.RECURSE('auth/kerberos')
 bld.RECURSE('lib/iniparser/src')
 bld.RECURSE('nsswitch')
 bld.RECURSE('nsswitch/libwbclient')
@@ -93,6 +96,7 @@ bld.RECURSE('librpc')
 bld.RECURSE('source4/client')
 bld.RECURSE('source4/libcli')
 bld.RECURSE('libcli/smb')
+bld.RECURSE('libcli/util')
 bld.RECURSE('libcli/cldap')
 bld.RECURSE('lib/subunit/c')
 bld.RECURSE('source4/kdc')
@@ -117,12 +121,11 @@ bld.RECURSE('source4/scripting')
 bld.RECURSE('pidl')
 bld.RECURSE('lib')
 bld.RECURSE('libds/common')
-if bld.env.enable_s3build:
-    bld.RECURSE('source3')
+bld.RECURSE('source3')
 
 bld.RECURSE('testsuite/headers')
 
 # install some extra empty directories
-bld.INSTALL_DIRS("", "${LOCKDIR} ${SYSCONFDIR} ${LOCKDIR} ${PIDDIR} ${LOCALSTATEDIR}/lib ${PRIVATEDIR}/smbd.tmp/messaging")
+bld.INSTALL_DIRS("", "${LOCKDIR} ${STATEDIR} ${CACHEDIR} ${SWATDIR} ${SYSCONFDIR} ${PIDDIR} ${LOCALSTATEDIR}/lib ${PRIVATEDIR}/smbd.tmp/messaging ${CODEPAGEDIR} ${LOGFILEBASE} ${SETUPDIR}")
 
 bld.SYMBOL_CHECK()