lib/util/charset: add back loading of charset modules

For autoconf builds these remain as modules, for waf builds they are
built into the charset library.

This is required to provide the CP850 charset when iconv is not available.

The charset modules static for the waf builds because with proper
shared libs, there isn't the same need for these to be in seperate .so
files.  The modules are also not produced if a system iconv is found,
except for developers, to allow testing of both code paths.

Andrew Bartlett

3 files changed, 76 insertions, 39 deletions

diff --git a/lib/util/charset/iconv.c b/lib/util/charset/iconv.c
index f63c4e6028d..fd8b8ae8eef 100644
--- a/lib/util/charset/iconv.c
+++ b/lib/util/charset/iconv.c
@@ -244,7 +244,7 @@ static int smb_iconv_t_destructor(smb_iconv_t hwd)
 }
 
 _PUBLIC_ smb_iconv_t smb_iconv_open_ex(TALLOC_CTX *mem_ctx, const char *tocode, 
-			      const char *fromcode, bool native_iconv)
+			      const char *fromcode, bool allow_native_iconv)
 {
 	smb_iconv_t ret;
 	const struct charset_functions *from=NULL, *to=NULL;
@@ -268,6 +268,7 @@ _PUBLIC_ smb_iconv_t smb_iconv_open_ex(TALLOC_CTX *mem_ctx, const char *tocode,
 		return ret;
 	}
 
+	/* check if we have a builtin function for this conversion */
 	for (i=0;i<ARRAY_SIZE(builtin_functions);i++) {
 		if (strcasecmp(fromcode, builtin_functions[i].name) == 0) {
 			from = &builtin_functions[i];
@@ -277,43 +278,68 @@ _PUBLIC_ smb_iconv_t smb_iconv_open_ex(TALLOC_CTX *mem_ctx, const char *tocode,
 		}
 	}
 
-	if (from == NULL) {
-		for (from=charsets; from; from=from->next) {
-			if (strcasecmp(from->name, fromcode) == 0) break;
-		}
-	}
-
-	if (to == NULL) {
-		for (to=charsets; to; to=to->next) {
-			if (strcasecmp(to->name, tocode) == 0) break;
-		}
-	}
-
 #ifdef HAVE_NATIVE_ICONV
-	if ((!from || !to) && !native_iconv) {
-		goto failed;
-	}
-	if (!from) {
-		ret->pull = sys_iconv;
+	/* the from and to varaibles indicate a samba module or
+	 * internal conversion, ret->pull and ret->push are
+	 * initialised only in this block for iconv based
+	 * conversions */
+
+	if (allow_native_iconv && from == NULL) {
 		ret->cd_pull = iconv_open("UTF-16LE", fromcode);
 		if (ret->cd_pull == (iconv_t)-1)
 			ret->cd_pull = iconv_open("UCS-2LE", fromcode);
-		if (ret->cd_pull == (iconv_t)-1) goto failed;
+		if (ret->cd_pull != (iconv_t)-1) {
+			ret->pull = sys_iconv;
+		}
 	}
-
-	if (!to) {
-		ret->push = sys_iconv;
+	
+	if (allow_native_iconv && to == NULL) {
 		ret->cd_push = iconv_open(tocode, "UTF-16LE");
 		if (ret->cd_push == (iconv_t)-1)
 			ret->cd_push = iconv_open(tocode, "UCS-2LE");
-		if (ret->cd_push == (iconv_t)-1) goto failed;
+		if (ret->cd_push != (iconv_t)-1) {
+			ret->push = sys_iconv;
+		}
 	}
-#else
-	if (!from || !to) {
-		goto failed;
+#endif
+
+	/* If iconv was unable to provide the conversion, or if use of
+	 * it was disabled, and it wasn't a builtin charset, try a
+	 * module */
+	if (ret->pull == NULL && from == NULL) {
+		from = find_charset_functions(fromcode);
+	}
+	
+	if (ret->push == NULL && to == NULL) {
+		to = find_charset_functions(tocode);
+	}
+
+	/* In the WAF builds, all charset modules are linked in at compile
+	 * time, as we have shared libs.  Using run-time loading as well will
+	 * cause dependency loops.  For the autoconf build, try loading from a module */
+#ifndef _SAMBA_WAF_BUILD_
+	/* check if there is a module available that can do this conversion */
+	if (from == NULL && ret->pull == NULL && NT_STATUS_IS_OK(smb_probe_module("charset", fromcode))) {
+		if (!(from = find_charset_functions(fromcode))) {
+			DEBUG(0, ("Module %s doesn't provide charset %s!\n", fromcode, fromcode));
+		}
+	}
+	
+	if (to == NULL && ret->push == NULL && NT_STATUS_IS_OK(smb_probe_module("charset", tocode))) {
+		if (!(to = find_charset_functions(tocode))) {
+			DEBUG(0, ("Module %s doesn't provide charset %s!\n", tocode, tocode));
+		}
 	}
 #endif
 
+	if (ret->pull == NULL && from == NULL) {
+		goto failed;
+	}
+	
+	if (ret->push == NULL && to == NULL) {
+		goto failed;
+	}
+
 	/* check for conversion to/from ucs2 */
 	if (is_utf16(fromcode) && to) {
 		ret->direct = to->push;
diff --git a/lib/util/charset/wscript_build b/lib/util/charset/wscript_build
index 7a9918046df..d659c7a4504 100644
--- a/lib/util/charset/wscript_build
+++ b/lib/util/charset/wscript_build
@@ -10,35 +10,39 @@ bld.SAMBA_SUBSYSTEM('charset',
                     deps='DYNCONFIG ICONV_WRAPPER',
                     public_deps='talloc')
 
+# In the WAF builds, all charset modules are linked in at compile
+# time, as we have shared libs.  Using run-time loading as well will
+# cause dependency loops
+
 bld.SAMBA_MODULE('charset_weird',
                  subsystem='charset',
                  source='weird.c',
-                 init_function='',
+                 init_function='charset_weird_init',
                  deps='samba-util',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_weird'),
+                 internal_module=True,
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_weird'))
 
 bld.SAMBA_MODULE('charset_CP850',
                  subsystem='charset',
                  source='CP850.c',
-                 init_function='',
+                 init_function='charset_CP850_init',
                  deps='samba-util',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_CP850'),
+                 internal_module=True,
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_CP850'))
 
 bld.SAMBA_MODULE('charset_CP437',
                  subsystem='charset',
                  source='CP437.c',
-                 init_function='',
+                 init_function='charset_CP437_init',
                  deps='samba-util',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_CP437'),
+                 internal_module=True,
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_CP437'))
 
 bld.SAMBA_MODULE('charset_macosxfs',
                  subsystem='charset',
                  source='charset_macosxfs.c',
-                 init_function='',
-                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('charset_macosxfs'),
+                 init_function='charset_macosxfs_init',
+                 internal_module=True,
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('charset_macosxfs'))
 
 
diff --git a/source3/wscript b/source3/wscript
index f7c12a83bf0..165146cf6f2 100644
--- a/source3/wscript
+++ b/source3/wscript
@@ -125,6 +125,7 @@ def configure(conf):
         conf.RECURSE('../lib/zlib')
         conf.RECURSE('../libcli/smbreadline')
         conf.RECURSE('../lib/util')
+        conf.RECURSE('../lib/util/charset')
 
         conf.ADD_EXTRA_INCLUDES('''#source3 #source3/include #lib/replace #lib''')
         if not conf.env.USING_SYSTEM_TDB:
@@ -1713,6 +1714,9 @@ main() {
     if conf.CHECK_HEADERS('gpfs_gpl.h'):
         conf.DEFINE('HAVE_GPFS', '1')
 
+    # Note that all charset 'modules' must actually be static, due to dependency loop issues 
+    # if we include the module loader in iconv
+
     default_static_modules=TO_LIST('''pdb_smbpasswd pdb_tdbsam pdb_wbc_sam
                                       auth_sam auth_unix auth_winbind auth_wbc auth_server
                                       auth_domain auth_builtin vfs_default
@@ -1721,16 +1725,19 @@ main() {
 
     default_shared_modules=TO_LIST('''vfs_recycle vfs_audit vfs_extd_audit vfs_full_audit vfs_netatalk
                                       vfs_fake_perms vfs_default_quota vfs_readonly vfs_cap
-                                      vfs_expand_msdfs vfs_shadow_copy vfs_shadow_copy2 charset_CP850
-                                      charset_CP437 auth_script vfs_readahead vfs_xattr_tdb
+                                      vfs_expand_msdfs vfs_shadow_copy vfs_shadow_copy2 
+                                      auth_script vfs_readahead vfs_xattr_tdb
                                       vfs_streams_xattr vfs_streams_depot vfs_acl_xattr vfs_acl_tdb
                                       vfs_smb_traffic_analyzer vfs_preopen vfs_catia vfs_scannedonly
                                       vfs_crossrename vfs_linux_xfs_sgid
                                       vfs_time_audit idmap_autorid''')
 
     if Options.options.developer:
-        default_static_modules.extend(TO_LIST('pdb_ads auth_netlogond'))
-        default_shared_modules.extend(TO_LIST('charset_weird perfcount_test'))
+        default_static_modules.extend(TO_LIST('pdb_ads auth_netlogond charset_weird'))
+        default_shared_modules.extend(TO_LIST('perfcount_test'))
+
+    if Options.options.developer or not conf.CONFIG_SET('HAVE_NATIVE_ICONV'):
+        default_static_modules.extend(TO_LIST('charset_CP850 charset_CP437'))
 
     if conf.env.toplevel_build:
         default_static_modules.extend(TO_LIST('pdb_samba4 auth_samba4'))
@@ -1754,7 +1761,7 @@ main() {
         default_static_modules.extend(TO_LIST('pdb_ldap idmap_ldap'))
 
     if conf.CONFIG_SET('DARWINOS'):
-	default_shared_modules.extend(TO_LIST('charset_macosxfs'))
+	default_static_modules.extend(TO_LIST('charset_macosxfs'))
 
     if conf.CONFIG_SET('HAVE_GPFS'):
 	default_shared_modules.extend(TO_LIST('vfs_gpfs vfs_gpfs_hsm_notify'))