tests: Add a test of non-ASCII character fidelity on various filesystem types (RHBZ#823887).

Thanks Laszlo Ersek.
(cherry picked from commit 8e8a5764293d1b6bc3dbfb9409722e4c13384aec)

5 files changed, 494 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index c17b54fe..66d49d94 100644
--- a/.gitignore
+++ b/.gitignore
@@ -384,6 +384,7 @@ pod2htm?.tmp
 /tests/c-api/tests.c
 /tests/c-api/test*.tmp
 /tests/c-api/test-user-cancel
+/tests/charsets/test-charset-fidelity
 /tests/data/100kallnewlines
 /tests/data/100kallspaces
 /tests/data/100kallzeroes
diff --git a/Makefile.am b/Makefile.am
index 0ddf13f1..87d2d422 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -43,6 +43,7 @@ SUBDIRS += tests/luks
 SUBDIRS += tests/md
 SUBDIRS += tests/ntfsclone
 SUBDIRS += tests/btrfs
+SUBDIRS += tests/charsets
 SUBDIRS += tests/xml
 SUBDIRS += tests/regressions
 endif
diff --git a/configure.ac b/configure.ac
index 5a969c28..ad1d7637 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1365,6 +1365,7 @@ AC_CONFIG_FILES([Makefile
                  test-tool/Makefile
                  tests/btrfs/Makefile
                  tests/c-api/Makefile
+                 tests/charsets/Makefile
                  tests/data/Makefile
                  tests/extra/Makefile
                  tests/guests/Makefile
diff --git a/tests/charsets/Makefile.am b/tests/charsets/Makefile.am
new file mode 100644
index 00000000..d18cb61a
--- /dev/null
+++ b/tests/charsets/Makefile.am
@@ -0,0 +1,38 @@
+# libguestfs
+# Copyright (C) 2009-2012 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+include $(top_srcdir)/subdir-rules.mk
+
+TESTS = \
+	test-charset-fidelity
+
+random_val := $(shell awk 'BEGIN{srand(); print 1+int(255*rand())}' < /dev/null)
+
+TESTS_ENVIRONMENT = \
+	MALLOC_PERTURB_=$(random_val) \
+	LIBGUESTFS_PATH=$(top_builddir)/appliance \
+	TMPDIR=$(top_builddir) \
+	$(VG)
+
+check_PROGRAMS = $(TESTS)
+
+test_charset_fidelity_SOURCES = test-charset-fidelity.c
+test_charset_fidelity_CFLAGS = \
+	-I$(top_srcdir)/src -I$(top_builddir)/src \
+	$(WARN_CFLAGS) $(WERROR_CFLAGS)
+test_charset_fidelity_LDADD = \
+	$(top_builddir)/src/libguestfs.la
diff --git a/tests/charsets/test-charset-fidelity.c b/tests/charsets/test-charset-fidelity.c
new file mode 100644
index 00000000..7332c542
--- /dev/null
+++ b/tests/charsets/test-charset-fidelity.c
@@ -0,0 +1,453 @@
+/* libguestfs
+ * Copyright (C) 2012 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/* Test fidelity of filenames on various filesystems.
+ * See RHBZ#823885 and RHBZ#823887.
+ * Thanks to Laszlo Ersek for suggestions for this test.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <error.h>
+
+#include "guestfs.h"
+
+#define STREQ(a,b) (strcmp((a),(b)) == 0)
+#define STRNEQ(a,b) (strcmp((a),(b)) != 0)
+#define STRCASENEQ(a,b) (strcasecmp((a),(b)) != 0)
+
+struct filesystem {
+  const char *fs_name;          /* Name of filesystem. */
+  int fs_case_insensitive;      /* True if filesystem is case insensitive. */
+  int fs_8bit_only;             /* True if fs only supports 8 bit chars. */
+  const char *fs_mount_options; /* Mount options, if required. */
+  const char *fs_feature;       /* Feature test, if required. */
+
+  /* Note these skip options indicate BUGS in the filesystems (not
+   * in libguestfs).  The filesystems should be able to pass these
+   * tests if they are working correctly.
+   */
+  int fs_skip_latin1;           /* Skip latin1 test. */
+  int fs_skip_latin2;           /* Skip latin2 test. */
+};
+
+static struct filesystem filesystems[] = {
+  { "ext2",  0, 0, NULL, NULL, 0, 0 },
+  { "ext3",  0, 0, NULL, NULL, 0, 0 },
+  { "ext4",  0, 0, NULL, NULL, 0, 0 },
+  { "btrfs", 0, 0, NULL, "btrfs", 0, 0 },
+  { "vfat",  1, 0, "utf8", NULL, 1, 1 },
+  { "msdos", 1, 1, NULL, NULL, 0, 0 },
+  /* In reality NTFS is case insensitive, but the ntfs-3g driver isn't. */
+  { "ntfs",  0, 0, NULL, "ntfs3g", 0, 0 },
+};
+
+static void test_filesystem (guestfs_h *g, const struct filesystem *fs);
+static void make_filesystem (guestfs_h *g, const struct filesystem *fs);
+static void mount_filesystem (guestfs_h *g, const struct filesystem *fs);
+static void unmount_filesystem (guestfs_h *g, const struct filesystem *fs);
+static void test_ascii (guestfs_h *g, const struct filesystem *fs);
+static void test_latin1 (guestfs_h *g, const struct filesystem *fs);
+static void test_latin2 (guestfs_h *g, const struct filesystem *fs);
+static void test_chinese (guestfs_h *g, const struct filesystem *fs);
+static void ignore_lost_and_found (char **);
+static size_t count_strings (char **);
+static int feature_available (guestfs_h *g, const char *feature);
+
+int
+main (int argc, char *argv[])
+{
+  guestfs_h *g;
+  char tmp[] = "/tmp/charsetXXXXXX";
+  int fd;
+  size_t i;
+  struct filesystem *fs;
+
+  g = guestfs_create ();
+  if (g == NULL)
+    error (EXIT_FAILURE, 0, "failed to create handle");
+
+  fd = mkstemp (tmp);
+  if (fd == -1)
+    error (EXIT_FAILURE, errno, "mkstemp");
+
+  if (ftruncate (fd, 1024 * 1024 * 1024) == -1)
+    error (EXIT_FAILURE, errno, "ftruncate: %s", tmp);
+
+  if (close (fd) == -1)
+    error (EXIT_FAILURE, errno, "close: %s", tmp);
+
+  if (guestfs_add_drive_opts (g, tmp, -1) == -1)
+    exit (EXIT_FAILURE);
+
+  if (guestfs_launch (g) == -1)
+    exit (EXIT_FAILURE);
+
+  if (guestfs_part_disk (g, "/dev/sda", "mbr") == -1)
+    exit (EXIT_FAILURE);
+
+  for (i = 0; i < sizeof filesystems / sizeof filesystems[0]; ++i) {
+    fs = &filesystems[i];
+    test_filesystem (g, fs);
+  }
+
+  guestfs_close (g);
+  unlink (tmp);
+
+  exit (EXIT_SUCCESS);
+}
+
+/* This function coordinates the test for each filesystem type. */
+static void
+test_filesystem (guestfs_h *g, const struct filesystem *fs)
+{
+  if (fs->fs_feature && !feature_available (g, fs->fs_feature)) {
+    printf ("skipped test of %s because %s feature not available\n",
+            fs->fs_name, fs->fs_feature);
+    return;
+  }
+
+  printf ("testing charset fidelity on %s\n", fs->fs_name);
+
+  make_filesystem (g, fs);
+  mount_filesystem (g, fs);
+
+  test_ascii (g, fs);
+
+  if (fs->fs_8bit_only)
+    goto out;
+
+  if (!fs->fs_skip_latin1)
+    test_latin1 (g, fs);
+  if (!fs->fs_skip_latin2)
+    test_latin2 (g, fs);
+  test_chinese (g, fs);
+
+ out:
+  unmount_filesystem (g, fs);
+}
+
+static void
+make_filesystem (guestfs_h *g, const struct filesystem *fs)
+{
+  if (guestfs_mkfs (g, fs->fs_name, "/dev/sda1") == -1)
+    exit (EXIT_FAILURE);
+}
+
+static void
+mount_filesystem (guestfs_h *g, const struct filesystem *fs)
+{
+  const char *mount_options;
+
+  mount_options = fs->fs_mount_options ? : "";
+  if (guestfs_mount_options (g, mount_options, "/dev/sda1", "/") == -1)
+    exit (EXIT_FAILURE);
+}
+
+static void
+unmount_filesystem (guestfs_h *g, const struct filesystem *fs)
+{
+  if (guestfs_umount (g, "/") == -1)
+    exit (EXIT_FAILURE);
+}
+
+static void
+test_ascii (guestfs_h *g, const struct filesystem *fs)
+{
+  char **files;
+  size_t count;
+
+  /* Create various ASCII-named files. */
+  if (guestfs_touch (g, "/ABC") == -1)
+    exit (EXIT_FAILURE);
+  if (guestfs_touch (g, "/def") == -1)
+    exit (EXIT_FAILURE);
+  if (guestfs_touch (g, "/abc") == -1)
+    exit (EXIT_FAILURE);
+
+  /* Read list of files, check for case sensitivity. */
+  files = guestfs_ls (g, "/");
+  if (files == NULL)
+    exit (EXIT_FAILURE);
+  ignore_lost_and_found (files);
+  count = count_strings (files);
+
+  if (fs->fs_case_insensitive) { /* case insensitive */
+    if (count != 2)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s is supposed to be case-insensitive, but %zu files "
+             "(instead of 2) were returned",
+             __func__, fs->fs_name, count);
+
+    if (STRCASENEQ (files[0], "abc") ||
+        STRCASENEQ (files[1], "def"))
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s returned unexpected filenames '%s' and '%s'",
+             __func__, fs->fs_name, files[0], files[1]);
+  }
+  else {                        /* case sensitive */
+    if (count != 3)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s is supposed to be case-sensitive, but %zu files "
+             "(instead of 3) were returned",
+             __func__, fs->fs_name, count);
+
+    if (STRNEQ (files[0], "ABC") == -1 ||
+        STRNEQ (files[1], "abc") == -1 ||
+        STRNEQ (files[2], "def") == -1)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s returned unexpected filenames '%s', '%s', '%s'",
+             __func__, fs->fs_name, files[0], files[1], files[2]);
+
+    if (guestfs_rm (g, "/abc") == -1)
+      exit (EXIT_FAILURE);
+  }
+
+  if (guestfs_rm (g, "/ABC") == -1)
+    exit (EXIT_FAILURE);
+  if (guestfs_rm (g, "/def") == -1)
+    exit (EXIT_FAILURE);
+}
+
+/* Note: This is testing characters in the Latin1 set, but the
+ * encoding is still UTF-8 as it must be for libguestfs.
+ */
+static void
+test_latin1 (guestfs_h *g, const struct filesystem *fs)
+{
+  /* LATIN CAPITAL LETTER O WITH TILDE */
+  const char O_tilde[] = { 0xc3, 0x95, 0 };
+  const char slash_O_tilde[] = { '/', 0xc3, 0x95, 0 };
+  /* LATIN SMALL LETTER O WITH TILDE */
+  const char o_tilde[] = { 0xc3, 0xb5, 0 };
+  const char slash_o_tilde[] = { '/', 0xc3, 0xb5, 0 };
+
+  char **files;
+  size_t count;
+
+  if (guestfs_touch (g, slash_O_tilde) == -1)
+    exit (EXIT_FAILURE);
+  if (guestfs_touch (g, slash_o_tilde) == -1)
+    exit (EXIT_FAILURE);
+
+  /* Read list of files, check for case sensitivity. */
+  files = guestfs_ls (g, "/");
+  if (files == NULL)
+    exit (EXIT_FAILURE);
+  ignore_lost_and_found (files);
+  count = count_strings (files);
+
+  if (fs->fs_case_insensitive) { /* case insensitive */
+    if (count != 1)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s is supposed to be case-insensitive, but %zu files "
+             "(instead of 1) were returned",
+             __func__, fs->fs_name, count);
+
+    if (memcmp (files[0], o_tilde, 3) != 0 &&
+        memcmp (files[0], O_tilde, 3) != 0)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s returned unexpected filename '%s'",
+             __func__, fs->fs_name, files[0]);
+  }
+  else {                        /* case sensitive */
+    if (count != 2)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s is supposed to be case-sensitive, but %zu files "
+             "(instead of 2) were returned",
+             __func__, fs->fs_name, count);
+
+    if (memcmp (files[0], O_tilde, 3) != 0 ||
+        memcmp (files[1], o_tilde, 3) != 0)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s returned unexpected filenames '%s' and '%s'",
+             __func__, fs->fs_name, files[0], files[1]);
+
+    if (guestfs_rm (g, slash_O_tilde) == -1)
+      exit (EXIT_FAILURE);
+  }
+
+  if (guestfs_rm (g, slash_o_tilde) == -1)
+    exit (EXIT_FAILURE);
+}
+
+/* Note: This is testing characters in the Latin2 set, but the
+ * encoding is still UTF-8 as it must be for libguestfs.
+ */
+static void
+test_latin2 (guestfs_h *g, const struct filesystem *fs)
+{
+  /* LATIN CAPITAL LETTER O WITH DOUBLE ACUTE */
+  const char O_dacute[] = { 0xc5, 0x90, 0 };
+  const char slash_O_dacute[] = { '/', 0xc5, 0x90, 0 };
+  /* LATIN SMALL LETTER O WITH DOUBLE ACUTE */
+  const char o_dacute[] = { 0xc5, 0x91, 0 };
+  const char slash_o_dacute[] = { '/', 0xc5, 0x91, 0 };
+
+  char **files;
+  size_t count;
+
+  if (guestfs_touch (g, slash_O_dacute) == -1)
+    exit (EXIT_FAILURE);
+  if (guestfs_touch (g, slash_o_dacute) == -1)
+    exit (EXIT_FAILURE);
+
+  /* Read list of files, check for case sensitivity. */
+  files = guestfs_ls (g, "/");
+  if (files == NULL)
+    exit (EXIT_FAILURE);
+  ignore_lost_and_found (files);
+  count = count_strings (files);
+
+  if (fs->fs_case_insensitive) { /* case insensitive */
+    if (count != 1)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s is supposed to be case-insensitive, but %zu files "
+             "(instead of 1) were returned",
+             __func__, fs->fs_name, count);
+
+    if (memcmp (files[0], o_dacute, 3) != 0 &&
+        memcmp (files[0], O_dacute, 3) != 0)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s returned unexpected filename '%s'",
+             __func__, fs->fs_name, files[0]);
+  }
+  else {                        /* case sensitive */
+    if (count != 2)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s is supposed to be case-sensitive, but %zu files "
+             "(instead of 2) were returned",
+             __func__, fs->fs_name, count);
+
+    if (memcmp (files[0], O_dacute, 3) != 0 ||
+        memcmp (files[1], o_dacute, 3) != 0)
+      error (EXIT_FAILURE, 0,
+             "error: %s: %s returned unexpected filenames '%s' and '%s'",
+             __func__, fs->fs_name, files[0], files[1]);
+
+    if (guestfs_rm (g, slash_O_dacute) == -1)
+      exit (EXIT_FAILURE);
+  }
+
+  if (guestfs_rm (g, slash_o_dacute) == -1)
+    exit (EXIT_FAILURE);
+}
+
+static void
+test_chinese (guestfs_h *g, const struct filesystem *fs)
+{
+  /* Various Simplified Chinese characters from:
+   * https://secure.wikimedia.org/wikipedia/en/wiki/Chinese_characters#Comparisons_of_traditional_Chinese.2C_simplified_Chinese.2C_and_Japanese
+   */
+  char filenames[][5] = {
+    { '/', 0xe7, 0x94, 0xb5, 0 },
+    { '/', 0xe4, 0xb9, 0xb0, 0 },
+    { '/', 0xe5, 0xbc, 0x80, 0 },
+    { '/', 0xe4, 0xb8, 0x9c, 0 },
+    { '/', 0xe8, 0xbd, 0xa6, 0 },
+    { '/', 0xe7, 0xba, 0xa2, 0 },
+  };
+  const size_t nr_filenames = sizeof filenames / sizeof filenames[0];
+  size_t i, j;
+  char **files;
+  size_t count;
+
+  for (i = 0; i < nr_filenames; ++i) {
+    if (guestfs_touch (g, filenames[i]) == -1)
+      exit (EXIT_FAILURE);
+  }
+
+  /* Check the filenames. */
+  files = guestfs_ls (g, "/");
+  if (files == NULL)
+    exit (EXIT_FAILURE);
+  ignore_lost_and_found (files);
+  count = count_strings (files);
+
+  if (count != nr_filenames)
+    error (EXIT_FAILURE, 0,
+           "error: %s: %s returned unexpected number of files "
+           "(%zu, expecting %zu)",
+           __func__, fs->fs_name, count, nr_filenames);
+
+  for (j = 0; j < count; ++j) {
+    for (i = 0; i < nr_filenames; ++i)
+      if (memcmp (files[j], &filenames[i][1], 4) == 0)
+        goto next;
+    error (EXIT_FAILURE, 0,
+           "error: %s: %s returned unexpected filename '%s'",
+           __func__, fs->fs_name, files[j]);
+
+  next:;
+  }
+
+  for (i = 0; i < nr_filenames; ++i)
+    if (guestfs_rm (g, filenames[i]) == -1)
+      exit (EXIT_FAILURE);
+}
+
+/* Remove 'lost+found' and (I guess in future) other similar files
+ * from the list.
+ */
+static void
+ignore_lost_and_found (char **files)
+{
+  size_t i, j;
+
+  for (i = j = 0; files[i] != NULL; ++i) {
+    if (STREQ (files[i], "lost+found"))
+      free (files[i]);
+    else
+      files[j++] = files[i];
+  }
+  files[j] = NULL;
+}
+
+static size_t
+count_strings (char **argv)
+{
+  size_t argc;
+
+  for (argc = 0; argv[argc] != NULL; ++argc)
+    ;
+  return argc;
+}
+
+static int
+feature_available (guestfs_h *g, const char *feature)
+{
+  /* If there's an error we should ignore it, so to do that we have to
+   * temporarily replace the error handler with a null one.
+   */
+  guestfs_error_handler_cb old_error_cb;
+  void *old_error_data;
+  old_error_cb = guestfs_get_error_handler (g, &old_error_data);
+  guestfs_set_error_handler (g, NULL, NULL);
+
+  const char *groups[] = { feature, NULL };
+  int r = guestfs_available (g, (char * const *) groups);
+
+  guestfs_set_error_handler (g, old_error_cb, old_error_data);
+
+  return r == 0 ? 1 : 0;
+}