New memory allocators that exit on OOM and handle multiplication overflow

Every place that does a regular malloc/calloc and aborts on failure
should use spice_malloc/spice_mallo0 instead, which is leaner and cleaner.

Allocations of dynamically sized arrays can use g_malloc_n or g_new etc
which correctly handle multiplication overflow if some of the arguments
are not trusted.

7 files changed, 263 insertions, 0 deletions

diff --git a/client/Makefile.am b/client/Makefile.am
index d01d03c9..887c7b5a 100644
--- a/client/Makefile.am
+++ b/client/Makefile.am
@@ -46,6 +46,7 @@ RED_COMMON_SRCS =	 		\
 	lz.cpp				\
 	monitor.cpp			\
 	monitor.h			\
+	mem.cpp				\
 	menu.cpp			\
 	menu.h				\
 	pixels_source.h			\
diff --git a/client/mem.cpp b/client/mem.cpp
new file mode 100644
index 00000000..b06bb3b6
--- /dev/null
+++ b/client/mem.cpp
@@ -0,0 +1,24 @@
+/*
+   Copyright (C) 2009 Red Hat, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License as
+   published by the Free Software Foundation; either version 2 of
+   the License, or (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "common.h"
+#include "utils.h"
+
+#define MALLOC_ERROR(format, ...) THROW(format, ## __VA_ARGS__)
+
+#include "../common/mem.c"
+
diff --git a/client/windows/redc.vcproj b/client/windows/redc.vcproj
index e7c2e150..214a88df 100644
--- a/client/windows/redc.vcproj
+++ b/client/windows/redc.vcproj
@@ -265,6 +265,10 @@
 				>

 			</File>

 			<File

+				RelativePath="..\mem.cpp"

+				>

+			</File>

+			<File

 				RelativePath="..\menu.cpp"

 				>

 			</File>

diff --git a/client/x11/Makefile.am b/client/x11/Makefile.am
index 7feedeaa..e2f5c21a 100644
--- a/client/x11/Makefile.am
+++ b/client/x11/Makefile.am
@@ -69,6 +69,7 @@ RED_COMMON_SRCS = 					\
 	$(top_srcdir)/client/lines.cpp			\
 	$(top_srcdir)/client/monitor.cpp		\
 	$(top_srcdir)/client/monitor.h			\
+	$(top_srcdir)/client/mem.cpp			\
 	$(top_srcdir)/client/menu.cpp			\
 	$(top_srcdir)/client/menu.h			\
 	$(top_srcdir)/client/pixels_source.h		\
diff --git a/common/mem.c b/common/mem.c
new file mode 100644
index 00000000..ad0cccf6
--- /dev/null
+++ b/common/mem.c
@@ -0,0 +1,150 @@
+/* -*- Mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+   Copyright (C) 2010 Red Hat, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License as
+   published by the Free Software Foundation; either version 2 of
+   the License, or (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "mem.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifndef MALLOC_ERROR
+#define MALLOC_ERROR(format, ...) {                             \
+    printf(format "\n", ## __VA_ARGS__);   \
+    abort();                                                    \
+}
+#endif
+
+char *spice_strdup(const char *str)
+{
+    char *copy;
+
+    copy = (char *)spice_malloc(strlen(str) + 1);
+    strcpy(copy, str);
+    return copy;
+}
+
+void *spice_memdup(const void *mem, size_t n_bytes)
+{
+    void *copy;
+
+    copy = spice_malloc(n_bytes);
+    memcpy(copy, mem, n_bytes);
+    return copy;
+}
+
+void *spice_malloc(size_t n_bytes)
+{
+    void *mem;
+
+    if (SPICE_LIKELY(n_bytes)) {
+        mem = malloc(n_bytes);
+
+        if (SPICE_LIKELY(mem != NULL)) {
+            return mem;
+        }
+
+        MALLOC_ERROR("spice_malloc: panic: unable to allocate %lu bytes\n",
+                     (unsigned long)n_bytes);
+    }
+    return NULL;
+}
+
+void *spice_malloc0(size_t n_bytes)
+{
+    void *mem;
+
+    if (SPICE_LIKELY(n_bytes)) {
+        mem = calloc(1, n_bytes);
+
+        if (SPICE_LIKELY(mem != NULL)) {
+            return mem;
+        }
+
+        MALLOC_ERROR("spice_malloc0: panic: unable to allocate %lu bytes\n",
+                     (unsigned long)n_bytes);
+    }
+    return NULL;
+}
+
+void *spice_realloc(void *mem, size_t n_bytes)
+{
+    if (SPICE_LIKELY(n_bytes)) {
+        mem = realloc(mem, n_bytes);
+
+        if (SPICE_LIKELY(mem != NULL)) {
+            return mem;
+        }
+
+        MALLOC_ERROR("spice_realloc: panic: unable to allocate %lu bytes\n",
+                     (unsigned long)n_bytes);
+    }
+
+    if (mem) {
+        free(mem);
+    }
+
+    return NULL;
+}
+
+#define SIZE_OVERFLOWS(a,b) (SPICE_UNLIKELY ((a) > SIZE_MAX / (b)))
+
+void *spice_malloc_n(size_t n_blocks, size_t n_block_bytes)
+{
+    if (SIZE_OVERFLOWS (n_blocks, n_block_bytes)) {
+        MALLOC_ERROR("spice_malloc_n: overflow allocating %lu*%lu bytes",
+                     (unsigned long)n_blocks, (unsigned long)n_block_bytes);
+    }
+
+    return spice_malloc(n_blocks * n_block_bytes);
+}
+
+void *spice_malloc_n_m(size_t n_blocks, size_t n_block_bytes, size_t extra_size)
+{
+    size_t size1, size2;
+    if (SIZE_OVERFLOWS (n_blocks, n_block_bytes)) {
+        MALLOC_ERROR("spice_malloc_n: overflow allocating %lu*%lu + %lubytes",
+                     (unsigned long)n_blocks, (unsigned long)n_block_bytes, (unsigned long)extra_size);
+    }
+    size1 = n_blocks * n_block_bytes;
+    size2 = size1 + extra_size;
+    if (size2 < size1) {
+        MALLOC_ERROR("spice_malloc_n: overflow allocating %lu*%lu + %lubytes",
+                     (unsigned long)n_blocks, (unsigned long)n_block_bytes, (unsigned long)extra_size);
+    }
+    return spice_malloc(size2);
+}
+
+
+void *spice_malloc0_n(size_t n_blocks, size_t n_block_bytes)
+{
+    if (SIZE_OVERFLOWS (n_blocks, n_block_bytes)) {
+        MALLOC_ERROR("spice_malloc0_n: overflow allocating %lu*%lu bytes",
+                     (unsigned long)n_blocks, (unsigned long)n_block_bytes);
+    }
+
+    return spice_malloc0 (n_blocks * n_block_bytes);
+}
+
+void *spice_realloc_n(void *mem, size_t n_blocks, size_t n_block_bytes)
+{
+    if (SIZE_OVERFLOWS (n_blocks, n_block_bytes)) {
+        MALLOC_ERROR("spice_realloc_n: overflow allocating %lu*%lu bytes",
+                     (unsigned long)n_blocks, (unsigned long)n_block_bytes);
+  }
+
+    return spice_realloc(mem, n_blocks * n_block_bytes);
+}
diff --git a/common/mem.h b/common/mem.h
new file mode 100644
index 00000000..f5fab20a
--- /dev/null
+++ b/common/mem.h
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+   Copyright (C) 2010 Red Hat, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License as
+   published by the Free Software Foundation; either version 2 of
+   the License, or (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _H_MEM
+#define _H_MEM
+
+#include <stdlib.h>
+#include <spice/macros.h>
+
+char *spice_strdup(const char *str) SPICE_GNUC_MALLOC;
+void *spice_memdup(const void *mem, size_t n_bytes) SPICE_GNUC_MALLOC;
+void *spice_malloc(size_t n_bytes) SPICE_GNUC_MALLOC SPICE_GNUC_ALLOC_SIZE(1);
+void *spice_malloc0(size_t n_bytes) SPICE_GNUC_MALLOC SPICE_GNUC_ALLOC_SIZE(1);
+void *spice_realloc(void *mem, size_t n_bytes) SPICE_GNUC_WARN_UNUSED_RESULT;
+void *spice_malloc_n(size_t n_blocks, size_t n_block_bytes) SPICE_GNUC_MALLOC SPICE_GNUC_ALLOC_SIZE2(1,2);
+void *spice_malloc_n_m(size_t n_blocks, size_t n_block_bytes, size_t extra_size) SPICE_GNUC_MALLOC;
+void *spice_malloc0_n(size_t n_blocks, size_t n_block_bytes) SPICE_GNUC_MALLOC SPICE_GNUC_ALLOC_SIZE2(1,2);
+void *spice_realloc_n(void *mem, size_t n_blocks, size_t n_block_bytes) SPICE_GNUC_WARN_UNUSED_RESULT;
+
+/* Optimise: avoid the call to the (slower) _n function if we can
+ * determine at compile-time that no overflow happens.
+ */
+#if defined (__GNUC__) && (__GNUC__ >= 2) && defined (__OPTIMIZE__)
+#  define _SPICE_NEW(struct_type, n_structs, func)        \
+    (struct_type *) (__extension__ ({                     \
+        size_t __n = (size_t) (n_structs);                \
+        size_t __s = sizeof (struct_type);                \
+        void *__p;                                        \
+        if (__s == 1)                                     \
+            __p = spice_##func (__n);                     \
+        else if (__builtin_constant_p (__n) &&            \
+                 __n <= SIZE_MAX / __s)                   \
+            __p = spice_##func (__n * __s);               \
+        else                                              \
+            __p = spice_##func##_n (__n, __s);            \
+        __p;                                              \
+    }))
+#  define _SPICE_RENEW(struct_type, mem, n_structs, func) \
+    (struct_type *) (__extension__ ({                             \
+        size_t __n = (size_t) (n_structs);                \
+        size_t __s = sizeof (struct_type);                \
+        void *__p = (void *) (mem);                       \
+        if (__s == 1)                                     \
+            __p = spice_##func (__p, __n);                \
+        else if (__builtin_constant_p (__n) &&            \
+                 __n <= SIZE_MAX / __s)                   \
+            __p = spice_##func (__p, __n * __s);          \
+        else                                              \
+            __p = spice_##func##_n (__p, __n, __s);       \
+        __p;                                              \
+    }))
+#else
+
+/* Unoptimised version: always call the _n() function. */
+
+#define _SPICE_NEW(struct_type, n_structs, func)                        \
+    ((struct_type *) spice_##func##_n ((n_structs), sizeof (struct_type)))
+#define _SPICE_RENEW(struct_type, mem, n_structs, func)                 \
+    ((struct_type *) spice_##func##_n (mem, (n_structs), sizeof (struct_type)))
+
+#endif
+
+#define spice_new(struct_type, n_structs) _SPICE_NEW(struct_type, n_structs, malloc)
+#define spice_new0(struct_type, n_structs) _SPICE_NEW(struct_type, n_structs, malloc0)
+#define spice_renew(struct_type, mem, n_structs) _SPICE_RENEW(struct_type, mem, n_structs, realloc)
+
+#endif
diff --git a/server/Makefile.am b/server/Makefile.am
index f91debd1..0b6cb13d 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -30,6 +30,7 @@ COMMON_SRCS = 						\
 	$(top_srcdir)/common/quic.c			\
 	$(top_srcdir)/common/lz.c			\
 	$(top_srcdir)/common/canvas_utils.c		\
+	$(top_srcdir)/common/mem.c			\
 	$(NULL)
 
 lib_LTLIBRARIES = libspice.la