[PATCH 4/6] sandbox: libc_malloc: fail allocations exceeding MALLOC_MAX_SIZE

[Ahmad Fatoum <a.fatoum@pengutronix.de>]

The libc allocator is a bit unpredictable, because overcommit can
mean that big allocations succeed initially, only to OOM later.

Let's thus enforce a maximum allocation limit on the barebox side in
alignment with the bare metal allocators.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 arch/sandbox/os/libc_malloc.c | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/arch/sandbox/os/libc_malloc.c b/arch/sandbox/os/libc_malloc.c
index e34156cd49e7..ac97fc37eee5 100644
--- a/arch/sandbox/os/libc_malloc.c
+++ b/arch/sandbox/os/libc_malloc.c
@@ -7,6 +7,8 @@
 #include <malloc.h>
 
 #define BAREBOX_ENOMEM 12
+#define BAREBOX_MALLOC_MAX_SIZE 0x40000000
+
 extern int barebox_errno;
 
 void barebox_malloc_stats(void)
@@ -15,7 +17,10 @@ void barebox_malloc_stats(void)
 
 void *barebox_memalign(size_t alignment, size_t bytes)
 {
-	void *mem = memalign(alignment, bytes);
+	void *mem = NULL;
+
+	if (alignment <= BAREBOX_MALLOC_MAX_SIZE && bytes <= BAREBOX_MALLOC_MAX_SIZE)
+		mem = memalign(alignment, bytes);
 	if (!mem)
 		barebox_errno = BAREBOX_ENOMEM;
 
@@ -25,7 +30,10 @@ void *barebox_memalign(size_t alignment, size_t bytes)
 void *barebox_malloc(size_t size)
 {
 
-	void *mem = malloc(size);
+	void *mem = NULL;
+
+	if (size <= BAREBOX_MALLOC_MAX_SIZE)
+		mem = malloc(size);
 	if (!mem)
 		barebox_errno = BAREBOX_ENOMEM;
 
@@ -44,7 +52,10 @@ void barebox_free(void *ptr)
 
 void *barebox_realloc(void *ptr, size_t size)
 {
-	void *mem = realloc(ptr, size);
+	void *mem = NULL;
+
+	if (size <= BAREBOX_MALLOC_MAX_SIZE)
+		mem = realloc(ptr, size);
 	if (!mem)
 		barebox_errno = BAREBOX_ENOMEM;
 
@@ -53,7 +64,12 @@ void *barebox_realloc(void *ptr, size_t size)
 
 void *barebox_calloc(size_t n, size_t elem_size)
 {
-	void *mem = calloc(n, elem_size);
+	size_t product;
+	void *mem = NULL;
+
+	if (!__builtin_add_overflow(n, elem_size, &product) &&
+	    product <= BAREBOX_MALLOC_MAX_SIZE)
+		mem = calloc(n, elem_size);
 	if (!mem)
 		barebox_errno = BAREBOX_ENOMEM;
 
-- 
2.39.5