From 7fabc4531340ab5b680ed5acb954cd3dd1cf5919 Mon Sep 17 00:00:00 2001 From: Christian Prochaska Date: Mon, 23 Jul 2018 14:53:03 +0200 Subject: [PATCH] libc: return MAP_FAILED if 'mmap()' failed Fixes #2924 --- repos/libports/src/lib/libc/file_operations.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/repos/libports/src/lib/libc/file_operations.cc b/repos/libports/src/lib/libc/file_operations.cc index 46502d7a06..7ae1ea14ac 100644 --- a/repos/libports/src/lib/libc/file_operations.cc +++ b/repos/libports/src/lib/libc/file_operations.cc @@ -430,6 +430,10 @@ extern "C" void *mmap(void *addr, ::size_t length, int prot, int flags, if (!addr && libc_fd == -1) { bool const executable = prot & PROT_EXEC; void *start = Libc::mem_alloc(executable)->alloc(length, PAGE_SHIFT); + if (!start) { + errno = ENOMEM; + return MAP_FAILED; + } mmap_registry()->insert(start, length, 0); return start; } @@ -439,7 +443,7 @@ extern "C" void *mmap(void *addr, ::size_t length, int prot, int flags, if (!fd || !fd->plugin || !fd->plugin->supports_mmap()) { Genode::warning("mmap not supported for file descriptor ", libc_fd); errno = EBADF; - return (void *)INVALID_FD; + return MAP_FAILED; } void *start = fd->plugin->mmap(addr, length, prot, flags, fd, offset);