unix: check for mmap error and act accordingly

At startup, a large chunk of virtual memory is used up by the heap. This
works fine in emulation (qemu-arm), but doesn't work so well on an
actual Raspberry Pi. Therefore, this commit reduces the requested amount
until a heap size is found that works on the system.

This can certainly be improved, but for now it's an important fix
because it allows TinyGo built binaries to actually run on a Raspberry
Pi with just 1GB RAM.

src/runtime/runtime_unix.go

@@ -16,8 +16,9 @@ func usleep(usec uint) int
 //export malloc
 func malloc(size uintptr) unsafe.Pointer
 
+// void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset);
 //export mmap
-func mmap(addr unsafe.Pointer, length, prot, flags, fd int, offset int) unsafe.Pointer
+func mmap(addr unsafe.Pointer, length uintptr, prot, flags, fd int, offset int) unsafe.Pointer
 
 //export abort
 func abort()

src/runtime/runtime_unix_heap.go

@@ -5,8 +5,10 @@
 
 package runtime
 
-var heapSize uintptr = 128 * 1024          // small amount to start
-const heapMaxSize = 1 * 1024 * 1024 * 1024 // 1GB for the entire heap
+import "unsafe"
+
+var heapSize uintptr = 128 * 1024 // small amount to start
+var heapMaxSize uintptr
 
 var heapStart, heapEnd uintptr
 
@@ -14,9 +16,21 @@ func preinit() {
 	// Allocate a large chunk of virtual memory. Because it is virtual, it won't
 	// really be allocated in RAM. Memory will only be allocated when it is
 	// first touched.
-	addr := mmap(nil, heapMaxSize, flag_PROT_READ|flag_PROT_WRITE, flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0)
-	heapStart = uintptr(addr)
-	heapEnd = heapStart + heapSize
+	heapMaxSize = 1 * 1024 * 1024 * 1024 // 1GB for the entire heap
+	for {
+		addr := mmap(nil, heapMaxSize, flag_PROT_READ|flag_PROT_WRITE, flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0)
+		if addr == unsafe.Pointer(^uintptr(0)) {
+			// Heap was too big to be mapped by mmap. Reduce the maximum size.
+			// We might want to make this a bit smarter than simply halving the
+			// heap size.
+			// This can happen on 32-bit systems.
+			heapMaxSize /= 2
+			continue
+		}
+		heapStart = uintptr(addr)
+		heapEnd = heapStart + heapSize
+		break
+	}
 }
 
 // growHeap tries to grow the heap size. It returns true if it succeeds, false