vm: fix a null dereference on out-of-memory

. also make other out-of-memory conditions less fatal
	. add a test case for a user program using all the memory
	  it can
	. remove some diagnostic prints for situations that are normal
	  when running out of memory so running the test isn't noisy

servers/pm/forkexit.c

@@ -75,7 +75,6 @@ int do_fork()
 
   /* Memory part of the forking. */
   if((s=vm_fork(rmp->mp_endpoint, next_child, &child_ep)) != OK) {
-	printf("PM: vm_fork failed: %d\n", s);
 	return s;
   }
 
@@ -176,7 +175,6 @@ int do_srv_fork()
 	panic("do_fork finds wrong child slot: %d", next_child);
 
   if((s=vm_fork(rmp->mp_endpoint, next_child, &child_ep)) != OK) {
-	printf("PM: vm_fork failed: %d\n", s);
 	return s;
   }
 

servers/vm/arch/i386/pagetable.c

@@ -478,7 +478,6 @@ int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end,
 				 * and pt_ptalloc leaves the directory
 				 * and other data in a consistent state.
 				 */
-				printf("pt_ptalloc_in_range: pt_ptalloc failed\n");
 				return r;
 			}
 		}
@@ -692,9 +691,6 @@ int pt_writemap(struct vmproc * vmp,
 		int pde = I386_VM_PDE(v);
 		int pte = I386_VM_PTE(v);
 
-		if(!v) { printf("VM: warning: making zero page for %d\n",
-			vmp->vm_endpoint); }
-
 		assert(!(v % I386_PAGE_SIZE));
 		assert(pte >= 0 && pte < I386_VM_PT_ENTRIES);
 		assert(pde >= 0 && pde < I386_VM_DIR_ENTRIES);
@@ -825,7 +821,7 @@ int pt_new(pt_t *pt)
  * its physical address as we'll need that in the future. Verify it's
  * page-aligned.
  */
-	int i;
+	int i, r;
 
 	/* Don't ever re-allocate/re-move a certain process slot's
 	 * page directory once it's been created. This is a fraction
@@ -847,8 +843,8 @@ int pt_new(pt_t *pt)
 	pt->pt_virtop = 0;
 
         /* Map in kernel. */
-        if(pt_mapkernel(pt) != OK)
-                panic("pt_new: pt_mapkernel failed");
+        if((r=pt_mapkernel(pt)) != OK)
+		return r;
 
 	return OK;
 }
@@ -1122,12 +1118,13 @@ int pt_mapkernel(pt_t *pt)
 
 	/* Kernel also wants various mappings of its own. */
 	for(i = 0; i < kernmappings; i++) {
-		if(pt_writemap(NULL, pt,
+		int r;
+		if((r=pt_writemap(NULL, pt,
 			kern_mappings[i].vir_addr,
 			kern_mappings[i].phys_addr,
 			kern_mappings[i].len,
-			kern_mappings[i].flags, 0) != OK) {
-			panic("pt_mapkernel: pt_writemap failed");
+			kern_mappings[i].flags, 0)) != OK) {
+			return r;
 		}
 	}
 

servers/vm/exit.c

@@ -120,7 +120,8 @@ int do_procctl(message *msg)
 				&& msg->m_source != VFS_PROC_NR)
 				return EPERM;
 			free_proc(vmp);
-			pt_new(&vmp->vm_pt);
+			if(pt_new(&vmp->vm_pt) != OK)
+				panic("VMPPARAM_CLEAR: pt_new failed");
 			pt_bind(&vmp->vm_pt, vmp);
 			return OK;
 		default:

servers/vm/fork.c

@@ -71,7 +71,6 @@ int do_fork(message *msg)
 #endif
 
   if(pt_new(&vmc->vm_pt) != OK) {
-	printf("VM: fork: pt_new failed\n");
 	return ENOMEM;
   }
 

servers/vm/region.c

@@ -431,9 +431,6 @@ static vir_bytes region_find_slot_range(struct vmproc *vmp,
 	}
 
 	if(!foundflag) {
-		printf("VM: region_find_slot: no 0x%lx bytes found for %d between 0x%lx and 0x%lx\n",
-			length, vmp->vm_endpoint, minv, maxv);
-		util_stacktrace();
 		return SLOT_FAIL;
 	}
 
@@ -537,8 +534,12 @@ mem_type_t *memtype;
 	}
 
 	/* If a new event is specified, invoke it. */
-	if(newregion->memtype->ev_new)
-		newregion->memtype->ev_new(newregion);
+	if(newregion->memtype->ev_new) {
+		if(newregion->memtype->ev_new(newregion) != OK) {
+			/* ev_new will have freed and removed the region */
+			return NULL;
+		}
+	}
 
 	if(mapflags & MF_PREALLOC) {
 		if(map_handle_memory(vmp, newregion, 0, length, 1) != OK) {

test/Makefile

@@ -23,7 +23,7 @@ OBJS.test57=test57loop.o
  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 \
 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 \
 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 \
-61 62
+61 62 64
 PROG+= test$(t)
 .endfor
   

test/run

@@ -14,7 +14,7 @@ badones=			# list of tests that failed
 tests="   1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 \
          21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 \
          41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 \
-         61 62 63 \
+         61 62 63 64 \
 	 sh1.sh sh2.sh interp.sh"
 tests_no=`expr 0`
 

test/test64.c

@@ -0,0 +1,48 @@
+
+/* Code to test reasonable response to out-of-memory condition
+ * of regular processes.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <dlfcn.h>
+
+#include <sys/mman.h>
+#include <sys/wait.h>
+
+#define MAX_ERROR 2
+
+#include "magic.h"
+#include "common.c"
+
+int main (int argc, char *argv[])
+{
+  pid_t f;
+  start(64);
+#define NADDRS 500
+#define LEN 4096
+  static void *addrs[NADDRS];
+  int i = 0;
+  int st;
+
+  if((f=fork()) == -1) {
+  	e(1);
+	exit(1);
+  }
+
+  if(f == 0) {
+  	/* child: use up as much memory as we can */
+	while((addrs[i++ % NADDRS] = minix_mmap(0, LEN, PROT_READ|PROT_WRITE,
+		MAP_PREALLOC|MAP_CONTIG|MAP_ANON, -1, 0)) != MAP_FAILED)
+			;
+	exit(0);
+  }
+
+  /* parent: wait for child */
+  if(waitpid(f, &st, 0) < 0) 
+	perror("waitpid");
+
+  quit();
+
+  return(0);
+}