VM: simplify slab allocator

. only keep a list of non-empty, non-full pages with slab objects
	. simplifies alloc/free operations and reduces list management overhead

servers/vm/arch/i386/pagetable.c

@@ -185,11 +185,9 @@ static u32_t findhole(void)
 /*===========================================================================*
  *				vm_freepages		     		     *
  *===========================================================================*/
-static void vm_freepages(vir_bytes vir, vir_bytes phys, int pages, int reason)
+void vm_freepages(vir_bytes vir, int pages)
 {
-	assert(reason >= 0 && reason < VMP_CATEGORIES);
 	assert(!(vir % I386_PAGE_SIZE)); 
-	assert(!(phys % I386_PAGE_SIZE)); 
 	extern char _end;	
 
 	if(vir < (vir_bytes) &_end) {
@@ -197,9 +195,9 @@ static void vm_freepages(vir_bytes vir, vir_bytes phys, int pages, int reason)
 		return;
 	}
 
-	free_mem(ABS2CLICK(phys), pages);
 	if(pt_writemap(vmprocess, &vmprocess->vm_pt, vir,
-		MAP_NONE, pages*I386_PAGE_SIZE, 0, WMF_OVERWRITE) != OK)
+		MAP_NONE, pages*I386_PAGE_SIZE, 0,
+		WMF_OVERWRITE | WMF_FREE) != OK)
 		panic("vm_freepages: pt_writemap failed");
 
 #if SANITYCHECKS
@@ -1080,8 +1078,7 @@ void pt_free(pt_t *pt)
 
 	for(i = 0; i < I386_VM_DIR_ENTRIES; i++)
 		if(pt->pt_pt[i])
-			vm_freepages((vir_bytes) pt->pt_pt[i],
+			vm_freepages((vir_bytes) pt->pt_pt[i], 1);
-				I386_VM_PFA(pt->pt_dir[i]), 1, VMP_PAGETABLE);
 
 	return;
 }

servers/vm/proto.h

@@ -82,6 +82,7 @@ int handle_memory(struct vmproc *vmp, vir_bytes mem, vir_bytes len, int
 
 /* $(ARCH)/pagetable.c */
 void pt_init();
+void vm_freepages(vir_bytes vir, int pages);
 void pt_init_mem(void);
 void pt_check(struct vmproc *vmp);
 int pt_new(pt_t *pt);

servers/vm/slaballoc.c

@@ -23,6 +23,8 @@
 
 #include <memory.h>
 
+#include <sys/param.h>
+
 #include "glo.h"
 #include "proto.h"
 #include "util.h"
@@ -72,6 +74,8 @@
 #define SETBIT(f, b)   {OFF(f,b); SLABDATAUSE(f, BITEL(f,b)|= BITPAT(b); (f)->sdh.nused++;); }
 #define CLEARBIT(f, b) {ON(f, b); SLABDATAUSE(f, BITEL(f,b)&=~BITPAT(b); (f)->sdh.nused--; (f)->sdh.freeguess = (b);); }
 
+#define OBJALIGN	8
+
 #define MINSIZE 8
 #define MAXSIZE (SLABSIZES-1+MINSIZE)
 #define USEELEMENTS (1+(VM_PAGE_SIZE/MINSIZE/8))
@@ -97,8 +101,6 @@ struct sdh {
 #if SANITYCHECKS
 	u32_t magic1;
 #endif
-	u8_t list;
-	u16_t nused;	/* Number of data items used in this slab. */
 	int freeguess;
 	struct slabdata *next, *prev;
 	elements_t usebits;
@@ -107,6 +109,7 @@ struct sdh {
 	int writable;	/* data item number or WRITABLE_* */
 	u32_t magic2;
 #endif
+	u16_t nused;	/* Number of data items used in this slab. */
 };
 
 #define DATABYTES	(VM_PAGE_SIZE-sizeof(struct sdh))
@@ -116,17 +119,11 @@ struct sdh {
 #define JUNK  0xdeadbeef
 #define NOJUNK 0xc0ffee
 
-#define LIST_UNUSED	0
-#define LIST_FREE	1
-#define LIST_USED	2
-#define LIST_FULL	3
-#define LIST_NUMBER	4
-
 static struct slabheader {
 	struct slabdata {
-		struct	sdh sdh;
 		u8_t 	data[DATABYTES];
-	} *list_head[LIST_NUMBER];
+		struct	sdh sdh;
+	} *list_head;
 } slabs[SLABSIZES];
 
 static int objstats(void *, int, struct slabheader **, struct slabdata
@@ -141,32 +138,12 @@ static int objstats(void *, int, struct slabheader **, struct slabdata
 	s = &slabs[i];			\
 }
 
-#define LH(sl, l) (sl)->list_head[l]
-
-/* move head of list l1 to list of l2 in slabheader sl. */
-#define MOVEHEAD(sl, l1, l2) {		\
-	struct slabdata *t;		\
-	assert(LH(sl,l1));		\
-	REMOVEHEAD(sl, l1, t);		\
-	ADDHEAD(t, sl, l2);		\
-}
-
-/* remove head of list 'list' in sl, assign it unlinked to 'to'. */
-#define REMOVEHEAD(sl, list, to) {	\
-	struct slabdata *dat;		\
-	dat = (to) = LH(sl, list);	\
-	assert(dat);			\
-	LH(sl, list) = dat->sdh.next;	\
-	UNLINKNODE(dat);		\
-}
-
 /* move slabdata nw to slabheader sl under list number l. */
-#define ADDHEAD(nw, sl, l) {			\
+#define ADDHEAD(nw, sl) {			\
 	SLABDATAUSE(nw,				\
-		(nw)->sdh.next = LH(sl, l);	\
+		(nw)->sdh.next = sl->list_head;	\
-		(nw)->sdh.prev = NULL;		\
+		(nw)->sdh.prev = NULL;);	\
-		(nw)->sdh.list = l;);		\
+	sl->list_head = nw;			\
-	LH(sl, l) = (nw);			\
 	if((nw)->sdh.next) {			\
 		SLABDATAUSE((nw)->sdh.next, \
 			(nw)->sdh.next->sdh.prev = (nw););	\
@@ -181,7 +158,7 @@ static int objstats(void *, int, struct slabheader **, struct slabdata
 	if(next) { SLABDATAUSE(next, next->sdh.prev = prev;); }	\
 }
 
-struct slabdata *newslabdata(int list)
+static struct slabdata *newslabdata()
 {
 	struct slabdata *n;
 	phys_bytes p;
@@ -202,7 +179,6 @@ struct slabdata *newslabdata(int list)
 #endif
 	n->sdh.nused = 0;
 	n->sdh.freeguess = 0;
-	n->sdh.list = list;
 
 #if SANITYCHECKS
 	n->sdh.writable = WRITABLE_HEADER;
@@ -218,9 +194,9 @@ struct slabdata *newslabdata(int list)
  *				checklist				     *
  *===========================================================================*/
 static int checklist(char *file, int line,
-	struct slabheader *s, int l, int bytes)
+	struct slabheader *s, int bytes)
 {
-	struct slabdata *n = s->list_head[l];
+	struct slabdata *n = s->list_head;
 	int ch = 0;
 
 	while(n) {
@@ -258,9 +234,7 @@ void slab_sanitycheck(char *file, int line)
 	int s;
 	for(s = 0; s < SLABSIZES; s++) {
 		int l;
-		for(l = 0; l < LIST_NUMBER; l++) {
+		checklist(file, line, &slabs[s], s + MINSIZE);
-			checklist(file, line, &slabs[s], l, s + MINSIZE);
-		}
 	}
 }
 
@@ -289,7 +263,10 @@ void *slaballoc(int bytes)
 	int i;
 	int count = 0;
 	struct slabheader *s;
-	struct slabdata *firstused;
+	struct slabdata *newslab;
+	char *ret;
+
+	bytes = roundup(bytes, OBJALIGN);
 
 	SLABSANITYCHECK(SCL_FUNCTIONS);
 
@@ -297,53 +274,42 @@ void *slaballoc(int bytes)
 	GETSLAB(bytes, s);
 	assert(s);
 
-	/* To make the common case more common, make space in the 'used'
+	if(!(newslab = s->list_head)) {
-	 * queue first.
-	 */
-	if(!LH(s, LIST_USED)) {
 		/* Make sure there is something on the freelist. */
-	SLABSANITYCHECK(SCL_DETAIL);
+		newslab = newslabdata();
-		if(!LH(s, LIST_FREE)) {
+		if(!newslab) return NULL;
-			struct slabdata *nd = newslabdata(LIST_FREE);
+		ADDHEAD(newslab, s);
-	SLABSANITYCHECK(SCL_DETAIL);
+		assert(newslab->sdh.nused == 0);
-			if(!nd) return NULL;
+	} else	assert(newslab->sdh.nused > 0);
-			ADDHEAD(nd, s, LIST_FREE);
+	assert(newslab->sdh.nused < ITEMSPERPAGE(bytes));
-	SLABSANITYCHECK(SCL_DETAIL);
-		}
-
 
-	SLABSANITYCHECK(SCL_DETAIL);
-		MOVEHEAD(s, LIST_FREE, LIST_USED);
 	SLABSANITYCHECK(SCL_DETAIL);
 
-	}
-	SLABSANITYCHECK(SCL_DETAIL);
-
-	assert(s);
-	firstused = LH(s, LIST_USED);
-	assert(firstused);
 #if SANITYCHECKS
-	assert(firstused->sdh.magic1 == MAGIC1);
+	assert(newslab->sdh.magic1 == MAGIC1);
-	assert(firstused->sdh.magic2 == MAGIC2);
+	assert(newslab->sdh.magic2 == MAGIC2);
 #endif
-	assert(firstused->sdh.nused < ITEMSPERPAGE(bytes));
 
-	for(i = firstused->sdh.freeguess;
+	for(i = newslab->sdh.freeguess;
 		count < ITEMSPERPAGE(bytes); count++, i++) {
-	SLABSANITYCHECK(SCL_DETAIL);
 		i = i % ITEMSPERPAGE(bytes);
 
-		if(!GETBIT(firstused, i)) {
+		if(!GETBIT(newslab, i)) 
-			char *ret;
+			break;
-			SETBIT(firstused, i);
-	SLABSANITYCHECK(SCL_DETAIL);
-			if(firstused->sdh.nused == ITEMSPERPAGE(bytes)) {
-	SLABSANITYCHECK(SCL_DETAIL);
-				MOVEHEAD(s, LIST_USED, LIST_FULL);
-	SLABSANITYCHECK(SCL_DETAIL);
 	}
-	SLABSANITYCHECK(SCL_DETAIL);
+
-			ret = ((char *) firstused->data) + i*bytes;
+	SLABSANITYCHECK(SCL_FUNCTIONS);
+
+	assert(count < ITEMSPERPAGE(bytes));
+	assert(i >= 0 && i < ITEMSPERPAGE(bytes));
+
+	SETBIT(newslab, i);
+	if(newslab->sdh.nused == ITEMSPERPAGE(bytes)) {
+		UNLINKNODE(newslab);
+		s->list_head = newslab->sdh.next;
+	}
+
+	ret = ((char *) newslab) + i*bytes;
 
 #if SANITYCHECKS
 #if MEMPROTECT
@@ -358,31 +324,22 @@ void *slaballoc(int bytes)
 #endif
 #endif
 
-			SLABSANITYCHECK(SCL_FUNCTIONS);
+	SLABDATAUSE(newslab, newslab->sdh.freeguess = i+1;);
-			SLABDATAUSE(firstused, firstused->sdh.freeguess = i+1;);
 
 #if SANITYCHECKS
 	if(bytes >= SLABSIZES+MINSIZE) {
 		printf("slaballoc: odd, bytes %d?\n", bytes);
 	}
+
 	if(!slabsane_f(__FILE__, __LINE__, ret, bytes))
 		panic("slaballoc: slabsane failed");
 #endif
 
+	assert(!((vir_bytes) ret % OBJALIGN));
+
 	return ret;
 }
 
-	SLABSANITYCHECK(SCL_DETAIL);
-
-	}
-	SLABSANITYCHECK(SCL_FUNCTIONS);
-
-	panic("slaballoc: no space in 'used' slabdata");
-
-	/* Not reached. */
-	return NULL;
-}
-
 /*===========================================================================*
  *				int objstats				     *
  *===========================================================================*/
@@ -404,6 +361,8 @@ static inline int objstats(void *mem, int bytes,
 	struct slabdata *f;
 	int i;
 
+	assert(!(bytes % OBJALIGN));
+
 	OBJSTATSCHECK((char *) mem >= (char *) VM_PAGE_SIZE);
 
 #if SANITYCHECKS
@@ -422,7 +381,6 @@ static inline int objstats(void *mem, int bytes,
 	OBJSTATSCHECK(f->sdh.magic1 == MAGIC1);
 	OBJSTATSCHECK(f->sdh.magic2 == MAGIC2);
 #endif
-	OBJSTATSCHECK(f->sdh.list == LIST_USED || f->sdh.list == LIST_FULL);
 
 	/* Make sure it's in range. */
 	OBJSTATSCHECK((char *) mem >= (char *) f->data);
@@ -453,6 +411,8 @@ void slabfree(void *mem, int bytes)
 	struct slabheader *s;
 	struct slabdata *f;
 
+	bytes = roundup(bytes, OBJALIGN);
+
 	SLABSANITYCHECK(SCL_FUNCTIONS);
 
 	if(objstats(mem, bytes, &s, &f, &i) != OK) {
@@ -486,24 +446,12 @@ void slabfree(void *mem, int bytes)
 
 	/* Check if this slab changes lists. */
 	if(f->sdh.nused == 0) {
-		/* Now become FREE; must've been USED */
-		assert(f->sdh.list == LIST_USED);
 		UNLINKNODE(f);
-		if(f == LH(s, LIST_USED))
+		if(f == s->list_head) s->list_head = f->sdh.next;
-			LH(s, LIST_USED) = f->sdh.next;
+		vm_freepages((vir_bytes) f, 1);
-		ADDHEAD(f, s, LIST_FREE);
 		SLABSANITYCHECK(SCL_DETAIL);
 	} else if(f->sdh.nused == ITEMSPERPAGE(bytes)-1) {
-		/* Now become USED; must've been FULL */
+		ADDHEAD(f, s);
-		assert(f->sdh.list == LIST_FULL);
-		UNLINKNODE(f);
-		if(f == LH(s, LIST_FULL))
-			LH(s, LIST_FULL) = f->sdh.next;
-		ADDHEAD(f, s, LIST_USED);
-		SLABSANITYCHECK(SCL_DETAIL);
-	} else {
-		/* Stay USED */
-		assert(f->sdh.list == LIST_USED);
 	}
 
 	SLABSANITYCHECK(SCL_FUNCTIONS);
@@ -520,6 +468,8 @@ void slablock(void *mem, int bytes)
 	struct slabheader *s;
 	struct slabdata *f;
 
+	bytes = roundup(bytes, OBJALIGN);
+
 	if(objstats(mem, bytes, &s, &f, &i) != OK)
 		panic("slablock objstats failed");
 
@@ -537,6 +487,8 @@ void slabunlock(void *mem, int bytes)
 	struct slabheader *s;
 	struct slabdata *f;
 
+	bytes = roundup(bytes, OBJALIGN);
+
 	if(objstats(mem, bytes, &s, &f, &i) != OK)
 		panic("slabunlock objstats failed");
 
@@ -557,10 +509,9 @@ void slabstats(void)
 	if(n%1000) return;
 	for(s = 0; s < SLABSIZES; s++) {
 		int l;
-		for(l = 0; l < LIST_NUMBER; l++) {
 		int b, t;
 		b = s + MINSIZE;
-			t = checklist(__FILE__, __LINE__, &slabs[s], l, b);
+		t = checklist(__FILE__, __LINE__, &slabs[s], b);
 
 		if(t > 0) {
 			int bytes = t * b;
@@ -568,7 +519,6 @@ void slabstats(void)
 			totalbytes += bytes;
 		}
 	}
-	}
 
 	if(pages > 0) {
 		printf("VMSTATS: %dK net used in slab objects in %d pages (%dkB): %d%% utilization\n",