minix/servers/vm/slaballoc.c
Ben Gras 2dd02cc560 mark pages whose refcount were >1 and drop to 1 and are
read/write writable in the pagetable right away instead of waiting for
a pagefault. minor optimization.

some a sanity check of SLAB-allocated pointers.

vm gets its own _exit and __exit like PM, so the stock (library) panic works.
2009-04-22 12:39:29 +00:00

456 lines
11 KiB
C

#define _SYSTEM 1
#include <minix/callnr.h>
#include <minix/com.h>
#include <minix/config.h>
#include <minix/const.h>
#include <minix/ds.h>
#include <minix/endpoint.h>
#include <minix/keymap.h>
#include <minix/minlib.h>
#include <minix/type.h>
#include <minix/ipc.h>
#include <minix/sysutil.h>
#include <minix/syslib.h>
#include <errno.h>
#include <string.h>
#include <env.h>
#include <memory.h>
#include "glo.h"
#include "proto.h"
#include "util.h"
#include "sanitycheck.h"
#define SLABSIZES 60
#define ITEMSPERPAGE(s, bytes) (DATABYTES / (bytes))
#define ELBITS (sizeof(element_t)*8)
#define BITPAT(b) (1UL << ((b) % ELBITS))
#define BITEL(f, b) (f)->sdh.usebits[(b)/ELBITS]
#define OFF(f, b) vm_assert(!GETBIT(f, b))
#define ON(f, b) vm_assert(GETBIT(f, b))
#define GETBIT(f, b) (BITEL(f,b) & BITPAT(b))
#define SETBIT(f, b) {OFF(f,b); (BITEL(f,b)|= BITPAT(b)); (f)->sdh.nused++; }
#define CLEARBIT(f, b) {ON(f, b); (BITEL(f,b)&=~BITPAT(b)); (f)->sdh.nused--; (f)->sdh.freeguess = (b); }
#define MINSIZE 8
#define MAXSIZE (SLABSIZES-1+MINSIZE)
#define USEELEMENTS (1+(VM_PAGE_SIZE/MINSIZE/8))
PRIVATE int pages = 0;
typedef u8_t element_t;
#define BITS_FULL (~(element_t)0)
typedef element_t elements_t[USEELEMENTS];
/* This file is too low-level to have global SANITYCHECKs everywhere,
* as the (other) data structures are often necessarily in an
* inconsistent state during a slaballoc() / slabfree(). So only do
* our own sanity checks here, with SLABSANITYCHECK.
*/
#if SANITYCHECKS
#define SLABSANITYCHECK(l) if((l) <= vm_sanitychecklevel) { \
slab_sanitycheck(__FILE__, __LINE__); }
#else
#define SLABSANITYCHECK(l)
#endif
struct sdh {
u8_t list;
u16_t nused; /* Number of data items used in this slab. */
#if SANITYCHECKS
u32_t magic;
#endif
int freeguess;
struct slabdata *next, *prev;
elements_t usebits;
phys_bytes phys;
};
#define DATABYTES (VM_PAGE_SIZE-sizeof(struct sdh))
#define MAGIC 0x1f5b842f
#define JUNK 0xdeadbeef
#define NOJUNK 0xc0ffee
#define LIST_UNUSED 0
#define LIST_FREE 1
#define LIST_USED 2
#define LIST_FULL 3
#define LIST_NUMBER 4
PRIVATE struct slabheader {
struct slabdata {
struct sdh sdh;
u8_t data[DATABYTES];
} *list_head[LIST_NUMBER];
} slabs[SLABSIZES];
FORWARD _PROTOTYPE( int objstats, (void *, int, struct slabheader **, struct slabdata **, int *));
#define GETSLAB(b, s) { \
int i; \
vm_assert((b) >= MINSIZE); \
i = (b) - MINSIZE; \
vm_assert((i) < SLABSIZES); \
vm_assert((i) >= 0); \
s = &slabs[i]; \
}
#define LH(sl, l) (sl)->list_head[l]
#define MOVEHEAD(sl, l1, l2) { \
struct slabdata *t; \
vm_assert(LH(sl,l1)); \
REMOVEHEAD(sl, l1, t); \
ADDHEAD(t, sl, l2); \
}
#define REMOVEHEAD(sl, list, to) { \
(to) = LH(sl, list); \
vm_assert(to); \
LH(sl, list) = (to)->sdh.next; \
if(LH(sl, list)) LH(sl, list) = NULL; \
vm_assert((to)->sdh.magic == MAGIC);\
vm_assert(!(to)->sdh.prev); \
}
#define ADDHEAD(nw, sl, l) { \
vm_assert((nw)->sdh.magic == MAGIC); \
(nw)->sdh.next = LH(sl, l); \
(nw)->sdh.prev = NULL; \
(nw)->sdh.list = l; \
LH(sl, l) = (nw); \
if((nw)->sdh.next) (nw)->sdh.next->sdh.prev = (nw); \
}
#define UNLINKNODE(n) { \
if((f)->sdh.prev) (f)->sdh.prev->sdh.next = (f)->sdh.next; \
if((f)->sdh.next) (f)->sdh.next->sdh.prev = (f)->sdh.prev; \
}
struct slabdata *newslabdata(int list)
{
struct slabdata *n;
phys_bytes p;
vm_assert(sizeof(*n) == VM_PAGE_SIZE);
if(!(n = vm_allocpages(&p, 1, VMP_SLAB)))
return NULL;
memset(n->sdh.usebits, 0, sizeof(n->sdh.usebits));
pages++;
n->sdh.phys = p;
#if SANITYCHECKS
n->sdh.magic = MAGIC;
#endif
n->sdh.nused = 0;
n->sdh.freeguess = 0;
n->sdh.list = list;
return n;
}
#if SANITYCHECKS
/*===========================================================================*
* checklist *
*===========================================================================*/
PRIVATE int checklist(char *file, int line,
struct slabheader *s, int l, int bytes)
{
struct slabdata *n = s->list_head[l];
int ch = 0;
while(n) {
int count = 0, i;
MYASSERT(n->sdh.list == l);
MYASSERT(n->sdh.magic == MAGIC);
if(n->sdh.prev)
MYASSERT(n->sdh.prev->sdh.next == n);
else
MYASSERT(s->list_head[l] == n);
if(n->sdh.next) MYASSERT(n->sdh.next->sdh.prev == n);
for(i = 0; i < USEELEMENTS*8; i++)
if(i >= ITEMSPERPAGE(s, bytes))
MYASSERT(!GETBIT(n, i));
else
if(GETBIT(n,i))
count++;
MYASSERT(count == n->sdh.nused);
ch += count;
n = n->sdh.next;
}
return ch;
}
/*===========================================================================*
* void slab_sanitycheck *
*===========================================================================*/
PUBLIC void slab_sanitycheck(char *file, int line)
{
int s;
for(s = 0; s < SLABSIZES; s++) {
int l;
for(l = 0; l < LIST_NUMBER; l++) {
checklist(file, line, &slabs[s], l, s + MINSIZE);
}
}
}
/*===========================================================================*
* int slabsane *
*===========================================================================*/
PUBLIC int slabsane(void *mem, int bytes)
{
struct slabheader *s;
struct slabdata *f;
int i;
return (objstats(mem, bytes, &s, &f, &i) == OK);
}
#endif
/*===========================================================================*
* void *slaballoc *
*===========================================================================*/
PUBLIC void *slaballoc(int bytes)
{
int i, n = 0;
struct slabheader *s;
struct slabdata *firstused;
SLABSANITYCHECK(SCL_FUNCTIONS);
/* Retrieve entry in slabs[]. */
GETSLAB(bytes, s);
vm_assert(s);
/* To make the common case more common, make space in the 'used'
* queue first.
*/
if(!LH(s, LIST_USED)) {
/* Make sure there is something on the freelist. */
SLABSANITYCHECK(SCL_DETAIL);
if(!LH(s, LIST_FREE)) {
struct slabdata *n = newslabdata(LIST_FREE);
SLABSANITYCHECK(SCL_DETAIL);
if(!n) return NULL;
ADDHEAD(n, s, LIST_FREE);
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_DETAIL);
MOVEHEAD(s, LIST_FREE, LIST_USED);
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_DETAIL);
vm_assert(s);
firstused = LH(s, LIST_USED);
vm_assert(firstused);
vm_assert(firstused->sdh.magic == MAGIC);
for(i = firstused->sdh.freeguess; n < ITEMSPERPAGE(s, bytes); n++, i++) {
SLABSANITYCHECK(SCL_DETAIL);
i = i % ITEMSPERPAGE(s, bytes);
if(!GETBIT(firstused, i)) {
struct slabdata *f;
char *ret;
SETBIT(firstused, i);
SLABSANITYCHECK(SCL_DETAIL);
if(firstused->sdh.nused == ITEMSPERPAGE(s, bytes)) {
SLABSANITYCHECK(SCL_DETAIL);
MOVEHEAD(s, LIST_USED, LIST_FULL);
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_DETAIL);
ret = ((char *) firstused->data) + i*bytes;
#if SANITYCHECKS
f = (struct slabdata *) ((char *) ret - (vir_bytes) ret % VM_PAGE_SIZE);
if(f->sdh.magic != MAGIC) {
printf("slaballoc bogus pointer 0x%lx, "
"rounded 0x%lx, bad magic 0x%lx\n",
ret, f, f->sdh.magic);
vm_panic("slaballoc check failed", NO_NUM);
}
*(u32_t *) ret = NOJUNK;
#endif
SLABSANITYCHECK(SCL_FUNCTIONS);
firstused->sdh.freeguess = i+1;
#if SANITYCHECKS
if(!slabsane(ret, bytes))
vm_panic("slaballoc: slabsane failed", NO_NUM);
#endif
return ret;
}
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_FUNCTIONS);
vm_panic("slaballoc: no space in 'used' slabdata", NO_NUM);
/* Not reached. */
return NULL;
}
/*===========================================================================*
* int objstats *
*===========================================================================*/
PRIVATE int objstats(void *mem, int bytes,
struct slabheader **sp, struct slabdata **fp, int *ip)
{
#define OBJSTATSCHECK(cond) \
if(!(cond)) { \
printf("VM:objstats: %s failed for ptr 0x%p, %d bytes\n", \
#cond, mem, bytes); \
return EINVAL; \
}
struct slabheader *s;
struct slabdata *f;
int i;
OBJSTATSCHECK((char *) mem >= (char *) VM_PAGE_SIZE);
#if SANITYCHECKS
if(*(u32_t *) mem == JUNK) {
util_stacktrace();
printf("VM: WARNING: JUNK seen in slab object\n");
}
#endif
/* Retrieve entry in slabs[]. */
GETSLAB(bytes, s);
/* Round address down to VM_PAGE_SIZE boundary to get header. */
f = (struct slabdata *) ((char *) mem - (vir_bytes) mem % VM_PAGE_SIZE);
#if SANITYCHECKS
OBJSTATSCHECK(f->sdh.magic == MAGIC);
#endif
OBJSTATSCHECK(f->sdh.list == LIST_USED || f->sdh.list == LIST_FULL);
/* Make sure it's in range. */
OBJSTATSCHECK((char *) mem >= (char *) f->data);
OBJSTATSCHECK((char *) mem < (char *) f->data + sizeof(f->data));
/* Get position. */
i = (char *) mem - (char *) f->data;
OBJSTATSCHECK(!(i % bytes));
i = i / bytes;
/* Make sure it is marked as allocated. */
OBJSTATSCHECK(GETBIT(f, i));
/* return values */
*ip = i;
*fp = f;
*sp = s;
return OK;
}
/*===========================================================================*
* void *slabfree *
*===========================================================================*/
PUBLIC void slabfree(void *mem, int bytes)
{
int i;
struct slabheader *s;
struct slabdata *f;
SLABSANITYCHECK(SCL_FUNCTIONS);
#if SANITYCHECKS
if(*(u32_t *) mem == JUNK) {
printf("VM: WARNING: likely double free, JUNK seen\n");
}
#endif
if(objstats(mem, bytes, &s, &f, &i) != OK) {
vm_panic("slabfree objstats failed", NO_NUM);
}
/* Free this data. */
CLEARBIT(f, i);
#if SANITYCHECKS
*(u32_t *) mem = JUNK;
#endif
/* Check if this slab changes lists. */
if(f->sdh.nused == 0) {
/* Now become FREE; must've been USED */
vm_assert(f->sdh.list == LIST_USED);
UNLINKNODE(f);
if(f == LH(s, LIST_USED))
LH(s, LIST_USED) = f->sdh.next;
ADDHEAD(f, s, LIST_FREE);
SLABSANITYCHECK(SCL_DETAIL);
} else if(f->sdh.nused == ITEMSPERPAGE(s, bytes)-1) {
/* Now become USED; must've been FULL */
vm_assert(f->sdh.list == LIST_FULL);
UNLINKNODE(f);
if(f == LH(s, LIST_FULL))
LH(s, LIST_FULL) = f->sdh.next;
ADDHEAD(f, s, LIST_USED);
SLABSANITYCHECK(SCL_DETAIL);
} else {
/* Stay USED */
vm_assert(f->sdh.list == LIST_USED);
}
SLABSANITYCHECK(SCL_FUNCTIONS);
return;
}
#if SANITYCHECKS
/*===========================================================================*
* void slabstats *
*===========================================================================*/
PUBLIC void slabstats(void)
{
int s, total = 0, totalbytes = 0;
static int n;
n++;
if(n%1000) return;
for(s = 0; s < SLABSIZES; s++) {
int l;
for(l = 0; l < LIST_NUMBER; l++) {
int b, t;
b = s + MINSIZE;
t = checklist(__FILE__, __LINE__, &slabs[s], l, b);
if(t > 0) {
int bytes = t * b;
printf("VMSTATS: %2d slabs: %d (%dkB)\n", b, t, bytes/1024);
totalbytes += bytes;
}
}
}
if(pages > 0) {
printf("VMSTATS: %dK net used in slab objects in %d pages (%dkB): %d%% utilization\n",
totalbytes/1024, pages, pages*VM_PAGE_SIZE/1024,
100 * totalbytes / (pages*VM_PAGE_SIZE));
}
}
#endif