minix/servers/vm/slaballoc.c

559 lines
13 KiB
C

#define _SYSTEM 1
#include <minix/callnr.h>
#include <minix/com.h>
#include <minix/config.h>
#include <minix/const.h>
#include <minix/ds.h>
#include <minix/endpoint.h>
#include <minix/keymap.h>
#include <minix/minlib.h>
#include <minix/type.h>
#include <minix/ipc.h>
#include <minix/sysutil.h>
#include <minix/syslib.h>
#include <minix/bitmap.h>
#include <minix/debug.h>
#include <assert.h>
#include <errno.h>
#include <string.h>
#include <env.h>
#include <memory.h>
#include "glo.h"
#include "proto.h"
#include "util.h"
#include "sanitycheck.h"
#define SLABSIZES 60
#define ITEMSPERPAGE(bytes) (DATABYTES / (bytes))
#define ELBITS (sizeof(element_t)*8)
#define BITPAT(b) (1UL << ((b) % ELBITS))
#define BITEL(f, b) (f)->sdh.usebits[(b)/ELBITS]
#define OFF(f, b) assert(!GETBIT(f, b))
#define ON(f, b) assert(GETBIT(f, b))
#if SANITYCHECKS
#define SLABDATAWRITABLE(data, wr) do { \
assert(data->sdh.writable == WRITABLE_NONE); \
assert(wr != WRITABLE_NONE); \
vm_pagelock(data, 0); \
data->sdh.writable = wr; \
} while(0)
#define SLABDATAUNWRITABLE(data) do { \
assert(data->sdh.writable != WRITABLE_NONE); \
data->sdh.writable = WRITABLE_NONE; \
vm_pagelock(data, 1); \
} while(0)
#define SLABDATAUSE(data, code) do { \
SLABDATAWRITABLE(data, WRITABLE_HEADER); \
code \
SLABDATAUNWRITABLE(data); \
} while(0)
#else
#define SLABDATAWRITABLE(data, wr)
#define SLABDATAUNWRITABLE(data)
#define SLABDATAUSE(data, code) do { code } while(0)
#endif
#define GETBIT(f, b) (BITEL(f,b) & BITPAT(b))
#define SETBIT(f, b) {OFF(f,b); SLABDATAUSE(f, BITEL(f,b)|= BITPAT(b); (f)->sdh.nused++;); }
#define CLEARBIT(f, b) {ON(f, b); SLABDATAUSE(f, BITEL(f,b)&=~BITPAT(b); (f)->sdh.nused--; (f)->sdh.freeguess = (b);); }
#define MINSIZE 8
#define MAXSIZE (SLABSIZES-1+MINSIZE)
#define USEELEMENTS (1+(VM_PAGE_SIZE/MINSIZE/8))
PRIVATE int pages = 0;
typedef u8_t element_t;
#define BITS_FULL (~(element_t)0)
typedef element_t elements_t[USEELEMENTS];
/* This file is too low-level to have global SANITYCHECKs everywhere,
* as the (other) data structures are often necessarily in an
* inconsistent state during a slaballoc() / slabfree(). So only do
* our own sanity checks here, with SLABSANITYCHECK.
*/
/* Special writable values. */
#define WRITABLE_NONE -2
#define WRITABLE_HEADER -1
struct sdh {
#if SANITYCHECKS
u32_t magic1;
#endif
u8_t list;
u16_t nused; /* Number of data items used in this slab. */
int freeguess;
struct slabdata *next, *prev;
elements_t usebits;
phys_bytes phys;
#if SANITYCHECKS
int writable; /* data item number or WRITABLE_* */
u32_t magic2;
#endif
};
#define DATABYTES (VM_PAGE_SIZE-sizeof(struct sdh))
#define MAGIC1 0x1f5b842f
#define MAGIC2 0x8bb5a420
#define JUNK 0xdeadbeef
#define NOJUNK 0xc0ffee
#define LIST_UNUSED 0
#define LIST_FREE 1
#define LIST_USED 2
#define LIST_FULL 3
#define LIST_NUMBER 4
PRIVATE struct slabheader {
struct slabdata {
struct sdh sdh;
u8_t data[DATABYTES];
} *list_head[LIST_NUMBER];
} slabs[SLABSIZES];
FORWARD _PROTOTYPE( int objstats, (void *, int, struct slabheader **, struct slabdata **, int *));
#define GETSLAB(b, s) { \
int i; \
assert((b) >= MINSIZE); \
i = (b) - MINSIZE; \
assert((i) < SLABSIZES); \
assert((i) >= 0); \
s = &slabs[i]; \
}
#define LH(sl, l) (sl)->list_head[l]
/* move head of list l1 to list of l2 in slabheader sl. */
#define MOVEHEAD(sl, l1, l2) { \
struct slabdata *t; \
assert(LH(sl,l1)); \
REMOVEHEAD(sl, l1, t); \
ADDHEAD(t, sl, l2); \
}
/* remove head of list 'list' in sl, assign it unlinked to 'to'. */
#define REMOVEHEAD(sl, list, to) { \
struct slabdata *dat; \
dat = (to) = LH(sl, list); \
assert(dat); \
LH(sl, list) = dat->sdh.next; \
UNLINKNODE(dat); \
}
/* move slabdata nw to slabheader sl under list number l. */
#define ADDHEAD(nw, sl, l) { \
SLABDATAUSE(nw, \
(nw)->sdh.next = LH(sl, l); \
(nw)->sdh.prev = NULL; \
(nw)->sdh.list = l;); \
LH(sl, l) = (nw); \
if((nw)->sdh.next) { \
SLABDATAUSE((nw)->sdh.next, \
(nw)->sdh.next->sdh.prev = (nw);); \
} \
}
#define UNLINKNODE(node) { \
struct slabdata *next, *prev; \
prev = (node)->sdh.prev; \
next = (node)->sdh.next; \
if(prev) { SLABDATAUSE(prev, prev->sdh.next = next;); } \
if(next) { SLABDATAUSE(next, next->sdh.prev = prev;); } \
}
struct slabdata *newslabdata(int list)
{
struct slabdata *n;
phys_bytes p;
assert(sizeof(*n) == VM_PAGE_SIZE);
if(!(n = vm_allocpage(&p, VMP_SLAB))) {
printf("newslabdata: vm_allocpage failed\n");
return NULL;
}
memset(n->sdh.usebits, 0, sizeof(n->sdh.usebits));
pages++;
n->sdh.phys = p;
#if SANITYCHECKS
n->sdh.magic1 = MAGIC1;
n->sdh.magic2 = MAGIC2;
#endif
n->sdh.nused = 0;
n->sdh.freeguess = 0;
n->sdh.list = list;
#if SANITYCHECKS
n->sdh.writable = WRITABLE_HEADER;
SLABDATAUNWRITABLE(n);
#endif
return n;
}
#if SANITYCHECKS
/*===========================================================================*
* checklist *
*===========================================================================*/
PRIVATE int checklist(char *file, int line,
struct slabheader *s, int l, int bytes)
{
struct slabdata *n = s->list_head[l];
int ch = 0;
while(n) {
int count = 0, i;
MYASSERT(n->sdh.magic1 == MAGIC1);
MYASSERT(n->sdh.magic2 == MAGIC2);
MYASSERT(n->sdh.list == l);
MYASSERT(usedpages_add(n->sdh.phys, VM_PAGE_SIZE) == OK);
if(n->sdh.prev)
MYASSERT(n->sdh.prev->sdh.next == n);
else
MYASSERT(s->list_head[l] == n);
if(n->sdh.next) MYASSERT(n->sdh.next->sdh.prev == n);
for(i = 0; i < USEELEMENTS*8; i++)
if(i >= ITEMSPERPAGE(bytes))
MYASSERT(!GETBIT(n, i));
else
if(GETBIT(n,i))
count++;
MYASSERT(count == n->sdh.nused);
ch += count;
n = n->sdh.next;
}
return ch;
}
/*===========================================================================*
* void slab_sanitycheck *
*===========================================================================*/
PUBLIC void slab_sanitycheck(char *file, int line)
{
int s;
for(s = 0; s < SLABSIZES; s++) {
int l;
for(l = 0; l < LIST_NUMBER; l++) {
checklist(file, line, &slabs[s], l, s + MINSIZE);
}
}
}
/*===========================================================================*
* int slabsane *
*===========================================================================*/
PUBLIC int slabsane_f(char *file, int line, void *mem, int bytes)
{
struct slabheader *s;
struct slabdata *f;
int i;
return (objstats(mem, bytes, &s, &f, &i) == OK);
}
#endif
static int nojunkwarning = 0;
/*===========================================================================*
* void *slaballoc *
*===========================================================================*/
PUBLIC void *slaballoc(int bytes)
{
int i;
int count = 0;
struct slabheader *s;
struct slabdata *firstused;
SLABSANITYCHECK(SCL_FUNCTIONS);
/* Retrieve entry in slabs[]. */
GETSLAB(bytes, s);
assert(s);
/* To make the common case more common, make space in the 'used'
* queue first.
*/
if(!LH(s, LIST_USED)) {
/* Make sure there is something on the freelist. */
SLABSANITYCHECK(SCL_DETAIL);
if(!LH(s, LIST_FREE)) {
struct slabdata *nd = newslabdata(LIST_FREE);
SLABSANITYCHECK(SCL_DETAIL);
if(!nd) return NULL;
ADDHEAD(nd, s, LIST_FREE);
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_DETAIL);
MOVEHEAD(s, LIST_FREE, LIST_USED);
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_DETAIL);
assert(s);
firstused = LH(s, LIST_USED);
assert(firstused);
#if SANITYCHECKS
assert(firstused->sdh.magic1 == MAGIC1);
assert(firstused->sdh.magic2 == MAGIC2);
#endif
assert(firstused->sdh.nused < ITEMSPERPAGE(bytes));
for(i = firstused->sdh.freeguess;
count < ITEMSPERPAGE(bytes); count++, i++) {
SLABSANITYCHECK(SCL_DETAIL);
i = i % ITEMSPERPAGE(bytes);
if(!GETBIT(firstused, i)) {
struct slabdata *f;
char *ret;
SETBIT(firstused, i);
SLABSANITYCHECK(SCL_DETAIL);
if(firstused->sdh.nused == ITEMSPERPAGE(bytes)) {
SLABSANITYCHECK(SCL_DETAIL);
MOVEHEAD(s, LIST_USED, LIST_FULL);
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_DETAIL);
ret = ((char *) firstused->data) + i*bytes;
#if SANITYCHECKS
nojunkwarning++;
slabunlock(ret, bytes);
nojunkwarning--;
assert(!nojunkwarning);
*(u32_t *) ret = NOJUNK;
slablock(ret, bytes);
#endif
SLABSANITYCHECK(SCL_FUNCTIONS);
SLABDATAUSE(firstused, firstused->sdh.freeguess = i+1;);
#if SANITYCHECKS
if(bytes >= SLABSIZES+MINSIZE) {
printf("slaballoc: odd, bytes %d?\n", bytes);
}
if(!slabsane_f(__FILE__, __LINE__, ret, bytes))
panic("slaballoc: slabsane failed");
#endif
return ret;
}
SLABSANITYCHECK(SCL_DETAIL);
}
SLABSANITYCHECK(SCL_FUNCTIONS);
panic("slaballoc: no space in 'used' slabdata");
/* Not reached. */
return NULL;
}
/*===========================================================================*
* int objstats *
*===========================================================================*/
PRIVATE int objstats(void *mem, int bytes,
struct slabheader **sp, struct slabdata **fp, int *ip)
{
#if SANITYCHECKS
#define OBJSTATSCHECK(cond) \
if(!(cond)) { \
printf("VM: objstats: %s failed for ptr 0x%p, %d bytes\n", \
#cond, mem, bytes); \
return EINVAL; \
}
#else
#define OBJSTATSCHECK(cond)
#endif
struct slabheader *s;
struct slabdata *f;
int i;
OBJSTATSCHECK((char *) mem >= (char *) VM_PAGE_SIZE);
#if SANITYCHECKS
if(*(u32_t *) mem == JUNK && !nojunkwarning) {
util_stacktrace();
printf("VM: WARNING: JUNK seen in slab object\n");
}
#endif
/* Retrieve entry in slabs[]. */
GETSLAB(bytes, s);
/* Round address down to VM_PAGE_SIZE boundary to get header. */
f = (struct slabdata *) ((char *) mem - (vir_bytes) mem % VM_PAGE_SIZE);
OBJSTATSCHECK(f->sdh.magic1 == MAGIC1);
OBJSTATSCHECK(f->sdh.magic2 == MAGIC2);
OBJSTATSCHECK(f->sdh.list == LIST_USED || f->sdh.list == LIST_FULL);
/* Make sure it's in range. */
OBJSTATSCHECK((char *) mem >= (char *) f->data);
OBJSTATSCHECK((char *) mem < (char *) f->data + sizeof(f->data));
/* Get position. */
i = (char *) mem - (char *) f->data;
OBJSTATSCHECK(!(i % bytes));
i = i / bytes;
/* Make sure it is marked as allocated. */
OBJSTATSCHECK(GETBIT(f, i));
/* return values */
*ip = i;
*fp = f;
*sp = s;
return OK;
}
/*===========================================================================*
* void *slabfree *
*===========================================================================*/
PUBLIC void slabfree(void *mem, int bytes)
{
int i;
struct slabheader *s;
struct slabdata *f;
SLABSANITYCHECK(SCL_FUNCTIONS);
if(objstats(mem, bytes, &s, &f, &i) != OK) {
panic("slabfree objstats failed");
}
#if SANITYCHECKS
if(*(u32_t *) mem == JUNK) {
printf("VM: WARNING: likely double free, JUNK seen\n");
}
slabunlock(mem, bytes);
*(u32_t *) mem = JUNK;
nojunkwarning++;
slablock(mem, bytes);
nojunkwarning--;
assert(!nojunkwarning);
#endif
/* Free this data. */
CLEARBIT(f, i);
/* Check if this slab changes lists. */
if(f->sdh.nused == 0) {
/* Now become FREE; must've been USED */
assert(f->sdh.list == LIST_USED);
UNLINKNODE(f);
if(f == LH(s, LIST_USED))
LH(s, LIST_USED) = f->sdh.next;
ADDHEAD(f, s, LIST_FREE);
SLABSANITYCHECK(SCL_DETAIL);
} else if(f->sdh.nused == ITEMSPERPAGE(bytes)-1) {
/* Now become USED; must've been FULL */
assert(f->sdh.list == LIST_FULL);
UNLINKNODE(f);
if(f == LH(s, LIST_FULL))
LH(s, LIST_FULL) = f->sdh.next;
ADDHEAD(f, s, LIST_USED);
SLABSANITYCHECK(SCL_DETAIL);
} else {
/* Stay USED */
assert(f->sdh.list == LIST_USED);
}
SLABSANITYCHECK(SCL_FUNCTIONS);
return;
}
/*===========================================================================*
* void *slablock *
*===========================================================================*/
PUBLIC void slablock(void *mem, int bytes)
{
int i;
struct slabheader *s;
struct slabdata *f;
if(objstats(mem, bytes, &s, &f, &i) != OK)
panic("slablock objstats failed");
SLABDATAUNWRITABLE(f);
return;
}
/*===========================================================================*
* void *slabunlock *
*===========================================================================*/
PUBLIC void slabunlock(void *mem, int bytes)
{
int i;
struct slabheader *s;
struct slabdata *f;
if(objstats(mem, bytes, &s, &f, &i) != OK)
panic("slablock objstats failed");
SLABDATAWRITABLE(f, i);
return;
}
#if SANITYCHECKS
/*===========================================================================*
* void slabstats *
*===========================================================================*/
PUBLIC void slabstats(void)
{
int s, total = 0, totalbytes = 0;
static int n;
n++;
if(n%1000) return;
for(s = 0; s < SLABSIZES; s++) {
int l;
for(l = 0; l < LIST_NUMBER; l++) {
int b, t;
b = s + MINSIZE;
t = checklist(__FILE__, __LINE__, &slabs[s], l, b);
if(t > 0) {
int bytes = t * b;
printf("VMSTATS: %2d slabs: %d (%dkB)\n", b, t, bytes/1024);
totalbytes += bytes;
}
}
}
if(pages > 0) {
printf("VMSTATS: %dK net used in slab objects in %d pages (%dkB): %d%% utilization\n",
totalbytes/1024, pages, pages*VM_PAGE_SIZE/1024,
100 * totalbytes / (pages*VM_PAGE_SIZE));
}
}
#endif