#define _SYSTEM 1
#define _POSIX_SOURCE 1

#include <minix/callnr.h>
#include <minix/com.h>
#include <minix/config.h>
#include <minix/const.h>
#include <minix/ds.h>
#include <minix/endpoint.h>
#include <minix/keymap.h>
#include <minix/minlib.h>
#include <minix/type.h>
#include <minix/ipc.h>
#include <minix/sysutil.h>
#include <minix/syslib.h>
#include <minix/safecopies.h>
#include <minix/cpufeature.h>
#include <minix/bitmap.h>
#include <minix/debug.h>

#include <errno.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <env.h>
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>

#include "proto.h"
#include "glo.h"
#include "util.h"
#include "vm.h"
#include "sanitycheck.h"

#include "memory.h"

static int vm_self_pages;

/* PDE used to map in kernel, kernel physical address. */
static int pagedir_pde = -1;
static u32_t pagedir_pde_val;

static multiboot_module_t *kern_mb_mod = NULL;
static size_t kern_size = 0;
static int kern_start_pde = -1;

/* 1MB page size available in hardware? */
static int bigpage_ok = 1;

/* Our process table entry. */
struct vmproc *vmprocess = &vmproc[VM_PROC_NR];

/* Spare memory, ready to go after initialization, to avoid a
 * circular dependency on allocating memory and writing it into VM's
 * page table.
 */
#define SPAREPAGEDIRS 11
#define STATIC_SPAREPAGEDIRS 10
#define SPAREPAGES 250
#define STATIC_SPAREPAGES 100
int missing_sparedirs = SPAREPAGEDIRS;
static struct {
	void *pagedir;
	phys_bytes phys;
} sparepagedirs[SPAREPAGEDIRS];

int missing_spares = SPAREPAGES;
static struct {
	void *page;
	phys_bytes phys;
} sparepages[SPAREPAGES];

extern char _end;	
#define is_staticaddr(v) ((vir_bytes) (v) < (vir_bytes) &_end)

#define MAX_KERNMAPPINGS 10
static struct {
	phys_bytes	phys_addr;	/* Physical addr. */
	phys_bytes	len;		/* Length in bytes. */
	vir_bytes	vir_addr;	/* Offset in page table. */
	int		flags;
} kern_mappings[MAX_KERNMAPPINGS];
int kernmappings = 0;

/* Clicks must be pages, as
 *  - they must be page aligned to map them
 *  - they must be a multiple of the page size
 *  - it's inconvenient to have them bigger than pages, because we often want
 *    just one page
 * May as well require them to be equal then.
 */
#if CLICK_SIZE != ARM_PAGE_SIZE
#error CLICK_SIZE must be page size.
#endif

/* Page table that contains pointers to all page directories. */
phys_bytes page_directories_phys;
u32_t *page_directories = NULL;

static char static_sparepagedirs[ARM_PAGEDIR_SIZE*STATIC_SPAREPAGEDIRS + ARM_PAGEDIR_SIZE] __aligned(ARM_PAGEDIR_SIZE);

static char static_sparepages[ARM_PAGE_SIZE*STATIC_SPAREPAGES] __aligned(ARM_PAGE_SIZE);

#if SANITYCHECKS
/*===========================================================================*
 *				pt_sanitycheck		     		     *
 *===========================================================================*/
void pt_sanitycheck(pt_t *pt, char *file, int line)
{
/* Basic pt sanity check. */
	int slot;

	MYASSERT(pt);
	MYASSERT(pt->pt_dir);
	MYASSERT(pt->pt_dir_phys);

	for(slot = 0; slot < ELEMENTS(vmproc); slot++) {
		if(pt == &vmproc[slot].vm_pt)
			break;
	}

	if(slot >= ELEMENTS(vmproc)) {
		panic("pt_sanitycheck: passed pt not in any proc");
	}

	MYASSERT(usedpages_add(pt->pt_dir_phys, ARM_PAGE_SIZE) == OK);
}
#endif

/*===========================================================================*
 *				findhole		     		     *
 *===========================================================================*/
static u32_t findhole(int pages)
{
/* Find a space in the virtual address space of VM. */
	u32_t curv;
	int pde = 0, try_restart;
	static u32_t lastv = 0;
	pt_t *pt = &vmprocess->vm_pt;
	vir_bytes vmin, vmax;
	u32_t holev;

	vmin = (vir_bytes) (&_end) & ARM_VM_ADDR_MASK; /* marks end of VM BSS */
	vmax = VM_STACKTOP;

	/* Input sanity check. */
	assert(vmin + ARM_PAGE_SIZE >= vmin);
	assert(vmax >= vmin + ARM_PAGE_SIZE);
	assert((vmin % ARM_PAGE_SIZE) == 0);
	assert((vmax % ARM_PAGE_SIZE) == 0);
	assert(pages > 0);

#if SANITYCHECKS
	curv = ((u32_t) random()) % ((vmax - vmin)/ARM_PAGE_SIZE);
	curv *= ARM_PAGE_SIZE;
	curv += vmin;
#else
	curv = lastv;
	if(curv < vmin || curv >= vmax)
		curv = vmin;
#endif
	try_restart = 1;

	/* Start looking for a free page starting at vmin. */
	while(curv < vmax) {
		int pte;
		int i, nohole;

		assert(curv >= vmin);
		assert(curv < vmax);

		holev = curv; /* the candidate hole */
		nohole = 0;
		for (i = 0; i < pages && !nohole; ++i) {
		    if(curv >= vmax) {
			break;
		    }

		    pde = ARM_VM_PDE(curv);
		    pte = ARM_VM_PTE(curv);

		    /* if page present, no hole */
		    if((pt->pt_dir[pde] & ARM_VM_PDE_PRESENT) &&
		       (pt->pt_pt[pde][pte] & ARM_VM_PTE_PRESENT))
			nohole = 1;

		    /* if not contiguous, no hole */
		    if (curv != holev + i * ARM_PAGE_SIZE)
			nohole = 1;

		    curv+=ARM_PAGE_SIZE;
		}

		/* there's a large enough hole */
		if (!nohole && i == pages) {
			lastv = curv;
			return holev;
		}

		/* Reset curv */
		if(curv >= vmax && try_restart) {
			curv = vmin;
			try_restart = 0;
		}
	}

	printf("VM: out of virtual address space in vm\n");

	return NO_MEM;
}

/*===========================================================================*
 *				vm_freepages		     		     *
 *===========================================================================*/
void vm_freepages(vir_bytes vir, int pages)
{
	assert(!(vir % ARM_PAGE_SIZE)); 

	if(is_staticaddr(vir)) {
		printf("VM: not freeing static page\n");
		return;
	}

	if(pt_writemap(vmprocess, &vmprocess->vm_pt, vir,
		MAP_NONE, pages*ARM_PAGE_SIZE, 0,
		WMF_OVERWRITE | WMF_FREE) != OK)
		panic("vm_freepages: pt_writemap failed");

	vm_self_pages--;

#if SANITYCHECKS
	/* If SANITYCHECKS are on, flush tlb so accessing freed pages is
	 * always trapped, also if not in tlb.
	 */
	if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
		panic("VMCTL_FLUSHTLB failed");
	}
#endif
}

/*===========================================================================*
 *				vm_getsparepage		     		     *
 *===========================================================================*/
static void *vm_getsparepage(phys_bytes *phys)
{
	int s;
	assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
	for(s = 0; s < SPAREPAGES; s++) {
		if(sparepages[s].page) {
			void *sp;
			sp = sparepages[s].page;
			*phys = sparepages[s].phys;
			sparepages[s].page = NULL;
			missing_spares++;
			assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
			return sp;
		}
	}
	return NULL;
}

/*===========================================================================*
 *				vm_getsparepagedir	      		     *
 *===========================================================================*/
static void *vm_getsparepagedir(phys_bytes *phys)
{
	int s;
	assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
	for(s = 0; s < SPAREPAGEDIRS; s++) {
		if(sparepagedirs[s].pagedir) {
			void *sp;
			sp = sparepagedirs[s].pagedir;
			*phys = sparepagedirs[s].phys;
			sparepagedirs[s].pagedir = NULL;
			missing_sparedirs++;
			assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
			return sp;
		}
	}
	return NULL;
}

/*===========================================================================*
 *				vm_checkspares		     		     *
 *===========================================================================*/
static void *vm_checkspares(void)
{
	int s, n = 0;
	static int total = 0, worst = 0;
	assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
	for(s = 0; s < SPAREPAGES && missing_spares > 0; s++)
	    if(!sparepages[s].page) {
		n++;
		if((sparepages[s].page = vm_allocpage(&sparepages[s].phys, 
			VMP_SPARE))) {
			missing_spares--;
			assert(missing_spares >= 0);
			assert(missing_spares <= SPAREPAGES);
		} else {
			printf("VM: warning: couldn't get new spare page\n");
		}
	}
	if(worst < n) worst = n;
	total += n;

	return NULL;
}

/*===========================================================================*
 *				vm_checksparedirs	       		     *
 *===========================================================================*/
static void *vm_checksparedirs(void)
{
	int s, n = 0;
	static int total = 0, worst = 0;
	assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
	for(s = 0; s < SPAREPAGEDIRS && missing_sparedirs > 0; s++)
	    if(!sparepagedirs[s].pagedir) {
		n++;
		if((sparepagedirs[s].pagedir = vm_allocpage(&sparepagedirs[s].phys,
			VMP_SPARE))) {
			missing_sparedirs--;
			assert(missing_sparedirs >= 0);
			assert(missing_sparedirs <= SPAREPAGEDIRS);
		} else {
			printf("VM: warning: couldn't get new spare pagedir\n");
		}
	}
	if(worst < n) worst = n;
	total += n;

	return NULL;
}

static int pt_init_done;

/*===========================================================================*
 *				vm_allocpage		     		     *
 *===========================================================================*/
void *vm_allocpage(phys_bytes *phys, int reason)
{
/* Allocate a page for use by VM itself. */
	phys_bytes newpage;
	vir_bytes loc;
	pt_t *pt;
	int r;
	static int level = 0;
	void *ret;
	u32_t mem_bytes, mem_clicks, mem_flags;

	pt = &vmprocess->vm_pt;
	assert(reason >= 0 && reason < VMP_CATEGORIES);

	level++;

	assert(level >= 1);
	assert(level <= 2);

	if(level > 1 || !pt_init_done) {
		void *s;

		if (reason == VMP_PAGEDIR)
			s=vm_getsparepagedir(phys);
		else
			s=vm_getsparepage(phys);

		level--;
		if(!s) {
			util_stacktrace();
			printf("VM: warning: out of spare pages\n");
		}
		if(!is_staticaddr(s)) vm_self_pages++;
		return s;
	}

	if (reason == VMP_PAGEDIR) {
		mem_bytes = ARM_PAGEDIR_SIZE;
		mem_flags = PAF_ALIGN16K;
	} else {
		mem_bytes = ARM_PAGE_SIZE;
		mem_flags = 0;
	}
	mem_clicks = mem_bytes / ARM_PAGE_SIZE * CLICKSPERPAGE;

	/* VM does have a pagetable, so get a page and map it in there.
	 * Where in our virtual address space can we put it?
	 */
	loc = findhole(mem_bytes / ARM_PAGE_SIZE);
	if(loc == NO_MEM) {
		level--;
		printf("VM: vm_allocpage: findhole failed\n");
		return NULL;
	}

	/* Allocate page of memory for use by VM. As VM
	 * is trusted, we don't have to pre-clear it.
	 */
	if((newpage = alloc_mem(mem_clicks, mem_flags)) == NO_MEM) {
		level--;
		printf("VM: vm_allocpage: alloc_mem failed\n");
		return NULL;
	}

	*phys = CLICK2ABS(newpage);

	/* Map this page into our address space. */
	if((r=pt_writemap(vmprocess, pt, loc, *phys, mem_bytes,
		ARM_VM_PTE_PRESENT | ARM_VM_PTE_USER | ARM_VM_PTE_RW |
		ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE, 0)) != OK) {
		free_mem(newpage, mem_clicks);
		printf("vm_allocpage writemap failed\n");
		level--;
		return NULL;
	}

	if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
		panic("VMCTL_FLUSHTLB failed: %d", r);
	}

	level--;

	/* Return user-space-ready pointer to it. */
	ret = (void *) loc;

	vm_self_pages++;
	return ret;
}

/*===========================================================================*
 *				vm_pagelock		     		     *
 *===========================================================================*/
void vm_pagelock(void *vir, int lockflag)
{
/* Mark a page allocated by vm_allocpage() unwritable, i.e. only for VM. */
	vir_bytes m = (vir_bytes) vir;
	int r;
	u32_t flags = ARM_VM_PTE_PRESENT | ARM_VM_PTE_USER;
	pt_t *pt;

	pt = &vmprocess->vm_pt;

	assert(!(m % ARM_PAGE_SIZE));

	if(!lockflag)
		flags |= ARM_VM_PTE_RW;
	else
		flags |= ARM_VM_PTE_RO;
	flags |= ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;

	/* Update flags. */
	if((r=pt_writemap(vmprocess, pt, m, 0, ARM_PAGE_SIZE,
		flags, WMF_OVERWRITE | WMF_WRITEFLAGSONLY)) != OK) {
		panic("vm_lockpage: pt_writemap failed");
	}

	if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
		panic("VMCTL_FLUSHTLB failed: %d", r);
	}

	return;
}

/*===========================================================================*
 *				vm_addrok		     		     *
 *===========================================================================*/
int vm_addrok(void *vir, int writeflag)
{
	pt_t *pt = &vmprocess->vm_pt;
	int pde, pte;
	vir_bytes v = (vir_bytes) vir;

	pde = ARM_VM_PDE(v);
	pte = ARM_VM_PTE(v);

	if(!(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT)) {
		printf("addr not ok: missing pde %d\n", pde);
		return 0;
	}

	if(!(pt->pt_pt[pde][pte] & ARM_VM_PTE_PRESENT)) {
		printf("addr not ok: missing pde %d / pte %d\n",
			pde, pte);
		return 0;
	}

	if(!writeflag &&
		!(pt->pt_pt[pde][pte] & ARM_VM_PTE_RO)) {
		printf("addr not ok: pde %d / pte %d present but writable\n",
			pde, pte);
		return 0;
	}

	return 1;
}

/*===========================================================================*
 *				pt_ptalloc		     		     *
 *===========================================================================*/
static int pt_ptalloc(pt_t *pt, int pde, u32_t flags)
{
/* Allocate a page table and write its address into the page directory. */
	int i;
	phys_bytes pt_phys;

	/* Argument must make sense. */
	assert(pde >= 0 && pde < ARM_VM_DIR_ENTRIES);
	assert(!(flags & ~(PTF_ALLFLAGS)));

	/* We don't expect to overwrite page directory entry, nor
	 * storage for the page table.
	 */
	assert(!(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT));
	assert(!pt->pt_pt[pde]);

	/* Get storage for the page table. */
        if(!(pt->pt_pt[pde] = vm_allocpage(&pt_phys, VMP_PAGETABLE)))
		return ENOMEM;

	for(i = 0; i < ARM_VM_PT_ENTRIES; i++)
		pt->pt_pt[pde][i] = 0;	/* Empty entry. */

	/* Make page directory entry.
	 * The PDE is always 'present,' 'writable,' and 'user accessible,'
	 * relying on the PTE for protection.
	 */
	pt->pt_dir[pde] = (pt_phys & ARM_VM_PDE_MASK)
		| ARM_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;

	return OK;
}

/*===========================================================================*
 *			    pt_ptalloc_in_range		     		     *
 *===========================================================================*/
int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end,
	u32_t flags, int verify)
{
/* Allocate all the page tables in the range specified. */
	int pde, first_pde, last_pde;

	first_pde = ARM_VM_PDE(start);
	last_pde = ARM_VM_PDE(end-1);
	assert(first_pde >= 0);
	assert(last_pde < ARM_VM_DIR_ENTRIES);

	/* Scan all page-directory entries in the range. */
	for(pde = first_pde; pde <= last_pde; pde++) {
		assert(!(pt->pt_dir[pde] & ARM_VM_BIGPAGE));
		if(!(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT)) {
			int r;
			if(verify) {
				printf("pt_ptalloc_in_range: no pde %d\n", pde);
				return EFAULT;
			}
			assert(!pt->pt_dir[pde]);
			if((r=pt_ptalloc(pt, pde, flags)) != OK) {
				/* Couldn't do (complete) mapping.
				 * Don't bother freeing any previously
				 * allocated page tables, they're
				 * still writable, don't point to nonsense,
				 * and pt_ptalloc leaves the directory
				 * and other data in a consistent state.
				 */
				printf("pt_ptalloc_in_range: pt_ptalloc failed\n");
				return r;
			}
		}
		assert(pt->pt_dir[pde]);
		assert(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT);
	}

	return OK;
}

static char *ptestr(u32_t pte)
{
#define FLAG(constant, name) {						\
	if(pte & (constant)) { strcat(str, name); strcat(str, " "); }	\
}

	static char str[30];
	if(!(pte & ARM_VM_PTE_PRESENT)) {
		return "not present";
	}
	str[0] = '\0';
	if(pte & ARM_VM_PTE_RO) {
	    strcat(str, "R ");
	} else {
	    strcat(str, "W ");
	}
	FLAG(ARM_VM_PTE_USER, "U");
	FLAG(ARM_VM_PTE_SUPER, "S");
	FLAG(ARM_VM_PTE_SHAREABLE, "SH");
	FLAG(ARM_VM_PTE_WB, "WB");
	FLAG(ARM_VM_PTE_WT, "WT");

	return str;
}

/*===========================================================================*
 *			     pt_map_in_range		     		     *
 *===========================================================================*/
int pt_map_in_range(struct vmproc *src_vmp, struct vmproc *dst_vmp,
	vir_bytes start, vir_bytes end)
{
/* Transfer all the mappings from the pt of the source process to the pt of
 * the destination process in the range specified.
 */
	int pde, pte;
	vir_bytes viraddr;
	pt_t *pt, *dst_pt;

	pt = &src_vmp->vm_pt;
	dst_pt = &dst_vmp->vm_pt;

	end = end ? end : VM_DATATOP;
	assert(start % ARM_PAGE_SIZE == 0);
	assert(end % ARM_PAGE_SIZE == 0);
	assert(ARM_VM_PDE(start) >= 0 && start <= end);
	assert(ARM_VM_PDE(end) < ARM_VM_DIR_ENTRIES);

#if LU_DEBUG
	printf("VM: pt_map_in_range: src = %d, dst = %d\n",
		src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
	printf("VM: pt_map_in_range: transferring from 0x%08x (pde %d pte %d) to 0x%08x (pde %d pte %d)\n",
		start, ARM_VM_PDE(start), ARM_VM_PTE(start),
		end, ARM_VM_PDE(end), ARM_VM_PTE(end));
#endif

	/* Scan all page-table entries in the range. */
	for(viraddr = start; viraddr <= end; viraddr += ARM_PAGE_SIZE) {
		pde = ARM_VM_PDE(viraddr);
		if(!(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT)) {
			if(viraddr == VM_DATATOP) break;
			continue;
		}
		pte = ARM_VM_PTE(viraddr);
		if(!(pt->pt_pt[pde][pte] & ARM_VM_PTE_PRESENT)) {
			if(viraddr == VM_DATATOP) break;
			continue;
		}

		/* Transfer the mapping. */
		dst_pt->pt_pt[pde][pte] = pt->pt_pt[pde][pte];

                if(viraddr == VM_DATATOP) break;
	}

	return OK;
}

/*===========================================================================*
 *				pt_ptmap		     		     *
 *===========================================================================*/
int pt_ptmap(struct vmproc *src_vmp, struct vmproc *dst_vmp)
{
/* Transfer mappings to page dir and page tables from source process and
 * destination process. Make sure all the mappings are above the stack, not
 * to corrupt valid mappings in the data segment of the destination process.
 */
	int pde, r;
	phys_bytes physaddr;
	vir_bytes viraddr;
	pt_t *pt;

	pt = &src_vmp->vm_pt;

#if LU_DEBUG
	printf("VM: pt_ptmap: src = %d, dst = %d\n",
		src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
#endif

	/* Transfer mapping to the page directory. */
	viraddr = (vir_bytes) pt->pt_dir;
	physaddr = pt->pt_dir_phys & ARM_VM_ADDR_MASK;
	if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, ARM_PAGEDIR_SIZE,
		ARM_VM_PTE_PRESENT | ARM_VM_PTE_USER | ARM_VM_PTE_RW |
		ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE,
		WMF_OVERWRITE)) != OK) {
		return r;
	}
#if LU_DEBUG
	printf("VM: pt_ptmap: transferred mapping to page dir: 0x%08x (0x%08x)\n",
		viraddr, physaddr);
#endif

	/* Scan all non-reserved page-directory entries. */
	for(pde=0; pde < ARM_VM_DIR_ENTRIES; pde++) {
		if(!(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT)) {
			continue;
		}

		/* Transfer mapping to the page table. */
		viraddr = (vir_bytes) pt->pt_pt[pde];
		physaddr = pt->pt_dir[pde] & ARM_VM_PDE_MASK;
		if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, ARM_PAGE_SIZE,
			ARM_VM_PTE_PRESENT | ARM_VM_PTE_USER | ARM_VM_PTE_RW |
			ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE,
			WMF_OVERWRITE)) != OK) {
			return r;
		}
	}

	return OK;
}

void pt_clearmapcache(void)
{
	/* Make sure kernel will invalidate tlb when using current
	 * pagetable (i.e. vm's) to make new mappings before new cr3
	 * is loaded.
	 */
	if(sys_vmctl(SELF, VMCTL_CLEARMAPCACHE, 0) != OK)
		panic("VMCTL_CLEARMAPCACHE failed");
}

/*===========================================================================*
 *				pt_writemap		     		     *
 *===========================================================================*/
int pt_writemap(struct vmproc * vmp,
			pt_t *pt,
			vir_bytes v,
			phys_bytes physaddr,
			size_t bytes,
			u32_t flags,
			u32_t writemapflags)
{
/* Write mapping into page table. Allocate a new page table if necessary. */
/* Page directory and table entries for this virtual address. */
	int p, pages;
	int verify = 0;
	int ret = OK;

#ifdef CONFIG_SMP
	int vminhibit_clear = 0;
	/* FIXME
	 * don't do it everytime, stop the process only on the first change and
	 * resume the execution on the last change. Do in a wrapper of this
	 * function
	 */
	if (vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
			!(vmp->vm_flags & VMF_EXITING)) {
		sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_SET, 0);
		vminhibit_clear = 1;
	}
#endif

	if(writemapflags & WMF_VERIFY)
		verify = 1;

	assert(!(bytes % ARM_PAGE_SIZE));
	assert(!(flags & ~(PTF_ALLFLAGS)));

	pages = bytes / ARM_PAGE_SIZE;

	/* MAP_NONE means to clear the mapping. It doesn't matter
	 * what's actually written into the PTE if ARM_VM_PRESENT
	 * isn't on, so we can just write MAP_NONE into it.
	 */
	assert(physaddr == MAP_NONE || (flags & ARM_VM_PTE_PRESENT));
	assert(physaddr != MAP_NONE || !flags);

	/* First make sure all the necessary page tables are allocated,
	 * before we start writing in any of them, because it's a pain
	 * to undo our work properly.
	 */
	ret = pt_ptalloc_in_range(pt, v, v + ARM_PAGE_SIZE*pages, flags, verify);
	if(ret != OK) {
		printf("VM: writemap: pt_ptalloc_in_range failed\n");
		goto resume_exit;
	}

	/* Now write in them. */
	for(p = 0; p < pages; p++) {
		u32_t entry;
		int pde = ARM_VM_PDE(v);
		int pte = ARM_VM_PTE(v);

		if(!v) { printf("VM: warning: making zero page for %d\n",
			vmp->vm_endpoint); }

		assert(!(v % ARM_PAGE_SIZE));
		assert(pte >= 0 && pte < ARM_VM_PT_ENTRIES);
		assert(pde >= 0 && pde < ARM_VM_DIR_ENTRIES);

		/* Page table has to be there. */
		assert(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT);

		/* We do not expect it to be a bigpage. */
		assert(!(pt->pt_dir[pde] & ARM_VM_BIGPAGE));

		/* Make sure page directory entry for this page table
		 * is marked present and page table entry is available.
		 */
		assert(pt->pt_pt[pde]);

#if SANITYCHECKS
		/* We don't expect to overwrite a page. */
		if(!(writemapflags & (WMF_OVERWRITE|WMF_VERIFY)))
			assert(!(pt->pt_pt[pde][pte] & ARM_VM_PTE_PRESENT));
#endif
		if(writemapflags & (WMF_WRITEFLAGSONLY|WMF_FREE)) {
			physaddr = pt->pt_pt[pde][pte] & ARM_VM_PTE_MASK;
		}

		if(writemapflags & WMF_FREE) {
			free_mem(ABS2CLICK(physaddr), 1);
		}

		/* Entry we will write. */
		entry = (physaddr & ARM_VM_PTE_MASK) | flags;

		if(verify) {
			u32_t maskedentry;
			maskedentry = pt->pt_pt[pde][pte];
			/* Verify pagetable entry. */
			if(entry & ARM_VM_PTE_RW) {
				/* If we expect a writable page, allow a readonly page. */
				maskedentry |= ARM_VM_PTE_RW;
			}
			if(maskedentry != entry) {
				printf("pt_writemap: mismatch: ");
				if((entry & ARM_VM_PTE_MASK) !=
					(maskedentry & ARM_VM_PTE_MASK)) {
					printf("pt_writemap: physaddr mismatch (0x%lx, 0x%lx); ",
						(long)entry, (long)maskedentry);
				} else printf("phys ok; ");
				printf(" flags: found %s; ",
					ptestr(pt->pt_pt[pde][pte]));
				printf(" masked %s; ",
					ptestr(maskedentry));
				printf(" expected %s\n", ptestr(entry));
				ret = EFAULT;
				goto resume_exit;
			}
		} else {
			/* Write pagetable entry. */
			pt->pt_pt[pde][pte] = entry;
		}

		physaddr += ARM_PAGE_SIZE;
		v += ARM_PAGE_SIZE;
	}

resume_exit:

#ifdef CONFIG_SMP
	if (vminhibit_clear) {
		assert(vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
			!(vmp->vm_flags & VMF_EXITING));
		sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_CLEAR, 0);
	}
#endif

	return ret;
}

/*===========================================================================*
 *				pt_checkrange		     		     *
 *===========================================================================*/
int pt_checkrange(pt_t *pt, vir_bytes v,  size_t bytes,
	int write)
{
	int p, pages;

	assert(!(bytes % ARM_PAGE_SIZE));

	pages = bytes / ARM_PAGE_SIZE;

	for(p = 0; p < pages; p++) {
		int pde = ARM_VM_PDE(v);
		int pte = ARM_VM_PTE(v);

		assert(!(v % ARM_PAGE_SIZE));
		assert(pte >= 0 && pte < ARM_VM_PT_ENTRIES);
		assert(pde >= 0 && pde < ARM_VM_DIR_ENTRIES);

		/* Page table has to be there. */
		if(!(pt->pt_dir[pde] & ARM_VM_PDE_PRESENT))
			return EFAULT;

		/* Make sure page directory entry for this page table
		 * is marked present and page table entry is available.
		 */
		assert((pt->pt_dir[pde] & ARM_VM_PDE_PRESENT) && pt->pt_pt[pde]);

		if(!(pt->pt_pt[pde][pte] & ARM_VM_PTE_PRESENT)) {
			return EFAULT;
		}

		if(write && (pt->pt_pt[pde][pte] & ARM_VM_PTE_RO)) {
			return EFAULT;
		}

		v += ARM_PAGE_SIZE;
	}

	return OK;
}

/*===========================================================================*
 *				pt_new			     		     *
 *===========================================================================*/
int pt_new(pt_t *pt)
{
/* Allocate a pagetable root. On ARM, allocate a page-aligned page directory
 * and set them to 0 (indicating no page tables are allocated). Lookup
 * its physical address as we'll need that in the future. Verify it's
 * page-aligned.
 */
	int i;

	/* Don't ever re-allocate/re-move a certain process slot's
	 * page directory once it's been created. This is a fraction
	 * faster, but also avoids having to invalidate the page
	 * mappings from in-kernel page tables pointing to
	 * the page directories (the page_directories data).
	 */
        if(!pt->pt_dir &&
          !(pt->pt_dir = vm_allocpage((phys_bytes *)&pt->pt_dir_phys, VMP_PAGEDIR))) {
		return ENOMEM;
	}
	assert(!((u32_t)pt->pt_dir_phys % ARM_PAGEDIR_SIZE));

	for(i = 0; i < ARM_VM_DIR_ENTRIES; i++) {
		pt->pt_dir[i] = 0; /* invalid entry (ARM_VM_PRESENT bit = 0) */
		pt->pt_pt[i] = NULL;
	}

	/* Where to start looking for free virtual address space? */
	pt->pt_virtop = 0;

        /* Map in kernel. */
        if(pt_mapkernel(pt) != OK)
                panic("pt_new: pt_mapkernel failed");

	return OK;
}

static int freepde(void)
{
	int p = kernel_boot_info.freepde_start++;
	assert(kernel_boot_info.freepde_start < ARM_VM_DIR_ENTRIES);
	return p;
}

/*===========================================================================*
 *                              pt_init                                      *
 *===========================================================================*/
void pt_init(void)
{
        pt_t *newpt;
        int s, r, p;
	vir_bytes sparepages_mem;
	vir_bytes sparepagedirs_mem;
	static u32_t currentpagedir[ARM_VM_DIR_ENTRIES];
	int m = kernel_boot_info.kern_mod;
	u32_t myttbr;

	/* Find what the physical location of the kernel is. */
	assert(m >= 0);
	assert(m < kernel_boot_info.mods_with_kernel);
	assert(kernel_boot_info.mods_with_kernel < MULTIBOOT_MAX_MODS);
	kern_mb_mod = &kernel_boot_info.module_list[m];
	kern_size = kern_mb_mod->mod_end - kern_mb_mod->mod_start;
	assert(!(kern_mb_mod->mod_start % ARM_BIG_PAGE_SIZE));
	assert(!(kernel_boot_info.vir_kern_start % ARM_BIG_PAGE_SIZE));
	kern_start_pde = kernel_boot_info.vir_kern_start / ARM_BIG_PAGE_SIZE;

        /* Get ourselves spare pages. */
        sparepages_mem = (vir_bytes) static_sparepages;
	assert(!(sparepages_mem % ARM_PAGE_SIZE));

        /* Get ourselves spare pagedirs. */
	sparepagedirs_mem = (vir_bytes) static_sparepagedirs;
	assert(!(sparepagedirs_mem % ARM_PAGEDIR_SIZE));

	/* Spare pages are used to allocate memory before VM has its own page
	 * table that things (i.e. arbitrary physical memory) can be mapped into.
	 * We get it by pre-allocating it in our bss (allocated and mapped in by
	 * the kernel) in static_sparepages. We also need the physical addresses
	 * though; we look them up now so they are ready for use.
	 */
        missing_sparedirs = 0;
        assert(STATIC_SPAREPAGEDIRS < SPAREPAGEDIRS);
        for(s = 0; s < SPAREPAGEDIRS; s++) {
		vir_bytes v = (sparepagedirs_mem + s*ARM_PAGEDIR_SIZE);;
		phys_bytes ph;
        	if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
	                ARM_PAGEDIR_SIZE, &ph)) != OK)
				panic("pt_init: sys_umap failed: %d", r);
        	if(s >= STATIC_SPAREPAGEDIRS) {
        		sparepagedirs[s].pagedir = NULL;
        		missing_sparedirs++;
        		continue;
        	}
        	sparepagedirs[s].pagedir = (void *) v;
        	sparepagedirs[s].phys = ph;
        }

        missing_spares = 0;
        assert(STATIC_SPAREPAGES < SPAREPAGES);
        for(s = 0; s < SPAREPAGES; s++) {
		vir_bytes v = (sparepages_mem + s*ARM_PAGE_SIZE);;
		phys_bytes ph;
		if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
	                ARM_PAGE_SIZE*SPAREPAGES, &ph)) != OK)
				panic("pt_init: sys_umap failed: %d", r);
		if(s >= STATIC_SPAREPAGES) {
        		sparepages[s].page = NULL;
        		missing_spares++;
        		continue;
        	}
        	sparepages[s].page = (void *) v;
        	sparepages[s].phys = ph;
        }

	/* 1MB pages available? */
	bigpage_ok = 1;

	/* Allocate us a page table in which to remember page directory
	 * pointers.
	 */
	if(!(page_directories = vm_allocpage(&page_directories_phys,
		VMP_PAGETABLE)))
                panic("no virt addr for vm mappings");

	memset(page_directories, 0, ARM_PAGE_SIZE);

	/* Now reserve another pde for kernel's own mappings. */
	{
		int kernmap_pde;
		phys_bytes addr, len;
		int flags, index = 0;
		u32_t offset = 0;

		kernmap_pde = freepde();
		offset = kernmap_pde * ARM_BIG_PAGE_SIZE;

		while(sys_vmctl_get_mapping(index, &addr, &len,
			&flags) == OK)  {
			vir_bytes vir;
			if(index >= MAX_KERNMAPPINGS)
                		panic("VM: too many kernel mappings: %d", index);
			kern_mappings[index].phys_addr = addr;
			kern_mappings[index].len = len;
			kern_mappings[index].flags = flags;
			kern_mappings[index].vir_addr = addr;
			kern_mappings[index].flags =
				ARM_VM_PTE_PRESENT;
			if(flags & VMMF_UNCACHED)
				kern_mappings[index].flags |= ARM_VM_PTE_DEVICE;
			else
				kern_mappings[index].flags |=
				    ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;
			if(flags & VMMF_USER)
				kern_mappings[index].flags |= ARM_VM_PTE_USER;
			else
				kern_mappings[index].flags |= ARM_VM_PTE_SUPER;
			if(flags & VMMF_WRITE)
				kern_mappings[index].flags |= ARM_VM_PTE_RW;
			else
				kern_mappings[index].flags |= ARM_VM_PTE_RO;
			if(addr % ARM_PAGE_SIZE)
                		panic("VM: addr unaligned: %d", addr);
			if(len % ARM_PAGE_SIZE)
                		panic("VM: len unaligned: %d", len);
			vir = offset;
			if(sys_vmctl_reply_mapping(index, vir) != OK)
                		panic("VM: reply failed");
			offset += len;
			index++;
			kernmappings++;
		}
	}

	/* Find a PDE below processes available for mapping in the
	 * page directories.
	 */
	pagedir_pde = freepde();
	pagedir_pde_val = (page_directories_phys & ARM_VM_PDE_MASK) |
			ARM_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;

	/* Allright. Now. We have to make our own page directory and page tables,
	 * that the kernel has already set up, accessible to us. It's easier to
	 * understand if we just copy all the required pages (i.e. page directory
	 * and page tables), and set up the pointers as if VM had done it itself.
	 *
	 * This allocation will happen without using any page table, and just
	 * uses spare pages.
	 */
        newpt = &vmprocess->vm_pt;
	if(pt_new(newpt) != OK)
		panic("vm pt_new failed");

	/* Get our current pagedir so we can see it. */
	if(sys_vmctl_get_pdbr(SELF, &myttbr) != OK)
		panic("VM: sys_vmctl_get_pdbr failed");
	if(sys_vircopy(NONE, myttbr, SELF,
		(vir_bytes) currentpagedir, ARM_PAGEDIR_SIZE) != OK)
		panic("VM: sys_vircopy failed");

	/* We have mapped in kernel ourselves; now copy mappings for VM
	 * that kernel made, including allocations for BSS. Skip identity
	 * mapping bits; just map in VM.
	 */
	for(p = 0; p < ARM_VM_DIR_ENTRIES; p++) {
		u32_t entry = currentpagedir[p];
		phys_bytes ptaddr_kern, ptaddr_us;

		/* BIGPAGEs are kernel mapping (do ourselves) or boot
		 * identity mapping (don't want).
		 */
		if(!(entry & ARM_VM_PDE_PRESENT)) continue;
		if((entry & ARM_VM_BIGPAGE)) continue;

		if(pt_ptalloc(newpt, p, 0) != OK)
			panic("pt_ptalloc failed");
		assert(newpt->pt_dir[p] & ARM_VM_PDE_PRESENT);

		ptaddr_kern = entry & ARM_VM_PDE_MASK;
		ptaddr_us = newpt->pt_dir[p] & ARM_VM_PDE_MASK;

		/* Copy kernel-initialized pagetable contents into our
		 * normally accessible pagetable.
		 */
                if(sys_abscopy(ptaddr_kern, ptaddr_us, ARM_PAGETABLE_SIZE) != OK)
			panic("pt_init: abscopy failed");
	}

	/* Inform kernel vm has a newly built page table. */
	assert(vmproc[VM_PROC_NR].vm_endpoint == VM_PROC_NR);
	pt_bind(newpt, &vmproc[VM_PROC_NR]);

	pt_init_done = 1;

        /* All OK. */
        return;
}

/*===========================================================================*
 *				pt_bind			     		     *
 *===========================================================================*/
int pt_bind(pt_t *pt, struct vmproc *who)
{
	int slot;
	u32_t phys;
	void *pdes;
	int i;
	int pages_per_pagedir = ARM_PAGEDIR_SIZE/ARM_PAGE_SIZE;

	/* Basic sanity checks. */
	assert(who);
	assert(who->vm_flags & VMF_INUSE);
	assert(pt);

	assert(pagedir_pde >= 0);

	slot = who->vm_slot;
	assert(slot >= 0);
	assert(slot < ELEMENTS(vmproc));
	assert(slot < ARM_VM_PT_ENTRIES / pages_per_pagedir);

	phys = pt->pt_dir_phys & ARM_VM_PTE_MASK;
	assert(pt->pt_dir_phys == phys);
	assert(!(pt->pt_dir_phys % ARM_PAGEDIR_SIZE));

	/* Update "page directory pagetable." */
	for (i = 0; i < pages_per_pagedir; i++)
	    page_directories[slot*pages_per_pagedir+i] =
		(phys+i*ARM_PAGE_SIZE) |
		ARM_VM_PTE_PRESENT | ARM_VM_PTE_RW |
		ARM_VM_PTE_USER;

	/* This is where the PDE's will be visible to the kernel
	 * in its address space.
	 */
	pdes = (void *) (pagedir_pde*ARM_BIG_PAGE_SIZE + 
			slot * ARM_PAGEDIR_SIZE);

#if 0
	printf("VM: slot %d endpoint %d has pde val 0x%lx at kernel address 0x%lx\n",
		slot, who->vm_endpoint, page_directories[slot], pdes);
#endif
	/* Tell kernel about new page table root. */
	return sys_vmctl_set_addrspace(who->vm_endpoint, pt->pt_dir_phys, pdes);
}

/*===========================================================================*
 *				pt_free			     		     *
 *===========================================================================*/
void pt_free(pt_t *pt)
{
/* Free memory associated with this pagetable. */
	int i;

	for(i = 0; i < ARM_VM_DIR_ENTRIES; i++)
		if(pt->pt_pt[i])
			vm_freepages((vir_bytes) pt->pt_pt[i], 1);

	return;
}

/*===========================================================================*
 *				pt_mapkernel		     		     *
 *===========================================================================*/
int pt_mapkernel(pt_t *pt)
{
	int i;
	int kern_pde = kern_start_pde;
	phys_bytes addr, mapped = 0;

        /* Any ARM page table needs to map in the kernel address space. */
	assert(bigpage_ok);
	assert(pagedir_pde >= 0);
	assert(kern_pde >= 0);

	/* pt_init() has made sure this is ok. */
	addr = kern_mb_mod->mod_start;

	/* Actually mapping in kernel */
	while(mapped < kern_size) {
		pt->pt_dir[kern_pde] = (addr & ARM_VM_PDE_MASK) |
			ARM_VM_SECTION |
			ARM_VM_SECTION_DOMAIN | ARM_VM_SECTION_WB |
			ARM_VM_SECTION_SHAREABLE | ARM_VM_SECTION_SUPER;
		kern_pde++;
		mapped += ARM_BIG_PAGE_SIZE;
		addr += ARM_BIG_PAGE_SIZE;
	}

	/* Kernel also wants to know about all page directories. */
	assert(pagedir_pde > kern_pde);
	pt->pt_dir[pagedir_pde] = pagedir_pde_val;

	/* Kernel also wants various mappings of its own. */
	for(i = 0; i < kernmappings; i++) {
		if(pt_writemap(NULL, pt,
			kern_mappings[i].vir_addr,
			kern_mappings[i].phys_addr,
			kern_mappings[i].len,
			kern_mappings[i].flags, 0) != OK) {
			panic("pt_mapkernel: pt_writemap failed");
		}
	}

	return OK;
}

/*===========================================================================*
 *				pt_cycle		     		     *
 *===========================================================================*/
void pt_cycle(void)
{
	vm_checkspares();
	vm_checksparedirs();
}

int get_vm_self_pages(void) { return vm_self_pages; }