#include "kernel/kernel.h" #include "kernel/proc.h" #include "kernel/vm.h" #include #include #include #include #include #include #include #include #include #include "arch_proto.h" #include "kernel/proto.h" #include "kernel/debug.h" #include "omap_timer.h" phys_bytes device_mem_vaddr = 0; #define HASPT(procptr) ((procptr)->p_seg.p_ttbr != 0) static int nfreepdes = 0; #define MAXFREEPDES 2 static int freepdes[MAXFREEPDES]; static u32_t phys_get32(phys_bytes v); void mem_clear_mapcache(void) { int i; for(i = 0; i < nfreepdes; i++) { struct proc *ptproc = get_cpulocal_var(ptproc); int pde = freepdes[i]; u32_t *ptv; assert(ptproc); ptv = ptproc->p_seg.p_ttbr_v; assert(ptv); ptv[pde] = 0; } } /* This function sets up a mapping from within the kernel's address * space to any other area of memory, either straight physical * memory (pr == NULL) or a process view of memory, in 1MB windows. * I.e., it maps in 1MB chunks of virtual (or physical) address space * to 1MB chunks of kernel virtual address space. * * It recognizes pr already being in memory as a special case (no * mapping required). * * The target (i.e. in-kernel) mapping area is one of the freepdes[] * VM has earlier already told the kernel about that is available. It is * identified as the 'pde' parameter. This value can be chosen freely * by the caller, as long as it is in range (i.e. 0 or higher and corresonds * to a known freepde slot). It is up to the caller to keep track of which * freepde's are in use, and to determine which ones are free to use. * * The logical number supplied by the caller is translated into an actual * pde number to be used, and a pointer to it (linear address) is returned * for actual use by phys_copy or memset. */ static phys_bytes createpde( const struct proc *pr, /* Requested process, NULL for physical. */ const phys_bytes linaddr,/* Address after segment translation. */ phys_bytes *bytes, /* Size of chunk, function may truncate it. */ int free_pde_idx, /* index of the free slot to use */ int *changed /* If mapping is made, this is set to 1. */ ) { u32_t pdeval; phys_bytes offset; int pde; assert(free_pde_idx >= 0 && free_pde_idx < nfreepdes); pde = freepdes[free_pde_idx]; assert(pde >= 0 && pde < 4096); if(pr && ((pr == get_cpulocal_var(ptproc)) || iskernelp(pr))) { /* Process memory is requested, and * it's a process that is already in current page table, or * the kernel, which is always there. * Therefore linaddr is valid directly, with the requested * size. */ return linaddr; } if(pr) { /* Requested address is in a process that is not currently * accessible directly. Grab the PDE entry of that process' * page table that corresponds to the requested address. */ assert(pr->p_seg.p_ttbr_v); pdeval = pr->p_seg.p_ttbr_v[ARM_VM_PDE(linaddr)]; } else { /* Requested address is physical. Make up the PDE entry. */ pdeval = (linaddr & ARM_VM_SECTION_MASK) | ARM_VM_SECTION | ARM_VM_SECTION_DOMAIN | ARM_VM_SECTION_DEVICE | ARM_VM_SECTION_USER; } /* Write the pde value that we need into a pde that the kernel * can access, into the currently loaded page table so it becomes * visible. */ assert(get_cpulocal_var(ptproc)->p_seg.p_ttbr_v); if(get_cpulocal_var(ptproc)->p_seg.p_ttbr_v[pde] != pdeval) { get_cpulocal_var(ptproc)->p_seg.p_ttbr_v[pde] = pdeval; *changed = 1; } /* Memory is now available, but only the 1MB window of virtual * address space that we have mapped; calculate how much of * the requested range is visible and return that in *bytes, * if that is less than the requested range. */ offset = linaddr & ARM_VM_OFFSET_MASK_1MB; /* Offset in 1MB window. */ *bytes = MIN(*bytes, ARM_SECTION_SIZE - offset); /* Return the linear address of the start of the new mapping. */ return ARM_SECTION_SIZE*pde + offset; } /*===========================================================================* * check_resumed_caller * *===========================================================================*/ static int check_resumed_caller(struct proc *caller) { /* Returns the result from VM if caller was resumed, otherwise OK. */ if (caller && (caller->p_misc_flags & MF_KCALL_RESUME)) { assert(caller->p_vmrequest.vmresult != VMSUSPEND); return caller->p_vmrequest.vmresult; } return OK; } /*===========================================================================* * lin_lin_copy * *===========================================================================*/ static int lin_lin_copy(struct proc *srcproc, vir_bytes srclinaddr, struct proc *dstproc, vir_bytes dstlinaddr, vir_bytes bytes) { u32_t addr; proc_nr_t procslot; assert(get_cpulocal_var(ptproc)); assert(get_cpulocal_var(proc_ptr)); assert(read_ttbr0() == get_cpulocal_var(ptproc)->p_seg.p_ttbr); procslot = get_cpulocal_var(ptproc)->p_nr; assert(procslot >= 0 && procslot < ARM_VM_DIR_ENTRIES); if(srcproc) assert(!RTS_ISSET(srcproc, RTS_SLOT_FREE)); if(dstproc) assert(!RTS_ISSET(dstproc, RTS_SLOT_FREE)); assert(!RTS_ISSET(get_cpulocal_var(ptproc), RTS_SLOT_FREE)); assert(get_cpulocal_var(ptproc)->p_seg.p_ttbr_v); if(srcproc) assert(!RTS_ISSET(srcproc, RTS_VMINHIBIT)); if(dstproc) assert(!RTS_ISSET(dstproc, RTS_VMINHIBIT)); while(bytes > 0) { phys_bytes srcptr, dstptr; vir_bytes chunk = bytes; int changed = 0; #ifdef CONFIG_SMP unsigned cpu = cpuid; if (srcproc && GET_BIT(srcproc->p_stale_tlb, cpu)) { changed = 1; UNSET_BIT(srcproc->p_stale_tlb, cpu); } if (dstproc && GET_BIT(dstproc->p_stale_tlb, cpu)) { changed = 1; UNSET_BIT(dstproc->p_stale_tlb, cpu); } #endif /* Set up 1MB ranges. */ srcptr = createpde(srcproc, srclinaddr, &chunk, 0, &changed); dstptr = createpde(dstproc, dstlinaddr, &chunk, 1, &changed); if(changed) { reload_ttbr0(); } /* Copy pages. */ PHYS_COPY_CATCH(srcptr, dstptr, chunk, addr); if(addr) { /* If addr is nonzero, a page fault was caught. * * phys_copy does all memory accesses word-aligned (rounded * down), so pagefaults can occur at a lower address than * the specified offsets. compute the lower bounds for sanity * check use. */ vir_bytes src_aligned = srcptr & ~0x3, dst_aligned = dstptr & ~0x3; if(addr >= src_aligned && addr < (srcptr + chunk)) { return EFAULT_SRC; } if(addr >= dst_aligned && addr < (dstptr + chunk)) { return EFAULT_DST; } panic("lin_lin_copy fault out of range"); /* Not reached. */ return EFAULT; } /* Update counter and addresses for next iteration, if any. */ bytes -= chunk; srclinaddr += chunk; dstlinaddr += chunk; } if(srcproc) assert(!RTS_ISSET(srcproc, RTS_SLOT_FREE)); if(dstproc) assert(!RTS_ISSET(dstproc, RTS_SLOT_FREE)); assert(!RTS_ISSET(get_cpulocal_var(ptproc), RTS_SLOT_FREE)); assert(get_cpulocal_var(ptproc)->p_seg.p_ttbr_v); return OK; } static u32_t phys_get32(phys_bytes addr) { const u32_t v; int r; if((r=lin_lin_copy(NULL, addr, proc_addr(SYSTEM), (phys_bytes) &v, sizeof(v))) != OK) { panic("lin_lin_copy for phys_get32 failed: %d", r); } return v; } /*===========================================================================* * umap_virtual * *===========================================================================*/ phys_bytes umap_virtual(rp, seg, vir_addr, bytes) register struct proc *rp; /* pointer to proc table entry for process */ int seg; /* T, D, or S segment */ vir_bytes vir_addr; /* virtual address in bytes within the seg */ vir_bytes bytes; /* # of bytes to be copied */ { phys_bytes phys = 0; if(vm_lookup(rp, vir_addr, &phys, NULL) != OK) { printf("SYSTEM:umap_virtual: vm_lookup of %s: seg 0x%x: 0x%lx failed\n", rp->p_name, seg, vir_addr); phys = 0; } else { if(phys == 0) panic("vm_lookup returned phys: %d", phys); } if(phys == 0) { printf("SYSTEM:umap_virtual: lookup failed\n"); return 0; } /* Now make sure addresses are contiguous in physical memory * so that the umap makes sense. */ if(bytes > 0 && vm_lookup_range(rp, vir_addr, NULL, bytes) != bytes) { printf("umap_virtual: %s: %lu at 0x%lx (vir 0x%lx) not contiguous\n", rp->p_name, bytes, vir_addr, vir_addr); return 0; } /* phys must be larger than 0 (or the caller will think the call * failed), and address must not cross a page boundary. */ assert(phys); return phys; } /*===========================================================================* * vm_lookup * *===========================================================================*/ int vm_lookup(const struct proc *proc, const vir_bytes virtual, phys_bytes *physical, u32_t *ptent) { u32_t *root, *pt; int pde, pte; u32_t pde_v, pte_v; assert(proc); assert(physical); assert(!isemptyp(proc)); assert(HASPT(proc)); /* Retrieve page directory entry. */ root = (u32_t *) proc->p_seg.p_ttbr; assert(!((u32_t) root % ARM_PAGEDIR_SIZE)); pde = ARM_VM_PDE(virtual); assert(pde >= 0 && pde < ARM_VM_DIR_ENTRIES); pde_v = phys_get32((u32_t) (root + pde)); if(!(pde_v & ARM_VM_PDE_PRESENT)) { return EFAULT; } /* We don't expect to ever see this. * LSC Impossible with the previous test. if(pde_v & ARM_VM_SECTION) { *physical = pde_v & ARM_VM_SECTION_MASK; if(ptent) *ptent = pde_v; *physical += virtual & ARM_VM_OFFSET_MASK_1MB; } else */ { /* Retrieve page table entry. */ pt = (u32_t *) (pde_v & ARM_VM_PDE_MASK); assert(!((u32_t) pt % ARM_PAGETABLE_SIZE)); pte = ARM_VM_PTE(virtual); assert(pte >= 0 && pte < ARM_VM_PT_ENTRIES); pte_v = phys_get32((u32_t) (pt + pte)); if(!(pte_v & ARM_VM_PTE_PRESENT)) { return EFAULT; } if(ptent) *ptent = pte_v; /* Actual address now known; retrieve it and add page offset. */ *physical = pte_v & ARM_VM_PTE_MASK; *physical += virtual % ARM_PAGE_SIZE; } return OK; } /*===========================================================================* * vm_lookup_range * *===========================================================================*/ size_t vm_lookup_range(const struct proc *proc, vir_bytes vir_addr, phys_bytes *phys_addr, size_t bytes) { /* Look up the physical address corresponding to linear virtual address * 'vir_addr' for process 'proc'. Return the size of the range covered * by contiguous physical memory starting from that address; this may * be anywhere between 0 and 'bytes' inclusive. If the return value is * nonzero, and 'phys_addr' is non-NULL, 'phys_addr' will be set to the * base physical address of the range. 'vir_addr' and 'bytes' need not * be page-aligned, but the caller must have verified that the given * linear range is valid for the given process at all. */ phys_bytes phys, next_phys; size_t len; assert(proc); assert(bytes > 0); assert(HASPT(proc)); /* Look up the first page. */ if (vm_lookup(proc, vir_addr, &phys, NULL) != OK) return 0; if (phys_addr != NULL) *phys_addr = phys; len = ARM_PAGE_SIZE - (vir_addr % ARM_PAGE_SIZE); vir_addr += len; next_phys = phys + len; /* Look up any next pages and test physical contiguity. */ while (len < bytes) { if (vm_lookup(proc, vir_addr, &phys, NULL) != OK) break; if (next_phys != phys) break; len += ARM_PAGE_SIZE; vir_addr += ARM_PAGE_SIZE; next_phys += ARM_PAGE_SIZE; } /* We might now have overshot the requested length somewhat. */ return MIN(bytes, len); } /*===========================================================================* * vm_suspend * *===========================================================================*/ static void vm_suspend(struct proc *caller, const struct proc *target, const vir_bytes linaddr, const vir_bytes len, const int type) { /* This range is not OK for this process. Set parameters * of the request and notify VM about the pending request. */ assert(!RTS_ISSET(caller, RTS_VMREQUEST)); assert(!RTS_ISSET(target, RTS_VMREQUEST)); RTS_SET(caller, RTS_VMREQUEST); caller->p_vmrequest.req_type = VMPTYPE_CHECK; caller->p_vmrequest.target = target->p_endpoint; caller->p_vmrequest.params.check.start = linaddr; caller->p_vmrequest.params.check.length = len; caller->p_vmrequest.params.check.writeflag = 1; caller->p_vmrequest.type = type; /* Connect caller on vmrequest wait queue. */ if(!(caller->p_vmrequest.nextrequestor = vmrequest)) if(OK != send_sig(VM_PROC_NR, SIGKMEM)) panic("send_sig failed"); vmrequest = caller; } /*===========================================================================* * vm_check_range * *===========================================================================*/ int vm_check_range(struct proc *caller, struct proc *target, vir_bytes vir_addr, size_t bytes) { /* Public interface to vm_suspend(), for use by kernel calls. On behalf * of 'caller', call into VM to check linear virtual address range of * process 'target', starting at 'vir_addr', for 'bytes' bytes. This * function assumes that it will called twice if VM returned an error * the first time (since nothing has changed in that case), and will * then return the error code resulting from the first call. Upon the * first call, a non-success error code is returned as well. */ int r; if ((caller->p_misc_flags & MF_KCALL_RESUME) && (r = caller->p_vmrequest.vmresult) != OK) return r; vm_suspend(caller, target, vir_addr, bytes, VMSTYPE_KERNELCALL); return VMSUSPEND; } /*===========================================================================* * delivermsg * *===========================================================================*/ void delivermsg(struct proc *rp) { int r = OK; assert(rp->p_misc_flags & MF_DELIVERMSG); assert(rp->p_delivermsg.m_source != NONE); if (copy_msg_to_user(&rp->p_delivermsg, (message *) rp->p_delivermsg_vir)) { printf("WARNING wrong user pointer 0x%08lx from " "process %s / %d\n", rp->p_delivermsg_vir, rp->p_name, rp->p_endpoint); r = EFAULT; } /* Indicate message has been delivered; address is 'used'. */ rp->p_delivermsg.m_source = NONE; rp->p_misc_flags &= ~MF_DELIVERMSG; if(!(rp->p_misc_flags & MF_CONTEXT_SET)) { rp->p_reg.retreg = r; } } /*===========================================================================* * vmmemset * *===========================================================================*/ int vm_memset(struct proc* caller, endpoint_t who, phys_bytes ph, int c, phys_bytes count) { u32_t pattern; struct proc *whoptr = NULL; phys_bytes cur_ph = ph; phys_bytes left = count; phys_bytes ptr, chunk, pfa = 0; int new_ttbr, r = OK; if ((r = check_resumed_caller(caller)) != OK) return r; /* NONE for physical, otherwise virtual */ if (who != NONE && !(whoptr = endpoint_lookup(who))) return ESRCH; c &= 0xFF; pattern = c | (c << 8) | (c << 16) | (c << 24); assert(get_cpulocal_var(ptproc)->p_seg.p_ttbr_v); assert(!catch_pagefaults); catch_pagefaults = 1; /* We can memset as many bytes as we have remaining, * or as many as remain in the 1MB chunk we mapped in. */ while (left > 0) { new_ttbr = 0; chunk = left; ptr = createpde(whoptr, cur_ph, &chunk, 0, &new_ttbr); if (new_ttbr) { reload_ttbr0(); } /* If a page fault happens, pfa is non-null */ if ((pfa = phys_memset(ptr, pattern, chunk))) { /* If a process pagefaults, VM may help out */ if (whoptr) { vm_suspend(caller, whoptr, ph, count, VMSTYPE_KERNELCALL); assert(catch_pagefaults); catch_pagefaults = 0; return VMSUSPEND; } /* Pagefault when phys copying ?! */ panic("vm_memset: pf %lx addr=%lx len=%lu\n", pfa , ptr, chunk); } cur_ph += chunk; left -= chunk; } assert(get_cpulocal_var(ptproc)->p_seg.p_ttbr_v); assert(catch_pagefaults); catch_pagefaults = 0; return OK; } /*===========================================================================* * virtual_copy_f * *===========================================================================*/ int virtual_copy_f(caller, src_addr, dst_addr, bytes, vmcheck) struct proc * caller; struct vir_addr *src_addr; /* source virtual address */ struct vir_addr *dst_addr; /* destination virtual address */ vir_bytes bytes; /* # of bytes to copy */ int vmcheck; /* if nonzero, can return VMSUSPEND */ { /* Copy bytes from virtual address src_addr to virtual address dst_addr. */ struct vir_addr *vir_addr[2]; /* virtual source and destination address */ int i, r; struct proc *procs[2]; assert((vmcheck && caller) || (!vmcheck && !caller)); /* Check copy count. */ if (bytes <= 0) return(EDOM); /* Do some more checks and map virtual addresses to physical addresses. */ vir_addr[_SRC_] = src_addr; vir_addr[_DST_] = dst_addr; for (i=_SRC_; i<=_DST_; i++) { endpoint_t proc_e = vir_addr[i]->proc_nr_e; int proc_nr; struct proc *p; if(proc_e == NONE) { p = NULL; } else { if(!isokendpt(proc_e, &proc_nr)) { printf("virtual_copy: no reasonable endpoint\n"); return ESRCH; } p = proc_addr(proc_nr); } procs[i] = p; } if ((r = check_resumed_caller(caller)) != OK) return r; if((r=lin_lin_copy(procs[_SRC_], vir_addr[_SRC_]->offset, procs[_DST_], vir_addr[_DST_]->offset, bytes)) != OK) { struct proc *target = NULL; phys_bytes lin; if(r != EFAULT_SRC && r != EFAULT_DST) panic("lin_lin_copy failed: %d", r); if(!vmcheck || !caller) { return r; } if(r == EFAULT_SRC) { lin = vir_addr[_SRC_]->offset; target = procs[_SRC_]; } else if(r == EFAULT_DST) { lin = vir_addr[_DST_]->offset; target = procs[_DST_]; } else { panic("r strange: %d", r); } assert(caller); assert(target); vm_suspend(caller, target, lin, bytes, VMSTYPE_KERNELCALL); return VMSUSPEND; } return OK; } /*===========================================================================* * data_copy * *===========================================================================*/ int data_copy(const endpoint_t from_proc, const vir_bytes from_addr, const endpoint_t to_proc, const vir_bytes to_addr, size_t bytes) { struct vir_addr src, dst; src.offset = from_addr; dst.offset = to_addr; src.proc_nr_e = from_proc; dst.proc_nr_e = to_proc; assert(src.proc_nr_e != NONE); assert(dst.proc_nr_e != NONE); return virtual_copy(&src, &dst, bytes); } /*===========================================================================* * data_copy_vmcheck * *===========================================================================*/ int data_copy_vmcheck(struct proc * caller, const endpoint_t from_proc, const vir_bytes from_addr, const endpoint_t to_proc, const vir_bytes to_addr, size_t bytes) { struct vir_addr src, dst; src.offset = from_addr; dst.offset = to_addr; src.proc_nr_e = from_proc; dst.proc_nr_e = to_proc; assert(src.proc_nr_e != NONE); assert(dst.proc_nr_e != NONE); return virtual_copy_vmcheck(caller, &src, &dst, bytes); } void memory_init(void) { assert(nfreepdes == 0); freepdes[nfreepdes++] = kinfo.freepde_start++; freepdes[nfreepdes++] = kinfo.freepde_start++; assert(kinfo.freepde_start < ARM_VM_DIR_ENTRIES); assert(nfreepdes == 2); assert(nfreepdes <= MAXFREEPDES); } /*===========================================================================* * arch_proc_init * *===========================================================================*/ void arch_proc_init(struct proc *pr, const u32_t ip, const u32_t sp, char *name) { arch_proc_reset(pr); strcpy(pr->p_name, name); /* set custom state we know */ pr->p_reg.pc = ip; pr->p_reg.sp = sp; } static int device_mem_mapping_index = -1, frclock_index = -1, usermapped_glo_index = -1, usermapped_index = -1, first_um_idx = -1; char *device_mem; /* defined in kernel.lds */ extern char usermapped_start, usermapped_end, usermapped_nonglo_start; int arch_phys_map(const int index, phys_bytes *addr, phys_bytes *len, int *flags) { static int first = 1; int freeidx = 0; u32_t glo_len = (u32_t) &usermapped_nonglo_start - (u32_t) &usermapped_start; if(first) { memset(&minix_kerninfo, 0, sizeof(minix_kerninfo)); device_mem_mapping_index = freeidx++; frclock_index = freeidx++; if(glo_len > 0) { usermapped_glo_index = freeidx++; } usermapped_index = freeidx++; first_um_idx = usermapped_index; if(usermapped_glo_index != -1) first_um_idx = usermapped_glo_index; first = 0; } if(index == usermapped_glo_index) { *addr = vir2phys(&usermapped_start); *len = glo_len; *flags = VMMF_USER | VMMF_GLO; return OK; } else if(index == usermapped_index) { *addr = vir2phys(&usermapped_nonglo_start); *len = (u32_t) &usermapped_end - (u32_t) &usermapped_nonglo_start; *flags = VMMF_USER; return OK; } else if (index == device_mem_mapping_index) { #ifdef DM37XX /* map device memory */ *addr = 0x48000000; *len = 0x02000000; #endif #ifdef AM335X /* map device memory until 0x5700 SGX */ *addr = 0x44000000; *len = 0x06000000; #endif *flags = VMMF_UNCACHED | VMMF_WRITE; return OK; } else if (index == frclock_index) { #ifdef DM37XX *addr = OMAP3_GPTIMER10_BASE; #endif #ifdef AM335X *addr = AM335X_DMTIMER7_BASE; #endif *len = ARM_PAGE_SIZE; *flags = VMMF_USER; return OK; } return EINVAL; } int arch_phys_map_reply(const int index, const vir_bytes addr) { if(index == first_um_idx) { u32_t usermapped_offset; assert(addr > (u32_t) &usermapped_start); usermapped_offset = addr - (u32_t) &usermapped_start; #define FIXEDPTR(ptr) (void *) ((u32_t)ptr + usermapped_offset) #define FIXPTR(ptr) ptr = FIXEDPTR(ptr) #define ASSIGN(minixstruct) minix_kerninfo.minixstruct = FIXEDPTR(&minixstruct) ASSIGN(kinfo); ASSIGN(machine); ASSIGN(kmessages); ASSIGN(loadinfo); /* adjust the pointers of the functions and the struct * itself to the user-accessible mapping */ minix_kerninfo.kerninfo_magic = KERNINFO_MAGIC; minix_kerninfo.minix_feature_flags = minix_feature_flags; minix_kerninfo_user = (vir_bytes) FIXEDPTR(&minix_kerninfo); return OK; } if (index == usermapped_index) { return OK; } else if (index == device_mem_mapping_index) { device_mem_vaddr = addr; return OK; } else if (index == frclock_index) { #ifdef DM37XX minix_kerninfo.minix_frclock = addr; #endif #ifdef AM335X minix_kerninfo.minix_frclock = addr; #endif return OK; } return EINVAL; } int arch_enable_paging(struct proc * caller) { assert(caller->p_seg.p_ttbr); /* load caller's page table */ switch_address_space(caller); device_mem = (char *) device_mem_vaddr; return OK; } void release_address_space(struct proc *pr) { pr->p_seg.p_ttbr_v = NULL; barrier(); }