From fac5fbfdbfcc17bd630b82a0931426801fefcfdf Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Wed, 15 Sep 2010 14:10:18 +0000 Subject: [PATCH] SMP - CPU local run queues - each CPU has its own runqueues - processes on BSP are put on the runqueues later after a switch to the final stack when cpuid works to avoid special cases - enqueue() and dequeue() use the run queues of the cpu the process is assigned to - pick_proc() uses the local run queues - printing of per-CPU run queues ('2') on serial console --- kernel/arch/i386/arch_smp.c | 6 ++- kernel/arch/i386/arch_system.c | 29 ++++++++++-- kernel/arch/i386/glo.h | 2 +- kernel/clock.c | 2 + kernel/cpulocals.h | 5 ++ kernel/debug.c | 34 +++++++++++++- kernel/main.c | 22 ++++++--- kernel/proc.c | 86 +++++++++++++++++----------------- kernel/proc.h | 2 - kernel/proto.h | 6 +++ 10 files changed, 134 insertions(+), 60 deletions(-) diff --git a/kernel/arch/i386/arch_smp.c b/kernel/arch/i386/arch_smp.c index 360786a33..f381b7b9a 100644 --- a/kernel/arch/i386/arch_smp.c +++ b/kernel/arch/i386/arch_smp.c @@ -215,8 +215,12 @@ PRIVATE void ap_finish_booting(void) } printf("CPU %d local APIC timer is ticking\n", cpu); + /* FIXME assign CPU local idle structure */ + get_cpulocal_var(proc_ptr) = proc_addr(IDLE); + get_cpulocal_var(bill_ptr) = proc_addr(IDLE); + BKL_UNLOCK(); - + ap_boot_finished(cpu); spinlock_unlock(&boot_lock); for(;;); diff --git a/kernel/arch/i386/arch_system.c b/kernel/arch/i386/arch_system.c index bf0f39934..84fd53892 100644 --- a/kernel/arch/i386/arch_system.c +++ b/kernel/arch/i386/arch_system.c @@ -370,19 +370,38 @@ PUBLIC void do_ser_debug() ser_debug(c); } -PRIVATE void ser_dump_queues(void) +PRIVATE void ser_dump_queue_cpu(unsigned cpu) { int q; + struct proc ** rdy_head; + + rdy_head = get_cpu_var(cpu, run_q_head); + for(q = 0; q < NR_SCHED_QUEUES; q++) { struct proc *p; - if(rdy_head[q]) + if(rdy_head[q]) { printf("%2d: ", q); - for(p = rdy_head[q]; p; p = p->p_nextready) { - printf("%s / %d ", p->p_name, p->p_endpoint); + for(p = rdy_head[q]; p; p = p->p_nextready) { + printf("%s / %d ", p->p_name, p->p_endpoint); + } + printf("\n"); } - printf("\n"); } +} +PRIVATE void ser_dump_queues(void) +{ +#ifdef CONFIG_SMP + unsigned cpu; + + printf("--- run queues ---\n"); + for (cpu = 0; cpu < ncpus; cpu++) { + printf("CPU %d :\n", cpu); + ser_dump_queue_cpu(cpu); + } +#else + ser_dump_queue_cpu(0); +#endif } PRIVATE void ser_dump_segs(void) diff --git a/kernel/arch/i386/glo.h b/kernel/arch/i386/glo.h index aaa910899..27437def8 100644 --- a/kernel/arch/i386/glo.h +++ b/kernel/arch/i386/glo.h @@ -2,7 +2,7 @@ #define __GLO_X86_H__ #include "kernel/kernel.h" -#include "proto.h" +#include "arch_proto.h" EXTERN int cpu_has_tsc; /* signal whether this cpu has time stamp register. This feature was introduced by Pentium */ diff --git a/kernel/clock.c b/kernel/clock.c index 3f848b3cb..e8920098b 100644 --- a/kernel/clock.c +++ b/kernel/clock.c @@ -194,6 +194,7 @@ PRIVATE void load_update(void) u16_t slot; int enqueued = 0, q; struct proc *p; + struct proc **rdy_head; /* Load average data is stored as a list of numbers in a circular * buffer. Each slot accumulates _LOAD_UNIT_SECS of samples of @@ -207,6 +208,7 @@ PRIVATE void load_update(void) kloadinfo.proc_last_slot = slot; } + rdy_head = get_cpulocal_var(run_q_head); /* Cumulation. How many processes are ready now? */ for(q = 0; q < NR_SCHED_QUEUES; q++) { for(p = rdy_head[q]; p != NULL; p = p->p_nextready) { diff --git a/kernel/cpulocals.h b/kernel/cpulocals.h index e16169a88..9c9be2db0 100644 --- a/kernel/cpulocals.h +++ b/kernel/cpulocals.h @@ -5,6 +5,7 @@ #ifndef __ASSEMBLY__ #include "kernel.h" +#include "proc.h" #ifdef CONFIG_SMP @@ -68,6 +69,10 @@ DECLARE_CPULOCAL(int, pagefault_handled); */ DECLARE_CPULOCAL(struct proc *, ptproc); +/* CPU private run queues */ +DECLARE_CPULOCAL(struct proc *, run_q_head[NR_SCHED_QUEUES]); /* ptrs to ready list headers */ +DECLARE_CPULOCAL(struct proc *, run_q_tail[NR_SCHED_QUEUES]); /* ptrs to ready list tails */ + DECLARE_CPULOCAL_END #endif /* __ASSEMBLY__ */ diff --git a/kernel/debug.c b/kernel/debug.c index f6b71205f..e476c5439 100644 --- a/kernel/debug.c +++ b/kernel/debug.c @@ -14,11 +14,14 @@ #define MAX_LOOP (NR_PROCS + NR_TASKS) -PUBLIC int -runqueues_ok(void) +PUBLIC int runqueues_ok_cpu(unsigned cpu) { int q, l = 0; register struct proc *xp; + struct proc **rdy_head, **rdy_tail; + + rdy_head = get_cpu_var(cpu, run_q_head); + rdy_tail = get_cpu_var(cpu, run_q_tail); for (xp = BEG_PROC_ADDR; xp < END_PROC_ADDR; ++xp) { xp->p_found = 0; @@ -109,6 +112,33 @@ runqueues_ok(void) return 1; } +#ifdef CONFIG_SMP +PRIVATE int runqueues_ok_all(void) +{ + unsigned c; + + for (c = 0 ; c < ncpus; c++) { + if (!runqueues_ok_cpu(c)) + return 0; + } + return 1; +} + +PUBLIC int runqueues_ok(void) +{ + return runqueues_ok_all(); +} + +#else + +PUBLIC int runqueues_ok(void) +{ + return runqueues_ok_cpu(0); +} + + +#endif + PUBLIC char * rtsflagstr(const int flags) { diff --git a/kernel/main.c b/kernel/main.c index 043926341..730cc30aa 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -32,8 +32,10 @@ /* Prototype declarations for PRIVATE functions. */ FORWARD _PROTOTYPE( void announce, (void)); +void ser_dump_queues(void); PUBLIC void bsp_finish_booting(void) { + int i; #if SPROFILE sprofiling = 0; /* we're not profiling until instructed to */ #endif /* SPROFILE */ @@ -49,6 +51,13 @@ PUBLIC void bsp_finish_booting(void) get_cpulocal_var(bill_ptr) = proc_addr(IDLE); /* it has to point somewhere */ announce(); /* print MINIX startup banner */ + /* + * we have access to the cpu local run queue, only now schedule the processes. + * We ignore the slots for the former kernel tasks + */ + for (i=0; i < NR_BOOT_PROCS - NR_TASKS; i++) { + RTS_UNSET(proc_addr(i), RTS_PROC_STOP); + } /* * enable timer interrupts and clock task on the boot CPU */ @@ -75,8 +84,10 @@ PUBLIC void bsp_finish_booting(void) cycles_accounting_init(); DEBUGEXTRA(("done\n")); - assert(runqueues_ok()); - +#ifdef CONFIG_SMP + cpu_set_flag(bsp_cpu_id, CPU_IS_READY); +#endif + switch_to_user(); NOT_REACHABLE; } @@ -246,11 +257,10 @@ PUBLIC int main(void) * done this; until then, don't let it run. */ if(ip->flags & PROC_FULLVM) - RTS_SET(rp, RTS_VMINHIBIT); + rp->p_rts_flags |= RTS_VMINHIBIT; - /* None of the kernel tasks run */ - if (rp->p_nr < 0) RTS_SET(rp, RTS_PROC_STOP); - RTS_UNSET(rp, RTS_SLOT_FREE); /* remove RTS_SLOT_FREE and schedule */ + rp->p_rts_flags |= RTS_PROC_STOP; + rp->p_rts_flags &= ~RTS_SLOT_FREE; alloc_segments(rp); DEBUGEXTRA(("done\n")); } diff --git a/kernel/proc.c b/kernel/proc.c index 70938a9d9..90d5d87ba 100644 --- a/kernel/proc.c +++ b/kernel/proc.c @@ -110,6 +110,18 @@ PUBLIC void proc_init(void) } } +PRIVATE void switch_address_space_idle(void) +{ +#ifdef CONFIG_SMP + /* + * currently we bet that VM is always alive and its pages available so + * when the CPU wakes up the kernel is mapped and no surprises happen. + * This is only a problem if more than 1 cpus are available + */ + switch_address_space(proc_addr(VM_PROC_NR)); +#endif +} + /*===========================================================================* * idle * *===========================================================================*/ @@ -121,6 +133,8 @@ PRIVATE void idle(void) * the CPU utiliziation of certain workloads with high precision. */ + switch_address_space_idle(); + /* start accounting for the idle time */ context_stop(proc_addr(KERNEL)); halt_cpu(); @@ -1209,19 +1223,21 @@ PUBLIC void enqueue( * responsible for inserting a process into one of the scheduling queues. * The mechanism is implemented here. The actual scheduling policy is * defined in sched() and pick_proc(). + * + * This function can be used x-cpu as it always uses the queues of the cpu the + * process is assigned to. */ int q = rp->p_priority; /* scheduling queue to use */ struct proc * p; - -#if DEBUG_RACE - /* With DEBUG_RACE, schedule everyone at the same priority level. */ - rp->p_priority = q = MIN_USER_Q; -#endif - + struct proc **rdy_head, **rdy_tail; + assert(proc_is_runnable(rp)); assert(q >= 0); + rdy_head = get_cpu_var(rp->p_cpu, run_q_head); + rdy_tail = get_cpu_var(rp->p_cpu, run_q_tail); + /* Now add the process to the queue. */ if (!rdy_head[q]) { /* add to empty queue */ rdy_head[q] = rdy_tail[q] = rp; /* create a new queue */ @@ -1245,7 +1261,7 @@ PUBLIC void enqueue( RTS_SET(p, RTS_PREEMPTED); /* calls dequeue() */ #if DEBUG_SANITYCHECKS - assert(runqueues_ok()); + assert(runqueues_ok_local()); #endif } @@ -1262,6 +1278,8 @@ PRIVATE void enqueue_head(struct proc *rp) { const int q = rp->p_priority; /* scheduling queue to use */ + struct proc **rdy_head, **rdy_tail; + assert(proc_ptr_ok(rp)); assert(proc_is_runnable(rp)); @@ -1274,6 +1292,9 @@ PRIVATE void enqueue_head(struct proc *rp) assert(q >= 0); + rdy_head = get_cpu_var(rp->p_cpu, run_q_head); + rdy_tail = get_cpu_var(rp->p_cpu, run_q_tail); + /* Now add the process to the queue. */ if (!rdy_head[q]) { /* add to empty queue */ rdy_head[q] = rdy_tail[q] = rp; /* create a new queue */ @@ -1284,7 +1305,7 @@ PRIVATE void enqueue_head(struct proc *rp) rdy_head[q] = rp; /* set new queue head */ #if DEBUG_SANITYCHECKS - assert(runqueues_ok()); + assert(runqueues_ok_local()); #endif } @@ -1297,23 +1318,31 @@ PUBLIC void dequeue(const struct proc *rp) /* A process must be removed from the scheduling queues, for example, because * it has blocked. If the currently active process is removed, a new process * is picked to run by calling pick_proc(). + * + * This function can operate x-cpu as it always removes the process from the + * queue of the cpu the process is currently assigned to. */ register int q = rp->p_priority; /* queue to use */ register struct proc **xpp; /* iterate over queue */ register struct proc *prev_xp; + struct proc **rdy_tail; + assert(proc_ptr_ok(rp)); assert(!proc_is_runnable(rp)); /* Side-effect for kernel: check if the task's stack still is ok? */ assert (!iskernelp(rp) || *priv(rp)->s_stack_guard == STACK_GUARD); + rdy_tail = get_cpu_var(rp->p_cpu, run_q_tail); + /* Now make sure that the process is not in its ready queue. Remove the * process if it is found. A process can be made unready even if it is not * running by being sent a signal that kills it. */ prev_xp = NULL; - for (xpp = &rdy_head[q]; *xpp; xpp = &(*xpp)->p_nextready) { + for (xpp = get_cpu_var_ptr(rp->p_cpu, run_q_head[q]); *xpp; + xpp = &(*xpp)->p_nextready) { if (*xpp == rp) { /* found process to remove */ *xpp = (*xpp)->p_nextready; /* replace with next chain */ if (rp == rdy_tail[q]) { /* queue tail removed */ @@ -1326,36 +1355,10 @@ PUBLIC void dequeue(const struct proc *rp) } #if DEBUG_SANITYCHECKS - assert(runqueues_ok()); + assert(runqueues_ok_local()); #endif } -#if DEBUG_RACE -/*===========================================================================* - * random_process * - *===========================================================================*/ -PRIVATE struct proc *random_process(struct proc *head) -{ - int i, n = 0; - struct proc *rp; - u64_t r; - read_tsc_64(&r); - - for(rp = head; rp; rp = rp->p_nextready) - n++; - - /* Use low-order word of TSC as pseudorandom value. */ - i = r.lo % n; - - for(rp = head; i--; rp = rp->p_nextready) - ; - - assert(rp); - - return rp; -} -#endif - /*===========================================================================* * pick_proc * *===========================================================================*/ @@ -1364,26 +1367,23 @@ PRIVATE struct proc * pick_proc(void) /* Decide who to run now. A new process is selected an returned. * When a billable process is selected, record it in 'bill_ptr', so that the * clock task can tell who to bill for system time. + * + * This functions always uses the run queues of the local cpu! */ register struct proc *rp; /* process to run */ + struct proc **rdy_head; int q; /* iterate over queues */ /* Check each of the scheduling queues for ready processes. The number of * queues is defined in proc.h, and priorities are set in the task table. * The lowest queue contains IDLE, which is always ready. */ + rdy_head = get_cpulocal_var(run_q_head); for (q=0; q < NR_SCHED_QUEUES; q++) { if(!(rp = rdy_head[q])) { TRACE(VF_PICKPROC, printf("queue %d empty\n", q);); continue; } - -#if DEBUG_RACE - rp = random_process(rdy_head[q]); -#endif - - TRACE(VF_PICKPROC, printf("found %s / %d on queue %d\n", - rp->p_name, rp->p_endpoint, q);); assert(proc_is_runnable(rp)); if (priv(rp)->s_flags & BILLABLE) get_cpulocal_var(bill_ptr) = rp; /* bill for system time */ diff --git a/kernel/proc.h b/kernel/proc.h index 2a9b91579..e1fa0fc88 100644 --- a/kernel/proc.h +++ b/kernel/proc.h @@ -252,8 +252,6 @@ struct proc { #ifndef __ASSEMBLY__ EXTERN struct proc proc[NR_TASKS + NR_PROCS]; /* process table */ -EXTERN struct proc *rdy_head[NR_SCHED_QUEUES]; /* ptrs to ready list headers */ -EXTERN struct proc *rdy_tail[NR_SCHED_QUEUES]; /* ptrs to ready list tails */ _PROTOTYPE( int mini_send, (struct proc *caller_ptr, endpoint_t dst_e, message *m_ptr, int flags)); diff --git a/kernel/proto.h b/kernel/proto.h index f0923ee5e..0ac33a28c 100644 --- a/kernel/proto.h +++ b/kernel/proto.h @@ -102,6 +102,12 @@ _PROTOTYPE( int disable_irq, (const irq_hook_t *hook) ); /* debug.c */ _PROTOTYPE( int runqueues_ok, (void) ); +#ifndef CONFIG_SMP +#define runqueues_ok_local runqueues_ok +#else +#define runqueues_ok_local() runqueues_ok_cpu(cpuid) +_PROTOTYPE( int runqueues_ok_cpu, (unsigned cpu)); +#endif _PROTOTYPE( char *rtsflagstr, (int flags) ); _PROTOTYPE( char *miscflagstr, (int flags) ); _PROTOTYPE( char *schedulerstr, (struct proc *scheduler) );