Final word on the locking fiasco?

Change pushcli / popcli so that they can never turn on
interrupts unexpectedly.  That is, if interrupts are on,
then pushcli(); popcli(); turns them off and back on, but
if they are off to begin with, then pushcli(); popcli(); is
a no-op.

I think our fundamental mistake was having a primitive
(release and then popcli nee spllo) that could turn
interrupts on at unexpected moments instead of being
explicit about when we want to start allowing interrupts.

With the new semantics, all the manual fiddling of ncli
to force interrupts off in certain sections goes away.
In return, we must explicitly mark the places where
we want to enable interrupts unconditionally, by calling sti().
There is only one: inside the scheduler loop.

main.c

@@ -12,19 +12,13 @@ static void mpmain(void) __attribute__((noreturn));
 int
 main(void)
 {
-  int bcpu, i;
   extern char edata[], end[];
 
   // clear BSS
   memset(edata, 0, end - edata);
 
-  // pushcli() every processor during bootstrap.
-  for(i=0; i<NCPU; i++)
-    cpus[i].ncli = 1;  // no interrupts during bootstrap
-
   mp_init(); // collect info about this machine
-  bcpu = mp_bcpu();
+  lapic_init(mp_bcpu());
-  lapic_init(bcpu);
   cprintf("\ncpu%d: starting xv6\n\n", cpu());
 
   pinit();         // process table
@@ -40,17 +34,13 @@ main(void)
   if(!ismp)
     timer_init();  // uniprocessor timer
   userinit();      // first user process
+  bootothers();    // start other processors
 
-  // Allocate scheduler stacks and boot the other CPUs.
+  // Finish setting up this processor in mpmain.
-  for(i=0; i<ncpu; i++)
-    cpus[i].stack = kalloc(KSTACKSIZE);
-  bootothers();
-
-  // Switch to our scheduler stack and continue with mpmain.
-  asm volatile("movl %0, %%esp" : : "r" (cpus[bcpu].stack+KSTACKSIZE));
   mpmain();
 }
 
+// Bootstrap processor gets here after setting up the hardware.
 // Additional processors start here.
 static void
 mpmain(void)
@@ -62,7 +52,6 @@ mpmain(void)
   setupsegs(0);
   cpuid(0, 0, 0, 0, 0);  // memory barrier
   cpus[cpu()].booted = 1;
-  popcli();
 
   scheduler();
 }
@@ -73,6 +62,7 @@ bootothers(void)
   extern uchar _binary_bootother_start[], _binary_bootother_size[];
   uchar *code;
   struct cpu *c;
+  char *stack;
 
   // Write bootstrap code to unused memory at 0x7000.
   code = (uchar*)0x7000;
@@ -83,7 +73,8 @@ bootothers(void)
       continue;
 
     // Fill in %esp, %eip and start code on cpu.
-    *(void**)(code-4) = c->stack + KSTACKSIZE;
+    stack = kalloc(KSTACKSIZE);
+    *(void**)(code-4) = stack + KSTACKSIZE;
     *(void**)(code-8) = mpmain;
     lapic_startap(c->apicid, (uint)code);
 

proc.c

@@ -179,7 +179,6 @@ userinit(void)
 }
 
 // Return currently running process.
-// XXX comment better
 struct proc*
 curproc(void)
 {
@@ -206,11 +205,13 @@ scheduler(void)
   struct cpu *c;
   int i;
 
+  c = &cpus[cpu()];
   for(;;){
+    // Enable interrupts on this processor.
+    sti();
+
     // Loop over process table looking for process to run.
     acquire(&proc_table_lock);
-    
-    c = &cpus[cpu()];
     for(i = 0; i < NPROC; i++){
       p = &proc[i];
       if(p->state != RUNNABLE)
@@ -229,8 +230,8 @@ scheduler(void)
       c->curproc = 0;
       setupsegs(0);
     }
-
     release(&proc_table_lock);
+
   }
 }
 

proc.h

@@ -56,9 +56,9 @@ struct cpu {
   struct context context;     // Switch here to enter scheduler
   struct taskstate ts;        // Used by x86 to find stack for interrupt
   struct segdesc gdt[NSEGS];  // x86 global descriptor table
-  char *stack;
   volatile int booted;        // Has the CPU started?
   int ncli;                   // Depth of pushcli nesting.
+  int intena;                 // Were interrupts enabled before pushcli? 
 };
 
 extern struct cpu cpus[NCPU];

spinlock.c

@@ -88,15 +88,19 @@ holding(struct spinlock *lock)
 }
 
 
-
+// Pushcli/popcli are like cli/sti except that they are matched:
-// XXX!
+// it takes two popcli to undo two pushcli.  Also, if interrupts
-// Better names?  Better functions?  
+// are off, then pushcli, popcli leaves them off.
 
 void
 pushcli(void)
 {
+  int eflags;
+  
+  eflags = read_eflags();
   cli();
-  cpus[cpu()].ncli++;
+  if(cpus[cpu()].ncli++ == 0)
+    cpus[cpu()].intena = eflags & FL_IF;
 }
 
 void
@@ -106,7 +110,7 @@ popcli(void)
     panic("popcli - interruptible");
   if(--cpus[cpu()].ncli < 0)
     panic("popcli");
-  if(cpus[cpu()].ncli == 0)
+  if(cpus[cpu()].ncli == 0 && cpus[cpu()].intena)
     sti();
 }
 

trap.c

@@ -44,9 +44,6 @@ trap(struct trapframe *tf)
     return;
   }
 
-  // No interrupts during interrupt handling.
-  pushcli();
-
   switch(tf->trapno){
   case IRQ_OFFSET + IRQ_TIMER:
     if(cpu() == 0){
@@ -84,8 +81,6 @@ trap(struct trapframe *tf)
     cp->killed = 1;
   }
 
-  popcli();
-
   // Force process exit if it has been killed and is in user space.
   // (If it is still executing in the kernel, let it keep running 
   // until it gets to the regular system call return.)