Userspace scheduling

- cotributed by Bjorn Swift

- In this first phase, scheduling is moved from the kernel to the PM
  server. The next steps are to a) moving scheduling to its own server
  and b) include useful information in the "out of quantum" message,
  so that the scheduler can make use of this information.

- The kernel process table now keeps record of who is responsible for
  scheduling each process (p_scheduler). When this pointer is NULL,
  the process will be scheduled by the kernel. If such a process runs
  out of quantum, the kernel will simply renew its quantum an requeue
  it.

- When PM loads, it will take over scheduling of all running
  processes, except system processes, using sys_schedctl().
  Essentially, this only results in taking over init. As children
  inherit a scheduler from their parent, user space programs forked by
  init will inherit PM (for now) as their scheduler.

 - Once a process has been assigned a scheduler, and runs out of
   quantum, its RTS_NO_QUANTUM flag will be set and the process
   dequeued. The kernel will send a message to the scheduler, on the
   process' behalf, informing the scheduler that it has run out of
   quantum. The scheduler can take what ever action it pleases, based
   on its policy, and then reschedule the process using the
   sys_schedule() system call.

- Balance queues does not work as before. While the old in-kernel
  function used to renew the quantum of processes in the highest
  priority run queue, the user-space implementation only acts on
  processes that have been bumped down to a lower priority queue.
  This approach reacts slower to changes than the old one, but saves
  us sending a sys_schedule message for each process every time we
  balance the queues. Currently, when processes are moved up a
  priority queue, their quantum is also renewed, but this can be
  fiddled with.

- do_nice has been removed from kernel. PM answers to get- and
  setpriority calls, updates it's own nice variable as well as the
  max_run_queue. This will be refactored once scheduling is moved to a
  separate server. We will probably have PM update it's local nice
  value and then send a message to whoever is scheduling the process.

- changes to fix an issue in do_fork() where processes could run out
  of quantum but bypassing the code path that handles it correctly.
  The future plan is to remove the policy from do_fork() and implement
  it in userspace too.

include/minix/com.h

@@ -20,6 +20,7 @@
  *    0xC00 -  0xCFF	Virtual Memory (VM) requests
  *    0xD00 -  0xDFF	IPC server requests
  *    0xE00 -  0xEFF	Common system messages (e.g. system signals)
+ *    0xF00 -  0xFFF    Scheduling messages
  *   0x1000 - 0x10FF	Notify messages
  *
  * Zero and negative values are widely used for OK and error responses.
@@ -308,7 +309,7 @@
 #  define SYS_FORK       (KERNEL_CALL + 0)	/* sys_fork() */
 #  define SYS_EXEC       (KERNEL_CALL + 1)	/* sys_exec() */
 #  define SYS_CLEAR	 (KERNEL_CALL + 2)	/* sys_clear() */
-#  define SYS_NICE       (KERNEL_CALL + 3)	/* sys_nice() */
+#  define SYS_SCHEDULE 	 (KERNEL_CALL + 3)	/* sys_schedule() */
 #  define SYS_PRIVCTL    (KERNEL_CALL + 4)	/* sys_privctl() */
 #  define SYS_TRACE      (KERNEL_CALL + 5)	/* sys_trace() */
 #  define SYS_KILL       (KERNEL_CALL + 6)	/* sys_kill() */
@@ -363,8 +364,10 @@
 #  define SYS_UPDATE	 (KERNEL_CALL + 52)	/* sys_update() */
 #  define SYS_EXIT	 (KERNEL_CALL + 53)	/* sys_exit() */
 
+#  define SYS_SCHEDCTL (KERNEL_CALL + 54)	/* sys_schedctl() */
+
 /* Total */
-#define NR_SYS_CALLS	54	/* number of system calls */ 
+#define NR_SYS_CALLS	55	/* number of system calls */
 
 #define SYS_CALL_MASK_SIZE BITMAP_CHUNKS(NR_SYS_CALLS)
 
@@ -1048,4 +1051,15 @@
 #	define SEMOP_OPS	m2_l1
 #	define SEMOP_SIZE	m2_i2
 
-#endif /* _MINIX_COM_H */ 
+/*===========================================================================*
+ *                Messages for Scheduling				     *
+ *===========================================================================*/
+#define SCHEDULING_BASE	0xF00
+
+#define SCHEDULING_NO_QUANTUM	(SCHEDULING_BASE+1)
+#	define SCHEDULING_ENDPOINT	m1_i1
+#	define SCHEDULING_PRIORITY	m1_i2
+#	define SCHEDULING_QUANTUM	m1_i3
+#endif
+
+/* _MINIX_COM_H */

include/minix/syslib.h

@@ -44,6 +44,9 @@ _PROTOTYPE( int sys_clear, (endpoint_t proc_ep));
 _PROTOTYPE( int sys_exit, (void));
 _PROTOTYPE( int sys_trace, (int req, endpoint_t proc_ep, long addr, long *data_p));
 
+_PROTOTYPE( int sys_schedule, (endpoint_t proc_ep, char priority, char quantum));
+_PROTOTYPE( int sys_schedctl, (endpoint_t proc_ep));
+
 /* Shorthands for sys_runctl() system call. */
 #define sys_stop(proc_ep) sys_runctl(proc_ep, RC_STOP, 0)
 #define sys_delay_stop(proc_ep) sys_runctl(proc_ep, RC_STOP, RC_DELAY)
@@ -55,7 +58,6 @@ _PROTOTYPE( int sys_privctl, (endpoint_t proc_ep, int req, void *p));
 _PROTOTYPE( int sys_privquery_mem, (endpoint_t proc_ep,
 	phys_bytes physstart, phys_bytes physlen));
 _PROTOTYPE( int sys_setgrant, (cp_grant_t *grants, int ngrants));
-_PROTOTYPE( int sys_nice, (endpoint_t proc_ep, int priority));
 
 _PROTOTYPE( int sys_int86, (struct reg86u *reg86p));
 _PROTOTYPE( int sys_vm_setbuf, (phys_bytes base, phys_bytes size,

kernel/arch/i386/system.c

@@ -326,11 +326,12 @@ PRIVATE void printslot(struct proc *pp, const int level)
 
 	COL
 
-	printf("%d: %s %d prio %d/%d time %d/%d cycles 0x%x%08x cr3 0x%lx rts %s misc %s",
+	printf("%d: %s %d prio %d time %d/%d cycles 0x%x%08x cr3 0x%lx rts %s misc %s sched %s",
 		proc_nr(pp), pp->p_name, pp->p_endpoint, 
-		pp->p_priority, pp->p_max_priority, pp->p_user_time,
+		pp->p_priority, pp->p_user_time,
 		pp->p_sys_time, pp->p_cycles.hi, pp->p_cycles.lo, pp->p_seg.p_cr3,
-		rtsflagstr(pp->p_rts_flags), miscflagstr(pp->p_misc_flags));
+		rtsflagstr(pp->p_rts_flags), miscflagstr(pp->p_misc_flags),
+		schedulerstr(pp->p_scheduler));
 
 	if((dep = P_BLOCKEDON(pp)) != NONE) {
 		printf(" blocked on: ");

kernel/clock.c

@@ -32,6 +32,7 @@
 #include "kernel.h"
 #include "proc.h"
 #include <minix/endpoint.h>
+#include <assert.h>
 
 #include "clock.h"
 
@@ -57,18 +58,6 @@ PRIVATE clock_t next_timeout;	/* realtime that next timer expires */
  */
 PRIVATE clock_t realtime = 0;		      /* real time clock */
 
-/*===========================================================================*
- *				init_clock				     *
- *===========================================================================*/
-PUBLIC void clock_init()
-{
-   
-	/* Set a watchdog timer to periodically balance the scheduling queues.
-	   Side-effect sets new timer */
-
-	balance_queues(NULL);
-}
-
 /*
  * The boot processor timer interrupt handler. In addition to non-boot cpus it
  * keeps real time and notifies the clock task if need be
@@ -86,6 +75,7 @@ PUBLIC int bsp_timer_int_handler(void)
 	realtime += ticks;
 
 	ap_timer_int_handler();
+	assert(!proc_is_runnable(proc_ptr) || proc_ptr->p_ticks_left > 0);
 
 	/* if a timer expired, notify the clock task */
 	if ((next_timeout <= realtime)) {
@@ -214,7 +204,6 @@ PUBLIC int ap_timer_int_handler(void)
 	}
 	if (! (priv(p)->s_flags & BILLABLE)) {
 		billp->p_sys_time += ticks;
-		billp->p_ticks_left -= ticks;
 	}
 
 	/* Decrement virtual timers, if applicable. We decrement both the
@@ -247,12 +236,8 @@ PUBLIC int ap_timer_int_handler(void)
 	/* Update load average. */
 	load_update();
 
-	/* check if the process is still runnable after checking the vtimer */
+	/* check if the processes still have some ticks left */
-	if (p->p_rts_flags == 0 && p->p_ticks_left <= 0 &&
+	check_ticks_left(p);
-			priv(p)->s_flags & PREEMPTIBLE) {
-		/* this dequeues the process */
-		RTS_SET(p, RTS_NO_QUANTUM);
-	}
 
 	return 1;
 }

kernel/config.h

@@ -37,7 +37,6 @@
 #define USE_IRQCTL     	   1	/* set an interrupt policy */
 #define USE_SEGCTL     	   1	/* set up a remote segment */
 #define USE_PRIVCTL    	   1	/* system privileges control */
-#define USE_NICE 	   1	/* change scheduling priority */
 #define USE_UMAP       	   1	/* map virtual to physical address */
 #define USE_VIRCOPY   	   1	/* copy using virtual addressing */ 
 #define USE_PHYSCOPY  	   1 	/* copy using physical addressing */

kernel/debug.c

@@ -150,3 +150,14 @@ miscflagstr(const int flags)
 	return str;
 }
 
+PUBLIC char *
+schedulerstr(struct proc *scheduler)
+{
+	if (scheduler != NULL)
+	{
+		return scheduler->p_name;
+	}
+
+	return "KERNEL";
+}
+

kernel/ipc.h

@@ -8,6 +8,7 @@
 
 /* Masks and flags for system calls. */
 #define NON_BLOCKING    0x0080  /* do not block if target not ready */
+#define FROM_KERNEL     0x0100  /* message from kernel on behalf of a process */
 
 #define WILLRECEIVE(target, source_ep) \
   ((RTS_ISSET(target, RTS_RECEIVING) && !RTS_ISSET(target, RTS_SENDING)) &&	\

kernel/main.c

@@ -79,7 +79,7 @@ PUBLIC void main()
 	DEBUGMAX(("initializing %s... ", ip->proc_name));
 	rp = proc_addr(ip->proc_nr);		/* get process pointer */
 	ip->endpoint = rp->p_endpoint;		/* ipc endpoint */
-	rp->p_max_priority = ip->priority;	/* max scheduling priority */
+	rp->p_scheduler = NULL;			/* no user space scheduler */
 	rp->p_priority = ip->priority;		/* current priority */
 	rp->p_quantum_size = ip->quantum;	/* quantum size in ticks */
 	rp->p_ticks_left = ip->quantum;		/* current credit */
@@ -218,10 +218,6 @@ PUBLIC void main()
   DEBUGMAX(("system_init()... "));
   system_init();
   DEBUGMAX(("done\n"));
-  /* Initialize timers handling */
-  DEBUGMAX(("clock_init()... "));
-  clock_init();
-  DEBUGMAX(("done\n"));
 
 #if SPROFILE
   sprofiling = 0;      /* we're not profiling until instructed to */

kernel/proc.c

@@ -43,8 +43,11 @@
 
 /* Scheduling and message passing functions */
 FORWARD _PROTOTYPE( void idle, (void));
+/**
+ * Made public for use in clock.c (for user-space scheduling)
 FORWARD _PROTOTYPE( int mini_send, (struct proc *caller_ptr, int dst_e,
 		message *m_ptr, int flags));
+*/
 FORWARD _PROTOTYPE( int mini_receive, (struct proc *caller_ptr, int src,
 		message *m_ptr, int flags));
 FORWARD _PROTOTYPE( int mini_senda, (struct proc *caller_ptr,
@@ -54,7 +57,6 @@ FORWARD _PROTOTYPE( int deadlock, (int function,
 FORWARD _PROTOTYPE( int try_async, (struct proc *caller_ptr));
 FORWARD _PROTOTYPE( int try_one, (struct proc *src_ptr, struct proc *dst_ptr,
 		int *postponed));
-FORWARD _PROTOTYPE( void sched, (struct proc *rp, int *queue, int *front));
 FORWARD _PROTOTYPE( struct proc * pick_proc, (void));
 FORWARD _PROTOTYPE( void enqueue_head, (struct proc *rp));
 
@@ -164,9 +166,16 @@ not_runnable_pick_new:
 		if (proc_is_runnable(proc_ptr))
 			enqueue_head(proc_ptr);
 	}
-	/* this enqueues the process again */
+
-	if (proc_no_quantum(proc_ptr))
+	/*
+	 * If this process is scheduled by the kernel, we renew it's quantum
+	 * and remove it's RTS_NO_QUANTUM flag.
+	 */
+	if (proc_no_quantum(proc_ptr) && (proc_ptr->p_scheduler == NULL)) {
+		/* give new quantum */
+		proc_ptr->p_ticks_left = proc_ptr->p_quantum_size;
 		RTS_UNSET(proc_ptr, RTS_NO_QUANTUM);
+	}
 
 	/*
 	 * if we have no process to run, set IDLE as the current process for
@@ -187,6 +196,7 @@ check_misc_flags:
 
 	assert(proc_ptr);
 	assert(proc_is_runnable(proc_ptr));
+	assert(proc_ptr->p_ticks_left > 0);
 	while (proc_ptr->p_misc_flags &
 		(MF_KCALL_RESUME | MF_DELIVERMSG |
 		 MF_SC_DEFER | MF_SC_TRACE | MF_SC_ACTIVE)) {
@@ -247,20 +257,26 @@ check_misc_flags:
 			break;
 		}
 
-		/*
-		 * the selected process might not be runnable anymore. We have
-		 * to checkit and schedule another one
-		 */
 		if (!proc_is_runnable(proc_ptr))
-			goto not_runnable_pick_new;
+			break;
 	}
+	/*
+	 * After handling the misc flags the selected process might not be
+	 * runnable anymore. We have to checkit and schedule another one
+	 */
+	if (!proc_is_runnable(proc_ptr))
+		goto not_runnable_pick_new;
+
 	TRACE(VF_SCHEDULING, printf("starting %s / %d\n",
 		proc_ptr->p_name, proc_ptr->p_endpoint););
 #if DEBUG_TRACE
 	proc_ptr->p_schedules++;
 #endif
 
+
 	proc_ptr = arch_finish_schedcheck();
+	assert(proc_ptr->p_ticks_left > 0);
+
 	cycles_accounting_stop(proc_addr(KERNEL));
 
 	NOREC_RETURN(schedch, proc_ptr);
@@ -533,7 +549,7 @@ proc_nr_t src_dst;				/* src or dst process */
 /*===========================================================================*
  *				mini_send				     * 
  *===========================================================================*/
-PRIVATE int mini_send(caller_ptr, dst_e, m_ptr, flags)
+PUBLIC int mini_send(caller_ptr, dst_e, m_ptr, flags)
 register struct proc *caller_ptr;	/* who is trying to send a message? */
 int dst_e;				/* to whom is message being sent? */
 message *m_ptr;				/* pointer to message buffer */
@@ -545,12 +561,23 @@ const int flags;
  */
   register struct proc *dst_ptr;
   register struct proc **xpp;
+  struct proc *msg_proc_ptr; /* Proc in which the message can be found */
   int dst_p;
   phys_bytes linaddr;
   vir_bytes addr;
   int r;
 
-  if(!(linaddr = umap_local(caller_ptr, D, (vir_bytes) m_ptr,
+  /* The kernel can send messages on behalf of other processes. In that case
+     we need to pass a pointer to the kernel, not the caller_ptr, to
+     umap_local as the kernel contains the message to be sent. */
+  if (flags & FROM_KERNEL) {
+	msg_proc_ptr = proc_addr(KERNEL);
+  }
+  else {
+	msg_proc_ptr = caller_ptr;
+  }
+
+  if(!(linaddr = umap_local(msg_proc_ptr, D, (vir_bytes) m_ptr,
 	sizeof(message)))) {
 	return EFAULT;
   }
@@ -1143,16 +1170,12 @@ PUBLIC void enqueue(
  * The mechanism is implemented here.   The actual scheduling policy is
  * defined in sched() and pick_proc().
  */
-  int q;	 				/* scheduling queue to use */
+  int q = rp->p_priority;	 		/* scheduling queue to use */
-  int front;					/* add to front or back */
 
   NOREC_ENTER(enqueuefunc);
 
   assert(proc_is_runnable(rp));
 
-  /* Determine where to insert to process. */
-  sched(rp, &q, &front);
-
   assert(q >= 0);
 
   /* Now add the process to the queue. */
@@ -1160,10 +1183,6 @@ PUBLIC void enqueue(
       rdy_head[q] = rdy_tail[q] = rp; 		/* create a new queue */
       rp->p_nextready = NULL;		/* mark new end */
   } 
-  else if (front) {				/* add to head of queue */
-      rp->p_nextready = rdy_head[q];		/* chain head of queue */
-      rdy_head[q] = rp;				/* set new queue head */
-  } 
   else {					/* add to tail of queue */
       rdy_tail[q]->p_nextready = rp;		/* chain tail of queue */	
       rdy_tail[q] = rp;				/* set new queue tail */
@@ -1207,9 +1226,7 @@ PRIVATE void enqueue_head(struct proc *rp)
    * the process was runnable without its quantum expired when dequeued. A
    * process with no time left should vahe been handled else and differently
    */
-#if 0
+  assert(rp->p_ticks_left > 0);
-  assert(rp->p_ticks_left);
-#endif
 
   assert(q >= 0);
 
@@ -1274,39 +1291,6 @@ PUBLIC void dequeue(const struct proc *rp)
   NOREC_RETURN(dequeuefunc, );
 }
 
-/*===========================================================================*
- *				sched					     * 
- *===========================================================================*/
-PRIVATE void sched(rp, queue, front)
-register struct proc *rp;			/* process to be scheduled */
-int *queue;					/* return: queue to use */
-int *front;					/* return: front or back */
-{
-/* This function determines the scheduling policy.  It is called whenever a
- * process must be added to one of the scheduling queues to decide where to
- * insert it.  As a side-effect the process' priority may be updated.  
- */
-  const int time_left = (rp->p_ticks_left > 0);	/* quantum fully consumed? */
-
-  /* Check whether the process has time left. Otherwise give a new quantum 
-   * and lower the process' priority, unless the process already is in the 
-   * lowest queue.  
-   */
-  if (! time_left) {				/* quantum consumed ? */
-      rp->p_ticks_left = rp->p_quantum_size; 	/* give new quantum */
-      if (rp->p_priority < (NR_SCHED_QUEUES-1)) {
-          rp->p_priority += 1;			/* lower priority */
-      }
-  }
-
-  /* If there is time left, the process is added to the front of its queue, 
-   * so that it can immediately run. The queue to use simply is always the
-   * process' current priority. 
-   */
-  *queue = rp->p_priority;
-  *front = time_left;
-}
-
 /*===========================================================================*
  *				pick_proc				     * 
  *===========================================================================*/
@@ -1338,52 +1322,6 @@ PRIVATE struct proc * pick_proc(void)
   return NULL;
 }
 
-/*===========================================================================*
- *				balance_queues				     *
- *===========================================================================*/
-#define Q_BALANCE_TICKS	 100
-PUBLIC void balance_queues(
-  timer_t *tp				/* watchdog timer pointer */
-)
-{
-/* Check entire process table and give all process a higher priority. This
- * effectively means giving a new quantum. If a process already is at its 
- * maximum priority, its quantum will be renewed.
- */
-  static timer_t queue_timer;			/* timer structure to use */
-  register struct proc* rp;			/* process table pointer  */
-  clock_t next_period;				/* time of next period  */
-  int ticks_added = 0;				/* total time added */
-
-  for (rp=BEG_PROC_ADDR; rp<END_PROC_ADDR; rp++) {
-      if (! isemptyp(rp)) {				/* check slot use */
-	  if (rp->p_priority > rp->p_max_priority) {	/* update priority? */
-	      int paused = 0;
-	      if (proc_is_runnable(rp)) {
-		RTS_SET(rp, RTS_PROC_STOP);		/* take off queue */
-		paused = 1;
-	      }
-	      ticks_added += rp->p_quantum_size;	/* do accounting */
-	      rp->p_priority -= 1;			/* raise priority */
-	      if(paused) {
-	      	RTS_UNSET(rp, RTS_PROC_STOP);
-	        assert(proc_is_runnable(rp));
-	      }
-	  }
-	  else {
-	      ticks_added += rp->p_quantum_size - rp->p_ticks_left;
-              rp->p_ticks_left = rp->p_quantum_size; 	/* give new quantum */
-	  }
-      }
-  }
-
-  /* Now schedule a new watchdog timer to balance the queues again.  The 
-   * period depends on the total amount of quantum ticks added.
-   */
-  next_period = MAX(Q_BALANCE_TICKS, ticks_added);	/* calculate next */
-  set_timer(&queue_timer, get_uptime() + next_period, balance_queues);
-}
-
 /*===========================================================================*
  *				endpoint_lookup				     *
  *===========================================================================*/
@@ -1447,3 +1385,51 @@ const int fatalflag;
 	return ok;
 }
 
+PRIVATE void notify_scheduler(struct proc *p)
+{
+	/* dequeue the process */
+	RTS_SET(p, RTS_NO_QUANTUM);
+	/*
+	 * Notify the process's scheduler that it has run out of
+	 * quantum. This is done by sending a message to the scheduler
+	 * on the process's behalf
+	 */
+	if (p->p_scheduler == p) {
+		/*
+		 * If a scheduler is scheduling itself, and runs out of
+		 * quantum, we don't send a message. The RTS_NO_QUANTUM
+		 * flag will be removed by schedcheck in proc.c.
+		 */
+	}
+	else if (p->p_scheduler != NULL) {
+		message m_no_quantum;
+		int err;
+
+		m_no_quantum.m_source = p->p_endpoint;
+		m_no_quantum.m_type   = SCHEDULING_NO_QUANTUM;
+
+		if ((err = mini_send(p, p->p_scheduler->p_endpoint,
+						&m_no_quantum, FROM_KERNEL))) {
+			panic("WARNING: Scheduling: mini_send returned %d\n", err);
+		}
+	}
+}
+
+PUBLIC void check_ticks_left(struct proc * p)
+{
+	if (p->p_ticks_left <= 0) {
+		p->p_ticks_left = 0;
+		if (priv(p)->s_flags & PREEMPTIBLE) {
+			/* this dequeues the process */
+			notify_scheduler(p);
+		}
+		else {
+			/*
+			 * non-preemptible processes only need their quantum to
+			 * be renewed. In fact, they by pass scheduling
+			 */
+			p->p_ticks_left = p->p_quantum_size;
+		}
+	}
+}
+

kernel/proc.h

@@ -25,10 +25,10 @@ struct proc {
   short p_rts_flags;		/* process is runnable only if zero */
   short p_misc_flags;		/* flags that do not suspend the process */
 
-  char p_priority;		/* current scheduling priority */
+  char p_priority;		/* current process priority */
-  char p_max_priority;		/* maximum scheduling priority */
   char p_ticks_left;		/* number of scheduling ticks left */
   char p_quantum_size;		/* quantum size in ticks */
+  struct proc *p_scheduler;	/* who should get out of quantum msg */
 
   struct mem_map p_memmap[NR_LOCAL_SEGS];   /* memory map (T, D, S) */
   struct pagefault p_pagefault;	/* valid if PAGEFAULT in p_rts_flags set */
@@ -224,8 +224,7 @@ struct proc {
 
 /* Scheduling priorities for p_priority. Values must start at zero (highest
  * priority) and increment.  Priorities of the processes in the boot image 
- * can be set in table.c. IDLE must have a queue for itself, to prevent low 
+ * can be set in table.c.
- * priority user processes to run round-robin with IDLE. 
  */
 #define NR_SCHED_QUEUES   16	/* MUST equal minimum priority + 1 */
 #define TASK_Q		   0	/* highest, used for kernel tasks */
@@ -259,6 +258,9 @@ EXTERN struct proc proc[NR_TASKS + NR_PROCS];	/* process table */
 EXTERN struct proc *rdy_head[NR_SCHED_QUEUES]; /* ptrs to ready list headers */
 EXTERN struct proc *rdy_tail[NR_SCHED_QUEUES]; /* ptrs to ready list tails */
 
+_PROTOTYPE( int mini_send, (struct proc *caller_ptr, int dst_e,
+		message *m_ptr, int flags));
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* PROC_H */

kernel/proto.h

@@ -12,7 +12,6 @@ struct proc;
 struct timer;
 
 /* clock.c */
-_PROTOTYPE( void clock_init, (void) );
 _PROTOTYPE( clock_t get_uptime, (void)					);
 _PROTOTYPE( void set_timer, (struct timer *tp, clock_t t, tmr_func_t f)	);
 _PROTOTYPE( void reset_timer, (struct timer *tp)			);
@@ -41,7 +40,6 @@ _PROTOTYPE( int do_ipc, (int call_nr, int src_dst,
 _PROTOTYPE( int mini_notify, (const struct proc *src, endpoint_t dst)	);
 _PROTOTYPE( void enqueue, (struct proc *rp)				);
 _PROTOTYPE( void dequeue, (const struct proc *rp)			);
-_PROTOTYPE( void balance_queues, (struct timer *tp)			);
 _PROTOTYPE( struct proc * schedcheck, (void)				);
 _PROTOTYPE( struct proc * arch_finish_schedcheck, (void)		);
 _PROTOTYPE( struct proc *endpoint_lookup, (endpoint_t ep)		);
@@ -52,6 +50,7 @@ _PROTOTYPE( int isokendpt_f, (const char *file, int line, endpoint_t e, int *p,
 _PROTOTYPE( int isokendpt_f, (endpoint_t e, int *p, int f)		);
 #define isokendpt_d(e, p, f) isokendpt_f((e), (p), (f))
 #endif
+_PROTOTYPE( void check_ticks_left, (struct proc *p));
 
 /* start.c */
 _PROTOTYPE( void cstart, (U16_t cs, U16_t ds, U16_t mds,
@@ -91,6 +90,7 @@ _PROTOTYPE( int disable_irq, (const irq_hook_t *hook)			);
 _PROTOTYPE( int runqueues_ok, (void) );
 _PROTOTYPE( char *rtsflagstr, (int flags) );
 _PROTOTYPE( char *miscflagstr, (int flags) );
+_PROTOTYPE( char *schedulerstr, (struct proc *scheduler) );
 
 /* system/do_safemap.c */
 _PROTOTYPE( int map_invoke_vm, (struct proc * caller, int req_type,

kernel/system.c

@@ -179,7 +179,6 @@ PUBLIC void system_init(void)
   map(SYS_EXEC, do_exec);		/* update process after execute */
   map(SYS_CLEAR, do_clear);		/* clean up after process exit */
   map(SYS_EXIT, do_exit);		/* a system process wants to exit */
-  map(SYS_NICE, do_nice);		/* set scheduling priority */
   map(SYS_PRIVCTL, do_privctl);		/* system privileges control */
   map(SYS_TRACE, do_trace);		/* request a trace operation */
   map(SYS_SETGRANT, do_setgrant);	/* get/set own parameters */
@@ -244,8 +243,12 @@ PUBLIC void system_init(void)
   map(SYS_SETMCONTEXT, do_setmcontext); /* set machine context */
   map(SYS_GETMCONTEXT, do_getmcontext); /* get machine context */
 #endif
-}
 
+  /* Scheduling */
+  map(SYS_SCHEDULE, do_schedule);	/* reschedule a process */
+  map(SYS_SCHEDCTL, do_schedctl);	/* change process scheduler */
+
+}
 /*===========================================================================*
  *				get_priv				     *
  *===========================================================================*/

kernel/system.h

@@ -64,11 +64,6 @@ _PROTOTYPE( int do_trace, (struct proc * caller, message *m_ptr) );
 #define do_trace do_unused
 #endif
 
-_PROTOTYPE( int do_nice, (struct proc * caller, message *m_ptr) );
-#if ! USE_NICE
-#define do_nice do_unused
-#endif
-
 _PROTOTYPE( int do_runctl, (struct proc * caller, message *m_ptr) );
 #if ! USE_RUNCTL
 #define do_runctl do_unused
@@ -210,5 +205,8 @@ _PROTOTYPE( int do_setmcontext, (struct proc * caller, message *m_ptr) );
 #define do_setmcontext do_unused
 #endif
 
+_PROTOTYPE( int do_schedule,    (struct proc * caller, message *m_ptr) );
+_PROTOTYPE( int do_schedctl, (struct proc * caller, message *m_ptr) );
+
 #endif	/* SYSTEM_H */
 

kernel/system/Makefile

@@ -30,7 +30,6 @@ OBJECTS	= \
 	do_clear.o \
 	do_exit.o \
 	do_trace.o \
-	do_nice.o \
 	do_runctl.o \
 	do_update.o \
 	do_times.o \
@@ -60,7 +59,9 @@ OBJECTS	= \
 	do_cprofile.o \
 	do_profbuf.o \
 	do_vmctl.o \
-	do_mcontext.o
+	do_mcontext.o \
+	do_schedule.o \
+	do_schedctl.o
 
 build $(SYSTEM):	$(SYSTEM)($(OBJECTS))
 	aal cr $@ *.o

kernel/system/do_fork.c

@@ -82,11 +82,24 @@ PUBLIC int do_fork(struct proc * caller, message * m_ptr)
 
   /* Parent and child have to share the quantum that the forked process had,
    * so that queued processes do not have to wait longer because of the fork.
-   * If the time left is odd, the child gets an extra tick.
    */
-  rpc->p_ticks_left = (rpc->p_ticks_left + 1) / 2;
+
+  /*
+   * we want to avoid having processes that loose their quantum without going
+   * through the standard path where the "out of quantum" is handled. We add
+   * some more time to such processes.
+   *
+   * This is a temporary solution until we are able to handle this in the
+   * userspace
+   */
+  if (rpp->p_ticks_left < 2)
+	  rpp->p_ticks_left = 2;
+
+  rpc->p_ticks_left =  rpp->p_ticks_left / 2;
   rpp->p_ticks_left =  rpp->p_ticks_left / 2;	
 
+  assert(rpc->p_ticks_left > 0 && rpp->p_ticks_left > 0);
+
   /* If the parent is a privileged process, take away the privileges from the 
    * child process and inhibit it from running by setting the NO_PRIV flag.
    * The caller should explicitely set the new privileges before executing.

kernel/system/do_schedctl.c

@@ -0,0 +1,21 @@
+#include "../system.h"
+#include <signal.h>
+#include <sys/sigcontext.h>
+#include <minix/endpoint.h>
+
+/*===========================================================================*
+ *			          do_schedctl			     *
+ *===========================================================================*/
+PUBLIC int do_schedctl(struct proc * caller, message * m_ptr)
+{
+	struct proc *p;
+
+	/* Only system processes can change process schedulers */
+	if (! (priv(caller)->s_flags & SYS_PROC))
+		return(EPERM);
+
+	p = proc_addr(_ENDPOINT_P(m_ptr->SCHEDULING_ENDPOINT));
+	p->p_scheduler = caller;
+
+	return(OK);
+}

kernel/system/do_schedule.c

@@ -0,0 +1,42 @@
+#include "../system.h"
+#include <signal.h>
+#include <sys/sigcontext.h>
+#include <minix/endpoint.h>
+
+/*===========================================================================*
+ *				do_schedule				     *
+ *===========================================================================*/
+PUBLIC int do_schedule(struct proc * caller, message * m_ptr)
+{
+	struct proc *p;
+
+	p = proc_addr(_ENDPOINT_P(m_ptr->SCHEDULING_ENDPOINT));
+
+	/* Make sure the priority number given is within the allowed range.*/
+	if (m_ptr->SCHEDULING_PRIORITY < TASK_Q ||
+		m_ptr->SCHEDULING_PRIORITY > NR_SCHED_QUEUES)
+		return(EINVAL);
+
+	/* Only system processes can schedule processes */
+	if (! (priv(caller)->s_flags & SYS_PROC))
+		return(EPERM);
+
+	/* Only this process' scheduler can schedule it */
+	if (caller != p->p_scheduler)
+		return(EPERM);
+
+	/* In some cases, we might be rescheduling a runnable process. In such
+	 * a case (i.e. if we are updating the priority) we set the NO_QUANTUM
+	 * flag before the generic unset to dequeue/enqueue the process
+	 */
+	if (proc_is_runnable(p))
+		RTS_SET(p, RTS_NO_QUANTUM);
+
+	/* Clear the scheduling bit and enqueue the process */
+	p->p_priority = m_ptr->SCHEDULING_PRIORITY;
+	p->p_ticks_left = m_ptr->SCHEDULING_QUANTUM;
+
+	RTS_UNSET(p, RTS_NO_QUANTUM);
+
+	return(OK);
+}

lib/libsys/Makefile

@@ -50,7 +50,6 @@ SRCS=  \
 	sys_kill.c \
 	sys_memset.c \
 	sys_newmap.c \
-	sys_nice.c \
 	sys_out.c \
 	sys_physcopy.c \
 	sys_readbios.c \
@@ -70,6 +69,8 @@ SRCS=  \
 	sys_setgrant.c \
 	sys_sprof.c \
 	sys_stime.c \
+        sys_schedule.c \
+        sys_schedctl.c \
 	sys_times.c \
 	sys_trace.c \
 	sys_umap.c \

lib/libsys/sys_nice.c

@@ -1,13 +0,0 @@
-#include "syslib.h"
-
-/*===========================================================================*
- *                                sys_nice			     	     *
- *===========================================================================*/
-PUBLIC int sys_nice(endpoint_t proc_ep, int prio)
-{
-  message m;
-
-  m.PR_ENDPT = proc_ep;
-  m.PR_PRIORITY = prio;
-  return(_kernel_call(SYS_NICE, &m));
-}

lib/libsys/sys_schedctl.c

@@ -0,0 +1,9 @@
+#include "syslib.h"
+
+PUBLIC int sys_schedctl(endpoint_t proc_ep)
+{
+	message m;
+
+	m.SCHEDULING_ENDPOINT = proc_ep;
+	return(_kernel_call(SYS_SCHEDCTL, &m));
+}

lib/libsys/sys_schedule.c

@@ -0,0 +1,11 @@
+#include "syslib.h"
+
+PUBLIC int sys_schedule(endpoint_t proc_ep, char priority, char quantum)
+{
+	message m;
+
+	m.SCHEDULING_ENDPOINT = proc_ep;
+	m.SCHEDULING_PRIORITY = priority;
+	m.SCHEDULING_QUANTUM  = quantum;
+	return(_kernel_call(SYS_SCHEDULE, &m));
+}

servers/is/dmp_kernel.c

@@ -398,9 +398,9 @@ PUBLIC void proctab_dmp()
 	size = rp->p_memmap[T].mem_len
 		+ ((rp->p_memmap[S].mem_phys + rp->p_memmap[S].mem_len) - data);
 	printf(" %5d %10d ", _ENDPOINT_G(rp->p_endpoint), rp->p_endpoint);
-	printf("%-8.8s %02u/%02u %02d/%02u %6lu %6lu ",
+	printf("%-8.8s %02u %02d/%02u %6lu %6lu ",
 	       rp->p_name,
-	       rp->p_priority, rp->p_max_priority,
+	       rp->p_priority,
 	       rp->p_ticks_left, rp->p_quantum_size, 
 	       rp->p_user_time, rp->p_sys_time);
 	PRINTRTS(rp);

servers/pm/Makefile

@@ -2,7 +2,7 @@
 PROG=	pm
 SRCS=	main.c forkexit.c break.c exec.c time.c timers.c alarm.c \
 	signal.c utility.c table.c trace.c getset.c misc.c \
-	profile.c dma.c mcontext.c
+	profile.c dma.c mcontext.c schedule.c
 
 DPADD+=	${LIBSYS} ${LIBTIMERS}
 LDADD+=	-lsys -ltimers

servers/pm/main.c

@@ -66,6 +66,7 @@ PUBLIC int main()
 
   /* SEF local startup. */
   sef_local_startup();
+  overtake_scheduling();	/* overtake all running processes */
 
   /* This is PM's main loop-  get work and do it, forever and forever. */
   while (TRUE) {
@@ -113,6 +114,10 @@ PUBLIC int main()
 		else
 			result= ENOSYS;
 		break;
+	case SCHEDULING_NO_QUANTUM:
+		/* This message was sent from the kernel, don't reply */
+		do_noquantum();
+		continue;
 	default:
 		/* Else, if the system call number is valid, perform the
 		 * call.
@@ -254,6 +259,11 @@ PRIVATE int sef_cb_init_fresh(int type, sef_init_info_t *info)
 		/* Get kernel endpoint identifier. */
 		rmp->mp_endpoint = ip->endpoint;
 
+		/* Get scheduling info */
+		rmp->mp_max_priority = ip->priority;
+		rmp->mp_priority = ip->priority;
+		rmp->mp_time_slice = ip->quantum;
+
 		/* Tell FS about this system process. */
 		mess.m_type = PM_INIT;
 		mess.PM_SLOT = ip->proc_nr;

servers/pm/misc.c

@@ -407,7 +407,7 @@ PUBLIC int do_reboot()
  *===========================================================================*/
 PUBLIC int do_getsetpriority()
 {
-	int r, arg_which, arg_who, arg_pri;
+	int r, arg_which, arg_who, arg_pri, new_q;
 	struct mproc *rmp;
 
 	arg_which = m_in.m1_i1;
@@ -439,8 +439,25 @@ PUBLIC int do_getsetpriority()
 	if (rmp->mp_nice > arg_pri && mp->mp_effuid != SUPER_USER)
 		return(EACCES);
 	
-	/* We're SET, and it's allowed. */
+	/* We're SET, and it's allowed.
-	if ((r = sys_nice(rmp->mp_endpoint, arg_pri)) != OK)
+	 *
+	 * The value passed in is currently between PRIO_MIN and PRIO_MAX.
+	 * We have to scale this between MIN_USER_Q and MAX_USER_Q to match
+	 * the kernel's scheduling queues.
+	 *
+	 * TODO: This assumes that we are the scheduler, this will be changed
+	 *       once the scheduler gets factored out of PM to its own server
+	 */
+	if (arg_pri < PRIO_MIN || arg_pri > PRIO_MAX) return(EINVAL);
+
+	new_q = MAX_USER_Q + (arg_pri-PRIO_MIN) * (MIN_USER_Q-MAX_USER_Q+1) /
+	    (PRIO_MAX-PRIO_MIN+1);
+	if (new_q < MAX_USER_Q) new_q = MAX_USER_Q;	/* shouldn't happen */
+	if (new_q > MIN_USER_Q) new_q = MIN_USER_Q;	/* shouldn't happen */
+
+	rmp->mp_max_priority = rmp->mp_priority = new_q;
+	if ((r = sys_schedule(rmp->mp_priority, rmp->mp_priority,
+		rmp->mp_time_slice)) != OK)
 		return(r);
 
 	rmp->mp_nice = arg_pri;

servers/pm/mproc.h

@@ -55,6 +55,11 @@ EXTERN struct mproc {
   /* Scheduling priority. */
   signed int mp_nice;		/* nice is PRIO_MIN..PRIO_MAX, standard 0. */
 
+  /* User space scheduling */
+  int mp_max_priority;		/* this process' highest allowed priority */
+  int mp_priority;		/* the process' current priority */
+  int mp_time_slice;		/* this process's scheduling queue */
+
   char mp_name[PROC_NAME_LEN];	/* process name */
 } mproc[NR_PROCS];
 

servers/pm/proto.h

@@ -60,6 +60,11 @@ _PROTOTYPE( int do_getepinfo, (void)					);
 _PROTOTYPE( int do_svrctl, (void)					);
 _PROTOTYPE( int do_getsetpriority, (void)				);
 
+/* schedule.c */
+_PROTOTYPE( void do_noquantum, (void)					);
+_PROTOTYPE( void overtake_scheduling, (void)				);
+_PROTOTYPE( void balance_queues, (struct timer *tp)			);
+
 /* profile.c */
 _PROTOTYPE( int do_sprofile, (void)					);
 _PROTOTYPE( int do_cprofile, (void)					);

servers/pm/schedule.c

@@ -0,0 +1,91 @@
+#include "pm.h"
+#include <minix/callnr.h>
+#include <minix/com.h>
+#include <minix/config.h>
+#include <minix/sysinfo.h>
+#include <minix/type.h>
+#include <machine/archtypes.h>
+#include <lib.h>
+#include "mproc.h"
+#include "../../kernel/proc.h" /* for MIN_USER_Q */
+
+PRIVATE timer_t sched_timer;
+
+/*===========================================================================*
+ *				do_noquantum				     *
+ *===========================================================================*/
+
+PUBLIC void do_noquantum(void)
+{
+	int rv, proc_nr_n;
+	register struct mproc *rmp;
+
+	if (pm_isokendpt(m_in.m_source, &proc_nr_n) != OK) {
+		printf("PM: WARNING: got an invalid endpoint in OOQ msg %u.\n",
+		m_in.m_source);
+		return;
+	}
+
+	rmp = &mproc[proc_nr_n];
+	if (rmp->mp_priority < MIN_USER_Q) {
+		rmp->mp_priority += 1; /* lower priority */
+	}
+
+	if ((rv = sys_schedule(rmp->mp_endpoint, rmp->mp_priority,
+		rmp->mp_time_slice))) {
+		printf("PM: An error occurred when trying to schedule %s: %d\n",
+		rmp->mp_name, rv);
+	}
+}
+
+/*===========================================================================*
+ *				overtake_scheduling			     *
+ *===========================================================================*/
+PUBLIC void overtake_scheduling(void)
+{
+	struct mproc *trmp;
+	int proc_nr;
+
+	tmr_inittimer(&sched_timer);
+
+	for (proc_nr=0, trmp=mproc; proc_nr < NR_PROCS; proc_nr++, trmp++) {
+		/* Don't overtake system processes. When the system starts,
+		 * this will typically only overtake init, from which other
+		 * user space processes will inherit. */
+		if (trmp->mp_flags & IN_USE && !(trmp->mp_flags & PRIV_PROC)) {
+			if (sys_schedctl(trmp->mp_endpoint))
+				printf("PM: Error while overtaking scheduling for %s\n",
+					trmp->mp_name);
+		}
+	}
+
+	pm_set_timer(&sched_timer, 100, balance_queues, 0);
+}
+
+/*===========================================================================*
+ *				balance_queues				     *
+ *===========================================================================*/
+
+PUBLIC void balance_queues(tp)
+struct timer *tp;
+{
+	struct mproc *rmp;
+	int proc_nr;
+	int rv;
+
+	for (proc_nr=0, rmp=mproc; proc_nr < NR_PROCS; proc_nr++, rmp++) {
+		if (rmp->mp_flags & IN_USE) {
+			if (rmp->mp_priority > rmp->mp_max_priority) {
+				rmp->mp_priority -= 1; /* increase priority */
+				if ((rv = sys_schedule(rmp->mp_endpoint,
+					rmp->mp_priority,
+					rmp->mp_time_slice))) {
+					printf("PM: An error occurred when balancing %s: %d\n",
+					rmp->mp_name, rv);
+				}
+			}
+		}
+	}
+
+	pm_set_timer(&sched_timer, 100, balance_queues, 0);
+}