diff --git a/src/cpu/o3/commit.hh b/src/cpu/o3/commit.hh
index 60b555269..49ff5cdad 100644
--- a/src/cpu/o3/commit.hh
+++ b/src/cpu/o3/commit.hh
@@ -187,11 +187,14 @@ class DefaultCommit
     /** Initializes stage by sending back the number of free entries. */
     void initStage();
 
-    /** Initializes the switching out of commit. */
-    void switchOut();
+    /** Initializes the draining of commit. */
+    void drain();
+
+    /** Resumes execution after draining. */
+    void resume();
 
     /** Completes the switch out of commit. */
-    void doSwitchOut();
+    void switchOut();
 
     /** Takes over from another CPU's thread. */
     void takeOverFrom();
@@ -383,8 +386,8 @@ class DefaultCommit
     /** Number of Active Threads */
     unsigned numThreads;
 
-    /** Is a switch out pending. */
-    bool switchPending;
+    /** Is a drain pending. */
+    bool drainPending;
 
     /** Is commit switched out. */
     bool switchedOut;
diff --git a/src/cpu/o3/commit_impl.hh b/src/cpu/o3/commit_impl.hh
index 06b8e8a95..2eb05afac 100644
--- a/src/cpu/o3/commit_impl.hh
+++ b/src/cpu/o3/commit_impl.hh
@@ -80,7 +80,7 @@ DefaultCommit<Impl>::DefaultCommit(Params *params)
       renameWidth(params->renameWidth),
       commitWidth(params->commitWidth),
       numThreads(params->numberOfThreads),
-      switchPending(false),
+      drainPending(false),
       switchedOut(false),
       trapLatency(params->trapLatency),
       fetchTrapLatency(params->fetchTrapLatency)
@@ -351,20 +351,26 @@ DefaultCommit<Impl>::initStage()
 
 template <class Impl>
 void
-DefaultCommit<Impl>::switchOut()
+DefaultCommit<Impl>::drain()
 {
-    switchPending = true;
+    drainPending = true;
 }
 
 template <class Impl>
 void
-DefaultCommit<Impl>::doSwitchOut()
+DefaultCommit<Impl>::switchOut()
 {
     switchedOut = true;
-    switchPending = false;
+    drainPending = false;
     rob->switchOut();
 }
 
+template <class Impl>
+void
+DefaultCommit<Impl>::resume()
+{
+}
+
 template <class Impl>
 void
 DefaultCommit<Impl>::takeOverFrom()
@@ -557,8 +563,9 @@ DefaultCommit<Impl>::tick()
     wroteToTimeBuffer = false;
     _nextStatus = Inactive;
 
-    if (switchPending && rob->isEmpty() && !iewStage->hasStoresToWB()) {
-        cpu->signalSwitched();
+    if (drainPending && rob->isEmpty() && !iewStage->hasStoresToWB()) {
+        cpu->signalDrained();
+        drainPending = false;
         return;
     }
 
diff --git a/src/cpu/o3/cpu.cc b/src/cpu/o3/cpu.cc
index fb7739db8..5bda57cf8 100644
--- a/src/cpu/o3/cpu.cc
+++ b/src/cpu/o3/cpu.cc
@@ -158,7 +158,7 @@ FullO3CPU<Impl>::FullO3CPU(Params *params)
       physmem(system->physmem),
 #endif // FULL_SYSTEM
       mem(params->mem),
-      switchCount(0),
+      drainCount(0),
       deferRegistration(params->deferRegistration),
       numThreads(number_of_threads)
 {
@@ -713,45 +713,72 @@ FullO3CPU<Impl>::haltContext(int tid)
 }
 
 template <class Impl>
-void
-FullO3CPU<Impl>::switchOut()
+bool
+FullO3CPU<Impl>::drain(Event *drain_event)
 {
-    switchCount = 0;
-    fetch.switchOut();
-    decode.switchOut();
-    rename.switchOut();
-    iew.switchOut();
-    commit.switchOut();
+    drainCount = 0;
+    drainEvent = drain_event;
+    fetch.drain();
+    decode.drain();
+    rename.drain();
+    iew.drain();
+    commit.drain();
 
     // Wake the CPU and record activity so everything can drain out if
     // the CPU is currently idle.
     wakeCPU();
     activityRec.activity();
+
+    return false;
 }
 
 template <class Impl>
 void
-FullO3CPU<Impl>::signalSwitched()
+FullO3CPU<Impl>::resume()
 {
-    if (++switchCount == NumStages) {
-        fetch.doSwitchOut();
-        rename.doSwitchOut();
-        commit.doSwitchOut();
-        instList.clear();
-        while (!removeList.empty()) {
-            removeList.pop();
-        }
+    if (_status == SwitchedOut)
+        return;
+    fetch.resume();
+    decode.resume();
+    rename.resume();
+    iew.resume();
+    commit.resume();
 
-#if USE_CHECKER
-        if (checker)
-            checker->switchOut();
-#endif
+    if (!tickEvent.scheduled())
+        tickEvent.schedule(curTick);
+    _status = Running;
+}
 
+template <class Impl>
+void
+FullO3CPU<Impl>::signalDrained()
+{
+    if (++drainCount == NumStages) {
         if (tickEvent.scheduled())
             tickEvent.squash();
-        _status = SwitchedOut;
+        _status = Drained;
+        drainEvent->process();
     }
-    assert(switchCount <= 5);
+    assert(drainCount <= 5);
+}
+
+template <class Impl>
+void
+FullO3CPU<Impl>::switchOut()
+{
+    fetch.switchOut();
+    rename.switchOut();
+    commit.switchOut();
+    instList.clear();
+    while (!removeList.empty()) {
+        removeList.pop();
+    }
+
+    _status = SwitchedOut;
+#if USE_CHECKER
+    if (checker)
+        checker->switchOut();
+#endif
 }
 
 template <class Impl>
diff --git a/src/cpu/o3/cpu.hh b/src/cpu/o3/cpu.hh
index bd0451601..cf3747601 100644
--- a/src/cpu/o3/cpu.hh
+++ b/src/cpu/o3/cpu.hh
@@ -57,6 +57,8 @@ class Checker;
 class ThreadContext;
 template <class>
 class O3ThreadContext;
+
+class Checkpoint;
 class MemObject;
 class Process;
 
@@ -109,6 +111,7 @@ class FullO3CPU : public BaseO3CPU
         Idle,
         Halted,
         Blocked,
+        Drained,
         SwitchedOut
     };
 
@@ -270,14 +273,21 @@ class FullO3CPU : public BaseO3CPU
      */
     virtual void syscall(int tid) { panic("Unimplemented!"); }
 
-    /** Switches out this CPU. */
-    void switchOut();
+    /** Starts draining the CPU's pipeline of all instructions in
+     * order to stop all memory accesses. */
+    virtual bool drain(Event *drain_event);
+
+    /** Resumes execution after a drain. */
+    virtual void resume();
 
     /** Signals to this CPU that a stage has completed switching out. */
-    void signalSwitched();
+    void signalDrained();
+
+    /** Switches out this CPU. */
+    virtual void switchOut();
 
     /** Takes over from another CPU. */
-    void takeOverFrom(BaseCPU *oldCPU);
+    virtual void takeOverFrom(BaseCPU *oldCPU);
 
     /** Get the current instruction sequence number, and increment it. */
     InstSeqNum getAndIncrementInstSeq()
@@ -550,8 +560,11 @@ class FullO3CPU : public BaseO3CPU
     /** Pointer to memory. */
     MemObject *mem;
 
-    /** Counter of how many stages have completed switching out. */
-    int switchCount;
+    /** Event to call process() on once draining has completed. */
+    Event *drainEvent;
+
+    /** Counter of how many stages have completed draining. */
+    int drainCount;
 
     /** Pointers to all of the threads in the CPU. */
     std::vector<Thread *> thread;
diff --git a/src/cpu/o3/decode.hh b/src/cpu/o3/decode.hh
index 1edf3335d..1e96f1884 100644
--- a/src/cpu/o3/decode.hh
+++ b/src/cpu/o3/decode.hh
@@ -109,8 +109,14 @@ class DefaultDecode
     /** Sets pointer to list of active threads. */
     void setActiveThreads(std::list<unsigned> *at_ptr);
 
+    /** Drains the decode stage. */
+    void drain();
+
+    /** Resumes execution after a drain. */
+    void resume() { }
+
     /** Switches out the decode stage. */
-    void switchOut();
+    void switchOut() { }
 
     /** Takes over from another CPU's thread. */
     void takeOverFrom();
diff --git a/src/cpu/o3/decode_impl.hh b/src/cpu/o3/decode_impl.hh
index 16be01784..71637883b 100644
--- a/src/cpu/o3/decode_impl.hh
+++ b/src/cpu/o3/decode_impl.hh
@@ -166,10 +166,10 @@ DefaultDecode<Impl>::setActiveThreads(list<unsigned> *at_ptr)
 
 template <class Impl>
 void
-DefaultDecode<Impl>::switchOut()
+DefaultDecode<Impl>::drain()
 {
-    // Decode can immediately switch out.
-    cpu->signalSwitched();
+    // Decode is done draining at any time.
+    cpu->signalDrained();
 }
 
 template <class Impl>
diff --git a/src/cpu/o3/fetch.hh b/src/cpu/o3/fetch.hh
index 848ebf39e..9611f0455 100644
--- a/src/cpu/o3/fetch.hh
+++ b/src/cpu/o3/fetch.hh
@@ -180,11 +180,14 @@ class DefaultFetch
     /** Processes cache completion event. */
     void processCacheCompletion(PacketPtr pkt);
 
-    /** Begins the switch out of the fetch stage. */
-    void switchOut();
+    /** Begins the drain of the fetch stage. */
+    void drain();
 
-    /** Completes the switch out of the fetch stage. */
-    void doSwitchOut();
+    /** Resumes execution after a drain. */
+    void resume();
+
+    /** Tells fetch stage to prepare to be switched out. */
+    void switchOut();
 
     /** Takes over from another CPU's thread. */
     void takeOverFrom();
@@ -421,6 +424,9 @@ class DefaultFetch
      */
     bool interruptPending;
 
+    /** Is there a drain pending. */
+    bool drainPending;
+
     /** Records if fetch is switched out. */
     bool switchedOut;
 
diff --git a/src/cpu/o3/fetch_impl.hh b/src/cpu/o3/fetch_impl.hh
index 60eb76d17..500b5304e 100644
--- a/src/cpu/o3/fetch_impl.hh
+++ b/src/cpu/o3/fetch_impl.hh
@@ -109,6 +109,7 @@ DefaultFetch<Impl>::DefaultFetch(Params *params)
       numThreads(params->numberOfThreads),
       numFetchingThreads(params->smtNumFetchingThreads),
       interruptPending(false),
+      drainPending(false),
       switchedOut(false)
 {
     if (numThreads > Impl::MaxThreads)
@@ -353,7 +354,8 @@ DefaultFetch<Impl>::processCacheCompletion(PacketPtr pkt)
     // to return.
     if (fetchStatus[tid] != IcacheWaitResponse ||
         pkt->req != memReq[tid] ||
-        isSwitchedOut()) {
+        isSwitchedOut() ||
+        drainPending) {
         ++fetchIcacheSquashes;
         delete pkt->req;
         delete pkt;
@@ -384,17 +386,25 @@ DefaultFetch<Impl>::processCacheCompletion(PacketPtr pkt)
 
 template <class Impl>
 void
-DefaultFetch<Impl>::switchOut()
+DefaultFetch<Impl>::drain()
 {
-    // Fetch is ready to switch out at any time.
-    switchedOut = true;
-    cpu->signalSwitched();
+    // Fetch is ready to drain at any time.
+    cpu->signalDrained();
+    drainPending = true;
 }
 
 template <class Impl>
 void
-DefaultFetch<Impl>::doSwitchOut()
+DefaultFetch<Impl>::resume()
 {
+    drainPending = false;
+}
+
+template <class Impl>
+void
+DefaultFetch<Impl>::switchOut()
+{
+    switchedOut = true;
     // Branch predictor needs to have its state cleared.
     branchPred.switchOut();
 }
@@ -498,7 +508,7 @@ DefaultFetch<Impl>::fetchCacheLine(Addr fetch_PC, Fault &ret_fault, unsigned tid
     unsigned flags = 0;
 #endif // FULL_SYSTEM
 
-    if (cacheBlocked || (interruptPending && flags == 0) || switchedOut) {
+    if (cacheBlocked || (interruptPending && flags == 0) || drainPending) {
         // Hold off fetch from getting new instructions when:
         // Cache is blocked, or
         // while an interrupt is pending and we're not in PAL mode, or
diff --git a/src/cpu/o3/iew.hh b/src/cpu/o3/iew.hh
index 9627609c2..774b6dcbd 100644
--- a/src/cpu/o3/iew.hh
+++ b/src/cpu/o3/iew.hh
@@ -143,11 +143,14 @@ class DefaultIEW
     /** Sets pointer to the scoreboard. */
     void setScoreboard(Scoreboard *sb_ptr);
 
-    /** Starts switch out of IEW stage. */
-    void switchOut();
+    /** Drains IEW stage. */
+    void drain();
+
+    /** Resumes execution after a drain. */
+    void resume();
 
     /** Completes switch out of IEW stage. */
-    void doSwitchOut();
+    void switchOut();
 
     /** Takes over from another CPU's thread. */
     void takeOverFrom();
diff --git a/src/cpu/o3/iew_impl.hh b/src/cpu/o3/iew_impl.hh
index 118038b65..c3aa748ae 100644
--- a/src/cpu/o3/iew_impl.hh
+++ b/src/cpu/o3/iew_impl.hh
@@ -355,15 +355,21 @@ DefaultIEW<Impl>::setScoreboard(Scoreboard *sb_ptr)
 
 template <class Impl>
 void
-DefaultIEW<Impl>::switchOut()
+DefaultIEW<Impl>::drain()
 {
-    // IEW is ready to switch out at any time.
-    cpu->signalSwitched();
+    // IEW is ready to drain at any time.
+    cpu->signalDrained();
 }
 
 template <class Impl>
 void
-DefaultIEW<Impl>::doSwitchOut()
+DefaultIEW<Impl>::resume()
+{
+}
+
+template <class Impl>
+void
+DefaultIEW<Impl>::switchOut()
 {
     // Clear any state.
     switchedOut = true;
diff --git a/src/cpu/o3/rename.hh b/src/cpu/o3/rename.hh
index 581fc8f81..538dd9bb4 100644
--- a/src/cpu/o3/rename.hh
+++ b/src/cpu/o3/rename.hh
@@ -157,12 +157,15 @@ class DefaultRename
     /** Sets pointer to the scoreboard. */
     void setScoreboard(Scoreboard *_scoreboard);
 
+    /** Drains the rename stage. */
+    void drain();
+
+    /** Resumes execution after a drain. */
+    void resume() { }
+
     /** Switches out the rename stage. */
     void switchOut();
 
-    /** Completes the switch out. */
-    void doSwitchOut();
-
     /** Takes over from another CPU's thread. */
     void takeOverFrom();
 
diff --git a/src/cpu/o3/rename_impl.hh b/src/cpu/o3/rename_impl.hh
index df8b7f9da..fddbae3db 100644
--- a/src/cpu/o3/rename_impl.hh
+++ b/src/cpu/o3/rename_impl.hh
@@ -258,15 +258,15 @@ DefaultRename<Impl>::setScoreboard(Scoreboard *_scoreboard)
 
 template <class Impl>
 void
-DefaultRename<Impl>::switchOut()
+DefaultRename<Impl>::drain()
 {
     // Rename is ready to switch out at any time.
-    cpu->signalSwitched();
+    cpu->signalDrained();
 }
 
 template <class Impl>
 void
-DefaultRename<Impl>::doSwitchOut()
+DefaultRename<Impl>::switchOut()
 {
     // Clear any state, fix up the rename map.
     for (int i = 0; i < numThreads; i++) {