kvm: Avoid synchronizing the TC on every KVM exit

Reduce the number of KVM->TC synchronizations by overloading the
getContext() method and only request an update when the TC is
requested as opposed to every time KVM returns to gem5.

src/cpu/base.hh

@@ -252,7 +252,7 @@ class BaseCPU : public MemObject
    int findContext(ThreadContext *tc);
 
    /// Given a thread num get tho thread context for it
-   ThreadContext *getContext(int tn) { return threadContexts[tn]; }
+   virtual ThreadContext *getContext(int tn) { return threadContexts[tn]; }
 
   public:
     typedef BaseCPUParams Params;

src/cpu/kvm/base.cc

@@ -72,7 +72,8 @@ BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
       _status(Idle),
       dataPort(name() + ".dcache_port", this),
       instPort(name() + ".icache_port", this),
-      contextDirty(true),
+      threadContextDirty(true),
+      kvmStateDirty(false),
       vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
       _kvmRun(NULL), mmioRing(NULL),
       pageSize(sysconf(_SC_PAGE_SIZE)),
@@ -205,6 +206,9 @@ BaseKvmCPU::regStats()
 void
 BaseKvmCPU::serializeThread(std::ostream &os, ThreadID tid)
 {
+    // Update the thread context so we have something to serialize.
+    syncThreadContext();
+
     assert(tid == 0);
     assert(_status == Idle);
     thread->serialize(os);
@@ -217,7 +221,7 @@ BaseKvmCPU::unserializeThread(Checkpoint *cp, const std::string &section,
     assert(tid == 0);
     assert(_status == Idle);
     thread->unserialize(cp, section);
-    contextDirty = true;
+    threadContextDirty = true;
 }
 
 unsigned int
@@ -263,10 +267,14 @@ BaseKvmCPU::drainResume()
 void
 BaseKvmCPU::switchOut()
 {
-    BaseCPU::switchOut();
-
     DPRINTF(Kvm, "switchOut\n");
 
+    // Make sure to update the thread context in case, the new CPU
+    // will need to access it.
+    syncThreadContext();
+
+    BaseCPU::switchOut();
+
     // We should have drained prior to executing a switchOut, which
     // means that the tick event shouldn't be scheduled and the CPU is
     // idle.
@@ -288,8 +296,9 @@ BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
     assert(_status == Idle);
     assert(threadContexts.size() == 1);
 
-    // Force a gem5 -> KVM context synchronization
-    contextDirty = true;
+    // The BaseCPU updated the thread context, make sure that we
+    // synchronize next time we enter start the CPU.
+    threadContextDirty = true;
 }
 
 void
@@ -368,6 +377,15 @@ BaseKvmCPU::haltContext(ThreadID thread_num)
     suspendContext(thread_num);
 }
 
+ThreadContext *
+BaseKvmCPU::getContext(int tn)
+{
+    assert(tn == 0);
+    syncThreadContext();
+    return tc;
+}
+
+
 Counter
 BaseKvmCPU::totalInsts() const
 {
@@ -394,14 +412,8 @@ BaseKvmCPU::tick()
 
     DPRINTF(KvmRun, "Entering KVM...\n");
 
-    if (contextDirty) {
-        contextDirty = false;
-        updateKvmState();
-    }
-
     Tick ticksToExecute(mainEventQueue.nextTick() - curTick());
     Tick ticksExecuted(kvmRun(ticksToExecute));
-    updateThreadContext();
 
     Tick delay(ticksExecuted + handleKvmExit());
 
@@ -423,6 +435,13 @@ BaseKvmCPU::kvmRun(Tick ticks)
     uint64_t baseCycles(hwCycles.read());
     uint64_t baseInstrs(hwInstructions.read());
 
+    // We might need to update the KVM state.
+    syncKvmState();
+    // Entering into KVM implies that we'll have to reload the thread
+    // context from KVM if we want to access it. Flag the KVM state as
+    // dirty with respect to the cached thread context.
+    kvmStateDirty = true;
+
     if (ticks < runTimer->resolution()) {
         DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
                 ticks, runTimer->resolution());
@@ -604,6 +623,30 @@ BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
 #endif
 }
 
+void
+BaseKvmCPU::syncThreadContext()
+{
+    if (!kvmStateDirty)
+        return;
+
+    assert(!threadContextDirty);
+
+    updateThreadContext();
+    kvmStateDirty = false;
+}
+
+void
+BaseKvmCPU::syncKvmState()
+{
+    if (!threadContextDirty)
+        return;
+
+    assert(!kvmStateDirty);
+
+    updateKvmState();
+    threadContextDirty = false;
+}
+
 Tick
 BaseKvmCPU::handleKvmExit()
 {

src/cpu/kvm/base.hh

@@ -102,13 +102,27 @@ class BaseKvmCPU : public BaseCPU
     void deallocateContext(ThreadID thread_num);
     void haltContext(ThreadID thread_num);
 
+    ThreadContext *getContext(int tn);
+
     Counter totalInsts() const;
     Counter totalOps() const;
 
     /** Dump the internal state to the terminal. */
     virtual void dump();
 
-    /** SimpleThread object, provides all the architectural state. */
+    /**
+     * A cached copy of a thread's state in the form of a SimpleThread
+     * object.
+     *
+     * Normally the actual thread state is stored in the KVM vCPU. If KVM has
+     * been running this copy is will be out of date. If we recently handled
+     * some events within gem5 that required state to be updated this could be
+     * the most up-to-date copy. When getContext() or updateThreadContext() is
+     * called this copy gets updated.  The method syncThreadContext can
+     * be used within a KVM CPU to update the thread context if the
+     * KVM state is dirty (i.e., the vCPU has been run since the last
+     * update).
+     */
     SimpleThread *thread;
 
     /** ThreadContext object, provides an interface for external
@@ -272,6 +286,17 @@ class BaseKvmCPU : public BaseCPU
      * and update gem5's thread state.
      */
     virtual void updateThreadContext() = 0;
+
+    /**
+     * Update a thread context if the KVM state is dirty with respect
+     * to the cached thread context.
+     */
+    void syncThreadContext();
+
+    /**
+     * Update the KVM if the thread context is dirty.
+     */
+    void syncKvmState();
     /** @} */
 
     /** @{ */
@@ -391,7 +416,13 @@ class BaseKvmCPU : public BaseCPU
      * Is the gem5 context dirty? Set to true to force an update of
      * the KVM vCPU state upon the next call to kvmRun().
      */
-    bool contextDirty;
+    bool threadContextDirty;
+
+    /**
+     * Is the KVM state dirty? Set to true to force an update of
+     * the KVM vCPU state upon the next call to kvmRun().
+     */
+    bool kvmStateDirty;
 
     /** KVM internal ID of the vCPU */
     const long vcpuID;