kvm: Add support for multi-system simulation
The introduction of parallel event queues added most of the support
needed to run multiple VMs (systems) within the same gem5
instance. This changeset fixes up signal delivery so that KVM's
control signals are delivered to the thread that executes the CPU's
event queue. Specifically:
* Timers and counters are now initialized from a separate method
(startupThread) that is scheduled as the first event in the
thread-specific event queue. This ensures that they are
initialized from the thread that is going to execute the CPUs
event queue and enables signal delivery to the right thread when
exiting from KVM.
* The POSIX-timer-based KVM timer (used to force exits from KVM) has
been updated to deliver signals to the thread that's executing KVM
instead of the process (thread is undefined in that case). This
assumes that the timer is instantiated from the thread that is
going to execute the KVM vCPU.
* Signal masking is now done using pthread_sigmask instead of
sigprocmask. The behavior of the latter is undefined in threaded
applications.
* Since signal masks can be inherited, make sure to actively unmask
the control signals when setting up the KVM signal mask.
There are currently no facilities to multiplex between multiple KVM
CPUs in the same event queue, we are therefore limited to
configurations where there is only one KVM CPU per event queue. In
practice, this means that multi-system configurations can be
simulated, but not multiple CPUs in a shared-memory configuration.
This commit is contained in:
Andreas Sandberg
parent
fd9343eb85
commit
0d6009e8dc
|
|
@ -63,7 +63,7 @@
|
|||
/* Used by some KVM macros */
|
||||
#define PAGE_SIZE pageSize
|
||||
|
||||
volatile bool timerOverflowed = false;
|
||||
static volatile __thread bool timerOverflowed = false;
|
||||
|
||||
BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
|
||||
: BaseCPU(params),
|
||||
|
|
@ -92,18 +92,6 @@ BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
|
|||
thread->setStatus(ThreadContext::Halted);
|
||||
tc = thread->getTC();
|
||||
threadContexts.push_back(tc);
|
||||
|
||||
setupCounters();
|
||||
|
||||
if (params->usePerfOverflow)
|
||||
runTimer.reset(new PerfKvmTimer(hwCycles,
|
||||
KVM_TIMER_SIGNAL,
|
||||
params->hostFactor,
|
||||
params->hostFreq));
|
||||
else
|
||||
runTimer.reset(new PosixKvmTimer(KVM_TIMER_SIGNAL, CLOCK_MONOTONIC,
|
||||
params->hostFactor,
|
||||
params->hostFreq));
|
||||
}
|
||||
|
||||
BaseKvmCPU::~BaseKvmCPU()
|
||||
|
|
@ -177,6 +165,35 @@ BaseKvmCPU::startup()
|
|||
}
|
||||
|
||||
thread->startup();
|
||||
|
||||
Event *startupEvent(
|
||||
new EventWrapper<BaseKvmCPU,
|
||||
&BaseKvmCPU::startupThread>(this, true));
|
||||
schedule(startupEvent, curTick());
|
||||
}
|
||||
|
||||
void
|
||||
BaseKvmCPU::startupThread()
|
||||
{
|
||||
// Do thread-specific initialization. We need to setup signal
|
||||
// delivery for counters and timers from within the thread that
|
||||
// will execute the event queue to ensure that signals are
|
||||
// delivered to the right threads.
|
||||
const BaseKvmCPUParams * const p(
|
||||
dynamic_cast<const BaseKvmCPUParams *>(params()));
|
||||
|
||||
setupCounters();
|
||||
|
||||
if (p->usePerfOverflow)
|
||||
runTimer.reset(new PerfKvmTimer(hwCycles,
|
||||
KVM_TIMER_SIGNAL,
|
||||
p->hostFactor,
|
||||
p->hostFreq));
|
||||
else
|
||||
runTimer.reset(new PosixKvmTimer(KVM_TIMER_SIGNAL, CLOCK_MONOTONIC,
|
||||
p->hostFactor,
|
||||
p->hostFreq));
|
||||
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -1044,6 +1061,13 @@ BaseKvmCPU::flushCoalescedMMIO()
|
|||
/**
|
||||
* Cycle timer overflow when running in KVM. Forces the KVM syscall to
|
||||
* exit with EINTR and allows us to run the event queue.
|
||||
*
|
||||
* @warn This function might not be called since some kernels don't
|
||||
* seem to deliver signals when the signal is only unmasked when
|
||||
* running in KVM. This doesn't matter though since we are only
|
||||
* interested in getting KVM to exit, which happens as expected. See
|
||||
* setupSignalHandler() and kvmRun() for details about KVM signal
|
||||
* handling.
|
||||
*/
|
||||
static void
|
||||
onTimerOverflow(int signo, siginfo_t *si, void *data)
|
||||
|
|
@ -1079,20 +1103,22 @@ BaseKvmCPU::setupSignalHandler()
|
|||
panic("KVM: Failed to setup vCPU instruction signal handler\n");
|
||||
|
||||
sigset_t sigset;
|
||||
if (sigprocmask(SIG_BLOCK, NULL, &sigset) == -1)
|
||||
if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
|
||||
panic("KVM: Failed get signal mask\n");
|
||||
|
||||
// Request KVM to setup the same signal mask as we're currently
|
||||
// running with. We'll sometimes need to mask the KVM_TIMER_SIGNAL
|
||||
// to cause immediate exits from KVM after servicing IO
|
||||
// requests. See kvmRun().
|
||||
// running with except for the KVM control signals. We'll
|
||||
// sometimes need to raise the KVM_TIMER_SIGNAL to cause immediate
|
||||
// exits from KVM after servicing IO requests. See kvmRun().
|
||||
sigdelset(&sigset, KVM_TIMER_SIGNAL);
|
||||
sigdelset(&sigset, KVM_INST_SIGNAL);
|
||||
setSignalMask(&sigset);
|
||||
|
||||
// Mask our control signals so they aren't delivered unless we're
|
||||
// actually executing inside KVM.
|
||||
sigaddset(&sigset, KVM_TIMER_SIGNAL);
|
||||
sigaddset(&sigset, KVM_INST_SIGNAL);
|
||||
if (sigprocmask(SIG_SETMASK, &sigset, NULL) == -1)
|
||||
if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
|
||||
panic("KVM: Failed mask the KVM control signals\n");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -618,6 +618,20 @@ class BaseKvmCPU : public BaseCPU
|
|||
*/
|
||||
bool discardPendingSignal(int signum) const;
|
||||
|
||||
/**
|
||||
* Thread-specific initialization.
|
||||
*
|
||||
* Some KVM-related initialization requires us to know the TID of
|
||||
* the thread that is going to execute our event queue. For
|
||||
* example, when setting up timers, we need to know the TID of the
|
||||
* thread executing in KVM in order to deliver the timer signal to
|
||||
* that thread. This method is called as the first event in this
|
||||
* SimObject's event queue.
|
||||
*
|
||||
* @see startup
|
||||
*/
|
||||
void startupThread();
|
||||
|
||||
/** Try to drain the CPU if a drain is pending */
|
||||
bool tryDrain();
|
||||
|
||||
|
|
|
|||
|
|
@ -40,12 +40,30 @@
|
|||
#include <algorithm>
|
||||
#include <csignal>
|
||||
#include <ctime>
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
|
||||
#include "base/misc.hh"
|
||||
#include "base/trace.hh"
|
||||
#include "cpu/kvm/timer.hh"
|
||||
#include "debug/KvmTimer.hh"
|
||||
|
||||
/* According to timer_create(2), the value SIGEV_THREAD_ID can be used
|
||||
* to specify which thread a timer signal gets delivered to. According
|
||||
* to the man page, the member sigev_notify_thread is used to specify
|
||||
* the TID. This member is currently not defined by default in
|
||||
* siginfo.h on x86, so we define it here as a workaround.
|
||||
*/
|
||||
#ifndef sigev_notify_thread_id
|
||||
#define sigev_notify_thread_id _sigev_un._tid
|
||||
#endif
|
||||
|
||||
static pid_t
|
||||
gettid()
|
||||
{
|
||||
return syscall(__NR_gettid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimum number of cycles that a host can spend in a KVM call (used
|
||||
* to calculate the resolution of some timers).
|
||||
|
|
@ -62,11 +80,9 @@ PosixKvmTimer::PosixKvmTimer(int signo, clockid_t clockID,
|
|||
{
|
||||
struct sigevent sev;
|
||||
|
||||
// TODO: We should request signal delivery to thread instead of
|
||||
// the process here. Unfortunately this seems to be broken, or at
|
||||
// least not work as specified in the man page.
|
||||
sev.sigev_notify = SIGEV_SIGNAL;
|
||||
sev.sigev_notify = SIGEV_THREAD_ID;
|
||||
sev.sigev_signo = signo;
|
||||
sev.sigev_notify_thread_id = gettid();
|
||||
sev.sigev_value.sival_ptr = NULL;
|
||||
|
||||
while (timer_create(clockID, &sev, &timer) == -1) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue