dev, arm: Add draining to the GIC model

The GIC model currently adds a delay to interrupts when posting them
to a target CPU. This means that an interrupt signal will be
represented by an event for a short period of time. We currently
ignore this when draining and serialize the tick when the interrupt
will fire. Upon loading the checkpoint, the simulated GIC reschedules
the pending events. This behaviour is undesirable when we implement
support for switching between in-kernel GIC emulation and gem5 GIC
emulation. In that case, the (kernel) GIC model gets a lot simpler if
we don't need to worry about in-flight interrupts from the gem5 GIC.

This changeset adds a draining check to force the GIC into a state
where all interrupts have been delivered prior to checkpointing/CPU
switching. It also removes the now redundant serialization of
interrupt events.

Change-Id: I8b8b080aa291ca029a3a7bdd1777f1fcd5b01179
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Curtis Dunham <curtis.dunham@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/2331
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>

src/dev/arm/gic_pl390.cc

@@ -76,7 +76,8 @@ Pl390::Pl390(const Params *p)
       cpuSgiPending {}, cpuSgiActive {},
       cpuSgiPendingExt {}, cpuSgiActiveExt {},
       cpuPpiPending {}, cpuPpiActive {},
-      irqEnable(false)
+      irqEnable(false),
+      pendingDelayedInterrupts(0)
 {
     for (int x = 0; x < CPU_MAX; x++) {
         iccrpr[x] = 0xff;
@@ -85,7 +86,7 @@ Pl390::Pl390(const Params *p)
         cpuBpr[x] = 0;
         // Initialize cpu highest int
         cpuHighestInt[x] = SPURIOUS_INT;
-        postIntEvent[x] = new PostIntEvent(x, p->platform);
+        postIntEvent[x] = new PostIntEvent(*this, x);
     }
     DPRINTF(Interrupt, "cpuEnabled[0]=%d cpuEnabled[1]=%d\n", cpuEnabled[0],
             cpuEnabled[1]);
@@ -812,10 +813,31 @@ Pl390::clearPPInt(uint32_t num, uint32_t cpu)
 void
 Pl390::postInt(uint32_t cpu, Tick when)
 {
-    if (!(postIntEvent[cpu]->scheduled()))
+    if (!(postIntEvent[cpu]->scheduled())) {
+        ++pendingDelayedInterrupts;
         eventq->schedule(postIntEvent[cpu], when);
+    }
 }
 
+void
+Pl390::postDelayedInt(uint32_t cpu)
+{
+    platform->intrctrl->post(cpu, ArmISA::INT_IRQ, 0);
+    --pendingDelayedInterrupts;
+    assert(pendingDelayedInterrupts >= 0);
+    if (pendingDelayedInterrupts == 0)
+        signalDrainDone();
+}
+
+DrainState
+Pl390::drain()
+{
+    if (pendingDelayedInterrupts == 0) {
+        return DrainState::Drained;
+    } else {
+        return DrainState::Draining;
+    }
+}
 
 void
 Pl390::serialize(CheckpointOut &cp) const
@@ -842,14 +864,6 @@ Pl390::serialize(CheckpointOut &cp) const
     SERIALIZE_ARRAY(cpuPpiActive, CPU_MAX);
     SERIALIZE_ARRAY(cpuPpiPending, CPU_MAX);
     SERIALIZE_SCALAR(irqEnable);
-    Tick interrupt_time[CPU_MAX];
-    for (uint32_t cpu = 0; cpu < CPU_MAX; cpu++) {
-        interrupt_time[cpu] = 0;
-        if (postIntEvent[cpu]->scheduled()) {
-            interrupt_time[cpu] = postIntEvent[cpu]->when();
-        }
-    }
-    SERIALIZE_ARRAY(interrupt_time, CPU_MAX);
     SERIALIZE_SCALAR(gem5ExtensionsEnabled);
 
     for (uint32_t i=0; i < bankedRegs.size(); ++i) {
@@ -895,6 +909,9 @@ Pl390::unserialize(CheckpointIn &cp)
     UNSERIALIZE_ARRAY(cpuPpiPending, CPU_MAX);
     UNSERIALIZE_SCALAR(irqEnable);
 
+    // Handle checkpoints from before we drained the GIC to prevent
+    // in-flight interrupts.
+    if (cp.entryExists(Serializable::currentSection(), "interrupt_time")) {
         Tick interrupt_time[CPU_MAX];
         UNSERIALIZE_ARRAY(interrupt_time, CPU_MAX);
 
@@ -902,6 +919,8 @@ Pl390::unserialize(CheckpointIn &cp)
             if (interrupt_time[cpu])
                 schedule(postIntEvent[cpu], interrupt_time[cpu]);
         }
+    }
+
     if (!UNSERIALIZE_OPT_SCALAR(gem5ExtensionsEnabled))
         gem5ExtensionsEnabled = false;
 

src/dev/arm/gic_pl390.hh

@@ -318,25 +318,32 @@ class Pl390 : public BaseGic
     int intNumToWord(int num) const { return num >> 5; }
     int intNumToBit(int num) const { return num % 32; }
 
-    /** Post an interrupt to a CPU
+    /**
+     * Post an interrupt to a CPU with a delay
      */
     void postInt(uint32_t cpu, Tick when);
 
+    /**
+     * Deliver a delayed interrupt to the target CPU
+     */
+    void postDelayedInt(uint32_t cpu);
+
     /** Event definition to post interrupt to CPU after a delay
     */
     class PostIntEvent : public Event
     {
       private:
+        Pl390 &parent;
         uint32_t cpu;
-        Platform *platform;
       public:
-        PostIntEvent( uint32_t c, Platform* p)
-            : cpu(c), platform(p)
+        PostIntEvent(Pl390 &_parent, uint32_t _cpu)
+            : parent(_parent), cpu(_cpu)
         { }
-        void process() { platform->intrctrl->post(cpu, ArmISA::INT_IRQ, 0);}
+        void process() { parent.postDelayedInt(cpu); }
         const char *description() const { return "Post Interrupt to CPU"; }
     };
     PostIntEvent *postIntEvent[CPU_MAX];
+    int pendingDelayedInterrupts;
 
   public:
     typedef Pl390Params Params;
@@ -347,6 +354,8 @@ class Pl390 : public BaseGic
     }
     Pl390(const Params *p);
 
+    DrainState drain() override;
+
     void serialize(CheckpointOut &cp) const override;
     void unserialize(CheckpointIn &cp) override;