sim: Handle cases where Drainable::resume() creates objects

There are cases where Drainable objects need to create new objects in
Drainable::resume(). In such cases, the local drain state will be
inherited from the DrainManager. We currently set the state to Running
as soon as we start resuming the simulator. This means that new
objects are created in the Running state rather than the Drained
state, which the resume code assumes. Depending on the traversal order
in DrainManager::resume(), this sometimes triggers a panic because the
object being resumed is in the wrong state.

This change introduces a new drain state, Resuming, that the
DrainManager enters as soon as it starts resuming the
simulator. Objects that are created while resuming are created in this
state. Such objects are then resumed in a subsequent pass over the
list of Drainable objects that need to be resumed. Once all objects
have been resumed, the simulator enters the Running state.

Change-Id: Ieee8645351ffbdec477e9cd2ff86fc795e459617
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Curtis Dunham <curtis.dunham@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/2600
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Reviewed-by: Weiping Liao <weipingliao@google.com>

src/sim/drain.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2015 ARM Limited
+ * Copyright (c) 2012, 2015, 2017 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -101,14 +101,33 @@ DrainManager::resume()
             "Resuming a system that isn't fully drained, this is untested and "
             "likely to break\n");
 
+    panic_if(_state == DrainState::Resuming,
+             "Resuming a system that is already trying to resume. This should "
+             "never happen.\n");
+
     panic_if(_count != 0,
              "Resume called in the middle of a drain cycle. %u objects "
              "left to drain.\n", _count);
 
-    DPRINTF(Drain, "Resuming %u objects.\n", drainableCount());
+    // At this point in time the DrainManager and all objects will be
+    // in the the Drained state. New objects (i.e., objects created
+    // while resuming) will inherit the Resuming state from the
+    // DrainManager, which means we have to resume objects until all
+    // objects are in the Running state.
+    _state = DrainState::Resuming;
+
+    do {
+        DPRINTF(Drain, "Resuming %u objects.\n", drainableCount());
+        for (auto *obj : _allDrainable) {
+            if (obj->drainState() != DrainState::Running) {
+                assert(obj->drainState() == DrainState::Drained ||
+                       obj->drainState() == DrainState::Resuming);
+                obj->dmDrainResume();
+            }
+        }
+    } while (!allInState(DrainState::Running));
+
     _state = DrainState::Running;
-    for (auto *obj : _allDrainable)
-        obj->dmDrainResume();
 }
 
 void
@@ -154,6 +173,17 @@ DrainManager::unregisterDrainable(Drainable *obj)
     _allDrainable.erase(o);
 }
 
+bool
+DrainManager::allInState(DrainState state) const
+{
+    for (const auto *obj : _allDrainable) {
+        if (obj->drainState() != state)
+            return false;
+    }
+
+    return true;
+}
+
 size_t
 DrainManager::drainableCount() const
 {
@@ -189,7 +219,8 @@ Drainable::dmDrain()
 void
 Drainable::dmDrainResume()
 {
-    panic_if(_drainState != DrainState::Drained,
+    panic_if(_drainState != DrainState::Drained &&
+             _drainState != DrainState::Resuming,
              "Trying to resume an object that hasn't been drained\n");
 
     _drainState = DrainState::Running;

src/sim/drain.hh

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2015 ARM Limited
+ * Copyright (c) 2012, 2015, 2017 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -58,7 +58,11 @@ class Drainable;
  * all objects have entered the Drained state.
  *
  * Before resuming simulation, the simulator calls resume() to
- * transfer the object to the Running state.
+ * transfer the object to the Running state. This in turn results in a
+ * call to drainResume() for all Drainable objects in the
+ * simulator. New Drainable objects may be created while resuming. In
+ * such cases, the new objects will be created in the Resuming state
+ * and later resumed.
  *
  * \note Even though the state of an object (visible to the rest of
  * the world through Drainable::getState()) could be used to determine
@@ -68,7 +72,8 @@ class Drainable;
 enum class DrainState {
     Running,  /** Running normally */
     Draining, /** Draining buffers pending serialization/handover */
-    Drained   /** Buffers drained, ready for serialization/handover */
+    Drained,  /** Buffers drained, ready for serialization/handover */
+    Resuming, /** Transient state while the simulator is resuming */
 };
 #endif
 
@@ -152,6 +157,12 @@ class DrainManager
     void unregisterDrainable(Drainable *obj);
 
   private:
+    /**
+     * Helper function to check if all Drainable objects are in a
+     * specific state.
+     */
+    bool allInState(DrainState state) const;
+
     /**
      * Thread-safe helper function to get the number of Drainable
      * objects in a system.
@@ -261,6 +272,7 @@ class Drainable
         switch (_drainState) {
           case DrainState::Running:
           case DrainState::Drained:
+          case DrainState::Resuming:
             return;
           case DrainState::Draining:
             _drainState = DrainState::Drained;