ruby: connects sm queues to the network

configs/example/memtest-ruby.py

@@ -97,15 +97,21 @@ class L2Cache(RubyCache):
     latency = 15
     size = 1048576
 
-# It would be nice to lump all the network nodes into a single list,
-# but for consistency with the old scripts I'm segregating them by
-# type.  I'm not sure if this is really necessary or not.
+#
+# The ruby network creation expects the list of nodes in the system to be
+# consistent with the NetDest list.  Therefore the l1 controller nodes must be
+# listed before the directory nodes and directory nodes before dma nodes, etc.
+#
  
 # net_nodes = []
 l1_cntrl_nodes = []
 dir_cntrl_nodes = []
 
-for cpu in cpus:
+#
+# Must create the individual controllers before the network to ensure the
+# controller constructors are called before the network constructor
+#
+for (i, cpu) in enumerate(cpus):
     l1_cntrl = L1Cache_Controller()
     cpu_seq = RubySequencer(controller = l1_cntrl,
                             icache = L1Cache(controller = l1_cntrl),
@@ -124,6 +130,10 @@ for cpu in cpus:
     l1_cntrl_nodes.append(l1_cntrl)
     dir_cntrl_nodes.append(dir_cntrl)
 
+#
+# Important: the topology constructor must be called before the network
+# constructor.
+#
 network = SimpleNetwork(topology = makeCrossbar(l1_cntrl_nodes + \
                                                 dir_cntrl_nodes))
 

src/mem/ruby/network/Network.cc

@@ -28,6 +28,7 @@
 
 #include "mem/protocol/MachineType.hh"
 #include "mem/ruby/network/Network.hh"
+#include "mem/ruby/network/simple/Topology.hh"
 
 Network::Network(const Params *p)
     : SimObject(p)
@@ -40,14 +41,24 @@ Network::Network(const Params *p)
     m_link_latency = p->link_latency;
     m_control_msg_size = p->control_msg_size;
 
+    //
+    // Total nodes/controllers in network
+    // Must make sure this is called after the State Machine constructors
+    //
+    m_nodes = MachineType_base_number(MachineType_NUM); 
+    assert(m_nodes != 0);
+
     assert(m_virtual_networks != 0);
     assert(m_topology_ptr != NULL);
+
+    //
+    // Initialize the controller's network pointers
+    //
+    m_topology_ptr->initNetworkPtr(this);
 }
 
 void Network::init()
 {
-  m_nodes = MachineType_base_number(MachineType_NUM); // Total nodes in network
-
   m_data_msg_size = RubySystem::getBlockSizeBytes() + m_control_msg_size;
 }
 

src/mem/ruby/network/simple/SimpleNetwork.cc

@@ -62,13 +62,12 @@ Network* Network::createNetwork(int nodes)
 SimpleNetwork::SimpleNetwork(const Params *p)
     : Network(p)
 {
-}
-
-void SimpleNetwork::init()
-{
-
-  Network::init();
-
+  //
+  // Note: the parent Network Object constructor is called before the 
+  // SimpleNetwork child constructor.  Therefore, the member variables
+  // used below should already be initialized.
+  //
+  
   m_endpoint_switches.setSize(m_nodes);
 
   m_in_use.setSize(m_virtual_networks);
@@ -91,10 +90,18 @@ void SimpleNetwork::init()
                    "fromNet node "+int_to_string(node)+" j "+int_to_string(j));
     }
   }
+}
 
-  // Setup the network switches
-  //  m_topology_ptr = new Topology(this, m_nodes);
-  m_topology_ptr->makeTopology();
+void SimpleNetwork::init()
+{
+
+  Network::init();
+
+  //
+  // The topology pointer should have already been initialized in the parent 
+  // class network constructor.
+  //
+  assert(m_topology_ptr != NULL);
   int number_of_switches = m_topology_ptr->numSwitches();
   for (int i=0; i<number_of_switches; i++) {
     m_switch_ptr_vector.insertAtBottom(new Switch(i, this));

src/mem/ruby/network/simple/Topology.cc

@@ -68,54 +68,41 @@ Topology::Topology(const Params *p)
 {
     m_print_config = p->print_config;
     m_number_of_switches = p->num_int_nodes;
-  // initialize component latencies record
-  m_component_latencies.setSize(0);
-  m_component_inter_switches.setSize(0);
-}
+    // initialize component latencies record
+    m_component_latencies.setSize(0);
+    m_component_inter_switches.setSize(0);
 
-void Topology::init()
-{
-    // need to defer this until init, to guarantee that constructors
-    // for all the controller objects have been called.
+    //
+    // Total nodes/controllers in network
+    // Must make sure this is called after the State Machine constructors
+    //
     m_nodes = MachineType_base_number(MachineType_NUM);
-}
+    assert(m_nodes > 1);
 
-void Topology::makeTopology()
-{
     if (m_nodes != params()->ext_links.size()) {
         fatal("m_nodes (%d) != ext_links vector length (%d)\n",
               m_nodes != params()->ext_links.size());
     }
 
-    
-
-  /*
-  if (m_nodes == 1) {
-    SwitchID id = newSwitchID();
-    addLink(0, id, m_network_ptr->getOffChipLinkLatency());
-    addLink(id, 1, m_network_ptr->getOffChipLinkLatency());
-    return;
-  }
-  */
-  assert(m_nodes > 1);
-
-  Vector< Vector < SwitchID > > nodePairs;  // node pairs extracted from the file
-  Vector<int> latencies;  // link latencies for each link extracted
-  Vector<int> bw_multis;  // bw multipliers for each link extracted
-  Vector<int> weights;  // link weights used to enfore e-cube deadlock free routing
-  Vector< SwitchID > int_network_switches;  // internal switches extracted from the file
-  Vector<bool> endpointConnectionExist;  // used to ensure all endpoints are connected to the network
-
+    //
+    // First create the links between the endpoints (i.e. controllers) and the
+    // network.
+    //
   for (vector<ExtLink*>::const_iterator i = params()->ext_links.begin();
        i != params()->ext_links.end(); ++i)
   {
       const ExtLinkParams *p = (*i)->params();
       AbstractController *c = p->ext_node;
+
+      // Store the controller pointers for later
+      m_controller_vector.insertAtBottom(c);
+
       int ext_idx1 =
           MachineType_base_number(c->getMachineType()) + c->getVersion();
       int ext_idx2 = ext_idx1 + m_nodes;
       int int_idx = p->int_node + 2*m_nodes;
 
+      // create the links in both directions
       addLink(ext_idx1, int_idx, p->latency, p->bw_multiplier, p->weight);
       addLink(int_idx, ext_idx2, p->latency, p->bw_multiplier, p->weight);
   }
@@ -126,12 +113,23 @@ void Topology::makeTopology()
       const IntLinkParams *p = (*i)->params();
       int a = p->node_a + 2*m_nodes;
       int b = p->node_b + 2*m_nodes;
+
+      // create the links in both directions
       addLink(a, b, p->latency, p->bw_multiplier, p->weight);
       addLink(b, a, p->latency, p->bw_multiplier, p->weight);
   }
 }
 
 
+void Topology::initNetworkPtr(Network* net_ptr)
+{
+    for (int cntrl = 0; cntrl < m_controller_vector.size(); cntrl++)
+    {
+        m_controller_vector[cntrl]->initNetworkPtr(net_ptr);
+    }
+}
+
+
 void Topology::createLinks(Network *net, bool isReconfiguration)
 {
   // Find maximum switchID

src/mem/ruby/network/simple/Topology.hh

@@ -95,13 +95,12 @@ public:
   // Destructor
   virtual ~Topology() {}
 
-  virtual void init();
-
   // Public Methods
-  void makeTopology();
   int numSwitches() const { return m_number_of_switches; }
   void createLinks(Network *net, bool isReconfiguration);
 
+  void initNetworkPtr(Network* net_ptr);
+
   const string getName() { return m_name; }
   void printStats(ostream& out) const {}
   void clearStats() {}
@@ -129,6 +128,8 @@ protected:
   NodeID m_nodes;
   int m_number_of_switches;
 
+  Vector<AbstractController*> m_controller_vector;
+
   Vector<SwitchID> m_links_src_vector;
   Vector<SwitchID> m_links_dest_vector;
   Vector<int> m_links_latency_vector;

src/mem/ruby/slicc_interface/AbstractController.hh

@@ -8,6 +8,7 @@
 #include "mem/ruby/common/Consumer.hh"
 #include "mem/protocol/MachineType.hh"
 #include "mem/ruby/common/Address.hh"
+#include "mem/ruby/network/Network.hh"
 
 class MessageBuffer;
 class Network;
@@ -26,6 +27,7 @@ public:
   virtual const MachineType getMachineType() const = 0;
   virtual void blockOnQueue(Address, MessageBuffer*) = 0;
   virtual void unblock(Address) = 0;
+  virtual void initNetworkPtr(Network* net_ptr) = 0;
 
   virtual void print(ostream & out) const = 0;
   virtual void printStats(ostream & out) const = 0;

src/mem/slicc/symbols/StateMachine.py

@@ -209,6 +209,7 @@ public:
     const string toString() const;
     const string getName() const;
     const MachineType getMachineType() const;
+    void initNetworkPtr(Network* net_ptr) { m_net_ptr = net_ptr; }
     void print(ostream& out) const;
     void printConfig(ostream& out) const;
     void wakeup();