Garnet: Stats at vnet granularity + code cleanup

This patch
(1) Moves redundant code from fixed and flexible networks to BaseGarnetNetwork.
(2) Prints network stats at vnet granularity.

src/mem/ruby/network/garnet/BaseGarnetNetwork.cc

@@ -43,10 +43,6 @@ BaseGarnetNetwork::BaseGarnetNetwork(const Params *p)
         fault_model = p->fault_model;
 
     m_ruby_start = 0;
-    m_flits_received = 0;
-    m_flits_injected = 0;
-    m_network_latency = 0.0;
-    m_queueing_latency = 0.0;
 
     // Currently Garnet only supports uniform bandwidth for all
     // links and network interfaces.
@@ -77,9 +73,17 @@ BaseGarnetNetwork::BaseGarnetNetwork(const Params *p)
 
     m_in_use.resize(m_virtual_networks);
     m_ordered.resize(m_virtual_networks);
+    m_flits_received.resize(m_virtual_networks);
+    m_flits_injected.resize(m_virtual_networks);
+    m_network_latency.resize(m_virtual_networks);
+    m_queueing_latency.resize(m_virtual_networks);
     for (int i = 0; i < m_virtual_networks; i++) {
         m_in_use[i] = false;
         m_ordered[i] = false;
+        m_flits_received[i] = 0;
+        m_flits_injected[i] = 0;
+        m_network_latency[i] = 0.0;
+        m_queueing_latency[i] = 0.0;
     }
 
     for (int node = 0; node < m_nodes; node++) {
@@ -101,3 +105,87 @@ BaseGarnetNetwork::init()
 {
     Network::init();
 }
+
+MessageBuffer*
+BaseGarnetNetwork::getToNetQueue(NodeID id, bool ordered, int network_num,
+                                 string vnet_type)
+{
+    checkNetworkAllocation(id, ordered, network_num, vnet_type);
+    return m_toNetQueues[id][network_num];
+}
+
+MessageBuffer*
+BaseGarnetNetwork::getFromNetQueue(NodeID id, bool ordered, int network_num,  
+                                   string vnet_type)
+{
+    checkNetworkAllocation(id, ordered, network_num, vnet_type);
+    return m_fromNetQueues[id][network_num];
+}
+
+void
+BaseGarnetNetwork::clearStats()
+{
+    m_ruby_start = g_eventQueue_ptr->getTime();
+}
+
+Time
+BaseGarnetNetwork::getRubyStartTime()
+{
+    return m_ruby_start;
+}
+
+void
+BaseGarnetNetwork::printStats(ostream& out) const
+{
+    out << endl;
+    out << "Network Stats" << endl;
+    out << "-------------" << endl;
+    out << endl;
+    printPerformanceStats(out);
+    printLinkStats(out);
+    printPowerStats(out);
+    m_topology_ptr->printStats(out);
+}
+
+void
+BaseGarnetNetwork::printPerformanceStats(ostream& out) const
+{
+    int total_flits_injected = 0;
+    int total_flits_received = 0;
+    int total_network_latency = 0.0;
+    int total_queueing_latency = 0.0;
+
+    for (int i = 0; i < m_virtual_networks; i++) {
+        if (!m_in_use[i])
+            continue;
+
+        out << "[Vnet " << i << "]: flits injected = "
+            << m_flits_injected[i] << endl;
+        out << "[Vnet " << i << "]: flits received = "
+            << m_flits_received[i] << endl;
+        out << "[Vnet " << i << "]: average network latency = "
+            << ((double) m_network_latency[i] / (double) m_flits_received[i])
+            << endl;
+        out << "[Vnet " << i << "]: average queueing (at source NI) latency = "
+            << ((double) m_queueing_latency[i] / (double) m_flits_received[i])
+            << endl;
+
+        out << endl;
+        total_flits_injected += m_flits_injected[i];
+        total_flits_received += m_flits_received[i];
+        total_network_latency += m_network_latency[i];
+        total_queueing_latency += m_queueing_latency[i];
+    }
+    out << "Total flits injected = " << total_flits_injected << endl;
+    out << "Total flits received = " << total_flits_received << endl;
+    out << "Average network latency = "
+        << ((double) total_network_latency/ (double) total_flits_received) << endl;
+    out << "Average queueing (at source NI) latency = "
+        << ((double) total_queueing_latency/ (double) total_flits_received) << endl;
+    out << "Average latency = "
+        << ((double)  (total_queueing_latency + total_network_latency) /
+            (double) total_flits_received)<< endl;
+    out << "-------------" << endl;
+    out << endl;
+}
+

src/mem/ruby/network/garnet/BaseGarnetNetwork.hh

@@ -54,15 +54,49 @@ class BaseGarnetNetwork : public Network
     bool isFaultModelEnabled() {return m_enable_fault_model;}
     FaultModel* fault_model;
 
+    void increment_injected_flits(int vnet) { m_flits_injected[vnet]++; }
+    void increment_received_flits(int vnet) { m_flits_received[vnet]++; }
+
+    void
+    increment_network_latency(Time latency, int vnet)
+    {
+        m_network_latency[vnet] += latency;
+    }
+
+    void
+    increment_queueing_latency(Time latency, int vnet)
+    {
+        m_queueing_latency[vnet] += latency;
+    }
+
+    // returns the queue requested for the given component
+    MessageBuffer* getToNetQueue(NodeID id, bool ordered, int network_num,
+                                 std::string vnet_type);
+    MessageBuffer* getFromNetQueue(NodeID id, bool ordered, int network_num,
+                                   std::string vnet_type);
+
+
+    bool isVNetOrdered(int vnet) { return m_ordered[vnet]; }
+    bool validVirtualNetwork(int vnet) { return m_in_use[vnet]; }
+    virtual void checkNetworkAllocation(NodeID id, bool ordered,
+        int network_num, std::string vnet_type) = 0;
+
+    Time getRubyStartTime();
+    void clearStats();
+    void printStats(std::ostream& out) const;
+    void printPerformanceStats(std::ostream& out) const;
+    virtual void printLinkStats(std::ostream& out) const = 0;
+    virtual void printPowerStats(std::ostream& out) const = 0;
+
   protected:
     int m_ni_flit_size;
     int m_vcs_per_vnet;
     bool m_enable_fault_model;
 
-    int m_flits_received;
+    std::vector<int> m_flits_received;
-    int m_flits_injected;
+    std::vector<int> m_flits_injected;
-    double m_network_latency;
+    std::vector<double> m_network_latency;
-    double m_queueing_latency;
+    std::vector<double> m_queueing_latency;
 
     std::vector<bool> m_in_use;
     std::vector<bool> m_ordered;

src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.cc

@@ -257,49 +257,17 @@ GarnetNetwork_d::checkNetworkAllocation(NodeID id, bool ordered,
         m_vnet_type[network_num] = CTRL_VNET_; // carries only ctrl packets
 }
 
-MessageBuffer*
-GarnetNetwork_d::getToNetQueue(NodeID id, bool ordered, int network_num,
-                               string vnet_type)
-{
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    return m_toNetQueues[id][network_num];
-}
-
-MessageBuffer*
-GarnetNetwork_d::getFromNetQueue(NodeID id, bool ordered, int network_num,  
-                                 string vnet_type)
-{
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    return m_fromNetQueues[id][network_num];
-}
-
 void
-GarnetNetwork_d::clearStats()
+GarnetNetwork_d::printLinkStats(ostream& out) const
-{
-    m_ruby_start = g_eventQueue_ptr->getTime();
-}
-
-Time
-GarnetNetwork_d::getRubyStartTime()
-{
-    return m_ruby_start;
-}
-
-void
-GarnetNetwork_d::printStats(ostream& out) const
 {
     double average_link_utilization = 0;
     vector<double> average_vc_load;
     average_vc_load.resize(m_virtual_networks*m_vcs_per_vnet);
 
-    for (int i = 0; i < m_virtual_networks*m_vcs_per_vnet; i++)
+    for (int i = 0; i < m_virtual_networks*m_vcs_per_vnet; i++) {
-    {
         average_vc_load[i] = 0;
     }
 
-    out << endl;
-    out << "Network Stats" << endl;
-    out << "-------------" << endl;
     out << endl;
     for (int i = 0; i < m_link_ptr_vector.size(); i++) {
         average_link_utilization +=
@@ -328,18 +296,14 @@ GarnetNetwork_d::printStats(ostream& out) const
             << " flits/cycle " << endl;
     }
     out << "-------------" << endl;
+    out << endl;
+}
 
-    out << "Total flits injected = " << m_flits_injected << endl;
+void
-    out << "Total flits received = " << m_flits_received << endl;
+GarnetNetwork_d::printPowerStats(ostream& out) const
-    out << "Average network latency = "
+{
-        << ((double) m_network_latency/ (double) m_flits_received)<< endl;
+    out << "Network Power" << endl;
-    out << "Average queueing (at source NI) latency = "
-        << ((double) m_queueing_latency/ (double) m_flits_received)<< endl;
-    out << "Average latency = "
-        << ((double)  (m_queueing_latency + m_network_latency) /
-            (double) m_flits_received)<< endl;
     out << "-------------" << endl;
-
     double m_total_link_power = 0.0;
     double m_dynamic_link_power = 0.0;
     double m_static_link_power = 0.0;
@@ -368,7 +332,7 @@ GarnetNetwork_d::printStats(ostream& out) const
     out << "Router Static Power = " << m_static_router_power << " W" << endl;
     out << "Total Router Power = " << m_total_router_power << " W " <<endl;
     out << "-------------" << endl;
-    m_topology_ptr->printStats(out);
+    out << endl;
 }
 
 void
@@ -377,7 +341,7 @@ GarnetNetwork_d::printConfig(ostream& out) const
     out << endl;
     out << "Network Configuration" << endl;
     out << "---------------------" << endl;
-    out << "network: GarnetNetwork_d" << endl;
+    out << "network: Garnet Fixed Pipeline" << endl;
     out << "topology: " << m_topology_ptr->getName() << endl;
     out << endl;
 

src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.hh

@@ -63,14 +63,8 @@ class GarnetNetwork_d : public BaseGarnetNetwork
     int getBuffersPerDataVC() {return m_buffers_per_data_vc; }
     int getBuffersPerCtrlVC() {return m_buffers_per_ctrl_vc; }
 
-    // returns the queue requested for the given component
+    void printLinkStats(std::ostream& out) const;
-    MessageBuffer* getToNetQueue(NodeID id, bool ordered, int network_num,
+    void printPowerStats(std::ostream& out) const;
-                                 std::string vnet_type);
-    MessageBuffer* getFromNetQueue(NodeID id, bool ordered, int network_num,
-                                   std::string vnet_type);
-
-    void clearStats();
-    void printStats(std::ostream& out) const;
     void printConfig(std::ostream& out) const;
     void print(std::ostream& out) const;
 
@@ -81,26 +75,6 @@ class GarnetNetwork_d : public BaseGarnetNetwork
         return m_vnet_type[vnet];
     }
 
-
-    inline void increment_injected_flits() { m_flits_injected++; }
-    inline void increment_received_flits() { m_flits_received++; }
-
-    inline void
-    increment_network_latency(Time latency)
-    {
-        m_network_latency += latency;
-    }
-
-    inline void
-    increment_queueing_latency(Time latency)
-    {
-        m_queueing_latency += latency;
-    }
-
-    bool isVNetOrdered(int vnet) { return m_ordered[vnet]; }
-    bool validVirtualNetwork(int vnet) { return m_in_use[vnet]; }
-    Time getRubyStartTime();
-
     void reset();
 
     // Methods used by Topology to setup the network

src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc

@@ -167,7 +167,7 @@ NetworkInterface_d::flitisizeMessage(MsgPtr msg_ptr, int vnet)
         }
 
         for (int i = 0; i < num_flits; i++) {
-            m_net_ptr->increment_injected_flits();
+            m_net_ptr->increment_injected_flits(vnet);
             flit_d *fl = new flit_d(i, vc, vnet, num_flits, new_msg_ptr);
             fl->set_delay(g_eventQueue_ptr->getTime() - msg_ptr->getTime());
             m_ni_buffers[vc]->insert(fl);
@@ -247,12 +247,13 @@ NetworkInterface_d::wakeup()
         creditQueue->insert(credit_flit);
         g_eventQueue_ptr->scheduleEvent(m_ni_credit_link, 1);
 
-        m_net_ptr->increment_received_flits();
+        int vnet = t_flit->get_vnet();
+        m_net_ptr->increment_received_flits(vnet);
         int network_delay = g_eventQueue_ptr->getTime() -
                             t_flit->get_enqueue_time();
         int queueing_delay = t_flit->get_delay();
-        m_net_ptr->increment_network_latency(network_delay);
+        m_net_ptr->increment_network_latency(network_delay, vnet);
-        m_net_ptr->increment_queueing_latency(queueing_delay);
+        m_net_ptr->increment_queueing_latency(queueing_delay, vnet);
         delete t_flit;
     }
 

src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.cc

@@ -181,7 +181,7 @@ GarnetNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
 
 void
 GarnetNetwork::checkNetworkAllocation(NodeID id, bool ordered,
-    int network_num)
+    int network_num, std::string vnet_type)
 {
     assert(id < m_nodes);
     assert(network_num < m_virtual_networks);
@@ -192,36 +192,8 @@ GarnetNetwork::checkNetworkAllocation(NodeID id, bool ordered,
     m_in_use[network_num] = true;
 }
 
-MessageBuffer*
-GarnetNetwork::getToNetQueue(NodeID id, bool ordered, int network_num,
-                             std::string vnet_type)
-{
-    checkNetworkAllocation(id, ordered, network_num);
-    return m_toNetQueues[id][network_num];
-}
-
-MessageBuffer*
-GarnetNetwork::getFromNetQueue(NodeID id, bool ordered, int network_num,
-                               std::string vnet_type)
-{
-    checkNetworkAllocation(id, ordered, network_num);
-    return m_fromNetQueues[id][network_num];
-}
-
 void
-GarnetNetwork::clearStats()
+GarnetNetwork::printLinkStats(ostream& out) const
-{
-    m_ruby_start = g_eventQueue_ptr->getTime();
-}
-
-Time
-GarnetNetwork::getRubyStartTime()
-{
-    return m_ruby_start;
-}
-
-void
-GarnetNetwork::printStats(ostream& out) const
 {
     double average_link_utilization = 0;
     vector<double> average_vc_load;
@@ -231,13 +203,12 @@ GarnetNetwork::printStats(ostream& out) const
         average_vc_load[i] = 0;
     }
 
-    out << endl;
-    out << "Network Stats" << endl;
-    out << "-------------" << endl;
     out << endl;
     for (int i = 0; i < m_link_ptr_vector.size(); i++) {
         average_link_utilization +=
-            m_link_ptr_vector[i]->getLinkUtilization();
+            (double(m_link_ptr_vector[i]->getLinkUtilization())) /
+            (double(g_eventQueue_ptr->getTime()-m_ruby_start));
+
         vector<int> vc_load = m_link_ptr_vector[i]->getVcLoad();
         for (int j = 0; j < vc_load.size(); j++) {
             assert(vc_load.size() == m_vcs_per_vnet*m_virtual_networks);
@@ -246,8 +217,8 @@ GarnetNetwork::printStats(ostream& out) const
     }
     average_link_utilization =
         average_link_utilization/m_link_ptr_vector.size();
-    out << "Average Link Utilization :: " << average_link_utilization <<
+    out << "Average Link Utilization :: " << average_link_utilization
-           " flits/cycle" <<endl;
+        << " flits/cycle" << endl;
     out << "-------------" << endl;
 
     for (int i = 0; i < m_vcs_per_vnet*m_virtual_networks; i++) {
@@ -256,23 +227,20 @@ GarnetNetwork::printStats(ostream& out) const
 
         average_vc_load[i] = (double(average_vc_load[i]) /
             (double(g_eventQueue_ptr->getTime()) - m_ruby_start));
-        out << "Average VC Load [" << i << "] = " << average_vc_load[i] <<
+        out << "Average VC Load [" << i << "] = " << average_vc_load[i]
-               " flits/cycle" << endl;
+            << " flits/cycle " << endl;
     }
     out << "-------------" << endl;
+    out << endl;
+}
 
-    out << "Total flits injected = " << m_flits_injected << endl;
+void
-    out << "Total flits received = " << m_flits_received << endl;
+GarnetNetwork::printPowerStats(ostream& out) const
-    out << "Average network latency = "
+{
-        << ((double) m_network_latency/ (double) m_flits_received)<< endl;
+    out << "Network Power" << endl;
-    out << "Average queueing (at source NI) latency = "
-        << ((double) m_queueing_latency/ (double) m_flits_received)<< endl;
-    out << "Average latency = "
-        << ((double)  (m_queueing_latency + m_network_latency) /
-            (double) m_flits_received)<< endl;
     out << "-------------" << endl;
-
+    out << "Orion does not work with flexible pipeline" << endl;
-    m_topology_ptr->printStats(out);
+    out << endl;
 }
 
 void
@@ -281,7 +249,7 @@ GarnetNetwork::printConfig(ostream& out) const
     out << endl;
     out << "Network Configuration" << endl;
     out << "---------------------" << endl;
-    out << "network: GARNET_NETWORK" << endl;
+    out << "network: Garnet Flexible Pipeline" << endl;
     out << "topology: " << m_topology_ptr->getName() << endl;
     out << endl;
 

src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.hh

@@ -59,40 +59,15 @@ class GarnetNetwork : public BaseGarnetNetwork
     int getBufferSize() { return m_buffer_size; }
     int getNumPipeStages() {return m_number_of_pipe_stages; }
 
-    // returns the queue requested for the given component
-    MessageBuffer* getToNetQueue(NodeID id, bool ordered, int network_num,
-                                 std::string vnet_type);
-    MessageBuffer* getFromNetQueue(NodeID id, bool ordered, int network_num,
-                                   std::string vnet_type);
-
-    void clearStats();
-    void printStats(std::ostream& out) const;
-    void printConfig(std::ostream& out) const;
-    void print(std::ostream& out) const;
-
-    inline void increment_injected_flits() { m_flits_injected++; }
-    inline void increment_received_flits() { m_flits_received++; }
-
-    inline void
-    increment_network_latency(Time latency)
-    {
-        m_network_latency += latency;
-    }
-
-    inline void
-    increment_queueing_latency(Time latency)
-    {
-        m_queueing_latency += latency;
-    }
-
-    bool isVNetOrdered(int vnet) { return m_ordered[vnet]; }
-    bool validVirtualNetwork(int vnet) { return m_in_use[vnet]; }
-
-    Time getRubyStartTime();
     int getNumNodes(){ return m_nodes; }
 
     void reset();
 
+    void printLinkStats(std::ostream& out) const;
+    void printPowerStats(std::ostream& out) const;
+    void printConfig(std::ostream& out) const;
+    void print(std::ostream& out) const;
+
     // Methods used by Topology to setup the network
     void makeOutLink(SwitchID src, NodeID dest, BasicLink* link, 
                      LinkDirection direction, 
@@ -108,7 +83,8 @@ class GarnetNetwork : public BaseGarnetNetwork
                           bool isReconfiguration);
 
   private:
-    void checkNetworkAllocation(NodeID id, bool ordered, int network_num);
+    void checkNetworkAllocation(NodeID id, bool ordered, int network_num,
+                                std::string vnet_type);
 
     GarnetNetwork(const GarnetNetwork& obj);
     GarnetNetwork& operator=(const GarnetNetwork& obj);

src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc

@@ -164,7 +164,7 @@ NetworkInterface::flitisizeMessage(MsgPtr msg_ptr, int vnet)
             net_msg_ptr->getInternalDestination().removeNetDest(personal_dest);
         }
         for (int i = 0; i < num_flits; i++) {
-            m_net_ptr->increment_injected_flits();
+            m_net_ptr->increment_injected_flits(vnet);
             flit *fl = new flit(i, vc, vnet, num_flits, new_msg_ptr);
             fl->set_delay(g_eventQueue_ptr->getTime() - msg_ptr->getTime());
             m_ni_buffers[vc]->insert(fl);
@@ -273,12 +273,13 @@ NetworkInterface::wakeup()
             inNetLink->release_vc_link(t_flit->get_vc(),
                 g_eventQueue_ptr->getTime() + 1);
         }
-        m_net_ptr->increment_received_flits();
+        int vnet = t_flit->get_vnet();
+        m_net_ptr->increment_received_flits(vnet);
         int network_delay = g_eventQueue_ptr->getTime() -
                             t_flit->get_enqueue_time();
         int queueing_delay = t_flit->get_delay();
-        m_net_ptr->increment_network_latency(network_delay);
+        m_net_ptr->increment_network_latency(network_delay, vnet);
-        m_net_ptr->increment_queueing_latency(queueing_delay);
+        m_net_ptr->increment_queueing_latency(queueing_delay, vnet);
         delete t_flit;
     }
 }