From a4b546c3a139aeb33f087422637ac06fc4477d11 Mon Sep 17 00:00:00 2001
From: Matthew Poremba <matthew.poremba@amd.com>
Date: Thu, 19 Jan 2017 11:58:59 -0500
Subject: [PATCH] ruby: Add occupancy stats to MessageBuffers

This patch is an updated version of /r/3297.

"The most important statistic for measuring memory hierarchy performance is
throughput, which is affected by independent variables, buffer sizing and
communication latency. It is difficult/impossible to debug performance issues
through series buffers without knowing which are the bottlenecks. For finite
buffers, this patch adds statistics for the average number of messages in the
buffer, the occupancy of the buffer slots, and number of message stalls."
---
 src/mem/ruby/network/MessageBuffer.cc | 45 +++++++++++++++++++++++++--
 src/mem/ruby/network/MessageBuffer.hh |  7 ++++-
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/src/mem/ruby/network/MessageBuffer.cc b/src/mem/ruby/network/MessageBuffer.cc
index d7e1d0384..573d8833a 100644
--- a/src/mem/ruby/network/MessageBuffer.cc
+++ b/src/mem/ruby/network/MessageBuffer.cc
@@ -57,6 +57,9 @@ MessageBuffer::MessageBuffer(const Params *p)
     m_stall_msg_map.clear();
     m_input_link_id = 0;
     m_vnet_id = 0;
+
+    m_buf_msgs = 0;
+    m_stall_time = 0;
 }
 
 unsigned int
@@ -196,6 +199,8 @@ MessageBuffer::enqueue(MsgPtr message, Tick current_time, Tick delta)
     // Insert the message into the priority heap
     m_prio_heap.push_back(message);
     push_heap(m_prio_heap.begin(), m_prio_heap.end(), greater<MsgPtr>());
+    // Increment the number of messages statistic
+    m_buf_msgs++;
 
     DPRINTF(RubyQueue, "Enqueue arrival_time: %lld, Message: %s\n",
             arrival_time, *(message.get()));
@@ -207,7 +212,7 @@ MessageBuffer::enqueue(MsgPtr message, Tick current_time, Tick delta)
 }
 
 Tick
-MessageBuffer::dequeue(Tick current_time)
+MessageBuffer::dequeue(Tick current_time, bool decrement_messages)
 {
     DPRINTF(RubyQueue, "Popping\n");
     assert(isReady(current_time));
@@ -219,6 +224,8 @@ MessageBuffer::dequeue(Tick current_time)
     message->updateDelayedTicks(current_time);
     Tick delay = message->getDelayedTicks();
 
+    m_stall_time = curTick() - message->getTime();
+
     // record previous size and time so the current buffer size isn't
     // adjusted until schd cycle
     if (m_time_last_time_pop < current_time) {
@@ -228,6 +235,11 @@ MessageBuffer::dequeue(Tick current_time)
 
     pop_heap(m_prio_heap.begin(), m_prio_heap.end(), greater<MsgPtr>());
     m_prio_heap.pop_back();
+    if (decrement_messages) {
+        // If the message will be removed from the queue, decrement the
+        // number of message in the queue.
+        m_buf_msgs--;
+    }
 
     return delay;
 }
@@ -324,7 +336,9 @@ MessageBuffer::stallMessage(Addr addr, Tick current_time)
     assert(getOffset(addr) == 0);
     MsgPtr message = m_prio_heap.front();
 
-    dequeue(current_time);
+    // Since the message will just be moved to stall map, indicate that the
+    // buffer should not decrement the m_buf_msgs statistic
+    dequeue(current_time, false);
 
     //
     // Note: no event is scheduled to analyze the map at a later time.
@@ -333,6 +347,7 @@ MessageBuffer::stallMessage(Addr addr, Tick current_time)
     //
     (m_stall_msg_map[addr]).push_back(message);
     m_stall_map_size++;
+    m_stall_count++;
 }
 
 void
@@ -362,6 +377,32 @@ MessageBuffer::regStats()
         .name(name() + ".not_avail_count")
         .desc("Number of times this buffer did not have N slots available")
         .flags(Stats::nozero);
+
+    m_buf_msgs
+        .name(name() + ".avg_buf_msgs")
+        .desc("Average number of messages in buffer")
+        .flags(Stats::nozero);
+
+    m_stall_count
+        .name(name() + ".num_msg_stalls")
+        .desc("Number of times messages were stalled")
+        .flags(Stats::nozero);
+
+    m_occupancy
+        .name(name() + ".avg_buf_occ")
+        .desc("Average occupancy of buffer capacity")
+        .flags(Stats::nozero);
+
+    m_stall_time
+        .name(name() + ".avg_stall_time")
+        .desc("Average number of cycles messages are stalled in this MB")
+        .flags(Stats::nozero);
+
+    if (m_max_size > 0) {
+        m_occupancy = m_buf_msgs / m_max_size;
+    } else {
+        m_occupancy = 0;
+    }
 }
 
 uint32_t
diff --git a/src/mem/ruby/network/MessageBuffer.hh b/src/mem/ruby/network/MessageBuffer.hh
index b1d04ab9a..fc8feb03e 100644
--- a/src/mem/ruby/network/MessageBuffer.hh
+++ b/src/mem/ruby/network/MessageBuffer.hh
@@ -100,7 +100,7 @@ class MessageBuffer : public SimObject
 
     //! Updates the delay cycles of the message at the head of the queue,
     //! removes it from the queue and returns its total delay.
-    Tick dequeue(Tick current_time);
+    Tick dequeue(Tick current_time, bool decrement_messages = true);
 
     void recycle(Tick current_time, Tick recycle_latency);
     bool isEmpty() const { return m_prio_heap.size() == 0; }
@@ -189,6 +189,11 @@ class MessageBuffer : public SimObject
 
     int m_input_link_id;
     int m_vnet_id;
+
+    Stats::Average m_buf_msgs;
+    Stats::Average m_stall_time;
+    Stats::Scalar m_stall_count;
+    Stats::Formula m_occupancy;
 };
 
 Tick random_time();