cache: fail SC when invalidated while waiting for bus

Corrects an oversight in cset f97b62be544f.  The fix there only
failed queued SCUpgradeReq packets that encountered an
invalidation, which meant that the upgrade had to reach the L2
cache.  To handle pending requests in the L1 we must similarly
fail StoreCondReq packets too.

src/mem/cache/cache_impl.hh

@@ -900,9 +900,10 @@ Cache<TagStore>::handleResponse(PacketPtr pkt)
                 assert(!target->pkt->req->isUncacheable());
                 missLatency[target->pkt->cmdToIndex()][0/*pkt->req->threadId()*/] +=
                     completion_time - target->recvTime;
-            } else if (target->pkt->cmd == MemCmd::StoreCondReq &&
+            } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
-                       pkt->cmd == MemCmd::UpgradeFailResp) {
                 // failed StoreCond upgrade
+                assert(target->pkt->cmd == MemCmd::StoreCondReq ||
+                       target->pkt->cmd == MemCmd::StoreCondFailReq);
                 completion_time = tags->getHitLatency() + pkt->finishTime;
                 target->pkt->req->setExtraData(0);
             } else {
@@ -1443,10 +1444,11 @@ Cache<TagStore>::getTimingPacket()
     PacketPtr tgt_pkt = mshr->getTarget()->pkt;
     PacketPtr pkt = NULL;
 
-    if (tgt_pkt->cmd == MemCmd::SCUpgradeFailReq) {
+    if (tgt_pkt->cmd == MemCmd::SCUpgradeFailReq ||
-        // SCUpgradeReq saw invalidation while queued in MSHR, so now
+        tgt_pkt->cmd == MemCmd::StoreCondFailReq) {
-        // that we are getting around to processing it, just treat it
+        // SCUpgradeReq or StoreCondReq saw invalidation while queued
-        // as if we got a failure response
+        // in MSHR, so now that we are getting around to processing
+        // it, just treat it as if we got a failure response
         pkt = new Packet(tgt_pkt);
         pkt->cmd = MemCmd::UpgradeFailResp;
         pkt->senderState = mshr;

src/mem/cache/mshr.cc

@@ -72,7 +72,10 @@ MSHR::TargetList::add(PacketPtr pkt, Tick readyTime,
             needsExclusive = true;
         }
 
-        if (pkt->isUpgrade()) {
+        // StoreCondReq is effectively an upgrade if it's in an MSHR
+        // since it would have been failed already if we didn't have a
+        // read-only copy
+        if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) {
             hasUpgrade = true;
         }
     }
@@ -98,6 +101,9 @@ replaceUpgrade(PacketPtr pkt)
     } else if (pkt->cmd == MemCmd::SCUpgradeReq) {
         pkt->cmd = MemCmd::SCUpgradeFailReq;
         DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n");
+    } else if (pkt->cmd == MemCmd::StoreCondReq) {
+        pkt->cmd = MemCmd::StoreCondFailReq;
+        DPRINTF(Cache, "Replacing StoreCondReq with StoreCondFailReq\n");
     }
 }
 

src/mem/packet.cc

@@ -123,6 +123,10 @@ MemCmd::commandInfo[] =
     { SET6(IsWrite, NeedsExclusive, IsLlsc,
            IsRequest, NeedsResponse, HasData),
             StoreCondResp, "StoreCondReq" },
+    /* StoreCondFailReq: generates failing StoreCondResp ASAP */
+    { SET6(IsWrite, NeedsExclusive, IsLlsc,
+           IsRequest, NeedsResponse, HasData),
+            StoreCondResp, "StoreCondFailReq" },
     /* StoreCondResp */
     { SET4(IsWrite, NeedsExclusive, IsLlsc, IsResponse),
             InvalidCmd, "StoreCondResp" },

src/mem/packet.hh

@@ -90,6 +90,7 @@ class MemCmd
         ReadExResp,
         LoadLockedReq,
         StoreCondReq,
+        StoreCondFailReq,       // Failed StoreCondReq in MSHR (never sent)
         StoreCondResp,
         SwapReq,
         SwapResp,