diff --git a/src/mem/bridge.cc b/src/mem/bridge.cc
index 085c97b53..b26da57e3 100644
--- a/src/mem/bridge.cc
+++ b/src/mem/bridge.cc
@@ -242,9 +242,8 @@ Bridge::BridgeSlavePort::schedTimingResp(PacketPtr pkt, Tick when)
     pkt->setDest(req_state->origSrc);
     delete req_state;
 
-    // the bridge assumes that at least one crossbar has set the
-    // destination field of the packet
-    assert(pkt->isDestValid());
+    // the bridge sets the destination irrespective of it is valid or
+    // not, as it is checked in the crossbar
     DPRINTF(Bridge, "response, new dest %d\n", pkt->getDest());
 
     // If we're about to put this packet at the head of the queue, we
diff --git a/src/mem/coherent_xbar.cc b/src/mem/coherent_xbar.cc
index ce5116de9..6ad7b1785 100644
--- a/src/mem/coherent_xbar.cc
+++ b/src/mem/coherent_xbar.cc
@@ -272,6 +272,8 @@ CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
 
     // determine the destination based on what is stored in the packet
     PortID slave_port_id = pkt->getDest();
+    assert(slave_port_id != InvalidPortID);
+    assert(slave_port_id < respLayers.size());
 
     // test if the crossbar should be considered occupied for the
     // current port
@@ -369,6 +371,7 @@ CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
 
     // get the destination from the packet
     PortID dest_port_id = pkt->getDest();
+    assert(dest_port_id != InvalidPortID);
 
     // determine if the response is from a snoop request we
     // created as the result of a normal request (in which case it
@@ -382,6 +385,7 @@ CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
     // is being passed on as a normal response since this is occupying
     // the response layer rather than the snoop response layer
     if (forwardAsSnoop) {
+        assert(dest_port_id < snoopLayers.size());
         if (!snoopLayers[dest_port_id]->tryTiming(src_port)) {
             DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
                     src_port->name(), pkt->cmdString(), pkt->getAddr());
@@ -390,6 +394,7 @@ CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
     } else {
         // get the master port that mirrors this slave port internally
         MasterPort* snoop_port = snoopRespPorts[slave_port_id];
+        assert(dest_port_id < respLayers.size());
         if (!respLayers[dest_port_id]->tryTiming(snoop_port)) {
             DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
                     snoop_port->name(), pkt->cmdString(), pkt->getAddr());
diff --git a/src/mem/noncoherent_xbar.cc b/src/mem/noncoherent_xbar.cc
index bc51be5a2..476b8314f 100644
--- a/src/mem/noncoherent_xbar.cc
+++ b/src/mem/noncoherent_xbar.cc
@@ -171,6 +171,8 @@ NoncoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
 
     // determine the destination based on what is stored in the packet
     PortID slave_port_id = pkt->getDest();
+    assert(slave_port_id != InvalidPortID);
+    assert(slave_port_id < respLayers.size());
 
     // test if the layer should be considered occupied for the current
     // port
diff --git a/src/mem/packet.hh b/src/mem/packet.hh
index dab1b1b95..9320d7886 100644
--- a/src/mem/packet.hh
+++ b/src/mem/packet.hh
@@ -544,15 +544,13 @@ class Packet : public Printable
     bool hadBadAddress() const { return cmd == MemCmd::BadAddressError; }
     void copyError(Packet *pkt) { assert(pkt->isError()); cmd = pkt->cmd; }
 
-    bool isSrcValid() const { return src != InvalidPortID; }
     /// Accessor function to get the source index of the packet.
-    PortID getSrc() const { assert(isSrcValid()); return src; }
+    PortID getSrc() const { return src; }
     /// Accessor function to set the source index of the packet.
     void setSrc(PortID _src) { src = _src; }
 
-    bool isDestValid() const { return dest != InvalidPortID; }
     /// Accessor function for the destination index of the packet.
-    PortID getDest() const { assert(isDestValid()); return dest; }
+    PortID getDest() const { return dest; }
     /// Accessor function to set the destination index of the packet.
     void setDest(PortID _dest) { dest = _dest; }
     /// Reset destination field, e.g. to turn a response into a request again.