add a backoff algorithm when nacks are received by devices

add seperate response buffers and request queue sizes in bus bridge
add delay to respond to a nack in the bus bridge

src/dev/i8254xGBe.cc:
src/dev/ide_ctrl.cc:
src/dev/ns_gige.cc:
src/dev/pcidev.hh:
src/dev/sinic.cc:
    add backoff delay parameters
src/dev/io_device.cc:
src/dev/io_device.hh:
    add a backoff algorithm when nacks are received.
src/mem/bridge.cc:
src/mem/bridge.hh:
    add seperate response buffers and request queue sizes
    add a new parameters to specify how long before a nack in ready to go after a packet that needs to be nacked is received
src/mem/cache/cache_impl.hh:
    assert on the
src/mem/tport.cc:
    add a friendly assert to make sure the packet was inserted into the list

--HG--
extra : convert_revision : 3595ad932015a4ce2bb72772da7850ad91bd09b1
This commit is contained in:
Ali Saidi 2007-05-09 18:20:24 -04:00
parent 37b45e3c8c
commit 3c608bf765
11 changed files with 250 additions and 61 deletions

View file

@ -1460,6 +1460,8 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(IGbE)
SimObjectParam<System *> system;
SimObjectParam<Platform *> platform;
Param<Tick> min_backoff_delay;
Param<Tick> max_backoff_delay;
SimObjectParam<PciConfigData *> configdata;
Param<uint32_t> pci_bus;
Param<uint32_t> pci_dev;
@ -1481,6 +1483,8 @@ BEGIN_INIT_SIM_OBJECT_PARAMS(IGbE)
INIT_PARAM(system, "System pointer"),
INIT_PARAM(platform, "Platform pointer"),
INIT_PARAM(min_backoff_delay, "Minimum delay after receving a nack packed"),
INIT_PARAM(max_backoff_delay, "Maximum delay after receving a nack packed"),
INIT_PARAM(configdata, "PCI Config data"),
INIT_PARAM(pci_bus, "PCI bus ID"),
INIT_PARAM(pci_dev, "PCI device number"),
@ -1505,6 +1509,8 @@ CREATE_SIM_OBJECT(IGbE)
params->name = getInstanceName();
params->platform = platform;
params->system = system;
params->min_backoff_delay = min_backoff_delay;
params->max_backoff_delay = max_backoff_delay;
params->configData = configdata;
params->busNum = pci_bus;
params->deviceNum = pci_dev;

View file

@ -751,6 +751,8 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(IdeController)
SimObjectParam<System *> system;
SimObjectParam<Platform *> platform;
Param<Tick> min_backoff_delay;
Param<Tick> max_backoff_delay;
SimObjectParam<PciConfigData *> configdata;
Param<uint32_t> pci_bus;
Param<uint32_t> pci_dev;
@ -765,6 +767,8 @@ BEGIN_INIT_SIM_OBJECT_PARAMS(IdeController)
INIT_PARAM(system, "System pointer"),
INIT_PARAM(platform, "Platform pointer"),
INIT_PARAM(min_backoff_delay, "Minimum delay after receving a nack packed"),
INIT_PARAM(max_backoff_delay, "Maximum delay after receving a nack packed"),
INIT_PARAM(configdata, "PCI Config data"),
INIT_PARAM(pci_bus, "PCI bus ID"),
INIT_PARAM(pci_dev, "PCI device number"),
@ -781,6 +785,8 @@ CREATE_SIM_OBJECT(IdeController)
params->name = getInstanceName();
params->platform = platform;
params->system = system;
params->min_backoff_delay = min_backoff_delay;
params->max_backoff_delay = max_backoff_delay;
params->configData = configdata;
params->busNum = pci_bus;
params->deviceNum = pci_dev;

View file

@ -93,7 +93,8 @@ BasicPioDevice::addressRanges(AddrRangeList &range_list)
DmaPort::DmaPort(DmaDevice *dev, System *s)
: Port(dev->name() + "-dmaport", dev), device(dev), sys(s),
pendingCount(0), actionInProgress(0), drainEvent(NULL)
pendingCount(0), actionInProgress(0), drainEvent(NULL),
backoffTime(0), inRetry(false), backoffEvent(this)
{ }
bool
@ -104,12 +105,27 @@ DmaPort::recvTiming(PacketPtr pkt)
if (pkt->result == Packet::Nacked) {
DPRINTF(DMA, "Received nacked Pkt %#x with State: %#x Addr: %#x\n",
pkt, pkt->senderState, pkt->getAddr());
if (backoffTime < device->minBackoffDelay)
backoffTime = device->minBackoffDelay;
else if (backoffTime < device->maxBackoffDelay)
backoffTime <<= 1;
if (backoffEvent.scheduled())
backoffEvent.reschedule(curTick + backoffTime);
else
backoffEvent.schedule(curTick + backoffTime);
DPRINTF(DMA, "Backoff time set to %d ticks\n", backoffTime);
pkt->reinitNacked();
sendDma(pkt, true);
queueDma(pkt, true);
} else if (pkt->senderState) {
DmaReqState *state;
DPRINTF(DMA, "Received response Pkt %#x with State: %#x Addr: %#x\n",
pkt, pkt->senderState, pkt->getAddr());
backoffTime >>= 2;
DPRINTF(DMA, "Received response Pkt %#x with State: %#x Addr: %#x size: %#x\n",
pkt, pkt->senderState, pkt->getAddr(), pkt->req->getSize());
state = dynamic_cast<DmaReqState*>(pkt->senderState);
pendingCount--;
@ -117,6 +133,7 @@ DmaPort::recvTiming(PacketPtr pkt)
assert(state);
state->numBytes += pkt->req->getSize();
assert(state->totBytes >= state->numBytes);
if (state->totBytes == state->numBytes) {
state->completionEvent->process();
delete state;
@ -136,7 +153,8 @@ DmaPort::recvTiming(PacketPtr pkt)
}
DmaDevice::DmaDevice(Params *p)
: PioDevice(p), dmaPort(NULL)
: PioDevice(p), dmaPort(NULL), minBackoffDelay(p->min_backoff_delay),
maxBackoffDelay(p->max_backoff_delay)
{ }
@ -165,19 +183,31 @@ DmaPort::drain(Event *de)
void
DmaPort::recvRetry()
{
assert(transmitList.size());
PacketPtr pkt = transmitList.front();
bool result = true;
while (result && transmitList.size()) {
do {
DPRINTF(DMA, "Retry on Packet %#x with senderState: %#x\n",
pkt, pkt->senderState);
result = sendTiming(pkt);
if (result) {
DPRINTF(DMA, "-- Done\n");
transmitList.pop_front();
inRetry = false;
} else {
inRetry = true;
DPRINTF(DMA, "-- Failed, queued\n");
}
} while (!backoffTime && result && transmitList.size());
if (transmitList.size() && backoffTime && !inRetry) {
DPRINTF(DMA, "Scheduling backoff for %d\n", curTick+backoffTime);
if (!backoffEvent.scheduled())
backoffEvent.schedule(backoffTime+curTick);
}
DPRINTF(DMA, "TransmitList: %d, backoffTime: %d inRetry: %d es: %d\n",
transmitList.size(), backoffTime, inRetry,
backoffEvent.scheduled());
}
@ -204,33 +234,61 @@ DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
assert(pendingCount >= 0);
pendingCount++;
sendDma(pkt);
queueDma(pkt);
}
}
void
DmaPort::queueDma(PacketPtr pkt, bool front)
{
if (front)
transmitList.push_front(pkt);
else
transmitList.push_back(pkt);
sendDma();
}
void
DmaPort::sendDma(PacketPtr pkt, bool front)
DmaPort::sendDma()
{
// some kind of selction between access methods
// more work is going to have to be done to make
// switching actually work
assert(transmitList.size());
PacketPtr pkt = transmitList.front();
System::MemoryMode state = sys->getMemoryMode();
if (state == System::Timing) {
if (backoffEvent.scheduled() || inRetry) {
DPRINTF(DMA, "Can't send immediately, waiting for retry or backoff timer\n");
return;
}
DPRINTF(DMA, "Attempting to send Packet %#x with addr: %#x\n",
pkt, pkt->getAddr());
if (transmitList.size() || !sendTiming(pkt)) {
if (front)
transmitList.push_front(pkt);
else
transmitList.push_back(pkt);
DPRINTF(DMA, "-- Failed: queued\n");
} else {
DPRINTF(DMA, "-- Done\n");
bool result;
do {
result = sendTiming(pkt);
if (result) {
transmitList.pop_front();
DPRINTF(DMA, "-- Done\n");
} else {
inRetry = true;
DPRINTF(DMA, "-- Failed: queued\n");
}
} while (result && !backoffTime && transmitList.size());
if (transmitList.size() && backoffTime && !inRetry &&
!backoffEvent.scheduled()) {
backoffEvent.schedule(backoffTime+curTick);
}
} else if (state == System::Atomic) {
transmitList.pop_front();
Tick lat;
lat = sendAtomic(pkt);
assert(pkt->senderState);

View file

@ -107,6 +107,14 @@ class DmaPort : public Port
* here.*/
Event *drainEvent;
/** time to wait between sending another packet, increases as NACKs are
* recived, decreases as responses are recived. */
Tick backoffTime;
/** If the port is currently waiting for a retry before it can send whatever
* it is that it's sending. */
bool inRetry;
virtual bool recvTiming(PacketPtr pkt);
virtual Tick recvAtomic(PacketPtr pkt)
{ panic("dma port shouldn't be used for pio access."); M5_DUMMY_RETURN }
@ -122,7 +130,11 @@ class DmaPort : public Port
AddrRangeList &snoop)
{ resp.clear(); snoop.clear(); }
void sendDma(PacketPtr pkt, bool front = false);
void queueDma(PacketPtr pkt, bool front = false);
void sendDma();
/** event to give us a kick every time we backoff time is reached. */
EventWrapper<DmaPort, &DmaPort::sendDma> backoffEvent;
public:
DmaPort(DmaDevice *dev, System *s);
@ -249,8 +261,17 @@ class BasicPioDevice : public PioDevice
class DmaDevice : public PioDevice
{
protected:
public:
struct Params : public PioDevice::Params
{
Tick min_backoff_delay;
Tick max_backoff_delay;
};
protected:
DmaPort *dmaPort;
Tick minBackoffDelay;
Tick maxBackoffDelay;
public:
DmaDevice(Params *p);

View file

@ -2812,6 +2812,8 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(NSGigE)
SimObjectParam<System *> system;
SimObjectParam<Platform *> platform;
Param<Tick> min_backoff_delay;
Param<Tick> max_backoff_delay;
SimObjectParam<PciConfigData *> configdata;
Param<uint32_t> pci_bus;
Param<uint32_t> pci_dev;
@ -2846,6 +2848,8 @@ BEGIN_INIT_SIM_OBJECT_PARAMS(NSGigE)
INIT_PARAM(system, "System pointer"),
INIT_PARAM(platform, "Platform pointer"),
INIT_PARAM(min_backoff_delay, "Minimum delay after receving a nack packed"),
INIT_PARAM(max_backoff_delay, "Maximum delay after receving a nack packed"),
INIT_PARAM(configdata, "PCI Config data"),
INIT_PARAM(pci_bus, "PCI bus ID"),
INIT_PARAM(pci_dev, "PCI device number"),
@ -2884,6 +2888,8 @@ CREATE_SIM_OBJECT(NSGigE)
params->name = getInstanceName();
params->platform = platform;
params->system = system;
params->min_backoff_delay = min_backoff_delay;
params->max_backoff_delay = max_backoff_delay;
params->configData = configdata;
params->busNum = pci_bus;
params->deviceNum = pci_dev;

View file

@ -105,7 +105,7 @@ class PciDev : public DmaDevice
};
public:
struct Params : public PioDevice::Params
struct Params : public DmaDevice::Params
{
/**
* A pointer to the object that contains the first 64 bytes of

View file

@ -1635,6 +1635,8 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS_WNS(Sinic, SinicDevice)
SimObjectParam<System *> system;
SimObjectParam<Platform *> platform;
Param<Tick> min_backoff_delay;
Param<Tick> max_backoff_delay;
SimObjectParam<PciConfigData *> configdata;
Param<uint32_t> pci_bus;
Param<uint32_t> pci_dev;
@ -1678,6 +1680,8 @@ BEGIN_INIT_SIM_OBJECT_PARAMS_WNS(Sinic, SinicDevice)
INIT_PARAM(system, "System pointer"),
INIT_PARAM(platform, "Platform pointer"),
INIT_PARAM(min_backoff_delay, "Minimum delay after receving a nack packed"),
INIT_PARAM(max_backoff_delay, "Maximum delay after receving a nack packed"),
INIT_PARAM(configdata, "PCI Config data"),
INIT_PARAM(pci_bus, "PCI bus ID"),
INIT_PARAM(pci_dev, "PCI device number"),
@ -1723,6 +1727,8 @@ CREATE_SIM_OBJECT_WNS(Sinic, SinicDevice)
params->name = getInstanceName();
params->platform = platform;
params->system = system;
params->min_backoff_delay = min_backoff_delay;
params->max_backoff_delay = max_backoff_delay;
params->configData = configdata;
params->busNum = pci_bus;
params->deviceNum = pci_dev;

View file

@ -43,22 +43,22 @@
Bridge::BridgePort::BridgePort(const std::string &_name,
Bridge *_bridge, BridgePort *_otherPort,
int _delay, int _queueLimit,
bool fix_partial_write)
int _delay, int _nack_delay, int _req_limit,
int _resp_limit, bool fix_partial_write)
: Port(_name), bridge(_bridge), otherPort(_otherPort),
delay(_delay), fixPartialWrite(fix_partial_write),
outstandingResponses(0), queuedRequests(0),
queueLimit(_queueLimit), sendEvent(this)
delay(_delay), nackDelay(_nack_delay), fixPartialWrite(fix_partial_write),
outstandingResponses(0), queuedRequests(0), inRetry(false),
reqQueueLimit(_req_limit), respQueueLimit(_resp_limit), sendEvent(this)
{
}
Bridge::Bridge(const std::string &n, int qsa, int qsb,
Tick _delay, int write_ack, bool fix_partial_write_a,
bool fix_partial_write_b)
: MemObject(n),
portA(n + "-portA", this, &portB, _delay, qsa, fix_partial_write_a),
portB(n + "-portB", this, &portA, _delay, qsa, fix_partial_write_b),
ackWrites(write_ack)
Bridge::Bridge(Params *p)
: MemObject(p->name),
portA(p->name + "-portA", this, &portB, p->delay, p->nack_delay,
p->req_size_a, p->resp_size_a, p->fix_partial_write_a),
portB(p->name + "-portB", this, &portA, p->delay, p->nack_delay,
p->req_size_b, p->resp_size_b, p->fix_partial_write_b),
ackWrites(p->write_ack), _params(p)
{
if (ackWrites)
panic("No support for acknowledging writes\n");
@ -94,11 +94,17 @@ Bridge::init()
}
bool
Bridge::BridgePort::queueFull()
Bridge::BridgePort::respQueueFull()
{
// use >= here because sendQueue could get larger because of
// nacks getting inserted
return queuedRequests + outstandingResponses >= queueLimit;
assert(outstandingResponses >= 0 && outstandingResponses <= respQueueLimit);
return outstandingResponses >= respQueueLimit;
}
bool
Bridge::BridgePort::reqQueueFull()
{
assert(queuedRequests >= 0 && queuedRequests <= reqQueueLimit);
return queuedRequests >= reqQueueLimit;
}
/** Function called by the port when the bus is receiving a Timing
@ -113,14 +119,14 @@ Bridge::BridgePort::recvTiming(PacketPtr pkt)
DPRINTF(BusBridge, "recvTiming: src %d dest %d addr 0x%x\n",
pkt->getSrc(), pkt->getDest(), pkt->getAddr());
if (pkt->isRequest() && otherPort->queueFull()) {
if (pkt->isRequest() && otherPort->reqQueueFull()) {
DPRINTF(BusBridge, "Remote queue full, nacking\n");
nackRequest(pkt);
return true;
}
if (pkt->needsResponse() && pkt->result != Packet::Nacked)
if (queueFull()) {
if (respQueueFull()) {
DPRINTF(BusBridge, "Local queue full, no space for response, nacking\n");
DPRINTF(BusBridge, "queue size: %d outreq: %d outstanding resp: %d\n",
sendQueue.size(), queuedRequests, outstandingResponses);
@ -144,12 +150,41 @@ Bridge::BridgePort::nackRequest(PacketPtr pkt)
pkt->setDest(pkt->getSrc());
//put it on the list to send
Tick readyTime = curTick + delay;
Tick readyTime = curTick + nackDelay;
PacketBuffer *buf = new PacketBuffer(pkt, readyTime, true);
// nothing on the list, add it and we're done
if (sendQueue.empty()) {
assert(!sendEvent.scheduled());
sendEvent.schedule(readyTime);
sendQueue.push_back(buf);
return;
}
sendQueue.push_back(buf);
assert(sendEvent.scheduled() || inRetry);
// does it go at the end?
if (readyTime >= sendQueue.back()->ready) {
sendQueue.push_back(buf);
return;
}
// ok, somewhere in the middle, fun
std::list<PacketBuffer*>::iterator i = sendQueue.begin();
std::list<PacketBuffer*>::iterator end = sendQueue.end();
std::list<PacketBuffer*>::iterator begin = sendQueue.begin();
bool done = false;
while (i != end && !done) {
if (readyTime < (*i)->ready) {
if (i == begin)
sendEvent.reschedule(readyTime);
sendQueue.insert(i,buf);
done = true;
}
i++;
}
assert(done);
}
@ -199,7 +234,6 @@ Bridge::BridgePort::trySend()
{
assert(!sendQueue.empty());
bool was_full = queueFull();
int pbs = peerBlockSize();
PacketBuffer *buf = sendQueue.front();
@ -245,15 +279,10 @@ Bridge::BridgePort::trySend()
DPRINTF(BusBridge, "Scheduling next send\n");
sendEvent.schedule(std::max(buf->ready, curTick + 1));
}
// Let things start sending again
if (was_full && !queueFull()) {
DPRINTF(BusBridge, "Queue was full, sending retry\n");
otherPort->sendRetry();
}
} else {
DPRINTF(BusBridge, " unsuccessful\n");
buf->undoPartialWriteFix();
inRetry = true;
}
DPRINTF(BusBridge, "trySend: queue size: %d outreq: %d outstanding resp: %d\n",
sendQueue.size(), queuedRequests, outstandingResponses);
@ -263,7 +292,12 @@ Bridge::BridgePort::trySend()
void
Bridge::BridgePort::recvRetry()
{
trySend();
inRetry = false;
Tick nextReady = sendQueue.front()->ready;
if (nextReady <= curTick)
trySend();
else
sendEvent.schedule(nextReady);
}
/** Function called by the port when the bus is receiving a Atomic
@ -309,9 +343,12 @@ Bridge::BridgePort::getDeviceAddressRanges(AddrRangeList &resp,
BEGIN_DECLARE_SIM_OBJECT_PARAMS(Bridge)
Param<int> queue_size_a;
Param<int> queue_size_b;
Param<int> req_size_a;
Param<int> req_size_b;
Param<int> resp_size_a;
Param<int> resp_size_b;
Param<Tick> delay;
Param<Tick> nack_delay;
Param<bool> write_ack;
Param<bool> fix_partial_write_a;
Param<bool> fix_partial_write_b;
@ -320,9 +357,12 @@ END_DECLARE_SIM_OBJECT_PARAMS(Bridge)
BEGIN_INIT_SIM_OBJECT_PARAMS(Bridge)
INIT_PARAM(queue_size_a, "The size of the queue for data coming into side a"),
INIT_PARAM(queue_size_b, "The size of the queue for data coming into side b"),
INIT_PARAM(req_size_a, "The size of the queue for requests coming into side a"),
INIT_PARAM(req_size_b, "The size of the queue for requests coming into side b"),
INIT_PARAM(resp_size_a, "The size of the queue for responses coming into side a"),
INIT_PARAM(resp_size_b, "The size of the queue for responses coming into side b"),
INIT_PARAM(delay, "The miminum delay to cross this bridge"),
INIT_PARAM(nack_delay, "The minimum delay to nack a packet"),
INIT_PARAM(write_ack, "Acknowledge any writes that are received."),
INIT_PARAM(fix_partial_write_a, "Fixup any partial block writes that are received"),
INIT_PARAM(fix_partial_write_b, "Fixup any partial block writes that are received")
@ -331,8 +371,18 @@ END_INIT_SIM_OBJECT_PARAMS(Bridge)
CREATE_SIM_OBJECT(Bridge)
{
return new Bridge(getInstanceName(), queue_size_a, queue_size_b, delay,
write_ack, fix_partial_write_a, fix_partial_write_b);
Bridge::Params *p = new Bridge::Params;
p->name = getInstanceName();
p->req_size_a = req_size_a;
p->req_size_b = req_size_b;
p->resp_size_a = resp_size_a;
p->resp_size_b = resp_size_b;
p->delay = delay;
p->nack_delay = nack_delay;
p->write_ack = write_ack;
p->fix_partial_write_a = fix_partial_write_a;
p->fix_partial_write_b = fix_partial_write_b;
return new Bridge(p);
}
REGISTER_SIM_OBJECT("Bridge", Bridge)

View file

@ -66,6 +66,9 @@ class Bridge : public MemObject
/** Minimum delay though this bridge. */
Tick delay;
/** Min delay to respond to a nack. */
Tick nackDelay;
bool fixPartialWrite;
class PacketBuffer : public Packet::SenderState {
@ -149,13 +152,20 @@ class Bridge : public MemObject
int outstandingResponses;
int queuedRequests;
/** If we're waiting for a retry to happen.*/
bool inRetry;
/** Max queue size for outbound packets */
int queueLimit;
int reqQueueLimit;
/** Max queue size for reserved responses. */
int respQueueLimit;
/**
* Is this side blocked from accepting outbound packets?
*/
bool queueFull();
bool respQueueFull();
bool reqQueueFull();
void queueForSendTiming(PacketPtr pkt);
@ -186,11 +196,10 @@ class Bridge : public MemObject
SendEvent sendEvent;
public:
/** Constructor for the BusPort.*/
BridgePort(const std::string &_name,
Bridge *_bridge, BridgePort *_otherPort,
int _delay, int _queueLimit, bool fix_partial_write);
BridgePort(const std::string &_name, Bridge *_bridge,
BridgePort *_otherPort, int _delay, int _nack_delay,
int _req_limit, int _resp_limit, bool fix_partial_write);
protected:
@ -226,14 +235,32 @@ class Bridge : public MemObject
bool ackWrites;
public:
struct Params
{
std::string name;
int req_size_a;
int req_size_b;
int resp_size_a;
int resp_size_b;
Tick delay;
Tick nack_delay;
bool write_ack;
bool fix_partial_write_a;
bool fix_partial_write_b;
};
protected:
Params *_params;
public:
const Params *params() const { return _params; }
/** A function used to return the port associated with this bus object. */
virtual Port *getPort(const std::string &if_name, int idx = -1);
virtual void init();
Bridge(const std::string &n, int qsa, int qsb, Tick _delay, int write_ack,
bool fix_partial_write_a, bool fix_partial_write_b);
Bridge(Params *p);
};
#endif //__MEM_BUS_HH__

View file

@ -1192,6 +1192,8 @@ template<class TagStore, class Coherence>
bool
Cache<TagStore,Coherence>::CpuSidePort::recvTiming(PacketPtr pkt)
{
assert(pkt->result != Packet::Nacked);
if (!pkt->req->isUncacheable()
&& pkt->isInvalidate()
&& !pkt->isRead() && !pkt->isWrite()) {
@ -1249,6 +1251,12 @@ template<class TagStore, class Coherence>
bool
Cache<TagStore,Coherence>::MemSidePort::recvTiming(PacketPtr pkt)
{
// this needs to be fixed so that the cache updates the mshr and sends the
// packet back out on the link, but it probably won't happen so until this
// gets fixed, just panic when it does
if (pkt->result == Packet::Nacked)
panic("Need to implement cache resending nacked packets!\n");
if (pkt->isRequest() && blocked)
{
DPRINTF(Cache,"Scheduling a retry while blocked\n");

View file

@ -128,6 +128,7 @@ SimpleTimingPort::sendTiming(PacketPtr pkt, Tick time)
}
i++;
}
assert(done);
}
void