sanchayanmaity/gem5
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge zizzer:/bk/m5

Browse source 
into  zeep.eecs.umich.edu:/z/saidi/work/m5

--HG--
extra : convert_revision : 3cc23080d19cc464a8ba7c1c93b6e5d45af7d463
This commit is contained in:
Ali Saidi 2005-11-02 14:56:18 -05:00
commit 3b66cb49ec
19 changed files with 936 additions and 845 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
SConscript
View file

@ -192,12 +192,15 @@ base_sources = Split('''
mem/cache/prefetch/prefetcher.cc
mem/cache/prefetch/tagged_prefetcher.cc
mem/cache/tags/base_tags.cc
mem/cache/tags/cache_tags.cc
mem/cache/tags/cache_tags.cc
mem/cache/tags/fa_lru.cc
mem/cache/tags/iic.cc
mem/cache/tags/lru.cc
mem/cache/tags/repl/gen.cc
mem/cache/tags/repl/repl.cc
mem/cache/tags/split.cc
mem/cache/tags/split_lru.cc
mem/cache/tags/split_lifo.cc
mem/functional/functional.cc
mem/timing/base_memory.cc
mem/timing/memory_builder.cc

17
arch/isa_parser.py
View file

@ -794,20 +794,19 @@ def defFormat(id, params, code, lineno):
##############
# Stack: a simple stack object. Used for both formats (formatStack)
# and default cases (defaultStack).
# and default cases (defaultStack). Simply wraps a list to give more
# stack-like syntax and enable initialization with an argument list
# (as opposed to an argument that's a list).
class Stack:
def __init__(self, initItem):
self.stack = [ initItem ]
class Stack(list):
def __init__(self, *items):
list.__init__(self, items)
def push(self, item):
self.stack.append(item);
def pop(self):
return self.stack.pop()
self.append(item);
def top(self):
return self.stack[-1]
return self[-1]
# The global format stack.
formatStack = Stack(NoFormat())

6
dev/ns_gige.cc
View file

@ -127,7 +127,7 @@ NSGigE::NSGigE(Params *p)
p->header_bus, 1,
p->dma_no_allocate);
} else if (p->payload_bus) {
pioInterface = newPioInterface(name() + ".pio2", p->hier,
pioInterface = newPioInterface(name() + ".pio", p->hier,
p->payload_bus, this,
&NSGigE::cacheAccess);
@ -2531,20 +2531,17 @@ NSGigE::recvPacket(PacketPtr packet)
if (!rxEnable) {
DPRINTF(Ethernet, "receive disabled...packet dropped\n");
interface->recvDone();
return true;
}
if (!rxFilterEnable) {
DPRINTF(Ethernet,
"receive packet filtering disabled . . . packet dropped\n");
interface->recvDone();
return true;
}
if (rxFilter(packet)) {
DPRINTF(Ethernet, "packet filtered...dropped\n");
interface->recvDone();
return true;
}
@ -2567,7 +2564,6 @@ NSGigE::recvPacket(PacketPtr packet)
}
rxFifo.push(packet);
interface->recvDone();
rxKick();
return true;

362
dev/sinic.cc
View file

@ -94,8 +94,8 @@ Device::Device(Params *p)
reset();
if (p->io_bus) {
pioInterface = newPioInterface(p->name, p->hier, p->io_bus, this,
&Device::cacheAccess);
pioInterface = newPioInterface(p->name + ".pio", p->hier, p->io_bus,
this, &Device::cacheAccess);
pioLatency = p->pio_latency * p->io_bus->clockRate;
@ -108,7 +108,8 @@ Device::Device(Params *p)
p->io_bus, 1,
p->dma_no_allocate);
} else if (p->payload_bus) {
pioInterface = newPioInterface(p->name, p->hier, p->payload_bus, this,
pioInterface = newPioInterface(p->name + ".pio", p->hier,
p->payload_bus, this,
&Device::cacheAccess);
pioLatency = p->pio_latency * p->payload_bus->clockRate;
@ -315,9 +316,26 @@ Device::writeConfig(int offset, int size, const uint8_t *data)
}
}
void
Device::prepareRead()
{
using namespace Regs;
// update rx registers
regs.RxDone = set_RxDone_Packets(regs.RxDone, rxFifo.packets());
regs.RxWait = regs.RxDone;
// update tx regsiters
regs.TxDone = set_TxDone_Packets(regs.TxDone, txFifo.packets());
regs.TxDone = set_TxDone_Full(regs.TxDone,
txFifo.avail() < regs.TxMaxCopy);
regs.TxDone = set_TxDone_Low(regs.TxDone,
txFifo.size() < regs.TxFifoMark);
regs.TxWait = regs.TxDone;
}
/**
* This reads the device registers, which are detailed in the NS83820
* spec sheet
* I/O read of device register
*/
Fault
Device::read(MemReqPtr &req, uint8_t *data)
@ -327,118 +345,115 @@ Device::read(MemReqPtr &req, uint8_t *data)
//The mask is to give you only the offset into the device register file
Addr daddr = req->paddr & 0xfff;
if (Regs::regSize(daddr) == 0)
panic("invalid address: da=%#x pa=%#x va=%#x size=%d",
if (!regValid(daddr))
panic("invalid register: da=%#x pa=%#x va=%#x size=%d",
daddr, req->paddr, req->vaddr, req->size);
if (req->size != Regs::regSize(daddr))
const Regs::Info &info = regInfo(daddr);
if (!info.read)
panic("reading write only register: %s: da=%#x pa=%#x va=%#x size=%d",
info.name, daddr, req->paddr, req->vaddr, req->size);
if (req->size != info.size)
panic("invalid size for reg %s: da=%#x pa=%#x va=%#x size=%d",
Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size);
info.name, daddr, req->paddr, req->vaddr, req->size);
DPRINTF(EthernetPIO, "read reg=%s da=%#x pa=%#x va=%#x size=%d\n",
Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size);
prepareRead();
uint32_t &reg32 = *(uint32_t *)data;
uint64_t &reg64 = *(uint64_t *)data;
switch (daddr) {
case Regs::Config:
reg32 = regs.Config;
break;
case Regs::RxMaxCopy:
reg32 = regs.RxMaxCopy;
break;
case Regs::TxMaxCopy:
reg32 = regs.TxMaxCopy;
break;
case Regs::RxThreshold:
reg32 = regs.RxThreshold;
break;
case Regs::TxThreshold:
reg32 = regs.TxThreshold;
break;
case Regs::IntrStatus:
reg32 = regs.IntrStatus;
devIntrClear();
break;
case Regs::IntrMask:
reg32 = regs.IntrMask;
break;
case Regs::RxData:
reg64 = regs.RxData;
break;
case Regs::RxDone:
case Regs::RxWait:
reg64 = Regs::set_RxDone_FifoLen(regs.RxDone,
min(rxFifo.packets(), 255));
break;
case Regs::TxData:
reg64 = regs.TxData;
break;
case Regs::TxDone:
case Regs::TxWait:
reg64 = Regs::set_TxDone_FifoLen(regs.TxDone,
min(txFifo.packets(), 255));
break;
case Regs::HwAddr:
reg64 = params()->eaddr;
break;
default:
panic("reading write only register %s: da=%#x pa=%#x va=%#x size=%d",
Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size);
uint64_t value = 0;
if (req->size == 4) {
uint32_t &reg = *(uint32_t *)data;
reg = regData32(daddr);
value = reg;
}
DPRINTF(EthernetPIO, "read reg=%s done val=%#x\n", Regs::regName(daddr),
Regs::regSize(daddr) == 4 ? reg32 : reg64);
if (req->size == 8) {
uint64_t &reg = *(uint64_t *)data;
reg = regData64(daddr);
value = reg;
}
DPRINTF(EthernetPIO, "read reg=%s da=%#x pa=%#x va=%#x size=%d val=%#x\n",
info.name, daddr, req->paddr, req->vaddr, req->size, value);
// reading the interrupt status register has the side effect of
// clearing it
if (daddr == Regs::IntrStatus)
devIntrClear();
return No_Fault;
}
/**
* IPR read of device register
*/
Fault
Device::iprRead(Addr daddr, uint64_t &result)
{
if (!regValid(daddr))
panic("invalid address: da=%#x", daddr);
const Regs::Info &info = regInfo(daddr);
if (!info.read)
panic("reading write only register %s: da=%#x", info.name, daddr);
DPRINTF(EthernetPIO, "read reg=%s da=%#x\n", info.name, daddr);
prepareRead();
if (info.size == 4)
result = regData32(daddr);
if (info.size == 8)
result = regData64(daddr);
DPRINTF(EthernetPIO, "IPR read reg=%s da=%#x val=%#x\n",
info.name, result);
return No_Fault;
}
/**
* I/O write of device register
*/
Fault
Device::write(MemReqPtr &req, const uint8_t *data)
{
assert(config.command & PCI_CMD_MSE);
//The mask is to give you only the offset into the device register file
Addr daddr = req->paddr & 0xfff;
if (Regs::regSize(daddr) == 0)
if (!regValid(daddr))
panic("invalid address: da=%#x pa=%#x va=%#x size=%d",
daddr, req->paddr, req->vaddr, req->size);
if (req->size != Regs::regSize(daddr))
panic("invalid size: reg=%s da=%#x pa=%#x va=%#x size=%d",
Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size);
const Regs::Info &info = regInfo(daddr);
if (!info.write)
panic("writing read only register %s: da=%#x", info.name, daddr);
if (req->size != info.size)
panic("invalid size for reg %s: da=%#x pa=%#x va=%#x size=%d",
info.name, daddr, req->paddr, req->vaddr, req->size);
uint32_t reg32 = *(uint32_t *)data;
uint64_t reg64 = *(uint64_t *)data;
DPRINTF(EthernetPIO, "write reg=%s val=%#x da=%#x pa=%#x va=%#x size=%d\n",
Regs::regName(daddr), Regs::regSize(daddr) == 4 ? reg32 : reg64,
daddr, req->paddr, req->vaddr, req->size);
info.name, info.size == 4 ? reg32 : reg64, daddr, req->paddr,
req->vaddr, req->size);
switch (daddr) {
case Regs::Config:
changeConfig(reg32);
break;
case Regs::RxThreshold:
regs.RxThreshold = reg32;
case Regs::Command:
command(reg32);
break;
case Regs::TxThreshold:
regs.TxThreshold = reg32;
case Regs::IntrStatus:
devIntrClear(regs.IntrStatus & reg32);
break;
case Regs::IntrMask:
@ -447,9 +462,10 @@ Device::write(MemReqPtr &req, const uint8_t *data)
case Regs::RxData:
if (rxState != rxIdle)
panic("receive machine busy with another request!");
panic("receive machine busy with another request! rxState=%s",
RxStateStrings[rxState]);
regs.RxDone = 0;
regs.RxDone = Regs::RxDone_Busy;
regs.RxData = reg64;
if (rxEnable) {
rxState = rxFifoBlock;
@ -459,19 +475,16 @@ Device::write(MemReqPtr &req, const uint8_t *data)
case Regs::TxData:
if (txState != txIdle)
panic("transmit machine busy with another request!");
panic("transmit machine busy with another request! txState=%s",
TxStateStrings[txState]);
regs.TxDone = 0;
regs.TxDone = Regs::TxDone_Busy;
regs.TxData = reg64;
if (txEnable) {
txState = txFifoBlock;
txKick();
}
break;
default:
panic("writing read only register %s: da=%#x pa=%#x va=%#x size=%d",
Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size);
}
return No_Fault;
@ -489,9 +502,25 @@ Device::devIntrPost(uint32_t interrupts)
"interrupt written to intStatus: intr=%#x status=%#x mask=%#x\n",
interrupts, regs.IntrStatus, regs.IntrMask);
if ((regs.IntrStatus & regs.IntrMask)) {
interrupts = regs.IntrStatus & regs.IntrMask;
// Intr_RxHigh is special, we only signal it if we've emptied the fifo
// and then filled it above the high watermark
if (rxEmpty)
rxEmpty = false;
else
interrupts &= ~Regs::Intr_RxHigh;
// Intr_TxLow is special, we only signal it if we've filled up the fifo
// and then dropped below the low watermark
if (txFull)
txFull = false;
else
interrupts &= ~Regs::Intr_TxLow;
if (interrupts) {
Tick when = curTick;
if ((regs.IntrStatus & regs.IntrMask & Regs::Intr_NoDelay) == 0)
if ((interrupts & Regs::Intr_NoDelay) == 0)
when += intrDelay;
cpuIntrPost(when);
}
@ -627,12 +656,6 @@ Device::changeConfig(uint32_t newconf)
regs.Config = newconf;
if ((changed & Regs::Config_Reset)) {
assert(regs.Config & Regs::Config_Reset);
reset();
regs.Config &= ~Regs::Config_Reset;
}
if ((changed & Regs::Config_IntEn)) {
cpuIntrEnable = regs.Config & Regs::Config_IntEn;
if (cpuIntrEnable) {
@ -656,20 +679,40 @@ Device::changeConfig(uint32_t newconf)
}
}
void
Device::command(uint32_t command)
{
if (command & Regs::Command_Reset)
reset();
}
void
Device::reset()
{
using namespace Regs;
memset(&regs, 0, sizeof(regs));
regs.Config = 0;
if (params()->dedicated)
regs.Config |= Config_Thread;
regs.IntrMask = Intr_RxHigh | Intr_RxDMA | Intr_TxLow;
regs.RxMaxCopy = params()->rx_max_copy;
regs.TxMaxCopy = params()->tx_max_copy;
regs.IntrMask = Intr_TxFifo | Intr_RxFifo | Intr_RxData;
regs.RxMaxIntr = params()->rx_max_intr;
regs.RxFifoSize = params()->rx_fifo_size;
regs.TxFifoSize = params()->tx_fifo_size;
regs.RxFifoMark = params()->rx_fifo_threshold;
regs.TxFifoMark = params()->tx_fifo_threshold;
regs.HwAddr = params()->eaddr;
rxState = rxIdle;
txState = txIdle;
rxFifo.clear();
txFifo.clear();
rxEmpty = false;
txFull = false;
}
void
@ -680,13 +723,18 @@ Device::rxDmaCopy()
physmem->dma_write(rxDmaAddr, (uint8_t *)rxDmaData, rxDmaLen);
DPRINTF(EthernetDMA, "rx dma write paddr=%#x len=%d\n",
rxDmaAddr, rxDmaLen);
DDUMP(EthernetDMA, rxDmaData, rxDmaLen);
DDUMP(EthernetData, rxDmaData, rxDmaLen);
}
void
Device::rxDmaDone()
{
rxDmaCopy();
// If the transmit state machine has a pending DMA, let it go first
if (txState == txBeginCopy)
txKick();
rxKick();
}
@ -706,6 +754,8 @@ Device::rxKick()
switch (rxState) {
case rxIdle:
if (rxPioRequest) {
DPRINTF(EthernetPIO, "rxIdle: PIO waiting responding at %d\n",
curTick + pioLatency);
pioInterface->respond(rxPioRequest, curTick);
rxPioRequest = 0;
}
@ -761,20 +811,20 @@ Device::rxKick()
break;
case rxBeginCopy:
if (dmaInterface && dmaInterface->busy())
goto exit;
rxDmaAddr = plat->pciToDma(Regs::get_RxData_Addr(regs.RxData));
rxDmaLen = min<int>(Regs::get_RxData_Len(regs.RxData), rxPktBytes);
rxDmaData = rxPacketBufPtr;
rxState = rxCopy;
if (dmaInterface) {
if (!dmaInterface->busy()) {
dmaInterface->doDMA(WriteInvalidate, rxDmaAddr, rxDmaLen,
curTick, &rxDmaEvent, true);
rxState = rxCopy;
}
dmaInterface->doDMA(WriteInvalidate, rxDmaAddr, rxDmaLen,
curTick, &rxDmaEvent, true);
goto exit;
}
rxState = rxCopy;
if (dmaWriteDelay != 0 || dmaWriteFactor != 0) {
Tick factor = ((rxDmaLen + ULL(63)) >> ULL(6)) * dmaWriteFactor;
Tick start = curTick + dmaWriteDelay + factor;
@ -802,7 +852,7 @@ Device::rxKick()
}
regs.RxDone |= Regs::RxDone_Complete;
devIntrPost(Regs::Intr_RxData);
devIntrPost(Regs::Intr_RxDMA);
rxState = rxIdle;
break;
@ -831,13 +881,18 @@ Device::txDmaCopy()
physmem->dma_read((uint8_t *)txDmaData, txDmaAddr, txDmaLen);
DPRINTF(EthernetDMA, "tx dma read paddr=%#x len=%d\n",
txDmaAddr, txDmaLen);
DDUMP(EthernetDMA, txDmaData, txDmaLen);
DDUMP(EthernetData, txDmaData, txDmaLen);
}
void
Device::txDmaDone()
{
txDmaCopy();
// If the receive state machine has a pending DMA, let it go first
if (rxState == rxBeginCopy)
rxKick();
txKick();
}
@ -849,6 +904,7 @@ Device::transmit()
return;
}
uint32_t interrupts;
PacketPtr packet = txFifo.front();
if (!interface->sendPacket(packet)) {
DPRINTF(Ethernet, "Packet Transmit: failed txFifo available %d\n",
@ -857,7 +913,6 @@ Device::transmit()
}
txFifo.pop();
#if TRACING_ON
if (DTRACE(Ethernet)) {
IpPtr ip(packet);
@ -872,17 +927,17 @@ Device::transmit()
}
#endif
DDUMP(Ethernet, packet->data, packet->length);
DDUMP(EthernetData, packet->data, packet->length);
txBytes += packet->length;
txPackets++;
DPRINTF(Ethernet, "Packet Transmit: successful txFifo Available %d\n",
txFifo.avail());
if (txFifo.size() <= params()->tx_fifo_threshold)
devIntrPost(Regs::Intr_TxFifo);
devIntrPost(Regs::Intr_TxDone);
interrupts = Regs::Intr_TxPacket;
if (txFifo.size() < regs.TxFifoMark)
interrupts |= Regs::Intr_TxLow;
devIntrPost(interrupts);
reschedule:
if (!txFifo.empty() && !txEvent.scheduled()) {
@ -907,6 +962,8 @@ Device::txKick()
switch (txState) {
case txIdle:
if (txPioRequest) {
DPRINTF(EthernetPIO, "txIdle: PIO waiting responding at %d\n",
curTick + pioLatency);
pioInterface->respond(txPioRequest, curTick + pioLatency);
txPioRequest = 0;
}
@ -929,21 +986,20 @@ Device::txKick()
break;
case txBeginCopy:
if (dmaInterface && dmaInterface->busy())
goto exit;
txDmaAddr = plat->pciToDma(Regs::get_TxData_Addr(regs.TxData));
txDmaLen = Regs::get_TxData_Len(regs.TxData);
txDmaData = txPacketBufPtr;
txState = txCopy;
if (dmaInterface) {
if (!dmaInterface->busy()) {
dmaInterface->doDMA(Read, txDmaAddr, txDmaLen,
curTick, &txDmaEvent, true);
txState = txCopy;
}
dmaInterface->doDMA(Read, txDmaAddr, txDmaLen,
curTick, &txDmaEvent, true);
goto exit;
}
txState = txCopy;
if (dmaReadDelay != 0 || dmaReadFactor != 0) {
Tick factor = ((txDmaLen + ULL(63)) >> ULL(6)) * dmaReadFactor;
Tick start = curTick + dmaReadDelay + factor;
@ -987,12 +1043,16 @@ Device::txKick()
}
}
txFifo.push(txPacket);
if (txFifo.avail() < regs.TxMaxCopy) {
devIntrPost(Regs::Intr_TxFull);
txFull = true;
}
txPacket = 0;
transmit();
}
regs.TxDone = txDmaLen | Regs::TxDone_Complete;
devIntrPost(Regs::Intr_TxData);
devIntrPost(Regs::Intr_TxDMA);
txState = txIdle;
break;
@ -1085,18 +1145,16 @@ Device::recvPacket(PacketPtr packet)
if (!rxEnable) {
DPRINTF(Ethernet, "receive disabled...packet dropped\n");
interface->recvDone();
return true;
}
if (rxFilter(packet)) {
DPRINTF(Ethernet, "packet filtered...dropped\n");
interface->recvDone();
return true;
}
if (rxFifo.size() >= params()->rx_fifo_threshold)
devIntrPost(Regs::Intr_RxFifo);
if (rxFifo.size() >= regs.RxFifoMark)
devIntrPost(Regs::Intr_RxHigh);
if (!rxFifo.push(packet)) {
DPRINTF(Ethernet,
@ -1104,8 +1162,7 @@ Device::recvPacket(PacketPtr packet)
return false;
}
interface->recvDone();
devIntrPost(Regs::Intr_RxDone);
devIntrPost(Regs::Intr_RxPacket);
rxKick();
return true;
}
@ -1163,22 +1220,23 @@ Device::serialize(ostream &os)
// Serialize the PciDev base class
Base::serialize(os);
if (rxDmaEvent.scheduled())
rxDmaCopy();
if (rxState == rxCopy)
panic("can't serialize with an in flight dma request rxState=%s",
RxStateStrings[rxState]);
if (txDmaEvent.scheduled())
txDmaCopy();
if (txState == txCopy)
panic("can't serialize with an in flight dma request txState=%s",
TxStateStrings[txState]);
/*
* Serialize the device registers
*/
SERIALIZE_SCALAR(regs.Config);
SERIALIZE_SCALAR(regs.RxMaxCopy);
SERIALIZE_SCALAR(regs.TxMaxCopy);
SERIALIZE_SCALAR(regs.RxThreshold);
SERIALIZE_SCALAR(regs.TxThreshold);
SERIALIZE_SCALAR(regs.IntrStatus);
SERIALIZE_SCALAR(regs.IntrMask);
SERIALIZE_SCALAR(regs.RxMaxCopy);
SERIALIZE_SCALAR(regs.TxMaxCopy);
SERIALIZE_SCALAR(regs.RxMaxIntr);
SERIALIZE_SCALAR(regs.RxData);
SERIALIZE_SCALAR(regs.RxDone);
SERIALIZE_SCALAR(regs.TxData);
@ -1189,6 +1247,7 @@ Device::serialize(ostream &os)
*/
int rxState = this->rxState;
SERIALIZE_SCALAR(rxState);
SERIALIZE_SCALAR(rxEmpty);
rxFifo.serialize("rxFifo", os);
bool rxPacketExists = rxPacket;
SERIALIZE_SCALAR(rxPacketExists);
@ -1205,6 +1264,7 @@ Device::serialize(ostream &os)
*/
int txState = this->txState;
SERIALIZE_SCALAR(txState);
SERIALIZE_SCALAR(txFull);
txFifo.serialize("txFifo", os);
bool txPacketExists = txPacket;
SERIALIZE_SCALAR(txPacketExists);
@ -1233,12 +1293,11 @@ Device::unserialize(Checkpoint *cp, const std::string &section)
* Unserialize the device registers
*/
UNSERIALIZE_SCALAR(regs.Config);
UNSERIALIZE_SCALAR(regs.RxMaxCopy);
UNSERIALIZE_SCALAR(regs.TxMaxCopy);
UNSERIALIZE_SCALAR(regs.RxThreshold);
UNSERIALIZE_SCALAR(regs.TxThreshold);
UNSERIALIZE_SCALAR(regs.IntrStatus);
UNSERIALIZE_SCALAR(regs.IntrMask);
UNSERIALIZE_SCALAR(regs.RxMaxCopy);
UNSERIALIZE_SCALAR(regs.TxMaxCopy);
UNSERIALIZE_SCALAR(regs.RxMaxIntr);
UNSERIALIZE_SCALAR(regs.RxData);
UNSERIALIZE_SCALAR(regs.RxDone);
UNSERIALIZE_SCALAR(regs.TxData);
@ -1249,6 +1308,7 @@ Device::unserialize(Checkpoint *cp, const std::string &section)
*/
int rxState;
UNSERIALIZE_SCALAR(rxState);
UNSERIALIZE_SCALAR(rxEmpty);
this->rxState = (RxState) rxState;
rxFifo.unserialize("rxFifo", cp, section);
bool rxPacketExists;
@ -1269,6 +1329,7 @@ Device::unserialize(Checkpoint *cp, const std::string &section)
*/
int txState;
UNSERIALIZE_SCALAR(txState);
UNSERIALIZE_SCALAR(txFull);
this->txState = (TxState) txState;
txFifo.unserialize("txFifo", cp, section);
bool txPacketExists;
@ -1310,15 +1371,19 @@ Device::cacheAccess(MemReqPtr &req)
Tick when = curTick + pioLatency;
switch (daddr) {
case Regs::RxDone:
case Regs::RxWait:
if (rxState != rxIdle) {
DPRINTF(EthernetPIO, "rxState=%s (not idle)... waiting\n",
TxStateStrings[txState]);
rxPioRequest = req;
when = 0;
}
break;
case Regs::TxDone:
case Regs::TxWait:
if (txState != txIdle) {
DPRINTF(EthernetPIO, "txState=%s (not idle)... waiting\n",
TxStateStrings[txState]);
txPioRequest = req;
when = 0;
}
@ -1387,6 +1452,7 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(Device)
Param<Tick> tx_delay;
Param<uint32_t> rx_max_copy;
Param<uint32_t> tx_max_copy;
Param<uint32_t> rx_max_intr;
Param<uint32_t> rx_fifo_size;
Param<uint32_t> tx_fifo_size;
Param<uint32_t> rx_fifo_threshold;
@ -1394,6 +1460,7 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(Device)
Param<bool> rx_filter;
Param<string> hardware_address;
Param<bool> dedicated;
END_DECLARE_SIM_OBJECT_PARAMS(Device)
@ -1426,13 +1493,15 @@ BEGIN_INIT_SIM_OBJECT_PARAMS(Device)
INIT_PARAM(tx_delay, "Transmit Delay"),
INIT_PARAM(rx_max_copy, "rx max copy"),
INIT_PARAM(tx_max_copy, "rx max copy"),
INIT_PARAM(rx_max_intr, "rx max intr"),
INIT_PARAM(rx_fifo_size, "max size in bytes of rxFifo"),
INIT_PARAM(tx_fifo_size, "max size in bytes of txFifo"),
INIT_PARAM(rx_fifo_threshold, "max size in bytes of rxFifo"),
INIT_PARAM(tx_fifo_threshold, "max size in bytes of txFifo"),
INIT_PARAM(rx_filter, "Enable Receive Filter"),
INIT_PARAM(hardware_address, "Ethernet Hardware Address")
INIT_PARAM(hardware_address, "Ethernet Hardware Address"),
INIT_PARAM(dedicated, "dedicate a kernel thread to the driver")
END_INIT_SIM_OBJECT_PARAMS(Device)
@ -1444,6 +1513,7 @@ CREATE_SIM_OBJECT(Device)
params->name = getInstanceName();
params->clock = clock;
params->mmu = mmu;
params->physmem = physmem;
params->configSpace = configspace;
@ -1468,6 +1538,7 @@ CREATE_SIM_OBJECT(Device)
params->rx_delay = rx_delay;
params->rx_max_copy = rx_max_copy;
params->tx_max_copy = tx_max_copy;
params->rx_max_intr = rx_max_intr;
params->rx_fifo_size = rx_fifo_size;
params->tx_fifo_size = tx_fifo_size;
params->rx_fifo_threshold = rx_fifo_threshold;
@ -1475,6 +1546,7 @@ CREATE_SIM_OBJECT(Device)
params->rx_filter = rx_filter;
params->eaddr = hardware_address;
params->dedicated = dedicated;
return new Device(params);
}

43
dev/sinic.hh
View file

@ -115,19 +115,31 @@ class Device : public Base
/** device register file */
struct {
uint32_t Config;
uint32_t RxMaxCopy;
uint32_t TxMaxCopy;
uint32_t RxThreshold;
uint32_t TxThreshold;
uint32_t IntrStatus;
uint32_t IntrMask;
uint64_t RxData;
uint64_t RxDone;
uint64_t TxData;
uint64_t TxDone;
uint32_t Config; // 0x00
uint32_t Command; // 0x04
uint32_t IntrStatus; // 0x08
uint32_t IntrMask; // 0x0c
uint32_t RxMaxCopy; // 0x10
uint32_t TxMaxCopy; // 0x14
uint32_t RxMaxIntr; // 0x18
uint32_t Reserved0; // 0x1c
uint32_t RxFifoSize; // 0x20
uint32_t TxFifoSize; // 0x24
uint32_t RxFifoMark; // 0x28
uint32_t TxFifoMark; // 0x2c
uint64_t RxData; // 0x30
uint64_t RxDone; // 0x38
uint64_t RxWait; // 0x40
uint64_t TxData; // 0x48
uint64_t TxDone; // 0x50
uint64_t TxWait; // 0x58
uint64_t HwAddr; // 0x60
} regs;
uint8_t &regData8(Addr daddr) { return *((uint8_t *)&regs + daddr); }
uint32_t &regData32(Addr daddr) { return *(uint32_t *)&regData8(daddr); }
uint64_t &regData64(Addr daddr) { return *(uint64_t *)&regData8(daddr); }
private:
Addr addr;
static const Addr size = Regs::Size;
@ -135,6 +147,7 @@ class Device : public Base
protected:
RxState rxState;
PacketFifo rxFifo;
bool rxEmpty;
PacketPtr rxPacket;
uint8_t *rxPacketBufPtr;
int rxPktBytes;
@ -145,6 +158,7 @@ class Device : public Base
TxState txState;
PacketFifo txFifo;
bool txFull;
PacketPtr txPacket;
uint8_t *txPacketBufPtr;
int txPktBytes;
@ -191,6 +205,7 @@ class Device : public Base
* device configuration
*/
void changeConfig(uint32_t newconfig);
void command(uint32_t command);
/**
* device ethernet interface
@ -212,7 +227,7 @@ class Device : public Base
void txDmaCopy();
void txDmaDone();
friend class EventWrapper<Device, &Device::txDmaDone>;
EventWrapper<Device, &Device::rxDmaDone> txDmaEvent;
EventWrapper<Device, &Device::txDmaDone> txDmaEvent;
Tick dmaReadDelay;
Tick dmaReadFactor;
@ -244,6 +259,8 @@ class Device : public Base
* Memory Interface
*/
public:
void prepareRead();
Fault iprRead(Addr daddr, uint64_t &result);
virtual Fault read(MemReqPtr &req, uint8_t *data);
virtual Fault write(MemReqPtr &req, const uint8_t *data);
Tick cacheAccess(MemReqPtr &req);
@ -308,6 +325,7 @@ class Device : public Base
Net::EthAddr eaddr;
uint32_t rx_max_copy;
uint32_t tx_max_copy;
uint32_t rx_max_intr;
uint32_t rx_fifo_size;
uint32_t tx_fifo_size;
uint32_t rx_fifo_threshold;
@ -317,6 +335,7 @@ class Device : public Base
Tick dma_write_delay;
Tick dma_write_factor;
bool dma_no_allocate;
bool dedicated;
};
protected:

195
dev/sinicreg.hh
View file

@ -57,23 +57,28 @@ namespace Regs {
// Registers
__SINIC_REG32(Config, 0x00); // 32: configuration register
__SINIC_REG32(RxMaxCopy, 0x04); // 32: max rx copy
__SINIC_REG32(TxMaxCopy, 0x08); // 32: max tx copy
__SINIC_REG32(RxThreshold, 0x0c); // 32: receive fifo threshold
__SINIC_REG32(TxThreshold, 0x10); // 32: transmit fifo threshold
__SINIC_REG32(IntrStatus, 0x14); // 32: interrupt status
__SINIC_REG32(IntrMask, 0x18); // 32: interrupt mask
__SINIC_REG32(RxData, 0x20); // 64: receive data
__SINIC_REG32(RxDone, 0x28); // 64: receive done
__SINIC_REG32(RxWait, 0x30); // 64: receive done (busy wait)
__SINIC_REG32(TxData, 0x38); // 64: transmit data
__SINIC_REG32(TxDone, 0x40); // 64: transmit done
__SINIC_REG32(TxWait, 0x48); // 64: transmit done (busy wait)
__SINIC_REG32(HwAddr, 0x50); // 64: mac address
__SINIC_REG32(Size, 0x58);
__SINIC_REG32(Command, 0x04); // 32: command register
__SINIC_REG32(IntrStatus, 0x08); // 32: interrupt status
__SINIC_REG32(IntrMask, 0x0c); // 32: interrupt mask
__SINIC_REG32(RxMaxCopy, 0x10); // 32: max bytes per rx copy
__SINIC_REG32(TxMaxCopy, 0x14); // 32: max bytes per tx copy
__SINIC_REG32(RxMaxIntr, 0x18); // 32: max receives per interrupt
__SINIC_REG32(Reserved0, 0x1c); // 32: reserved
__SINIC_REG32(RxFifoSize, 0x20); // 32: rx fifo capacity in bytes
__SINIC_REG32(TxFifoSize, 0x24); // 32: tx fifo capacity in bytes
__SINIC_REG32(RxFifoMark, 0x28); // 32: rx fifo high watermark
__SINIC_REG32(TxFifoMark, 0x2c); // 32: tx fifo low watermark
__SINIC_REG32(RxData, 0x30); // 64: receive data
__SINIC_REG32(RxDone, 0x38); // 64: receive done
__SINIC_REG32(RxWait, 0x40); // 64: receive done (busy wait)
__SINIC_REG32(TxData, 0x48); // 64: transmit data
__SINIC_REG32(TxDone, 0x50); // 64: transmit done
__SINIC_REG32(TxWait, 0x58); // 64: transmit done (busy wait)
__SINIC_REG32(HwAddr, 0x60); // 64: mac address
__SINIC_REG32(Size, 0x68); // register addres space size
// Config register bits
__SINIC_VAL32(Config_Reset, 31, 1); // reset chip
__SINIC_VAL32(Config_Thread, 8, 1); // enable receive filter
__SINIC_VAL32(Config_Filter, 7, 1); // enable receive filter
__SINIC_VAL32(Config_Vlan, 6, 1); // enable vlan tagging
__SINIC_VAL32(Config_Virtual, 5, 1); // enable virtual addressing
@ -83,105 +88,103 @@ __SINIC_VAL32(Config_IntEn, 2, 1); // enable interrupts
__SINIC_VAL32(Config_TxEn, 1, 1); // enable transmit
__SINIC_VAL32(Config_RxEn, 0, 1); // enable receive
// Command register bits
__SINIC_VAL32(Command_Reset, 0, 1); // reset chip
// Interrupt register bits
__SINIC_VAL32(Intr_TxFifo, 5, 1); // Fifo oflow/uflow/threshold
__SINIC_VAL32(Intr_TxData, 4, 1); // DMA Completed w/ interrupt
__SINIC_VAL32(Intr_TxDone, 3, 1); // Packet transmitted
__SINIC_VAL32(Intr_RxFifo, 2, 1); // Fifo oflow/uflow/threshold
__SINIC_VAL32(Intr_RxData, 1, 1); // DMA Completed w/ interrupt
__SINIC_VAL32(Intr_RxDone, 0, 1); // Packet received
__SINIC_REG32(Intr_All, 0x3f);
__SINIC_REG32(Intr_NoDelay, 0x24);
__SINIC_REG32(Intr_Res, ~0x3f);
__SINIC_VAL32(Intr_TxLow, 7, 1); // tx fifo dropped below watermark
__SINIC_VAL32(Intr_TxFull, 6, 1); // tx fifo full
__SINIC_VAL32(Intr_TxDMA, 5, 1); // tx dma completed w/ interrupt
__SINIC_VAL32(Intr_TxPacket, 4, 1); // packet transmitted
__SINIC_VAL32(Intr_RxHigh, 3, 1); // rx fifo above high watermark
__SINIC_VAL32(Intr_RxEmpty, 2, 1); // rx fifo empty
__SINIC_VAL32(Intr_RxDMA, 1, 1); // rx dma completed w/ interrupt
__SINIC_VAL32(Intr_RxPacket, 0, 1); // packet received
__SINIC_REG32(Intr_All, 0xff); // all valid interrupts
__SINIC_REG32(Intr_NoDelay, 0xcc); // interrupts that shouldn't be coalesced
__SINIC_REG32(Intr_Res, ~0xff); // reserved interrupt bits
// RX Data Description
__SINIC_VAL64(RxData_Len, 40, 20); // 0 - 1M
__SINIC_VAL64(RxData_Addr, 0, 40); // Address 1TB
// TX Data Description
__SINIC_VAL64(TxData_More, 63, 1);
__SINIC_VAL64(TxData_Checksum, 62, 1);
__SINIC_VAL64(TxData_More, 63, 1); // Packet not complete (will dma more)
__SINIC_VAL64(TxData_Checksum, 62, 1); // do checksum
__SINIC_VAL64(TxData_Len, 40, 20); // 0 - 1M
__SINIC_VAL64(TxData_Addr, 0, 40); // Address 1TB
// RX Done/Busy Information
__SINIC_VAL64(RxDone_Complete, 63, 1);
__SINIC_VAL64(RxDone_IpPacket, 45, 1);
__SINIC_VAL64(RxDone_TcpPacket, 44, 1);
__SINIC_VAL64(RxDone_UdpPacket, 43, 1);
__SINIC_VAL64(RxDone_IpError, 42, 1);
__SINIC_VAL64(RxDone_TcpError, 41, 1);
__SINIC_VAL64(RxDone_UdpError, 40, 1);
__SINIC_VAL64(RxDone_More, 32, 1);
__SINIC_VAL64(RxDone_FifoLen, 20, 8); // up to 255 packets
__SINIC_VAL64(RxDone_Packets, 32, 16); // number of packets in rx fifo
__SINIC_VAL64(RxDone_Busy, 31, 1); // receive dma busy copying
__SINIC_VAL64(RxDone_Complete, 30, 1); // valid data (packet complete)
__SINIC_VAL64(RxDone_More, 29, 1); // Packet has more data (dma again)
__SINIC_VAL64(RxDone_TcpError, 25, 1); // TCP packet error (bad checksum)
__SINIC_VAL64(RxDone_UdpError, 24, 1); // UDP packet error (bad checksum)
__SINIC_VAL64(RxDone_IpError, 23, 1); // IP packet error (bad checksum)
__SINIC_VAL64(RxDone_TcpPacket, 22, 1); // this is a TCP packet
__SINIC_VAL64(RxDone_UdpPacket, 21, 1); // this is a UDP packet
__SINIC_VAL64(RxDone_IpPacket, 20, 1); // this is an IP packet
__SINIC_VAL64(RxDone_CopyLen, 0, 20); // up to 256k
// TX Done/Busy Information
__SINIC_VAL64(TxDone_Complete, 63, 1);
__SINIC_VAL64(TxDone_FifoLen, 20, 8); // up to 255 packets
__SINIC_VAL64(TxDone_CopyLen, 0, 20); // up to 256k
__SINIC_VAL64(TxDone_Packets, 32, 16); // number of packets in tx fifo
__SINIC_VAL64(TxDone_Busy, 31, 1); // transmit dma busy copying
__SINIC_VAL64(TxDone_Complete, 30, 1); // valid data (packet complete)
__SINIC_VAL64(TxDone_Full, 29, 1); // tx fifo is full
__SINIC_VAL64(TxDone_Low, 28, 1); // tx fifo is below the watermark
__SINIC_VAL64(TxDone_CopyLen, 0, 20); // up to 256k
inline int
regSize(int offset)
struct Info
{
static const char sizes[] = {
4,
4,
4,
4,
4,
4,
4,
0,
8, 0,
8, 0,
8, 0,
8, 0,
8, 0,
8, 0,
8, 0
};
if (offset & 0x3)
return 0;
if (offset >= Size)
return 0;
return sizes[offset / 4];
}
inline const char *
regName(int offset)
{
static const char *names[] = {
"Config",
"RxMaxCopy",
"TxMaxCopy",
"RxThreshold",
"TxThreshold",
"IntrStatus",
"IntrMask",
"invalid",
"RxData", "invalid",
"RxDone", "invalid",
"RxWait", "invalid",
"TxData", "invalid",
"TxDone", "invalid",
"TxWait", "invalid",
"HwAddr", "invalid"
};
if (offset & 0x3)
return "invalid";
if (offset >= Size)
return "invalid";
return names[offset / 4];
}
uint8_t size;
bool read;
bool write;
const char *name;
};
/* namespace Regs */ }
inline const Regs::Info&
regInfo(Addr daddr)
{
static Regs::Info info [] = {
{ 4, true, true, "Config" },
{ 4, false, true, "Command" },
{ 4, true, true, "IntrStatus" },
{ 4, true, true, "IntrMask" },
{ 4, true, false, "RxMaxCopy" },
{ 4, true, false, "TxMaxCopy" },
{ 4, true, false, "RxMaxIntr" },
{ 0, false, false, "invalid" },
{ 4, true, false, "RxFifoSize" },
{ 4, true, false, "TxFifoSize" },
{ 4, true, false, "RxFifoMark" },
{ 4, true, false, "TxFifoMark" },
{ 8, true, true, "RxData" }, { 0, false, false, "invalid" },
{ 8, true, false, "RxDone" }, { 0, false, false, "invalid" },
{ 8, true, false, "RxWait" }, { 0, false, false, "invalid" },
{ 8, true, true, "TxData" }, { 0, false, false, "invalid" },
{ 8, true, false, "TxDone" }, { 0, false, false, "invalid" },
{ 8, true, false, "TxWait" }, { 0, false, false, "invalid" },
{ 8, true, false, "HwAddr" }, { 0, false, false, "invalid" }
};
return info[daddr / 4];
}
inline bool
regValid(Addr daddr)
{
if (daddr > Regs::Size)
return false;
if (regInfo(daddr).size == 0)
return false;
return true;
}
/* namespace Sinic */ }
#endif // __DEV_SINICREG_HH__

103
python/m5/config.py
View file

@ -755,7 +755,7 @@ class ParamDesc(object):
class VectorParamValue(list):
def ini_str(self):
return ' '.join([str(v) for v in self])
return ' '.join([v.ini_str() for v in self])
def unproxy(self, base):
return [v.unproxy(base) for v in self]
@ -825,6 +825,40 @@ VectorParam = ParamFactory(VectorParamDesc)
#
#####################################################################
# superclass for "numeric" parameter values, to emulate math
# operations in a type-safe way. e.g., a Latency times an int returns
# a new Latency object.
class NumericParamValue(ParamValue):
def __str__(self):
return str(self.value)
def __float__(self):
return float(self.value)
# hook for bounds checking
def _check(self):
return
def __mul__(self, other):
newobj = self.__class__(self)
newobj.value *= other
newobj._check()
return newobj
__rmul__ = __mul__
def __div__(self, other):
newobj = self.__class__(self)
newobj.value /= other
newobj._check()
return newobj
def __sub__(self, other):
newobj = self.__class__(self)
newobj.value -= other
newobj._check()
return newobj
class Range(ParamValue):
type = int # default; can be overridden in subclasses
def __init__(self, *args, **kwargs):
@ -891,19 +925,20 @@ class CheckedIntType(type):
# class is subclassed to generate parameter classes with specific
# bounds. Initialization of the min and max bounds is done in the
# metaclass CheckedIntType.__init__.
class CheckedInt(long,ParamValue):
class CheckedInt(NumericParamValue):
__metaclass__ = CheckedIntType
def __new__(cls, value):
if isinstance(value, str):
value = toInteger(value)
self = long.__new__(cls, value)
if not cls.min <= self <= cls.max:
def _check(self):
if not self.min <= self.value <= self.max:
raise TypeError, 'Integer param out of bounds %d < %d < %d' % \
(cls.min, self, cls.max)
return self
(self.min, self.value, self.max)
def __init__(self, value):
if isinstance(value, str):
self.value = toInteger(value)
elif isinstance(value, (int, long)):
self.value = long(value)
self._check()
class Int(CheckedInt): size = 32; unsigned = False
class Unsigned(CheckedInt): size = 32; unsigned = True
@ -930,19 +965,26 @@ class Float(ParamValue, float):
class MemorySize(CheckedInt):
size = 64
unsigned = True
def __new__(cls, value):
return super(MemorySize, cls).__new__(cls, toMemorySize(value))
def __init__(self, value):
if isinstance(value, MemorySize):
self.value = value.value
else:
self.value = toMemorySize(value)
self._check()
class Addr(CheckedInt):
size = 64
unsigned = True
def __new__(cls, value):
try:
value = long(toMemorySize(value))
except TypeError:
value = long(value)
return super(Addr, cls).__new__(cls, value)
def __init__(self, value):
if isinstance(value, Addr):
self.value = value.value
else:
try:
self.value = toMemorySize(value)
except TypeError:
self.value = long(value)
self._check()
class AddrRange(Range):
type = Addr
@ -1123,29 +1165,6 @@ def tick_check(float_ticks):
#raise ValueError
return int_ticks
# superclass for "numeric" parameter values, to emulate math
# operations in a type-safe way. e.g., a Latency times an int returns
# a new Latency object.
class NumericParamValue(ParamValue):
def __str__(self):
return str(self.value)
def __float__(self):
return float(self.value)
def __mul__(self, other):
newobj = self.__class__(self)
newobj.value *= other
return newobj
__rmul__ = __mul__
def __div__(self, other):
newobj = self.__class__(self)
newobj.value /= other
return newobj
def getLatency(value):
if isinstance(value, Latency) or isinstance(value, Clock):
return value.value

22
python/m5/convert.py
View file

@ -89,17 +89,9 @@ def toFloat(value):
else:
return float(value)
def toLong(value):
value = toFloat(value)
result = int(value)
if value != result:
raise ValueError, "cannot convert '%s' to long" % value
return result
def toInteger(value):
value = toFloat(value)
result = int(value)
result = long(value)
if value != result:
raise ValueError, "cannot convert '%s' to integer" % value
@ -220,16 +212,16 @@ def toMemorySize(value):
raise TypeError, "wrong type '%s' should be str" % type(value)
if value.endswith('PB'):
return float(value[:-2]) * pebi
return long(value[:-2]) * pebi
elif value.endswith('TB'):
return float(value[:-2]) * tebi
return long(value[:-2]) * tebi
elif value.endswith('GB'):
return float(value[:-2]) * gibi
return long(value[:-2]) * gibi
elif value.endswith('MB'):
return float(value[:-2]) * mebi
return long(value[:-2]) * mebi
elif value.endswith('kB'):
return float(value[:-2]) * kibi
return long(value[:-2]) * kibi
elif value.endswith('B'):
return float(value[:-1])
return long(value[:-1])
raise ValueError, "cannot convert '%s' to memory size" % value

3
python/m5/objects/Ethernet.py
View file

@ -84,6 +84,7 @@ class EtherDevBase(PciDevice):
rx_filter = Param.Bool(True, "Enable Receive Filter")
intr_delay = Param.Latency('10us', "Interrupt Propagation Delay")
dedicated = Param.Bool(False, "dedicate a kernel thread to the driver")
class NSGigE(EtherDevBase):
type = 'NSGigE'
@ -91,7 +92,6 @@ class NSGigE(EtherDevBase):
dma_data_free = Param.Bool(False, "DMA of Data is free")
dma_desc_free = Param.Bool(False, "DMA of Descriptors is free")
dedicated = Param.Bool(False, "dedicate a kernel thread to the driver")
class NSGigEInt(EtherInt):
type = 'NSGigEInt'
@ -102,6 +102,7 @@ class Sinic(EtherDevBase):
rx_max_copy = Param.MemorySize('1514B', "rx max copy")
tx_max_copy = Param.MemorySize('16kB', "tx max copy")
rx_max_intr = Param.UInt32(10, "max rx packets per interrupt")
rx_fifo_threshold = Param.MemorySize('48kB', "rx fifo high threshold")
tx_fifo_threshold = Param.MemorySize('16kB', "tx fifo low threshold")

3
test/genini.py
View file

@ -64,8 +64,7 @@ for path in pathlist:
AddToPath(path)
for arg in args:
AddToPath(os.path.dirname(arg))
execfile(arg)
m5execfile(arg, globals())
if globals().has_key('root') and isinstance(root, Root):
instantiate(root)

4
util/pbs/pbs.py
View file

@ -147,6 +147,8 @@ class qsub:
flags.append('e')
if len(flags):
self.cmd.append('-m ' + flags)
else:
self.cmd.append('-mn')
if self.name:
self.cmd.append("-N%s" % self.name)
@ -158,7 +160,7 @@ class qsub:
self.cmd.append('-q' + self.queue)
if self.afterok:
self.cmd.append('-Wdepend=afterok:%s' % self.after)
self.cmd.append('-Wdepend=afterok:%s' % self.afterok)
self.cmd.extend(args)
self.script = script

7
util/pbs/send.py
View file

@ -98,7 +98,7 @@ Usage:
try:
import getopt
opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lq:Rt:v')
opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v')
except getopt.GetoptError:
sys.exit(usage)
@ -115,6 +115,7 @@ docpts = False
doruns = True
runflag = False
node_type = 'FAST'
update = True
for opt,arg in opts:
if opt == '-C':
@ -134,6 +135,8 @@ for opt,arg in opts:
jfile = arg
if opt == '-l':
listonly = True
if opt == '-n':
update = False
if opt == '-q':
queue = arg
if opt == '-R':
@ -154,7 +157,7 @@ from job import JobDir, date
conf = jobfile.JobFile(jfile)
if not listonly and not onlyecho and isdir(conf.linkdir):
if update and not listonly and not onlyecho and isdir(conf.linkdir):
if verbose:
print 'Checking for outdated files in Link directory'
if not isdir(conf.basedir):

16
util/qdo
View file

@ -45,7 +45,11 @@ optparser.add_option('-o', dest='stdout_file',
help='command stdout output file')
optparser.add_option('-l', dest='save_log', action='store_true',
help='save qsub output log file')
optparser.add_option('-q', dest='qsub_timeout', type='int',
optparser.add_option('-N', dest='job_name',
help='qsub job name')
optparser.add_option('-q', dest='dest_queue',
help='qsub destination queue')
optparser.add_option('--qwait', dest='qsub_timeout', type='int',
help='qsub queue wait timeout', default=30*60)
optparser.add_option('-t', dest='cmd_timeout', type='int',
help='command execution timeout', default=600*60)
@ -56,6 +60,12 @@ if cmd == []:
print >>sys.stderr, "%s: missing command" % progname
sys.exit(1)
# If we want to do this, need to add check here to make sure cmd[0] is
# a valid PBS job name, else qsub will die on us.
#
#if not options.job_name:
# options.job_name = cmd[0]
cwd = os.getcwd()
# Deal with systems where /n is a symlink to /.automount
@ -137,6 +147,10 @@ if False and len(cmd) > 50:
print "%s: running %s on poolfs" % (progname, cmd[0])
else:
shell_cmd = 'qsub -I -S /bin/sh'
if options.job_name:
shell_cmd += ' -N "%s"' % options.job_name
if options.dest_queue:
shell_cmd += ' -q ' + options.dest_queue
shell = Shell(shell_cmd)

72
util/stats/db.py
View file

@ -103,6 +103,20 @@ class Node(object):
def __str__(self):
return self.name
class Result(object):
def __init__(self, x, y):
self.data = {}
self.x = x
self.y = y
def __contains__(self, run):
return run in self.data
def __getitem__(self, run):
if run not in self.data:
self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ]
return self.data[run]
class Database(object):
def __init__(self):
self.host = 'zizzer.pool'
@ -135,7 +149,23 @@ class Database(object):
self.runs = None
self.bins = None
self.ticks = None
self.__dict__['get'] = type(self).sum
self.method = 'sum'
self._method = type(self).sum
def get(self, job, stat):
run = self.allRunNames.get(job.name, None)
if run is None:
print 'run "%s" not found' % job
return None
from info import scalar, vector, value, values, total, len
stat.system = self[job.system]
if scalar(stat):
return value(stat, run.run)
if vector(stat):
return values(stat, run.run)
return None
def query(self, sql):
self.cursor.execute(sql)
@ -203,7 +233,7 @@ class Database(object):
self.query('select * from stats')
import info
for result in self.cursor.fetchall():
stat = info.NewStat(StatData(result))
stat = info.NewStat(self, StatData(result))
self.append(stat)
self.allStats.append(stat)
self.allStatIds[stat.stat] = stat
@ -421,30 +451,17 @@ class Database(object):
def stdev(self, stat, bins, ticks):
return self.outer('stddev', 'sum', stat, bins, ticks)
def __getattribute__(self, attr):
if attr != 'get':
return super(Database, self).__getattribute__(attr)
if self.__dict__['get'] == type(self).sum:
return 'sum'
elif self.__dict__['get'] == type(self).avg:
return 'avg'
elif self.__dict__['get'] == type(self).stdev:
return 'stdev'
else:
return ''
def __setattr__(self, attr, value):
if attr != 'get':
super(Database, self).__setattr__(attr, value)
super(Database, self).__setattr__(attr, value)
if attr != 'method':
return
if value == 'sum':
self.__dict__['get'] = type(self).sum
self._method = self.sum
elif value == 'avg':
self.__dict__['get'] = type(self).avg
self._method = self.avg
elif value == 'stdev':
self.__dict__['get'] = type(self).stdev
self._method = self.stdev
else:
raise AttributeError, "can only set get to: sum | avg | stdev"
@ -453,10 +470,12 @@ class Database(object):
bins = self.bins
if ticks is None:
ticks = self.ticks
sql = self.__dict__['get'](self, stat, bins, ticks)
sql = self._method(self, stat, bins, ticks)
self.query(sql)
runs = {}
xmax = 0
ymax = 0
for x in self.cursor.fetchall():
data = Data(x)
if not runs.has_key(data.run):
@ -464,8 +483,17 @@ class Database(object):
if not runs[data.run].has_key(data.x):
runs[data.run][data.x] = {}
xmax = max(xmax, data.x)
ymax = max(ymax, data.y)
runs[data.run][data.x][data.y] = data.data
return runs
results = Result(xmax + 1, ymax + 1)
for run,data in runs.iteritems():
result = results[run]
for x,ydata in data.iteritems():
for y,data in ydata.iteritems():
result[x][y] = data
return results
def __getitem__(self, key):
return self.stattop[key]

99
util/stats/display.py
View file

@ -26,7 +26,7 @@
class Value:
def __init__(self, value, precision, percent = False):
self.value = value
self.value = float(value)
self.precision = precision
self.percent = percent
def __str__(self):
@ -90,61 +90,60 @@ class Print:
class VectorDisplay:
def display(self):
if not self.value:
return
p = Print()
p.flags = self.flags
p.precision = self.precision
if isinstance(self.value, (list, tuple)):
if not len(self.value):
return
mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
mycdf = 0.0
value = self.value
if display_all:
subnames = [ '[%d]' % i for i in range(len(value)) ]
else:
subnames = [''] * len(value)
if self.__dict__.has_key('subnames'):
for i,each in enumerate(self.subnames):
if len(each) > 0:
subnames[i] = '.%s' % each
subdescs = [self.desc]*len(value)
if self.__dict__.has_key('subdescs'):
for i in xrange(min(len(value), len(self.subdescs))):
subdescs[i] = self.subdescs[i]
for val,sname,sdesc in map(None, value, subnames, subdescs):
if mytotal > 0.0:
mypdf = float(val) / float(mytotal)
mycdf += mypdf
if (self.flags & flags_pdf):
p.pdf = mypdf
p.cdf = mycdf
if len(sname) == 0:
continue
p.name = self.name + sname
p.desc = sdesc
p.value = val
p.display()
if (self.flags & flags_total):
if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
p.name = self.name + '.total'
p.desc = self.desc
p.value = mytotal
p.display()
else:
if not isinstance(self.value, (list, tuple)):
p.name = self.name
p.desc = self.desc
p.value = self.value
p.display()
return
mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
mycdf = 0.0
value = self.value
if display_all:
subnames = [ '[%d]' % i for i in range(len(value)) ]
else:
subnames = [''] * len(value)
if self.__dict__.has_key('subnames'):
for i,each in enumerate(self.subnames):
if len(each) > 0:
subnames[i] = '.%s' % each
subdescs = [self.desc]*len(value)
if self.__dict__.has_key('subdescs'):
for i in xrange(min(len(value), len(self.subdescs))):
subdescs[i] = self.subdescs[i]
for val,sname,sdesc in map(None, value, subnames, subdescs):
if mytotal > 0.0:
mypdf = float(val) / float(mytotal)
mycdf += mypdf
if (self.flags & flags_pdf):
p.pdf = mypdf
p.cdf = mycdf
if len(sname) == 0:
continue
p.name = self.name + sname
p.desc = sdesc
p.value = val
p.display()
if (self.flags & flags_total):
if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
p.name = self.name + '.total'
p.desc = self.desc
p.value = mytotal
p.display()

665
util/stats/info.py
View file

@ -27,391 +27,347 @@
from __future__ import division
import operator, re, types
source = None
display_run = 0
global globalTicks
globalTicks = None
def unproxy(proxy):
if hasattr(proxy, '__unproxy__'):
return proxy.__unproxy__()
def total(f):
if isinstance(f, FormulaStat):
v = f.value
else:
v = f
return proxy
f = FormulaStat()
if isinstance(v, (list, tuple)):
f.value = reduce(operator.add, v)
else:
f.value = v
def scalar(stat):
stat = unproxy(stat)
assert(stat.__scalar__() != stat.__vector__())
return stat.__scalar__()
return f
def vector(stat):
stat = unproxy(stat)
assert(stat.__scalar__() != stat.__vector__())
return stat.__vector__()
def unaryop(op, f):
if isinstance(f, FormulaStat):
v = f.value
else:
v = f
if isinstance(v, (list, tuple)):
return map(op, v)
else:
return op(v)
def zerodiv(lv, rv):
if rv == 0.0:
return 0.0
else:
return operator.truediv(lv, rv)
def wrapop(op, lv, rv):
if isinstance(lv, str):
return lv
if isinstance(rv, str):
return rv
return op(lv, rv)
def same(lrun, rrun):
for lx,rx in zip(lrun.keys(),rrun.keys()):
if lx != rx:
print 'lx != rx'
print lx, rx
print lrun.keys()
print rrun.keys()
return False
for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()):
if ly != ry:
print 'ly != ry'
print ly, ry
print lrun[lx].keys()
print rrun[rx].keys()
return False
return True
def binaryop(op, lf, rf):
result = {}
if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat):
lv = lf.value
rv = rf.value
theruns = []
for r in lv.keys():
if rv.has_key(r):
if same(lv[r], rv[r]):
theruns.append(r)
else:
raise AttributeError
for run in theruns:
result[run] = {}
for x in lv[run].keys():
result[run][x] = {}
for y in lv[run][x].keys():
result[run][x][y] = wrapop(op, lv[run][x][y],
rv[run][x][y])
elif isinstance(lf, FormulaStat):
lv = lf.value
for run in lv.keys():
result[run] = {}
for x in lv[run].keys():
result[run][x] = {}
for y in lv[run][x].keys():
result[run][x][y] = wrapop(op, lv[run][x][y], rf)
elif isinstance(rf, FormulaStat):
rv = rf.value
for run in rv.keys():
result[run] = {}
for x in rv[run].keys():
result[run][x] = {}
for y in rv[run][x].keys():
result[run][x][y] = wrapop(op, lf, rv[run][x][y])
def value(stat, *args):
stat = unproxy(stat)
return stat.__value__(*args)
def values(stat, run):
stat = unproxy(stat)
result = []
for i in xrange(len(stat)):
val = value(stat, run, i)
if val is None:
return None
result.append(val)
return result
def sums(x, y):
if isinstance(x, (list, tuple)):
return map(lambda x, y: x + y, x, y)
else:
return x + y
def total(stat, run):
return sum(values(stat, run))
def alltrue(seq):
return reduce(lambda x, y: x and y, seq)
def len(stat):
stat = unproxy(stat)
return stat.__len__()
def allfalse(seq):
return not reduce(lambda x, y: x or y, seq)
class Value(object):
def __scalar__(self):
raise AttributeError, "must define __scalar__ for %s" % (type (self))
def __vector__(self):
raise AttributeError, "must define __vector__ for %s" % (type (self))
def enumerate(seq):
return map(None, range(len(seq)), seq)
def __add__(self, other):
return BinaryProxy(operator.__add__, self, other)
def __sub__(self, other):
return BinaryProxy(operator.__sub__, self, other)
def __mul__(self, other):
return BinaryProxy(operator.__mul__, self, other)
def __div__(self, other):
return BinaryProxy(operator.__div__, self, other)
def __truediv__(self, other):
return BinaryProxy(operator.__truediv__, self, other)
def __floordiv__(self, other):
return BinaryProxy(operator.__floordiv__, self, other)
def cmp(a, b):
if a < b:
return -1
elif a == b:
return 0
else:
return 1
def __radd__(self, other):
return BinaryProxy(operator.__add__, other, self)
def __rsub__(self, other):
return BinaryProxy(operator.__sub__, other, self)
def __rmul__(self, other):
return BinaryProxy(operator.__mul__, other, self)
def __rdiv__(self, other):
return BinaryProxy(operator.__div__, other, self)
def __rtruediv__(self, other):
return BinaryProxy(operator.__truediv__, other, self)
def __rfloordiv__(self, other):
return BinaryProxy(operator.__floordiv__, other, self)
class Statistic(object):
def __neg__(self):
return UnaryProxy(operator.__neg__, self)
def __pos__(self):
return UnaryProxy(operator.__pos__, self)
def __abs__(self):
return UnaryProxy(operator.__abs__, self)
def __init__(self, data):
self.__dict__.update(data.__dict__)
if not self.__dict__.has_key('value'):
self.__dict__['value'] = None
if not self.__dict__.has_key('bins'):
self.__dict__['bins'] = None
if not self.__dict__.has_key('ticks'):
self.__dict__['ticks'] = None
if 'vc' not in self.__dict__:
self.vc = {}
class Scalar(Value):
def __scalar__(self):
return True
def __getattribute__(self, attr):
if attr == 'ticks':
if self.__dict__['ticks'] != globalTicks:
self.__dict__['value'] = None
self.__dict__['ticks'] = globalTicks
return self.__dict__['ticks']
if attr == 'value':
if self.__dict__['ticks'] != globalTicks:
if self.__dict__['ticks'] != None and \
len(self.__dict__['ticks']) == 1:
self.vc[self.__dict__['ticks'][0]] = self.__dict__['value']
self.__dict__['ticks'] = globalTicks
if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]):
self.__dict__['value'] = self.vc[globalTicks[0]]
else:
self.__dict__['value'] = None
if self.__dict__['value'] == None:
self.__dict__['value'] = self.getValue()
return self.__dict__['value']
else:
return super(Statistic, self).__getattribute__(attr)
def __setattr__(self, attr, value):
if attr == 'bins' or attr == 'ticks':
if attr == 'bins':
if value is not None:
value = source.getBin(value)
#elif attr == 'ticks' and type(value) is str:
# value = [ int(x) for x in value.split() ]
self.__dict__[attr] = value
self.__dict__['value'] = None
self.vc = {}
else:
super(Statistic, self).__setattr__(attr, value)
def getValue(self):
raise AttributeError, 'getValue() must be defined'
def zero(self):
def __vector__(self):
return False
def __ne__(self, other):
return not (self == other)
def __value__(self, run):
raise AttributeError, '__value__ must be defined'
def __str__(self):
return '%f' % (float(self))
class VectorItemProxy(Value):
def __init__(self, proxy, index):
self.proxy = proxy
self.index = index
class FormulaStat(object):
def __add__(self, other):
f = FormulaStat()
f.value = binaryop(operator.add, self, other)
return f
def __sub__(self, other):
f = FormulaStat()
f.value = binaryop(operator.sub, self, other)
return f
def __mul__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mul, self, other)
return f
def __truediv__(self, other):
f = FormulaStat()
f.value = binaryop(zerodiv, self, other)
return f
def __mod__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mod, self, other)
return f
def __radd__(self, other):
f = FormulaStat()
f.value = binaryop(operator.add, other, self)
return f
def __rsub__(self, other):
f = FormulaStat()
f.value = binaryop(operator.sub, other, self)
return f
def __rmul__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mul, other, self)
return f
def __rtruediv__(self, other):
f = FormulaStat()
f.value = binaryop(zerodiv, other, self)
return f
def __rmod__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mod, other, self)
return f
def __neg__(self):
f = FormulaStat()
f.value = unaryop(operator.neg, self)
return f
def __getitem__(self, idx):
f = FormulaStat()
f.value = {}
for key in self.value.keys():
f.value[key] = {}
f.value[key][0] = {}
f.value[key][0][0] = self.value[key][idx][0]
return f
def __scalar__(self):
return True
def __float__(self):
if isinstance(self.value, FormulaStat):
return float(self.value)
if not self.value.has_key(display_run):
return (1e300*1e300)
if len(self.value[display_run]) == 1:
return self.value[display_run][0][0]
else:
#print self.value[display_run]
return self.value[display_run][4][0]
#raise ValueError
def __vector__(self):
return False
def display(self):
import display
d = display.VectorDisplay()
d.flags = 0
d.precision = 1
d.name = 'formula'
d.desc = 'formula'
val = self.value[display_run]
d.value = [ val[x][0] for x in val.keys() ]
d.display()
def __value__(self, run):
return value(self.proxy, run, self.index)
class Vector(Value):
def __scalar__(self):
return False
class Scalar(Statistic,FormulaStat):
def getValue(self):
return source.data(self, self.bins, self.ticks)
def __vector__(self):
return True
def display(self):
def __value__(self, run, index):
raise AttributeError, '__value__ must be defined'
def __getitem__(self, index):
return VectorItemProxy(self, index)
class ScalarConstant(Scalar):
def __init__(self, constant):
self.constant = constant
def __value__(self, run):
return self.constant
class VectorConstant(Vector):
def __init__(self, constant):
self.constant = constant
def __value__(self, run, index):
return self.constant[index]
def __len__(self):
return len(self.constant)
def WrapValue(value):
if isinstance(value, (int, long, float)):
return ScalarConstant(value)
if isinstance(value, (list, tuple)):
return VectorConstant(value)
if isinstance(value, Value):
return value
raise AttributeError, 'Only values can be wrapped'
class Statistic(object):
def __getattr__(self, attr):
if attr in ('data', 'x', 'y'):
result = self.source.data(self, self.bins, self.ticks)
self.data = result.data
self.x = result.x
self.y = result.y
return super(Statistic, self).__getattribute__(attr)
def __setattr__(self, attr, value):
if attr == 'stat':
raise AttributeError, '%s is read only' % stat
if attr in ('source', 'bins', 'ticks'):
if getattr(self, attr) != value:
if hasattr(self, 'data'):
delattr(self, 'data')
super(Statistic, self).__setattr__(attr, value)
class ValueProxy(Value):
def __getattr__(self, attr):
if attr == '__value__':
if scalar(self):
return self.__scalarvalue__
if vector(self):
return self.__vectorvalue__
if attr == '__len__':
if vector(self):
return self.__vectorlen__
return super(ValueProxy, self).__getattribute__(attr)
class UnaryProxy(ValueProxy):
def __init__(self, op, arg):
self.op = op
self.arg = WrapValue(arg)
def __scalar__(self):
return scalar(self.arg)
def __vector__(self):
return vector(self.arg)
def __scalarvalue__(self, run):
val = value(self.arg, run)
if val is None:
return None
return self.op(val)
def __vectorvalue__(self, run, index):
val = value(self.arg, run, index)
if val is None:
return None
return self.op(val)
def __vectorlen__(self):
return len(unproxy(self.arg))
class BinaryProxy(ValueProxy):
def __init__(self, op, arg0, arg1):
super(BinaryProxy, self).__init__()
self.op = op
self.arg0 = WrapValue(arg0)
self.arg1 = WrapValue(arg1)
def __scalar__(self):
return scalar(self.arg0) and scalar(self.arg1)
def __vector__(self):
return vector(self.arg0) or vector(self.arg1)
def __scalarvalue__(self, run):
val0 = value(self.arg0, run)
val1 = value(self.arg1, run)
if val0 is None or val1 is None:
return None
return self.op(val0, val1)
def __vectorvalue__(self, run, index):
if scalar(self.arg0):
val0 = value(self.arg0, run)
if vector(self.arg0):
val0 = value(self.arg0, run, index)
if scalar(self.arg1):
val1 = value(self.arg1, run)
if vector(self.arg1):
val1 = value(self.arg1, run, index)
if val0 is None or val1 is None:
return None
return self.op(val0, val1)
def __vectorlen__(self):
if vector(self.arg0) and scalar(self.arg1):
return len(self.arg0)
if scalar(self.arg0) and vector(self.arg1):
return len(self.arg1)
len0 = len(self.arg0)
len1 = len(self.arg1)
if len0 != len1:
raise AttributeError, \
"vectors of different lengths %d != %d" % (len0, len1)
return len0
class Proxy(Value):
def __init__(self, name, dict):
self.name = name
self.dict = dict
def __unproxy__(self):
return unproxy(self.dict[self.name])
def __getitem__(self, index):
return ItemProxy(self, index)
def __getattr__(self, attr):
return AttrProxy(self, attr)
class ItemProxy(Proxy):
def __init__(self, proxy, index):
self.proxy = proxy
self.index = index
def __unproxy__(self):
return unproxy(unproxy(self.proxy)[self.index])
class AttrProxy(Proxy):
def __init__(self, proxy, attr):
self.proxy = proxy
self.attr = attr
def __unproxy__(self):
return unproxy(getattr(unproxy(self.proxy), self.attr))
class ProxyGroup(object):
def __init__(self, dict=None, **kwargs):
self.__dict__['dict'] = {}
if dict is not None:
self.dict.update(dict)
if kwargs:
self.dict.update(kwargs)
def __getattr__(self, name):
return Proxy(name, self.dict)
def __setattr__(self, attr, value):
self.dict[attr] = value
class ScalarStat(Statistic,Scalar):
def __value__(self, run):
if run not in self.data:
return None
return self.data[run][0][0]
def display(self, run=None):
import display
p = display.Print()
p.name = self.name
p.desc = self.desc
p.value = float(self)
p.value = value(self, run)
p.flags = self.flags
p.precision = self.precision
if display.all or (self.flags & flags.printable):
p.display()
def comparable(self, other):
return self.name == other.name
class VectorStat(Statistic,Vector):
def __value__(self, run, item):
if run not in self.data:
return None
return self.data[run][item][0]
def __eq__(self, other):
return self.value == other.value
def __len__(self):
return self.x
def __isub__(self, other):
self.value -= other.value
return self
def __iadd__(self, other):
self.value += other.value
return self
def __itruediv__(self, other):
if not other:
return self
self.value /= other
return self
class Vector(Statistic,FormulaStat):
def getValue(self):
return source.data(self, self.bins, self.ticks);
def display(self):
def display(self, run=None):
import display
if not display.all and not (self.flags & flags.printable):
return
d = display.VectorDisplay()
d.__dict__.update(self.__dict__)
d.name = self.name
d.desc = self.desc
d.value = [ value(self, run, i) for i in xrange(len(self)) ]
d.flags = self.flags
d.precision = self.precision
d.display()
def comparable(self, other):
return self.name == other.name and \
len(self.value) == len(other.value)
class Formula(Value):
def __getattribute__(self, attr):
if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ):
return super(Formula, self).__getattribute__(attr)
def __eq__(self, other):
if isinstance(self.value, (list, tuple)) != \
isinstance(other.value, (list, tuple)):
return False
if isinstance(self.value, (list, tuple)):
if len(self.value) != len(other.value):
return False
else:
for v1,v2 in zip(self.value, other.value):
if v1 != v2:
return False
return True
else:
return self.value == other.value
def __isub__(self, other):
self.value = binaryop(operator.sub, self.value, other.value)
return self
def __iadd__(self, other):
self.value = binaryop(operator.add, self.value, other.value)
return self
def __itruediv__(self, other):
if not other:
return self
if isinstance(self.value, (list, tuple)):
for i in xrange(len(self.value)):
self.value[i] /= other
else:
self.value /= other
return self
class Formula(Vector):
def getValue(self):
formula = re.sub(':', '__', self.formula)
x = eval(formula, source.stattop)
return x.value
value = eval(formula, self.source.stattop)
return getattr(value, attr)
def comparable(self, other):
return self.name == other.name and \
compare(self.dist, other.dist)
def __eq__(self, other):
return self.value == other.value
def __isub__(self, other):
return self
def __iadd__(self, other):
return self
def __itruediv__(self, other):
if not other:
return self
return self
class SimpleDist(object):
class SimpleDist(Statistic):
def __init__(self, sums, squares, samples):
self.sums = sums
self.squares = squares
self.samples = samples
def getValue(self):
return 0.0
def display(self, name, desc, flags, precision):
import display
p = display.Print()
@ -482,9 +438,6 @@ class FullDist(SimpleDist):
self.bsize = bsize
self.size = size
def getValue(self):
return 0.0
def display(self, name, desc, flags, precision):
import display
p = display.Print()
@ -574,9 +527,6 @@ class FullDist(SimpleDist):
return self
class Dist(Statistic):
def getValue(self):
return 0.0
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@ -606,9 +556,6 @@ class Dist(Statistic):
return self
class VectorDist(Statistic):
def getValue(self):
return 0.0
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@ -694,9 +641,6 @@ class VectorDist(Statistic):
return self
class Vector2d(Statistic):
def getValue(self):
return 0.0
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@ -748,20 +692,25 @@ class Vector2d(Statistic):
return self
return self
def NewStat(data):
def NewStat(source, data):
stat = None
if data.type == 'SCALAR':
stat = Scalar(data)
stat = ScalarStat()
elif data.type == 'VECTOR':
stat = Vector(data)
stat = VectorStat()
elif data.type == 'DIST':
stat = Dist(data)
stat = Dist()
elif data.type == 'VECTORDIST':
stat = VectorDist(data)
stat = VectorDist()
elif data.type == 'VECTOR2D':
stat = Vector2d(data)
stat = Vector2d()
elif data.type == 'FORMULA':
stat = Formula(data)
stat = Formula()
stat.__dict__['source'] = source
stat.__dict__['bins'] = None
stat.__dict__['ticks'] = None
stat.__dict__.update(data.__dict__)
return stat

52
util/stats/output.py
View file

@ -26,24 +26,8 @@
#
# Authors: Nathan Binkert
class dbinfo(object):
def get(self, job, stat):
import info
run = info.source.allRunNames.get(job.name, None)
if run is None:
print 'run "%s" not found' % job
return None
stat.system = info.source[job.system]
info.display_run = run.run;
val = float(stat)
if val == 1e300*1e300:
return None
return val
class StatOutput(object):
def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None):
def __init__(self, name, jobfile, info, stat=None, binstats=None):
self.name = name
self.jobfile = jobfile
self.stat = stat
@ -141,7 +125,7 @@ class StatOutput(object):
data = zeros((len(groupopts), len(baropts)), Float)
data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ]
enabled = False
stacked = None
stacked = 0
for g,gopt in enumerate(groupopts):
for b,bopt in enumerate(baropts):
job = self.jobfile.job(options + [ gopt, bopt ])
@ -149,17 +133,28 @@ class StatOutput(object):
continue
val = self.info.get(job, self.stat)
if val is None:
val = 0.0
curstacked = isinstance(val, (list, tuple))
if stacked is None:
stacked = curstacked
else:
if stacked != curstacked:
raise ValueError, "some stats stacked, some not"
if isinstance(val, (list, tuple)):
if len(val) == 1:
val = val[0]
else:
stacked = len(val)
data[g][b] = val
if stacked == 0:
for i in xrange(len(groupopts)):
for j in xrange(len(baropts)):
if data[i][j] is None:
data[i][j] = 0.0
else:
for i in xrange(len(groupopts)):
for j in xrange(len(baropts)):
val = data[i][j]
if val is None:
data[i][j] = [] * stacked
elif len(val) != stacked:
raise ValueError, "some stats stacked, some not"
data = array(data)
if data.sum() == 0:
continue
@ -167,7 +162,10 @@ class StatOutput(object):
bar_descs = [ opt.desc for opt in baropts ]
group_descs = [ opt.desc for opt in groupopts ]
if stacked:
legend = self.info.rcategories
try:
legend = self.info.rcategories
except:
legend = [ str(i) for i in xrange(stacked) ]
else:
legend = bar_descs

13
util/stats/profile.py
View file

@ -316,7 +316,7 @@ class Profile(object):
symbols.tree.dot(dot, threshold=threshold)
dot.write(symbols.filename[:-3] + 'dot')
def write_txt(self, jobfile=None, jobs=None):
def write_txt(self, jobfile=None, jobs=None, limit=None):
if jobs is None:
jobs = [ job for job in jobfile.jobs() ]
@ -327,7 +327,7 @@ class Profile(object):
continue
output = file(symbols.filename[:-3] + 'txt', 'w')
symbols.display(output)
symbols.display(output, limit)
def display(self, jobfile=None, jobs=None, limit=None):
if jobs is None:
@ -453,6 +453,8 @@ if __name__ == '__main__':
else:
profile = PCProfile()
if not categorize:
profile.categorize = None
profile.inputdir(jobfile.rootdir)
if graph:
@ -470,11 +472,6 @@ if __name__ == '__main__':
profile.cpu = cpu
profile.write_dot(jobfile=jobfile, threshold=threshold)
if not categorize:
for cpu in cpus:
profile.cpu = cpu
profile.categorize = None
if textout:
for cpu in cpus:
profile.cpu = cpu
@ -482,5 +479,7 @@ if __name__ == '__main__':
if not graph and not textout and not dodot:
for cpu in cpus:
if not categorize:
profile.categorize = None
profile.cpu = cpu
profile.display(jobfile=jobfile, limit=numsyms)

94
util/stats/stats.py
View file

@ -98,29 +98,27 @@ def commands(options, command, args):
raise CommandException
import db, info
info.source = db.Database()
info.source.host = options.host
info.source.db = options.db
info.source.passwd = options.passwd
info.source.user = options.user
info.source.connect()
#info.source.update_dict(globals())
import db
source = db.Database()
source.host = options.host
source.db = options.db
source.passwd = options.passwd
source.user = options.user
source.connect()
#source.update_dict(globals())
if type(options.get) is str:
info.source.get = options.get
if type(options.method) is str:
source.method = options.method
if options.runs is None:
runs = info.source.allRuns
runs = source.allRuns
else:
rx = re.compile(options.runs)
runs = []
for run in info.source.allRuns:
for run in source.allRuns:
if rx.match(run.name):
runs.append(run)
info.display_run = runs[0].run
if command == 'runs':
user = None
opts, args = getopts(args, '-u')
@ -129,14 +127,14 @@ def commands(options, command, args):
for o,a in opts:
if o == '-u':
user = a
info.source.listRuns(user)
source.listRuns(user)
return
if command == 'stats':
if len(args) == 0:
info.source.listStats()
source.listStats()
elif len(args) == 1:
info.source.listStats(args[0])
source.listStats(args[0])
else:
raise CommandException
@ -144,9 +142,9 @@ def commands(options, command, args):
if command == 'bins':
if len(args) == 0:
info.source.listBins()
source.listBins()
elif len(args) == 1:
info.source.listBins(args[0])
source.listBins(args[0])
else:
raise CommandException
@ -154,9 +152,9 @@ def commands(options, command, args):
if command == 'formulas':
if len(args) == 0:
info.source.listFormulas()
source.listFormulas()
elif len(args) == 1:
info.source.listFormulas(args[0])
source.listFormulas(args[0])
else:
raise CommandException
@ -166,7 +164,7 @@ def commands(options, command, args):
if len(args):
raise CommandException
info.source.listTicks(runs)
source.listTicks(runs)
return
if command == 'stability':
@ -177,8 +175,8 @@ def commands(options, command, args):
merge = int(args[0])
except ValueError:
usage()
stats = info.source.getStat(args[1])
info.source.get = "sum"
stats = source.getStat(args[1])
source.method = 'sum'
def disp(*args):
print "%-35s %12s %12s %4s %5s %5s %5s %10s" % args
@ -195,18 +193,17 @@ def commands(options, command, args):
#loop through all the selected runs
for run in runs:
info.display_run = run.run;
runTicks = info.source.retTicks([ run ])
runTicks = source.retTicks([ run ])
#throw away the first one, it's 0
runTicks.pop(0)
info.globalTicks = runTicks
source.ticks = runTicks
avg = 0
stdev = 0
numoutsideavg = 0
numoutside1std = 0
numoutside2std = 0
pairRunTicks = []
if float(stat) == 1e300*1e300:
if value(stat, run.run) == 1e300*1e300:
continue
for t in range(0, len(runTicks)-(merge-1), merge):
tempPair = []
@ -215,17 +212,17 @@ def commands(options, command, args):
pairRunTicks.append(tempPair)
#loop through all the various ticks for each run
for tick in pairRunTicks:
info.globalTicks = tick
avg += float(stat)
source.ticks = tick
avg += value(stat, run.run)
avg /= len(pairRunTicks)
for tick in pairRunTicks:
info.globalTicks = tick
val = float(stat)
source.ticks = tick
val = value(stat, run.run)
stdev += pow((val-avg),2)
stdev = math.sqrt(stdev / len(pairRunTicks))
for tick in pairRunTicks:
info.globalTicks = tick
val = float(stat)
source.ticks = tick
val = value(stat, run.run)
if (val < (avg * .9)) or (val > (avg * 1.1)):
numoutsideavg += 1
if (val < (avg - stdev)) or (val > (avg + stdev)):
@ -264,9 +261,9 @@ def commands(options, command, args):
if options.ticks:
if not options.graph:
print 'only displaying sample %s' % options.ticks
info.globalTicks = [ int(x) for x in options.ticks.split() ]
source.ticks = [ int(x) for x in options.ticks.split() ]
from output import StatOutput
import output
def display():
if options.graph:
@ -280,12 +277,12 @@ def commands(options, command, args):
raise CommandException
if command == 'stat':
stats = info.source.getStat(args[0])
stats = source.getStat(args[0])
if command == 'formula':
stats = eval(args[0])
for stat in stats:
output = StatOutput(stat.name, options.jobfile)
output = output.StatOutput(stat.name, options.jobfile, source)
output.stat = stat
output.label = stat.name
display()
@ -295,12 +292,11 @@ def commands(options, command, args):
if len(args):
raise CommandException
system = info.source.__dict__[options.system]
from proxy import ProxyGroup
sim_ticks = info.source['sim_ticks']
sim_seconds = info.source['sim_seconds']
proxy = ProxyGroup(system = info.source[options.system])
system = source.__dict__[options.system]
from info import ProxyGroup
sim_ticks = source['sim_ticks']
sim_seconds = source['sim_seconds']
proxy = ProxyGroup(system = source[options.system])
system = proxy.system
etherdev = system.tsunami.etherdev0
@ -309,7 +305,7 @@ def commands(options, command, args):
packets = etherdev.rxPackets + etherdev.txPackets
bps = etherdev.rxBandwidth + etherdev.txBandwidth
output = StatOutput(command, options.jobfile)
output = output.StatOutput(command, options.jobfile, source)
if command == 'usertime':
import copy
@ -460,7 +456,7 @@ if __name__ == '__main__':
options.user = getpass.getuser()
options.runs = None
options.system = 'client'
options.get = None
options.method = None
options.binned = False
options.graph = False
options.ticks = False
@ -469,7 +465,7 @@ if __name__ == '__main__':
options.jobfile = None
options.all = False
opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:')
opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:')
for o,a in opts:
if o == '-B':
options.binned = True
@ -477,8 +473,6 @@ if __name__ == '__main__':
options.printmode = 'E'
if o == '-F':
options.printmode = 'F'
if o == '-G':
options.get = a
if o == '-a':
options.all = True
if o == '-d':
@ -492,6 +486,8 @@ if __name__ == '__main__':
jobfilename = None
if o == '-j':
jobfilename = a
if o == '-m':
options.method = a
if o == '-p':
options.passwd = getpass.getpass()
if o == '-r':