diff --git a/SConscript b/SConscript index de320aa40..677a06d1f 100644 --- a/SConscript +++ b/SConscript @@ -192,12 +192,15 @@ base_sources = Split(''' mem/cache/prefetch/prefetcher.cc mem/cache/prefetch/tagged_prefetcher.cc mem/cache/tags/base_tags.cc - mem/cache/tags/cache_tags.cc + mem/cache/tags/cache_tags.cc mem/cache/tags/fa_lru.cc mem/cache/tags/iic.cc mem/cache/tags/lru.cc mem/cache/tags/repl/gen.cc mem/cache/tags/repl/repl.cc + mem/cache/tags/split.cc + mem/cache/tags/split_lru.cc + mem/cache/tags/split_lifo.cc mem/functional/functional.cc mem/timing/base_memory.cc mem/timing/memory_builder.cc diff --git a/arch/isa_parser.py b/arch/isa_parser.py index eaef4b798..8f4c6bce7 100755 --- a/arch/isa_parser.py +++ b/arch/isa_parser.py @@ -794,20 +794,19 @@ def defFormat(id, params, code, lineno): ############## # Stack: a simple stack object. Used for both formats (formatStack) -# and default cases (defaultStack). +# and default cases (defaultStack). Simply wraps a list to give more +# stack-like syntax and enable initialization with an argument list +# (as opposed to an argument that's a list). -class Stack: - def __init__(self, initItem): - self.stack = [ initItem ] +class Stack(list): + def __init__(self, *items): + list.__init__(self, items) def push(self, item): - self.stack.append(item); - - def pop(self): - return self.stack.pop() + self.append(item); def top(self): - return self.stack[-1] + return self[-1] # The global format stack. formatStack = Stack(NoFormat()) diff --git a/dev/ns_gige.cc b/dev/ns_gige.cc index f6d15b02f..0537c344a 100644 --- a/dev/ns_gige.cc +++ b/dev/ns_gige.cc @@ -127,7 +127,7 @@ NSGigE::NSGigE(Params *p) p->header_bus, 1, p->dma_no_allocate); } else if (p->payload_bus) { - pioInterface = newPioInterface(name() + ".pio2", p->hier, + pioInterface = newPioInterface(name() + ".pio", p->hier, p->payload_bus, this, &NSGigE::cacheAccess); @@ -2531,20 +2531,17 @@ NSGigE::recvPacket(PacketPtr packet) if (!rxEnable) { DPRINTF(Ethernet, "receive disabled...packet dropped\n"); - interface->recvDone(); return true; } if (!rxFilterEnable) { DPRINTF(Ethernet, "receive packet filtering disabled . . . packet dropped\n"); - interface->recvDone(); return true; } if (rxFilter(packet)) { DPRINTF(Ethernet, "packet filtered...dropped\n"); - interface->recvDone(); return true; } @@ -2567,7 +2564,6 @@ NSGigE::recvPacket(PacketPtr packet) } rxFifo.push(packet); - interface->recvDone(); rxKick(); return true; diff --git a/dev/sinic.cc b/dev/sinic.cc index 0f2fd20ea..1a1456e5f 100644 --- a/dev/sinic.cc +++ b/dev/sinic.cc @@ -94,8 +94,8 @@ Device::Device(Params *p) reset(); if (p->io_bus) { - pioInterface = newPioInterface(p->name, p->hier, p->io_bus, this, - &Device::cacheAccess); + pioInterface = newPioInterface(p->name + ".pio", p->hier, p->io_bus, + this, &Device::cacheAccess); pioLatency = p->pio_latency * p->io_bus->clockRate; @@ -108,7 +108,8 @@ Device::Device(Params *p) p->io_bus, 1, p->dma_no_allocate); } else if (p->payload_bus) { - pioInterface = newPioInterface(p->name, p->hier, p->payload_bus, this, + pioInterface = newPioInterface(p->name + ".pio", p->hier, + p->payload_bus, this, &Device::cacheAccess); pioLatency = p->pio_latency * p->payload_bus->clockRate; @@ -315,9 +316,26 @@ Device::writeConfig(int offset, int size, const uint8_t *data) } } +void +Device::prepareRead() +{ + using namespace Regs; + + // update rx registers + regs.RxDone = set_RxDone_Packets(regs.RxDone, rxFifo.packets()); + regs.RxWait = regs.RxDone; + + // update tx regsiters + regs.TxDone = set_TxDone_Packets(regs.TxDone, txFifo.packets()); + regs.TxDone = set_TxDone_Full(regs.TxDone, + txFifo.avail() < regs.TxMaxCopy); + regs.TxDone = set_TxDone_Low(regs.TxDone, + txFifo.size() < regs.TxFifoMark); + regs.TxWait = regs.TxDone; +} + /** - * This reads the device registers, which are detailed in the NS83820 - * spec sheet + * I/O read of device register */ Fault Device::read(MemReqPtr &req, uint8_t *data) @@ -327,118 +345,115 @@ Device::read(MemReqPtr &req, uint8_t *data) //The mask is to give you only the offset into the device register file Addr daddr = req->paddr & 0xfff; - if (Regs::regSize(daddr) == 0) - panic("invalid address: da=%#x pa=%#x va=%#x size=%d", + if (!regValid(daddr)) + panic("invalid register: da=%#x pa=%#x va=%#x size=%d", daddr, req->paddr, req->vaddr, req->size); - if (req->size != Regs::regSize(daddr)) + const Regs::Info &info = regInfo(daddr); + if (!info.read) + panic("reading write only register: %s: da=%#x pa=%#x va=%#x size=%d", + info.name, daddr, req->paddr, req->vaddr, req->size); + + if (req->size != info.size) panic("invalid size for reg %s: da=%#x pa=%#x va=%#x size=%d", - Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size); + info.name, daddr, req->paddr, req->vaddr, req->size); - DPRINTF(EthernetPIO, "read reg=%s da=%#x pa=%#x va=%#x size=%d\n", - Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size); + prepareRead(); - uint32_t ®32 = *(uint32_t *)data; - uint64_t ®64 = *(uint64_t *)data; - - switch (daddr) { - case Regs::Config: - reg32 = regs.Config; - break; - - case Regs::RxMaxCopy: - reg32 = regs.RxMaxCopy; - break; - - case Regs::TxMaxCopy: - reg32 = regs.TxMaxCopy; - break; - - case Regs::RxThreshold: - reg32 = regs.RxThreshold; - break; - - case Regs::TxThreshold: - reg32 = regs.TxThreshold; - break; - - case Regs::IntrStatus: - reg32 = regs.IntrStatus; - devIntrClear(); - break; - - case Regs::IntrMask: - reg32 = regs.IntrMask; - break; - - case Regs::RxData: - reg64 = regs.RxData; - break; - - case Regs::RxDone: - case Regs::RxWait: - reg64 = Regs::set_RxDone_FifoLen(regs.RxDone, - min(rxFifo.packets(), 255)); - break; - - case Regs::TxData: - reg64 = regs.TxData; - break; - - case Regs::TxDone: - case Regs::TxWait: - reg64 = Regs::set_TxDone_FifoLen(regs.TxDone, - min(txFifo.packets(), 255)); - break; - - case Regs::HwAddr: - reg64 = params()->eaddr; - break; - - default: - panic("reading write only register %s: da=%#x pa=%#x va=%#x size=%d", - Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size); + uint64_t value = 0; + if (req->size == 4) { + uint32_t ® = *(uint32_t *)data; + reg = regData32(daddr); + value = reg; } - DPRINTF(EthernetPIO, "read reg=%s done val=%#x\n", Regs::regName(daddr), - Regs::regSize(daddr) == 4 ? reg32 : reg64); + if (req->size == 8) { + uint64_t ® = *(uint64_t *)data; + reg = regData64(daddr); + value = reg; + } + + DPRINTF(EthernetPIO, "read reg=%s da=%#x pa=%#x va=%#x size=%d val=%#x\n", + info.name, daddr, req->paddr, req->vaddr, req->size, value); + + // reading the interrupt status register has the side effect of + // clearing it + if (daddr == Regs::IntrStatus) + devIntrClear(); return No_Fault; } +/** + * IPR read of device register + */ +Fault +Device::iprRead(Addr daddr, uint64_t &result) +{ + if (!regValid(daddr)) + panic("invalid address: da=%#x", daddr); + + const Regs::Info &info = regInfo(daddr); + if (!info.read) + panic("reading write only register %s: da=%#x", info.name, daddr); + + DPRINTF(EthernetPIO, "read reg=%s da=%#x\n", info.name, daddr); + + prepareRead(); + + if (info.size == 4) + result = regData32(daddr); + + if (info.size == 8) + result = regData64(daddr); + + DPRINTF(EthernetPIO, "IPR read reg=%s da=%#x val=%#x\n", + info.name, result); + + return No_Fault; +} + +/** + * I/O write of device register + */ Fault Device::write(MemReqPtr &req, const uint8_t *data) { assert(config.command & PCI_CMD_MSE); + + //The mask is to give you only the offset into the device register file Addr daddr = req->paddr & 0xfff; - if (Regs::regSize(daddr) == 0) + if (!regValid(daddr)) panic("invalid address: da=%#x pa=%#x va=%#x size=%d", daddr, req->paddr, req->vaddr, req->size); - if (req->size != Regs::regSize(daddr)) - panic("invalid size: reg=%s da=%#x pa=%#x va=%#x size=%d", - Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size); + const Regs::Info &info = regInfo(daddr); + if (!info.write) + panic("writing read only register %s: da=%#x", info.name, daddr); + + if (req->size != info.size) + panic("invalid size for reg %s: da=%#x pa=%#x va=%#x size=%d", + info.name, daddr, req->paddr, req->vaddr, req->size); uint32_t reg32 = *(uint32_t *)data; uint64_t reg64 = *(uint64_t *)data; DPRINTF(EthernetPIO, "write reg=%s val=%#x da=%#x pa=%#x va=%#x size=%d\n", - Regs::regName(daddr), Regs::regSize(daddr) == 4 ? reg32 : reg64, - daddr, req->paddr, req->vaddr, req->size); - + info.name, info.size == 4 ? reg32 : reg64, daddr, req->paddr, + req->vaddr, req->size); switch (daddr) { case Regs::Config: changeConfig(reg32); break; - case Regs::RxThreshold: - regs.RxThreshold = reg32; + case Regs::Command: + command(reg32); break; - case Regs::TxThreshold: - regs.TxThreshold = reg32; + case Regs::IntrStatus: + devIntrClear(regs.IntrStatus & reg32); break; case Regs::IntrMask: @@ -447,9 +462,10 @@ Device::write(MemReqPtr &req, const uint8_t *data) case Regs::RxData: if (rxState != rxIdle) - panic("receive machine busy with another request!"); + panic("receive machine busy with another request! rxState=%s", + RxStateStrings[rxState]); - regs.RxDone = 0; + regs.RxDone = Regs::RxDone_Busy; regs.RxData = reg64; if (rxEnable) { rxState = rxFifoBlock; @@ -459,19 +475,16 @@ Device::write(MemReqPtr &req, const uint8_t *data) case Regs::TxData: if (txState != txIdle) - panic("transmit machine busy with another request!"); + panic("transmit machine busy with another request! txState=%s", + TxStateStrings[txState]); - regs.TxDone = 0; + regs.TxDone = Regs::TxDone_Busy; regs.TxData = reg64; if (txEnable) { txState = txFifoBlock; txKick(); } break; - - default: - panic("writing read only register %s: da=%#x pa=%#x va=%#x size=%d", - Regs::regName(daddr), daddr, req->paddr, req->vaddr, req->size); } return No_Fault; @@ -489,9 +502,25 @@ Device::devIntrPost(uint32_t interrupts) "interrupt written to intStatus: intr=%#x status=%#x mask=%#x\n", interrupts, regs.IntrStatus, regs.IntrMask); - if ((regs.IntrStatus & regs.IntrMask)) { + interrupts = regs.IntrStatus & regs.IntrMask; + + // Intr_RxHigh is special, we only signal it if we've emptied the fifo + // and then filled it above the high watermark + if (rxEmpty) + rxEmpty = false; + else + interrupts &= ~Regs::Intr_RxHigh; + + // Intr_TxLow is special, we only signal it if we've filled up the fifo + // and then dropped below the low watermark + if (txFull) + txFull = false; + else + interrupts &= ~Regs::Intr_TxLow; + + if (interrupts) { Tick when = curTick; - if ((regs.IntrStatus & regs.IntrMask & Regs::Intr_NoDelay) == 0) + if ((interrupts & Regs::Intr_NoDelay) == 0) when += intrDelay; cpuIntrPost(when); } @@ -627,12 +656,6 @@ Device::changeConfig(uint32_t newconf) regs.Config = newconf; - if ((changed & Regs::Config_Reset)) { - assert(regs.Config & Regs::Config_Reset); - reset(); - regs.Config &= ~Regs::Config_Reset; - } - if ((changed & Regs::Config_IntEn)) { cpuIntrEnable = regs.Config & Regs::Config_IntEn; if (cpuIntrEnable) { @@ -656,20 +679,40 @@ Device::changeConfig(uint32_t newconf) } } +void +Device::command(uint32_t command) +{ + if (command & Regs::Command_Reset) + reset(); +} + void Device::reset() { using namespace Regs; + memset(®s, 0, sizeof(regs)); + + regs.Config = 0; + if (params()->dedicated) + regs.Config |= Config_Thread; + regs.IntrMask = Intr_RxHigh | Intr_RxDMA | Intr_TxLow; regs.RxMaxCopy = params()->rx_max_copy; regs.TxMaxCopy = params()->tx_max_copy; - regs.IntrMask = Intr_TxFifo | Intr_RxFifo | Intr_RxData; + regs.RxMaxIntr = params()->rx_max_intr; + regs.RxFifoSize = params()->rx_fifo_size; + regs.TxFifoSize = params()->tx_fifo_size; + regs.RxFifoMark = params()->rx_fifo_threshold; + regs.TxFifoMark = params()->tx_fifo_threshold; + regs.HwAddr = params()->eaddr; rxState = rxIdle; txState = txIdle; rxFifo.clear(); txFifo.clear(); + rxEmpty = false; + txFull = false; } void @@ -680,13 +723,18 @@ Device::rxDmaCopy() physmem->dma_write(rxDmaAddr, (uint8_t *)rxDmaData, rxDmaLen); DPRINTF(EthernetDMA, "rx dma write paddr=%#x len=%d\n", rxDmaAddr, rxDmaLen); - DDUMP(EthernetDMA, rxDmaData, rxDmaLen); + DDUMP(EthernetData, rxDmaData, rxDmaLen); } void Device::rxDmaDone() { rxDmaCopy(); + + // If the transmit state machine has a pending DMA, let it go first + if (txState == txBeginCopy) + txKick(); + rxKick(); } @@ -706,6 +754,8 @@ Device::rxKick() switch (rxState) { case rxIdle: if (rxPioRequest) { + DPRINTF(EthernetPIO, "rxIdle: PIO waiting responding at %d\n", + curTick + pioLatency); pioInterface->respond(rxPioRequest, curTick); rxPioRequest = 0; } @@ -761,20 +811,20 @@ Device::rxKick() break; case rxBeginCopy: + if (dmaInterface && dmaInterface->busy()) + goto exit; + rxDmaAddr = plat->pciToDma(Regs::get_RxData_Addr(regs.RxData)); rxDmaLen = min(Regs::get_RxData_Len(regs.RxData), rxPktBytes); rxDmaData = rxPacketBufPtr; + rxState = rxCopy; if (dmaInterface) { - if (!dmaInterface->busy()) { - dmaInterface->doDMA(WriteInvalidate, rxDmaAddr, rxDmaLen, - curTick, &rxDmaEvent, true); - rxState = rxCopy; - } + dmaInterface->doDMA(WriteInvalidate, rxDmaAddr, rxDmaLen, + curTick, &rxDmaEvent, true); goto exit; } - rxState = rxCopy; if (dmaWriteDelay != 0 || dmaWriteFactor != 0) { Tick factor = ((rxDmaLen + ULL(63)) >> ULL(6)) * dmaWriteFactor; Tick start = curTick + dmaWriteDelay + factor; @@ -802,7 +852,7 @@ Device::rxKick() } regs.RxDone |= Regs::RxDone_Complete; - devIntrPost(Regs::Intr_RxData); + devIntrPost(Regs::Intr_RxDMA); rxState = rxIdle; break; @@ -831,13 +881,18 @@ Device::txDmaCopy() physmem->dma_read((uint8_t *)txDmaData, txDmaAddr, txDmaLen); DPRINTF(EthernetDMA, "tx dma read paddr=%#x len=%d\n", txDmaAddr, txDmaLen); - DDUMP(EthernetDMA, txDmaData, txDmaLen); + DDUMP(EthernetData, txDmaData, txDmaLen); } void Device::txDmaDone() { txDmaCopy(); + + // If the receive state machine has a pending DMA, let it go first + if (rxState == rxBeginCopy) + rxKick(); + txKick(); } @@ -849,6 +904,7 @@ Device::transmit() return; } + uint32_t interrupts; PacketPtr packet = txFifo.front(); if (!interface->sendPacket(packet)) { DPRINTF(Ethernet, "Packet Transmit: failed txFifo available %d\n", @@ -857,7 +913,6 @@ Device::transmit() } txFifo.pop(); - #if TRACING_ON if (DTRACE(Ethernet)) { IpPtr ip(packet); @@ -872,17 +927,17 @@ Device::transmit() } #endif - DDUMP(Ethernet, packet->data, packet->length); + DDUMP(EthernetData, packet->data, packet->length); txBytes += packet->length; txPackets++; DPRINTF(Ethernet, "Packet Transmit: successful txFifo Available %d\n", txFifo.avail()); - if (txFifo.size() <= params()->tx_fifo_threshold) - devIntrPost(Regs::Intr_TxFifo); - - devIntrPost(Regs::Intr_TxDone); + interrupts = Regs::Intr_TxPacket; + if (txFifo.size() < regs.TxFifoMark) + interrupts |= Regs::Intr_TxLow; + devIntrPost(interrupts); reschedule: if (!txFifo.empty() && !txEvent.scheduled()) { @@ -907,6 +962,8 @@ Device::txKick() switch (txState) { case txIdle: if (txPioRequest) { + DPRINTF(EthernetPIO, "txIdle: PIO waiting responding at %d\n", + curTick + pioLatency); pioInterface->respond(txPioRequest, curTick + pioLatency); txPioRequest = 0; } @@ -929,21 +986,20 @@ Device::txKick() break; case txBeginCopy: + if (dmaInterface && dmaInterface->busy()) + goto exit; + txDmaAddr = plat->pciToDma(Regs::get_TxData_Addr(regs.TxData)); txDmaLen = Regs::get_TxData_Len(regs.TxData); txDmaData = txPacketBufPtr; + txState = txCopy; if (dmaInterface) { - if (!dmaInterface->busy()) { - dmaInterface->doDMA(Read, txDmaAddr, txDmaLen, - curTick, &txDmaEvent, true); - txState = txCopy; - } - + dmaInterface->doDMA(Read, txDmaAddr, txDmaLen, + curTick, &txDmaEvent, true); goto exit; } - txState = txCopy; if (dmaReadDelay != 0 || dmaReadFactor != 0) { Tick factor = ((txDmaLen + ULL(63)) >> ULL(6)) * dmaReadFactor; Tick start = curTick + dmaReadDelay + factor; @@ -987,12 +1043,16 @@ Device::txKick() } } txFifo.push(txPacket); + if (txFifo.avail() < regs.TxMaxCopy) { + devIntrPost(Regs::Intr_TxFull); + txFull = true; + } txPacket = 0; transmit(); } regs.TxDone = txDmaLen | Regs::TxDone_Complete; - devIntrPost(Regs::Intr_TxData); + devIntrPost(Regs::Intr_TxDMA); txState = txIdle; break; @@ -1085,18 +1145,16 @@ Device::recvPacket(PacketPtr packet) if (!rxEnable) { DPRINTF(Ethernet, "receive disabled...packet dropped\n"); - interface->recvDone(); return true; } if (rxFilter(packet)) { DPRINTF(Ethernet, "packet filtered...dropped\n"); - interface->recvDone(); return true; } - if (rxFifo.size() >= params()->rx_fifo_threshold) - devIntrPost(Regs::Intr_RxFifo); + if (rxFifo.size() >= regs.RxFifoMark) + devIntrPost(Regs::Intr_RxHigh); if (!rxFifo.push(packet)) { DPRINTF(Ethernet, @@ -1104,8 +1162,7 @@ Device::recvPacket(PacketPtr packet) return false; } - interface->recvDone(); - devIntrPost(Regs::Intr_RxDone); + devIntrPost(Regs::Intr_RxPacket); rxKick(); return true; } @@ -1163,22 +1220,23 @@ Device::serialize(ostream &os) // Serialize the PciDev base class Base::serialize(os); - if (rxDmaEvent.scheduled()) - rxDmaCopy(); + if (rxState == rxCopy) + panic("can't serialize with an in flight dma request rxState=%s", + RxStateStrings[rxState]); - if (txDmaEvent.scheduled()) - txDmaCopy(); + if (txState == txCopy) + panic("can't serialize with an in flight dma request txState=%s", + TxStateStrings[txState]); /* * Serialize the device registers */ SERIALIZE_SCALAR(regs.Config); - SERIALIZE_SCALAR(regs.RxMaxCopy); - SERIALIZE_SCALAR(regs.TxMaxCopy); - SERIALIZE_SCALAR(regs.RxThreshold); - SERIALIZE_SCALAR(regs.TxThreshold); SERIALIZE_SCALAR(regs.IntrStatus); SERIALIZE_SCALAR(regs.IntrMask); + SERIALIZE_SCALAR(regs.RxMaxCopy); + SERIALIZE_SCALAR(regs.TxMaxCopy); + SERIALIZE_SCALAR(regs.RxMaxIntr); SERIALIZE_SCALAR(regs.RxData); SERIALIZE_SCALAR(regs.RxDone); SERIALIZE_SCALAR(regs.TxData); @@ -1189,6 +1247,7 @@ Device::serialize(ostream &os) */ int rxState = this->rxState; SERIALIZE_SCALAR(rxState); + SERIALIZE_SCALAR(rxEmpty); rxFifo.serialize("rxFifo", os); bool rxPacketExists = rxPacket; SERIALIZE_SCALAR(rxPacketExists); @@ -1205,6 +1264,7 @@ Device::serialize(ostream &os) */ int txState = this->txState; SERIALIZE_SCALAR(txState); + SERIALIZE_SCALAR(txFull); txFifo.serialize("txFifo", os); bool txPacketExists = txPacket; SERIALIZE_SCALAR(txPacketExists); @@ -1233,12 +1293,11 @@ Device::unserialize(Checkpoint *cp, const std::string §ion) * Unserialize the device registers */ UNSERIALIZE_SCALAR(regs.Config); - UNSERIALIZE_SCALAR(regs.RxMaxCopy); - UNSERIALIZE_SCALAR(regs.TxMaxCopy); - UNSERIALIZE_SCALAR(regs.RxThreshold); - UNSERIALIZE_SCALAR(regs.TxThreshold); UNSERIALIZE_SCALAR(regs.IntrStatus); UNSERIALIZE_SCALAR(regs.IntrMask); + UNSERIALIZE_SCALAR(regs.RxMaxCopy); + UNSERIALIZE_SCALAR(regs.TxMaxCopy); + UNSERIALIZE_SCALAR(regs.RxMaxIntr); UNSERIALIZE_SCALAR(regs.RxData); UNSERIALIZE_SCALAR(regs.RxDone); UNSERIALIZE_SCALAR(regs.TxData); @@ -1249,6 +1308,7 @@ Device::unserialize(Checkpoint *cp, const std::string §ion) */ int rxState; UNSERIALIZE_SCALAR(rxState); + UNSERIALIZE_SCALAR(rxEmpty); this->rxState = (RxState) rxState; rxFifo.unserialize("rxFifo", cp, section); bool rxPacketExists; @@ -1269,6 +1329,7 @@ Device::unserialize(Checkpoint *cp, const std::string §ion) */ int txState; UNSERIALIZE_SCALAR(txState); + UNSERIALIZE_SCALAR(txFull); this->txState = (TxState) txState; txFifo.unserialize("txFifo", cp, section); bool txPacketExists; @@ -1310,15 +1371,19 @@ Device::cacheAccess(MemReqPtr &req) Tick when = curTick + pioLatency; switch (daddr) { - case Regs::RxDone: + case Regs::RxWait: if (rxState != rxIdle) { + DPRINTF(EthernetPIO, "rxState=%s (not idle)... waiting\n", + TxStateStrings[txState]); rxPioRequest = req; when = 0; } break; - case Regs::TxDone: + case Regs::TxWait: if (txState != txIdle) { + DPRINTF(EthernetPIO, "txState=%s (not idle)... waiting\n", + TxStateStrings[txState]); txPioRequest = req; when = 0; } @@ -1387,6 +1452,7 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(Device) Param tx_delay; Param rx_max_copy; Param tx_max_copy; + Param rx_max_intr; Param rx_fifo_size; Param tx_fifo_size; Param rx_fifo_threshold; @@ -1394,6 +1460,7 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(Device) Param rx_filter; Param hardware_address; + Param dedicated; END_DECLARE_SIM_OBJECT_PARAMS(Device) @@ -1426,13 +1493,15 @@ BEGIN_INIT_SIM_OBJECT_PARAMS(Device) INIT_PARAM(tx_delay, "Transmit Delay"), INIT_PARAM(rx_max_copy, "rx max copy"), INIT_PARAM(tx_max_copy, "rx max copy"), + INIT_PARAM(rx_max_intr, "rx max intr"), INIT_PARAM(rx_fifo_size, "max size in bytes of rxFifo"), INIT_PARAM(tx_fifo_size, "max size in bytes of txFifo"), INIT_PARAM(rx_fifo_threshold, "max size in bytes of rxFifo"), INIT_PARAM(tx_fifo_threshold, "max size in bytes of txFifo"), INIT_PARAM(rx_filter, "Enable Receive Filter"), - INIT_PARAM(hardware_address, "Ethernet Hardware Address") + INIT_PARAM(hardware_address, "Ethernet Hardware Address"), + INIT_PARAM(dedicated, "dedicate a kernel thread to the driver") END_INIT_SIM_OBJECT_PARAMS(Device) @@ -1444,6 +1513,7 @@ CREATE_SIM_OBJECT(Device) params->name = getInstanceName(); params->clock = clock; + params->mmu = mmu; params->physmem = physmem; params->configSpace = configspace; @@ -1468,6 +1538,7 @@ CREATE_SIM_OBJECT(Device) params->rx_delay = rx_delay; params->rx_max_copy = rx_max_copy; params->tx_max_copy = tx_max_copy; + params->rx_max_intr = rx_max_intr; params->rx_fifo_size = rx_fifo_size; params->tx_fifo_size = tx_fifo_size; params->rx_fifo_threshold = rx_fifo_threshold; @@ -1475,6 +1546,7 @@ CREATE_SIM_OBJECT(Device) params->rx_filter = rx_filter; params->eaddr = hardware_address; + params->dedicated = dedicated; return new Device(params); } diff --git a/dev/sinic.hh b/dev/sinic.hh index 924c6eeeb..e01015061 100644 --- a/dev/sinic.hh +++ b/dev/sinic.hh @@ -115,19 +115,31 @@ class Device : public Base /** device register file */ struct { - uint32_t Config; - uint32_t RxMaxCopy; - uint32_t TxMaxCopy; - uint32_t RxThreshold; - uint32_t TxThreshold; - uint32_t IntrStatus; - uint32_t IntrMask; - uint64_t RxData; - uint64_t RxDone; - uint64_t TxData; - uint64_t TxDone; + uint32_t Config; // 0x00 + uint32_t Command; // 0x04 + uint32_t IntrStatus; // 0x08 + uint32_t IntrMask; // 0x0c + uint32_t RxMaxCopy; // 0x10 + uint32_t TxMaxCopy; // 0x14 + uint32_t RxMaxIntr; // 0x18 + uint32_t Reserved0; // 0x1c + uint32_t RxFifoSize; // 0x20 + uint32_t TxFifoSize; // 0x24 + uint32_t RxFifoMark; // 0x28 + uint32_t TxFifoMark; // 0x2c + uint64_t RxData; // 0x30 + uint64_t RxDone; // 0x38 + uint64_t RxWait; // 0x40 + uint64_t TxData; // 0x48 + uint64_t TxDone; // 0x50 + uint64_t TxWait; // 0x58 + uint64_t HwAddr; // 0x60 } regs; + uint8_t ®Data8(Addr daddr) { return *((uint8_t *)®s + daddr); } + uint32_t ®Data32(Addr daddr) { return *(uint32_t *)®Data8(daddr); } + uint64_t ®Data64(Addr daddr) { return *(uint64_t *)®Data8(daddr); } + private: Addr addr; static const Addr size = Regs::Size; @@ -135,6 +147,7 @@ class Device : public Base protected: RxState rxState; PacketFifo rxFifo; + bool rxEmpty; PacketPtr rxPacket; uint8_t *rxPacketBufPtr; int rxPktBytes; @@ -145,6 +158,7 @@ class Device : public Base TxState txState; PacketFifo txFifo; + bool txFull; PacketPtr txPacket; uint8_t *txPacketBufPtr; int txPktBytes; @@ -191,6 +205,7 @@ class Device : public Base * device configuration */ void changeConfig(uint32_t newconfig); + void command(uint32_t command); /** * device ethernet interface @@ -212,7 +227,7 @@ class Device : public Base void txDmaCopy(); void txDmaDone(); friend class EventWrapper; - EventWrapper txDmaEvent; + EventWrapper txDmaEvent; Tick dmaReadDelay; Tick dmaReadFactor; @@ -244,6 +259,8 @@ class Device : public Base * Memory Interface */ public: + void prepareRead(); + Fault iprRead(Addr daddr, uint64_t &result); virtual Fault read(MemReqPtr &req, uint8_t *data); virtual Fault write(MemReqPtr &req, const uint8_t *data); Tick cacheAccess(MemReqPtr &req); @@ -308,6 +325,7 @@ class Device : public Base Net::EthAddr eaddr; uint32_t rx_max_copy; uint32_t tx_max_copy; + uint32_t rx_max_intr; uint32_t rx_fifo_size; uint32_t tx_fifo_size; uint32_t rx_fifo_threshold; @@ -317,6 +335,7 @@ class Device : public Base Tick dma_write_delay; Tick dma_write_factor; bool dma_no_allocate; + bool dedicated; }; protected: diff --git a/dev/sinicreg.hh b/dev/sinicreg.hh index 78b175f71..12f545255 100644 --- a/dev/sinicreg.hh +++ b/dev/sinicreg.hh @@ -57,23 +57,28 @@ namespace Regs { // Registers __SINIC_REG32(Config, 0x00); // 32: configuration register -__SINIC_REG32(RxMaxCopy, 0x04); // 32: max rx copy -__SINIC_REG32(TxMaxCopy, 0x08); // 32: max tx copy -__SINIC_REG32(RxThreshold, 0x0c); // 32: receive fifo threshold -__SINIC_REG32(TxThreshold, 0x10); // 32: transmit fifo threshold -__SINIC_REG32(IntrStatus, 0x14); // 32: interrupt status -__SINIC_REG32(IntrMask, 0x18); // 32: interrupt mask -__SINIC_REG32(RxData, 0x20); // 64: receive data -__SINIC_REG32(RxDone, 0x28); // 64: receive done -__SINIC_REG32(RxWait, 0x30); // 64: receive done (busy wait) -__SINIC_REG32(TxData, 0x38); // 64: transmit data -__SINIC_REG32(TxDone, 0x40); // 64: transmit done -__SINIC_REG32(TxWait, 0x48); // 64: transmit done (busy wait) -__SINIC_REG32(HwAddr, 0x50); // 64: mac address -__SINIC_REG32(Size, 0x58); +__SINIC_REG32(Command, 0x04); // 32: command register +__SINIC_REG32(IntrStatus, 0x08); // 32: interrupt status +__SINIC_REG32(IntrMask, 0x0c); // 32: interrupt mask +__SINIC_REG32(RxMaxCopy, 0x10); // 32: max bytes per rx copy +__SINIC_REG32(TxMaxCopy, 0x14); // 32: max bytes per tx copy +__SINIC_REG32(RxMaxIntr, 0x18); // 32: max receives per interrupt +__SINIC_REG32(Reserved0, 0x1c); // 32: reserved +__SINIC_REG32(RxFifoSize, 0x20); // 32: rx fifo capacity in bytes +__SINIC_REG32(TxFifoSize, 0x24); // 32: tx fifo capacity in bytes +__SINIC_REG32(RxFifoMark, 0x28); // 32: rx fifo high watermark +__SINIC_REG32(TxFifoMark, 0x2c); // 32: tx fifo low watermark +__SINIC_REG32(RxData, 0x30); // 64: receive data +__SINIC_REG32(RxDone, 0x38); // 64: receive done +__SINIC_REG32(RxWait, 0x40); // 64: receive done (busy wait) +__SINIC_REG32(TxData, 0x48); // 64: transmit data +__SINIC_REG32(TxDone, 0x50); // 64: transmit done +__SINIC_REG32(TxWait, 0x58); // 64: transmit done (busy wait) +__SINIC_REG32(HwAddr, 0x60); // 64: mac address +__SINIC_REG32(Size, 0x68); // register addres space size // Config register bits -__SINIC_VAL32(Config_Reset, 31, 1); // reset chip +__SINIC_VAL32(Config_Thread, 8, 1); // enable receive filter __SINIC_VAL32(Config_Filter, 7, 1); // enable receive filter __SINIC_VAL32(Config_Vlan, 6, 1); // enable vlan tagging __SINIC_VAL32(Config_Virtual, 5, 1); // enable virtual addressing @@ -83,105 +88,103 @@ __SINIC_VAL32(Config_IntEn, 2, 1); // enable interrupts __SINIC_VAL32(Config_TxEn, 1, 1); // enable transmit __SINIC_VAL32(Config_RxEn, 0, 1); // enable receive +// Command register bits +__SINIC_VAL32(Command_Reset, 0, 1); // reset chip + // Interrupt register bits -__SINIC_VAL32(Intr_TxFifo, 5, 1); // Fifo oflow/uflow/threshold -__SINIC_VAL32(Intr_TxData, 4, 1); // DMA Completed w/ interrupt -__SINIC_VAL32(Intr_TxDone, 3, 1); // Packet transmitted -__SINIC_VAL32(Intr_RxFifo, 2, 1); // Fifo oflow/uflow/threshold -__SINIC_VAL32(Intr_RxData, 1, 1); // DMA Completed w/ interrupt -__SINIC_VAL32(Intr_RxDone, 0, 1); // Packet received -__SINIC_REG32(Intr_All, 0x3f); -__SINIC_REG32(Intr_NoDelay, 0x24); -__SINIC_REG32(Intr_Res, ~0x3f); +__SINIC_VAL32(Intr_TxLow, 7, 1); // tx fifo dropped below watermark +__SINIC_VAL32(Intr_TxFull, 6, 1); // tx fifo full +__SINIC_VAL32(Intr_TxDMA, 5, 1); // tx dma completed w/ interrupt +__SINIC_VAL32(Intr_TxPacket, 4, 1); // packet transmitted +__SINIC_VAL32(Intr_RxHigh, 3, 1); // rx fifo above high watermark +__SINIC_VAL32(Intr_RxEmpty, 2, 1); // rx fifo empty +__SINIC_VAL32(Intr_RxDMA, 1, 1); // rx dma completed w/ interrupt +__SINIC_VAL32(Intr_RxPacket, 0, 1); // packet received +__SINIC_REG32(Intr_All, 0xff); // all valid interrupts +__SINIC_REG32(Intr_NoDelay, 0xcc); // interrupts that shouldn't be coalesced +__SINIC_REG32(Intr_Res, ~0xff); // reserved interrupt bits // RX Data Description __SINIC_VAL64(RxData_Len, 40, 20); // 0 - 1M __SINIC_VAL64(RxData_Addr, 0, 40); // Address 1TB // TX Data Description -__SINIC_VAL64(TxData_More, 63, 1); -__SINIC_VAL64(TxData_Checksum, 62, 1); +__SINIC_VAL64(TxData_More, 63, 1); // Packet not complete (will dma more) +__SINIC_VAL64(TxData_Checksum, 62, 1); // do checksum __SINIC_VAL64(TxData_Len, 40, 20); // 0 - 1M __SINIC_VAL64(TxData_Addr, 0, 40); // Address 1TB // RX Done/Busy Information -__SINIC_VAL64(RxDone_Complete, 63, 1); -__SINIC_VAL64(RxDone_IpPacket, 45, 1); -__SINIC_VAL64(RxDone_TcpPacket, 44, 1); -__SINIC_VAL64(RxDone_UdpPacket, 43, 1); -__SINIC_VAL64(RxDone_IpError, 42, 1); -__SINIC_VAL64(RxDone_TcpError, 41, 1); -__SINIC_VAL64(RxDone_UdpError, 40, 1); -__SINIC_VAL64(RxDone_More, 32, 1); -__SINIC_VAL64(RxDone_FifoLen, 20, 8); // up to 255 packets +__SINIC_VAL64(RxDone_Packets, 32, 16); // number of packets in rx fifo +__SINIC_VAL64(RxDone_Busy, 31, 1); // receive dma busy copying +__SINIC_VAL64(RxDone_Complete, 30, 1); // valid data (packet complete) +__SINIC_VAL64(RxDone_More, 29, 1); // Packet has more data (dma again) +__SINIC_VAL64(RxDone_TcpError, 25, 1); // TCP packet error (bad checksum) +__SINIC_VAL64(RxDone_UdpError, 24, 1); // UDP packet error (bad checksum) +__SINIC_VAL64(RxDone_IpError, 23, 1); // IP packet error (bad checksum) +__SINIC_VAL64(RxDone_TcpPacket, 22, 1); // this is a TCP packet +__SINIC_VAL64(RxDone_UdpPacket, 21, 1); // this is a UDP packet +__SINIC_VAL64(RxDone_IpPacket, 20, 1); // this is an IP packet __SINIC_VAL64(RxDone_CopyLen, 0, 20); // up to 256k // TX Done/Busy Information -__SINIC_VAL64(TxDone_Complete, 63, 1); -__SINIC_VAL64(TxDone_FifoLen, 20, 8); // up to 255 packets -__SINIC_VAL64(TxDone_CopyLen, 0, 20); // up to 256k +__SINIC_VAL64(TxDone_Packets, 32, 16); // number of packets in tx fifo +__SINIC_VAL64(TxDone_Busy, 31, 1); // transmit dma busy copying +__SINIC_VAL64(TxDone_Complete, 30, 1); // valid data (packet complete) +__SINIC_VAL64(TxDone_Full, 29, 1); // tx fifo is full +__SINIC_VAL64(TxDone_Low, 28, 1); // tx fifo is below the watermark +__SINIC_VAL64(TxDone_CopyLen, 0, 20); // up to 256k -inline int -regSize(int offset) +struct Info { - static const char sizes[] = { - 4, - 4, - 4, - 4, - 4, - 4, - 4, - 0, - 8, 0, - 8, 0, - 8, 0, - 8, 0, - 8, 0, - 8, 0, - 8, 0 - }; - - if (offset & 0x3) - return 0; - - if (offset >= Size) - return 0; - - return sizes[offset / 4]; -} - -inline const char * -regName(int offset) -{ - static const char *names[] = { - "Config", - "RxMaxCopy", - "TxMaxCopy", - "RxThreshold", - "TxThreshold", - "IntrStatus", - "IntrMask", - "invalid", - "RxData", "invalid", - "RxDone", "invalid", - "RxWait", "invalid", - "TxData", "invalid", - "TxDone", "invalid", - "TxWait", "invalid", - "HwAddr", "invalid" - }; - - if (offset & 0x3) - return "invalid"; - - if (offset >= Size) - return "invalid"; - - return names[offset / 4]; -} + uint8_t size; + bool read; + bool write; + const char *name; +}; /* namespace Regs */ } + +inline const Regs::Info& +regInfo(Addr daddr) +{ + static Regs::Info info [] = { + { 4, true, true, "Config" }, + { 4, false, true, "Command" }, + { 4, true, true, "IntrStatus" }, + { 4, true, true, "IntrMask" }, + { 4, true, false, "RxMaxCopy" }, + { 4, true, false, "TxMaxCopy" }, + { 4, true, false, "RxMaxIntr" }, + { 0, false, false, "invalid" }, + { 4, true, false, "RxFifoSize" }, + { 4, true, false, "TxFifoSize" }, + { 4, true, false, "RxFifoMark" }, + { 4, true, false, "TxFifoMark" }, + { 8, true, true, "RxData" }, { 0, false, false, "invalid" }, + { 8, true, false, "RxDone" }, { 0, false, false, "invalid" }, + { 8, true, false, "RxWait" }, { 0, false, false, "invalid" }, + { 8, true, true, "TxData" }, { 0, false, false, "invalid" }, + { 8, true, false, "TxDone" }, { 0, false, false, "invalid" }, + { 8, true, false, "TxWait" }, { 0, false, false, "invalid" }, + { 8, true, false, "HwAddr" }, { 0, false, false, "invalid" } + }; + + return info[daddr / 4]; +} + +inline bool +regValid(Addr daddr) +{ + if (daddr > Regs::Size) + return false; + + if (regInfo(daddr).size == 0) + return false; + + return true; +} + /* namespace Sinic */ } #endif // __DEV_SINICREG_HH__ diff --git a/python/m5/config.py b/python/m5/config.py index 3c49421d3..e1970e672 100644 --- a/python/m5/config.py +++ b/python/m5/config.py @@ -755,7 +755,7 @@ class ParamDesc(object): class VectorParamValue(list): def ini_str(self): - return ' '.join([str(v) for v in self]) + return ' '.join([v.ini_str() for v in self]) def unproxy(self, base): return [v.unproxy(base) for v in self] @@ -825,6 +825,40 @@ VectorParam = ParamFactory(VectorParamDesc) # ##################################################################### +# superclass for "numeric" parameter values, to emulate math +# operations in a type-safe way. e.g., a Latency times an int returns +# a new Latency object. +class NumericParamValue(ParamValue): + def __str__(self): + return str(self.value) + + def __float__(self): + return float(self.value) + + # hook for bounds checking + def _check(self): + return + + def __mul__(self, other): + newobj = self.__class__(self) + newobj.value *= other + newobj._check() + return newobj + + __rmul__ = __mul__ + + def __div__(self, other): + newobj = self.__class__(self) + newobj.value /= other + newobj._check() + return newobj + + def __sub__(self, other): + newobj = self.__class__(self) + newobj.value -= other + newobj._check() + return newobj + class Range(ParamValue): type = int # default; can be overridden in subclasses def __init__(self, *args, **kwargs): @@ -891,19 +925,20 @@ class CheckedIntType(type): # class is subclassed to generate parameter classes with specific # bounds. Initialization of the min and max bounds is done in the # metaclass CheckedIntType.__init__. -class CheckedInt(long,ParamValue): +class CheckedInt(NumericParamValue): __metaclass__ = CheckedIntType - def __new__(cls, value): - if isinstance(value, str): - value = toInteger(value) - - self = long.__new__(cls, value) - - if not cls.min <= self <= cls.max: + def _check(self): + if not self.min <= self.value <= self.max: raise TypeError, 'Integer param out of bounds %d < %d < %d' % \ - (cls.min, self, cls.max) - return self + (self.min, self.value, self.max) + + def __init__(self, value): + if isinstance(value, str): + self.value = toInteger(value) + elif isinstance(value, (int, long)): + self.value = long(value) + self._check() class Int(CheckedInt): size = 32; unsigned = False class Unsigned(CheckedInt): size = 32; unsigned = True @@ -930,19 +965,26 @@ class Float(ParamValue, float): class MemorySize(CheckedInt): size = 64 unsigned = True - def __new__(cls, value): - return super(MemorySize, cls).__new__(cls, toMemorySize(value)) + def __init__(self, value): + if isinstance(value, MemorySize): + self.value = value.value + else: + self.value = toMemorySize(value) + self._check() class Addr(CheckedInt): size = 64 unsigned = True - def __new__(cls, value): - try: - value = long(toMemorySize(value)) - except TypeError: - value = long(value) - return super(Addr, cls).__new__(cls, value) + def __init__(self, value): + if isinstance(value, Addr): + self.value = value.value + else: + try: + self.value = toMemorySize(value) + except TypeError: + self.value = long(value) + self._check() class AddrRange(Range): type = Addr @@ -1123,29 +1165,6 @@ def tick_check(float_ticks): #raise ValueError return int_ticks -# superclass for "numeric" parameter values, to emulate math -# operations in a type-safe way. e.g., a Latency times an int returns -# a new Latency object. -class NumericParamValue(ParamValue): - def __str__(self): - return str(self.value) - - def __float__(self): - return float(self.value) - - def __mul__(self, other): - newobj = self.__class__(self) - newobj.value *= other - return newobj - - __rmul__ = __mul__ - - def __div__(self, other): - newobj = self.__class__(self) - newobj.value /= other - return newobj - - def getLatency(value): if isinstance(value, Latency) or isinstance(value, Clock): return value.value diff --git a/python/m5/convert.py b/python/m5/convert.py index 9d9f4efa7..73181e985 100644 --- a/python/m5/convert.py +++ b/python/m5/convert.py @@ -89,17 +89,9 @@ def toFloat(value): else: return float(value) -def toLong(value): - value = toFloat(value) - result = int(value) - if value != result: - raise ValueError, "cannot convert '%s' to long" % value - - return result - def toInteger(value): value = toFloat(value) - result = int(value) + result = long(value) if value != result: raise ValueError, "cannot convert '%s' to integer" % value @@ -220,16 +212,16 @@ def toMemorySize(value): raise TypeError, "wrong type '%s' should be str" % type(value) if value.endswith('PB'): - return float(value[:-2]) * pebi + return long(value[:-2]) * pebi elif value.endswith('TB'): - return float(value[:-2]) * tebi + return long(value[:-2]) * tebi elif value.endswith('GB'): - return float(value[:-2]) * gibi + return long(value[:-2]) * gibi elif value.endswith('MB'): - return float(value[:-2]) * mebi + return long(value[:-2]) * mebi elif value.endswith('kB'): - return float(value[:-2]) * kibi + return long(value[:-2]) * kibi elif value.endswith('B'): - return float(value[:-1]) + return long(value[:-1]) raise ValueError, "cannot convert '%s' to memory size" % value diff --git a/python/m5/objects/Ethernet.py b/python/m5/objects/Ethernet.py index 3ad6b57f9..a2a8b217f 100644 --- a/python/m5/objects/Ethernet.py +++ b/python/m5/objects/Ethernet.py @@ -84,6 +84,7 @@ class EtherDevBase(PciDevice): rx_filter = Param.Bool(True, "Enable Receive Filter") intr_delay = Param.Latency('10us', "Interrupt Propagation Delay") + dedicated = Param.Bool(False, "dedicate a kernel thread to the driver") class NSGigE(EtherDevBase): type = 'NSGigE' @@ -91,7 +92,6 @@ class NSGigE(EtherDevBase): dma_data_free = Param.Bool(False, "DMA of Data is free") dma_desc_free = Param.Bool(False, "DMA of Descriptors is free") - dedicated = Param.Bool(False, "dedicate a kernel thread to the driver") class NSGigEInt(EtherInt): type = 'NSGigEInt' @@ -102,6 +102,7 @@ class Sinic(EtherDevBase): rx_max_copy = Param.MemorySize('1514B', "rx max copy") tx_max_copy = Param.MemorySize('16kB', "tx max copy") + rx_max_intr = Param.UInt32(10, "max rx packets per interrupt") rx_fifo_threshold = Param.MemorySize('48kB', "rx fifo high threshold") tx_fifo_threshold = Param.MemorySize('16kB', "tx fifo low threshold") diff --git a/test/genini.py b/test/genini.py index ac789e5d6..2af81fe2b 100755 --- a/test/genini.py +++ b/test/genini.py @@ -64,8 +64,7 @@ for path in pathlist: AddToPath(path) for arg in args: - AddToPath(os.path.dirname(arg)) - execfile(arg) + m5execfile(arg, globals()) if globals().has_key('root') and isinstance(root, Root): instantiate(root) diff --git a/util/pbs/pbs.py b/util/pbs/pbs.py index 292ddf1f9..21c2cb89d 100755 --- a/util/pbs/pbs.py +++ b/util/pbs/pbs.py @@ -147,6 +147,8 @@ class qsub: flags.append('e') if len(flags): self.cmd.append('-m ' + flags) + else: + self.cmd.append('-mn') if self.name: self.cmd.append("-N%s" % self.name) @@ -158,7 +160,7 @@ class qsub: self.cmd.append('-q' + self.queue) if self.afterok: - self.cmd.append('-Wdepend=afterok:%s' % self.after) + self.cmd.append('-Wdepend=afterok:%s' % self.afterok) self.cmd.extend(args) self.script = script diff --git a/util/pbs/send.py b/util/pbs/send.py index 1f48b2b0e..54bba931d 100755 --- a/util/pbs/send.py +++ b/util/pbs/send.py @@ -98,7 +98,7 @@ Usage: try: import getopt - opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lq:Rt:v') + opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v') except getopt.GetoptError: sys.exit(usage) @@ -115,6 +115,7 @@ docpts = False doruns = True runflag = False node_type = 'FAST' +update = True for opt,arg in opts: if opt == '-C': @@ -134,6 +135,8 @@ for opt,arg in opts: jfile = arg if opt == '-l': listonly = True + if opt == '-n': + update = False if opt == '-q': queue = arg if opt == '-R': @@ -154,7 +157,7 @@ from job import JobDir, date conf = jobfile.JobFile(jfile) -if not listonly and not onlyecho and isdir(conf.linkdir): +if update and not listonly and not onlyecho and isdir(conf.linkdir): if verbose: print 'Checking for outdated files in Link directory' if not isdir(conf.basedir): diff --git a/util/qdo b/util/qdo index 3a475b420..339d944a7 100755 --- a/util/qdo +++ b/util/qdo @@ -45,7 +45,11 @@ optparser.add_option('-o', dest='stdout_file', help='command stdout output file') optparser.add_option('-l', dest='save_log', action='store_true', help='save qsub output log file') -optparser.add_option('-q', dest='qsub_timeout', type='int', +optparser.add_option('-N', dest='job_name', + help='qsub job name') +optparser.add_option('-q', dest='dest_queue', + help='qsub destination queue') +optparser.add_option('--qwait', dest='qsub_timeout', type='int', help='qsub queue wait timeout', default=30*60) optparser.add_option('-t', dest='cmd_timeout', type='int', help='command execution timeout', default=600*60) @@ -56,6 +60,12 @@ if cmd == []: print >>sys.stderr, "%s: missing command" % progname sys.exit(1) +# If we want to do this, need to add check here to make sure cmd[0] is +# a valid PBS job name, else qsub will die on us. +# +#if not options.job_name: +# options.job_name = cmd[0] + cwd = os.getcwd() # Deal with systems where /n is a symlink to /.automount @@ -137,6 +147,10 @@ if False and len(cmd) > 50: print "%s: running %s on poolfs" % (progname, cmd[0]) else: shell_cmd = 'qsub -I -S /bin/sh' + if options.job_name: + shell_cmd += ' -N "%s"' % options.job_name + if options.dest_queue: + shell_cmd += ' -q ' + options.dest_queue shell = Shell(shell_cmd) diff --git a/util/stats/db.py b/util/stats/db.py index ab005e67b..1ece6df88 100644 --- a/util/stats/db.py +++ b/util/stats/db.py @@ -103,6 +103,20 @@ class Node(object): def __str__(self): return self.name +class Result(object): + def __init__(self, x, y): + self.data = {} + self.x = x + self.y = y + + def __contains__(self, run): + return run in self.data + + def __getitem__(self, run): + if run not in self.data: + self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ] + return self.data[run] + class Database(object): def __init__(self): self.host = 'zizzer.pool' @@ -135,7 +149,23 @@ class Database(object): self.runs = None self.bins = None self.ticks = None - self.__dict__['get'] = type(self).sum + self.method = 'sum' + self._method = type(self).sum + + def get(self, job, stat): + run = self.allRunNames.get(job.name, None) + if run is None: + print 'run "%s" not found' % job + return None + + from info import scalar, vector, value, values, total, len + stat.system = self[job.system] + if scalar(stat): + return value(stat, run.run) + if vector(stat): + return values(stat, run.run) + + return None def query(self, sql): self.cursor.execute(sql) @@ -203,7 +233,7 @@ class Database(object): self.query('select * from stats') import info for result in self.cursor.fetchall(): - stat = info.NewStat(StatData(result)) + stat = info.NewStat(self, StatData(result)) self.append(stat) self.allStats.append(stat) self.allStatIds[stat.stat] = stat @@ -421,30 +451,17 @@ class Database(object): def stdev(self, stat, bins, ticks): return self.outer('stddev', 'sum', stat, bins, ticks) - def __getattribute__(self, attr): - if attr != 'get': - return super(Database, self).__getattribute__(attr) - - if self.__dict__['get'] == type(self).sum: - return 'sum' - elif self.__dict__['get'] == type(self).avg: - return 'avg' - elif self.__dict__['get'] == type(self).stdev: - return 'stdev' - else: - return '' - def __setattr__(self, attr, value): - if attr != 'get': - super(Database, self).__setattr__(attr, value) + super(Database, self).__setattr__(attr, value) + if attr != 'method': return if value == 'sum': - self.__dict__['get'] = type(self).sum + self._method = self.sum elif value == 'avg': - self.__dict__['get'] = type(self).avg + self._method = self.avg elif value == 'stdev': - self.__dict__['get'] = type(self).stdev + self._method = self.stdev else: raise AttributeError, "can only set get to: sum | avg | stdev" @@ -453,10 +470,12 @@ class Database(object): bins = self.bins if ticks is None: ticks = self.ticks - sql = self.__dict__['get'](self, stat, bins, ticks) + sql = self._method(self, stat, bins, ticks) self.query(sql) runs = {} + xmax = 0 + ymax = 0 for x in self.cursor.fetchall(): data = Data(x) if not runs.has_key(data.run): @@ -464,8 +483,17 @@ class Database(object): if not runs[data.run].has_key(data.x): runs[data.run][data.x] = {} + xmax = max(xmax, data.x) + ymax = max(ymax, data.y) runs[data.run][data.x][data.y] = data.data - return runs + + results = Result(xmax + 1, ymax + 1) + for run,data in runs.iteritems(): + result = results[run] + for x,ydata in data.iteritems(): + for y,data in ydata.iteritems(): + result[x][y] = data + return results def __getitem__(self, key): return self.stattop[key] diff --git a/util/stats/display.py b/util/stats/display.py index 629684ca4..fbcff5c70 100644 --- a/util/stats/display.py +++ b/util/stats/display.py @@ -26,7 +26,7 @@ class Value: def __init__(self, value, precision, percent = False): - self.value = value + self.value = float(value) self.precision = precision self.percent = percent def __str__(self): @@ -90,61 +90,60 @@ class Print: class VectorDisplay: def display(self): + if not self.value: + return + p = Print() p.flags = self.flags p.precision = self.precision - if isinstance(self.value, (list, tuple)): - if not len(self.value): - return - - mytotal = reduce(lambda x,y: float(x) + float(y), self.value) - mycdf = 0.0 - - value = self.value - - if display_all: - subnames = [ '[%d]' % i for i in range(len(value)) ] - else: - subnames = [''] * len(value) - - if self.__dict__.has_key('subnames'): - for i,each in enumerate(self.subnames): - if len(each) > 0: - subnames[i] = '.%s' % each - - subdescs = [self.desc]*len(value) - if self.__dict__.has_key('subdescs'): - for i in xrange(min(len(value), len(self.subdescs))): - subdescs[i] = self.subdescs[i] - - for val,sname,sdesc in map(None, value, subnames, subdescs): - if mytotal > 0.0: - mypdf = float(val) / float(mytotal) - mycdf += mypdf - if (self.flags & flags_pdf): - p.pdf = mypdf - p.cdf = mycdf - - if len(sname) == 0: - continue - - p.name = self.name + sname - p.desc = sdesc - p.value = val - p.display() - - if (self.flags & flags_total): - if (p.__dict__.has_key('pdf')): del p.__dict__['pdf'] - if (p.__dict__.has_key('cdf')): del p.__dict__['cdf'] - p.name = self.name + '.total' - p.desc = self.desc - p.value = mytotal - p.display() - - else: + if not isinstance(self.value, (list, tuple)): p.name = self.name p.desc = self.desc p.value = self.value p.display() + return + mytotal = reduce(lambda x,y: float(x) + float(y), self.value) + mycdf = 0.0 + + value = self.value + + if display_all: + subnames = [ '[%d]' % i for i in range(len(value)) ] + else: + subnames = [''] * len(value) + + if self.__dict__.has_key('subnames'): + for i,each in enumerate(self.subnames): + if len(each) > 0: + subnames[i] = '.%s' % each + + subdescs = [self.desc]*len(value) + if self.__dict__.has_key('subdescs'): + for i in xrange(min(len(value), len(self.subdescs))): + subdescs[i] = self.subdescs[i] + + for val,sname,sdesc in map(None, value, subnames, subdescs): + if mytotal > 0.0: + mypdf = float(val) / float(mytotal) + mycdf += mypdf + if (self.flags & flags_pdf): + p.pdf = mypdf + p.cdf = mycdf + + if len(sname) == 0: + continue + + p.name = self.name + sname + p.desc = sdesc + p.value = val + p.display() + + if (self.flags & flags_total): + if (p.__dict__.has_key('pdf')): del p.__dict__['pdf'] + if (p.__dict__.has_key('cdf')): del p.__dict__['cdf'] + p.name = self.name + '.total' + p.desc = self.desc + p.value = mytotal + p.display() diff --git a/util/stats/info.py b/util/stats/info.py index ae5d3211f..889af6d53 100644 --- a/util/stats/info.py +++ b/util/stats/info.py @@ -27,391 +27,347 @@ from __future__ import division import operator, re, types -source = None -display_run = 0 -global globalTicks -globalTicks = None +def unproxy(proxy): + if hasattr(proxy, '__unproxy__'): + return proxy.__unproxy__() -def total(f): - if isinstance(f, FormulaStat): - v = f.value - else: - v = f + return proxy - f = FormulaStat() - if isinstance(v, (list, tuple)): - f.value = reduce(operator.add, v) - else: - f.value = v +def scalar(stat): + stat = unproxy(stat) + assert(stat.__scalar__() != stat.__vector__()) + return stat.__scalar__() - return f +def vector(stat): + stat = unproxy(stat) + assert(stat.__scalar__() != stat.__vector__()) + return stat.__vector__() -def unaryop(op, f): - if isinstance(f, FormulaStat): - v = f.value - else: - v = f - - if isinstance(v, (list, tuple)): - return map(op, v) - else: - return op(v) - -def zerodiv(lv, rv): - if rv == 0.0: - return 0.0 - else: - return operator.truediv(lv, rv) - -def wrapop(op, lv, rv): - if isinstance(lv, str): - return lv - - if isinstance(rv, str): - return rv - - return op(lv, rv) - -def same(lrun, rrun): - for lx,rx in zip(lrun.keys(),rrun.keys()): - if lx != rx: - print 'lx != rx' - print lx, rx - print lrun.keys() - print rrun.keys() - return False - for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()): - if ly != ry: - print 'ly != ry' - print ly, ry - print lrun[lx].keys() - print rrun[rx].keys() - return False - return True - - -def binaryop(op, lf, rf): - result = {} - - if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat): - lv = lf.value - rv = rf.value - - theruns = [] - for r in lv.keys(): - if rv.has_key(r): - if same(lv[r], rv[r]): - theruns.append(r) - else: - raise AttributeError - - for run in theruns: - result[run] = {} - for x in lv[run].keys(): - result[run][x] = {} - for y in lv[run][x].keys(): - result[run][x][y] = wrapop(op, lv[run][x][y], - rv[run][x][y]) - elif isinstance(lf, FormulaStat): - lv = lf.value - for run in lv.keys(): - result[run] = {} - for x in lv[run].keys(): - result[run][x] = {} - for y in lv[run][x].keys(): - result[run][x][y] = wrapop(op, lv[run][x][y], rf) - elif isinstance(rf, FormulaStat): - rv = rf.value - for run in rv.keys(): - result[run] = {} - for x in rv[run].keys(): - result[run][x] = {} - for y in rv[run][x].keys(): - result[run][x][y] = wrapop(op, lf, rv[run][x][y]) +def value(stat, *args): + stat = unproxy(stat) + return stat.__value__(*args) +def values(stat, run): + stat = unproxy(stat) + result = [] + for i in xrange(len(stat)): + val = value(stat, run, i) + if val is None: + return None + result.append(val) return result -def sums(x, y): - if isinstance(x, (list, tuple)): - return map(lambda x, y: x + y, x, y) - else: - return x + y +def total(stat, run): + return sum(values(stat, run)) -def alltrue(seq): - return reduce(lambda x, y: x and y, seq) +def len(stat): + stat = unproxy(stat) + return stat.__len__() -def allfalse(seq): - return not reduce(lambda x, y: x or y, seq) +class Value(object): + def __scalar__(self): + raise AttributeError, "must define __scalar__ for %s" % (type (self)) + def __vector__(self): + raise AttributeError, "must define __vector__ for %s" % (type (self)) -def enumerate(seq): - return map(None, range(len(seq)), seq) + def __add__(self, other): + return BinaryProxy(operator.__add__, self, other) + def __sub__(self, other): + return BinaryProxy(operator.__sub__, self, other) + def __mul__(self, other): + return BinaryProxy(operator.__mul__, self, other) + def __div__(self, other): + return BinaryProxy(operator.__div__, self, other) + def __truediv__(self, other): + return BinaryProxy(operator.__truediv__, self, other) + def __floordiv__(self, other): + return BinaryProxy(operator.__floordiv__, self, other) -def cmp(a, b): - if a < b: - return -1 - elif a == b: - return 0 - else: - return 1 + def __radd__(self, other): + return BinaryProxy(operator.__add__, other, self) + def __rsub__(self, other): + return BinaryProxy(operator.__sub__, other, self) + def __rmul__(self, other): + return BinaryProxy(operator.__mul__, other, self) + def __rdiv__(self, other): + return BinaryProxy(operator.__div__, other, self) + def __rtruediv__(self, other): + return BinaryProxy(operator.__truediv__, other, self) + def __rfloordiv__(self, other): + return BinaryProxy(operator.__floordiv__, other, self) -class Statistic(object): + def __neg__(self): + return UnaryProxy(operator.__neg__, self) + def __pos__(self): + return UnaryProxy(operator.__pos__, self) + def __abs__(self): + return UnaryProxy(operator.__abs__, self) - def __init__(self, data): - self.__dict__.update(data.__dict__) - if not self.__dict__.has_key('value'): - self.__dict__['value'] = None - if not self.__dict__.has_key('bins'): - self.__dict__['bins'] = None - if not self.__dict__.has_key('ticks'): - self.__dict__['ticks'] = None - if 'vc' not in self.__dict__: - self.vc = {} +class Scalar(Value): + def __scalar__(self): + return True - def __getattribute__(self, attr): - if attr == 'ticks': - if self.__dict__['ticks'] != globalTicks: - self.__dict__['value'] = None - self.__dict__['ticks'] = globalTicks - return self.__dict__['ticks'] - if attr == 'value': - if self.__dict__['ticks'] != globalTicks: - if self.__dict__['ticks'] != None and \ - len(self.__dict__['ticks']) == 1: - self.vc[self.__dict__['ticks'][0]] = self.__dict__['value'] - self.__dict__['ticks'] = globalTicks - if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]): - self.__dict__['value'] = self.vc[globalTicks[0]] - else: - self.__dict__['value'] = None - if self.__dict__['value'] == None: - self.__dict__['value'] = self.getValue() - return self.__dict__['value'] - else: - return super(Statistic, self).__getattribute__(attr) - - def __setattr__(self, attr, value): - if attr == 'bins' or attr == 'ticks': - if attr == 'bins': - if value is not None: - value = source.getBin(value) - #elif attr == 'ticks' and type(value) is str: - # value = [ int(x) for x in value.split() ] - - self.__dict__[attr] = value - self.__dict__['value'] = None - self.vc = {} - else: - super(Statistic, self).__setattr__(attr, value) - - def getValue(self): - raise AttributeError, 'getValue() must be defined' - - def zero(self): + def __vector__(self): return False - def __ne__(self, other): - return not (self == other) + def __value__(self, run): + raise AttributeError, '__value__ must be defined' - def __str__(self): - return '%f' % (float(self)) +class VectorItemProxy(Value): + def __init__(self, proxy, index): + self.proxy = proxy + self.index = index -class FormulaStat(object): - def __add__(self, other): - f = FormulaStat() - f.value = binaryop(operator.add, self, other) - return f - def __sub__(self, other): - f = FormulaStat() - f.value = binaryop(operator.sub, self, other) - return f - def __mul__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mul, self, other) - return f - def __truediv__(self, other): - f = FormulaStat() - f.value = binaryop(zerodiv, self, other) - return f - def __mod__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mod, self, other) - return f - def __radd__(self, other): - f = FormulaStat() - f.value = binaryop(operator.add, other, self) - return f - def __rsub__(self, other): - f = FormulaStat() - f.value = binaryop(operator.sub, other, self) - return f - def __rmul__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mul, other, self) - return f - def __rtruediv__(self, other): - f = FormulaStat() - f.value = binaryop(zerodiv, other, self) - return f - def __rmod__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mod, other, self) - return f - def __neg__(self): - f = FormulaStat() - f.value = unaryop(operator.neg, self) - return f - def __getitem__(self, idx): - f = FormulaStat() - f.value = {} - for key in self.value.keys(): - f.value[key] = {} - f.value[key][0] = {} - f.value[key][0][0] = self.value[key][idx][0] - return f + def __scalar__(self): + return True - def __float__(self): - if isinstance(self.value, FormulaStat): - return float(self.value) - if not self.value.has_key(display_run): - return (1e300*1e300) - if len(self.value[display_run]) == 1: - return self.value[display_run][0][0] - else: - #print self.value[display_run] - return self.value[display_run][4][0] - #raise ValueError + def __vector__(self): + return False - def display(self): - import display - d = display.VectorDisplay() - d.flags = 0 - d.precision = 1 - d.name = 'formula' - d.desc = 'formula' - val = self.value[display_run] - d.value = [ val[x][0] for x in val.keys() ] - d.display() + def __value__(self, run): + return value(self.proxy, run, self.index) +class Vector(Value): + def __scalar__(self): + return False -class Scalar(Statistic,FormulaStat): - def getValue(self): - return source.data(self, self.bins, self.ticks) + def __vector__(self): + return True - def display(self): + def __value__(self, run, index): + raise AttributeError, '__value__ must be defined' + + def __getitem__(self, index): + return VectorItemProxy(self, index) + +class ScalarConstant(Scalar): + def __init__(self, constant): + self.constant = constant + def __value__(self, run): + return self.constant + +class VectorConstant(Vector): + def __init__(self, constant): + self.constant = constant + def __value__(self, run, index): + return self.constant[index] + def __len__(self): + return len(self.constant) + +def WrapValue(value): + if isinstance(value, (int, long, float)): + return ScalarConstant(value) + if isinstance(value, (list, tuple)): + return VectorConstant(value) + if isinstance(value, Value): + return value + + raise AttributeError, 'Only values can be wrapped' + +class Statistic(object): + def __getattr__(self, attr): + if attr in ('data', 'x', 'y'): + result = self.source.data(self, self.bins, self.ticks) + self.data = result.data + self.x = result.x + self.y = result.y + return super(Statistic, self).__getattribute__(attr) + + def __setattr__(self, attr, value): + if attr == 'stat': + raise AttributeError, '%s is read only' % stat + if attr in ('source', 'bins', 'ticks'): + if getattr(self, attr) != value: + if hasattr(self, 'data'): + delattr(self, 'data') + + super(Statistic, self).__setattr__(attr, value) + +class ValueProxy(Value): + def __getattr__(self, attr): + if attr == '__value__': + if scalar(self): + return self.__scalarvalue__ + if vector(self): + return self.__vectorvalue__ + if attr == '__len__': + if vector(self): + return self.__vectorlen__ + return super(ValueProxy, self).__getattribute__(attr) + +class UnaryProxy(ValueProxy): + def __init__(self, op, arg): + self.op = op + self.arg = WrapValue(arg) + + def __scalar__(self): + return scalar(self.arg) + + def __vector__(self): + return vector(self.arg) + + def __scalarvalue__(self, run): + val = value(self.arg, run) + if val is None: + return None + return self.op(val) + + def __vectorvalue__(self, run, index): + val = value(self.arg, run, index) + if val is None: + return None + return self.op(val) + + def __vectorlen__(self): + return len(unproxy(self.arg)) + +class BinaryProxy(ValueProxy): + def __init__(self, op, arg0, arg1): + super(BinaryProxy, self).__init__() + self.op = op + self.arg0 = WrapValue(arg0) + self.arg1 = WrapValue(arg1) + + def __scalar__(self): + return scalar(self.arg0) and scalar(self.arg1) + + def __vector__(self): + return vector(self.arg0) or vector(self.arg1) + + def __scalarvalue__(self, run): + val0 = value(self.arg0, run) + val1 = value(self.arg1, run) + if val0 is None or val1 is None: + return None + return self.op(val0, val1) + + def __vectorvalue__(self, run, index): + if scalar(self.arg0): + val0 = value(self.arg0, run) + if vector(self.arg0): + val0 = value(self.arg0, run, index) + if scalar(self.arg1): + val1 = value(self.arg1, run) + if vector(self.arg1): + val1 = value(self.arg1, run, index) + + if val0 is None or val1 is None: + return None + + return self.op(val0, val1) + + def __vectorlen__(self): + if vector(self.arg0) and scalar(self.arg1): + return len(self.arg0) + if scalar(self.arg0) and vector(self.arg1): + return len(self.arg1) + + len0 = len(self.arg0) + len1 = len(self.arg1) + + if len0 != len1: + raise AttributeError, \ + "vectors of different lengths %d != %d" % (len0, len1) + + return len0 + +class Proxy(Value): + def __init__(self, name, dict): + self.name = name + self.dict = dict + + def __unproxy__(self): + return unproxy(self.dict[self.name]) + + def __getitem__(self, index): + return ItemProxy(self, index) + + def __getattr__(self, attr): + return AttrProxy(self, attr) + +class ItemProxy(Proxy): + def __init__(self, proxy, index): + self.proxy = proxy + self.index = index + + def __unproxy__(self): + return unproxy(unproxy(self.proxy)[self.index]) + +class AttrProxy(Proxy): + def __init__(self, proxy, attr): + self.proxy = proxy + self.attr = attr + + def __unproxy__(self): + return unproxy(getattr(unproxy(self.proxy), self.attr)) + +class ProxyGroup(object): + def __init__(self, dict=None, **kwargs): + self.__dict__['dict'] = {} + + if dict is not None: + self.dict.update(dict) + + if kwargs: + self.dict.update(kwargs) + + def __getattr__(self, name): + return Proxy(name, self.dict) + + def __setattr__(self, attr, value): + self.dict[attr] = value + +class ScalarStat(Statistic,Scalar): + def __value__(self, run): + if run not in self.data: + return None + return self.data[run][0][0] + + def display(self, run=None): import display p = display.Print() p.name = self.name p.desc = self.desc - p.value = float(self) + p.value = value(self, run) p.flags = self.flags p.precision = self.precision if display.all or (self.flags & flags.printable): p.display() - def comparable(self, other): - return self.name == other.name +class VectorStat(Statistic,Vector): + def __value__(self, run, item): + if run not in self.data: + return None + return self.data[run][item][0] - def __eq__(self, other): - return self.value == other.value + def __len__(self): + return self.x - def __isub__(self, other): - self.value -= other.value - return self - - def __iadd__(self, other): - self.value += other.value - return self - - def __itruediv__(self, other): - if not other: - return self - self.value /= other - return self - -class Vector(Statistic,FormulaStat): - def getValue(self): - return source.data(self, self.bins, self.ticks); - - def display(self): + def display(self, run=None): import display - if not display.all and not (self.flags & flags.printable): - return - d = display.VectorDisplay() - d.__dict__.update(self.__dict__) + d.name = self.name + d.desc = self.desc + d.value = [ value(self, run, i) for i in xrange(len(self)) ] + d.flags = self.flags + d.precision = self.precision d.display() - def comparable(self, other): - return self.name == other.name and \ - len(self.value) == len(other.value) +class Formula(Value): + def __getattribute__(self, attr): + if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ): + return super(Formula, self).__getattribute__(attr) - def __eq__(self, other): - if isinstance(self.value, (list, tuple)) != \ - isinstance(other.value, (list, tuple)): - return False - - if isinstance(self.value, (list, tuple)): - if len(self.value) != len(other.value): - return False - else: - for v1,v2 in zip(self.value, other.value): - if v1 != v2: - return False - return True - else: - return self.value == other.value - - def __isub__(self, other): - self.value = binaryop(operator.sub, self.value, other.value) - return self - - def __iadd__(self, other): - self.value = binaryop(operator.add, self.value, other.value) - return self - - def __itruediv__(self, other): - if not other: - return self - if isinstance(self.value, (list, tuple)): - for i in xrange(len(self.value)): - self.value[i] /= other - else: - self.value /= other - return self - -class Formula(Vector): - def getValue(self): formula = re.sub(':', '__', self.formula) - x = eval(formula, source.stattop) - return x.value + value = eval(formula, self.source.stattop) + return getattr(value, attr) - def comparable(self, other): - return self.name == other.name and \ - compare(self.dist, other.dist) - - def __eq__(self, other): - return self.value == other.value - - def __isub__(self, other): - return self - - def __iadd__(self, other): - return self - - def __itruediv__(self, other): - if not other: - return self - return self - -class SimpleDist(object): +class SimpleDist(Statistic): def __init__(self, sums, squares, samples): self.sums = sums self.squares = squares self.samples = samples - def getValue(self): - return 0.0 - def display(self, name, desc, flags, precision): import display p = display.Print() @@ -482,9 +438,6 @@ class FullDist(SimpleDist): self.bsize = bsize self.size = size - def getValue(self): - return 0.0 - def display(self, name, desc, flags, precision): import display p = display.Print() @@ -574,9 +527,6 @@ class FullDist(SimpleDist): return self class Dist(Statistic): - def getValue(self): - return 0.0 - def display(self): import display if not display.all and not (self.flags & flags.printable): @@ -606,9 +556,6 @@ class Dist(Statistic): return self class VectorDist(Statistic): - def getValue(self): - return 0.0 - def display(self): import display if not display.all and not (self.flags & flags.printable): @@ -694,9 +641,6 @@ class VectorDist(Statistic): return self class Vector2d(Statistic): - def getValue(self): - return 0.0 - def display(self): import display if not display.all and not (self.flags & flags.printable): @@ -748,20 +692,25 @@ class Vector2d(Statistic): return self return self -def NewStat(data): +def NewStat(source, data): stat = None if data.type == 'SCALAR': - stat = Scalar(data) + stat = ScalarStat() elif data.type == 'VECTOR': - stat = Vector(data) + stat = VectorStat() elif data.type == 'DIST': - stat = Dist(data) + stat = Dist() elif data.type == 'VECTORDIST': - stat = VectorDist(data) + stat = VectorDist() elif data.type == 'VECTOR2D': - stat = Vector2d(data) + stat = Vector2d() elif data.type == 'FORMULA': - stat = Formula(data) + stat = Formula() + + stat.__dict__['source'] = source + stat.__dict__['bins'] = None + stat.__dict__['ticks'] = None + stat.__dict__.update(data.__dict__) return stat diff --git a/util/stats/output.py b/util/stats/output.py index f793749f7..cf76f291e 100644 --- a/util/stats/output.py +++ b/util/stats/output.py @@ -26,24 +26,8 @@ # # Authors: Nathan Binkert -class dbinfo(object): - def get(self, job, stat): - import info - - run = info.source.allRunNames.get(job.name, None) - if run is None: - print 'run "%s" not found' % job - return None - - stat.system = info.source[job.system] - info.display_run = run.run; - val = float(stat) - if val == 1e300*1e300: - return None - return val - class StatOutput(object): - def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None): + def __init__(self, name, jobfile, info, stat=None, binstats=None): self.name = name self.jobfile = jobfile self.stat = stat @@ -141,7 +125,7 @@ class StatOutput(object): data = zeros((len(groupopts), len(baropts)), Float) data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ] enabled = False - stacked = None + stacked = 0 for g,gopt in enumerate(groupopts): for b,bopt in enumerate(baropts): job = self.jobfile.job(options + [ gopt, bopt ]) @@ -149,17 +133,28 @@ class StatOutput(object): continue val = self.info.get(job, self.stat) - if val is None: - val = 0.0 - curstacked = isinstance(val, (list, tuple)) - if stacked is None: - stacked = curstacked - else: - if stacked != curstacked: - raise ValueError, "some stats stacked, some not" + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = val[0] + else: + stacked = len(val) data[g][b] = val + if stacked == 0: + for i in xrange(len(groupopts)): + for j in xrange(len(baropts)): + if data[i][j] is None: + data[i][j] = 0.0 + else: + for i in xrange(len(groupopts)): + for j in xrange(len(baropts)): + val = data[i][j] + if val is None: + data[i][j] = [] * stacked + elif len(val) != stacked: + raise ValueError, "some stats stacked, some not" + data = array(data) if data.sum() == 0: continue @@ -167,7 +162,10 @@ class StatOutput(object): bar_descs = [ opt.desc for opt in baropts ] group_descs = [ opt.desc for opt in groupopts ] if stacked: - legend = self.info.rcategories + try: + legend = self.info.rcategories + except: + legend = [ str(i) for i in xrange(stacked) ] else: legend = bar_descs diff --git a/util/stats/profile.py b/util/stats/profile.py index cf946452b..57b854a30 100644 --- a/util/stats/profile.py +++ b/util/stats/profile.py @@ -316,7 +316,7 @@ class Profile(object): symbols.tree.dot(dot, threshold=threshold) dot.write(symbols.filename[:-3] + 'dot') - def write_txt(self, jobfile=None, jobs=None): + def write_txt(self, jobfile=None, jobs=None, limit=None): if jobs is None: jobs = [ job for job in jobfile.jobs() ] @@ -327,7 +327,7 @@ class Profile(object): continue output = file(symbols.filename[:-3] + 'txt', 'w') - symbols.display(output) + symbols.display(output, limit) def display(self, jobfile=None, jobs=None, limit=None): if jobs is None: @@ -453,6 +453,8 @@ if __name__ == '__main__': else: profile = PCProfile() + if not categorize: + profile.categorize = None profile.inputdir(jobfile.rootdir) if graph: @@ -470,11 +472,6 @@ if __name__ == '__main__': profile.cpu = cpu profile.write_dot(jobfile=jobfile, threshold=threshold) - if not categorize: - for cpu in cpus: - profile.cpu = cpu - profile.categorize = None - if textout: for cpu in cpus: profile.cpu = cpu @@ -482,5 +479,7 @@ if __name__ == '__main__': if not graph and not textout and not dodot: for cpu in cpus: + if not categorize: + profile.categorize = None profile.cpu = cpu profile.display(jobfile=jobfile, limit=numsyms) diff --git a/util/stats/stats.py b/util/stats/stats.py index 3f6d162bc..61f54dede 100755 --- a/util/stats/stats.py +++ b/util/stats/stats.py @@ -98,29 +98,27 @@ def commands(options, command, args): raise CommandException - import db, info - info.source = db.Database() - info.source.host = options.host - info.source.db = options.db - info.source.passwd = options.passwd - info.source.user = options.user - info.source.connect() - #info.source.update_dict(globals()) + import db + source = db.Database() + source.host = options.host + source.db = options.db + source.passwd = options.passwd + source.user = options.user + source.connect() + #source.update_dict(globals()) - if type(options.get) is str: - info.source.get = options.get + if type(options.method) is str: + source.method = options.method if options.runs is None: - runs = info.source.allRuns + runs = source.allRuns else: rx = re.compile(options.runs) runs = [] - for run in info.source.allRuns: + for run in source.allRuns: if rx.match(run.name): runs.append(run) - info.display_run = runs[0].run - if command == 'runs': user = None opts, args = getopts(args, '-u') @@ -129,14 +127,14 @@ def commands(options, command, args): for o,a in opts: if o == '-u': user = a - info.source.listRuns(user) + source.listRuns(user) return if command == 'stats': if len(args) == 0: - info.source.listStats() + source.listStats() elif len(args) == 1: - info.source.listStats(args[0]) + source.listStats(args[0]) else: raise CommandException @@ -144,9 +142,9 @@ def commands(options, command, args): if command == 'bins': if len(args) == 0: - info.source.listBins() + source.listBins() elif len(args) == 1: - info.source.listBins(args[0]) + source.listBins(args[0]) else: raise CommandException @@ -154,9 +152,9 @@ def commands(options, command, args): if command == 'formulas': if len(args) == 0: - info.source.listFormulas() + source.listFormulas() elif len(args) == 1: - info.source.listFormulas(args[0]) + source.listFormulas(args[0]) else: raise CommandException @@ -166,7 +164,7 @@ def commands(options, command, args): if len(args): raise CommandException - info.source.listTicks(runs) + source.listTicks(runs) return if command == 'stability': @@ -177,8 +175,8 @@ def commands(options, command, args): merge = int(args[0]) except ValueError: usage() - stats = info.source.getStat(args[1]) - info.source.get = "sum" + stats = source.getStat(args[1]) + source.method = 'sum' def disp(*args): print "%-35s %12s %12s %4s %5s %5s %5s %10s" % args @@ -195,18 +193,17 @@ def commands(options, command, args): #loop through all the selected runs for run in runs: - info.display_run = run.run; - runTicks = info.source.retTicks([ run ]) + runTicks = source.retTicks([ run ]) #throw away the first one, it's 0 runTicks.pop(0) - info.globalTicks = runTicks + source.ticks = runTicks avg = 0 stdev = 0 numoutsideavg = 0 numoutside1std = 0 numoutside2std = 0 pairRunTicks = [] - if float(stat) == 1e300*1e300: + if value(stat, run.run) == 1e300*1e300: continue for t in range(0, len(runTicks)-(merge-1), merge): tempPair = [] @@ -215,17 +212,17 @@ def commands(options, command, args): pairRunTicks.append(tempPair) #loop through all the various ticks for each run for tick in pairRunTicks: - info.globalTicks = tick - avg += float(stat) + source.ticks = tick + avg += value(stat, run.run) avg /= len(pairRunTicks) for tick in pairRunTicks: - info.globalTicks = tick - val = float(stat) + source.ticks = tick + val = value(stat, run.run) stdev += pow((val-avg),2) stdev = math.sqrt(stdev / len(pairRunTicks)) for tick in pairRunTicks: - info.globalTicks = tick - val = float(stat) + source.ticks = tick + val = value(stat, run.run) if (val < (avg * .9)) or (val > (avg * 1.1)): numoutsideavg += 1 if (val < (avg - stdev)) or (val > (avg + stdev)): @@ -264,9 +261,9 @@ def commands(options, command, args): if options.ticks: if not options.graph: print 'only displaying sample %s' % options.ticks - info.globalTicks = [ int(x) for x in options.ticks.split() ] + source.ticks = [ int(x) for x in options.ticks.split() ] - from output import StatOutput + import output def display(): if options.graph: @@ -280,12 +277,12 @@ def commands(options, command, args): raise CommandException if command == 'stat': - stats = info.source.getStat(args[0]) + stats = source.getStat(args[0]) if command == 'formula': stats = eval(args[0]) for stat in stats: - output = StatOutput(stat.name, options.jobfile) + output = output.StatOutput(stat.name, options.jobfile, source) output.stat = stat output.label = stat.name display() @@ -295,12 +292,11 @@ def commands(options, command, args): if len(args): raise CommandException - system = info.source.__dict__[options.system] - - from proxy import ProxyGroup - sim_ticks = info.source['sim_ticks'] - sim_seconds = info.source['sim_seconds'] - proxy = ProxyGroup(system = info.source[options.system]) + system = source.__dict__[options.system] + from info import ProxyGroup + sim_ticks = source['sim_ticks'] + sim_seconds = source['sim_seconds'] + proxy = ProxyGroup(system = source[options.system]) system = proxy.system etherdev = system.tsunami.etherdev0 @@ -309,7 +305,7 @@ def commands(options, command, args): packets = etherdev.rxPackets + etherdev.txPackets bps = etherdev.rxBandwidth + etherdev.txBandwidth - output = StatOutput(command, options.jobfile) + output = output.StatOutput(command, options.jobfile, source) if command == 'usertime': import copy @@ -460,7 +456,7 @@ if __name__ == '__main__': options.user = getpass.getuser() options.runs = None options.system = 'client' - options.get = None + options.method = None options.binned = False options.graph = False options.ticks = False @@ -469,7 +465,7 @@ if __name__ == '__main__': options.jobfile = None options.all = False - opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:') + opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:') for o,a in opts: if o == '-B': options.binned = True @@ -477,8 +473,6 @@ if __name__ == '__main__': options.printmode = 'E' if o == '-F': options.printmode = 'F' - if o == '-G': - options.get = a if o == '-a': options.all = True if o == '-d': @@ -492,6 +486,8 @@ if __name__ == '__main__': jobfilename = None if o == '-j': jobfilename = a + if o == '-m': + options.method = a if o == '-p': options.passwd = getpass.getpass() if o == '-r':