Major cleanup of the statistics handling code
util/stats/db.py:
Build a result object as the result of a query operation so it is
easier to populate and contains a bit more information than just
a big dict. Also change the next level data into a matrix instead
of a dict of dicts.
Move the "get" function into the Database object. (The get function
is used by the output parsing function as the interface for accessing
backend storage, same interface for profile stuff.)
Change the old get variable to the method variable, it describes how
the get works, (whether using sum, stdev, etc.)
util/stats/display.py:
Clean up the display functions, mostly formatting.
Handle values the way they should be now.
util/stats/info.py:
Totally re-work how values are accessed from their data store.
Access individual values on demand instead of calculating everything
and passing up a huge result from the bottom.
This impacts the way that proxying works, and in general, everything
is now esentially a proxy for the lower level database. Provide new
operators: unproxy, scalar, vector, value, values, total, and len which
retrieve the proper result from the object they are called on.
Move the ProxyGroup stuff (proxies of proxies!) here from the now gone
proxy.py file and integrate the shared parts of the code. The ProxyGroup
stuff allows you to write formulas without specifying the statistics
until evaluation time.
Get rid of global variables!
util/stats/output.py:
Move the dbinfo stuff into the Database itself. Each source should
have it's own get() function for accessing it's data.
This get() function behaves a bit differently than before in that it
can return vectors as well, deal with these vectors and with no result
conditions better.
util/stats/stats.py:
the info module no longer has the source global variable, just
create the database source and pass it around as necessary
--HG--
extra : convert_revision : 8e5aa228e5d3ae8068ef9c40f65b3a2f9e7c0cff
This commit is contained in:
Nathan Binkert
parent
8ab674582e
commit
e00237e49e
5 changed files with 476 additions and 506 deletions
|
|
@ -103,6 +103,20 @@ class Node(object):
|
|||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
class Result(object):
|
||||
def __init__(self, x, y):
|
||||
self.data = {}
|
||||
self.x = x
|
||||
self.y = y
|
||||
|
||||
def __contains__(self, run):
|
||||
return run in self.data
|
||||
|
||||
def __getitem__(self, run):
|
||||
if run not in self.data:
|
||||
self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ]
|
||||
return self.data[run]
|
||||
|
||||
class Database(object):
|
||||
def __init__(self):
|
||||
self.host = 'zizzer.pool'
|
||||
|
|
@ -135,7 +149,23 @@ class Database(object):
|
|||
self.runs = None
|
||||
self.bins = None
|
||||
self.ticks = None
|
||||
self.__dict__['get'] = type(self).sum
|
||||
self.method = 'sum'
|
||||
self._method = type(self).sum
|
||||
|
||||
def get(self, job, stat):
|
||||
run = self.allRunNames.get(job.name, None)
|
||||
if run is None:
|
||||
print 'run "%s" not found' % job
|
||||
return None
|
||||
|
||||
from info import scalar, vector, value, total, len
|
||||
stat.system = self[job.system]
|
||||
if scalar(stat):
|
||||
return value(stat, run.run)
|
||||
if vector(stat):
|
||||
return values(stat, run.run)
|
||||
|
||||
return None
|
||||
|
||||
def query(self, sql):
|
||||
self.cursor.execute(sql)
|
||||
|
|
@ -203,7 +233,7 @@ class Database(object):
|
|||
self.query('select * from stats')
|
||||
import info
|
||||
for result in self.cursor.fetchall():
|
||||
stat = info.NewStat(StatData(result))
|
||||
stat = info.NewStat(self, StatData(result))
|
||||
self.append(stat)
|
||||
self.allStats.append(stat)
|
||||
self.allStatIds[stat.stat] = stat
|
||||
|
|
@ -421,30 +451,17 @@ class Database(object):
|
|||
def stdev(self, stat, bins, ticks):
|
||||
return self.outer('stddev', 'sum', stat, bins, ticks)
|
||||
|
||||
def __getattribute__(self, attr):
|
||||
if attr != 'get':
|
||||
return super(Database, self).__getattribute__(attr)
|
||||
|
||||
if self.__dict__['get'] == type(self).sum:
|
||||
return 'sum'
|
||||
elif self.__dict__['get'] == type(self).avg:
|
||||
return 'avg'
|
||||
elif self.__dict__['get'] == type(self).stdev:
|
||||
return 'stdev'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def __setattr__(self, attr, value):
|
||||
if attr != 'get':
|
||||
super(Database, self).__setattr__(attr, value)
|
||||
if attr != 'method':
|
||||
return
|
||||
|
||||
if value == 'sum':
|
||||
self.__dict__['get'] = type(self).sum
|
||||
self._method = self.sum
|
||||
elif value == 'avg':
|
||||
self.__dict__['get'] = type(self).avg
|
||||
self._method = self.avg
|
||||
elif value == 'stdev':
|
||||
self.__dict__['get'] = type(self).stdev
|
||||
self._method = self.stdev
|
||||
else:
|
||||
raise AttributeError, "can only set get to: sum | avg | stdev"
|
||||
|
||||
|
|
@ -453,10 +470,12 @@ class Database(object):
|
|||
bins = self.bins
|
||||
if ticks is None:
|
||||
ticks = self.ticks
|
||||
sql = self.__dict__['get'](self, stat, bins, ticks)
|
||||
sql = self._method(self, stat, bins, ticks)
|
||||
self.query(sql)
|
||||
|
||||
runs = {}
|
||||
xmax = 0
|
||||
ymax = 0
|
||||
for x in self.cursor.fetchall():
|
||||
data = Data(x)
|
||||
if not runs.has_key(data.run):
|
||||
|
|
@ -464,8 +483,17 @@ class Database(object):
|
|||
if not runs[data.run].has_key(data.x):
|
||||
runs[data.run][data.x] = {}
|
||||
|
||||
xmax = max(xmax, data.x)
|
||||
ymax = max(ymax, data.y)
|
||||
runs[data.run][data.x][data.y] = data.data
|
||||
return runs
|
||||
|
||||
results = Result(xmax + 1, ymax + 1)
|
||||
for run,data in runs.iteritems():
|
||||
result = results[run]
|
||||
for x,ydata in data.iteritems():
|
||||
for y,data in ydata.iteritems():
|
||||
result[x][y] = data
|
||||
return results
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.stattop[key]
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@
|
|||
|
||||
class Value:
|
||||
def __init__(self, value, precision, percent = False):
|
||||
self.value = value
|
||||
self.value = float(value)
|
||||
self.precision = precision
|
||||
self.percent = percent
|
||||
def __str__(self):
|
||||
|
|
@ -90,12 +90,18 @@ class Print:
|
|||
|
||||
class VectorDisplay:
|
||||
def display(self):
|
||||
if not self.value:
|
||||
return
|
||||
|
||||
p = Print()
|
||||
p.flags = self.flags
|
||||
p.precision = self.precision
|
||||
|
||||
if isinstance(self.value, (list, tuple)):
|
||||
if not len(self.value):
|
||||
if not isinstance(self.value, (list, tuple)):
|
||||
p.name = self.name
|
||||
p.desc = self.desc
|
||||
p.value = self.value
|
||||
p.display()
|
||||
return
|
||||
|
||||
mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
|
||||
|
|
@ -141,10 +147,3 @@ class VectorDisplay:
|
|||
p.desc = self.desc
|
||||
p.value = mytotal
|
||||
p.display()
|
||||
|
||||
else:
|
||||
p.name = self.name
|
||||
p.desc = self.desc
|
||||
p.value = self.value
|
||||
p.display()
|
||||
|
||||
|
|
|
|||
|
|
@ -27,391 +27,347 @@
|
|||
from __future__ import division
|
||||
import operator, re, types
|
||||
|
||||
source = None
|
||||
display_run = 0
|
||||
global globalTicks
|
||||
globalTicks = None
|
||||
def unproxy(proxy):
|
||||
if hasattr(proxy, '__unproxy__'):
|
||||
return proxy.__unproxy__()
|
||||
|
||||
def total(f):
|
||||
if isinstance(f, FormulaStat):
|
||||
v = f.value
|
||||
else:
|
||||
v = f
|
||||
return proxy
|
||||
|
||||
f = FormulaStat()
|
||||
if isinstance(v, (list, tuple)):
|
||||
f.value = reduce(operator.add, v)
|
||||
else:
|
||||
f.value = v
|
||||
def scalar(stat):
|
||||
stat = unproxy(stat)
|
||||
assert(stat.__scalar__() != stat.__vector__())
|
||||
return stat.__scalar__()
|
||||
|
||||
return f
|
||||
def vector(stat):
|
||||
stat = unproxy(stat)
|
||||
assert(stat.__scalar__() != stat.__vector__())
|
||||
return stat.__vector__()
|
||||
|
||||
def unaryop(op, f):
|
||||
if isinstance(f, FormulaStat):
|
||||
v = f.value
|
||||
else:
|
||||
v = f
|
||||
|
||||
if isinstance(v, (list, tuple)):
|
||||
return map(op, v)
|
||||
else:
|
||||
return op(v)
|
||||
|
||||
def zerodiv(lv, rv):
|
||||
if rv == 0.0:
|
||||
return 0.0
|
||||
else:
|
||||
return operator.truediv(lv, rv)
|
||||
|
||||
def wrapop(op, lv, rv):
|
||||
if isinstance(lv, str):
|
||||
return lv
|
||||
|
||||
if isinstance(rv, str):
|
||||
return rv
|
||||
|
||||
return op(lv, rv)
|
||||
|
||||
def same(lrun, rrun):
|
||||
for lx,rx in zip(lrun.keys(),rrun.keys()):
|
||||
if lx != rx:
|
||||
print 'lx != rx'
|
||||
print lx, rx
|
||||
print lrun.keys()
|
||||
print rrun.keys()
|
||||
return False
|
||||
for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()):
|
||||
if ly != ry:
|
||||
print 'ly != ry'
|
||||
print ly, ry
|
||||
print lrun[lx].keys()
|
||||
print rrun[rx].keys()
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def binaryop(op, lf, rf):
|
||||
result = {}
|
||||
|
||||
if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat):
|
||||
lv = lf.value
|
||||
rv = rf.value
|
||||
|
||||
theruns = []
|
||||
for r in lv.keys():
|
||||
if rv.has_key(r):
|
||||
if same(lv[r], rv[r]):
|
||||
theruns.append(r)
|
||||
else:
|
||||
raise AttributeError
|
||||
|
||||
for run in theruns:
|
||||
result[run] = {}
|
||||
for x in lv[run].keys():
|
||||
result[run][x] = {}
|
||||
for y in lv[run][x].keys():
|
||||
result[run][x][y] = wrapop(op, lv[run][x][y],
|
||||
rv[run][x][y])
|
||||
elif isinstance(lf, FormulaStat):
|
||||
lv = lf.value
|
||||
for run in lv.keys():
|
||||
result[run] = {}
|
||||
for x in lv[run].keys():
|
||||
result[run][x] = {}
|
||||
for y in lv[run][x].keys():
|
||||
result[run][x][y] = wrapop(op, lv[run][x][y], rf)
|
||||
elif isinstance(rf, FormulaStat):
|
||||
rv = rf.value
|
||||
for run in rv.keys():
|
||||
result[run] = {}
|
||||
for x in rv[run].keys():
|
||||
result[run][x] = {}
|
||||
for y in rv[run][x].keys():
|
||||
result[run][x][y] = wrapop(op, lf, rv[run][x][y])
|
||||
def value(stat, *args):
|
||||
stat = unproxy(stat)
|
||||
return stat.__value__(*args)
|
||||
|
||||
def values(stat, run):
|
||||
stat = unproxy(stat)
|
||||
result = []
|
||||
for i in xrange(len(stat)):
|
||||
val = value(stat, run.run, i)
|
||||
if val is None:
|
||||
return None
|
||||
result.append(val)
|
||||
return result
|
||||
|
||||
def sums(x, y):
|
||||
if isinstance(x, (list, tuple)):
|
||||
return map(lambda x, y: x + y, x, y)
|
||||
else:
|
||||
return x + y
|
||||
def total(stat, run):
|
||||
return sum(values(stat, run))
|
||||
|
||||
def alltrue(seq):
|
||||
return reduce(lambda x, y: x and y, seq)
|
||||
def len(stat):
|
||||
stat = unproxy(stat)
|
||||
return stat.__len__()
|
||||
|
||||
def allfalse(seq):
|
||||
return not reduce(lambda x, y: x or y, seq)
|
||||
class Value(object):
|
||||
def __scalar__(self):
|
||||
raise AttributeError, "must define __scalar__ for %s" % (type (self))
|
||||
def __vector__(self):
|
||||
raise AttributeError, "must define __vector__ for %s" % (type (self))
|
||||
|
||||
def enumerate(seq):
|
||||
return map(None, range(len(seq)), seq)
|
||||
def __add__(self, other):
|
||||
return BinaryProxy(operator.__add__, self, other)
|
||||
def __sub__(self, other):
|
||||
return BinaryProxy(operator.__sub__, self, other)
|
||||
def __mul__(self, other):
|
||||
return BinaryProxy(operator.__mul__, self, other)
|
||||
def __div__(self, other):
|
||||
return BinaryProxy(operator.__div__, self, other)
|
||||
def __truediv__(self, other):
|
||||
return BinaryProxy(operator.__truediv__, self, other)
|
||||
def __floordiv__(self, other):
|
||||
return BinaryProxy(operator.__floordiv__, self, other)
|
||||
|
||||
def cmp(a, b):
|
||||
if a < b:
|
||||
return -1
|
||||
elif a == b:
|
||||
return 0
|
||||
else:
|
||||
return 1
|
||||
def __radd__(self, other):
|
||||
return BinaryProxy(operator.__add__, other, self)
|
||||
def __rsub__(self, other):
|
||||
return BinaryProxy(operator.__sub__, other, self)
|
||||
def __rmul__(self, other):
|
||||
return BinaryProxy(operator.__mul__, other, self)
|
||||
def __rdiv__(self, other):
|
||||
return BinaryProxy(operator.__div__, other, self)
|
||||
def __rtruediv__(self, other):
|
||||
return BinaryProxy(operator.__truediv__, other, self)
|
||||
def __rfloordiv__(self, other):
|
||||
return BinaryProxy(operator.__floordiv__, other, self)
|
||||
|
||||
def __neg__(self):
|
||||
return UnaryProxy(operator.__neg__, self)
|
||||
def __pos__(self):
|
||||
return UnaryProxy(operator.__pos__, self)
|
||||
def __abs__(self):
|
||||
return UnaryProxy(operator.__abs__, self)
|
||||
|
||||
class Scalar(Value):
|
||||
def __scalar__(self):
|
||||
return True
|
||||
|
||||
def __vector__(self):
|
||||
return False
|
||||
|
||||
def __value__(self, run):
|
||||
raise AttributeError, '__value__ must be defined'
|
||||
|
||||
class VectorItemProxy(Value):
|
||||
def __init__(self, proxy, index):
|
||||
self.proxy = proxy
|
||||
self.index = index
|
||||
|
||||
def __scalar__(self):
|
||||
return True
|
||||
|
||||
def __vector__(self):
|
||||
return False
|
||||
|
||||
def __value__(self, run):
|
||||
return value(self.proxy, run, self.index)
|
||||
|
||||
class Vector(Value):
|
||||
def __scalar__(self):
|
||||
return False
|
||||
|
||||
def __vector__(self):
|
||||
return True
|
||||
|
||||
def __value__(self, run, index):
|
||||
raise AttributeError, '__value__ must be defined'
|
||||
|
||||
def __getitem__(self, index):
|
||||
return VectorItemProxy(self, index)
|
||||
|
||||
class ScalarConstant(Scalar):
|
||||
def __init__(self, constant):
|
||||
self.constant = constant
|
||||
def __value__(self, run):
|
||||
return self.constant
|
||||
|
||||
class VectorConstant(Vector):
|
||||
def __init__(self, constant):
|
||||
self.constant = constant
|
||||
def __value__(self, run, index):
|
||||
return self.constant[index]
|
||||
def __len__(self):
|
||||
return len(self.constant)
|
||||
|
||||
def WrapValue(value):
|
||||
if isinstance(value, (int, long, float)):
|
||||
return ScalarConstant(value)
|
||||
if isinstance(value, (list, tuple)):
|
||||
return VectorConstant(value)
|
||||
if isinstance(value, Value):
|
||||
return value
|
||||
|
||||
raise AttributeError, 'Only values can be wrapped'
|
||||
|
||||
class Statistic(object):
|
||||
|
||||
def __init__(self, data):
|
||||
self.__dict__.update(data.__dict__)
|
||||
if not self.__dict__.has_key('value'):
|
||||
self.__dict__['value'] = None
|
||||
if not self.__dict__.has_key('bins'):
|
||||
self.__dict__['bins'] = None
|
||||
if not self.__dict__.has_key('ticks'):
|
||||
self.__dict__['ticks'] = None
|
||||
if 'vc' not in self.__dict__:
|
||||
self.vc = {}
|
||||
|
||||
def __getattribute__(self, attr):
|
||||
if attr == 'ticks':
|
||||
if self.__dict__['ticks'] != globalTicks:
|
||||
self.__dict__['value'] = None
|
||||
self.__dict__['ticks'] = globalTicks
|
||||
return self.__dict__['ticks']
|
||||
if attr == 'value':
|
||||
if self.__dict__['ticks'] != globalTicks:
|
||||
if self.__dict__['ticks'] != None and \
|
||||
len(self.__dict__['ticks']) == 1:
|
||||
self.vc[self.__dict__['ticks'][0]] = self.__dict__['value']
|
||||
self.__dict__['ticks'] = globalTicks
|
||||
if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]):
|
||||
self.__dict__['value'] = self.vc[globalTicks[0]]
|
||||
else:
|
||||
self.__dict__['value'] = None
|
||||
if self.__dict__['value'] == None:
|
||||
self.__dict__['value'] = self.getValue()
|
||||
return self.__dict__['value']
|
||||
else:
|
||||
def __getattr__(self, attr):
|
||||
if attr in ('data', 'x', 'y'):
|
||||
result = self.source.data(self, self.bins, self.ticks)
|
||||
self.data = result.data
|
||||
self.x = result.x
|
||||
self.y = result.y
|
||||
return super(Statistic, self).__getattribute__(attr)
|
||||
|
||||
def __setattr__(self, attr, value):
|
||||
if attr == 'bins' or attr == 'ticks':
|
||||
if attr == 'bins':
|
||||
if value is not None:
|
||||
value = source.getBin(value)
|
||||
#elif attr == 'ticks' and type(value) is str:
|
||||
# value = [ int(x) for x in value.split() ]
|
||||
if attr == 'stat':
|
||||
raise AttributeError, '%s is read only' % stat
|
||||
if attr in ('source', 'bins', 'ticks'):
|
||||
if getattr(self, attr) != value:
|
||||
if hasattr(self, 'data'):
|
||||
delattr(self, 'data')
|
||||
|
||||
self.__dict__[attr] = value
|
||||
self.__dict__['value'] = None
|
||||
self.vc = {}
|
||||
else:
|
||||
super(Statistic, self).__setattr__(attr, value)
|
||||
|
||||
def getValue(self):
|
||||
raise AttributeError, 'getValue() must be defined'
|
||||
class ValueProxy(Value):
|
||||
def __getattr__(self, attr):
|
||||
if attr == '__value__':
|
||||
if scalar(self):
|
||||
return self.__scalarvalue__
|
||||
if vector(self):
|
||||
return self.__vectorvalue__
|
||||
if attr == '__len__':
|
||||
if vector(self):
|
||||
return self.__vectorlen__
|
||||
return super(ValueProxy, self).__getattribute__(attr)
|
||||
|
||||
def zero(self):
|
||||
return False
|
||||
class UnaryProxy(ValueProxy):
|
||||
def __init__(self, op, arg):
|
||||
self.op = op
|
||||
self.arg = WrapValue(arg)
|
||||
|
||||
def __ne__(self, other):
|
||||
return not (self == other)
|
||||
def __scalar__(self):
|
||||
return scalar(self.arg)
|
||||
|
||||
def __str__(self):
|
||||
return '%f' % (float(self))
|
||||
def __vector__(self):
|
||||
return vector(self.arg)
|
||||
|
||||
class FormulaStat(object):
|
||||
def __add__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.add, self, other)
|
||||
return f
|
||||
def __sub__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.sub, self, other)
|
||||
return f
|
||||
def __mul__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.mul, self, other)
|
||||
return f
|
||||
def __truediv__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(zerodiv, self, other)
|
||||
return f
|
||||
def __mod__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.mod, self, other)
|
||||
return f
|
||||
def __radd__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.add, other, self)
|
||||
return f
|
||||
def __rsub__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.sub, other, self)
|
||||
return f
|
||||
def __rmul__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.mul, other, self)
|
||||
return f
|
||||
def __rtruediv__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(zerodiv, other, self)
|
||||
return f
|
||||
def __rmod__(self, other):
|
||||
f = FormulaStat()
|
||||
f.value = binaryop(operator.mod, other, self)
|
||||
return f
|
||||
def __neg__(self):
|
||||
f = FormulaStat()
|
||||
f.value = unaryop(operator.neg, self)
|
||||
return f
|
||||
def __getitem__(self, idx):
|
||||
f = FormulaStat()
|
||||
f.value = {}
|
||||
for key in self.value.keys():
|
||||
f.value[key] = {}
|
||||
f.value[key][0] = {}
|
||||
f.value[key][0][0] = self.value[key][idx][0]
|
||||
return f
|
||||
def __scalarvalue__(self, run):
|
||||
val = value(self.arg, run)
|
||||
if val is None:
|
||||
return None
|
||||
return self.op(val)
|
||||
|
||||
def __float__(self):
|
||||
if isinstance(self.value, FormulaStat):
|
||||
return float(self.value)
|
||||
if not self.value.has_key(display_run):
|
||||
return (1e300*1e300)
|
||||
if len(self.value[display_run]) == 1:
|
||||
return self.value[display_run][0][0]
|
||||
else:
|
||||
#print self.value[display_run]
|
||||
return self.value[display_run][4][0]
|
||||
#raise ValueError
|
||||
def __vectorvalue__(self, run, index):
|
||||
val = value(self.arg, run, index)
|
||||
if val is None:
|
||||
return None
|
||||
return self.op(val)
|
||||
|
||||
def display(self):
|
||||
import display
|
||||
d = display.VectorDisplay()
|
||||
d.flags = 0
|
||||
d.precision = 1
|
||||
d.name = 'formula'
|
||||
d.desc = 'formula'
|
||||
val = self.value[display_run]
|
||||
d.value = [ val[x][0] for x in val.keys() ]
|
||||
d.display()
|
||||
def __vectorlen__(self):
|
||||
return len(unproxy(self.arg))
|
||||
|
||||
class BinaryProxy(ValueProxy):
|
||||
def __init__(self, op, arg0, arg1):
|
||||
super(BinaryProxy, self).__init__()
|
||||
self.op = op
|
||||
self.arg0 = WrapValue(arg0)
|
||||
self.arg1 = WrapValue(arg1)
|
||||
|
||||
class Scalar(Statistic,FormulaStat):
|
||||
def getValue(self):
|
||||
return source.data(self, self.bins, self.ticks)
|
||||
def __scalar__(self):
|
||||
return scalar(self.arg0) and scalar(self.arg1)
|
||||
|
||||
def display(self):
|
||||
def __vector__(self):
|
||||
return vector(self.arg0) or vector(self.arg1)
|
||||
|
||||
def __scalarvalue__(self, run):
|
||||
val0 = value(self.arg0, run)
|
||||
val1 = value(self.arg1, run)
|
||||
if val0 is None or val1 is None:
|
||||
return None
|
||||
return self.op(val0, val1)
|
||||
|
||||
def __vectorvalue__(self, run, index):
|
||||
if scalar(self.arg0):
|
||||
val0 = value(self.arg0, run)
|
||||
if vector(self.arg0):
|
||||
val0 = value(self.arg0, run, index)
|
||||
if scalar(self.arg1):
|
||||
val1 = value(self.arg1, run)
|
||||
if vector(self.arg1):
|
||||
val1 = value(self.arg1, run, index)
|
||||
|
||||
if val0 is None or val1 is None:
|
||||
return None
|
||||
|
||||
return self.op(val0, val1)
|
||||
|
||||
def __vectorlen__(self):
|
||||
if vector(self.arg0) and scalar(self.arg1):
|
||||
return len(self.arg0)
|
||||
if scalar(self.arg0) and vector(self.arg1):
|
||||
return len(self.arg1)
|
||||
|
||||
len0 = len(self.arg0)
|
||||
len1 = len(self.arg1)
|
||||
|
||||
if len0 != len1:
|
||||
raise AttributeError, \
|
||||
"vectors of different lengths %d != %d" % (len0, len1)
|
||||
|
||||
return len0
|
||||
|
||||
class Proxy(Value):
|
||||
def __init__(self, name, dict):
|
||||
self.name = name
|
||||
self.dict = dict
|
||||
|
||||
def __unproxy__(self):
|
||||
return unproxy(self.dict[self.name])
|
||||
|
||||
def __getitem__(self, index):
|
||||
return ItemProxy(self, index)
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return AttrProxy(self, attr)
|
||||
|
||||
class ItemProxy(Proxy):
|
||||
def __init__(self, proxy, index):
|
||||
self.proxy = proxy
|
||||
self.index = index
|
||||
|
||||
def __unproxy__(self):
|
||||
return unproxy(unproxy(self.proxy)[self.index])
|
||||
|
||||
class AttrProxy(Proxy):
|
||||
def __init__(self, proxy, attr):
|
||||
self.proxy = proxy
|
||||
self.attr = attr
|
||||
|
||||
def __unproxy__(self):
|
||||
return unproxy(getattr(unproxy(self.proxy), self.attr))
|
||||
|
||||
class ProxyGroup(object):
|
||||
def __init__(self, dict=None, **kwargs):
|
||||
self.__dict__['dict'] = {}
|
||||
|
||||
if dict is not None:
|
||||
self.dict.update(dict)
|
||||
|
||||
if kwargs:
|
||||
self.dict.update(kwargs)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return Proxy(name, self.dict)
|
||||
|
||||
def __setattr__(self, attr, value):
|
||||
self.dict[attr] = value
|
||||
|
||||
class ScalarStat(Statistic,Scalar):
|
||||
def __value__(self, run):
|
||||
if run not in self.data:
|
||||
return None
|
||||
return self.data[run][0][0]
|
||||
|
||||
def display(self, run=None):
|
||||
import display
|
||||
p = display.Print()
|
||||
p.name = self.name
|
||||
p.desc = self.desc
|
||||
p.value = float(self)
|
||||
p.value = value(self, run)
|
||||
p.flags = self.flags
|
||||
p.precision = self.precision
|
||||
if display.all or (self.flags & flags.printable):
|
||||
p.display()
|
||||
|
||||
def comparable(self, other):
|
||||
return self.name == other.name
|
||||
class VectorStat(Statistic,Vector):
|
||||
def __value__(self, run, item):
|
||||
if run not in self.data:
|
||||
return None
|
||||
return self.data[run][item][0]
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.value == other.value
|
||||
def __len__(self):
|
||||
return self.x
|
||||
|
||||
def __isub__(self, other):
|
||||
self.value -= other.value
|
||||
return self
|
||||
|
||||
def __iadd__(self, other):
|
||||
self.value += other.value
|
||||
return self
|
||||
|
||||
def __itruediv__(self, other):
|
||||
if not other:
|
||||
return self
|
||||
self.value /= other
|
||||
return self
|
||||
|
||||
class Vector(Statistic,FormulaStat):
|
||||
def getValue(self):
|
||||
return source.data(self, self.bins, self.ticks);
|
||||
|
||||
def display(self):
|
||||
def display(self, run=None):
|
||||
import display
|
||||
if not display.all and not (self.flags & flags.printable):
|
||||
return
|
||||
|
||||
d = display.VectorDisplay()
|
||||
d.__dict__.update(self.__dict__)
|
||||
d.name = self.name
|
||||
d.desc = self.desc
|
||||
d.value = [ value(self, run, i) for i in xrange(len(self)) ]
|
||||
d.flags = self.flags
|
||||
d.precision = self.precision
|
||||
d.display()
|
||||
|
||||
def comparable(self, other):
|
||||
return self.name == other.name and \
|
||||
len(self.value) == len(other.value)
|
||||
class Formula(Value):
|
||||
def __getattribute__(self, attr):
|
||||
if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ):
|
||||
return super(Formula, self).__getattribute__(attr)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(self.value, (list, tuple)) != \
|
||||
isinstance(other.value, (list, tuple)):
|
||||
return False
|
||||
|
||||
if isinstance(self.value, (list, tuple)):
|
||||
if len(self.value) != len(other.value):
|
||||
return False
|
||||
else:
|
||||
for v1,v2 in zip(self.value, other.value):
|
||||
if v1 != v2:
|
||||
return False
|
||||
return True
|
||||
else:
|
||||
return self.value == other.value
|
||||
|
||||
def __isub__(self, other):
|
||||
self.value = binaryop(operator.sub, self.value, other.value)
|
||||
return self
|
||||
|
||||
def __iadd__(self, other):
|
||||
self.value = binaryop(operator.add, self.value, other.value)
|
||||
return self
|
||||
|
||||
def __itruediv__(self, other):
|
||||
if not other:
|
||||
return self
|
||||
if isinstance(self.value, (list, tuple)):
|
||||
for i in xrange(len(self.value)):
|
||||
self.value[i] /= other
|
||||
else:
|
||||
self.value /= other
|
||||
return self
|
||||
|
||||
class Formula(Vector):
|
||||
def getValue(self):
|
||||
formula = re.sub(':', '__', self.formula)
|
||||
x = eval(formula, source.stattop)
|
||||
return x.value
|
||||
value = eval(formula, self.source.stattop)
|
||||
return getattr(value, attr)
|
||||
|
||||
def comparable(self, other):
|
||||
return self.name == other.name and \
|
||||
compare(self.dist, other.dist)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.value == other.value
|
||||
|
||||
def __isub__(self, other):
|
||||
return self
|
||||
|
||||
def __iadd__(self, other):
|
||||
return self
|
||||
|
||||
def __itruediv__(self, other):
|
||||
if not other:
|
||||
return self
|
||||
return self
|
||||
|
||||
class SimpleDist(object):
|
||||
class SimpleDist(Statistic):
|
||||
def __init__(self, sums, squares, samples):
|
||||
self.sums = sums
|
||||
self.squares = squares
|
||||
self.samples = samples
|
||||
|
||||
def getValue(self):
|
||||
return 0.0
|
||||
|
||||
def display(self, name, desc, flags, precision):
|
||||
import display
|
||||
p = display.Print()
|
||||
|
|
@ -482,9 +438,6 @@ class FullDist(SimpleDist):
|
|||
self.bsize = bsize
|
||||
self.size = size
|
||||
|
||||
def getValue(self):
|
||||
return 0.0
|
||||
|
||||
def display(self, name, desc, flags, precision):
|
||||
import display
|
||||
p = display.Print()
|
||||
|
|
@ -574,9 +527,6 @@ class FullDist(SimpleDist):
|
|||
return self
|
||||
|
||||
class Dist(Statistic):
|
||||
def getValue(self):
|
||||
return 0.0
|
||||
|
||||
def display(self):
|
||||
import display
|
||||
if not display.all and not (self.flags & flags.printable):
|
||||
|
|
@ -606,9 +556,6 @@ class Dist(Statistic):
|
|||
return self
|
||||
|
||||
class VectorDist(Statistic):
|
||||
def getValue(self):
|
||||
return 0.0
|
||||
|
||||
def display(self):
|
||||
import display
|
||||
if not display.all and not (self.flags & flags.printable):
|
||||
|
|
@ -694,9 +641,6 @@ class VectorDist(Statistic):
|
|||
return self
|
||||
|
||||
class Vector2d(Statistic):
|
||||
def getValue(self):
|
||||
return 0.0
|
||||
|
||||
def display(self):
|
||||
import display
|
||||
if not display.all and not (self.flags & flags.printable):
|
||||
|
|
@ -748,20 +692,25 @@ class Vector2d(Statistic):
|
|||
return self
|
||||
return self
|
||||
|
||||
def NewStat(data):
|
||||
def NewStat(source, data):
|
||||
stat = None
|
||||
if data.type == 'SCALAR':
|
||||
stat = Scalar(data)
|
||||
stat = ScalarStat()
|
||||
elif data.type == 'VECTOR':
|
||||
stat = Vector(data)
|
||||
stat = VectorStat()
|
||||
elif data.type == 'DIST':
|
||||
stat = Dist(data)
|
||||
stat = Dist()
|
||||
elif data.type == 'VECTORDIST':
|
||||
stat = VectorDist(data)
|
||||
stat = VectorDist()
|
||||
elif data.type == 'VECTOR2D':
|
||||
stat = Vector2d(data)
|
||||
stat = Vector2d()
|
||||
elif data.type == 'FORMULA':
|
||||
stat = Formula(data)
|
||||
stat = Formula()
|
||||
|
||||
stat.__dict__['source'] = source
|
||||
stat.__dict__['bins'] = None
|
||||
stat.__dict__['ticks'] = None
|
||||
stat.__dict__.update(data.__dict__)
|
||||
|
||||
return stat
|
||||
|
||||
|
|
|
|||
|
|
@ -26,24 +26,8 @@
|
|||
#
|
||||
# Authors: Nathan Binkert
|
||||
|
||||
class dbinfo(object):
|
||||
def get(self, job, stat):
|
||||
import info
|
||||
|
||||
run = info.source.allRunNames.get(job.name, None)
|
||||
if run is None:
|
||||
print 'run "%s" not found' % job
|
||||
return None
|
||||
|
||||
stat.system = info.source[job.system]
|
||||
info.display_run = run.run;
|
||||
val = float(stat)
|
||||
if val == 1e300*1e300:
|
||||
return None
|
||||
return val
|
||||
|
||||
class StatOutput(object):
|
||||
def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None):
|
||||
def __init__(self, name, jobfile, info, stat=None, binstats=None):
|
||||
self.name = name
|
||||
self.jobfile = jobfile
|
||||
self.stat = stat
|
||||
|
|
@ -141,7 +125,7 @@ class StatOutput(object):
|
|||
data = zeros((len(groupopts), len(baropts)), Float)
|
||||
data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ]
|
||||
enabled = False
|
||||
stacked = None
|
||||
stacked = 0
|
||||
for g,gopt in enumerate(groupopts):
|
||||
for b,bopt in enumerate(baropts):
|
||||
job = self.jobfile.job(options + [ gopt, bopt ])
|
||||
|
|
@ -149,17 +133,28 @@ class StatOutput(object):
|
|||
continue
|
||||
|
||||
val = self.info.get(job, self.stat)
|
||||
if val is None:
|
||||
val = 0.0
|
||||
curstacked = isinstance(val, (list, tuple))
|
||||
if stacked is None:
|
||||
stacked = curstacked
|
||||
if isinstance(val, (list, tuple)):
|
||||
if len(val) == 1:
|
||||
val = val[0]
|
||||
else:
|
||||
if stacked != curstacked:
|
||||
raise ValueError, "some stats stacked, some not"
|
||||
stacked = len(val)
|
||||
|
||||
data[g][b] = val
|
||||
|
||||
if stacked == 0:
|
||||
for i in xrange(len(groupopts)):
|
||||
for j in xrange(len(baropts)):
|
||||
if data[i][j] is None:
|
||||
data[i][j] = 0.0
|
||||
else:
|
||||
for i in xrange(len(groupopts)):
|
||||
for j in xrange(len(baropts)):
|
||||
val = data[i][j]
|
||||
if val is None:
|
||||
data[i][j] = [] * stacked
|
||||
elif len(val) != stacked:
|
||||
raise ValueError, "some stats stacked, some not"
|
||||
|
||||
data = array(data)
|
||||
if data.sum() == 0:
|
||||
continue
|
||||
|
|
@ -167,7 +162,10 @@ class StatOutput(object):
|
|||
bar_descs = [ opt.desc for opt in baropts ]
|
||||
group_descs = [ opt.desc for opt in groupopts ]
|
||||
if stacked:
|
||||
try:
|
||||
legend = self.info.rcategories
|
||||
except:
|
||||
legend = [ str(i) for i in xrange(stacked) ]
|
||||
else:
|
||||
legend = bar_descs
|
||||
|
||||
|
|
|
|||
|
|
@ -98,29 +98,27 @@ def commands(options, command, args):
|
|||
|
||||
raise CommandException
|
||||
|
||||
import db, info
|
||||
info.source = db.Database()
|
||||
info.source.host = options.host
|
||||
info.source.db = options.db
|
||||
info.source.passwd = options.passwd
|
||||
info.source.user = options.user
|
||||
info.source.connect()
|
||||
#info.source.update_dict(globals())
|
||||
import db
|
||||
source = db.Database()
|
||||
source.host = options.host
|
||||
source.db = options.db
|
||||
source.passwd = options.passwd
|
||||
source.user = options.user
|
||||
source.connect()
|
||||
#source.update_dict(globals())
|
||||
|
||||
if type(options.get) is str:
|
||||
info.source.get = options.get
|
||||
if type(options.method) is str:
|
||||
source.method = options.method
|
||||
|
||||
if options.runs is None:
|
||||
runs = info.source.allRuns
|
||||
runs = source.allRuns
|
||||
else:
|
||||
rx = re.compile(options.runs)
|
||||
runs = []
|
||||
for run in info.source.allRuns:
|
||||
for run in source.allRuns:
|
||||
if rx.match(run.name):
|
||||
runs.append(run)
|
||||
|
||||
info.display_run = runs[0].run
|
||||
|
||||
if command == 'runs':
|
||||
user = None
|
||||
opts, args = getopts(args, '-u')
|
||||
|
|
@ -129,14 +127,14 @@ def commands(options, command, args):
|
|||
for o,a in opts:
|
||||
if o == '-u':
|
||||
user = a
|
||||
info.source.listRuns(user)
|
||||
source.listRuns(user)
|
||||
return
|
||||
|
||||
if command == 'stats':
|
||||
if len(args) == 0:
|
||||
info.source.listStats()
|
||||
source.listStats()
|
||||
elif len(args) == 1:
|
||||
info.source.listStats(args[0])
|
||||
source.listStats(args[0])
|
||||
else:
|
||||
raise CommandException
|
||||
|
||||
|
|
@ -144,9 +142,9 @@ def commands(options, command, args):
|
|||
|
||||
if command == 'bins':
|
||||
if len(args) == 0:
|
||||
info.source.listBins()
|
||||
source.listBins()
|
||||
elif len(args) == 1:
|
||||
info.source.listBins(args[0])
|
||||
source.listBins(args[0])
|
||||
else:
|
||||
raise CommandException
|
||||
|
||||
|
|
@ -154,9 +152,9 @@ def commands(options, command, args):
|
|||
|
||||
if command == 'formulas':
|
||||
if len(args) == 0:
|
||||
info.source.listFormulas()
|
||||
source.listFormulas()
|
||||
elif len(args) == 1:
|
||||
info.source.listFormulas(args[0])
|
||||
source.listFormulas(args[0])
|
||||
else:
|
||||
raise CommandException
|
||||
|
||||
|
|
@ -166,7 +164,7 @@ def commands(options, command, args):
|
|||
if len(args):
|
||||
raise CommandException
|
||||
|
||||
info.source.listTicks(runs)
|
||||
source.listTicks(runs)
|
||||
return
|
||||
|
||||
if command == 'stability':
|
||||
|
|
@ -177,8 +175,8 @@ def commands(options, command, args):
|
|||
merge = int(args[0])
|
||||
except ValueError:
|
||||
usage()
|
||||
stats = info.source.getStat(args[1])
|
||||
info.source.get = "sum"
|
||||
stats = source.getStat(args[1])
|
||||
source.method = 'sum'
|
||||
|
||||
def disp(*args):
|
||||
print "%-20s %12s %12s %4s %5s %5s %5s %10s" % args
|
||||
|
|
@ -195,18 +193,17 @@ def commands(options, command, args):
|
|||
|
||||
#loop through all the selected runs
|
||||
for run in runs:
|
||||
info.display_run = run.run;
|
||||
runTicks = info.source.retTicks([ run ])
|
||||
runTicks = source.retTicks([ run ])
|
||||
#throw away the first one, it's 0
|
||||
runTicks.pop(0)
|
||||
info.globalTicks = runTicks
|
||||
source.ticks = runTicks
|
||||
avg = 0
|
||||
stdev = 0
|
||||
numoutsideavg = 0
|
||||
numoutside1std = 0
|
||||
numoutside2std = 0
|
||||
pairRunTicks = []
|
||||
if float(stat) == 1e300*1e300:
|
||||
if value(stat, run.run) == 1e300*1e300:
|
||||
continue
|
||||
for t in range(0, len(runTicks)-(merge-1), merge):
|
||||
tempPair = []
|
||||
|
|
@ -215,17 +212,17 @@ def commands(options, command, args):
|
|||
pairRunTicks.append(tempPair)
|
||||
#loop through all the various ticks for each run
|
||||
for tick in pairRunTicks:
|
||||
info.globalTicks = tick
|
||||
avg += float(stat)
|
||||
source.ticks = tick
|
||||
avg += value(stat, run.run)
|
||||
avg /= len(pairRunTicks)
|
||||
for tick in pairRunTicks:
|
||||
info.globalTicks = tick
|
||||
val = float(stat)
|
||||
source.ticks = tick
|
||||
val = value(stat, run.run)
|
||||
stdev += pow((val-avg),2)
|
||||
stdev = math.sqrt(stdev / len(pairRunTicks))
|
||||
for tick in pairRunTicks:
|
||||
info.globalTicks = tick
|
||||
val = float(stat)
|
||||
source.ticks = tick
|
||||
val = value(stat, run.run)
|
||||
if (val < (avg * .9)) or (val > (avg * 1.1)):
|
||||
numoutsideavg += 1
|
||||
if (val < (avg - stdev)) or (val > (avg + stdev)):
|
||||
|
|
@ -264,9 +261,9 @@ def commands(options, command, args):
|
|||
if options.ticks:
|
||||
if not options.graph:
|
||||
print 'only displaying sample %s' % options.ticks
|
||||
info.globalTicks = [ int(x) for x in options.ticks.split() ]
|
||||
source.ticks = [ int(x) for x in options.ticks.split() ]
|
||||
|
||||
from output import StatOutput
|
||||
import output
|
||||
|
||||
def display():
|
||||
if options.graph:
|
||||
|
|
@ -280,12 +277,12 @@ def commands(options, command, args):
|
|||
raise CommandException
|
||||
|
||||
if command == 'stat':
|
||||
stats = info.source.getStat(args[0])
|
||||
stats = source.getStat(args[0])
|
||||
if command == 'formula':
|
||||
stats = eval(args[0])
|
||||
|
||||
for stat in stats:
|
||||
output = StatOutput(stat.name, options.jobfile)
|
||||
output = output.StatOutput(stat.name, options.jobfile, source)
|
||||
output.stat = stat
|
||||
output.label = stat.name
|
||||
display()
|
||||
|
|
@ -295,12 +292,11 @@ def commands(options, command, args):
|
|||
if len(args):
|
||||
raise CommandException
|
||||
|
||||
system = info.source.__dict__[options.system]
|
||||
|
||||
from proxy import ProxyGroup
|
||||
sim_ticks = info.source['sim_ticks']
|
||||
sim_seconds = info.source['sim_seconds']
|
||||
proxy = ProxyGroup(system = info.source[options.system])
|
||||
system = source.__dict__[options.system]
|
||||
from info import ProxyGroup
|
||||
sim_ticks = source['sim_ticks']
|
||||
sim_seconds = source['sim_seconds']
|
||||
proxy = ProxyGroup(system = source[options.system])
|
||||
system = proxy.system
|
||||
|
||||
etherdev = system.tsunami.etherdev0
|
||||
|
|
@ -309,7 +305,7 @@ def commands(options, command, args):
|
|||
packets = etherdev.rxPackets + etherdev.txPackets
|
||||
bps = etherdev.rxBandwidth + etherdev.txBandwidth
|
||||
|
||||
output = StatOutput(command, options.jobfile)
|
||||
output = output.StatOutput(command, options.jobfile, source)
|
||||
|
||||
if command == 'usertime':
|
||||
import copy
|
||||
|
|
@ -460,7 +456,7 @@ if __name__ == '__main__':
|
|||
options.user = getpass.getuser()
|
||||
options.runs = None
|
||||
options.system = 'client'
|
||||
options.get = None
|
||||
options.method = None
|
||||
options.binned = False
|
||||
options.graph = False
|
||||
options.ticks = False
|
||||
|
|
@ -469,7 +465,7 @@ if __name__ == '__main__':
|
|||
options.jobfile = None
|
||||
options.all = False
|
||||
|
||||
opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:')
|
||||
opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:')
|
||||
for o,a in opts:
|
||||
if o == '-B':
|
||||
options.binned = True
|
||||
|
|
@ -477,8 +473,6 @@ if __name__ == '__main__':
|
|||
options.printmode = 'E'
|
||||
if o == '-F':
|
||||
options.printmode = 'F'
|
||||
if o == '-G':
|
||||
options.get = a
|
||||
if o == '-a':
|
||||
options.all = True
|
||||
if o == '-d':
|
||||
|
|
@ -492,6 +486,8 @@ if __name__ == '__main__':
|
|||
jobfilename = None
|
||||
if o == '-j':
|
||||
jobfilename = a
|
||||
if o == '-m':
|
||||
options.method = a
|
||||
if o == '-p':
|
||||
options.passwd = getpass.getpass()
|
||||
if o == '-r':
|
||||
|
|
|
|||
Loading…
Reference in a new issue