Major cleanup of the statistics handling code

util/stats/db.py:
    Build a result object as the result of a query operation so it is
    easier to populate and contains a bit more information than just
    a big dict.  Also change the next level data into a matrix instead
    of a dict of dicts.
    Move the "get" function into the Database object.  (The get function
    is used by the output parsing function as the interface for accessing
    backend storage, same interface for profile stuff.)
    Change the old get variable to the method variable, it describes how
    the get works, (whether using sum, stdev, etc.)
util/stats/display.py:
    Clean up the display functions, mostly formatting.
    Handle values the way they should be now.
util/stats/info.py:
    Totally re-work how values are accessed from their data store.
    Access individual values on demand instead of calculating everything
    and passing up a huge result from the bottom.
    This impacts the way that proxying works, and in general, everything
    is now esentially a proxy for the lower level database.  Provide new
    operators: unproxy, scalar, vector, value, values, total, and len which
    retrieve the proper result from the object they are called on.
    Move the ProxyGroup stuff (proxies of proxies!) here from the now gone
    proxy.py file and integrate the shared parts of the code.  The ProxyGroup
    stuff allows you to write formulas without specifying the statistics
    until evaluation time.

    Get rid of global variables!
util/stats/output.py:
    Move the dbinfo stuff into the Database itself.  Each source should
    have it's own get() function for accessing it's data.
    This get() function behaves a bit differently than before in that it
    can return vectors as well, deal with these vectors and with no result
    conditions better.
util/stats/stats.py:
    the info module no longer has the source global variable, just
    create the database source and pass it around as necessary

--HG--
extra : convert_revision : 8e5aa228e5d3ae8068ef9c40f65b3a2f9e7c0cff
This commit is contained in:
Nathan Binkert 2005-10-21 16:29:13 -04:00
parent 8ab674582e
commit e00237e49e
5 changed files with 476 additions and 506 deletions

View file

@ -103,6 +103,20 @@ class Node(object):
def __str__(self):
return self.name
class Result(object):
def __init__(self, x, y):
self.data = {}
self.x = x
self.y = y
def __contains__(self, run):
return run in self.data
def __getitem__(self, run):
if run not in self.data:
self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ]
return self.data[run]
class Database(object):
def __init__(self):
self.host = 'zizzer.pool'
@ -135,7 +149,23 @@ class Database(object):
self.runs = None
self.bins = None
self.ticks = None
self.__dict__['get'] = type(self).sum
self.method = 'sum'
self._method = type(self).sum
def get(self, job, stat):
run = self.allRunNames.get(job.name, None)
if run is None:
print 'run "%s" not found' % job
return None
from info import scalar, vector, value, total, len
stat.system = self[job.system]
if scalar(stat):
return value(stat, run.run)
if vector(stat):
return values(stat, run.run)
return None
def query(self, sql):
self.cursor.execute(sql)
@ -203,7 +233,7 @@ class Database(object):
self.query('select * from stats')
import info
for result in self.cursor.fetchall():
stat = info.NewStat(StatData(result))
stat = info.NewStat(self, StatData(result))
self.append(stat)
self.allStats.append(stat)
self.allStatIds[stat.stat] = stat
@ -421,30 +451,17 @@ class Database(object):
def stdev(self, stat, bins, ticks):
return self.outer('stddev', 'sum', stat, bins, ticks)
def __getattribute__(self, attr):
if attr != 'get':
return super(Database, self).__getattribute__(attr)
if self.__dict__['get'] == type(self).sum:
return 'sum'
elif self.__dict__['get'] == type(self).avg:
return 'avg'
elif self.__dict__['get'] == type(self).stdev:
return 'stdev'
else:
return ''
def __setattr__(self, attr, value):
if attr != 'get':
super(Database, self).__setattr__(attr, value)
super(Database, self).__setattr__(attr, value)
if attr != 'method':
return
if value == 'sum':
self.__dict__['get'] = type(self).sum
self._method = self.sum
elif value == 'avg':
self.__dict__['get'] = type(self).avg
self._method = self.avg
elif value == 'stdev':
self.__dict__['get'] = type(self).stdev
self._method = self.stdev
else:
raise AttributeError, "can only set get to: sum | avg | stdev"
@ -453,10 +470,12 @@ class Database(object):
bins = self.bins
if ticks is None:
ticks = self.ticks
sql = self.__dict__['get'](self, stat, bins, ticks)
sql = self._method(self, stat, bins, ticks)
self.query(sql)
runs = {}
xmax = 0
ymax = 0
for x in self.cursor.fetchall():
data = Data(x)
if not runs.has_key(data.run):
@ -464,8 +483,17 @@ class Database(object):
if not runs[data.run].has_key(data.x):
runs[data.run][data.x] = {}
xmax = max(xmax, data.x)
ymax = max(ymax, data.y)
runs[data.run][data.x][data.y] = data.data
return runs
results = Result(xmax + 1, ymax + 1)
for run,data in runs.iteritems():
result = results[run]
for x,ydata in data.iteritems():
for y,data in ydata.iteritems():
result[x][y] = data
return results
def __getitem__(self, key):
return self.stattop[key]

View file

@ -26,7 +26,7 @@
class Value:
def __init__(self, value, precision, percent = False):
self.value = value
self.value = float(value)
self.precision = precision
self.percent = percent
def __str__(self):
@ -90,61 +90,60 @@ class Print:
class VectorDisplay:
def display(self):
if not self.value:
return
p = Print()
p.flags = self.flags
p.precision = self.precision
if isinstance(self.value, (list, tuple)):
if not len(self.value):
return
mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
mycdf = 0.0
value = self.value
if display_all:
subnames = [ '[%d]' % i for i in range(len(value)) ]
else:
subnames = [''] * len(value)
if self.__dict__.has_key('subnames'):
for i,each in enumerate(self.subnames):
if len(each) > 0:
subnames[i] = '.%s' % each
subdescs = [self.desc]*len(value)
if self.__dict__.has_key('subdescs'):
for i in xrange(min(len(value), len(self.subdescs))):
subdescs[i] = self.subdescs[i]
for val,sname,sdesc in map(None, value, subnames, subdescs):
if mytotal > 0.0:
mypdf = float(val) / float(mytotal)
mycdf += mypdf
if (self.flags & flags_pdf):
p.pdf = mypdf
p.cdf = mycdf
if len(sname) == 0:
continue
p.name = self.name + sname
p.desc = sdesc
p.value = val
p.display()
if (self.flags & flags_total):
if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
p.name = self.name + '.total'
p.desc = self.desc
p.value = mytotal
p.display()
else:
if not isinstance(self.value, (list, tuple)):
p.name = self.name
p.desc = self.desc
p.value = self.value
p.display()
return
mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
mycdf = 0.0
value = self.value
if display_all:
subnames = [ '[%d]' % i for i in range(len(value)) ]
else:
subnames = [''] * len(value)
if self.__dict__.has_key('subnames'):
for i,each in enumerate(self.subnames):
if len(each) > 0:
subnames[i] = '.%s' % each
subdescs = [self.desc]*len(value)
if self.__dict__.has_key('subdescs'):
for i in xrange(min(len(value), len(self.subdescs))):
subdescs[i] = self.subdescs[i]
for val,sname,sdesc in map(None, value, subnames, subdescs):
if mytotal > 0.0:
mypdf = float(val) / float(mytotal)
mycdf += mypdf
if (self.flags & flags_pdf):
p.pdf = mypdf
p.cdf = mycdf
if len(sname) == 0:
continue
p.name = self.name + sname
p.desc = sdesc
p.value = val
p.display()
if (self.flags & flags_total):
if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
p.name = self.name + '.total'
p.desc = self.desc
p.value = mytotal
p.display()

View file

@ -27,391 +27,347 @@
from __future__ import division
import operator, re, types
source = None
display_run = 0
global globalTicks
globalTicks = None
def unproxy(proxy):
if hasattr(proxy, '__unproxy__'):
return proxy.__unproxy__()
def total(f):
if isinstance(f, FormulaStat):
v = f.value
else:
v = f
return proxy
f = FormulaStat()
if isinstance(v, (list, tuple)):
f.value = reduce(operator.add, v)
else:
f.value = v
def scalar(stat):
stat = unproxy(stat)
assert(stat.__scalar__() != stat.__vector__())
return stat.__scalar__()
return f
def vector(stat):
stat = unproxy(stat)
assert(stat.__scalar__() != stat.__vector__())
return stat.__vector__()
def unaryop(op, f):
if isinstance(f, FormulaStat):
v = f.value
else:
v = f
if isinstance(v, (list, tuple)):
return map(op, v)
else:
return op(v)
def zerodiv(lv, rv):
if rv == 0.0:
return 0.0
else:
return operator.truediv(lv, rv)
def wrapop(op, lv, rv):
if isinstance(lv, str):
return lv
if isinstance(rv, str):
return rv
return op(lv, rv)
def same(lrun, rrun):
for lx,rx in zip(lrun.keys(),rrun.keys()):
if lx != rx:
print 'lx != rx'
print lx, rx
print lrun.keys()
print rrun.keys()
return False
for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()):
if ly != ry:
print 'ly != ry'
print ly, ry
print lrun[lx].keys()
print rrun[rx].keys()
return False
return True
def binaryop(op, lf, rf):
result = {}
if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat):
lv = lf.value
rv = rf.value
theruns = []
for r in lv.keys():
if rv.has_key(r):
if same(lv[r], rv[r]):
theruns.append(r)
else:
raise AttributeError
for run in theruns:
result[run] = {}
for x in lv[run].keys():
result[run][x] = {}
for y in lv[run][x].keys():
result[run][x][y] = wrapop(op, lv[run][x][y],
rv[run][x][y])
elif isinstance(lf, FormulaStat):
lv = lf.value
for run in lv.keys():
result[run] = {}
for x in lv[run].keys():
result[run][x] = {}
for y in lv[run][x].keys():
result[run][x][y] = wrapop(op, lv[run][x][y], rf)
elif isinstance(rf, FormulaStat):
rv = rf.value
for run in rv.keys():
result[run] = {}
for x in rv[run].keys():
result[run][x] = {}
for y in rv[run][x].keys():
result[run][x][y] = wrapop(op, lf, rv[run][x][y])
def value(stat, *args):
stat = unproxy(stat)
return stat.__value__(*args)
def values(stat, run):
stat = unproxy(stat)
result = []
for i in xrange(len(stat)):
val = value(stat, run.run, i)
if val is None:
return None
result.append(val)
return result
def sums(x, y):
if isinstance(x, (list, tuple)):
return map(lambda x, y: x + y, x, y)
else:
return x + y
def total(stat, run):
return sum(values(stat, run))
def alltrue(seq):
return reduce(lambda x, y: x and y, seq)
def len(stat):
stat = unproxy(stat)
return stat.__len__()
def allfalse(seq):
return not reduce(lambda x, y: x or y, seq)
class Value(object):
def __scalar__(self):
raise AttributeError, "must define __scalar__ for %s" % (type (self))
def __vector__(self):
raise AttributeError, "must define __vector__ for %s" % (type (self))
def enumerate(seq):
return map(None, range(len(seq)), seq)
def __add__(self, other):
return BinaryProxy(operator.__add__, self, other)
def __sub__(self, other):
return BinaryProxy(operator.__sub__, self, other)
def __mul__(self, other):
return BinaryProxy(operator.__mul__, self, other)
def __div__(self, other):
return BinaryProxy(operator.__div__, self, other)
def __truediv__(self, other):
return BinaryProxy(operator.__truediv__, self, other)
def __floordiv__(self, other):
return BinaryProxy(operator.__floordiv__, self, other)
def cmp(a, b):
if a < b:
return -1
elif a == b:
return 0
else:
return 1
def __radd__(self, other):
return BinaryProxy(operator.__add__, other, self)
def __rsub__(self, other):
return BinaryProxy(operator.__sub__, other, self)
def __rmul__(self, other):
return BinaryProxy(operator.__mul__, other, self)
def __rdiv__(self, other):
return BinaryProxy(operator.__div__, other, self)
def __rtruediv__(self, other):
return BinaryProxy(operator.__truediv__, other, self)
def __rfloordiv__(self, other):
return BinaryProxy(operator.__floordiv__, other, self)
class Statistic(object):
def __neg__(self):
return UnaryProxy(operator.__neg__, self)
def __pos__(self):
return UnaryProxy(operator.__pos__, self)
def __abs__(self):
return UnaryProxy(operator.__abs__, self)
def __init__(self, data):
self.__dict__.update(data.__dict__)
if not self.__dict__.has_key('value'):
self.__dict__['value'] = None
if not self.__dict__.has_key('bins'):
self.__dict__['bins'] = None
if not self.__dict__.has_key('ticks'):
self.__dict__['ticks'] = None
if 'vc' not in self.__dict__:
self.vc = {}
class Scalar(Value):
def __scalar__(self):
return True
def __getattribute__(self, attr):
if attr == 'ticks':
if self.__dict__['ticks'] != globalTicks:
self.__dict__['value'] = None
self.__dict__['ticks'] = globalTicks
return self.__dict__['ticks']
if attr == 'value':
if self.__dict__['ticks'] != globalTicks:
if self.__dict__['ticks'] != None and \
len(self.__dict__['ticks']) == 1:
self.vc[self.__dict__['ticks'][0]] = self.__dict__['value']
self.__dict__['ticks'] = globalTicks
if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]):
self.__dict__['value'] = self.vc[globalTicks[0]]
else:
self.__dict__['value'] = None
if self.__dict__['value'] == None:
self.__dict__['value'] = self.getValue()
return self.__dict__['value']
else:
return super(Statistic, self).__getattribute__(attr)
def __setattr__(self, attr, value):
if attr == 'bins' or attr == 'ticks':
if attr == 'bins':
if value is not None:
value = source.getBin(value)
#elif attr == 'ticks' and type(value) is str:
# value = [ int(x) for x in value.split() ]
self.__dict__[attr] = value
self.__dict__['value'] = None
self.vc = {}
else:
super(Statistic, self).__setattr__(attr, value)
def getValue(self):
raise AttributeError, 'getValue() must be defined'
def zero(self):
def __vector__(self):
return False
def __ne__(self, other):
return not (self == other)
def __value__(self, run):
raise AttributeError, '__value__ must be defined'
def __str__(self):
return '%f' % (float(self))
class VectorItemProxy(Value):
def __init__(self, proxy, index):
self.proxy = proxy
self.index = index
class FormulaStat(object):
def __add__(self, other):
f = FormulaStat()
f.value = binaryop(operator.add, self, other)
return f
def __sub__(self, other):
f = FormulaStat()
f.value = binaryop(operator.sub, self, other)
return f
def __mul__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mul, self, other)
return f
def __truediv__(self, other):
f = FormulaStat()
f.value = binaryop(zerodiv, self, other)
return f
def __mod__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mod, self, other)
return f
def __radd__(self, other):
f = FormulaStat()
f.value = binaryop(operator.add, other, self)
return f
def __rsub__(self, other):
f = FormulaStat()
f.value = binaryop(operator.sub, other, self)
return f
def __rmul__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mul, other, self)
return f
def __rtruediv__(self, other):
f = FormulaStat()
f.value = binaryop(zerodiv, other, self)
return f
def __rmod__(self, other):
f = FormulaStat()
f.value = binaryop(operator.mod, other, self)
return f
def __neg__(self):
f = FormulaStat()
f.value = unaryop(operator.neg, self)
return f
def __getitem__(self, idx):
f = FormulaStat()
f.value = {}
for key in self.value.keys():
f.value[key] = {}
f.value[key][0] = {}
f.value[key][0][0] = self.value[key][idx][0]
return f
def __scalar__(self):
return True
def __float__(self):
if isinstance(self.value, FormulaStat):
return float(self.value)
if not self.value.has_key(display_run):
return (1e300*1e300)
if len(self.value[display_run]) == 1:
return self.value[display_run][0][0]
else:
#print self.value[display_run]
return self.value[display_run][4][0]
#raise ValueError
def __vector__(self):
return False
def display(self):
import display
d = display.VectorDisplay()
d.flags = 0
d.precision = 1
d.name = 'formula'
d.desc = 'formula'
val = self.value[display_run]
d.value = [ val[x][0] for x in val.keys() ]
d.display()
def __value__(self, run):
return value(self.proxy, run, self.index)
class Vector(Value):
def __scalar__(self):
return False
class Scalar(Statistic,FormulaStat):
def getValue(self):
return source.data(self, self.bins, self.ticks)
def __vector__(self):
return True
def display(self):
def __value__(self, run, index):
raise AttributeError, '__value__ must be defined'
def __getitem__(self, index):
return VectorItemProxy(self, index)
class ScalarConstant(Scalar):
def __init__(self, constant):
self.constant = constant
def __value__(self, run):
return self.constant
class VectorConstant(Vector):
def __init__(self, constant):
self.constant = constant
def __value__(self, run, index):
return self.constant[index]
def __len__(self):
return len(self.constant)
def WrapValue(value):
if isinstance(value, (int, long, float)):
return ScalarConstant(value)
if isinstance(value, (list, tuple)):
return VectorConstant(value)
if isinstance(value, Value):
return value
raise AttributeError, 'Only values can be wrapped'
class Statistic(object):
def __getattr__(self, attr):
if attr in ('data', 'x', 'y'):
result = self.source.data(self, self.bins, self.ticks)
self.data = result.data
self.x = result.x
self.y = result.y
return super(Statistic, self).__getattribute__(attr)
def __setattr__(self, attr, value):
if attr == 'stat':
raise AttributeError, '%s is read only' % stat
if attr in ('source', 'bins', 'ticks'):
if getattr(self, attr) != value:
if hasattr(self, 'data'):
delattr(self, 'data')
super(Statistic, self).__setattr__(attr, value)
class ValueProxy(Value):
def __getattr__(self, attr):
if attr == '__value__':
if scalar(self):
return self.__scalarvalue__
if vector(self):
return self.__vectorvalue__
if attr == '__len__':
if vector(self):
return self.__vectorlen__
return super(ValueProxy, self).__getattribute__(attr)
class UnaryProxy(ValueProxy):
def __init__(self, op, arg):
self.op = op
self.arg = WrapValue(arg)
def __scalar__(self):
return scalar(self.arg)
def __vector__(self):
return vector(self.arg)
def __scalarvalue__(self, run):
val = value(self.arg, run)
if val is None:
return None
return self.op(val)
def __vectorvalue__(self, run, index):
val = value(self.arg, run, index)
if val is None:
return None
return self.op(val)
def __vectorlen__(self):
return len(unproxy(self.arg))
class BinaryProxy(ValueProxy):
def __init__(self, op, arg0, arg1):
super(BinaryProxy, self).__init__()
self.op = op
self.arg0 = WrapValue(arg0)
self.arg1 = WrapValue(arg1)
def __scalar__(self):
return scalar(self.arg0) and scalar(self.arg1)
def __vector__(self):
return vector(self.arg0) or vector(self.arg1)
def __scalarvalue__(self, run):
val0 = value(self.arg0, run)
val1 = value(self.arg1, run)
if val0 is None or val1 is None:
return None
return self.op(val0, val1)
def __vectorvalue__(self, run, index):
if scalar(self.arg0):
val0 = value(self.arg0, run)
if vector(self.arg0):
val0 = value(self.arg0, run, index)
if scalar(self.arg1):
val1 = value(self.arg1, run)
if vector(self.arg1):
val1 = value(self.arg1, run, index)
if val0 is None or val1 is None:
return None
return self.op(val0, val1)
def __vectorlen__(self):
if vector(self.arg0) and scalar(self.arg1):
return len(self.arg0)
if scalar(self.arg0) and vector(self.arg1):
return len(self.arg1)
len0 = len(self.arg0)
len1 = len(self.arg1)
if len0 != len1:
raise AttributeError, \
"vectors of different lengths %d != %d" % (len0, len1)
return len0
class Proxy(Value):
def __init__(self, name, dict):
self.name = name
self.dict = dict
def __unproxy__(self):
return unproxy(self.dict[self.name])
def __getitem__(self, index):
return ItemProxy(self, index)
def __getattr__(self, attr):
return AttrProxy(self, attr)
class ItemProxy(Proxy):
def __init__(self, proxy, index):
self.proxy = proxy
self.index = index
def __unproxy__(self):
return unproxy(unproxy(self.proxy)[self.index])
class AttrProxy(Proxy):
def __init__(self, proxy, attr):
self.proxy = proxy
self.attr = attr
def __unproxy__(self):
return unproxy(getattr(unproxy(self.proxy), self.attr))
class ProxyGroup(object):
def __init__(self, dict=None, **kwargs):
self.__dict__['dict'] = {}
if dict is not None:
self.dict.update(dict)
if kwargs:
self.dict.update(kwargs)
def __getattr__(self, name):
return Proxy(name, self.dict)
def __setattr__(self, attr, value):
self.dict[attr] = value
class ScalarStat(Statistic,Scalar):
def __value__(self, run):
if run not in self.data:
return None
return self.data[run][0][0]
def display(self, run=None):
import display
p = display.Print()
p.name = self.name
p.desc = self.desc
p.value = float(self)
p.value = value(self, run)
p.flags = self.flags
p.precision = self.precision
if display.all or (self.flags & flags.printable):
p.display()
def comparable(self, other):
return self.name == other.name
class VectorStat(Statistic,Vector):
def __value__(self, run, item):
if run not in self.data:
return None
return self.data[run][item][0]
def __eq__(self, other):
return self.value == other.value
def __len__(self):
return self.x
def __isub__(self, other):
self.value -= other.value
return self
def __iadd__(self, other):
self.value += other.value
return self
def __itruediv__(self, other):
if not other:
return self
self.value /= other
return self
class Vector(Statistic,FormulaStat):
def getValue(self):
return source.data(self, self.bins, self.ticks);
def display(self):
def display(self, run=None):
import display
if not display.all and not (self.flags & flags.printable):
return
d = display.VectorDisplay()
d.__dict__.update(self.__dict__)
d.name = self.name
d.desc = self.desc
d.value = [ value(self, run, i) for i in xrange(len(self)) ]
d.flags = self.flags
d.precision = self.precision
d.display()
def comparable(self, other):
return self.name == other.name and \
len(self.value) == len(other.value)
class Formula(Value):
def __getattribute__(self, attr):
if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ):
return super(Formula, self).__getattribute__(attr)
def __eq__(self, other):
if isinstance(self.value, (list, tuple)) != \
isinstance(other.value, (list, tuple)):
return False
if isinstance(self.value, (list, tuple)):
if len(self.value) != len(other.value):
return False
else:
for v1,v2 in zip(self.value, other.value):
if v1 != v2:
return False
return True
else:
return self.value == other.value
def __isub__(self, other):
self.value = binaryop(operator.sub, self.value, other.value)
return self
def __iadd__(self, other):
self.value = binaryop(operator.add, self.value, other.value)
return self
def __itruediv__(self, other):
if not other:
return self
if isinstance(self.value, (list, tuple)):
for i in xrange(len(self.value)):
self.value[i] /= other
else:
self.value /= other
return self
class Formula(Vector):
def getValue(self):
formula = re.sub(':', '__', self.formula)
x = eval(formula, source.stattop)
return x.value
value = eval(formula, self.source.stattop)
return getattr(value, attr)
def comparable(self, other):
return self.name == other.name and \
compare(self.dist, other.dist)
def __eq__(self, other):
return self.value == other.value
def __isub__(self, other):
return self
def __iadd__(self, other):
return self
def __itruediv__(self, other):
if not other:
return self
return self
class SimpleDist(object):
class SimpleDist(Statistic):
def __init__(self, sums, squares, samples):
self.sums = sums
self.squares = squares
self.samples = samples
def getValue(self):
return 0.0
def display(self, name, desc, flags, precision):
import display
p = display.Print()
@ -482,9 +438,6 @@ class FullDist(SimpleDist):
self.bsize = bsize
self.size = size
def getValue(self):
return 0.0
def display(self, name, desc, flags, precision):
import display
p = display.Print()
@ -574,9 +527,6 @@ class FullDist(SimpleDist):
return self
class Dist(Statistic):
def getValue(self):
return 0.0
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@ -606,9 +556,6 @@ class Dist(Statistic):
return self
class VectorDist(Statistic):
def getValue(self):
return 0.0
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@ -694,9 +641,6 @@ class VectorDist(Statistic):
return self
class Vector2d(Statistic):
def getValue(self):
return 0.0
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@ -748,20 +692,25 @@ class Vector2d(Statistic):
return self
return self
def NewStat(data):
def NewStat(source, data):
stat = None
if data.type == 'SCALAR':
stat = Scalar(data)
stat = ScalarStat()
elif data.type == 'VECTOR':
stat = Vector(data)
stat = VectorStat()
elif data.type == 'DIST':
stat = Dist(data)
stat = Dist()
elif data.type == 'VECTORDIST':
stat = VectorDist(data)
stat = VectorDist()
elif data.type == 'VECTOR2D':
stat = Vector2d(data)
stat = Vector2d()
elif data.type == 'FORMULA':
stat = Formula(data)
stat = Formula()
stat.__dict__['source'] = source
stat.__dict__['bins'] = None
stat.__dict__['ticks'] = None
stat.__dict__.update(data.__dict__)
return stat

View file

@ -26,24 +26,8 @@
#
# Authors: Nathan Binkert
class dbinfo(object):
def get(self, job, stat):
import info
run = info.source.allRunNames.get(job.name, None)
if run is None:
print 'run "%s" not found' % job
return None
stat.system = info.source[job.system]
info.display_run = run.run;
val = float(stat)
if val == 1e300*1e300:
return None
return val
class StatOutput(object):
def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None):
def __init__(self, name, jobfile, info, stat=None, binstats=None):
self.name = name
self.jobfile = jobfile
self.stat = stat
@ -141,7 +125,7 @@ class StatOutput(object):
data = zeros((len(groupopts), len(baropts)), Float)
data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ]
enabled = False
stacked = None
stacked = 0
for g,gopt in enumerate(groupopts):
for b,bopt in enumerate(baropts):
job = self.jobfile.job(options + [ gopt, bopt ])
@ -149,17 +133,28 @@ class StatOutput(object):
continue
val = self.info.get(job, self.stat)
if val is None:
val = 0.0
curstacked = isinstance(val, (list, tuple))
if stacked is None:
stacked = curstacked
else:
if stacked != curstacked:
raise ValueError, "some stats stacked, some not"
if isinstance(val, (list, tuple)):
if len(val) == 1:
val = val[0]
else:
stacked = len(val)
data[g][b] = val
if stacked == 0:
for i in xrange(len(groupopts)):
for j in xrange(len(baropts)):
if data[i][j] is None:
data[i][j] = 0.0
else:
for i in xrange(len(groupopts)):
for j in xrange(len(baropts)):
val = data[i][j]
if val is None:
data[i][j] = [] * stacked
elif len(val) != stacked:
raise ValueError, "some stats stacked, some not"
data = array(data)
if data.sum() == 0:
continue
@ -167,7 +162,10 @@ class StatOutput(object):
bar_descs = [ opt.desc for opt in baropts ]
group_descs = [ opt.desc for opt in groupopts ]
if stacked:
legend = self.info.rcategories
try:
legend = self.info.rcategories
except:
legend = [ str(i) for i in xrange(stacked) ]
else:
legend = bar_descs

View file

@ -98,29 +98,27 @@ def commands(options, command, args):
raise CommandException
import db, info
info.source = db.Database()
info.source.host = options.host
info.source.db = options.db
info.source.passwd = options.passwd
info.source.user = options.user
info.source.connect()
#info.source.update_dict(globals())
import db
source = db.Database()
source.host = options.host
source.db = options.db
source.passwd = options.passwd
source.user = options.user
source.connect()
#source.update_dict(globals())
if type(options.get) is str:
info.source.get = options.get
if type(options.method) is str:
source.method = options.method
if options.runs is None:
runs = info.source.allRuns
runs = source.allRuns
else:
rx = re.compile(options.runs)
runs = []
for run in info.source.allRuns:
for run in source.allRuns:
if rx.match(run.name):
runs.append(run)
info.display_run = runs[0].run
if command == 'runs':
user = None
opts, args = getopts(args, '-u')
@ -129,14 +127,14 @@ def commands(options, command, args):
for o,a in opts:
if o == '-u':
user = a
info.source.listRuns(user)
source.listRuns(user)
return
if command == 'stats':
if len(args) == 0:
info.source.listStats()
source.listStats()
elif len(args) == 1:
info.source.listStats(args[0])
source.listStats(args[0])
else:
raise CommandException
@ -144,9 +142,9 @@ def commands(options, command, args):
if command == 'bins':
if len(args) == 0:
info.source.listBins()
source.listBins()
elif len(args) == 1:
info.source.listBins(args[0])
source.listBins(args[0])
else:
raise CommandException
@ -154,9 +152,9 @@ def commands(options, command, args):
if command == 'formulas':
if len(args) == 0:
info.source.listFormulas()
source.listFormulas()
elif len(args) == 1:
info.source.listFormulas(args[0])
source.listFormulas(args[0])
else:
raise CommandException
@ -166,7 +164,7 @@ def commands(options, command, args):
if len(args):
raise CommandException
info.source.listTicks(runs)
source.listTicks(runs)
return
if command == 'stability':
@ -177,8 +175,8 @@ def commands(options, command, args):
merge = int(args[0])
except ValueError:
usage()
stats = info.source.getStat(args[1])
info.source.get = "sum"
stats = source.getStat(args[1])
source.method = 'sum'
def disp(*args):
print "%-20s %12s %12s %4s %5s %5s %5s %10s" % args
@ -195,18 +193,17 @@ def commands(options, command, args):
#loop through all the selected runs
for run in runs:
info.display_run = run.run;
runTicks = info.source.retTicks([ run ])
runTicks = source.retTicks([ run ])
#throw away the first one, it's 0
runTicks.pop(0)
info.globalTicks = runTicks
source.ticks = runTicks
avg = 0
stdev = 0
numoutsideavg = 0
numoutside1std = 0
numoutside2std = 0
pairRunTicks = []
if float(stat) == 1e300*1e300:
if value(stat, run.run) == 1e300*1e300:
continue
for t in range(0, len(runTicks)-(merge-1), merge):
tempPair = []
@ -215,17 +212,17 @@ def commands(options, command, args):
pairRunTicks.append(tempPair)
#loop through all the various ticks for each run
for tick in pairRunTicks:
info.globalTicks = tick
avg += float(stat)
source.ticks = tick
avg += value(stat, run.run)
avg /= len(pairRunTicks)
for tick in pairRunTicks:
info.globalTicks = tick
val = float(stat)
source.ticks = tick
val = value(stat, run.run)
stdev += pow((val-avg),2)
stdev = math.sqrt(stdev / len(pairRunTicks))
for tick in pairRunTicks:
info.globalTicks = tick
val = float(stat)
source.ticks = tick
val = value(stat, run.run)
if (val < (avg * .9)) or (val > (avg * 1.1)):
numoutsideavg += 1
if (val < (avg - stdev)) or (val > (avg + stdev)):
@ -264,9 +261,9 @@ def commands(options, command, args):
if options.ticks:
if not options.graph:
print 'only displaying sample %s' % options.ticks
info.globalTicks = [ int(x) for x in options.ticks.split() ]
source.ticks = [ int(x) for x in options.ticks.split() ]
from output import StatOutput
import output
def display():
if options.graph:
@ -280,12 +277,12 @@ def commands(options, command, args):
raise CommandException
if command == 'stat':
stats = info.source.getStat(args[0])
stats = source.getStat(args[0])
if command == 'formula':
stats = eval(args[0])
for stat in stats:
output = StatOutput(stat.name, options.jobfile)
output = output.StatOutput(stat.name, options.jobfile, source)
output.stat = stat
output.label = stat.name
display()
@ -295,12 +292,11 @@ def commands(options, command, args):
if len(args):
raise CommandException
system = info.source.__dict__[options.system]
from proxy import ProxyGroup
sim_ticks = info.source['sim_ticks']
sim_seconds = info.source['sim_seconds']
proxy = ProxyGroup(system = info.source[options.system])
system = source.__dict__[options.system]
from info import ProxyGroup
sim_ticks = source['sim_ticks']
sim_seconds = source['sim_seconds']
proxy = ProxyGroup(system = source[options.system])
system = proxy.system
etherdev = system.tsunami.etherdev0
@ -309,7 +305,7 @@ def commands(options, command, args):
packets = etherdev.rxPackets + etherdev.txPackets
bps = etherdev.rxBandwidth + etherdev.txBandwidth
output = StatOutput(command, options.jobfile)
output = output.StatOutput(command, options.jobfile, source)
if command == 'usertime':
import copy
@ -460,7 +456,7 @@ if __name__ == '__main__':
options.user = getpass.getuser()
options.runs = None
options.system = 'client'
options.get = None
options.method = None
options.binned = False
options.graph = False
options.ticks = False
@ -469,7 +465,7 @@ if __name__ == '__main__':
options.jobfile = None
options.all = False
opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:')
opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:')
for o,a in opts:
if o == '-B':
options.binned = True
@ -477,8 +473,6 @@ if __name__ == '__main__':
options.printmode = 'E'
if o == '-F':
options.printmode = 'F'
if o == '-G':
options.get = a
if o == '-a':
options.all = True
if o == '-d':
@ -492,6 +486,8 @@ if __name__ == '__main__':
jobfilename = None
if o == '-j':
jobfilename = a
if o == '-m':
options.method = a
if o == '-p':
options.passwd = getpass.getpass()
if o == '-r':