diff --git a/util/pbs/job.py b/util/pbs/job.py index f370862de..e2636c111 100755 --- a/util/pbs/job.py +++ b/util/pbs/job.py @@ -29,10 +29,21 @@ # Steve Reinhardt # Ali Saidi -import os, os.path, shutil, signal, socket, sys, time +import os, os.path, shutil, signal, socket, sys from os import environ as env from os.path import join as joinpath, expanduser +def date(): + import time + return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime()) + +def cleandir(dir): + for root, dirs, files in os.walk(dir, False): + for name in files: + os.remove(joinpath(root, name)) + for name in dirs: + os.rmdir(joinpath(root, name)) + class rsync: def __init__(self): self.sudo = False @@ -61,25 +72,76 @@ class rsync: return os.spawnvp(os.P_WAIT, args[0], args) -def cleandir(dir): - for root, dirs, files in os.walk(dir, False): - for name in files: - os.remove(joinpath(root, name)) - for name in dirs: - os.rmdir(joinpath(root, name)) +class JobDir(object): + def __init__(self, dir): + self.dir = dir -def date(): - return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime()) + def file(self, filename): + return joinpath(self.dir, filename) -def remfile(file): - if os.path.isfile(file): - os.unlink(file) + def create(self): + if os.path.exists(self.dir): + if not os.path.isdir(self.dir): + sys.exit('%s is not a directory. Cannot build job' % self.dir) + else: + os.mkdir(self.dir) -def readval(filename): - file = open(filename, 'r') - value = file.readline().strip() - file.close() - return value + def exists(self): + return os.path.isdir(self.dir) + + def clean(self): + cleandir(self.dir) + + def hasfile(self, filename): + return os.path.isfile(self.file(filename)) + + def echofile(self, filename, string): + filename = self.file(filename) + try: + f = file(filename, 'w') + print >>f, string + f.flush() + f.close() + except IOError,e: + sys.exit(e) + + def rmfile(self, filename): + filename = self.file(filename) + if os.path.isfile(filename): + os.unlink(filename) + + def readval(self, filename): + filename = self.file(filename) + f = file(filename, 'r') + value = f.readline().strip() + f.close() + return value + + def setstatus(self, string): + filename = self.file('.status') + try: + f = file(filename, 'a') + print >>f, string + f.flush() + f.close() + except IOError,e: + sys.exit(e) + + def getstatus(self): + filename = self.file('.status') + try: + f = file(filename, 'r') + except IOError, e: + return 'none' + + # fast forward to the end + for line in f: pass + + # the first word on the last line is the status + return line.split(' ')[0] + + def __str__(self): + return self.dir if __name__ == '__main__': rootdir = env.setdefault('ROOTDIR', os.getcwd()) @@ -97,29 +159,27 @@ if __name__ == '__main__': workbase = "/tmp/" workdir = joinpath(workbase, '%s.%s' % (env['USER'], pbs_jobid)) - - def echofile(filename, string): - try: - f = file(joinpath(outdir, filename), 'w') - print >>f, string - f.flush() - f.close() - except IOError,e: - sys.exit(e) + host = socket.gethostname() os.umask(0022) - echofile('.start', date()) - echofile('.pbs_jobid', pbs_jobid) - echofile('.pbs_jobname', pbs_jobid) - echofile('.host', socket.gethostname()) + jobdir = JobDir(outdir) + + started = date() + jobdir.echofile('.running', started) + jobdir.rmfile('.queued') + jobdir.echofile('.pbs_jobid', pbs_jobid) + jobdir.echofile('.pbs_jobname', pbs_jobid) + jobdir.echofile('.host', host) + + jobdir.setstatus('running on %s on %s' % (host, started)) if os.path.isdir(workdir): cleandir(workdir) else: os.mkdir(workdir) - if os.path.isdir('/z/dist'): + if False and os.path.isdir('/z/dist'): sync = rsync() sync.delete = True sync.sudo = True @@ -130,13 +190,13 @@ if __name__ == '__main__': except OSError,e: sys.exit(e) - os.symlink(joinpath(outdir, 'output'), 'status.out') + os.symlink(jobdir.file('output'), 'status.out') args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.py') ] if not len(args): sys.exit("no arguments") - print 'starting job... %s' % date() + print 'starting job... %s' % started print ' '.join(args) print sys.stdout.flush() @@ -145,7 +205,7 @@ if __name__ == '__main__': if not childpid: # Execute command sys.stdin.close() - fd = os.open(joinpath(outdir, "output"), + fd = os.open(jobdir.file("output"), os.O_WRONLY | os.O_CREAT | os.O_TRUNC) os.dup2(fd, sys.stdout.fileno()) os.dup2(fd, sys.stderr.fileno()) @@ -170,12 +230,15 @@ if __name__ == '__main__': thepid,ec = os.waitpid(childpid, 0) if ec: print 'Exit code ', ec - echofile('.failure', date()) + status = 'failure' else: - echofile('.success', date()) + status = 'success' done = 1 except OSError: pass - print '\njob complete... %s' % date() - echofile('.stop', date()) + complete = date() + print '\njob complete... %s' % complete + jobdir.echofile('.%s' % status, complete) + jobdir.rmfile('.running') + jobdir.setstatus('%s on %s' % (status, complete)) diff --git a/util/pbs/jobfile.py b/util/pbs/jobfile.py index 83eb81358..d36b5ee6d 100644 --- a/util/pbs/jobfile.py +++ b/util/pbs/jobfile.py @@ -26,67 +26,485 @@ # # Authors: Nathan Binkert -from os.path import expanduser, isfile, join as joinpath import sys -def crossproduct(options): - number = len(options) - indexes = [ 0 ] * number - maxes = [ len(opt) for opt in options ] - def next(): - for i in xrange(number - 1, -1, -1): - indexes[i] += 1 - if indexes[i] < maxes[i]: - return False +class ternary(object): + def __new__(cls, *args): + if len(args) > 1: + raise TypeError, \ + '%s() takes at most 1 argument (%d given)' % \ + (cls.__name__, len(args)) - indexes[i] = 0 + if args: + if not isinstance(args[0], (bool, ternary)): + raise TypeError, \ + '%s() argument must be True, False, or Any' % \ + cls.__name__ + return args[0] + return super(ternary, cls).__new__(cls) + + def __bool__(self): return True - done = False - while not done: - result = [] - for i in xrange(number): - result.append(options[i][indexes[i]]) - yield result - done = next() + def __neg__(self): + return self -class JobFile(object): - def __init__(self, jfile): - self.data = {} - jfile = expanduser(jfile) - if not isfile(jfile): - for p in sys.path: - if isfile(joinpath(p, jfile)): - jfile = joinpath(p, jfile) - break + def __eq__(self, other): + return True - execfile(jfile, self.data) - self.options = self.data['options'] - self.environment = self.data['environment'] - self.jobinfo = {} - self.jobs = [] - for job in crossproduct(self.options): - jobname = '.'.join([ id[0] for id in job ]) - self.jobs.append(jobname) - list = [] - for info in job: - for item in info[1:]: - list.append(item) - self.jobinfo[jobname] = list + def __ne__(self, other): + return False - def env(self, jobname): - env = {} - for key,val in self.jobinfo[jobname]: - env[key] = val + def __str__(self): + return 'Any' - for key,val in self.environment: - env[key] = val - return env + def __repr__(self): + return 'Any' - def printinfo(self, jobname): - print '%s:' % jobname - for key,val in self.jobinfo[jobname]: - print ' %s = %s' % (key, val) +Any = ternary() - for key,val in self.environment: - print ' %s = %s' % (key, val) +class Flags(dict): + def __init__(self, *args, **kwargs): + super(Flags, self).__init__() + self.update(*args, **kwargs) + + def __getattr__(self, attr): + return self[attr] + + def __setattr__(self, attr, value): + self[attr] = value + + def __setitem__(self, item, value): + return super(Flags, self).__setitem__(item, ternary(value)) + + def __getitem__(self, item): + if item not in self: + return False + return super(Flags, self).__getitem__(item) + + def update(self, *args, **kwargs): + for arg in args: + if isinstance(arg, Flags): + super(Flags, self).update(arg) + elif isinstance(arg, dict): + for key,val in kwargs.iteritems(): + self[key] = val + else: + raise AttributeError, \ + 'flags not of type %s or %s, but %s' % \ + (Flags, dict, type(arg)) + + for key,val in kwargs.iteritems(): + self[key] = val + + def match(self, *args, **kwargs): + match = Flags(*args, **kwargs) + + for key,value in match.iteritems(): + if self[key] != value: + return False + + return True + +def crossproduct(items): + if not isinstance(items, (list, tuple)): + raise AttributeError, 'crossproduct works only on sequences' + + if not items: + yield None + return + + current = items[0] + remainder = items[1:] + + if not hasattr(current, '__iter__'): + current = [ current ] + + for item in current: + for rem in crossproduct(remainder): + data = [ item ] + if rem: + data += rem + yield data + +def flatten(items): + if not isinstance(items, (list, tuple)): + yield items + return + + for item in items: + for flat in flatten(item): + yield flat + +class Data(object): + def __init__(self, name, desc, **kwargs): + self.name = name + self.desc = desc + self.system = None + self.flags = Flags() + self.env = {} + for k,v in kwargs.iteritems(): + setattr(self, k, v) + + def update(self, obj): + if not isinstance(obj, Data): + raise AttributeError, "can only update from Data object" + + self.env.update(obj.env) + self.flags.update(obj.flags) + if obj.system: + if self.system and self.system != obj.system: + raise AttributeError, \ + "conflicting values for system: '%s'/'%s'" % \ + (self.system, obj.system) + self.system = obj.system + + def printinfo(self): + if self.name: + print 'name: %s' % self.name + if self.desc: + print 'desc: %s' % self.desc + if self.system: + print 'system: %s' % self.system + + def printverbose(self): + print 'flags:' + keys = self.flags.keys() + keys.sort() + for key in keys: + print ' %s = %s' % (key, self.flags[key]) + print 'env:' + keys = self.env.keys() + keys.sort() + for key in keys: + print ' %s = %s' % (key, self.env[key]) + print + + def __str__(self): + return self.name + +class Job(Data): + def __init__(self, options): + super(Job, self).__init__('', '') + self.setoptions(options) + + self.checkpoint = False + opts = [] + for opt in options: + cpt = opt.group.checkpoint + if not cpt: + self.checkpoint = True + continue + if isinstance(cpt, Option): + opt = cpt.clone(suboptions=False) + else: + opt = opt.clone(suboptions=False) + + opts.append(opt) + + if not opts: + self.checkpoint = False + + if self.checkpoint: + self.checkpoint = Job(opts) + + def clone(self): + return Job(self.options) + + def __getattribute__(self, attr): + if attr == 'name': + names = [ ] + for opt in self.options: + if opt.name: + names.append(opt.name) + return ':'.join(names) + + if attr == 'desc': + descs = [ ] + for opt in self.options: + if opt.desc: + descs.append(opt.desc) + return ', '.join(descs) + + return super(Job, self).__getattribute__(attr) + + def setoptions(self, options): + config = options[0].config + for opt in options: + if opt.config != config: + raise AttributeError, \ + "All options are not from the same Configuration" + + self.config = config + self.groups = [ opt.group for opt in options ] + self.options = options + + self.update(self.config) + for group in self.groups: + self.update(group) + + for option in self.options: + self.update(option) + if option._suboption: + self.update(option._suboption) + + def printinfo(self): + super(Job, self).printinfo() + if self.checkpoint: + print 'checkpoint: %s' % self.checkpoint.name + print 'config: %s' % self.config.name + print 'groups: %s' % [ g.name for g in self.groups ] + print 'options: %s' % [ o.name for o in self.options ] + super(Job, self).printverbose() + +class SubOption(Data): + def __init__(self, name, desc, **kwargs): + super(SubOption, self).__init__(name, desc, **kwargs) + self.number = None + +class Option(Data): + def __init__(self, name, desc, **kwargs): + super(Option, self).__init__(name, desc, **kwargs) + self._suboptions = [] + self._suboption = None + self.number = None + + def __getattribute__(self, attr): + if attr == 'name': + name = self.__dict__[attr] + if self._suboption is not None: + name = '%s:%s' % (name, self._suboption.name) + return name + + if attr == 'desc': + desc = self.__dict__[attr] + if self._suboption is not None: + desc = '%s, %s' % (desc, self._suboption.desc) + return desc + + return super(Option, self).__getattribute__(attr) + + def suboption(self, name, desc, **kwargs): + subo = SubOption(name, desc, **kwargs) + subo.config = self.config + subo.group = self.group + subo.option = self + subo.number = len(self._suboptions) + self._suboptions.append(subo) + return subo + + def clone(self, suboptions=True): + option = Option(self.__dict__['name'], self.__dict__['desc']) + option.update(self) + option.group = self.group + option.config = self.config + option.number = self.number + if suboptions: + option._suboptions.extend(self._suboptions) + option._suboption = self._suboption + return option + + def subopts(self): + if not self._suboptions: + return [ self ] + + subopts = [] + for subo in self._suboptions: + option = self.clone() + option._suboption = subo + subopts.append(option) + + return subopts + + def printinfo(self): + super(Option, self).printinfo() + print 'config: %s' % self.config.name + super(Option, self).printverbose() + +class Group(Data): + def __init__(self, name, desc, **kwargs): + super(Group, self).__init__(name, desc, **kwargs) + self._options = [] + self.checkpoint = False + self.number = None + + def option(self, name, desc, **kwargs): + opt = Option(name, desc, **kwargs) + opt.config = self.config + opt.group = self + opt.number = len(self._options) + self._options.append(opt) + return opt + + def options(self): + return self._options + + def subopts(self): + subopts = [] + for opt in self._options: + for subo in opt.subopts(): + subopts.append(subo) + return subopts + + def printinfo(self): + super(Group, self).printinfo() + print 'config: %s' % self.config.name + print 'options: %s' % [ o.name for o in self._options ] + super(Group, self).printverbose() + +class Configuration(Data): + def __init__(self, name, desc, **kwargs): + super(Configuration, self).__init__(name, desc, **kwargs) + self._groups = [] + + def group(self, name, desc, **kwargs): + grp = Group(name, desc, **kwargs) + grp.config = self + grp.number = len(self._groups) + self._groups.append(grp) + return grp + + def groups(self, flags=Flags(), sign=True): + if not flags: + return self._groups + + return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ] + + def checkchildren(self, kids): + for kid in kids: + if kid.config != self: + raise AttributeError, "child from the wrong configuration" + + def sortgroups(self, groups): + groups = [ (grp.number, grp) for grp in groups ] + groups.sort() + return [ grp[1] for grp in groups ] + + def options(self, groups = None, checkpoint = False): + if groups is None: + groups = self._groups + self.checkchildren(groups) + groups = self.sortgroups(groups) + if checkpoint: + groups = [ grp for grp in groups if grp.checkpoint ] + optgroups = [ g.options() for g in groups ] + else: + optgroups = [ g.subopts() for g in groups ] + for options in crossproduct(optgroups): + for opt in options: + cpt = opt.group.checkpoint + if not isinstance(cpt, bool) and cpt != opt: + if checkpoint: + break + else: + yield options + else: + if checkpoint: + yield options + + def checkpoints(self, groups = None): + for options in self.options(groups, True): + yield Job(options) + + def jobs(self, groups = None): + for options in self.options(groups, False): + yield Job(options) + + def alljobs(self, groups = None): + for options in self.options(groups, True): + yield Job(options) + for options in self.options(groups, False): + yield Job(options) + + def find(self, jobname): + for job in self.alljobs(): + if job.name == jobname: + return job + else: + raise AttributeError, "job '%s' not found" % jobname + + def job(self, options): + self.checkchildren(options) + options = [ (opt.group.number, opt) for opt in options ] + options.sort() + options = [ opt[1] for opt in options ] + job = Job(options) + return job + + def printinfo(self): + super(Configuration, self).printinfo() + print 'groups: %s' % [ g.name for g in self._grouips ] + super(Configuration, self).printverbose() + +def JobFile(jobfile): + from os.path import expanduser, isfile, join as joinpath + filename = expanduser(jobfile) + + # Can't find filename in the current path, search sys.path + if not isfile(filename): + for path in sys.path: + testname = joinpath(path, filename) + if isfile(testname): + filename = testname + break + else: + raise AttributeError, \ + "Could not find file '%s'" % jobfile + + data = {} + execfile(filename, data) + if 'conf' not in data: + raise ImportError, 'cannot import name conf from %s' % jobfile + conf = data['conf'] + import jobfile + if not isinstance(conf, Configuration): + raise AttributeError, \ + 'conf in jobfile: %s (%s) is not type %s' % \ + (jobfile, type(conf), Configuration) + return conf + +if __name__ == '__main__': + from jobfile import * + import sys + + usage = 'Usage: %s [-b] [-c] [-v] ' % sys.argv[0] + + try: + import getopt + opts, args = getopt.getopt(sys.argv[1:], '-bcv') + except getopt.GetoptError: + sys.exit(usage) + + if len(args) != 1: + raise AttributeError, usage + + both = False + checkpoint = False + verbose = False + for opt,arg in opts: + if opt == '-b': + both = True + checkpoint = True + if opt == '-c': + checkpoint = True + if opt == '-v': + verbose = True + + jobfile = args[0] + conf = JobFile(jobfile) + + if both: + gen = conf.alljobs() + elif checkpoint: + gen = conf.checkpoints() + else: + gen = conf.jobs() + + for job in gen: + if not verbose: + cpt = '' + if job.checkpoint: + cpt = job.checkpoint.name + print job.name, cpt + else: + job.printinfo() diff --git a/util/pbs/send.py b/util/pbs/send.py index ecb0be0ec..c66fb1c05 100755 --- a/util/pbs/send.py +++ b/util/pbs/send.py @@ -96,7 +96,7 @@ Usage: try: import getopt - opts, args = getopt.getopt(sys.argv[1:], '-cd:efhj:lq:v') + opts, args = getopt.getopt(sys.argv[1:], '-CRcd:efhj:lq:v') except getopt.GetoptError: sys.exit(usage) @@ -107,13 +107,18 @@ force = False listonly = False queue = '' verbose = False -rootdir = nfspath(os.getcwd()) -jfile = 'test.py' +jfile = 'Base/test.py' +docpts = False +doruns = True +runflag = False + for opt,arg in opts: + if opt == '-C': + docpts = True + if opt == '-R': + runflag = True if opt == '-c': clean = True - if opt == '-d': - rootdir = arg if opt == '-e': onlyecho = True if opt == '-f': @@ -130,95 +135,123 @@ for opt,arg in opts: if opt == '-v': verbose = True -basedir = joinpath(rootdir, 'Base') -linkdir = joinpath(rootdir, 'Link') +if docpts: + doruns = runflag for arg in args: exprs.append(re.compile(arg)) -if not listonly and not onlyecho and isdir(linkdir): +import jobfile, pbs +from job import JobDir, date + +conf = jobfile.JobFile(jfile) + +if not listonly and not onlyecho and isdir(conf.linkdir): if verbose: print 'Checking for outdated files in Link directory' - syncdir(linkdir, basedir) - -import job, jobfile, pbs - -test = jobfile.JobFile(joinpath(basedir, jfile)) + syncdir(conf.linkdir, conf.basedir) +jobnames = {} joblist = [] -for jobname in test.jobs: - if not exprs: - joblist.append(jobname) + +if docpts and doruns: + gen = conf.alljobs() +elif docpts: + gen = conf.checkpoints() +elif doruns: + gen = conf.jobs() + +for job in gen: + if job.name in jobnames: continue - for expr in exprs: - if expr.match(jobname): - joblist.append(jobname) - break + if exprs: + for expr in exprs: + if expr.match(job.name): + joblist.append(job) + break + else: + joblist.append(job) if listonly: if verbose: - for jobname in joblist: - test.printinfo(jobname) + for job in joblist: + job.printinfo() else: - for jobname in joblist: - print jobname + for job in joblist: + print job.name sys.exit(0) if not onlyecho: - jl = [] - for jobname in joblist: - jobdir = joinpath(rootdir, jobname) - if os.path.exists(jobname): + newlist = [] + for job in joblist: + jobdir = JobDir(joinpath(conf.rootdir, job.name)) + if jobdir.exists(): if not force: - if os.path.isfile(joinpath(jobdir, '.success')): + status = jobdir.getstatus() + if status == 'queued': continue - if os.path.isfile(joinpath(jobdir, '.start')) and \ - not os.path.isfile(joinpath(jobdir, '.stop')): + if status == 'running': + continue + + if status == 'success': continue if not clean: - sys.exit('job directory not clean!') + sys.exit('job directory %s not clean!' % jobdir) - job.cleandir(jobdir) - else: - os.mkdir(jobdir) - jl.append(jobname) - joblist = jl + jobdir.clean() + newlist.append(job) + joblist = newlist -def setname(jobid, jobname): - # since pbs can handle jobnames of 15 characters or less, don't - # use the raj hack. - if len(jobname) <= 15: - return +class NameHack(object): + def __init__(self, host='pbs.pool', port=24465): + self.host = host + self.port = port + self.socket = None - import socket - s = socket.socket() - # Connect to pbs.pool and send the jobid/jobname pair to port - # 24465 (Raj didn't realize that there are only 64k ports and - # setup inetd to point to port 90001) - s.connect(("pbs.pool", 24465)) - s.send("%s %s\n" % (jobid, jobname)) - s.close() + def setname(self, jobid, jobname): + try: + jobid = int(jobid) + except ValueError: + jobid = int(jobid.strip().split('.')[0]) -for jobname in joblist: - jobdir = joinpath(rootdir, jobname) + jobname = jobname.strip() + # since pbs can handle jobnames of 15 characters or less, + # don't use the raj hack. + if len(jobname) <= 15: + return - if not onlyecho and not os.path.isdir(jobdir): - sys.exit('%s is not a directory. Cannot build job' % jobdir) + if self.socket is None: + import socket + self.socket = socket.socket() + # Connect to pbs.pool and send the jobid/jobname pair to port + # 24465 (Raj didn't realize that there are only 64k ports and + # setup inetd to point to port 90001) + self.socket.connect((self.host, self.port)) - print 'Job name: %s' % jobname + self.socket.send("%s %s\n" % (jobid, jobname)) + +namehack = NameHack() + +for job in joblist: + jobdir = JobDir(joinpath(conf.rootdir, job.name)) + + if not onlyecho: + jobdir.create() + + print 'Job name: %s' % job.name print 'Job directory: %s' % jobdir qsub = pbs.qsub() qsub.pbshost = 'simpool.eecs.umich.edu' - qsub.stdout = joinpath(jobdir, 'jobout') - qsub.name = jobname[:15] + qsub.stdout = jobdir.file('jobout') + qsub.name = job.name[:15] qsub.join = True qsub.node_type = 'FAST' - qsub.env['ROOTDIR'] = rootdir - qsub.env['JOBNAME'] = jobname + qsub.env['ROOTDIR'] = conf.rootdir + qsub.env['JOBNAME'] = job.name if len(queue): qsub.queue = queue qsub.build(joinpath(progpath, 'job.py')) @@ -231,6 +264,9 @@ for jobname in joblist: if ec == 0: jobid = qsub.result print 'PBS Jobid: %s' % jobid - setname(jobid, jobname) + namehack.setname(jobid, job.name) + queued = date() + jobdir.echofile('.queued', queued) + jobdir.setstatus('queued on %s' % queued) else: print 'PBS Failed' diff --git a/util/stats/__init__.py b/util/stats/__init__.py new file mode 100644 index 000000000..b9968bf48 --- /dev/null +++ b/util/stats/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + diff --git a/util/stats/barchart.py b/util/stats/barchart.py new file mode 100644 index 000000000..a2cbea816 --- /dev/null +++ b/util/stats/barchart.py @@ -0,0 +1,246 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert +# Lisa Hsu + +import matplotlib, pylab +from matplotlib.numerix import array, arange, reshape, shape, transpose, zeros +from matplotlib.numerix import Float + +matplotlib.interactive(False) + +class BarChart(object): + def __init__(self, **kwargs): + self.init(**kwargs) + + def init(self, **kwargs): + self.colormap = 'jet' + self.inputdata = None + self.chartdata = None + self.xlabel = None + self.ylabel = None + self.legend = None + self.xticks = None + self.yticks = None + self.title = None + + for key,value in kwargs.iteritems(): + self.__setattr__(key, value) + + def gen_colors(self, count): + cmap = matplotlib.cm.get_cmap(self.colormap) + if count == 1: + return cmap([ 0.5 ]) + else: + return cmap(arange(count) / float(count - 1)) + + # The input data format does not match the data format that the + # graph function takes because it is intuitive. The conversion + # from input data format to chart data format depends on the + # dimensionality of the input data. Check here for the + # dimensionality and correctness of the input data + def set_data(self, data): + if data is None: + self.inputdata = None + self.chartdata = None + return + + data = array(data) + dim = len(shape(data)) + if dim not in (1, 2, 3): + raise AttributeError, "Input data must be a 1, 2, or 3d matrix" + self.inputdata = data + + # If the input data is a 1d matrix, then it describes a + # standard bar chart. + if dim == 1: + self.chartdata = array([[data]]) + + # If the input data is a 2d matrix, then it describes a bar + # chart with groups. The matrix being an array of groups of + # bars. + if dim == 2: + self.chartdata = transpose([data], axes=(2,0,1)) + + # If the input data is a 3d matrix, then it describes an array + # of groups of bars with each bar being an array of stacked + # values. + if dim == 3: + self.chartdata = transpose(data, axes=(1,2,0)) + + def get_data(self): + return self.inputdata + + data = property(get_data, set_data) + + # Graph the chart data. + # Input is a 3d matrix that describes a plot that has multiple + # groups, multiple bars in each group, and multiple values stacked + # in each bar. The underlying bar() function expects a sequence of + # bars in the same stack location and same group location, so the + # organization of the matrix is that the inner most sequence + # represents one of these bar groups, then those are grouped + # together to make one full stack of bars in each group, and then + # the outer most layer describes the groups. Here is an example + # data set and how it gets plotted as a result. + # + # e.g. data = [[[10,11,12], [13,14,15], [16,17,18], [19,20,21]], + # [[22,23,24], [25,26,27], [28,29,30], [31,32,33]]] + # + # will plot like this: + # + # 19 31 20 32 21 33 + # 16 28 17 29 18 30 + # 13 25 14 26 15 27 + # 10 22 11 23 12 24 + # + # Because this arrangement is rather conterintuitive, the rearrange + # function takes various matricies and arranges them to fit this + # profile. + # + # This code deals with one of the dimensions in the matrix being + # one wide. + # + def graph(self): + if self.chartdata is None: + raise AttributeError, "Data not set for bar chart!" + + self.figure = pylab.figure() + self.axes = self.figure.add_subplot(111) + + dim = len(shape(self.inputdata)) + cshape = shape(self.chartdata) + if dim == 1: + colors = self.gen_colors(cshape[2]) + colors = [ [ colors ] * cshape[1] ] * cshape[0] + + if dim == 2: + colors = self.gen_colors(cshape[0]) + colors = [ [ [ c ] * cshape[2] ] * cshape[1] for c in colors ] + + if dim == 3: + colors = self.gen_colors(cshape[1]) + colors = [ [ [ c ] * cshape[2] for c in colors ] ] * cshape[0] + + colors = array(colors) + + bars_in_group = len(self.chartdata) + if bars_in_group < 5: + width = 1.0 / ( bars_in_group + 1) + center = width / 2 + else: + width = .8 / bars_in_group + center = .1 + + bars = [] + for i,stackdata in enumerate(self.chartdata): + bottom = array([0] * len(stackdata[0])) + stack = [] + for j,bardata in enumerate(stackdata): + bardata = array(bardata) + ind = arange(len(bardata)) + i * width + center + bar = self.axes.bar(ind, bardata, width, bottom=bottom, + color=colors[i][j]) + stack.append(bar) + bottom += bardata + bars.append(stack) + + if self.xlabel is not None: + self.axes.set_xlabel(self.xlabel) + + if self.ylabel is not None: + self.axes.set_ylabel(self.ylabel) + + if self.yticks is not None: + ymin, ymax = self.axes.get_ylim() + nticks = float(len(self.yticks)) + ticks = arange(nticks) / (nticks - 1) * (ymax - ymin) + ymin + self.axes.set_yticks(ticks) + self.axes.set_yticklabels(self.yticks) + + if self.xticks is not None: + self.axes.set_xticks(arange(cshape[2]) + .5) + self.axes.set_xticklabels(self.xticks) + + if self.legend is not None: + if dim == 1: + lbars = bars[0][0] + if dim == 2: + lbars = [ bars[i][0][0] for i in xrange(len(bars))] + if dim == 3: + number = len(bars[0]) + lbars = [ bars[0][number - j - 1][0] for j in xrange(number)] + + self.axes.legend(lbars, self.legend, loc='best') + + if self.title is not None: + self.axes.set_title(self.title) + + def savefig(self, name): + self.figure.savefig(name) + +if __name__ == '__main__': + import random, sys + + dim = 3 + number = 5 + + args = sys.argv[1:] + if len(args) > 3: + sys.exit("invalid number of arguments") + elif len(args) > 0: + myshape = [ int(x) for x in args ] + else: + myshape = [ 3, 4, 8 ] + + # generate a data matrix of the given shape + size = reduce(lambda x,y: x*y, myshape) + #data = [ random.randrange(size - i) + 10 for i in xrange(size) ] + data = [ float(i)/100.0 for i in xrange(size) ] + data = reshape(data, myshape) + + # setup some test bar charts + if True: + chart1 = BarChart() + chart1.data = data + + chart1.xlabel = 'Benchmark' + chart1.ylabel = 'Bandwidth (GBps)' + chart1.legend = [ 'x%d' % x for x in xrange(myshape[-1]) ] + chart1.xticks = [ 'xtick%d' % x for x in xrange(myshape[0]) ] + chart1.title = 'this is the title' + chart1.graph() + #chart1.savefig('/tmp/test1.png') + + if False: + chart2 = BarChart() + chart2.data = data + chart2.colormap = 'gray' + chart2.graph() + #chart2.savefig('/tmp/test2.png') + + pylab.show() diff --git a/util/categories.py b/util/stats/categories.py similarity index 98% rename from util/categories.py rename to util/stats/categories.py index 877e548ed..0c359b856 100644 --- a/util/categories.py +++ b/util/stats/categories.py @@ -1,3 +1,31 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + categories = { 'CALL_PALrdunique_' : 'interrupt', # 'Call_Pal_Callsys' : 'interrupt', # diff --git a/util/stats/db.py b/util/stats/db.py index 6e1ccec95..ab005e67b 100644 --- a/util/stats/db.py +++ b/util/stats/db.py @@ -101,7 +101,7 @@ class Node(object): def __init__(self, name): self.name = name def __str__(self): - return name + return self.name class Database(object): def __init__(self): @@ -466,3 +466,6 @@ class Database(object): runs[data.run][data.x][data.y] = data.data return runs + + def __getitem__(self, key): + return self.stattop[key] diff --git a/util/stats/orderdict.py b/util/stats/orderdict.py new file mode 100644 index 000000000..816355ae2 --- /dev/null +++ b/util/stats/orderdict.py @@ -0,0 +1,78 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +__all__ = [ 'orderdict' ] + +class orderdict(dict): + def __init__(self, d = {}): + self._keys = d.keys() + super(orderdict, self).__init__(d) + + def __setitem__(self, key, item): + super(orderdict, self).__setitem__(key, item) + if not hasattr(self, '_keys'): + self._keys = [key,] + if key not in self._keys: + self._keys.append(key) + + def __delitem__(self, key): + super(orderdict, self).__delitem__(key) + self._keys.remove(key) + + def clear(self): + super(orderdict, self).clear() + self._keys = [] + + def items(self): + for i in self._keys: + yield i, self[i] + + def keys(self): + return self._keys + + def popitem(self): + if len(self._keys) == 0: + raise KeyError('dictionary is empty') + else: + key = self._keys[-1] + val = self[key] + del self[key] + return key, val + + def setdefault(self, key, failobj = None): + super(orderdict, self).setdefault(key, failobj) + if key not in self._keys: + self._keys.append(key) + + def update(self, d): + for key in d.keys(): + if not self.has_key(key): + self._keys.append(key) + super(orderdict, self).update(d) + + def values(self): + for i in self._keys: + yield self[i] diff --git a/util/stats/output.py b/util/stats/output.py new file mode 100644 index 000000000..44dba5d15 --- /dev/null +++ b/util/stats/output.py @@ -0,0 +1,182 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +class dbinfo(object): + def get(self, job, stat): + import info + + run = info.source.allRunNames.get(job.name, None) + if run is None: + print 'run "%s" not found' % job + return None + + stat.system = info.source[job.system] + info.display_run = run.run; + val = float(stat) + if val == 1e300*1e300: + return None + return val + +class StatOutput(object): + def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None): + self.name = name + self.jobfile = jobfile + self.stat = stat + self.binstats = None + self.label = self.name + self.invert = False + self.info = info + + def printdata(self, bin = None, printmode = 'G'): + import info + + if bin: + print '%s %s stats' % (self.name, bin) + + if self.binstats: + for stat in self.binstats: + stat.bins = bin + + if printmode == 'G': + valformat = '%g' + elif printmode != 'F' and value > 1e6: + valformat = '%0.5e' + else: + valformat = '%f' + + for job in self.jobfile.jobs(): + value = self.info.get(job, self.stat) + if value is None: + return + + if not isinstance(value, list): + value = [ value ] + + if self.invert: + for i,val in enumerate(value): + if val != 0.0: + value[i] = 1 / val + + valstring = ', '.join([ valformat % val for val in value ]) + print '%-50s %s' % (job.name + ':', valstring) + + def display(self, binned = False, printmode = 'G'): + if binned and self.binstats: + self.printdata('kernel', printmode) + self.printdata('idle', printmode) + self.printdata('user', printmode) + self.printdata('interrupt', printmode) + + print '%s total stats' % self.name + self.printdata(printmode=printmode) + + def graph(self, graphdir): + from os.path import expanduser, join as joinpath + from barchart import BarChart + from matplotlib.numerix import Float, zeros + import re + + confgroups = self.jobfile.groups() + ngroups = len(confgroups) + skiplist = [ False ] * ngroups + groupopts = None + baropts = None + groups = [] + for i,group in enumerate(confgroups): + if group.flags.graph_group: + if groupopts is not None: + raise AttributeError, \ + 'Two groups selected for graph group' + groupopts = group.subopts() + skiplist[i] = True + elif group.flags.graph_bars: + if baropts is not None: + raise AttributeError, \ + 'Two groups selected for graph bars' + baropts = group.subopts() + skiplist[i] = True + else: + groups.append(group) + + if groupopts is None: + raise AttributeError, 'No group selected for graph group' + + if baropts is None: + raise AttributeError, 'No group selected for graph bars' + + directory = expanduser(graphdir) + html = file(joinpath(directory, '%s.html' % self.name), 'w') + print >>html, '' + print >>html, 'Graphs for %s' % self.name + print >>html, '' + + for options in self.jobfile.options(groups): + data = zeros((len(groupopts), len(baropts)), Float) + data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ] + enabled = False + stacked = None + for g,gopt in enumerate(groupopts): + for b,bopt in enumerate(baropts): + job = self.jobfile.job(options + [ gopt, bopt ]) + + val = self.info.get(job, self.stat) + if val is None: + val = 0.0 + curstacked = isinstance(val, (list, tuple)) + if stacked is None: + stacked = curstacked + else: + if stacked != curstacked: + raise ValueError, "some stats stacked, some not" + + data[g][b] = val + + bar_descs = [ opt.desc for opt in baropts ] + group_descs = [ opt.desc for opt in groupopts ] + if stacked: + legend = self.info.rcategories + else: + legend = bar_descs + + chart = BarChart(data=data, xlabel='Benchmark', ylabel=self.label, + legend=legend, xticks=group_descs) + chart.graph() + + names = [ opt.name for opt in options ] + descs = [ opt.desc for opt in options ] + + filename = '%s-%s.png' % (self.name, ':'.join(names)) + desc = ' '.join(descs) + filepath = joinpath(directory, filename) + chart.savefig(filepath) + filename = re.sub(':', '%3A', filename) + print >>html, '''%s

''' % (desc, filename) + + print >>html, '' + print >>html, '' + html.close() diff --git a/util/stats/profile.py b/util/stats/profile.py new file mode 100644 index 000000000..65a03e9aa --- /dev/null +++ b/util/stats/profile.py @@ -0,0 +1,146 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from orderdict import orderdict +import output + +class ProfileData(object): + def __init__(self): + self.data = {} + self.total = {} + self.runs = orderdict() + self.runlist = [] + + def addvalue(self, run, cpu, symbol, value): + value = float(value) + self.data[run, cpu, symbol] = self.getvalue(run, cpu, symbol) + value + self.total[run, cpu] = self.gettotal(run, cpu) + value + if run not in self.runs: + self.runs[run] = orderdict() + + if cpu not in self.runs[run]: + self.runs[run][cpu] = {} + + if symbol not in self.runs[run][cpu]: + self.runs[run][cpu][symbol] = 0 + + self.runs[run][cpu][symbol] += value + + def getvalue(self, run, cpu, symbol): + return self.data.get((run, cpu, symbol), 0) + + def gettotal(self, run, cpu): + return self.total.get((run, cpu), 0) + +class Profile(object): + default_order = ['ste', 'hte', 'htd', 'ocm', 'occ', 'ocp'] + + # This list controls the order of values in stacked bar data output + default_categories = [ 'interrupt', + 'driver', + 'stack', + 'bufmgt', + 'copy', + 'user', + 'other', + 'idle'] + + def __init__(self, run_order=[], categories=[], stacknames=[]): + if not run_order: + run_order = Profile.default_order + if not categories: + categories = Profile.default_categories + + self.run_order = run_order + self.categories = categories + self.rcategories = [] + self.rcategories.extend(categories) + self.rcategories.reverse() + self.stacknames = stacknames + self.prof = ProfileData() + self.categorize = True + self.showidle = True + self.maxsymlen = 0 + + def category(self, symbol): + from categories import categories, categories_re + if categories.has_key(symbol): + return categories[symbol] + for regexp, cat in categories_re: + if regexp.match(symbol): + return cat + return 'other' + + # Parse input file and put the results in the given run and cpu + def parsefile(self, run, cpu, filename): + fd = file(filename) + + for line in fd: + (symbol, count) = line.split() + if symbol == "0x0": + continue + count = int(count) + + if self.categorize: + symbol = self.category(symbol) + if symbol == 'idle' and not self.showidle: + continue + + if symbol not in self.categories: + symbol = 'other' + + self.maxsymlen = max(self.maxsymlen, len(symbol)) + self.prof.addvalue(run, cpu, symbol, count) + + fd.close() + + # Read in files + def inputdir(self, directory): + import os, os.path, re + from os.path import expanduser, join as joinpath + + directory = expanduser(directory) + label_ex = re.compile(r'm5prof\.(.*)') + for root,dirs,files in os.walk(directory): + for name in files: + match = label_ex.match(name) + if not match: + continue + + filename = joinpath(root, name) + prefix = os.path.commonprefix([root, directory]) + dirname = root[len(prefix)+1:] + self.parsefile(dirname, match.group(1), filename) + + def get(self, job, stat): + if job.system is None: + raise AttributeError, 'The job must have a system set' + + cpu = '%s.full0' % job.system + values = [] + for cat in self.categories: + values.append(self.prof.getvalue(job.name, cpu, cat)) + return values diff --git a/util/stats/stats.py b/util/stats/stats.py index be9767d6e..c53d77d54 100755 --- a/util/stats/stats.py +++ b/util/stats/stats.py @@ -29,10 +29,9 @@ from __future__ import division import re, sys, math - def usage(): print '''\ -Usage: %s [-E] [-F] [-d ] [-g ] [-h ] [-p] +Usage: %s [-E] [-F] [ -G ] [-d ] [-g ] [-h ] [-p] [-s ] [-r ] [-T ] [-u ] [command args] @@ -61,143 +60,6 @@ def getopts(list, flags): return opts, args -def printval(name, value, invert = False): - if invert and value != 0.0: - value = 1 / value - - if value == (1e300*1e300): - return - - if printval.mode == 'G': - print '%s: %g' % (name, value) - elif printval.mode != 'F' and value > 1e6: - print '%s: %0.5e' % (name, value) - else: - print '%s: %f' % (name, value) - -printval.mode = 'G' - -def unique(list): - set = {} - map(set.__setitem__, list, []) - return set.keys() - -#benchmarks = [ 'm', 's', 'snt', 'nb1', 'w1', 'w2', 'w3', 'w4', 'nm', 'ns', 'nw1', 'nw2', 'nw3' ] - -def graphdata(runs, options, tag, label, value): - import info - - bench_system = { - 'm' : 'client', - 's' : 'client', - 'snt' : 'client', - 'nb1' : 'server', - 'nb2' : 'server', - 'nt1' : 'server', - 'nt2' : 'server', - 'w1' : 'server', - 'w2' : 'server', - 'w3' : 'server', - 'w4' : 'server', - 'w1s' : 'server', - 'w2s' : 'server', - 'w3s' : 'server', - 'ns' : 'natbox', - 'nm' : 'natbox', - 'nw1' : 'natbox', - 'nw2' : 'natbox', - 'nw3' : 'natbox' - } - - system_configs = { - 's1' : 'Uni 4GHz', - 'm1' : 'Uni 6GHz', - 'f1' : 'Uni 8GHz', - 'q1' : 'Uni 10GHz', - 's2' : 'Dual 4GHz', - 'm2' : 'Dual 6GHz', - 's4' : 'Quad 4GHz', - 'm4' : 'Quad 6GHz' } - - configs = ['ste', 'hte', 'htd', 'ocm', 'occ', 'ocp' ] - benchmarks = [ 'm', 'snt', 'w2', 'nm', 'nw2' ] - caches = [ '0', '2', '4' ] - - names = [] - for bench in benchmarks: - if bench_system[bench] != options.system: - continue - - for cache in caches: - names.append([bench, cache]) - - for bench,cache in names: - base = '%s.%s' % (bench, cache) - fname = 'data/uni.%s.%s.dat' % (tag, base) - f = open(fname, 'w') - print >>f, '#set TITLE = ' - print >>f, '#set ylbl = %s' % label - #print >>f, '#set sublabels = %s' % ' '.join(configs) - print >>f, '#set sublabels = ste hte htd ocm occ ocs' - - for speed in ('s1', 'm1', 'f1', 'q1'): - label = system_configs[speed] - print >>f, '"%s"' % label, - for conf in configs: - name = '%s.%s.%s.%s' % (conf, bench, cache, speed) - run = info.source.allRunNames[name] - info.display_run = run.run; - val = float(value) - if val == 1e300*1e300: - print >>f, 0.0, - else: - print >>f, "%f" % val, - print >>f - f.close() - - configs = ['ste', 'hte', 'htd', 'ocm', 'occ', 'ocp' ] - benchmarks = [ 'w2'] - caches = [ '0', '2', '4' ] - - names = [] - for bench in benchmarks: - if bench_system[bench] != options.system: - continue - - for cache in caches: - names.append([bench, cache]) - - for bench,cache in names: - base = '%s.%s' % (bench, cache) - fname = 'data/mp.%s.%s.dat' % (tag, base) - f = open(fname, 'w') - print >>f, '#set TITLE = ' - print >>f, '#set ylbl = %s' % label - #print >>f, '#set sublabels = %s' % ' '.join(configs) - print >>f, '#set sublabels = ste hte htd ocm occ ocs' - - for speed in ('s2', 'm2', 's4', 'm4'): - label = system_configs[speed] - print >>f, '"%s"' % label, - for conf in configs: - name = '%s.%s.%s.%s' % (conf, bench, cache, speed) - run = info.source.allRunNames[name] - info.display_run = run.run; - val = float(value) - if val == 1e300*1e300: - print >>f, 0.0, - else: - print >>f, "%f" % val, - print >>f - f.close() - -def printdata(runs, value, invert = False): - import info - for run in runs: - info.display_run = run.run; - val = float(value) - printval(run.name, val) - class CommandException(Exception): pass @@ -243,7 +105,7 @@ def commands(options, command, args): info.source.passwd = options.passwd info.source.user = options.user info.source.connect() - info.source.update_dict(globals()) + #info.source.update_dict(globals()) if type(options.get) is str: info.source.get = options.get @@ -270,6 +132,43 @@ def commands(options, command, args): info.source.listRuns(user) return + if command == 'stats': + if len(args) == 0: + info.source.listStats() + elif len(args) == 1: + info.source.listStats(args[0]) + else: + raise CommandException + + return + + if command == 'bins': + if len(args) == 0: + info.source.listBins() + elif len(args) == 1: + info.source.listBins(args[0]) + else: + raise CommandException + + return + + if command == 'formulas': + if len(args) == 0: + info.source.listFormulas() + elif len(args) == 1: + info.source.listFormulas(args[0]) + else: + raise CommandException + + return + + if command == 'samples': + if len(args): + raise CommandException + + info.source.listTicks(runs) + return + if command == 'stability': if len(args) < 2: raise CommandException @@ -281,16 +180,19 @@ def commands(options, command, args): stats = info.source.getStat(args[1]) info.source.get = "sum" + def disp(*args): + print "%-20s %12s %12s %4s %5s %5s %5s %10s" % args + + # temporary variable containing a bunch of dashes + d = '-' * 100 #loop through all the stats selected for stat in stats: - print "%s:" % stat.name - print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \ - ("run name", "average", "stdev", ">10%", ">1SDV", ">2SDV", "SAMP", "CV") - print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \ - ("--------------------", "------------", - "------------", "----", "-----", "-----", "-----", "----------") + disp("run name", "average", "stdev", ">10%", ">1SDV", ">2SDV", + "SAMP", "CV") + disp(d[:20], d[:12], d[:12], d[:4], d[:5], d[:5], d[:5], d[:10]) + #loop through all the selected runs for run in runs: info.display_run = run.run; @@ -331,130 +233,63 @@ def commands(options, command, args): if (val < (avg - (2*stdev))) or (val > (avg + (2*stdev))): numoutside2std += 1 if avg > 1000: - print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \ - (run.name, "%.1f" % avg, "%.1f" % stdev, - "%d" % numoutsideavg, "%d" % numoutside1std, - "%d" % numoutside2std, "%d" % len(pairRunTicks), - "%.3f" % (stdev/avg*100)) + disp(run.name, "%.1f" % avg, "%.1f" % stdev, + "%d" % numoutsideavg, "%d" % numoutside1std, + "%d" % numoutside2std, "%d" % len(pairRunTicks), + "%.3f" % (stdev/avg*100)) elif avg > 100: - print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \ - (run.name, "%.1f" % avg, "%.1f" % stdev, - "%d" % numoutsideavg, "%d" % numoutside1std, - "%d" % numoutside2std, "%d" % len(pairRunTicks), - "%.5f" % (stdev/avg*100)) + disp(run.name, "%.1f" % avg, "%.1f" % stdev, + "%d" % numoutsideavg, "%d" % numoutside1std, + "%d" % numoutside2std, "%d" % len(pairRunTicks), + "%.5f" % (stdev/avg*100)) else: - print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \ - (run.name, "%.5f" % avg, "%.5f" % stdev, - "%d" % numoutsideavg, "%d" % numoutside1std, - "%d" % numoutside2std, "%d" % len(pairRunTicks), - "%.7f" % (stdev/avg*100)) + disp(run.name, "%.5f" % avg, "%.5f" % stdev, + "%d" % numoutsideavg, "%d" % numoutside1std, + "%d" % numoutside2std, "%d" % len(pairRunTicks), + "%.7f" % (stdev/avg*100)) return - if command == 'stats': - if len(args) == 0: - info.source.listStats() - elif len(args) == 1: - info.source.listStats(args[0]) - else: - raise CommandException - - return - - if command == 'stat': - if len(args) != 1: - raise CommandException - - stats = info.source.getStat(args[0]) - for stat in stats: - if options.graph: - graphdata(runs, options, stat.name, stat.name, stat) - else: - if options.ticks: - print 'only displaying sample %s' % options.ticks - info.globalTicks = [ int(x) for x in options.ticks.split() ] - - if options.binned: - print 'kernel ticks' - stat.bins = 'kernel' - printdata(runs, stat) - - print 'idle ticks' - stat.bins = 'idle' - printdata(runs, stat) - - print 'user ticks' - stat.bins = 'user' - printdata(runs, stat) - - print 'interrupt ticks' - stat.bins = 'interrupt' - printdata(runs, stat) - - print 'total ticks' - - stat.bins = None - print stat.name - printdata(runs, stat) - return - - if command == 'formula': - if len(args) != 1: - raise CommandException - - stats = eval(args[0]) - for stat in stats: - if options.graph: - graphdata(runs, options, stat.name, stat.name, stat) - else: - if options.binned: - print 'kernel ticks' - stat.bins = 'kernel' - printdata(runs, stat) - - print 'idle ticks' - stat.bins = 'idle' - printdata(runs, stat) - - print 'user ticks' - stat.bins = 'user' - printdata(runs, stat) - - print 'interrupt ticks' - stat.bins = 'interrupt' - printdata(runs, stat) - - print 'total ticks' - - stat.bins = None - print args[0] - printdata(runs, stat) - return - - if command == 'bins': - if len(args) == 0: - info.source.listBins() - elif len(args) == 1: - info.source.listBins(args[0]) - else: - raise CommandException - - return - - if command == 'formulas': - if len(args) == 0: - info.source.listFormulas() - elif len(args) == 1: - info.source.listFormulas(args[0]) - else: - raise CommandException - - return - - if command == 'samples': + if command == 'all': if len(args): raise CommandException - info.source.listTicks(runs) + all = [ 'bps', 'rxbps', 'txbps', 'bpt', + 'misses', 'mpkb', + 'ipkb', + 'pps', 'bpp', 'txbpp', 'rxbpp', + 'rtp', 'rtb' ] + for command in all: + commands(options, command, args) + + if options.ticks: + if not options.graph: + print 'only displaying sample %s' % options.ticks + info.globalTicks = [ int(x) for x in options.ticks.split() ] + + from output import StatOutput + + def display(): + if options.graph: + output.graph(options.graphdir) + else: + output.display(options.binned, options.printmode) + + + if command == 'stat' or command == 'formula': + if len(args) != 1: + raise CommandException + + if command == 'stat': + stats = info.source.getStat(args[0]) + if command == 'formula': + stats = eval(args[0]) + + for stat in stats: + output = StatOutput(stat.name, options.jobfile) + output.stat = stat + output.label = stat.name + display() + return if len(args): @@ -462,273 +297,151 @@ def commands(options, command, args): system = info.source.__dict__[options.system] + from proxy import ProxyGroup + sim_ticks = info.source['sim_ticks'] + sim_seconds = info.source['sim_seconds'] + proxy = ProxyGroup(system = info.source[options.system]) + system = proxy.system + + etherdev = system.tsunami.etherdev0 + bytes = etherdev.rxBytes + etherdev.txBytes + kbytes = bytes / 1024 + packets = etherdev.rxPackets + etherdev.txPackets + bps = etherdev.rxBandwidth + etherdev.txBandwidth + + output = StatOutput(command, options.jobfile) + if command == 'usertime': import copy - kernel = copy.copy(system.full0.numCycles) - kernel.bins = 'kernel' - user = copy.copy(system.full0.numCycles) user.bins = 'user' - if options.graph: - graphdata(runs, options, 'usertime', 'User Fraction', - user / system.full0.numCycles) - else: - printdata(runs, user / system.full0.numCycles) + output.stat = user / system.full0.numCycles + output.label = 'User Fraction' + + display() return if command == 'ticks': - if options.binned: - print 'kernel ticks' - system.full0.numCycles.bins = 'kernel' - printdata(runs, system.full0.numCycles) - - print 'idle ticks' - system.full0.numCycles.bins = 'idle' - printdata(runs, system.full0.numCycles) - - print 'user ticks' - system.full0.numCycles.bins = 'user' - printdata(runs, system.full0.numCycles) - - print 'total ticks' - - system.full0.numCycles.bins = None - printdata(runs, system.full0.numCycles) - return - - if command == 'packets': - packets = system.tsunami.etherdev0.rxPackets - if options.graph: - graphdata(runs, options, 'packets', 'Packets', packets) - else: - printdata(runs, packets) - return - - if command == 'ppt' or command == 'tpp': - ppt = system.tsunami.etherdev0.rxPackets / sim_ticks - printdata(runs, ppt, command == 'tpp') - return - - if command == 'pps': - pps = system.tsunami.etherdev0.rxPackets / sim_seconds - if options.graph: - graphdata(runs, options, 'pps', 'Packets/s', pps) - else: - printdata(runs, pps) - return - - if command == 'bpt' or command == 'tpb': - bytes = system.tsunami.etherdev0.rxBytes + system.tsunami.etherdev0.txBytes - bpt = bytes / sim_ticks * 8 - if options.graph: - graphdata(runs, options, 'bpt', 'bps / Hz', bpt) - else: - printdata(runs, bpt, command == 'tpb') - return - - if command == 'bptb' or command == 'tpbb': - bytes = system.tsunami.etherdev0.rxBytes + system.tsunami.etherdev0.txBytes - - print 'kernel stats' - bytes.bins = 'kernel' - printdata(runs, bytes / ticks) - - print 'idle stats' - bytes.bins = 'idle' - printdata(runs, bytes / ticks) - - print 'user stats' - bytes.bins = 'user' - printdata(runs, bytes / ticks) + output.stat = system.full0.numCycles + output.binstats = [ system.full0.numCycles ] + display() return if command == 'bytes': - stat = system.tsunami.etherdev0.rxBytes + system.tsunami.etherdev0.txBytes + output.stat = bytes + display() + return - if options.binned: - print '%s kernel stats' % stat.name - stat.bins = 'kernel' - printdata(runs, stat) + if command == 'packets': + output.stat = packets + display() + return - print '%s idle stats' % stat.name - stat.bins = 'idle' - printdata(runs, stat) + if command == 'ppt' or command == 'tpp': + output.stat = packets / sim_ticks + output.invert = command == 'tpp' + display() + return - print '%s user stats' % stat.name - stat.bins = 'user' - printdata(runs, stat) + if command == 'pps': + output.stat = packets / sim_seconds + output.label = 'Packets/s' + display() + return - print '%s total stats' % stat.name - stat.bins = None - - printdata(runs, stat) + if command == 'bpt' or command == 'tpb': + output.stat = bytes / sim_ticks * 8 + output.label = 'bps / Hz' + output.invert = command == 'tpb' + display() return if command == 'rxbps': - gbps = system.tsunami.etherdev0.rxBandwidth / 1e9 - if options.graph: - graphdata(runs, options, 'rxbps', 'Bandwidth (Gbps)', gbps) - else: - printdata(runs, gbps) + output.stat = etherdev.rxBandwidth / 1e9 + output.label = 'Bandwidth (Gbps)' + display() return if command == 'txbps': - gbps = system.tsunami.etherdev0.txBandwidth / 1e9 - if options.graph: - graphdata(runs, options, 'txbps', 'Bandwidth (Gbps)', gbps) - else: - printdata(runs, gbps) + output.stat = etherdev.txBandwidth / 1e9 + output.label = 'Bandwidth (Gbps)' + display() return if command == 'bps': - rxbps = system.tsunami.etherdev0.rxBandwidth - txbps = system.tsunami.etherdev0.txBandwidth - gbps = (rxbps + txbps) / 1e9 - if options.graph: - graphdata(runs, options, 'bps', 'Bandwidth (Gbps)', gbps) - else: - printdata(runs, gbps) + output.stat = bps / 1e9 + output.label = 'Bandwidth (Gbps)' + display() return + if command == 'bpp': + output.stat = bytes / packets + output.label = 'Bytes / Packet' + display() + return + + if command == 'rxbpp': + output.stat = etherdev.rxBytes / etherdev.rxPackets + output.label = 'Receive Bytes / Packet' + display() + return + + if command == 'txbpp': + output.stat = etherdev.txBytes / etherdev.txPackets + output.label = 'Transmit Bytes / Packet' + display() + return + + if command == 'rtp': + output.stat = etherdev.rxPackets / etherdev.txPackets + output.label = 'rxPackets / txPackets' + display() + return + + if command == 'rtb': + output.stat = etherdev.rxBytes / etherdev.txBytes + output.label = 'rxBytes / txBytes' + display() + return + + misses = system.l2.overall_mshr_misses + if command == 'misses': - stat = system.l2.overall_mshr_misses - if options.binned: - print '%s kernel stats' % stat.name - stat.bins = 'kernel' - printdata(runs, stat) - - print '%s idle stats' % stat.name - stat.bins = 'idle' - printdata(runs, stat) - - print '%s user stats' % stat.name - stat.bins = 'user' - printdata(runs, stat) - - print '%s total stats' % stat.name - - stat.bins = None - if options.graph: - graphdata(runs, options, 'misses', 'Overall MSHR Misses', stat) - else: - printdata(runs, stat) + output.stat = misses + output.label = 'Overall MSHR Misses' + display() return if command == 'mpkb': - misses = system.l2.overall_mshr_misses - rxbytes = system.tsunami.etherdev0.rxBytes - txbytes = system.tsunami.etherdev0.txBytes - - if options.binned: - print 'mpkb kernel stats' - misses.bins = 'kernel' - mpkb = misses / ((rxbytes + txbytes) / 1024) - printdata(runs, mpkb) - - print 'mpkb idle stats' - misses.bins = 'idle' - mpkb = misses / ((rxbytes + txbytes) / 1024) - printdata(runs, mpkb) - - print 'mpkb user stats' - misses.bins = 'user' - mpkb = misses / ((rxbytes + txbytes) / 1024) - printdata(runs, mpkb) - - print 'mpkb total stats' - - mpkb = misses / ((rxbytes + txbytes) / 1024) - misses.bins = None - if options.graph: - graphdata(runs, options, 'mpkb', 'Misses / KB', mpkb) - else: - printdata(runs, mpkb) + output.stat = misses / (bytes / 1024) + output.binstats = [ misses ] + output.label = 'Misses / KB' + display() return if command == 'ipkb': interrupts = system.full0.kern.faults[4] - rxbytes = system.tsunami.etherdev0.rxBytes - txbytes = system.tsunami.etherdev0.txBytes - - if options.binned: - print 'ipkb kernel stats' - interrupts.bins = 'kernel' - ipkb = interrupts / ((rxbytes + txbytes) / 1024) - printdata(runs, ipkb) - - print 'ipkb idle stats' - interrupts.bins = 'idle' - ipkb = interrupts / ((rxbytes + txbytes) / 1024) - printdata(runs, ipkb) - - print 'ipkb user stats' - interrupts.bins = 'user' - ipkb = interrupts / ((rxbytes + txbytes) / 1024) - printdata(runs, ipkb) - - print 'ipkb total stats' - - ipkb = interrupts / ((rxbytes + txbytes) / 1024) - interrupts.bins = None - if options.graph: - graphdata(runs, options, 'ipkb', 'Interrupts / KB', ipkb) - else: - printdata(runs, ipkb) + output.stat = interrupts / kbytes + output.binstats = [ interrupts ] + output.label = 'Interrupts / KB' + display() return if command == 'execute': - printdata(runs, system.full0.ISSUE__count) + output.stat = system.full0.ISSUE__count + display() return if command == 'commit': - printdata(runs, system.full0.COM__count) + output.stat = system.full0.COM__count + display() return if command == 'fetch': - printdata(runs, system.full0.FETCH__count) - return - - if command == 'bpp': - ed = system.tsunami.etherdev0 - bpp = (ed.rxBytes + ed.txBytes) / (ed.rxPackets + ed.txPackets) - if options.graph: - graphdata(runs, options, 'bpp', 'Bytes / Packet', bpp) - else: - printdata(runs, bpp) - return - - if command == 'rxbpp': - bpp = system.tsunami.etherdev0.rxBytes / system.tsunami.etherdev0.rxPackets - if options.graph: - graphdata(runs, options, 'rxbpp', 'Receive Bytes / Packet', bpp) - else: - printdata(runs, bpp) - return - - if command == 'txbpp': - bpp = system.tsunami.etherdev0.txBytes / system.tsunami.etherdev0.txPackets - if options.graph: - graphdata(runs, options, 'txbpp', 'Transmit Bytes / Packet', bpp) - else: - printdata(runs, bpp) - return - - if command == 'rtp': - rtp = system.tsunami.etherdev0.rxPackets / system.tsunami.etherdev0.txPackets - if options.graph: - graphdata(runs, options, 'rtp', 'rxPackets / txPackets', rtp) - else: - printdata(runs, rtp) - return - - if command == 'rtb': - rtb = system.tsunami.etherdev0.rxBytes / system.tsunami.etherdev0.txBytes - if options.graph: - graphdata(runs, options, 'rtb', 'rxBytes / txBytes', rtb) - else: - printdata(runs, rtb) + output.stat = system.full0.FETCH__count + display() return raise CommandException @@ -738,9 +451,10 @@ class Options: pass if __name__ == '__main__': import getpass + from jobfile import JobFile options = Options() - options.host = 'zizzer.pool' + options.host = None options.db = None options.passwd = '' options.user = getpass.getuser() @@ -750,23 +464,31 @@ if __name__ == '__main__': options.binned = False options.graph = False options.ticks = False + options.printmode = 'G' + options.jobfile = None + options.all = False - opts, args = getopts(sys.argv[1:], '-6BEFGd:g:h:pr:s:u:T:') + opts, args = getopts(sys.argv[1:], '-BEFG:ad:g:h:j:pr:s:u:T:') for o,a in opts: if o == '-B': options.binned = True if o == '-E': - printval.mode = 'E' + options.printmode = 'E' if o == '-F': - printval.mode = 'F' + options.printmode = 'F' if o == '-G': - options.graph = True; + options.get = a + if o == '-a': + options.all = True if o == '-d': options.db = a if o == '-g': - options.get = a + options.graph = True; + options.graphdir = a if o == '-h': options.host = a + if o == '-j': + options.jobfile = JobFile(a) if o == '-p': options.passwd = getpass.getpass() if o == '-r': @@ -778,6 +500,18 @@ if __name__ == '__main__': if o == '-T': options.ticks = a + if options.jobfile: + if not options.host: + options.host = options.jobfile.dbhost + if not options.db: + options.db = options.jobfile.statdb + + if not options.host: + sys.exit('Database server must be provided from a jobfile or -h') + + if not options.db: + sys.exit('Database name must be provided from a jobfile or -d') + if len(args) == 0: usage()