Make pbs submission scripts available to all.

Fix up configuration scrupts to have better support for
running on the simulation pool.

--HG--
extra : convert_revision : 0178c8600b193d6c0ca69163fb735a7fa0e70782
This commit is contained in:
Nathan Binkert 2005-02-05 13:49:17 -05:00
parent b78d7c2f16
commit fc64ab8102
4 changed files with 561 additions and 0 deletions

183
util/pbs/job.py Executable file
View file

@ -0,0 +1,183 @@
#!/usr/bin/env python
# Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
# Steve Reinhardt
# Ali Saidi
import os, os.path, shutil, signal, socket, sys, time
from os import environ as env
from os.path import join as joinpath, expanduser
class rsync:
def __init__(self):
self.sudo = False
self.rsync = 'rsync'
self.compress = False
self.archive = True
self.delete = False
self.options = ''
def do(self, src, dst):
args = []
if self.sudo:
args.append('sudo')
args.append(self.rsync)
if (self.archive):
args.append('-a')
if (self.compress):
args.append('-z')
if (self.delete):
args.append('--delete')
if len(self.options):
args.append(self.options)
args.append(src)
args.append(dst)
return os.spawnvp(os.P_WAIT, args[0], args)
def cleandir(dir):
for root, dirs, files in os.walk(dir, False):
for name in files:
os.remove(joinpath(root, name))
for name in dirs:
os.rmdir(joinpath(root, name))
def date():
return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime())
def remfile(file):
if os.path.isfile(file):
os.unlink(file)
def readval(filename):
file = open(filename, 'r')
value = file.readline().strip()
file.close()
return value
if __name__ == '__main__':
rootdir = env.setdefault('ROOTDIR', os.getcwd())
jobid = env['PBS_JOBID']
jobname = env['PBS_JOBNAME']
jobdir = joinpath(rootdir, jobname)
basedir = joinpath(rootdir, 'Base')
user = env['USER']
env['POOLJOB'] = 'True'
env['OUTPUT_DIR'] = jobdir
env['JOBFILE'] = joinpath(basedir, 'test.py')
env['JOBNAME'] = jobname
def echofile(filename, string):
try:
f = file(joinpath(jobdir, filename), 'w')
print >>f, string
f.flush()
f.close()
except IOError,e:
sys.exit(e)
if os.path.isdir("/work"):
workbase = "/work"
else:
workbase = "/tmp/"
workdir = joinpath(workbase, '%s.%s' % (user, jobid))
os.umask(0022)
echofile('.start', date())
echofile('.jobid', jobid)
echofile('.host', socket.gethostname())
if os.path.isdir(workdir):
cleandir(workdir)
else:
os.mkdir(workdir)
if os.path.isdir('/z/dist'):
sync = rsync()
sync.delete = True
sync.sudo = True
sync.do('poolfs::dist/m5/', '/z/dist/m5/')
try:
os.chdir(workdir)
except OSError,e:
sys.exit(e)
os.symlink(joinpath(jobdir, 'output'), 'status.out')
args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.mpy') ]
if not len(args):
sys.exit("no arguments")
print 'starting job... %s' % date()
print ' '.join(args)
print
sys.stdout.flush()
childpid = os.fork()
if not childpid:
# Execute command
sys.stdin.close()
fd = os.open(joinpath(jobdir, "output"),
os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
os.dup2(fd, sys.stdout.fileno())
os.dup2(fd, sys.stderr.fileno())
os.execvp(args[0], args)
def handler(signum, frame):
if childpid != 0:
os.kill(childpid, signum)
signal.signal(signal.SIGHUP, handler)
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGQUIT, handler)
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGSTOP, handler)
signal.signal(signal.SIGCONT, handler)
signal.signal(signal.SIGUSR1, handler)
signal.signal(signal.SIGUSR2, handler)
done = 0
while not done:
try:
thepid,ec = os.waitpid(childpid, 0)
if ec:
print 'Exit code ', ec
echofile('.failure', date())
else:
echofile('.success', date())
done = 1
except OSError:
pass
print '\njob complete... %s' % date()
echofile('.stop', date())

83
util/pbs/jobfile.py Normal file
View file

@ -0,0 +1,83 @@
# Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
from os.path import expanduser
def crossproduct(options):
number = len(options)
indexes = [ 0 ] * number
maxes = [ len(opt) for opt in options ]
def next():
for i in xrange(number - 1, -1, -1):
indexes[i] += 1
if indexes[i] < maxes[i]:
return False
indexes[i] = 0
return True
done = False
while not done:
result = []
for i in xrange(number):
result.append(options[i][indexes[i]])
yield result
done = next()
class JobFile(object):
def __init__(self, file):
self.data = {}
execfile(expanduser(file), self.data)
self.options = self.data['options']
self.environment = self.data['environment']
self.jobinfo = {}
self.jobs = []
for job in crossproduct(self.options):
jobname = '.'.join([ id[0] for id in job ])
self.jobs.append(jobname)
list = []
for info in job:
for item in info[1:]:
list.append(item)
self.jobinfo[jobname] = list
def env(self, jobname):
env = {}
for key,val in self.jobinfo[jobname]:
env[key] = val
for key,val in self.environment:
env[key] = val
return env
def printinfo(self, jobname):
print '%s:' % jobname
for key,val in self.jobinfo[jobname]:
print ' %s = %s' % (key, val)
for key,val in self.environment:
print ' %s = %s' % (key, val)

126
util/pbs/pbs.py Executable file
View file

@ -0,0 +1,126 @@
# Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
import os, re, sys
def ssh(host, script, tty = False, user = ''):
args = [ 'ssh', '-x' ]
if user:
args.append('-l' + user)
if tty:
args.append('-t')
args.append(host)
args.append(script)
return os.spawnvp(os.P_WAIT, args[0], args)
class qsub:
def __init__(self):
self.hold = False
self.join = False
self.keep_stdout = False
self.keep_stderr = False
self.node_type = ''
self.mail_abort = False
self.mail_begin = False
self.mail_end = False
self.name = ''
self.stdout = ''
self.priority = 0
self.queue = ''
self.pbshost = ''
self.qsub = 'qsub'
self.env = {}
self.onlyecho = False
self.verbose = False
def do(self, script, ):
args = [self.qsub]
if self.env:
arg = '-v'
arg += ','.join([ '%s=%s' % i for i in self.env.iteritems() ])
args.append(arg)
if self.hold:
args.append('-h')
if len(self.stdout):
args.append('-olocalhost:' + self.stdout)
if self.keep_stdout and self.keep_stderr:
args.append('-koe')
elif self.keep_stdout:
args.append('-ko')
elif self.keep_stderr:
args.append('-ke')
else:
args.append('-kn')
if self.join:
args.append('-joe')
if len(self.node_type):
args.append('-lnodes=' + self.node_type)
if self.mail_abort or self.mail_begin or self.mail_end:
flags = ''
if self.mail_abort:
flags.append('a')
if self.mail_begin:
flags.append('b')
if self.mail_end:
flags.append('e')
if len(flags):
args.append('-m ' + flags)
if len(self.name):
args.append("-N%s" % self.name)
if self.priority != 0:
args.append('-p' + self.priority)
if len(self.queue):
args.append('-q' + self.queue)
args.append(script)
if self.verbose or self.onlyecho:
print >>sys.stderr, 'PBS Command: ', ' '.join(args)
if self.onlyecho:
return 0
print >>sys.stderr, 'PBS Jobid: ',
ec = os.spawnvp(os.P_WAIT, args[0], args)
if ec != 0 and len(self.pbshost):
ec = ssh(self.pbshost, ' '.join(args))
return ec

169
util/pbs/send.py Executable file
View file

@ -0,0 +1,169 @@
#!/usr/bin/env python
# Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ali Saidi
# Nathan Binkert
import os, os.path, re, sys
from os import environ as env, listdir
from os.path import basename, isdir, isfile, islink, join as joinpath
from filecmp import cmp as filecmp
from shutil import copyfile
progname = basename(sys.argv[0])
usage = """\
Usage:
%(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
-c clean directory if job can be run
-e only echo pbs command info, don't actually send the job
-f force the job to run regardless of state
-q <queue> submit job to the named queue
-v be verbose
%(progname)s -l [-v] <regexp>
-l list job names, don't submit
-v be verbose (list job parameters)
%(progname)s -h
-h display this help
""" % locals()
try:
import getopt
opts, args = getopt.getopt(sys.argv[1:], '-cefhlq:v')
except getopt.GetoptError:
sys.exit(usage)
clean = False
onlyecho = False
exprs = []
force = False
listonly = False
queue = ''
verbose = False
for o,a in opts:
if o == '-c':
clean = True
if o == '-e':
onlyecho = True
if o == '-f':
force = True
if o == '-h':
print usage
sys.exit(0)
if o == '-l':
listonly = True
if o == '-q':
queue = a
if o == '-v':
verbose = True
for arg in args:
exprs.append(re.compile(arg))
if not listonly and not onlyecho and isdir('Link'):
print 'Checking for outdated files in Link directory'
entries = listdir('Link')
for entry in entries:
link = joinpath('Link', entry)
if not islink(link):
continue
base = joinpath('Base', entry)
if not isfile(base) or not filecmp(link, base):
print '%s is different than source %s...copying' % (base, link)
copyfile(link, base)
import job, jobfile, pbs
test = jobfile.JobFile(joinpath('Base', 'test.py'))
joblist = []
for jobname in test.jobs:
if not exprs:
joblist.append(jobname)
continue
for expr in exprs:
if expr.match(jobname):
joblist.append(jobname)
break
if listonly:
if verbose:
for jobname in joblist:
test.printinfo(jobname)
else:
for jobname in joblist:
print jobname
sys.exit(0)
if not onlyecho:
jl = []
for jobname in joblist:
if os.path.exists(jobname):
if not force:
if os.path.isfile(joinpath(jobname, '.success')):
continue
if os.path.isfile(joinpath(jobname, '.start')) and \
not os.path.isfile(joinpath(jobname, '.stop')):
continue
if not clean:
sys.exit('job directory not clean!')
job.cleandir(jobname)
else:
os.mkdir(jobname)
jl.append(jobname)
joblist = jl
rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
for jobname in joblist:
jobdir = joinpath(rootdir, jobname)
if not onlyecho and not os.path.isdir(jobdir):
sys.exit('%s is not a directory. Cannot build job' % jobdir)
print >>sys.stderr, 'Job name: %s' % jobname
print >>sys.stderr, 'Job directory: %s' % jobdir
qsub = pbs.qsub()
qsub.pbshost = 'simpool.eecs.umich.edu'
qsub.stdout = joinpath(jobdir, 'jobout')
qsub.name = jobname
qsub.join = True
qsub.node_type = 'FAST'
qsub.onlyecho = onlyecho
qsub.env['ROOTDIR'] = rootdir
qsub.verbose = verbose
if len(queue):
qsub.queue = queue
qsub.do(joinpath('Base', 'job.py'))
print >>sys.stderr, ''