More fixes to the pbs stuff to make it more robust.

sim/pyconfig/SConscript:
    Embed the jobfile.py script into the binary so that we don't
    need to copy it into the Base directory every time.
test/genini.py:
    Add the util/pbs directory to the path so we can get to
    jobfile.py
    Add a -I argument to set to add to the path.
util/pbs/pbs.py:
    Create a MyPOpen class.  This is a lot like the popen2.Popen3 class
    in the python library except that my version allows redirection of
    standard in and standard out to a file instead of a pipe.
    Use this popen class to execute qsub or ssh qsub.  This was important
    for the ssh version of qsub because we need to pipe the script into
    standard in of ssh so that the script can get to the qsub command.
    (Otherwise we have a problem discovering the path.)
util/pbs/send.py:
    Tweak the script so it figures out paths in NFS correctly.
    Use the new system for running qsub.

--HG--
extra : convert_revision : 1289915ba99cec6fd464b71215c32d2197ff2824
This commit is contained in:
Nathan Binkert 2005-02-09 23:55:21 -05:00
parent 4a3e33fb6d
commit 8efd7d9063
4 changed files with 132 additions and 56 deletions

View file

@ -170,7 +170,7 @@ EmbedMap %(name)s("%(fname)s",
/* namespace */ } /* namespace */ }
''' '''
embedded_py_files = ['m5config.py'] embedded_py_files = ['m5config.py', '../../util/pbs/jobfile.py']
objpath = os.path.join(env['SRCDIR'], 'objects') objpath = os.path.join(env['SRCDIR'], 'objects')
for root, dirs, files in os.walk(objpath, topdown=True): for root, dirs, files in os.walk(objpath, topdown=True):
for i,dir in enumerate(dirs): for i,dir in enumerate(dirs):

View file

@ -31,23 +31,26 @@ from os.path import join as joinpath, realpath
mypath = sys.path[0] mypath = sys.path[0]
sys.path.append(joinpath(mypath, '..')) sys.path.append(joinpath(mypath, '..'))
sys.path.append(joinpath(mypath, '../configs/kernel')) sys.path.append(joinpath(mypath, '../configs/kernel'))
sys.path.append(joinpath(mypath, '../util/pbs'))
sys.path.append(joinpath(mypath, '../sim/pyconfig')) sys.path.append(joinpath(mypath, '../sim/pyconfig'))
from importer import mpy_exec, mpy_execfile, AddToPath from importer import mpy_exec, mpy_execfile, AddToPath
from m5config import * from m5config import *
try: try:
opts, args = getopt.getopt(sys.argv[1:], '-E:') opts, args = getopt.getopt(sys.argv[1:], '-E:I:')
for o,a in opts: for opt,arg in opts:
if o == '-E': if opt == '-E':
offset = a.find('=') offset = arg.find('=')
if offset == -1: if offset == -1:
name = a name = arg
value = True value = True
else: else:
name = a[:offset] name = arg[:offset]
value = a[offset+1:] value = arg[offset+1:]
env[name] = value env[name] = value
if opt == '-I':
AddToPath(arg)
except getopt.GetoptError: except getopt.GetoptError:
sys.exit('Improper Usage') sys.exit('Improper Usage')

View file

@ -26,18 +26,76 @@
# #
# Authors: Nathan Binkert # Authors: Nathan Binkert
import os, re, sys import os, popen2, re, sys
def ssh(host, script, tty = False, user = ''): class MyPOpen(object):
args = [ 'ssh', '-x' ] def __init__(self, cmd, input = None, output = None, bufsize = -1):
if user: self.sts = -1
args.append('-l' + user)
if tty:
args.append('-t')
args.append(host)
args.append(script)
return os.spawnvp(os.P_WAIT, args[0], args) if input is None:
p2c_read, p2c_write = os.pipe()
self.tochild = os.fdopen(p2c_write, 'w', bufsize)
else:
p2c_write = None
if isinstance(input, file):
p2c_read = input.fileno()
elif isinstance(input, str):
input = file(input, 'r')
p2c_read = input.fileno()
elif isinstance(input, int):
p2c_read = input
else:
raise AttributeError
if output is None:
c2p_read, c2p_write = os.pipe()
self.fromchild = os.fdopen(c2p_read, 'r', bufsize)
else:
c2p_read = None
if isinstance(output, file):
c2p_write = output.fileno()
elif isinstance(output, str):
output = file(output, 'w')
c2p_write = output.fileno()
elif isinstance(output, int):
c2p_write = output
else:
raise AttributeError
self.pid = os.fork()
if self.pid == 0:
os.dup2(p2c_read, 0)
os.dup2(c2p_write, 1)
os.dup2(c2p_write, 2)
if isinstance(cmd, basestring):
cmd = ['/bin/sh', '-c', cmd]
if False:
for i in range(3, MAXFD):
try:
os.close(i)
except OSError:
pass
try:
os.execvp(cmd[0], cmd)
finally:
os._exit(1)
os.close(p2c_read)
os.close(c2p_write)
def poll(self):
if self.sts < 0:
pid, sts = os.waitpid(self.pid, os.WNOHANG)
if pid == self.pid:
self.sts = sts
return self.sts
def wait(self):
if self.sts < 0:
pid, sts = os.waitpid(self.pid, 0)
if pid == self.pid:
self.sts = sts
return self.sts
class qsub: class qsub:
def __init__(self): def __init__(self):
@ -56,37 +114,35 @@ class qsub:
self.pbshost = '' self.pbshost = ''
self.qsub = 'qsub' self.qsub = 'qsub'
self.env = {} self.env = {}
self.onlyecho = False
self.verbose = False
def do(self, script, ): def build(self, script, args = []):
args = [self.qsub] self.cmd = [ self.qsub ]
if self.env: if self.env:
arg = '-v' arg = '-v'
arg += ','.join([ '%s=%s' % i for i in self.env.iteritems() ]) arg += ','.join([ '%s=%s' % i for i in self.env.iteritems() ])
args.append(arg) self.cmd.append(arg)
if self.hold: if self.hold:
args.append('-h') self.cmd.append('-h')
if len(self.stdout): if len(self.stdout):
args.append('-olocalhost:' + self.stdout) self.cmd.append('-olocalhost:' + self.stdout)
if self.keep_stdout and self.keep_stderr: if self.keep_stdout and self.keep_stderr:
args.append('-koe') self.cmd.append('-koe')
elif self.keep_stdout: elif self.keep_stdout:
args.append('-ko') self.cmd.append('-ko')
elif self.keep_stderr: elif self.keep_stderr:
args.append('-ke') self.cmd.append('-ke')
else: else:
args.append('-kn') self.cmd.append('-kn')
if self.join: if self.join:
args.append('-joe') self.cmd.append('-joe')
if len(self.node_type): if len(self.node_type):
args.append('-lnodes=' + self.node_type) self.cmd.append('-lnodes=' + self.node_type)
if self.mail_abort or self.mail_begin or self.mail_end: if self.mail_abort or self.mail_begin or self.mail_end:
flags = '' flags = ''
@ -97,30 +153,32 @@ class qsub:
if self.mail_end: if self.mail_end:
flags.append('e') flags.append('e')
if len(flags): if len(flags):
args.append('-m ' + flags) self.cmd.append('-m ' + flags)
if len(self.name): if len(self.name):
args.append("-N%s" % self.name) self.cmd.append("-N%s" % self.name)
if self.priority != 0: if self.priority != 0:
args.append('-p' + self.priority) self.cmd.append('-p' + self.priority)
if len(self.queue): if len(self.queue):
args.append('-q' + self.queue) self.cmd.append('-q' + self.queue)
args.append(script) self.cmd.extend(args)
self.script = script
self.command = ' '.join(self.cmd + [ self.script ])
if self.verbose or self.onlyecho: def do(self):
print >>sys.stderr, 'PBS Command: ', ' '.join(args) pbs = MyPOpen(self.cmd + [ self.script ])
self.result = pbs.fromchild.read()
ec = pbs.wait()
if self.onlyecho: if ec != 0 and self.pbshost:
return 0 cmd = ' '.join(self.cmd + [ '-' ])
cmd = [ 'ssh', '-x', self.pbshost, cmd ]
print >>sys.stderr, 'PBS Jobid: ', self.command = ' '.join(cmd)
ssh = MyPOpen(cmd, input = self.script)
ec = os.spawnvp(os.P_WAIT, args[0], args) self.result = ssh.fromchild.read()
ec = ssh.wait()
if ec != 0 and len(self.pbshost):
ec = ssh(self.pbshost, ' '.join(args))
return ec return ec

View file

@ -28,12 +28,20 @@
# Authors: Ali Saidi # Authors: Ali Saidi
# Nathan Binkert # Nathan Binkert
import os, os.path, re, sys import os, os.path, re, socket, sys
from os import environ as env, listdir from os import environ as env, listdir
from os.path import basename, isdir, isfile, islink, join as joinpath from os.path import basename, isdir, isfile, islink, join as joinpath
from filecmp import cmp as filecmp from filecmp import cmp as filecmp
from shutil import copyfile from shutil import copyfile
def nfspath(dir):
if dir.startswith('/.automount/'):
dir = '/n/%s' % dir[12:]
elif not dir.startswith('/n/'):
dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
return dir
progpath = nfspath(sys.path[0])
progname = basename(sys.argv[0]) progname = basename(sys.argv[0])
usage = """\ usage = """\
Usage: Usage:
@ -65,7 +73,7 @@ force = False
listonly = False listonly = False
queue = '' queue = ''
verbose = False verbose = False
rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd()) rootdir = nfspath(os.getcwd())
for opt,arg in opts: for opt,arg in opts:
if opt == '-c': if opt == '-c':
clean = True clean = True
@ -92,6 +100,7 @@ for arg in args:
exprs.append(re.compile(arg)) exprs.append(re.compile(arg))
if not listonly and not onlyecho and isdir(linkdir): if not listonly and not onlyecho and isdir(linkdir):
if verbose:
print 'Checking for outdated files in Link directory' print 'Checking for outdated files in Link directory'
entries = listdir(linkdir) entries = listdir(linkdir)
for entry in entries: for entry in entries:
@ -156,8 +165,8 @@ for jobname in joblist:
if not onlyecho and not os.path.isdir(jobdir): if not onlyecho and not os.path.isdir(jobdir):
sys.exit('%s is not a directory. Cannot build job' % jobdir) sys.exit('%s is not a directory. Cannot build job' % jobdir)
print >>sys.stderr, 'Job name: %s' % jobname print 'Job name: %s' % jobname
print >>sys.stderr, 'Job directory: %s' % jobdir print 'Job directory: %s' % jobdir
qsub = pbs.qsub() qsub = pbs.qsub()
qsub.pbshost = 'simpool.eecs.umich.edu' qsub.pbshost = 'simpool.eecs.umich.edu'
@ -165,11 +174,17 @@ for jobname in joblist:
qsub.name = jobname qsub.name = jobname
qsub.join = True qsub.join = True
qsub.node_type = 'FAST' qsub.node_type = 'FAST'
qsub.onlyecho = onlyecho
qsub.env['ROOTDIR'] = rootdir qsub.env['ROOTDIR'] = rootdir
qsub.verbose = verbose
if len(queue): if len(queue):
qsub.queue = queue qsub.queue = queue
qsub.build(joinpath(progpath, 'job.py'))
qsub.do(joinpath(basedir, 'job.py')) if verbose:
print >>sys.stderr, '' print 'PBS Command: %s' % qsub.command
if not onlyecho:
ec = qsub.do()
if ec == 0:
print 'PBS Jobid: %s' % qsub.result
else:
print 'PBS Failed'