Adding gem5 source to svn.
git-svn-id: https://www4.informatik.uni-erlangen.de/i4svn/danceos/trunk/devel/fail@1819 8c4709b5-6ec9-48aa-a5cd-a96041d1645a
This commit is contained in:
friemel
306
simulators/gem5/util/batch/send.py
Executable file
306
simulators/gem5/util/batch/send.py
Executable file
@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2006 The Regents of The University of Michigan
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
# Authors: Kevin Lim
|
||||
|
||||
import os, os.path, re, socket, sys
|
||||
from os import environ as env, listdir
|
||||
from os.path import basename, isdir, isfile, islink, join as joinpath, normpath
|
||||
from filecmp import cmp as filecmp
|
||||
from shutil import copy
|
||||
|
||||
def nfspath(dir):
|
||||
if dir.startswith('/.automount/'):
|
||||
dir = '/n/%s' % dir[12:]
|
||||
elif not dir.startswith('/n/'):
|
||||
dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
|
||||
return dir
|
||||
|
||||
def syncdir(srcdir, destdir):
|
||||
srcdir = normpath(srcdir)
|
||||
destdir = normpath(destdir)
|
||||
if not isdir(destdir):
|
||||
sys.exit('destination directory "%s" does not exist' % destdir)
|
||||
|
||||
for root, dirs, files in os.walk(srcdir):
|
||||
root = normpath(root)
|
||||
prefix = os.path.commonprefix([root, srcdir])
|
||||
root = root[len(prefix):]
|
||||
if root.startswith('/'):
|
||||
root = root[1:]
|
||||
for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']:
|
||||
dirs.remove(rem)
|
||||
|
||||
for entry in dirs:
|
||||
newdir = joinpath(destdir, root, entry)
|
||||
if not isdir(newdir):
|
||||
os.mkdir(newdir)
|
||||
print 'mkdir', newdir
|
||||
|
||||
for i,d in enumerate(dirs):
|
||||
if islink(joinpath(srcdir, root, d)):
|
||||
dirs[i] = joinpath(d, '.')
|
||||
|
||||
for entry in files:
|
||||
dest = normpath(joinpath(destdir, root, entry))
|
||||
src = normpath(joinpath(srcdir, root, entry))
|
||||
if not isfile(dest) or not filecmp(src, dest):
|
||||
print 'copy %s %s' % (dest, src)
|
||||
copy(src, dest)
|
||||
|
||||
progpath = nfspath(sys.path[0])
|
||||
progname = basename(sys.argv[0])
|
||||
usage = """\
|
||||
Usage:
|
||||
%(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp>
|
||||
-c clean directory if job can be run
|
||||
-C submit the checkpointing runs
|
||||
-d Make jobs be dependent on the completion of the checkpoint runs
|
||||
-e only echo pbs command info, don't actually send the job
|
||||
-f force the job to run regardless of state
|
||||
-q <queue> submit job to the named queue
|
||||
-j <jobfile> specify the jobfile (default is <rootdir>/Test.py)
|
||||
-v be verbose
|
||||
|
||||
%(progname)s [-j <jobfile>] -l [-v] <regexp>
|
||||
-j <jobfile> specify the jobfile (default is <rootdir>/Test.py)
|
||||
-l list job names, don't submit
|
||||
-v be verbose (list job parameters)
|
||||
|
||||
%(progname)s -h
|
||||
-h display this help
|
||||
""" % locals()
|
||||
|
||||
try:
|
||||
import getopt
|
||||
opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v')
|
||||
except getopt.GetoptError:
|
||||
sys.exit(usage)
|
||||
|
||||
depend = False
|
||||
clean = False
|
||||
onlyecho = False
|
||||
exprs = []
|
||||
force = False
|
||||
listonly = False
|
||||
queue = ''
|
||||
verbose = False
|
||||
jfile = 'Test.py'
|
||||
docpts = False
|
||||
doruns = True
|
||||
runflag = False
|
||||
node_type = 'FAST'
|
||||
update = True
|
||||
|
||||
for opt,arg in opts:
|
||||
if opt == '-C':
|
||||
docpts = True
|
||||
if opt == '-c':
|
||||
clean = True
|
||||
if opt == '-d':
|
||||
depend = True
|
||||
if opt == '-e':
|
||||
onlyecho = True
|
||||
if opt == '-f':
|
||||
force = True
|
||||
if opt == '-h':
|
||||
print usage
|
||||
sys.exit(0)
|
||||
if opt == '-j':
|
||||
jfile = arg
|
||||
if opt == '-l':
|
||||
listonly = True
|
||||
if opt == '-n':
|
||||
update = False
|
||||
if opt == '-q':
|
||||
queue = arg
|
||||
if opt == '-R':
|
||||
runflag = True
|
||||
if opt == '-t':
|
||||
node_type = arg
|
||||
if opt == '-v':
|
||||
verbose = True
|
||||
|
||||
if docpts:
|
||||
doruns = runflag
|
||||
|
||||
for arg in args:
|
||||
exprs.append(re.compile(arg))
|
||||
|
||||
import jobfile, batch
|
||||
from job import JobDir, date
|
||||
|
||||
conf = jobfile.JobFile(jfile)
|
||||
|
||||
if update and not listonly and not onlyecho and isdir(conf.linkdir):
|
||||
if verbose:
|
||||
print 'Checking for outdated files in Link directory'
|
||||
if not isdir(conf.basedir):
|
||||
os.mkdir(conf.basedir)
|
||||
syncdir(conf.linkdir, conf.basedir)
|
||||
|
||||
jobnames = {}
|
||||
joblist = []
|
||||
|
||||
if docpts and doruns:
|
||||
gen = conf.alljobs()
|
||||
elif docpts:
|
||||
gen = conf.checkpoints()
|
||||
elif doruns:
|
||||
gen = conf.jobs()
|
||||
|
||||
for job in gen:
|
||||
if job.name in jobnames:
|
||||
continue
|
||||
|
||||
if exprs:
|
||||
for expr in exprs:
|
||||
if expr.match(job.name):
|
||||
joblist.append(job)
|
||||
break
|
||||
else:
|
||||
joblist.append(job)
|
||||
|
||||
if listonly:
|
||||
if verbose:
|
||||
for job in joblist:
|
||||
job.printinfo()
|
||||
else:
|
||||
for job in joblist:
|
||||
print job.name
|
||||
sys.exit(0)
|
||||
|
||||
if not onlyecho:
|
||||
newlist = []
|
||||
for job in joblist:
|
||||
jobdir = JobDir(joinpath(conf.rootdir, job.name))
|
||||
if jobdir.exists():
|
||||
if not force:
|
||||
status = jobdir.getstatus()
|
||||
if status == 'queued':
|
||||
continue
|
||||
|
||||
if status == 'running':
|
||||
continue
|
||||
|
||||
if status == 'success':
|
||||
continue
|
||||
|
||||
if not clean:
|
||||
sys.exit('job directory %s not clean!' % jobdir)
|
||||
|
||||
jobdir.clean()
|
||||
newlist.append(job)
|
||||
joblist = newlist
|
||||
|
||||
class NameHack(object):
|
||||
def __init__(self, host='pbs.pool', port=24465):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.socket = None
|
||||
|
||||
def setname(self, jobid, jobname):
|
||||
try:
|
||||
jobid = int(jobid)
|
||||
except ValueError:
|
||||
jobid = int(jobid.strip().split('.')[0])
|
||||
|
||||
jobname = jobname.strip()
|
||||
# since pbs can handle jobnames of 15 characters or less,
|
||||
# don't use the raj hack.
|
||||
if len(jobname) <= 15:
|
||||
return
|
||||
|
||||
if self.socket is None:
|
||||
import socket
|
||||
self.socket = socket.socket()
|
||||
# Connect to pbs.pool and send the jobid/jobname pair to port
|
||||
# 24465 (Raj didn't realize that there are only 64k ports and
|
||||
# setup inetd to point to port 90001)
|
||||
self.socket.connect((self.host, self.port))
|
||||
|
||||
self.socket.send("%s %s\n" % (jobid, jobname))
|
||||
|
||||
namehack = NameHack()
|
||||
|
||||
rootdir = conf.rootdir
|
||||
script = joinpath(rootdir, 'Base', 'job.py')
|
||||
|
||||
for job in joblist:
|
||||
jobdir = JobDir(joinpath(rootdir, job.name))
|
||||
if depend:
|
||||
cptdir = JobDir(joinpath(rootdir, job.checkpoint.name))
|
||||
path = str(cptdir)
|
||||
if not isdir(path) or not isfile(joinpath(path, '.success')):
|
||||
continue
|
||||
|
||||
cptjob = cptdir.readval('.batch_jobid')
|
||||
|
||||
if not onlyecho:
|
||||
jobdir.create()
|
||||
os.chdir(str(jobdir))
|
||||
os.environ['PWD'] = str(jobdir)
|
||||
|
||||
print 'Job name: %s' % job.name
|
||||
print 'Job directory: %s' % jobdir
|
||||
|
||||
|
||||
qsub = batch.oarsub()
|
||||
qsub.oarhost = 'poolfs.eecs.umich.edu'
|
||||
#qsub.stdout = jobdir.file('jobout')
|
||||
qsub.name = job.name
|
||||
qsub.walltime = '50'
|
||||
#qsub.join = True
|
||||
#qsub.node_type = node_type
|
||||
#qsub.env['ROOTDIR'] = conf.rootdir
|
||||
#qsub.env['JOBNAME'] = job.name
|
||||
#if depend:
|
||||
# qsub.afterok = cptjob
|
||||
#if queue:
|
||||
# qsub.queue = queue
|
||||
qsub.properties = "64bit = 'Yes' or 64bit = 'No'"
|
||||
qsub.build(script)
|
||||
|
||||
if verbose:
|
||||
print 'cwd: %s' % qsub.command
|
||||
print 'PBS Command: %s' % qsub.command
|
||||
|
||||
if not onlyecho:
|
||||
ec = qsub.do()
|
||||
if ec == 0:
|
||||
jobid = qsub.result
|
||||
print 'OAR Jobid: %s' % jobid
|
||||
#namehack.setname(jobid, job.name)
|
||||
queued = date()
|
||||
jobdir.echofile('.batch_jobid', jobid)
|
||||
jobdir.echofile('.batch_jobname', job.name)
|
||||
jobdir.echofile('.queued', queued)
|
||||
jobdir.setstatus('queued on %s' % queued)
|
||||
else:
|
||||
print 'OAR Failed'
|
||||
print
|
||||
print
|
||||
Reference in New Issue
Block a user