Link in Python interpreter.

Use embedded zip archive to carry Python code instead
of homegrown embedded string/file mechanism.
Do argument parsing in Python instead of C++.

SConstruct:
    Add Python interpreter include path & library.
    Define two new simple builders which copy &
    concatenate files, respectively, for use by
    the Python embedded zipfile code.
src/SConscript:
    Encapsulate environment creation in a function.
    Add code to append Python zip archive to final executable.
    Eliminate references to obsolete files.
src/python/SConscript:
    Rewrite to generate embedded zip archive of Python code
    (replacing old "embedded string" mechanism).
src/python/m5/__init__.py:
    Move main arg-parsing loop here (out of C++ main()).
src/python/m5/config.py:
    Minor fix (version incompatibility?).
src/sim/main.cc:
    Invoke embedded Python interpreter to parse args
    and generate config.ini, replacing C++ arg parsing code.

--HG--
extra : convert_revision : 72d21236b2bee139ff39ba4cf031a4a1f8560029
This commit is contained in:
Steve Reinhardt 2006-05-30 13:11:34 -04:00
parent d308055afc
commit 0337db3388
6 changed files with 254 additions and 369 deletions

View file

@ -169,7 +169,13 @@ if sys.platform == 'cygwin':
env.Append(CCFLAGS=Split("-Wno-uninitialized"))
env.Append(CPPPATH=[Dir('ext/dnet')])
# Default libraries
# Environment args for linking in Python interpreter.
# Should really have an option for setting the version instead of
# having 2.4 hardwired in here...
env.Append(CPPPATH='/usr/include/python2.4')
env.Append(LIBS='python2.4')
# Other default libraries
env.Append(LIBS=['z'])
# Platform-specific configuration. Note again that we assume that all
@ -310,6 +316,32 @@ config_builder = Builder(emitter = config_emitter, action = config_action)
env.Append(BUILDERS = { 'ConfigFile' : config_builder })
###################################################
#
# Define a SCons builder for copying files. This is used by the
# Python zipfile code in src/python/SConscript, but is placed up here
# since it's potentially more generally applicable.
#
###################################################
copy_builder = Builder(action = Copy("$TARGET", "$SOURCE"))
env.Append(BUILDERS = { 'CopyFile' : copy_builder })
###################################################
#
# Define a simple SCons builder to concatenate files.
#
# Used to append the Python zip archive to the executable.
#
###################################################
concat_builder = Builder(action = Action(['cat $SOURCES > $TARGET',
'chmod +x $TARGET']))
env.Append(BUILDERS = { 'Concat' : concat_builder })
# base help text
help_text = '''
Usage: scons [scons options] [build options] [target(s)]

View file

@ -46,9 +46,7 @@ Import('env')
base_sources = Split('''
base/circlebuf.cc
base/copyright.cc
base/cprintf.cc
base/embedfile.cc
base/fast_alloc.cc
base/fifo_buffer.cc
base/hostinfo.cc
@ -99,9 +97,6 @@ base_sources = Split('''
mem/port.cc
mem/request.cc
python/pyconfig.cc
python/embedded_py.cc
sim/builder.cc
sim/configfile.cc
sim/debug.cc
@ -356,43 +351,45 @@ def make_objs(sources, env):
# files.
env.Append(CPPPATH='.')
# List of constructed environments to pass back to SConstruct
envList = []
# Function to create a new build environment as clone of current
# environment 'env' with modified object suffix and optional stripped
# binary. Additional keyword arguments are appended to corresponding
# build environment vars.
def makeEnv(label, objsfx, strip = False, **kwargs):
newEnv = env.Copy(OBJSUFFIX=objsfx)
newEnv.Label = label
newEnv.Append(**kwargs)
exe = 'm5.' + label # final executable
bin = exe + '.bin' # executable w/o appended Python zip archive
newEnv.Program(bin, make_objs(sources, newEnv))
if strip:
stripped_bin = bin + '.stripped'
newEnv.Command(stripped_bin, bin, 'strip $SOURCE -o $TARGET')
bin = stripped_bin
targets = newEnv.Concat(exe, [bin, 'python/m5py.zip'])
newEnv.M5Binary = targets[0]
envList.append(newEnv)
# Debug binary
debugEnv = env.Copy(OBJSUFFIX='.do')
debugEnv.Label = 'debug'
debugEnv.Append(CCFLAGS=Split('-g3 -gdwarf-2 -O0'))
debugEnv.Append(CPPDEFINES='DEBUG')
tlist = debugEnv.Program(target = 'm5.debug',
source = make_objs(sources, debugEnv))
debugEnv.M5Binary = tlist[0]
makeEnv('debug', '.do',
CCFLAGS = Split('-g3 -gdwarf-2 -O0'),
CPPDEFINES = 'DEBUG')
# Optimized binary
optEnv = env.Copy()
optEnv.Label = 'opt'
optEnv.Append(CCFLAGS=Split('-g -O3'))
tlist = optEnv.Program(target = 'm5.opt',
source = make_objs(sources, optEnv))
optEnv.M5Binary = tlist[0]
makeEnv('opt', '.o',
CCFLAGS = Split('-g -O3'))
# "Fast" binary
fastEnv = env.Copy(OBJSUFFIX='.fo')
fastEnv.Label = 'fast'
fastEnv.Append(CCFLAGS=Split('-O3'))
fastEnv.Append(CPPDEFINES='NDEBUG')
fastEnv.Program(target = 'm5.fast.unstripped',
source = make_objs(sources, fastEnv))
tlist = fastEnv.Command(target = 'm5.fast',
source = 'm5.fast.unstripped',
action = 'strip $SOURCE -o $TARGET')
fastEnv.M5Binary = tlist[0]
makeEnv('fast', '.fo', strip = True,
CCFLAGS = Split('-O3'),
CPPDEFINES = 'NDEBUG')
# Profiled binary
profEnv = env.Copy(OBJSUFFIX='.po')
profEnv.Label = 'prof'
profEnv.Append(CCFLAGS=Split('-O3 -g -pg'), LINKFLAGS='-pg')
tlist = profEnv.Program(target = 'm5.prof',
source = make_objs(sources, profEnv))
profEnv.M5Binary = tlist[0]
envList = [debugEnv, optEnv, fastEnv, profEnv]
makeEnv('prof', '.po',
CCFLAGS = Split('-O3 -g -pg'),
LINKFLAGS = '-pg')
Return('envList')

View file

@ -27,126 +27,55 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os, os.path, re, sys
from zipfile import PyZipFile
# handy function for path joins
def join(*args):
return os.path.normpath(os.path.join(*args))
Import('env')
import scons_helper
# This SConscript is in charge of collecting .py files and generating a zip archive that is appended to the m5 binary.
def WriteEmbeddedPyFile(target, source, path, name, ext, filename):
if isinstance(source, str):
source = file(source, 'r')
# Copy .py source files here (relative to src/python in the build
# directory).
pyzip_root = 'zip'
if isinstance(target, str):
target = file(target, 'w')
# List of files & directories to include in the zip file. To include
# a package, list only the root directory of the package, not any
# internal .py files (else they will get the path stripped off when
# they are imported into the zip file).
pyzip_files = []
print >>target, "AddModule(%s, %s, %s, %s, '''\\" % \
(`path`, `name`, `ext`, `filename`)
# List of additional files on which the zip archive depends, but which
# are not included in pyzip_files... i.e. individual .py files within
# a package.
pyzip_dep_files = []
for line in source:
line = line
# escape existing backslashes
line = line.replace('\\', '\\\\')
# escape existing triple quotes
line = line.replace("'''", r"\'\'\'")
# Add the specified package to the zip archive. Adds the directory to
# pyzip_files and all included .py files to pyzip_dep_files.
def addPkg(pkgdir):
pyzip_files.append(join(pyzip_root, pkgdir))
origdir = os.getcwd()
srcdir = join(Dir('.').srcnode().abspath, pkgdir)
os.chdir(srcdir)
for path, dirs, files in os.walk('.'):
for i,dir in enumerate(dirs):
if dir == 'SCCS':
del dirs[i]
break
print >>target, line,
for f in files:
if f.endswith('.py'):
source = join(pkgdir, path, f)
target = join(pyzip_root, source)
pyzip_dep_files.append(target)
env.CopyFile(target, source)
print >>target, "''')"
print >>target
def WriteCFile(target, source, name):
if isinstance(source, str):
source = file(source, 'r')
if isinstance(target, str):
target = file(target, 'w')
print >>target, 'const char %s_string[] = {' % name
count = 0
from array import array
try:
while True:
foo = array('B')
foo.fromfile(source, 10000)
l = [ str(i) for i in foo.tolist() ]
count += len(l)
for i in xrange(0,9999,20):
print >>target, ','.join(l[i:i+20]) + ','
except EOFError:
l = [ str(i) for i in foo.tolist() ]
count += len(l)
for i in xrange(0,len(l),20):
print >>target, ','.join(l[i:i+20]) + ','
print >>target, ','.join(l[i:]) + ','
print >>target, '};'
print >>target, 'const int %s_length = %d;' % (name, count)
print >>target
def splitpath(path):
dir,file = os.path.split(path)
path = []
assert(file)
while dir:
dir,base = os.path.split(dir)
path.insert(0, base)
return path, file
def MakeEmbeddedPyFile(target, source, env):
target = file(str(target[0]), 'w')
tree = {}
for src in source:
src = str(src)
path,pyfile = splitpath(src)
node = tree
for dir in path:
if not node.has_key(dir):
node[dir] = { }
node = node[dir]
name,ext = pyfile.split('.')
if name == '__init__':
node['.hasinit'] = True
node[pyfile] = (src,name,ext,src)
done = False
while not done:
done = True
for name,entry in tree.items():
if not isinstance(entry, dict): continue
if entry.has_key('.hasinit'): continue
done = False
del tree[name]
for key,val in entry.iteritems():
if tree.has_key(key):
raise NameError, \
"dir already has %s can't add it again" % key
tree[key] = val
files = []
def populate(node, path = []):
names = node.keys()
names.sort()
for name in names:
if name == '.hasinit':
continue
entry = node[name]
if isinstance(entry, dict):
if not entry.has_key('.hasinit'):
raise NameError, 'package directory missing __init__.py'
populate(entry, path + [ name ])
else:
pyfile,name,ext,filename = entry
files.append((pyfile, path, name, ext, filename))
populate(tree)
for pyfile, path, name, ext, filename in files:
WriteEmbeddedPyFile(target, pyfile, path, name, ext, filename)
os.chdir(origdir)
# Generate Python file that contains a dict specifying the current
# build_env flags.
def MakeDefinesPyFile(target, source, env):
f = file(str(target[0]), 'w')
print >>f, "import __main__"
@ -154,54 +83,21 @@ def MakeDefinesPyFile(target, source, env):
print >>f, source[0]
f.close()
CFileCounter = 0
def MakePythonCFile(target, source, env):
global CFileCounter
target = file(str(target[0]), 'w')
print >>target, '''\
#include "base/embedfile.hh"
namespace {
'''
for src in source:
src = str(src)
fname = os.path.basename(src)
name = 'embedded_file%d' % CFileCounter
CFileCounter += 1
WriteCFile(target, src, name)
print >>target, '''\
EmbedMap %(name)s("%(fname)s",
%(name)s_string, %(name)s_length);
''' % locals()
print >>target, '''\
/* namespace */ }
'''
# base list of .py files to embed
embedded_py_files = [ os.path.join(env['ROOT'], 'util/pbs/jobfile.py') ]
# add all .py files in python/m5
objpath = os.path.join(env['SRCDIR'], 'python', 'm5')
for root, dirs, files in os.walk(objpath, topdown=True):
for i,dir in enumerate(dirs):
if dir == 'SCCS':
del dirs[i]
break
assert(root.startswith(objpath))
for f in files:
if f.endswith('.py'):
embedded_py_files.append(os.path.join(root, f))
embedfile_hh = os.path.join(env['SRCDIR'], 'base/embedfile.hh')
optionDict = dict([(opt, env[opt]) for opt in env.ExportOptions])
env.Command('defines.py', Value(optionDict), MakeDefinesPyFile)
env.Command('embedded_py.py', embedded_py_files, MakeEmbeddedPyFile)
env.Depends('embedded_py.cc', embedfile_hh)
env.Command('embedded_py.cc',
['string_importer.py', 'defines.py', 'embedded_py.py'],
MakePythonCFile)
# Now specify the packages & files for the zip archive.
addPkg('m5')
pyzip_files.append('defines.py')
pyzip_files.append(join(env['ROOT'], 'util/pbs/jobfile.py'))
# Action function to build the zip archive. Uses the PyZipFile module
# included in the standard Python library.
def buildPyZip(target, source, env):
pzf = PyZipFile(str(target[0]), 'w')
for s in source:
pzf.writepy(str(s))
# Add the zip file target to the environment.
env.Command('m5py.zip', pyzip_files, buildPyZip)
env.Depends('m5py.zip', pyzip_dep_files)

View file

@ -24,7 +24,53 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import sys, os
import sys, os, time
import __main__
briefCopyright = '''
Copyright (c) 2001-2006
The Regents of The University of Michigan
All Rights Reserved
'''
fullCopyright = '''
Copyright (c) 2001-2006
The Regents of The University of Michigan
All Rights Reserved
Permission is granted to use, copy, create derivative works and
redistribute this software and such derivative works for any purpose,
so long as the copyright notice above, this grant of permission, and
the disclaimer below appear in all copies made; and so long as the
name of The University of Michigan is not used in any advertising or
publicity pertaining to the use or distribution of this software
without specific, written prior authorization.
THIS SOFTWARE IS PROVIDED AS IS, WITHOUT REPRESENTATION FROM THE
UNIVERSITY OF MICHIGAN AS TO ITS FITNESS FOR ANY PURPOSE, AND WITHOUT
WARRANTY BY THE UNIVERSITY OF MICHIGAN OF ANY KIND, EITHER EXPRESS OR
IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE REGENTS OF
THE UNIVERSITY OF MICHIGAN SHALL NOT BE LIABLE FOR ANY DAMAGES,
INCLUDING DIRECT, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WITH RESPECT TO ANY CLAIM ARISING OUT OF OR IN CONNECTION
WITH THE USE OF THE SOFTWARE, EVEN IF IT HAS BEEN OR IS HEREAFTER
ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
'''
def sayHello(f):
print >> f, "M5 Simulator System"
print >> f, briefCopyright
print >> f, "M5 compiled on", __main__.compileDate
hostname = os.environ.get('HOSTNAME')
if not hostname:
hostname = os.environ.get('HOST')
if hostname:
print >> f, "M5 executing on", hostname
print >> f, "M5 simulation started", time.ctime()
sayHello(sys.stderr)
# define this here so we can use it right away if necessary
def panic(string):
@ -72,3 +118,63 @@ from config import *
# import the built-in object definitions
from objects import *
args_left = sys.argv[1:]
configfile_found = False
while args_left:
arg = args_left.pop(0)
if arg.startswith('--'):
# if arg starts with '--', parse as a special python option
# of the format --<python var>=<string value>
try:
(var, val) = arg.split('=', 1)
except ValueError:
panic("Could not parse configuration argument '%s'\n"
"Expecting --<variable>=<value>\n" % arg);
eval("%s = %s" % (var, repr(val)))
elif arg.startswith('-'):
# if the arg starts with '-', it should be a simulator option
# with a format similar to getopt.
optchar = arg[1]
if len(arg) > 2:
args_left.insert(0, arg[2:])
if optchar == 'd':
outdir = args_left.pop(0)
elif optchar == 'h':
showBriefHelp(sys.stderr)
sys.exit(1)
elif optchar == 'E':
env_str = args_left.pop(0)
split_result = env_str.split('=', 1)
var = split_result[0]
if len(split_result == 2):
val = split_result[1]
else:
val = True
env[var] = val
elif optchar == 'I':
AddToPath(args_left.pop(0))
elif optchar == 'P':
eval(args_left.pop(0))
else:
showBriefHelp(sys.stderr)
panic("invalid argument '%s'\n" % arg_str)
else:
# In any other case, treat the option as a configuration file
# name and load it.
if not arg.endswith('.py'):
panic("Config file '%s' must end in '.py'\n" % arg)
configfile_found = True
m5execfile(arg, globals())
if not configfile_found:
panic("no configuration file specified!")
if globals().has_key('root') and isinstance(root, Root):
sys.stdout = file('config.ini', 'w')
instantiate(root)
else:
print 'Instantiation skipped: no root object found.'

View file

@ -794,7 +794,7 @@ class ParamFactory(object):
# E.g., Param.Int(5, "number of widgets")
def __call__(self, *args, **kwargs):
caller_frame = inspect.stack()[1][0]
caller_frame = inspect.currentframe().f_back
ptype = None
try:
ptype = eval(self.ptype_str,

View file

@ -29,6 +29,8 @@
///
/// @file sim/main.cc
///
#include <Python.h> // must be before system headers... see Python docs
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
@ -40,8 +42,6 @@
#include <string>
#include <vector>
#include "base/copyright.hh"
#include "base/embedfile.hh"
#include "base/inifile.hh"
#include "base/misc.hh"
#include "base/output.hh"
@ -51,7 +51,6 @@
#include "base/time.hh"
#include "cpu/base.hh"
#include "cpu/smt.hh"
#include "python/pyconfig.hh"
#include "sim/async.hh"
#include "sim/builder.hh"
#include "sim/configfile.hh"
@ -111,50 +110,6 @@ abortHandler(int sigtype)
/// Simulator executable name
char *myProgName = "";
/// Show brief help message.
void
showBriefHelp(ostream &out)
{
char *prog = basename(myProgName);
ccprintf(out, "Usage:\n");
ccprintf(out,
"%s [-d <dir>] [-E <var>[=<val>]] [-I <dir>] [-P <python>]\n"
" [--<var>=<val>] <config file>\n"
"\n"
" -d set the output directory to <dir>\n"
" -E set the environment variable <var> to <val> (or 'True')\n"
" -I add the directory <dir> to python's path\n"
" -P execute <python> directly in the configuration\n"
" --var=val set the python variable <var> to '<val>'\n"
" <configfile> config file name (ends in .py)\n\n",
prog);
ccprintf(out, "%s -X\n -X extract embedded files\n\n", prog);
ccprintf(out, "%s -h\n -h print short help\n\n", prog);
}
/// Print welcome message.
void
sayHello(ostream &out)
{
extern const char *compileDate; // from date.cc
ccprintf(out, "M5 Simulator System\n");
// display copyright
ccprintf(out, "%s\n", briefCopyright);
ccprintf(out, "M5 compiled on %d\n", compileDate);
char *host = getenv("HOSTNAME");
if (!host)
host = getenv("HOST");
if (host)
ccprintf(out, "M5 executing on %s\n", host);
ccprintf(out, "M5 simulation started %s\n", Time::start);
}
///
/// Echo the command line for posterity in such a way that it can be
/// used to rerun the same simulation (given the same .ini files).
@ -191,19 +146,7 @@ echoCommandLine(int argc, char **argv, ostream &out)
out << endl << endl;
}
char *
getOptionString(int &index, int argc, char **argv)
{
char *option = argv[index] + 2;
if (*option != '\0')
return option;
// We didn't find an argument, it must be in the next variable.
if (++index >= argc)
panic("option string for option '%s' not found", argv[index - 1]);
return argv[index];
}
#include "config/python_build_env.hh"
int
main(int argc, char **argv)
@ -218,121 +161,37 @@ main(int argc, char **argv)
signal(SIGINT, exitNowHandler); // dump final stats and exit
signal(SIGABRT, abortHandler);
bool configfile_found = false;
PythonConfig pyconfig;
string outdir;
// Python embedded interpreter invocation
Py_SetProgramName(argv[0]);
const char *fileName = Py_GetProgramFullPath();
Py_Initialize();
PySys_SetArgv(argc, argv);
if (argc < 2) {
showBriefHelp(cerr);
exit(1);
}
// loadSwigModules();
sayHello(cerr);
// Set Python module path to include current file to find embedded
// zip archive
PyRun_SimpleString("import sys");
string pathCmd = csprintf("sys.path[1:1] = ['%s']", fileName);
PyRun_SimpleString(pathCmd.c_str());
// Parse command-line options.
// Since most of the complex options are handled through the
// config database, we don't mess with getopts, and just parse
// manually.
for (int i = 1; i < argc; ++i) {
char *arg_str = argv[i];
// Pass compile timestamp string to Python
extern const char *compileDate; // from date.cc
string setCompileDate = csprintf("compileDate = '%s'", compileDate);
PyRun_SimpleString(setCompileDate.c_str());
// if arg starts with '--', parse as a special python option
// of the format --<python var>=<string value>, if the arg
// starts with '-', it should be a simulator option with a
// format similar to getopt. In any other case, treat the
// option as a configuration file name and load it.
if (arg_str[0] == '-' && arg_str[1] == '-') {
string str = &arg_str[2];
string var, val;
// PyRun_InteractiveLoop(stdin, "stdin");
// m5/__init__.py currently contains main argv parsing loop etc.,
// and will write out config.ini file before returning.
PyImport_ImportModule("defines");
PyImport_ImportModule("m5");
Py_Finalize();
if (!split_first(str, var, val, '='))
panic("Could not parse configuration argument '%s'\n"
"Expecting --<variable>=<value>\n", arg_str);
pyconfig.setVariable(var, val);
} else if (arg_str[0] == '-') {
char *option;
string var, val;
// switch on second char
switch (arg_str[1]) {
case 'd':
outdir = getOptionString(i, argc, argv);
break;
case 'h':
showBriefHelp(cerr);
exit(1);
case 'E':
option = getOptionString(i, argc, argv);
if (!split_first(option, var, val, '='))
val = "True";
if (setenv(var.c_str(), val.c_str(), true) == -1)
panic("setenv: %s\n", strerror(errno));
break;
case 'I':
option = getOptionString(i, argc, argv);
pyconfig.addPath(option);
break;
case 'P':
option = getOptionString(i, argc, argv);
pyconfig.writeLine(option);
break;
case 'X': {
list<EmbedFile> lst;
EmbedMap::all(lst);
list<EmbedFile>::iterator i = lst.begin();
list<EmbedFile>::iterator end = lst.end();
while (i != end) {
cprintf("Embedded File: %s\n", i->name);
cout.write(i->data, i->length);
++i;
}
return 0;
}
default:
showBriefHelp(cerr);
panic("invalid argument '%s'\n", arg_str);
}
} else {
string file(arg_str);
string base, ext;
if (!split_last(file, base, ext, '.') || ext != "py")
panic("Config file '%s' must end in '.py'\n", file);
pyconfig.load(file);
configfile_found = true;
}
}
if (outdir.empty()) {
char *env = getenv("OUTPUT_DIR");
outdir = env ? env : ".";
}
simout.setDirectory(outdir);
char *env = getenv("CONFIG_OUTPUT");
if (!env)
env = "config.out";
configStream = simout.find(env);
if (!configfile_found)
panic("no configuration file specified!");
configStream = simout.find("config.out");
// The configuration database is now complete; start processing it.
IniFile inifile;
if (!pyconfig.output(inifile))
panic("Error processing python code");
inifile.load("config.ini");
// Initialize statistics database
Stats::InitSimStats();
@ -346,11 +205,6 @@ main(int argc, char **argv)
ParamContext::parseAllContexts(inifile);
ParamContext::checkAllContexts();
// Print hello message to stats file if it's actually a file. If
// it's not (i.e. it's cout or cerr) then we already did it above.
if (simout.isFile(*outputStream))
sayHello(*outputStream);
// Echo command line and all parameter settings to stats file as well.
echoCommandLine(argc, argv, *outputStream);
ParamContext::showAllContexts(*configStream);