sanchayanmaity/gem5
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

init: don't build files that centralize python and swig code

Browse source 
Instead of putting all object files into m5/object/__init__.py, interrogate
the importer to find out what should be imported.
Instead of creating a single file that lists all of the embedded python
modules, use static object construction to put those objects onto a list.
Do something similar for embedded swig (C++) code.
This commit is contained in:
Nathan Binkert 2010-09-09 14:15:42 -07:00
parent 710ed8f492
commit e6ee56c657
5 changed files with 218 additions and 170 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

179
src/SConscript
View file

@ -51,6 +51,8 @@ Export('env')
build_env = [(opt, env[opt]) for opt in export_vars]
from m5.util import code_formatter
########################################################################
# Code for adding source files of various types
#
@ -142,8 +144,8 @@ class PySource(SourceFile):
self.arcname = joinpath(*arcpath)
self.abspath = abspath
self.compiled = File(self.filename + 'c')
self.assembly = File(self.filename + '.s')
self.symname = "PyEMB_" + PySource.invalid_sym_char.sub('_', modpath)
self.cpp = File(self.filename + '.cc')
self.symname = PySource.invalid_sym_char.sub('_', modpath)
PySource.modules[modpath] = self
PySource.tnodes[self.tnode] = self
@ -446,24 +448,6 @@ env.Command('python/m5/info.py',
makeInfoPyFile)
PySource('m5', 'python/m5/info.py')
# Generate the __init__.py file for m5.objects
def makeObjectsInitFile(target, source, env):
code = code_formatter()
code('''\
from params import *
from m5.SimObject import *
''')
for module in source:
code('from $0 import *', module.get_contents())
code.write(str(target[0]))
# Generate an __init__.py file for the objects package
env.Command('python/m5/objects/__init__.py',
map(Value, SimObject.modnames),
makeObjectsInitFile)
PySource('m5.objects', 'python/m5/objects/__init__.py')
########################################################################
#
# Create all of the SimObject param headers and enum headers
@ -632,39 +616,32 @@ env.Command(params_file, map(Value, names), buildParams)
env.Depends(params_file, params_hh_files + params_i_files + depends)
SwigSource('m5.objects', params_file)
# Generate the main swig init file
def makeEmbeddedSwigInit(target, source, env):
code = code_formatter()
module = source[0].get_contents()
code('''\
#include "sim/init.hh"
extern "C" {
void init_${module}();
}
EmbeddedSwig embed_swig_${module}(init_${module});
''')
code.write(str(target[0]))
# Build all swig modules
for swig in SwigSource.all:
env.Command([swig.cc_source.tnode, swig.py_source.tnode], swig.tnode,
'$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} '
'-o ${TARGETS[0]} $SOURCES')
init_file = 'python/swig/init_%s.cc' % swig.module
env.Command(init_file, Value(swig.module), makeEmbeddedSwigInit)
Source(init_file)
env.Depends(swig.py_source.tnode, swig.tnode)
env.Depends(swig.cc_source.tnode, swig.tnode)
# Generate the main swig init file
def makeSwigInit(target, source, env):
code = code_formatter()
code('extern "C" {')
code.indent()
for module in source:
code('void init_$0();', module.get_contents())
code.dedent()
code('}')
code('void initSwig() {')
code.indent()
for module in source:
code('init_$0();', module.get_contents())
code.dedent()
code('}')
code.write(str(target[0]))
env.Command('python/swig/init.cc',
map(Value, sorted(s.module for s in SwigSource.all)),
makeSwigInit)
Source('python/swig/init.cc')
def getFlags(source_flags):
flagsMap = {}
flagsList = []
@ -892,13 +869,17 @@ env.Command('base/traceflags.hh', flags, traceFlagsHH)
env.Command('base/traceflags.cc', flags, traceFlagsCC)
Source('base/traceflags.cc')
# embed python files. All .py files that have been indicated by a
# Embed python files. All .py files that have been indicated by a
# PySource() call in a SConscript need to be embedded into the M5
# library. To do that, we compile the file to byte code, marshal the
# byte code, compress it, and then generate an assembly file that
# inserts the result into the data section with symbols indicating the
# beginning, and end (and with the size at the end)
def objectifyPyFile(target, source, env):
# byte code, compress it, and then generate a c++ file that
# inserts the result into an array.
def embedPyFile(target, source, env):
def c_str(string):
if string is None:
return "0"
return '"%s"' % string
'''Action function to compile a .py into a code object, marshal
it, compress it, and stick it into an asm file so the code appears
as just bytes with a label in the data section'''
@ -910,90 +891,40 @@ def objectifyPyFile(target, source, env):
marshalled = marshal.dumps(compiled)
compressed = zlib.compress(marshalled)
data = compressed
sym = pysource.symname
# Some C/C++ compilers prepend an underscore to global symbol
# names, so if they're going to do that, we need to prepend that
# leading underscore to globals in the assembly file.
if env['LEADING_UNDERSCORE']:
sym = '_' + pysource.symname
else:
sym = pysource.symname
step = 16
code = code_formatter()
code('''\
.data
.globl ${sym}_beg
.globl ${sym}_end
${sym}_beg:''')
#include "sim/init.hh"
namespace {
const char data_${sym}[] = {
''')
code.indent()
step = 16
for i in xrange(0, len(data), step):
x = array.array('B', data[i:i+step])
bytes = ','.join([str(d) for d in x])
code('.byte $bytes')
code('${sym}_end:')
code('.long $0', len(marshalled))
code(''.join('%d,' % d for d in x))
code.dedent()
code('''};
EmbeddedPython embedded_${sym}(
${{c_str(pysource.arcname)}},
${{c_str(pysource.abspath)}},
${{c_str(pysource.modpath)}},
data_${sym},
${{len(data)}},
${{len(marshalled)}});
/* namespace */ }
''')
code.write(str(target[0]))
for source in PySource.all:
env.Command(source.assembly, source.tnode, objectifyPyFile)
Source(source.assembly)
# Generate init_python.cc which creates a bunch of EmbeddedPyModule
# structs that describe the embedded python code. One such struct
# contains information about the importer that python uses to get at
# the embedded files, and then there's a list of all of the rest that
# the importer uses to load the rest on demand.
def pythonInit(target, source, env):
code = code_formatter()
def dump_mod(sym, endchar=','):
def c_str(string):
if string is None:
return "0"
return '"%s"' % string
pysource = PySource.symnames[sym]
arcname = c_str(pysource.arcname)
abspath = c_str(pysource.abspath)
modpath = c_str(pysource.modpath)
code.indent()
code('''\
{ $arcname,
$abspath,
$modpath,
${sym}_beg, ${sym}_end,
${sym}_end - ${sym}_beg,
*(int *)${sym}_end }$endchar
''')
code.dedent()
code('#include "sim/init.hh"')
for sym in source:
sym = sym.get_contents()
code('extern const char ${sym}_beg[], ${sym}_end[];')
code('const EmbeddedPyModule embeddedPyImporter = ')
dump_mod("PyEMB_importer", endchar=';')
code()
code('const EmbeddedPyModule embeddedPyModules[] = {')
for i,sym in enumerate(source):
sym = sym.get_contents()
if sym == "PyEMB_importer":
# Skip the importer since we've already exported it
continue
dump_mod(sym)
code(' { 0, 0, 0, 0, 0, 0, 0 }')
code('};')
code.write(str(target[0]))
env.Command('sim/init_python.cc',
map(Value, (s.symname for s in PySource.all)),
pythonInit)
Source('sim/init_python.cc')
env.Command(source.cpp, source.tnode, embedPyFile)
Source(source.cpp)
########################################################################
#

1
src/python/SConscript
View file

@ -49,6 +49,7 @@ PySource('m5', 'm5/simulate.py')
PySource('m5', 'm5/stats.py')
PySource('m5', 'm5/ticks.py')
PySource('m5', 'm5/trace.py')
PySource('m5.objects', 'm5/objects/__init__.py')
PySource('m5.util', 'm5/util/__init__.py')
PySource('m5.util', 'm5/util/attrdict.py')
PySource('m5.util', 'm5/util/code_formatter.py')

39
src/python/m5/objects/__init__.py Normal file
View file

@ -0,0 +1,39 @@
# Copyright (c) 2010 The Hewlett-Packard Development Company
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
from m5.objects.params import *
from m5.SimObject import *
try:
modules = __loader__.modules
except NameError:
modules = { }
for module in modules.iterkeys():
if module.startswith('m5.objects.') and module != 'm5.objects.params':
exec "from %s import *" % module

135
src/sim/init.cc
View file

@ -33,6 +33,7 @@
#include <marshal.h>
#include <signal.h>
#include <list>
#include <iostream>
#include <string>
#include <zlib.h>
@ -104,69 +105,121 @@ initSignals()
signal(SIGABRT, abortHandler);
}
/*
* Uncompress and unmarshal the code object stored in the
* EmbeddedPyModule
*/
PyObject *
getCode(const EmbeddedPyModule *pymod)
{
assert(pymod->zlen == pymod->code_end - pymod->code);
Bytef *marshalled = new Bytef[pymod->mlen];
uLongf unzlen = pymod->mlen;
int ret = uncompress(marshalled, &unzlen, (const Bytef *)pymod->code,
pymod->zlen);
if (ret != Z_OK)
panic("Could not uncompress code: %s\n", zError(ret));
assert(unzlen == (uLongf)pymod->mlen);
return PyMarshal_ReadObjectFromString((char *)marshalled, pymod->mlen);
}
// The python library is totally messed up with respect to constness,
// so make a simple macro to make life a little easier
#define PyCC(x) (const_cast<char *>(x))
EmbeddedPython *EmbeddedPython::importer = NULL;
PyObject *EmbeddedPython::importerModule = NULL;
EmbeddedPython::EmbeddedPython(const char *filename, const char *abspath,
const char *modpath, const char *code, int zlen, int len)
: filename(filename), abspath(abspath), modpath(modpath), code(code),
zlen(zlen), len(len)
{
// if we've added the importer keep track of it because we need it
// to bootstrap.
if (string(modpath) == string("importer"))
importer = this;
else
getList().push_back(this);
}
list<EmbeddedPython *> &
EmbeddedPython::getList()
{
static list<EmbeddedPython *> the_list;
return the_list;
}
/*
* Uncompress and unmarshal the code object stored in the
* EmbeddedPython
*/
PyObject *
EmbeddedPython::getCode() const
{
Bytef marshalled[len];
uLongf unzlen = len;
int ret = uncompress(marshalled, &unzlen, (const Bytef *)code, zlen);
if (ret != Z_OK)
panic("Could not uncompress code: %s\n", zError(ret));
assert(unzlen == (uLongf)len);
return PyMarshal_ReadObjectFromString((char *)marshalled, len);
}
bool
EmbeddedPython::addModule() const
{
PyObject *code = getCode();
PyObject *result = PyObject_CallMethod(importerModule, PyCC("add_module"),
PyCC("sssO"), filename, abspath, modpath, code);
if (!result) {
PyErr_Print();
return false;
}
Py_DECREF(result);
return true;
}
/*
* Load and initialize all of the python parts of M5, including Swig
* and the embedded module importer.
*/
int
initM5Python()
EmbeddedPython::initAll()
{
extern void initSwig();
// initialize SWIG modules. initSwig() is autogenerated and calls
// all of the individual swig initialization functions.
initSwig();
// Load the importer module
PyObject *code = getCode(&embeddedPyImporter);
PyObject *module = PyImport_ExecCodeModule(PyCC("importer"), code);
if (!module) {
PyObject *code = importer->getCode();
importerModule = PyImport_ExecCodeModule(PyCC("importer"), code);
if (!importerModule) {
PyErr_Print();
return 1;
}
// Load the rest of the embedded python files into the embedded
// python importer
const EmbeddedPyModule *pymod = &embeddedPyModules[0];
while (pymod->filename) {
PyObject *code = getCode(pymod);
PyObject *result = PyObject_CallMethod(module, PyCC("add_module"),
PyCC("sssO"), pymod->filename, pymod->abspath, pymod->modpath,
code);
if (!result) {
PyErr_Print();
list<EmbeddedPython *>::iterator i = getList().begin();
list<EmbeddedPython *>::iterator end = getList().end();
for (; i != end; ++i)
if (!(*i)->addModule())
return 1;
}
Py_DECREF(result);
++pymod;
}
return 0;
}
EmbeddedSwig::EmbeddedSwig(void (*init_func)())
: initFunc(init_func)
{
getList().push_back(this);
}
list<EmbeddedSwig *> &
EmbeddedSwig::getList()
{
static list<EmbeddedSwig *> the_list;
return the_list;
}
void
EmbeddedSwig::initAll()
{
// initialize SWIG modules. initSwig() is autogenerated and calls
// all of the individual swig initialization functions.
list<EmbeddedSwig *>::iterator i = getList().begin();
list<EmbeddedSwig *>::iterator end = getList().end();
for (; i != end; ++i)
(*i)->initFunc();
}
int
initM5Python()
{
EmbeddedSwig::initAll();
return EmbeddedPython::initAll();
}
/*
* Start up the M5 simulator. This mostly vectors into the python
* main function.

34
src/sim/init.hh
View file

@ -34,19 +34,43 @@
/*
* Data structure describing an embedded python file.
*/
struct EmbeddedPyModule
#include <list>
#ifndef PyObject_HEAD
struct _object;
typedef _object PyObject;
#endif
struct EmbeddedPython
{
const char *filename;
const char *abspath;
const char *modpath;
const char *code;
const char *code_end;
int zlen;
int mlen;
int len;
EmbeddedPython(const char *filename, const char *abspath,
const char *modpath, const char *code, int zlen, int len);
PyObject *getCode() const;
bool addModule() const;
static EmbeddedPython *importer;
static PyObject *importerModule;
static std::list<EmbeddedPython *> &getList();
static int initAll();
};
extern const EmbeddedPyModule embeddedPyImporter;
extern const EmbeddedPyModule embeddedPyModules[];
struct EmbeddedSwig
{
void (*initFunc)();
EmbeddedSwig(void (*init_func)());
static std::list<EmbeddedSwig *> &getList();
static void initAll();
};
void initSignals();
int initM5Python();