init: don't build files that centralize python and swig code

Instead of putting all object files into m5/object/__init__.py, interrogate
the importer to find out what should be imported.
Instead of creating a single file that lists all of the embedded python
modules, use static object construction to put those objects onto a list.
Do something similar for embedded swig (C++) code.

src/SConscript

@@ -51,6 +51,8 @@ Export('env')
 
 build_env = [(opt, env[opt]) for opt in export_vars]
 
+from m5.util import code_formatter
+
 ########################################################################
 # Code for adding source files of various types
 #
@@ -142,8 +144,8 @@ class PySource(SourceFile):
         self.arcname = joinpath(*arcpath)
         self.abspath = abspath
         self.compiled = File(self.filename + 'c')
-        self.assembly = File(self.filename + '.s')
+        self.cpp = File(self.filename + '.cc')
-        self.symname = "PyEMB_" + PySource.invalid_sym_char.sub('_', modpath)
+        self.symname = PySource.invalid_sym_char.sub('_', modpath)
 
         PySource.modules[modpath] = self
         PySource.tnodes[self.tnode] = self
@@ -446,24 +448,6 @@ env.Command('python/m5/info.py',
             makeInfoPyFile)
 PySource('m5', 'python/m5/info.py')
 
-# Generate the __init__.py file for m5.objects
-def makeObjectsInitFile(target, source, env):
-    code = code_formatter()
-    code('''\
-from params import *
-from m5.SimObject import *
-''')
-
-    for module in source:
-        code('from $0 import *', module.get_contents())
-    code.write(str(target[0]))
-
-# Generate an __init__.py file for the objects package
-env.Command('python/m5/objects/__init__.py',
-            map(Value, SimObject.modnames),
-            makeObjectsInitFile)
-PySource('m5.objects', 'python/m5/objects/__init__.py')
-
 ########################################################################
 #
 # Create all of the SimObject param headers and enum headers
@@ -632,39 +616,32 @@ env.Command(params_file, map(Value, names), buildParams)
 env.Depends(params_file, params_hh_files + params_i_files + depends)
 SwigSource('m5.objects', params_file)
 
+# Generate the main swig init file
+def makeEmbeddedSwigInit(target, source, env):
+    code = code_formatter()
+    module = source[0].get_contents()
+    code('''\
+#include "sim/init.hh"
+
+extern "C" {
+    void init_${module}();
+}
+
+EmbeddedSwig embed_swig_${module}(init_${module});
+''')
+    code.write(str(target[0]))
+    
 # Build all swig modules
 for swig in SwigSource.all:
     env.Command([swig.cc_source.tnode, swig.py_source.tnode], swig.tnode,
                 '$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} '
                 '-o ${TARGETS[0]} $SOURCES')
+    init_file = 'python/swig/init_%s.cc' % swig.module
+    env.Command(init_file, Value(swig.module), makeEmbeddedSwigInit)
+    Source(init_file)
     env.Depends(swig.py_source.tnode, swig.tnode)
     env.Depends(swig.cc_source.tnode, swig.tnode)
 
-# Generate the main swig init file
-def makeSwigInit(target, source, env):
-    code = code_formatter()
-
-    code('extern "C" {')
-    code.indent()
-    for module in source:
-        code('void init_$0();', module.get_contents())
-    code.dedent()
-    code('}')
-
-    code('void initSwig() {')
-    code.indent()
-    for module in source:
-        code('init_$0();', module.get_contents())
-    code.dedent()
-    code('}')
-
-    code.write(str(target[0]))
-
-env.Command('python/swig/init.cc',
-            map(Value, sorted(s.module for s in SwigSource.all)),
-            makeSwigInit)
-Source('python/swig/init.cc')
-
 def getFlags(source_flags):
     flagsMap = {}
     flagsList = []
@@ -892,13 +869,17 @@ env.Command('base/traceflags.hh', flags, traceFlagsHH)
 env.Command('base/traceflags.cc', flags, traceFlagsCC)
 Source('base/traceflags.cc')
 
-# embed python files.  All .py files that have been indicated by a
+# Embed python files.  All .py files that have been indicated by a
 # PySource() call in a SConscript need to be embedded into the M5
 # library.  To do that, we compile the file to byte code, marshal the
-# byte code, compress it, and then generate an assembly file that
+# byte code, compress it, and then generate a c++ file that
-# inserts the result into the data section with symbols indicating the
+# inserts the result into an array.
-# beginning, and end (and with the size at the end)
+def embedPyFile(target, source, env):
-def objectifyPyFile(target, source, env):
+    def c_str(string):
+        if string is None:
+            return "0"
+        return '"%s"' % string
+
     '''Action function to compile a .py into a code object, marshal
     it, compress it, and stick it into an asm file so the code appears
     as just bytes with a label in the data section'''
@@ -910,90 +891,40 @@ def objectifyPyFile(target, source, env):
     marshalled = marshal.dumps(compiled)
     compressed = zlib.compress(marshalled)
     data = compressed
+    sym = pysource.symname
 
-    # Some C/C++ compilers prepend an underscore to global symbol
-    # names, so if they're going to do that, we need to prepend that
-    # leading underscore to globals in the assembly file.
-    if env['LEADING_UNDERSCORE']:
-        sym = '_' + pysource.symname
-    else:
-        sym = pysource.symname
-
-    step = 16
     code = code_formatter()
     code('''\
-.data
+#include "sim/init.hh"
-.globl ${sym}_beg
-.globl ${sym}_end
-${sym}_beg:''')
 
+namespace {
+
+const char data_${sym}[] = {
+''')
+    code.indent()
+    step = 16
     for i in xrange(0, len(data), step):
         x = array.array('B', data[i:i+step])
-        bytes = ','.join([str(d) for d in x])
+        code(''.join('%d,' % d for d in x))
-        code('.byte $bytes')
+    code.dedent()
-    code('${sym}_end:')
+    
-    code('.long $0', len(marshalled))
+    code('''};
 
+EmbeddedPython embedded_${sym}(
+    ${{c_str(pysource.arcname)}},
+    ${{c_str(pysource.abspath)}},
+    ${{c_str(pysource.modpath)}},
+    data_${sym},
+    ${{len(data)}},
+    ${{len(marshalled)}});
+
+/* namespace */ }
+''')
     code.write(str(target[0]))
 
 for source in PySource.all:
-    env.Command(source.assembly, source.tnode, objectifyPyFile)
+    env.Command(source.cpp, source.tnode, embedPyFile)
-    Source(source.assembly)
+    Source(source.cpp)
-
-# Generate init_python.cc which creates a bunch of EmbeddedPyModule
-# structs that describe the embedded python code.  One such struct
-# contains information about the importer that python uses to get at
-# the embedded files, and then there's a list of all of the rest that
-# the importer uses to load the rest on demand.
-def pythonInit(target, source, env):
-    code = code_formatter()
-
-    def dump_mod(sym, endchar=','):
-        def c_str(string):
-            if string is None:
-                return "0"
-            return '"%s"' % string
-
-        pysource = PySource.symnames[sym]
-        arcname = c_str(pysource.arcname)
-        abspath = c_str(pysource.abspath)
-        modpath = c_str(pysource.modpath)
-        code.indent()
-        code('''\
-{ $arcname,
-  $abspath,
-  $modpath,
-  ${sym}_beg, ${sym}_end,
-  ${sym}_end - ${sym}_beg,
-  *(int *)${sym}_end }$endchar
-''')
-        code.dedent()
-
-    code('#include "sim/init.hh"')
-    for sym in source:
-        sym = sym.get_contents()
-        code('extern const char ${sym}_beg[], ${sym}_end[];')
-
-    code('const EmbeddedPyModule embeddedPyImporter = ')
-    dump_mod("PyEMB_importer", endchar=';')
-    code()
-
-    code('const EmbeddedPyModule embeddedPyModules[] = {')
-    for i,sym in enumerate(source):
-        sym = sym.get_contents()
-        if sym == "PyEMB_importer":
-            # Skip the importer since we've already exported it
-            continue
-        dump_mod(sym)
-    code('    { 0, 0, 0, 0, 0, 0, 0 }')
-    code('};')
-
-    code.write(str(target[0]))
-
-env.Command('sim/init_python.cc',
-            map(Value, (s.symname for s in PySource.all)),
-            pythonInit)
-Source('sim/init_python.cc')
 
 ########################################################################
 #

src/python/SConscript

@@ -49,6 +49,7 @@ PySource('m5', 'm5/simulate.py')
 PySource('m5', 'm5/stats.py')
 PySource('m5', 'm5/ticks.py')
 PySource('m5', 'm5/trace.py')
+PySource('m5.objects', 'm5/objects/__init__.py')
 PySource('m5.util', 'm5/util/__init__.py')
 PySource('m5.util', 'm5/util/attrdict.py')
 PySource('m5.util', 'm5/util/code_formatter.py')

src/python/m5/objects/__init__.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2010 The Hewlett-Packard Development Company
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
+from m5.objects.params import *
+from m5.SimObject import *
+
+try:
+    modules = __loader__.modules
+except NameError:
+    modules = { }
+
+for module in modules.iterkeys():
+    if module.startswith('m5.objects.') and module != 'm5.objects.params':
+        exec "from %s import *" % module

src/sim/init.cc

@@ -33,6 +33,7 @@
 #include <marshal.h>
 #include <signal.h>
 
+#include <list>
 #include <iostream>
 #include <string>
 #include <zlib.h>
@@ -104,69 +105,121 @@ initSignals()
     signal(SIGABRT, abortHandler);
 }
 
-/*
- * Uncompress and unmarshal the code object stored in the
- * EmbeddedPyModule
- */
-PyObject *
-getCode(const EmbeddedPyModule *pymod)
-{
-    assert(pymod->zlen == pymod->code_end - pymod->code);
-    Bytef *marshalled = new Bytef[pymod->mlen];
-    uLongf unzlen = pymod->mlen;
-    int ret = uncompress(marshalled, &unzlen, (const Bytef *)pymod->code,
-        pymod->zlen);
-    if (ret != Z_OK)
-        panic("Could not uncompress code: %s\n", zError(ret));
-    assert(unzlen == (uLongf)pymod->mlen);
-
-    return PyMarshal_ReadObjectFromString((char *)marshalled, pymod->mlen);
-}
-
 // The python library is totally messed up with respect to constness,
 // so make a simple macro to make life a little easier
 #define PyCC(x) (const_cast<char *>(x))
 
+EmbeddedPython *EmbeddedPython::importer = NULL;
+PyObject *EmbeddedPython::importerModule = NULL;
+EmbeddedPython::EmbeddedPython(const char *filename, const char *abspath,
+    const char *modpath, const char *code, int zlen, int len)
+    : filename(filename), abspath(abspath), modpath(modpath), code(code),
+      zlen(zlen), len(len)
+{
+    // if we've added the importer keep track of it because we need it
+    // to bootstrap.
+    if (string(modpath) == string("importer"))
+        importer = this;
+    else
+        getList().push_back(this);
+}
+
+list<EmbeddedPython *> &
+EmbeddedPython::getList()
+{
+    static list<EmbeddedPython *> the_list;
+    return the_list;
+}
+
+/*
+ * Uncompress and unmarshal the code object stored in the
+ * EmbeddedPython
+ */
+PyObject *
+EmbeddedPython::getCode() const
+{
+    Bytef marshalled[len];
+    uLongf unzlen = len;
+    int ret = uncompress(marshalled, &unzlen, (const Bytef *)code, zlen);
+    if (ret != Z_OK)
+        panic("Could not uncompress code: %s\n", zError(ret));
+    assert(unzlen == (uLongf)len);
+
+    return PyMarshal_ReadObjectFromString((char *)marshalled, len);
+}
+
+bool
+EmbeddedPython::addModule() const
+{
+    PyObject *code = getCode();
+    PyObject *result = PyObject_CallMethod(importerModule, PyCC("add_module"),
+        PyCC("sssO"), filename, abspath, modpath, code);
+    if (!result) {
+        PyErr_Print();
+        return false;
+    }
+
+    Py_DECREF(result);
+    return true;
+}
+
 /*
  * Load and initialize all of the python parts of M5, including Swig
  * and the embedded module importer.
  */
 int
-initM5Python()
+EmbeddedPython::initAll()
 {
-    extern void initSwig();
-
-    // initialize SWIG modules.  initSwig() is autogenerated and calls
-    // all of the individual swig initialization functions.
-    initSwig();
-
     // Load the importer module
-    PyObject *code = getCode(&embeddedPyImporter);
+    PyObject *code = importer->getCode();
-    PyObject *module = PyImport_ExecCodeModule(PyCC("importer"), code);
+    importerModule = PyImport_ExecCodeModule(PyCC("importer"), code);
-    if (!module) {
+    if (!importerModule) {
         PyErr_Print();
         return 1;
     }
 
     // Load the rest of the embedded python files into the embedded
     // python importer
-    const EmbeddedPyModule *pymod = &embeddedPyModules[0];
+    list<EmbeddedPython *>::iterator i = getList().begin();
-    while (pymod->filename) {
+    list<EmbeddedPython *>::iterator end = getList().end();
-        PyObject *code = getCode(pymod);
+    for (; i != end; ++i)
-        PyObject *result = PyObject_CallMethod(module, PyCC("add_module"),
+        if (!(*i)->addModule())
-            PyCC("sssO"), pymod->filename, pymod->abspath, pymod->modpath,
-            code);
-        if (!result) {
-            PyErr_Print();
             return 1;
-        }
-        Py_DECREF(result);
-        ++pymod;
-    }
 
     return 0;
 }
 
+EmbeddedSwig::EmbeddedSwig(void (*init_func)())
+    : initFunc(init_func)
+{
+    getList().push_back(this);
+}
+
+list<EmbeddedSwig *> &
+EmbeddedSwig::getList()
+{
+    static list<EmbeddedSwig *> the_list;
+    return the_list;
+}
+
+void
+EmbeddedSwig::initAll()
+{
+    // initialize SWIG modules.  initSwig() is autogenerated and calls
+    // all of the individual swig initialization functions.
+    list<EmbeddedSwig *>::iterator i = getList().begin();
+    list<EmbeddedSwig *>::iterator end = getList().end();
+    for (; i != end; ++i)
+        (*i)->initFunc();
+}
+
+int
+initM5Python()
+{
+    EmbeddedSwig::initAll();
+    return EmbeddedPython::initAll();
+}
+
 /*
  * Start up the M5 simulator.  This mostly vectors into the python
  * main function.

src/sim/init.hh

@@ -34,19 +34,43 @@
 /*
  * Data structure describing an embedded python file.
  */
-struct EmbeddedPyModule
+#include <list>
+
+#ifndef PyObject_HEAD
+struct _object;
+typedef _object PyObject;
+#endif
+
+struct EmbeddedPython
 {
     const char *filename;
     const char *abspath;
     const char *modpath;
     const char *code;
-    const char *code_end;
     int zlen;
-    int mlen;
+    int len;
+
+    EmbeddedPython(const char *filename, const char *abspath,
+        const char *modpath, const char *code, int zlen, int len);
+
+    PyObject *getCode() const;
+    bool addModule() const;
+
+    static EmbeddedPython *importer;
+    static PyObject *importerModule;
+    static std::list<EmbeddedPython *> &getList();
+    static int initAll();
 };
 
-extern const EmbeddedPyModule embeddedPyImporter;
+struct EmbeddedSwig
-extern const EmbeddedPyModule embeddedPyModules[];
+{
+    void (*initFunc)();
+
+    EmbeddedSwig(void (*init_func)());
+
+    static std::list<EmbeddedSwig *> &getList();
+    static void initAll();
+};
 
 void initSignals();
 int initM5Python();