stats: move code that loops over all stats into python

src/base/SConscript

@@ -71,9 +71,7 @@ Source('loader/object_file.cc')
 Source('loader/raw_object.cc')
 Source('loader/symtab.cc')
 
-Source('stats/output.cc')
 Source('stats/text.cc')
-Source('stats/visit.cc')
 
 if env['USE_MYSQL']:
     Source('mysql.cc')

src/base/statistics.cc

@@ -433,62 +433,8 @@ Formula::str() const
     return root ? root->str() : "";
 }
 
-void
-enable()
-{
-    typedef list<Info *>::iterator iter_t;
-
-    iter_t i, end = statsList().end();
-    for (i = statsList().begin(); i != end; ++i) {
-        Info *info = *i;
-        assert(info);
-        if (!info->check() || !info->baseCheck())
-            panic("stat check failed for '%s' %d\n", info->name, info->id);
-    }
-
-    off_t j = 0;
-    for (i = statsList().begin(); i != end; ++i) {
-        Info *info = *i;
-        if (!(info->flags & display))
-            info->name = "__Stat" + to_string(j++);
-    }
-
-    statsList().sort(Info::less);
-
-    for (i = statsList().begin(); i != end; ++i) {
-        Info *info = *i;
-        info->enable();
-    }
-}
-
-void
-prepare()
-{
-    list<Info *>::iterator i = statsList().begin();
-    list<Info *>::iterator end = statsList().end();
-    while (i != end) {
-        Info *info = *i;
-        info->prepare();
-        ++i;
-    }
-}
-
 CallbackQueue resetQueue;
 
-void
-reset()
-{
-    list<Info *>::iterator i = statsList().begin();
-    list<Info *>::iterator end = statsList().end();
-    while (i != end) {
-        Info *info = *i;
-        info->reset();
-        ++i;
-    }
-
-    resetQueue.process();
-}
-
 void
 registerResetCallback(Callback *cb)
 {
@@ -496,3 +442,9 @@ registerResetCallback(Callback *cb)
 }
 
 } // namespace Stats
+
+void
+debugDumpStats()
+{
+    Stats::dump();
+}

src/base/statistics.hh

@@ -60,8 +60,8 @@
 #include <vector>
 
 #include "base/stats/info.hh"
+#include "base/stats/output.hh"
 #include "base/stats/types.hh"
-#include "base/stats/visit.hh"
 #include "base/cast.hh"
 #include "base/cprintf.hh"
 #include "base/intmath.hh"
@@ -90,7 +90,7 @@ class InfoProxy : public Base
     void prepare() { s.prepare(); }
     void reset() { s.reset(); }
     void
-    visit(Visit &visitor)
+    visit(Output &visitor)
     {
         visitor.visit(*static_cast<Base *>(this));
     }
@@ -738,7 +738,7 @@ class ProxyInfo : public ScalarInfo
     void reset() { }
     bool zero() const { return value() == 0; }
 
-    void visit(Visit &visitor) { visitor.visit(*this); }
+    void visit(Output &visitor) { visitor.visit(*this); }
 };
 
 template <class T>
@@ -2933,28 +2933,10 @@ sum(Temp val)
     return NodePtr(new SumNode<std::plus<Result> >(val));
 }
 
-/**
+/** Dump all statistics data to the registered outputs */
- * Enable the statistics package.  Before the statistics package is
- * enabled, all statistics must be created and initialized and once
- * the package is enabled, no more statistics can be created.
- */
-void enable();
-
-/**
- * Prepare all stats for data access.  This must be done before
- * dumping and serialization.
- */
-void prepare();
-
-/**
- * Dump all statistics data to the registered outputs
- */
 void dump();
-
-/**
- * Reset all statistics to the base state
- */
 void reset();
+
 /**
  * Register a callback that should be called whenever statistics are
  * reset

src/base/stats/info.hh

@@ -62,7 +62,7 @@ const FlagsType nonan =         0x0200;
 const FlagsType __reserved =    init | display;
 
 struct StorageParams;
-struct Visit;
+struct Output;
 
 class Info
 {
@@ -129,7 +129,7 @@ class Info
     /**
      * Visitor entry for outputing statistics data
      */
-    virtual void visit(Visit &visitor) = 0;
+    virtual void visit(Output &visitor) = 0;
 
     /**
      * Checks if the first stat's name is alphabetically less than the second.

src/base/stats/mysql.cc

@@ -395,7 +395,8 @@ MySql::configure()
 
     list<Info *>::const_iterator i, end = statsList().end();
     for (i = statsList().begin(); i != end; ++i) {
-        (*i)->visit(*this);
+        Info *info = *i;
+        info->visit(*this);
     }
 
     for (i = statsList().begin(); i != end; ++i) {
@@ -606,7 +607,7 @@ MySql::valid() const
 }
 
 void
-MySql::output()
+MySql::begin()
 {
     assert(valid());
 
@@ -615,20 +616,22 @@ MySql::output()
 
     // store sample #
     newdata.tick = curTick();
+}
 
-    MySQL::Connection &mysql = run->conn();
+void
-
+MySql::end()
-    list<Info *>::const_iterator i, end = statsList().end();
+{
-    for (i = statsList().begin(); i != end; ++i) {
-        Info *stat = *i;
-        stat->visit(*this);
-        if (mysql.commit())
-            panic("could not commit transaction\n%s\n", mysql.error);
-    }
-
     newdata.flush();
 }
 
+void
+MySql::commit()
+{
+    MySQL::Connection &mysql = run->conn();
+    if (mysql.commit())
+        panic("could not commit transaction\n%s\n", mysql.error);
+}
+
 void
 MySql::output(const ScalarInfo &info)
 {
@@ -641,6 +644,8 @@ MySql::output(const ScalarInfo &info)
     newdata.data = info.value();
 
     newdata.insert();
+
+    commit();
 }
 
 void
@@ -659,6 +664,8 @@ MySql::output(const VectorInfo &info)
         newdata.data = cvec[x];
         newdata.insert();
     }
+
+    commit();
 }
 
 void
@@ -708,6 +715,8 @@ MySql::output(const DistData &data, const DistParams *params)
             newdata.insert();
         }
     }
+
+    commit();
 }
 
 void
@@ -719,6 +728,8 @@ MySql::output(const DistInfo &info)
     newdata.stat = find(info.id);
     newdata.y = 0;
     output(info.data, safe_cast<const DistParams *>(info.storageParams));
+
+    commit();
 }
 
 void
@@ -735,6 +746,8 @@ MySql::output(const VectorDistInfo &info)
         output(info.data[y],
                safe_cast<const DistParams *>(info.storageParams));
     }
+
+    commit();
 }
 
 void
@@ -754,6 +767,8 @@ MySql::output(const Vector2dInfo &info)
             newdata.insert();
         }
     }
+
+    commit();
 }
 
 void
@@ -821,20 +836,17 @@ MySql::visit(const FormulaInfo &info)
         output(info);
 }
 
-bool
+Output *
 initMySQL(string host, string user, string password, string database,
     string project, string name, string sample)
 {
-    extern list<Output *> OutputList;
     static MySql mysql;
 
-    if (mysql.connected())
+    if (mysql.connected()) {
-        return false;
+        mysql.connect(host, user, password, database, name, sample, project);
+    }
 
-    mysql.connect(host, user, password, database, name, sample, project);
+    return &mysql;
-    OutputList.push_back(&mysql);
-
-    return true;
 }
 
 } // namespace Stats

src/base/stats/mysql.hh

@@ -147,10 +147,12 @@ class MySql : public Output
 
     // Implement Output
     virtual bool valid() const;
-    virtual void output();
+    virtual void begin();
+    virtual void end();
 
   protected:
     // Output helper
+    void commit();
     void output(const ScalarInfo &info);
     void output(const VectorInfo &info);
     void output(const DistInfo &info);
@@ -169,17 +171,17 @@ class MySql : public Output
     void configure(const FormulaInfo &info);
 };
 
-bool initMySQL(std::string host, std::string database, std::string user,
+Output *initMySQL(std::string host, std::string database, std::string user,
     std::string passwd, std::string project, std::string name,
     std::string sample);
 
 #if !USE_MYSQL
-inline bool
+inline Output *
 initMySQL(std::string host, std::string user, std::string password,
     std::string database, std::string project, std::string name,
     std::string sample)
 {
-    return false;
+    return NULL;
 }
 #endif
 

src/base/stats/output.cc

@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Regents of The University of Michigan
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met: redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer;
- * redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution;
- * neither the name of the copyright holders nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Authors: Nathan Binkert
- */
-
-#include <list>
-
-#include "base/stats/output.hh"
-#include "base/statistics.hh"
-#include "base/types.hh"
-#include "sim/eventq.hh"
-
-using namespace std;
-
-namespace Stats {
-
-Tick lastDump(0);
-list<Output *> OutputList;
-
-void
-dump()
-{
-    assert(lastDump <= curTick());
-    if (lastDump == curTick())
-        return;
-    lastDump = curTick();
-
-    prepare();
-
-    list<Output *>::iterator i = OutputList.begin();
-    list<Output *>::iterator end = OutputList.end();
-    for (; i != end; ++i) {
-        Output *output = *i;
-        if (!output->valid())
-            continue;
-
-        output->output();
-    }
-}
-
-} // namespace Stats
-
-void
-debugDumpStats()
-{
-    Stats::dump();
-}
-

src/base/stats/output.hh

@@ -31,17 +31,31 @@
 #ifndef __BASE_STATS_OUTPUT_HH__
 #define __BASE_STATS_OUTPUT_HH__
 
+#include <list>
 #include <string>
 
-#include "base/stats/visit.hh"
-
 namespace Stats {
 
-struct Output : public Visit
+class Info;
+class ScalarInfo;
+class VectorInfo;
+class DistInfo;
+class VectorDistInfo;
+class Vector2dInfo;
+class FormulaInfo;
+
+struct Output
 {
-    inline void operator()() { output(); }
+    virtual void begin() = 0;
-    virtual void output() = 0;
+    virtual void end() = 0;
     virtual bool valid() const = 0;
+
+    virtual void visit(const ScalarInfo &info) = 0;
+    virtual void visit(const VectorInfo &info) = 0;
+    virtual void visit(const DistInfo &info) = 0;
+    virtual void visit(const VectorDistInfo &info) = 0;
+    virtual void visit(const Vector2dInfo &info) = 0;
+    virtual void visit(const FormulaInfo &info) = 0;
 };
 
 } // namespace Stats

src/base/stats/text.cc

@@ -48,7 +48,6 @@
 
 #include "base/stats/info.hh"
 #include "base/stats/text.hh"
-#include "base/stats/visit.hh"
 #include "base/cast.hh"
 #include "base/misc.hh"
 #include "base/str.hh"
@@ -138,12 +137,14 @@ Text::valid() const
 }
 
 void
-Text::output()
+Text::begin()
 {
     ccprintf(*stream, "\n---------- Begin Simulation Statistics ----------\n");
-    list<Info *>::const_iterator i, end = statsList().end();
+}
-    for (i = statsList().begin(); i != end; ++i)
+
-        (*i)->visit(*this);
+void
+Text::end()
+{
     ccprintf(*stream, "\n---------- End Simulation Statistics   ----------\n");
     stream->flush();
 }
@@ -580,23 +581,19 @@ Text::visit(const FormulaInfo &info)
     visit((const VectorInfo &)info);
 }
 
-bool
+Output *
 initText(const string &filename, bool desc)
 {
     static Text text;
     static bool connected = false;
 
-    if (connected)
+    if (!connected) {
-        return false;
+        text.open(*simout.find(filename));
+        text.descriptions = desc;
+        connected = true;
+    }
 
-    extern list<Output *> OutputList;
+    return &text;
-
-    text.open(*simout.find(filename));
-    text.descriptions = desc;
-    OutputList.push_back(&text);
-    connected = true;
-
-    return true;
 }
 
 } // namespace Stats

src/base/stats/text.hh

@@ -70,10 +70,11 @@ class Text : public Output
 
     // Implement Output
     virtual bool valid() const;
-    virtual void output();
+    virtual void begin();
+    virtual void end();
 };
 
-bool initText(const std::string &filename, bool desc);
+Output *initText(const std::string &filename, bool desc);
 
 } // namespace Stats
 

src/base/stats/visit.cc

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Regents of The University of Michigan
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met: redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer;
- * redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution;
- * neither the name of the copyright holders nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Authors: Nathan Binkert
- */
-
-#include "base/stats/visit.hh"
-
-namespace Stats {
-
-Visit::Visit()
-{}
-
-Visit::~Visit()
-{}
-
-} // namespace Stats

src/base/stats/visit.hh

@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Regents of The University of Michigan
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met: redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer;
- * redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution;
- * neither the name of the copyright holders nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Authors: Nathan Binkert
- */
-
-#ifndef __BASE_STATS_VISIT_HH__
-#define __BASE_STATS_VISIT_HH__
-
-namespace Stats {
-
-class Info;
-class ScalarInfo;
-class VectorInfo;
-class DistInfo;
-class VectorDistInfo;
-class Vector2dInfo;
-class FormulaInfo;
-
-struct Visit
-{
-    Visit();
-    virtual ~Visit();
-
-    virtual void visit(const ScalarInfo &info) = 0;
-    virtual void visit(const VectorInfo &info) = 0;
-    virtual void visit(const DistInfo &info) = 0;
-    virtual void visit(const VectorDistInfo &info) = 0;
-    virtual void visit(const Vector2dInfo &info) = 0;
-    virtual void visit(const FormulaInfo &info) = 0;
-};
-
-} // namespace Stats
-
-#endif // __BASE_STATS_VISIT_HH__

src/python/m5/simulate.py

@@ -131,7 +131,7 @@ def curTick():
     return internal.core.curTick()
 
 # Python exit handlers happen in reverse order.  We want to dump stats last.
-atexit.register(internal.stats.dump)
+atexit.register(stats.dump)
 
 # register our C++ exit callback function with Python
 atexit.register(internal.core.doExitCleanup)

src/python/m5/stats/__init__.py

@@ -27,13 +27,17 @@
 #
 # Authors: Nathan Binkert
 
+import m5
+
 from m5 import internal
 from m5.internal.stats import schedStatEvent as schedEvent
 from m5.objects import Root
-from m5.util import attrdict
+from m5.util import attrdict, fatal
 
+outputList = []
 def initText(filename, desc=True):
-    internal.stats.initText(filename, desc)
+    output = internal.stats.initText(filename, desc)
+    outputList.append(output)
 
 def initMySQL(host, database, user='', passwd='', project='test', name='test',
               sample='0'):
@@ -41,8 +45,9 @@ def initMySQL(host, database, user='', passwd='', project='test', name='test',
         import getpass
         user = getpass.getuser()
 
-    internal.stats.initMySQL(host, database, user, passwd, project, name,
+    output = internal.stats.initMySQL(host, database, user, passwd,
-                             sample)
+                                      project, name, sample)
+    outputList.append(output)
 
 def initSimStats():
     internal.stats.initSimStats()
@@ -70,6 +75,13 @@ def enable():
         else:
             fatal("unknown stat type %s", stat)
 
+    for stat in stats_list:
+        if not stat.check() or not stat.baseCheck():
+            fatal("stat check failed for '%s' %d\n", stat.name, stat.id)
+
+        if not (stat.flags & flags.display):
+            stat.name = "__Stat%06d" % stat.id
+
     def less(stat1, stat2):
         v1 = stat1.name.split('.')
         v2 = stat2.name.split('.')
@@ -78,24 +90,49 @@ def enable():
     stats_list.sort(less)
     for stat in stats_list:
         stats_dict[stat.name] = stat
+        stat.enable()
 
-    internal.stats.enable()
+def prepare():
+    '''Prepare all stats for data access.  This must be done before
+    dumping and serialization.'''
 
+    for stat in stats_list:
+        stat.prepare()
+
+lastDump = 0
 def dump():
-    # Currently prepare happens in the dump, but we should maybe move
+    '''Dump all statistics data to the registered outputs'''
-    # that out.
 
-    #internal.stats.prepare()
+    curTick = m5.curTick()
-    internal.stats.dump()
+
+    global lastDump
+    assert lastDump <= curTick
+    if lastDump == curTick:
+        return
+    lastDump = curTick
+
+    prepare()
+
+    for output in outputList:
+        if output.valid():
+            output.begin()
+            for stat in stats_list:
+                output.visit(stat)
+            output.end()
 
 def reset():
+    '''Reset all statistics to the base state'''
+
     # call reset stats on all SimObjects
     root = Root.getInstance()
     if root:
         for obj in root.descendants(): obj.resetStats()
 
     # call any other registered stats reset callbacks
-    internal.stats.reset()
+    for stat in stats_list:
+        stat.reset()
+
+    internal.stats.processResetQueue()
 
 flags = attrdict({
     'none'    : 0x0000,

src/python/swig/stats.i

@@ -39,6 +39,8 @@
 #include "base/stats/mysql.hh"
 #include "base/stats/text.hh"
 #include "base/stats/types.hh"
+#include "base/callback.hh"
+#include "base/misc.hh"
 #include "base/statistics.hh"
 #include "sim/core.hh"
 #include "sim/stat_control.hh"
@@ -63,6 +65,48 @@ Stats_Info_flags_set(Info *info, FlagsType flags)
     info->flags = flags;
 }
 
+inline void
+processResetQueue()
+{
+    extern CallbackQueue resetQueue;
+    resetQueue.process();
+}
+
+inline char *
+PCC(const char *string)
+{
+    return const_cast<char *>(string);
+}
+
+void
+call_module_function(const char *module_name, const char *func_name)
+{
+    PyObject *module = PyImport_ImportModule(PCC(module_name));
+    if (module == NULL)
+        panic("Could not import %s", module);
+
+    PyObject *result = PyObject_CallMethod(module, PCC(func_name), PCC(""));
+    if (result == NULL) {
+        PyErr_Print();
+        panic("failure on call to function %s", func_name);
+    }
+
+    Py_DECREF(module);
+    Py_DECREF(result);
+}
+
+void
+dump()
+{
+    call_module_function("m5.stats", "dump");
+}
+
+void
+reset()
+{
+    call_module_function("m5.stats", "reset");
+}
+
 } // namespace Stats
 %}
 
@@ -72,9 +116,10 @@ Stats_Info_flags_set(Info *info, FlagsType flags)
 
 %ignore Stats::Info::flags;
 
-%import "base/stats/types.hh"
+%import  "base/stats/types.hh"
 
 %include "base/stats/info.hh"
+%include "base/stats/output.hh"
 
 namespace std {
 %template(list_info) list<Stats::Info *>;
@@ -95,18 +140,15 @@ template <class T> T cast_info(Info *info);
 %template(dynamic_FormulaInfo) cast_info<FormulaInfo *>;
 
 void initSimStats();
-void initText(const std::string &filename, bool desc);
+Output *initText(const std::string &filename, bool desc);
-void initMySQL(std::string host, std::string database, std::string user,
+Output *initMySQL(std::string host, std::string database, std::string user,
     std::string passwd, std::string project, std::string name,
     std::string sample);
 
 void schedStatEvent(bool dump, bool reset,
                     Tick when = curTick(), Tick repeat = 0);
 
-void enable();
+void processResetQueue();
-void prepare();
-void dump();
-void reset();
 
 std::list<Info *> &statsList();