Add python output support to the statistics package!

base/statistics.cc:
base/statistics.hh:
    -  add python output support to the statistics package
    -  each statistic type has a python() member function that takes
    a Python object to which the stat will output it's python
    representation
    -  add getStatData hack so that the StatData pointer can be looked
    up by the proxies with their opaque pointer to the stat they're
    proxying for.  This is necessary because the proxy really proxies
    for the bin and not the stat.  Be nice to figure out how to get
    rid of it.  The hack is used so that the str() function of a
    proxy can properly name itself.
    -  To print formula stats, every stat has a str() function that
    converts that stat to a string that python can execute to get
    a value.
test/Makefile:
    add python stuff
test/stattest.cc:
    add more tests and test python support

--HG--
extra : convert_revision : 513814ab0a125606897f2c57dccdf22879032ef9

base/statistics.cc

@@ -35,9 +35,12 @@
 
 #include "base/callback.hh"
 #include "base/cprintf.hh"
+#include "base/hostinfo.hh"
 #include "base/misc.hh"
+#include "base/python.hh"
 #include "base/statistics.hh"
 #include "base/str.hh"
+#include "base/time.hh"
 #include "base/trace.hh"
 
 #ifdef __M5_NAN
@@ -82,6 +85,8 @@ namespace Database
       public:
         void dump(ostream &stream, const string &name, DisplayMode mode);
         void display(ostream &stream, DisplayMode mode);
+        void python(ostream &stream, const string &name);
+        void python(Python &py, const string &name, const string &bin);
 
         StatData *find(void *stat);
         void mapStat(void *stat, StatData *data);
@@ -101,6 +106,9 @@ Data::dump(ostream &stream, const string &name, DisplayMode mode)
     MainBin *orig = MainBin::curBin();
 
     switch (mode) {
+      case mode_python:
+        python(stream, name);
+        break;
       case mode_m5:
       case mode_simplescalar:
         display(stream, mode);
@@ -149,6 +157,53 @@ Data::display(ostream &stream, DisplayMode mode)
     }
 }
 
+void
+Data::python(ostream &stream, const string &name)
+{
+    Python py(stream);
+
+    ccprintf(stream, "import sys\n");
+    ccprintf(stream, "sys.path.append('.')\n");
+    ccprintf(stream, "from m5stats import *\n\n");
+
+    if (bins.empty()) {
+        python(py, name, "");
+    } else {
+        list<MainBin *>::iterator i = bins.begin();
+        list<MainBin *>::iterator end = bins.end();
+
+        while (i != end) {
+            (*i)->activate();
+            python(py, name, (*i)->name());
+            ++i;
+        }
+    }
+
+    py.next();
+    ccprintf(stream, "if __name__ == '__main__':\n");
+    ccprintf(stream, "    program_display()\n");
+}
+
+void
+Data::python(Python &py, const string &name, const string &bin)
+{
+    py.start("collections.append");
+    py.start("Collection");
+    py.qarg(name);
+    py.qarg(bin);
+    py.qarg(hostname());
+    py.qarg(Time::start.date());
+    list_t::iterator i = allStats.begin();
+    list_t::iterator end = allStats.end();
+    while (i != end) {
+        StatData *stat = *i;
+        stat->python(py);
+        ++i;
+    }
+    py.end();
+    py.end();
+}
+
 StatData *
 Data::find(void *stat)
 {
@@ -272,6 +327,12 @@ DataAccess::find() const
     return Database::StatDB().find(const_cast<void *>((const void *)this));
 }
 
+const StatData *
+getStatData(const void *stat)
+{
+    return Database::StatDB().find(const_cast<void *>(stat));
+}
+
 void
 DataAccess::map(StatData *data)
 {
@@ -901,6 +962,165 @@ VectorDistDataBase::display(ostream &stream, DisplayMode mode) const
     }
 }
 
+void
+ScalarDataBase::python(Python &py) const
+{
+    py.start("Scalar");
+    py.qarg(name);
+    py.qarg(desc);
+    py.kwarg("binned", binned());
+    py.kwarg("precision", precision);
+    py.kwarg("flags", flags);
+    if (prereq)
+        py.qkwarg("prereq", prereq->name);
+    py.kwarg("value", val());
+    py.end();
+}
+
+void
+VectorDataBase::python(Python &py) const
+{
+    const_cast<VectorDataBase *>(this)->update();
+
+    py.start("Vector");
+    py.qarg(name);
+    py.qarg(desc);
+    py.kwarg("binned", binned());
+    py.kwarg("precision", precision);
+    py.kwarg("flags", flags);
+    if (prereq)
+        py.qkwarg("prereq", prereq->name);
+    py.kwarg("value", val());
+    if (!subnames.empty())
+        py.qkwarg("subnames", subnames);
+    if (!subdescs.empty())
+        py.qkwarg("subdescs", subdescs);
+    py.end();
+}
+
+void
+DistDataData::python(Python &py, const string &name) const
+{
+    string s = name.empty() ? "" : name + "=";
+
+    if (samples == 0 || fancy)
+        s += "SimpleDist";
+    else
+        s += "FullDist";
+
+    py.start(s);
+    py.arg(sum);
+    py.arg(squares);
+    py.arg(samples);
+    if (samples && !fancy) {
+        py.arg(min_val);
+        py.arg(min_val);
+        py.arg(underflow);
+        py.arg(vec);
+        py.arg(overflow);
+        py.arg(min);
+        py.arg(max);
+        py.arg(bucket_size);
+        py.arg(size);
+    }
+    py.end();
+}
+
+void
+FormulaDataBase::python(Python &py) const
+{
+    const_cast<FormulaDataBase *>(this)->update();
+
+    py.start("Formula");
+    py.qarg(name);
+    py.qarg(desc);
+    py.kwarg("binned", binned());
+    py.kwarg("precision", precision);
+    py.kwarg("flags", flags);
+    if (prereq)
+        py.qkwarg("prereq", prereq->name);
+    py.qkwarg("formula", str());
+    if (!subnames.empty())
+        py.qkwarg("subnames", subnames);
+    if (!subdescs.empty())
+        py.qkwarg("subdescs", subdescs);
+    py.end();
+}
+
+void
+DistDataBase::python(Python &py) const
+{
+    const_cast<DistDataBase *>(this)->update();
+
+    py.start("Dist");
+    py.qarg(name);
+    py.qarg(desc);
+    py.kwarg("binned", binned());
+    py.kwarg("precision", precision);
+    py.kwarg("flags", flags);
+    if (prereq)
+        py.qkwarg("prereq", prereq->name);
+    data.python(py, "dist");
+    py.end();
+}
+
+void
+VectorDistDataBase::python(Python &py) const
+{
+    const_cast<VectorDistDataBase *>(this)->update();
+
+    py.start("VectorDist");
+    py.qarg(name);
+    py.qarg(desc);
+    py.kwarg("binned", binned());
+    py.kwarg("precision", precision);
+    py.kwarg("flags", flags);
+    if (prereq)
+        py.qkwarg("prereq", prereq->name);
+    if (!subnames.empty())
+        py.qkwarg("subnames", subnames);
+    if (!subdescs.empty())
+        py.qkwarg("subdescs", subdescs);
+
+    py.tuple("dist");
+    typedef std::vector<DistDataData>::const_iterator iter;
+    iter i = data.begin();
+    iter end = data.end();
+    while (i != end) {
+        i->python(py, "");
+        ++i;
+    }
+    py.endTuple();
+    py.end();
+}
+
+void
+Vector2dDataBase::python(Python &py) const
+{
+    const_cast<Vector2dDataBase *>(this)->update();
+
+    py.start("Vector2d");
+    py.qarg(name);
+    py.qarg(desc);
+    py.kwarg("binned", binned());
+    py.kwarg("precision", precision);
+    py.kwarg("flags", flags);
+    if (prereq)
+        py.qkwarg("prereq", prereq->name);
+
+    py.kwarg("value", vec);
+    if (!subnames.empty())
+        py.qkwarg("subnames", subnames);
+    if (!subdescs.empty())
+        py.qkwarg("subdescs", subdescs);
+    if (!y_subnames.empty())
+        py.qkwarg("ysubnames", y_subnames);
+
+    py.kwarg("x", x);
+    py.kwarg("y", y);
+    py.end();
+}
+
 void
 FormulaBase::val(rvec_t &vec) const
 {
@@ -949,6 +1169,12 @@ FormulaBase::update(StatData *)
 {
 }
 
+string
+FormulaBase::str() const
+{
+    return root ? root->str() : "";
+}
+
 Formula::Formula()
 {
     setInit();

base/statistics.hh

@@ -75,6 +75,7 @@ float __nan();
 #endif
 
 class Callback;
+class Python;
 
 /** The current simulated cycle. */
 extern Tick curTick;
@@ -162,6 +163,7 @@ struct StatData
      * @param stream The stream to print to.
      */
     virtual void display(std::ostream &stream, DisplayMode mode) const = 0;
+    virtual void python(Python &py) const = 0;
     bool dodisplay() const { return !prereq || !prereq->zero(); }
 
     /**
@@ -200,6 +202,7 @@ struct ScalarDataBase : public StatData
     virtual result_t total() const = 0;
 
     virtual void display(std::ostream &stream, DisplayMode mode) const;
+    virtual void python(Python &py) const;
 };
 
 template <class T>
@@ -226,6 +229,7 @@ struct VectorDataBase : public StatData
     mutable std::vector<std::string> subdescs;
 
     virtual void display(std::ostream &stream, DisplayMode mode) const;
+    virtual void python(Python &py) const;
 
     virtual size_t size() const  = 0;
     virtual const rvec_t &val() const  = 0;
@@ -288,6 +292,8 @@ struct DistDataData
     int bucket_size;
     int size;
     bool fancy;
+
+    void python(Python &py, const std::string &name) const;
 };
 
 struct DistDataBase : public StatData
@@ -296,6 +302,7 @@ struct DistDataBase : public StatData
     DistDataData data;
 
     virtual void display(std::ostream &stream, DisplayMode mode) const;
+    virtual void python(Python &py) const;
     virtual void update() = 0;
 };
 
@@ -328,6 +335,7 @@ struct VectorDistDataBase : public StatData
 
     virtual size_t size() const = 0;
     virtual void display(std::ostream &stream, DisplayMode mode) const;
+    virtual void python(Python &py) const;
     virtual void update()
     {
         int s = size();
@@ -374,6 +382,7 @@ struct Vector2dDataBase : public StatData
     mutable int y;
 
     virtual void display(std::ostream &stream, DisplayMode mode) const;
+    virtual void python(Python &py) const;
     virtual void update()
     {
         if (subnames.size() < x)
@@ -989,6 +998,8 @@ class VectorBase : public DataAccess
     void update(StatData *data) {}
 };
 
+const StatData * getStatData(const void *stat);
+
 /**
  * A proxy class to access the stat at a given index in a VectorBase stat.
  * Behaves like a ScalarBase.
@@ -1011,6 +1022,8 @@ class ScalarProxy
     params_t *params;
     /** The index to access in the parent VectorBase. */
     int index;
+    /** Keep a pointer to the original stat so was can get data */
+    void *stat;
 
   protected:
     /**
@@ -1048,14 +1061,14 @@ class ScalarProxy
      * @param p The params to use.
      * @param i The index to access.
      */
-    ScalarProxy(bin_t &b, params_t &p, int i)
+    ScalarProxy(bin_t &b, params_t &p, int i, void *s)
-        : bin(&b), params(&p), index(i)  {}
+        : bin(&b), params(&p), index(i), stat(s)  {}
     /**
      * Create a copy of the provided ScalarProxy.
      * @param sp The proxy to copy.
      */
     ScalarProxy(const ScalarProxy &sp)
-        : bin(sp.bin), params(sp.params), index(sp.index) {}
+        : bin(sp.bin), params(sp.params), index(sp.index), stat(sp.stat) {}
     /**
      * Set this proxy equal to the provided one.
      * @param sp The proxy to copy.
@@ -1065,6 +1078,7 @@ class ScalarProxy
         bin = sp.bin;
         params = sp.params;
         index = sp.index;
+        stat = sp.stat;
         return *this;
     }
 
@@ -1126,6 +1140,14 @@ class ScalarProxy
      * This stat has no state.  Nothing to reset
      */
     void reset() {  }
+
+  public:
+    const StatData *statData() const { return getStatData(stat); }
+    std::string str() const
+    {
+        return csprintf("%s[%d]", statData()->name, index);
+
+    }
 };
 
 template <typename T, template <typename T> class Storage, class Bin>
@@ -1133,7 +1155,7 @@ inline ScalarProxy<T, Storage, Bin>
 VectorBase<T, Storage, Bin>::operator[](int index)
 {
     assert (index >= 0 && index < size());
-    return ScalarProxy<T, Storage, Bin>(bin, params, index);
+    return ScalarProxy<T, Storage, Bin>(bin, params, index, this);
 }
 
 template <typename T, template <typename T> class Storage, class Bin>
@@ -1207,6 +1229,7 @@ class VectorProxy
     params_t *params;
     int offset;
     int len;
+    void *stat;
 
   private:
     mutable rvec_t *vec;
@@ -1243,14 +1266,19 @@ class VectorProxy
     }
 
   public:
-    VectorProxy(bin_t &b, params_t &p, int o, int l)
+    VectorProxy(bin_t &b, params_t &p, int o, int l, void *s)
-        : bin(&b), params(&p), offset(o), len(l), vec(NULL)
+        : bin(&b), params(&p), offset(o), len(l), stat(s), vec(NULL)
-        { }
+    {
+    }
+
     VectorProxy(const VectorProxy &sp)
         : bin(sp.bin), params(sp.params), offset(sp.offset), len(sp.len),
-          vec(NULL)
+          stat(sp.stat), vec(NULL)
-        { }
+    {
-    ~VectorProxy() {
+    }
+
+    ~VectorProxy()
+    {
         if (vec)
             delete vec;
     }
@@ -1261,6 +1289,7 @@ class VectorProxy
         params = sp.params;
         offset = sp.offset;
         len = sp.len;
+        stat = sp.stat;
         if (vec)
             delete vec;
         vec = NULL;
@@ -1270,7 +1299,8 @@ class VectorProxy
     ScalarProxy<T, Storage, Bin> operator[](int index)
     {
         assert (index >= 0 && index < size());
-        return ScalarProxy<T, Storage, Bin>(*bin, *params, offset + index);
+        return ScalarProxy<T, Storage, Bin>(*bin, *params, offset + index,
+                                            stat);
     }
 
     size_t size() const { return len; }
@@ -1293,7 +1323,7 @@ Vector2dBase<T, Storage, Bin>::operator[](int index)
 {
     int offset = index * y;
     assert (index >= 0 && offset < size());
-    return VectorProxy<T, Storage, Bin>(bin, params, offset, y);
+    return VectorProxy<T, Storage, Bin>(bin, params, offset, y, this);
 }
 
 //////////////////////////////////////////////////////////////////////
@@ -1834,6 +1864,11 @@ class Node : public RefCounted
      *@return True is stat is binned.
      */
     virtual bool binned() const = 0;
+
+    /**
+     *
+     */
+    virtual std::string str() const = 0;
 };
 
 /** Reference counting pointer to a function Node. */
@@ -1860,6 +1895,11 @@ class ScalarStatNode : public Node
      *@return True is stat is binned.
      */
     virtual bool binned() const { return data->binned(); }
+
+    /**
+     *
+     */
+    virtual std::string str() const { return data->name; }
 };
 
 template <typename T, template <typename T> class Storage, class Bin>
@@ -1885,6 +1925,11 @@ class ScalarProxyNode : public Node
      *@return True is stat is binned.
      */
     virtual bool binned() const { return proxy.binned(); }
+
+    /**
+     *
+     */
+    virtual std::string str() const { return proxy.str(); }
 };
 
 class VectorStatNode : public Node
@@ -1903,6 +1948,8 @@ class VectorStatNode : public Node
      *@return True is stat is binned.
      */
     virtual bool binned() const { return data->binned(); }
+
+    virtual std::string str() const { return data->name; }
 };
 
 template <typename T>
@@ -1922,6 +1969,8 @@ class ConstNode : public Node
      *@return False since constants aren't binned.
      */
     virtual bool binned() const { return false; }
+
+    virtual std::string str() const { return to_string(data[0]); }
 };
 
 template <typename T>
@@ -1945,6 +1994,7 @@ class FunctorNode : public Node
      *@return False since Functors aren't binned
      */
     virtual bool binned() const { return false; }
+    virtual std::string str() const { return to_string(functor()); }
 };
 
 template <typename T>
@@ -1968,6 +2018,46 @@ class ScalarNode : public Node
      *@return False since Scalar's aren't binned
      */
     virtual bool binned() const { return false; }
+    virtual std::string str() const { return to_string(scalar); }
+};
+
+template <class Op>
+struct OpString;
+
+template<>
+struct OpString<std::plus<result_t> >
+{
+    static std::string str() { return "+"; }
+};
+
+template<>
+struct OpString<std::minus<result_t> >
+{
+    static std::string str() { return "-"; }
+};
+
+template<>
+struct OpString<std::multiplies<result_t> >
+{
+    static std::string str() { return "*"; }
+};
+
+template<>
+struct OpString<std::divides<result_t> >
+{
+    static std::string str() { return "/"; }
+};
+
+template<>
+struct OpString<std::modulus<result_t> >
+{
+    static std::string str() { return "%"; }
+};
+
+template<>
+struct OpString<std::negate<result_t> >
+{
+    static std::string str() { return "-"; }
 };
 
 template <class Op>
@@ -2005,6 +2095,11 @@ class UnaryNode : public Node
      *@return True if child of node is binned.
      */
     virtual bool binned() const { return l->binned(); }
+
+    virtual std::string str() const
+    {
+        return OpString<Op>::str() + l->str();
+    }
 };
 
 template <class Op>
@@ -2070,6 +2165,11 @@ class BinaryNode : public Node
      *@return True if either child of node is binned.
      */
     virtual bool binned() const { return (l->binned() || r->binned()); }
+
+    virtual std::string str() const
+    {
+        return csprintf("(%s %s %s)", l->str(), OpString<Op>::str(), r->str());
+    }
 };
 
 template <class Op>
@@ -2116,6 +2216,11 @@ class SumNode : public Node
      *@return True if child of node is binned.
      */
     virtual bool binned() const { return l->binned(); }
+
+    virtual std::string str() const
+    {
+        return csprintf("total(%s)", l->str());
+    }
 };
 
 //////////////////////////////////////////////////////////////////////
@@ -2125,6 +2230,9 @@ class SumNode : public Node
 //////////////////////////////////////////////////////////////////////
 struct MainBin
 {
+    class BinBase;
+    friend class MainBin::BinBase;
+
   private:
     std::string _name;
     char *mem;
@@ -2778,6 +2886,45 @@ class FormulaBase : public DataAccess
      *
      */
     void update(StatData *);
+
+    std::string str() const;
+};
+
+class FormulaDataBase : public VectorDataBase
+{
+  public:
+    virtual std::string str() const = 0;
+    virtual bool check() const { return true; }
+    virtual void python(Python &py) const;
+};
+
+template <class T>
+class FormulaData : public FormulaDataBase
+{
+  protected:
+    T &s;
+    mutable rvec_t vec;
+
+  public:
+    FormulaData(T &stat) : s(stat) {}
+
+    virtual bool binned() const { return s.binned(); }
+    virtual bool zero() const { return s.zero(); }
+    virtual void reset() { s.reset(); }
+
+    virtual size_t size() const { return s.size(); }
+    virtual const rvec_t &val() const
+    {
+        s.val(vec);
+        return vec;
+    }
+    virtual result_t total() const { return s.total(); }
+    virtual void update()
+    {
+        VectorDataBase::update();
+        s.update(this);
+    }
+    virtual std::string str() const { return s.str(); }
 };
 
 class Temp;
@@ -2827,6 +2974,8 @@ class FormulaNode : public Node
     virtual const rvec_t &val() const { formula.val(vec); return vec; }
     virtual result_t total() const { return formula.total(); }
     virtual bool binned() const { return formula.binned(); }
+
+    virtual std::string str() const { return formula.str(); }
 };
 
 /**

test/Makefile

@@ -52,7 +52,8 @@ offtest: offtest.o
 rangetest: rangetest.o str.o
 	$(CXX) $(LFLAGS) -o $@ $^
 
-stattest: cprintf.o hostinfo.o misc.o statistics.o stattest.o str.o
+stattest: cprintf.o hostinfo.o misc.o python.o statistics.o stattest.o \
+	str.o time.o
 	$(CXX) $(LFLAGS) -o $@ $^
 
 strnumtest: strnumtest.o str.o

test/stattest.cc

@@ -28,6 +28,7 @@
 
 #include <iomanip>
 #include <iostream>
+#include <fstream>
 #include <string>
 #include <unistd.h>
 
@@ -48,7 +49,7 @@ Average<> s3;
 Scalar<Counter, MainBin> s4;
 Vector<Counter, MainBin> s5;
 Distribution<Counter, MainBin> s6;
-Vector<> s7;
+Vector<Counter, MainBin> s7;
 AverageVector<> s8;
 StandardDeviation<> s9;
 AverageDeviation<> s10;
@@ -65,6 +66,7 @@ Formula f3;
 Formula f4;
 Formula f5;
 Formula f6;
+Formula f7;
 
 MainBin bin1("bin1");
 MainBin bin2("bin2");
@@ -131,7 +133,7 @@ main(int argc, char *argv[])
     s3
         .name("Stat03")
         .desc("this is statistic 3")
-        .prereq(s11)
+        .prereq(f7)
         ;
 
     s4
@@ -253,18 +255,20 @@ main(int argc, char *argv[])
         .desc("this is formula 6")
         ;
 
-    check();
-
-    bin1.activate();
-
     f1 = s1 + s2;
-    f2 = (-s1) / (-s2) * -s3 + ULL(100) + s4;
+    f2 = (-s1) / (-s2) * (-s3 + ULL(100) + s4);
     f3 = sum(s5) * s7;
     f4 = functor(testfunc);
     TestClass testclass;
     f5 = functor(testclass);
     f6 += constant(10.0);
     f6 += s5[3];
+    f7 = constant(1);
+
+    check();
+    reset();
+
+    bin1.activate();
 
     s16[1][0] = 1;
     s16[0][1] = 3;
@@ -477,6 +481,14 @@ main(int argc, char *argv[])
     s6.sample(99);
     s6.sample(99);
 
+    s7[0] = 700;
+    s7[1] = 600;
+    s7[2] = 500;
+    s7[3] = 400;
+    s7[4] = 300;
+    s7[5] = 200;
+    s7[6] = 100;
+
     s9.sample(100);
     s9.sample(100);
     s9.sample(100);
@@ -497,20 +509,11 @@ main(int argc, char *argv[])
 
     s12.sample(100);
 
-    bin1.activate();
+//    dump(cout, mode_simplescalar);
-    cout << "dump 1" << endl;
+    ofstream file("/tmp/stats.py");
-    dump(cout);
+    dump(file, "stattest", mode_python);
-    cout << endl << endl;
+    file.close();
 
-    bin2.activate();
-    cout << "dump 2" << endl;
-    dump(cout);
-    cout << endl << endl;
-
-    cout << "dump 3" << endl;
-    reset();
-    dump(cout);
-    cout << endl << endl;
 
     return 0;
 }