gem5/base/stats/mysql.cc

/*
 * Copyright (c) 2004 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <cassert>
#include <map>
#include <sstream>
#include <string>
#include <vector>

#include "base/misc.hh"
#include "base/mysql.hh"
#include "base/statistics.hh"
#include "base/stats/flags.hh"
#include "base/stats/mysql.hh"
#include "base/stats/mysql_run.hh"
#include "base/stats/statdb.hh"
#include "base/stats/types.hh"
#include "base/str.hh"
#include "sim/host.hh"

using namespace std;

namespace Stats {

MySqlRun MySqlDB;

bool
MySqlConnected()
{
    return MySqlDB.connected();
}

void
MySqlRun::connect(const string &host, const string &user, const string &passwd,
                  const string &db, const string &name, const string &sample,
                  const string &project)
{
    if (connected())
        panic("can only get one database connection at this time!");

    mysql.connect(host, user, passwd, db);
    if (mysql.error)
        panic("could not connect to database server\n%s\n", mysql.error);

    mysql.query("LOCK TABLES runs WRITE");
    if (mysql.error)
        panic("could not lock tables\n%s\n", mysql.error);

    remove(name);
//    cleanup();
    setup(name, sample, user, project);
    mysql.query("UNLOCK TABLES");
    if (mysql.error)
        panic("could not unlock tables\n%s\n", mysql.error);
}

void
MySqlRun::setup(const string &name, const string &sample, const string &user,
                const string &project)
{
    assert(mysql.connected());

    stringstream insert;
    ccprintf(insert,
             "INSERT INTO "
             "runs(rn_name,rn_sample,rn_user,rn_project,rn_date,rn_expire)"
             "values(\"%s\", \"%s\", \"%s\", \"%s\", NOW(),"
             "DATE_ADD(CURDATE(), INTERVAL 31 DAY))",
             name, sample, user, project);

    mysql.query(insert);
    if (mysql.error)
        panic("could not get a run\n%s\n", mysql.error);

    run_id = mysql.insert_id();
}

void
MySqlRun::remove(const string &name)
{
    assert(mysql.connected());
    stringstream sql;
    ccprintf(sql, "DELETE FROM runs WHERE rn_name=\"%s\"", name);
    mysql.query(sql);
    if (mysql.error)
        panic("could not delete run\n%s\n", mysql.error);
}

void
MySqlRun::cleanup()
{
    assert(mysql.connected());

    mysql.query("DELETE data "
                "FROM data "
                "LEFT JOIN runs ON dt_run=rn_id "
                "WHERE rn_id IS NULL");

    mysql.query("DELETE formula_ref "
                "FROM formula_ref "
                "LEFT JOIN runs ON fr_run=rn_id "
                "WHERE rn_id IS NULL");

    mysql.query("DELETE formulas "
                "FROM formulas "
                "LEFT JOIN formula_ref ON fm_stat=fr_stat "
                "WHERE fr_stat IS NULL");

    mysql.query("DELETE stats "
                "FROM stats "
                "LEFT JOIN data ON st_id=dt_stat "
                "WHERE dt_stat IS NULL");

    mysql.query("DELETE subdata "
                "FROM subdata "
                "LEFT JOIN data ON sd_stat=dt_stat "
                "WHERE dt_stat IS NULL");

    mysql.query("DELETE bins "
                "FROM bins "
                "LEFT JOIN data ON bn_id=dt_bin "
                "WHERE dt_bin IS NULL");

    mysql.query("DELETE events"
                "FROM events"
                "LEFT JOIN runs ON ev_run=rn_id"
                "WHERE rn_id IS NULL");

    mysql.query("DELETE event_names"
                "FROM event_names"
                "LEFT JOIN events ON en_id=ev_event"
                "WHERE ev_event IS NULL");
}

void
SetupStat::init()
{
    name = "";
    descr = "";
    type = "";
    print = false;
    prereq = 0;
    prec = -1;
    nozero = false;
    nonan = false;
    total = false;
    pdf = false;
    cdf = false;
    min = 0;
    max = 0;
    bktsize = 0;
    size = 0;
}

unsigned
SetupStat::setup()
{
    MySQL::Connection &mysql = MySqlDB.conn();

    stringstream insert;
    ccprintf(insert,
             "INSERT INTO "
             "stats(st_name, st_descr, st_type, st_print, st_prereq, "
             "st_prec, st_nozero, st_nonan, st_total, st_pdf, st_cdf, "
             "st_min, st_max, st_bktsize, st_size)"
             "values(\"%s\",\"%s\",\"%s\","
             "        %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)",
             name, descr, type, print, prereq, (int)prec, nozero, nonan,
             total, pdf, cdf,
             min, max, bktsize, size);

    mysql.query(insert);
    if (!mysql.error)
        return mysql.insert_id();

    stringstream select;
    ccprintf(select, "SELECT * FROM stats WHERE st_name=\"%s\"", name);

    mysql.query(select);
    MySQL::Result result = mysql.store_result();
    if (!result)
        panic("could not find stat\n%s\n", mysql.error);


    assert(result.num_fields() == 16);
    MySQL::Row row = result.fetch_row();
    if (!row)
        panic("could not get stat row\n%s\n", mysql.error);

    bool tb;
    int8_t ti8;
    uint16_t tu16;
    int64_t ti64;
    uint64_t tu64;

    if (name != (char *)row[1])
        panic("failed stat check on %s:name. %s != %s\n",
              name, name, row[1]);

    if (descr != (char *)row[2])
        panic("failed stat check on %s:descr. %s != %s\n",
              name, descr, row[2]);

    if (type != (char *)row[3])
        panic("failed stat check on %s:type. %s != %s\n",
              name, type, row[3]);

    if (!to_number(row[4], tb) || print != tb)
        panic("failed stat check on %s:print. %d != %d\n",
              name, print, tb);

    if (!to_number(row[6], ti8) || prec != ti8)
        panic("failed stat check on %s:prec. %d != %d\n",
              name, prec, ti8);

    if (!to_number(row[7], tb) || nozero != tb)
        panic("failed stat check on %s:nozero. %d != %d\n",
              name, nozero, tb);

    if (!to_number(row[8], tb) || nonan != tb)
        panic("failed stat check on %s:nonan. %d != %d\n",
              name, nonan, tb);

    if (!to_number(row[9], tb) || total != tb)
        panic("failed stat check on %s:total. %d != %d\n",
              name, total, tb);

    if (!to_number(row[10], tb) || pdf != tb)
        panic("failed stat check on %s:pdf. %d != %d\n",
              name, pdf, tb);

    if (!to_number(row[11], tb) || cdf != tb)
        panic("failed stat check on %s:cdf. %d != %d\n",
              name, cdf, tb);

    if (!to_number(row[12], ti64) || min != ti64)
        panic("failed stat check on %s:min. %d != %d\n",
              name, min, ti64);

    if (!to_number(row[13], ti64) || max != ti64)
        panic("failed stat check on %s:max. %d != %d\n",
              name, max, ti64);

    if (!to_number(row[14], tu64) || bktsize != tu64)
        panic("failed stat check on %s:bktsize. %d != %d\n",
              name, bktsize, tu64);

    if (!to_number(row[15], tu16) || size != tu16)
        panic("failed stat check on %s:size. %d != %d\n",
              name, size, tu16);

    to_number(row[5], prereq);
    uint16_t statid;
    to_number(row[0], statid);
    return statid;
}

unsigned
SetupBin(const string &bin)
{
    static map<string, int> binmap;

    using namespace MySQL;
    map<string,int>::const_iterator i = binmap.find(bin);
    if (i != binmap.end())
        return (*i).second;

    Connection &mysql = MySqlDB.conn();
    assert(mysql.connected());

    mysql.query("LOCK TABLES bins WRITE");
    if (mysql.error)
        panic("could not lock bin table\n%s\n", mysql.error);

    uint16_t bin_id;

    stringstream select;
    stringstream insert;
    ccprintf(select, "SELECT bn_id FROM bins WHERE bn_name=\"%s\"", bin);

    mysql.query(select);
    MySQL::Result result = mysql.store_result();
    if (result) {
        assert(result.num_fields() == 1);
        MySQL::Row row = result.fetch_row();
        if (row) {
            to_number(row[0], bin_id);
            goto exit;
        }
    }

    ccprintf(insert, "INSERT INTO bins(bn_name) values(\"%s\")", bin);

    mysql.query(insert);
    if (mysql.error)
        panic("could not get a bin\n%s\n", mysql.error);

    bin_id = mysql.insert_id();
    binmap.insert(make_pair(bin, bin_id));

  exit:
    mysql.query("UNLOCK TABLES");
    if (mysql.error)
        panic("could not unlock tables\n%s\n", mysql.error);

    return bin_id;
}

InsertData::InsertData()
{
    query = new char[maxsize + 1];
    size = 0;
    flush();
}

InsertData::~InsertData()
{
    delete [] query;
}

void
InsertData::flush()
{
    if (size) {
        MySQL::Connection &mysql = MySqlDB.conn();
        assert(mysql.connected());
        mysql.query(query);
        if (mysql.error)
            panic("could not insert data\n%s\n", mysql.error);
    }

    query[0] = '\0';
    size = 0;
    first = true;
    strcpy(query, "INSERT INTO "
           "data(dt_stat,dt_x,dt_y,dt_run,dt_tick,dt_bin,dt_data) "
           "values");
    size = strlen(query);
}

void
InsertData::insert()
{
    if (size + 1024 > maxsize)
        flush();

    if (!first) {
        query[size++] = ',';
        query[size] = '\0';
    }

    first = false;

    size += sprintf(query + size, "(%u,%d,%d,%u,%llu,%u,\"%f\")",
                    stat, x, y, MySqlDB.run(), (unsigned long long)tick,
                    bin, data);
}

struct InsertSubData
{
    uint16_t stat;
    int16_t x;
    int16_t y;
    string name;
    string descr;

    void setup();
};

void
InsertSubData::setup()
{
    MySQL::Connection &mysql = MySqlDB.conn();
    assert(mysql.connected());
    stringstream insert;
    ccprintf(insert,
             "INSERT INTO subdata(sd_stat,sd_x,sd_y,sd_name,sd_descr) "
             "values(%d,%d,%d,\"%s\",\"%s\")",
             stat, x, y, name, descr);

    mysql.query(insert);
//    if (mysql.error)
//	panic("could not insert subdata\n%s\n", mysql.error);
}

void
InsertFormula(uint16_t stat, const string &formula)
{
    MySQL::Connection &mysql = MySqlDB.conn();
    assert(mysql.connected());
    stringstream insert_formula;
    ccprintf(insert_formula,
             "INSERT INTO formulas(fm_stat,fm_formula) values(%d, \"%s\")",
             stat, formula);

    mysql.query(insert_formula);
//    if (mysql.error)
//	panic("could not insert formula\n%s\n", mysql.error);

    stringstream insert_ref;
    ccprintf(insert_ref,
             "INSERT INTO formula_ref(fr_stat,fr_run) values(%d, %d)",
             stat, MySqlDB.run());

    mysql.query(insert_ref);
//    if (mysql.error)
//	panic("could not insert formula reference\n%s\n", mysql.error);
}

void
UpdatePrereq(uint16_t stat, uint16_t prereq)
{
    MySQL::Connection &mysql = MySqlDB.conn();
    assert(mysql.connected());
    stringstream update;
    ccprintf(update, "UPDATE stats SET st_prereq=%d WHERE st_id=%d",
             prereq, stat);
    mysql.query(update);
    if (mysql.error)
        panic("could not update prereq\n%s\n", mysql.error);
}

void
MySql::configure()
{
    /*
     * set up all stats!
     */
    using namespace Database;

    MySQL::Connection &mysql = MySqlDB.conn();
    mysql.query("LOCK TABLES "
                "stats WRITE, "
                "bins WRITE, "
                "subdata WRITE, "
                "formulas WRITE, "
                "formula_ref WRITE");
    if (mysql.error)
        panic("could not lock tables\n%s\n", mysql.error);

    stat_list_t::const_iterator i, end = stats().end();
    for (i = stats().begin(); i != end; ++i)
        (*i)->visit(*this);

    for (i = stats().begin(); i != end; ++i) {
        StatData *data = *i;
        if (data->prereq) {
            uint16_t stat_id = find(data->id);
            uint16_t prereq_id = find(data->prereq->id);
            assert(stat_id && prereq_id);

            UpdatePrereq(stat_id, prereq_id);
        }
    }

    mysql.query("UNLOCK TABLES");
    if (mysql.error)
        panic("could not unlock tables\n%s\n", mysql.error);

    configured = true;
}


bool
MySql::configure(const StatData &data, string type)
{
    stat.init();
    stat.name = data.name;
    stat.descr = data.desc;
    stat.type = type;
    stat.print = data.flags & print;
    stat.prec = data.precision;
    stat.nozero = data.flags & nozero;
    stat.nonan = data.flags & nonan;
    stat.total = data.flags & total;
    stat.pdf = data.flags & pdf;
    stat.cdf = data.flags & cdf;

    return stat.print;
}

void
MySql::configure(const ScalarData &data)
{
    if (!configure(data, "SCALAR"))
        return;

    insert(data.id, stat.setup());
}

void
MySql::configure(const VectorData &data)
{
    if (!configure(data, "VECTOR"))
        return;

    uint16_t statid = stat.setup();

    if (!data.subnames.empty()) {
        InsertSubData subdata;
        subdata.stat = statid;
        subdata.y = 0;
        for (int i = 0; i < data.subnames.size(); ++i) {
            subdata.x = i;
            subdata.name = data.subnames[i];
            subdata.descr = data.subdescs.empty() ? "" : data.subdescs[i];

            if (!subdata.name.empty() || !subdata.descr.empty())
                subdata.setup();
        }
    }

    insert(data.id, statid);
}

void
MySql::configure(const DistData &data)
{
    if (!configure(data, "DIST"))
        return;

    if (!data.data.fancy) {
        stat.size = data.data.size;
        stat.min = data.data.min;
        stat.max = data.data.max;
        stat.bktsize = data.data.bucket_size;
    }
    insert(data.id, stat.setup());
}

void
MySql::configure(const VectorDistData &data)
{
    if (!configure(data, "VECTORDIST"))
        return;

    if (!data.data[0].fancy) {
        stat.size = data.data[0].size;
        stat.min = data.data[0].min;
        stat.max = data.data[0].max;
        stat.bktsize = data.data[0].bucket_size;
    }

    uint16_t statid = stat.setup();

    if (!data.subnames.empty()) {
        InsertSubData subdata;
        subdata.stat = statid;
        subdata.y = 0;
        for (int i = 0; i < data.subnames.size(); ++i) {
            subdata.x = i;
            subdata.name = data.subnames[i];
            subdata.descr = data.subdescs.empty() ? "" : data.subdescs[i];
            if (!subdata.name.empty() || !subdata.descr.empty())
                subdata.setup();
        }
    }

    insert(data.id, statid);
}

void
MySql::configure(const Vector2dData &data)
{
    if (!configure(data, "VECTOR2D"))
        return;

    uint16_t statid = stat.setup();

    if (!data.subnames.empty()) {
        InsertSubData subdata;
        subdata.stat = statid;
        subdata.y = -1;
        for (int i = 0; i < data.subnames.size(); ++i) {
            subdata.x = i;
            subdata.name = data.subnames[i];
            subdata.descr = data.subdescs.empty() ? "" : data.subdescs[i];
            if (!subdata.name.empty() || !subdata.descr.empty())
                subdata.setup();
        }
    }

    if (!data.y_subnames.empty()) {
        InsertSubData subdata;
        subdata.stat = statid;
        subdata.x = -1;
        subdata.descr = "";
        for (int i = 0; i < data.y_subnames.size(); ++i) {
            subdata.y = i;
            subdata.name = data.y_subnames[i];
            if (!subdata.name.empty())
                subdata.setup();
        }
    }

    insert(data.id, statid);
}

void
MySql::configure(const FormulaData &data)
{
    configure(data, "FORMULA");
    insert(data.id, stat.setup());
    InsertFormula(find(data.id), data.str());
}

void
MySql::output(MainBin *bin)
{
    if (bin) {
        bin->activate();
        newdata.bin = SetupBin(bin->name());
    } else {
        newdata.bin = 0;
    }

    Database::stat_list_t::const_iterator i, end = Database::stats().end();
    for (i = Database::stats().begin(); i != end; ++i) {
        StatData *stat = *i;
        if (bin && stat->binned() || !bin && !stat->binned())
            stat->visit(*this);
    }
}

bool
MySql::valid() const
{
    return MySqlDB.connected();
}

void
MySql::output()
{
    using namespace Database;
    assert(valid());

    if (!configured)
        configure();

    // store sample #
    newdata.tick = curTick;

    output(NULL);
    if (!bins().empty()) {
        bin_list_t::iterator i, end = bins().end();
        for (i = bins().begin(); i != end; ++i)
            output(*i);
    }

    newdata.flush();
}

void
MySql::output(const ScalarData &data)
{
    if (!(data.flags & print))
        return;

    newdata.stat = find(data.id);
    newdata.x = 0;
    newdata.y = 0;
    newdata.data = data.value();

    newdata.insert();
}

void
MySql::output(const VectorData &data)
{
    if (!(data.flags & print))
        return;

    newdata.stat = find(data.id);
    newdata.y = 0;

    const VCounter &cvec = data.value();
    int size = data.size();
    for (int x = 0; x < size; x++) {
        newdata.x = x;
        newdata.data = cvec[x];
        newdata.insert();
    }
}

void
MySql::output(const DistDataData &data)
{
    const int db_sum = -1;
    const int db_squares = -2;
    const int db_samples = -3;
    const int db_min_val = -4;
    const int db_max_val = -5;
    const int db_underflow = -6;
    const int db_overflow = -7;

    newdata.x = db_sum;
    newdata.data = data.sum;
    newdata.insert();

    newdata.x = db_squares;
    newdata.data = data.squares;
    newdata.insert();

    newdata.x = db_samples;
    newdata.data = data.samples;
    newdata.insert();

    if (data.samples && !data.fancy) {
        newdata.x = db_min_val;
        newdata.data = data.min_val;
        newdata.insert();

        newdata.x = db_max_val;
        newdata.data = data.max_val;
        newdata.insert();

        newdata.x = db_underflow;
        newdata.data = data.underflow;
        newdata.insert();

        newdata.x = db_overflow;
        newdata.data = data.overflow;
        newdata.insert();

        int size = data.cvec.size();
        for (int x = 0; x < size; x++) {
            newdata.x = x;
            newdata.data = data.cvec[x];
            newdata.insert();
        }
    }
}


void
MySql::output(const DistData &data)
{
    if (!(data.flags & print))
        return;

    newdata.stat = find(data.id);
    newdata.y = 0;
    output(data.data);
}

void
MySql::output(const VectorDistData &data)
{
    if (!(data.flags & print))
        return;

    newdata.stat = find(data.id);

    int size = data.data.size();
    for (int y = 0; y < size; ++y) {
        newdata.y = y;
        output(data.data[y]);
    }
}

void
MySql::output(const Vector2dData &data)
{
    if (!(data.flags & print))
        return;

    newdata.stat = find(data.id);

    int index = 0;
    for (int x = 0; x < data.x; x++) {
        newdata.x = x;
        for (int y = 0; y < data.y; y++) {
            newdata.y = y;
            newdata.data = data.cvec[index++];
            newdata.insert();
        }
    }
}

void
MySql::output(const FormulaData &data)
{
}

/*
 * Implement the visitor
 */
void
MySql::visit(const ScalarData &data)
{
    if (!configured)
        configure(data);
    else
        output(data);
}

void
MySql::visit(const VectorData &data)
{
    if (!configured)
        configure(data);
    else
        output(data);
}

void
MySql::visit(const DistData &data)
{
    if (!configured)
        configure(data);
    else
        output(data);
}

void
MySql::visit(const VectorDistData &data)
{
    if (!configured)
        configure(data);
    else
        output(data);
}

void
MySql::visit(const Vector2dData &data)
{
    if (!configured)
        configure(data);
    else
        output(data);
}

void
MySql::visit(const FormulaData &data)
{
    if (!configured)
        configure(data);
    else
        output(data);
}

/* namespace Stats */ }