a928a438b8
Used cppclean to help identify useless includes and removed them. This involved erroneously included headers, but also cases where forward declarations could have been used rather than a full include.
646 lines
21 KiB
C++
646 lines
21 KiB
C++
/*
|
|
* Copyright (c) 2015-2016 ARM Limited
|
|
* All rights reserved
|
|
*
|
|
* The license below extends only to copyright in the software and shall
|
|
* not be construed as granting a license to any other intellectual
|
|
* property including but not limited to intellectual property relating
|
|
* to a hardware implementation of the functionality of the software
|
|
* licensed hereunder. You may use the software subject to the license
|
|
* terms below provided that you ensure that this notice is replicated
|
|
* unmodified and in its entirety in all distributions of the software,
|
|
* modified or unmodified, in source code or in binary form.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met: redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer;
|
|
* redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution;
|
|
* neither the name of the copyright holders nor the names of its
|
|
* contributors may be used to endorse or promote products derived from
|
|
* this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* Authors: Gabor Dozsa
|
|
*/
|
|
|
|
/* @file
|
|
* The interface class for dist gem5 simulations.
|
|
*
|
|
* dist-gem5 is an extension to gem5 to enable parallel simulation of a
|
|
* distributed system (e.g. simulation of a pool of machines
|
|
* connected by Ethernet links). A dist gem5 run consists of seperate gem5
|
|
* processes running in parallel. Each gem5 process executes
|
|
* the simulation of a component of the simulated distributed system.
|
|
* (An example component can be a dist-core board with an Ethernet NIC.)
|
|
* The DistIface class below provides services to transfer data and
|
|
* control messages among the gem5 processes. The main such services are
|
|
* as follows.
|
|
*
|
|
* 1. Send a data packet coming from a simulated Ethernet link. The packet
|
|
* will be transferred to (all) the target(s) gem5 processes. The send
|
|
* operation is always performed by the simulation thread, i.e. the gem5
|
|
* thread that is processing the event queue associated with the simulated
|
|
* Ethernet link.
|
|
*
|
|
* 2. Spawn a receiver thread to process messages coming in from the
|
|
* from other gem5 processes. Each simulated Ethernet link has its own
|
|
* associated receiver thread. The receiver thread saves the incoming packet
|
|
* and schedule an appropriate receive event in the event queue.
|
|
*
|
|
* 3. Schedule a global barrier event periodically to keep the gem5
|
|
* processes in sync.
|
|
* Periodic barrier event to keep peer gem5 processes in sync. The basic idea
|
|
* is that no gem5 process can go ahead further than the simulated link
|
|
* transmission delay to ensure that a corresponding receive event can always
|
|
* be scheduled for any message coming in from a peer gem5 process.
|
|
*
|
|
*
|
|
*
|
|
* This interface is an abstract class. It can work with various low level
|
|
* send/receive service implementations (e.g. TCP/IP, MPI,...). A TCP
|
|
* stream socket version is implemented in src/dev/net/tcp_iface.[hh,cc].
|
|
*/
|
|
#ifndef __DEV_DIST_IFACE_HH__
|
|
#define __DEV_DIST_IFACE_HH__
|
|
|
|
#include <array>
|
|
#include <mutex>
|
|
#include <queue>
|
|
#include <thread>
|
|
#include <utility>
|
|
|
|
#include "base/misc.hh"
|
|
#include "dev/net/dist_packet.hh"
|
|
#include "dev/net/etherpkt.hh"
|
|
#include "sim/core.hh"
|
|
#include "sim/drain.hh"
|
|
#include "sim/global_event.hh"
|
|
#include "sim/serialize.hh"
|
|
|
|
class EventManager;
|
|
class System;
|
|
class ThreadContext;
|
|
|
|
/**
|
|
* The interface class to talk to peer gem5 processes.
|
|
*/
|
|
class DistIface : public Drainable, public Serializable
|
|
{
|
|
public:
|
|
typedef DistHeaderPkt::Header Header;
|
|
|
|
protected:
|
|
typedef DistHeaderPkt::MsgType MsgType;
|
|
typedef DistHeaderPkt::ReqType ReqType;
|
|
|
|
private:
|
|
class SyncEvent;
|
|
/** @class Sync
|
|
* This class implements global sync operations among gem5 peer processes.
|
|
*
|
|
* @note This class is used as a singleton object (shared by all DistIface
|
|
* objects).
|
|
*/
|
|
class Sync : public Serializable
|
|
{
|
|
protected:
|
|
/**
|
|
* The lock to protect access to the Sync object.
|
|
*/
|
|
std::mutex lock;
|
|
/**
|
|
* Condition variable for the simulation thread to wait on
|
|
* until all receiver threads completes the current global
|
|
* synchronisation.
|
|
*/
|
|
std::condition_variable cv;
|
|
/**
|
|
* Number of receiver threads that not yet completed the current global
|
|
* synchronisation.
|
|
*/
|
|
unsigned waitNum;
|
|
/**
|
|
* Flag is set if exit is permitted upon sync completion
|
|
*/
|
|
bool doExit;
|
|
/**
|
|
* Flag is set if taking a ckpt is permitted upon sync completion
|
|
*/
|
|
bool doCkpt;
|
|
/**
|
|
* Flag is set if sync is to stop upon sync completion
|
|
*/
|
|
bool doStopSync;
|
|
/**
|
|
* The repeat value for the next periodic sync
|
|
*/
|
|
Tick nextRepeat;
|
|
/**
|
|
* Tick for the next periodic sync (if the event is not scheduled yet)
|
|
*/
|
|
Tick nextAt;
|
|
/**
|
|
* Flag is set if the sync is aborted (e.g. due to connection lost)
|
|
*/
|
|
bool isAbort;
|
|
|
|
friend class SyncEvent;
|
|
|
|
public:
|
|
/**
|
|
* Initialize periodic sync params.
|
|
*
|
|
* @param start Start tick for dist synchronisation
|
|
* @param repeat Frequency of dist synchronisation
|
|
*
|
|
*/
|
|
void init(Tick start, Tick repeat);
|
|
/**
|
|
* Core method to perform a full dist sync.
|
|
*
|
|
* @return true if the sync completes, false if it gets aborted
|
|
*/
|
|
virtual bool run(bool same_tick) = 0;
|
|
/**
|
|
* Callback when the receiver thread gets a sync ack message.
|
|
*
|
|
* @return false if the receiver thread needs to stop (e.g.
|
|
* simulation is to exit)
|
|
*/
|
|
virtual bool progress(Tick send_tick,
|
|
Tick next_repeat,
|
|
ReqType do_ckpt,
|
|
ReqType do_exit,
|
|
ReqType do_stop_sync) = 0;
|
|
/**
|
|
* Abort processing an on-going sync event (in case of an error, e.g.
|
|
* lost connection to a peer gem5)
|
|
*/
|
|
void abort();
|
|
|
|
virtual void requestCkpt(ReqType req) = 0;
|
|
virtual void requestExit(ReqType req) = 0;
|
|
virtual void requestStopSync(ReqType req) = 0;
|
|
|
|
void drainComplete();
|
|
|
|
virtual void serialize(CheckpointOut &cp) const override = 0;
|
|
virtual void unserialize(CheckpointIn &cp) override = 0;
|
|
};
|
|
|
|
class SyncNode: public Sync
|
|
{
|
|
private:
|
|
/**
|
|
* Exit requested
|
|
*/
|
|
ReqType needExit;
|
|
/**
|
|
* Ckpt requested
|
|
*/
|
|
ReqType needCkpt;
|
|
/**
|
|
* Sync stop requested
|
|
*/
|
|
ReqType needStopSync;
|
|
|
|
public:
|
|
|
|
SyncNode();
|
|
~SyncNode() {}
|
|
bool run(bool same_tick) override;
|
|
bool progress(Tick max_req_tick,
|
|
Tick next_repeat,
|
|
ReqType do_ckpt,
|
|
ReqType do_exit,
|
|
ReqType do_stop_sync) override;
|
|
|
|
void requestCkpt(ReqType req) override;
|
|
void requestExit(ReqType req) override;
|
|
void requestStopSync(ReqType req) override;
|
|
|
|
void serialize(CheckpointOut &cp) const override;
|
|
void unserialize(CheckpointIn &cp) override;
|
|
};
|
|
|
|
class SyncSwitch: public Sync
|
|
{
|
|
private:
|
|
/**
|
|
* Counter for recording exit requests
|
|
*/
|
|
unsigned numExitReq;
|
|
/**
|
|
* Counter for recording ckpt requests
|
|
*/
|
|
unsigned numCkptReq;
|
|
/**
|
|
* Counter for recording stop sync requests
|
|
*/
|
|
unsigned numStopSyncReq;
|
|
/**
|
|
* Number of connected simulated nodes
|
|
*/
|
|
unsigned numNodes;
|
|
|
|
public:
|
|
SyncSwitch(int num_nodes);
|
|
~SyncSwitch() {}
|
|
|
|
bool run(bool same_tick) override;
|
|
bool progress(Tick max_req_tick,
|
|
Tick next_repeat,
|
|
ReqType do_ckpt,
|
|
ReqType do_exit,
|
|
ReqType do_stop_sync) override;
|
|
|
|
void requestCkpt(ReqType) override {
|
|
panic("Switch requested checkpoint");
|
|
}
|
|
void requestExit(ReqType) override {
|
|
panic("Switch requested exit");
|
|
}
|
|
void requestStopSync(ReqType) override {
|
|
panic("Switch requested stop sync");
|
|
}
|
|
|
|
void serialize(CheckpointOut &cp) const override;
|
|
void unserialize(CheckpointIn &cp) override;
|
|
};
|
|
|
|
/**
|
|
* The global event to schedule periodic dist sync. It is used as a
|
|
* singleton object.
|
|
*
|
|
* The periodic synchronisation works as follows.
|
|
* 1. A SyncEvent is scheduled as a global event when startup() is
|
|
* called.
|
|
* 2. The process() method of the SyncEvent initiates a new barrier
|
|
* for each simulated Ethernet link.
|
|
* 3. Simulation thread(s) then waits until all receiver threads
|
|
* complete the ongoing barrier. The global sync event is done.
|
|
*/
|
|
class SyncEvent : public GlobalSyncEvent
|
|
{
|
|
private:
|
|
/**
|
|
* Flag to set when the system is draining
|
|
*/
|
|
bool _draining;
|
|
public:
|
|
/**
|
|
* Only the firstly instantiated DistIface object will
|
|
* call this constructor.
|
|
*/
|
|
SyncEvent() : GlobalSyncEvent(Sim_Exit_Pri, 0), _draining(false) {}
|
|
|
|
~SyncEvent() {}
|
|
/**
|
|
* Schedule the first periodic sync event.
|
|
*/
|
|
void start();
|
|
/**
|
|
* This is a global event so process() will only be called by
|
|
* exactly one simulation thread. (See further comments in the .cc
|
|
* file.)
|
|
*/
|
|
void process() override;
|
|
|
|
bool draining() const { return _draining; }
|
|
void draining(bool fl) { _draining = fl; }
|
|
};
|
|
/**
|
|
* Class to encapsulate information about data packets received.
|
|
|
|
* @note The main purpose of the class to take care of scheduling receive
|
|
* done events for the simulated network link and store incoming packets
|
|
* until they can be received by the simulated network link.
|
|
*/
|
|
class RecvScheduler : public Serializable
|
|
{
|
|
private:
|
|
/**
|
|
* Received packet descriptor. This information is used by the receive
|
|
* thread to schedule receive events and by the simulation thread to
|
|
* process those events.
|
|
*/
|
|
struct Desc : public Serializable
|
|
{
|
|
EthPacketPtr packet;
|
|
Tick sendTick;
|
|
Tick sendDelay;
|
|
|
|
Desc() : sendTick(0), sendDelay(0) {}
|
|
Desc(EthPacketPtr p, Tick s, Tick d) :
|
|
packet(p), sendTick(s), sendDelay(d) {}
|
|
Desc(const Desc &d) :
|
|
packet(d.packet), sendTick(d.sendTick), sendDelay(d.sendDelay) {}
|
|
|
|
void serialize(CheckpointOut &cp) const override;
|
|
void unserialize(CheckpointIn &cp) override;
|
|
};
|
|
/**
|
|
* The queue to store the receive descriptors.
|
|
*/
|
|
std::queue<Desc> descQueue;
|
|
/**
|
|
* The tick when the most recent receive event was processed.
|
|
*
|
|
* @note This information is necessary to simulate possible receiver
|
|
* link contention when calculating the receive tick for the next
|
|
* incoming data packet (see the calcReceiveTick() method)
|
|
*/
|
|
Tick prevRecvTick;
|
|
/**
|
|
* The receive done event for the simulated Ethernet link.
|
|
*
|
|
* @note This object is constructed by the simulated network link. We
|
|
* schedule this object for each incoming data packet.
|
|
*/
|
|
Event *recvDone;
|
|
/**
|
|
* The link delay in ticks for the simulated Ethernet link.
|
|
*
|
|
* @note This value is used for calculating the receive ticks for
|
|
* incoming data packets.
|
|
*/
|
|
Tick linkDelay;
|
|
/**
|
|
* The event manager associated with the simulated Ethernet link.
|
|
*
|
|
* @note It is used to access the event queue for scheduling receive
|
|
* done events for the link.
|
|
*/
|
|
EventManager *eventManager;
|
|
/**
|
|
* Calculate the tick to schedule the next receive done event.
|
|
*
|
|
* @param send_tick The tick the packet was sent.
|
|
* @param send_delay The simulated delay at the sender side.
|
|
* @param prev_recv_tick Tick when the last receive event was
|
|
* processed.
|
|
*
|
|
* @note This method tries to take into account possible receiver link
|
|
* contention and adjust receive tick for the incoming packets
|
|
* accordingly.
|
|
*/
|
|
Tick calcReceiveTick(Tick send_tick,
|
|
Tick send_delay,
|
|
Tick prev_recv_tick);
|
|
|
|
/**
|
|
* Flag to set if receive ticks for pending packets need to be
|
|
* recalculated due to changed link latencies at a resume
|
|
*/
|
|
bool ckptRestore;
|
|
|
|
public:
|
|
/**
|
|
* Scheduler for the incoming data packets.
|
|
*
|
|
* @param em The event manager associated with the simulated Ethernet
|
|
* link.
|
|
*/
|
|
RecvScheduler(EventManager *em) :
|
|
prevRecvTick(0), recvDone(nullptr), linkDelay(0),
|
|
eventManager(em), ckptRestore(false) {}
|
|
|
|
/**
|
|
* Initialize network link parameters.
|
|
*
|
|
* @note This method is called from the receiver thread (see
|
|
* recvThreadFunc()).
|
|
*/
|
|
void init(Event *recv_done, Tick link_delay);
|
|
/**
|
|
* Fetch the next packet that is to be received by the simulated network
|
|
* link.
|
|
*
|
|
* @note This method is called from the process() method of the receive
|
|
* done event associated with the network link.
|
|
*/
|
|
EthPacketPtr popPacket();
|
|
/**
|
|
* Push a newly arrived packet into the desc queue.
|
|
*/
|
|
void pushPacket(EthPacketPtr new_packet,
|
|
Tick send_tick,
|
|
Tick send_delay);
|
|
|
|
void serialize(CheckpointOut &cp) const override;
|
|
void unserialize(CheckpointIn &cp) override;
|
|
/**
|
|
* Adjust receive ticks for pending packets when restoring from a
|
|
* checkpoint
|
|
*
|
|
* @note Link speed and delay parameters may change at resume.
|
|
*/
|
|
void resumeRecvTicks();
|
|
};
|
|
/**
|
|
* Tick to schedule the first dist sync event.
|
|
* This is just as optimization : we do not need any dist sync
|
|
* event until the simulated NIC is brought up by the OS.
|
|
*/
|
|
Tick syncStart;
|
|
/**
|
|
* Frequency of dist sync events in ticks.
|
|
*/
|
|
Tick syncRepeat;
|
|
/**
|
|
* Receiver thread pointer.
|
|
* Each DistIface object must have exactly one receiver thread.
|
|
*/
|
|
std::thread *recvThread;
|
|
/**
|
|
* Meta information about data packets received.
|
|
*/
|
|
RecvScheduler recvScheduler;
|
|
/**
|
|
* Use pseudoOp to start synchronization.
|
|
*/
|
|
bool syncStartOnPseudoOp;
|
|
|
|
protected:
|
|
/**
|
|
* The rank of this process among the gem5 peers.
|
|
*/
|
|
unsigned rank;
|
|
/**
|
|
* The number of gem5 processes comprising this dist simulation.
|
|
*/
|
|
unsigned size;
|
|
/**
|
|
* Number of DistIface objects (i.e. dist links in this gem5 process)
|
|
*/
|
|
static unsigned distIfaceNum;
|
|
/**
|
|
* Unique id for the dist link
|
|
*/
|
|
unsigned distIfaceId;
|
|
|
|
bool isMaster;
|
|
|
|
private:
|
|
/**
|
|
* Number of receiver threads (in this gem5 process)
|
|
*/
|
|
static unsigned recvThreadsNum;
|
|
/**
|
|
* The singleton Sync object to perform dist synchronisation.
|
|
*/
|
|
static Sync *sync;
|
|
/**
|
|
* The singleton SyncEvent object to schedule periodic dist sync.
|
|
*/
|
|
static SyncEvent *syncEvent;
|
|
/**
|
|
* The very first DistIface object created becomes the master. We need
|
|
* a master to co-ordinate the global synchronisation.
|
|
*/
|
|
static DistIface *master;
|
|
/**
|
|
* System pointer used to wakeup sleeping threads when stopping sync.
|
|
*/
|
|
static System *sys;
|
|
/**
|
|
* Is this node a switch?
|
|
*/
|
|
static bool isSwitch;
|
|
|
|
private:
|
|
/**
|
|
* Send out a data packet to the remote end.
|
|
* @param header Meta info about the packet (which needs to be transferred
|
|
* to the destination alongside the packet).
|
|
* @param packet Pointer to the packet to send.
|
|
*/
|
|
virtual void sendPacket(const Header &header, const EthPacketPtr &packet) = 0;
|
|
/**
|
|
* Send out a control command to the remote end.
|
|
* @param header Meta info describing the command (e.g. sync request)
|
|
*/
|
|
virtual void sendCmd(const Header &header) = 0;
|
|
/**
|
|
* Receive a header (i.e. meta info describing a data packet or a control command)
|
|
* from the remote end.
|
|
* @param header The meta info structure to store the incoming header.
|
|
*/
|
|
virtual bool recvHeader(Header &header) = 0;
|
|
/**
|
|
* Receive a packet from the remote end.
|
|
* @param header Meta info about the incoming packet (obtanied by a previous
|
|
* call to the recvHedaer() method).
|
|
* @param Pointer to packet received.
|
|
*/
|
|
virtual void recvPacket(const Header &header, EthPacketPtr &packet) = 0;
|
|
/**
|
|
* Init hook for the underlaying transport
|
|
*/
|
|
virtual void initTransport() = 0;
|
|
/**
|
|
* spawn the receiver thread.
|
|
* @param recv_done The receive done event associated with the simulated
|
|
* Ethernet link.
|
|
* @param link_delay The link delay for the simulated Ethernet link.
|
|
*/
|
|
void spawnRecvThread(const Event *recv_done, Tick link_delay);
|
|
/**
|
|
* The function executed by a receiver thread.
|
|
*/
|
|
void recvThreadFunc(Event *recv_done, Tick link_delay);
|
|
|
|
public:
|
|
|
|
/**
|
|
* ctor
|
|
* @param dist_rank Rank of this gem5 process within the dist run
|
|
* @param sync_start Start tick for dist synchronisation
|
|
* @param sync_repeat Frequency for dist synchronisation
|
|
* @param em The event manager associated with the simulated Ethernet link
|
|
*/
|
|
DistIface(unsigned dist_rank,
|
|
unsigned dist_size,
|
|
Tick sync_start,
|
|
Tick sync_repeat,
|
|
EventManager *em,
|
|
bool use_pseudo_op,
|
|
bool is_switch,
|
|
int num_nodes);
|
|
|
|
virtual ~DistIface();
|
|
/**
|
|
* Send out an Ethernet packet.
|
|
* @param pkt The Ethernet packet to send.
|
|
* @param send_delay The delay in ticks for the send completion event.
|
|
*/
|
|
void packetOut(EthPacketPtr pkt, Tick send_delay);
|
|
/**
|
|
* Fetch the packet scheduled to be received next by the simulated
|
|
* network link.
|
|
*
|
|
* @note This method is called within the process() method of the link
|
|
* receive done event. It also schedules the next receive event if the
|
|
* receive queue is not empty.
|
|
*/
|
|
EthPacketPtr packetIn() { return recvScheduler.popPacket(); }
|
|
|
|
DrainState drain() override;
|
|
void drainResume() override;
|
|
void init(const Event *e, Tick link_delay);
|
|
void startup();
|
|
|
|
void serialize(CheckpointOut &cp) const override;
|
|
void unserialize(CheckpointIn &cp) override;
|
|
/**
|
|
* Initiate the exit from the simulation.
|
|
* @param delay Delay param from the m5 exit command. If Delay is zero
|
|
* then a collaborative exit is requested (i.e. all nodes have to call
|
|
* this method before the distributed simulation can exit). If Delay is
|
|
* not zero then exit is requested asap (and it will happen at the next
|
|
* sync tick).
|
|
* @return False if we are in distributed mode (i.e. exit can happen only
|
|
* at sync), True otherwise.
|
|
*/
|
|
static bool readyToExit(Tick delay);
|
|
/**
|
|
* Initiate taking a checkpoint
|
|
* @param delay Delay param from the m5 checkpoint command. If Delay is
|
|
* zero then a collaborative checkpoint is requested (i.e. all nodes have
|
|
* to call this method before the checkpoint can be taken). If Delay is
|
|
* not zero then a checkpoint is requested asap (and it will happen at the
|
|
* next sync tick).
|
|
* @return False if we are in dist mode (i.e. exit can happen only at
|
|
* sync), True otherwise.
|
|
*/
|
|
static bool readyToCkpt(Tick delay, Tick period);
|
|
/**
|
|
* Getter for the dist rank param.
|
|
*/
|
|
static uint64_t rankParam();
|
|
/**
|
|
* Getter for the dist size param.
|
|
*/
|
|
static uint64_t sizeParam();
|
|
/**
|
|
* Trigger the master to start/stop synchronization.
|
|
*/
|
|
static void toggleSync(ThreadContext *tc);
|
|
};
|
|
|
|
#endif
|