sanchayanmaity/gem5
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

ruby: add support for clusters

Browse source 
A cluster over here means a set of controllers that can be accessed only by a
certain set of cores.  For example,  consider a two level hierarchy. Assume
there are 4 L1 controllers (private) and 2 L2 controllers.  We can have two
different hierarchies here:

a. the address space is partitioned between the two L2 controllers.  Each L1
controller accesses both the L2 controllers.  In this case, each L1 controller
is a cluster initself.

b. both the L2 controllers can cache any address.  An L1 controller has access
to only one of the L2 controllers.  In this case, each L2 controller
along with the L1 controllers that access it, form a cluster.

This patch allows for each controller to have a cluster ID, which is 0 by
default.  By setting the cluster ID properly,  one can instantiate hierarchies
with clusters.  Note that the coherence protocol might have to be changed as
well.
This commit is contained in:
Nilay Vaish 2014-01-04 00:03:31 -06:00
parent 9853ef6651
commit 5b1804e3bd
16 changed files with 107 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/mem/protocol/MESI_CMP_directory-L1cache.sm
View file

@ -500,7 +500,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Type := CoherenceRequestType:GETS; out_msg.Type := CoherenceRequestType:GETS;
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
DPRINTF(RubySlicc, "address: %s, destination: %s\n", DPRINTF(RubySlicc, "address: %s, destination: %s\n",
address, out_msg.Destination); address, out_msg.Destination);
out_msg.MessageSize := MessageSizeType:Control; out_msg.MessageSize := MessageSizeType:Control;
@ -518,7 +518,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Type := CoherenceRequestType:GETS; out_msg.Type := CoherenceRequestType:GETS;
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
DPRINTF(RubySlicc, "address: %s, destination: %s\n", DPRINTF(RubySlicc, "address: %s, destination: %s\n",
address, out_msg.Destination); address, out_msg.Destination);
out_msg.MessageSize := MessageSizeType:Control; out_msg.MessageSize := MessageSizeType:Control;
@ -535,7 +535,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Type := CoherenceRequestType:GET_INSTR; out_msg.Type := CoherenceRequestType:GET_INSTR;
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
DPRINTF(RubySlicc, "address: %s, destination: %s\n", DPRINTF(RubySlicc, "address: %s, destination: %s\n",
address, out_msg.Destination); address, out_msg.Destination);
out_msg.MessageSize := MessageSizeType:Control; out_msg.MessageSize := MessageSizeType:Control;
@ -555,7 +555,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.Destination.add( out_msg.Destination.add(
mapAddressToRange(address, MachineType:L2Cache, mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Control; out_msg.MessageSize := MessageSizeType:Control;
out_msg.Prefetch := in_msg.Prefetch; out_msg.Prefetch := in_msg.Prefetch;
out_msg.AccessMode := in_msg.AccessMode; out_msg.AccessMode := in_msg.AccessMode;
@ -574,7 +574,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
DPRINTF(RubySlicc, "%s\n", machineID); DPRINTF(RubySlicc, "%s\n", machineID);
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
DPRINTF(RubySlicc, "address: %s, destination: %s\n", DPRINTF(RubySlicc, "address: %s, destination: %s\n",
address, out_msg.Destination); address, out_msg.Destination);
out_msg.MessageSize := MessageSizeType:Control; out_msg.MessageSize := MessageSizeType:Control;
@ -593,10 +593,8 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
DPRINTF(RubySlicc, "%s\n", machineID); DPRINTF(RubySlicc, "%s\n", machineID);
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
MachineType:L2Cache, l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_low_bit,
l2_select_num_bits));
DPRINTF(RubySlicc, "address: %s, destination: %s\n", DPRINTF(RubySlicc, "address: %s, destination: %s\n",
address, out_msg.Destination); address, out_msg.Destination);
@ -614,7 +612,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Type := CoherenceRequestType:UPGRADE; out_msg.Type := CoherenceRequestType:UPGRADE;
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
DPRINTF(RubySlicc, "address: %s, destination: %s\n", DPRINTF(RubySlicc, "address: %s, destination: %s\n",
address, out_msg.Destination); address, out_msg.Destination);
out_msg.MessageSize := MessageSizeType:Control; out_msg.MessageSize := MessageSizeType:Control;
@ -648,7 +646,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Dirty := cache_entry.Dirty; out_msg.Dirty := cache_entry.Dirty;
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Response_Data; out_msg.MessageSize := MessageSizeType:Response_Data;
} }
} }
@ -677,7 +675,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Dirty := tbe.Dirty; out_msg.Dirty := tbe.Dirty;
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Response_Data; out_msg.MessageSize := MessageSizeType:Response_Data;
} }
} }
@ -703,7 +701,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Dirty := cache_entry.Dirty; out_msg.Dirty := cache_entry.Dirty;
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Writeback_Data; out_msg.MessageSize := MessageSizeType:Writeback_Data;
} }
} }
@ -717,7 +715,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Dirty := tbe.Dirty; out_msg.Dirty := tbe.Dirty;
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Writeback_Data; out_msg.MessageSize := MessageSizeType:Writeback_Data;
} }
} }
@ -751,7 +749,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Dirty := cache_entry.Dirty; out_msg.Dirty := cache_entry.Dirty;
out_msg.Requestor:= machineID; out_msg.Requestor:= machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
if (cache_entry.Dirty) { if (cache_entry.Dirty) {
out_msg.MessageSize := MessageSizeType:Writeback_Data; out_msg.MessageSize := MessageSizeType:Writeback_Data;
} else { } else {
@ -766,7 +764,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Type := CoherenceResponseType:UNBLOCK; out_msg.Type := CoherenceResponseType:UNBLOCK;
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Response_Control; out_msg.MessageSize := MessageSizeType:Response_Control;
DPRINTF(RubySlicc, "%s\n", address); DPRINTF(RubySlicc, "%s\n", address);
} }
@ -778,7 +776,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
out_msg.Type := CoherenceResponseType:EXCLUSIVE_UNBLOCK; out_msg.Type := CoherenceResponseType:EXCLUSIVE_UNBLOCK;
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Response_Control; out_msg.MessageSize := MessageSizeType:Response_Control;
DPRINTF(RubySlicc, "%s\n", address); DPRINTF(RubySlicc, "%s\n", address);

25
src/mem/protocol/MESI_CMP_directory-dir.sm
View file

@ -39,7 +39,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
MemoryControl * memBuffer, MemoryControl * memBuffer,
Cycles to_mem_ctrl_latency = 1, Cycles to_mem_ctrl_latency = 1,
Cycles directory_latency = 6, Cycles directory_latency = 6,
int l2_select_num_bits
{ {
MessageBuffer requestToDir, network="From", virtual_network="0", MessageBuffer requestToDir, network="From", virtual_network="0",
ordered="false", vnet_type="request"; ordered="false", vnet_type="request";
@ -83,6 +82,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
structure(Entry, desc="...", interface="AbstractEntry") { structure(Entry, desc="...", interface="AbstractEntry") {
State DirectoryState, desc="Directory state"; State DirectoryState, desc="Directory state";
DataBlock DataBlk, desc="data for the block"; DataBlock DataBlk, desc="data for the block";
MachineID Owner;
} }
// TBE entries for DMA requests // TBE entries for DMA requests
@ -102,8 +102,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
// ** OBJECTS ** // ** OBJECTS **
int l2_select_low_bit, default="RubySystem::getBlockSizeBits()";
TBETable TBEs, template="<Directory_TBE>", constructor="m_number_of_TBEs"; TBETable TBEs, template="<Directory_TBE>", constructor="m_number_of_TBEs";
void set_tbe(TBE tbe); void set_tbe(TBE tbe);
@ -262,6 +260,9 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
out_msg.DataBlk := in_msg.DataBlk; out_msg.DataBlk := in_msg.DataBlk;
out_msg.Dirty := false; out_msg.Dirty := false;
out_msg.MessageSize := MessageSizeType:Response_Data; out_msg.MessageSize := MessageSizeType:Response_Data;
Entry e := getDirectoryEntry(in_msg.Addr);
e.Owner := in_msg.OriginalRequestorMachId;
} }
} }
} }
@ -409,12 +410,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
action(inv_sendCacheInvalidate, "inv", desc="Invalidate a cache block") { action(inv_sendCacheInvalidate, "inv", desc="Invalidate a cache block") {
peek(requestNetwork_in, RequestMsg) { peek(requestNetwork_in, RequestMsg) {
enqueue(responseNetwork_out, ResponseMsg, latency=directory_latency) { enqueue(responseNetwork_out, ResponseMsg, latency=directory_latency) {
out_msg.Addr := address; out_msg.Addr := address;
out_msg.Type := CoherenceResponseType:INV; out_msg.Type := CoherenceResponseType:INV;
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(getDirectoryEntry(address).Owner);
l2_select_low_bit, l2_select_num_bits)); out_msg.MessageSize := MessageSizeType:Response_Control;
out_msg.MessageSize := MessageSizeType:Response_Control;
} }
} }
} }
@ -483,6 +483,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
j_popIncomingRequestQueue; j_popIncomingRequestQueue;
} }
transition(M, Fetch) {
inv_sendCacheInvalidate;
z_stallAndWaitRequest;
}
transition(IM, Memory_Data, M) { transition(IM, Memory_Data, M) {
d_sendData; d_sendData;
l_popMemQueue; l_popMemQueue;
@ -492,6 +497,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
transition(M, CleanReplacement, I) { transition(M, CleanReplacement, I) {
a_sendAck; a_sendAck;
k_popIncomingResponseQueue; k_popIncomingResponseQueue;
kd_wakeUpDependents;
} }
transition(M, Data, MI) { transition(M, Data, MI) {
@ -578,5 +584,4 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
l_popMemQueue; l_popMemQueue;
kd_wakeUpDependents; kd_wakeUpDependents;
} }
} }

30
src/mem/protocol/MOESI_CMP_directory-L1cache.sm
View file

@ -420,7 +420,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.RequestorMachine := MachineType:L1Cache; out_msg.RequestorMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Request_Control; out_msg.MessageSize := MessageSizeType:Request_Control;
out_msg.AccessMode := in_msg.AccessMode; out_msg.AccessMode := in_msg.AccessMode;
out_msg.Prefetch := in_msg.Prefetch; out_msg.Prefetch := in_msg.Prefetch;
@ -436,7 +436,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.RequestorMachine := MachineType:L1Cache; out_msg.RequestorMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Request_Control; out_msg.MessageSize := MessageSizeType:Request_Control;
out_msg.AccessMode := in_msg.AccessMode; out_msg.AccessMode := in_msg.AccessMode;
out_msg.Prefetch := in_msg.Prefetch; out_msg.Prefetch := in_msg.Prefetch;
@ -452,7 +452,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.RequestorMachine := MachineType:L1Cache; out_msg.RequestorMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Writeback_Control; out_msg.MessageSize := MessageSizeType:Writeback_Control;
} }
} }
@ -465,7 +465,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.RequestorMachine := MachineType:L1Cache; out_msg.RequestorMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Writeback_Control; out_msg.MessageSize := MessageSizeType:Writeback_Control;
} }
} }
@ -478,7 +478,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.RequestorMachine := MachineType:L1Cache; out_msg.RequestorMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Writeback_Control; out_msg.MessageSize := MessageSizeType:Writeback_Control;
} }
} }
@ -493,7 +493,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.DataBlk := cache_entry.DataBlk; out_msg.DataBlk := cache_entry.DataBlk;
// out_msg.Dirty := cache_entry.Dirty; // out_msg.Dirty := cache_entry.Dirty;
out_msg.Dirty := false; out_msg.Dirty := false;
@ -528,7 +528,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.DataBlk := cache_entry.DataBlk; out_msg.DataBlk := cache_entry.DataBlk;
out_msg.Dirty := cache_entry.Dirty; out_msg.Dirty := cache_entry.Dirty;
out_msg.Acks := 0; // irrelevant out_msg.Acks := 0; // irrelevant
@ -546,7 +546,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.DataBlk := cache_entry.DataBlk; out_msg.DataBlk := cache_entry.DataBlk;
out_msg.Dirty := cache_entry.Dirty; out_msg.Dirty := cache_entry.Dirty;
out_msg.Acks := in_msg.Acks; out_msg.Acks := in_msg.Acks;
@ -591,7 +591,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.Acks := 0 - 1; // -1 out_msg.Acks := 0 - 1; // -1
out_msg.MessageSize := MessageSizeType:Response_Control; out_msg.MessageSize := MessageSizeType:Response_Control;
} }
@ -606,7 +606,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Unblock_Control; out_msg.MessageSize := MessageSizeType:Unblock_Control;
} }
} }
@ -618,7 +618,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.MessageSize := MessageSizeType:Unblock_Control; out_msg.MessageSize := MessageSizeType:Unblock_Control;
} }
} }
@ -715,7 +715,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.Dirty := false; out_msg.Dirty := false;
out_msg.Acks := 1; out_msg.Acks := 1;
out_msg.MessageSize := MessageSizeType:Response_Control; out_msg.MessageSize := MessageSizeType:Response_Control;
@ -748,7 +748,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.DataBlk := tbe.DataBlk; out_msg.DataBlk := tbe.DataBlk;
// out_msg.Dirty := tbe.Dirty; // out_msg.Dirty := tbe.Dirty;
out_msg.Dirty := false; out_msg.Dirty := false;
@ -782,7 +782,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.DataBlk := tbe.DataBlk; out_msg.DataBlk := tbe.DataBlk;
out_msg.Dirty := tbe.Dirty; out_msg.Dirty := tbe.Dirty;
out_msg.Acks := in_msg.Acks; out_msg.Acks := in_msg.Acks;
@ -800,7 +800,7 @@ machine(L1Cache, "Directory protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.SenderMachine := MachineType:L1Cache; out_msg.SenderMachine := MachineType:L1Cache;
out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache, out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)); l2_select_low_bit, l2_select_num_bits, intToID(0)));
out_msg.Dirty := tbe.Dirty; out_msg.Dirty := tbe.Dirty;
if (tbe.Dirty) { if (tbe.Dirty) {
out_msg.Type := CoherenceResponseType:WRITEBACK_DIRTY_DATA; out_msg.Type := CoherenceResponseType:WRITEBACK_DIRTY_DATA;

53
src/mem/protocol/MOESI_CMP_token-L1cache.sm
View file

@ -406,7 +406,7 @@ machine(L1Cache, "Token protocol")
} else if (machineIDToMachineType(sender) == MachineType:L2Cache) { } else if (machineIDToMachineType(sender) == MachineType:L2Cache) {
if (sender == mapAddressToRange(addr, MachineType:L2Cache, if (sender == mapAddressToRange(addr, MachineType:L2Cache,
l2_select_low_bit, l2_select_num_bits)) { l2_select_low_bit, l2_select_num_bits, intToID(0))) {
return false; return false;
} else { } else {
return true; return true;
@ -540,9 +540,8 @@ machine(L1Cache, "Token protocol")
if ( machineIDToMachineType(in_msg.Sender) == MachineType:L2Cache ) { if ( machineIDToMachineType(in_msg.Sender) == MachineType:L2Cache ) {
if (in_msg.Sender == mapAddressToRange(in_msg.Addr, if (in_msg.Sender == mapAddressToRange(in_msg.Addr,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0))) {
l2_select_num_bits)) {
// came from an off-chip L2 cache // came from an off-chip L2 cache
if (is_valid(tbe)) { if (is_valid(tbe)) {
@ -748,9 +747,8 @@ machine(L1Cache, "Token protocol")
//out_msg.Destination.addNetDest(getAllPertinentL2Banks(address)); //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Destination.add(map_Address_to_Directory(address)); out_msg.Destination.add(map_Address_to_Directory(address));
out_msg.MessageSize := MessageSizeType:Persistent_Control; out_msg.MessageSize := MessageSizeType:Persistent_Control;
@ -792,9 +790,8 @@ machine(L1Cache, "Token protocol")
out_msg.Type := CoherenceRequestType:GETS; out_msg.Type := CoherenceRequestType:GETS;
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.RetryNum := tbe.IssueCount; out_msg.RetryNum := tbe.IssueCount;
if (tbe.IssueCount == 0) { if (tbe.IssueCount == 0) {
@ -869,9 +866,8 @@ machine(L1Cache, "Token protocol")
//out_msg.Destination.addNetDest(getAllPertinentL2Banks(address)); //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Destination.add(map_Address_to_Directory(address)); out_msg.Destination.add(map_Address_to_Directory(address));
out_msg.MessageSize := MessageSizeType:Persistent_Control; out_msg.MessageSize := MessageSizeType:Persistent_Control;
@ -914,9 +910,8 @@ machine(L1Cache, "Token protocol")
out_msg.Requestor := machineID; out_msg.Requestor := machineID;
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.RetryNum := tbe.IssueCount; out_msg.RetryNum := tbe.IssueCount;
@ -991,9 +986,8 @@ machine(L1Cache, "Token protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Tokens := cache_entry.Tokens; out_msg.Tokens := cache_entry.Tokens;
out_msg.DataBlk := cache_entry.DataBlk; out_msg.DataBlk := cache_entry.DataBlk;
@ -1016,9 +1010,8 @@ machine(L1Cache, "Token protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Tokens := cache_entry.Tokens; out_msg.Tokens := cache_entry.Tokens;
out_msg.DataBlk := cache_entry.DataBlk; out_msg.DataBlk := cache_entry.DataBlk;
@ -1039,9 +1032,8 @@ machine(L1Cache, "Token protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Tokens := cache_entry.Tokens; out_msg.Tokens := cache_entry.Tokens;
out_msg.DataBlk := cache_entry.DataBlk; out_msg.DataBlk := cache_entry.DataBlk;
@ -1384,10 +1376,8 @@ machine(L1Cache, "Token protocol")
out_msg.Sender := machineID; out_msg.Sender := machineID;
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.MessageSize := MessageSizeType:Response_Control; out_msg.MessageSize := MessageSizeType:Response_Control;
} }
} }
@ -1427,9 +1417,8 @@ machine(L1Cache, "Token protocol")
//out_msg.Destination.addNetDest(getAllPertinentL2Banks(address)); //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Destination.add(map_Address_to_Directory(address)); out_msg.Destination.add(map_Address_to_Directory(address));
out_msg.MessageSize := MessageSizeType:Persistent_Control; out_msg.MessageSize := MessageSizeType:Persistent_Control;

25
src/mem/protocol/MOESI_CMP_token-dir.sm
View file

@ -443,9 +443,8 @@ machine(Directory, "Token protocol")
//out_msg.Destination.addNetDest(getAllPertinentL2Banks(address)); //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Destination.add(map_Address_to_Directory(address)); out_msg.Destination.add(map_Address_to_Directory(address));
out_msg.MessageSize := MessageSizeType:Persistent_Control; out_msg.MessageSize := MessageSizeType:Persistent_Control;
@ -485,9 +484,8 @@ machine(Directory, "Token protocol")
// //
out_msg.Destination.broadcast(MachineType:L1Cache); out_msg.Destination.broadcast(MachineType:L1Cache);
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.RetryNum := 0; out_msg.RetryNum := 0;
out_msg.MessageSize := MessageSizeType:Broadcast_Control; out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@ -513,9 +511,8 @@ machine(Directory, "Token protocol")
//out_msg.Destination.addNetDest(getAllPertinentL2Banks(address)); //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Destination.add(map_Address_to_Directory(address)); out_msg.Destination.add(map_Address_to_Directory(address));
out_msg.MessageSize := MessageSizeType:Persistent_Control; out_msg.MessageSize := MessageSizeType:Persistent_Control;
@ -551,9 +548,8 @@ machine(Directory, "Token protocol")
// //
out_msg.Destination.broadcast(MachineType:L1Cache); out_msg.Destination.broadcast(MachineType:L1Cache);
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.RetryNum := 0; out_msg.RetryNum := 0;
out_msg.MessageSize := MessageSizeType:Broadcast_Control; out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@ -730,9 +726,8 @@ machine(Directory, "Token protocol")
//out_msg.Destination.addNetDest(getAllPertinentL2Banks(address)); //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
out_msg.Destination.add(mapAddressToRange(address, out_msg.Destination.add(mapAddressToRange(address,
MachineType:L2Cache, MachineType:L2Cache, l2_select_low_bit,
l2_select_low_bit, l2_select_num_bits, intToID(0)));
l2_select_num_bits));
out_msg.Destination.add(map_Address_to_Directory(address)); out_msg.Destination.add(map_Address_to_Directory(address));
out_msg.MessageSize := MessageSizeType:Persistent_Control; out_msg.MessageSize := MessageSizeType:Persistent_Control;

3
src/mem/protocol/RubySlicc_ComponentMapping.sm
View file

@ -30,7 +30,8 @@
// Mapping functions // Mapping functions
int machineCount(MachineType machType); int machineCount(MachineType machType);
MachineID mapAddressToRange(Address addr, MachineType type, int low, int high); MachineID mapAddressToRange(Address addr, MachineType type,
int low, int high, NodeID n);
NetDest broadcast(MachineType type); NetDest broadcast(MachineType type);
MachineID map_Address_to_DMA(Address addr); MachineID map_Address_to_DMA(Address addr);
MachineID map_Address_to_Directory(Address addr); MachineID map_Address_to_Directory(Address addr);

2
src/mem/protocol/RubySlicc_Defines.sm
View file

@ -31,4 +31,4 @@
NodeID id; NodeID id;
NodeID version; NodeID version;
MachineID machineID; MachineID machineID;
NodeID clusterID;

6
src/mem/ruby/common/NetDest.cc
View file

@ -102,7 +102,7 @@ NetDest::broadcast()
void void
NetDest::broadcast(MachineType machineType) NetDest::broadcast(MachineType machineType)
{ {
for (int i = 0; i < MachineType_base_count(machineType); i++) { for (NodeID i = 0; i < MachineType_base_count(machineType); i++) {
MachineID mach = {machineType, i}; MachineID mach = {machineType, i};
add(mach); add(mach);
} }
@ -146,7 +146,7 @@ NetDest::smallestElement() const
{ {
assert(count() > 0); assert(count() > 0);
for (int i = 0; i < m_bits.size(); i++) { for (int i = 0; i < m_bits.size(); i++) {
for (int j = 0; j < m_bits[i].getSize(); j++) { for (NodeID j = 0; j < m_bits[i].getSize(); j++) {
if (m_bits[i].isElement(j)) { if (m_bits[i].isElement(j)) {
MachineID mach = {MachineType_from_base_level(i), j}; MachineID mach = {MachineType_from_base_level(i), j};
return mach; return mach;
@ -160,7 +160,7 @@ MachineID
NetDest::smallestElement(MachineType machine) const NetDest::smallestElement(MachineType machine) const
{ {
int size = m_bits[MachineType_base_level(machine)].getSize(); int size = m_bits[MachineType_base_level(machine)].getSize();
for (int j = 0; j < size; j++) { for (NodeID j = 0; j < size; j++) {
if (m_bits[MachineType_base_level(machine)].isElement(j)) { if (m_bits[MachineType_base_level(machine)].isElement(j)) {
MachineID mach = {machine, j}; MachineID mach = {machine, j};
return mach; return mach;

6
src/mem/ruby/common/TypeDefines.hh
View file

@ -37,8 +37,8 @@ typedef int64 Time;
typedef uint64 physical_address_t; typedef uint64 physical_address_t;
typedef int64 Index; // what the address bit ripper returns typedef int64 Index; // what the address bit ripper returns
typedef int LinkID; typedef unsigned int LinkID;
typedef int NodeID; typedef unsigned int NodeID;
typedef int SwitchID; typedef unsigned int SwitchID;
#endif #endif

4
src/mem/ruby/network/Topology.cc
View file

@ -129,7 +129,7 @@ Topology::createLinks(Network *net)
SwitchID max_switch_id = 0; SwitchID max_switch_id = 0;
for (LinkMap::const_iterator i = m_link_map.begin(); for (LinkMap::const_iterator i = m_link_map.begin();
i != m_link_map.end(); ++i) { i != m_link_map.end(); ++i) {
std::pair<int, int> src_dest = (*i).first; std::pair<SwitchID, SwitchID> src_dest = (*i).first;
max_switch_id = max(max_switch_id, src_dest.first); max_switch_id = max(max_switch_id, src_dest.first);
max_switch_id = max(max_switch_id, src_dest.second); max_switch_id = max(max_switch_id, src_dest.second);
} }
@ -310,7 +310,7 @@ shortest_path_to_node(SwitchID src, SwitchID next, const Matrix& weights,
max_machines = MachineType_base_number(MachineType_NUM); max_machines = MachineType_base_number(MachineType_NUM);
for (int m = 0; m < machines; m++) { for (int m = 0; m < machines; m++) {
for (int i = 0; i < MachineType_base_count((MachineType)m); i++) { for (NodeID i = 0; i < MachineType_base_count((MachineType)m); i++) {
// we use "d+max_machines" below since the "destination" // we use "d+max_machines" below since the "destination"
// switches for the machines are numbered // switches for the machines are numbered
// [MachineType_base_number(MachineType_NUM)... // [MachineType_base_number(MachineType_NUM)...

2
src/mem/ruby/network/Topology.hh
View file

@ -59,7 +59,7 @@ struct LinkEntry
LinkDirection direction; LinkDirection direction;
}; };
typedef std::map<std::pair<int, int>, LinkEntry> LinkMap; typedef std::map<std::pair<SwitchID, SwitchID>, LinkEntry> LinkMap;
class Topology class Topology
{ {

2
src/mem/ruby/slicc_interface/AbstractController.cc
View file

@ -35,6 +35,8 @@ AbstractController::AbstractController(const Params *p)
m_request_count(0) m_request_count(0)
{ {
m_version = p->version; m_version = p->version;
m_clusterID = p->cluster_id;
m_transitions_per_cycle = p->transitions_per_cycle; m_transitions_per_cycle = p->transitions_per_cycle;
m_buffer_size = p->buffer_size; m_buffer_size = p->buffer_size;
m_recycle_latency = p->recycle_latency; m_recycle_latency = p->recycle_latency;

12
src/mem/ruby/slicc_interface/AbstractController.hh
View file

@ -56,7 +56,7 @@ class AbstractController : public ClockedObject, public Consumer
void init(); void init();
const Params *params() const { return (const Params *)_params; } const Params *params() const { return (const Params *)_params; }
const int & getVersion() const { return m_version; } const NodeID getVersion() const { return m_version; }
void initNetworkPtr(Network* net_ptr) { m_net_ptr = net_ptr; } void initNetworkPtr(Network* net_ptr) { m_net_ptr = net_ptr; }
// return instance name // return instance name
@ -133,13 +133,12 @@ class AbstractController : public ClockedObject, public Consumer
void wakeUpAllBuffers(); void wakeUpAllBuffers();
protected: protected:
int m_transitions_per_cycle;
int m_buffer_size;
Cycles m_recycle_latency;
std::string m_name; std::string m_name;
NodeID m_version; NodeID m_version;
Network* m_net_ptr;
MachineID m_machineID; MachineID m_machineID;
NodeID m_clusterID;
Network* m_net_ptr;
bool m_is_blocking; bool m_is_blocking;
std::map<Address, MessageBuffer*> m_block_map; std::map<Address, MessageBuffer*> m_block_map;
typedef std::vector<MessageBuffer*> MsgVecType; typedef std::vector<MessageBuffer*> MsgVecType;
@ -148,6 +147,9 @@ class AbstractController : public ClockedObject, public Consumer
unsigned int m_in_ports; unsigned int m_in_ports;
unsigned int m_cur_in_port; unsigned int m_cur_in_port;
int m_number_of_TBEs; int m_number_of_TBEs;
int m_transitions_per_cycle;
int m_buffer_size;
Cycles m_recycle_latency;
//! Map from physical network number to the Message Buffer. //! Map from physical network number to the Message Buffer.
std::map<uint32_t, MessageBuffer*> peerQueueMap; std::map<uint32_t, MessageBuffer*> peerQueueMap;

3
src/mem/ruby/slicc_interface/Controller.py
View file

@ -36,7 +36,8 @@ class RubyController(ClockedObject):
cxx_header = "mem/ruby/slicc_interface/AbstractController.hh" cxx_header = "mem/ruby/slicc_interface/AbstractController.hh"
abstract = True abstract = True
version = Param.Int("") version = Param.Int("")
cntrl_id = Param.Int("") cluster_id = Param.UInt32(0, "Id of this controller's cluster")
transitions_per_cycle = \ transitions_per_cycle = \
Param.Int(32, "no. of SLICC state machine transitions per cycle") Param.Int(32, "no. of SLICC state machine transitions per cycle")
buffer_size = Param.Int(0, "max buffer size 0 means infinite") buffer_size = Param.Int(0, "max buffer size 0 means infinite")

10
src/mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh
View file

@ -58,7 +58,7 @@ inline NetDest
broadcast(MachineType type) broadcast(MachineType type)
{ {
NetDest dest; NetDest dest;
for (int i = 0; i < MachineType_base_count(type); i++) { for (NodeID i = 0; i < MachineType_base_count(type); i++) {
MachineID mach = {type, i}; MachineID mach = {type, i};
dest.add(mach); dest.add(mach);
} }
@ -67,12 +67,14 @@ broadcast(MachineType type)
inline MachineID inline MachineID
mapAddressToRange(const Address & addr, MachineType type, int low_bit, mapAddressToRange(const Address & addr, MachineType type, int low_bit,
int num_bits) int num_bits, int cluster_id = 0)
{ {
MachineID mach = {type, 0}; MachineID mach = {type, 0};
if (num_bits == 0) if (num_bits == 0)
return mach; mach.num = cluster_id;
mach.num = addr.bitSelect(low_bit, low_bit + num_bits - 1); else
mach.num = addr.bitSelect(low_bit, low_bit + num_bits - 1)
+ (1 << num_bits) * cluster_id;
return mach; return mach;
} }

2
src/mem/slicc/ast/ObjDeclAST.py
View file

@ -70,6 +70,8 @@ class ObjDeclAST(DeclAST):
c_code = "m_version" c_code = "m_version"
elif self.ident == "machineID": elif self.ident == "machineID":
c_code = "m_machineID" c_code = "m_machineID"
elif self.ident == "clusterID":
c_code = "m_clusterID"
elif machine: elif machine:
c_code = "(*m_%s_%s_ptr)" % (machine.ident, self.ident) c_code = "(*m_%s_%s_ptr)" % (machine.ident, self.ident)
else: else: