ruby: add support for clusters

A cluster over here means a set of controllers that can be accessed only by a
certain set of cores.  For example,  consider a two level hierarchy. Assume
there are 4 L1 controllers (private) and 2 L2 controllers.  We can have two
different hierarchies here:

a. the address space is partitioned between the two L2 controllers.  Each L1
controller accesses both the L2 controllers.  In this case, each L1 controller
is a cluster initself.

b. both the L2 controllers can cache any address.  An L1 controller has access
to only one of the L2 controllers.  In this case, each L2 controller
along with the L1 controllers that access it, form a cluster.

This patch allows for each controller to have a cluster ID, which is 0 by
default.  By setting the cluster ID properly,  one can instantiate hierarchies
with clusters.  Note that the coherence protocol might have to be changed as
well.

src/mem/protocol/MESI_CMP_directory-L1cache.sm

@@ -500,7 +500,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GETS;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -518,7 +518,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GETS;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                      l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -535,7 +535,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GET_INSTR;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -555,7 +555,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
               out_msg.Requestor := machineID;
               out_msg.Destination.add(
                   mapAddressToRange(address, MachineType:L2Cache,
-                                    l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
               out_msg.MessageSize := MessageSizeType:Control;
               out_msg.Prefetch := in_msg.Prefetch;
               out_msg.AccessMode := in_msg.AccessMode;
@@ -574,7 +574,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Requestor := machineID;
         DPRINTF(RubySlicc, "%s\n", machineID);
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -593,10 +593,8 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
               out_msg.Requestor := machineID;
               DPRINTF(RubySlicc, "%s\n", machineID);
 
-              out_msg.Destination.add(mapAddressToRange(address,
-                                                        MachineType:L2Cache,
-                                                        l2_select_low_bit,
-                                                        l2_select_num_bits));
+              out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
 
               DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                       address, out_msg.Destination);
@@ -614,7 +612,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:UPGRADE;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -648,7 +646,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Data;
     }
   }
@@ -677,7 +675,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := tbe.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Data;
     }
   }
@@ -703,7 +701,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Data;
     }
   }
@@ -717,7 +715,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := tbe.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Data;
     }
   }
@@ -751,7 +749,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Requestor:= machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       if (cache_entry.Dirty) {
         out_msg.MessageSize := MessageSizeType:Writeback_Data;
       } else {
@@ -766,7 +764,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Type := CoherenceResponseType:UNBLOCK;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Control;
       DPRINTF(RubySlicc, "%s\n", address);
     }
@@ -778,7 +776,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Type := CoherenceResponseType:EXCLUSIVE_UNBLOCK;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Control;
       DPRINTF(RubySlicc, "%s\n", address);
 

src/mem/protocol/MESI_CMP_directory-dir.sm

@@ -39,7 +39,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
    MemoryControl * memBuffer,
    Cycles to_mem_ctrl_latency = 1,
    Cycles directory_latency = 6,
-   int l2_select_num_bits
 {
   MessageBuffer requestToDir, network="From", virtual_network="0",
         ordered="false", vnet_type="request";
@@ -83,6 +82,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
     DataBlock DataBlk,             desc="data for the block";
+    MachineID Owner;
   }
 
   // TBE entries for DMA requests
@@ -102,8 +102,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
 
 
   // ** OBJECTS **
-
-  int l2_select_low_bit, default="RubySystem::getBlockSizeBits()";
   TBETable TBEs, template="<Directory_TBE>", constructor="m_number_of_TBEs";
 
   void set_tbe(TBE tbe);
@@ -262,6 +260,9 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
         out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false;
         out_msg.MessageSize := MessageSizeType:Response_Data;
+
+        Entry e := getDirectoryEntry(in_msg.Addr);
+        e.Owner := in_msg.OriginalRequestorMachId;
       }
     }
   }
@@ -409,12 +410,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   action(inv_sendCacheInvalidate, "inv", desc="Invalidate a cache block") {
     peek(requestNetwork_in, RequestMsg) {
       enqueue(responseNetwork_out, ResponseMsg, latency=directory_latency) {
-      out_msg.Addr := address;
-      out_msg.Type := CoherenceResponseType:INV;
-      out_msg.Sender := machineID;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                     l2_select_low_bit, l2_select_num_bits));
-      out_msg.MessageSize := MessageSizeType:Response_Control;
+        out_msg.Addr := address;
+        out_msg.Type := CoherenceResponseType:INV;
+        out_msg.Sender := machineID;
+        out_msg.Destination.add(getDirectoryEntry(address).Owner);
+        out_msg.MessageSize := MessageSizeType:Response_Control;
       }
     }
   }
@@ -483,6 +483,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
     j_popIncomingRequestQueue;
   }
 
+  transition(M, Fetch) {
+    inv_sendCacheInvalidate;
+    z_stallAndWaitRequest;
+  }
+
   transition(IM, Memory_Data, M) {
     d_sendData;
     l_popMemQueue;
@@ -492,6 +497,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   transition(M, CleanReplacement, I) {
     a_sendAck;
     k_popIncomingResponseQueue;
+    kd_wakeUpDependents;
   }
 
   transition(M, Data, MI) {
@@ -578,5 +584,4 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
     l_popMemQueue;
     kd_wakeUpDependents;
   }
-
 }

src/mem/protocol/MOESI_CMP_directory-L1cache.sm

@@ -420,7 +420,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -436,7 +436,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -452,7 +452,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -465,7 +465,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -478,7 +478,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -493,7 +493,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := cache_entry.DataBlk;
           // out_msg.Dirty := cache_entry.Dirty;
           out_msg.Dirty := false;
@@ -528,7 +528,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.DataBlk := cache_entry.DataBlk;
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Acks := 0; // irrelevant
@@ -546,7 +546,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := cache_entry.DataBlk;
           out_msg.Dirty := cache_entry.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -591,7 +591,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.Acks := 0 - 1; // -1
           out_msg.MessageSize := MessageSizeType:Response_Control;
         }
@@ -606,7 +606,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -618,7 +618,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -715,7 +715,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.Dirty := false;
         out_msg.Acks := 1;
         out_msg.MessageSize := MessageSizeType:Response_Control;
@@ -748,7 +748,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := tbe.DataBlk;
           // out_msg.Dirty := tbe.Dirty;
           out_msg.Dirty := false;
@@ -782,7 +782,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := tbe.DataBlk;
           out_msg.Dirty := tbe.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -800,7 +800,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.Dirty := tbe.Dirty;
       if (tbe.Dirty) {
         out_msg.Type := CoherenceResponseType:WRITEBACK_DIRTY_DATA;

src/mem/protocol/MOESI_CMP_token-L1cache.sm

@@ -406,7 +406,7 @@ machine(L1Cache, "Token protocol")
     } else if (machineIDToMachineType(sender) == MachineType:L2Cache) {
 
       if (sender == mapAddressToRange(addr, MachineType:L2Cache,
-                      l2_select_low_bit, l2_select_num_bits)) {
+                      l2_select_low_bit, l2_select_num_bits, intToID(0))) {
         return false;
       } else {
         return  true;
@@ -540,9 +540,8 @@ machine(L1Cache, "Token protocol")
         if ( machineIDToMachineType(in_msg.Sender) == MachineType:L2Cache ) {
 
           if (in_msg.Sender == mapAddressToRange(in_msg.Addr,
-                                                 MachineType:L2Cache,
-                                                 l2_select_low_bit,
-                                                 l2_select_num_bits)) {
+                                 MachineType:L2Cache, l2_select_low_bit,
+                                 l2_select_num_bits, intToID(0))) {
 
             // came from an off-chip L2 cache
             if (is_valid(tbe)) {
@@ -748,9 +747,8 @@ machine(L1Cache, "Token protocol")
             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
             out_msg.Destination.add(mapAddressToRange(address,
-                                                      MachineType:L2Cache,
-                                                      l2_select_low_bit,
-                                                      l2_select_num_bits));
+                                      MachineType:L2Cache, l2_select_low_bit,
+                                      l2_select_num_bits, intToID(0)));
 
             out_msg.Destination.add(map_Address_to_Directory(address));
             out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -792,9 +790,8 @@ machine(L1Cache, "Token protocol")
           out_msg.Type := CoherenceRequestType:GETS;
           out_msg.Requestor := machineID;
           out_msg.Destination.add(mapAddressToRange(address,
-                                                    MachineType:L2Cache,
-                                                    l2_select_low_bit,
-                                                    l2_select_num_bits));
+                                    MachineType:L2Cache, l2_select_low_bit,
+                                    l2_select_num_bits, intToID(0)));
 
           out_msg.RetryNum := tbe.IssueCount;
           if (tbe.IssueCount == 0) {
@@ -869,9 +866,8 @@ machine(L1Cache, "Token protocol")
             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
             out_msg.Destination.add(mapAddressToRange(address,
-                                                      MachineType:L2Cache,
-                                                      l2_select_low_bit,
-                                                      l2_select_num_bits));
+                                      MachineType:L2Cache, l2_select_low_bit,
+                                      l2_select_num_bits, intToID(0)));
 
             out_msg.Destination.add(map_Address_to_Directory(address));
             out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -914,9 +910,8 @@ machine(L1Cache, "Token protocol")
           out_msg.Requestor := machineID;
 
           out_msg.Destination.add(mapAddressToRange(address,
-                                                    MachineType:L2Cache,
-                                                    l2_select_low_bit,
-                                                    l2_select_num_bits));
+                                    MachineType:L2Cache, l2_select_low_bit,
+                                    l2_select_num_bits, intToID(0)));
 
           out_msg.RetryNum := tbe.IssueCount;
 
@@ -991,9 +986,8 @@ machine(L1Cache, "Token protocol")
       out_msg.Sender := machineID;
 
       out_msg.Destination.add(mapAddressToRange(address,
-                                                MachineType:L2Cache,
-                                                l2_select_low_bit,
-                                                l2_select_num_bits));
+                                MachineType:L2Cache, l2_select_low_bit,
+                                l2_select_num_bits, intToID(0)));
 
       out_msg.Tokens := cache_entry.Tokens;
       out_msg.DataBlk := cache_entry.DataBlk;
@@ -1016,9 +1010,8 @@ machine(L1Cache, "Token protocol")
         out_msg.Sender := machineID;
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Tokens := cache_entry.Tokens;
         out_msg.DataBlk := cache_entry.DataBlk;
@@ -1039,9 +1032,8 @@ machine(L1Cache, "Token protocol")
         out_msg.Sender := machineID;
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Tokens := cache_entry.Tokens;
         out_msg.DataBlk := cache_entry.DataBlk;
@@ -1384,10 +1376,8 @@ machine(L1Cache, "Token protocol")
        out_msg.Sender := machineID;
 
        out_msg.Destination.add(mapAddressToRange(address,
-                                                 MachineType:L2Cache,
-                                                 l2_select_low_bit,
-                                                 l2_select_num_bits));
-
+                                 MachineType:L2Cache, l2_select_low_bit,
+                                 l2_select_num_bits, intToID(0)));
        out_msg.MessageSize := MessageSizeType:Response_Control;
     }
   }
@@ -1427,9 +1417,8 @@ machine(L1Cache, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;

src/mem/protocol/MOESI_CMP_token-dir.sm

@@ -443,9 +443,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -485,9 +484,8 @@ machine(Directory, "Token protocol")
         //
         out_msg.Destination.broadcast(MachineType:L1Cache);
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.RetryNum := 0;
         out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@@ -513,9 +511,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -551,9 +548,8 @@ machine(Directory, "Token protocol")
         //
         out_msg.Destination.broadcast(MachineType:L1Cache);
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.RetryNum := 0;
         out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@@ -730,9 +726,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;

src/mem/protocol/RubySlicc_ComponentMapping.sm

@@ -30,7 +30,8 @@
 // Mapping functions
 
 int machineCount(MachineType machType);
-MachineID mapAddressToRange(Address addr, MachineType type, int low, int high);
+MachineID mapAddressToRange(Address addr, MachineType type,
+                            int low, int high, NodeID n);
 NetDest broadcast(MachineType type);
 MachineID map_Address_to_DMA(Address addr);
 MachineID map_Address_to_Directory(Address addr);

src/mem/protocol/RubySlicc_Defines.sm

@@ -31,4 +31,4 @@
 NodeID id;
 NodeID version;
 MachineID machineID;
-
+NodeID clusterID;

src/mem/ruby/common/NetDest.cc

@@ -102,7 +102,7 @@ NetDest::broadcast()
 void
 NetDest::broadcast(MachineType machineType)
 {
-    for (int i = 0; i < MachineType_base_count(machineType); i++) {
+    for (NodeID i = 0; i < MachineType_base_count(machineType); i++) {
         MachineID mach = {machineType, i};
         add(mach);
     }
@@ -146,7 +146,7 @@ NetDest::smallestElement() const
 {
     assert(count() > 0);
     for (int i = 0; i < m_bits.size(); i++) {
-        for (int j = 0; j < m_bits[i].getSize(); j++) {
+        for (NodeID j = 0; j < m_bits[i].getSize(); j++) {
             if (m_bits[i].isElement(j)) {
                 MachineID mach = {MachineType_from_base_level(i), j};
                 return mach;
@@ -160,7 +160,7 @@ MachineID
 NetDest::smallestElement(MachineType machine) const
 {
     int size = m_bits[MachineType_base_level(machine)].getSize();
-    for (int j = 0; j < size; j++) {
+    for (NodeID j = 0; j < size; j++) {
         if (m_bits[MachineType_base_level(machine)].isElement(j)) {
             MachineID mach = {machine, j};
             return mach;

src/mem/ruby/common/TypeDefines.hh

@@ -37,8 +37,8 @@ typedef int64 Time;
 typedef uint64 physical_address_t;
 
 typedef int64 Index;            // what the address bit ripper returns
-typedef int LinkID;
-typedef int NodeID;
-typedef int SwitchID;
+typedef unsigned int LinkID;
+typedef unsigned int NodeID;
+typedef unsigned int SwitchID;
 
 #endif

src/mem/ruby/network/Topology.cc

@@ -129,7 +129,7 @@ Topology::createLinks(Network *net)
     SwitchID max_switch_id = 0;
     for (LinkMap::const_iterator i = m_link_map.begin();
          i != m_link_map.end(); ++i) {
-        std::pair<int, int> src_dest = (*i).first;
+        std::pair<SwitchID, SwitchID> src_dest = (*i).first;
         max_switch_id = max(max_switch_id, src_dest.first);
         max_switch_id = max(max_switch_id, src_dest.second);        
     }
@@ -310,7 +310,7 @@ shortest_path_to_node(SwitchID src, SwitchID next, const Matrix& weights,
     max_machines = MachineType_base_number(MachineType_NUM);
 
     for (int m = 0; m < machines; m++) {
-        for (int i = 0; i < MachineType_base_count((MachineType)m); i++) {
+        for (NodeID i = 0; i < MachineType_base_count((MachineType)m); i++) {
             // we use "d+max_machines" below since the "destination"
             // switches for the machines are numbered
             // [MachineType_base_number(MachineType_NUM)...

src/mem/ruby/network/Topology.hh

@@ -59,7 +59,7 @@ struct LinkEntry
     LinkDirection direction;
 };
 
-typedef std::map<std::pair<int, int>, LinkEntry> LinkMap;
+typedef std::map<std::pair<SwitchID, SwitchID>, LinkEntry> LinkMap;
 
 class Topology
 {

src/mem/ruby/slicc_interface/AbstractController.cc

@@ -35,6 +35,8 @@ AbstractController::AbstractController(const Params *p)
     m_request_count(0)
 {
     m_version = p->version;
+    m_clusterID = p->cluster_id;
+
     m_transitions_per_cycle = p->transitions_per_cycle;
     m_buffer_size = p->buffer_size;
     m_recycle_latency = p->recycle_latency;

src/mem/ruby/slicc_interface/AbstractController.hh

@@ -56,7 +56,7 @@ class AbstractController : public ClockedObject, public Consumer
     void init();
     const Params *params() const { return (const Params *)_params; }
 
-    const int & getVersion() const { return m_version; }
+    const NodeID getVersion() const { return m_version; }
     void initNetworkPtr(Network* net_ptr) { m_net_ptr = net_ptr; }
 
     // return instance name
@@ -133,13 +133,12 @@ class AbstractController : public ClockedObject, public Consumer
     void wakeUpAllBuffers();
 
   protected:
-    int m_transitions_per_cycle;
-    int m_buffer_size;
-    Cycles m_recycle_latency;
     std::string m_name;
     NodeID m_version;
-    Network* m_net_ptr;
     MachineID m_machineID;
+    NodeID m_clusterID;
+
+    Network* m_net_ptr;
     bool m_is_blocking;
     std::map<Address, MessageBuffer*> m_block_map;
     typedef std::vector<MessageBuffer*> MsgVecType;
@@ -148,6 +147,9 @@ class AbstractController : public ClockedObject, public Consumer
     unsigned int m_in_ports;
     unsigned int m_cur_in_port;
     int m_number_of_TBEs;
+    int m_transitions_per_cycle;
+    int m_buffer_size;
+    Cycles m_recycle_latency;
 
     //! Map from physical network number to the Message Buffer.
     std::map<uint32_t, MessageBuffer*> peerQueueMap;

src/mem/ruby/slicc_interface/Controller.py

@@ -36,7 +36,8 @@ class RubyController(ClockedObject):
     cxx_header = "mem/ruby/slicc_interface/AbstractController.hh"
     abstract = True
     version = Param.Int("")
-    cntrl_id = Param.Int("")
+    cluster_id = Param.UInt32(0, "Id of this controller's cluster")
+
     transitions_per_cycle = \
         Param.Int(32, "no. of  SLICC state machine transitions per cycle")
     buffer_size = Param.Int(0, "max buffer size 0 means infinite")

src/mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh

@@ -58,7 +58,7 @@ inline NetDest
 broadcast(MachineType type)
 {
     NetDest dest;
-    for (int i = 0; i < MachineType_base_count(type); i++) {
+    for (NodeID i = 0; i < MachineType_base_count(type); i++) {
         MachineID mach = {type, i};
         dest.add(mach);
     }
@@ -67,12 +67,14 @@ broadcast(MachineType type)
 
 inline MachineID
 mapAddressToRange(const Address & addr, MachineType type, int low_bit,
-                  int num_bits)
+                  int num_bits, int cluster_id = 0)
 {
     MachineID mach = {type, 0};
     if (num_bits == 0)
-        return mach;
-    mach.num = addr.bitSelect(low_bit, low_bit + num_bits - 1);
+        mach.num = cluster_id;
+    else
+        mach.num = addr.bitSelect(low_bit, low_bit + num_bits - 1)
+            + (1 << num_bits) * cluster_id;
     return mach;
 }
 

src/mem/slicc/ast/ObjDeclAST.py

@@ -70,6 +70,8 @@ class ObjDeclAST(DeclAST):
             c_code = "m_version"
         elif self.ident == "machineID":
             c_code = "m_machineID"
+        elif self.ident == "clusterID":
+            c_code = "m_clusterID"
         elif machine:
             c_code = "(*m_%s_%s_ptr)" % (machine.ident, self.ident)
         else: